Schema v8 binds authority to builds (ADR-0068): acquisition names the host build and only retained, unretired builds may hold a lease; no commit may name a retired build; upgrade_barrier retires a build and fences every holder in one transaction, while build-less pre-v8 hosts keep working until the first barrier. Tests cover a writer surviving a failed and then a real online migration, old-writer fencing on every path, whole-history recovery by the new build, and the M7 qualification fixture (19.6 s SIGKILL-to-ready under the production lease, no lost acknowledgements). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
780 lines
42 KiB
C++
780 lines
42 KiB
C++
// Real PostgreSQL + filesystem M7.4 checkpoint publication/recovery (ADR-0061).
|
|
#include "evo/persistence/checkpoint_store.hpp"
|
|
#include "evo/persistence/sha256.hpp"
|
|
#include "fixtures/environmental_region.hpp"
|
|
#include <chrono>
|
|
#include <cstdio>
|
|
#include <cstdlib>
|
|
#include <filesystem>
|
|
#include <fstream>
|
|
#include <optional>
|
|
#include <source_location>
|
|
#include <stdexcept>
|
|
#include <string>
|
|
#include <sys/stat.h>
|
|
#include <thread>
|
|
#include <unistd.h>
|
|
|
|
void run_region_checkpoint_store_tests();
|
|
|
|
namespace {
|
|
using namespace evo::persistence;
|
|
using namespace std::chrono_literals;
|
|
namespace fs = std::filesystem;
|
|
void check(bool value, std::source_location at = std::source_location::current()) {
|
|
if (!value)
|
|
throw std::runtime_error("region checkpoint store test line " + std::to_string(at.line()));
|
|
}
|
|
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
|
|
std::string env(const char* key) {
|
|
const auto* value = std::getenv(key);
|
|
if (!value)
|
|
throw std::runtime_error("missing test environment");
|
|
return value;
|
|
}
|
|
std::unique_ptr<Postgres> connect_as(const std::string& user) {
|
|
PgOptions options{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), user, {}, {}};
|
|
auto made = Postgres::connect(options, 5s);
|
|
CHECK(made);
|
|
return std::move(*made);
|
|
}
|
|
template <std::size_t N> std::array<std::uint8_t, N> filled(std::uint8_t value) {
|
|
std::array<std::uint8_t, N> out{};
|
|
out.fill(value);
|
|
return out;
|
|
}
|
|
std::string hex(std::span<const std::uint8_t> bytes) {
|
|
constexpr char digits[] = "0123456789abcdef";
|
|
std::string out;
|
|
for (const auto b : bytes) {
|
|
out.push_back(digits[b >> 4U]);
|
|
out.push_back(digits[b & 0x0FU]);
|
|
}
|
|
return out;
|
|
}
|
|
std::vector<std::uint8_t> bytes_of(std::string_view s) {
|
|
return {s.begin(), s.end()};
|
|
}
|
|
// Private scratch root; the CI script runs each binary under its own /tmp dir.
|
|
fs::path scratch(const char* name) {
|
|
const auto root = fs::path(env("PGHOST")) / name;
|
|
fs::remove_all(root);
|
|
fs::create_directory(root);
|
|
fs::permissions(root, fs::perms::owner_all, fs::perm_options::replace);
|
|
return root;
|
|
}
|
|
const Sha256Digest build = filled<32>(0xB1);
|
|
constexpr std::size_t budget = 8000000;
|
|
|
|
void sha256_vectors_and_database_agreement() {
|
|
// FIPS 180-2 / NIST example vectors.
|
|
CHECK(hex(*sha256({})) == "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
|
|
CHECK(hex(*sha256(bytes_of("abc"))) ==
|
|
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad");
|
|
CHECK(hex(*sha256(bytes_of("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"))) ==
|
|
"248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1");
|
|
// Independent implementation: PostgreSQL computes the commit digest.
|
|
auto db = connect_as(env("PGUSER"));
|
|
std::vector<std::uint8_t> data(2000); // Hex parameter stays within the 4KiB adapter bound.
|
|
for (std::size_t i = 0; i < data.size(); ++i)
|
|
data[i] = static_cast<std::uint8_t>((i * 131U) ^ (i >> 3U));
|
|
const std::array<std::string, 1> p{"\\x" + hex(data)};
|
|
auto rows = db->query("SELECT encode(sha256($1::bytea), 'hex')", p, 5s);
|
|
CHECK(rows && (*rows)[0][0] == hex(*sha256(data)));
|
|
}
|
|
|
|
void object_store_is_immutable_and_verified() {
|
|
const auto root = scratch("objects");
|
|
CHECK(!FilesystemObjectStore::open(root / "absent", {1, 1}, 1024));
|
|
CHECK(!FilesystemObjectStore::open(root, {0, 1}, 1024));
|
|
fs::create_directory_symlink(root, root.parent_path() / "objects-link");
|
|
CHECK(!FilesystemObjectStore::open(root.parent_path() / "objects-link", {1, 1}, 1024));
|
|
fs::remove(root.parent_path() / "objects-link");
|
|
auto store = FilesystemObjectStore::open(root, {1, 2}, 1024);
|
|
CHECK(store);
|
|
const auto dir = (*store)->directory();
|
|
CHECK(dir == root / "w1" / "r2");
|
|
CHECK((fs::status(dir).permissions() & fs::perms::all) == fs::perms::owner_all);
|
|
|
|
const auto payload = bytes_of("immutable payload bytes");
|
|
auto put = (*store)->put(payload);
|
|
CHECK(put && put->bytes == payload.size() && put->sha256 == *sha256(payload));
|
|
auto again = (*store)->put(payload); // Idempotent for identical bytes.
|
|
CHECK(again && *again == *put);
|
|
auto got = (*store)->get(*put);
|
|
CHECK(got && *got == payload);
|
|
const auto file = dir / hex(put->sha256);
|
|
CHECK((fs::status(file).permissions() & fs::perms::all) == fs::perms::owner_read ||
|
|
(fs::status(file).permissions() & fs::perms::all) ==
|
|
(fs::perms::owner_read | fs::perms::owner_write));
|
|
|
|
CHECK((*store)->put({}).error() == ObjectError::invalid);
|
|
CHECK((*store)->put(std::vector<std::uint8_t>(1025)).error() == ObjectError::bounds);
|
|
CHECK((*store)->get({put->sha256, 1025}).error() == ObjectError::bounds);
|
|
CHECK((*store)->get({filled<32>(7), 5}).error() == ObjectError::missing);
|
|
CHECK((*store)->get({put->sha256, put->bytes - 1}).error() == ObjectError::corrupt);
|
|
|
|
// A crash mid-upload leaves only a temporary name no key can resolve.
|
|
std::ofstream(dir / (".tmp-" + hex(filled<32>(9)) + "-1-0")) << "partial";
|
|
CHECK((*store)->get({filled<32>(9), 7}).error() == ObjectError::missing);
|
|
|
|
// Corruption is detected, and never repaired by overwriting.
|
|
{
|
|
std::fstream f(file, std::ios::in | std::ios::out | std::ios::binary);
|
|
f.seekp(3);
|
|
f.put('X');
|
|
}
|
|
CHECK((*store)->get(*put).error() == ObjectError::corrupt);
|
|
CHECK((*store)->put(payload).error() == ObjectError::corrupt);
|
|
fs::remove(file);
|
|
fs::create_symlink(dir / "elsewhere", file);
|
|
CHECK((*store)->get(*put).error() == ObjectError::corrupt);
|
|
fs::remove(file);
|
|
CHECK((*store)->put(payload) && (*store)->get(*put));
|
|
fs::resize_file(file, payload.size() + 1);
|
|
CHECK((*store)->get(*put).error() == ObjectError::corrupt);
|
|
fs::remove(file);
|
|
|
|
if (::geteuid() != 0) { // Permission faults cannot be simulated as root.
|
|
fs::permissions(dir, fs::perms::owner_read | fs::perms::owner_exec,
|
|
fs::perm_options::replace);
|
|
CHECK((*store)->put(payload).error() == ObjectError::io);
|
|
fs::permissions(dir, fs::perms::owner_all, fs::perm_options::replace);
|
|
}
|
|
}
|
|
|
|
// Review P1: a name made visible by an install whose directory sync failed is not
|
|
// durable. Neither a retry that finds it nor a namespace reopen may report success
|
|
// until a directory sync actually succeeds.
|
|
void directory_sync_failures_are_never_acknowledged() {
|
|
const auto root = scratch("syncs");
|
|
ObjectStoreFaults faults;
|
|
faults.fail_directory_syncs = 1; // Creating w1 under the root.
|
|
CHECK(FilesystemObjectStore::open(root, {1, 5}, 1024, &faults).error() == ObjectError::io);
|
|
CHECK(fs::exists(root / "w1")); // Created, but its entry is unconfirmed.
|
|
faults.fail_directory_syncs = 1; // Reopen finds it and must sync again.
|
|
CHECK(FilesystemObjectStore::open(root, {1, 5}, 1024, &faults).error() == ObjectError::io);
|
|
auto store = FilesystemObjectStore::open(root, {1, 5}, 1024, &faults);
|
|
CHECK(store && faults.directory_syncs == 2); // root->w1 and w1->r5.
|
|
|
|
const auto payload = bytes_of("durable only after a directory sync succeeds");
|
|
const VerifiedObject object{*sha256(payload), payload.size()};
|
|
faults.fail_directory_syncs = 1;
|
|
CHECK((*store)->put(payload).error() == ObjectError::io);
|
|
CHECK((*store)->get(object)); // Visible, yet never acknowledged.
|
|
faults.fail_directory_syncs = 3; // Retries while the directory still cannot sync.
|
|
for (int attempt = 0; attempt < 3; ++attempt)
|
|
CHECK((*store)->put(payload).error() == ObjectError::io);
|
|
CHECK(faults.directory_syncs == 2);
|
|
auto retried = (*store)->put(payload);
|
|
CHECK(retried && *retried == object && faults.directory_syncs == 3);
|
|
// Every reuse re-establishes durability rather than trusting the visible name.
|
|
CHECK((*store)->put(payload) && faults.directory_syncs == 4);
|
|
}
|
|
|
|
struct Region {
|
|
RegionKey key{7, 3};
|
|
std::unique_ptr<Postgres> owner, runtime;
|
|
std::unique_ptr<FilesystemObjectStore> objects;
|
|
};
|
|
Region provisioned(const char* name, std::uint64_t region, std::uint8_t genesis_seed) {
|
|
Region r;
|
|
r.key.region = region;
|
|
r.owner = connect_as(env("PGUSER"));
|
|
r.runtime = connect_as(env("EVO_PG_RUNTIME_USER"));
|
|
auto objects =
|
|
FilesystemObjectStore::open(scratch(name), r.key, max_environmental_payload_bytes);
|
|
CHECK(objects);
|
|
r.objects = std::move(*objects);
|
|
CHECK(r.owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
|
|
"(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING",
|
|
{}, 5s));
|
|
auto opening = evo::test::region_community();
|
|
auto genesis = provision_environmental_region(*r.owner, *r.objects, r.key,
|
|
evo::test::region_identity.frame, *opening,
|
|
filled<16>(genesis_seed), build, budget, 5s);
|
|
CHECK(genesis && genesis->head.sequence == 0);
|
|
return r;
|
|
}
|
|
std::size_t count_objects(const fs::path& dir) {
|
|
std::size_t n = 0;
|
|
for (const auto& entry : fs::directory_iterator(dir))
|
|
n += entry.path().filename().string().starts_with(".tmp-") ? 0 : 1;
|
|
return n;
|
|
}
|
|
|
|
using EnvBatch = RegionBatch<evo::processes::EnvironmentalCommunity>;
|
|
using LineageBatch = RegionBatch<evo::processes::LineageRegion>;
|
|
// Runs up to `ticks` more ticks (stopping early if an outcome seals it), then flushes.
|
|
template <typename Batch> void run_batch(Batch& b, std::uint64_t ticks, bool deaths = false) {
|
|
for (std::uint64_t i = 0; i < ticks && !b.sealed(); ++i)
|
|
CHECK(b.advance(evo::test::region_input(b.end_tick() + 1, deaths)));
|
|
b.seal();
|
|
}
|
|
|
|
void publish_recover_and_continue() {
|
|
auto r = provisioned("publish", 3, 0xC0);
|
|
const std::array<Sha256Digest, 1> retained{build};
|
|
auto lease = acquire_region_lease(*r.runtime, r.key, build, 20000ms, 5s);
|
|
CHECK(lease);
|
|
// Genesis recovers to the exact opening state.
|
|
auto recovered =
|
|
recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s);
|
|
CHECK(recovered && recovered->state->progress().tick == 0 && recovered->decisions.empty());
|
|
auto head = recovered->committed;
|
|
|
|
// Genesis cannot be re-provisioned or published over.
|
|
auto opening = evo::test::region_community();
|
|
CHECK(!provision_environmental_region(*r.owner, *r.objects, r.key, 9, *opening,
|
|
filled<16>(0xC9), build, budget, 5s));
|
|
|
|
auto original = evo::test::region_community();
|
|
EnvBatch batch{*original};
|
|
// An unsealed batch is still being simulated: refused before any upload.
|
|
CHECK(batch.advance(evo::test::region_input(1)));
|
|
const auto before = count_objects(r.objects->directory());
|
|
auto open = publish_environmental_batch(*r.runtime, *r.objects, *lease, head, batch,
|
|
filled<16>(0xCA), build, budget, 5s);
|
|
CHECK(!open && open.error().kind == PublicationError::candidate);
|
|
CHECK(count_objects(r.objects->directory()) == before);
|
|
std::uint8_t seed = 0xC1;
|
|
for (int n = 0; n < 3; ++n) {
|
|
run_batch(batch, n == 1 ? 64 : 15);
|
|
const auto started = std::chrono::steady_clock::now();
|
|
auto published = publish_environmental_batch(*r.runtime, *r.objects, *lease, head, batch,
|
|
filled<16>(seed++), build, budget, 5s);
|
|
const auto us = std::chrono::duration_cast<std::chrono::microseconds>(
|
|
std::chrono::steady_clock::now() - started)
|
|
.count();
|
|
CHECK(published && published->committed.head.sequence == head.head.sequence + 1);
|
|
CHECK(published->committed.head.end_tick == original->progress().tick);
|
|
CHECK(published->decisions.empty());
|
|
const auto m = *decode_manifest(published->committed.manifest);
|
|
CHECK(m.manifest_version == 2 &&
|
|
m.batch.trace_bytes == trace_bytes_for(m.end_tick - m.begin_tick));
|
|
std::printf("batch_publish_us batch=%d ticks=%llu payload_bytes=%llu us=%lld\n", n,
|
|
static_cast<unsigned long long>(m.end_tick - m.begin_tick),
|
|
static_cast<unsigned long long>(m.payload_bytes), static_cast<long long>(us));
|
|
CHECK(batch.confirm(m.end_tick));
|
|
head = std::move(published->committed);
|
|
// The trace object is retained, bound to this link of the chain.
|
|
auto trace = r.objects->get({m.batch.trace_sha256, m.batch.trace_bytes});
|
|
auto decoded = trace ? decode_trace(*trace) : std::unexpected{BatchError::format};
|
|
CHECK(decoded && decoded->header.begin_tick == m.begin_tick &&
|
|
decoded->header.predecessor_digest == m.predecessor_digest);
|
|
}
|
|
// State advanced outside the recorded batch cannot be published: the replay of
|
|
// the recorded inputs no longer reproduces it.
|
|
auto drifted = evo::test::region_community();
|
|
EnvBatch drift{*drifted};
|
|
run_batch(drift, 3);
|
|
CHECK(drifted->advance(4, evo::test::region_input(4)));
|
|
auto diverged =
|
|
publish_environmental_batch(*r.runtime, *r.objects, *lease, recovered->committed, drift,
|
|
filled<16>(0xCD), build, budget, 5s);
|
|
CHECK(!diverged && diverged.error().kind == PublicationError::candidate);
|
|
// A batch replayed from the wrong predecessor cannot reproduce its state.
|
|
run_batch(batch, 4);
|
|
auto stale = publish_environmental_batch(*r.runtime, *r.objects, *lease, recovered->committed,
|
|
batch, filled<16>(0xCB), build, budget, 5s);
|
|
CHECK(!stale && stale.error().kind == PublicationError::candidate);
|
|
|
|
// A new process recovers the committed head; its trajectory matches the original's.
|
|
auto second = FilesystemObjectStore::open(r.objects->directory().parent_path().parent_path(),
|
|
r.key, max_environmental_payload_bytes);
|
|
CHECK(second);
|
|
auto restored =
|
|
recover_environmental_checkpoint(*r.runtime, **second, r.key, retained, budget, 5s);
|
|
CHECK(restored && restored->committed.head == head.head);
|
|
CHECK(restored->state->progress().tick == 95);
|
|
auto reference = evo::test::region_community();
|
|
evo::test::advance_region(*reference, 95);
|
|
CHECK(reference->snapshot() == restored->state->snapshot());
|
|
for (std::uint64_t tick = 96; tick <= 140; ++tick) {
|
|
auto a = reference->advance(tick, evo::test::region_input(tick, true));
|
|
auto b = restored->state->advance(tick, evo::test::region_input(tick, true));
|
|
CHECK(a && b && *a == *b);
|
|
}
|
|
CHECK(reference->snapshot() == restored->state->snapshot());
|
|
|
|
// Retained-build allowlist: a host without this build cannot restore it.
|
|
const std::array<Sha256Digest, 1> other{filled<32>(0xB2)};
|
|
auto foreign =
|
|
recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, other, budget, 5s);
|
|
CHECK(!foreign && foreign.error().kind == RecoveryError::unsupported);
|
|
// Unretained build is refused by the database; the uploads stay orphans.
|
|
auto refused = publish_environmental_batch(*r.runtime, *r.objects, *lease, head, batch,
|
|
filled<16>(0xCC), filled<32>(0xB2), budget, 5s);
|
|
CHECK(!refused && refused.error().kind == PublicationError::rejected &&
|
|
refused.error().store.kind == LeaseStoreError::unsupported);
|
|
CHECK(release_region_lease(*r.runtime, r.key, *lease, 5s));
|
|
}
|
|
|
|
void failures_leave_the_previous_head() {
|
|
auto r = provisioned("failures", 4, 0xD0);
|
|
const std::array<Sha256Digest, 1> retained{build};
|
|
auto lease = acquire_region_lease(*r.runtime, r.key, build, 20000ms, 5s);
|
|
CHECK(lease);
|
|
auto head = load_region_checkpoint(*r.runtime, r.key, 5s);
|
|
CHECK(head);
|
|
auto state = evo::test::region_community();
|
|
EnvBatch batch{*state};
|
|
run_batch(batch, 8);
|
|
auto first = publish_environmental_batch(*r.runtime, *r.objects, *lease, *head, batch,
|
|
filled<16>(0xD1), build, budget, 5s);
|
|
CHECK(first && batch.confirm(8));
|
|
const auto manifest = *decode_manifest(first->committed.manifest);
|
|
const auto object = r.objects->directory() / hex(manifest.payload_sha256);
|
|
|
|
// Upload failure: no manifest is sent; the head does not move.
|
|
run_batch(batch, 8);
|
|
if (::geteuid() != 0) {
|
|
fs::permissions(r.objects->directory(), fs::perms::owner_read | fs::perms::owner_exec,
|
|
fs::perm_options::replace);
|
|
auto upload = publish_environmental_batch(*r.runtime, *r.objects, *lease, first->committed,
|
|
batch, filled<16>(0xD2), build, budget, 5s);
|
|
fs::permissions(r.objects->directory(), fs::perms::owner_all, fs::perm_options::replace);
|
|
CHECK(!upload && upload.error().kind == PublicationError::upload);
|
|
CHECK(load_region_head(*r.runtime, r.key, 5s)->sequence == 1);
|
|
}
|
|
|
|
// Manifest transaction failure after verified uploads: fenced writer, orphans retained.
|
|
auto stale = *lease;
|
|
stale.holder[0] ^= 1U;
|
|
auto fenced = publish_environmental_batch(*r.runtime, *r.objects, stale, first->committed,
|
|
batch, filled<16>(0xD3), build, budget, 5s);
|
|
CHECK(!fenced && fenced.error().kind == PublicationError::rejected &&
|
|
fenced.error().store.kind == LeaseStoreError::fenced);
|
|
CHECK(fenced.error().prepared.commit == filled<16>(0xD3));
|
|
auto orphan_bytes = state->snapshot();
|
|
auto orphan = r.objects->get({*sha256(*orphan_bytes), orphan_bytes->size()});
|
|
CHECK(orphan && *orphan == *orphan_bytes); // Uploaded, verified, unreferenced.
|
|
auto recovered =
|
|
recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s);
|
|
CHECK(recovered && recovered->state->progress().tick == 8); // Never the orphan.
|
|
|
|
// Missing and corrupt committed payloads quarantine; no older head is selected.
|
|
const auto saved = r.objects->get({manifest.payload_sha256, manifest.payload_bytes});
|
|
CHECK(saved);
|
|
fs::remove(object);
|
|
auto missing =
|
|
recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s);
|
|
CHECK(!missing && missing.error().kind == RecoveryError::missing_payload);
|
|
// Staging replays from the committed predecessor bytes; without them it stops.
|
|
auto no_base = publish_environmental_batch(*r.runtime, *r.objects, *lease, first->committed,
|
|
batch, filled<16>(0xD4), build, budget, 5s);
|
|
CHECK(!no_base && no_base.error().kind == PublicationError::predecessor &&
|
|
no_base.error().object == ObjectError::missing);
|
|
CHECK(r.objects->put(*saved));
|
|
fs::permissions(object, fs::perms::owner_all, fs::perm_options::replace);
|
|
{
|
|
std::fstream f(object, std::ios::in | std::ios::out | std::ios::binary);
|
|
f.seekp(100);
|
|
f.put(static_cast<char>((*saved)[100] ^ 1U));
|
|
}
|
|
auto corrupt =
|
|
recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s);
|
|
CHECK(!corrupt && corrupt.error().kind == RecoveryError::corrupt_payload);
|
|
fs::remove(object);
|
|
CHECK(r.objects->put(*saved));
|
|
CHECK(recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s));
|
|
|
|
// Committed history is immutable even to the owner (trigger), and the head
|
|
// cannot be repointed (composite foreign key).
|
|
auto tamper = r.owner->query("UPDATE evolution.region_commits SET manifest = "
|
|
"overlay(manifest placing '\\x02'::bytea from 121 for 1) "
|
|
"WHERE world_id = 7 AND region_id = 4 AND commit_sequence = 1",
|
|
{}, 5s);
|
|
CHECK(!tamper && std::string(tamper.error().sqlstate.data()) == "EVL09");
|
|
auto erase = r.owner->query("DELETE FROM evolution.region_commits WHERE world_id = 7 AND "
|
|
"region_id = 4 AND commit_sequence = 1",
|
|
{}, 5s);
|
|
CHECK(!erase && std::string(erase.error().sqlstate.data()) == "EVL09");
|
|
auto repoint = r.owner->query("UPDATE evolution.region_heads SET digest = sha256('\\x00') "
|
|
"WHERE world_id = 7 AND region_id = 4",
|
|
{}, 5s);
|
|
CHECK(!repoint && std::string(repoint.error().sqlstate.data()) == "23503");
|
|
// Defense in depth: even with the trigger and digest constraint removed, recovery
|
|
// re-digests the manifest and quarantines. Both guards are restored afterwards.
|
|
CHECK(r.owner->query("ALTER TABLE evolution.region_commits DISABLE TRIGGER "
|
|
"region_commits_immutable",
|
|
{}, 5s));
|
|
CHECK(r.owner->query("ALTER TABLE evolution.region_commits DROP CONSTRAINT "
|
|
"region_commits_manifest_digest",
|
|
{}, 5s));
|
|
// Flip a payload-digest byte: only the manifest digest check can notice it.
|
|
CHECK(r.owner->query("UPDATE evolution.region_commits SET manifest = "
|
|
"set_byte(manifest, 248, get_byte(manifest, 248) # 1) "
|
|
"WHERE world_id = 7 AND region_id = 4 AND commit_sequence = 1",
|
|
{}, 5s));
|
|
auto tampered =
|
|
recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s);
|
|
CHECK(!tampered && tampered.error().kind == RecoveryError::manifest);
|
|
const std::array<std::string, 1> original{"\\x" + hex(first->committed.manifest)};
|
|
CHECK(r.owner->query("UPDATE evolution.region_commits SET manifest = $1::bytea "
|
|
"WHERE world_id = 7 AND region_id = 4 AND commit_sequence = 1",
|
|
original, 5s));
|
|
CHECK(r.owner->query("ALTER TABLE evolution.region_commits ADD CONSTRAINT "
|
|
"region_commits_manifest_digest CHECK (commit_digest = sha256(manifest))",
|
|
{}, 5s));
|
|
CHECK(r.owner->query("ALTER TABLE evolution.region_commits ENABLE TRIGGER "
|
|
"region_commits_immutable",
|
|
{}, 5s));
|
|
CHECK(recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s));
|
|
|
|
// Recovery does not create genesis for an unknown region.
|
|
auto none =
|
|
recover_environmental_checkpoint(*r.runtime, *r.objects, {7, 99}, retained, budget, 5s);
|
|
CHECK(!none && none.error().kind == RecoveryError::not_provisioned);
|
|
}
|
|
|
|
// History-capable profile: deaths commit with the state that caused them, the
|
|
// archive survives recovery, and a region never switches profile.
|
|
void lineage_publish_and_recover() {
|
|
const RegionKey key{7, 6};
|
|
auto owner = connect_as(env("PGUSER"));
|
|
auto runtime = connect_as(env("EVO_PG_RUNTIME_USER"));
|
|
auto objects = FilesystemObjectStore::open(scratch("lineage"), key, max_lineage_payload_bytes);
|
|
CHECK(objects);
|
|
CHECK(owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
|
|
"(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING",
|
|
{}, 5s));
|
|
auto opening = evo::test::lineage_region();
|
|
auto genesis =
|
|
provision_lineage_region(*owner, **objects, key, evo::test::region_identity.frame, *opening,
|
|
filled<16>(0xE0), build, budget, 5s);
|
|
CHECK(genesis);
|
|
CHECK(decode_manifest(genesis->manifest)->profile ==
|
|
CheckpointProfile::environmental_lineage_v1);
|
|
auto lease = acquire_region_lease(*runtime, key, build, 20000ms, 5s);
|
|
CHECK(lease);
|
|
|
|
auto region = evo::test::lineage_region();
|
|
LineageBatch batch{*region};
|
|
run_batch(batch, 64, true); // The consumer's death at tick 20 seals the batch there.
|
|
CHECK(batch.end_tick() == 20);
|
|
auto first = publish_lineage_batch(*runtime, **objects, *lease, *genesis, batch,
|
|
filled<16>(0xE1), build, budget, 5s);
|
|
CHECK(first && first->committed.head.end_tick == 20);
|
|
CHECK(first->decisions.size() == 1 &&
|
|
first->decisions[0] == BatchDecision{20, Actor::consumer, Outcome::death, 11});
|
|
CHECK(batch.confirm(20));
|
|
// The environmental path refuses a lineage predecessor before uploading.
|
|
auto plain = evo::test::region_community();
|
|
EnvBatch plain_batch{*plain};
|
|
run_batch(plain_batch, 30);
|
|
auto crossed = publish_environmental_batch(*runtime, **objects, *lease, first->committed,
|
|
plain_batch, filled<16>(0xE2), build, budget, 5s);
|
|
CHECK(!crossed && crossed.error().kind == PublicationError::candidate);
|
|
// The database refuses a profile switch even when the manifest is well formed.
|
|
auto switched = *decode_manifest(first->committed.manifest);
|
|
auto next = successor_manifest(switched, first->committed.head.digest, filled<16>(0xE3), *lease,
|
|
30, build, 100, filled<32>(0xA2),
|
|
{trace_bytes_for(10), filled<32>(0x7C), 0, {}});
|
|
CHECK(next);
|
|
next->profile = CheckpointProfile::environmental_v1;
|
|
next->history_profile = 0;
|
|
auto switched_bytes = encode_manifest(*next);
|
|
CHECK(switched_bytes);
|
|
auto refused = publish_region_checkpoint(*runtime, *switched_bytes, {}, 5s);
|
|
CHECK(!refused && refused.error().kind == LeaseStoreError::unsupported);
|
|
|
|
const std::array<Sha256Digest, 1> retained{build};
|
|
auto recovered = recover_lineage_checkpoint(*runtime, **objects, key, retained, budget, 5s);
|
|
CHECK(recovered && recovered->committed.head == first->committed.head);
|
|
CHECK(recovered->decisions.size() == 1 && recovered->decisions[0].commit_sequence == 1);
|
|
const auto consumer = recovered->state->archive().organism(11);
|
|
CHECK(consumer && !consumer->alive && consumer->died.value == 20);
|
|
CHECK(recovered->state->archive().ancestry(2)->size() == 2); // Retained ancestry.
|
|
CHECK(recovered->state->snapshot() == region->snapshot());
|
|
LineageBatch resumed{*recovered->state};
|
|
for (std::uint64_t tick = 21; tick <= 60; ++tick) {
|
|
auto a = region->advance(tick, evo::test::region_input(tick));
|
|
auto b = resumed.advance(evo::test::region_input(tick));
|
|
CHECK(a && b && *a == *b);
|
|
}
|
|
CHECK(recovered->state->snapshot() == region->snapshot());
|
|
resumed.seal();
|
|
auto second = publish_lineage_batch(*runtime, **objects, *lease, recovered->committed, resumed,
|
|
filled<16>(0xE4), build, budget, 5s);
|
|
CHECK(second && second->committed.head.sequence == 2 && second->committed.head.end_tick == 60 &&
|
|
second->decisions.empty());
|
|
// A host restoring the other profile refuses rather than misreading the payload.
|
|
auto wrong = recover_environmental_checkpoint(*runtime, **objects, key, retained, budget, 5s);
|
|
CHECK(!wrong && wrong.error().kind == RecoveryError::unsupported);
|
|
CHECK(release_region_lease(*runtime, key, *lease, 5s));
|
|
}
|
|
|
|
std::string runtime_user() {
|
|
return env("EVO_PG_RUNTIME_USER");
|
|
}
|
|
// Polls resolution on a fresh connection until the prepared commit is committed.
|
|
ResolvedCommit await_resolution(const PreparedCommit& prepared, Resolution wanted) {
|
|
auto db = connect_as(runtime_user());
|
|
for (int attempt = 0; attempt < 200; ++attempt) {
|
|
auto resolved = resolve_prepared(*db, prepared, 5s);
|
|
CHECK(resolved);
|
|
if (resolved->status == wanted)
|
|
return *resolved;
|
|
std::this_thread::sleep_for(25ms);
|
|
}
|
|
CHECK(false);
|
|
return {};
|
|
}
|
|
// Holds the lease row lock so a publication's outcome becomes ambiguous to its client.
|
|
struct LeaseRowLock {
|
|
std::unique_ptr<Postgres> db;
|
|
LeaseRowLock(RegionKey key) : db(connect_as(env("PGUSER"))) {
|
|
const std::array<std::string, 2> p{std::to_string(key.world), std::to_string(key.region)};
|
|
CHECK(db->query("BEGIN", {}, 5s));
|
|
CHECK(db->query("SELECT 1 FROM evolution.region_leases WHERE world_id = $1::evolution.u64 "
|
|
"AND region_id = $2::evolution.u64 FOR UPDATE",
|
|
p, 5s));
|
|
}
|
|
// Terminates backends of the runtime login queued behind this lock.
|
|
void kill_waiters() {
|
|
const std::array<std::string, 1> p{runtime_user()};
|
|
CHECK(db->query("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE "
|
|
"usename = $1 AND wait_event_type = 'Lock'",
|
|
p, 5s));
|
|
}
|
|
void release() {
|
|
CHECK(db->query("ROLLBACK", {}, 5s));
|
|
}
|
|
};
|
|
|
|
// M7.5 durability: permanent outcomes commit atomically with their checkpoint,
|
|
// ambiguity is resolved by exact bytes, and nothing is ever applied twice.
|
|
void durable_outcomes_and_ambiguity() {
|
|
auto r = provisioned("outcomes", 7, 0xF0);
|
|
const std::array<Sha256Digest, 1> retained{build};
|
|
auto lease = acquire_region_lease(*r.runtime, r.key, build, 20000ms, 5s);
|
|
CHECK(lease);
|
|
auto genesis = load_region_checkpoint(*r.runtime, r.key, 5s);
|
|
CHECK(genesis);
|
|
auto state = evo::test::region_community();
|
|
EnvBatch batch{*state};
|
|
run_batch(batch, 64, true); // Sealed by the consumer's death at tick 20.
|
|
CHECK(batch.end_tick() == 20);
|
|
auto died = publish_environmental_batch(*r.runtime, *r.objects, *lease, *genesis, batch,
|
|
filled<16>(0xF1), build, budget, 5s);
|
|
CHECK(died && died->decisions.size() == 1 &&
|
|
died->decisions[0] == BatchDecision{20, Actor::consumer, Outcome::death, 0});
|
|
CHECK(batch.confirm(20));
|
|
const CommittedDecision final_death{1, died->decisions[0]};
|
|
auto decisions = load_region_decisions(*r.runtime, r.key, 5s);
|
|
CHECK(decisions && decisions->size() == 1 && (*decisions)[0] == final_death);
|
|
|
|
// Outbox: at-least-once delivery, idempotent acknowledgement, no simulation effect.
|
|
auto pending = pending_region_outbox(*r.runtime, r.key, 64, 5s);
|
|
CHECK(pending && pending->size() == 1 && (*pending)[0] == final_death);
|
|
CHECK(pending_region_outbox(*r.runtime, r.key, 64, 5s)->size() == 1); // Redelivered.
|
|
auto ack = acknowledge_region_outbox(*r.runtime, r.key, died->decisions[0], 5s);
|
|
CHECK(ack && *ack);
|
|
ack = acknowledge_region_outbox(*r.runtime, r.key, died->decisions[0], 5s);
|
|
CHECK(ack && !*ack);
|
|
CHECK(pending_region_outbox(*r.runtime, r.key, 64, 5s)->empty());
|
|
CHECK(!pending_region_outbox(*r.runtime, r.key, 65, 5s));
|
|
const BatchDecision never{21, Actor::producer, Outcome::death, 0};
|
|
CHECK(!acknowledge_region_outbox(*r.runtime, r.key, never, 5s));
|
|
|
|
// Commit succeeded but the response was lost: the exact retry returns the same
|
|
// receipt and records nothing twice; resolution confirms it under any epoch.
|
|
auto death_block = encode_decisions(died->decisions, 0, 20);
|
|
CHECK(death_block);
|
|
const PreparedCommit lost{r.key, filled<16>(0xF1), died->committed.manifest, *death_block};
|
|
auto retried = retry_prepared(*r.runtime, lost, 5s);
|
|
CHECK(retried && *retried == died->committed.head);
|
|
auto resolved = resolve_prepared(*r.runtime, lost, 5s);
|
|
CHECK(resolved && resolved->status == Resolution::committed &&
|
|
resolved->receipt == died->committed.head);
|
|
CHECK(load_region_decisions(*r.runtime, r.key, 5s)->size() == 1);
|
|
// Changed bytes under the same ID conflict, whether manifest or decisions differ.
|
|
auto changed = lost;
|
|
changed.manifest[288] ^= 1U; // Trace digest.
|
|
CHECK(retry_prepared(*r.runtime, changed, 5s).error().kind == LeaseStoreError::commit_conflict);
|
|
CHECK(resolve_prepared(*r.runtime, changed, 5s)->status == Resolution::conflict);
|
|
auto undecided = lost;
|
|
undecided.decisions.clear(); // No longer what the manifest digests.
|
|
CHECK(retry_prepared(*r.runtime, undecided, 5s).error().kind == LeaseStoreError::invalid);
|
|
CHECK(resolve_prepared(*r.runtime, undecided, 5s)->status == Resolution::conflict);
|
|
auto elsewhere = lost;
|
|
elsewhere.key.region = 3;
|
|
CHECK(resolve_prepared(*r.runtime, elsewhere, 5s)->status == Resolution::conflict);
|
|
auto unknown_id = lost;
|
|
unknown_id.commit = filled<16>(0xFE);
|
|
CHECK(resolve_prepared(*r.runtime, unknown_id, 5s)->status == Resolution::absent);
|
|
|
|
// The committed dead stay dead: a later batch cannot finalize the death again.
|
|
const auto head = died->committed;
|
|
const std::array again{BatchDecision{25, Actor::consumer, Outcome::death, 0}};
|
|
auto again_bytes = encode_decisions(again, 20, 30);
|
|
CHECK(again_bytes);
|
|
auto relapse = successor_manifest(
|
|
*decode_manifest(head.manifest), head.head.digest, filled<16>(0xF2), *lease, 30, build, 100,
|
|
filled<32>(0xA2), {trace_bytes_for(10), filled<32>(0x7C), 1, *sha256(*again_bytes)});
|
|
CHECK(relapse);
|
|
auto relapse_bytes = encode_manifest(*relapse);
|
|
auto repeated = publish_region_checkpoint(*r.runtime, *relapse_bytes, *again_bytes, 5s);
|
|
CHECK(!repeated && repeated.error().kind == LeaseStoreError::outcome_final);
|
|
// A block of the right size that is not the one the manifest digests is refused.
|
|
auto forged_block = *again_bytes;
|
|
forged_block[0] ^= 1U; // Tick 25 -> 24: still well formed, different bytes.
|
|
CHECK(publish_region_checkpoint(*r.runtime, *relapse_bytes, forged_block, 5s).error().kind ==
|
|
LeaseStoreError::invalid);
|
|
// Partial or reordered decision blocks are refused whole; nothing is recorded.
|
|
CHECK(publish_region_checkpoint(*r.runtime, *relapse_bytes, {}, 5s).error().kind ==
|
|
LeaseStoreError::invalid);
|
|
const std::array two{BatchDecision{25, Actor::producer, Outcome::death, 0},
|
|
BatchDecision{24, Actor::consumer, Outcome::death, 0}};
|
|
std::vector<std::uint8_t> reordered(two.size() * decision_bytes);
|
|
auto ordered = encode_decisions(std::array{two[1], two[0]}, 20, 30);
|
|
CHECK(ordered);
|
|
std::copy_n(ordered->begin() + 24, 24, reordered.begin());
|
|
std::copy_n(ordered->begin(), 24, reordered.begin() + 24);
|
|
auto reorder = *relapse;
|
|
reorder.batch = {trace_bytes_for(10), filled<32>(0x7C), 2, *sha256(reordered)};
|
|
CHECK(publish_region_checkpoint(*r.runtime, *encode_manifest(reorder), reordered, 5s)
|
|
.error()
|
|
.kind == LeaseStoreError::invalid);
|
|
// A gap: a link from an older predecessor is not the committed head's successor.
|
|
auto gap = successor_manifest(*decode_manifest(genesis->manifest), genesis->head.digest,
|
|
filled<16>(0xF3), *lease, 30, build, 100, filled<32>(0xA2),
|
|
{trace_bytes_for(30), filled<32>(0x7C), 0, {}});
|
|
CHECK(publish_region_checkpoint(*r.runtime, *encode_manifest(*gap), {}, 5s).error().kind ==
|
|
LeaseStoreError::head_mismatch);
|
|
CHECK(load_region_head(*r.runtime, r.key, 5s)->sequence == 1);
|
|
CHECK(load_region_decisions(*r.runtime, r.key, 5s)->size() == 1);
|
|
|
|
// Keep authority comfortably valid for the ambiguity cases (sanitizer builds are slow).
|
|
CHECK(renew_region_lease(*r.runtime, r.key, *lease, 20000ms, 5s));
|
|
// Ambiguous outcome that did commit: the client times out while queued, the
|
|
// server finishes after the lock is released. Resolution adopts the receipt.
|
|
run_batch(batch, 10);
|
|
std::optional<LeaseRowLock> lock{std::in_place, r.key};
|
|
auto ambiguous_db = connect_as(runtime_user());
|
|
auto ambiguous = publish_environmental_batch(*ambiguous_db, *r.objects, *lease, head, batch,
|
|
filled<16>(0xF4), build, budget, 300ms);
|
|
CHECK(!ambiguous && ambiguous.error().kind == PublicationError::unknown);
|
|
CHECK(batch.sealed()); // Still paused: never assume rollback.
|
|
lock->release();
|
|
lock.reset();
|
|
const auto committed = await_resolution(ambiguous.error().prepared, Resolution::committed);
|
|
CHECK(committed.receipt.sequence == 2 && committed.receipt.end_tick == 30);
|
|
auto adopted = retry_prepared(*r.runtime, ambiguous.error().prepared, 5s);
|
|
CHECK(adopted && *adopted == committed.receipt);
|
|
CHECK(batch.confirm(30));
|
|
|
|
// Ambiguous outcome that did not commit: the orphaned request is terminated
|
|
// before it reaches the row. Absent, so the same bytes are retried while authorized.
|
|
auto after = load_region_checkpoint(*r.runtime, r.key, 5s);
|
|
CHECK(after);
|
|
run_batch(batch, 5);
|
|
lock.emplace(r.key);
|
|
auto orphan_db = connect_as(runtime_user());
|
|
auto orphaned = publish_environmental_batch(*orphan_db, *r.objects, *lease, *after, batch,
|
|
filled<16>(0xF5), build, budget, 300ms);
|
|
CHECK(!orphaned && orphaned.error().kind == PublicationError::unknown);
|
|
lock->kill_waiters();
|
|
lock->release();
|
|
lock.reset();
|
|
std::this_thread::sleep_for(200ms);
|
|
CHECK(resolve_prepared(*r.runtime, orphaned.error().prepared, 5s)->status ==
|
|
Resolution::absent);
|
|
auto recommitted = retry_prepared(*r.runtime, orphaned.error().prepared, 5s);
|
|
CHECK(recommitted && recommitted->sequence == 3 && recommitted->end_tick == 35);
|
|
CHECK(batch.confirm(35));
|
|
|
|
// Recovery reconciles durable outcomes with the restored lifecycle.
|
|
auto recovered =
|
|
recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s);
|
|
CHECK(recovered && recovered->decisions.size() == 1 && recovered->decisions[0] == final_death &&
|
|
recovered->state->progress().consumer_terminal == 1);
|
|
CHECK(recovered->state->snapshot() == state->snapshot());
|
|
// A durable death the state does not show is a history disagreement: quarantine.
|
|
CHECK(r.owner->query("INSERT INTO evolution.region_decisions(world_id, region_id, outcome, "
|
|
"actor, tick, organism, commit_sequence) VALUES (7, 7, 1, 0, 30, 0, 2)",
|
|
{}, 5s));
|
|
auto disagreed =
|
|
recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s);
|
|
CHECK(!disagreed && disagreed.error().kind == RecoveryError::history);
|
|
auto undo = r.owner->query("DELETE FROM evolution.region_decisions WHERE world_id = 7 AND "
|
|
"region_id = 7 AND actor = 0",
|
|
{}, 5s);
|
|
CHECK(!undo && std::string(undo.error().sqlstate.data()) == "EVL09"); // Stays quarantined.
|
|
CHECK(release_region_lease(*r.runtime, r.key, *lease, 5s));
|
|
}
|
|
// Publication cannot acknowledge a checkpoint whose objects are not yet durable.
|
|
void publication_waits_for_directory_durability() {
|
|
auto r = provisioned("sync-publish", 9, 0xA8);
|
|
ObjectStoreFaults faults;
|
|
auto faulty = FilesystemObjectStore::open(r.objects->directory().parent_path().parent_path(),
|
|
r.key, max_environmental_payload_bytes, &faults);
|
|
CHECK(faulty);
|
|
auto lease = acquire_region_lease(*r.runtime, r.key, build, 20000ms, 5s);
|
|
CHECK(lease);
|
|
auto genesis = load_region_checkpoint(*r.runtime, r.key, 5s);
|
|
CHECK(genesis);
|
|
auto state = evo::test::region_community();
|
|
EnvBatch batch{*state};
|
|
run_batch(batch, 8);
|
|
for (const std::uint32_t failures : {1U, 1U, 2U}) {
|
|
faults.fail_directory_syncs = failures;
|
|
auto attempt = publish_environmental_batch(*r.runtime, **faulty, *lease, *genesis, batch,
|
|
filled<16>(0xA9), build, budget, 5s);
|
|
CHECK(!attempt && attempt.error().kind == PublicationError::upload &&
|
|
attempt.error().object == ObjectError::io);
|
|
CHECK(load_region_head(*r.runtime, r.key, 5s)->sequence == 0); // Nothing sent.
|
|
}
|
|
faults.fail_directory_syncs = 0;
|
|
auto published = publish_environmental_batch(*r.runtime, **faulty, *lease, *genesis, batch,
|
|
filled<16>(0xA9), build, budget, 5s);
|
|
CHECK(published && published->committed.head.sequence == 1);
|
|
CHECK(release_region_lease(*r.runtime, r.key, *lease, 5s));
|
|
}
|
|
|
|
// Review P1: a committed death with no durable decision (what an accepted pre-v5
|
|
// upgrade would leave, or a host that skipped derivation) is never served as a
|
|
// recovered region. The database cannot see payload deaths; recovery must.
|
|
void undecided_death_is_quarantined() {
|
|
auto r = provisioned("undecided", 10, 0xB6);
|
|
auto lease = acquire_region_lease(*r.runtime, r.key, build, 20000ms, 5s);
|
|
CHECK(lease);
|
|
auto genesis = load_region_checkpoint(*r.runtime, r.key, 5s);
|
|
CHECK(genesis);
|
|
auto state = evo::test::region_community();
|
|
EnvBatch batch{*state};
|
|
run_batch(batch, 64, true); // The consumer dies at tick 20.
|
|
CHECK(batch.end_tick() == 20 && state->progress().consumer_terminal == 1);
|
|
const auto previous = *decode_manifest(genesis->manifest);
|
|
auto payload = state->snapshot();
|
|
auto trace =
|
|
encode_trace({1, r.key, previous.commit, genesis->head.digest, 0, 20}, batch.steps());
|
|
CHECK(payload && trace && r.objects->put(*payload) && r.objects->put(*trace));
|
|
auto manifest = successor_manifest(previous, genesis->head.digest, filled<16>(0xB7), *lease, 20,
|
|
build, payload->size(), *sha256(*payload),
|
|
{trace->size(), *sha256(*trace), 0, {}});
|
|
CHECK(manifest);
|
|
auto bytes = encode_manifest(*manifest);
|
|
CHECK(bytes && publish_region_checkpoint(*r.runtime, *bytes, {}, 5s));
|
|
const std::array<Sha256Digest, 1> retained{build};
|
|
auto recovered =
|
|
recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s);
|
|
CHECK(!recovered && recovered.error().kind == RecoveryError::history);
|
|
CHECK(release_region_lease(*r.runtime, r.key, *lease, 5s));
|
|
}
|
|
|
|
} // namespace
|
|
|
|
void run_region_checkpoint_store_tests() {
|
|
sha256_vectors_and_database_agreement();
|
|
object_store_is_immutable_and_verified();
|
|
directory_sync_failures_are_never_acknowledged();
|
|
publish_recover_and_continue();
|
|
failures_leave_the_previous_head();
|
|
lineage_publish_and_recover();
|
|
durable_outcomes_and_ambiguity();
|
|
publication_waits_for_directory_durability();
|
|
undecided_death_is_quarantined();
|
|
}
|