Files
aevolution/tests/region_clone_tests.cpp
T
dingoandClaude Opus 5.5 703f67834f Complete M7.9a region clones and retained-history audit
Clone a verified committed head into an isolated world and database
(ADR-0067): world re-identification with a proven byte round trip, a
schema v7 clone genesis naming the source commit and digest with digested
inherited outcomes, a read-only source session, an owner-only atomic
target transaction and the evo-world-clone tool. audit_region_history
re-verifies every retained link. Real-process tests cover trajectory
equality after the cut, source isolation, killed clones and retention.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-29 06:47:45 -04:00

482 lines
25 KiB
C++

// M7.9a region clones and retained history (ADR-0067) against two real PostgreSQL
// databases: the source (PGDATABASE, runtime EVO_PG_RUNTIME_USER) and an isolated
// clone database (EVO_PG_CLONE_DATABASE, its own runtime EVO_PG_CLONE_RUNTIME_USER).
// Real evo-server and evo-world-clone processes. Run by tools/ci/postgres-check.sh.
#include "evo/persistence/region_clone.hpp"
#include "evo/persistence/sha256.hpp"
#include "fixtures/server_process.hpp"
#include <algorithm>
#include <fstream>
#include <source_location>
#include <thread>
void run_region_clone_tests();
namespace {
using namespace evo::test::process;
using namespace evo::persistence;
void check(bool value, std::source_location at = std::source_location::current()) {
if (!value)
throw std::runtime_error("region clone test line " + std::to_string(at.line()));
}
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
constexpr std::size_t budget = 8000000;
const char* const build_hex = "b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1";
Sha256Digest build() {
Sha256Digest b{};
b.fill(0xB1);
return b;
}
template <std::size_t N> std::array<std::uint8_t, N> filled(std::uint8_t value) {
std::array<std::uint8_t, N> out{};
out.fill(value);
return out;
}
std::string hex(std::span<const std::uint8_t> bytes) {
constexpr char digits[] = "0123456789abcdef";
std::string out;
for (const auto b : bytes) {
out.push_back(digits[b >> 4U]);
out.push_back(digits[b & 0x0FU]);
}
return out;
}
std::unique_ptr<Postgres> connect_to(const std::string& database, const std::string& user) {
auto made = Postgres::connect({env("PGHOST"), {}, env("PGPORT"), database, user, {}, {}}, 5s);
CHECK(made);
return std::move(*made);
}
std::string source_db() {
return env("PGDATABASE");
}
std::string clone_db() {
return env("EVO_PG_CLONE_DATABASE");
}
std::string sqlstate(const PgFailure& f) {
return f.sqlstate.data();
}
std::unique_ptr<FilesystemObjectStore> store(const ServerRegion& r) {
auto made = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes);
CHECK(made);
return std::move(*made);
}
// An empty object root for a clone region.
ServerRegion clone_target(RegionKey key, const std::string& name) {
ServerRegion r{key, fs::path(env("PGHOST")) / name};
fs::remove_all(r.root);
fs::create_directory(r.root);
fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace);
return r;
}
void retain_in_clone_database() {
CHECK(connect_to(clone_db(), env("PGUSER"))
->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
"(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING",
{}, 5s));
}
std::vector<std::string> with(std::vector<std::string> args, const std::string& flag,
const std::string& value) {
const auto at = std::ranges::find(args, flag);
CHECK(at != args.end() && at + 1 != args.end());
*(at + 1) = value;
return args;
}
// evo-server for a region of the clone database, with its own runtime login.
std::vector<std::string> clone_server_args(const ServerRegion& r) {
return with(with(server_args(r), "--pg-database", clone_db()), "--pg-user",
env("EVO_PG_CLONE_RUNTIME_USER"));
}
std::vector<std::string> clone_args(const ServerRegion& from, const ServerRegion& to,
std::vector<std::string> extra = {}) {
std::vector<std::string> args{env("EVO_WORLD_CLONE_BIN"),
"clone",
"--source-world",
std::to_string(from.key.world),
"--source-region",
std::to_string(from.key.region),
"--target-world",
std::to_string(to.key.world),
"--target-region",
std::to_string(to.key.region),
"--source-pg-host",
env("PGHOST"),
"--source-pg-port",
env("PGPORT"),
"--source-pg-database",
source_db(),
"--source-pg-user",
env("EVO_PG_RUNTIME_USER"),
"--target-pg-host",
env("PGHOST"),
"--target-pg-port",
env("PGPORT"),
"--target-pg-database",
clone_db(),
"--target-pg-user",
env("PGUSER"),
"--source-objects",
from.root.string(),
"--target-objects",
to.root.string(),
"--retained",
build_hex,
"--audit-max",
"1000"};
args.insert(args.end(), extra.begin(), extra.end());
return args;
}
int run_tool(const std::vector<std::string>& args, std::vector<std::string>* lines = nullptr) {
Child tool{args};
const auto code = tool.wait_exit(60s);
if (lines != nullptr)
*lines = tool.lines();
return code;
}
// Runs a fresh owner process over inputs first..last (consumer death at 20) and drains.
void run_inputs(const std::vector<std::string>& args, std::uint64_t first, std::uint64_t last) {
Child server{args};
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
server.send(input_lines(first, last, true));
CHECK(server.wait_line("ack " + request_hex(last) + " " + std::to_string(last) + " ", 30s));
server.signal(SIGTERM);
CHECK(server.wait_exit(20s) == 0);
}
std::optional<RegionHead> maybe_head(const std::string& database, RegionKey key) {
auto committed = load_region_checkpoint(*connect_to(database, env("PGUSER")), key, 5s);
if (!committed)
return std::nullopt;
return committed->head;
}
RegionHead head_in(const std::string& database, RegionKey key) {
const auto head = maybe_head(database, key);
CHECK(head.has_value());
return head.value_or(RegionHead{});
}
std::unique_ptr<evo::processes::LineageRegion>
recovered_state(const std::string& database, const std::string& user, const ServerRegion& r) {
const std::array<Sha256Digest, 1> retained{build()};
auto recovered = recover_lineage_checkpoint(*connect_to(database, user), *store(r), r.key,
retained, budget, 5s);
CHECK(recovered);
return std::move(recovered->state);
}
// Consistent clone into an isolated world and database; the clone continues exactly
// as its source does, and inherits its outcomes and ancestry.
void clone_matches_the_source_trajectory() {
retain_in_clone_database();
const auto source = provision(71, 0xE8);
run_inputs(server_args(source), 1, 30); // Consumer dies at tick 20.
const auto cut = head_in(source_db(), source.key);
CHECK(cut.end_tick == 30 && cut.sequence >= 2);
const auto target = clone_target({8, 71}, "clone-71");
std::vector<std::string> out;
CHECK(run_tool(clone_args(source, target), &out) == 0);
CHECK(out.size() == 1 &&
out[0].starts_with("cloned source=7/71 sequence=" + std::to_string(cut.sequence) +
" tick=30 target=8/71 outcomes=1 digest="));
// The clone is a verified region of world 8, recovered with the clone runtime.
const auto clone_user = env("EVO_PG_CLONE_RUNTIME_USER");
auto cloned = recovered_state(clone_db(), clone_user, target);
CHECK(cloned->community().world() == 8 && cloned->community().progress().tick == 30);
const auto consumer = cloned->archive().organism(11);
CHECK(consumer && !consumer->alive && consumer->died.value == 20);
CHECK(cloned->archive().ancestry(2)->size() == 2); // Retained ancestry.
{
const std::array<Sha256Digest, 1> retained{build()};
auto clone_reader = connect_to(clone_db(), clone_user);
auto decisions = load_region_decisions(*clone_reader, target.key, 5s);
CHECK(decisions && decisions->size() == 1 && (*decisions)[0].commit_sequence == 0 &&
(*decisions)[0].decision == BatchDecision{20, Actor::consumer, Outcome::death, 11});
auto audited = audit_region_history(*clone_reader, *store(target), target.key, retained,
budget, 16, 5s);
CHECK(audited && audited->commits == 1 && audited->clone && audited->end_tick == 30 &&
audited->outcomes == 1 && audited->origin_commit == cut.commit &&
audited->origin_digest == cut.digest);
// No outbox rows: the source already delivered those outcomes.
auto pending = pending_region_outbox(*clone_reader, target.key, 64, 5s);
CHECK(pending && pending->empty());
}
// Baseline versus clone: the same inputs from the cut give the same world.
run_inputs(server_args(source), 31, 90);
run_inputs(clone_server_args(target), 31, 90);
auto baseline = recovered_state(source_db(), env("EVO_PG_RUNTIME_USER"), source);
auto restored = recovered_state(clone_db(), clone_user, target);
CHECK(baseline->community().progress().tick == 90 &&
restored->community().progress().tick == 90);
CHECK(restored->rebind_world(7));
CHECK(restored->snapshot() == baseline->snapshot());
// The clone chain continues from its own genesis, which names the source cut.
const auto clone_head = head_in(clone_db(), target.key);
CHECK(clone_head.sequence >= 1 && clone_head.end_tick == 90);
}
// A clone has no path to write its source: read-only source session, separate
// credentials and database, a different world identity and immutable origins.
void clone_cannot_write_its_source() {
const auto source = provision(72, 0xE9);
run_inputs(server_args(source), 1, 25);
const auto before = head_in(source_db(), source.key);
CHECK(before.end_tick == 25);
const std::array<Sha256Digest, 1> retained{build()};
auto reader = connect_to(source_db(), env("EVO_PG_RUNTIME_USER"));
auto owner_db = connect_to(source_db(), env("PGUSER"));
auto source_objects = store(source);
const auto same_db = clone_target({10, 72}, "clone-72");
CHECK(clone_region(*reader, *source_objects, source.key, *owner_db, *store(same_db), {7, 99},
filled<16>(0xEC), retained, budget, 5s)
.error()
.kind == CloneError::identity);
// A world that already holds an original region is not an isolated clone world.
{
const RegionKey original{9, 1};
auto region = evo::test::lineage_region();
CHECK(region->rebind_world(9));
const auto root = clone_target(original, "original-9");
CHECK(provision_lineage_region(*owner_db, *store(root), original,
evo::test::region_identity.frame, *region, filled<16>(0xF8),
build(), budget, 5s));
const auto mixed = clone_target({9, 72}, "mixed-9");
auto refused = clone_region(*reader, *source_objects, source.key, *owner_db, *store(mixed),
mixed.key, filled<16>(0xED), retained, budget, 5s);
CHECK(!refused && refused.error().kind == CloneError::rejected &&
sqlstate(refused.error().store.cause) == "22023");
CHECK(!maybe_head(source_db(), mixed.key));
}
// Same-database clone into a fresh world: the database also checks the origin.
auto cloned = clone_region(*reader, *source_objects, source.key, *owner_db, *store(same_db),
same_db.key, filled<16>(0xEE), retained, budget, 5s);
CHECK(cloned && cloned->inherited_outcomes == 1 && cloned->origin.tick == 25 &&
cloned->origin.sequence == before.sequence && cloned->origin.commit == before.commit);
// The source session is read-only for good, even through SECURITY DEFINER functions.
auto denied = acquire_region_lease(*reader, source.key, 4000ms, 5s);
CHECK(!denied && sqlstate(denied.error().cause) == "25006");
// Authority over the clone is not authority over the source.
auto runtime = connect_to(source_db(), env("EVO_PG_RUNTIME_USER"));
auto lease = acquire_region_lease(*runtime, same_db.key, 4000ms, 5s);
CHECK(lease);
auto source_head = load_region_checkpoint(*runtime, source.key, 5s);
CHECK(source_head);
const auto source_manifest = decode_manifest(source_head->manifest);
CHECK(source_manifest);
auto forged = successor_manifest(*source_manifest, source_head->head.digest, filled<16>(0xEF),
*lease, 26, build(), 100, filled<32>(0xA4),
{trace_bytes_for(1), filled<32>(0x7C), 0, {}});
CHECK(forged);
auto published = publish_region_checkpoint(*runtime, *encode_manifest(*forged), {}, 5s);
CHECK(!published && published.error().kind == LeaseStoreError::fenced);
// A forged origin is refused when the source is in the same database.
auto wrong = clone_manifest(*source_manifest, filled<32>(0x55), {11, 72}, filled<16>(0xF6), 100,
filled<32>(0xA5), {0, {}, 0, {}});
CHECK(wrong);
const std::array<std::string, 5> p{"\\x" + hex(*encode_manifest(*wrong)), "7", "72",
std::to_string(before.sequence), "\\x"};
const std::string provision_clone =
"SELECT evolution.provision_region_clone($1::bytea, $2::evolution.u64, "
"$3::evolution.u64, $4::evolution.u64, $5::bytea)";
auto forged_origin = owner_db->query(provision_clone, p, 5s);
CHECK(!forged_origin && sqlstate(forged_origin.error()) == "22023");
// Only the owner provisions clones; the runtime role cannot.
auto runtime_clone = runtime->query(provision_clone, p, 5s);
CHECK(!runtime_clone && sqlstate(runtime_clone.error()) == "42501");
// A clone world never receives an original region, and origins are immutable.
{
auto region = evo::test::lineage_region();
CHECK(region->rebind_world(10));
const auto extra = clone_target({10, 73}, "clone-10-73");
auto plain = provision_lineage_region(*owner_db, *store(extra), extra.key,
evo::test::region_identity.frame, *region,
filled<16>(0xF7), build(), budget, 5s);
CHECK(!plain && plain.error().kind == PublicationError::rejected);
}
auto rewrite = owner_db->query(
"UPDATE evolution.region_origins SET source_tick = 0 WHERE world_id = 10", {}, 5s);
CHECK(!rewrite && sqlstate(rewrite.error()) == "EVL09");
// The clone database's runtime cannot reach the source database's functions.
auto foreign = connect_to(source_db(), env("EVO_PG_CLONE_RUNTIME_USER"));
auto crossed = acquire_region_lease(*foreign, source.key, 4000ms, 5s);
CHECK(!crossed && sqlstate(crossed.error().cause) == "42501");
// Nothing above moved the source.
const auto after = head_in(source_db(), source.key);
CHECK(after.commit == before.commit && decisions_of(source.key) == 1);
CHECK(release_region_lease(*runtime, same_db.key, *lease, 5s));
}
std::vector<char> read_file(const fs::path& p) {
std::ifstream in(p, std::ios::binary);
return {std::istreambuf_iterator<char>(in), {}};
}
void write_file(const fs::path& p, const std::vector<char>& bytes) {
fs::remove(p);
std::ofstream(p, std::ios::binary)
.write(bytes.data(), static_cast<std::streamsize>(bytes.size()));
fs::permissions(p, fs::perms::owner_read | fs::perms::owner_write, fs::perm_options::replace);
}
// A clone killed before its target transaction leaves the source untouched and the
// target unprovisioned (never ready); a retry completes. Bad inputs are refused.
void failed_clone_changes_nothing() {
const auto source = provision(73, 0xEA);
run_inputs(server_args(source), 1, 25);
const auto before = head_in(source_db(), source.key);
const auto target = clone_target({8, 73}, "clone-73");
CHECK(run_tool(clone_args(source, target, {"--crash-at", "before-transaction"})) ==
128 + SIGKILL);
CHECK(!maybe_head(clone_db(), target.key));
CHECK(object_count(target) == 1); // The verified upload is an unreferenced orphan.
const auto after = head_in(source_db(), source.key);
CHECK(after.commit == before.commit && decisions_of(source.key) == 1);
{
Child server{clone_server_args(target)};
CHECK(server.wait_exit(20s) == 4);
CHECK(server.printed("phase quarantined") && !server.printed("ready 1"));
}
// Refusals before anything is read or written.
CHECK(run_tool(with(clone_args(source, target), "--target-pg-database", source_db())) == 2);
CHECK(run_tool(with(clone_args(source, target), "--target-pg-user",
env("EVO_PG_RUNTIME_USER"))) == 2);
CHECK(run_tool(with(clone_args(source, target), "--target-world", "7")) == 4);
CHECK(run_tool({env("EVO_WORLD_CLONE_BIN")}) == 2);
CHECK(!maybe_head(clone_db(), target.key));
// Retry: the same verified bytes, one object, now provisioned.
CHECK(run_tool(clone_args(source, target)) == 0);
CHECK(object_count(target) == 1);
const auto cloned = head_in(clone_db(), target.key);
CHECK(cloned.sequence == 0 && cloned.end_tick == 25);
// A quarantined source (damaged head payload) is never cloned.
const auto m = decode_manifest(
load_region_checkpoint(*connect_to(source_db(), env("PGUSER")), source.key, 5s)->manifest);
CHECK(m);
const auto payload = source.objects() / hex(m->payload_sha256);
const auto original = read_file(payload);
auto flipped = original;
flipped[flipped.size() / 2] = static_cast<char>(flipped[flipped.size() / 2] ^ 1);
write_file(payload, flipped);
const auto other = clone_target({8, 173}, "clone-173");
CHECK(run_tool(clone_args(source, other)) == 3);
CHECK(!maybe_head(clone_db(), other.key));
CHECK(!fs::exists(other.objects()) || object_count(other) == 0);
write_file(payload, original);
}
// Every committed link stays verifiable across owners and kills; losing any
// historical evidence is detected even though head recovery still succeeds.
void history_is_retained() {
const auto source = provision(74, 0xEB);
{
Child first{server_args(source)};
CHECK(first.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
first.send(input_lines(1, 100, true));
CHECK(first.wait_line("ack " + request_hex(100) + " 100 ", 30s));
first.signal(SIGKILL);
CHECK(first.wait_exit(20s) == 128 + SIGKILL);
}
run_inputs(server_args(source), 101, 200); // The successor waits out the lease.
const auto head = head_in(source_db(), source.key);
CHECK(head.end_tick == 200 && head.sequence >= 4);
const std::array<Sha256Digest, 1> retained{build()};
auto reader = connect_to(source_db(), env("EVO_PG_RUNTIME_USER"));
auto objects = store(source);
auto audited = audit_region_history(*reader, *objects, source.key, retained, budget, 1000, 5s);
CHECK(audited && audited->commits == head.sequence + 1 && audited->end_tick == 200 &&
audited->outcomes == 1 && !audited->clone);
std::vector<std::string> out;
CHECK(run_tool({env("EVO_WORLD_CLONE_BIN"), "audit", "--world", "7", "--region", "74",
"--pg-host", env("PGHOST"), "--pg-port", env("PGPORT"), "--pg-database",
source_db(), "--pg-user", env("EVO_PG_RUNTIME_USER"), "--objects",
source.root.string(), "--retained", build_hex},
&out) == 0);
CHECK(out.size() == 1 && out[0] == "audited commits=" + std::to_string(head.sequence + 1) +
" tick=200 outcomes=1 profile=2 clone=0");
auto bounded =
audit_region_history(*reader, *objects, source.key, retained, budget, head.sequence, 5s);
CHECK(!bounded && bounded.error().kind == AuditError::bounds);
// Remove the first published link's trace: the head alone still recovers, but
// the audit names exactly the link whose evidence is gone.
auto first = load_region_commit(*reader, source.key, 1, 5s);
CHECK(first);
const auto m = decode_manifest(first->manifest);
CHECK(m);
const auto trace = source.objects() / hex(m->batch.trace_sha256);
const auto kept = read_file(trace);
fs::remove(trace);
auto missing = audit_region_history(*reader, *objects, source.key, retained, budget, 1000, 5s);
CHECK(!missing && missing.error().kind == AuditError::tail && missing.error().sequence == 1);
CHECK(recover_lineage_checkpoint(*reader, *objects, source.key, retained, budget, 5s));
write_file(trace, kept);
CHECK(audit_region_history(*reader, *objects, source.key, retained, budget, 1000, 5s));
}
// Environmental profile: the clone genesis digests its inherited outcomes. This
// profile names no organism, so only that digest pins the inherited death's tick.
void environmental_clone_binds_inherited_outcomes() {
const RegionKey key{7, 75};
const auto source = clone_target(key, "env-75");
auto owner_db = connect_to(source_db(), env("PGUSER"));
auto runtime = connect_to(source_db(), env("EVO_PG_RUNTIME_USER"));
auto source_objects = store(source);
auto genesis = provision_environmental_region(
*owner_db, *source_objects, key, evo::test::region_identity.frame,
*evo::test::region_community(), filled<16>(0xF9), build(), budget, 5s);
CHECK(genesis);
auto lease = acquire_region_lease(*runtime, key, 20000ms, 5s);
CHECK(lease);
auto state = evo::test::region_community();
RegionBatch<evo::processes::EnvironmentalCommunity> batch{*state};
while (!batch.sealed())
CHECK(batch.advance(evo::test::region_input(batch.end_tick() + 1, true)));
CHECK(batch.end_tick() == 20);
auto published = publish_environmental_batch(*runtime, *source_objects, *lease, *genesis, batch,
filled<16>(0xFA), build(), budget, 5s);
CHECK(published && published->decisions.size() == 1);
CHECK(release_region_lease(*runtime, key, *lease, 5s));
const std::array<Sha256Digest, 1> retained{build()};
const auto target = clone_target({8, 75}, "clone-env-75");
auto reader = connect_to(source_db(), env("EVO_PG_RUNTIME_USER"));
auto clone_owner = connect_to(clone_db(), env("PGUSER"));
auto cloned = clone_region(*reader, *source_objects, key, *clone_owner, *store(target),
target.key, filled<16>(0xFB), retained, budget, 5s);
CHECK(cloned && cloned->inherited_outcomes == 1 && cloned->origin.tick == 20);
auto clone_runtime = connect_to(clone_db(), env("EVO_PG_CLONE_RUNTIME_USER"));
const auto recover = [&] {
return recover_environmental_checkpoint(*clone_runtime, *store(target), target.key,
retained, budget, 5s);
};
auto recovered = recover();
CHECK(recovered && recovered->state->progress().tick == 20 &&
recovered->state->progress().consumer_terminal == 1 && recovered->decisions.size() == 1);
// Break glass (owner disables immutability) to move the inherited death one tick.
const auto set_tick = [&](int from, int to) {
const std::array<std::string, 2> p{std::to_string(from), std::to_string(to)};
CHECK(clone_owner->query("ALTER TABLE evolution.region_decisions DISABLE TRIGGER "
"region_decisions_immutable",
{}, 5s));
CHECK(clone_owner->query("UPDATE evolution.region_decisions SET tick = $2::evolution.u64 "
"WHERE world_id = 8 AND region_id = 75 AND "
"tick = $1::evolution.u64",
p, 5s));
CHECK(clone_owner->query("ALTER TABLE evolution.region_decisions ENABLE TRIGGER "
"region_decisions_immutable",
{}, 5s));
};
set_tick(20, 19);
auto moved = recover();
CHECK(!moved && moved.error().kind == RecoveryError::history);
auto audited =
audit_region_history(*clone_runtime, *store(target), target.key, retained, budget, 8, 5s);
CHECK(!audited && audited.error().kind == AuditError::history && audited.error().sequence == 0);
set_tick(19, 20);
CHECK(recover());
}
} // namespace
void run_region_clone_tests() {
clone_matches_the_source_trajectory();
clone_cannot_write_its_source();
failed_clone_changes_nothing();
history_is_retained();
environmental_clone_binds_inherited_outcomes();
}