// M7.9a region clones and retained history (ADR-0067) against two real PostgreSQL // databases: the source (PGDATABASE, runtime EVO_PG_RUNTIME_USER) and an isolated // clone database (EVO_PG_CLONE_DATABASE, its own runtime EVO_PG_CLONE_RUNTIME_USER). // Real evo-server and evo-world-clone processes. Run by tools/ci/postgres-check.sh. #include "evo/persistence/region_clone.hpp" #include "evo/persistence/sha256.hpp" #include "fixtures/server_process.hpp" #include #include #include #include void run_region_clone_tests(); namespace { using namespace evo::test::process; using namespace evo::persistence; void check(bool value, std::source_location at = std::source_location::current()) { if (!value) throw std::runtime_error("region clone test line " + std::to_string(at.line())); } #define CHECK(...) check(static_cast(__VA_ARGS__)) constexpr std::size_t budget = 8000000; const char* const build_hex = "b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1"; Sha256Digest build() { Sha256Digest b{}; b.fill(0xB1); return b; } template std::array filled(std::uint8_t value) { std::array out{}; out.fill(value); return out; } std::string hex(std::span bytes) { constexpr char digits[] = "0123456789abcdef"; std::string out; for (const auto b : bytes) { out.push_back(digits[b >> 4U]); out.push_back(digits[b & 0x0FU]); } return out; } std::unique_ptr connect_to(const std::string& database, const std::string& user) { auto made = Postgres::connect({env("PGHOST"), {}, env("PGPORT"), database, user, {}, {}}, 5s); CHECK(made); return std::move(*made); } std::string source_db() { return env("PGDATABASE"); } std::string clone_db() { return env("EVO_PG_CLONE_DATABASE"); } std::string sqlstate(const PgFailure& f) { return f.sqlstate.data(); } std::unique_ptr store(const ServerRegion& r) { auto made = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes); CHECK(made); return std::move(*made); } // An empty object root for a clone region. ServerRegion clone_target(RegionKey key, const std::string& name) { ServerRegion r{key, fs::path(env("PGHOST")) / name}; fs::remove_all(r.root); fs::create_directory(r.root); fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace); return r; } void retain_in_clone_database() { CHECK(connect_to(clone_db(), env("PGUSER")) ->query("INSERT INTO evolution.retained_builds(build_identity) VALUES " "(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING", {}, 5s)); } std::vector with(std::vector args, const std::string& flag, const std::string& value) { const auto at = std::ranges::find(args, flag); CHECK(at != args.end() && at + 1 != args.end()); *(at + 1) = value; return args; } // evo-server for a region of the clone database, with its own runtime login. std::vector clone_server_args(const ServerRegion& r) { return with(with(server_args(r), "--pg-database", clone_db()), "--pg-user", env("EVO_PG_CLONE_RUNTIME_USER")); } std::vector clone_args(const ServerRegion& from, const ServerRegion& to, std::vector extra = {}) { std::vector args{env("EVO_WORLD_CLONE_BIN"), "clone", "--source-world", std::to_string(from.key.world), "--source-region", std::to_string(from.key.region), "--target-world", std::to_string(to.key.world), "--target-region", std::to_string(to.key.region), "--source-pg-host", env("PGHOST"), "--source-pg-port", env("PGPORT"), "--source-pg-database", source_db(), "--source-pg-user", env("EVO_PG_RUNTIME_USER"), "--target-pg-host", env("PGHOST"), "--target-pg-port", env("PGPORT"), "--target-pg-database", clone_db(), "--target-pg-user", env("PGUSER"), "--source-objects", from.root.string(), "--target-objects", to.root.string(), "--retained", build_hex, "--audit-max", "1000"}; args.insert(args.end(), extra.begin(), extra.end()); return args; } int run_tool(const std::vector& args, std::vector* lines = nullptr) { Child tool{args}; const auto code = tool.wait_exit(60s); if (lines != nullptr) *lines = tool.lines(); return code; } // Runs a fresh owner process over inputs first..last (consumer death at 20) and drains. void run_inputs(const std::vector& args, std::uint64_t first, std::uint64_t last) { Child server{args}; CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s)); server.send(input_lines(first, last, true)); CHECK(server.wait_line("ack " + request_hex(last) + " " + std::to_string(last) + " ", 30s)); server.signal(SIGTERM); CHECK(server.wait_exit(20s) == 0); } std::optional maybe_head(const std::string& database, RegionKey key) { auto committed = load_region_checkpoint(*connect_to(database, env("PGUSER")), key, 5s); if (!committed) return std::nullopt; return committed->head; } RegionHead head_in(const std::string& database, RegionKey key) { const auto head = maybe_head(database, key); CHECK(head.has_value()); return head.value_or(RegionHead{}); } std::unique_ptr recovered_state(const std::string& database, const std::string& user, const ServerRegion& r) { const std::array retained{build()}; auto recovered = recover_lineage_checkpoint(*connect_to(database, user), *store(r), r.key, retained, budget, 5s); CHECK(recovered); return std::move(recovered->state); } // Consistent clone into an isolated world and database; the clone continues exactly // as its source does, and inherits its outcomes and ancestry. void clone_matches_the_source_trajectory() { retain_in_clone_database(); const auto source = provision(71, 0xE8); run_inputs(server_args(source), 1, 30); // Consumer dies at tick 20. const auto cut = head_in(source_db(), source.key); CHECK(cut.end_tick == 30 && cut.sequence >= 2); const auto target = clone_target({8, 71}, "clone-71"); std::vector out; CHECK(run_tool(clone_args(source, target), &out) == 0); CHECK(out.size() == 1 && out[0].starts_with("cloned source=7/71 sequence=" + std::to_string(cut.sequence) + " tick=30 target=8/71 outcomes=1 digest=")); // The clone is a verified region of world 8, recovered with the clone runtime. const auto clone_user = env("EVO_PG_CLONE_RUNTIME_USER"); auto cloned = recovered_state(clone_db(), clone_user, target); CHECK(cloned->community().world() == 8 && cloned->community().progress().tick == 30); const auto consumer = cloned->archive().organism(11); CHECK(consumer && !consumer->alive && consumer->died.value == 20); CHECK(cloned->archive().ancestry(2)->size() == 2); // Retained ancestry. { const std::array retained{build()}; auto clone_reader = connect_to(clone_db(), clone_user); auto decisions = load_region_decisions(*clone_reader, target.key, 5s); CHECK(decisions && decisions->size() == 1 && (*decisions)[0].commit_sequence == 0 && (*decisions)[0].decision == BatchDecision{20, Actor::consumer, Outcome::death, 11}); auto audited = audit_region_history(*clone_reader, *store(target), target.key, retained, budget, 16, 5s); CHECK(audited && audited->commits == 1 && audited->clone && audited->end_tick == 30 && audited->outcomes == 1 && audited->origin_commit == cut.commit && audited->origin_digest == cut.digest); // No outbox rows: the source already delivered those outcomes. auto pending = pending_region_outbox(*clone_reader, target.key, 64, 5s); CHECK(pending && pending->empty()); } // Baseline versus clone: the same inputs from the cut give the same world. run_inputs(server_args(source), 31, 90); run_inputs(clone_server_args(target), 31, 90); auto baseline = recovered_state(source_db(), env("EVO_PG_RUNTIME_USER"), source); auto restored = recovered_state(clone_db(), clone_user, target); CHECK(baseline->community().progress().tick == 90 && restored->community().progress().tick == 90); CHECK(restored->rebind_world(7)); CHECK(restored->snapshot() == baseline->snapshot()); // The clone chain continues from its own genesis, which names the source cut. const auto clone_head = head_in(clone_db(), target.key); CHECK(clone_head.sequence >= 1 && clone_head.end_tick == 90); } // A clone has no path to write its source: read-only source session, separate // credentials and database, a different world identity and immutable origins. void clone_cannot_write_its_source() { const auto source = provision(72, 0xE9); run_inputs(server_args(source), 1, 25); const auto before = head_in(source_db(), source.key); CHECK(before.end_tick == 25); const std::array retained{build()}; auto reader = connect_to(source_db(), env("EVO_PG_RUNTIME_USER")); auto owner_db = connect_to(source_db(), env("PGUSER")); auto source_objects = store(source); const auto same_db = clone_target({10, 72}, "clone-72"); CHECK(clone_region(*reader, *source_objects, source.key, *owner_db, *store(same_db), {7, 99}, filled<16>(0xEC), retained, budget, 5s) .error() .kind == CloneError::identity); // A world that already holds an original region is not an isolated clone world. { const RegionKey original{9, 1}; auto region = evo::test::lineage_region(); CHECK(region->rebind_world(9)); const auto root = clone_target(original, "original-9"); CHECK(provision_lineage_region(*owner_db, *store(root), original, evo::test::region_identity.frame, *region, filled<16>(0xF8), build(), budget, 5s)); const auto mixed = clone_target({9, 72}, "mixed-9"); auto refused = clone_region(*reader, *source_objects, source.key, *owner_db, *store(mixed), mixed.key, filled<16>(0xED), retained, budget, 5s); CHECK(!refused && refused.error().kind == CloneError::rejected && sqlstate(refused.error().store.cause) == "22023"); CHECK(!maybe_head(source_db(), mixed.key)); } // Same-database clone into a fresh world: the database also checks the origin. auto cloned = clone_region(*reader, *source_objects, source.key, *owner_db, *store(same_db), same_db.key, filled<16>(0xEE), retained, budget, 5s); CHECK(cloned && cloned->inherited_outcomes == 1 && cloned->origin.tick == 25 && cloned->origin.sequence == before.sequence && cloned->origin.commit == before.commit); // The source session is read-only for good, even through SECURITY DEFINER functions. auto denied = acquire_region_lease(*reader, source.key, 4000ms, 5s); CHECK(!denied && sqlstate(denied.error().cause) == "25006"); // Authority over the clone is not authority over the source. auto runtime = connect_to(source_db(), env("EVO_PG_RUNTIME_USER")); auto lease = acquire_region_lease(*runtime, same_db.key, 4000ms, 5s); CHECK(lease); auto source_head = load_region_checkpoint(*runtime, source.key, 5s); CHECK(source_head); const auto source_manifest = decode_manifest(source_head->manifest); CHECK(source_manifest); auto forged = successor_manifest(*source_manifest, source_head->head.digest, filled<16>(0xEF), *lease, 26, build(), 100, filled<32>(0xA4), {trace_bytes_for(1), filled<32>(0x7C), 0, {}}); CHECK(forged); auto published = publish_region_checkpoint(*runtime, *encode_manifest(*forged), {}, 5s); CHECK(!published && published.error().kind == LeaseStoreError::fenced); // A forged origin is refused when the source is in the same database. auto wrong = clone_manifest(*source_manifest, filled<32>(0x55), {11, 72}, filled<16>(0xF6), 100, filled<32>(0xA5), {0, {}, 0, {}}); CHECK(wrong); const std::array p{"\\x" + hex(*encode_manifest(*wrong)), "7", "72", std::to_string(before.sequence), "\\x"}; const std::string provision_clone = "SELECT evolution.provision_region_clone($1::bytea, $2::evolution.u64, " "$3::evolution.u64, $4::evolution.u64, $5::bytea)"; auto forged_origin = owner_db->query(provision_clone, p, 5s); CHECK(!forged_origin && sqlstate(forged_origin.error()) == "22023"); // Only the owner provisions clones; the runtime role cannot. auto runtime_clone = runtime->query(provision_clone, p, 5s); CHECK(!runtime_clone && sqlstate(runtime_clone.error()) == "42501"); // A clone world never receives an original region, and origins are immutable. { auto region = evo::test::lineage_region(); CHECK(region->rebind_world(10)); const auto extra = clone_target({10, 73}, "clone-10-73"); auto plain = provision_lineage_region(*owner_db, *store(extra), extra.key, evo::test::region_identity.frame, *region, filled<16>(0xF7), build(), budget, 5s); CHECK(!plain && plain.error().kind == PublicationError::rejected); } auto rewrite = owner_db->query( "UPDATE evolution.region_origins SET source_tick = 0 WHERE world_id = 10", {}, 5s); CHECK(!rewrite && sqlstate(rewrite.error()) == "EVL09"); // The clone database's runtime cannot reach the source database's functions. auto foreign = connect_to(source_db(), env("EVO_PG_CLONE_RUNTIME_USER")); auto crossed = acquire_region_lease(*foreign, source.key, 4000ms, 5s); CHECK(!crossed && sqlstate(crossed.error().cause) == "42501"); // Nothing above moved the source. const auto after = head_in(source_db(), source.key); CHECK(after.commit == before.commit && decisions_of(source.key) == 1); CHECK(release_region_lease(*runtime, same_db.key, *lease, 5s)); } std::vector read_file(const fs::path& p) { std::ifstream in(p, std::ios::binary); return {std::istreambuf_iterator(in), {}}; } void write_file(const fs::path& p, const std::vector& bytes) { fs::remove(p); std::ofstream(p, std::ios::binary) .write(bytes.data(), static_cast(bytes.size())); fs::permissions(p, fs::perms::owner_read | fs::perms::owner_write, fs::perm_options::replace); } // A clone killed before its target transaction leaves the source untouched and the // target unprovisioned (never ready); a retry completes. Bad inputs are refused. void failed_clone_changes_nothing() { const auto source = provision(73, 0xEA); run_inputs(server_args(source), 1, 25); const auto before = head_in(source_db(), source.key); const auto target = clone_target({8, 73}, "clone-73"); CHECK(run_tool(clone_args(source, target, {"--crash-at", "before-transaction"})) == 128 + SIGKILL); CHECK(!maybe_head(clone_db(), target.key)); CHECK(object_count(target) == 1); // The verified upload is an unreferenced orphan. const auto after = head_in(source_db(), source.key); CHECK(after.commit == before.commit && decisions_of(source.key) == 1); { Child server{clone_server_args(target)}; CHECK(server.wait_exit(20s) == 4); CHECK(server.printed("phase quarantined") && !server.printed("ready 1")); } // Refusals before anything is read or written. CHECK(run_tool(with(clone_args(source, target), "--target-pg-database", source_db())) == 2); CHECK(run_tool(with(clone_args(source, target), "--target-pg-user", env("EVO_PG_RUNTIME_USER"))) == 2); CHECK(run_tool(with(clone_args(source, target), "--target-world", "7")) == 4); CHECK(run_tool({env("EVO_WORLD_CLONE_BIN")}) == 2); CHECK(!maybe_head(clone_db(), target.key)); // Retry: the same verified bytes, one object, now provisioned. CHECK(run_tool(clone_args(source, target)) == 0); CHECK(object_count(target) == 1); const auto cloned = head_in(clone_db(), target.key); CHECK(cloned.sequence == 0 && cloned.end_tick == 25); // A quarantined source (damaged head payload) is never cloned. const auto m = decode_manifest( load_region_checkpoint(*connect_to(source_db(), env("PGUSER")), source.key, 5s)->manifest); CHECK(m); const auto payload = source.objects() / hex(m->payload_sha256); const auto original = read_file(payload); auto flipped = original; flipped[flipped.size() / 2] = static_cast(flipped[flipped.size() / 2] ^ 1); write_file(payload, flipped); const auto other = clone_target({8, 173}, "clone-173"); CHECK(run_tool(clone_args(source, other)) == 3); CHECK(!maybe_head(clone_db(), other.key)); CHECK(!fs::exists(other.objects()) || object_count(other) == 0); write_file(payload, original); } // Every committed link stays verifiable across owners and kills; losing any // historical evidence is detected even though head recovery still succeeds. void history_is_retained() { const auto source = provision(74, 0xEB); { Child first{server_args(source)}; CHECK(first.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s)); first.send(input_lines(1, 100, true)); CHECK(first.wait_line("ack " + request_hex(100) + " 100 ", 30s)); first.signal(SIGKILL); CHECK(first.wait_exit(20s) == 128 + SIGKILL); } run_inputs(server_args(source), 101, 200); // The successor waits out the lease. const auto head = head_in(source_db(), source.key); CHECK(head.end_tick == 200 && head.sequence >= 4); const std::array retained{build()}; auto reader = connect_to(source_db(), env("EVO_PG_RUNTIME_USER")); auto objects = store(source); auto audited = audit_region_history(*reader, *objects, source.key, retained, budget, 1000, 5s); CHECK(audited && audited->commits == head.sequence + 1 && audited->end_tick == 200 && audited->outcomes == 1 && !audited->clone); std::vector out; CHECK(run_tool({env("EVO_WORLD_CLONE_BIN"), "audit", "--world", "7", "--region", "74", "--pg-host", env("PGHOST"), "--pg-port", env("PGPORT"), "--pg-database", source_db(), "--pg-user", env("EVO_PG_RUNTIME_USER"), "--objects", source.root.string(), "--retained", build_hex}, &out) == 0); CHECK(out.size() == 1 && out[0] == "audited commits=" + std::to_string(head.sequence + 1) + " tick=200 outcomes=1 profile=2 clone=0"); auto bounded = audit_region_history(*reader, *objects, source.key, retained, budget, head.sequence, 5s); CHECK(!bounded && bounded.error().kind == AuditError::bounds); // Remove the first published link's trace: the head alone still recovers, but // the audit names exactly the link whose evidence is gone. auto first = load_region_commit(*reader, source.key, 1, 5s); CHECK(first); const auto m = decode_manifest(first->manifest); CHECK(m); const auto trace = source.objects() / hex(m->batch.trace_sha256); const auto kept = read_file(trace); fs::remove(trace); auto missing = audit_region_history(*reader, *objects, source.key, retained, budget, 1000, 5s); CHECK(!missing && missing.error().kind == AuditError::tail && missing.error().sequence == 1); CHECK(recover_lineage_checkpoint(*reader, *objects, source.key, retained, budget, 5s)); write_file(trace, kept); CHECK(audit_region_history(*reader, *objects, source.key, retained, budget, 1000, 5s)); } // Environmental profile: the clone genesis digests its inherited outcomes. This // profile names no organism, so only that digest pins the inherited death's tick. void environmental_clone_binds_inherited_outcomes() { const RegionKey key{7, 75}; const auto source = clone_target(key, "env-75"); auto owner_db = connect_to(source_db(), env("PGUSER")); auto runtime = connect_to(source_db(), env("EVO_PG_RUNTIME_USER")); auto source_objects = store(source); auto genesis = provision_environmental_region( *owner_db, *source_objects, key, evo::test::region_identity.frame, *evo::test::region_community(), filled<16>(0xF9), build(), budget, 5s); CHECK(genesis); auto lease = acquire_region_lease(*runtime, key, 20000ms, 5s); CHECK(lease); auto state = evo::test::region_community(); RegionBatch batch{*state}; while (!batch.sealed()) CHECK(batch.advance(evo::test::region_input(batch.end_tick() + 1, true))); CHECK(batch.end_tick() == 20); auto published = publish_environmental_batch(*runtime, *source_objects, *lease, *genesis, batch, filled<16>(0xFA), build(), budget, 5s); CHECK(published && published->decisions.size() == 1); CHECK(release_region_lease(*runtime, key, *lease, 5s)); const std::array retained{build()}; const auto target = clone_target({8, 75}, "clone-env-75"); auto reader = connect_to(source_db(), env("EVO_PG_RUNTIME_USER")); auto clone_owner = connect_to(clone_db(), env("PGUSER")); auto cloned = clone_region(*reader, *source_objects, key, *clone_owner, *store(target), target.key, filled<16>(0xFB), retained, budget, 5s); CHECK(cloned && cloned->inherited_outcomes == 1 && cloned->origin.tick == 20); auto clone_runtime = connect_to(clone_db(), env("EVO_PG_CLONE_RUNTIME_USER")); const auto recover = [&] { return recover_environmental_checkpoint(*clone_runtime, *store(target), target.key, retained, budget, 5s); }; auto recovered = recover(); CHECK(recovered && recovered->state->progress().tick == 20 && recovered->state->progress().consumer_terminal == 1 && recovered->decisions.size() == 1); // Break glass (owner disables immutability) to move the inherited death one tick. const auto set_tick = [&](int from, int to) { const std::array p{std::to_string(from), std::to_string(to)}; CHECK(clone_owner->query("ALTER TABLE evolution.region_decisions DISABLE TRIGGER " "region_decisions_immutable", {}, 5s)); CHECK(clone_owner->query("UPDATE evolution.region_decisions SET tick = $2::evolution.u64 " "WHERE world_id = 8 AND region_id = 75 AND " "tick = $1::evolution.u64", p, 5s)); CHECK(clone_owner->query("ALTER TABLE evolution.region_decisions ENABLE TRIGGER " "region_decisions_immutable", {}, 5s)); }; set_tick(20, 19); auto moved = recover(); CHECK(!moved && moved.error().kind == RecoveryError::history); auto audited = audit_region_history(*clone_runtime, *store(target), target.key, retained, budget, 8, 5s); CHECK(!audited && audited.error().kind == AuditError::history && audited.error().sequence == 0); set_tick(19, 20); CHECK(recover()); } } // namespace void run_region_clone_tests() { clone_matches_the_source_trajectory(); clone_cannot_write_its_source(); failed_clone_changes_nothing(); history_is_retained(); environmental_clone_binds_inherited_outcomes(); }