// M8.9 upgrade rehearsal on clones (ADR-0076) with real evo-server and evo-world-clone // processes: one cut is cloned into the isolated clone database several times; the // current build (b1) and a candidate build (b2) each run the same deterministic inputs // on their own clone, accelerated, to the same tick; `compare` must find the same // world. A clone fed a different input stream, or stopped at another tick, must be // reported as different. Run by tools/ci/postgres-check.sh. #include "evo/processes/development_opening.hpp" #include "fixtures/server_process.hpp" #include #include #include void run_region_upgrade_tests(); namespace { using namespace evo::test::process; using evo::persistence::RegionKey; void check(bool value, std::source_location at = std::source_location::current()) { if (!value) throw std::runtime_error("region upgrade test line " + std::to_string(at.line())); } #define CHECK(...) check(static_cast(__VA_ARGS__)) std::string b1_hex() { std::string out; for (int i = 0; i < 32; ++i) out += "b1"; return out; } std::string b2_hex() { std::string out; for (int i = 0; i < 32; ++i) out += "b2"; return out; } constexpr std::uint64_t region = 95; ServerRegion clone_root(std::uint64_t world) { ServerRegion r{{world, region}, fs::path(env("PGHOST")) / ("upgrade-" + std::to_string(world))}; fs::remove_all(r.root); fs::create_directory(r.root); fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace); return r; } std::vector pg(const std::string& prefix, const std::string& database, const std::string& user) { return {prefix + "host", env("PGHOST"), prefix + "port", env("PGPORT"), prefix + "database", database, prefix + "user", user}; } int run(const std::vector& args, std::vector* lines = nullptr) { Child tool{args}; const auto code = tool.wait_exit(60s); if (lines != nullptr) *lines = tool.lines(); return code; } void clone_to(const ServerRegion& source, const ServerRegion& target) { std::vector args{env("EVO_WORLD_CLONE_BIN"), "clone", "--source-world", std::to_string(source.key.world), "--source-region", std::to_string(region), "--target-world", std::to_string(target.key.world)}; for (auto& a : pg("--source-pg-", env("PGDATABASE"), env("EVO_PG_RUNTIME_USER"))) args.push_back(a); for (auto& a : pg("--target-pg-", env("EVO_PG_CLONE_DATABASE"), env("PGUSER"))) args.push_back(a); for (const auto& a : std::vector{"--source-objects", source.root.string(), "--target-objects", target.root.string(), "--retained", b1_hex()}) args.push_back(a); CHECK(run(args) == 0); } // evo-server on a clone of the clone database with the given build. std::vector clone_server(const ServerRegion& r, const std::string& build, const std::vector& extra) { std::vector args{env("EVO_SERVER_BIN"), "--profile", "lineage", "--world", std::to_string(r.key.world), "--region", std::to_string(region), "--frame", "9"}; for (auto& a : pg("--pg-", env("EVO_PG_CLONE_DATABASE"), env("EVO_PG_CLONE_RUNTIME_USER"))) args.push_back(a); for (const auto& a : std::vector{"--objects", r.root.string(), "--build", build, "--retained", b1_hex(), "--flush-ms", "1000", "--lease-profile", "fast"}) args.push_back(a); args.insert(args.end(), extra.begin(), extra.end()); return args; } bool acked(const Child& c, std::uint64_t tick) { for (const auto& line : c.lines()) { unsigned long long t = 0, s = 0; char request[40]; if (std::sscanf(line.c_str(), "ack %39s %llu %llu", request, &t, &s) == 3 && t >= tick) return true; } return false; } // Runs a synthetic, accelerated owner to `until` and drains; returns wall ms from // readiness to the acknowledgement of `until`. long long accelerated(const ServerRegion& r, const std::string& build, std::uint64_t until) { Child server{clone_server(r, build, {"--synthetic-inputs", "development-v1", "--synthetic-until", std::to_string(until), "--accelerate", "20"})}; CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s)); const auto start = Clock::now(); CHECK(server.wait_for([&](const std::string&) { return acked(server, until); }, 30s)); const auto ms = std::chrono::duration_cast(Clock::now() - start).count(); CHECK(!acked(server, until + 1)); // The driver stops at `until`. server.signal(SIGTERM); CHECK(server.wait_exit(20s) == 0); return ms; } // The development inputs, with the temperature shifted: a different input stream. std::string shifted_lines(std::uint64_t first, std::uint64_t last) { std::string out; for (auto tick = first; tick <= last; ++tick) { const auto in = evo::processes::development_input(tick); std::string line = "input " + request_hex(tick, 0x5A) + " " + std::to_string(in.temperature_mK + 1000) + " " + std::to_string(in.source_enabled); for (const auto b : in.bite) line += " " + std::to_string(b); for (const auto t : in.terminal) line += " " + std::to_string(t); out += line + "\n"; } return out; } int compare(std::uint64_t a, const ServerRegion& ra, std::uint64_t b, const ServerRegion& rb, std::vector& lines) { std::vector args{env("EVO_WORLD_CLONE_BIN"), "compare", "--world", std::to_string(a), "--region", std::to_string(region), "--other-world", std::to_string(b)}; for (auto& x : pg("--pg-", env("EVO_PG_CLONE_DATABASE"), env("PGUSER"))) args.push_back(x); for (const auto& x : std::vector{"--objects", ra.root.string(), "--other-objects", rb.root.string(), "--retained", b1_hex(), "--retained", b2_hex()}) args.push_back(x); return run(args, &lines); } bool printed(const std::vector& lines, std::string_view prefix) { return std::ranges::any_of(lines, [&](const std::string& l) { return l.starts_with(prefix); }); } void candidate_build_matches_on_clones() { // The live source: provisioned and run past the tick-20 death, then stopped. const auto source = provision(region, 0xD7); { Child owner{server_args( source, {"--synthetic-inputs", "development-v1", "--synthetic-until", "30"})}; CHECK(owner.wait_for([&](const std::string&) { return acked(owner, 30); }, 30s)); owner.signal(SIGTERM); CHECK(owner.wait_exit(20s) == 0); } CHECK(head_of(source.key).tick == 30); auto clone_owner = evo::persistence::Postgres::connect( {env("PGHOST"), {}, env("PGPORT"), env("EVO_PG_CLONE_DATABASE"), env("PGUSER"), {}, {}}, 5s); CHECK(clone_owner); CHECK((*clone_owner) ->query("INSERT INTO evolution.retained_builds(build_identity) VALUES " "(decode(repeat('b1', 32), 'hex')), (decode(repeat('b2', 32), 'hex')) " "ON CONFLICT DO NOTHING", {}, 5s)); const auto reference = clone_root(951), candidate = clone_root(952), diverged = clone_root(953), shorter = clone_root(954); for (const auto* r : {&reference, &candidate, &diverged, &shorter}) clone_to(source, *r); // Current and candidate builds, the same inputs, accelerated x20 to tick 150: // 120 ticks are 6 s of region time. const auto reference_ms = accelerated(reference, b1_hex(), 150); const auto candidate_ms = accelerated(candidate, b2_hex(), 150); CHECK(reference_ms < 4000 && candidate_ms < 4000); std::printf("qualification upgrade_compare ticks=120 accelerate=20 reference_ms=%lld " "candidate_ms=%lld realtime_ms=6000\n", reference_ms, candidate_ms); std::vector lines; CHECK(compare(951, reference, 952, candidate, lines) == 0); CHECK(printed(lines, "compared tick=150 outcomes=1 ")); // A different input stream from the same cut is a different world. { Child owner{clone_server(diverged, b2_hex(), {})}; CHECK(owner.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s)); owner.send(shifted_lines(31, 150)); CHECK(owner.wait_for([&](const std::string&) { return acked(owner, 150); }, 30s)); owner.signal(SIGTERM); CHECK(owner.wait_exit(20s) == 0); } CHECK(compare(951, reference, 953, diverged, lines) == 6); CHECK(printed(lines, "compare differs state")); // Heads at different ticks are not compared. static_cast(accelerated(shorter, b1_hex(), 100)); CHECK(compare(951, reference, 954, shorter, lines) == 6); CHECK(printed(lines, "compare differs ticks")); // A world is not compared with itself, and rehearsal flags need the driver. CHECK(compare(951, reference, 951, reference, lines) == 2); CHECK(run(clone_server(reference, b1_hex(), {"--accelerate", "20"})) == 2); CHECK(run(clone_server(reference, b1_hex(), {"--synthetic-until", "10"})) == 2); } } // namespace void run_region_upgrade_tests() { candidate_build_matches_on_clones(); }