Drill-only --synthetic-until/--accelerate and evo-world-clone compare (compare_region_histories) prove a candidate build runs the same world on clones of one cut before any barrier. upgrade-check.sh rehearses a failed migration under load and the version barrier with real hosts; upgrade-barrier.sh retires an old build in dev/stage only once the new digest is Ready. Nodes: aevolution.io/stores pins every objects-volume pod, aevolution.io/compute is preference-only (Alertmanager first), checked by policy and scheduling-check.sh. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
217 lines
10 KiB
C++
217 lines
10 KiB
C++
// M8.9 upgrade rehearsal on clones (ADR-0076) with real evo-server and evo-world-clone
|
|
// processes: one cut is cloned into the isolated clone database several times; the
|
|
// current build (b1) and a candidate build (b2) each run the same deterministic inputs
|
|
// on their own clone, accelerated, to the same tick; `compare` must find the same
|
|
// world. A clone fed a different input stream, or stopped at another tick, must be
|
|
// reported as different. Run by tools/ci/postgres-check.sh.
|
|
#include "evo/processes/development_opening.hpp"
|
|
#include "fixtures/server_process.hpp"
|
|
#include <algorithm>
|
|
#include <cstdio>
|
|
#include <source_location>
|
|
|
|
void run_region_upgrade_tests();
|
|
|
|
namespace {
|
|
using namespace evo::test::process;
|
|
using evo::persistence::RegionKey;
|
|
void check(bool value, std::source_location at = std::source_location::current()) {
|
|
if (!value)
|
|
throw std::runtime_error("region upgrade test line " + std::to_string(at.line()));
|
|
}
|
|
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
|
|
std::string b1_hex() {
|
|
std::string out;
|
|
for (int i = 0; i < 32; ++i)
|
|
out += "b1";
|
|
return out;
|
|
}
|
|
std::string b2_hex() {
|
|
std::string out;
|
|
for (int i = 0; i < 32; ++i)
|
|
out += "b2";
|
|
return out;
|
|
}
|
|
constexpr std::uint64_t region = 95;
|
|
|
|
ServerRegion clone_root(std::uint64_t world) {
|
|
ServerRegion r{{world, region}, fs::path(env("PGHOST")) / ("upgrade-" + std::to_string(world))};
|
|
fs::remove_all(r.root);
|
|
fs::create_directory(r.root);
|
|
fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace);
|
|
return r;
|
|
}
|
|
std::vector<std::string> pg(const std::string& prefix, const std::string& database,
|
|
const std::string& user) {
|
|
return {prefix + "host", env("PGHOST"), prefix + "port", env("PGPORT"),
|
|
prefix + "database", database, prefix + "user", user};
|
|
}
|
|
int run(const std::vector<std::string>& args, std::vector<std::string>* lines = nullptr) {
|
|
Child tool{args};
|
|
const auto code = tool.wait_exit(60s);
|
|
if (lines != nullptr)
|
|
*lines = tool.lines();
|
|
return code;
|
|
}
|
|
void clone_to(const ServerRegion& source, const ServerRegion& target) {
|
|
std::vector<std::string> args{env("EVO_WORLD_CLONE_BIN"), "clone",
|
|
"--source-world", std::to_string(source.key.world),
|
|
"--source-region", std::to_string(region),
|
|
"--target-world", std::to_string(target.key.world)};
|
|
for (auto& a : pg("--source-pg-", env("PGDATABASE"), env("EVO_PG_RUNTIME_USER")))
|
|
args.push_back(a);
|
|
for (auto& a : pg("--target-pg-", env("EVO_PG_CLONE_DATABASE"), env("PGUSER")))
|
|
args.push_back(a);
|
|
for (const auto& a :
|
|
std::vector<std::string>{"--source-objects", source.root.string(), "--target-objects",
|
|
target.root.string(), "--retained", b1_hex()})
|
|
args.push_back(a);
|
|
CHECK(run(args) == 0);
|
|
}
|
|
// evo-server on a clone of the clone database with the given build.
|
|
std::vector<std::string> clone_server(const ServerRegion& r, const std::string& build,
|
|
const std::vector<std::string>& extra) {
|
|
std::vector<std::string> args{env("EVO_SERVER_BIN"),
|
|
"--profile",
|
|
"lineage",
|
|
"--world",
|
|
std::to_string(r.key.world),
|
|
"--region",
|
|
std::to_string(region),
|
|
"--frame",
|
|
"9"};
|
|
for (auto& a : pg("--pg-", env("EVO_PG_CLONE_DATABASE"), env("EVO_PG_CLONE_RUNTIME_USER")))
|
|
args.push_back(a);
|
|
for (const auto& a :
|
|
std::vector<std::string>{"--objects", r.root.string(), "--build", build, "--retained",
|
|
b1_hex(), "--flush-ms", "1000", "--lease-profile", "fast"})
|
|
args.push_back(a);
|
|
args.insert(args.end(), extra.begin(), extra.end());
|
|
return args;
|
|
}
|
|
bool acked(const Child& c, std::uint64_t tick) {
|
|
for (const auto& line : c.lines()) {
|
|
unsigned long long t = 0, s = 0;
|
|
char request[40];
|
|
if (std::sscanf(line.c_str(), "ack %39s %llu %llu", request, &t, &s) == 3 && t >= tick)
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
// Runs a synthetic, accelerated owner to `until` and drains; returns wall ms from
|
|
// readiness to the acknowledgement of `until`.
|
|
long long accelerated(const ServerRegion& r, const std::string& build, std::uint64_t until) {
|
|
Child server{clone_server(r, build,
|
|
{"--synthetic-inputs", "development-v1", "--synthetic-until",
|
|
std::to_string(until), "--accelerate", "20"})};
|
|
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
|
|
const auto start = Clock::now();
|
|
CHECK(server.wait_for([&](const std::string&) { return acked(server, until); }, 30s));
|
|
const auto ms =
|
|
std::chrono::duration_cast<std::chrono::milliseconds>(Clock::now() - start).count();
|
|
CHECK(!acked(server, until + 1)); // The driver stops at `until`.
|
|
server.signal(SIGTERM);
|
|
CHECK(server.wait_exit(20s) == 0);
|
|
return ms;
|
|
}
|
|
// The development inputs, with the temperature shifted: a different input stream.
|
|
std::string shifted_lines(std::uint64_t first, std::uint64_t last) {
|
|
std::string out;
|
|
for (auto tick = first; tick <= last; ++tick) {
|
|
const auto in = evo::processes::development_input(tick);
|
|
std::string line = "input " + request_hex(tick, 0x5A) + " " +
|
|
std::to_string(in.temperature_mK + 1000) + " " +
|
|
std::to_string(in.source_enabled);
|
|
for (const auto b : in.bite)
|
|
line += " " + std::to_string(b);
|
|
for (const auto t : in.terminal)
|
|
line += " " + std::to_string(t);
|
|
out += line + "\n";
|
|
}
|
|
return out;
|
|
}
|
|
int compare(std::uint64_t a, const ServerRegion& ra, std::uint64_t b, const ServerRegion& rb,
|
|
std::vector<std::string>& lines) {
|
|
std::vector<std::string> args{env("EVO_WORLD_CLONE_BIN"),
|
|
"compare",
|
|
"--world",
|
|
std::to_string(a),
|
|
"--region",
|
|
std::to_string(region),
|
|
"--other-world",
|
|
std::to_string(b)};
|
|
for (auto& x : pg("--pg-", env("EVO_PG_CLONE_DATABASE"), env("PGUSER")))
|
|
args.push_back(x);
|
|
for (const auto& x :
|
|
std::vector<std::string>{"--objects", ra.root.string(), "--other-objects",
|
|
rb.root.string(), "--retained", b1_hex(), "--retained", b2_hex()})
|
|
args.push_back(x);
|
|
return run(args, &lines);
|
|
}
|
|
bool printed(const std::vector<std::string>& lines, std::string_view prefix) {
|
|
return std::ranges::any_of(lines, [&](const std::string& l) { return l.starts_with(prefix); });
|
|
}
|
|
|
|
void candidate_build_matches_on_clones() {
|
|
// The live source: provisioned and run past the tick-20 death, then stopped.
|
|
const auto source = provision(region, 0xD7);
|
|
{
|
|
Child owner{server_args(
|
|
source, {"--synthetic-inputs", "development-v1", "--synthetic-until", "30"})};
|
|
CHECK(owner.wait_for([&](const std::string&) { return acked(owner, 30); }, 30s));
|
|
owner.signal(SIGTERM);
|
|
CHECK(owner.wait_exit(20s) == 0);
|
|
}
|
|
CHECK(head_of(source.key).tick == 30);
|
|
auto clone_owner = evo::persistence::Postgres::connect(
|
|
{env("PGHOST"), {}, env("PGPORT"), env("EVO_PG_CLONE_DATABASE"), env("PGUSER"), {}, {}},
|
|
5s);
|
|
CHECK(clone_owner);
|
|
CHECK((*clone_owner)
|
|
->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
|
|
"(decode(repeat('b1', 32), 'hex')), (decode(repeat('b2', 32), 'hex')) "
|
|
"ON CONFLICT DO NOTHING",
|
|
{}, 5s));
|
|
const auto reference = clone_root(951), candidate = clone_root(952), diverged = clone_root(953),
|
|
shorter = clone_root(954);
|
|
for (const auto* r : {&reference, &candidate, &diverged, &shorter})
|
|
clone_to(source, *r);
|
|
|
|
// Current and candidate builds, the same inputs, accelerated x20 to tick 150:
|
|
// 120 ticks are 6 s of region time.
|
|
const auto reference_ms = accelerated(reference, b1_hex(), 150);
|
|
const auto candidate_ms = accelerated(candidate, b2_hex(), 150);
|
|
CHECK(reference_ms < 4000 && candidate_ms < 4000);
|
|
std::printf("qualification upgrade_compare ticks=120 accelerate=20 reference_ms=%lld "
|
|
"candidate_ms=%lld realtime_ms=6000\n",
|
|
reference_ms, candidate_ms);
|
|
std::vector<std::string> lines;
|
|
CHECK(compare(951, reference, 952, candidate, lines) == 0);
|
|
CHECK(printed(lines, "compared tick=150 outcomes=1 "));
|
|
|
|
// A different input stream from the same cut is a different world.
|
|
{
|
|
Child owner{clone_server(diverged, b2_hex(), {})};
|
|
CHECK(owner.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
|
|
owner.send(shifted_lines(31, 150));
|
|
CHECK(owner.wait_for([&](const std::string&) { return acked(owner, 150); }, 30s));
|
|
owner.signal(SIGTERM);
|
|
CHECK(owner.wait_exit(20s) == 0);
|
|
}
|
|
CHECK(compare(951, reference, 953, diverged, lines) == 6);
|
|
CHECK(printed(lines, "compare differs state"));
|
|
// Heads at different ticks are not compared.
|
|
static_cast<void>(accelerated(shorter, b1_hex(), 100));
|
|
CHECK(compare(951, reference, 954, shorter, lines) == 6);
|
|
CHECK(printed(lines, "compare differs ticks"));
|
|
// A world is not compared with itself, and rehearsal flags need the driver.
|
|
CHECK(compare(951, reference, 951, reference, lines) == 2);
|
|
CHECK(run(clone_server(reference, b1_hex(), {"--accelerate", "20"})) == 2);
|
|
CHECK(run(clone_server(reference, b1_hex(), {"--synthetic-until", "10"})) == 2);
|
|
}
|
|
} // namespace
|
|
|
|
void run_region_upgrade_tests() {
|
|
candidate_build_matches_on_clones();
|
|
}
|