Files
aevolution/tests/region_migration_tests.cpp
T
dingoandClaude Opus 5.5 4bc5d5eee7
Foundation / checks (checks) (push) Successful in 5s
Foundation / checks (client) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 4s
Foundation / checks (postgres) (push) Successful in 5s
Foundation / checks (network) (push) Successful in 5s
Foundation / checks (tsan) (push) Successful in 5s
Foundation / checks (deploy) (push) Successful in 4m29s
Foundation / postgres-tools (push) Successful in 24s
Foundation / server-image (push) Successful in 4m33s
Foundation / image (push) Successful in 4m59s
feat(cellular): session v14, bundle 15 and rules-13 wildlife
Session v14 carries the rules-13 revamp: a body block in the owner's view
(members, emergence threshold, exclusions, Emerge open, payable energy,
mass and the 20 traits), become, emerge and channel exclusions on actions,
become on mutation choices, catalog offers (id, chance, cost) and the
emerged ending. Older sessions never see a v14 field. The client shows
Stay / Become offspring offers named from the shared catalog, continuing
as a bud, the organism and its Emerge button, kept channels, the body and
the emerged end screen.

Bundle 15 (rules 13 on owner format 11) is durable: schema 60 admits it
raw-only, stores become as bit 15 of the choice's operator, widens the
action input and mutation results, and publishes the 14 -> 15 revamp
profile migration. evo-world-clone opens cellular-v10 (cellular-v9 plus the
revamp upgrade from --revamp-tuning), resumable and recoverable.

Wildlife counts reserve matter as energy, waits for the bud fee, grows
organisms one short of emerging, and hunts by the region step's rules-13
engulf test. The play summary adds organisms, offers and the spent pool.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-26 07:31:32 -04:00

750 lines
41 KiB
C++

// M7.9b schema migration, upgrade barrier and durability qualification (ADR-0068)
// against real PostgreSQL, real evo-server processes and the checked-in migrations
// (EVO_SOURCE_ROOT). Run by tools/ci/postgres-check.sh; prints measured latencies.
#include "evo/persistence/region_clone.hpp"
#include "evo/persistence/sha256.hpp"
#include "fixtures/server_process.hpp"
#include <algorithm>
#include <fstream>
#include <iterator>
#include <source_location>
#include <sstream>
#include <thread>
void run_region_migration_tests();
namespace {
using namespace evo::test::process;
using namespace evo::persistence;
void check(bool value, std::source_location at = std::source_location::current()) {
if (!value)
throw std::runtime_error("region migration test line " + std::to_string(at.line()));
}
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
constexpr std::size_t budget = 8000000;
const std::string b1_text = [] {
std::string s;
for (int i = 0; i < 32; ++i)
s += "b1";
return s;
}();
const std::string b2_text = [] {
std::string s;
for (int i = 0; i < 32; ++i)
s += "b2";
return s;
}();
Sha256Digest digest_of(std::uint8_t b) {
Sha256Digest d{};
d.fill(b);
return d;
}
const Sha256Digest b1 = digest_of(0xB1), b2 = digest_of(0xB2);
template <std::size_t N> std::array<std::uint8_t, N> filled(std::uint8_t value) {
std::array<std::uint8_t, N> out{};
out.fill(value);
return out;
}
std::unique_ptr<Postgres> connect_to(const std::string& database, const std::string& user) {
auto made = Postgres::connect({env("PGHOST"), {}, env("PGPORT"), database, user, {}, {}}, 5s);
CHECK(made);
return std::move(*made);
}
std::string sqlstate(const PgFailure& f) {
return f.sqlstate.data();
}
std::string scalar(Postgres& db, const std::string& sql) {
auto rows = db.query(sql, {}, 5s);
CHECK(rows && rows->size() == 1 && (*rows)[0].size() == 1);
return (*rows)[0][0].value_or("");
}
int shell(const std::string& command) {
Child c{{"/bin/bash", "-c", command}};
return c.wait_exit(120s);
}
std::vector<std::string> with(std::vector<std::string> args, const std::string& flag,
const std::string& value) {
const auto at = std::ranges::find(args, flag);
CHECK(at != args.end() && at + 1 != args.end());
*(at + 1) = value;
return args;
}
std::unique_ptr<FilesystemObjectStore> store(const ServerRegion& r) {
auto made = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes);
CHECK(made);
return std::move(*made);
}
std::optional<Uuid> parse_uuid(const std::string& text) {
std::string compact;
for (const char c : text)
if (c != '-')
compact.push_back(c);
if (compact.size() != 32)
return std::nullopt;
Uuid out{};
for (std::size_t i = 0; i < 16; ++i)
out[i] = static_cast<std::uint8_t>(std::stoul(compact.substr(2 * i, 2), nullptr, 16));
return out;
}
// A pre-v8 writer: the build-less acquisition every earlier host used.
std::expected<LeaseGrant, std::string> legacy_acquire(Postgres& db, RegionKey key) {
const std::array<std::string, 2> p{std::to_string(key.world), std::to_string(key.region)};
auto rows = db.query("SELECT * FROM evolution.acquire_region_lease($1::evolution.u64, "
"$2::evolution.u64, 20000)",
p, 5s);
if (!rows)
return std::unexpected{sqlstate(rows.error())};
CHECK(rows->size() == 1 && (*rows)[0].size() == 4);
const auto& r = (*rows)[0];
const auto holder = parse_uuid(r[0].value_or(""));
CHECK(holder.has_value());
return LeaseGrant{holder.value_or(Uuid{}), std::stoull(r[1].value_or("0")),
std::stoull(r[2].value_or("0")), 20000ms};
}
const char* const sha_function =
"s(){ sha256sum \"$EVO_SOURCE_ROOT/migrations/$1\" | cut -d' ' -f1; }; ";
std::string history(Postgres& owner) {
return scalar(owner, "SELECT string_agg(version::text, ',' ORDER BY version) FROM "
"evolution.schema_migrations");
}
// An existing v7 database is upgraded while an older writer holds a lease. A
// migration that fails leaves v7 exactly and the writer unaffected; the real one
// commits under it. The barrier then retires the old build: the writer is fenced
// and can never regain authority, while the new build recovers the whole history.
void upgrade_across_migration_and_barrier() {
const std::string database = "evo_online";
const std::string runtime_user = env("EVO_PG_RUNTIME_USER");
CHECK(shell(std::string(sha_function) + "cd \"$EVO_SOURCE_ROOT/migrations\" && createdb " +
database + " && PGDATABASE=" + database +
" psql -X -q -v ON_ERROR_STOP=1 -1 -v migration_sha=$(s 0001_region_bootstrap.sql)"
" -v migration_sha_2=$(s 0002_region_fencing.sql)"
" -v migration_sha_3=$(s 0003_region_checkpoints.sql)"
" -v migration_sha_4=$(s 0004_lineage_profile.sql)"
" -v migration_sha_5=$(s 0005_batch_decisions.sql)"
" -v migration_sha_6=$(s 0006_commit_lookup.sql)"
" -v migration_sha_7=$(s 0007_region_clones.sql)"
" -v runtime_role=evo_runtime_test -f 0001_region_bootstrap.sql"
" -f 0002_region_fencing.sql -f 0003_region_checkpoints.sql"
" -f 0004_lineage_profile.sql -f 0005_batch_decisions.sql"
" -f 0006_commit_lookup.sql -f 0007_region_clones.sql > /dev/null") == 0);
auto owner_db = connect_to(database, env("PGUSER"));
CHECK(history(*owner_db) == "1,2,3,4,5,6,7");
CHECK(owner_db->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
"(decode(repeat('b1', 32), 'hex')), (decode(repeat('b2', 32), 'hex'))",
{}, 5s));
const ServerRegion r{{7, 91}, fs::path(env("PGHOST")) / "online-91"};
fs::remove_all(r.root);
fs::create_directory(r.root);
fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace);
auto objects = store(r);
auto genesis =
provision_lineage_region(*owner_db, *objects, r.key, evo::test::region_identity.frame,
*evo::test::lineage_region(), filled<16>(0xDA), b1, budget, 5s);
CHECK(genesis);
// The old writer: build-less lease, publishing with build b1.
auto runtime = connect_to(database, runtime_user);
auto lease = legacy_acquire(*runtime, r.key);
CHECK(lease);
auto state = evo::test::lineage_region();
RegionBatch<evo::processes::LineageRegion> batch{*state};
CommittedCheckpoint head = *genesis;
std::uint8_t commit_seed = 0xDB;
const auto publish_next = [&](std::uint64_t ticks) {
for (std::uint64_t i = 0; i < ticks && !batch.sealed(); ++i)
CHECK(batch.advance(evo::test::region_input(batch.end_tick() + 1, true)));
batch.seal();
return publish_lineage_batch(*runtime, *objects, *lease, head, batch,
filled<16>(commit_seed++), b1, budget, 5s);
};
const auto confirm = [&](const std::expected<PublishedBatch, PublicationFailure>& done) {
CHECK(done && batch.confirm(done->committed.head.end_tick));
head = done->committed;
};
confirm(publish_next(64)); // Ticks 1..20: the consumer's death seals it.
CHECK(head.head.end_tick == 20);
// A v8 migration that fails at its last statement changes nothing.
CHECK(shell(std::string(sha_function) + "cd \"$EVO_SOURCE_ROOT\" && PGDATABASE=" + database +
" psql -X -q -v ON_ERROR_STOP=1 -v migration_sha=$(s 0001_region_bootstrap.sql)"
" -v migration_sha_2=$(s 0002_region_fencing.sql)"
" -v migration_sha_3=$(s 0003_region_checkpoints.sql)"
" -v migration_sha_4=$(s 0004_lineage_profile.sql)"
" -v migration_sha_5=$(s 0005_batch_decisions.sql)"
" -v migration_sha_6=$(s 0006_commit_lookup.sql)"
" -v migration_sha_7=$(s 0007_region_clones.sql) -v migration_sha_8=invalid"
" -v migration_sha_9=$(s 0009_serialized_authority.sql)"
" -v migration_sha_10=$(s 0010_account_control.sql)"
" -v migration_sha_11=$(s 0011_region_actions.sql)"
" -v migration_sha_12=$(s 0012_probe_handoff.sql)"
" -v migration_sha_13=$(s 0013_probe_incoming.sql)"
" -v migration_sha_14=$(s 0014_probe_overdue.sql)"
" -v migration_sha_15=$(s 0015_life_reservations.sql)"
" -v migration_sha_16=$(s 0016_primitive_publication.sql)"
" -v migration_sha_17=$(s 0017_life_request_outcomes.sql)"
" -v migration_sha_18=$(s 0018_cellular_profile_admission.sql)"
" -v migration_sha_19=$(s 0019_cellular_action_records.sql)"
" -v migration_sha_20=$(s 0020_cellular_publication.sql)"
" -v migration_sha_21=$(s 0021_cellular_trace_and_recovery.sql)"
" -v migration_sha_22=$(s 0022_cellular_birth_trace_bound.sql)"
" -v migration_sha_23=$(s 0023_cellular_birth_publication.sql)"
" -v migration_sha_24=$(s 0024_primitive_life_owner_lookup.sql)"
" -v migration_sha_25=$(s 0025_cellular_v2_bundle.sql)"
" -v migration_sha_26=$(s 0026_cellular_profile_migration.sql)"
" -v migration_sha_27=$(s 0027_cellular_v3_migration.sql)"
" -v migration_sha_28=$(s 0028_cellular_v3_trace_bounds.sql)"
" -v migration_sha_29=$(s 0029_cellular_mutation_reservations.sql)"
" -v migration_sha_30=$(s 0030_cellular_birth_publication_binding.sql)"
" -v migration_sha_31=$(s 0031_fix_cellular_birth_publication_record.sql)"
" -v migration_sha_32=$(s 0032_fix_cellular_v3_manifest_normalization.sql)"
" -v migration_sha_33=$(s 0033_fix_cellular_action_tick_offset.sql)"
" -v migration_sha_34=$(s 0034_cellular_v4_engulf_bundle.sql)"
" -v migration_sha_35=$(s 0035_cellular_mutation_publication.sql)"
" -v migration_sha_36=$(s 0036_cellular_v5_budding_bundle.sql)"
" -v migration_sha_37=$(s 0037_cellular_life_endings.sql)"
" -v migration_sha_38=$(s 0038_cellular_v6_presence_bundle.sql)"
" -v migration_sha_39=$(s 0039_account_identities.sql)"
" -v migration_sha_40=$(s 0040_cellular_mutation_decline.sql)"
" -v migration_sha_41=$(s 0041_cellular_v7_balance_bundle.sql)"
" -v migration_sha_42=$(s 0042_cellular_v8_reproduction_bundle.sql)"
" -v migration_sha_43=$(s 0043_cellular_v9_foraging_bundle.sql)"
" -v migration_sha_44=$(s 0044_exact_population_publication.sql)"
" -v migration_sha_45=$(s 0045_owned_population_intervals.sql)"
" -v migration_sha_46=$(s 0046_boundary_accounting_upgrade.sql)"
" -v migration_sha_47=$(s 0047_cellular_migration_freeze.sql)"
" -v migration_sha_48=$(s 0048_cellular_migration_staging.sql)"
" -v migration_sha_49=$(s 0049_cellular_migration_publication.sql)"
" -v migration_sha_50=$(s 0050_world_region_catalog.sql)"
" -v migration_sha_51=$(s 0051_database_history_domain.sql)"
" -v migration_sha_52=$(s 0052_census_pending_migrations.sql)"
" -v migration_sha_53=$(s 0053_funded_cellular_maturation.sql)"
" -v migration_sha_54=$(s 0054_maturing_cellular_migration.sql)"
" -v migration_sha_55=$(s 0055_retained_cellular_plans.sql)"
" -v migration_sha_56=$(s 0056_cellular_environment_publication.sql)"
" -v migration_sha_57=$(s 0057_photosynthesis_bundle.sql)"
" -v migration_sha_58=$(s 0058_liquid_bundle.sql)"
" -v migration_sha_59=$(s 0059_durable_mutation_queue.sql)"
" -v migration_sha_60=$(s 0060_cellular_revamp_bundle.sql)"
" -v runtime_role=evo_runtime_test -f migrations/apply.sql"
" > /dev/null 2>&1") != 0);
CHECK(history(*owner_db) == "1,2,3,4,5,6,7");
CHECK(scalar(*owner_db, "SELECT count(*) FROM information_schema.columns WHERE "
"table_schema = 'evolution' AND column_name = 'retired_at'") == "0");
CHECK(scalar(*owner_db,
"SELECT (to_regprocedure('evolution.acquire_region_lease("
"evolution.u64, evolution.u64, integer, bytea)') IS NULL)::text") == "true");
CHECK(renew_region_lease(*runtime, r.key, *lease, 20000ms, 5s));
confirm(publish_next(20)); // 21..40, still v7.
// The real migration commits while the writer holds its lease; it keeps going.
CHECK(shell("PGDATABASE=" + database +
" EVO_PG_RUNTIME_ROLE=evo_runtime_test bash \"$EVO_SOURCE_ROOT/"
"tools/migrate-postgres.sh\" > /dev/null") == 0);
CHECK(history(*owner_db) == "1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,"
"25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,"
"47,48,49,50,51,52,53,54,55,56,57,58,59,60");
CHECK(scalar(*owner_db, "SELECT (c.condeferrable AND c.condeferred AND "
"c.confrelid = 'evolution.region_commits'::regclass)::text "
"FROM pg_catalog.pg_constraint c WHERE c.conrelid = "
"'evolution.primitive_life_bindings'::regclass AND "
"c.conname = 'primitive_life_bindings_commit_id_fkey'") == "true");
CHECK(scalar(*owner_db,
"SELECT (t.tgdeferrable AND t.tginitdeferred)::text FROM pg_catalog.pg_trigger t "
"WHERE t.tgrelid = 'evolution.primitive_life_bindings'::regclass AND "
"t.tgname = 'life_binding_publication_evidence' AND NOT t.tgisinternal") ==
"true");
CHECK(scalar(*owner_db,
"SELECT (to_regprocedure('evolution.primitive_life_owner("
"evolution.u64,evolution.u64,uuid,evolution.u64,uuid)') IS NOT NULL)::text") ==
"true");
CHECK(scalar(*owner_db, "SELECT has_function_privilege('evo_runtime_login', "
"'evolution.primitive_life_owner(evolution.u64,evolution.u64,uuid,"
"evolution.u64,uuid)', 'EXECUTE')::text") == "true");
CHECK(scalar(*owner_db, "SELECT has_table_privilege('evo_runtime_login', "
"'evolution.life_reservations', 'SELECT')::text") == "false");
CHECK(scalar(*owner_db, "SELECT count(*) FROM information_schema.columns WHERE "
"table_schema = 'evolution' AND table_name = 'region_actions' AND "
"column_name IN ('input_bytes', 'result')") == "2");
CHECK(scalar(*owner_db, "SELECT (to_regprocedure('evolution.find_region_action_record("
"evolution.u64,evolution.u64,uuid,evolution.u64,uuid,evolution.u64)') "
"IS NOT NULL)::text") == "true");
CHECK(scalar(*owner_db, "SELECT evolution.valid_cellular_action_input(decode('01' || "
"repeat('00', 23), 'hex'), 1)::text") == "true");
CHECK(scalar(*owner_db, "SELECT evolution.valid_cellular_action_input(decode(repeat('00', 24), "
"'hex'), 1)::text") == "false");
// Schema v34 (ADR-0124): target kind 2 is engulf, with no coordinates. Schema v36
// (ADR-0125): kind 3 is a bud, with no coordinates; kind 4 is unknown.
CHECK(scalar(*owner_db,
"SELECT evolution.valid_cellular_action_input(decode('01' || "
"repeat('00', 11) || '02' || repeat('00', 11), 'hex'), 1)::text") == "true");
CHECK(scalar(*owner_db, "SELECT evolution.valid_cellular_action_input(decode('01' || "
"repeat('00', 11) || '02' || repeat('00', 3) || '01' || "
"repeat('00', 7), 'hex'), 1)::text") == "false");
CHECK(scalar(*owner_db,
"SELECT evolution.valid_cellular_action_input(decode('01' || "
"repeat('00', 11) || '03' || repeat('00', 11), 'hex'), 1)::text") == "true");
CHECK(scalar(*owner_db, "SELECT evolution.valid_cellular_action_input(decode('01' || "
"repeat('00', 11) || '03' || repeat('00', 3) || '01' || "
"repeat('00', 7), 'hex'), 1)::text") == "false");
CHECK(scalar(*owner_db,
"SELECT evolution.valid_cellular_action_input(decode('01' || "
"repeat('00', 11) || '04' || repeat('00', 11), 'hex'), 1)::text") == "false");
CHECK(scalar(*owner_db,
"SELECT evolution.check_cellular_action_block(decode(repeat('01', 16) || "
"'0100000000000000' || '0100000000000000' || repeat('00', 16), 'hex'), "
"0, 1)::text") == "1");
CHECK(
scalar(*owner_db,
"SELECT "
"(to_regprocedure('evolution.publish_cellular_checkpoint(bytea,bytea,bytea,bytea)') "
"IS NOT NULL)::text") == "true");
CHECK(
scalar(
*owner_db,
"SELECT has_function_privilege('evo_runtime_login', "
"'evolution.publish_cellular_checkpoint(bytea,bytea,bytea,bytea)', 'EXECUTE')::text") ==
"true");
CHECK(scalar(*owner_db,
"SELECT has_function_privilege('evo_runtime_login', "
"'evolution.publish_cellular_checkpoint(bytea,bytea,bytea)', 'EXECUTE')::text") ==
"false");
CHECK(scalar(*owner_db,
"SELECT (to_regprocedure('evolution.load_cellular_publication_births("
"evolution.u64,evolution.u64,uuid,evolution.u64,uuid)') IS NOT NULL)::text") ==
"true");
CHECK(scalar(*owner_db,
"SELECT has_function_privilege('evo_runtime_login', "
"'evolution.resolve_cellular_region_commit(evolution.u64,evolution.u64,uuid,"
"bytea,bytea,bytea,bytea)', 'EXECUTE')::text") == "true");
CHECK(scalar(*owner_db, "SELECT has_table_privilege('evo_runtime_login', "
"'evolution.cellular_publications', 'INSERT')::text") == "false");
CHECK(scalar(*owner_db,
"SELECT has_function_privilege('evo_runtime_login', "
"'evolution.publish_region_checkpoint_core(bytea,bytea)', 'EXECUTE')::text") ==
"false");
CHECK(renew_region_lease(*runtime, r.key, *lease, 20000ms, 5s));
confirm(publish_next(20)); // 41..60, now v8.
CHECK(head.head.end_tick == 60);
// Barrier guards: the last current build cannot be retired; runtime cannot call it.
const std::string retire_b1 = "SELECT evolution.upgrade_barrier(decode(repeat('b1', 32), "
"'hex'))::text";
auto denied = runtime->query(retire_b1, {}, 5s);
CHECK(!denied && sqlstate(denied.error()) == "42501");
// The barrier: b1 retired and every holder fenced, in one transaction.
CHECK(scalar(*owner_db, retire_b1) == "1");
auto last_b2 = owner_db->query(
"SELECT evolution.upgrade_barrier(decode(repeat('b2', 32), 'hex'))", {}, 5s);
CHECK(!last_b2 && sqlstate(last_b2.error()) == "22023");
// The old writer is fenced and cannot regain authority by any path.
auto after = publish_next(10);
CHECK(!after && after.error().kind == PublicationError::rejected &&
after.error().store.kind == LeaseStoreError::fenced);
auto renewed = renew_region_lease(*runtime, r.key, *lease, 20000ms, 5s);
CHECK(!renewed && renewed.error().kind == LeaseStoreError::fenced);
auto legacy = legacy_acquire(*runtime, r.key);
CHECK(!legacy && legacy.error() == "EVL06");
auto retired = acquire_region_lease(*runtime, r.key, b1, 20000ms, 5s);
CHECK(!retired && retired.error().kind == LeaseStoreError::unsupported);
// Authority for b2 still cannot publish a link naming the retired build.
auto current = acquire_region_lease(*runtime, r.key, b2, 20000ms, 5s);
CHECK(current);
lease = *current;
auto stale_build = publish_lineage_batch(*runtime, *objects, *lease, head, batch,
filled<16>(commit_seed++), b1, budget, 5s);
CHECK(!stale_build && stale_build.error().kind == PublicationError::rejected &&
stale_build.error().store.kind == LeaseStoreError::unsupported);
CHECK(release_region_lease(*runtime, r.key, *lease, 5s));
const auto before_new = head.head;
// The new build resumes from the committed head, and b1 history stays readable.
auto args = with(with(server_args(r), "--pg-database", database), "--build", b2_text);
args.insert(args.end(), {"--retained", b1_text});
{
Child server{args};
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
CHECK(server.printed("phase running epoch=3 committed=60 "));
server.send(input_lines(61, 90, true));
CHECK(server.wait_line("ack " + request_hex(90) + " 90 ", 30s));
server.signal(SIGTERM);
CHECK(server.wait_exit(20s) == 0);
}
auto final_head = load_region_checkpoint(*owner_db, r.key, 5s);
CHECK(final_head && final_head->head.end_tick == 90 &&
final_head->head.sequence > before_new.sequence);
const auto final_manifest = decode_manifest(final_head->manifest);
CHECK(final_manifest && final_manifest->build == b2);
const std::array<Sha256Digest, 2> both{b1, b2};
const std::array<Sha256Digest, 1> only_new{b2};
auto audited = audit_region_history(*runtime, *objects, r.key, both, budget, 100, 5s);
CHECK(audited && audited->commits == final_head->head.sequence + 1 && audited->outcomes == 1);
auto without_old = audit_region_history(*runtime, *objects, r.key, only_new, budget, 100, 5s);
CHECK(!without_old && without_old.error().kind == AuditError::unsupported &&
without_old.error().sequence == 0);
// A host of the retired build never serves, not even once the region is free.
{
Child old_host{with(with(server_args(r), "--pg-database", database), "--build", b1_text)};
CHECK(old_host.wait_exit(20s) == 4);
CHECK(old_host.printed("phase quarantined") && !old_host.printed("ready 1"));
}
CHECK(load_region_checkpoint(*owner_db, r.key, 5s)->head == final_head->head);
}
// Resource profile of a live process from /proc (M8.1 budgets): peak resident
// set (VmHWM, KiB) and consumed CPU time (user + system, ms).
struct ProcessProfile {
std::uint64_t peak_rss_kib{}, cpu_ms{};
};
ProcessProfile profile_of(pid_t pid) {
ProcessProfile out;
std::ifstream status("/proc/" + std::to_string(pid) + "/status");
for (std::string line; std::getline(status, line);)
if (line.starts_with("VmHWM:"))
out.peak_rss_kib = std::stoull(line.substr(6));
std::ifstream stat("/proc/" + std::to_string(pid) + "/stat");
std::string text((std::istreambuf_iterator<char>(stat)), {});
// Fields after the parenthesized command: state is field 3; utime/stime 14/15.
std::istringstream fields(text.substr(text.rfind(')') + 2));
std::vector<std::string> f{std::istream_iterator<std::string>(fields), {}};
CHECK(f.size() > 13 && out.peak_rss_kib > 0);
const auto ticks = std::stoull(f[11]) + std::stoull(f[12]);
out.cpu_ms = ticks * 1000 / static_cast<std::uint64_t>(::sysconf(_SC_CLK_TCK));
return out;
}
std::uint64_t percentile(std::vector<std::uint64_t> v, unsigned p) {
CHECK(!v.empty());
std::ranges::sort(v);
return v[std::min(v.size() - 1, (v.size() * p) / 100)];
}
std::uint64_t ms_between(Clock::time_point a, Clock::time_point b) {
return static_cast<std::uint64_t>(
std::chrono::duration_cast<std::chrono::milliseconds>(b - a).count());
}
// Sends inputs first..last paced at the 50 ms tick; returns send times.
std::vector<Clock::time_point> paced(Child& server, std::uint64_t first, std::uint64_t last) {
std::vector<Clock::time_point> sent;
const auto start = Clock::now();
for (auto tick = first; tick <= last; ++tick) {
const auto due = start + std::chrono::milliseconds{50 * (tick - first)};
if (const auto now = Clock::now(); due > now)
server.pump(std::chrono::duration_cast<std::chrono::milliseconds>(due - now));
server.send(input_line(tick, true));
sent.push_back(Clock::now());
}
return sent;
}
std::optional<Clock::time_point> ack_time(const Child& server, std::uint64_t tick) {
const auto prefix = "ack " + request_hex(tick) + " " + std::to_string(tick) + " ";
for (std::size_t i = 0; i < server.lines().size(); ++i)
if (starts(server.lines()[i], prefix))
return server.times()[i];
return std::nullopt;
}
Clock::time_point ready_time(const Child& server) {
for (std::size_t i = 0; i < server.lines().size(); ++i)
if (server.lines()[i] == "ready 1")
return server.times()[i];
CHECK(false);
return {};
}
// The M7 exit fixture: production lease policy (20 s lease, 5 s renewal) and
// flush interval (1 s), one lineage region on a local socket and filesystem.
// Measures acknowledgement latency, worker-kill-to-ready recovery against the
// proposed 60 s target, and the cost of verification, audit and clone.
void durability_qualification() {
const auto r = provision(93, 0xDC);
const auto args =
with(with(server_args(r), "--lease-profile", "default"), "--flush-ms", "1000");
std::vector<std::uint64_t> latency, kill_to_ready;
std::uint64_t death_latency = 0, peak_rss_kib = 0;
std::vector<std::uint64_t> cpu_ms, busy_ms; // Per owner: CPU used, wall time serving.
std::uint64_t next = 1;
std::uint64_t epoch = 1;
for (int round = 0; round < 3; ++round) {
Child server{args};
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 60s));
CHECK(server.printed("phase running epoch=" + std::to_string(epoch) + " "));
const auto first = next;
const auto last = next + 99;
const auto sent = paced(server, first, last);
CHECK(server.wait_line("ack " + request_hex(last) + " " + std::to_string(last) + " ", 30s));
for (auto tick = first; tick <= last; ++tick) {
const auto at = ack_time(server, tick);
CHECK(at.has_value());
const auto ms = ms_between(sent[tick - first], at.value_or(Clock::time_point{}));
if (tick == 20)
death_latency = ms;
else
latency.push_back(ms);
}
// Mid-batch force kill: more inputs are in flight, none acknowledged yet.
server.send(input_lines(last + 1, last + 10, true));
server.pump(200ms);
const auto acked = max_acked(server);
const auto used = profile_of(server.pid());
peak_rss_kib = std::max(peak_rss_kib, used.peak_rss_kib);
cpu_ms.push_back(used.cpu_ms);
busy_ms.push_back(ms_between(sent.front(), Clock::now()));
server.signal(SIGKILL);
const auto killed = Clock::now();
CHECK(server.wait_exit(20s) == 128 + SIGKILL);
const auto committed = head_of(r.key);
CHECK(acked <= committed.tick && acked >= last); // No acknowledged input lost.
next = committed.tick + 1;
++epoch;
if (round == 2)
break;
Child successor{args};
CHECK(successor.wait_for([](const std::string& l) { return l == "ready 1"; }, 60s));
kill_to_ready.push_back(ms_between(killed, ready_time(successor)));
CHECK(successor.printed("phase running epoch=" + std::to_string(epoch) +
" committed=" + std::to_string(committed.tick) + " "));
successor.signal(SIGTERM);
CHECK(successor.wait_exit(20s) == 0);
++epoch;
}
CHECK(decisions_of(r.key) == 1);
for (const auto ms : kill_to_ready)
CHECK(ms < 60000); // The declared target for this fixture.
std::printf("qualification ack_latency_ms flush=1000 n=%zu p50=%llu p99=%llu max=%llu "
"death=%llu\n",
latency.size(), static_cast<unsigned long long>(percentile(latency, 50)),
static_cast<unsigned long long>(percentile(latency, 99)),
static_cast<unsigned long long>(percentile(latency, 100)),
static_cast<unsigned long long>(death_latency));
std::printf("qualification kill_to_ready_ms lease=20000 renew=5000 runs=%zu first=%llu "
"second=%llu target=60000\n",
kill_to_ready.size(), static_cast<unsigned long long>(kill_to_ready[0]),
static_cast<unsigned long long>(kill_to_ready[1]));
std::uint64_t cpu_total = 0, busy_total = 0;
for (std::size_t i = 0; i < cpu_ms.size(); ++i) {
cpu_total += cpu_ms[i];
busy_total += busy_ms[i];
}
std::printf(
"qualification server_profile owners=%zu peak_rss_kib=%llu cpu_ms=%llu "
"wall_ms=%llu cpu_permille=%llu\n",
cpu_ms.size(), static_cast<unsigned long long>(peak_rss_kib),
static_cast<unsigned long long>(cpu_total), static_cast<unsigned long long>(busy_total),
static_cast<unsigned long long>(busy_total == 0 ? 0 : cpu_total * 1000 / busy_total));
// Verification cost on the final head (no lease needed to read).
auto reader = connect_to(env("PGDATABASE"), env("EVO_PG_RUNTIME_USER"));
auto objects = store(r);
const std::array<Sha256Digest, 1> retained{b1};
std::vector<std::uint64_t> verify_us;
std::uint64_t payload_bytes = 0;
for (int i = 0; i < 10; ++i) {
const auto start = Clock::now();
auto recovered = recover_lineage_checkpoint(*reader, *objects, r.key, retained, budget, 5s);
CHECK(recovered);
payload_bytes = recovered->manifest.payload_bytes;
verify_us.push_back(static_cast<std::uint64_t>(
std::chrono::duration_cast<std::chrono::microseconds>(Clock::now() - start).count()));
}
const auto audit_start = Clock::now();
auto audited = audit_region_history(*reader, *objects, r.key, retained, budget, 1000, 5s);
const auto audit_ms = ms_between(audit_start, Clock::now());
CHECK(audited);
auto clone_owner = connect_to(env("EVO_PG_CLONE_DATABASE"), env("PGUSER"));
const ServerRegion target{{8, 93}, fs::path(env("PGHOST")) / "qualify-clone-93"};
fs::remove_all(target.root);
fs::create_directory(target.root);
fs::permissions(target.root, fs::perms::owner_all, fs::perm_options::replace);
const auto clone_start = Clock::now();
auto cloned = clone_region(*reader, *objects, r.key, *clone_owner, *store(target), target.key,
filled<16>(0xDD), retained, budget, 5s);
const auto clone_ms = ms_between(clone_start, Clock::now());
CHECK(cloned);
std::printf("qualification verify_head_us payload_bytes=%llu n=10 p50=%llu max=%llu\n",
static_cast<unsigned long long>(payload_bytes),
static_cast<unsigned long long>(percentile(verify_us, 50)),
static_cast<unsigned long long>(percentile(verify_us, 100)));
std::printf("qualification audit_ms commits=%llu ticks=%llu ms=%llu clone_ms=%llu\n",
static_cast<unsigned long long>(audited->commits),
static_cast<unsigned long long>(audited->end_tick),
static_cast<unsigned long long>(audit_ms),
static_cast<unsigned long long>(clone_ms));
}
// Review fixes (schema v9): builds, authority and world classification follow one
// lock order. Each case holds a transaction open, proves the contender is blocked
// on a lock in the database, commits, and checks the contender's outcome.
struct Race {
std::string database;
// Commits the holder's open transaction once `contender` is waiting on a lock.
template <typename F> auto run(Postgres& holder, F contender) {
std::optional<decltype(contender())> result;
std::thread thread([&] { result.emplace(contender()); });
auto observer = connect_to(database, env("PGUSER"));
const std::array<std::string, 1> p{database};
bool waiting = false;
for (int i = 0; i < 500 && !waiting; ++i) {
auto rows = observer->query("SELECT count(*)::text FROM pg_catalog.pg_stat_activity "
"WHERE datname = $1 AND wait_event_type = 'Lock'",
p, 5s);
waiting = rows && rows->size() == 1 && (*rows)[0][0].value_or("0") != "0";
if (!waiting)
std::this_thread::sleep_for(20ms);
}
CHECK(holder.query("COMMIT", {}, 5s));
thread.join();
CHECK(waiting);
if (!result.has_value())
throw std::runtime_error("race contender produced no result");
return std::move(*result);
}
};
std::string bytea_of(std::span<const std::uint8_t> bytes) {
constexpr char digits[] = "0123456789abcdef";
std::string out = "\\x";
for (const auto b : bytes) {
out.push_back(digits[b >> 4U]);
out.push_back(digits[b & 0x0FU]);
}
return out;
}
std::string build_literal(std::uint8_t b) {
char out[64];
std::snprintf(out, sizeof out, "decode(repeat('%02x', 32), 'hex')", b);
return out;
}
void concurrent_authority_and_classification() {
const std::string database = "evo_race";
CHECK(shell("createdb " + database + " && PGDATABASE=" + database +
" EVO_PG_RUNTIME_ROLE=evo_runtime_test bash \"$EVO_SOURCE_ROOT/"
"tools/migrate-postgres.sh\" > /dev/null") == 0);
auto owner_db = connect_to(database, env("PGUSER"));
auto cellular = genesis_manifest({7, 90}, 9, filled<16>(0x70), 50000, digest_of(0xC4), 100,
filled<32>(0xA9), CheckpointProfile::cellular_lives_v1);
CHECK(cellular);
const auto cellular_bytes = encode_manifest(*cellular);
CHECK(cellular_bytes);
const std::array cellular_parameter{bytea_of(*cellular_bytes)};
const auto parsed_cellular =
owner_db->query("SELECT (evolution.parse_checkpoint_manifest($1::bytea)).profile::text",
cellular_parameter, 5s);
CHECK(parsed_cellular && parsed_cellular->size() == 1 &&
(*parsed_cellular)[0][0].value_or("") == "4");
// encode_manifest refuses unsupported bundles, so corrupt the encoded archive
// version (u32 slot 8 of the bundle at offset 160) to reach the SQL parser.
auto wrong_cellular_bytes = *cellular_bytes;
++wrong_cellular_bytes[160 + (7 * 4)];
const std::array wrong_cellular_parameter{bytea_of(wrong_cellular_bytes)};
const auto refused_cellular = owner_db->query(
"SELECT evolution.parse_checkpoint_manifest($1::bytea)", wrong_cellular_parameter, 5s);
CHECK(!refused_cellular && sqlstate(refused_cellular.error()) == "EVL06");
CHECK(owner_db->query("INSERT INTO evolution.retained_builds(build_identity) SELECT "
"decode(repeat(b, 32), 'hex') FROM unnest(ARRAY['c1', 'c2', 'c3', "
"'c4']) AS b",
{}, 5s));
const Sha256Digest c2 = digest_of(0xC2), c3 = digest_of(0xC3), c4 = digest_of(0xC4);
const RegionKey key{7, 1};
auto genesis = genesis_manifest(key, 9, filled<16>(0x61), 50000, c4, 100, filled<32>(0xA0));
CHECK(genesis);
const std::array<std::string, 1> gp{bytea_of(*encode_manifest(*genesis))};
CHECK(owner_db->query("SELECT evolution.provision_region_checkpoint($1::bytea)", gp, 5s));
Race race{database};
const auto barrier_holding_lease_row = [&](std::uint8_t build) {
CHECK(owner_db->query("BEGIN", {}, 5s));
CHECK(owner_db->query("SELECT 1 FROM evolution.region_leases WHERE world_id = 7 AND "
"region_id = 1 FOR UPDATE",
{}, 5s));
CHECK(scalar(*owner_db, "SELECT evolution.upgrade_barrier(" + build_literal(build) +
")::text") == "0");
};
// Build-less acquisition queued behind the first barrier is refused after it.
barrier_holding_lease_row(0xC1);
auto legacy = race.run(*owner_db, [&] {
auto runtime = connect_to(database, env("EVO_PG_RUNTIME_USER"));
return legacy_acquire(*runtime, key);
});
CHECK(!legacy && legacy.error() == "EVL06");
// Build-aware acquisition for the build being retired is refused after COMMIT.
barrier_holding_lease_row(0xC2);
auto retired = race.run(*owner_db, [&] {
auto runtime = connect_to(database, env("EVO_PG_RUNTIME_USER"));
return acquire_region_lease(*runtime, key, c2, 20000ms, 5s);
});
CHECK(!retired && retired.error().kind == LeaseStoreError::unsupported);
CHECK(scalar(*owner_db, "SELECT (holder IS NULL)::text FROM evolution.region_leases WHERE "
"world_id = 7 AND region_id = 1") == "true");
// The new build acquires; the rejected one never held anything to renew.
auto runtime = connect_to(database, env("EVO_PG_RUNTIME_USER"));
auto current = acquire_region_lease(*runtime, key, c3, 20000ms, 5s);
CHECK(current);
auto stale = renew_region_lease(*runtime, key, {filled<16>(0x77), 1, 9, 20000ms}, 20000ms, 5s);
CHECK(!stale && stale.error().kind == LeaseStoreError::fenced);
CHECK(release_region_lease(*runtime, key, *current, 5s));
// Two barriers for the last two current builds: exactly one may succeed.
CHECK(owner_db->query("BEGIN", {}, 5s));
CHECK(scalar(*owner_db,
"SELECT evolution.upgrade_barrier(" + build_literal(0xC3) + ")::text") == "0");
auto last = race.run(*owner_db, [&] {
auto other = connect_to(database, env("PGUSER"));
return other->query("SELECT evolution.upgrade_barrier(" + build_literal(0xC4) + ")", {},
5s);
});
CHECK(!last && sqlstate(last.error()) == "22023");
CHECK(scalar(*owner_db, "SELECT count(*)::text FROM evolution.retained_builds WHERE "
"retired_at IS NULL") == "1");
// Classification: a synthetic source cut (absent from this database) and clone
// manifests naming it; the database never reads payload objects.
const auto source_cut = [&](std::uint64_t world, std::uint8_t seed) {
auto m =
genesis_manifest({world, 1}, 9, filled<16>(seed), 50000, c4, 100, filled<32>(0xA1));
CHECK(m);
return *m;
};
const auto clone_sql = [&](Postgres& db, RegionKey target, std::uint64_t source_world,
std::uint8_t seed) {
auto m = clone_manifest(source_cut(source_world, static_cast<std::uint8_t>(seed + 1)),
filled<32>(0xD5), target, filled<16>(seed), 100, filled<32>(0xA2),
{0, {}, 0, {}});
CHECK(m);
const std::array<std::string, 3> p{bytea_of(*encode_manifest(*m)),
std::to_string(source_world), "1"};
return db.query("SELECT evolution.provision_region_clone($1::bytea, $2::evolution.u64, "
"$3::evolution.u64, 0, '\\x'::bytea)",
p, 5s);
};
const auto contender = [&](RegionKey target, std::uint64_t source_world, std::uint8_t seed) {
return [&, target, source_world, seed] {
auto other = connect_to(database, env("PGUSER"));
return clone_sql(*other, target, source_world, seed);
};
};
// Different sources into one target world: exactly one source is accepted.
CHECK(owner_db->query("BEGIN", {}, 5s));
CHECK(clone_sql(*owner_db, {2, 1}, 10, 0x62));
auto mixed = race.run(*owner_db, contender({2, 2}, 11, 0x64));
CHECK(!mixed && sqlstate(mixed.error()) == "22023");
// An original region racing a clone into the same world.
auto original = genesis_manifest({3, 1}, 9, filled<16>(0x66), 50000, c4, 100, filled<32>(0xA0));
CHECK(original);
const std::array<std::string, 1> op{bytea_of(*encode_manifest(*original))};
CHECK(owner_db->query("BEGIN", {}, 5s));
CHECK(owner_db->query("SELECT evolution.provision_region_checkpoint($1::bytea)", op, 5s));
auto against_original = race.run(*owner_db, contender({3, 2}, 10, 0x68));
CHECK(!against_original && sqlstate(against_original.error()) == "22023");
// Same source, different regions: both succeed.
CHECK(owner_db->query("BEGIN", {}, 5s));
CHECK(clone_sql(*owner_db, {4, 1}, 10, 0x6A));
auto same_source = race.run(*owner_db, contender({4, 2}, 10, 0x6C));
CHECK(same_source);
CHECK(scalar(*owner_db, "SELECT string_agg(world_id || '/' || region_id || '<' || "
"source_world, ',' ORDER BY world_id, region_id) FROM "
"evolution.region_origins") == "2/1<10,4/1<10,4/2<10");
}
} // namespace
void run_region_migration_tests() {
concurrent_authority_and_classification();
durability_qualification();
upgrade_across_migration_and_barrier();
}