// M7.9b schema migration, upgrade barrier and durability qualification (ADR-0068) // against real PostgreSQL, real evo-server processes and the checked-in migrations // (EVO_SOURCE_ROOT). Run by tools/ci/postgres-check.sh; prints measured latencies. #include "evo/persistence/region_clone.hpp" #include "evo/persistence/sha256.hpp" #include "fixtures/server_process.hpp" #include #include #include #include #include #include void run_region_migration_tests(); namespace { using namespace evo::test::process; using namespace evo::persistence; void check(bool value, std::source_location at = std::source_location::current()) { if (!value) throw std::runtime_error("region migration test line " + std::to_string(at.line())); } #define CHECK(...) check(static_cast(__VA_ARGS__)) constexpr std::size_t budget = 8000000; const std::string b1_text = [] { std::string s; for (int i = 0; i < 32; ++i) s += "b1"; return s; }(); const std::string b2_text = [] { std::string s; for (int i = 0; i < 32; ++i) s += "b2"; return s; }(); Sha256Digest digest_of(std::uint8_t b) { Sha256Digest d{}; d.fill(b); return d; } const Sha256Digest b1 = digest_of(0xB1), b2 = digest_of(0xB2); template std::array filled(std::uint8_t value) { std::array out{}; out.fill(value); return out; } std::unique_ptr connect_to(const std::string& database, const std::string& user) { auto made = Postgres::connect({env("PGHOST"), {}, env("PGPORT"), database, user, {}, {}}, 5s); CHECK(made); return std::move(*made); } std::string sqlstate(const PgFailure& f) { return f.sqlstate.data(); } std::string scalar(Postgres& db, const std::string& sql) { auto rows = db.query(sql, {}, 5s); CHECK(rows && rows->size() == 1 && (*rows)[0].size() == 1); return (*rows)[0][0].value_or(""); } int shell(const std::string& command) { Child c{{"/bin/bash", "-c", command}}; return c.wait_exit(120s); } std::vector with(std::vector args, const std::string& flag, const std::string& value) { const auto at = std::ranges::find(args, flag); CHECK(at != args.end() && at + 1 != args.end()); *(at + 1) = value; return args; } std::unique_ptr store(const ServerRegion& r) { auto made = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes); CHECK(made); return std::move(*made); } std::optional parse_uuid(const std::string& text) { std::string compact; for (const char c : text) if (c != '-') compact.push_back(c); if (compact.size() != 32) return std::nullopt; Uuid out{}; for (std::size_t i = 0; i < 16; ++i) out[i] = static_cast(std::stoul(compact.substr(2 * i, 2), nullptr, 16)); return out; } // A pre-v8 writer: the build-less acquisition every earlier host used. std::expected legacy_acquire(Postgres& db, RegionKey key) { const std::array p{std::to_string(key.world), std::to_string(key.region)}; auto rows = db.query("SELECT * FROM evolution.acquire_region_lease($1::evolution.u64, " "$2::evolution.u64, 20000)", p, 5s); if (!rows) return std::unexpected{sqlstate(rows.error())}; CHECK(rows->size() == 1 && (*rows)[0].size() == 4); const auto& r = (*rows)[0]; const auto holder = parse_uuid(r[0].value_or("")); CHECK(holder.has_value()); return LeaseGrant{holder.value_or(Uuid{}), std::stoull(r[1].value_or("0")), std::stoull(r[2].value_or("0")), 20000ms}; } const char* const sha_function = "s(){ sha256sum \"$EVO_SOURCE_ROOT/migrations/$1\" | cut -d' ' -f1; }; "; std::string history(Postgres& owner) { return scalar(owner, "SELECT string_agg(version::text, ',' ORDER BY version) FROM " "evolution.schema_migrations"); } // An existing v7 database is upgraded while an older writer holds a lease. A // migration that fails leaves v7 exactly and the writer unaffected; the real one // commits under it. The barrier then retires the old build: the writer is fenced // and can never regain authority, while the new build recovers the whole history. void upgrade_across_migration_and_barrier() { const std::string database = "evo_online"; const std::string runtime_user = env("EVO_PG_RUNTIME_USER"); CHECK(shell(std::string(sha_function) + "cd \"$EVO_SOURCE_ROOT/migrations\" && createdb " + database + " && PGDATABASE=" + database + " psql -X -q -v ON_ERROR_STOP=1 -1 -v migration_sha=$(s 0001_region_bootstrap.sql)" " -v migration_sha_2=$(s 0002_region_fencing.sql)" " -v migration_sha_3=$(s 0003_region_checkpoints.sql)" " -v migration_sha_4=$(s 0004_lineage_profile.sql)" " -v migration_sha_5=$(s 0005_batch_decisions.sql)" " -v migration_sha_6=$(s 0006_commit_lookup.sql)" " -v migration_sha_7=$(s 0007_region_clones.sql)" " -v runtime_role=evo_runtime_test -f 0001_region_bootstrap.sql" " -f 0002_region_fencing.sql -f 0003_region_checkpoints.sql" " -f 0004_lineage_profile.sql -f 0005_batch_decisions.sql" " -f 0006_commit_lookup.sql -f 0007_region_clones.sql > /dev/null") == 0); auto owner_db = connect_to(database, env("PGUSER")); CHECK(history(*owner_db) == "1,2,3,4,5,6,7"); CHECK(owner_db->query("INSERT INTO evolution.retained_builds(build_identity) VALUES " "(decode(repeat('b1', 32), 'hex')), (decode(repeat('b2', 32), 'hex'))", {}, 5s)); const ServerRegion r{{7, 91}, fs::path(env("PGHOST")) / "online-91"}; fs::remove_all(r.root); fs::create_directory(r.root); fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace); auto objects = store(r); auto genesis = provision_lineage_region(*owner_db, *objects, r.key, evo::test::region_identity.frame, *evo::test::lineage_region(), filled<16>(0xDA), b1, budget, 5s); CHECK(genesis); // The old writer: build-less lease, publishing with build b1. auto runtime = connect_to(database, runtime_user); auto lease = legacy_acquire(*runtime, r.key); CHECK(lease); auto state = evo::test::lineage_region(); RegionBatch batch{*state}; CommittedCheckpoint head = *genesis; std::uint8_t commit_seed = 0xDB; const auto publish_next = [&](std::uint64_t ticks) { for (std::uint64_t i = 0; i < ticks && !batch.sealed(); ++i) CHECK(batch.advance(evo::test::region_input(batch.end_tick() + 1, true))); batch.seal(); return publish_lineage_batch(*runtime, *objects, *lease, head, batch, filled<16>(commit_seed++), b1, budget, 5s); }; const auto confirm = [&](const std::expected& done) { CHECK(done && batch.confirm(done->committed.head.end_tick)); head = done->committed; }; confirm(publish_next(64)); // Ticks 1..20: the consumer's death seals it. CHECK(head.head.end_tick == 20); // A v8 migration that fails at its last statement changes nothing. CHECK(shell(std::string(sha_function) + "cd \"$EVO_SOURCE_ROOT\" && PGDATABASE=" + database + " psql -X -q -v ON_ERROR_STOP=1 -v migration_sha=$(s 0001_region_bootstrap.sql)" " -v migration_sha_2=$(s 0002_region_fencing.sql)" " -v migration_sha_3=$(s 0003_region_checkpoints.sql)" " -v migration_sha_4=$(s 0004_lineage_profile.sql)" " -v migration_sha_5=$(s 0005_batch_decisions.sql)" " -v migration_sha_6=$(s 0006_commit_lookup.sql)" " -v migration_sha_7=$(s 0007_region_clones.sql) -v migration_sha_8=invalid" " -v migration_sha_9=$(s 0009_serialized_authority.sql)" " -v migration_sha_10=$(s 0010_account_control.sql)" " -v migration_sha_11=$(s 0011_region_actions.sql)" " -v migration_sha_12=$(s 0012_probe_handoff.sql)" " -v migration_sha_13=$(s 0013_probe_incoming.sql)" " -v migration_sha_14=$(s 0014_probe_overdue.sql)" " -v migration_sha_15=$(s 0015_life_reservations.sql)" " -v migration_sha_16=$(s 0016_primitive_publication.sql)" " -v migration_sha_17=$(s 0017_life_request_outcomes.sql)" " -v migration_sha_18=$(s 0018_cellular_profile_admission.sql)" " -v migration_sha_19=$(s 0019_cellular_action_records.sql)" " -v migration_sha_20=$(s 0020_cellular_publication.sql)" " -v migration_sha_21=$(s 0021_cellular_trace_and_recovery.sql)" " -v migration_sha_22=$(s 0022_cellular_birth_trace_bound.sql)" " -v migration_sha_23=$(s 0023_cellular_birth_publication.sql)" " -v migration_sha_24=$(s 0024_primitive_life_owner_lookup.sql)" " -v migration_sha_25=$(s 0025_cellular_v2_bundle.sql)" " -v migration_sha_26=$(s 0026_cellular_profile_migration.sql)" " -v migration_sha_27=$(s 0027_cellular_v3_migration.sql)" " -v migration_sha_28=$(s 0028_cellular_v3_trace_bounds.sql)" " -v migration_sha_29=$(s 0029_cellular_mutation_reservations.sql)" " -v migration_sha_30=$(s 0030_cellular_birth_publication_binding.sql)" " -v migration_sha_31=$(s 0031_fix_cellular_birth_publication_record.sql)" " -v migration_sha_32=$(s 0032_fix_cellular_v3_manifest_normalization.sql)" " -v migration_sha_33=$(s 0033_fix_cellular_action_tick_offset.sql)" " -v migration_sha_34=$(s 0034_cellular_v4_engulf_bundle.sql)" " -v migration_sha_35=$(s 0035_cellular_mutation_publication.sql)" " -v migration_sha_36=$(s 0036_cellular_v5_budding_bundle.sql)" " -v migration_sha_37=$(s 0037_cellular_life_endings.sql)" " -v migration_sha_38=$(s 0038_cellular_v6_presence_bundle.sql)" " -v migration_sha_39=$(s 0039_account_identities.sql)" " -v migration_sha_40=$(s 0040_cellular_mutation_decline.sql)" " -v migration_sha_41=$(s 0041_cellular_v7_balance_bundle.sql)" " -v migration_sha_42=$(s 0042_cellular_v8_reproduction_bundle.sql)" " -v migration_sha_43=$(s 0043_cellular_v9_foraging_bundle.sql)" " -v migration_sha_44=$(s 0044_exact_population_publication.sql)" " -v migration_sha_45=$(s 0045_owned_population_intervals.sql)" " -v migration_sha_46=$(s 0046_boundary_accounting_upgrade.sql)" " -v migration_sha_47=$(s 0047_cellular_migration_freeze.sql)" " -v migration_sha_48=$(s 0048_cellular_migration_staging.sql)" " -v migration_sha_49=$(s 0049_cellular_migration_publication.sql)" " -v migration_sha_50=$(s 0050_world_region_catalog.sql)" " -v migration_sha_51=$(s 0051_database_history_domain.sql)" " -v migration_sha_52=$(s 0052_census_pending_migrations.sql)" " -v migration_sha_53=$(s 0053_funded_cellular_maturation.sql)" " -v migration_sha_54=$(s 0054_maturing_cellular_migration.sql)" " -v migration_sha_55=$(s 0055_retained_cellular_plans.sql)" " -v migration_sha_56=$(s 0056_cellular_environment_publication.sql)" " -v migration_sha_57=$(s 0057_photosynthesis_bundle.sql)" " -v migration_sha_58=$(s 0058_liquid_bundle.sql)" " -v migration_sha_59=$(s 0059_durable_mutation_queue.sql)" " -v migration_sha_60=$(s 0060_cellular_revamp_bundle.sql)" " -v migration_sha_61=$(s 0061_admin_read_views.sql)" " -v migration_sha_62=$(s 0062_session_windows.sql)" " -v migration_sha_63=$(s 0063_admin_commands.sql)" " -v migration_sha_64=$(s 0064_account_admin.sql)" " -v runtime_role=evo_runtime_test -f migrations/apply.sql" " > /dev/null 2>&1") != 0); CHECK(history(*owner_db) == "1,2,3,4,5,6,7"); CHECK(scalar(*owner_db, "SELECT count(*) FROM information_schema.columns WHERE " "table_schema = 'evolution' AND column_name = 'retired_at'") == "0"); CHECK(scalar(*owner_db, "SELECT (to_regprocedure('evolution.acquire_region_lease(" "evolution.u64, evolution.u64, integer, bytea)') IS NULL)::text") == "true"); CHECK(renew_region_lease(*runtime, r.key, *lease, 20000ms, 5s)); confirm(publish_next(20)); // 21..40, still v7. // The real migration commits while the writer holds its lease; it keeps going. CHECK(shell("PGDATABASE=" + database + " EVO_PG_RUNTIME_ROLE=evo_runtime_test bash \"$EVO_SOURCE_ROOT/" "tools/migrate-postgres.sh\" > /dev/null") == 0); CHECK(history(*owner_db) == "1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24," "25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46," "47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64"); CHECK(scalar(*owner_db, "SELECT (c.condeferrable AND c.condeferred AND " "c.confrelid = 'evolution.region_commits'::regclass)::text " "FROM pg_catalog.pg_constraint c WHERE c.conrelid = " "'evolution.primitive_life_bindings'::regclass AND " "c.conname = 'primitive_life_bindings_commit_id_fkey'") == "true"); CHECK(scalar(*owner_db, "SELECT (t.tgdeferrable AND t.tginitdeferred)::text FROM pg_catalog.pg_trigger t " "WHERE t.tgrelid = 'evolution.primitive_life_bindings'::regclass AND " "t.tgname = 'life_binding_publication_evidence' AND NOT t.tgisinternal") == "true"); CHECK(scalar(*owner_db, "SELECT (to_regprocedure('evolution.primitive_life_owner(" "evolution.u64,evolution.u64,uuid,evolution.u64,uuid)') IS NOT NULL)::text") == "true"); CHECK(scalar(*owner_db, "SELECT has_function_privilege('evo_runtime_login', " "'evolution.primitive_life_owner(evolution.u64,evolution.u64,uuid," "evolution.u64,uuid)', 'EXECUTE')::text") == "true"); CHECK(scalar(*owner_db, "SELECT has_table_privilege('evo_runtime_login', " "'evolution.life_reservations', 'SELECT')::text") == "false"); CHECK(scalar(*owner_db, "SELECT count(*) FROM information_schema.columns WHERE " "table_schema = 'evolution' AND table_name = 'region_actions' AND " "column_name IN ('input_bytes', 'result')") == "2"); CHECK(scalar(*owner_db, "SELECT (to_regprocedure('evolution.find_region_action_record(" "evolution.u64,evolution.u64,uuid,evolution.u64,uuid,evolution.u64)') " "IS NOT NULL)::text") == "true"); CHECK(scalar(*owner_db, "SELECT evolution.valid_cellular_action_input(decode('01' || " "repeat('00', 23), 'hex'), 1)::text") == "true"); CHECK(scalar(*owner_db, "SELECT evolution.valid_cellular_action_input(decode(repeat('00', 24), " "'hex'), 1)::text") == "false"); // Schema v34 (ADR-0124): target kind 2 is engulf, with no coordinates. Schema v36 // (ADR-0125): kind 3 is a bud, with no coordinates; kind 4 is unknown. CHECK(scalar(*owner_db, "SELECT evolution.valid_cellular_action_input(decode('01' || " "repeat('00', 11) || '02' || repeat('00', 11), 'hex'), 1)::text") == "true"); CHECK(scalar(*owner_db, "SELECT evolution.valid_cellular_action_input(decode('01' || " "repeat('00', 11) || '02' || repeat('00', 3) || '01' || " "repeat('00', 7), 'hex'), 1)::text") == "false"); CHECK(scalar(*owner_db, "SELECT evolution.valid_cellular_action_input(decode('01' || " "repeat('00', 11) || '03' || repeat('00', 11), 'hex'), 1)::text") == "true"); CHECK(scalar(*owner_db, "SELECT evolution.valid_cellular_action_input(decode('01' || " "repeat('00', 11) || '03' || repeat('00', 3) || '01' || " "repeat('00', 7), 'hex'), 1)::text") == "false"); CHECK(scalar(*owner_db, "SELECT evolution.valid_cellular_action_input(decode('01' || " "repeat('00', 11) || '04' || repeat('00', 11), 'hex'), 1)::text") == "false"); CHECK(scalar(*owner_db, "SELECT evolution.check_cellular_action_block(decode(repeat('01', 16) || " "'0100000000000000' || '0100000000000000' || repeat('00', 16), 'hex'), " "0, 1)::text") == "1"); CHECK( scalar(*owner_db, "SELECT " "(to_regprocedure('evolution.publish_cellular_checkpoint(bytea,bytea,bytea,bytea)') " "IS NOT NULL)::text") == "true"); CHECK( scalar( *owner_db, "SELECT has_function_privilege('evo_runtime_login', " "'evolution.publish_cellular_checkpoint(bytea,bytea,bytea,bytea)', 'EXECUTE')::text") == "true"); CHECK(scalar(*owner_db, "SELECT has_function_privilege('evo_runtime_login', " "'evolution.publish_cellular_checkpoint(bytea,bytea,bytea)', 'EXECUTE')::text") == "false"); CHECK(scalar(*owner_db, "SELECT (to_regprocedure('evolution.load_cellular_publication_births(" "evolution.u64,evolution.u64,uuid,evolution.u64,uuid)') IS NOT NULL)::text") == "true"); CHECK(scalar(*owner_db, "SELECT has_function_privilege('evo_runtime_login', " "'evolution.resolve_cellular_region_commit(evolution.u64,evolution.u64,uuid," "bytea,bytea,bytea,bytea)', 'EXECUTE')::text") == "true"); CHECK(scalar(*owner_db, "SELECT has_table_privilege('evo_runtime_login', " "'evolution.cellular_publications', 'INSERT')::text") == "false"); CHECK(scalar(*owner_db, "SELECT has_function_privilege('evo_runtime_login', " "'evolution.publish_region_checkpoint_core(bytea,bytea)', 'EXECUTE')::text") == "false"); CHECK(renew_region_lease(*runtime, r.key, *lease, 20000ms, 5s)); confirm(publish_next(20)); // 41..60, now v8. CHECK(head.head.end_tick == 60); // Barrier guards: the last current build cannot be retired; runtime cannot call it. const std::string retire_b1 = "SELECT evolution.upgrade_barrier(decode(repeat('b1', 32), " "'hex'))::text"; auto denied = runtime->query(retire_b1, {}, 5s); CHECK(!denied && sqlstate(denied.error()) == "42501"); // The barrier: b1 retired and every holder fenced, in one transaction. CHECK(scalar(*owner_db, retire_b1) == "1"); auto last_b2 = owner_db->query( "SELECT evolution.upgrade_barrier(decode(repeat('b2', 32), 'hex'))", {}, 5s); CHECK(!last_b2 && sqlstate(last_b2.error()) == "22023"); // The old writer is fenced and cannot regain authority by any path. auto after = publish_next(10); CHECK(!after && after.error().kind == PublicationError::rejected && after.error().store.kind == LeaseStoreError::fenced); auto renewed = renew_region_lease(*runtime, r.key, *lease, 20000ms, 5s); CHECK(!renewed && renewed.error().kind == LeaseStoreError::fenced); auto legacy = legacy_acquire(*runtime, r.key); CHECK(!legacy && legacy.error() == "EVL06"); auto retired = acquire_region_lease(*runtime, r.key, b1, 20000ms, 5s); CHECK(!retired && retired.error().kind == LeaseStoreError::unsupported); // Authority for b2 still cannot publish a link naming the retired build. auto current = acquire_region_lease(*runtime, r.key, b2, 20000ms, 5s); CHECK(current); lease = *current; auto stale_build = publish_lineage_batch(*runtime, *objects, *lease, head, batch, filled<16>(commit_seed++), b1, budget, 5s); CHECK(!stale_build && stale_build.error().kind == PublicationError::rejected && stale_build.error().store.kind == LeaseStoreError::unsupported); CHECK(release_region_lease(*runtime, r.key, *lease, 5s)); const auto before_new = head.head; // The new build resumes from the committed head, and b1 history stays readable. auto args = with(with(server_args(r), "--pg-database", database), "--build", b2_text); args.insert(args.end(), {"--retained", b1_text}); { Child server{args}; CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s)); CHECK(server.printed("phase running epoch=3 committed=60 ")); server.send(input_lines(61, 90, true)); CHECK(server.wait_line("ack " + request_hex(90) + " 90 ", 30s)); server.signal(SIGTERM); CHECK(server.wait_exit(20s) == 0); } auto final_head = load_region_checkpoint(*owner_db, r.key, 5s); CHECK(final_head && final_head->head.end_tick == 90 && final_head->head.sequence > before_new.sequence); const auto final_manifest = decode_manifest(final_head->manifest); CHECK(final_manifest && final_manifest->build == b2); const std::array both{b1, b2}; const std::array only_new{b2}; auto audited = audit_region_history(*runtime, *objects, r.key, both, budget, 100, 5s); CHECK(audited && audited->commits == final_head->head.sequence + 1 && audited->outcomes == 1); auto without_old = audit_region_history(*runtime, *objects, r.key, only_new, budget, 100, 5s); CHECK(!without_old && without_old.error().kind == AuditError::unsupported && without_old.error().sequence == 0); // A host of the retired build never serves, not even once the region is free. { Child old_host{with(with(server_args(r), "--pg-database", database), "--build", b1_text)}; CHECK(old_host.wait_exit(20s) == 4); CHECK(old_host.printed("phase quarantined") && !old_host.printed("ready 1")); } CHECK(load_region_checkpoint(*owner_db, r.key, 5s)->head == final_head->head); } // Resource profile of a live process from /proc (M8.1 budgets): peak resident // set (VmHWM, KiB) and consumed CPU time (user + system, ms). struct ProcessProfile { std::uint64_t peak_rss_kib{}, cpu_ms{}; }; ProcessProfile profile_of(pid_t pid) { ProcessProfile out; std::ifstream status("/proc/" + std::to_string(pid) + "/status"); for (std::string line; std::getline(status, line);) if (line.starts_with("VmHWM:")) out.peak_rss_kib = std::stoull(line.substr(6)); std::ifstream stat("/proc/" + std::to_string(pid) + "/stat"); std::string text((std::istreambuf_iterator(stat)), {}); // Fields after the parenthesized command: state is field 3; utime/stime 14/15. std::istringstream fields(text.substr(text.rfind(')') + 2)); std::vector f{std::istream_iterator(fields), {}}; CHECK(f.size() > 13 && out.peak_rss_kib > 0); const auto ticks = std::stoull(f[11]) + std::stoull(f[12]); out.cpu_ms = ticks * 1000 / static_cast(::sysconf(_SC_CLK_TCK)); return out; } std::uint64_t percentile(std::vector v, unsigned p) { CHECK(!v.empty()); std::ranges::sort(v); return v[std::min(v.size() - 1, (v.size() * p) / 100)]; } std::uint64_t ms_between(Clock::time_point a, Clock::time_point b) { return static_cast( std::chrono::duration_cast(b - a).count()); } // Sends inputs first..last paced at the 50 ms tick; returns send times. std::vector paced(Child& server, std::uint64_t first, std::uint64_t last) { std::vector sent; const auto start = Clock::now(); for (auto tick = first; tick <= last; ++tick) { const auto due = start + std::chrono::milliseconds{50 * (tick - first)}; if (const auto now = Clock::now(); due > now) server.pump(std::chrono::duration_cast(due - now)); server.send(input_line(tick, true)); sent.push_back(Clock::now()); } return sent; } std::optional ack_time(const Child& server, std::uint64_t tick) { const auto prefix = "ack " + request_hex(tick) + " " + std::to_string(tick) + " "; for (std::size_t i = 0; i < server.lines().size(); ++i) if (starts(server.lines()[i], prefix)) return server.times()[i]; return std::nullopt; } Clock::time_point ready_time(const Child& server) { for (std::size_t i = 0; i < server.lines().size(); ++i) if (server.lines()[i] == "ready 1") return server.times()[i]; CHECK(false); return {}; } // The M7 exit fixture: production lease policy (20 s lease, 5 s renewal) and // flush interval (1 s), one lineage region on a local socket and filesystem. // Measures acknowledgement latency, worker-kill-to-ready recovery against the // proposed 60 s target, and the cost of verification, audit and clone. void durability_qualification() { const auto r = provision(93, 0xDC); const auto args = with(with(server_args(r), "--lease-profile", "default"), "--flush-ms", "1000"); std::vector latency, kill_to_ready; std::uint64_t death_latency = 0, peak_rss_kib = 0; std::vector cpu_ms, busy_ms; // Per owner: CPU used, wall time serving. std::uint64_t next = 1; std::uint64_t epoch = 1; for (int round = 0; round < 3; ++round) { Child server{args}; CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 60s)); CHECK(server.printed("phase running epoch=" + std::to_string(epoch) + " ")); const auto first = next; const auto last = next + 99; const auto sent = paced(server, first, last); CHECK(server.wait_line("ack " + request_hex(last) + " " + std::to_string(last) + " ", 30s)); for (auto tick = first; tick <= last; ++tick) { const auto at = ack_time(server, tick); CHECK(at.has_value()); const auto ms = ms_between(sent[tick - first], at.value_or(Clock::time_point{})); if (tick == 20) death_latency = ms; else latency.push_back(ms); } // Mid-batch force kill: more inputs are in flight, none acknowledged yet. server.send(input_lines(last + 1, last + 10, true)); server.pump(200ms); const auto acked = max_acked(server); const auto used = profile_of(server.pid()); peak_rss_kib = std::max(peak_rss_kib, used.peak_rss_kib); cpu_ms.push_back(used.cpu_ms); busy_ms.push_back(ms_between(sent.front(), Clock::now())); server.signal(SIGKILL); const auto killed = Clock::now(); CHECK(server.wait_exit(20s) == 128 + SIGKILL); const auto committed = head_of(r.key); CHECK(acked <= committed.tick && acked >= last); // No acknowledged input lost. next = committed.tick + 1; ++epoch; if (round == 2) break; Child successor{args}; CHECK(successor.wait_for([](const std::string& l) { return l == "ready 1"; }, 60s)); kill_to_ready.push_back(ms_between(killed, ready_time(successor))); CHECK(successor.printed("phase running epoch=" + std::to_string(epoch) + " committed=" + std::to_string(committed.tick) + " ")); successor.signal(SIGTERM); CHECK(successor.wait_exit(20s) == 0); ++epoch; } CHECK(decisions_of(r.key) == 1); for (const auto ms : kill_to_ready) CHECK(ms < 60000); // The declared target for this fixture. std::printf("qualification ack_latency_ms flush=1000 n=%zu p50=%llu p99=%llu max=%llu " "death=%llu\n", latency.size(), static_cast(percentile(latency, 50)), static_cast(percentile(latency, 99)), static_cast(percentile(latency, 100)), static_cast(death_latency)); std::printf("qualification kill_to_ready_ms lease=20000 renew=5000 runs=%zu first=%llu " "second=%llu target=60000\n", kill_to_ready.size(), static_cast(kill_to_ready[0]), static_cast(kill_to_ready[1])); std::uint64_t cpu_total = 0, busy_total = 0; for (std::size_t i = 0; i < cpu_ms.size(); ++i) { cpu_total += cpu_ms[i]; busy_total += busy_ms[i]; } std::printf( "qualification server_profile owners=%zu peak_rss_kib=%llu cpu_ms=%llu " "wall_ms=%llu cpu_permille=%llu\n", cpu_ms.size(), static_cast(peak_rss_kib), static_cast(cpu_total), static_cast(busy_total), static_cast(busy_total == 0 ? 0 : cpu_total * 1000 / busy_total)); // Verification cost on the final head (no lease needed to read). auto reader = connect_to(env("PGDATABASE"), env("EVO_PG_RUNTIME_USER")); auto objects = store(r); const std::array retained{b1}; std::vector verify_us; std::uint64_t payload_bytes = 0; for (int i = 0; i < 10; ++i) { const auto start = Clock::now(); auto recovered = recover_lineage_checkpoint(*reader, *objects, r.key, retained, budget, 5s); CHECK(recovered); payload_bytes = recovered->manifest.payload_bytes; verify_us.push_back(static_cast( std::chrono::duration_cast(Clock::now() - start).count())); } const auto audit_start = Clock::now(); auto audited = audit_region_history(*reader, *objects, r.key, retained, budget, 1000, 5s); const auto audit_ms = ms_between(audit_start, Clock::now()); CHECK(audited); auto clone_owner = connect_to(env("EVO_PG_CLONE_DATABASE"), env("PGUSER")); const ServerRegion target{{8, 93}, fs::path(env("PGHOST")) / "qualify-clone-93"}; fs::remove_all(target.root); fs::create_directory(target.root); fs::permissions(target.root, fs::perms::owner_all, fs::perm_options::replace); const auto clone_start = Clock::now(); auto cloned = clone_region(*reader, *objects, r.key, *clone_owner, *store(target), target.key, filled<16>(0xDD), retained, budget, 5s); const auto clone_ms = ms_between(clone_start, Clock::now()); CHECK(cloned); std::printf("qualification verify_head_us payload_bytes=%llu n=10 p50=%llu max=%llu\n", static_cast(payload_bytes), static_cast(percentile(verify_us, 50)), static_cast(percentile(verify_us, 100))); std::printf("qualification audit_ms commits=%llu ticks=%llu ms=%llu clone_ms=%llu\n", static_cast(audited->commits), static_cast(audited->end_tick), static_cast(audit_ms), static_cast(clone_ms)); } // Review fixes (schema v9): builds, authority and world classification follow one // lock order. Each case holds a transaction open, proves the contender is blocked // on a lock in the database, commits, and checks the contender's outcome. struct Race { std::string database; // Commits the holder's open transaction once `contender` is waiting on a lock. template auto run(Postgres& holder, F contender) { std::optional result; std::thread thread([&] { result.emplace(contender()); }); auto observer = connect_to(database, env("PGUSER")); const std::array p{database}; bool waiting = false; for (int i = 0; i < 500 && !waiting; ++i) { auto rows = observer->query("SELECT count(*)::text FROM pg_catalog.pg_stat_activity " "WHERE datname = $1 AND wait_event_type = 'Lock'", p, 5s); waiting = rows && rows->size() == 1 && (*rows)[0][0].value_or("0") != "0"; if (!waiting) std::this_thread::sleep_for(20ms); } CHECK(holder.query("COMMIT", {}, 5s)); thread.join(); CHECK(waiting); if (!result.has_value()) throw std::runtime_error("race contender produced no result"); return std::move(*result); } }; std::string bytea_of(std::span bytes) { constexpr char digits[] = "0123456789abcdef"; std::string out = "\\x"; for (const auto b : bytes) { out.push_back(digits[b >> 4U]); out.push_back(digits[b & 0x0FU]); } return out; } std::string build_literal(std::uint8_t b) { char out[64]; std::snprintf(out, sizeof out, "decode(repeat('%02x', 32), 'hex')", b); return out; } void concurrent_authority_and_classification() { const std::string database = "evo_race"; CHECK(shell("createdb " + database + " && PGDATABASE=" + database + " EVO_PG_RUNTIME_ROLE=evo_runtime_test bash \"$EVO_SOURCE_ROOT/" "tools/migrate-postgres.sh\" > /dev/null") == 0); auto owner_db = connect_to(database, env("PGUSER")); auto cellular = genesis_manifest({7, 90}, 9, filled<16>(0x70), 50000, digest_of(0xC4), 100, filled<32>(0xA9), CheckpointProfile::cellular_lives_v1); CHECK(cellular); const auto cellular_bytes = encode_manifest(*cellular); CHECK(cellular_bytes); const std::array cellular_parameter{bytea_of(*cellular_bytes)}; const auto parsed_cellular = owner_db->query("SELECT (evolution.parse_checkpoint_manifest($1::bytea)).profile::text", cellular_parameter, 5s); CHECK(parsed_cellular && parsed_cellular->size() == 1 && (*parsed_cellular)[0][0].value_or("") == "4"); // encode_manifest refuses unsupported bundles, so corrupt the encoded archive // version (u32 slot 8 of the bundle at offset 160) to reach the SQL parser. auto wrong_cellular_bytes = *cellular_bytes; ++wrong_cellular_bytes[160 + (7 * 4)]; const std::array wrong_cellular_parameter{bytea_of(wrong_cellular_bytes)}; const auto refused_cellular = owner_db->query( "SELECT evolution.parse_checkpoint_manifest($1::bytea)", wrong_cellular_parameter, 5s); CHECK(!refused_cellular && sqlstate(refused_cellular.error()) == "EVL06"); CHECK(owner_db->query("INSERT INTO evolution.retained_builds(build_identity) SELECT " "decode(repeat(b, 32), 'hex') FROM unnest(ARRAY['c1', 'c2', 'c3', " "'c4']) AS b", {}, 5s)); const Sha256Digest c2 = digest_of(0xC2), c3 = digest_of(0xC3), c4 = digest_of(0xC4); const RegionKey key{7, 1}; auto genesis = genesis_manifest(key, 9, filled<16>(0x61), 50000, c4, 100, filled<32>(0xA0)); CHECK(genesis); const std::array gp{bytea_of(*encode_manifest(*genesis))}; CHECK(owner_db->query("SELECT evolution.provision_region_checkpoint($1::bytea)", gp, 5s)); Race race{database}; const auto barrier_holding_lease_row = [&](std::uint8_t build) { CHECK(owner_db->query("BEGIN", {}, 5s)); CHECK(owner_db->query("SELECT 1 FROM evolution.region_leases WHERE world_id = 7 AND " "region_id = 1 FOR UPDATE", {}, 5s)); CHECK(scalar(*owner_db, "SELECT evolution.upgrade_barrier(" + build_literal(build) + ")::text") == "0"); }; // Build-less acquisition queued behind the first barrier is refused after it. barrier_holding_lease_row(0xC1); auto legacy = race.run(*owner_db, [&] { auto runtime = connect_to(database, env("EVO_PG_RUNTIME_USER")); return legacy_acquire(*runtime, key); }); CHECK(!legacy && legacy.error() == "EVL06"); // Build-aware acquisition for the build being retired is refused after COMMIT. barrier_holding_lease_row(0xC2); auto retired = race.run(*owner_db, [&] { auto runtime = connect_to(database, env("EVO_PG_RUNTIME_USER")); return acquire_region_lease(*runtime, key, c2, 20000ms, 5s); }); CHECK(!retired && retired.error().kind == LeaseStoreError::unsupported); CHECK(scalar(*owner_db, "SELECT (holder IS NULL)::text FROM evolution.region_leases WHERE " "world_id = 7 AND region_id = 1") == "true"); // The new build acquires; the rejected one never held anything to renew. auto runtime = connect_to(database, env("EVO_PG_RUNTIME_USER")); auto current = acquire_region_lease(*runtime, key, c3, 20000ms, 5s); CHECK(current); auto stale = renew_region_lease(*runtime, key, {filled<16>(0x77), 1, 9, 20000ms}, 20000ms, 5s); CHECK(!stale && stale.error().kind == LeaseStoreError::fenced); CHECK(release_region_lease(*runtime, key, *current, 5s)); // Two barriers for the last two current builds: exactly one may succeed. CHECK(owner_db->query("BEGIN", {}, 5s)); CHECK(scalar(*owner_db, "SELECT evolution.upgrade_barrier(" + build_literal(0xC3) + ")::text") == "0"); auto last = race.run(*owner_db, [&] { auto other = connect_to(database, env("PGUSER")); return other->query("SELECT evolution.upgrade_barrier(" + build_literal(0xC4) + ")", {}, 5s); }); CHECK(!last && sqlstate(last.error()) == "22023"); CHECK(scalar(*owner_db, "SELECT count(*)::text FROM evolution.retained_builds WHERE " "retired_at IS NULL") == "1"); // Classification: a synthetic source cut (absent from this database) and clone // manifests naming it; the database never reads payload objects. const auto source_cut = [&](std::uint64_t world, std::uint8_t seed) { auto m = genesis_manifest({world, 1}, 9, filled<16>(seed), 50000, c4, 100, filled<32>(0xA1)); CHECK(m); return *m; }; const auto clone_sql = [&](Postgres& db, RegionKey target, std::uint64_t source_world, std::uint8_t seed) { auto m = clone_manifest(source_cut(source_world, static_cast(seed + 1)), filled<32>(0xD5), target, filled<16>(seed), 100, filled<32>(0xA2), {0, {}, 0, {}}); CHECK(m); const std::array p{bytea_of(*encode_manifest(*m)), std::to_string(source_world), "1"}; return db.query("SELECT evolution.provision_region_clone($1::bytea, $2::evolution.u64, " "$3::evolution.u64, 0, '\\x'::bytea)", p, 5s); }; const auto contender = [&](RegionKey target, std::uint64_t source_world, std::uint8_t seed) { return [&, target, source_world, seed] { auto other = connect_to(database, env("PGUSER")); return clone_sql(*other, target, source_world, seed); }; }; // Different sources into one target world: exactly one source is accepted. CHECK(owner_db->query("BEGIN", {}, 5s)); CHECK(clone_sql(*owner_db, {2, 1}, 10, 0x62)); auto mixed = race.run(*owner_db, contender({2, 2}, 11, 0x64)); CHECK(!mixed && sqlstate(mixed.error()) == "22023"); // An original region racing a clone into the same world. auto original = genesis_manifest({3, 1}, 9, filled<16>(0x66), 50000, c4, 100, filled<32>(0xA0)); CHECK(original); const std::array op{bytea_of(*encode_manifest(*original))}; CHECK(owner_db->query("BEGIN", {}, 5s)); CHECK(owner_db->query("SELECT evolution.provision_region_checkpoint($1::bytea)", op, 5s)); auto against_original = race.run(*owner_db, contender({3, 2}, 10, 0x68)); CHECK(!against_original && sqlstate(against_original.error()) == "22023"); // Same source, different regions: both succeed. CHECK(owner_db->query("BEGIN", {}, 5s)); CHECK(clone_sql(*owner_db, {4, 1}, 10, 0x6A)); auto same_source = race.run(*owner_db, contender({4, 2}, 10, 0x6C)); CHECK(same_source); CHECK(scalar(*owner_db, "SELECT string_agg(world_id || '/' || region_id || '<' || " "source_world, ',' ORDER BY world_id, region_id) FROM " "evolution.region_origins") == "2/1<10,4/1<10,4/2<10"); } } // namespace void run_region_migration_tests() { concurrent_authority_and_classification(); durability_qualification(); upgrade_across_migration_and_barrier(); }