Foundation / checks (ecs) (push) Successful in 2m30s
Foundation / checks (deploy) (push) Successful in 4m0s
Foundation / checks (tsan) (push) Successful in 10m37s
Foundation / checks (client) (push) Failing after 13m56s
Foundation / checks (network) (push) Successful in 19m9s
Foundation / checks (checks) (push) Successful in 42m54s
Foundation / checks (postgres) (push) Successful in 44m9s
Foundation / postgres-tools (push) Skipped
Foundation / server-image (push) Skipped
Foundation / image (push) Skipped
M11.5c (ADR-0125): v10 cellular actions, the canonical input and the trace carry target kind 3 (bud), matched to durable actions; below rules 4 the owner ignores it. The v5 version bundle is admitted in C++ and by schema v36, which also admits kind 3 in SQL action inputs; the provisioner offers cellular-v3. A PostgreSQL test publishes a bud from a fed founder and recovers it after a host restart. The publication tests now use commit ids no other suite produces. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
719 lines
38 KiB
C++
719 lines
38 KiB
C++
// M7.9b schema migration, upgrade barrier and durability qualification (ADR-0068)
|
|
// against real PostgreSQL, real evo-server processes and the checked-in migrations
|
|
// (EVO_SOURCE_ROOT). Run by tools/ci/postgres-check.sh; prints measured latencies.
|
|
#include "evo/persistence/region_clone.hpp"
|
|
#include "evo/persistence/sha256.hpp"
|
|
#include "fixtures/server_process.hpp"
|
|
#include <algorithm>
|
|
#include <fstream>
|
|
#include <iterator>
|
|
#include <source_location>
|
|
#include <sstream>
|
|
#include <thread>
|
|
|
|
void run_region_migration_tests();
|
|
|
|
namespace {
|
|
using namespace evo::test::process;
|
|
using namespace evo::persistence;
|
|
void check(bool value, std::source_location at = std::source_location::current()) {
|
|
if (!value)
|
|
throw std::runtime_error("region migration test line " + std::to_string(at.line()));
|
|
}
|
|
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
|
|
constexpr std::size_t budget = 8000000;
|
|
const std::string b1_text = [] {
|
|
std::string s;
|
|
for (int i = 0; i < 32; ++i)
|
|
s += "b1";
|
|
return s;
|
|
}();
|
|
const std::string b2_text = [] {
|
|
std::string s;
|
|
for (int i = 0; i < 32; ++i)
|
|
s += "b2";
|
|
return s;
|
|
}();
|
|
Sha256Digest digest_of(std::uint8_t b) {
|
|
Sha256Digest d{};
|
|
d.fill(b);
|
|
return d;
|
|
}
|
|
const Sha256Digest b1 = digest_of(0xB1), b2 = digest_of(0xB2);
|
|
template <std::size_t N> std::array<std::uint8_t, N> filled(std::uint8_t value) {
|
|
std::array<std::uint8_t, N> out{};
|
|
out.fill(value);
|
|
return out;
|
|
}
|
|
std::unique_ptr<Postgres> connect_to(const std::string& database, const std::string& user) {
|
|
auto made = Postgres::connect({env("PGHOST"), {}, env("PGPORT"), database, user, {}, {}}, 5s);
|
|
CHECK(made);
|
|
return std::move(*made);
|
|
}
|
|
std::string sqlstate(const PgFailure& f) {
|
|
return f.sqlstate.data();
|
|
}
|
|
std::string scalar(Postgres& db, const std::string& sql) {
|
|
auto rows = db.query(sql, {}, 5s);
|
|
CHECK(rows && rows->size() == 1 && (*rows)[0].size() == 1);
|
|
return (*rows)[0][0].value_or("");
|
|
}
|
|
int shell(const std::string& command) {
|
|
Child c{{"/bin/bash", "-c", command}};
|
|
return c.wait_exit(120s);
|
|
}
|
|
std::vector<std::string> with(std::vector<std::string> args, const std::string& flag,
|
|
const std::string& value) {
|
|
const auto at = std::ranges::find(args, flag);
|
|
CHECK(at != args.end() && at + 1 != args.end());
|
|
*(at + 1) = value;
|
|
return args;
|
|
}
|
|
std::unique_ptr<FilesystemObjectStore> store(const ServerRegion& r) {
|
|
auto made = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes);
|
|
CHECK(made);
|
|
return std::move(*made);
|
|
}
|
|
std::optional<Uuid> parse_uuid(const std::string& text) {
|
|
std::string compact;
|
|
for (const char c : text)
|
|
if (c != '-')
|
|
compact.push_back(c);
|
|
if (compact.size() != 32)
|
|
return std::nullopt;
|
|
Uuid out{};
|
|
for (std::size_t i = 0; i < 16; ++i)
|
|
out[i] = static_cast<std::uint8_t>(std::stoul(compact.substr(2 * i, 2), nullptr, 16));
|
|
return out;
|
|
}
|
|
// A pre-v8 writer: the build-less acquisition every earlier host used.
|
|
std::expected<LeaseGrant, std::string> legacy_acquire(Postgres& db, RegionKey key) {
|
|
const std::array<std::string, 2> p{std::to_string(key.world), std::to_string(key.region)};
|
|
auto rows = db.query("SELECT * FROM evolution.acquire_region_lease($1::evolution.u64, "
|
|
"$2::evolution.u64, 20000)",
|
|
p, 5s);
|
|
if (!rows)
|
|
return std::unexpected{sqlstate(rows.error())};
|
|
CHECK(rows->size() == 1 && (*rows)[0].size() == 4);
|
|
const auto& r = (*rows)[0];
|
|
const auto holder = parse_uuid(r[0].value_or(""));
|
|
CHECK(holder.has_value());
|
|
return LeaseGrant{holder.value_or(Uuid{}), std::stoull(r[1].value_or("0")),
|
|
std::stoull(r[2].value_or("0")), 20000ms};
|
|
}
|
|
const char* const sha_function =
|
|
"s(){ sha256sum \"$EVO_SOURCE_ROOT/migrations/$1\" | cut -d' ' -f1; }; ";
|
|
std::string history(Postgres& owner) {
|
|
return scalar(owner, "SELECT string_agg(version::text, ',' ORDER BY version) FROM "
|
|
"evolution.schema_migrations");
|
|
}
|
|
|
|
// An existing v7 database is upgraded while an older writer holds a lease. A
|
|
// migration that fails leaves v7 exactly and the writer unaffected; the real one
|
|
// commits under it. The barrier then retires the old build: the writer is fenced
|
|
// and can never regain authority, while the new build recovers the whole history.
|
|
void upgrade_across_migration_and_barrier() {
|
|
const std::string database = "evo_online";
|
|
const std::string runtime_user = env("EVO_PG_RUNTIME_USER");
|
|
CHECK(shell(std::string(sha_function) + "cd \"$EVO_SOURCE_ROOT/migrations\" && createdb " +
|
|
database + " && PGDATABASE=" + database +
|
|
" psql -X -q -v ON_ERROR_STOP=1 -1 -v migration_sha=$(s 0001_region_bootstrap.sql)"
|
|
" -v migration_sha_2=$(s 0002_region_fencing.sql)"
|
|
" -v migration_sha_3=$(s 0003_region_checkpoints.sql)"
|
|
" -v migration_sha_4=$(s 0004_lineage_profile.sql)"
|
|
" -v migration_sha_5=$(s 0005_batch_decisions.sql)"
|
|
" -v migration_sha_6=$(s 0006_commit_lookup.sql)"
|
|
" -v migration_sha_7=$(s 0007_region_clones.sql)"
|
|
" -v runtime_role=evo_runtime_test -f 0001_region_bootstrap.sql"
|
|
" -f 0002_region_fencing.sql -f 0003_region_checkpoints.sql"
|
|
" -f 0004_lineage_profile.sql -f 0005_batch_decisions.sql"
|
|
" -f 0006_commit_lookup.sql -f 0007_region_clones.sql > /dev/null") == 0);
|
|
auto owner_db = connect_to(database, env("PGUSER"));
|
|
CHECK(history(*owner_db) == "1,2,3,4,5,6,7");
|
|
CHECK(owner_db->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
|
|
"(decode(repeat('b1', 32), 'hex')), (decode(repeat('b2', 32), 'hex'))",
|
|
{}, 5s));
|
|
const ServerRegion r{{7, 91}, fs::path(env("PGHOST")) / "online-91"};
|
|
fs::remove_all(r.root);
|
|
fs::create_directory(r.root);
|
|
fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace);
|
|
auto objects = store(r);
|
|
auto genesis =
|
|
provision_lineage_region(*owner_db, *objects, r.key, evo::test::region_identity.frame,
|
|
*evo::test::lineage_region(), filled<16>(0xDA), b1, budget, 5s);
|
|
CHECK(genesis);
|
|
|
|
// The old writer: build-less lease, publishing with build b1.
|
|
auto runtime = connect_to(database, runtime_user);
|
|
auto lease = legacy_acquire(*runtime, r.key);
|
|
CHECK(lease);
|
|
auto state = evo::test::lineage_region();
|
|
RegionBatch<evo::processes::LineageRegion> batch{*state};
|
|
CommittedCheckpoint head = *genesis;
|
|
std::uint8_t commit_seed = 0xDB;
|
|
const auto publish_next = [&](std::uint64_t ticks) {
|
|
for (std::uint64_t i = 0; i < ticks && !batch.sealed(); ++i)
|
|
CHECK(batch.advance(evo::test::region_input(batch.end_tick() + 1, true)));
|
|
batch.seal();
|
|
return publish_lineage_batch(*runtime, *objects, *lease, head, batch,
|
|
filled<16>(commit_seed++), b1, budget, 5s);
|
|
};
|
|
const auto confirm = [&](const std::expected<PublishedBatch, PublicationFailure>& done) {
|
|
CHECK(done && batch.confirm(done->committed.head.end_tick));
|
|
head = done->committed;
|
|
};
|
|
confirm(publish_next(64)); // Ticks 1..20: the consumer's death seals it.
|
|
CHECK(head.head.end_tick == 20);
|
|
|
|
// A v8 migration that fails at its last statement changes nothing.
|
|
CHECK(shell(std::string(sha_function) + "cd \"$EVO_SOURCE_ROOT\" && PGDATABASE=" + database +
|
|
" psql -X -q -v ON_ERROR_STOP=1 -v migration_sha=$(s 0001_region_bootstrap.sql)"
|
|
" -v migration_sha_2=$(s 0002_region_fencing.sql)"
|
|
" -v migration_sha_3=$(s 0003_region_checkpoints.sql)"
|
|
" -v migration_sha_4=$(s 0004_lineage_profile.sql)"
|
|
" -v migration_sha_5=$(s 0005_batch_decisions.sql)"
|
|
" -v migration_sha_6=$(s 0006_commit_lookup.sql)"
|
|
" -v migration_sha_7=$(s 0007_region_clones.sql) -v migration_sha_8=invalid"
|
|
" -v migration_sha_9=$(s 0009_serialized_authority.sql)"
|
|
" -v migration_sha_10=$(s 0010_account_control.sql)"
|
|
" -v migration_sha_11=$(s 0011_region_actions.sql)"
|
|
" -v migration_sha_12=$(s 0012_probe_handoff.sql)"
|
|
" -v migration_sha_13=$(s 0013_probe_incoming.sql)"
|
|
" -v migration_sha_14=$(s 0014_probe_overdue.sql)"
|
|
" -v migration_sha_15=$(s 0015_life_reservations.sql)"
|
|
" -v migration_sha_16=$(s 0016_primitive_publication.sql)"
|
|
" -v migration_sha_17=$(s 0017_life_request_outcomes.sql)"
|
|
" -v migration_sha_18=$(s 0018_cellular_profile_admission.sql)"
|
|
" -v migration_sha_19=$(s 0019_cellular_action_records.sql)"
|
|
" -v migration_sha_20=$(s 0020_cellular_publication.sql)"
|
|
" -v migration_sha_21=$(s 0021_cellular_trace_and_recovery.sql)"
|
|
" -v migration_sha_22=$(s 0022_cellular_birth_trace_bound.sql)"
|
|
" -v migration_sha_23=$(s 0023_cellular_birth_publication.sql)"
|
|
" -v migration_sha_24=$(s 0024_primitive_life_owner_lookup.sql)"
|
|
" -v migration_sha_25=$(s 0025_cellular_v2_bundle.sql)"
|
|
" -v migration_sha_26=$(s 0026_cellular_profile_migration.sql)"
|
|
" -v migration_sha_27=$(s 0027_cellular_v3_migration.sql)"
|
|
" -v migration_sha_28=$(s 0028_cellular_v3_trace_bounds.sql)"
|
|
" -v migration_sha_29=$(s 0029_cellular_mutation_reservations.sql)"
|
|
" -v migration_sha_30=$(s 0030_cellular_birth_publication_binding.sql)"
|
|
" -v runtime_role=evo_runtime_test -f migrations/apply.sql"
|
|
" > /dev/null 2>&1") != 0);
|
|
CHECK(history(*owner_db) == "1,2,3,4,5,6,7");
|
|
CHECK(scalar(*owner_db, "SELECT count(*) FROM information_schema.columns WHERE "
|
|
"table_schema = 'evolution' AND column_name = 'retired_at'") == "0");
|
|
CHECK(scalar(*owner_db,
|
|
"SELECT (to_regprocedure('evolution.acquire_region_lease("
|
|
"evolution.u64, evolution.u64, integer, bytea)') IS NULL)::text") == "true");
|
|
CHECK(renew_region_lease(*runtime, r.key, *lease, 20000ms, 5s));
|
|
confirm(publish_next(20)); // 21..40, still v7.
|
|
|
|
// The real migration commits while the writer holds its lease; it keeps going.
|
|
CHECK(shell("PGDATABASE=" + database +
|
|
" EVO_PG_RUNTIME_ROLE=evo_runtime_test bash \"$EVO_SOURCE_ROOT/"
|
|
"tools/migrate-postgres.sh\" > /dev/null") == 0);
|
|
CHECK(history(*owner_db) == "1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,"
|
|
"25,26,27,28,29,30,31,32,33,34,35,36");
|
|
CHECK(scalar(*owner_db, "SELECT (c.condeferrable AND c.condeferred AND "
|
|
"c.confrelid = 'evolution.region_commits'::regclass)::text "
|
|
"FROM pg_catalog.pg_constraint c WHERE c.conrelid = "
|
|
"'evolution.primitive_life_bindings'::regclass AND "
|
|
"c.conname = 'primitive_life_bindings_commit_id_fkey'") == "true");
|
|
CHECK(scalar(*owner_db,
|
|
"SELECT (t.tgdeferrable AND t.tginitdeferred)::text FROM pg_catalog.pg_trigger t "
|
|
"WHERE t.tgrelid = 'evolution.primitive_life_bindings'::regclass AND "
|
|
"t.tgname = 'life_binding_publication_evidence' AND NOT t.tgisinternal") ==
|
|
"true");
|
|
CHECK(scalar(*owner_db,
|
|
"SELECT (to_regprocedure('evolution.primitive_life_owner("
|
|
"evolution.u64,evolution.u64,uuid,evolution.u64,uuid)') IS NOT NULL)::text") ==
|
|
"true");
|
|
CHECK(scalar(*owner_db, "SELECT has_function_privilege('evo_runtime_login', "
|
|
"'evolution.primitive_life_owner(evolution.u64,evolution.u64,uuid,"
|
|
"evolution.u64,uuid)', 'EXECUTE')::text") == "true");
|
|
CHECK(scalar(*owner_db, "SELECT has_table_privilege('evo_runtime_login', "
|
|
"'evolution.life_reservations', 'SELECT')::text") == "false");
|
|
CHECK(scalar(*owner_db, "SELECT count(*) FROM information_schema.columns WHERE "
|
|
"table_schema = 'evolution' AND table_name = 'region_actions' AND "
|
|
"column_name IN ('input_bytes', 'result')") == "2");
|
|
CHECK(scalar(*owner_db, "SELECT (to_regprocedure('evolution.find_region_action_record("
|
|
"evolution.u64,evolution.u64,uuid,evolution.u64,uuid,evolution.u64)') "
|
|
"IS NOT NULL)::text") == "true");
|
|
CHECK(scalar(*owner_db, "SELECT evolution.valid_cellular_action_input(decode('01' || "
|
|
"repeat('00', 23), 'hex'), 1)::text") == "true");
|
|
CHECK(scalar(*owner_db, "SELECT evolution.valid_cellular_action_input(decode(repeat('00', 24), "
|
|
"'hex'), 1)::text") == "false");
|
|
// Schema v34 (ADR-0124): target kind 2 is engulf, with no coordinates. Schema v36
|
|
// (ADR-0125): kind 3 is a bud, with no coordinates; kind 4 is unknown.
|
|
CHECK(scalar(*owner_db,
|
|
"SELECT evolution.valid_cellular_action_input(decode('01' || "
|
|
"repeat('00', 11) || '02' || repeat('00', 11), 'hex'), 1)::text") == "true");
|
|
CHECK(scalar(*owner_db, "SELECT evolution.valid_cellular_action_input(decode('01' || "
|
|
"repeat('00', 11) || '02' || repeat('00', 3) || '01' || "
|
|
"repeat('00', 7), 'hex'), 1)::text") == "false");
|
|
CHECK(scalar(*owner_db,
|
|
"SELECT evolution.valid_cellular_action_input(decode('01' || "
|
|
"repeat('00', 11) || '03' || repeat('00', 11), 'hex'), 1)::text") == "true");
|
|
CHECK(scalar(*owner_db, "SELECT evolution.valid_cellular_action_input(decode('01' || "
|
|
"repeat('00', 11) || '03' || repeat('00', 3) || '01' || "
|
|
"repeat('00', 7), 'hex'), 1)::text") == "false");
|
|
CHECK(scalar(*owner_db,
|
|
"SELECT evolution.valid_cellular_action_input(decode('01' || "
|
|
"repeat('00', 11) || '04' || repeat('00', 11), 'hex'), 1)::text") == "false");
|
|
CHECK(scalar(*owner_db,
|
|
"SELECT evolution.check_cellular_action_block(decode(repeat('01', 16) || "
|
|
"'0100000000000000' || '0100000000000000' || repeat('00', 16), 'hex'), "
|
|
"0, 1)::text") == "1");
|
|
CHECK(
|
|
scalar(*owner_db,
|
|
"SELECT "
|
|
"(to_regprocedure('evolution.publish_cellular_checkpoint(bytea,bytea,bytea,bytea)') "
|
|
"IS NOT NULL)::text") == "true");
|
|
CHECK(
|
|
scalar(
|
|
*owner_db,
|
|
"SELECT has_function_privilege('evo_runtime_login', "
|
|
"'evolution.publish_cellular_checkpoint(bytea,bytea,bytea,bytea)', 'EXECUTE')::text") ==
|
|
"true");
|
|
CHECK(scalar(*owner_db,
|
|
"SELECT has_function_privilege('evo_runtime_login', "
|
|
"'evolution.publish_cellular_checkpoint(bytea,bytea,bytea)', 'EXECUTE')::text") ==
|
|
"false");
|
|
CHECK(scalar(*owner_db,
|
|
"SELECT (to_regprocedure('evolution.load_cellular_publication_births("
|
|
"evolution.u64,evolution.u64,uuid,evolution.u64,uuid)') IS NOT NULL)::text") ==
|
|
"true");
|
|
CHECK(scalar(*owner_db,
|
|
"SELECT has_function_privilege('evo_runtime_login', "
|
|
"'evolution.resolve_cellular_region_commit(evolution.u64,evolution.u64,uuid,"
|
|
"bytea,bytea,bytea,bytea)', 'EXECUTE')::text") == "true");
|
|
CHECK(scalar(*owner_db, "SELECT has_table_privilege('evo_runtime_login', "
|
|
"'evolution.cellular_publications', 'INSERT')::text") == "false");
|
|
CHECK(scalar(*owner_db,
|
|
"SELECT has_function_privilege('evo_runtime_login', "
|
|
"'evolution.publish_region_checkpoint_core(bytea,bytea)', 'EXECUTE')::text") ==
|
|
"false");
|
|
CHECK(renew_region_lease(*runtime, r.key, *lease, 20000ms, 5s));
|
|
confirm(publish_next(20)); // 41..60, now v8.
|
|
CHECK(head.head.end_tick == 60);
|
|
|
|
// Barrier guards: the last current build cannot be retired; runtime cannot call it.
|
|
const std::string retire_b1 = "SELECT evolution.upgrade_barrier(decode(repeat('b1', 32), "
|
|
"'hex'))::text";
|
|
auto denied = runtime->query(retire_b1, {}, 5s);
|
|
CHECK(!denied && sqlstate(denied.error()) == "42501");
|
|
// The barrier: b1 retired and every holder fenced, in one transaction.
|
|
CHECK(scalar(*owner_db, retire_b1) == "1");
|
|
auto last_b2 = owner_db->query(
|
|
"SELECT evolution.upgrade_barrier(decode(repeat('b2', 32), 'hex'))", {}, 5s);
|
|
CHECK(!last_b2 && sqlstate(last_b2.error()) == "22023");
|
|
// The old writer is fenced and cannot regain authority by any path.
|
|
auto after = publish_next(10);
|
|
CHECK(!after && after.error().kind == PublicationError::rejected &&
|
|
after.error().store.kind == LeaseStoreError::fenced);
|
|
auto renewed = renew_region_lease(*runtime, r.key, *lease, 20000ms, 5s);
|
|
CHECK(!renewed && renewed.error().kind == LeaseStoreError::fenced);
|
|
auto legacy = legacy_acquire(*runtime, r.key);
|
|
CHECK(!legacy && legacy.error() == "EVL06");
|
|
auto retired = acquire_region_lease(*runtime, r.key, b1, 20000ms, 5s);
|
|
CHECK(!retired && retired.error().kind == LeaseStoreError::unsupported);
|
|
// Authority for b2 still cannot publish a link naming the retired build.
|
|
auto current = acquire_region_lease(*runtime, r.key, b2, 20000ms, 5s);
|
|
CHECK(current);
|
|
lease = *current;
|
|
auto stale_build = publish_lineage_batch(*runtime, *objects, *lease, head, batch,
|
|
filled<16>(commit_seed++), b1, budget, 5s);
|
|
CHECK(!stale_build && stale_build.error().kind == PublicationError::rejected &&
|
|
stale_build.error().store.kind == LeaseStoreError::unsupported);
|
|
CHECK(release_region_lease(*runtime, r.key, *lease, 5s));
|
|
const auto before_new = head.head;
|
|
|
|
// The new build resumes from the committed head, and b1 history stays readable.
|
|
auto args = with(with(server_args(r), "--pg-database", database), "--build", b2_text);
|
|
args.insert(args.end(), {"--retained", b1_text});
|
|
{
|
|
Child server{args};
|
|
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
|
|
CHECK(server.printed("phase running epoch=3 committed=60 "));
|
|
server.send(input_lines(61, 90, true));
|
|
CHECK(server.wait_line("ack " + request_hex(90) + " 90 ", 30s));
|
|
server.signal(SIGTERM);
|
|
CHECK(server.wait_exit(20s) == 0);
|
|
}
|
|
auto final_head = load_region_checkpoint(*owner_db, r.key, 5s);
|
|
CHECK(final_head && final_head->head.end_tick == 90 &&
|
|
final_head->head.sequence > before_new.sequence);
|
|
const auto final_manifest = decode_manifest(final_head->manifest);
|
|
CHECK(final_manifest && final_manifest->build == b2);
|
|
const std::array<Sha256Digest, 2> both{b1, b2};
|
|
const std::array<Sha256Digest, 1> only_new{b2};
|
|
auto audited = audit_region_history(*runtime, *objects, r.key, both, budget, 100, 5s);
|
|
CHECK(audited && audited->commits == final_head->head.sequence + 1 && audited->outcomes == 1);
|
|
auto without_old = audit_region_history(*runtime, *objects, r.key, only_new, budget, 100, 5s);
|
|
CHECK(!without_old && without_old.error().kind == AuditError::unsupported &&
|
|
without_old.error().sequence == 0);
|
|
// A host of the retired build never serves, not even once the region is free.
|
|
{
|
|
Child old_host{with(with(server_args(r), "--pg-database", database), "--build", b1_text)};
|
|
CHECK(old_host.wait_exit(20s) == 4);
|
|
CHECK(old_host.printed("phase quarantined") && !old_host.printed("ready 1"));
|
|
}
|
|
CHECK(load_region_checkpoint(*owner_db, r.key, 5s)->head == final_head->head);
|
|
}
|
|
|
|
// Resource profile of a live process from /proc (M8.1 budgets): peak resident
|
|
// set (VmHWM, KiB) and consumed CPU time (user + system, ms).
|
|
struct ProcessProfile {
|
|
std::uint64_t peak_rss_kib{}, cpu_ms{};
|
|
};
|
|
ProcessProfile profile_of(pid_t pid) {
|
|
ProcessProfile out;
|
|
std::ifstream status("/proc/" + std::to_string(pid) + "/status");
|
|
for (std::string line; std::getline(status, line);)
|
|
if (line.starts_with("VmHWM:"))
|
|
out.peak_rss_kib = std::stoull(line.substr(6));
|
|
std::ifstream stat("/proc/" + std::to_string(pid) + "/stat");
|
|
std::string text((std::istreambuf_iterator<char>(stat)), {});
|
|
// Fields after the parenthesized command: state is field 3; utime/stime 14/15.
|
|
std::istringstream fields(text.substr(text.rfind(')') + 2));
|
|
std::vector<std::string> f{std::istream_iterator<std::string>(fields), {}};
|
|
CHECK(f.size() > 13 && out.peak_rss_kib > 0);
|
|
const auto ticks = std::stoull(f[11]) + std::stoull(f[12]);
|
|
out.cpu_ms = ticks * 1000 / static_cast<std::uint64_t>(::sysconf(_SC_CLK_TCK));
|
|
return out;
|
|
}
|
|
|
|
std::uint64_t percentile(std::vector<std::uint64_t> v, unsigned p) {
|
|
CHECK(!v.empty());
|
|
std::ranges::sort(v);
|
|
return v[std::min(v.size() - 1, (v.size() * p) / 100)];
|
|
}
|
|
std::uint64_t ms_between(Clock::time_point a, Clock::time_point b) {
|
|
return static_cast<std::uint64_t>(
|
|
std::chrono::duration_cast<std::chrono::milliseconds>(b - a).count());
|
|
}
|
|
// Sends inputs first..last paced at the 50 ms tick; returns send times.
|
|
std::vector<Clock::time_point> paced(Child& server, std::uint64_t first, std::uint64_t last) {
|
|
std::vector<Clock::time_point> sent;
|
|
const auto start = Clock::now();
|
|
for (auto tick = first; tick <= last; ++tick) {
|
|
const auto due = start + std::chrono::milliseconds{50 * (tick - first)};
|
|
if (const auto now = Clock::now(); due > now)
|
|
server.pump(std::chrono::duration_cast<std::chrono::milliseconds>(due - now));
|
|
server.send(input_line(tick, true));
|
|
sent.push_back(Clock::now());
|
|
}
|
|
return sent;
|
|
}
|
|
std::optional<Clock::time_point> ack_time(const Child& server, std::uint64_t tick) {
|
|
const auto prefix = "ack " + request_hex(tick) + " " + std::to_string(tick) + " ";
|
|
for (std::size_t i = 0; i < server.lines().size(); ++i)
|
|
if (starts(server.lines()[i], prefix))
|
|
return server.times()[i];
|
|
return std::nullopt;
|
|
}
|
|
Clock::time_point ready_time(const Child& server) {
|
|
for (std::size_t i = 0; i < server.lines().size(); ++i)
|
|
if (server.lines()[i] == "ready 1")
|
|
return server.times()[i];
|
|
CHECK(false);
|
|
return {};
|
|
}
|
|
|
|
// The M7 exit fixture: production lease policy (20 s lease, 5 s renewal) and
|
|
// flush interval (1 s), one lineage region on a local socket and filesystem.
|
|
// Measures acknowledgement latency, worker-kill-to-ready recovery against the
|
|
// proposed 60 s target, and the cost of verification, audit and clone.
|
|
void durability_qualification() {
|
|
const auto r = provision(93, 0xDC);
|
|
const auto args =
|
|
with(with(server_args(r), "--lease-profile", "default"), "--flush-ms", "1000");
|
|
std::vector<std::uint64_t> latency, kill_to_ready;
|
|
std::uint64_t death_latency = 0, peak_rss_kib = 0;
|
|
std::vector<std::uint64_t> cpu_ms, busy_ms; // Per owner: CPU used, wall time serving.
|
|
std::uint64_t next = 1;
|
|
std::uint64_t epoch = 1;
|
|
for (int round = 0; round < 3; ++round) {
|
|
Child server{args};
|
|
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 60s));
|
|
CHECK(server.printed("phase running epoch=" + std::to_string(epoch) + " "));
|
|
const auto first = next;
|
|
const auto last = next + 99;
|
|
const auto sent = paced(server, first, last);
|
|
CHECK(server.wait_line("ack " + request_hex(last) + " " + std::to_string(last) + " ", 30s));
|
|
for (auto tick = first; tick <= last; ++tick) {
|
|
const auto at = ack_time(server, tick);
|
|
CHECK(at.has_value());
|
|
const auto ms = ms_between(sent[tick - first], at.value_or(Clock::time_point{}));
|
|
if (tick == 20)
|
|
death_latency = ms;
|
|
else
|
|
latency.push_back(ms);
|
|
}
|
|
// Mid-batch force kill: more inputs are in flight, none acknowledged yet.
|
|
server.send(input_lines(last + 1, last + 10, true));
|
|
server.pump(200ms);
|
|
const auto acked = max_acked(server);
|
|
const auto used = profile_of(server.pid());
|
|
peak_rss_kib = std::max(peak_rss_kib, used.peak_rss_kib);
|
|
cpu_ms.push_back(used.cpu_ms);
|
|
busy_ms.push_back(ms_between(sent.front(), Clock::now()));
|
|
server.signal(SIGKILL);
|
|
const auto killed = Clock::now();
|
|
CHECK(server.wait_exit(20s) == 128 + SIGKILL);
|
|
const auto committed = head_of(r.key);
|
|
CHECK(acked <= committed.tick && acked >= last); // No acknowledged input lost.
|
|
next = committed.tick + 1;
|
|
++epoch;
|
|
if (round == 2)
|
|
break;
|
|
Child successor{args};
|
|
CHECK(successor.wait_for([](const std::string& l) { return l == "ready 1"; }, 60s));
|
|
kill_to_ready.push_back(ms_between(killed, ready_time(successor)));
|
|
CHECK(successor.printed("phase running epoch=" + std::to_string(epoch) +
|
|
" committed=" + std::to_string(committed.tick) + " "));
|
|
successor.signal(SIGTERM);
|
|
CHECK(successor.wait_exit(20s) == 0);
|
|
++epoch;
|
|
}
|
|
CHECK(decisions_of(r.key) == 1);
|
|
for (const auto ms : kill_to_ready)
|
|
CHECK(ms < 60000); // The declared target for this fixture.
|
|
std::printf("qualification ack_latency_ms flush=1000 n=%zu p50=%llu p99=%llu max=%llu "
|
|
"death=%llu\n",
|
|
latency.size(), static_cast<unsigned long long>(percentile(latency, 50)),
|
|
static_cast<unsigned long long>(percentile(latency, 99)),
|
|
static_cast<unsigned long long>(percentile(latency, 100)),
|
|
static_cast<unsigned long long>(death_latency));
|
|
std::printf("qualification kill_to_ready_ms lease=20000 renew=5000 runs=%zu first=%llu "
|
|
"second=%llu target=60000\n",
|
|
kill_to_ready.size(), static_cast<unsigned long long>(kill_to_ready[0]),
|
|
static_cast<unsigned long long>(kill_to_ready[1]));
|
|
|
|
std::uint64_t cpu_total = 0, busy_total = 0;
|
|
for (std::size_t i = 0; i < cpu_ms.size(); ++i) {
|
|
cpu_total += cpu_ms[i];
|
|
busy_total += busy_ms[i];
|
|
}
|
|
std::printf(
|
|
"qualification server_profile owners=%zu peak_rss_kib=%llu cpu_ms=%llu "
|
|
"wall_ms=%llu cpu_permille=%llu\n",
|
|
cpu_ms.size(), static_cast<unsigned long long>(peak_rss_kib),
|
|
static_cast<unsigned long long>(cpu_total), static_cast<unsigned long long>(busy_total),
|
|
static_cast<unsigned long long>(busy_total == 0 ? 0 : cpu_total * 1000 / busy_total));
|
|
|
|
// Verification cost on the final head (no lease needed to read).
|
|
auto reader = connect_to(env("PGDATABASE"), env("EVO_PG_RUNTIME_USER"));
|
|
auto objects = store(r);
|
|
const std::array<Sha256Digest, 1> retained{b1};
|
|
std::vector<std::uint64_t> verify_us;
|
|
std::uint64_t payload_bytes = 0;
|
|
for (int i = 0; i < 10; ++i) {
|
|
const auto start = Clock::now();
|
|
auto recovered = recover_lineage_checkpoint(*reader, *objects, r.key, retained, budget, 5s);
|
|
CHECK(recovered);
|
|
payload_bytes = recovered->manifest.payload_bytes;
|
|
verify_us.push_back(static_cast<std::uint64_t>(
|
|
std::chrono::duration_cast<std::chrono::microseconds>(Clock::now() - start).count()));
|
|
}
|
|
const auto audit_start = Clock::now();
|
|
auto audited = audit_region_history(*reader, *objects, r.key, retained, budget, 1000, 5s);
|
|
const auto audit_ms = ms_between(audit_start, Clock::now());
|
|
CHECK(audited);
|
|
auto clone_owner = connect_to(env("EVO_PG_CLONE_DATABASE"), env("PGUSER"));
|
|
const ServerRegion target{{8, 93}, fs::path(env("PGHOST")) / "qualify-clone-93"};
|
|
fs::remove_all(target.root);
|
|
fs::create_directory(target.root);
|
|
fs::permissions(target.root, fs::perms::owner_all, fs::perm_options::replace);
|
|
const auto clone_start = Clock::now();
|
|
auto cloned = clone_region(*reader, *objects, r.key, *clone_owner, *store(target), target.key,
|
|
filled<16>(0xDD), retained, budget, 5s);
|
|
const auto clone_ms = ms_between(clone_start, Clock::now());
|
|
CHECK(cloned);
|
|
std::printf("qualification verify_head_us payload_bytes=%llu n=10 p50=%llu max=%llu\n",
|
|
static_cast<unsigned long long>(payload_bytes),
|
|
static_cast<unsigned long long>(percentile(verify_us, 50)),
|
|
static_cast<unsigned long long>(percentile(verify_us, 100)));
|
|
std::printf("qualification audit_ms commits=%llu ticks=%llu ms=%llu clone_ms=%llu\n",
|
|
static_cast<unsigned long long>(audited->commits),
|
|
static_cast<unsigned long long>(audited->end_tick),
|
|
static_cast<unsigned long long>(audit_ms),
|
|
static_cast<unsigned long long>(clone_ms));
|
|
}
|
|
// Review fixes (schema v9): builds, authority and world classification follow one
|
|
// lock order. Each case holds a transaction open, proves the contender is blocked
|
|
// on a lock in the database, commits, and checks the contender's outcome.
|
|
struct Race {
|
|
std::string database;
|
|
// Commits the holder's open transaction once `contender` is waiting on a lock.
|
|
template <typename F> auto run(Postgres& holder, F contender) {
|
|
std::optional<decltype(contender())> result;
|
|
std::thread thread([&] { result.emplace(contender()); });
|
|
auto observer = connect_to(database, env("PGUSER"));
|
|
const std::array<std::string, 1> p{database};
|
|
bool waiting = false;
|
|
for (int i = 0; i < 500 && !waiting; ++i) {
|
|
auto rows = observer->query("SELECT count(*)::text FROM pg_catalog.pg_stat_activity "
|
|
"WHERE datname = $1 AND wait_event_type = 'Lock'",
|
|
p, 5s);
|
|
waiting = rows && rows->size() == 1 && (*rows)[0][0].value_or("0") != "0";
|
|
if (!waiting)
|
|
std::this_thread::sleep_for(20ms);
|
|
}
|
|
CHECK(holder.query("COMMIT", {}, 5s));
|
|
thread.join();
|
|
CHECK(waiting);
|
|
if (!result.has_value())
|
|
throw std::runtime_error("race contender produced no result");
|
|
return std::move(*result);
|
|
}
|
|
};
|
|
std::string bytea_of(std::span<const std::uint8_t> bytes) {
|
|
constexpr char digits[] = "0123456789abcdef";
|
|
std::string out = "\\x";
|
|
for (const auto b : bytes) {
|
|
out.push_back(digits[b >> 4U]);
|
|
out.push_back(digits[b & 0x0FU]);
|
|
}
|
|
return out;
|
|
}
|
|
std::string build_literal(std::uint8_t b) {
|
|
char out[64];
|
|
std::snprintf(out, sizeof out, "decode(repeat('%02x', 32), 'hex')", b);
|
|
return out;
|
|
}
|
|
void concurrent_authority_and_classification() {
|
|
const std::string database = "evo_race";
|
|
CHECK(shell("createdb " + database + " && PGDATABASE=" + database +
|
|
" EVO_PG_RUNTIME_ROLE=evo_runtime_test bash \"$EVO_SOURCE_ROOT/"
|
|
"tools/migrate-postgres.sh\" > /dev/null") == 0);
|
|
auto owner_db = connect_to(database, env("PGUSER"));
|
|
auto cellular = genesis_manifest({7, 90}, 9, filled<16>(0x70), 50000, digest_of(0xC4), 100,
|
|
filled<32>(0xA9), CheckpointProfile::cellular_lives_v1);
|
|
CHECK(cellular);
|
|
const auto cellular_bytes = encode_manifest(*cellular);
|
|
CHECK(cellular_bytes);
|
|
const std::array cellular_parameter{bytea_of(*cellular_bytes)};
|
|
const auto parsed_cellular =
|
|
owner_db->query("SELECT (evolution.parse_checkpoint_manifest($1::bytea)).profile::text",
|
|
cellular_parameter, 5s);
|
|
CHECK(parsed_cellular && parsed_cellular->size() == 1 &&
|
|
(*parsed_cellular)[0][0].value_or("") == "4");
|
|
// encode_manifest refuses unsupported bundles, so corrupt the encoded archive
|
|
// version (u32 slot 8 of the bundle at offset 160) to reach the SQL parser.
|
|
auto wrong_cellular_bytes = *cellular_bytes;
|
|
++wrong_cellular_bytes[160 + (7 * 4)];
|
|
const std::array wrong_cellular_parameter{bytea_of(wrong_cellular_bytes)};
|
|
const auto refused_cellular = owner_db->query(
|
|
"SELECT evolution.parse_checkpoint_manifest($1::bytea)", wrong_cellular_parameter, 5s);
|
|
CHECK(!refused_cellular && sqlstate(refused_cellular.error()) == "EVL06");
|
|
CHECK(owner_db->query("INSERT INTO evolution.retained_builds(build_identity) SELECT "
|
|
"decode(repeat(b, 32), 'hex') FROM unnest(ARRAY['c1', 'c2', 'c3', "
|
|
"'c4']) AS b",
|
|
{}, 5s));
|
|
const Sha256Digest c2 = digest_of(0xC2), c3 = digest_of(0xC3), c4 = digest_of(0xC4);
|
|
const RegionKey key{7, 1};
|
|
auto genesis = genesis_manifest(key, 9, filled<16>(0x61), 50000, c4, 100, filled<32>(0xA0));
|
|
CHECK(genesis);
|
|
const std::array<std::string, 1> gp{bytea_of(*encode_manifest(*genesis))};
|
|
CHECK(owner_db->query("SELECT evolution.provision_region_checkpoint($1::bytea)", gp, 5s));
|
|
Race race{database};
|
|
const auto barrier_holding_lease_row = [&](std::uint8_t build) {
|
|
CHECK(owner_db->query("BEGIN", {}, 5s));
|
|
CHECK(owner_db->query("SELECT 1 FROM evolution.region_leases WHERE world_id = 7 AND "
|
|
"region_id = 1 FOR UPDATE",
|
|
{}, 5s));
|
|
CHECK(scalar(*owner_db, "SELECT evolution.upgrade_barrier(" + build_literal(build) +
|
|
")::text") == "0");
|
|
};
|
|
// Build-less acquisition queued behind the first barrier is refused after it.
|
|
barrier_holding_lease_row(0xC1);
|
|
auto legacy = race.run(*owner_db, [&] {
|
|
auto runtime = connect_to(database, env("EVO_PG_RUNTIME_USER"));
|
|
return legacy_acquire(*runtime, key);
|
|
});
|
|
CHECK(!legacy && legacy.error() == "EVL06");
|
|
// Build-aware acquisition for the build being retired is refused after COMMIT.
|
|
barrier_holding_lease_row(0xC2);
|
|
auto retired = race.run(*owner_db, [&] {
|
|
auto runtime = connect_to(database, env("EVO_PG_RUNTIME_USER"));
|
|
return acquire_region_lease(*runtime, key, c2, 20000ms, 5s);
|
|
});
|
|
CHECK(!retired && retired.error().kind == LeaseStoreError::unsupported);
|
|
CHECK(scalar(*owner_db, "SELECT (holder IS NULL)::text FROM evolution.region_leases WHERE "
|
|
"world_id = 7 AND region_id = 1") == "true");
|
|
// The new build acquires; the rejected one never held anything to renew.
|
|
auto runtime = connect_to(database, env("EVO_PG_RUNTIME_USER"));
|
|
auto current = acquire_region_lease(*runtime, key, c3, 20000ms, 5s);
|
|
CHECK(current);
|
|
auto stale = renew_region_lease(*runtime, key, {filled<16>(0x77), 1, 9, 20000ms}, 20000ms, 5s);
|
|
CHECK(!stale && stale.error().kind == LeaseStoreError::fenced);
|
|
CHECK(release_region_lease(*runtime, key, *current, 5s));
|
|
// Two barriers for the last two current builds: exactly one may succeed.
|
|
CHECK(owner_db->query("BEGIN", {}, 5s));
|
|
CHECK(scalar(*owner_db,
|
|
"SELECT evolution.upgrade_barrier(" + build_literal(0xC3) + ")::text") == "0");
|
|
auto last = race.run(*owner_db, [&] {
|
|
auto other = connect_to(database, env("PGUSER"));
|
|
return other->query("SELECT evolution.upgrade_barrier(" + build_literal(0xC4) + ")", {},
|
|
5s);
|
|
});
|
|
CHECK(!last && sqlstate(last.error()) == "22023");
|
|
CHECK(scalar(*owner_db, "SELECT count(*)::text FROM evolution.retained_builds WHERE "
|
|
"retired_at IS NULL") == "1");
|
|
|
|
// Classification: a synthetic source cut (absent from this database) and clone
|
|
// manifests naming it; the database never reads payload objects.
|
|
const auto source_cut = [&](std::uint64_t world, std::uint8_t seed) {
|
|
auto m =
|
|
genesis_manifest({world, 1}, 9, filled<16>(seed), 50000, c4, 100, filled<32>(0xA1));
|
|
CHECK(m);
|
|
return *m;
|
|
};
|
|
const auto clone_sql = [&](Postgres& db, RegionKey target, std::uint64_t source_world,
|
|
std::uint8_t seed) {
|
|
auto m = clone_manifest(source_cut(source_world, static_cast<std::uint8_t>(seed + 1)),
|
|
filled<32>(0xD5), target, filled<16>(seed), 100, filled<32>(0xA2),
|
|
{0, {}, 0, {}});
|
|
CHECK(m);
|
|
const std::array<std::string, 3> p{bytea_of(*encode_manifest(*m)),
|
|
std::to_string(source_world), "1"};
|
|
return db.query("SELECT evolution.provision_region_clone($1::bytea, $2::evolution.u64, "
|
|
"$3::evolution.u64, 0, '\\x'::bytea)",
|
|
p, 5s);
|
|
};
|
|
const auto contender = [&](RegionKey target, std::uint64_t source_world, std::uint8_t seed) {
|
|
return [&, target, source_world, seed] {
|
|
auto other = connect_to(database, env("PGUSER"));
|
|
return clone_sql(*other, target, source_world, seed);
|
|
};
|
|
};
|
|
// Different sources into one target world: exactly one source is accepted.
|
|
CHECK(owner_db->query("BEGIN", {}, 5s));
|
|
CHECK(clone_sql(*owner_db, {2, 1}, 10, 0x62));
|
|
auto mixed = race.run(*owner_db, contender({2, 2}, 11, 0x64));
|
|
CHECK(!mixed && sqlstate(mixed.error()) == "22023");
|
|
// An original region racing a clone into the same world.
|
|
auto original = genesis_manifest({3, 1}, 9, filled<16>(0x66), 50000, c4, 100, filled<32>(0xA0));
|
|
CHECK(original);
|
|
const std::array<std::string, 1> op{bytea_of(*encode_manifest(*original))};
|
|
CHECK(owner_db->query("BEGIN", {}, 5s));
|
|
CHECK(owner_db->query("SELECT evolution.provision_region_checkpoint($1::bytea)", op, 5s));
|
|
auto against_original = race.run(*owner_db, contender({3, 2}, 10, 0x68));
|
|
CHECK(!against_original && sqlstate(against_original.error()) == "22023");
|
|
// Same source, different regions: both succeed.
|
|
CHECK(owner_db->query("BEGIN", {}, 5s));
|
|
CHECK(clone_sql(*owner_db, {4, 1}, 10, 0x6A));
|
|
auto same_source = race.run(*owner_db, contender({4, 2}, 10, 0x6C));
|
|
CHECK(same_source);
|
|
CHECK(scalar(*owner_db, "SELECT string_agg(world_id || '/' || region_id || '<' || "
|
|
"source_world, ',' ORDER BY world_id, region_id) FROM "
|
|
"evolution.region_origins") == "2/1<10,4/1<10,4/2<10");
|
|
}
|
|
} // namespace
|
|
|
|
void run_region_migration_tests() {
|
|
concurrent_authority_and_classification();
|
|
durability_qualification();
|
|
upgrade_across_migration_and_barrier();
|
|
}
|