Complete M7.9b upgrade barrier and durability qualification

Schema v8 binds authority to builds (ADR-0068): acquisition names the
host build and only retained, unretired builds may hold a lease; no
commit may name a retired build; upgrade_barrier retires a build and
fences every holder in one transaction, while build-less pre-v8 hosts
keep working until the first barrier. Tests cover a writer surviving a
failed and then a real online migration, old-writer fencing on every
path, whole-history recovery by the new build, and the M7 qualification
fixture (19.6 s SIGKILL-to-ready under the production lease, no lost
acknowledgements).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-09-29 07:31:04 -04:00
co-authored by Claude Opus 5.5
parent 703f67834f
commit f59cca2c3b
24 changed files with 893 additions and 52 deletions
+11 -1
View File
@@ -46,7 +46,7 @@ inline std::unique_ptr<persistence::Postgres> owner() {
inline std::string request_hex(std::uint64_t tick, std::uint8_t tag = 0x9B) {
char out[33];
std::snprintf(out, sizeof out, "%02x000000000000000000000000%06llx", tag,
static_cast<unsigned long long>(tick));
static_cast<unsigned long long>(tick & 0xFFFFFFU)); // Test ticks < 2^24.
return out;
}
inline std::string input_line(std::uint64_t tick, bool consumer_dies, std::uint8_t tag = 0x9B) {
@@ -150,6 +150,14 @@ class Child {
[[nodiscard]] const std::vector<std::string>& lines() const noexcept {
return lines_;
}
// Arrival time of each line (when it was read), for latency measurement.
[[nodiscard]] const std::vector<Clock::time_point>& times() const noexcept {
return times_;
}
// Reads whatever arrives for `duration`, stamping each line.
void pump(std::chrono::milliseconds duration) {
wait_for([](const std::string&) { return false; }, duration);
}
[[nodiscard]] bool printed(std::string_view prefix) const {
for (const auto& l : lines_)
if (starts(l, prefix))
@@ -169,6 +177,7 @@ class Child {
buffer_.append(chunk, static_cast<std::size_t>(n));
for (auto end = buffer_.find('\n'); end != std::string::npos; end = buffer_.find('\n')) {
lines_.push_back(buffer_.substr(0, end));
times_.push_back(Clock::now());
buffer_.erase(0, end + 1);
}
return true;
@@ -177,6 +186,7 @@ class Child {
int in_{-1}, out_{-1};
std::string buffer_;
std::vector<std::string> lines_;
std::vector<Clock::time_point> times_;
};
struct ServerRegion {
+3
View File
@@ -10,6 +10,7 @@ void run_region_server_process_tests();
void run_region_recovery_tests();
void run_region_ops_tests();
void run_region_clone_tests();
void run_region_migration_tests();
namespace {
using namespace evo::persistence;
using namespace std::chrono_literals;
@@ -90,6 +91,8 @@ int main() {
std::puts("region ops integration passed");
run_region_clone_tests();
std::puts("region clone and history integration passed");
run_region_migration_tests();
std::puts("region migration, barrier and qualification integration passed");
return 0;
} catch (const std::exception& e) {
std::fprintf(stderr, "%s\n", e.what());
+6 -6
View File
@@ -215,7 +215,7 @@ template <typename Batch> void run_batch(Batch& b, std::uint64_t ticks, bool dea
void publish_recover_and_continue() {
auto r = provisioned("publish", 3, 0xC0);
const std::array<Sha256Digest, 1> retained{build};
auto lease = acquire_region_lease(*r.runtime, r.key, 20000ms, 5s);
auto lease = acquire_region_lease(*r.runtime, r.key, build, 20000ms, 5s);
CHECK(lease);
// Genesis recovers to the exact opening state.
auto recovered =
@@ -313,7 +313,7 @@ void publish_recover_and_continue() {
void failures_leave_the_previous_head() {
auto r = provisioned("failures", 4, 0xD0);
const std::array<Sha256Digest, 1> retained{build};
auto lease = acquire_region_lease(*r.runtime, r.key, 20000ms, 5s);
auto lease = acquire_region_lease(*r.runtime, r.key, build, 20000ms, 5s);
CHECK(lease);
auto head = load_region_checkpoint(*r.runtime, r.key, 5s);
CHECK(head);
@@ -446,7 +446,7 @@ void lineage_publish_and_recover() {
CHECK(genesis);
CHECK(decode_manifest(genesis->manifest)->profile ==
CheckpointProfile::environmental_lineage_v1);
auto lease = acquire_region_lease(*runtime, key, 20000ms, 5s);
auto lease = acquire_region_lease(*runtime, key, build, 20000ms, 5s);
CHECK(lease);
auto region = evo::test::lineage_region();
@@ -548,7 +548,7 @@ struct LeaseRowLock {
void durable_outcomes_and_ambiguity() {
auto r = provisioned("outcomes", 7, 0xF0);
const std::array<Sha256Digest, 1> retained{build};
auto lease = acquire_region_lease(*r.runtime, r.key, 20000ms, 5s);
auto lease = acquire_region_lease(*r.runtime, r.key, build, 20000ms, 5s);
CHECK(lease);
auto genesis = load_region_checkpoint(*r.runtime, r.key, 5s);
CHECK(genesis);
@@ -711,7 +711,7 @@ void publication_waits_for_directory_durability() {
auto faulty = FilesystemObjectStore::open(r.objects->directory().parent_path().parent_path(),
r.key, max_environmental_payload_bytes, &faults);
CHECK(faulty);
auto lease = acquire_region_lease(*r.runtime, r.key, 20000ms, 5s);
auto lease = acquire_region_lease(*r.runtime, r.key, build, 20000ms, 5s);
CHECK(lease);
auto genesis = load_region_checkpoint(*r.runtime, r.key, 5s);
CHECK(genesis);
@@ -738,7 +738,7 @@ void publication_waits_for_directory_durability() {
// recovered region. The database cannot see payload deaths; recovery must.
void undecided_death_is_quarantined() {
auto r = provisioned("undecided", 10, 0xB6);
auto lease = acquire_region_lease(*r.runtime, r.key, 20000ms, 5s);
auto lease = acquire_region_lease(*r.runtime, r.key, build, 20000ms, 5s);
CHECK(lease);
auto genesis = load_region_checkpoint(*r.runtime, r.key, 5s);
CHECK(genesis);
+4 -4
View File
@@ -251,11 +251,11 @@ void clone_cannot_write_its_source() {
CHECK(cloned && cloned->inherited_outcomes == 1 && cloned->origin.tick == 25 &&
cloned->origin.sequence == before.sequence && cloned->origin.commit == before.commit);
// The source session is read-only for good, even through SECURITY DEFINER functions.
auto denied = acquire_region_lease(*reader, source.key, 4000ms, 5s);
auto denied = acquire_region_lease(*reader, source.key, build(), 4000ms, 5s);
CHECK(!denied && sqlstate(denied.error().cause) == "25006");
// Authority over the clone is not authority over the source.
auto runtime = connect_to(source_db(), env("EVO_PG_RUNTIME_USER"));
auto lease = acquire_region_lease(*runtime, same_db.key, 4000ms, 5s);
auto lease = acquire_region_lease(*runtime, same_db.key, build(), 4000ms, 5s);
CHECK(lease);
auto source_head = load_region_checkpoint(*runtime, source.key, 5s);
CHECK(source_head);
@@ -296,7 +296,7 @@ void clone_cannot_write_its_source() {
CHECK(!rewrite && sqlstate(rewrite.error()) == "EVL09");
// The clone database's runtime cannot reach the source database's functions.
auto foreign = connect_to(source_db(), env("EVO_PG_CLONE_RUNTIME_USER"));
auto crossed = acquire_region_lease(*foreign, source.key, 4000ms, 5s);
auto crossed = acquire_region_lease(*foreign, source.key, build(), 4000ms, 5s);
CHECK(!crossed && sqlstate(crossed.error().cause) == "42501");
// Nothing above moved the source.
const auto after = head_in(source_db(), source.key);
@@ -420,7 +420,7 @@ void environmental_clone_binds_inherited_outcomes() {
*owner_db, *source_objects, key, evo::test::region_identity.frame,
*evo::test::region_community(), filled<16>(0xF9), build(), budget, 5s);
CHECK(genesis);
auto lease = acquire_region_lease(*runtime, key, 20000ms, 5s);
auto lease = acquire_region_lease(*runtime, key, build(), 20000ms, 5s);
CHECK(lease);
auto state = evo::test::region_community();
RegionBatch<evo::processes::EnvironmentalCommunity> batch{*state};
+6
View File
@@ -305,6 +305,12 @@ void unrestorable_head_quarantines() {
auto c = config(r);
c.build = filled<32>(0xB2);
c.retained_builds = {c.build}; // Not the build that produced the head.
// A current build in the database (so acquisition succeeds, ADR-0068): the
// quarantine comes from recovery refusing the head, not from the lease.
CHECK(connect_as(env("PGUSER"))
->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
"(decode(repeat('b2', 32), 'hex')) ON CONFLICT DO NOTHING",
{}, 5s));
auto h = host<LineageRegion>(r, c);
const auto& s = h->step(std::chrono::steady_clock::now());
CHECK(s.phase == HostPhase::quarantined && !s.ready && h->state() == nullptr);
+20 -20
View File
@@ -178,7 +178,7 @@ void runtime_cannot_bypass_the_guard() {
// A SECURITY DEFINER function ignores a hostile caller search_path.
CHECK(db->query("SET search_path = pg_temp, public", {}, 5s));
CHECK(db->query("CREATE TEMP TABLE region_leases(x int)", {}, 5s));
auto lease = acquire_region_lease(*db, key, 20000ms, 5s);
auto lease = acquire_region_lease(*db, key, build, 20000ms, 5s);
CHECK(lease && lease->epoch == 1 && lease->frame == 7);
CHECK(release_region_lease(*db, key, *lease, 5s));
}
@@ -187,13 +187,13 @@ void acquire_renew_release_and_publish() {
auto admin = owner();
const auto key = provision(*admin, 20);
auto db = runtime();
CHECK(
failed(acquire_region_lease(*db, {1, 999}, 20000ms, 5s), LeaseStoreError::not_provisioned));
CHECK(failed(acquire_region_lease(*db, key, 999ms, 5s), LeaseStoreError::invalid));
CHECK(failed(acquire_region_lease(*db, key, 60001ms, 5s), LeaseStoreError::invalid));
auto lease = acquire_region_lease(*db, key, 20000ms, 5s);
CHECK(failed(acquire_region_lease(*db, {1, 999}, build, 20000ms, 5s),
LeaseStoreError::not_provisioned));
CHECK(failed(acquire_region_lease(*db, key, build, 999ms, 5s), LeaseStoreError::invalid));
CHECK(failed(acquire_region_lease(*db, key, build, 60001ms, 5s), LeaseStoreError::invalid));
auto lease = acquire_region_lease(*db, key, build, 20000ms, 5s);
CHECK(lease && lease->epoch == 1 && lease->duration == 20000ms && lease->holder != Uuid{});
CHECK(failed(acquire_region_lease(*db, key, 20000ms, 5s), LeaseStoreError::busy));
CHECK(failed(acquire_region_lease(*db, key, build, 20000ms, 5s), LeaseStoreError::busy));
auto renewed = renew_region_lease(*db, key, *lease, 20000ms, 5s);
CHECK(renewed && *renewed == 20000ms);
auto forged = *lease;
@@ -244,7 +244,7 @@ void acquire_renew_release_and_publish() {
CHECK(failed(release_region_lease(*db, key, *lease, 5s), LeaseStoreError::fenced));
CHECK(failed(renew_region_lease(*db, key, *lease, 20000ms, 5s), LeaseStoreError::fenced));
CHECK(failed(publish(*db, link(*head, *lease, 22, 65)), LeaseStoreError::fenced));
auto next = acquire_region_lease(*db, key, 20000ms, 5s);
auto next = acquire_region_lease(*db, key, build, 20000ms, 5s);
CHECK(next && next->epoch == 2 && next->holder != lease->holder);
auto final_head = publish(*db, link(*head, *next, 22, 65));
CHECK(final_head && final_head->head.sequence == 2);
@@ -258,16 +258,16 @@ void database_time_expiry_and_paused_writer() {
const LeasePolicy fast{1000ms, 200ms, 100ms, 100ms, 300ms};
LeaseAuthority a{fast};
auto sent = LeaseClock::now();
auto lease_a = acquire_region_lease(*worker_a, key, fast.duration, 5s);
auto lease_a = acquire_region_lease(*worker_a, key, build, fast.duration, 5s);
CHECK(lease_a && a.acquired(*lease_a, sent, LeaseClock::now()));
CHECK(failed(acquire_region_lease(*worker_b, key, 20000ms, 5s), LeaseStoreError::busy));
CHECK(failed(acquire_region_lease(*worker_b, key, build, 20000ms, 5s), LeaseStoreError::busy));
const auto genesis = committed(*worker_a, key);
std::this_thread::sleep_for(1200ms); // Worker A is paused past its lease.
CHECK(a.state(LeaseClock::now()) == LeaseState::fenced);
// Expiry is decided by database time; the successor acquires a newer epoch.
wait_until_expired(*admin, key);
auto lease_b = acquire_region_lease(*worker_b, key, 20000ms, 5s);
auto lease_b = acquire_region_lease(*worker_b, key, build, 20000ms, 5s);
CHECK(lease_b && lease_b->epoch == lease_a->epoch + 1);
auto head_b = publish(*worker_b, link(genesis, *lease_b, 31, 3));
CHECK(head_b);
@@ -282,7 +282,7 @@ void database_time_expiry_and_paused_writer() {
// Expired renewal cannot revive a lease even without a successor.
CHECK(release_region_lease(*worker_b, key, *lease_b, 5s));
auto lease_c = acquire_region_lease(*worker_a, key, 1000ms, 5s);
auto lease_c = acquire_region_lease(*worker_a, key, build, 1000ms, 5s);
CHECK(lease_c && lease_c->epoch == lease_b->epoch + 1);
wait_until_expired(*admin, key);
CHECK(
@@ -295,7 +295,7 @@ void queued_writer_uses_post_lock_database_time() {
auto admin = owner();
const auto key = provision(*admin, 40);
auto worker = runtime();
auto lease = acquire_region_lease(*worker, key, 1000ms, 5s);
auto lease = acquire_region_lease(*worker, key, build, 1000ms, 5s);
CHECK(lease);
const auto genesis = committed(*worker, key);
auto blocker = owner();
@@ -325,7 +325,7 @@ void partitioned_renewal_fences_locally() {
const LeasePolicy fast{1000ms, 200ms, 100ms, 100ms, 300ms};
LeaseAuthority authority{fast};
const auto sent = LeaseClock::now();
auto lease = acquire_region_lease(*worker, key, fast.duration, 5s);
auto lease = acquire_region_lease(*worker, key, build, fast.duration, 5s);
CHECK(lease && authority.acquired(*lease, sent, LeaseClock::now()));
auto blocker = owner();
const std::array<std::string, 1> region{"50"};
@@ -346,7 +346,7 @@ void partitioned_renewal_fences_locally() {
CHECK(blocker->query("ROLLBACK", {}, 5s));
// The orphaned renewal reached the row only after database expiry: rejected.
auto successor = runtime();
auto next = acquire_region_lease(*successor, key, 20000ms, 5s);
auto next = acquire_region_lease(*successor, key, build, 20000ms, 5s);
CHECK(next && next->epoch == lease->epoch + 1);
}
@@ -365,7 +365,7 @@ void contenders_serialize_on_the_database() {
for (std::size_t i = 0; i < 2; ++i)
threads.emplace_back([&, i] {
start.arrive_and_wait();
results[i] = acquire_region_lease(*workers[i], key, 20000ms, 5s);
results[i] = acquire_region_lease(*workers[i], key, build, 20000ms, 5s);
});
for (auto& t : threads)
t.join();
@@ -380,7 +380,7 @@ void contenders_serialize_on_the_database() {
}
// The same holder racing two commits from the same predecessor: exactly one wins.
auto lease = acquire_region_lease(*workers[0], key, 20000ms, 5s);
auto lease = acquire_region_lease(*workers[0], key, build, 20000ms, 5s);
CHECK(lease);
auto head = committed(*workers[0], key);
for (std::uint8_t round = 0; round < 32; ++round) {
@@ -413,9 +413,9 @@ void exhaustion_fails_closed() {
"WHERE world_id = 1 AND region_id = 70",
{}, 5s));
auto db = runtime();
CHECK(failed(acquire_region_lease(*db, key, 20000ms, 5s), LeaseStoreError::exhausted));
CHECK(failed(acquire_region_lease(*db, key, build, 20000ms, 5s), LeaseStoreError::exhausted));
const auto other = provision(*admin, 71);
auto lease = acquire_region_lease(*db, other, 20000ms, 5s);
auto lease = acquire_region_lease(*db, other, build, 20000ms, 5s);
CHECK(lease);
// Owner-only fixture: a chain link at the last sequence (the head FK requires it).
CHECK(admin->query(
@@ -493,7 +493,7 @@ void measure_round_trips() {
for (int i = 0; i < n; ++i) {
std::expected<LeaseGrant, LeaseStoreFailure> lease = std::unexpected{LeaseStoreFailure{}};
acquire.samples.push_back(
timed([&] { lease = acquire_region_lease(*db, key, 20000ms, 5s); }));
timed([&] { lease = acquire_region_lease(*db, key, build, 20000ms, 5s); }));
CHECK(lease);
renew.samples.push_back(
timed([&] { CHECK(renew_region_lease(*db, key, *lease, 20000ms, 5s)); }));
+403
View File
@@ -0,0 +1,403 @@
// M7.9b schema migration, upgrade barrier and durability qualification (ADR-0068)
// against real PostgreSQL, real evo-server processes and the checked-in migrations
// (EVO_SOURCE_ROOT). Run by tools/ci/postgres-check.sh; prints measured latencies.
#include "evo/persistence/region_clone.hpp"
#include "evo/persistence/sha256.hpp"
#include "fixtures/server_process.hpp"
#include <algorithm>
#include <source_location>
#include <thread>
void run_region_migration_tests();
namespace {
using namespace evo::test::process;
using namespace evo::persistence;
void check(bool value, std::source_location at = std::source_location::current()) {
if (!value)
throw std::runtime_error("region migration test line " + std::to_string(at.line()));
}
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
constexpr std::size_t budget = 8000000;
const std::string b1_text = [] {
std::string s;
for (int i = 0; i < 32; ++i)
s += "b1";
return s;
}();
const std::string b2_text = [] {
std::string s;
for (int i = 0; i < 32; ++i)
s += "b2";
return s;
}();
Sha256Digest digest_of(std::uint8_t b) {
Sha256Digest d{};
d.fill(b);
return d;
}
const Sha256Digest b1 = digest_of(0xB1), b2 = digest_of(0xB2);
template <std::size_t N> std::array<std::uint8_t, N> filled(std::uint8_t value) {
std::array<std::uint8_t, N> out{};
out.fill(value);
return out;
}
std::unique_ptr<Postgres> connect_to(const std::string& database, const std::string& user) {
auto made = Postgres::connect({env("PGHOST"), {}, env("PGPORT"), database, user, {}, {}}, 5s);
CHECK(made);
return std::move(*made);
}
std::string sqlstate(const PgFailure& f) {
return f.sqlstate.data();
}
std::string scalar(Postgres& db, const std::string& sql) {
auto rows = db.query(sql, {}, 5s);
CHECK(rows && rows->size() == 1 && (*rows)[0].size() == 1);
return (*rows)[0][0].value_or("");
}
int shell(const std::string& command) {
Child c{{"/bin/bash", "-c", command}};
return c.wait_exit(120s);
}
std::vector<std::string> with(std::vector<std::string> args, const std::string& flag,
const std::string& value) {
const auto at = std::ranges::find(args, flag);
CHECK(at != args.end() && at + 1 != args.end());
*(at + 1) = value;
return args;
}
std::unique_ptr<FilesystemObjectStore> store(const ServerRegion& r) {
auto made = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes);
CHECK(made);
return std::move(*made);
}
std::optional<Uuid> parse_uuid(const std::string& text) {
std::string compact;
for (const char c : text)
if (c != '-')
compact.push_back(c);
if (compact.size() != 32)
return std::nullopt;
Uuid out{};
for (std::size_t i = 0; i < 16; ++i)
out[i] = static_cast<std::uint8_t>(std::stoul(compact.substr(2 * i, 2), nullptr, 16));
return out;
}
// A pre-v8 writer: the build-less acquisition every earlier host used.
std::expected<LeaseGrant, std::string> legacy_acquire(Postgres& db, RegionKey key) {
const std::array<std::string, 2> p{std::to_string(key.world), std::to_string(key.region)};
auto rows = db.query("SELECT * FROM evolution.acquire_region_lease($1::evolution.u64, "
"$2::evolution.u64, 20000)",
p, 5s);
if (!rows)
return std::unexpected{sqlstate(rows.error())};
CHECK(rows->size() == 1 && (*rows)[0].size() == 4);
const auto& r = (*rows)[0];
const auto holder = parse_uuid(r[0].value_or(""));
CHECK(holder.has_value());
return LeaseGrant{holder.value_or(Uuid{}), std::stoull(r[1].value_or("0")),
std::stoull(r[2].value_or("0")), 20000ms};
}
const char* const sha_function =
"s(){ sha256sum \"$EVO_SOURCE_ROOT/migrations/$1\" | cut -d' ' -f1; }; ";
std::string history(Postgres& owner) {
return scalar(owner, "SELECT string_agg(version::text, ',' ORDER BY version) FROM "
"evolution.schema_migrations");
}
// An existing v7 database is upgraded while an older writer holds a lease. A
// migration that fails leaves v7 exactly and the writer unaffected; the real one
// commits under it. The barrier then retires the old build: the writer is fenced
// and can never regain authority, while the new build recovers the whole history.
void upgrade_across_migration_and_barrier() {
const std::string database = "evo_online";
const std::string runtime_user = env("EVO_PG_RUNTIME_USER");
CHECK(shell(std::string(sha_function) + "cd \"$EVO_SOURCE_ROOT/migrations\" && createdb " +
database + " && PGDATABASE=" + database +
" psql -X -q -v ON_ERROR_STOP=1 -1 -v migration_sha=$(s 0001_region_bootstrap.sql)"
" -v migration_sha_2=$(s 0002_region_fencing.sql)"
" -v migration_sha_3=$(s 0003_region_checkpoints.sql)"
" -v migration_sha_4=$(s 0004_lineage_profile.sql)"
" -v migration_sha_5=$(s 0005_batch_decisions.sql)"
" -v migration_sha_6=$(s 0006_commit_lookup.sql)"
" -v migration_sha_7=$(s 0007_region_clones.sql)"
" -v runtime_role=evo_runtime_test -f 0001_region_bootstrap.sql"
" -f 0002_region_fencing.sql -f 0003_region_checkpoints.sql"
" -f 0004_lineage_profile.sql -f 0005_batch_decisions.sql"
" -f 0006_commit_lookup.sql -f 0007_region_clones.sql > /dev/null") == 0);
auto owner_db = connect_to(database, env("PGUSER"));
CHECK(history(*owner_db) == "1,2,3,4,5,6,7");
CHECK(owner_db->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
"(decode(repeat('b1', 32), 'hex')), (decode(repeat('b2', 32), 'hex'))",
{}, 5s));
const ServerRegion r{{7, 91}, fs::path(env("PGHOST")) / "online-91"};
fs::remove_all(r.root);
fs::create_directory(r.root);
fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace);
auto objects = store(r);
auto genesis =
provision_lineage_region(*owner_db, *objects, r.key, evo::test::region_identity.frame,
*evo::test::lineage_region(), filled<16>(0xDA), b1, budget, 5s);
CHECK(genesis);
// The old writer: build-less lease, publishing with build b1.
auto runtime = connect_to(database, runtime_user);
auto lease = legacy_acquire(*runtime, r.key);
CHECK(lease);
auto state = evo::test::lineage_region();
RegionBatch<evo::processes::LineageRegion> batch{*state};
CommittedCheckpoint head = *genesis;
std::uint8_t commit_seed = 0xDB;
const auto publish_next = [&](std::uint64_t ticks) {
for (std::uint64_t i = 0; i < ticks && !batch.sealed(); ++i)
CHECK(batch.advance(evo::test::region_input(batch.end_tick() + 1, true)));
batch.seal();
return publish_lineage_batch(*runtime, *objects, *lease, head, batch,
filled<16>(commit_seed++), b1, budget, 5s);
};
const auto confirm = [&](const std::expected<PublishedBatch, PublicationFailure>& done) {
CHECK(done && batch.confirm(done->committed.head.end_tick));
head = done->committed;
};
confirm(publish_next(64)); // Ticks 1..20: the consumer's death seals it.
CHECK(head.head.end_tick == 20);
// A v8 migration that fails at its last statement changes nothing.
CHECK(shell(std::string(sha_function) + "cd \"$EVO_SOURCE_ROOT\" && PGDATABASE=" + database +
" psql -X -q -v ON_ERROR_STOP=1 -v migration_sha=$(s 0001_region_bootstrap.sql)"
" -v migration_sha_2=$(s 0002_region_fencing.sql)"
" -v migration_sha_3=$(s 0003_region_checkpoints.sql)"
" -v migration_sha_4=$(s 0004_lineage_profile.sql)"
" -v migration_sha_5=$(s 0005_batch_decisions.sql)"
" -v migration_sha_6=$(s 0006_commit_lookup.sql)"
" -v migration_sha_7=$(s 0007_region_clones.sql) -v migration_sha_8=invalid"
" -v runtime_role=evo_runtime_test -f migrations/apply.sql"
" > /dev/null 2>&1") != 0);
CHECK(history(*owner_db) == "1,2,3,4,5,6,7");
CHECK(scalar(*owner_db, "SELECT count(*) FROM information_schema.columns WHERE "
"table_schema = 'evolution' AND column_name = 'retired_at'") == "0");
CHECK(scalar(*owner_db,
"SELECT (to_regprocedure('evolution.acquire_region_lease("
"evolution.u64, evolution.u64, integer, bytea)') IS NULL)::text") == "true");
CHECK(renew_region_lease(*runtime, r.key, *lease, 20000ms, 5s));
confirm(publish_next(20)); // 21..40, still v7.
// The real migration commits while the writer holds its lease; it keeps going.
CHECK(shell("PGDATABASE=" + database +
" EVO_PG_RUNTIME_ROLE=evo_runtime_test bash \"$EVO_SOURCE_ROOT/"
"tools/migrate-postgres.sh\" > /dev/null") == 0);
CHECK(history(*owner_db) == "1,2,3,4,5,6,7,8");
CHECK(renew_region_lease(*runtime, r.key, *lease, 20000ms, 5s));
confirm(publish_next(20)); // 41..60, now v8.
CHECK(head.head.end_tick == 60);
// Barrier guards: the last current build cannot be retired; runtime cannot call it.
const std::string retire_b1 = "SELECT evolution.upgrade_barrier(decode(repeat('b1', 32), "
"'hex'))::text";
auto denied = runtime->query(retire_b1, {}, 5s);
CHECK(!denied && sqlstate(denied.error()) == "42501");
// The barrier: b1 retired and every holder fenced, in one transaction.
CHECK(scalar(*owner_db, retire_b1) == "1");
auto last_b2 = owner_db->query(
"SELECT evolution.upgrade_barrier(decode(repeat('b2', 32), 'hex'))", {}, 5s);
CHECK(!last_b2 && sqlstate(last_b2.error()) == "22023");
// The old writer is fenced and cannot regain authority by any path.
auto after = publish_next(10);
CHECK(!after && after.error().kind == PublicationError::rejected &&
after.error().store.kind == LeaseStoreError::fenced);
auto renewed = renew_region_lease(*runtime, r.key, *lease, 20000ms, 5s);
CHECK(!renewed && renewed.error().kind == LeaseStoreError::fenced);
auto legacy = legacy_acquire(*runtime, r.key);
CHECK(!legacy && legacy.error() == "EVL06");
auto retired = acquire_region_lease(*runtime, r.key, b1, 20000ms, 5s);
CHECK(!retired && retired.error().kind == LeaseStoreError::unsupported);
// Authority for b2 still cannot publish a link naming the retired build.
auto current = acquire_region_lease(*runtime, r.key, b2, 20000ms, 5s);
CHECK(current);
lease = *current;
auto stale_build = publish_lineage_batch(*runtime, *objects, *lease, head, batch,
filled<16>(commit_seed++), b1, budget, 5s);
CHECK(!stale_build && stale_build.error().kind == PublicationError::rejected &&
stale_build.error().store.kind == LeaseStoreError::unsupported);
CHECK(release_region_lease(*runtime, r.key, *lease, 5s));
const auto before_new = head.head;
// The new build resumes from the committed head, and b1 history stays readable.
auto args = with(with(server_args(r), "--pg-database", database), "--build", b2_text);
args.insert(args.end(), {"--retained", b1_text});
{
Child server{args};
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
CHECK(server.printed("phase running epoch=3 committed=60 "));
server.send(input_lines(61, 90, true));
CHECK(server.wait_line("ack " + request_hex(90) + " 90 ", 30s));
server.signal(SIGTERM);
CHECK(server.wait_exit(20s) == 0);
}
auto final_head = load_region_checkpoint(*owner_db, r.key, 5s);
CHECK(final_head && final_head->head.end_tick == 90 &&
final_head->head.sequence > before_new.sequence);
const auto final_manifest = decode_manifest(final_head->manifest);
CHECK(final_manifest && final_manifest->build == b2);
const std::array<Sha256Digest, 2> both{b1, b2};
const std::array<Sha256Digest, 1> only_new{b2};
auto audited = audit_region_history(*runtime, *objects, r.key, both, budget, 100, 5s);
CHECK(audited && audited->commits == final_head->head.sequence + 1 && audited->outcomes == 1);
auto without_old = audit_region_history(*runtime, *objects, r.key, only_new, budget, 100, 5s);
CHECK(!without_old && without_old.error().kind == AuditError::unsupported &&
without_old.error().sequence == 0);
// A host of the retired build never serves, not even once the region is free.
{
Child old_host{with(with(server_args(r), "--pg-database", database), "--build", b1_text)};
CHECK(old_host.wait_exit(20s) == 4);
CHECK(old_host.printed("phase quarantined") && !old_host.printed("ready 1"));
}
CHECK(load_region_checkpoint(*owner_db, r.key, 5s)->head == final_head->head);
}
std::uint64_t percentile(std::vector<std::uint64_t> v, unsigned p) {
CHECK(!v.empty());
std::ranges::sort(v);
return v[std::min(v.size() - 1, (v.size() * p) / 100)];
}
std::uint64_t ms_between(Clock::time_point a, Clock::time_point b) {
return static_cast<std::uint64_t>(
std::chrono::duration_cast<std::chrono::milliseconds>(b - a).count());
}
// Sends inputs first..last paced at the 50 ms tick; returns send times.
std::vector<Clock::time_point> paced(Child& server, std::uint64_t first, std::uint64_t last) {
std::vector<Clock::time_point> sent;
const auto start = Clock::now();
for (auto tick = first; tick <= last; ++tick) {
const auto due = start + std::chrono::milliseconds{50 * (tick - first)};
if (const auto now = Clock::now(); due > now)
server.pump(std::chrono::duration_cast<std::chrono::milliseconds>(due - now));
server.send(input_line(tick, true));
sent.push_back(Clock::now());
}
return sent;
}
std::optional<Clock::time_point> ack_time(const Child& server, std::uint64_t tick) {
const auto prefix = "ack " + request_hex(tick) + " " + std::to_string(tick) + " ";
for (std::size_t i = 0; i < server.lines().size(); ++i)
if (starts(server.lines()[i], prefix))
return server.times()[i];
return std::nullopt;
}
Clock::time_point ready_time(const Child& server) {
for (std::size_t i = 0; i < server.lines().size(); ++i)
if (server.lines()[i] == "ready 1")
return server.times()[i];
CHECK(false);
return {};
}
// The M7 exit fixture: production lease policy (20 s lease, 5 s renewal) and
// flush interval (1 s), one lineage region on a local socket and filesystem.
// Measures acknowledgement latency, worker-kill-to-ready recovery against the
// proposed 60 s target, and the cost of verification, audit and clone.
void durability_qualification() {
const auto r = provision(93, 0xDC);
const auto args =
with(with(server_args(r), "--lease-profile", "default"), "--flush-ms", "1000");
std::vector<std::uint64_t> latency, kill_to_ready;
std::uint64_t death_latency = 0;
std::uint64_t next = 1;
std::uint64_t epoch = 1;
for (int round = 0; round < 3; ++round) {
Child server{args};
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 60s));
CHECK(server.printed("phase running epoch=" + std::to_string(epoch) + " "));
const auto first = next;
const auto last = next + 99;
const auto sent = paced(server, first, last);
CHECK(server.wait_line("ack " + request_hex(last) + " " + std::to_string(last) + " ", 30s));
for (auto tick = first; tick <= last; ++tick) {
const auto at = ack_time(server, tick);
CHECK(at.has_value());
const auto ms = ms_between(sent[tick - first], at.value_or(Clock::time_point{}));
if (tick == 20)
death_latency = ms;
else
latency.push_back(ms);
}
// Mid-batch force kill: more inputs are in flight, none acknowledged yet.
server.send(input_lines(last + 1, last + 10, true));
server.pump(200ms);
const auto acked = max_acked(server);
server.signal(SIGKILL);
const auto killed = Clock::now();
CHECK(server.wait_exit(20s) == 128 + SIGKILL);
const auto committed = head_of(r.key);
CHECK(acked <= committed.tick && acked >= last); // No acknowledged input lost.
next = committed.tick + 1;
++epoch;
if (round == 2)
break;
Child successor{args};
CHECK(successor.wait_for([](const std::string& l) { return l == "ready 1"; }, 60s));
kill_to_ready.push_back(ms_between(killed, ready_time(successor)));
CHECK(successor.printed("phase running epoch=" + std::to_string(epoch) +
" committed=" + std::to_string(committed.tick) + " "));
successor.signal(SIGTERM);
CHECK(successor.wait_exit(20s) == 0);
++epoch;
}
CHECK(decisions_of(r.key) == 1);
for (const auto ms : kill_to_ready)
CHECK(ms < 60000); // The declared target for this fixture.
std::printf("qualification ack_latency_ms flush=1000 n=%zu p50=%llu p99=%llu max=%llu "
"death=%llu\n",
latency.size(), static_cast<unsigned long long>(percentile(latency, 50)),
static_cast<unsigned long long>(percentile(latency, 99)),
static_cast<unsigned long long>(percentile(latency, 100)),
static_cast<unsigned long long>(death_latency));
std::printf("qualification kill_to_ready_ms lease=20000 renew=5000 runs=%zu first=%llu "
"second=%llu target=60000\n",
kill_to_ready.size(), static_cast<unsigned long long>(kill_to_ready[0]),
static_cast<unsigned long long>(kill_to_ready[1]));
// Verification cost on the final head (no lease needed to read).
auto reader = connect_to(env("PGDATABASE"), env("EVO_PG_RUNTIME_USER"));
auto objects = store(r);
const std::array<Sha256Digest, 1> retained{b1};
std::vector<std::uint64_t> verify_us;
std::uint64_t payload_bytes = 0;
for (int i = 0; i < 10; ++i) {
const auto start = Clock::now();
auto recovered = recover_lineage_checkpoint(*reader, *objects, r.key, retained, budget, 5s);
CHECK(recovered);
payload_bytes = recovered->manifest.payload_bytes;
verify_us.push_back(static_cast<std::uint64_t>(
std::chrono::duration_cast<std::chrono::microseconds>(Clock::now() - start).count()));
}
const auto audit_start = Clock::now();
auto audited = audit_region_history(*reader, *objects, r.key, retained, budget, 1000, 5s);
const auto audit_ms = ms_between(audit_start, Clock::now());
CHECK(audited);
auto clone_owner = connect_to(env("EVO_PG_CLONE_DATABASE"), env("PGUSER"));
const ServerRegion target{{8, 93}, fs::path(env("PGHOST")) / "qualify-clone-93"};
fs::remove_all(target.root);
fs::create_directory(target.root);
fs::permissions(target.root, fs::perms::owner_all, fs::perm_options::replace);
const auto clone_start = Clock::now();
auto cloned = clone_region(*reader, *objects, r.key, *clone_owner, *store(target), target.key,
filled<16>(0xDD), retained, budget, 5s);
const auto clone_ms = ms_between(clone_start, Clock::now());
CHECK(cloned);
std::printf("qualification verify_head_us payload_bytes=%llu n=10 p50=%llu max=%llu\n",
static_cast<unsigned long long>(payload_bytes),
static_cast<unsigned long long>(percentile(verify_us, 50)),
static_cast<unsigned long long>(percentile(verify_us, 100)));
std::printf("qualification audit_ms commits=%llu ticks=%llu ms=%llu clone_ms=%llu\n",
static_cast<unsigned long long>(audited->commits),
static_cast<unsigned long long>(audited->end_tick),
static_cast<unsigned long long>(audit_ms),
static_cast<unsigned long long>(clone_ms));
}
} // namespace
void run_region_migration_tests() {
durability_qualification();
upgrade_across_migration_and_barrier();
}
+1 -1
View File
@@ -193,7 +193,7 @@ void tail_verification_before_readiness() {
id.fill(v);
return id;
};
auto lease = acquire_region_lease(**runtime, r.key, 20000ms, 5s);
auto lease = acquire_region_lease(**runtime, r.key, build, 20000ms, 5s);
CHECK(lease);
auto head = load_region_checkpoint(**runtime, r.key, 5s);
CHECK(head);