M12.4: retain migration source freezes and explicit aborts
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m24s
Foundation / checks (network) (push) Successful in 20m44s
Foundation / checks (client) (push) Successful in 22m3s
Foundation / checks (tsan) (push) Successful in 23m1s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 24m7s
Foundation / checks (postgres) (push) Canceled after 24m3s

This commit is contained in:
dingo
2026-10-18 09:08:50 -04:00
parent 1b5c0cdf7c
commit c625a4cb68
25 changed files with 1342 additions and 18 deletions
@@ -0,0 +1,255 @@
#include "evo/persistence/cellular_migration_store.hpp"
#include "evo/persistence/sha256.hpp"
#include <algorithm>
#include <charconv>
#include <string_view>
namespace evo::persistence {
namespace {
using std::chrono::milliseconds;
LeaseStoreFailure failure(const PgFailure& cause) {
const std::string_view state(cause.sqlstate.data(), 5);
auto kind = LeaseStoreError::store;
if (cause.kind == PgError::unknown)
kind = LeaseStoreError::unknown;
else if (state == "22023" || cause.kind == PgError::bounds || cause.kind == PgError::config)
kind = LeaseStoreError::invalid;
else if (state == "EVL02")
kind = LeaseStoreError::fenced;
else if (state == "EVL03")
kind = LeaseStoreError::head_mismatch;
else if (state == "EVL04")
kind = LeaseStoreError::not_provisioned;
else if (state == "EVL06")
kind = LeaseStoreError::unsupported;
else if (state == "EVL07" || state == "EVL18")
kind = LeaseStoreError::commit_conflict;
return {kind, cause};
}
std::string hex(std::span<const std::uint8_t> bytes) {
constexpr std::string_view digits = "0123456789abcdef";
std::string out;
for (const auto byte : bytes) {
out += digits[byte >> 4U];
out += digits[byte & 15U];
}
return out;
}
std::string uuid(const Uuid& id) {
const auto h = hex(id);
return h.substr(0, 8) + '-' + h.substr(8, 4) + '-' + h.substr(12, 4) + '-' + h.substr(16, 4) +
'-' + h.substr(20);
}
std::optional<std::vector<std::uint8_t>> unhex(std::string_view value, std::size_t cap) {
if (value.size() % 2 != 0 || value.size() / 2 > cap)
return std::nullopt;
std::vector<std::uint8_t> out;
out.reserve(value.size() / 2);
for (std::size_t at = 0; at < value.size(); at += 2) {
unsigned byte{};
const auto [end, error] =
std::from_chars(value.data() + at, value.data() + at + 2, byte, 16);
if (error != std::errc{} || end != value.data() + at + 2 || byte > 255)
return std::nullopt;
out.push_back(static_cast<std::uint8_t>(byte));
}
return out;
}
struct Record {
std::vector<std::uint8_t> trace, source, destination;
bool aborted{};
};
std::optional<Record> parse(const PgRows::value_type& row) {
if (row.size() != 4 || (row[3].value_or("") != "true" && row[3].value_or("") != "false"))
return std::nullopt;
auto trace = unhex(row[0].value_or(""), server::max_migration_trace_bytes);
auto source = unhex(row[1].value_or(""), checkpoint_manifest_bytes);
auto destination = unhex(row[2].value_or(""), checkpoint_manifest_bytes);
if (!trace || !source || !destination)
return std::nullopt;
return Record{std::move(*trace), std::move(*source), std::move(*destination),
row[3].value_or("") == "true"};
}
bool bound(const CommittedCheckpoint& c, const CheckpointManifest& m) {
const auto digest = sha256(c.manifest);
return digest && *digest == c.head.digest && m.commit == c.head.commit &&
m.frame == c.head.frame && m.sequence == c.head.sequence &&
m.end_tick == c.head.end_tick;
}
bool cuts(const CheckpointManifest& s, const CheckpointManifest& d,
const Sha256Digest& source_digest, const Sha256Digest& destination_digest,
const server::CellularMigrationTrace& t) {
const auto profile = t.representation == server::CellularMigrationRepresentation::individual
? CheckpointProfile::cellular_lives_v1
: CheckpointProfile::cellular_population_v1;
return supported(s) && supported(d) && encode_manifest(s) && encode_manifest(d) &&
s.cellular_versions == cellular_v10_versions &&
d.cellular_versions == cellular_v10_versions && s.profile == profile &&
d.profile == profile && s.sequence > 0 && d.sequence > 0 &&
s.key == RegionKey{t.world, t.request.source.region} &&
d.key == RegionKey{t.world, t.request.destination.region} && s.frame == t.frame &&
d.frame == t.frame && s.end_tick == t.tick && d.end_tick == t.tick &&
s.commit == t.request.source.predecessor &&
d.commit == t.request.destination.predecessor &&
source_digest == t.request.source.predecessor_digest &&
destination_digest == t.request.destination.predecessor_digest;
}
std::expected<server::CellularMigrationTrace, CheckpointError>
derive(const CheckpointManifest& s, const CheckpointManifest& d,
std::span<const std::uint8_t> source, std::span<const std::uint8_t> destination,
const server::CellularMigrationRequest& request) {
if (s.profile == CheckpointProfile::cellular_lives_v1) {
auto src = restore_cellular_payload(source, s),
dst = restore_cellular_payload(destination, d);
if (!src || !dst)
return std::unexpected(CheckpointError::payload);
auto pair = server::prepare_migration_trace(**src, **dst, request);
if (!pair)
return std::unexpected(CheckpointError::payload);
return std::move(pair->trace);
}
auto src = restore_population_payload(source, s),
dst = restore_population_payload(destination, d);
if (!src || !dst)
return std::unexpected(CheckpointError::payload);
auto pair = server::prepare_migration_trace(**src, **dst, request);
if (!pair)
return std::unexpected(CheckpointError::payload);
return std::move(pair->trace);
}
std::optional<CellularMigrationFreeze> checked_record(const Record& record, RegionKey key,
const LeaseGrant& lease) {
auto trace = server::decode_migration_trace(record.trace);
const auto source = decode_manifest(record.source),
destination = decode_manifest(record.destination);
const auto s = sha256(record.source), d = sha256(record.destination);
if (!trace || !source || !destination || !s || !d ||
key != RegionKey{trace->world, trace->request.source.region} ||
lease.frame != trace->frame || !cuts(*source, *destination, *s, *d, *trace))
return std::nullopt;
return CellularMigrationFreeze{std::move(*trace), record.aborted};
}
constexpr auto columns = "encode(trace,'hex'),encode(source_manifest,'hex'),"
"encode(destination_manifest,'hex'),aborted::text";
CellularMigrationFailure failed(PublicationError kind, LeaseStoreFailure store = {},
ObjectError object = {}, CellularMigrationPrepared prepared = {}) {
return {kind, store, object, std::move(prepared)};
}
} // namespace
std::expected<CellularMigrationFreeze, CellularMigrationFailure>
retry_cellular_migration_freeze(Postgres& db, const LeaseGrant& lease,
const CellularMigrationPrepared& prepared, milliseconds timeout) {
const auto trace = server::decode_migration_trace(prepared.trace);
if (!trace || prepared.key != RegionKey{trace->world, trace->request.source.region} ||
lease.frame != trace->frame)
return std::unexpected(failed(PublicationError::candidate));
const std::array<std::string, 5> args{std::to_string(prepared.key.world),
std::to_string(prepared.key.region), uuid(lease.holder),
std::to_string(lease.epoch), "\\x" + hex(prepared.trace)};
auto rows =
db.query(std::string("SELECT ") + columns +
" FROM evolution.prepare_cellular_migration_freeze("
"$1::evolution.u64,$2::evolution.u64,$3::uuid,$4::evolution.u64,$5::bytea)",
args, timeout);
if (!rows) {
const auto store = failure(rows.error());
return std::unexpected(failed(store.kind == LeaseStoreError::unknown
? PublicationError::unknown
: PublicationError::rejected,
store, {}, prepared));
}
const auto record = rows->size() == 1 ? parse(rows->front()) : std::nullopt;
const auto result = record ? checked_record(*record, prepared.key, lease) : std::nullopt;
if (!result || record->trace != prepared.trace)
return std::unexpected(failed(PublicationError::unknown, {}, {}, prepared));
return *result;
}
std::expected<CellularMigrationFreeze, CellularMigrationFailure>
prepare_cellular_migration_freeze(Postgres& db, const FilesystemObjectStore& source_objects,
const FilesystemObjectStore& destination_objects,
const LeaseGrant& lease, const CommittedCheckpoint& source,
const CommittedCheckpoint& destination,
const server::CellularMigrationRequest& request,
milliseconds timeout, const PublishHooks& hooks) {
const auto s = decode_manifest(source.manifest), d = decode_manifest(destination.manifest);
if (!s || !d || !bound(source, *s) || !bound(destination, *d))
return std::unexpected(failed(PublicationError::predecessor));
auto src = source_objects.get({s->payload_sha256, s->payload_bytes});
auto dst = destination_objects.get({d->payload_sha256, d->payload_bytes});
if (!src || !dst)
return std::unexpected(
failed(PublicationError::predecessor, {}, !src ? src.error() : dst.error()));
auto trace = derive(*s, *d, *src, *dst, request);
if (!trace || !cuts(*s, *d, source.head.digest, destination.head.digest, *trace) ||
lease.frame != s->frame)
return std::unexpected(failed(PublicationError::candidate));
auto encoded = server::encode_migration_trace(*trace);
if (!encoded)
return std::unexpected(failed(PublicationError::candidate));
auto capsule = source_objects.put(*dst);
if (!capsule)
return std::unexpected(failed(PublicationError::upload, {}, capsule.error()));
auto readback = source_objects.get(*capsule);
if (!readback || *readback != *dst ||
*capsule != VerifiedObject{d->payload_sha256, d->payload_bytes})
return std::unexpected(failed(PublicationError::verify, {},
readback ? ObjectError::corrupt : readback.error()));
if (hooks.before_transaction)
hooks.before_transaction();
return retry_cellular_migration_freeze(db, lease, {s->key, std::move(*encoded)}, timeout);
}
std::expected<CellularMigrationFreeze, LeaseStoreFailure>
abort_cellular_migration_freeze(Postgres& db, RegionKey key, const LeaseGrant& lease,
const Uuid& request, milliseconds timeout) {
const std::array<std::string, 5> args{std::to_string(key.world), std::to_string(key.region),
uuid(lease.holder), std::to_string(lease.epoch),
uuid(request)};
auto rows =
db.query(std::string("SELECT ") + columns +
" FROM evolution.abort_cellular_migration_freeze("
"$1::evolution.u64,$2::evolution.u64,$3::uuid,$4::evolution.u64,$5::uuid)",
args, timeout);
if (!rows)
return std::unexpected(failure(rows.error()));
const auto record = rows->size() == 1 ? parse(rows->front()) : std::nullopt;
auto result = record ? checked_record(*record, key, lease) : std::nullopt;
if (!result || !result->aborted || result->trace.request.request != request)
return std::unexpected(LeaseStoreFailure{LeaseStoreError::unknown, {}});
return *result;
}
std::expected<std::optional<CellularMigrationFreeze>, RecoveryFailure>
load_cellular_migration_freeze(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
const LeaseGrant& lease, milliseconds timeout) {
const auto fail = [](RecoveryError kind) { return std::unexpected(RecoveryFailure{kind, {}}); };
const std::array<std::string, 4> args{std::to_string(key.world), std::to_string(key.region),
uuid(lease.holder), std::to_string(lease.epoch)};
auto rows = db.query(std::string("SELECT ") + columns +
" FROM evolution.load_cellular_migration_freeze("
"$1::evolution.u64,$2::evolution.u64,$3::uuid,$4::evolution.u64)",
args, timeout);
if (!rows)
return std::unexpected(RecoveryFailure{RecoveryError::store, failure(rows.error())});
if (rows->empty())
return std::optional<CellularMigrationFreeze>{};
const auto record = rows->size() == 1 ? parse(rows->front()) : std::nullopt;
auto result = record ? checked_record(*record, key, lease) : std::nullopt;
if (!result || result->aborted)
return fail(RecoveryError::history);
const auto s = decode_manifest(record->source), d = decode_manifest(record->destination);
if (!s || !d)
return fail(RecoveryError::manifest);
auto src = objects.get({s->payload_sha256, s->payload_bytes});
auto dst = objects.get({d->payload_sha256, d->payload_bytes});
if (!src || !dst)
return fail(RecoveryError::tail);
const auto replay = derive(*s, *d, *src, *dst, result->trace.request);
if (!replay || *replay != result->trace)
return fail(RecoveryError::history);
return result;
}
} // namespace evo::persistence