M12.4: retain read-only destination migration stages
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m25s
Foundation / checks (network) (push) Successful in 20m44s
Foundation / checks (client) (push) Successful in 21m57s
Foundation / checks (tsan) (push) Successful in 22m46s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 30m2s
Foundation / checks (postgres) (push) Canceled after 29m56s

This commit is contained in:
dingo
2026-10-18 09:32:50 -04:00
parent c625a4cb68
commit 0515702964
21 changed files with 1023 additions and 21 deletions
@@ -252,4 +252,179 @@ load_cellular_migration_freeze(Postgres& db, const FilesystemObjectStore& object
return fail(RecoveryError::history);
return result;
}
namespace {
constexpr auto stage_columns = "encode(trace,'hex'),encode(source_manifest,'hex'),"
"encode(destination_manifest,'hex'),aborted::text,"
"encode(ghost_sha256,'hex'),ghost_bytes::text";
struct StageRecord {
Record cut;
VerifiedObject ghost;
};
std::optional<StageRecord> parse_stage(const PgRows::value_type& row) {
if (row.size() != 6)
return std::nullopt;
auto cut = parse(PgRows::value_type(row.begin(), row.begin() + 4));
const auto digest = unhex(row[4].value_or(""), 32);
const auto size = row[5].value_or("");
std::uint64_t bytes{};
const auto [end, error] = std::from_chars(size.data(), size.data() + size.size(), bytes);
if (!cut || !digest || digest->size() != 32 || error != std::errc{} ||
end != size.data() + size.size() || bytes == 0)
return std::nullopt;
VerifiedObject ghost{{}, bytes};
std::ranges::copy(*digest, ghost.sha256.begin());
return StageRecord{std::move(*cut), ghost};
}
std::optional<CellularMigrationStage> checked_stage(const StageRecord& record, RegionKey key,
const LeaseGrant& lease) {
const auto trace = server::decode_migration_trace(record.cut.trace);
if (!trace || key != RegionKey{trace->world, trace->request.destination.region})
return std::nullopt;
const auto freeze =
checked_record(record.cut, {trace->world, trace->request.source.region}, lease);
const auto destination = decode_manifest(record.cut.destination);
if (!freeze || !destination || record.ghost.sha256 == Sha256Digest{} ||
record.ghost.bytes > max_payload_bytes(destination->profile))
return std::nullopt;
return CellularMigrationStage{*freeze, record.ghost};
}
std::optional<std::vector<std::uint8_t>>
ghost_candidate(const CheckpointManifest& s, const CheckpointManifest& d,
std::span<const std::uint8_t> source, std::span<const std::uint8_t> destination,
const server::CellularMigrationTrace& trace) {
if (s.profile == CheckpointProfile::cellular_lives_v1) {
auto src = restore_cellular_payload(source, s),
dst = restore_cellular_payload(destination, d);
if (!src || !dst)
return std::nullopt;
auto pair = server::replay_migration_trace(**src, **dst, trace.request, trace);
if (!pair)
return std::nullopt;
return pair->destination->snapshot();
}
auto src = restore_population_payload(source, s),
dst = restore_population_payload(destination, d);
if (!src || !dst)
return std::nullopt;
auto pair = server::replay_migration_trace(**src, **dst, trace.request, trace);
if (!pair)
return std::nullopt;
return pair->destination->snapshot();
}
CellularMigrationStageFailure stage_failed(PublicationError kind, LeaseStoreFailure store = {},
ObjectError object = {},
CellularMigrationStagePrepared prepared = {}) {
return {kind, store, object, std::move(prepared)};
}
} // namespace
std::expected<CellularMigrationStage, CellularMigrationStageFailure>
retry_cellular_migration_stage(Postgres& db, const LeaseGrant& lease,
const CellularMigrationStagePrepared& prepared,
milliseconds timeout) {
const auto trace = server::decode_migration_trace(prepared.trace);
if (!trace || prepared.key != RegionKey{trace->world, trace->request.destination.region} ||
lease.frame != trace->frame || prepared.ghost.sha256 == Sha256Digest{} ||
prepared.ghost.bytes == 0)
return std::unexpected(stage_failed(PublicationError::candidate));
const std::array<std::string, 7> args{std::to_string(prepared.key.world),
std::to_string(prepared.key.region),
uuid(lease.holder),
std::to_string(lease.epoch),
"\\x" + hex(prepared.trace),
"\\x" + hex(prepared.ghost.sha256),
std::to_string(prepared.ghost.bytes)};
auto rows = db.query(std::string("SELECT ") + stage_columns +
" FROM evolution.stage_cellular_migration($1::evolution.u64,"
"$2::evolution.u64,$3::uuid,$4::evolution.u64,$5::bytea,$6::bytea,"
"$7::evolution.u64)",
args, timeout);
if (!rows) {
const auto store = failure(rows.error());
return std::unexpected(stage_failed(store.kind == LeaseStoreError::unknown
? PublicationError::unknown
: PublicationError::rejected,
store, {}, prepared));
}
const auto record = rows->size() == 1 ? parse_stage(rows->front()) : std::nullopt;
const auto result = record ? checked_stage(*record, prepared.key, lease) : std::nullopt;
if (!result || record->cut.trace != prepared.trace || result->ghost != prepared.ghost)
return std::unexpected(stage_failed(PublicationError::unknown, {}, {}, prepared));
return *result;
}
std::expected<CellularMigrationStage, CellularMigrationStageFailure>
stage_cellular_migration(Postgres& db, const FilesystemObjectStore& source_objects,
const FilesystemObjectStore& destination_objects, const LeaseGrant& lease,
const CommittedCheckpoint& source, const CommittedCheckpoint& destination,
const server::CellularMigrationTrace& trace, milliseconds timeout,
const PublishHooks& hooks) {
const auto s = decode_manifest(source.manifest), d = decode_manifest(destination.manifest);
if (!s || !d || !bound(source, *s) || !bound(destination, *d) ||
!cuts(*s, *d, source.head.digest, destination.head.digest, trace) ||
lease.frame != d->frame)
return std::unexpected(stage_failed(PublicationError::predecessor));
auto src = source_objects.get({s->payload_sha256, s->payload_bytes});
auto dst = destination_objects.get({d->payload_sha256, d->payload_bytes});
if (!src || !dst)
return std::unexpected(
stage_failed(PublicationError::predecessor, {}, !src ? src.error() : dst.error()));
const auto candidate = ghost_candidate(*s, *d, *src, *dst, trace);
const auto encoded = server::encode_migration_trace(trace);
if (!candidate || !encoded)
return std::unexpected(stage_failed(PublicationError::candidate));
auto capsule = destination_objects.put(*src);
if (!capsule)
return std::unexpected(stage_failed(PublicationError::upload, {}, capsule.error()));
auto capsule_bytes = destination_objects.get(*capsule);
if (!capsule_bytes || *capsule_bytes != *src ||
*capsule != VerifiedObject{s->payload_sha256, s->payload_bytes})
return std::unexpected(
stage_failed(PublicationError::verify, {},
capsule_bytes ? ObjectError::corrupt : capsule_bytes.error()));
auto ghost = destination_objects.put(*candidate);
if (!ghost)
return std::unexpected(stage_failed(PublicationError::upload, {}, ghost.error()));
auto readback = destination_objects.get(*ghost);
if (!readback || *readback != *candidate)
return std::unexpected(stage_failed(PublicationError::verify, {},
readback ? ObjectError::corrupt : readback.error()));
if (hooks.before_transaction)
hooks.before_transaction();
return retry_cellular_migration_stage(db, lease, {d->key, *encoded, *ghost}, timeout);
}
std::expected<std::optional<CellularMigrationStage>, RecoveryFailure>
load_cellular_migration_stage(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
const LeaseGrant& lease, milliseconds timeout) {
const auto fail = [](RecoveryError kind) { return std::unexpected(RecoveryFailure{kind, {}}); };
const std::array<std::string, 4> args{std::to_string(key.world), std::to_string(key.region),
uuid(lease.holder), std::to_string(lease.epoch)};
auto rows = db.query(std::string("SELECT ") + stage_columns +
" FROM evolution.load_cellular_migration_stage($1::evolution.u64,"
"$2::evolution.u64,$3::uuid,$4::evolution.u64)",
args, timeout);
if (!rows)
return std::unexpected(RecoveryFailure{RecoveryError::store, failure(rows.error())});
if (rows->empty())
return std::optional<CellularMigrationStage>{};
const auto record = rows->size() == 1 ? parse_stage(rows->front()) : std::nullopt;
auto result = record ? checked_stage(*record, key, lease) : std::nullopt;
if (!result || result->freeze.aborted)
return fail(RecoveryError::history);
const auto s = decode_manifest(record->cut.source),
d = decode_manifest(record->cut.destination);
if (!s || !d)
return fail(RecoveryError::manifest);
auto src = objects.get({s->payload_sha256, s->payload_bytes});
auto dst = objects.get({d->payload_sha256, d->payload_bytes});
auto ghost = objects.get(result->ghost);
if (!src || !dst || !ghost)
return fail(RecoveryError::tail);
const auto candidate = ghost_candidate(*s, *d, *src, *dst, result->freeze.trace);
if (!candidate || *candidate != *ghost)
return fail(RecoveryError::history);
return result;
}
} // namespace evo::persistence