M12.4: retain read-only destination migration stages
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m25s
Foundation / checks (network) (push) Successful in 20m44s
Foundation / checks (client) (push) Successful in 21m57s
Foundation / checks (tsan) (push) Successful in 22m46s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 30m2s
Foundation / checks (postgres) (push) Canceled after 29m56s
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m25s
Foundation / checks (network) (push) Successful in 20m44s
Foundation / checks (client) (push) Successful in 21m57s
Foundation / checks (tsan) (push) Successful in 22m46s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 30m2s
Foundation / checks (postgres) (push) Canceled after 29m56s
This commit is contained in:
@@ -252,4 +252,179 @@ load_cellular_migration_freeze(Postgres& db, const FilesystemObjectStore& object
|
||||
return fail(RecoveryError::history);
|
||||
return result;
|
||||
}
|
||||
|
||||
namespace {
|
||||
constexpr auto stage_columns = "encode(trace,'hex'),encode(source_manifest,'hex'),"
|
||||
"encode(destination_manifest,'hex'),aborted::text,"
|
||||
"encode(ghost_sha256,'hex'),ghost_bytes::text";
|
||||
struct StageRecord {
|
||||
Record cut;
|
||||
VerifiedObject ghost;
|
||||
};
|
||||
std::optional<StageRecord> parse_stage(const PgRows::value_type& row) {
|
||||
if (row.size() != 6)
|
||||
return std::nullopt;
|
||||
auto cut = parse(PgRows::value_type(row.begin(), row.begin() + 4));
|
||||
const auto digest = unhex(row[4].value_or(""), 32);
|
||||
const auto size = row[5].value_or("");
|
||||
std::uint64_t bytes{};
|
||||
const auto [end, error] = std::from_chars(size.data(), size.data() + size.size(), bytes);
|
||||
if (!cut || !digest || digest->size() != 32 || error != std::errc{} ||
|
||||
end != size.data() + size.size() || bytes == 0)
|
||||
return std::nullopt;
|
||||
VerifiedObject ghost{{}, bytes};
|
||||
std::ranges::copy(*digest, ghost.sha256.begin());
|
||||
return StageRecord{std::move(*cut), ghost};
|
||||
}
|
||||
std::optional<CellularMigrationStage> checked_stage(const StageRecord& record, RegionKey key,
|
||||
const LeaseGrant& lease) {
|
||||
const auto trace = server::decode_migration_trace(record.cut.trace);
|
||||
if (!trace || key != RegionKey{trace->world, trace->request.destination.region})
|
||||
return std::nullopt;
|
||||
const auto freeze =
|
||||
checked_record(record.cut, {trace->world, trace->request.source.region}, lease);
|
||||
const auto destination = decode_manifest(record.cut.destination);
|
||||
if (!freeze || !destination || record.ghost.sha256 == Sha256Digest{} ||
|
||||
record.ghost.bytes > max_payload_bytes(destination->profile))
|
||||
return std::nullopt;
|
||||
return CellularMigrationStage{*freeze, record.ghost};
|
||||
}
|
||||
std::optional<std::vector<std::uint8_t>>
|
||||
ghost_candidate(const CheckpointManifest& s, const CheckpointManifest& d,
|
||||
std::span<const std::uint8_t> source, std::span<const std::uint8_t> destination,
|
||||
const server::CellularMigrationTrace& trace) {
|
||||
if (s.profile == CheckpointProfile::cellular_lives_v1) {
|
||||
auto src = restore_cellular_payload(source, s),
|
||||
dst = restore_cellular_payload(destination, d);
|
||||
if (!src || !dst)
|
||||
return std::nullopt;
|
||||
auto pair = server::replay_migration_trace(**src, **dst, trace.request, trace);
|
||||
if (!pair)
|
||||
return std::nullopt;
|
||||
return pair->destination->snapshot();
|
||||
}
|
||||
auto src = restore_population_payload(source, s),
|
||||
dst = restore_population_payload(destination, d);
|
||||
if (!src || !dst)
|
||||
return std::nullopt;
|
||||
auto pair = server::replay_migration_trace(**src, **dst, trace.request, trace);
|
||||
if (!pair)
|
||||
return std::nullopt;
|
||||
return pair->destination->snapshot();
|
||||
}
|
||||
CellularMigrationStageFailure stage_failed(PublicationError kind, LeaseStoreFailure store = {},
|
||||
ObjectError object = {},
|
||||
CellularMigrationStagePrepared prepared = {}) {
|
||||
return {kind, store, object, std::move(prepared)};
|
||||
}
|
||||
} // namespace
|
||||
|
||||
std::expected<CellularMigrationStage, CellularMigrationStageFailure>
|
||||
retry_cellular_migration_stage(Postgres& db, const LeaseGrant& lease,
|
||||
const CellularMigrationStagePrepared& prepared,
|
||||
milliseconds timeout) {
|
||||
const auto trace = server::decode_migration_trace(prepared.trace);
|
||||
if (!trace || prepared.key != RegionKey{trace->world, trace->request.destination.region} ||
|
||||
lease.frame != trace->frame || prepared.ghost.sha256 == Sha256Digest{} ||
|
||||
prepared.ghost.bytes == 0)
|
||||
return std::unexpected(stage_failed(PublicationError::candidate));
|
||||
const std::array<std::string, 7> args{std::to_string(prepared.key.world),
|
||||
std::to_string(prepared.key.region),
|
||||
uuid(lease.holder),
|
||||
std::to_string(lease.epoch),
|
||||
"\\x" + hex(prepared.trace),
|
||||
"\\x" + hex(prepared.ghost.sha256),
|
||||
std::to_string(prepared.ghost.bytes)};
|
||||
auto rows = db.query(std::string("SELECT ") + stage_columns +
|
||||
" FROM evolution.stage_cellular_migration($1::evolution.u64,"
|
||||
"$2::evolution.u64,$3::uuid,$4::evolution.u64,$5::bytea,$6::bytea,"
|
||||
"$7::evolution.u64)",
|
||||
args, timeout);
|
||||
if (!rows) {
|
||||
const auto store = failure(rows.error());
|
||||
return std::unexpected(stage_failed(store.kind == LeaseStoreError::unknown
|
||||
? PublicationError::unknown
|
||||
: PublicationError::rejected,
|
||||
store, {}, prepared));
|
||||
}
|
||||
const auto record = rows->size() == 1 ? parse_stage(rows->front()) : std::nullopt;
|
||||
const auto result = record ? checked_stage(*record, prepared.key, lease) : std::nullopt;
|
||||
if (!result || record->cut.trace != prepared.trace || result->ghost != prepared.ghost)
|
||||
return std::unexpected(stage_failed(PublicationError::unknown, {}, {}, prepared));
|
||||
return *result;
|
||||
}
|
||||
|
||||
std::expected<CellularMigrationStage, CellularMigrationStageFailure>
|
||||
stage_cellular_migration(Postgres& db, const FilesystemObjectStore& source_objects,
|
||||
const FilesystemObjectStore& destination_objects, const LeaseGrant& lease,
|
||||
const CommittedCheckpoint& source, const CommittedCheckpoint& destination,
|
||||
const server::CellularMigrationTrace& trace, milliseconds timeout,
|
||||
const PublishHooks& hooks) {
|
||||
const auto s = decode_manifest(source.manifest), d = decode_manifest(destination.manifest);
|
||||
if (!s || !d || !bound(source, *s) || !bound(destination, *d) ||
|
||||
!cuts(*s, *d, source.head.digest, destination.head.digest, trace) ||
|
||||
lease.frame != d->frame)
|
||||
return std::unexpected(stage_failed(PublicationError::predecessor));
|
||||
auto src = source_objects.get({s->payload_sha256, s->payload_bytes});
|
||||
auto dst = destination_objects.get({d->payload_sha256, d->payload_bytes});
|
||||
if (!src || !dst)
|
||||
return std::unexpected(
|
||||
stage_failed(PublicationError::predecessor, {}, !src ? src.error() : dst.error()));
|
||||
const auto candidate = ghost_candidate(*s, *d, *src, *dst, trace);
|
||||
const auto encoded = server::encode_migration_trace(trace);
|
||||
if (!candidate || !encoded)
|
||||
return std::unexpected(stage_failed(PublicationError::candidate));
|
||||
auto capsule = destination_objects.put(*src);
|
||||
if (!capsule)
|
||||
return std::unexpected(stage_failed(PublicationError::upload, {}, capsule.error()));
|
||||
auto capsule_bytes = destination_objects.get(*capsule);
|
||||
if (!capsule_bytes || *capsule_bytes != *src ||
|
||||
*capsule != VerifiedObject{s->payload_sha256, s->payload_bytes})
|
||||
return std::unexpected(
|
||||
stage_failed(PublicationError::verify, {},
|
||||
capsule_bytes ? ObjectError::corrupt : capsule_bytes.error()));
|
||||
auto ghost = destination_objects.put(*candidate);
|
||||
if (!ghost)
|
||||
return std::unexpected(stage_failed(PublicationError::upload, {}, ghost.error()));
|
||||
auto readback = destination_objects.get(*ghost);
|
||||
if (!readback || *readback != *candidate)
|
||||
return std::unexpected(stage_failed(PublicationError::verify, {},
|
||||
readback ? ObjectError::corrupt : readback.error()));
|
||||
if (hooks.before_transaction)
|
||||
hooks.before_transaction();
|
||||
return retry_cellular_migration_stage(db, lease, {d->key, *encoded, *ghost}, timeout);
|
||||
}
|
||||
|
||||
std::expected<std::optional<CellularMigrationStage>, RecoveryFailure>
|
||||
load_cellular_migration_stage(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
const LeaseGrant& lease, milliseconds timeout) {
|
||||
const auto fail = [](RecoveryError kind) { return std::unexpected(RecoveryFailure{kind, {}}); };
|
||||
const std::array<std::string, 4> args{std::to_string(key.world), std::to_string(key.region),
|
||||
uuid(lease.holder), std::to_string(lease.epoch)};
|
||||
auto rows = db.query(std::string("SELECT ") + stage_columns +
|
||||
" FROM evolution.load_cellular_migration_stage($1::evolution.u64,"
|
||||
"$2::evolution.u64,$3::uuid,$4::evolution.u64)",
|
||||
args, timeout);
|
||||
if (!rows)
|
||||
return std::unexpected(RecoveryFailure{RecoveryError::store, failure(rows.error())});
|
||||
if (rows->empty())
|
||||
return std::optional<CellularMigrationStage>{};
|
||||
const auto record = rows->size() == 1 ? parse_stage(rows->front()) : std::nullopt;
|
||||
auto result = record ? checked_stage(*record, key, lease) : std::nullopt;
|
||||
if (!result || result->freeze.aborted)
|
||||
return fail(RecoveryError::history);
|
||||
const auto s = decode_manifest(record->cut.source),
|
||||
d = decode_manifest(record->cut.destination);
|
||||
if (!s || !d)
|
||||
return fail(RecoveryError::manifest);
|
||||
auto src = objects.get({s->payload_sha256, s->payload_bytes});
|
||||
auto dst = objects.get({d->payload_sha256, d->payload_bytes});
|
||||
auto ghost = objects.get(result->ghost);
|
||||
if (!src || !dst || !ghost)
|
||||
return fail(RecoveryError::tail);
|
||||
const auto candidate = ghost_candidate(*s, *d, *src, *dst, result->freeze.trace);
|
||||
if (!candidate || *candidate != *ghost)
|
||||
return fail(RecoveryError::history);
|
||||
return result;
|
||||
}
|
||||
} // namespace evo::persistence
|
||||
|
||||
Reference in New Issue
Block a user