M12.4: retain read-only destination migration stages
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m25s
Foundation / checks (network) (push) Successful in 20m44s
Foundation / checks (client) (push) Successful in 21m57s
Foundation / checks (tsan) (push) Successful in 22m46s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 30m2s
Foundation / checks (postgres) (push) Canceled after 29m56s
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m25s
Foundation / checks (network) (push) Successful in 20m44s
Foundation / checks (client) (push) Successful in 21m57s
Foundation / checks (tsan) (push) Successful in 22m46s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 30m2s
Foundation / checks (postgres) (push) Canceled after 29m56s
This commit is contained in:
@@ -252,4 +252,179 @@ load_cellular_migration_freeze(Postgres& db, const FilesystemObjectStore& object
|
||||
return fail(RecoveryError::history);
|
||||
return result;
|
||||
}
|
||||
|
||||
namespace {
|
||||
constexpr auto stage_columns = "encode(trace,'hex'),encode(source_manifest,'hex'),"
|
||||
"encode(destination_manifest,'hex'),aborted::text,"
|
||||
"encode(ghost_sha256,'hex'),ghost_bytes::text";
|
||||
struct StageRecord {
|
||||
Record cut;
|
||||
VerifiedObject ghost;
|
||||
};
|
||||
std::optional<StageRecord> parse_stage(const PgRows::value_type& row) {
|
||||
if (row.size() != 6)
|
||||
return std::nullopt;
|
||||
auto cut = parse(PgRows::value_type(row.begin(), row.begin() + 4));
|
||||
const auto digest = unhex(row[4].value_or(""), 32);
|
||||
const auto size = row[5].value_or("");
|
||||
std::uint64_t bytes{};
|
||||
const auto [end, error] = std::from_chars(size.data(), size.data() + size.size(), bytes);
|
||||
if (!cut || !digest || digest->size() != 32 || error != std::errc{} ||
|
||||
end != size.data() + size.size() || bytes == 0)
|
||||
return std::nullopt;
|
||||
VerifiedObject ghost{{}, bytes};
|
||||
std::ranges::copy(*digest, ghost.sha256.begin());
|
||||
return StageRecord{std::move(*cut), ghost};
|
||||
}
|
||||
std::optional<CellularMigrationStage> checked_stage(const StageRecord& record, RegionKey key,
|
||||
const LeaseGrant& lease) {
|
||||
const auto trace = server::decode_migration_trace(record.cut.trace);
|
||||
if (!trace || key != RegionKey{trace->world, trace->request.destination.region})
|
||||
return std::nullopt;
|
||||
const auto freeze =
|
||||
checked_record(record.cut, {trace->world, trace->request.source.region}, lease);
|
||||
const auto destination = decode_manifest(record.cut.destination);
|
||||
if (!freeze || !destination || record.ghost.sha256 == Sha256Digest{} ||
|
||||
record.ghost.bytes > max_payload_bytes(destination->profile))
|
||||
return std::nullopt;
|
||||
return CellularMigrationStage{*freeze, record.ghost};
|
||||
}
|
||||
std::optional<std::vector<std::uint8_t>>
|
||||
ghost_candidate(const CheckpointManifest& s, const CheckpointManifest& d,
|
||||
std::span<const std::uint8_t> source, std::span<const std::uint8_t> destination,
|
||||
const server::CellularMigrationTrace& trace) {
|
||||
if (s.profile == CheckpointProfile::cellular_lives_v1) {
|
||||
auto src = restore_cellular_payload(source, s),
|
||||
dst = restore_cellular_payload(destination, d);
|
||||
if (!src || !dst)
|
||||
return std::nullopt;
|
||||
auto pair = server::replay_migration_trace(**src, **dst, trace.request, trace);
|
||||
if (!pair)
|
||||
return std::nullopt;
|
||||
return pair->destination->snapshot();
|
||||
}
|
||||
auto src = restore_population_payload(source, s),
|
||||
dst = restore_population_payload(destination, d);
|
||||
if (!src || !dst)
|
||||
return std::nullopt;
|
||||
auto pair = server::replay_migration_trace(**src, **dst, trace.request, trace);
|
||||
if (!pair)
|
||||
return std::nullopt;
|
||||
return pair->destination->snapshot();
|
||||
}
|
||||
CellularMigrationStageFailure stage_failed(PublicationError kind, LeaseStoreFailure store = {},
|
||||
ObjectError object = {},
|
||||
CellularMigrationStagePrepared prepared = {}) {
|
||||
return {kind, store, object, std::move(prepared)};
|
||||
}
|
||||
} // namespace
|
||||
|
||||
std::expected<CellularMigrationStage, CellularMigrationStageFailure>
|
||||
retry_cellular_migration_stage(Postgres& db, const LeaseGrant& lease,
|
||||
const CellularMigrationStagePrepared& prepared,
|
||||
milliseconds timeout) {
|
||||
const auto trace = server::decode_migration_trace(prepared.trace);
|
||||
if (!trace || prepared.key != RegionKey{trace->world, trace->request.destination.region} ||
|
||||
lease.frame != trace->frame || prepared.ghost.sha256 == Sha256Digest{} ||
|
||||
prepared.ghost.bytes == 0)
|
||||
return std::unexpected(stage_failed(PublicationError::candidate));
|
||||
const std::array<std::string, 7> args{std::to_string(prepared.key.world),
|
||||
std::to_string(prepared.key.region),
|
||||
uuid(lease.holder),
|
||||
std::to_string(lease.epoch),
|
||||
"\\x" + hex(prepared.trace),
|
||||
"\\x" + hex(prepared.ghost.sha256),
|
||||
std::to_string(prepared.ghost.bytes)};
|
||||
auto rows = db.query(std::string("SELECT ") + stage_columns +
|
||||
" FROM evolution.stage_cellular_migration($1::evolution.u64,"
|
||||
"$2::evolution.u64,$3::uuid,$4::evolution.u64,$5::bytea,$6::bytea,"
|
||||
"$7::evolution.u64)",
|
||||
args, timeout);
|
||||
if (!rows) {
|
||||
const auto store = failure(rows.error());
|
||||
return std::unexpected(stage_failed(store.kind == LeaseStoreError::unknown
|
||||
? PublicationError::unknown
|
||||
: PublicationError::rejected,
|
||||
store, {}, prepared));
|
||||
}
|
||||
const auto record = rows->size() == 1 ? parse_stage(rows->front()) : std::nullopt;
|
||||
const auto result = record ? checked_stage(*record, prepared.key, lease) : std::nullopt;
|
||||
if (!result || record->cut.trace != prepared.trace || result->ghost != prepared.ghost)
|
||||
return std::unexpected(stage_failed(PublicationError::unknown, {}, {}, prepared));
|
||||
return *result;
|
||||
}
|
||||
|
||||
std::expected<CellularMigrationStage, CellularMigrationStageFailure>
|
||||
stage_cellular_migration(Postgres& db, const FilesystemObjectStore& source_objects,
|
||||
const FilesystemObjectStore& destination_objects, const LeaseGrant& lease,
|
||||
const CommittedCheckpoint& source, const CommittedCheckpoint& destination,
|
||||
const server::CellularMigrationTrace& trace, milliseconds timeout,
|
||||
const PublishHooks& hooks) {
|
||||
const auto s = decode_manifest(source.manifest), d = decode_manifest(destination.manifest);
|
||||
if (!s || !d || !bound(source, *s) || !bound(destination, *d) ||
|
||||
!cuts(*s, *d, source.head.digest, destination.head.digest, trace) ||
|
||||
lease.frame != d->frame)
|
||||
return std::unexpected(stage_failed(PublicationError::predecessor));
|
||||
auto src = source_objects.get({s->payload_sha256, s->payload_bytes});
|
||||
auto dst = destination_objects.get({d->payload_sha256, d->payload_bytes});
|
||||
if (!src || !dst)
|
||||
return std::unexpected(
|
||||
stage_failed(PublicationError::predecessor, {}, !src ? src.error() : dst.error()));
|
||||
const auto candidate = ghost_candidate(*s, *d, *src, *dst, trace);
|
||||
const auto encoded = server::encode_migration_trace(trace);
|
||||
if (!candidate || !encoded)
|
||||
return std::unexpected(stage_failed(PublicationError::candidate));
|
||||
auto capsule = destination_objects.put(*src);
|
||||
if (!capsule)
|
||||
return std::unexpected(stage_failed(PublicationError::upload, {}, capsule.error()));
|
||||
auto capsule_bytes = destination_objects.get(*capsule);
|
||||
if (!capsule_bytes || *capsule_bytes != *src ||
|
||||
*capsule != VerifiedObject{s->payload_sha256, s->payload_bytes})
|
||||
return std::unexpected(
|
||||
stage_failed(PublicationError::verify, {},
|
||||
capsule_bytes ? ObjectError::corrupt : capsule_bytes.error()));
|
||||
auto ghost = destination_objects.put(*candidate);
|
||||
if (!ghost)
|
||||
return std::unexpected(stage_failed(PublicationError::upload, {}, ghost.error()));
|
||||
auto readback = destination_objects.get(*ghost);
|
||||
if (!readback || *readback != *candidate)
|
||||
return std::unexpected(stage_failed(PublicationError::verify, {},
|
||||
readback ? ObjectError::corrupt : readback.error()));
|
||||
if (hooks.before_transaction)
|
||||
hooks.before_transaction();
|
||||
return retry_cellular_migration_stage(db, lease, {d->key, *encoded, *ghost}, timeout);
|
||||
}
|
||||
|
||||
std::expected<std::optional<CellularMigrationStage>, RecoveryFailure>
|
||||
load_cellular_migration_stage(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
const LeaseGrant& lease, milliseconds timeout) {
|
||||
const auto fail = [](RecoveryError kind) { return std::unexpected(RecoveryFailure{kind, {}}); };
|
||||
const std::array<std::string, 4> args{std::to_string(key.world), std::to_string(key.region),
|
||||
uuid(lease.holder), std::to_string(lease.epoch)};
|
||||
auto rows = db.query(std::string("SELECT ") + stage_columns +
|
||||
" FROM evolution.load_cellular_migration_stage($1::evolution.u64,"
|
||||
"$2::evolution.u64,$3::uuid,$4::evolution.u64)",
|
||||
args, timeout);
|
||||
if (!rows)
|
||||
return std::unexpected(RecoveryFailure{RecoveryError::store, failure(rows.error())});
|
||||
if (rows->empty())
|
||||
return std::optional<CellularMigrationStage>{};
|
||||
const auto record = rows->size() == 1 ? parse_stage(rows->front()) : std::nullopt;
|
||||
auto result = record ? checked_stage(*record, key, lease) : std::nullopt;
|
||||
if (!result || result->freeze.aborted)
|
||||
return fail(RecoveryError::history);
|
||||
const auto s = decode_manifest(record->cut.source),
|
||||
d = decode_manifest(record->cut.destination);
|
||||
if (!s || !d)
|
||||
return fail(RecoveryError::manifest);
|
||||
auto src = objects.get({s->payload_sha256, s->payload_bytes});
|
||||
auto dst = objects.get({d->payload_sha256, d->payload_bytes});
|
||||
auto ghost = objects.get(result->ghost);
|
||||
if (!src || !dst || !ghost)
|
||||
return fail(RecoveryError::tail);
|
||||
const auto candidate = ghost_candidate(*s, *d, *src, *dst, result->freeze.trace);
|
||||
if (!candidate || *candidate != *ghost)
|
||||
return fail(RecoveryError::history);
|
||||
return result;
|
||||
}
|
||||
} // namespace evo::persistence
|
||||
|
||||
@@ -416,7 +416,8 @@ void CellularRegionHost::refresh() {
|
||||
status_.ready = status_.phase == HostPhase::running && state_ && committed_ && grant &&
|
||||
authority_.may_originate(LeaseClock::now()) && !interval_reload_ &&
|
||||
!interval_request_.has_value() && !migration_reload_ && !migration_frozen_ &&
|
||||
!migration_request_.has_value() && !migration_abort_.has_value();
|
||||
!migration_request_.has_value() && !migration_stage_request_.has_value() &&
|
||||
!migration_abort_.has_value();
|
||||
status_.migration_frozen = migration_frozen_;
|
||||
const auto& interval = admitted_interval_;
|
||||
status_.ecological_pending_ticks =
|
||||
@@ -430,7 +431,8 @@ void CellularRegionHost::refresh() {
|
||||
}
|
||||
|
||||
void CellularRegionHost::reconcile_migration_freeze() {
|
||||
if ((!migration_reload_ && !migration_frozen_ && !migration_request_ && !migration_abort_) ||
|
||||
if ((!migration_reload_ && !migration_frozen_ && !migration_request_ &&
|
||||
!migration_stage_request_ && !migration_abort_) ||
|
||||
LeaseClock::now() < migration_poll_at_)
|
||||
return;
|
||||
const auto grant = authority_.grant();
|
||||
@@ -460,6 +462,30 @@ void CellularRegionHost::reconcile_migration_freeze() {
|
||||
}
|
||||
migration_request_.reset();
|
||||
}
|
||||
if (migration_stage_request_) {
|
||||
const auto staged = persistence::retry_cellular_migration_stage(
|
||||
*db_, *grant, *migration_stage_request_, config_.lease.commit_timeout);
|
||||
if (!staged) {
|
||||
if (staged.error().kind == persistence::PublicationError::unknown) {
|
||||
db_.reset();
|
||||
++status_.unknown_outcomes;
|
||||
return;
|
||||
}
|
||||
// A concurrent source abort can win before the uncertain stage.
|
||||
// Reload authoritative pointers before making the region ready.
|
||||
if (staged.error().store.kind != LeaseStoreError::commit_conflict) {
|
||||
authority_.fence();
|
||||
status_.phase = staged.error().store.kind == LeaseStoreError::fenced ||
|
||||
staged.error().store.kind == LeaseStoreError::head_mismatch
|
||||
? HostPhase::acquiring
|
||||
: HostPhase::quarantined;
|
||||
state_.reset();
|
||||
committed_.reset();
|
||||
return;
|
||||
}
|
||||
}
|
||||
migration_stage_request_.reset();
|
||||
}
|
||||
if (migration_abort_) {
|
||||
auto result = persistence::abort_cellular_migration_freeze(
|
||||
*db_, config_.key, *grant, *migration_abort_, config_.lease.commit_timeout);
|
||||
@@ -492,9 +518,65 @@ void CellularRegionHost::reconcile_migration_freeze() {
|
||||
return;
|
||||
}
|
||||
migration_frozen_ = frozen->has_value();
|
||||
if (!migration_frozen_) {
|
||||
const auto staged = persistence::load_cellular_migration_stage(
|
||||
*db_, *objects_, config_.key, *grant, config_.lease.commit_timeout);
|
||||
if (!staged) {
|
||||
migration_reload_ = true;
|
||||
if (staged.error().kind == persistence::RecoveryError::store) {
|
||||
db_.reset();
|
||||
++status_.store_failures;
|
||||
} else {
|
||||
authority_.fence();
|
||||
status_.phase = HostPhase::quarantined;
|
||||
}
|
||||
return;
|
||||
}
|
||||
migration_frozen_ = staged->has_value();
|
||||
}
|
||||
migration_reload_ = false;
|
||||
}
|
||||
|
||||
std::expected<void, CellularPublishFailure>
|
||||
CellularRegionHost::stage_migration(const persistence::CommittedCheckpoint& source,
|
||||
const persistence::FilesystemObjectStore& source_objects,
|
||||
const CellularMigrationTrace& trace) {
|
||||
const auto grant = authority_.grant();
|
||||
if (!grant.has_value() || !status_.ready || pending_ || !state_ || !committed_ ||
|
||||
!queued_births_.empty() || !queued_mutations_.empty() || mutations_uncertain_ ||
|
||||
!pending_endings_.empty() || interval_request_ || admitted_interval_ || interval_reload_ ||
|
||||
!population_interval_safe(*state_) || !authority_.may_begin_commit(LeaseClock::now()) ||
|
||||
!database())
|
||||
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
|
||||
migration_reload_ = true;
|
||||
refresh();
|
||||
const auto staged =
|
||||
persistence::stage_cellular_migration(*db_, source_objects, *objects_, *grant, source,
|
||||
*committed_, trace, config_.lease.commit_timeout);
|
||||
if (!staged) {
|
||||
if (staged.error().kind == persistence::PublicationError::unknown) {
|
||||
migration_stage_request_ = staged.error().prepared;
|
||||
db_.reset();
|
||||
++status_.unknown_outcomes;
|
||||
} else if (staged.error().store.kind == LeaseStoreError::fenced ||
|
||||
staged.error().store.kind == LeaseStoreError::head_mismatch) {
|
||||
authority_.fence();
|
||||
state_.reset();
|
||||
committed_.reset();
|
||||
status_.phase = HostPhase::acquiring;
|
||||
}
|
||||
migration_poll_at_ = {};
|
||||
refresh();
|
||||
return std::unexpected(CellularPublishFailure{CellularHostError::publication,
|
||||
staged.error().kind, staged.error().store,
|
||||
staged.error().object});
|
||||
}
|
||||
migration_poll_at_ = {};
|
||||
reconcile_migration_freeze();
|
||||
refresh();
|
||||
return {};
|
||||
}
|
||||
|
||||
std::expected<void, CellularPublishFailure> CellularRegionHost::prepare_migration_freeze(
|
||||
const persistence::CommittedCheckpoint& destination,
|
||||
const persistence::FilesystemObjectStore& destination_objects,
|
||||
@@ -573,7 +655,8 @@ const HostStatus& CellularRegionHost::step() {
|
||||
acquire();
|
||||
if (!pending_ && state_ && status_.phase == HostPhase::running)
|
||||
reconcile_migration_freeze();
|
||||
if (migration_reload_ || migration_frozen_ || migration_request_ || migration_abort_) {
|
||||
if (migration_reload_ || migration_frozen_ || migration_request_ || migration_stage_request_ ||
|
||||
migration_abort_) {
|
||||
refresh();
|
||||
return status_;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user