M12.4: retain read-only destination migration stages
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m25s
Foundation / checks (network) (push) Successful in 20m44s
Foundation / checks (client) (push) Successful in 21m57s
Foundation / checks (tsan) (push) Successful in 22m46s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 30m2s
Foundation / checks (postgres) (push) Canceled after 29m56s

This commit is contained in:
dingo
2026-10-18 09:32:50 -04:00
parent c625a4cb68
commit 0515702964
21 changed files with 1023 additions and 21 deletions
+86 -3
View File
@@ -416,7 +416,8 @@ void CellularRegionHost::refresh() {
status_.ready = status_.phase == HostPhase::running && state_ && committed_ && grant &&
authority_.may_originate(LeaseClock::now()) && !interval_reload_ &&
!interval_request_.has_value() && !migration_reload_ && !migration_frozen_ &&
!migration_request_.has_value() && !migration_abort_.has_value();
!migration_request_.has_value() && !migration_stage_request_.has_value() &&
!migration_abort_.has_value();
status_.migration_frozen = migration_frozen_;
const auto& interval = admitted_interval_;
status_.ecological_pending_ticks =
@@ -430,7 +431,8 @@ void CellularRegionHost::refresh() {
}
void CellularRegionHost::reconcile_migration_freeze() {
if ((!migration_reload_ && !migration_frozen_ && !migration_request_ && !migration_abort_) ||
if ((!migration_reload_ && !migration_frozen_ && !migration_request_ &&
!migration_stage_request_ && !migration_abort_) ||
LeaseClock::now() < migration_poll_at_)
return;
const auto grant = authority_.grant();
@@ -460,6 +462,30 @@ void CellularRegionHost::reconcile_migration_freeze() {
}
migration_request_.reset();
}
if (migration_stage_request_) {
const auto staged = persistence::retry_cellular_migration_stage(
*db_, *grant, *migration_stage_request_, config_.lease.commit_timeout);
if (!staged) {
if (staged.error().kind == persistence::PublicationError::unknown) {
db_.reset();
++status_.unknown_outcomes;
return;
}
// A concurrent source abort can win before the uncertain stage.
// Reload authoritative pointers before making the region ready.
if (staged.error().store.kind != LeaseStoreError::commit_conflict) {
authority_.fence();
status_.phase = staged.error().store.kind == LeaseStoreError::fenced ||
staged.error().store.kind == LeaseStoreError::head_mismatch
? HostPhase::acquiring
: HostPhase::quarantined;
state_.reset();
committed_.reset();
return;
}
}
migration_stage_request_.reset();
}
if (migration_abort_) {
auto result = persistence::abort_cellular_migration_freeze(
*db_, config_.key, *grant, *migration_abort_, config_.lease.commit_timeout);
@@ -492,9 +518,65 @@ void CellularRegionHost::reconcile_migration_freeze() {
return;
}
migration_frozen_ = frozen->has_value();
if (!migration_frozen_) {
const auto staged = persistence::load_cellular_migration_stage(
*db_, *objects_, config_.key, *grant, config_.lease.commit_timeout);
if (!staged) {
migration_reload_ = true;
if (staged.error().kind == persistence::RecoveryError::store) {
db_.reset();
++status_.store_failures;
} else {
authority_.fence();
status_.phase = HostPhase::quarantined;
}
return;
}
migration_frozen_ = staged->has_value();
}
migration_reload_ = false;
}
std::expected<void, CellularPublishFailure>
CellularRegionHost::stage_migration(const persistence::CommittedCheckpoint& source,
const persistence::FilesystemObjectStore& source_objects,
const CellularMigrationTrace& trace) {
const auto grant = authority_.grant();
if (!grant.has_value() || !status_.ready || pending_ || !state_ || !committed_ ||
!queued_births_.empty() || !queued_mutations_.empty() || mutations_uncertain_ ||
!pending_endings_.empty() || interval_request_ || admitted_interval_ || interval_reload_ ||
!population_interval_safe(*state_) || !authority_.may_begin_commit(LeaseClock::now()) ||
!database())
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
migration_reload_ = true;
refresh();
const auto staged =
persistence::stage_cellular_migration(*db_, source_objects, *objects_, *grant, source,
*committed_, trace, config_.lease.commit_timeout);
if (!staged) {
if (staged.error().kind == persistence::PublicationError::unknown) {
migration_stage_request_ = staged.error().prepared;
db_.reset();
++status_.unknown_outcomes;
} else if (staged.error().store.kind == LeaseStoreError::fenced ||
staged.error().store.kind == LeaseStoreError::head_mismatch) {
authority_.fence();
state_.reset();
committed_.reset();
status_.phase = HostPhase::acquiring;
}
migration_poll_at_ = {};
refresh();
return std::unexpected(CellularPublishFailure{CellularHostError::publication,
staged.error().kind, staged.error().store,
staged.error().object});
}
migration_poll_at_ = {};
reconcile_migration_freeze();
refresh();
return {};
}
std::expected<void, CellularPublishFailure> CellularRegionHost::prepare_migration_freeze(
const persistence::CommittedCheckpoint& destination,
const persistence::FilesystemObjectStore& destination_objects,
@@ -573,7 +655,8 @@ const HostStatus& CellularRegionHost::step() {
acquire();
if (!pending_ && state_ && status_.phase == HostPhase::running)
reconcile_migration_freeze();
if (migration_reload_ || migration_frozen_ || migration_request_ || migration_abort_) {
if (migration_reload_ || migration_frozen_ || migration_request_ || migration_stage_request_ ||
migration_abort_) {
refresh();
return status_;
}