M12.4: retain migration source freezes and explicit aborts
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m24s
Foundation / checks (network) (push) Successful in 20m44s
Foundation / checks (client) (push) Successful in 22m3s
Foundation / checks (tsan) (push) Successful in 23m1s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 24m7s
Foundation / checks (postgres) (push) Canceled after 24m3s

This commit is contained in:
dingo
2026-10-18 09:08:50 -04:00
parent 1b5c0cdf7c
commit c625a4cb68
25 changed files with 1342 additions and 18 deletions
+145 -1
View File
@@ -199,6 +199,9 @@ void CellularRegionHost::acquire() {
}
if (!recover_state(*lease) || !committed_.has_value() || !state_)
return;
migration_reload_ = true;
migration_frozen_ = false;
migration_poll_at_ = {};
interval_reload_ = population_mode();
admitted_interval_.reset();
interval_request_.reset();
@@ -412,7 +415,9 @@ void CellularRegionHost::refresh() {
const auto& grant = authority_.grant();
status_.ready = status_.phase == HostPhase::running && state_ && committed_ && grant &&
authority_.may_originate(LeaseClock::now()) && !interval_reload_ &&
!interval_request_.has_value();
!interval_request_.has_value() && !migration_reload_ && !migration_frozen_ &&
!migration_request_.has_value() && !migration_abort_.has_value();
status_.migration_frozen = migration_frozen_;
const auto& interval = admitted_interval_;
status_.ecological_pending_ticks =
interval.has_value() ? interval->end_tick - interval->begin_tick : 0;
@@ -424,6 +429,139 @@ void CellularRegionHost::refresh() {
: 0;
}
void CellularRegionHost::reconcile_migration_freeze() {
if ((!migration_reload_ && !migration_frozen_ && !migration_request_ && !migration_abort_) ||
LeaseClock::now() < migration_poll_at_)
return;
const auto grant = authority_.grant();
if (!grant || !authority_.may_begin_commit(LeaseClock::now()) || !database())
return;
migration_poll_at_ = LeaseClock::now() + std::chrono::seconds{1};
if (migration_request_) {
auto result = persistence::retry_cellular_migration_freeze(
*db_, *grant, *migration_request_, config_.lease.commit_timeout);
if (!result) {
if (result.error().kind == persistence::PublicationError::unknown) {
db_.reset();
++status_.unknown_outcomes;
return;
}
if (result.error().store.kind == LeaseStoreError::fenced ||
result.error().store.kind == LeaseStoreError::head_mismatch) {
authority_.fence();
state_.reset();
committed_.reset();
status_.phase = HostPhase::acquiring;
} else {
authority_.fence();
status_.phase = HostPhase::quarantined;
}
return;
}
migration_request_.reset();
}
if (migration_abort_) {
auto result = persistence::abort_cellular_migration_freeze(
*db_, config_.key, *grant, *migration_abort_, config_.lease.commit_timeout);
if (!result) {
if (result.error().kind == LeaseStoreError::unknown) {
db_.reset();
++status_.unknown_outcomes;
return;
}
authority_.fence();
status_.phase = result.error().kind == LeaseStoreError::fenced ? HostPhase::acquiring
: HostPhase::quarantined;
state_.reset();
committed_.reset();
return;
}
migration_abort_.reset();
}
const auto frozen = persistence::load_cellular_migration_freeze(
*db_, *objects_, config_.key, *grant, config_.lease.commit_timeout);
if (!frozen) {
migration_reload_ = true;
if (frozen.error().kind == persistence::RecoveryError::store) {
db_.reset();
++status_.store_failures;
} else {
authority_.fence();
status_.phase = HostPhase::quarantined;
}
return;
}
migration_frozen_ = frozen->has_value();
migration_reload_ = false;
}
std::expected<void, CellularPublishFailure> CellularRegionHost::prepare_migration_freeze(
const persistence::CommittedCheckpoint& destination,
const persistence::FilesystemObjectStore& destination_objects,
const CellularMigrationRequest& request) {
const auto grant = authority_.grant();
if (!grant.has_value() || !status_.ready || pending_ || !state_ || !committed_ ||
!queued_births_.empty() || !queued_mutations_.empty() || mutations_uncertain_ ||
!pending_endings_.empty() || interval_request_ || admitted_interval_ || interval_reload_ ||
!population_interval_safe(*state_) || !authority_.may_begin_commit(LeaseClock::now()) ||
!database())
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
migration_reload_ = true;
refresh();
const auto result = persistence::prepare_cellular_migration_freeze(
*db_, *objects_, destination_objects, *grant, *committed_, destination, request,
config_.lease.commit_timeout);
if (!result) {
if (result.error().kind == persistence::PublicationError::unknown) {
migration_request_ = result.error().prepared;
db_.reset();
++status_.unknown_outcomes;
} else if (result.error().store.kind == LeaseStoreError::head_mismatch ||
result.error().store.kind == LeaseStoreError::fenced) {
authority_.fence();
state_.reset();
committed_.reset();
status_.phase = HostPhase::acquiring;
}
migration_poll_at_ = {};
refresh();
return std::unexpected(CellularPublishFailure{CellularHostError::publication,
result.error().kind, result.error().store,
result.error().object});
}
migration_poll_at_ = {};
reconcile_migration_freeze();
refresh();
return {};
}
std::expected<void, CellularPublishFailure>
CellularRegionHost::abort_migration_freeze(const persistence::Uuid& request) {
const auto grant = authority_.grant();
if (!grant || !state_ || pending_ || status_.phase != HostPhase::running ||
!authority_.may_begin_commit(LeaseClock::now()) || !database())
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
migration_reload_ = true;
refresh();
const auto result = persistence::abort_cellular_migration_freeze(
*db_, config_.key, *grant, request, config_.lease.commit_timeout);
if (!result) {
if (result.error().kind == LeaseStoreError::unknown) {
migration_abort_ = request;
db_.reset();
++status_.unknown_outcomes;
}
migration_poll_at_ = {};
refresh();
return std::unexpected(
CellularPublishFailure{CellularHostError::publication, {}, result.error(), {}});
}
migration_poll_at_ = {};
reconcile_migration_freeze();
refresh();
return {};
}
const HostStatus& CellularRegionHost::step() {
if (status_.phase == HostPhase::quarantined || status_.phase == HostPhase::stopped)
return status_;
@@ -433,6 +571,12 @@ const HostStatus& CellularRegionHost::step() {
resolve();
else if (!state_)
acquire();
if (!pending_ && state_ && status_.phase == HostPhase::running)
reconcile_migration_freeze();
if (migration_reload_ || migration_frozen_ || migration_request_ || migration_abort_) {
refresh();
return status_;
}
if (!pending_ && state_ && status_.phase == HostPhase::running)
reconcile_interval();
if (!pending_ && state_ && mutations_uncertain_ && status_.phase == HostPhase::running)