M12.4: retain migration source freezes and explicit aborts
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m24s
Foundation / checks (network) (push) Successful in 20m44s
Foundation / checks (client) (push) Successful in 22m3s
Foundation / checks (tsan) (push) Successful in 23m1s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 24m7s
Foundation / checks (postgres) (push) Canceled after 24m3s
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m24s
Foundation / checks (network) (push) Successful in 20m44s
Foundation / checks (client) (push) Successful in 22m3s
Foundation / checks (tsan) (push) Successful in 23m1s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 24m7s
Foundation / checks (postgres) (push) Canceled after 24m3s
This commit is contained in:
@@ -199,6 +199,9 @@ void CellularRegionHost::acquire() {
|
||||
}
|
||||
if (!recover_state(*lease) || !committed_.has_value() || !state_)
|
||||
return;
|
||||
migration_reload_ = true;
|
||||
migration_frozen_ = false;
|
||||
migration_poll_at_ = {};
|
||||
interval_reload_ = population_mode();
|
||||
admitted_interval_.reset();
|
||||
interval_request_.reset();
|
||||
@@ -412,7 +415,9 @@ void CellularRegionHost::refresh() {
|
||||
const auto& grant = authority_.grant();
|
||||
status_.ready = status_.phase == HostPhase::running && state_ && committed_ && grant &&
|
||||
authority_.may_originate(LeaseClock::now()) && !interval_reload_ &&
|
||||
!interval_request_.has_value();
|
||||
!interval_request_.has_value() && !migration_reload_ && !migration_frozen_ &&
|
||||
!migration_request_.has_value() && !migration_abort_.has_value();
|
||||
status_.migration_frozen = migration_frozen_;
|
||||
const auto& interval = admitted_interval_;
|
||||
status_.ecological_pending_ticks =
|
||||
interval.has_value() ? interval->end_tick - interval->begin_tick : 0;
|
||||
@@ -424,6 +429,139 @@ void CellularRegionHost::refresh() {
|
||||
: 0;
|
||||
}
|
||||
|
||||
void CellularRegionHost::reconcile_migration_freeze() {
|
||||
if ((!migration_reload_ && !migration_frozen_ && !migration_request_ && !migration_abort_) ||
|
||||
LeaseClock::now() < migration_poll_at_)
|
||||
return;
|
||||
const auto grant = authority_.grant();
|
||||
if (!grant || !authority_.may_begin_commit(LeaseClock::now()) || !database())
|
||||
return;
|
||||
migration_poll_at_ = LeaseClock::now() + std::chrono::seconds{1};
|
||||
if (migration_request_) {
|
||||
auto result = persistence::retry_cellular_migration_freeze(
|
||||
*db_, *grant, *migration_request_, config_.lease.commit_timeout);
|
||||
if (!result) {
|
||||
if (result.error().kind == persistence::PublicationError::unknown) {
|
||||
db_.reset();
|
||||
++status_.unknown_outcomes;
|
||||
return;
|
||||
}
|
||||
if (result.error().store.kind == LeaseStoreError::fenced ||
|
||||
result.error().store.kind == LeaseStoreError::head_mismatch) {
|
||||
authority_.fence();
|
||||
state_.reset();
|
||||
committed_.reset();
|
||||
status_.phase = HostPhase::acquiring;
|
||||
} else {
|
||||
authority_.fence();
|
||||
status_.phase = HostPhase::quarantined;
|
||||
}
|
||||
return;
|
||||
}
|
||||
migration_request_.reset();
|
||||
}
|
||||
if (migration_abort_) {
|
||||
auto result = persistence::abort_cellular_migration_freeze(
|
||||
*db_, config_.key, *grant, *migration_abort_, config_.lease.commit_timeout);
|
||||
if (!result) {
|
||||
if (result.error().kind == LeaseStoreError::unknown) {
|
||||
db_.reset();
|
||||
++status_.unknown_outcomes;
|
||||
return;
|
||||
}
|
||||
authority_.fence();
|
||||
status_.phase = result.error().kind == LeaseStoreError::fenced ? HostPhase::acquiring
|
||||
: HostPhase::quarantined;
|
||||
state_.reset();
|
||||
committed_.reset();
|
||||
return;
|
||||
}
|
||||
migration_abort_.reset();
|
||||
}
|
||||
const auto frozen = persistence::load_cellular_migration_freeze(
|
||||
*db_, *objects_, config_.key, *grant, config_.lease.commit_timeout);
|
||||
if (!frozen) {
|
||||
migration_reload_ = true;
|
||||
if (frozen.error().kind == persistence::RecoveryError::store) {
|
||||
db_.reset();
|
||||
++status_.store_failures;
|
||||
} else {
|
||||
authority_.fence();
|
||||
status_.phase = HostPhase::quarantined;
|
||||
}
|
||||
return;
|
||||
}
|
||||
migration_frozen_ = frozen->has_value();
|
||||
migration_reload_ = false;
|
||||
}
|
||||
|
||||
std::expected<void, CellularPublishFailure> CellularRegionHost::prepare_migration_freeze(
|
||||
const persistence::CommittedCheckpoint& destination,
|
||||
const persistence::FilesystemObjectStore& destination_objects,
|
||||
const CellularMigrationRequest& request) {
|
||||
const auto grant = authority_.grant();
|
||||
if (!grant.has_value() || !status_.ready || pending_ || !state_ || !committed_ ||
|
||||
!queued_births_.empty() || !queued_mutations_.empty() || mutations_uncertain_ ||
|
||||
!pending_endings_.empty() || interval_request_ || admitted_interval_ || interval_reload_ ||
|
||||
!population_interval_safe(*state_) || !authority_.may_begin_commit(LeaseClock::now()) ||
|
||||
!database())
|
||||
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
|
||||
migration_reload_ = true;
|
||||
refresh();
|
||||
const auto result = persistence::prepare_cellular_migration_freeze(
|
||||
*db_, *objects_, destination_objects, *grant, *committed_, destination, request,
|
||||
config_.lease.commit_timeout);
|
||||
if (!result) {
|
||||
if (result.error().kind == persistence::PublicationError::unknown) {
|
||||
migration_request_ = result.error().prepared;
|
||||
db_.reset();
|
||||
++status_.unknown_outcomes;
|
||||
} else if (result.error().store.kind == LeaseStoreError::head_mismatch ||
|
||||
result.error().store.kind == LeaseStoreError::fenced) {
|
||||
authority_.fence();
|
||||
state_.reset();
|
||||
committed_.reset();
|
||||
status_.phase = HostPhase::acquiring;
|
||||
}
|
||||
migration_poll_at_ = {};
|
||||
refresh();
|
||||
return std::unexpected(CellularPublishFailure{CellularHostError::publication,
|
||||
result.error().kind, result.error().store,
|
||||
result.error().object});
|
||||
}
|
||||
migration_poll_at_ = {};
|
||||
reconcile_migration_freeze();
|
||||
refresh();
|
||||
return {};
|
||||
}
|
||||
|
||||
std::expected<void, CellularPublishFailure>
|
||||
CellularRegionHost::abort_migration_freeze(const persistence::Uuid& request) {
|
||||
const auto grant = authority_.grant();
|
||||
if (!grant || !state_ || pending_ || status_.phase != HostPhase::running ||
|
||||
!authority_.may_begin_commit(LeaseClock::now()) || !database())
|
||||
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
|
||||
migration_reload_ = true;
|
||||
refresh();
|
||||
const auto result = persistence::abort_cellular_migration_freeze(
|
||||
*db_, config_.key, *grant, request, config_.lease.commit_timeout);
|
||||
if (!result) {
|
||||
if (result.error().kind == LeaseStoreError::unknown) {
|
||||
migration_abort_ = request;
|
||||
db_.reset();
|
||||
++status_.unknown_outcomes;
|
||||
}
|
||||
migration_poll_at_ = {};
|
||||
refresh();
|
||||
return std::unexpected(
|
||||
CellularPublishFailure{CellularHostError::publication, {}, result.error(), {}});
|
||||
}
|
||||
migration_poll_at_ = {};
|
||||
reconcile_migration_freeze();
|
||||
refresh();
|
||||
return {};
|
||||
}
|
||||
|
||||
const HostStatus& CellularRegionHost::step() {
|
||||
if (status_.phase == HostPhase::quarantined || status_.phase == HostPhase::stopped)
|
||||
return status_;
|
||||
@@ -433,6 +571,12 @@ const HostStatus& CellularRegionHost::step() {
|
||||
resolve();
|
||||
else if (!state_)
|
||||
acquire();
|
||||
if (!pending_ && state_ && status_.phase == HostPhase::running)
|
||||
reconcile_migration_freeze();
|
||||
if (migration_reload_ || migration_frozen_ || migration_request_ || migration_abort_) {
|
||||
refresh();
|
||||
return status_;
|
||||
}
|
||||
if (!pending_ && state_ && status_.phase == HostPhase::running)
|
||||
reconcile_interval();
|
||||
if (!pending_ && state_ && mutations_uncertain_ && status_.phase == HostPhase::running)
|
||||
|
||||
Reference in New Issue
Block a user