M12.4: retain read-only destination migration stages
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m25s
Foundation / checks (network) (push) Successful in 20m44s
Foundation / checks (client) (push) Successful in 21m57s
Foundation / checks (tsan) (push) Successful in 22m46s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 30m2s
Foundation / checks (postgres) (push) Canceled after 29m56s
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m25s
Foundation / checks (network) (push) Successful in 20m44s
Foundation / checks (client) (push) Successful in 21m57s
Foundation / checks (tsan) (push) Successful in 22m46s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 30m2s
Foundation / checks (postgres) (push) Canceled after 29m56s
This commit is contained in:
@@ -416,7 +416,8 @@ void CellularRegionHost::refresh() {
|
||||
status_.ready = status_.phase == HostPhase::running && state_ && committed_ && grant &&
|
||||
authority_.may_originate(LeaseClock::now()) && !interval_reload_ &&
|
||||
!interval_request_.has_value() && !migration_reload_ && !migration_frozen_ &&
|
||||
!migration_request_.has_value() && !migration_abort_.has_value();
|
||||
!migration_request_.has_value() && !migration_stage_request_.has_value() &&
|
||||
!migration_abort_.has_value();
|
||||
status_.migration_frozen = migration_frozen_;
|
||||
const auto& interval = admitted_interval_;
|
||||
status_.ecological_pending_ticks =
|
||||
@@ -430,7 +431,8 @@ void CellularRegionHost::refresh() {
|
||||
}
|
||||
|
||||
void CellularRegionHost::reconcile_migration_freeze() {
|
||||
if ((!migration_reload_ && !migration_frozen_ && !migration_request_ && !migration_abort_) ||
|
||||
if ((!migration_reload_ && !migration_frozen_ && !migration_request_ &&
|
||||
!migration_stage_request_ && !migration_abort_) ||
|
||||
LeaseClock::now() < migration_poll_at_)
|
||||
return;
|
||||
const auto grant = authority_.grant();
|
||||
@@ -460,6 +462,30 @@ void CellularRegionHost::reconcile_migration_freeze() {
|
||||
}
|
||||
migration_request_.reset();
|
||||
}
|
||||
if (migration_stage_request_) {
|
||||
const auto staged = persistence::retry_cellular_migration_stage(
|
||||
*db_, *grant, *migration_stage_request_, config_.lease.commit_timeout);
|
||||
if (!staged) {
|
||||
if (staged.error().kind == persistence::PublicationError::unknown) {
|
||||
db_.reset();
|
||||
++status_.unknown_outcomes;
|
||||
return;
|
||||
}
|
||||
// A concurrent source abort can win before the uncertain stage.
|
||||
// Reload authoritative pointers before making the region ready.
|
||||
if (staged.error().store.kind != LeaseStoreError::commit_conflict) {
|
||||
authority_.fence();
|
||||
status_.phase = staged.error().store.kind == LeaseStoreError::fenced ||
|
||||
staged.error().store.kind == LeaseStoreError::head_mismatch
|
||||
? HostPhase::acquiring
|
||||
: HostPhase::quarantined;
|
||||
state_.reset();
|
||||
committed_.reset();
|
||||
return;
|
||||
}
|
||||
}
|
||||
migration_stage_request_.reset();
|
||||
}
|
||||
if (migration_abort_) {
|
||||
auto result = persistence::abort_cellular_migration_freeze(
|
||||
*db_, config_.key, *grant, *migration_abort_, config_.lease.commit_timeout);
|
||||
@@ -492,9 +518,65 @@ void CellularRegionHost::reconcile_migration_freeze() {
|
||||
return;
|
||||
}
|
||||
migration_frozen_ = frozen->has_value();
|
||||
if (!migration_frozen_) {
|
||||
const auto staged = persistence::load_cellular_migration_stage(
|
||||
*db_, *objects_, config_.key, *grant, config_.lease.commit_timeout);
|
||||
if (!staged) {
|
||||
migration_reload_ = true;
|
||||
if (staged.error().kind == persistence::RecoveryError::store) {
|
||||
db_.reset();
|
||||
++status_.store_failures;
|
||||
} else {
|
||||
authority_.fence();
|
||||
status_.phase = HostPhase::quarantined;
|
||||
}
|
||||
return;
|
||||
}
|
||||
migration_frozen_ = staged->has_value();
|
||||
}
|
||||
migration_reload_ = false;
|
||||
}
|
||||
|
||||
std::expected<void, CellularPublishFailure>
|
||||
CellularRegionHost::stage_migration(const persistence::CommittedCheckpoint& source,
|
||||
const persistence::FilesystemObjectStore& source_objects,
|
||||
const CellularMigrationTrace& trace) {
|
||||
const auto grant = authority_.grant();
|
||||
if (!grant.has_value() || !status_.ready || pending_ || !state_ || !committed_ ||
|
||||
!queued_births_.empty() || !queued_mutations_.empty() || mutations_uncertain_ ||
|
||||
!pending_endings_.empty() || interval_request_ || admitted_interval_ || interval_reload_ ||
|
||||
!population_interval_safe(*state_) || !authority_.may_begin_commit(LeaseClock::now()) ||
|
||||
!database())
|
||||
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
|
||||
migration_reload_ = true;
|
||||
refresh();
|
||||
const auto staged =
|
||||
persistence::stage_cellular_migration(*db_, source_objects, *objects_, *grant, source,
|
||||
*committed_, trace, config_.lease.commit_timeout);
|
||||
if (!staged) {
|
||||
if (staged.error().kind == persistence::PublicationError::unknown) {
|
||||
migration_stage_request_ = staged.error().prepared;
|
||||
db_.reset();
|
||||
++status_.unknown_outcomes;
|
||||
} else if (staged.error().store.kind == LeaseStoreError::fenced ||
|
||||
staged.error().store.kind == LeaseStoreError::head_mismatch) {
|
||||
authority_.fence();
|
||||
state_.reset();
|
||||
committed_.reset();
|
||||
status_.phase = HostPhase::acquiring;
|
||||
}
|
||||
migration_poll_at_ = {};
|
||||
refresh();
|
||||
return std::unexpected(CellularPublishFailure{CellularHostError::publication,
|
||||
staged.error().kind, staged.error().store,
|
||||
staged.error().object});
|
||||
}
|
||||
migration_poll_at_ = {};
|
||||
reconcile_migration_freeze();
|
||||
refresh();
|
||||
return {};
|
||||
}
|
||||
|
||||
std::expected<void, CellularPublishFailure> CellularRegionHost::prepare_migration_freeze(
|
||||
const persistence::CommittedCheckpoint& destination,
|
||||
const persistence::FilesystemObjectStore& destination_objects,
|
||||
@@ -573,7 +655,8 @@ const HostStatus& CellularRegionHost::step() {
|
||||
acquire();
|
||||
if (!pending_ && state_ && status_.phase == HostPhase::running)
|
||||
reconcile_migration_freeze();
|
||||
if (migration_reload_ || migration_frozen_ || migration_request_ || migration_abort_) {
|
||||
if (migration_reload_ || migration_frozen_ || migration_request_ || migration_stage_request_ ||
|
||||
migration_abort_) {
|
||||
refresh();
|
||||
return status_;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user