M12.4: retain read-only destination migration stages
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m25s
Foundation / checks (network) (push) Successful in 20m44s
Foundation / checks (client) (push) Successful in 21m57s
Foundation / checks (tsan) (push) Successful in 22m46s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 30m2s
Foundation / checks (postgres) (push) Canceled after 29m56s

This commit is contained in:
dingo
2026-10-18 09:32:50 -04:00
parent c625a4cb68
commit 0515702964
21 changed files with 1023 additions and 21 deletions
@@ -252,4 +252,179 @@ load_cellular_migration_freeze(Postgres& db, const FilesystemObjectStore& object
return fail(RecoveryError::history);
return result;
}
namespace {
constexpr auto stage_columns = "encode(trace,'hex'),encode(source_manifest,'hex'),"
"encode(destination_manifest,'hex'),aborted::text,"
"encode(ghost_sha256,'hex'),ghost_bytes::text";
struct StageRecord {
Record cut;
VerifiedObject ghost;
};
std::optional<StageRecord> parse_stage(const PgRows::value_type& row) {
if (row.size() != 6)
return std::nullopt;
auto cut = parse(PgRows::value_type(row.begin(), row.begin() + 4));
const auto digest = unhex(row[4].value_or(""), 32);
const auto size = row[5].value_or("");
std::uint64_t bytes{};
const auto [end, error] = std::from_chars(size.data(), size.data() + size.size(), bytes);
if (!cut || !digest || digest->size() != 32 || error != std::errc{} ||
end != size.data() + size.size() || bytes == 0)
return std::nullopt;
VerifiedObject ghost{{}, bytes};
std::ranges::copy(*digest, ghost.sha256.begin());
return StageRecord{std::move(*cut), ghost};
}
std::optional<CellularMigrationStage> checked_stage(const StageRecord& record, RegionKey key,
const LeaseGrant& lease) {
const auto trace = server::decode_migration_trace(record.cut.trace);
if (!trace || key != RegionKey{trace->world, trace->request.destination.region})
return std::nullopt;
const auto freeze =
checked_record(record.cut, {trace->world, trace->request.source.region}, lease);
const auto destination = decode_manifest(record.cut.destination);
if (!freeze || !destination || record.ghost.sha256 == Sha256Digest{} ||
record.ghost.bytes > max_payload_bytes(destination->profile))
return std::nullopt;
return CellularMigrationStage{*freeze, record.ghost};
}
std::optional<std::vector<std::uint8_t>>
ghost_candidate(const CheckpointManifest& s, const CheckpointManifest& d,
std::span<const std::uint8_t> source, std::span<const std::uint8_t> destination,
const server::CellularMigrationTrace& trace) {
if (s.profile == CheckpointProfile::cellular_lives_v1) {
auto src = restore_cellular_payload(source, s),
dst = restore_cellular_payload(destination, d);
if (!src || !dst)
return std::nullopt;
auto pair = server::replay_migration_trace(**src, **dst, trace.request, trace);
if (!pair)
return std::nullopt;
return pair->destination->snapshot();
}
auto src = restore_population_payload(source, s),
dst = restore_population_payload(destination, d);
if (!src || !dst)
return std::nullopt;
auto pair = server::replay_migration_trace(**src, **dst, trace.request, trace);
if (!pair)
return std::nullopt;
return pair->destination->snapshot();
}
CellularMigrationStageFailure stage_failed(PublicationError kind, LeaseStoreFailure store = {},
ObjectError object = {},
CellularMigrationStagePrepared prepared = {}) {
return {kind, store, object, std::move(prepared)};
}
} // namespace
std::expected<CellularMigrationStage, CellularMigrationStageFailure>
retry_cellular_migration_stage(Postgres& db, const LeaseGrant& lease,
const CellularMigrationStagePrepared& prepared,
milliseconds timeout) {
const auto trace = server::decode_migration_trace(prepared.trace);
if (!trace || prepared.key != RegionKey{trace->world, trace->request.destination.region} ||
lease.frame != trace->frame || prepared.ghost.sha256 == Sha256Digest{} ||
prepared.ghost.bytes == 0)
return std::unexpected(stage_failed(PublicationError::candidate));
const std::array<std::string, 7> args{std::to_string(prepared.key.world),
std::to_string(prepared.key.region),
uuid(lease.holder),
std::to_string(lease.epoch),
"\\x" + hex(prepared.trace),
"\\x" + hex(prepared.ghost.sha256),
std::to_string(prepared.ghost.bytes)};
auto rows = db.query(std::string("SELECT ") + stage_columns +
" FROM evolution.stage_cellular_migration($1::evolution.u64,"
"$2::evolution.u64,$3::uuid,$4::evolution.u64,$5::bytea,$6::bytea,"
"$7::evolution.u64)",
args, timeout);
if (!rows) {
const auto store = failure(rows.error());
return std::unexpected(stage_failed(store.kind == LeaseStoreError::unknown
? PublicationError::unknown
: PublicationError::rejected,
store, {}, prepared));
}
const auto record = rows->size() == 1 ? parse_stage(rows->front()) : std::nullopt;
const auto result = record ? checked_stage(*record, prepared.key, lease) : std::nullopt;
if (!result || record->cut.trace != prepared.trace || result->ghost != prepared.ghost)
return std::unexpected(stage_failed(PublicationError::unknown, {}, {}, prepared));
return *result;
}
std::expected<CellularMigrationStage, CellularMigrationStageFailure>
stage_cellular_migration(Postgres& db, const FilesystemObjectStore& source_objects,
const FilesystemObjectStore& destination_objects, const LeaseGrant& lease,
const CommittedCheckpoint& source, const CommittedCheckpoint& destination,
const server::CellularMigrationTrace& trace, milliseconds timeout,
const PublishHooks& hooks) {
const auto s = decode_manifest(source.manifest), d = decode_manifest(destination.manifest);
if (!s || !d || !bound(source, *s) || !bound(destination, *d) ||
!cuts(*s, *d, source.head.digest, destination.head.digest, trace) ||
lease.frame != d->frame)
return std::unexpected(stage_failed(PublicationError::predecessor));
auto src = source_objects.get({s->payload_sha256, s->payload_bytes});
auto dst = destination_objects.get({d->payload_sha256, d->payload_bytes});
if (!src || !dst)
return std::unexpected(
stage_failed(PublicationError::predecessor, {}, !src ? src.error() : dst.error()));
const auto candidate = ghost_candidate(*s, *d, *src, *dst, trace);
const auto encoded = server::encode_migration_trace(trace);
if (!candidate || !encoded)
return std::unexpected(stage_failed(PublicationError::candidate));
auto capsule = destination_objects.put(*src);
if (!capsule)
return std::unexpected(stage_failed(PublicationError::upload, {}, capsule.error()));
auto capsule_bytes = destination_objects.get(*capsule);
if (!capsule_bytes || *capsule_bytes != *src ||
*capsule != VerifiedObject{s->payload_sha256, s->payload_bytes})
return std::unexpected(
stage_failed(PublicationError::verify, {},
capsule_bytes ? ObjectError::corrupt : capsule_bytes.error()));
auto ghost = destination_objects.put(*candidate);
if (!ghost)
return std::unexpected(stage_failed(PublicationError::upload, {}, ghost.error()));
auto readback = destination_objects.get(*ghost);
if (!readback || *readback != *candidate)
return std::unexpected(stage_failed(PublicationError::verify, {},
readback ? ObjectError::corrupt : readback.error()));
if (hooks.before_transaction)
hooks.before_transaction();
return retry_cellular_migration_stage(db, lease, {d->key, *encoded, *ghost}, timeout);
}
std::expected<std::optional<CellularMigrationStage>, RecoveryFailure>
load_cellular_migration_stage(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
const LeaseGrant& lease, milliseconds timeout) {
const auto fail = [](RecoveryError kind) { return std::unexpected(RecoveryFailure{kind, {}}); };
const std::array<std::string, 4> args{std::to_string(key.world), std::to_string(key.region),
uuid(lease.holder), std::to_string(lease.epoch)};
auto rows = db.query(std::string("SELECT ") + stage_columns +
" FROM evolution.load_cellular_migration_stage($1::evolution.u64,"
"$2::evolution.u64,$3::uuid,$4::evolution.u64)",
args, timeout);
if (!rows)
return std::unexpected(RecoveryFailure{RecoveryError::store, failure(rows.error())});
if (rows->empty())
return std::optional<CellularMigrationStage>{};
const auto record = rows->size() == 1 ? parse_stage(rows->front()) : std::nullopt;
auto result = record ? checked_stage(*record, key, lease) : std::nullopt;
if (!result || result->freeze.aborted)
return fail(RecoveryError::history);
const auto s = decode_manifest(record->cut.source),
d = decode_manifest(record->cut.destination);
if (!s || !d)
return fail(RecoveryError::manifest);
auto src = objects.get({s->payload_sha256, s->payload_bytes});
auto dst = objects.get({d->payload_sha256, d->payload_bytes});
auto ghost = objects.get(result->ghost);
if (!src || !dst || !ghost)
return fail(RecoveryError::tail);
const auto candidate = ghost_candidate(*s, *d, *src, *dst, result->freeze.trace);
if (!candidate || *candidate != *ghost)
return fail(RecoveryError::history);
return result;
}
} // namespace evo::persistence
+86 -3
View File
@@ -416,7 +416,8 @@ void CellularRegionHost::refresh() {
status_.ready = status_.phase == HostPhase::running && state_ && committed_ && grant &&
authority_.may_originate(LeaseClock::now()) && !interval_reload_ &&
!interval_request_.has_value() && !migration_reload_ && !migration_frozen_ &&
!migration_request_.has_value() && !migration_abort_.has_value();
!migration_request_.has_value() && !migration_stage_request_.has_value() &&
!migration_abort_.has_value();
status_.migration_frozen = migration_frozen_;
const auto& interval = admitted_interval_;
status_.ecological_pending_ticks =
@@ -430,7 +431,8 @@ void CellularRegionHost::refresh() {
}
void CellularRegionHost::reconcile_migration_freeze() {
if ((!migration_reload_ && !migration_frozen_ && !migration_request_ && !migration_abort_) ||
if ((!migration_reload_ && !migration_frozen_ && !migration_request_ &&
!migration_stage_request_ && !migration_abort_) ||
LeaseClock::now() < migration_poll_at_)
return;
const auto grant = authority_.grant();
@@ -460,6 +462,30 @@ void CellularRegionHost::reconcile_migration_freeze() {
}
migration_request_.reset();
}
if (migration_stage_request_) {
const auto staged = persistence::retry_cellular_migration_stage(
*db_, *grant, *migration_stage_request_, config_.lease.commit_timeout);
if (!staged) {
if (staged.error().kind == persistence::PublicationError::unknown) {
db_.reset();
++status_.unknown_outcomes;
return;
}
// A concurrent source abort can win before the uncertain stage.
// Reload authoritative pointers before making the region ready.
if (staged.error().store.kind != LeaseStoreError::commit_conflict) {
authority_.fence();
status_.phase = staged.error().store.kind == LeaseStoreError::fenced ||
staged.error().store.kind == LeaseStoreError::head_mismatch
? HostPhase::acquiring
: HostPhase::quarantined;
state_.reset();
committed_.reset();
return;
}
}
migration_stage_request_.reset();
}
if (migration_abort_) {
auto result = persistence::abort_cellular_migration_freeze(
*db_, config_.key, *grant, *migration_abort_, config_.lease.commit_timeout);
@@ -492,9 +518,65 @@ void CellularRegionHost::reconcile_migration_freeze() {
return;
}
migration_frozen_ = frozen->has_value();
if (!migration_frozen_) {
const auto staged = persistence::load_cellular_migration_stage(
*db_, *objects_, config_.key, *grant, config_.lease.commit_timeout);
if (!staged) {
migration_reload_ = true;
if (staged.error().kind == persistence::RecoveryError::store) {
db_.reset();
++status_.store_failures;
} else {
authority_.fence();
status_.phase = HostPhase::quarantined;
}
return;
}
migration_frozen_ = staged->has_value();
}
migration_reload_ = false;
}
std::expected<void, CellularPublishFailure>
CellularRegionHost::stage_migration(const persistence::CommittedCheckpoint& source,
const persistence::FilesystemObjectStore& source_objects,
const CellularMigrationTrace& trace) {
const auto grant = authority_.grant();
if (!grant.has_value() || !status_.ready || pending_ || !state_ || !committed_ ||
!queued_births_.empty() || !queued_mutations_.empty() || mutations_uncertain_ ||
!pending_endings_.empty() || interval_request_ || admitted_interval_ || interval_reload_ ||
!population_interval_safe(*state_) || !authority_.may_begin_commit(LeaseClock::now()) ||
!database())
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
migration_reload_ = true;
refresh();
const auto staged =
persistence::stage_cellular_migration(*db_, source_objects, *objects_, *grant, source,
*committed_, trace, config_.lease.commit_timeout);
if (!staged) {
if (staged.error().kind == persistence::PublicationError::unknown) {
migration_stage_request_ = staged.error().prepared;
db_.reset();
++status_.unknown_outcomes;
} else if (staged.error().store.kind == LeaseStoreError::fenced ||
staged.error().store.kind == LeaseStoreError::head_mismatch) {
authority_.fence();
state_.reset();
committed_.reset();
status_.phase = HostPhase::acquiring;
}
migration_poll_at_ = {};
refresh();
return std::unexpected(CellularPublishFailure{CellularHostError::publication,
staged.error().kind, staged.error().store,
staged.error().object});
}
migration_poll_at_ = {};
reconcile_migration_freeze();
refresh();
return {};
}
std::expected<void, CellularPublishFailure> CellularRegionHost::prepare_migration_freeze(
const persistence::CommittedCheckpoint& destination,
const persistence::FilesystemObjectStore& destination_objects,
@@ -573,7 +655,8 @@ const HostStatus& CellularRegionHost::step() {
acquire();
if (!pending_ && state_ && status_.phase == HostPhase::running)
reconcile_migration_freeze();
if (migration_reload_ || migration_frozen_ || migration_request_ || migration_abort_) {
if (migration_reload_ || migration_frozen_ || migration_request_ || migration_stage_request_ ||
migration_abort_) {
refresh();
return status_;
}