Add fenced cellular profile migration
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 1m56s
Foundation / checks (postgres) (push) Canceled after 0s
Foundation / checks (tsan) (push) Canceled after 0s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (client) (push) Canceled after 5m17s
Foundation / checks (checks) (push) Canceled after 5m21s
Foundation / checks (network) (push) Canceled after 3m14s

This commit is contained in:
dingo
2026-10-11 05:12:55 -04:00
parent beffec40d7
commit 66b8088f44
10 changed files with 600 additions and 10 deletions
@@ -23,6 +23,11 @@ CellularPublicationFailure failure(PublicationError kind, ObjectError object = {
CellularPreparedCommit prepared = {}) {
return {kind, store, object, std::move(prepared)};
}
CellularProfileMigrationFailure migration_failure(PublicationError kind, ObjectError object = {},
LeaseStoreFailure store = {},
CellularProfileMigrationPrepared prepared = {}) {
return {kind, store, object, std::move(prepared)};
}
std::expected<void, CellularPublicationFailure> put_verified(const FilesystemObjectStore& objects,
std::span<const std::uint8_t> bytes,
VerifiedObject expected) {
@@ -138,6 +143,95 @@ provision_cellular_region(Postgres& owner, const FilesystemObjectStore& objects,
return *confirmed;
}
std::expected<CommittedCheckpoint, CellularProfileMigrationFailure>
retry_cellular_profile_migration(Postgres& db, const CellularProfileMigrationPrepared& prepared,
milliseconds timeout) {
const auto manifest = decode_manifest(prepared.manifest);
if (!manifest || manifest->key != prepared.key || manifest->commit != prepared.commit ||
manifest->profile != CheckpointProfile::cellular_lives_v1 ||
manifest->cellular_versions != cellular_v2_versions ||
manifest->begin_tick != manifest->end_tick || manifest->batch != BatchDigests{})
return std::unexpected{migration_failure(PublicationError::candidate, {}, {}, prepared)};
const auto confirm_commit = [&](const RegionHead& head) {
const auto digest = sha256(prepared.manifest);
if (!digest || head.sequence != manifest->sequence || head.commit != manifest->commit ||
head.digest != *digest || head.end_tick != manifest->end_tick ||
head.frame != manifest->frame)
return std::expected<CommittedCheckpoint, CellularProfileMigrationFailure>{
std::unexpected{migration_failure(PublicationError::receipt, {}, {}, prepared)}};
return std::expected<CommittedCheckpoint, CellularProfileMigrationFailure>{
CommittedCheckpoint{head, prepared.manifest}};
};
auto published = publish_cellular_profile_migration(db, prepared.manifest, timeout);
if (published)
return confirm_commit(*published);
if (published.error().kind == LeaseStoreError::unknown) {
const auto stored = load_region_commit(db, prepared.key, manifest->sequence, timeout);
if (stored && stored->manifest == prepared.manifest)
return confirm_commit(stored->head);
if (!stored && stored.error().kind != LeaseStoreError::not_provisioned)
return std::unexpected{
migration_failure(PublicationError::unknown, {}, stored.error(), prepared)};
return std::unexpected{
migration_failure(PublicationError::unknown, {}, published.error(), prepared)};
}
return std::unexpected{
migration_failure(PublicationError::rejected, {}, published.error(), prepared)};
}
std::expected<CommittedCheckpoint, CellularProfileMigrationFailure>
migrate_cellular_profile_to_v2(Postgres& db, const FilesystemObjectStore& objects,
const LeaseGrant& lease, const CommittedCheckpoint& predecessor,
Uuid commit, const Sha256Digest& build, milliseconds timeout) {
const auto previous = decode_manifest(predecessor.manifest);
const auto previous_digest = sha256(predecessor.manifest);
if (!previous || !previous_digest ||
previous->profile != CheckpointProfile::cellular_lives_v1 ||
previous->cellular_versions != cellular_v1_versions ||
predecessor.head.digest != *previous_digest ||
predecessor.head.sequence != previous->sequence ||
predecessor.head.commit != previous->commit ||
predecessor.head.end_tick != previous->end_tick || lease.frame != previous->frame ||
previous->sequence == 0 || previous->end_tick == 0)
return std::unexpected{migration_failure(PublicationError::candidate)};
auto source_payload = objects.get({previous->payload_sha256, previous->payload_bytes});
if (!source_payload)
return std::unexpected{
migration_failure(PublicationError::predecessor, source_payload.error())};
const auto source_digest = sha256(*source_payload);
if (!source_digest || *source_digest != previous->payload_sha256)
return std::unexpected{
migration_failure(PublicationError::predecessor, ObjectError::corrupt)};
auto source = restore_cellular_payload(*source_payload, *previous);
if (!source || (*source)->format_version() != server::cellular_lives_legacy_format ||
(*source)->rules_version() != 1)
return std::unexpected{migration_failure(PublicationError::predecessor)};
auto migrated = (*source)->migrate_to_v2();
if (!migrated || (*migrated)->tick() != (*source)->tick() ||
(*migrated)->current_total() != (*source)->current_total())
return std::unexpected{migration_failure(PublicationError::candidate)};
const auto payload = (*migrated)->snapshot();
const auto payload_digest = sha256(payload);
if (!payload_digest)
return std::unexpected{migration_failure(PublicationError::digest)};
auto manifest = cellular_profile_migration_manifest(*previous, *previous_digest, commit, lease,
build, payload.size(), *payload_digest);
auto encoded = manifest ? encode_manifest(*manifest)
: std::expected<std::vector<std::uint8_t>, CheckpointError>{
std::unexpected{CheckpointError::format}};
if (!encoded || !restore_cellular_payload(payload, *manifest))
return std::unexpected{migration_failure(PublicationError::candidate)};
if (auto stored = put_verified(objects, payload, {*payload_digest, payload.size()}); !stored)
return std::unexpected{migration_failure(stored.error().kind, stored.error().object)};
return retry_cellular_profile_migration(
db, CellularProfileMigrationPrepared{previous->key, commit, std::move(*encoded)}, timeout);
}
std::expected<CommittedCheckpoint, CellularPublicationFailure>
retry_cellular_prepared(Postgres& db, const CellularPreparedCommit& prepared,
milliseconds timeout) {
@@ -268,6 +362,61 @@ recover_cellular_checkpoint(Postgres& db, const FilesystemObjectStore& objects,
return fail_with(RecoveryError::payload);
std::vector<server::CellularActionRecord> actions;
if (manifest->sequence != 0 && manifest->begin_tick == manifest->end_tick) {
auto migration = load_cellular_profile_migration(db, key, lease, manifest->commit, timeout);
if (!migration)
return std::unexpected{RecoveryFailure{RecoveryError::store, migration.error()}};
if (!*migration || (*migration)->target_sequence != manifest->sequence ||
(*migration)->target_commit != manifest->commit ||
(*migration)->target_manifest_sha256 != *digest ||
(*migration)->target_payload_sha256 != manifest->payload_sha256 ||
(*migration)->target_sequence == 0)
return fail_with(RecoveryError::history);
auto predecessor = load_region_commit(db, key, manifest->sequence - 1, timeout);
if (!predecessor)
return std::unexpected{RecoveryFailure{RecoveryError::tail, predecessor.error()}};
const auto source_manifest_digest = sha256(predecessor->manifest);
auto source_manifest = decode_manifest(predecessor->manifest);
const auto source_manifest_bytes =
encode_manifest(source_manifest ? *source_manifest : CheckpointManifest{});
const auto target_manifest_bytes = encode_manifest(*manifest);
if (!source_manifest_digest || !source_manifest || !source_manifest_bytes ||
!target_manifest_bytes || *source_manifest_digest != manifest->predecessor_digest ||
predecessor->head.digest != *source_manifest_digest ||
predecessor->head.commit != manifest->predecessor_commit ||
predecessor->head.sequence != manifest->sequence - 1 ||
source_manifest->cellular_versions != cellular_v1_versions ||
source_manifest->profile != CheckpointProfile::cellular_lives_v1 ||
source_manifest->end_tick != manifest->end_tick ||
(*migration)->source_sequence != source_manifest->sequence ||
(*migration)->source_commit != source_manifest->commit ||
(*migration)->source_manifest_sha256 != *source_manifest_digest ||
(*migration)->source_payload_sha256 != source_manifest->payload_sha256 ||
(*migration)->source_versions.size() != 36 ||
!std::ranges::equal((*migration)->source_versions,
std::span{*source_manifest_bytes}.subspan(160, 36)) ||
(*migration)->target_versions.size() != 36 ||
!std::ranges::equal((*migration)->target_versions,
std::span{*target_manifest_bytes}.subspan(160, 36)))
return fail_with(RecoveryError::history);
auto source_bytes =
objects.get({source_manifest->payload_sha256, source_manifest->payload_bytes});
if (!source_bytes)
return fail_with(source_bytes.error() == ObjectError::io ? RecoveryError::store
: RecoveryError::tail);
const auto source_payload_digest = sha256(*source_bytes);
if (!source_payload_digest || *source_payload_digest != source_manifest->payload_sha256)
return fail_with(RecoveryError::tail);
auto source_state = restore_cellular_payload(*source_bytes, *source_manifest);
if (!source_state ||
(*source_state)->format_version() != server::cellular_lives_legacy_format)
return fail_with(RecoveryError::tail);
auto migrated = (*source_state)->migrate_to_v2();
if (!migrated || (*migrated)->tick() != manifest->end_tick ||
(*migrated)->snapshot() != *payload)
return fail_with(RecoveryError::history);
return RecoveredCellularCheckpoint{std::move(*committed), *manifest, std::move(*state), {}};
}
if (manifest->sequence != 0) {
auto action_bytes =
load_cellular_publication_actions(db, key, lease, manifest->commit, timeout);