Add fenced cellular profile migration
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 1m56s
Foundation / checks (postgres) (push) Canceled after 0s
Foundation / checks (tsan) (push) Canceled after 0s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (client) (push) Canceled after 5m17s
Foundation / checks (checks) (push) Canceled after 5m21s
Foundation / checks (network) (push) Canceled after 3m14s
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 1m56s
Foundation / checks (postgres) (push) Canceled after 0s
Foundation / checks (tsan) (push) Canceled after 0s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (client) (push) Canceled after 5m17s
Foundation / checks (checks) (push) Canceled after 5m21s
Foundation / checks (network) (push) Canceled after 3m14s
This commit is contained in:
@@ -23,6 +23,11 @@ CellularPublicationFailure failure(PublicationError kind, ObjectError object = {
|
||||
CellularPreparedCommit prepared = {}) {
|
||||
return {kind, store, object, std::move(prepared)};
|
||||
}
|
||||
CellularProfileMigrationFailure migration_failure(PublicationError kind, ObjectError object = {},
|
||||
LeaseStoreFailure store = {},
|
||||
CellularProfileMigrationPrepared prepared = {}) {
|
||||
return {kind, store, object, std::move(prepared)};
|
||||
}
|
||||
std::expected<void, CellularPublicationFailure> put_verified(const FilesystemObjectStore& objects,
|
||||
std::span<const std::uint8_t> bytes,
|
||||
VerifiedObject expected) {
|
||||
@@ -138,6 +143,95 @@ provision_cellular_region(Postgres& owner, const FilesystemObjectStore& objects,
|
||||
return *confirmed;
|
||||
}
|
||||
|
||||
std::expected<CommittedCheckpoint, CellularProfileMigrationFailure>
|
||||
retry_cellular_profile_migration(Postgres& db, const CellularProfileMigrationPrepared& prepared,
|
||||
milliseconds timeout) {
|
||||
const auto manifest = decode_manifest(prepared.manifest);
|
||||
if (!manifest || manifest->key != prepared.key || manifest->commit != prepared.commit ||
|
||||
manifest->profile != CheckpointProfile::cellular_lives_v1 ||
|
||||
manifest->cellular_versions != cellular_v2_versions ||
|
||||
manifest->begin_tick != manifest->end_tick || manifest->batch != BatchDigests{})
|
||||
return std::unexpected{migration_failure(PublicationError::candidate, {}, {}, prepared)};
|
||||
|
||||
const auto confirm_commit = [&](const RegionHead& head) {
|
||||
const auto digest = sha256(prepared.manifest);
|
||||
if (!digest || head.sequence != manifest->sequence || head.commit != manifest->commit ||
|
||||
head.digest != *digest || head.end_tick != manifest->end_tick ||
|
||||
head.frame != manifest->frame)
|
||||
return std::expected<CommittedCheckpoint, CellularProfileMigrationFailure>{
|
||||
std::unexpected{migration_failure(PublicationError::receipt, {}, {}, prepared)}};
|
||||
return std::expected<CommittedCheckpoint, CellularProfileMigrationFailure>{
|
||||
CommittedCheckpoint{head, prepared.manifest}};
|
||||
};
|
||||
|
||||
auto published = publish_cellular_profile_migration(db, prepared.manifest, timeout);
|
||||
if (published)
|
||||
return confirm_commit(*published);
|
||||
|
||||
if (published.error().kind == LeaseStoreError::unknown) {
|
||||
const auto stored = load_region_commit(db, prepared.key, manifest->sequence, timeout);
|
||||
if (stored && stored->manifest == prepared.manifest)
|
||||
return confirm_commit(stored->head);
|
||||
if (!stored && stored.error().kind != LeaseStoreError::not_provisioned)
|
||||
return std::unexpected{
|
||||
migration_failure(PublicationError::unknown, {}, stored.error(), prepared)};
|
||||
return std::unexpected{
|
||||
migration_failure(PublicationError::unknown, {}, published.error(), prepared)};
|
||||
}
|
||||
return std::unexpected{
|
||||
migration_failure(PublicationError::rejected, {}, published.error(), prepared)};
|
||||
}
|
||||
|
||||
std::expected<CommittedCheckpoint, CellularProfileMigrationFailure>
|
||||
migrate_cellular_profile_to_v2(Postgres& db, const FilesystemObjectStore& objects,
|
||||
const LeaseGrant& lease, const CommittedCheckpoint& predecessor,
|
||||
Uuid commit, const Sha256Digest& build, milliseconds timeout) {
|
||||
const auto previous = decode_manifest(predecessor.manifest);
|
||||
const auto previous_digest = sha256(predecessor.manifest);
|
||||
if (!previous || !previous_digest ||
|
||||
previous->profile != CheckpointProfile::cellular_lives_v1 ||
|
||||
previous->cellular_versions != cellular_v1_versions ||
|
||||
predecessor.head.digest != *previous_digest ||
|
||||
predecessor.head.sequence != previous->sequence ||
|
||||
predecessor.head.commit != previous->commit ||
|
||||
predecessor.head.end_tick != previous->end_tick || lease.frame != previous->frame ||
|
||||
previous->sequence == 0 || previous->end_tick == 0)
|
||||
return std::unexpected{migration_failure(PublicationError::candidate)};
|
||||
|
||||
auto source_payload = objects.get({previous->payload_sha256, previous->payload_bytes});
|
||||
if (!source_payload)
|
||||
return std::unexpected{
|
||||
migration_failure(PublicationError::predecessor, source_payload.error())};
|
||||
const auto source_digest = sha256(*source_payload);
|
||||
if (!source_digest || *source_digest != previous->payload_sha256)
|
||||
return std::unexpected{
|
||||
migration_failure(PublicationError::predecessor, ObjectError::corrupt)};
|
||||
auto source = restore_cellular_payload(*source_payload, *previous);
|
||||
if (!source || (*source)->format_version() != server::cellular_lives_legacy_format ||
|
||||
(*source)->rules_version() != 1)
|
||||
return std::unexpected{migration_failure(PublicationError::predecessor)};
|
||||
auto migrated = (*source)->migrate_to_v2();
|
||||
if (!migrated || (*migrated)->tick() != (*source)->tick() ||
|
||||
(*migrated)->current_total() != (*source)->current_total())
|
||||
return std::unexpected{migration_failure(PublicationError::candidate)};
|
||||
|
||||
const auto payload = (*migrated)->snapshot();
|
||||
const auto payload_digest = sha256(payload);
|
||||
if (!payload_digest)
|
||||
return std::unexpected{migration_failure(PublicationError::digest)};
|
||||
auto manifest = cellular_profile_migration_manifest(*previous, *previous_digest, commit, lease,
|
||||
build, payload.size(), *payload_digest);
|
||||
auto encoded = manifest ? encode_manifest(*manifest)
|
||||
: std::expected<std::vector<std::uint8_t>, CheckpointError>{
|
||||
std::unexpected{CheckpointError::format}};
|
||||
if (!encoded || !restore_cellular_payload(payload, *manifest))
|
||||
return std::unexpected{migration_failure(PublicationError::candidate)};
|
||||
if (auto stored = put_verified(objects, payload, {*payload_digest, payload.size()}); !stored)
|
||||
return std::unexpected{migration_failure(stored.error().kind, stored.error().object)};
|
||||
return retry_cellular_profile_migration(
|
||||
db, CellularProfileMigrationPrepared{previous->key, commit, std::move(*encoded)}, timeout);
|
||||
}
|
||||
|
||||
std::expected<CommittedCheckpoint, CellularPublicationFailure>
|
||||
retry_cellular_prepared(Postgres& db, const CellularPreparedCommit& prepared,
|
||||
milliseconds timeout) {
|
||||
@@ -268,6 +362,61 @@ recover_cellular_checkpoint(Postgres& db, const FilesystemObjectStore& objects,
|
||||
return fail_with(RecoveryError::payload);
|
||||
|
||||
std::vector<server::CellularActionRecord> actions;
|
||||
if (manifest->sequence != 0 && manifest->begin_tick == manifest->end_tick) {
|
||||
auto migration = load_cellular_profile_migration(db, key, lease, manifest->commit, timeout);
|
||||
if (!migration)
|
||||
return std::unexpected{RecoveryFailure{RecoveryError::store, migration.error()}};
|
||||
if (!*migration || (*migration)->target_sequence != manifest->sequence ||
|
||||
(*migration)->target_commit != manifest->commit ||
|
||||
(*migration)->target_manifest_sha256 != *digest ||
|
||||
(*migration)->target_payload_sha256 != manifest->payload_sha256 ||
|
||||
(*migration)->target_sequence == 0)
|
||||
return fail_with(RecoveryError::history);
|
||||
auto predecessor = load_region_commit(db, key, manifest->sequence - 1, timeout);
|
||||
if (!predecessor)
|
||||
return std::unexpected{RecoveryFailure{RecoveryError::tail, predecessor.error()}};
|
||||
const auto source_manifest_digest = sha256(predecessor->manifest);
|
||||
auto source_manifest = decode_manifest(predecessor->manifest);
|
||||
const auto source_manifest_bytes =
|
||||
encode_manifest(source_manifest ? *source_manifest : CheckpointManifest{});
|
||||
const auto target_manifest_bytes = encode_manifest(*manifest);
|
||||
if (!source_manifest_digest || !source_manifest || !source_manifest_bytes ||
|
||||
!target_manifest_bytes || *source_manifest_digest != manifest->predecessor_digest ||
|
||||
predecessor->head.digest != *source_manifest_digest ||
|
||||
predecessor->head.commit != manifest->predecessor_commit ||
|
||||
predecessor->head.sequence != manifest->sequence - 1 ||
|
||||
source_manifest->cellular_versions != cellular_v1_versions ||
|
||||
source_manifest->profile != CheckpointProfile::cellular_lives_v1 ||
|
||||
source_manifest->end_tick != manifest->end_tick ||
|
||||
(*migration)->source_sequence != source_manifest->sequence ||
|
||||
(*migration)->source_commit != source_manifest->commit ||
|
||||
(*migration)->source_manifest_sha256 != *source_manifest_digest ||
|
||||
(*migration)->source_payload_sha256 != source_manifest->payload_sha256 ||
|
||||
(*migration)->source_versions.size() != 36 ||
|
||||
!std::ranges::equal((*migration)->source_versions,
|
||||
std::span{*source_manifest_bytes}.subspan(160, 36)) ||
|
||||
(*migration)->target_versions.size() != 36 ||
|
||||
!std::ranges::equal((*migration)->target_versions,
|
||||
std::span{*target_manifest_bytes}.subspan(160, 36)))
|
||||
return fail_with(RecoveryError::history);
|
||||
auto source_bytes =
|
||||
objects.get({source_manifest->payload_sha256, source_manifest->payload_bytes});
|
||||
if (!source_bytes)
|
||||
return fail_with(source_bytes.error() == ObjectError::io ? RecoveryError::store
|
||||
: RecoveryError::tail);
|
||||
const auto source_payload_digest = sha256(*source_bytes);
|
||||
if (!source_payload_digest || *source_payload_digest != source_manifest->payload_sha256)
|
||||
return fail_with(RecoveryError::tail);
|
||||
auto source_state = restore_cellular_payload(*source_bytes, *source_manifest);
|
||||
if (!source_state ||
|
||||
(*source_state)->format_version() != server::cellular_lives_legacy_format)
|
||||
return fail_with(RecoveryError::tail);
|
||||
auto migrated = (*source_state)->migrate_to_v2();
|
||||
if (!migrated || (*migrated)->tick() != manifest->end_tick ||
|
||||
(*migrated)->snapshot() != *payload)
|
||||
return fail_with(RecoveryError::history);
|
||||
return RecoveredCellularCheckpoint{std::move(*committed), *manifest, std::move(*state), {}};
|
||||
}
|
||||
if (manifest->sequence != 0) {
|
||||
auto action_bytes =
|
||||
load_cellular_publication_actions(db, key, lease, manifest->commit, timeout);
|
||||
|
||||
Reference in New Issue
Block a user