M12.4: admit and replay durable boundary-accounting upgrades
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m23s
Foundation / checks (network) (push) Successful in 21m4s
Foundation / checks (client) (push) Successful in 22m15s
Foundation / checks (tsan) (push) Successful in 23m3s
Foundation / checks (postgres) (push) Successful in 51m33s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 55m18s
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m23s
Foundation / checks (network) (push) Successful in 21m4s
Foundation / checks (client) (push) Successful in 22m15s
Foundation / checks (tsan) (push) Successful in 23m3s
Foundation / checks (postgres) (push) Successful in 51m33s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 55m18s
This commit is contained in:
@@ -236,9 +236,13 @@ retry_cellular_profile_migration(Postgres& db, const CellularProfileMigrationPre
|
||||
milliseconds timeout) {
|
||||
const auto manifest = decode_manifest(prepared.manifest);
|
||||
if (!manifest || manifest->key != prepared.key || manifest->commit != prepared.commit ||
|
||||
manifest->profile != CheckpointProfile::cellular_lives_v1 ||
|
||||
(manifest->cellular_versions != cellular_v2_versions &&
|
||||
manifest->cellular_versions != cellular_v3_versions) ||
|
||||
!supported(*manifest) || !encode_manifest(*manifest) ||
|
||||
!((manifest->profile == CheckpointProfile::cellular_lives_v1 &&
|
||||
(manifest->cellular_versions == cellular_v2_versions ||
|
||||
manifest->cellular_versions == cellular_v3_versions)) ||
|
||||
((manifest->profile == CheckpointProfile::cellular_lives_v1 ||
|
||||
manifest->profile == CheckpointProfile::cellular_population_v1) &&
|
||||
manifest->cellular_versions == cellular_v10_versions)) ||
|
||||
manifest->begin_tick != manifest->end_tick || manifest->batch != BatchDigests{})
|
||||
return std::unexpected{migration_failure(PublicationError::candidate, {}, {}, prepared)};
|
||||
|
||||
@@ -271,6 +275,43 @@ retry_cellular_profile_migration(Postgres& db, const CellularProfileMigrationPre
|
||||
migration_failure(PublicationError::rejected, {}, published.error(), prepared)};
|
||||
}
|
||||
|
||||
std::expected<CommittedCheckpoint, CellularProfileMigrationFailure>
|
||||
migrate_cellular_boundary_profile(Postgres& db, const FilesystemObjectStore& objects,
|
||||
const LeaseGrant& lease, const CommittedCheckpoint& predecessor,
|
||||
Uuid commit, const Sha256Digest& build, milliseconds timeout,
|
||||
const PublishHooks& hooks) {
|
||||
const auto previous = decode_manifest(predecessor.manifest);
|
||||
const auto previous_digest = sha256(predecessor.manifest);
|
||||
if (!previous || !previous_digest || predecessor.head.digest != *previous_digest ||
|
||||
predecessor.head.sequence != previous->sequence ||
|
||||
predecessor.head.commit != previous->commit ||
|
||||
predecessor.head.end_tick != previous->end_tick ||
|
||||
predecessor.head.frame != previous->frame)
|
||||
return std::unexpected(migration_failure(PublicationError::predecessor));
|
||||
auto source = objects.get({previous->payload_sha256, previous->payload_bytes});
|
||||
if (!source)
|
||||
return std::unexpected(migration_failure(PublicationError::predecessor, source.error()));
|
||||
auto payload = cellular_boundary_upgrade_payload(*source, *previous);
|
||||
if (!payload)
|
||||
return std::unexpected(migration_failure(PublicationError::candidate));
|
||||
const auto digest = sha256(*payload);
|
||||
if (!digest)
|
||||
return std::unexpected(migration_failure(PublicationError::digest));
|
||||
auto manifest = cellular_boundary_upgrade_manifest(*previous, *previous_digest, commit, lease,
|
||||
build, payload->size(), *digest);
|
||||
if (!manifest)
|
||||
return std::unexpected(migration_failure(PublicationError::candidate));
|
||||
auto encoded = encode_manifest(*manifest);
|
||||
if (!encoded)
|
||||
return std::unexpected(migration_failure(PublicationError::candidate));
|
||||
if (auto stored = put_verified(objects, *payload, {*digest, payload->size()}); !stored)
|
||||
return std::unexpected(migration_failure(stored.error().kind, stored.error().object));
|
||||
if (hooks.before_transaction)
|
||||
hooks.before_transaction();
|
||||
return retry_cellular_profile_migration(db, {previous->key, commit, std::move(*encoded)},
|
||||
timeout);
|
||||
}
|
||||
|
||||
std::expected<CommittedCheckpoint, CellularProfileMigrationFailure>
|
||||
migrate_cellular_profile_to_v2(Postgres& db, const FilesystemObjectStore& objects,
|
||||
const LeaseGrant& lease, const CommittedCheckpoint& predecessor,
|
||||
@@ -500,6 +541,80 @@ publish_cellular_batch(Postgres& db, const FilesystemObjectStore& objects, const
|
||||
return PublishedCellularBatch{std::move(*committed), std::move(published_results)};
|
||||
}
|
||||
|
||||
std::expected<void, RecoveryFailure>
|
||||
verify_cellular_no_tick_checkpoint(Postgres& db, const FilesystemObjectStore& objects,
|
||||
const LeaseGrant& lease, const CommittedCheckpoint& committed,
|
||||
const CheckpointManifest& m,
|
||||
std::span<const std::uint8_t> payload, milliseconds timeout) {
|
||||
const auto fail = [](RecoveryError kind) { return std::unexpected(RecoveryFailure{kind, {}}); };
|
||||
const auto digest = sha256(committed.manifest);
|
||||
if (!digest || *digest != committed.head.digest || m.commit != committed.head.commit ||
|
||||
m.sequence != committed.head.sequence || m.end_tick != committed.head.end_tick ||
|
||||
m.frame != committed.head.frame || lease.frame != m.frame || m.sequence == 0 ||
|
||||
m.begin_tick != m.end_tick || m.batch != BatchDigests{})
|
||||
return fail(RecoveryError::manifest);
|
||||
auto evidence = load_cellular_profile_migration(db, m.key, lease, m.commit, timeout);
|
||||
if (!evidence)
|
||||
return std::unexpected(RecoveryFailure{RecoveryError::store, evidence.error()});
|
||||
const auto optional_record = std::move(*evidence);
|
||||
if (!optional_record.has_value())
|
||||
return verify_population_profile_transition(db, objects, lease, committed, m, payload,
|
||||
timeout);
|
||||
const auto& record = *optional_record;
|
||||
auto predecessor = load_region_commit(db, m.key, m.sequence - 1, timeout);
|
||||
if (!predecessor)
|
||||
return std::unexpected(RecoveryFailure{RecoveryError::tail, predecessor.error()});
|
||||
const auto source_digest = sha256(predecessor->manifest);
|
||||
const auto source = decode_manifest(predecessor->manifest);
|
||||
const auto target_bytes = encode_manifest(m);
|
||||
const auto payload_digest = sha256(payload);
|
||||
if (!source || !source_digest || !target_bytes || *target_bytes != committed.manifest ||
|
||||
payload.size() != m.payload_bytes || !payload_digest ||
|
||||
*payload_digest != m.payload_sha256 || *source_digest != predecessor->head.digest ||
|
||||
source->commit != predecessor->head.commit ||
|
||||
source->sequence != predecessor->head.sequence ||
|
||||
source->end_tick != predecessor->head.end_tick ||
|
||||
source->frame != predecessor->head.frame || record.source_sequence != source->sequence ||
|
||||
record.source_commit != source->commit || record.source_manifest_sha256 != *source_digest ||
|
||||
record.source_payload_sha256 != source->payload_sha256 ||
|
||||
record.target_sequence != m.sequence || record.target_commit != m.commit ||
|
||||
record.target_manifest_sha256 != *digest ||
|
||||
record.target_payload_sha256 != m.payload_sha256 ||
|
||||
!std::ranges::equal(record.source_versions,
|
||||
std::span{predecessor->manifest}.subspan(160, 36)) ||
|
||||
!std::ranges::equal(record.target_versions, std::span{*target_bytes}.subspan(160, 36)))
|
||||
return fail(RecoveryError::history);
|
||||
// Reconstruct with the recorded writer's lease fields. Loading the immutable
|
||||
// evidence above is fenced by the CURRENT lease, which may be a newer epoch.
|
||||
const LeaseGrant recorded{m.holder, m.epoch, m.frame, milliseconds{0}};
|
||||
const bool boundary = m.cellular_versions == cellular_v10_versions;
|
||||
auto expected =
|
||||
boundary ? cellular_boundary_upgrade_manifest(*source, *source_digest, m.commit, recorded,
|
||||
m.build, m.payload_bytes, m.payload_sha256)
|
||||
: cellular_profile_migration_manifest(*source, *source_digest, m.commit, recorded,
|
||||
m.build, m.payload_bytes, m.payload_sha256);
|
||||
if (!expected || *expected != m)
|
||||
return fail(RecoveryError::history);
|
||||
auto bytes = objects.get({source->payload_sha256, source->payload_bytes});
|
||||
if (!bytes)
|
||||
return fail(RecoveryError::tail);
|
||||
if (boundary) {
|
||||
auto upgraded = cellular_boundary_upgrade_payload(*bytes, *source);
|
||||
if (!upgraded || !std::ranges::equal(*upgraded, payload))
|
||||
return fail(RecoveryError::history);
|
||||
} else {
|
||||
auto state = restore_cellular_payload(*bytes, *source);
|
||||
if (!state)
|
||||
return fail(RecoveryError::tail);
|
||||
auto upgraded = source->cellular_versions == cellular_v1_versions
|
||||
? (*state)->migrate_to_v2()
|
||||
: (*state)->migrate_to_v3();
|
||||
if (!upgraded || !std::ranges::equal((*upgraded)->snapshot(), payload))
|
||||
return fail(RecoveryError::history);
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
std::expected<RecoveredCellularCheckpoint, RecoveryFailure>
|
||||
recover_cellular_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
const LeaseGrant& lease, std::span<const Sha256Digest> retained_builds,
|
||||
@@ -538,75 +653,13 @@ recover_cellular_checkpoint(Postgres& db, const FilesystemObjectStore& objects,
|
||||
std::vector<server::CellularActionRecord> actions;
|
||||
std::vector<server::CellularMutationActionRecord> mutation_actions_recovered;
|
||||
std::vector<server::CellularMutationResult> mutation_results_recovered;
|
||||
if (manifest->sequence != 0 && manifest->begin_tick == manifest->end_tick &&
|
||||
manifest->cellular_versions == cellular_v9_versions) {
|
||||
auto verified = verify_population_profile_transition(db, objects, lease, *committed,
|
||||
*manifest, *payload, timeout);
|
||||
if (manifest->sequence != 0 && manifest->begin_tick == manifest->end_tick) {
|
||||
auto verified = verify_cellular_no_tick_checkpoint(db, objects, lease, *committed,
|
||||
*manifest, *payload, timeout);
|
||||
if (!verified)
|
||||
return std::unexpected(verified.error());
|
||||
return RecoveredCellularCheckpoint{
|
||||
std::move(*committed), *manifest, std::move(*state), {}, {}, {}};
|
||||
} else if (manifest->sequence != 0 && manifest->begin_tick == manifest->end_tick) {
|
||||
auto migration = load_cellular_profile_migration(db, key, lease, manifest->commit, timeout);
|
||||
if (!migration)
|
||||
return std::unexpected{RecoveryFailure{RecoveryError::store, migration.error()}};
|
||||
const auto migration_record = migration->value_or(CellularProfileMigrationRecord{});
|
||||
if (!*migration || migration_record.target_sequence != manifest->sequence ||
|
||||
migration_record.target_commit != manifest->commit ||
|
||||
migration_record.target_manifest_sha256 != *digest ||
|
||||
migration_record.target_payload_sha256 != manifest->payload_sha256 ||
|
||||
migration_record.target_sequence == 0)
|
||||
return fail_with(RecoveryError::history);
|
||||
auto predecessor = load_region_commit(db, key, manifest->sequence - 1, timeout);
|
||||
if (!predecessor)
|
||||
return std::unexpected{RecoveryFailure{RecoveryError::tail, predecessor.error()}};
|
||||
const auto source_manifest_digest = sha256(predecessor->manifest);
|
||||
auto source_manifest = decode_manifest(predecessor->manifest);
|
||||
const auto source_manifest_bytes =
|
||||
encode_manifest(source_manifest ? *source_manifest : CheckpointManifest{});
|
||||
const auto target_manifest_bytes = encode_manifest(*manifest);
|
||||
if (!source_manifest_digest || !source_manifest || !source_manifest_bytes ||
|
||||
!target_manifest_bytes || *source_manifest_digest != manifest->predecessor_digest ||
|
||||
predecessor->head.digest != *source_manifest_digest ||
|
||||
predecessor->head.commit != manifest->predecessor_commit ||
|
||||
predecessor->head.sequence != manifest->sequence - 1 ||
|
||||
source_manifest->profile != CheckpointProfile::cellular_lives_v1 ||
|
||||
(source_manifest->cellular_versions != cellular_v1_versions &&
|
||||
source_manifest->cellular_versions != cellular_v2_versions) ||
|
||||
((source_manifest->cellular_versions == cellular_v1_versions) !=
|
||||
(manifest->cellular_versions == cellular_v2_versions)) ||
|
||||
source_manifest->end_tick != manifest->end_tick ||
|
||||
migration_record.source_sequence != source_manifest->sequence ||
|
||||
migration_record.source_commit != source_manifest->commit ||
|
||||
migration_record.source_manifest_sha256 != *source_manifest_digest ||
|
||||
migration_record.source_payload_sha256 != source_manifest->payload_sha256 ||
|
||||
migration_record.source_versions.size() != 36 ||
|
||||
!std::ranges::equal(migration_record.source_versions,
|
||||
std::span{*source_manifest_bytes}.subspan(160, 36)) ||
|
||||
migration_record.target_versions.size() != 36 ||
|
||||
!std::ranges::equal(migration_record.target_versions,
|
||||
std::span{*target_manifest_bytes}.subspan(160, 36)))
|
||||
return fail_with(RecoveryError::history);
|
||||
auto source_bytes =
|
||||
objects.get({source_manifest->payload_sha256, source_manifest->payload_bytes});
|
||||
if (!source_bytes)
|
||||
return fail_with(source_bytes.error() == ObjectError::io ? RecoveryError::store
|
||||
: RecoveryError::tail);
|
||||
const auto source_payload_digest = sha256(*source_bytes);
|
||||
if (!source_payload_digest || *source_payload_digest != source_manifest->payload_sha256)
|
||||
return fail_with(RecoveryError::tail);
|
||||
auto source_state = restore_cellular_payload(*source_bytes, *source_manifest);
|
||||
if (!source_state ||
|
||||
(*source_state)->format_version() != source_manifest->cellular_versions.envelope)
|
||||
return fail_with(RecoveryError::tail);
|
||||
auto migrated = source_manifest->cellular_versions == cellular_v1_versions
|
||||
? (*source_state)->migrate_to_v2()
|
||||
: (*source_state)->migrate_to_v3();
|
||||
if (!migrated || (*migrated)->tick() != manifest->end_tick ||
|
||||
(*migrated)->snapshot() != *payload)
|
||||
return fail_with(RecoveryError::history);
|
||||
return RecoveredCellularCheckpoint{
|
||||
std::move(*committed), *manifest, std::move(*state), {}, {}, {}};
|
||||
}
|
||||
if (manifest->sequence != 0) {
|
||||
auto action_bytes =
|
||||
|
||||
Reference in New Issue
Block a user