M12.4: admit and replay durable boundary-accounting upgrades
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m23s
Foundation / checks (network) (push) Successful in 21m4s
Foundation / checks (client) (push) Successful in 22m15s
Foundation / checks (tsan) (push) Successful in 23m3s
Foundation / checks (postgres) (push) Successful in 51m33s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 55m18s

This commit is contained in:
dingo
2026-10-18 08:13:36 -04:00
parent 2416735e2b
commit 1b5c0cdf7c
21 changed files with 812 additions and 98 deletions
+121 -68
View File
@@ -236,9 +236,13 @@ retry_cellular_profile_migration(Postgres& db, const CellularProfileMigrationPre
milliseconds timeout) {
const auto manifest = decode_manifest(prepared.manifest);
if (!manifest || manifest->key != prepared.key || manifest->commit != prepared.commit ||
manifest->profile != CheckpointProfile::cellular_lives_v1 ||
(manifest->cellular_versions != cellular_v2_versions &&
manifest->cellular_versions != cellular_v3_versions) ||
!supported(*manifest) || !encode_manifest(*manifest) ||
!((manifest->profile == CheckpointProfile::cellular_lives_v1 &&
(manifest->cellular_versions == cellular_v2_versions ||
manifest->cellular_versions == cellular_v3_versions)) ||
((manifest->profile == CheckpointProfile::cellular_lives_v1 ||
manifest->profile == CheckpointProfile::cellular_population_v1) &&
manifest->cellular_versions == cellular_v10_versions)) ||
manifest->begin_tick != manifest->end_tick || manifest->batch != BatchDigests{})
return std::unexpected{migration_failure(PublicationError::candidate, {}, {}, prepared)};
@@ -271,6 +275,43 @@ retry_cellular_profile_migration(Postgres& db, const CellularProfileMigrationPre
migration_failure(PublicationError::rejected, {}, published.error(), prepared)};
}
std::expected<CommittedCheckpoint, CellularProfileMigrationFailure>
migrate_cellular_boundary_profile(Postgres& db, const FilesystemObjectStore& objects,
const LeaseGrant& lease, const CommittedCheckpoint& predecessor,
Uuid commit, const Sha256Digest& build, milliseconds timeout,
const PublishHooks& hooks) {
const auto previous = decode_manifest(predecessor.manifest);
const auto previous_digest = sha256(predecessor.manifest);
if (!previous || !previous_digest || predecessor.head.digest != *previous_digest ||
predecessor.head.sequence != previous->sequence ||
predecessor.head.commit != previous->commit ||
predecessor.head.end_tick != previous->end_tick ||
predecessor.head.frame != previous->frame)
return std::unexpected(migration_failure(PublicationError::predecessor));
auto source = objects.get({previous->payload_sha256, previous->payload_bytes});
if (!source)
return std::unexpected(migration_failure(PublicationError::predecessor, source.error()));
auto payload = cellular_boundary_upgrade_payload(*source, *previous);
if (!payload)
return std::unexpected(migration_failure(PublicationError::candidate));
const auto digest = sha256(*payload);
if (!digest)
return std::unexpected(migration_failure(PublicationError::digest));
auto manifest = cellular_boundary_upgrade_manifest(*previous, *previous_digest, commit, lease,
build, payload->size(), *digest);
if (!manifest)
return std::unexpected(migration_failure(PublicationError::candidate));
auto encoded = encode_manifest(*manifest);
if (!encoded)
return std::unexpected(migration_failure(PublicationError::candidate));
if (auto stored = put_verified(objects, *payload, {*digest, payload->size()}); !stored)
return std::unexpected(migration_failure(stored.error().kind, stored.error().object));
if (hooks.before_transaction)
hooks.before_transaction();
return retry_cellular_profile_migration(db, {previous->key, commit, std::move(*encoded)},
timeout);
}
std::expected<CommittedCheckpoint, CellularProfileMigrationFailure>
migrate_cellular_profile_to_v2(Postgres& db, const FilesystemObjectStore& objects,
const LeaseGrant& lease, const CommittedCheckpoint& predecessor,
@@ -500,6 +541,80 @@ publish_cellular_batch(Postgres& db, const FilesystemObjectStore& objects, const
return PublishedCellularBatch{std::move(*committed), std::move(published_results)};
}
std::expected<void, RecoveryFailure>
verify_cellular_no_tick_checkpoint(Postgres& db, const FilesystemObjectStore& objects,
const LeaseGrant& lease, const CommittedCheckpoint& committed,
const CheckpointManifest& m,
std::span<const std::uint8_t> payload, milliseconds timeout) {
const auto fail = [](RecoveryError kind) { return std::unexpected(RecoveryFailure{kind, {}}); };
const auto digest = sha256(committed.manifest);
if (!digest || *digest != committed.head.digest || m.commit != committed.head.commit ||
m.sequence != committed.head.sequence || m.end_tick != committed.head.end_tick ||
m.frame != committed.head.frame || lease.frame != m.frame || m.sequence == 0 ||
m.begin_tick != m.end_tick || m.batch != BatchDigests{})
return fail(RecoveryError::manifest);
auto evidence = load_cellular_profile_migration(db, m.key, lease, m.commit, timeout);
if (!evidence)
return std::unexpected(RecoveryFailure{RecoveryError::store, evidence.error()});
const auto optional_record = std::move(*evidence);
if (!optional_record.has_value())
return verify_population_profile_transition(db, objects, lease, committed, m, payload,
timeout);
const auto& record = *optional_record;
auto predecessor = load_region_commit(db, m.key, m.sequence - 1, timeout);
if (!predecessor)
return std::unexpected(RecoveryFailure{RecoveryError::tail, predecessor.error()});
const auto source_digest = sha256(predecessor->manifest);
const auto source = decode_manifest(predecessor->manifest);
const auto target_bytes = encode_manifest(m);
const auto payload_digest = sha256(payload);
if (!source || !source_digest || !target_bytes || *target_bytes != committed.manifest ||
payload.size() != m.payload_bytes || !payload_digest ||
*payload_digest != m.payload_sha256 || *source_digest != predecessor->head.digest ||
source->commit != predecessor->head.commit ||
source->sequence != predecessor->head.sequence ||
source->end_tick != predecessor->head.end_tick ||
source->frame != predecessor->head.frame || record.source_sequence != source->sequence ||
record.source_commit != source->commit || record.source_manifest_sha256 != *source_digest ||
record.source_payload_sha256 != source->payload_sha256 ||
record.target_sequence != m.sequence || record.target_commit != m.commit ||
record.target_manifest_sha256 != *digest ||
record.target_payload_sha256 != m.payload_sha256 ||
!std::ranges::equal(record.source_versions,
std::span{predecessor->manifest}.subspan(160, 36)) ||
!std::ranges::equal(record.target_versions, std::span{*target_bytes}.subspan(160, 36)))
return fail(RecoveryError::history);
// Reconstruct with the recorded writer's lease fields. Loading the immutable
// evidence above is fenced by the CURRENT lease, which may be a newer epoch.
const LeaseGrant recorded{m.holder, m.epoch, m.frame, milliseconds{0}};
const bool boundary = m.cellular_versions == cellular_v10_versions;
auto expected =
boundary ? cellular_boundary_upgrade_manifest(*source, *source_digest, m.commit, recorded,
m.build, m.payload_bytes, m.payload_sha256)
: cellular_profile_migration_manifest(*source, *source_digest, m.commit, recorded,
m.build, m.payload_bytes, m.payload_sha256);
if (!expected || *expected != m)
return fail(RecoveryError::history);
auto bytes = objects.get({source->payload_sha256, source->payload_bytes});
if (!bytes)
return fail(RecoveryError::tail);
if (boundary) {
auto upgraded = cellular_boundary_upgrade_payload(*bytes, *source);
if (!upgraded || !std::ranges::equal(*upgraded, payload))
return fail(RecoveryError::history);
} else {
auto state = restore_cellular_payload(*bytes, *source);
if (!state)
return fail(RecoveryError::tail);
auto upgraded = source->cellular_versions == cellular_v1_versions
? (*state)->migrate_to_v2()
: (*state)->migrate_to_v3();
if (!upgraded || !std::ranges::equal((*upgraded)->snapshot(), payload))
return fail(RecoveryError::history);
}
return {};
}
std::expected<RecoveredCellularCheckpoint, RecoveryFailure>
recover_cellular_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
const LeaseGrant& lease, std::span<const Sha256Digest> retained_builds,
@@ -538,75 +653,13 @@ recover_cellular_checkpoint(Postgres& db, const FilesystemObjectStore& objects,
std::vector<server::CellularActionRecord> actions;
std::vector<server::CellularMutationActionRecord> mutation_actions_recovered;
std::vector<server::CellularMutationResult> mutation_results_recovered;
if (manifest->sequence != 0 && manifest->begin_tick == manifest->end_tick &&
manifest->cellular_versions == cellular_v9_versions) {
auto verified = verify_population_profile_transition(db, objects, lease, *committed,
*manifest, *payload, timeout);
if (manifest->sequence != 0 && manifest->begin_tick == manifest->end_tick) {
auto verified = verify_cellular_no_tick_checkpoint(db, objects, lease, *committed,
*manifest, *payload, timeout);
if (!verified)
return std::unexpected(verified.error());
return RecoveredCellularCheckpoint{
std::move(*committed), *manifest, std::move(*state), {}, {}, {}};
} else if (manifest->sequence != 0 && manifest->begin_tick == manifest->end_tick) {
auto migration = load_cellular_profile_migration(db, key, lease, manifest->commit, timeout);
if (!migration)
return std::unexpected{RecoveryFailure{RecoveryError::store, migration.error()}};
const auto migration_record = migration->value_or(CellularProfileMigrationRecord{});
if (!*migration || migration_record.target_sequence != manifest->sequence ||
migration_record.target_commit != manifest->commit ||
migration_record.target_manifest_sha256 != *digest ||
migration_record.target_payload_sha256 != manifest->payload_sha256 ||
migration_record.target_sequence == 0)
return fail_with(RecoveryError::history);
auto predecessor = load_region_commit(db, key, manifest->sequence - 1, timeout);
if (!predecessor)
return std::unexpected{RecoveryFailure{RecoveryError::tail, predecessor.error()}};
const auto source_manifest_digest = sha256(predecessor->manifest);
auto source_manifest = decode_manifest(predecessor->manifest);
const auto source_manifest_bytes =
encode_manifest(source_manifest ? *source_manifest : CheckpointManifest{});
const auto target_manifest_bytes = encode_manifest(*manifest);
if (!source_manifest_digest || !source_manifest || !source_manifest_bytes ||
!target_manifest_bytes || *source_manifest_digest != manifest->predecessor_digest ||
predecessor->head.digest != *source_manifest_digest ||
predecessor->head.commit != manifest->predecessor_commit ||
predecessor->head.sequence != manifest->sequence - 1 ||
source_manifest->profile != CheckpointProfile::cellular_lives_v1 ||
(source_manifest->cellular_versions != cellular_v1_versions &&
source_manifest->cellular_versions != cellular_v2_versions) ||
((source_manifest->cellular_versions == cellular_v1_versions) !=
(manifest->cellular_versions == cellular_v2_versions)) ||
source_manifest->end_tick != manifest->end_tick ||
migration_record.source_sequence != source_manifest->sequence ||
migration_record.source_commit != source_manifest->commit ||
migration_record.source_manifest_sha256 != *source_manifest_digest ||
migration_record.source_payload_sha256 != source_manifest->payload_sha256 ||
migration_record.source_versions.size() != 36 ||
!std::ranges::equal(migration_record.source_versions,
std::span{*source_manifest_bytes}.subspan(160, 36)) ||
migration_record.target_versions.size() != 36 ||
!std::ranges::equal(migration_record.target_versions,
std::span{*target_manifest_bytes}.subspan(160, 36)))
return fail_with(RecoveryError::history);
auto source_bytes =
objects.get({source_manifest->payload_sha256, source_manifest->payload_bytes});
if (!source_bytes)
return fail_with(source_bytes.error() == ObjectError::io ? RecoveryError::store
: RecoveryError::tail);
const auto source_payload_digest = sha256(*source_bytes);
if (!source_payload_digest || *source_payload_digest != source_manifest->payload_sha256)
return fail_with(RecoveryError::tail);
auto source_state = restore_cellular_payload(*source_bytes, *source_manifest);
if (!source_state ||
(*source_state)->format_version() != source_manifest->cellular_versions.envelope)
return fail_with(RecoveryError::tail);
auto migrated = source_manifest->cellular_versions == cellular_v1_versions
? (*source_state)->migrate_to_v2()
: (*source_state)->migrate_to_v3();
if (!migrated || (*migrated)->tick() != manifest->end_tick ||
(*migrated)->snapshot() != *payload)
return fail_with(RecoveryError::history);
return RecoveredCellularCheckpoint{
std::move(*committed), *manifest, std::move(*state), {}, {}, {}};
}
if (manifest->sequence != 0) {
auto action_bytes =