Recover cellular checkpoints with action evidence
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (checks) (push) Failing after 41s
Foundation / checks (postgres) (push) Canceled after 0s
Foundation / checks (tsan) (push) Canceled after 0s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (client) (push) Canceled after 1m6s
Foundation / checks (network) (push) Canceled after 29s
Foundation / checks (ecs) (push) Canceled after 58s

This commit is contained in:
dingo
2026-10-10 23:23:17 -04:00
parent ad6e53c2e9
commit 13273b46f4
7 changed files with 256 additions and 0 deletions
@@ -8,6 +8,13 @@ namespace evo::persistence {
namespace {
using std::chrono::milliseconds;
std::uint64_t input_tick(const server::CellularActionRecord& action) {
std::uint64_t tick{};
for (std::size_t i = 0; i < 8; ++i)
tick |= static_cast<std::uint64_t>(action.input[i]) << (i * 8U);
return tick;
}
CellularPublicationFailure failure(PublicationError kind, ObjectError object = {},
LeaseStoreFailure store = {},
CellularPreparedCommit prepared = {}) {
@@ -150,4 +157,115 @@ publish_cellular_batch(Postgres& db, const FilesystemObjectStore& objects,
return std::unexpected{committed.error()};
return PublishedCellularBatch{std::move(*committed)};
}
std::expected<RecoveredCellularCheckpoint, RecoveryFailure>
recover_cellular_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
const LeaseGrant& lease,
std::span<const Sha256Digest> retained_builds,
milliseconds timeout) {
const auto fail_with = [](RecoveryError kind) {
return std::unexpected{RecoveryFailure{kind, {}}};
};
auto committed = load_region_checkpoint(db, key, timeout);
if (!committed)
return std::unexpected{RecoveryFailure{
committed.error().kind == LeaseStoreError::not_provisioned
? RecoveryError::not_provisioned
: RecoveryError::store,
committed.error()}};
const auto digest = sha256(committed->manifest);
auto manifest = decode_manifest(committed->manifest);
if (!digest || *digest != committed->head.digest || !manifest || manifest->key != key ||
manifest->sequence != committed->head.sequence ||
manifest->commit != committed->head.commit || manifest->end_tick != committed->head.end_tick ||
manifest->frame != committed->head.frame)
return fail_with(RecoveryError::manifest);
if (!supported(*manifest) || manifest->profile != CheckpointProfile::cellular_lives_v1 ||
std::ranges::find(retained_builds, manifest->build) == retained_builds.end())
return fail_with(RecoveryError::unsupported);
if (lease.frame != manifest->frame)
return fail_with(RecoveryError::unsupported);
auto payload = objects.get({manifest->payload_sha256, manifest->payload_bytes});
if (!payload)
return fail_with(payload.error() == ObjectError::io ? RecoveryError::store
: RecoveryError::corrupt_payload);
auto state = restore_cellular_payload(*payload, *manifest);
if (!state)
return fail_with(RecoveryError::payload);
std::vector<server::CellularActionRecord> actions;
if (manifest->sequence != 0) {
auto action_bytes = load_cellular_publication_actions(db, key, lease, manifest->commit, timeout);
if (!action_bytes)
return std::unexpected{RecoveryFailure{RecoveryError::store, action_bytes.error()}};
if (!*action_bytes)
return fail_with(RecoveryError::history);
auto trace_bytes = objects.get({manifest->batch.trace_sha256, manifest->batch.trace_bytes});
if (!trace_bytes)
return fail_with(trace_bytes.error() == ObjectError::io ? RecoveryError::store
: RecoveryError::tail);
auto trace = server::decode_cellular_batch(*trace_bytes);
if (!trace)
return fail_with(RecoveryError::tail);
if (trace->header.world != key.world || trace->header.region != key.region ||
trace->header.frame != manifest->frame ||
trace->header.predecessor != manifest->predecessor_commit ||
trace->header.predecessor_digest != manifest->predecessor_digest ||
trace->header.begin_tick != manifest->begin_tick ||
trace->header.end_tick != manifest->end_tick)
return fail_with(RecoveryError::tail);
auto predecessor = load_region_commit(db, key, manifest->sequence - 1, timeout);
if (!predecessor)
return std::unexpected{RecoveryFailure{RecoveryError::tail, predecessor.error()}};
const auto predecessor_digest = sha256(predecessor->manifest);
auto previous = decode_manifest(predecessor->manifest);
if (!predecessor_digest || *predecessor_digest != manifest->predecessor_digest ||
predecessor->head.digest != manifest->predecessor_digest ||
predecessor->head.commit != manifest->predecessor_commit ||
predecessor->head.sequence != manifest->sequence - 1 || !previous ||
previous->profile != CheckpointProfile::cellular_lives_v1 ||
previous->end_tick != manifest->begin_tick)
return fail_with(RecoveryError::tail);
auto base_bytes = objects.get({previous->payload_sha256, previous->payload_bytes});
if (!base_bytes)
return fail_with(base_bytes.error() == ObjectError::io ? RecoveryError::store
: RecoveryError::tail);
auto base = restore_cellular_payload(*base_bytes, *previous);
if (!base)
return fail_with(RecoveryError::tail);
auto batch_actions = server::decode_cellular_action_block(
**action_bytes, manifest->begin_tick, manifest->end_tick);
if (!batch_actions || !server::match_cellular_actions_to_trace(**base, *trace, *batch_actions))
return fail_with(RecoveryError::history);
auto replay = server::replay_cellular_batch(**base, *trace);
if (!replay || (*replay)->snapshot() != *payload)
return fail_with(RecoveryError::tail);
std::vector<ActionRef> refs;
refs.reserve(batch_actions->size());
for (const auto& action : *batch_actions)
refs.push_back({action.account, action.action_id});
auto records = find_region_action_records(db, key, lease, refs, timeout);
if (!records)
return std::unexpected{RecoveryFailure{RecoveryError::store, records.error()}};
for (std::size_t i = 0; i < batch_actions->size(); ++i) {
if (!(*records)[i])
return fail_with(RecoveryError::history);
const auto& record = *(*records)[i];
const auto& action = (*batch_actions)[i];
if (record.kind != 2 || record.result != 5 || !record.cellular_input ||
*record.cellular_input !=
std::vector<std::uint8_t>(action.input.begin(), action.input.end()) ||
record.action.region != key.region || record.action.tick != input_tick(action) ||
record.action.commit_sequence != manifest->sequence)
return fail_with(RecoveryError::history);
}
actions = std::move(*batch_actions);
}
return RecoveredCellularCheckpoint{std::move(*committed), *manifest, std::move(*state),
std::move(actions)};
}
} // namespace evo::persistence