Recover cellular checkpoints with action evidence
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (checks) (push) Failing after 41s
Foundation / checks (postgres) (push) Canceled after 0s
Foundation / checks (tsan) (push) Canceled after 0s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (client) (push) Canceled after 1m6s
Foundation / checks (network) (push) Canceled after 29s
Foundation / checks (ecs) (push) Canceled after 58s
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (checks) (push) Failing after 41s
Foundation / checks (postgres) (push) Canceled after 0s
Foundation / checks (tsan) (push) Canceled after 0s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (client) (push) Canceled after 1m6s
Foundation / checks (network) (push) Canceled after 29s
Foundation / checks (ecs) (push) Canceled after 58s
This commit is contained in:
@@ -8,6 +8,13 @@ namespace evo::persistence {
|
||||
namespace {
|
||||
using std::chrono::milliseconds;
|
||||
|
||||
std::uint64_t input_tick(const server::CellularActionRecord& action) {
|
||||
std::uint64_t tick{};
|
||||
for (std::size_t i = 0; i < 8; ++i)
|
||||
tick |= static_cast<std::uint64_t>(action.input[i]) << (i * 8U);
|
||||
return tick;
|
||||
}
|
||||
|
||||
CellularPublicationFailure failure(PublicationError kind, ObjectError object = {},
|
||||
LeaseStoreFailure store = {},
|
||||
CellularPreparedCommit prepared = {}) {
|
||||
@@ -150,4 +157,115 @@ publish_cellular_batch(Postgres& db, const FilesystemObjectStore& objects,
|
||||
return std::unexpected{committed.error()};
|
||||
return PublishedCellularBatch{std::move(*committed)};
|
||||
}
|
||||
|
||||
std::expected<RecoveredCellularCheckpoint, RecoveryFailure>
|
||||
recover_cellular_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
const LeaseGrant& lease,
|
||||
std::span<const Sha256Digest> retained_builds,
|
||||
milliseconds timeout) {
|
||||
const auto fail_with = [](RecoveryError kind) {
|
||||
return std::unexpected{RecoveryFailure{kind, {}}};
|
||||
};
|
||||
auto committed = load_region_checkpoint(db, key, timeout);
|
||||
if (!committed)
|
||||
return std::unexpected{RecoveryFailure{
|
||||
committed.error().kind == LeaseStoreError::not_provisioned
|
||||
? RecoveryError::not_provisioned
|
||||
: RecoveryError::store,
|
||||
committed.error()}};
|
||||
const auto digest = sha256(committed->manifest);
|
||||
auto manifest = decode_manifest(committed->manifest);
|
||||
if (!digest || *digest != committed->head.digest || !manifest || manifest->key != key ||
|
||||
manifest->sequence != committed->head.sequence ||
|
||||
manifest->commit != committed->head.commit || manifest->end_tick != committed->head.end_tick ||
|
||||
manifest->frame != committed->head.frame)
|
||||
return fail_with(RecoveryError::manifest);
|
||||
if (!supported(*manifest) || manifest->profile != CheckpointProfile::cellular_lives_v1 ||
|
||||
std::ranges::find(retained_builds, manifest->build) == retained_builds.end())
|
||||
return fail_with(RecoveryError::unsupported);
|
||||
if (lease.frame != manifest->frame)
|
||||
return fail_with(RecoveryError::unsupported);
|
||||
|
||||
auto payload = objects.get({manifest->payload_sha256, manifest->payload_bytes});
|
||||
if (!payload)
|
||||
return fail_with(payload.error() == ObjectError::io ? RecoveryError::store
|
||||
: RecoveryError::corrupt_payload);
|
||||
auto state = restore_cellular_payload(*payload, *manifest);
|
||||
if (!state)
|
||||
return fail_with(RecoveryError::payload);
|
||||
|
||||
std::vector<server::CellularActionRecord> actions;
|
||||
if (manifest->sequence != 0) {
|
||||
auto action_bytes = load_cellular_publication_actions(db, key, lease, manifest->commit, timeout);
|
||||
if (!action_bytes)
|
||||
return std::unexpected{RecoveryFailure{RecoveryError::store, action_bytes.error()}};
|
||||
if (!*action_bytes)
|
||||
return fail_with(RecoveryError::history);
|
||||
|
||||
auto trace_bytes = objects.get({manifest->batch.trace_sha256, manifest->batch.trace_bytes});
|
||||
if (!trace_bytes)
|
||||
return fail_with(trace_bytes.error() == ObjectError::io ? RecoveryError::store
|
||||
: RecoveryError::tail);
|
||||
auto trace = server::decode_cellular_batch(*trace_bytes);
|
||||
if (!trace)
|
||||
return fail_with(RecoveryError::tail);
|
||||
if (trace->header.world != key.world || trace->header.region != key.region ||
|
||||
trace->header.frame != manifest->frame ||
|
||||
trace->header.predecessor != manifest->predecessor_commit ||
|
||||
trace->header.predecessor_digest != manifest->predecessor_digest ||
|
||||
trace->header.begin_tick != manifest->begin_tick ||
|
||||
trace->header.end_tick != manifest->end_tick)
|
||||
return fail_with(RecoveryError::tail);
|
||||
|
||||
auto predecessor = load_region_commit(db, key, manifest->sequence - 1, timeout);
|
||||
if (!predecessor)
|
||||
return std::unexpected{RecoveryFailure{RecoveryError::tail, predecessor.error()}};
|
||||
const auto predecessor_digest = sha256(predecessor->manifest);
|
||||
auto previous = decode_manifest(predecessor->manifest);
|
||||
if (!predecessor_digest || *predecessor_digest != manifest->predecessor_digest ||
|
||||
predecessor->head.digest != manifest->predecessor_digest ||
|
||||
predecessor->head.commit != manifest->predecessor_commit ||
|
||||
predecessor->head.sequence != manifest->sequence - 1 || !previous ||
|
||||
previous->profile != CheckpointProfile::cellular_lives_v1 ||
|
||||
previous->end_tick != manifest->begin_tick)
|
||||
return fail_with(RecoveryError::tail);
|
||||
auto base_bytes = objects.get({previous->payload_sha256, previous->payload_bytes});
|
||||
if (!base_bytes)
|
||||
return fail_with(base_bytes.error() == ObjectError::io ? RecoveryError::store
|
||||
: RecoveryError::tail);
|
||||
auto base = restore_cellular_payload(*base_bytes, *previous);
|
||||
if (!base)
|
||||
return fail_with(RecoveryError::tail);
|
||||
auto batch_actions = server::decode_cellular_action_block(
|
||||
**action_bytes, manifest->begin_tick, manifest->end_tick);
|
||||
if (!batch_actions || !server::match_cellular_actions_to_trace(**base, *trace, *batch_actions))
|
||||
return fail_with(RecoveryError::history);
|
||||
auto replay = server::replay_cellular_batch(**base, *trace);
|
||||
if (!replay || (*replay)->snapshot() != *payload)
|
||||
return fail_with(RecoveryError::tail);
|
||||
|
||||
std::vector<ActionRef> refs;
|
||||
refs.reserve(batch_actions->size());
|
||||
for (const auto& action : *batch_actions)
|
||||
refs.push_back({action.account, action.action_id});
|
||||
auto records = find_region_action_records(db, key, lease, refs, timeout);
|
||||
if (!records)
|
||||
return std::unexpected{RecoveryFailure{RecoveryError::store, records.error()}};
|
||||
for (std::size_t i = 0; i < batch_actions->size(); ++i) {
|
||||
if (!(*records)[i])
|
||||
return fail_with(RecoveryError::history);
|
||||
const auto& record = *(*records)[i];
|
||||
const auto& action = (*batch_actions)[i];
|
||||
if (record.kind != 2 || record.result != 5 || !record.cellular_input ||
|
||||
*record.cellular_input !=
|
||||
std::vector<std::uint8_t>(action.input.begin(), action.input.end()) ||
|
||||
record.action.region != key.region || record.action.tick != input_tick(action) ||
|
||||
record.action.commit_sequence != manifest->sequence)
|
||||
return fail_with(RecoveryError::history);
|
||||
}
|
||||
actions = std::move(*batch_actions);
|
||||
}
|
||||
return RecoveredCellularCheckpoint{std::move(*committed), *manifest, std::move(*state),
|
||||
std::move(actions)};
|
||||
}
|
||||
} // namespace evo::persistence
|
||||
|
||||
Reference in New Issue
Block a user