#include "evo/persistence/population_checkpoint_store.hpp" #include "evo/persistence/sha256.hpp" #include #include namespace evo::persistence { namespace { using std::chrono::milliseconds; PopulationPublicationFailure failure(PublicationError kind, ObjectError object = {}, LeaseStoreFailure store = {}, PopulationPreparedCommit prepared = {}) { return {kind, store, object, std::move(prepared)}; } bool bound(const CommittedCheckpoint& checkpoint, const CheckpointManifest& m) { const auto digest = sha256(checkpoint.manifest); return digest && *digest == checkpoint.head.digest && m.commit == checkpoint.head.commit && m.sequence == checkpoint.head.sequence && m.end_tick == checkpoint.head.end_tick && m.frame == checkpoint.head.frame; } bool canonical(const PopulationPreparedCommit& prepared, const CheckpointManifest& m) { if (!supported(m) || m.key != prepared.key || m.commit != prepared.commit || m.sequence == 0 || population_envelope_version(m.cellular_versions) == 0 || !encode_manifest(m)) return false; if (prepared.trace.empty()) return (m.profile == CheckpointProfile::cellular_lives_v1 || m.profile == CheckpointProfile::cellular_population_v1) && m.begin_tick == m.end_tick && m.batch == BatchDigests{}; const auto trace = server::decode_population_batch(prepared.trace); const auto digest = sha256(prepared.trace); if (!trace || !digest || trace->format_version != population_envelope_version(m.cellular_versions) || m.profile != CheckpointProfile::cellular_population_v1 || m.batch != BatchDigests{prepared.trace.size(), *digest, 0, {}}) return false; const auto& h = trace->header; return h.world == m.key.world && h.region == m.key.region && h.frame == m.frame && h.predecessor == m.predecessor_commit && h.predecessor_digest == m.predecessor_digest && h.begin_tick == m.begin_tick && h.end_tick == m.end_tick; } std::expected put(const FilesystemObjectStore& objects, std::span bytes) { const auto digest = sha256(bytes); if (!digest) return std::unexpected(failure(PublicationError::digest)); auto object = objects.put(bytes); if (!object) return std::unexpected(failure(PublicationError::upload, object.error())); auto readback = objects.get(*object); if (!readback || object->bytes != bytes.size() || object->sha256 != *digest || !std::ranges::equal(*readback, bytes)) return std::unexpected( failure(PublicationError::verify, readback ? ObjectError::corrupt : readback.error())); return {}; } std::expected transition(Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease, const CommittedCheckpoint& predecessor, Uuid commit, const Sha256Digest& build, server::CellularTierPolicy policy, bool to_population, milliseconds timeout, const PublishHooks& hooks) { auto previous = decode_manifest(predecessor.manifest); const auto previous_digest = sha256(predecessor.manifest); if (!previous || !previous_digest || !bound(predecessor, *previous) || previous->profile != (to_population ? CheckpointProfile::cellular_lives_v1 : CheckpointProfile::cellular_population_v1) || population_envelope_version(previous->cellular_versions) == 0) return std::unexpected(failure(PublicationError::predecessor)); auto source = objects.get({previous->payload_sha256, previous->payload_bytes}); if (!source) return std::unexpected(failure(PublicationError::predecessor, source.error())); std::vector payload; if (to_population) { auto state = restore_cellular_payload(*source, *previous); if (!state) return std::unexpected(failure(PublicationError::predecessor)); if (!server::population_interval_safe(**state)) return std::unexpected(failure(PublicationError::candidate)); auto population = server::CellularPopulation::create(std::move(*state), policy); if (!population || (*population)->state().snapshot() != *source) return std::unexpected(failure(PublicationError::candidate)); payload = (*population)->snapshot(); } else { auto state = restore_population_payload(*source, *previous); if (!state) return std::unexpected(failure(PublicationError::predecessor)); payload = (*state)->state().snapshot(); } const auto digest = sha256(payload); if (!digest) return std::unexpected(failure(PublicationError::digest)); auto manifest = population_profile_transition_manifest(*previous, *previous_digest, commit, lease, build, payload.size(), *digest); if (!manifest) return std::unexpected(failure(PublicationError::candidate)); auto encoded = encode_manifest(*manifest); if (!encoded) return std::unexpected(failure(PublicationError::candidate)); if (auto uploaded = put(objects, payload); !uploaded) return std::unexpected(uploaded.error()); if (hooks.before_transaction) hooks.before_transaction(); return retry_population_prepared(db, {previous->key, commit, std::move(*encoded), {}}, timeout); } } // namespace std::expected migrate_cellular_to_population( Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease, const CommittedCheckpoint& predecessor, Uuid commit, const Sha256Digest& build, server::CellularTierPolicy policy, milliseconds timeout, const PublishHooks& hooks) { return transition(db, objects, lease, predecessor, commit, build, policy, true, timeout, hooks); } std::expected materialize_population_profile(Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease, const CommittedCheckpoint& predecessor, Uuid commit, const Sha256Digest& build, milliseconds timeout, const PublishHooks& hooks) { return transition(db, objects, lease, predecessor, commit, build, {}, false, timeout, hooks); } std::expected retry_population_prepared(Postgres& db, const PopulationPreparedCommit& prepared, milliseconds timeout) { const auto m = decode_manifest(prepared.manifest); if (!m || !canonical(prepared, *m)) return std::unexpected(failure(PublicationError::candidate)); auto result = prepared.trace.empty() ? publish_population_profile_transition(db, prepared.manifest, timeout) : publish_population_checkpoint(db, prepared.manifest, prepared.trace, timeout); if (!result) return std::unexpected(failure(result.error().kind == LeaseStoreError::unknown ? PublicationError::unknown : PublicationError::rejected, {}, result.error(), prepared)); const auto digest = sha256(prepared.manifest); if (!digest || result->digest != *digest || result->commit != m->commit || result->sequence != m->sequence || result->end_tick != m->end_tick || result->frame != m->frame) return std::unexpected(failure(PublicationError::receipt)); return CommittedCheckpoint{*result, prepared.manifest}; } std::expected resolve_population_prepared(Postgres& db, const PopulationPreparedCommit& prepared, milliseconds timeout) { const auto manifest = decode_manifest(prepared.manifest); if (!manifest || !canonical(prepared, *manifest)) return std::unexpected(LeaseStoreFailure{LeaseStoreError::invalid, {}}); return resolve_population_commit(db, prepared.key, prepared.commit, prepared.manifest, prepared.trace, timeout); } std::expected publish_population_batch( Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease, const CommittedCheckpoint& predecessor, const server::CellularPopulationBatch& batch, const server::CellularPopulation& candidate, Uuid commit, const Sha256Digest& build, milliseconds timeout, const PublishHooks& hooks) { const auto previous = decode_manifest(predecessor.manifest); const auto previous_digest = sha256(predecessor.manifest); const auto trace = server::encode_population_batch(batch); if (!previous || !previous_digest || !trace || !bound(predecessor, *previous) || previous->profile != CheckpointProfile::cellular_population_v1) return std::unexpected(failure(PublicationError::predecessor)); auto source = objects.get({previous->payload_sha256, previous->payload_bytes}); if (!source) return std::unexpected(failure(PublicationError::predecessor, source.error())); auto base = restore_population_payload(*source, *previous); if (!base) return std::unexpected(failure(PublicationError::predecessor)); if (!batch.conversion && !server::population_interval_safe((*base)->state())) return std::unexpected(failure(PublicationError::candidate)); auto replay = server::replay_population_batch(**base, batch); const auto payload = candidate.snapshot(); if (!replay || (*replay)->snapshot() != payload) return std::unexpected(failure(PublicationError::candidate)); const auto payload_digest = sha256(payload), trace_digest = sha256(*trace); if (!payload_digest || !trace_digest) return std::unexpected(failure(PublicationError::digest)); auto m = population_successor_manifest(*previous, *previous_digest, commit, lease, batch.header.end_tick, build, payload.size(), *payload_digest, {trace->size(), *trace_digest, 0, {}}); if (!m) return std::unexpected(failure(PublicationError::candidate)); auto encoded = encode_manifest(*m); if (!encoded) return std::unexpected(failure(PublicationError::candidate)); PopulationPreparedCommit prepared{previous->key, commit, std::move(*encoded), *trace}; if (!canonical(prepared, *m)) return std::unexpected(failure(PublicationError::candidate)); if (auto uploaded = put(objects, payload); !uploaded) return std::unexpected(uploaded.error()); if (auto uploaded = put(objects, *trace); !uploaded) return std::unexpected(uploaded.error()); if (hooks.before_transaction) hooks.before_transaction(); return retry_population_prepared(db, prepared, timeout); } std::expected verify_population_profile_transition(Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease, const CommittedCheckpoint& committed, const CheckpointManifest& m, std::span payload, milliseconds timeout) { const auto fail = [](RecoveryError kind) { return std::unexpected(RecoveryFailure{kind, {}}); }; if (!bound(committed, m) || m.sequence == 0 || m.begin_tick != m.end_tick || m.batch != BatchDigests{}) return fail(RecoveryError::manifest); auto evidence = load_population_evidence(db, m.key, lease, m.commit, timeout); if (!evidence) return std::unexpected(RecoveryFailure{RecoveryError::store, evidence.error()}); const auto record_optional = std::move(*evidence); if (!record_optional.has_value()) return fail(RecoveryError::history); const auto& record = *record_optional; if (!record.trace.empty() || !record.source_commit.has_value()) return fail(RecoveryError::history); auto predecessor = load_region_commit(db, m.key, m.sequence - 1, timeout); if (!predecessor) return std::unexpected(RecoveryFailure{RecoveryError::tail, predecessor.error()}); auto source = decode_manifest(predecessor->manifest); if (!source || !bound(*predecessor, *source) || !supported(*source) || source->commit != m.predecessor_commit || predecessor->head.digest != m.predecessor_digest || source->key != m.key || source->frame != m.frame || source->end_tick != m.end_tick || population_envelope_version(source->cellular_versions) == 0 || m.cellular_versions != source->cellular_versions || record.source_commit != source->commit || record.source_manifest_sha256 != predecessor->head.digest || record.source_payload_sha256 != source->payload_sha256) return fail(RecoveryError::tail); auto bytes = objects.get({source->payload_sha256, source->payload_bytes}); if (!bytes) return fail(RecoveryError::tail); if (m.profile == CheckpointProfile::cellular_population_v1 && source->profile == CheckpointProfile::cellular_lives_v1) { auto raw = restore_cellular_payload(*bytes, *source); auto population = restore_population_payload(payload, m); if (!raw || !population) return fail(RecoveryError::payload); auto expected = server::CellularPopulation::create(std::move(*raw), (*population)->policy()); if (!expected || !std::ranges::equal((*expected)->snapshot(), payload)) return fail(RecoveryError::history); } else if (m.profile == CheckpointProfile::cellular_lives_v1 && source->profile == CheckpointProfile::cellular_population_v1) { auto population = restore_population_payload(*bytes, *source); if (!population || !restore_cellular_payload(payload, m) || !std::ranges::equal((*population)->state().snapshot(), payload)) return fail(RecoveryError::history); } else return fail(RecoveryError::unsupported); return {}; } namespace { std::expected recover_population_loaded(Postgres& db, const FilesystemObjectStore& objects, RegionKey key, const LeaseGrant& lease, std::span retained_builds, milliseconds timeout, std::expected committed) { const auto fail = [](RecoveryError kind) { return std::unexpected(RecoveryFailure{kind, {}}); }; if (!committed) return std::unexpected( RecoveryFailure{committed.error().kind == LeaseStoreError::not_provisioned ? RecoveryError::not_provisioned : RecoveryError::store, committed.error()}); auto m = decode_manifest(committed->manifest); if (!m || !bound(*committed, *m) || m->key != key) return fail(RecoveryError::manifest); if (!supported(*m) || m->profile != CheckpointProfile::cellular_population_v1 || lease.frame != m->frame || std::ranges::find(retained_builds, m->build) == retained_builds.end()) return fail(RecoveryError::unsupported); auto bytes = objects.get({m->payload_sha256, m->payload_bytes}); if (!bytes) return fail(bytes.error() == ObjectError::io ? RecoveryError::store : bytes.error() == ObjectError::missing ? RecoveryError::missing_payload : RecoveryError::corrupt_payload); auto population = restore_population_payload(*bytes, *m); if (!population) return fail(RecoveryError::payload); if (m->batch == BatchDigests{}) { if (auto verified = verify_cellular_no_tick_checkpoint(db, objects, lease, *committed, *m, *bytes, timeout); !verified) return std::unexpected(verified.error()); } else { auto evidence = load_population_evidence(db, key, lease, m->commit, timeout); if (!evidence) return std::unexpected(RecoveryFailure{RecoveryError::store, evidence.error()}); const auto record_optional = std::move(*evidence); if (!record_optional.has_value()) return fail(RecoveryError::history); const auto& record = *record_optional; if (record.source_commit.has_value()) return fail(RecoveryError::history); auto trace_bytes = objects.get({m->batch.trace_sha256, m->batch.trace_bytes}); if (!trace_bytes || *trace_bytes != record.trace) return fail(RecoveryError::tail); PopulationPreparedCommit prepared{key, m->commit, committed->manifest, *trace_bytes}; if (!canonical(prepared, *m)) return fail(RecoveryError::tail); auto predecessor = load_region_commit(db, key, m->sequence - 1, timeout); if (!predecessor) return std::unexpected(RecoveryFailure{RecoveryError::tail, predecessor.error()}); auto previous = decode_manifest(predecessor->manifest); if (!previous || !bound(*predecessor, *previous) || previous->commit != m->predecessor_commit || predecessor->head.digest != m->predecessor_digest || previous->key != key || previous->frame != m->frame || previous->sequence + 1 != m->sequence || previous->end_tick != m->begin_tick) return fail(RecoveryError::tail); auto source_bytes = objects.get({previous->payload_sha256, previous->payload_bytes}); if (!source_bytes) return fail(RecoveryError::tail); auto base = restore_population_payload(*source_bytes, *previous); auto trace = server::decode_population_batch(*trace_bytes); if (!base || !trace) return fail(RecoveryError::tail); auto replay = server::replay_population_batch(**base, *trace); if (!replay || (*replay)->snapshot() != *bytes) return fail(RecoveryError::tail); } return RecoveredPopulationCheckpoint{std::move(*committed), *m, std::move(*population)}; } } // namespace std::expected recover_population_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key, const LeaseGrant& lease, std::span retained_builds, milliseconds timeout) { return recover_population_loaded(db, objects, key, lease, retained_builds, timeout, load_region_checkpoint(db, key, timeout)); } std::expected recover_population_commit(Postgres& db, const FilesystemObjectStore& objects, RegionKey key, std::uint64_t sequence, const LeaseGrant& lease, std::span retained_builds, milliseconds timeout) { return recover_population_loaded(db, objects, key, lease, retained_builds, timeout, load_region_commit(db, key, sequence, timeout)); } } // namespace evo::persistence