M12.3: publish and recover durable exact populations
Foundation / checks (ecs) (push) Successful in 2m41s
Foundation / checks (tsan) (push) Failing after 18m12s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (network) (push) Canceled after 20m41s
Foundation / checks (postgres) (push) Canceled after 20m43s
Foundation / checks (checks) (push) Canceled after 20m49s
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (client) (push) Failing after 17m4s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / checks (ecs) (push) Successful in 2m41s
Foundation / checks (tsan) (push) Failing after 18m12s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (network) (push) Canceled after 20m41s
Foundation / checks (postgres) (push) Canceled after 20m43s
Foundation / checks (checks) (push) Canceled after 20m49s
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (client) (push) Failing after 17m4s
Foundation / postgres-tools (push) Canceled after 0s
This commit is contained in:
@@ -0,0 +1,315 @@
|
||||
#include "evo/persistence/population_checkpoint_store.hpp"
|
||||
#include "evo/persistence/sha256.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <utility>
|
||||
|
||||
namespace evo::persistence {
|
||||
namespace {
|
||||
using std::chrono::milliseconds;
|
||||
PopulationPublicationFailure failure(PublicationError kind, ObjectError object = {},
|
||||
LeaseStoreFailure store = {},
|
||||
PopulationPreparedCommit prepared = {}) {
|
||||
return {kind, store, object, std::move(prepared)};
|
||||
}
|
||||
bool bound(const CommittedCheckpoint& checkpoint, const CheckpointManifest& m) {
|
||||
const auto digest = sha256(checkpoint.manifest);
|
||||
return digest && *digest == checkpoint.head.digest && m.commit == checkpoint.head.commit &&
|
||||
m.sequence == checkpoint.head.sequence && m.end_tick == checkpoint.head.end_tick &&
|
||||
m.frame == checkpoint.head.frame;
|
||||
}
|
||||
bool canonical(const PopulationPreparedCommit& prepared, const CheckpointManifest& m) {
|
||||
if (!supported(m) || m.key != prepared.key || m.commit != prepared.commit || m.sequence == 0 ||
|
||||
m.cellular_versions != cellular_v9_versions || !encode_manifest(m))
|
||||
return false;
|
||||
if (prepared.trace.empty())
|
||||
return (m.profile == CheckpointProfile::cellular_lives_v1 ||
|
||||
m.profile == CheckpointProfile::cellular_population_v1) &&
|
||||
m.begin_tick == m.end_tick && m.batch == BatchDigests{};
|
||||
const auto trace = server::decode_population_batch(prepared.trace);
|
||||
const auto digest = sha256(prepared.trace);
|
||||
if (!trace || !digest || m.profile != CheckpointProfile::cellular_population_v1 ||
|
||||
m.batch != BatchDigests{prepared.trace.size(), *digest, 0, {}})
|
||||
return false;
|
||||
const auto& h = trace->header;
|
||||
return h.world == m.key.world && h.region == m.key.region && h.frame == m.frame &&
|
||||
h.predecessor == m.predecessor_commit && h.predecessor_digest == m.predecessor_digest &&
|
||||
h.begin_tick == m.begin_tick && h.end_tick == m.end_tick;
|
||||
}
|
||||
std::expected<void, PopulationPublicationFailure> put(const FilesystemObjectStore& objects,
|
||||
std::span<const std::uint8_t> bytes) {
|
||||
const auto digest = sha256(bytes);
|
||||
if (!digest)
|
||||
return std::unexpected(failure(PublicationError::digest));
|
||||
auto object = objects.put(bytes);
|
||||
if (!object)
|
||||
return std::unexpected(failure(PublicationError::upload, object.error()));
|
||||
auto readback = objects.get(*object);
|
||||
if (!readback || object->bytes != bytes.size() || object->sha256 != *digest ||
|
||||
!std::ranges::equal(*readback, bytes))
|
||||
return std::unexpected(
|
||||
failure(PublicationError::verify, readback ? ObjectError::corrupt : readback.error()));
|
||||
return {};
|
||||
}
|
||||
std::expected<CommittedCheckpoint, PopulationPublicationFailure>
|
||||
transition(Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease,
|
||||
const CommittedCheckpoint& predecessor, Uuid commit, const Sha256Digest& build,
|
||||
server::CellularTierPolicy policy, bool to_population, milliseconds timeout,
|
||||
const PublishHooks& hooks) {
|
||||
auto previous = decode_manifest(predecessor.manifest);
|
||||
const auto previous_digest = sha256(predecessor.manifest);
|
||||
if (!previous || !previous_digest || !bound(predecessor, *previous) ||
|
||||
previous->profile != (to_population ? CheckpointProfile::cellular_lives_v1
|
||||
: CheckpointProfile::cellular_population_v1) ||
|
||||
previous->cellular_versions != cellular_v9_versions)
|
||||
return std::unexpected(failure(PublicationError::predecessor));
|
||||
auto source = objects.get({previous->payload_sha256, previous->payload_bytes});
|
||||
if (!source)
|
||||
return std::unexpected(failure(PublicationError::predecessor, source.error()));
|
||||
std::vector<std::uint8_t> payload;
|
||||
if (to_population) {
|
||||
auto state = restore_cellular_payload(*source, *previous);
|
||||
if (!state)
|
||||
return std::unexpected(failure(PublicationError::predecessor));
|
||||
auto population = server::CellularPopulation::create(std::move(*state), policy);
|
||||
if (!population || (*population)->state().snapshot() != *source)
|
||||
return std::unexpected(failure(PublicationError::candidate));
|
||||
payload = (*population)->snapshot();
|
||||
} else {
|
||||
auto state = restore_population_payload(*source, *previous);
|
||||
if (!state)
|
||||
return std::unexpected(failure(PublicationError::predecessor));
|
||||
payload = (*state)->state().snapshot();
|
||||
}
|
||||
const auto digest = sha256(payload);
|
||||
if (!digest)
|
||||
return std::unexpected(failure(PublicationError::digest));
|
||||
auto manifest = population_profile_transition_manifest(*previous, *previous_digest, commit,
|
||||
lease, build, payload.size(), *digest);
|
||||
if (!manifest)
|
||||
return std::unexpected(failure(PublicationError::candidate));
|
||||
auto encoded = encode_manifest(*manifest);
|
||||
if (!encoded)
|
||||
return std::unexpected(failure(PublicationError::candidate));
|
||||
if (auto uploaded = put(objects, payload); !uploaded)
|
||||
return std::unexpected(uploaded.error());
|
||||
if (hooks.before_transaction)
|
||||
hooks.before_transaction();
|
||||
return retry_population_prepared(db, {previous->key, commit, std::move(*encoded), {}}, timeout);
|
||||
}
|
||||
} // namespace
|
||||
|
||||
std::expected<CommittedCheckpoint, PopulationPublicationFailure> migrate_cellular_to_population(
|
||||
Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease,
|
||||
const CommittedCheckpoint& predecessor, Uuid commit, const Sha256Digest& build,
|
||||
server::CellularTierPolicy policy, milliseconds timeout, const PublishHooks& hooks) {
|
||||
return transition(db, objects, lease, predecessor, commit, build, policy, true, timeout, hooks);
|
||||
}
|
||||
std::expected<CommittedCheckpoint, PopulationPublicationFailure>
|
||||
materialize_population_profile(Postgres& db, const FilesystemObjectStore& objects,
|
||||
const LeaseGrant& lease, const CommittedCheckpoint& predecessor,
|
||||
Uuid commit, const Sha256Digest& build, milliseconds timeout,
|
||||
const PublishHooks& hooks) {
|
||||
return transition(db, objects, lease, predecessor, commit, build, {}, false, timeout, hooks);
|
||||
}
|
||||
std::expected<CommittedCheckpoint, PopulationPublicationFailure>
|
||||
retry_population_prepared(Postgres& db, const PopulationPreparedCommit& prepared,
|
||||
milliseconds timeout) {
|
||||
const auto m = decode_manifest(prepared.manifest);
|
||||
if (!m || !canonical(prepared, *m))
|
||||
return std::unexpected(failure(PublicationError::candidate));
|
||||
auto result =
|
||||
prepared.trace.empty()
|
||||
? publish_population_profile_transition(db, prepared.manifest, timeout)
|
||||
: publish_population_checkpoint(db, prepared.manifest, prepared.trace, timeout);
|
||||
if (!result)
|
||||
return std::unexpected(failure(result.error().kind == LeaseStoreError::unknown
|
||||
? PublicationError::unknown
|
||||
: PublicationError::rejected,
|
||||
{}, result.error(), prepared));
|
||||
const auto digest = sha256(prepared.manifest);
|
||||
if (!digest || result->digest != *digest || result->commit != m->commit ||
|
||||
result->sequence != m->sequence || result->end_tick != m->end_tick ||
|
||||
result->frame != m->frame)
|
||||
return std::unexpected(failure(PublicationError::receipt));
|
||||
return CommittedCheckpoint{*result, prepared.manifest};
|
||||
}
|
||||
std::expected<ResolvedCommit, LeaseStoreFailure>
|
||||
resolve_population_prepared(Postgres& db, const PopulationPreparedCommit& prepared,
|
||||
milliseconds timeout) {
|
||||
const auto manifest = decode_manifest(prepared.manifest);
|
||||
if (!manifest || !canonical(prepared, *manifest))
|
||||
return std::unexpected(LeaseStoreFailure{LeaseStoreError::invalid, {}});
|
||||
return resolve_population_commit(db, prepared.key, prepared.commit, prepared.manifest,
|
||||
prepared.trace, timeout);
|
||||
}
|
||||
std::expected<CommittedCheckpoint, PopulationPublicationFailure> publish_population_batch(
|
||||
Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease,
|
||||
const CommittedCheckpoint& predecessor, const server::CellularPopulationBatch& batch,
|
||||
const server::CellularPopulation& candidate, Uuid commit, const Sha256Digest& build,
|
||||
milliseconds timeout, const PublishHooks& hooks) {
|
||||
const auto previous = decode_manifest(predecessor.manifest);
|
||||
const auto previous_digest = sha256(predecessor.manifest);
|
||||
const auto trace = server::encode_population_batch(batch);
|
||||
if (!previous || !previous_digest || !trace || !bound(predecessor, *previous) ||
|
||||
previous->profile != CheckpointProfile::cellular_population_v1)
|
||||
return std::unexpected(failure(PublicationError::predecessor));
|
||||
auto source = objects.get({previous->payload_sha256, previous->payload_bytes});
|
||||
if (!source)
|
||||
return std::unexpected(failure(PublicationError::predecessor, source.error()));
|
||||
auto base = restore_population_payload(*source, *previous);
|
||||
if (!base)
|
||||
return std::unexpected(failure(PublicationError::predecessor));
|
||||
auto replay = server::replay_population_batch(**base, batch);
|
||||
const auto payload = candidate.snapshot();
|
||||
if (!replay || (*replay)->snapshot() != payload)
|
||||
return std::unexpected(failure(PublicationError::candidate));
|
||||
const auto payload_digest = sha256(payload), trace_digest = sha256(*trace);
|
||||
if (!payload_digest || !trace_digest)
|
||||
return std::unexpected(failure(PublicationError::digest));
|
||||
auto m = population_successor_manifest(*previous, *previous_digest, commit, lease,
|
||||
batch.header.end_tick, build, payload.size(),
|
||||
*payload_digest, {trace->size(), *trace_digest, 0, {}});
|
||||
if (!m)
|
||||
return std::unexpected(failure(PublicationError::candidate));
|
||||
auto encoded = encode_manifest(*m);
|
||||
if (!encoded)
|
||||
return std::unexpected(failure(PublicationError::candidate));
|
||||
PopulationPreparedCommit prepared{previous->key, commit, std::move(*encoded), *trace};
|
||||
if (!canonical(prepared, *m))
|
||||
return std::unexpected(failure(PublicationError::candidate));
|
||||
if (auto uploaded = put(objects, payload); !uploaded)
|
||||
return std::unexpected(uploaded.error());
|
||||
if (auto uploaded = put(objects, *trace); !uploaded)
|
||||
return std::unexpected(uploaded.error());
|
||||
if (hooks.before_transaction)
|
||||
hooks.before_transaction();
|
||||
return retry_population_prepared(db, prepared, timeout);
|
||||
}
|
||||
|
||||
std::expected<void, RecoveryFailure>
|
||||
verify_population_profile_transition(Postgres& db, const FilesystemObjectStore& objects,
|
||||
const LeaseGrant& lease, const CommittedCheckpoint& committed,
|
||||
const CheckpointManifest& m,
|
||||
std::span<const std::uint8_t> payload, milliseconds timeout) {
|
||||
const auto fail = [](RecoveryError kind) { return std::unexpected(RecoveryFailure{kind, {}}); };
|
||||
if (!bound(committed, m) || m.sequence == 0 || m.begin_tick != m.end_tick ||
|
||||
m.batch != BatchDigests{})
|
||||
return fail(RecoveryError::manifest);
|
||||
auto evidence = load_population_evidence(db, m.key, lease, m.commit, timeout);
|
||||
if (!evidence)
|
||||
return std::unexpected(RecoveryFailure{RecoveryError::store, evidence.error()});
|
||||
const auto record_optional = std::move(*evidence);
|
||||
if (!record_optional.has_value())
|
||||
return fail(RecoveryError::history);
|
||||
const auto& record = *record_optional;
|
||||
if (!record.trace.empty() || !record.source_commit.has_value())
|
||||
return fail(RecoveryError::history);
|
||||
auto predecessor = load_region_commit(db, m.key, m.sequence - 1, timeout);
|
||||
if (!predecessor)
|
||||
return std::unexpected(RecoveryFailure{RecoveryError::tail, predecessor.error()});
|
||||
auto source = decode_manifest(predecessor->manifest);
|
||||
if (!source || !bound(*predecessor, *source) || !supported(*source) ||
|
||||
source->commit != m.predecessor_commit ||
|
||||
predecessor->head.digest != m.predecessor_digest || source->key != m.key ||
|
||||
source->frame != m.frame || source->end_tick != m.end_tick ||
|
||||
source->cellular_versions != cellular_v9_versions ||
|
||||
m.cellular_versions != cellular_v9_versions || record.source_commit != source->commit ||
|
||||
record.source_manifest_sha256 != predecessor->head.digest ||
|
||||
record.source_payload_sha256 != source->payload_sha256)
|
||||
return fail(RecoveryError::tail);
|
||||
auto bytes = objects.get({source->payload_sha256, source->payload_bytes});
|
||||
if (!bytes)
|
||||
return fail(RecoveryError::tail);
|
||||
if (m.profile == CheckpointProfile::cellular_population_v1 &&
|
||||
source->profile == CheckpointProfile::cellular_lives_v1) {
|
||||
auto raw = restore_cellular_payload(*bytes, *source);
|
||||
auto population = restore_population_payload(payload, m);
|
||||
if (!raw || !population)
|
||||
return fail(RecoveryError::payload);
|
||||
auto expected =
|
||||
server::CellularPopulation::create(std::move(*raw), (*population)->policy());
|
||||
if (!expected || !std::ranges::equal((*expected)->snapshot(), payload))
|
||||
return fail(RecoveryError::history);
|
||||
} else if (m.profile == CheckpointProfile::cellular_lives_v1 &&
|
||||
source->profile == CheckpointProfile::cellular_population_v1) {
|
||||
auto population = restore_population_payload(*bytes, *source);
|
||||
if (!population || !restore_cellular_payload(payload, m) ||
|
||||
!std::ranges::equal((*population)->state().snapshot(), payload))
|
||||
return fail(RecoveryError::history);
|
||||
} else
|
||||
return fail(RecoveryError::unsupported);
|
||||
return {};
|
||||
}
|
||||
std::expected<RecoveredPopulationCheckpoint, RecoveryFailure>
|
||||
recover_population_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
const LeaseGrant& lease,
|
||||
std::span<const Sha256Digest> retained_builds, milliseconds timeout) {
|
||||
const auto fail = [](RecoveryError kind) { return std::unexpected(RecoveryFailure{kind, {}}); };
|
||||
auto committed = load_region_checkpoint(db, key, timeout);
|
||||
if (!committed)
|
||||
return std::unexpected(
|
||||
RecoveryFailure{committed.error().kind == LeaseStoreError::not_provisioned
|
||||
? RecoveryError::not_provisioned
|
||||
: RecoveryError::store,
|
||||
committed.error()});
|
||||
auto m = decode_manifest(committed->manifest);
|
||||
if (!m || !bound(*committed, *m) || m->key != key)
|
||||
return fail(RecoveryError::manifest);
|
||||
if (!supported(*m) || m->profile != CheckpointProfile::cellular_population_v1 ||
|
||||
lease.frame != m->frame ||
|
||||
std::ranges::find(retained_builds, m->build) == retained_builds.end())
|
||||
return fail(RecoveryError::unsupported);
|
||||
auto bytes = objects.get({m->payload_sha256, m->payload_bytes});
|
||||
if (!bytes)
|
||||
return fail(bytes.error() == ObjectError::io ? RecoveryError::store
|
||||
: bytes.error() == ObjectError::missing ? RecoveryError::missing_payload
|
||||
: RecoveryError::corrupt_payload);
|
||||
auto population = restore_population_payload(*bytes, *m);
|
||||
if (!population)
|
||||
return fail(RecoveryError::payload);
|
||||
if (m->batch == BatchDigests{}) {
|
||||
if (auto verified = verify_population_profile_transition(db, objects, lease, *committed, *m,
|
||||
*bytes, timeout);
|
||||
!verified)
|
||||
return std::unexpected(verified.error());
|
||||
} else {
|
||||
auto evidence = load_population_evidence(db, key, lease, m->commit, timeout);
|
||||
if (!evidence)
|
||||
return std::unexpected(RecoveryFailure{RecoveryError::store, evidence.error()});
|
||||
const auto record_optional = std::move(*evidence);
|
||||
if (!record_optional.has_value())
|
||||
return fail(RecoveryError::history);
|
||||
const auto& record = *record_optional;
|
||||
if (record.source_commit.has_value())
|
||||
return fail(RecoveryError::history);
|
||||
auto trace_bytes = objects.get({m->batch.trace_sha256, m->batch.trace_bytes});
|
||||
if (!trace_bytes || *trace_bytes != record.trace)
|
||||
return fail(RecoveryError::tail);
|
||||
PopulationPreparedCommit prepared{key, m->commit, committed->manifest, *trace_bytes};
|
||||
if (!canonical(prepared, *m))
|
||||
return fail(RecoveryError::tail);
|
||||
auto predecessor = load_region_commit(db, key, m->sequence - 1, timeout);
|
||||
if (!predecessor)
|
||||
return std::unexpected(RecoveryFailure{RecoveryError::tail, predecessor.error()});
|
||||
auto previous = decode_manifest(predecessor->manifest);
|
||||
if (!previous || !bound(*predecessor, *previous) ||
|
||||
previous->commit != m->predecessor_commit ||
|
||||
predecessor->head.digest != m->predecessor_digest || previous->key != key ||
|
||||
previous->frame != m->frame || previous->sequence + 1 != m->sequence ||
|
||||
previous->end_tick != m->begin_tick)
|
||||
return fail(RecoveryError::tail);
|
||||
auto source_bytes = objects.get({previous->payload_sha256, previous->payload_bytes});
|
||||
if (!source_bytes)
|
||||
return fail(RecoveryError::tail);
|
||||
auto base = restore_population_payload(*source_bytes, *previous);
|
||||
auto trace = server::decode_population_batch(*trace_bytes);
|
||||
if (!base || !trace)
|
||||
return fail(RecoveryError::tail);
|
||||
auto replay = server::replay_population_batch(**base, *trace);
|
||||
if (!replay || (*replay)->snapshot() != *bytes)
|
||||
return fail(RecoveryError::tail);
|
||||
}
|
||||
return RecoveredPopulationCheckpoint{std::move(*committed), *m, std::move(*population)};
|
||||
}
|
||||
} // namespace evo::persistence
|
||||
Reference in New Issue
Block a user