M12.3: publish and recover durable exact populations
Foundation / checks (ecs) (push) Successful in 2m41s
Foundation / checks (tsan) (push) Failing after 18m12s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (network) (push) Canceled after 20m41s
Foundation / checks (postgres) (push) Canceled after 20m43s
Foundation / checks (checks) (push) Canceled after 20m49s
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (client) (push) Failing after 17m4s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / checks (ecs) (push) Successful in 2m41s
Foundation / checks (tsan) (push) Failing after 18m12s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (network) (push) Canceled after 20m41s
Foundation / checks (postgres) (push) Canceled after 20m43s
Foundation / checks (checks) (push) Canceled after 20m49s
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (client) (push) Failing after 17m4s
Foundation / postgres-tools (push) Canceled after 0s
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
#include "evo/persistence/checkpoint_store.hpp"
|
||||
#include "evo/persistence/population_checkpoint_store.hpp"
|
||||
#include "evo/persistence/sha256.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
@@ -537,7 +538,15 @@ recover_cellular_checkpoint(Postgres& db, const FilesystemObjectStore& objects,
|
||||
std::vector<server::CellularActionRecord> actions;
|
||||
std::vector<server::CellularMutationActionRecord> mutation_actions_recovered;
|
||||
std::vector<server::CellularMutationResult> mutation_results_recovered;
|
||||
if (manifest->sequence != 0 && manifest->begin_tick == manifest->end_tick) {
|
||||
if (manifest->sequence != 0 && manifest->begin_tick == manifest->end_tick &&
|
||||
manifest->cellular_versions == cellular_v9_versions) {
|
||||
auto verified = verify_population_profile_transition(db, objects, lease, *committed,
|
||||
*manifest, *payload, timeout);
|
||||
if (!verified)
|
||||
return std::unexpected(verified.error());
|
||||
return RecoveredCellularCheckpoint{
|
||||
std::move(*committed), *manifest, std::move(*state), {}, {}, {}};
|
||||
} else if (manifest->sequence != 0 && manifest->begin_tick == manifest->end_tick) {
|
||||
auto migration = load_cellular_profile_migration(db, key, lease, manifest->commit, timeout);
|
||||
if (!migration)
|
||||
return std::unexpected{RecoveryFailure{RecoveryError::store, migration.error()}};
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
#include "evo/persistence/region_checkpoint.hpp"
|
||||
|
||||
namespace evo::persistence {
|
||||
namespace {
|
||||
std::expected<CheckpointManifest, CheckpointError>
|
||||
next_population(const CheckpointManifest& previous, const Sha256Digest& previous_digest,
|
||||
Uuid commit, const LeaseGrant& lease, std::uint64_t end_tick,
|
||||
const Sha256Digest& build, std::uint64_t bytes, const Sha256Digest& digest,
|
||||
BatchDigests batch, bool transition) {
|
||||
if (!supported(previous) || !encode_manifest(previous) || previous.sequence == 0 ||
|
||||
previous.sequence == ~std::uint64_t{0} || previous.end_tick == 0 ||
|
||||
previous.cellular_versions != cellular_v9_versions || previous_digest == Sha256Digest{} ||
|
||||
lease.frame != previous.frame || lease.epoch == 0 || lease.holder == Uuid{} ||
|
||||
commit == Uuid{} || commit == previous.commit || end_tick < previous.end_tick ||
|
||||
end_tick - previous.end_tick > max_checkpoint_ticks ||
|
||||
(previous.profile != CheckpointProfile::cellular_lives_v1 &&
|
||||
previous.profile != CheckpointProfile::cellular_population_v1) ||
|
||||
(!transition && previous.profile != CheckpointProfile::cellular_population_v1))
|
||||
return std::unexpected(CheckpointError::chain);
|
||||
auto next = previous;
|
||||
if (transition) {
|
||||
next.profile = previous.profile == CheckpointProfile::cellular_lives_v1
|
||||
? CheckpointProfile::cellular_population_v1
|
||||
: CheckpointProfile::cellular_lives_v1;
|
||||
next.history_profile = next.profile == CheckpointProfile::cellular_population_v1
|
||||
? population_history_profile
|
||||
: cellular_revision_history_profile;
|
||||
}
|
||||
next.commit = commit;
|
||||
++next.sequence;
|
||||
next.predecessor_commit = previous.commit;
|
||||
next.predecessor_digest = previous_digest;
|
||||
next.begin_tick = previous.end_tick;
|
||||
next.end_tick = end_tick;
|
||||
next.holder = lease.holder;
|
||||
next.epoch = lease.epoch;
|
||||
next.build = build;
|
||||
next.payload_bytes = bytes;
|
||||
next.payload_sha256 = digest;
|
||||
next.batch = batch;
|
||||
if (!encode_manifest(next))
|
||||
return std::unexpected(CheckpointError::format);
|
||||
return next;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
std::expected<CheckpointManifest, CheckpointError>
|
||||
population_profile_transition_manifest(const CheckpointManifest& previous,
|
||||
const Sha256Digest& previous_digest, Uuid commit,
|
||||
const LeaseGrant& lease, const Sha256Digest& build,
|
||||
std::uint64_t bytes, const Sha256Digest& digest) {
|
||||
return next_population(previous, previous_digest, commit, lease, previous.end_tick, build,
|
||||
bytes, digest, {}, true);
|
||||
}
|
||||
std::expected<CheckpointManifest, CheckpointError> population_successor_manifest(
|
||||
const CheckpointManifest& previous, const Sha256Digest& previous_digest, Uuid commit,
|
||||
const LeaseGrant& lease, std::uint64_t end_tick, const Sha256Digest& build, std::uint64_t bytes,
|
||||
const Sha256Digest& digest, const BatchDigests& batch) {
|
||||
if (batch.trace_bytes == 0)
|
||||
return std::unexpected(CheckpointError::chain);
|
||||
return next_population(previous, previous_digest, commit, lease, end_tick, build, bytes, digest,
|
||||
batch, false);
|
||||
}
|
||||
std::expected<std::unique_ptr<server::CellularPopulation>, CheckpointError>
|
||||
restore_population_payload(std::span<const std::uint8_t> payload, const CheckpointManifest& m) {
|
||||
if (!supported(m) || m.profile != CheckpointProfile::cellular_population_v1)
|
||||
return std::unexpected(CheckpointError::unsupported);
|
||||
if (!encode_manifest(m) || payload.size() != m.payload_bytes)
|
||||
return std::unexpected(CheckpointError::format);
|
||||
auto state = server::CellularPopulation::restore(payload, m.key.world, m.key.region, m.frame,
|
||||
m.end_tick);
|
||||
if (!state)
|
||||
return std::unexpected(state.error() == server::CellularPopulationError::unsupported
|
||||
? CheckpointError::unsupported
|
||||
: CheckpointError::payload);
|
||||
return std::move(*state);
|
||||
}
|
||||
} // namespace evo::persistence
|
||||
@@ -0,0 +1,315 @@
|
||||
#include "evo/persistence/population_checkpoint_store.hpp"
|
||||
#include "evo/persistence/sha256.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <utility>
|
||||
|
||||
namespace evo::persistence {
|
||||
namespace {
|
||||
using std::chrono::milliseconds;
|
||||
PopulationPublicationFailure failure(PublicationError kind, ObjectError object = {},
|
||||
LeaseStoreFailure store = {},
|
||||
PopulationPreparedCommit prepared = {}) {
|
||||
return {kind, store, object, std::move(prepared)};
|
||||
}
|
||||
bool bound(const CommittedCheckpoint& checkpoint, const CheckpointManifest& m) {
|
||||
const auto digest = sha256(checkpoint.manifest);
|
||||
return digest && *digest == checkpoint.head.digest && m.commit == checkpoint.head.commit &&
|
||||
m.sequence == checkpoint.head.sequence && m.end_tick == checkpoint.head.end_tick &&
|
||||
m.frame == checkpoint.head.frame;
|
||||
}
|
||||
bool canonical(const PopulationPreparedCommit& prepared, const CheckpointManifest& m) {
|
||||
if (!supported(m) || m.key != prepared.key || m.commit != prepared.commit || m.sequence == 0 ||
|
||||
m.cellular_versions != cellular_v9_versions || !encode_manifest(m))
|
||||
return false;
|
||||
if (prepared.trace.empty())
|
||||
return (m.profile == CheckpointProfile::cellular_lives_v1 ||
|
||||
m.profile == CheckpointProfile::cellular_population_v1) &&
|
||||
m.begin_tick == m.end_tick && m.batch == BatchDigests{};
|
||||
const auto trace = server::decode_population_batch(prepared.trace);
|
||||
const auto digest = sha256(prepared.trace);
|
||||
if (!trace || !digest || m.profile != CheckpointProfile::cellular_population_v1 ||
|
||||
m.batch != BatchDigests{prepared.trace.size(), *digest, 0, {}})
|
||||
return false;
|
||||
const auto& h = trace->header;
|
||||
return h.world == m.key.world && h.region == m.key.region && h.frame == m.frame &&
|
||||
h.predecessor == m.predecessor_commit && h.predecessor_digest == m.predecessor_digest &&
|
||||
h.begin_tick == m.begin_tick && h.end_tick == m.end_tick;
|
||||
}
|
||||
std::expected<void, PopulationPublicationFailure> put(const FilesystemObjectStore& objects,
|
||||
std::span<const std::uint8_t> bytes) {
|
||||
const auto digest = sha256(bytes);
|
||||
if (!digest)
|
||||
return std::unexpected(failure(PublicationError::digest));
|
||||
auto object = objects.put(bytes);
|
||||
if (!object)
|
||||
return std::unexpected(failure(PublicationError::upload, object.error()));
|
||||
auto readback = objects.get(*object);
|
||||
if (!readback || object->bytes != bytes.size() || object->sha256 != *digest ||
|
||||
!std::ranges::equal(*readback, bytes))
|
||||
return std::unexpected(
|
||||
failure(PublicationError::verify, readback ? ObjectError::corrupt : readback.error()));
|
||||
return {};
|
||||
}
|
||||
std::expected<CommittedCheckpoint, PopulationPublicationFailure>
|
||||
transition(Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease,
|
||||
const CommittedCheckpoint& predecessor, Uuid commit, const Sha256Digest& build,
|
||||
server::CellularTierPolicy policy, bool to_population, milliseconds timeout,
|
||||
const PublishHooks& hooks) {
|
||||
auto previous = decode_manifest(predecessor.manifest);
|
||||
const auto previous_digest = sha256(predecessor.manifest);
|
||||
if (!previous || !previous_digest || !bound(predecessor, *previous) ||
|
||||
previous->profile != (to_population ? CheckpointProfile::cellular_lives_v1
|
||||
: CheckpointProfile::cellular_population_v1) ||
|
||||
previous->cellular_versions != cellular_v9_versions)
|
||||
return std::unexpected(failure(PublicationError::predecessor));
|
||||
auto source = objects.get({previous->payload_sha256, previous->payload_bytes});
|
||||
if (!source)
|
||||
return std::unexpected(failure(PublicationError::predecessor, source.error()));
|
||||
std::vector<std::uint8_t> payload;
|
||||
if (to_population) {
|
||||
auto state = restore_cellular_payload(*source, *previous);
|
||||
if (!state)
|
||||
return std::unexpected(failure(PublicationError::predecessor));
|
||||
auto population = server::CellularPopulation::create(std::move(*state), policy);
|
||||
if (!population || (*population)->state().snapshot() != *source)
|
||||
return std::unexpected(failure(PublicationError::candidate));
|
||||
payload = (*population)->snapshot();
|
||||
} else {
|
||||
auto state = restore_population_payload(*source, *previous);
|
||||
if (!state)
|
||||
return std::unexpected(failure(PublicationError::predecessor));
|
||||
payload = (*state)->state().snapshot();
|
||||
}
|
||||
const auto digest = sha256(payload);
|
||||
if (!digest)
|
||||
return std::unexpected(failure(PublicationError::digest));
|
||||
auto manifest = population_profile_transition_manifest(*previous, *previous_digest, commit,
|
||||
lease, build, payload.size(), *digest);
|
||||
if (!manifest)
|
||||
return std::unexpected(failure(PublicationError::candidate));
|
||||
auto encoded = encode_manifest(*manifest);
|
||||
if (!encoded)
|
||||
return std::unexpected(failure(PublicationError::candidate));
|
||||
if (auto uploaded = put(objects, payload); !uploaded)
|
||||
return std::unexpected(uploaded.error());
|
||||
if (hooks.before_transaction)
|
||||
hooks.before_transaction();
|
||||
return retry_population_prepared(db, {previous->key, commit, std::move(*encoded), {}}, timeout);
|
||||
}
|
||||
} // namespace
|
||||
|
||||
std::expected<CommittedCheckpoint, PopulationPublicationFailure> migrate_cellular_to_population(
|
||||
Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease,
|
||||
const CommittedCheckpoint& predecessor, Uuid commit, const Sha256Digest& build,
|
||||
server::CellularTierPolicy policy, milliseconds timeout, const PublishHooks& hooks) {
|
||||
return transition(db, objects, lease, predecessor, commit, build, policy, true, timeout, hooks);
|
||||
}
|
||||
std::expected<CommittedCheckpoint, PopulationPublicationFailure>
|
||||
materialize_population_profile(Postgres& db, const FilesystemObjectStore& objects,
|
||||
const LeaseGrant& lease, const CommittedCheckpoint& predecessor,
|
||||
Uuid commit, const Sha256Digest& build, milliseconds timeout,
|
||||
const PublishHooks& hooks) {
|
||||
return transition(db, objects, lease, predecessor, commit, build, {}, false, timeout, hooks);
|
||||
}
|
||||
std::expected<CommittedCheckpoint, PopulationPublicationFailure>
|
||||
retry_population_prepared(Postgres& db, const PopulationPreparedCommit& prepared,
|
||||
milliseconds timeout) {
|
||||
const auto m = decode_manifest(prepared.manifest);
|
||||
if (!m || !canonical(prepared, *m))
|
||||
return std::unexpected(failure(PublicationError::candidate));
|
||||
auto result =
|
||||
prepared.trace.empty()
|
||||
? publish_population_profile_transition(db, prepared.manifest, timeout)
|
||||
: publish_population_checkpoint(db, prepared.manifest, prepared.trace, timeout);
|
||||
if (!result)
|
||||
return std::unexpected(failure(result.error().kind == LeaseStoreError::unknown
|
||||
? PublicationError::unknown
|
||||
: PublicationError::rejected,
|
||||
{}, result.error(), prepared));
|
||||
const auto digest = sha256(prepared.manifest);
|
||||
if (!digest || result->digest != *digest || result->commit != m->commit ||
|
||||
result->sequence != m->sequence || result->end_tick != m->end_tick ||
|
||||
result->frame != m->frame)
|
||||
return std::unexpected(failure(PublicationError::receipt));
|
||||
return CommittedCheckpoint{*result, prepared.manifest};
|
||||
}
|
||||
std::expected<ResolvedCommit, LeaseStoreFailure>
|
||||
resolve_population_prepared(Postgres& db, const PopulationPreparedCommit& prepared,
|
||||
milliseconds timeout) {
|
||||
const auto manifest = decode_manifest(prepared.manifest);
|
||||
if (!manifest || !canonical(prepared, *manifest))
|
||||
return std::unexpected(LeaseStoreFailure{LeaseStoreError::invalid, {}});
|
||||
return resolve_population_commit(db, prepared.key, prepared.commit, prepared.manifest,
|
||||
prepared.trace, timeout);
|
||||
}
|
||||
std::expected<CommittedCheckpoint, PopulationPublicationFailure> publish_population_batch(
|
||||
Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease,
|
||||
const CommittedCheckpoint& predecessor, const server::CellularPopulationBatch& batch,
|
||||
const server::CellularPopulation& candidate, Uuid commit, const Sha256Digest& build,
|
||||
milliseconds timeout, const PublishHooks& hooks) {
|
||||
const auto previous = decode_manifest(predecessor.manifest);
|
||||
const auto previous_digest = sha256(predecessor.manifest);
|
||||
const auto trace = server::encode_population_batch(batch);
|
||||
if (!previous || !previous_digest || !trace || !bound(predecessor, *previous) ||
|
||||
previous->profile != CheckpointProfile::cellular_population_v1)
|
||||
return std::unexpected(failure(PublicationError::predecessor));
|
||||
auto source = objects.get({previous->payload_sha256, previous->payload_bytes});
|
||||
if (!source)
|
||||
return std::unexpected(failure(PublicationError::predecessor, source.error()));
|
||||
auto base = restore_population_payload(*source, *previous);
|
||||
if (!base)
|
||||
return std::unexpected(failure(PublicationError::predecessor));
|
||||
auto replay = server::replay_population_batch(**base, batch);
|
||||
const auto payload = candidate.snapshot();
|
||||
if (!replay || (*replay)->snapshot() != payload)
|
||||
return std::unexpected(failure(PublicationError::candidate));
|
||||
const auto payload_digest = sha256(payload), trace_digest = sha256(*trace);
|
||||
if (!payload_digest || !trace_digest)
|
||||
return std::unexpected(failure(PublicationError::digest));
|
||||
auto m = population_successor_manifest(*previous, *previous_digest, commit, lease,
|
||||
batch.header.end_tick, build, payload.size(),
|
||||
*payload_digest, {trace->size(), *trace_digest, 0, {}});
|
||||
if (!m)
|
||||
return std::unexpected(failure(PublicationError::candidate));
|
||||
auto encoded = encode_manifest(*m);
|
||||
if (!encoded)
|
||||
return std::unexpected(failure(PublicationError::candidate));
|
||||
PopulationPreparedCommit prepared{previous->key, commit, std::move(*encoded), *trace};
|
||||
if (!canonical(prepared, *m))
|
||||
return std::unexpected(failure(PublicationError::candidate));
|
||||
if (auto uploaded = put(objects, payload); !uploaded)
|
||||
return std::unexpected(uploaded.error());
|
||||
if (auto uploaded = put(objects, *trace); !uploaded)
|
||||
return std::unexpected(uploaded.error());
|
||||
if (hooks.before_transaction)
|
||||
hooks.before_transaction();
|
||||
return retry_population_prepared(db, prepared, timeout);
|
||||
}
|
||||
|
||||
std::expected<void, RecoveryFailure>
|
||||
verify_population_profile_transition(Postgres& db, const FilesystemObjectStore& objects,
|
||||
const LeaseGrant& lease, const CommittedCheckpoint& committed,
|
||||
const CheckpointManifest& m,
|
||||
std::span<const std::uint8_t> payload, milliseconds timeout) {
|
||||
const auto fail = [](RecoveryError kind) { return std::unexpected(RecoveryFailure{kind, {}}); };
|
||||
if (!bound(committed, m) || m.sequence == 0 || m.begin_tick != m.end_tick ||
|
||||
m.batch != BatchDigests{})
|
||||
return fail(RecoveryError::manifest);
|
||||
auto evidence = load_population_evidence(db, m.key, lease, m.commit, timeout);
|
||||
if (!evidence)
|
||||
return std::unexpected(RecoveryFailure{RecoveryError::store, evidence.error()});
|
||||
const auto record_optional = std::move(*evidence);
|
||||
if (!record_optional.has_value())
|
||||
return fail(RecoveryError::history);
|
||||
const auto& record = *record_optional;
|
||||
if (!record.trace.empty() || !record.source_commit.has_value())
|
||||
return fail(RecoveryError::history);
|
||||
auto predecessor = load_region_commit(db, m.key, m.sequence - 1, timeout);
|
||||
if (!predecessor)
|
||||
return std::unexpected(RecoveryFailure{RecoveryError::tail, predecessor.error()});
|
||||
auto source = decode_manifest(predecessor->manifest);
|
||||
if (!source || !bound(*predecessor, *source) || !supported(*source) ||
|
||||
source->commit != m.predecessor_commit ||
|
||||
predecessor->head.digest != m.predecessor_digest || source->key != m.key ||
|
||||
source->frame != m.frame || source->end_tick != m.end_tick ||
|
||||
source->cellular_versions != cellular_v9_versions ||
|
||||
m.cellular_versions != cellular_v9_versions || record.source_commit != source->commit ||
|
||||
record.source_manifest_sha256 != predecessor->head.digest ||
|
||||
record.source_payload_sha256 != source->payload_sha256)
|
||||
return fail(RecoveryError::tail);
|
||||
auto bytes = objects.get({source->payload_sha256, source->payload_bytes});
|
||||
if (!bytes)
|
||||
return fail(RecoveryError::tail);
|
||||
if (m.profile == CheckpointProfile::cellular_population_v1 &&
|
||||
source->profile == CheckpointProfile::cellular_lives_v1) {
|
||||
auto raw = restore_cellular_payload(*bytes, *source);
|
||||
auto population = restore_population_payload(payload, m);
|
||||
if (!raw || !population)
|
||||
return fail(RecoveryError::payload);
|
||||
auto expected =
|
||||
server::CellularPopulation::create(std::move(*raw), (*population)->policy());
|
||||
if (!expected || !std::ranges::equal((*expected)->snapshot(), payload))
|
||||
return fail(RecoveryError::history);
|
||||
} else if (m.profile == CheckpointProfile::cellular_lives_v1 &&
|
||||
source->profile == CheckpointProfile::cellular_population_v1) {
|
||||
auto population = restore_population_payload(*bytes, *source);
|
||||
if (!population || !restore_cellular_payload(payload, m) ||
|
||||
!std::ranges::equal((*population)->state().snapshot(), payload))
|
||||
return fail(RecoveryError::history);
|
||||
} else
|
||||
return fail(RecoveryError::unsupported);
|
||||
return {};
|
||||
}
|
||||
std::expected<RecoveredPopulationCheckpoint, RecoveryFailure>
|
||||
recover_population_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
const LeaseGrant& lease,
|
||||
std::span<const Sha256Digest> retained_builds, milliseconds timeout) {
|
||||
const auto fail = [](RecoveryError kind) { return std::unexpected(RecoveryFailure{kind, {}}); };
|
||||
auto committed = load_region_checkpoint(db, key, timeout);
|
||||
if (!committed)
|
||||
return std::unexpected(
|
||||
RecoveryFailure{committed.error().kind == LeaseStoreError::not_provisioned
|
||||
? RecoveryError::not_provisioned
|
||||
: RecoveryError::store,
|
||||
committed.error()});
|
||||
auto m = decode_manifest(committed->manifest);
|
||||
if (!m || !bound(*committed, *m) || m->key != key)
|
||||
return fail(RecoveryError::manifest);
|
||||
if (!supported(*m) || m->profile != CheckpointProfile::cellular_population_v1 ||
|
||||
lease.frame != m->frame ||
|
||||
std::ranges::find(retained_builds, m->build) == retained_builds.end())
|
||||
return fail(RecoveryError::unsupported);
|
||||
auto bytes = objects.get({m->payload_sha256, m->payload_bytes});
|
||||
if (!bytes)
|
||||
return fail(bytes.error() == ObjectError::io ? RecoveryError::store
|
||||
: bytes.error() == ObjectError::missing ? RecoveryError::missing_payload
|
||||
: RecoveryError::corrupt_payload);
|
||||
auto population = restore_population_payload(*bytes, *m);
|
||||
if (!population)
|
||||
return fail(RecoveryError::payload);
|
||||
if (m->batch == BatchDigests{}) {
|
||||
if (auto verified = verify_population_profile_transition(db, objects, lease, *committed, *m,
|
||||
*bytes, timeout);
|
||||
!verified)
|
||||
return std::unexpected(verified.error());
|
||||
} else {
|
||||
auto evidence = load_population_evidence(db, key, lease, m->commit, timeout);
|
||||
if (!evidence)
|
||||
return std::unexpected(RecoveryFailure{RecoveryError::store, evidence.error()});
|
||||
const auto record_optional = std::move(*evidence);
|
||||
if (!record_optional.has_value())
|
||||
return fail(RecoveryError::history);
|
||||
const auto& record = *record_optional;
|
||||
if (record.source_commit.has_value())
|
||||
return fail(RecoveryError::history);
|
||||
auto trace_bytes = objects.get({m->batch.trace_sha256, m->batch.trace_bytes});
|
||||
if (!trace_bytes || *trace_bytes != record.trace)
|
||||
return fail(RecoveryError::tail);
|
||||
PopulationPreparedCommit prepared{key, m->commit, committed->manifest, *trace_bytes};
|
||||
if (!canonical(prepared, *m))
|
||||
return fail(RecoveryError::tail);
|
||||
auto predecessor = load_region_commit(db, key, m->sequence - 1, timeout);
|
||||
if (!predecessor)
|
||||
return std::unexpected(RecoveryFailure{RecoveryError::tail, predecessor.error()});
|
||||
auto previous = decode_manifest(predecessor->manifest);
|
||||
if (!previous || !bound(*predecessor, *previous) ||
|
||||
previous->commit != m->predecessor_commit ||
|
||||
predecessor->head.digest != m->predecessor_digest || previous->key != key ||
|
||||
previous->frame != m->frame || previous->sequence + 1 != m->sequence ||
|
||||
previous->end_tick != m->begin_tick)
|
||||
return fail(RecoveryError::tail);
|
||||
auto source_bytes = objects.get({previous->payload_sha256, previous->payload_bytes});
|
||||
if (!source_bytes)
|
||||
return fail(RecoveryError::tail);
|
||||
auto base = restore_population_payload(*source_bytes, *previous);
|
||||
auto trace = server::decode_population_batch(*trace_bytes);
|
||||
if (!base || !trace)
|
||||
return fail(RecoveryError::tail);
|
||||
auto replay = server::replay_population_batch(**base, *trace);
|
||||
if (!replay || (*replay)->snapshot() != *bytes)
|
||||
return fail(RecoveryError::tail);
|
||||
}
|
||||
return RecoveredPopulationCheckpoint{std::move(*committed), *m, std::move(*population)};
|
||||
}
|
||||
} // namespace evo::persistence
|
||||
@@ -60,12 +60,20 @@ bool structural(const CheckpointManifest& m) noexcept {
|
||||
return false;
|
||||
const auto& b = m.batch;
|
||||
const bool primitive = m.profile == CheckpointProfile::primitive_lives_v1;
|
||||
const bool cellular = m.profile == CheckpointProfile::cellular_lives_v1;
|
||||
const bool population = m.profile == CheckpointProfile::cellular_population_v1;
|
||||
const bool cellular = m.profile == CheckpointProfile::cellular_lives_v1 || population;
|
||||
const bool cellular_migration = cellular && m.sequence > 0 &&
|
||||
(m.cellular_versions == cellular_v2_versions ||
|
||||
m.cellular_versions == cellular_v3_versions) &&
|
||||
m.cellular_versions == cellular_v3_versions ||
|
||||
m.cellular_versions == cellular_v9_versions) &&
|
||||
m.begin_tick == m.end_tick && m.end_tick > 0 &&
|
||||
b == BatchDigests{};
|
||||
const bool population_conversion = population && m.begin_tick == m.end_tick && m.end_tick > 0 &&
|
||||
b.trace_bytes >= server::population_batch_header_bytes + 16;
|
||||
if (population && (m.sequence == 0 || m.cellular_versions != cellular_v9_versions ||
|
||||
m.history_profile != population_history_profile || b.decision_count != 0 ||
|
||||
!zero(b.decisions_sha256)))
|
||||
return false;
|
||||
if ((!cellular && m.cellular_versions != cellular_v1_versions) ||
|
||||
(cellular && m.cellular_versions != cellular_v1_versions &&
|
||||
m.cellular_versions != cellular_v2_versions &&
|
||||
@@ -103,7 +111,7 @@ bool structural(const CheckpointManifest& m) noexcept {
|
||||
}
|
||||
const bool link = !zero(m.predecessor_commit) && m.predecessor_commit != m.commit &&
|
||||
!zero(m.predecessor_digest) && !zero(m.holder) && m.epoch != 0 &&
|
||||
(m.end_tick > m.begin_tick || cellular_migration) &&
|
||||
(m.end_tick > m.begin_tick || cellular_migration || population_conversion) &&
|
||||
m.end_tick - m.begin_tick <= max_checkpoint_ticks;
|
||||
if (m.manifest_version == 1)
|
||||
return link && no_batch;
|
||||
@@ -146,6 +154,9 @@ bool structural(const CheckpointManifest& m) noexcept {
|
||||
(b.trace_bytes - server::primitive_batch_header_bytes) %
|
||||
server::primitive_birth_bytes ==
|
||||
0
|
||||
: population
|
||||
? b.trace_bytes >= server::population_batch_header_bytes + 16 &&
|
||||
b.trace_bytes <= server::max_population_batch_bytes && b.trace_bytes % 8 == 0
|
||||
: cellular ? (cellular_current_trace || cellular_v2_trace || cellular_legacy_trace)
|
||||
: b.trace_bytes == trace_bytes_for(ticks);
|
||||
return link && trace_shape && !zero(b.trace_sha256) &&
|
||||
@@ -155,6 +166,11 @@ bool structural(const CheckpointManifest& m) noexcept {
|
||||
} // namespace
|
||||
|
||||
bool supported(const CheckpointManifest& m) noexcept {
|
||||
if (m.profile == CheckpointProfile::cellular_population_v1)
|
||||
return m.manifest_version == 2 && m.history_profile == population_history_profile &&
|
||||
m.cellular_versions == cellular_v9_versions &&
|
||||
m.versions == environmental_v1_versions && m.tick_microseconds == 50000 &&
|
||||
m.rng_streams == 0 && m.durability == DurabilityPolicy::single_volume_development;
|
||||
if (m.profile == CheckpointProfile::primitive_lives_v1)
|
||||
return m.manifest_version == 2 && m.history_profile == primitive_history_profile &&
|
||||
m.versions == primitive_v1_versions && m.tick_microseconds == 50000 &&
|
||||
@@ -208,7 +224,8 @@ encode_manifest(const CheckpointManifest& m) {
|
||||
w.u(m.epoch, 8);
|
||||
w.u(m.tick_microseconds, 8);
|
||||
const auto versions =
|
||||
m.profile == CheckpointProfile::cellular_lives_v1
|
||||
(m.profile == CheckpointProfile::cellular_lives_v1 ||
|
||||
m.profile == CheckpointProfile::cellular_population_v1)
|
||||
? std::array{m.cellular_versions.rules, m.cellular_versions.envelope,
|
||||
m.cellular_versions.habitat, m.cellular_versions.coordinate,
|
||||
m.cellular_versions.genome, m.cellular_versions.development,
|
||||
@@ -265,7 +282,8 @@ decode_manifest(std::span<const std::uint8_t> bytes) {
|
||||
std::array<std::uint32_t, 9> versions{};
|
||||
for (auto& v : versions)
|
||||
v = static_cast<std::uint32_t>(r.u(4));
|
||||
if (m.profile == CheckpointProfile::cellular_lives_v1) {
|
||||
if (m.profile == CheckpointProfile::cellular_lives_v1 ||
|
||||
m.profile == CheckpointProfile::cellular_population_v1) {
|
||||
m.cellular_versions = {versions[0], versions[1], versions[2], versions[3], versions[4],
|
||||
versions[5], versions[6], versions[7], versions[8]};
|
||||
} else {
|
||||
@@ -326,7 +344,8 @@ clone_manifest(const CheckpointManifest& source, const Sha256Digest& source_dige
|
||||
RegionKey target, Uuid commit, std::uint64_t payload_bytes,
|
||||
const Sha256Digest& payload_sha256, const BatchDigests& inherited) {
|
||||
if (!supported(source) || source.profile == CheckpointProfile::primitive_lives_v1 ||
|
||||
source.profile == CheckpointProfile::cellular_lives_v1)
|
||||
source.profile == CheckpointProfile::cellular_lives_v1 ||
|
||||
source.profile == CheckpointProfile::cellular_population_v1)
|
||||
return std::unexpected{CheckpointError::unsupported};
|
||||
if (!structural(source) || target.world == source.key.world || commit == source.commit ||
|
||||
inherited.trace_bytes != 0 || !zero(inherited.trace_sha256))
|
||||
|
||||
@@ -463,7 +463,7 @@ std::expected<ResolvedCommit, LeaseStoreFailure> resolve_cellular_region_commit(
|
||||
p, timeout);
|
||||
if (!rows)
|
||||
return std::unexpected{classify(rows.error())};
|
||||
if (rows->size() != 1 || (*rows)[0].size() != 6 || !(*rows)[0][0])
|
||||
if (rows->size() != 1 || (*rows)[0].size() != 6 || !(*rows)[0][0].has_value())
|
||||
return std::unexpected{malformed(false)};
|
||||
const auto status = (*rows)[0][0].value_or(std::string{});
|
||||
if (status == "absent")
|
||||
@@ -1502,4 +1502,100 @@ std::expected<bool, AccountIdentityError> revoke_account_grants(Postgres& db, st
|
||||
return std::unexpected{AccountIdentityError::unknown};
|
||||
return *revoked == "true";
|
||||
}
|
||||
} // namespace evo::persistence
|
||||
std::expected<RegionHead, LeaseStoreFailure>
|
||||
publish_population_checkpoint(Postgres& db, std::span<const std::uint8_t> manifest,
|
||||
std::span<const std::uint8_t> trace, milliseconds timeout) {
|
||||
const std::array<std::string, 2> parameters{bytea(manifest), bytea(trace)};
|
||||
auto rows =
|
||||
db.query("SELECT * FROM evolution.publish_population_checkpoint($1::bytea,$2::bytea)",
|
||||
parameters, timeout);
|
||||
if (!rows)
|
||||
return std::unexpected(classify(rows.error()));
|
||||
auto head = parse_head(*rows);
|
||||
if (!head)
|
||||
return std::unexpected(malformed(true));
|
||||
return *head;
|
||||
}
|
||||
std::expected<RegionHead, LeaseStoreFailure>
|
||||
publish_population_profile_transition(Postgres& db, std::span<const std::uint8_t> manifest,
|
||||
milliseconds timeout) {
|
||||
const std::array<std::string, 1> parameters{bytea(manifest)};
|
||||
auto rows = db.query("SELECT * FROM evolution.publish_population_profile_transition($1::bytea)",
|
||||
parameters, timeout);
|
||||
if (!rows)
|
||||
return std::unexpected(classify(rows.error()));
|
||||
auto head = parse_head(*rows);
|
||||
if (!head)
|
||||
return std::unexpected(malformed(true));
|
||||
return *head;
|
||||
}
|
||||
std::expected<ResolvedCommit, LeaseStoreFailure>
|
||||
resolve_population_commit(Postgres& db, RegionKey key, const Uuid& commit,
|
||||
std::span<const std::uint8_t> manifest,
|
||||
std::span<const std::uint8_t> trace, milliseconds timeout) {
|
||||
const std::array<std::string, 5> parameters{std::to_string(key.world),
|
||||
std::to_string(key.region), uuid_text(commit),
|
||||
bytea(manifest), bytea(trace)};
|
||||
auto rows = db.query("SELECT * FROM evolution.resolve_population_commit($1::evolution.u64,"
|
||||
"$2::evolution.u64,$3::uuid,$4::bytea,$5::bytea)",
|
||||
parameters, timeout);
|
||||
if (!rows)
|
||||
return std::unexpected(classify(rows.error()));
|
||||
if (rows->size() != 1 || (*rows)[0].size() != 6 || !(*rows)[0][0])
|
||||
return std::unexpected(malformed(false));
|
||||
const auto status_field = (*rows)[0][0];
|
||||
if (!status_field.has_value())
|
||||
return std::unexpected(malformed(false));
|
||||
const auto& status = *status_field;
|
||||
if (status == "absent")
|
||||
return ResolvedCommit{Resolution::absent, {}};
|
||||
if (status == "conflict")
|
||||
return ResolvedCommit{Resolution::conflict, {}};
|
||||
PgRows receipt{PgRows::value_type((*rows)[0].begin() + 1, (*rows)[0].end())};
|
||||
const auto head = status == "committed" ? parse_head(receipt) : std::nullopt;
|
||||
if (!head)
|
||||
return std::unexpected(malformed(false));
|
||||
return ResolvedCommit{Resolution::committed, *head};
|
||||
}
|
||||
std::expected<std::optional<PopulationEvidence>, LeaseStoreFailure>
|
||||
load_population_evidence(Postgres& db, RegionKey key, const LeaseGrant& lease, const Uuid& commit,
|
||||
milliseconds timeout) {
|
||||
const std::array<std::string, 5> parameters{std::to_string(key.world),
|
||||
std::to_string(key.region), uuid_text(lease.holder),
|
||||
std::to_string(lease.epoch), uuid_text(commit)};
|
||||
auto rows = db.query(
|
||||
"SELECT encode(trace,'hex'),source_commit::text,encode(source_manifest_sha256,'hex'),"
|
||||
"encode(source_payload_sha256,'hex') FROM "
|
||||
"evolution.load_population_evidence($1::evolution.u64,"
|
||||
"$2::evolution.u64,$3::uuid,$4::evolution.u64,$5::uuid)",
|
||||
parameters, timeout);
|
||||
if (!rows)
|
||||
return std::unexpected(classify(rows.error()));
|
||||
if (rows->empty())
|
||||
return std::optional<PopulationEvidence>{};
|
||||
if (rows->size() != 1 || (*rows)[0].size() != 4 || !(*rows)[0][0].has_value())
|
||||
return std::unexpected(malformed(false));
|
||||
const auto& row = (*rows)[0];
|
||||
const auto trace_field = row[0];
|
||||
if (!trace_field.has_value())
|
||||
return std::unexpected(malformed(false));
|
||||
auto trace = unhex_bytes(*trace_field);
|
||||
if (!trace)
|
||||
return std::unexpected(malformed(false));
|
||||
PopulationEvidence evidence{std::move(*trace), {}, {}, {}};
|
||||
if (evidence.trace.empty()) {
|
||||
const auto source = parse_uuid(row[1]);
|
||||
const auto manifest_field = row[2], payload_field = row[3];
|
||||
const auto manifest =
|
||||
manifest_field.has_value() ? unhex<32>(*manifest_field) : std::nullopt;
|
||||
const auto payload = payload_field.has_value() ? unhex<32>(*payload_field) : std::nullopt;
|
||||
if (!source || !manifest || !payload)
|
||||
return std::unexpected(malformed(false));
|
||||
evidence.source_commit = *source;
|
||||
evidence.source_manifest_sha256 = *manifest;
|
||||
evidence.source_payload_sha256 = *payload;
|
||||
} else if (row[1] || row[2] || row[3])
|
||||
return std::unexpected(malformed(false));
|
||||
return std::optional<PopulationEvidence>{std::move(evidence)};
|
||||
}
|
||||
} // namespace evo::persistence
|
||||
|
||||
Reference in New Issue
Block a user