Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 1m47s
Foundation / checks (network) (push) Successful in 9m30s
Foundation / checks (client) (push) Successful in 12m19s
Foundation / checks (tsan) (push) Successful in 7m43s
Foundation / checks (checks) (push) Successful in 31m29s
Foundation / checks (postgres) (push) Successful in 36m39s
Foundation / postgres-tools (push) Successful in 19s
Foundation / image (push) Successful in 2m15s
Foundation / server-image (push) Successful in 20m37s
Each batch's actions are recorded in the same statement as its checkpoint: one region_actions row per (world, account, action id) and the block in an append-only table, compared on exact retries and resolution. The region host attaches actions to the ticks they ride and reports acknowledgements and drops, keeping unknown-outcome batches unresolved. Outcomes come from the durable record via ActionLedger, so crashes before or after a commit, unknown publications and cross-region moves all resolve to one answer; a repeated id keeps its first record instead of stalling a region. ADR-0092. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
185 lines
10 KiB
C++
185 lines
10 KiB
C++
#include "evo/persistence/checkpoint_store.hpp"
|
|
#include "checkpoint_detail.hpp"
|
|
|
|
namespace evo::persistence {
|
|
namespace {
|
|
using namespace detail;
|
|
using std::chrono::milliseconds;
|
|
|
|
template <typename Codec>
|
|
std::expected<CommittedCheckpoint, PublicationFailure>
|
|
provision(Postgres& owner, const FilesystemObjectStore& objects, RegionKey key, std::uint64_t frame,
|
|
const typename Codec::State& opening, Uuid commit, const Sha256Digest& build,
|
|
std::size_t budget, milliseconds timeout) {
|
|
auto payload = Codec::snapshot(opening);
|
|
if (!payload || Codec::tick(opening) != 0)
|
|
return std::unexpected{fail(PublicationError::candidate)};
|
|
const auto digest = sha256(*payload);
|
|
if (!digest)
|
|
return std::unexpected{fail(PublicationError::digest)};
|
|
auto manifest = genesis_manifest(key, frame, commit, Codec::quantum(opening), build,
|
|
payload->size(), *digest, Codec::profile);
|
|
auto bytes = manifest ? encode_manifest(*manifest)
|
|
: std::expected<std::vector<std::uint8_t>, CheckpointError>{
|
|
std::unexpected{CheckpointError::format}};
|
|
if (!bytes || !Codec::restore(*payload, *manifest, budget))
|
|
return std::unexpected{fail(PublicationError::candidate)};
|
|
if (auto stored = put_verified(objects, *payload, {*digest, payload->size()}); !stored)
|
|
return std::unexpected{stored.error()};
|
|
const std::array<std::string, 1> p{hex_parameter(*bytes)};
|
|
auto done = owner.query("SELECT evolution.provision_region_checkpoint($1::bytea)", p, timeout);
|
|
if (!done) {
|
|
const bool unknown = done.error().kind == PgError::unknown;
|
|
return std::unexpected{PublicationFailure{
|
|
unknown ? PublicationError::unknown : PublicationError::rejected,
|
|
{unknown ? LeaseStoreError::unknown : LeaseStoreError::store, done.error()},
|
|
{},
|
|
{}}};
|
|
}
|
|
auto committed = load_region_checkpoint(owner, key, timeout);
|
|
if (!committed)
|
|
return std::unexpected{
|
|
PublicationFailure{PublicationError::unknown, committed.error(), {}, {}}};
|
|
return confirm(committed->head, std::move(*bytes), *manifest, {});
|
|
}
|
|
|
|
template <typename Codec>
|
|
std::expected<PublishedBatch, PublicationFailure>
|
|
publish_batch(Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease,
|
|
const CommittedCheckpoint& predecessor,
|
|
const RegionBatch<typename Codec::State>& batch, Uuid commit,
|
|
const Sha256Digest& build, std::size_t budget, milliseconds timeout,
|
|
const PublishHooks& hooks, std::span<const ActionRecord> actions) {
|
|
const auto previous = decode_manifest(predecessor.manifest);
|
|
if (!previous || previous->profile != Codec::profile || !batch.sealed() ||
|
|
batch.steps().empty() || batch.begin_tick() != previous->end_tick)
|
|
return std::unexpected{fail(PublicationError::candidate)};
|
|
// Replay the recorded inputs from the committed predecessor's verified bytes.
|
|
auto base = objects.get({previous->payload_sha256, previous->payload_bytes});
|
|
if (!base)
|
|
return std::unexpected{fail(PublicationError::predecessor, base.error())};
|
|
auto replay = Codec::restore(*base, *previous, budget);
|
|
if (!replay)
|
|
return std::unexpected{fail(PublicationError::predecessor)};
|
|
for (const auto& step : batch.steps()) {
|
|
auto report = (*replay)->advance(step.report.tick, step.input);
|
|
if (!report || *report != step.report)
|
|
return std::unexpected{fail(PublicationError::candidate)};
|
|
}
|
|
auto payload = Codec::snapshot(batch.state());
|
|
const auto replayed = Codec::snapshot(**replay);
|
|
if (!payload || !replayed || *payload != *replayed)
|
|
return std::unexpected{fail(PublicationError::candidate)};
|
|
|
|
const auto organisms = Codec::organisms(batch.state());
|
|
const auto decisions = derive_decisions(batch.steps(), organisms.producer, organisms.consumer);
|
|
auto decision_bytes = encode_decisions(decisions, batch.begin_tick(), batch.end_tick());
|
|
const TraceHeader header{static_cast<std::uint32_t>(Codec::profile),
|
|
previous->key,
|
|
previous->commit,
|
|
predecessor.head.digest,
|
|
batch.begin_tick(),
|
|
batch.end_tick()};
|
|
auto trace = encode_trace(header, batch.steps());
|
|
auto action_bytes = encode_actions(actions, batch.begin_tick(), batch.end_tick());
|
|
if (!decision_bytes || !trace || !action_bytes)
|
|
return std::unexpected{fail(PublicationError::candidate)};
|
|
const auto payload_digest = sha256(*payload);
|
|
const auto trace_digest = sha256(*trace);
|
|
const auto decisions_digest = sha256(*decision_bytes);
|
|
if (!payload_digest || !trace_digest || !decisions_digest)
|
|
return std::unexpected{fail(PublicationError::digest)};
|
|
const BatchDigests digests{trace->size(), *trace_digest,
|
|
static_cast<std::uint32_t>(decisions.size()),
|
|
decisions.empty() ? Sha256Digest{} : *decisions_digest};
|
|
auto manifest =
|
|
successor_manifest(*previous, predecessor.head.digest, commit, lease, batch.end_tick(),
|
|
build, payload->size(), *payload_digest, digests);
|
|
auto bytes = manifest ? encode_manifest(*manifest)
|
|
: std::expected<std::vector<std::uint8_t>, CheckpointError>{
|
|
std::unexpected{CheckpointError::format}};
|
|
if (!bytes || !Codec::restore(*payload, *manifest, budget))
|
|
return std::unexpected{fail(PublicationError::candidate)};
|
|
if (auto stored = put_verified(objects, *payload, {*payload_digest, payload->size()}); !stored)
|
|
return std::unexpected{stored.error()};
|
|
if (auto stored = put_verified(objects, *trace, {*trace_digest, trace->size()}); !stored)
|
|
return std::unexpected{stored.error()};
|
|
|
|
PreparedCommit prepared{previous->key, commit, *bytes, *decision_bytes, *action_bytes};
|
|
if (hooks.before_transaction)
|
|
hooks.before_transaction();
|
|
auto published = retry_prepared(db, prepared, timeout);
|
|
auto committed = confirm(published, std::move(*bytes), *manifest, std::move(prepared));
|
|
if (!committed)
|
|
return std::unexpected{committed.error()};
|
|
return PublishedBatch{std::move(*committed), decisions};
|
|
}
|
|
|
|
} // namespace
|
|
|
|
std::expected<CommittedCheckpoint, PublicationFailure> provision_environmental_region(
|
|
Postgres& owner, const FilesystemObjectStore& objects, RegionKey key, std::uint64_t frame,
|
|
const processes::EnvironmentalCommunity& opening, Uuid commit, const Sha256Digest& build,
|
|
std::size_t restore_budget, milliseconds timeout) {
|
|
return provision<EnvironmentalCodec>(owner, objects, key, frame, opening, commit, build,
|
|
restore_budget, timeout);
|
|
}
|
|
std::expected<PublishedBatch, PublicationFailure>
|
|
publish_environmental_batch(Postgres& db, const FilesystemObjectStore& objects,
|
|
const LeaseGrant& lease, const CommittedCheckpoint& predecessor,
|
|
const RegionBatch<processes::EnvironmentalCommunity>& batch,
|
|
Uuid commit, const Sha256Digest& build, std::size_t restore_budget,
|
|
milliseconds timeout, const PublishHooks& hooks,
|
|
std::span<const ActionRecord> actions) {
|
|
return publish_batch<EnvironmentalCodec>(db, objects, lease, predecessor, batch, commit, build,
|
|
restore_budget, timeout, hooks, actions);
|
|
}
|
|
std::expected<RecoveredCheckpoint, RecoveryFailure>
|
|
recover_environmental_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
|
std::span<const Sha256Digest> retained_builds,
|
|
std::size_t restore_budget, milliseconds timeout) {
|
|
return recover<EnvironmentalCodec>(db, objects, key, retained_builds, restore_budget, timeout);
|
|
}
|
|
|
|
std::expected<CommittedCheckpoint, PublicationFailure>
|
|
provision_lineage_region(Postgres& owner, const FilesystemObjectStore& objects, RegionKey key,
|
|
std::uint64_t frame, const processes::LineageRegion& opening, Uuid commit,
|
|
const Sha256Digest& build, std::size_t restore_budget,
|
|
milliseconds timeout) {
|
|
return provision<LineageCodec>(owner, objects, key, frame, opening, commit, build,
|
|
restore_budget, timeout);
|
|
}
|
|
std::expected<PublishedBatch, PublicationFailure>
|
|
publish_lineage_batch(Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease,
|
|
const CommittedCheckpoint& predecessor,
|
|
const RegionBatch<processes::LineageRegion>& batch, Uuid commit,
|
|
const Sha256Digest& build, std::size_t restore_budget, milliseconds timeout,
|
|
const PublishHooks& hooks, std::span<const ActionRecord> actions) {
|
|
return publish_batch<LineageCodec>(db, objects, lease, predecessor, batch, commit, build,
|
|
restore_budget, timeout, hooks, actions);
|
|
}
|
|
std::expected<RecoveredLineageCheckpoint, RecoveryFailure>
|
|
recover_lineage_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
|
std::span<const Sha256Digest> retained_builds,
|
|
std::size_t restore_budget, milliseconds timeout) {
|
|
return recover<LineageCodec>(db, objects, key, retained_builds, restore_budget, timeout);
|
|
}
|
|
|
|
std::expected<ResolvedCommit, LeaseStoreFailure>
|
|
resolve_prepared(Postgres& db, const PreparedCommit& prepared, milliseconds timeout) {
|
|
if (!prepared.actions.empty())
|
|
return resolve_region_commit(db, prepared.key, prepared.commit, prepared.manifest,
|
|
prepared.decisions, prepared.actions, timeout);
|
|
return resolve_region_commit(db, prepared.key, prepared.commit, prepared.manifest,
|
|
prepared.decisions, timeout);
|
|
}
|
|
std::expected<RegionHead, LeaseStoreFailure>
|
|
retry_prepared(Postgres& db, const PreparedCommit& prepared, milliseconds timeout) {
|
|
// Without actions the schema v5 publication is used, exactly as before v11.
|
|
if (!prepared.actions.empty())
|
|
return publish_region_checkpoint(db, prepared.manifest, prepared.decisions,
|
|
prepared.actions, timeout);
|
|
return publish_region_checkpoint(db, prepared.manifest, prepared.decisions, timeout);
|
|
}
|
|
} // namespace evo::persistence
|