M12.6: derive replayed census history from immutable commits
Foundation / checks (deploy) (push) Successful in 6s
Foundation / checks (ecs) (push) Successful in 2m25s
Foundation / checks (network) (push) Successful in 20m50s
Foundation / checks (client) (push) Successful in 22m7s
Foundation / checks (tsan) (push) Successful in 24m42s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (postgres) (push) Canceled after 35m33s
Foundation / checks (checks) (push) Canceled after 35m43s

This commit is contained in:
dingo
2026-10-18 14:36:14 -04:00
parent da39fc8095
commit a180c41ae7
23 changed files with 732 additions and 18 deletions
+20 -4
View File
@@ -624,14 +624,15 @@ verify_cellular_no_tick_checkpoint(Postgres& db, const FilesystemObjectStore& ob
return {};
}
namespace {
std::expected<RecoveredCellularCheckpoint, RecoveryFailure>
recover_cellular_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
const LeaseGrant& lease, std::span<const Sha256Digest> retained_builds,
milliseconds timeout) {
recover_cellular_loaded(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
const LeaseGrant& lease, std::span<const Sha256Digest> retained_builds,
milliseconds timeout,
std::expected<CommittedCheckpoint, LeaseStoreFailure> committed) {
const auto fail_with = [](RecoveryError kind) {
return std::unexpected{RecoveryFailure{kind, {}}};
};
auto committed = load_region_checkpoint(db, key, timeout);
if (!committed)
return std::unexpected{
RecoveryFailure{committed.error().kind == LeaseStoreError::not_provisioned
@@ -802,4 +803,19 @@ recover_cellular_checkpoint(Postgres& db, const FilesystemObjectStore& objects,
std::move(mutation_actions_recovered),
std::move(mutation_results_recovered)};
}
} // namespace
std::expected<RecoveredCellularCheckpoint, RecoveryFailure>
recover_cellular_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
const LeaseGrant& lease, std::span<const Sha256Digest> retained_builds,
milliseconds timeout) {
return recover_cellular_loaded(db, objects, key, lease, retained_builds, timeout,
load_region_checkpoint(db, key, timeout));
}
std::expected<RecoveredCellularCheckpoint, RecoveryFailure>
recover_cellular_commit(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
std::uint64_t sequence, const LeaseGrant& lease,
std::span<const Sha256Digest> retained_builds, milliseconds timeout) {
return recover_cellular_loaded(db, objects, key, lease, retained_builds, timeout,
load_region_commit(db, key, sequence, timeout));
}
} // namespace evo::persistence
+111
View File
@@ -0,0 +1,111 @@
#include "evo/persistence/cellular_history_store.hpp"
#include "evo/persistence/sha256.hpp"
#include <algorithm>
#include <utility>
namespace evo::persistence {
namespace {
bool bound(const CommittedCheckpoint& checkpoint, RegionKey key) {
const auto digest = sha256(checkpoint.manifest);
const auto manifest = decode_manifest(checkpoint.manifest);
const auto& head = checkpoint.head;
return digest && *digest == head.digest && manifest && manifest->key == key &&
manifest->frame == head.frame && manifest->sequence == head.sequence &&
manifest->commit == head.commit && manifest->end_tick == head.end_tick;
}
auto failed(RecoveryFailure failure) {
return std::unexpected(CellularHistoryFailure{CellularHistoryError::recovery, failure});
}
auto failed(RecoveryError kind) {
return failed(RecoveryFailure{kind, {}});
}
std::expected<CellularCensusHistoryEntry, CellularHistoryFailure>
observe_commit(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
const LeaseGrant& lease, std::uint64_t sequence,
std::span<const Sha256Digest> builds, std::chrono::milliseconds timeout) {
auto cut = load_region_commit(db, key, sequence, timeout);
if (!cut)
return failed(RecoveryFailure{RecoveryError::store, cut.error()});
if (!bound(*cut, key) || cut->head.sequence != sequence)
return failed(RecoveryError::manifest);
const auto manifest = decode_manifest(cut->manifest);
if (!manifest)
return failed(RecoveryError::manifest);
auto observation = [&]() -> std::expected<server::CellularCensus, RecoveryFailure> {
if (manifest->profile == CheckpointProfile::cellular_lives_v1) {
auto recovered =
recover_cellular_commit(db, objects, key, sequence, lease, builds, timeout);
if (!recovered)
return std::unexpected(recovered.error());
if (recovered->committed.head != cut->head ||
recovered->committed.manifest != cut->manifest)
return std::unexpected(RecoveryFailure{RecoveryError::manifest, {}});
auto census = server::CellularCensus::observe(*recovered->state);
if (census)
return std::move(*census);
return std::unexpected(
RecoveryFailure{census.error() == server::CellularCensusError::unsupported
? RecoveryError::unsupported
: RecoveryError::payload,
{}});
}
if (manifest->profile == CheckpointProfile::cellular_population_v1) {
auto recovered =
recover_population_commit(db, objects, key, sequence, lease, builds, timeout);
if (!recovered)
return std::unexpected(recovered.error());
if (recovered->committed.head != cut->head ||
recovered->committed.manifest != cut->manifest)
return std::unexpected(RecoveryFailure{RecoveryError::manifest, {}});
auto census = server::CellularCensus::observe(*recovered->state);
if (census)
return std::move(*census);
return std::unexpected(
RecoveryFailure{census.error() == server::CellularCensusError::unsupported
? RecoveryError::unsupported
: RecoveryError::payload,
{}});
}
return std::unexpected(RecoveryFailure{RecoveryError::unsupported, {}});
}();
if (!observation)
return failed(observation.error());
return CellularCensusHistoryEntry{std::move(*cut), std::move(*observation)};
}
} // namespace
std::expected<CellularCensusHistoryPage, CellularHistoryFailure> load_cellular_census_history(
Postgres& db, const FilesystemObjectStore& objects, RegionKey key, const LeaseGrant& lease,
const RegionHead& watermark, std::uint64_t first_sequence, std::size_t limit,
std::span<const Sha256Digest> retained_builds, std::chrono::milliseconds timeout) {
if (!db.transaction_idle() || key.world == 0 || key.region == 0 || watermark.frame == 0 ||
watermark.frame != lease.frame || first_sequence > watermark.sequence || limit == 0 ||
limit > max_cellular_history_page || timeout <= std::chrono::milliseconds::zero())
return std::unexpected(CellularHistoryFailure{CellularHistoryError::invalid_request, {}});
auto upper = load_region_commit(db, key, watermark.sequence, timeout);
if (!upper)
return failed(RecoveryFailure{RecoveryError::store, upper.error()});
if (upper->head != watermark || !bound(*upper, key))
return std::unexpected(CellularHistoryFailure{CellularHistoryError::cut_mismatch, {}});
CellularCensusHistoryPage page{key, watermark, {}, {}};
page.entries.reserve(limit);
const auto last = first_sequence + std::min(static_cast<std::uint64_t>(limit - 1),
watermark.sequence - first_sequence);
for (auto sequence = first_sequence;; ++sequence) {
auto entry = observe_commit(db, objects, key, lease, sequence, retained_builds, timeout);
if (!entry)
return std::unexpected(entry.error());
page.entries.push_back(std::move(*entry));
if (sequence == last)
break;
}
if (last < watermark.sequence)
page.next_sequence = last + 1;
const auto confirmed = load_region_commit(db, key, watermark.sequence, timeout);
if (!confirmed)
return failed(RecoveryFailure{RecoveryError::store, confirmed.error()});
if (confirmed->head != watermark || confirmed->manifest != upper->manifest)
return std::unexpected(CellularHistoryFailure{CellularHistoryError::cut_mismatch, {}});
return page;
}
} // namespace evo::persistence
@@ -248,12 +248,13 @@ verify_population_profile_transition(Postgres& db, const FilesystemObjectStore&
return fail(RecoveryError::unsupported);
return {};
}
namespace {
std::expected<RecoveredPopulationCheckpoint, RecoveryFailure>
recover_population_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
const LeaseGrant& lease,
std::span<const Sha256Digest> retained_builds, milliseconds timeout) {
recover_population_loaded(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
const LeaseGrant& lease, std::span<const Sha256Digest> retained_builds,
milliseconds timeout,
std::expected<CommittedCheckpoint, LeaseStoreFailure> committed) {
const auto fail = [](RecoveryError kind) { return std::unexpected(RecoveryFailure{kind, {}}); };
auto committed = load_region_checkpoint(db, key, timeout);
if (!committed)
return std::unexpected(
RecoveryFailure{committed.error().kind == LeaseStoreError::not_provisioned
@@ -319,4 +320,19 @@ recover_population_checkpoint(Postgres& db, const FilesystemObjectStore& objects
}
return RecoveredPopulationCheckpoint{std::move(*committed), *m, std::move(*population)};
}
} // namespace
std::expected<RecoveredPopulationCheckpoint, RecoveryFailure>
recover_population_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
const LeaseGrant& lease,
std::span<const Sha256Digest> retained_builds, milliseconds timeout) {
return recover_population_loaded(db, objects, key, lease, retained_builds, timeout,
load_region_checkpoint(db, key, timeout));
}
std::expected<RecoveredPopulationCheckpoint, RecoveryFailure>
recover_population_commit(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
std::uint64_t sequence, const LeaseGrant& lease,
std::span<const Sha256Digest> retained_builds, milliseconds timeout) {
return recover_population_loaded(db, objects, key, lease, retained_builds, timeout,
load_region_commit(db, key, sequence, timeout));
}
} // namespace evo::persistence
+4
View File
@@ -119,6 +119,10 @@ struct Postgres::Impl {
};
Postgres::Postgres(std::unique_ptr<Impl> impl) : impl_(std::move(impl)) {}
Postgres::~Postgres() = default;
bool Postgres::transaction_idle() const noexcept {
return impl_->connection && PQstatus(impl_->connection.get()) == CONNECTION_OK &&
PQtransactionStatus(impl_->connection.get()) == PQTRANS_IDLE;
}
std::expected<PgOptions, PgError> read_pg_service(const std::string& file,
std::string_view section) {
constexpr std::size_t limit = 4096;