M12.6: derive replayed census history from immutable commits
Foundation / checks (deploy) (push) Successful in 6s
Foundation / checks (ecs) (push) Successful in 2m25s
Foundation / checks (network) (push) Successful in 20m50s
Foundation / checks (client) (push) Successful in 22m7s
Foundation / checks (tsan) (push) Successful in 24m42s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (postgres) (push) Canceled after 35m33s
Foundation / checks (checks) (push) Canceled after 35m43s
Foundation / checks (deploy) (push) Successful in 6s
Foundation / checks (ecs) (push) Successful in 2m25s
Foundation / checks (network) (push) Successful in 20m50s
Foundation / checks (client) (push) Successful in 22m7s
Foundation / checks (tsan) (push) Successful in 24m42s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (postgres) (push) Canceled after 35m33s
Foundation / checks (checks) (push) Canceled after 35m43s
This commit is contained in:
@@ -624,14 +624,15 @@ verify_cellular_no_tick_checkpoint(Postgres& db, const FilesystemObjectStore& ob
|
||||
return {};
|
||||
}
|
||||
|
||||
namespace {
|
||||
std::expected<RecoveredCellularCheckpoint, RecoveryFailure>
|
||||
recover_cellular_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
const LeaseGrant& lease, std::span<const Sha256Digest> retained_builds,
|
||||
milliseconds timeout) {
|
||||
recover_cellular_loaded(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
const LeaseGrant& lease, std::span<const Sha256Digest> retained_builds,
|
||||
milliseconds timeout,
|
||||
std::expected<CommittedCheckpoint, LeaseStoreFailure> committed) {
|
||||
const auto fail_with = [](RecoveryError kind) {
|
||||
return std::unexpected{RecoveryFailure{kind, {}}};
|
||||
};
|
||||
auto committed = load_region_checkpoint(db, key, timeout);
|
||||
if (!committed)
|
||||
return std::unexpected{
|
||||
RecoveryFailure{committed.error().kind == LeaseStoreError::not_provisioned
|
||||
@@ -802,4 +803,19 @@ recover_cellular_checkpoint(Postgres& db, const FilesystemObjectStore& objects,
|
||||
std::move(mutation_actions_recovered),
|
||||
std::move(mutation_results_recovered)};
|
||||
}
|
||||
} // namespace
|
||||
std::expected<RecoveredCellularCheckpoint, RecoveryFailure>
|
||||
recover_cellular_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
const LeaseGrant& lease, std::span<const Sha256Digest> retained_builds,
|
||||
milliseconds timeout) {
|
||||
return recover_cellular_loaded(db, objects, key, lease, retained_builds, timeout,
|
||||
load_region_checkpoint(db, key, timeout));
|
||||
}
|
||||
std::expected<RecoveredCellularCheckpoint, RecoveryFailure>
|
||||
recover_cellular_commit(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
std::uint64_t sequence, const LeaseGrant& lease,
|
||||
std::span<const Sha256Digest> retained_builds, milliseconds timeout) {
|
||||
return recover_cellular_loaded(db, objects, key, lease, retained_builds, timeout,
|
||||
load_region_commit(db, key, sequence, timeout));
|
||||
}
|
||||
} // namespace evo::persistence
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
#include "evo/persistence/cellular_history_store.hpp"
|
||||
#include "evo/persistence/sha256.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <utility>
|
||||
|
||||
namespace evo::persistence {
|
||||
namespace {
|
||||
bool bound(const CommittedCheckpoint& checkpoint, RegionKey key) {
|
||||
const auto digest = sha256(checkpoint.manifest);
|
||||
const auto manifest = decode_manifest(checkpoint.manifest);
|
||||
const auto& head = checkpoint.head;
|
||||
return digest && *digest == head.digest && manifest && manifest->key == key &&
|
||||
manifest->frame == head.frame && manifest->sequence == head.sequence &&
|
||||
manifest->commit == head.commit && manifest->end_tick == head.end_tick;
|
||||
}
|
||||
auto failed(RecoveryFailure failure) {
|
||||
return std::unexpected(CellularHistoryFailure{CellularHistoryError::recovery, failure});
|
||||
}
|
||||
auto failed(RecoveryError kind) {
|
||||
return failed(RecoveryFailure{kind, {}});
|
||||
}
|
||||
std::expected<CellularCensusHistoryEntry, CellularHistoryFailure>
|
||||
observe_commit(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
const LeaseGrant& lease, std::uint64_t sequence,
|
||||
std::span<const Sha256Digest> builds, std::chrono::milliseconds timeout) {
|
||||
auto cut = load_region_commit(db, key, sequence, timeout);
|
||||
if (!cut)
|
||||
return failed(RecoveryFailure{RecoveryError::store, cut.error()});
|
||||
if (!bound(*cut, key) || cut->head.sequence != sequence)
|
||||
return failed(RecoveryError::manifest);
|
||||
const auto manifest = decode_manifest(cut->manifest);
|
||||
if (!manifest)
|
||||
return failed(RecoveryError::manifest);
|
||||
auto observation = [&]() -> std::expected<server::CellularCensus, RecoveryFailure> {
|
||||
if (manifest->profile == CheckpointProfile::cellular_lives_v1) {
|
||||
auto recovered =
|
||||
recover_cellular_commit(db, objects, key, sequence, lease, builds, timeout);
|
||||
if (!recovered)
|
||||
return std::unexpected(recovered.error());
|
||||
if (recovered->committed.head != cut->head ||
|
||||
recovered->committed.manifest != cut->manifest)
|
||||
return std::unexpected(RecoveryFailure{RecoveryError::manifest, {}});
|
||||
auto census = server::CellularCensus::observe(*recovered->state);
|
||||
if (census)
|
||||
return std::move(*census);
|
||||
return std::unexpected(
|
||||
RecoveryFailure{census.error() == server::CellularCensusError::unsupported
|
||||
? RecoveryError::unsupported
|
||||
: RecoveryError::payload,
|
||||
{}});
|
||||
}
|
||||
if (manifest->profile == CheckpointProfile::cellular_population_v1) {
|
||||
auto recovered =
|
||||
recover_population_commit(db, objects, key, sequence, lease, builds, timeout);
|
||||
if (!recovered)
|
||||
return std::unexpected(recovered.error());
|
||||
if (recovered->committed.head != cut->head ||
|
||||
recovered->committed.manifest != cut->manifest)
|
||||
return std::unexpected(RecoveryFailure{RecoveryError::manifest, {}});
|
||||
auto census = server::CellularCensus::observe(*recovered->state);
|
||||
if (census)
|
||||
return std::move(*census);
|
||||
return std::unexpected(
|
||||
RecoveryFailure{census.error() == server::CellularCensusError::unsupported
|
||||
? RecoveryError::unsupported
|
||||
: RecoveryError::payload,
|
||||
{}});
|
||||
}
|
||||
return std::unexpected(RecoveryFailure{RecoveryError::unsupported, {}});
|
||||
}();
|
||||
if (!observation)
|
||||
return failed(observation.error());
|
||||
return CellularCensusHistoryEntry{std::move(*cut), std::move(*observation)};
|
||||
}
|
||||
} // namespace
|
||||
std::expected<CellularCensusHistoryPage, CellularHistoryFailure> load_cellular_census_history(
|
||||
Postgres& db, const FilesystemObjectStore& objects, RegionKey key, const LeaseGrant& lease,
|
||||
const RegionHead& watermark, std::uint64_t first_sequence, std::size_t limit,
|
||||
std::span<const Sha256Digest> retained_builds, std::chrono::milliseconds timeout) {
|
||||
if (!db.transaction_idle() || key.world == 0 || key.region == 0 || watermark.frame == 0 ||
|
||||
watermark.frame != lease.frame || first_sequence > watermark.sequence || limit == 0 ||
|
||||
limit > max_cellular_history_page || timeout <= std::chrono::milliseconds::zero())
|
||||
return std::unexpected(CellularHistoryFailure{CellularHistoryError::invalid_request, {}});
|
||||
auto upper = load_region_commit(db, key, watermark.sequence, timeout);
|
||||
if (!upper)
|
||||
return failed(RecoveryFailure{RecoveryError::store, upper.error()});
|
||||
if (upper->head != watermark || !bound(*upper, key))
|
||||
return std::unexpected(CellularHistoryFailure{CellularHistoryError::cut_mismatch, {}});
|
||||
CellularCensusHistoryPage page{key, watermark, {}, {}};
|
||||
page.entries.reserve(limit);
|
||||
const auto last = first_sequence + std::min(static_cast<std::uint64_t>(limit - 1),
|
||||
watermark.sequence - first_sequence);
|
||||
for (auto sequence = first_sequence;; ++sequence) {
|
||||
auto entry = observe_commit(db, objects, key, lease, sequence, retained_builds, timeout);
|
||||
if (!entry)
|
||||
return std::unexpected(entry.error());
|
||||
page.entries.push_back(std::move(*entry));
|
||||
if (sequence == last)
|
||||
break;
|
||||
}
|
||||
if (last < watermark.sequence)
|
||||
page.next_sequence = last + 1;
|
||||
const auto confirmed = load_region_commit(db, key, watermark.sequence, timeout);
|
||||
if (!confirmed)
|
||||
return failed(RecoveryFailure{RecoveryError::store, confirmed.error()});
|
||||
if (confirmed->head != watermark || confirmed->manifest != upper->manifest)
|
||||
return std::unexpected(CellularHistoryFailure{CellularHistoryError::cut_mismatch, {}});
|
||||
return page;
|
||||
}
|
||||
} // namespace evo::persistence
|
||||
@@ -248,12 +248,13 @@ verify_population_profile_transition(Postgres& db, const FilesystemObjectStore&
|
||||
return fail(RecoveryError::unsupported);
|
||||
return {};
|
||||
}
|
||||
namespace {
|
||||
std::expected<RecoveredPopulationCheckpoint, RecoveryFailure>
|
||||
recover_population_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
const LeaseGrant& lease,
|
||||
std::span<const Sha256Digest> retained_builds, milliseconds timeout) {
|
||||
recover_population_loaded(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
const LeaseGrant& lease, std::span<const Sha256Digest> retained_builds,
|
||||
milliseconds timeout,
|
||||
std::expected<CommittedCheckpoint, LeaseStoreFailure> committed) {
|
||||
const auto fail = [](RecoveryError kind) { return std::unexpected(RecoveryFailure{kind, {}}); };
|
||||
auto committed = load_region_checkpoint(db, key, timeout);
|
||||
if (!committed)
|
||||
return std::unexpected(
|
||||
RecoveryFailure{committed.error().kind == LeaseStoreError::not_provisioned
|
||||
@@ -319,4 +320,19 @@ recover_population_checkpoint(Postgres& db, const FilesystemObjectStore& objects
|
||||
}
|
||||
return RecoveredPopulationCheckpoint{std::move(*committed), *m, std::move(*population)};
|
||||
}
|
||||
} // namespace
|
||||
std::expected<RecoveredPopulationCheckpoint, RecoveryFailure>
|
||||
recover_population_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
const LeaseGrant& lease,
|
||||
std::span<const Sha256Digest> retained_builds, milliseconds timeout) {
|
||||
return recover_population_loaded(db, objects, key, lease, retained_builds, timeout,
|
||||
load_region_checkpoint(db, key, timeout));
|
||||
}
|
||||
std::expected<RecoveredPopulationCheckpoint, RecoveryFailure>
|
||||
recover_population_commit(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
std::uint64_t sequence, const LeaseGrant& lease,
|
||||
std::span<const Sha256Digest> retained_builds, milliseconds timeout) {
|
||||
return recover_population_loaded(db, objects, key, lease, retained_builds, timeout,
|
||||
load_region_commit(db, key, sequence, timeout));
|
||||
}
|
||||
} // namespace evo::persistence
|
||||
|
||||
@@ -119,6 +119,10 @@ struct Postgres::Impl {
|
||||
};
|
||||
Postgres::Postgres(std::unique_ptr<Impl> impl) : impl_(std::move(impl)) {}
|
||||
Postgres::~Postgres() = default;
|
||||
bool Postgres::transaction_idle() const noexcept {
|
||||
return impl_->connection && PQstatus(impl_->connection.get()) == CONNECTION_OK &&
|
||||
PQtransactionStatus(impl_->connection.get()) == PQTRANS_IDLE;
|
||||
}
|
||||
std::expected<PgOptions, PgError> read_pg_service(const std::string& file,
|
||||
std::string_view section) {
|
||||
constexpr std::size_t limit = 4096;
|
||||
|
||||
Reference in New Issue
Block a user