M12.3: publish and recover durable exact populations
Foundation / checks (ecs) (push) Successful in 2m41s
Foundation / checks (tsan) (push) Failing after 18m12s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (network) (push) Canceled after 20m41s
Foundation / checks (postgres) (push) Canceled after 20m43s
Foundation / checks (checks) (push) Canceled after 20m49s
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (client) (push) Failing after 17m4s
Foundation / postgres-tools (push) Canceled after 0s

This commit is contained in:
dingo
2026-10-17 17:10:17 -04:00
parent d5d10a0cc7
commit 627411983f
35 changed files with 2637 additions and 102 deletions
@@ -0,0 +1,54 @@
#pragma once
#include "evo/persistence/checkpoint_store.hpp"
namespace evo::persistence {
struct PopulationPreparedCommit {
RegionKey key{};
Uuid commit{};
std::vector<std::uint8_t> manifest, trace; // Empty trace means a profile transition.
};
struct PopulationPublicationFailure {
PublicationError kind{};
LeaseStoreFailure store{};
ObjectError object{};
PopulationPreparedCommit prepared;
};
struct RecoveredPopulationCheckpoint {
CommittedCheckpoint committed;
CheckpointManifest manifest;
std::unique_ptr<server::CellularPopulation> state;
};
// Explicit no-tick transitions, under the same lease/head fence as publication.
[[nodiscard]] std::expected<CommittedCheckpoint, PopulationPublicationFailure>
migrate_cellular_to_population(Postgres&, const FilesystemObjectStore&, const LeaseGrant&,
const CommittedCheckpoint&, Uuid commit, const Sha256Digest& build,
server::CellularTierPolicy, std::chrono::milliseconds timeout,
const PublishHooks& hooks = {});
[[nodiscard]] std::expected<CommittedCheckpoint, PopulationPublicationFailure>
materialize_population_profile(Postgres&, const FilesystemObjectStore&, const LeaseGrant&,
const CommittedCheckpoint&, Uuid commit, const Sha256Digest& build,
std::chrono::milliseconds timeout, const PublishHooks& hooks = {});
[[nodiscard]] std::expected<CommittedCheckpoint, PopulationPublicationFailure>
publish_population_batch(Postgres&, const FilesystemObjectStore&, const LeaseGrant&,
const CommittedCheckpoint&, const server::CellularPopulationBatch&,
const server::CellularPopulation& candidate, Uuid commit,
const Sha256Digest& build, std::chrono::milliseconds timeout,
const PublishHooks& hooks = {});
[[nodiscard]] std::expected<CommittedCheckpoint, PopulationPublicationFailure>
retry_population_prepared(Postgres&, const PopulationPreparedCommit&,
std::chrono::milliseconds timeout);
[[nodiscard]] std::expected<ResolvedCommit, LeaseStoreFailure>
resolve_population_prepared(Postgres&, const PopulationPreparedCommit&,
std::chrono::milliseconds timeout);
[[nodiscard]] std::expected<RecoveredPopulationCheckpoint, RecoveryFailure>
recover_population_checkpoint(Postgres&, const FilesystemObjectStore&, RegionKey, const LeaseGrant&,
std::span<const Sha256Digest> retained_builds,
std::chrono::milliseconds timeout);
// Used by raw cellular recovery after a population -> individual profile transition.
[[nodiscard]] std::expected<void, RecoveryFailure>
verify_population_profile_transition(Postgres&, const FilesystemObjectStore&, const LeaseGrant&,
const CommittedCheckpoint&, const CheckpointManifest&,
std::span<const std::uint8_t> payload,
std::chrono::milliseconds timeout);
} // namespace evo::persistence
+21 -1
View File
@@ -5,6 +5,7 @@
#include "evo/processes/lineage_region.hpp"
#include "evo/server/lifecycle/cellular_batch.hpp"
#include "evo/server/lifecycle/cellular_lives.hpp"
#include "evo/server/lifecycle/cellular_population_batch.hpp"
#include "evo/server/lifecycle/primitive_batch.hpp"
#include <cstddef>
#include <cstdint>
@@ -25,12 +26,14 @@ enum class CheckpointProfile : std::uint8_t {
environmental_lineage_v1 = 2, // LineageRegion container; history_profile 1.
primitive_lives_v1 = 3, // Funded creation state; history_profile 2.
cellular_lives_v1 = 4, // Complete cellular state; history_profile 3.
cellular_population_v1 = 5, // Population envelope 1; nested cellular bundle 9.
};
// history_profile 1: lineage archive format 1 with genome schema 1.
inline constexpr std::uint32_t lineage_history_profile = 1;
// Archive format 1, genome schema 2, development 2, founder-life rules 1.
inline constexpr std::uint32_t primitive_history_profile = 2;
inline constexpr std::uint32_t cellular_history_profile = 3;
inline constexpr std::uint32_t population_history_profile = 5;
inline constexpr std::uint32_t cellular_revision_history_profile = 4;
// Declared durability, verified at deployment. Development: one volume, no host-loss claim.
enum class DurabilityPolicy : std::uint8_t { unknown = 0, single_volume_development = 1 };
@@ -119,7 +122,9 @@ inline constexpr std::size_t max_lineage_payload_bytes = processes::max_lineage_
: profile == CheckpointProfile::environmental_lineage_v1 ? max_lineage_payload_bytes
: profile == CheckpointProfile::primitive_lives_v1 ? server::max_primitive_lives_bytes
: profile == CheckpointProfile::cellular_lives_v1 ? server::max_cellular_lives_bytes
: 0;
: profile == CheckpointProfile::cellular_population_v1
? server::max_cellular_population_bytes
: 0;
}
inline constexpr std::uint64_t max_checkpoint_ticks = 64;
@@ -189,4 +194,19 @@ restore_primitive_payload(std::span<const std::uint8_t> payload,
const CheckpointManifest& manifest);
[[nodiscard]] std::expected<std::unique_ptr<server::CellularLives>, CheckpointError>
restore_cellular_payload(std::span<const std::uint8_t> payload, const CheckpointManifest& manifest);
[[nodiscard]] std::expected<std::unique_ptr<server::CellularPopulation>, CheckpointError>
restore_population_payload(std::span<const std::uint8_t> payload,
const CheckpointManifest& manifest);
// Dedicated profile 4/bundle 9 <-> profile 5 transition, no biological tick.
[[nodiscard]] std::expected<CheckpointManifest, CheckpointError>
population_profile_transition_manifest(const CheckpointManifest& predecessor,
const Sha256Digest& predecessor_digest, Uuid commit,
const LeaseGrant& lease, const Sha256Digest& build,
std::uint64_t payload_bytes,
const Sha256Digest& payload_sha256);
// Population manual conversion may publish at the same tick with a nonempty trace.
[[nodiscard]] std::expected<CheckpointManifest, CheckpointError> population_successor_manifest(
const CheckpointManifest& predecessor, const Sha256Digest& predecessor_digest, Uuid commit,
const LeaseGrant& lease, std::uint64_t end_tick, const Sha256Digest& build,
std::uint64_t payload_bytes, const Sha256Digest& payload_sha256, const BatchDigests& batch);
} // namespace evo::persistence
@@ -217,4 +217,23 @@ pending_region_outbox(Postgres& db, RegionKey key, std::uint32_t limit,
[[nodiscard]] std::expected<bool, LeaseStoreFailure>
acknowledge_region_outbox(Postgres& db, RegionKey key, const BatchDecision& decision,
std::chrono::milliseconds timeout);
struct PopulationEvidence {
std::vector<std::uint8_t> trace;
std::optional<Uuid> source_commit;
Sha256Digest source_manifest_sha256{}, source_payload_sha256{};
};
[[nodiscard]] std::expected<RegionHead, LeaseStoreFailure>
publish_population_checkpoint(Postgres&, std::span<const std::uint8_t> manifest,
std::span<const std::uint8_t> trace,
std::chrono::milliseconds timeout);
[[nodiscard]] std::expected<RegionHead, LeaseStoreFailure>
publish_population_profile_transition(Postgres&, std::span<const std::uint8_t> manifest,
std::chrono::milliseconds timeout);
[[nodiscard]] std::expected<ResolvedCommit, LeaseStoreFailure>
resolve_population_commit(Postgres&, RegionKey, const Uuid& commit,
std::span<const std::uint8_t> manifest,
std::span<const std::uint8_t> trace, std::chrono::milliseconds timeout);
[[nodiscard]] std::expected<std::optional<PopulationEvidence>, LeaseStoreFailure>
load_population_evidence(Postgres&, RegionKey, const LeaseGrant&, const Uuid& commit,
std::chrono::milliseconds timeout);
} // namespace evo::persistence
+50 -2
View File
@@ -3,6 +3,7 @@
#include "evo/network/session_protocol.hpp"
#include "evo/persistence/checkpoint_store.hpp"
#include "evo/persistence/life_reservation_store.hpp"
#include "evo/persistence/population_checkpoint_store.hpp"
#include "evo/server/lifecycle/cellular_batch.hpp"
#include "evo/server/ops.hpp"
@@ -13,6 +14,7 @@
#include <memory>
#include <optional>
#include <span>
#include <utility>
#include <vector>
namespace evo::server {
@@ -24,6 +26,7 @@ struct CellularHostConfig {
persistence::LeasePolicy lease{};
persistence::Sha256Digest build{};
std::vector<persistence::Sha256Digest> retained_builds;
std::optional<CellularTierPolicy> population_policy{}; // Opt-in profile-5 entry.
};
enum class CellularHostError : std::uint8_t {
configuration,
@@ -74,7 +77,7 @@ struct CellularMutationReply {
[[nodiscard]] network::CellularMutationOffer
offer_view(const CellularLives& state, const CellularOwnedLife& life, std::uint64_t tick);
// Single-writer profile-4 authority. The caller supplies a complete bounded trace;
// Single-writer profile-4/5 authority. The caller supplies a complete bounded trace;
// the host replays it from its recovered state and advances only on a confirmed head.
class CellularRegionHost {
public:
@@ -87,6 +90,9 @@ class CellularRegionHost {
// Acquires/renews authority, recovers fully, or resolves the exact pending commit.
const HostStatus& step();
// Explicit no-tick fenced transition. Uncertain publication pauses the host.
[[nodiscard]] std::expected<void, CellularPublishFailure> set_population_mode(bool enabled);
[[nodiscard]] bool population_mode() const noexcept;
[[nodiscard]] std::expected<persistence::PublishedCellularBatch, CellularPublishFailure>
publish(const CellularBatch& batch, std::span<const CellularActionRecord> actions = {},
std::span<const CellularMutationActionRecord> mutation_actions = {});
@@ -140,6 +146,9 @@ class CellularRegionHost {
void upkeep();
void resolve();
void refresh();
bool recover_state(const persistence::LeaseGrant&);
[[nodiscard]] std::expected<persistence::PublishedCellularBatch, CellularPublishFailure>
publish_population(const CellularBatch&);
void finalize_births(std::span<const CellularTickIntents::Birth> births);
void queue_birth(const CellularTickIntents::Birth& birth);
void reload_mutations();
@@ -158,7 +167,46 @@ class CellularRegionHost {
std::unique_ptr<persistence::FilesystemObjectStore> objects_;
std::unique_ptr<persistence::Postgres> db_;
persistence::LeaseAuthority authority_;
std::unique_ptr<CellularLives> state_;
// Exactly one owner. The common interface is a const biological view, never
// a second independently spendable cellular copy alongside a population.
struct OwnedState {
std::unique_ptr<CellularLives> individual;
std::unique_ptr<CellularPopulation> population;
OwnedState() = default;
OwnedState(std::unique_ptr<CellularLives> state) : individual(std::move(state)) {}
OwnedState(std::unique_ptr<CellularPopulation> state) : population(std::move(state)) {}
OwnedState& operator=(std::unique_ptr<CellularLives> state) {
population.reset();
individual = std::move(state);
return *this;
}
OwnedState& operator=(std::unique_ptr<CellularPopulation> state) {
individual.reset();
population = std::move(state);
return *this;
}
void reset() {
individual.reset();
population.reset();
}
const CellularLives* get() const {
return population ? &population->state() : individual.get();
}
const CellularLives* operator->() const {
return get();
}
const CellularLives& operator*() const {
return *get();
}
explicit operator bool() const {
return get() != nullptr;
}
};
[[nodiscard]] std::expected<persistence::CommittedCheckpoint, CellularPublishFailure>
accept_population_publication(
std::expected<persistence::CommittedCheckpoint, persistence::PopulationPublicationFailure>,
OwnedState candidate);
OwnedState state_;
std::optional<persistence::CommittedCheckpoint> committed_;
std::unique_ptr<Pending> pending_;
std::vector<FinalCellularAction> finalized_actions_;
@@ -10,6 +10,7 @@
#include <vector>
namespace evo::server {
struct CellularPopulationBatch;
// ADR-0139: exact local cohorts retain individuals, never mean reserves or samples.
// The single owned CellularLives store backs BOTH representations, exactly once.
struct CellularCohortMember {
@@ -72,6 +73,8 @@ class CellularPopulation {
std::span<const CellularMutationSelection> selections = {});
private:
friend std::expected<std::unique_ptr<CellularPopulation>, CellularPopulationError>
execute_population_batch(const CellularPopulation&, CellularPopulationBatch&, bool);
explicit CellularPopulation(std::unique_ptr<CellularLives> state, CellularTierPolicy policy);
void apply_policy();
std::unique_ptr<CellularLives> state_;
@@ -0,0 +1,49 @@
#pragma once
#include "evo/server/lifecycle/cellular_batch.hpp"
#include "evo/server/lifecycle/cellular_population.hpp"
namespace evo::server {
struct CellularPopulationBoundary {
std::uint64_t sequence{};
std::vector<CellularCohortMember> members;
bool operator==(const CellularPopulationBoundary&) const = default;
};
struct CellularPopulationConversion {
std::uint64_t genome{};
bool to_cohort{};
bool operator==(const CellularPopulationConversion&) const = default;
};
struct CellularPopulationBatch {
CellularBatchHeader header;
// Exactly one: 1..64 unattended biological ticks, or a no-tick whole-group conversion.
CellularBatch biology;
std::optional<CellularPopulationConversion> conversion;
std::vector<CellularPopulationBoundary> boundaries;
bool operator==(const CellularPopulationBatch&) const = default;
};
inline constexpr std::size_t population_batch_header_bytes = 136;
inline constexpr std::size_t max_population_batch_bytes =
population_batch_header_bytes + max_cellular_batch_bytes +
max_cellular_batch_ticks * (16 + 16 * biology::max_cellular_tick_lives);
struct CellularPopulationExecution {
CellularPopulationBatch batch;
std::unique_ptr<CellularPopulation> state;
};
// Build an unattended interval from a detached biological copy. Controlled cells
// receive frozen-only intentions; the unchanged wildlife controller supplies the
// rest. The returned recorded intentions replay without rerunning that controller.
[[nodiscard]] std::expected<CellularBatch, CellularPopulationError>
unattended_population_inputs(const CellularLives&, const CellularBatchHeader&);
[[nodiscard]] std::expected<CellularPopulationExecution, CellularPopulationError>
prepare_population_batch(const CellularPopulation& base, CellularBatch biology);
[[nodiscard]] std::expected<CellularPopulationExecution, CellularPopulationError>
prepare_population_conversion(const CellularPopulation& base, CellularBatchHeader header,
CellularPopulationConversion conversion);
[[nodiscard]] std::expected<std::unique_ptr<CellularPopulation>, CellularPopulationError>
replay_population_batch(const CellularPopulation& base, const CellularPopulationBatch& batch);
[[nodiscard]] std::expected<std::vector<std::uint8_t>, CellularPopulationError>
encode_population_batch(const CellularPopulationBatch& batch);
[[nodiscard]] std::expected<CellularPopulationBatch, CellularPopulationError>
decode_population_batch(std::span<const std::uint8_t> bytes);
} // namespace evo::server