M12.3: publish and recover durable exact populations
Foundation / checks (ecs) (push) Successful in 2m41s
Foundation / checks (tsan) (push) Failing after 18m12s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (network) (push) Canceled after 20m41s
Foundation / checks (postgres) (push) Canceled after 20m43s
Foundation / checks (checks) (push) Canceled after 20m49s
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (client) (push) Failing after 17m4s
Foundation / postgres-tools (push) Canceled after 0s

This commit is contained in:
dingo
2026-10-17 17:10:17 -04:00
parent d5d10a0cc7
commit 627411983f
35 changed files with 2637 additions and 102 deletions
@@ -0,0 +1,296 @@
#include "evo/server/lifecycle/cellular_population_batch.hpp"
#include "evo/server/lifecycle/wildlife_controller.hpp"
#include <algorithm>
#include <utility>
namespace evo::server {
namespace {
constexpr std::uint64_t magic = 0x315442504F564541ULL; // "AEVOPBT1"
bool shape(const CellularPopulationBatch& batch, bool recording) {
const auto& h = batch.header;
const auto zero = [](const auto& bytes) {
return std::ranges::all_of(bytes, [](auto byte) { return byte == 0; });
};
if (h.world == 0 || h.region == 0 || h.frame == 0 || zero(h.predecessor) ||
zero(h.predecessor_digest) || h.end_tick < h.begin_tick)
return false;
const auto count = batch.conversion ? std::size_t{1} : batch.biology.ticks.size();
if (count == 0 || count > max_cellular_batch_ticks ||
(!recording && batch.boundaries.size() != count))
return false;
if (batch.conversion) {
if (h.begin_tick == 0 || h.begin_tick != h.end_tick || batch.conversion->genome == 0 ||
!batch.biology.ticks.empty() || batch.biology.header != CellularBatchHeader{})
return false;
} else {
if (batch.biology.header != h || batch.biology.format_version != cellular_batch_format ||
h.end_tick - h.begin_tick != count || !encode_cellular_batch(batch.biology))
return false;
for (const auto& tick : batch.biology.ticks)
if (!tick.births.empty() || !tick.mutations.empty())
return false;
}
for (const auto& boundary : batch.boundaries) {
if (boundary.members.size() > biology::max_cellular_tick_lives)
return false;
std::uint64_t previous = 0;
for (const auto& member : boundary.members) {
if (member.life <= previous || member.genome == 0)
return false;
previous = member.life;
}
}
return true;
}
void put(std::vector<std::uint8_t>& out, std::uint64_t value, unsigned width = 8) {
for (unsigned i = 0; i < width; ++i) {
out.push_back(static_cast<std::uint8_t>(value & 255U));
value >>= 8U;
}
}
class Reader {
public:
explicit Reader(std::span<const std::uint8_t> bytes) : bytes_(bytes) {}
std::uint64_t get(unsigned width = 8) {
std::uint64_t value = 0;
for (unsigned i = 0; i < width; ++i)
value |= std::uint64_t{bytes_[at_++]} << (8U * i);
return value;
}
template <std::size_t N> std::array<std::uint8_t, N> array() {
std::array<std::uint8_t, N> value{};
for (auto& byte : value)
byte = bytes_[at_++];
return value;
}
std::size_t at() const {
return at_;
}
private:
std::span<const std::uint8_t> bytes_;
std::size_t at_{};
};
} // namespace
std::expected<CellularBatch, CellularPopulationError>
unattended_population_inputs(const CellularLives& base, const CellularBatchHeader& header) {
if (header.world != base.world() || header.region != base.region() ||
header.frame != base.frame() || header.begin_tick != base.tick() ||
header.end_tick <= header.begin_tick ||
header.end_tick - header.begin_tick > max_cellular_batch_ticks)
return std::unexpected(CellularPopulationError::identity);
auto working = CellularLives::restore(base.snapshot(), base.world(), base.region(),
base.frame(), base.tick());
if (!working)
return std::unexpected(CellularPopulationError::format);
CellularBatch batch{header, {}};
while ((*working)->tick() < header.end_tick) {
auto intents = wildlife_intents(**working, (*working)->tick() + 1);
if (!intents)
return std::unexpected(CellularPopulationError::ineligible);
for (const auto& life : (*working)->lives()) {
if (life.account == LifeAccount{})
continue;
CellularLifeIntent frozen{life.identity.life, {}};
frozen.intent.frozen = true;
intents->push_back(frozen);
}
std::ranges::sort(*intents, {}, &CellularLifeIntent::life_id);
if (!(*working)->advance_tick(*intents))
return std::unexpected(CellularPopulationError::ineligible);
batch.ticks.push_back({std::move(*intents), {}, {}});
}
if (!encode_cellular_batch(batch))
return std::unexpected(CellularPopulationError::format);
return batch;
}
std::expected<std::unique_ptr<CellularPopulation>, CellularPopulationError>
execute_population_batch(const CellularPopulation& base, CellularPopulationBatch& batch,
bool record) {
if (!shape(batch, record) || batch.header.world != base.state().world() ||
batch.header.region != base.state().region() ||
batch.header.frame != base.state().frame() ||
batch.header.begin_tick != base.state().tick())
return std::unexpected(CellularPopulationError::identity);
auto candidate =
CellularPopulation::restore(base.snapshot(), base.state().world(), base.state().region(),
base.state().frame(), base.state().tick());
if (!candidate)
return std::unexpected(candidate.error());
if (record)
batch.boundaries.clear();
const auto check_boundary = [&](std::size_t index) {
CellularPopulationBoundary actual{(*candidate)->conversion_sequence_,
(*candidate)->members_};
if (record) {
batch.boundaries.push_back(std::move(actual));
return true;
}
return batch.boundaries[index] == actual;
};
if (batch.conversion) {
const auto converted = batch.conversion->to_cohort
? (*candidate)->collapse(batch.conversion->genome)
: (*candidate)->materialize(batch.conversion->genome);
if (!converted || !check_boundary(0))
return std::unexpected(CellularPopulationError::ineligible);
} else {
for (std::size_t index = 0; index < batch.biology.ticks.size(); ++index) {
const auto& input = batch.biology.ticks[index];
for (const auto& life : (*candidate)->state().lives()) {
if (life.account == LifeAccount{})
continue;
const auto intent = std::ranges::find(input.lives, life.identity.life,
&CellularLifeIntent::life_id);
biology::CellularStepIntent frozen;
frozen.frozen = true;
if (intent == input.lives.end() || intent->intent != frozen)
return std::unexpected(CellularPopulationError::ineligible);
}
CellularBatch one{batch.biology.header, {input}, cellular_batch_format};
one.header.begin_tick = (*candidate)->state().tick();
one.header.end_tick = one.header.begin_tick + 1;
auto biology = replay_cellular_batch((*candidate)->state(), one);
if (!biology)
return std::unexpected(CellularPopulationError::ineligible);
(*candidate)->state_ = std::move(*biology);
std::erase_if((*candidate)->members_, [&](const auto& member) {
return std::ranges::find((*candidate)->state().lives(), member.life,
[](const auto& life) { return life.identity.life; }) ==
(*candidate)->state().lives().end();
});
// Verify the persisted policy's exact result against the recorded sample;
// no controller or random materialization is run during replay.
if ((*candidate)->policy_.automatic) {
if ((*candidate)->conversion_sequence_ == ~std::uint64_t{0})
return std::unexpected(CellularPopulationError::overflow);
(*candidate)->apply_policy();
}
if (!check_boundary(index))
return std::unexpected(CellularPopulationError::ineligible);
}
}
return candidate;
}
std::expected<CellularPopulationExecution, CellularPopulationError>
prepare_population_batch(const CellularPopulation& base, CellularBatch biology) {
CellularPopulationBatch batch{biology.header, std::move(biology), {}, {}};
auto state = execute_population_batch(base, batch, true);
if (!state)
return std::unexpected(state.error());
return CellularPopulationExecution{std::move(batch), std::move(*state)};
}
std::expected<CellularPopulationExecution, CellularPopulationError>
prepare_population_conversion(const CellularPopulation& base, CellularBatchHeader header,
CellularPopulationConversion conversion) {
CellularPopulationBatch batch{header, {}, conversion, {}};
auto state = execute_population_batch(base, batch, true);
if (!state)
return std::unexpected(state.error());
return CellularPopulationExecution{std::move(batch), std::move(*state)};
}
std::expected<std::unique_ptr<CellularPopulation>, CellularPopulationError>
replay_population_batch(const CellularPopulation& base, const CellularPopulationBatch& batch) {
auto copy = batch;
return execute_population_batch(base, copy, false);
}
std::expected<std::vector<std::uint8_t>, CellularPopulationError>
encode_population_batch(const CellularPopulationBatch& batch) {
if (!shape(batch, false))
return std::unexpected(CellularPopulationError::format);
std::vector<std::uint8_t> biological;
if (!batch.conversion) {
auto encoded = encode_cellular_batch(batch.biology);
if (!encoded)
return std::unexpected(CellularPopulationError::format);
biological = std::move(*encoded);
}
std::vector<std::uint8_t> out;
put(out, magic);
put(out, 1, 4);
put(out, batch.conversion ? 1 : 0, 4);
const auto& h = batch.header;
for (const auto value : {h.world, h.region, h.frame})
put(out, value);
out.insert(out.end(), h.predecessor.begin(), h.predecessor.end());
out.insert(out.end(), h.predecessor_digest.begin(), h.predecessor_digest.end());
put(out, h.begin_tick);
put(out, h.end_tick);
put(out, batch.boundaries.size(), 4);
put(out, 0, 4);
put(out, biological.size());
put(out, batch.conversion ? batch.conversion->genome : 0);
put(out, batch.conversion && batch.conversion->to_cohort ? 1 : 0);
out.insert(out.end(), biological.begin(), biological.end());
for (const auto& boundary : batch.boundaries) {
put(out, boundary.sequence);
put(out, boundary.members.size());
for (const auto& member : boundary.members) {
put(out, member.life);
put(out, member.genome);
}
}
return out;
}
std::expected<CellularPopulationBatch, CellularPopulationError>
decode_population_batch(std::span<const std::uint8_t> bytes) {
if (bytes.size() < population_batch_header_bytes || bytes.size() > max_population_batch_bytes)
return std::unexpected(CellularPopulationError::format);
Reader in(bytes);
if (in.get() != magic || in.get(4) != 1)
return std::unexpected(CellularPopulationError::unsupported);
const auto kind = in.get(4);
CellularPopulationBatch batch;
auto& h = batch.header;
h.world = in.get();
h.region = in.get();
h.frame = in.get();
h.predecessor = in.array<16>();
h.predecessor_digest = in.array<32>();
h.begin_tick = in.get();
h.end_tick = in.get();
const auto count = in.get(4);
if (in.get(4) != 0 || kind > 1 || count == 0 || count > max_cellular_batch_ticks)
return std::unexpected(CellularPopulationError::format);
const auto size = in.get(), genome = in.get(), to_cohort = in.get();
if (size > max_cellular_batch_bytes || size > bytes.size() - population_batch_header_bytes ||
to_cohort > 1 || (kind == 0 && (genome != 0 || to_cohort != 0)) || (kind == 1 && size != 0))
return std::unexpected(CellularPopulationError::format);
if (kind == 1)
batch.conversion = {genome, to_cohort == 1};
else {
auto biological = decode_cellular_batch(
bytes.subspan(population_batch_header_bytes, static_cast<std::size_t>(size)));
if (!biological)
return std::unexpected(CellularPopulationError::format);
batch.biology = std::move(*biological);
}
std::size_t at = population_batch_header_bytes + static_cast<std::size_t>(size);
for (std::uint64_t index = 0; index < count; ++index) {
if (bytes.size() - at < 16)
return std::unexpected(CellularPopulationError::format);
Reader boundary_reader(bytes.subspan(at));
CellularPopulationBoundary boundary;
boundary.sequence = boundary_reader.get();
const auto members = boundary_reader.get();
if (members > biology::max_cellular_tick_lives || members * 16 > bytes.size() - at - 16)
return std::unexpected(CellularPopulationError::format);
for (std::uint64_t member = 0; member < members; ++member)
boundary.members.push_back({boundary_reader.get(), boundary_reader.get()});
at += boundary_reader.at();
batch.boundaries.push_back(std::move(boundary));
}
if (at != bytes.size() || !shape(batch, false))
return std::unexpected(CellularPopulationError::format);
const auto encoded = encode_population_batch(batch);
if (!encoded || !std::ranges::equal(*encoded, bytes))
return std::unexpected(CellularPopulationError::format);
return batch;
}
} // namespace evo::server