feat(server): commit cellular batches on a background thread
Foundation / checks (checks) (push) Successful in 5s
Foundation / checks (client) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 4s
Foundation / checks (network) (push) Successful in 4s
Foundation / checks (postgres) (push) Successful in 4s
Foundation / checks (tsan) (push) Successful in 4s
Foundation / checks (deploy) (push) Successful in 4m28s
Foundation / postgres-tools (push) Successful in 23s
Foundation / server-image (push) Successful in 4m27s
Foundation / image (push) Successful in 4m31s

The rules-12 loop blocked for ~411 ms per 20-tick commit, so players saw
one-second bursts. The host now commits a batch on a worker thread with
its own connection (begin_publish / poll_publish), using the builder's
already-simulated candidate. The next batch builds on that candidate
under a provisional head and is rebased when the commit lands. One commit
is in flight at a time; births and mutation choices wait for a batch that
starts on the durable head, and host reconciliation pauses meanwhile.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-10-22 08:41:56 -04:00
co-authored by Claude Opus 5.5
parent 4112bef52c
commit ff333d97c2
6 changed files with 403 additions and 37 deletions
+102 -26
View File
@@ -942,6 +942,20 @@ int serve_cellular(Options& o) {
std::vector<network::CarriedAction> batch_actions;
std::uint64_t pace_epoch = 0, pace_tick = 0, flushes_logged = 0;
std::vector<network::CarriedAction> awaiting_outcomes;
// The batch being committed on the host's worker thread while the next one builds on its
// candidate: its carried actions, birth refusals and operator timing.
std::vector<network::CarriedAction> flight_actions;
std::vector<server::CellularBirthRefusal> flight_refusals;
std::size_t flight_ticks = 0;
long long flight_handoff_ms = 0;
auto flight_start = std::chrono::steady_clock::now();
const auto drop_carried = [&](std::vector<network::CarriedAction>& carried) {
bridge->dropped(carried, true);
std::erase_if(awaiting_outcomes, [&](const auto& action) {
return std::ranges::find(carried, action) != carried.end();
});
carried.clear();
};
// A tick that cannot be built is retried every 50 ms; name each distinct cause once.
std::optional<std::pair<std::string_view, server::CellularSessionBatchError>> stalled;
const auto report_stall = [&](std::string_view where, server::CellularSessionBatchError error,
@@ -1008,6 +1022,35 @@ int serve_cellular(Options& o) {
host.drain();
return finish(0);
}
// A finished background commit lands first, so this step sees its head.
if (auto landed = host.poll_publish(); landed) {
const auto commit_ms = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now() - flight_start)
.count();
// Operator evidence: the loop is blocked only for the hand-off, not the commit.
if (flight_handoff_ms > 50 || !*landed || ++flushes_logged % 30 == 0)
std::printf("cellular commit ticks=%zu ms=%lld handoff_ms=%lld ok=%d\n",
flight_ticks, static_cast<long long>(commit_ms), flight_handoff_ms,
*landed ? 1 : 0);
if (*landed) {
host.refuse_births(flight_refusals);
flight_actions.clear();
if (batch_builder && !batch_builder->rebase(host.committed()->head)) {
drop_carried(batch_actions);
batch_builder.reset();
}
} else {
// An unresolved outcome keeps its actions for the host to finalize.
if (host.status().phase == server::HostPhase::resolving)
flight_actions.clear();
else
drop_carried(flight_actions);
// The open batch continued from a state that is not durable.
drop_carried(batch_actions);
batch_builder.reset();
}
flight_refusals.clear();
}
const auto& status = host.step();
const auto wall =
static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::seconds>(
@@ -1115,34 +1158,60 @@ int serve_cellular(Options& o) {
}
const auto flush_ticks = std::clamp<std::uint64_t>(
static_cast<std::uint64_t>(o.host.flush_interval.count()) / 50, 1, 60);
// Births and mutation choices wait for a batch that starts on the durable head.
const bool admissions_waiting = !held_births.empty() || !host.queued_mutations().empty();
// Write-behind commit: the open batch goes to the host's worker thread and the next
// batch continues from its candidate under a provisional head, rebased when the commit
// lands. One commit is in flight; meanwhile the open batch simply keeps growing.
const auto flush = [&] {
if (!batch_builder || batch_builder->batch().ticks.empty()) {
batch_builder.reset();
if (!host.commit_in_flight())
batch_builder.reset();
return;
}
const auto flush_start = std::chrono::steady_clock::now();
const auto flush_ticks_held = batch_builder->batch().ticks.size();
auto published = host.publish(batch_builder->batch(), batch_builder->actions(),
batch_builder->mutation_actions());
// Operator evidence for commit stalls: every slow commit, and one in 60 otherwise.
const auto flush_ms = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now() - flush_start)
.count();
if (flush_ms > 50 || ++flushes_logged % 60 == 0)
std::printf("cellular commit ticks=%zu ms=%lld ok=%d\n", flush_ticks_held,
static_cast<long long>(flush_ms), published ? 1 : 0);
if (published)
host.refuse_births(batch_builder->refused_births());
if (!published && host.status().phase != server::HostPhase::resolving) {
bridge->dropped(batch_actions, true);
std::erase_if(awaiting_outcomes, [&](const auto& action) {
return std::ranges::find(batch_actions, action) != batch_actions.end();
});
if (host.commit_in_flight())
return;
const auto handoff_start = std::chrono::steady_clock::now();
const auto& candidate = batch_builder->candidate();
std::optional<server::CellularSessionBatchBuilder> next;
if (!admissions_waiting && candidate.tick() != UINT64_MAX) {
persistence::RegionHead provisional{candidate.frame(),
host.committed()->head.sequence + 1,
{},
{},
candidate.tick()};
provisional.commit.fill(0xFF);
provisional.digest.fill(0xFF);
if (auto created =
server::CellularSessionBatchBuilder::create(candidate, provisional);
created)
next.emplace(std::move(*created));
}
flight_ticks = batch_builder->batch().ticks.size();
const auto refusals = batch_builder->refused_births();
flight_refusals.assign(refusals.begin(), refusals.end());
const auto actions = batch_builder->actions();
const auto mutation_actions = batch_builder->mutation_actions();
auto begun =
host.begin_publish(batch_builder->batch(), {actions.begin(), actions.end()},
{mutation_actions.begin(), mutation_actions.end()},
batch_builder->take_candidate());
batch_builder.reset();
flight_actions = std::move(batch_actions);
batch_actions.clear();
deliver_finalized();
state = host.state(); // Publication replaced the committed state.
flight_start = std::chrono::steady_clock::now();
flight_handoff_ms =
std::chrono::duration_cast<std::chrono::milliseconds>(flight_start - handoff_start)
.count();
if (!begun) {
std::printf("cellular commit refused handoff_ms=%lld error=%u\n", flight_handoff_ms,
static_cast<unsigned>(begun.error().kind));
drop_carried(flight_actions);
flight_refusals.clear();
return;
}
if (next)
batch_builder = std::move(next);
};
// A batch cannot outlive the authority or the raw mode it was built under; when demand
// ends, what it holds is committed.
@@ -1168,11 +1237,16 @@ int serve_cellular(Options& o) {
// large: commit every flush interval (ADR-0089: 1 s), not every tick. Players
// see the batch's working state at once; outcomes become final at its commit.
// Births and mutation choices commit promptly, in their own fresh batch.
if (batch_builder && (!held_births.empty() || !host.queued_mutations().empty()))
if (batch_builder && admissions_waiting)
flush();
const auto* current = batch_builder ? &batch_builder->candidate() : state;
// A failed commit can leave the host without state until it recovers.
if (current && host.committed() && paced->due && needed &&
// A failed commit can leave the host without state until it recovers. While a
// commit is in flight the open batch is the only place to simulate, and it is
// bounded below the batch limit.
const bool can_build = batch_builder ? batch_builder->batch().ticks.size() + 4 <
server::max_cellular_batch_ticks
: !host.commit_in_flight();
if (current && host.committed() && paced->due && needed && can_build &&
current->tick() != UINT64_MAX) {
const auto tick = current->tick() + 1;
std::vector<server::CellularLifeIntent> baseline;
@@ -1227,8 +1301,10 @@ int serve_cellular(Options& o) {
if (!batch_builder) {
bridge->dropped(actions, true);
} else {
// Births and mutation choices only ever enter a fresh batch.
const bool fresh = batch_builder->batch().ticks.empty();
// Births and mutation choices only ever enter a fresh batch on the
// durable head.
const bool fresh =
batch_builder->batch().ticks.empty() && !host.commit_in_flight();
const std::span<const server::CellularTickIntents::Birth> births =
fresh ? std::span<const server::CellularTickIntents::Birth>{held_births}
: std::span<const server::CellularTickIntents::Birth>{};
@@ -12,7 +12,7 @@ spec:
env:
- {name: REGION_SESSION, value: "dev"}
- {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"}
- {name: REGION_SESSION_WINDOW, value: "dev-20261022T120212-e2b4030c:1792684932"}
- {name: REGION_SESSION_WINDOW, value: "dev-20261022T123452-6a08d01f:1792686892"}
- {name: REGION_M10_MAX_GRANT_LIFETIME, value: "14400"}
ports:
# QUIC (ADR-0090): UDP only; nothing listens on TCP 7400.
@@ -114,6 +114,22 @@ class CellularRegionHost {
[[nodiscard]] std::expected<std::optional<persistence::CommittedCheckpoint>,
CellularPublishFailure>
advance_unattended(std::uint64_t due_ticks);
// Write-behind commit (rules-12 regions, Cellular_Region_Movement_Plan.md): commits a raw
// batch on a worker thread with its own connection while the caller keeps simulating.
// `candidate` is the state the batch's builder already computed, so nothing is replayed.
// At most one commit is in flight; its result is applied on this thread by poll_publish.
[[nodiscard]] std::expected<void, CellularPublishFailure>
begin_publish(CellularBatch batch, std::vector<CellularActionRecord> actions,
std::vector<CellularMutationActionRecord> mutation_actions,
std::unique_ptr<CellularLives> candidate);
[[nodiscard]] bool commit_in_flight() const noexcept {
return in_flight_ != nullptr;
}
// The finished background commit's outcome, applied exactly as publish applies its own;
// nullopt while none has finished.
[[nodiscard]] std::optional<
std::expected<persistence::PublishedCellularBatch, CellularPublishFailure>>
poll_publish();
[[nodiscard]] std::expected<persistence::PublishedCellularBatch, CellularPublishFailure>
publish(const CellularBatch& batch, std::span<const CellularActionRecord> actions = {},
std::span<const CellularMutationActionRecord> mutation_actions = {});
@@ -160,6 +176,13 @@ class CellularRegionHost {
private:
struct Pending;
struct InFlight;
[[nodiscard]] std::expected<persistence::PublishedCellularBatch, CellularPublishFailure>
finish_publication(const CellularBatch& batch, std::span<const CellularActionRecord> actions,
std::span<const CellularMutationActionRecord> mutation_actions,
std::unique_ptr<CellularLives> candidate,
std::expected<persistence::PublishedCellularBatch,
persistence::CellularPublicationFailure> published);
CellularRegionHost(CellularHostConfig config, CellularConnect connect,
std::unique_ptr<persistence::FilesystemObjectStore> objects);
bool database();
@@ -190,6 +213,9 @@ class CellularRegionHost {
CellularConnect connect_;
std::unique_ptr<persistence::FilesystemObjectStore> objects_;
std::unique_ptr<persistence::Postgres> db_;
// The write-behind commit and its own connection (libpq connections are not shared).
std::unique_ptr<InFlight> in_flight_;
std::unique_ptr<persistence::Postgres> commit_db_;
persistence::LeaseAuthority authority_;
// Exactly one owner. The common interface is a const biological view, never
// a second independently spendable cellular copy alongside a population.
@@ -82,6 +82,19 @@ class CellularSessionBatchBuilder {
[[nodiscard]] const CellularLives& candidate() const noexcept {
return *working_;
}
// Hands the working state to a commit; the builder is spent afterwards.
[[nodiscard]] std::unique_ptr<CellularLives> take_candidate() noexcept {
return std::move(working_);
}
// A batch started on an uncommitted predecessor (a commit still in flight) takes that
// predecessor's real head once it is known. The head must end where this batch begins.
[[nodiscard]] bool rebase(const persistence::RegionHead& head) noexcept {
if (head.frame != batch_.header.frame || head.end_tick != batch_.header.begin_tick)
return false;
batch_.header.predecessor = head.commit;
batch_.header.predecessor_digest = head.digest;
return true;
}
// The owner's own refusal behind the last `biology` error, for operator logs.
[[nodiscard]] std::optional<CellularLivesError> last_lives_error() const noexcept {
return last_lives_error_;
+124 -9
View File
@@ -5,8 +5,10 @@
#include <algorithm>
#include <array>
#include <atomic>
#include <chrono>
#include <limits>
#include <thread>
#include <utility>
namespace evo::server {
@@ -18,6 +20,22 @@ struct CellularRegionHost::Pending {
std::vector<CellularMutationActionRecord> mutation_actions;
std::optional<persistence::PopulationPreparedCommit> population_prepared{};
};
// A write-behind commit: everything the worker reads is owned here and untouched by the host
// thread until `done`, and the worker touches nothing else.
struct CellularRegionHost::InFlight {
CellularBatch batch;
std::vector<CellularActionRecord> actions;
std::vector<CellularMutationActionRecord> mutation_actions;
std::unique_ptr<CellularLives> candidate;
persistence::CommittedCheckpoint predecessor;
persistence::LeaseGrant grant;
persistence::Uuid commit{};
std::optional<
std::expected<persistence::PublishedCellularBatch, persistence::CellularPublicationFailure>>
result;
std::atomic<bool> done{false};
std::thread worker;
};
namespace {
using persistence::LeaseClock;
@@ -155,7 +173,10 @@ CellularRegionHost::CellularRegionHost(CellularHostConfig config, CellularConnec
: config_{std::move(config)}, connect_{std::move(connect)}, objects_{std::move(objects)},
authority_{config_.lease} {}
CellularRegionHost::~CellularRegionHost() = default;
CellularRegionHost::~CellularRegionHost() {
if (in_flight_ && in_flight_->worker.joinable())
in_flight_->worker.join();
}
std::expected<std::unique_ptr<CellularRegionHost>, CellularHostError>
CellularRegionHost::create(CellularHostConfig config, CellularConnect connect,
@@ -676,8 +697,13 @@ const HostStatus& CellularRegionHost::step() {
upkeep();
if (pending_)
resolve();
else if (!state_)
else if (!state_ && !in_flight_)
acquire();
// A write-behind commit owns the next head: nothing else commits or reloads meanwhile.
if (in_flight_) {
refresh();
return status_;
}
if (!pending_ && state_ && status_.phase == HostPhase::running)
reconcile_migration_freeze();
if (migration_reload_ || migration_frozen_ || migration_request_ || migration_stage_request_ ||
@@ -867,7 +893,7 @@ CellularRegionHost::publish(const CellularBatch& batch,
std::span<const CellularActionRecord> actions,
std::span<const CellularMutationActionRecord> mutation_actions) {
const auto grant = authority_.grant();
if (!status_.ready || mutations_uncertain_ || !state_ || !committed_ || !grant ||
if (in_flight_ || !status_.ready || mutations_uncertain_ || !state_ || !committed_ || !grant ||
!authority_.may_begin_commit(LeaseClock::now()))
return std::unexpected{CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}}};
if (population_mode()) {
@@ -891,11 +917,94 @@ CellularRegionHost::publish(const CellularBatch& batch,
auto published = persistence::publish_cellular_batch(
*db_, *objects_, grant.value_or(persistence::LeaseGrant{}), *committed_, batch, **candidate,
*commit, config_.build, config_.lease.commit_timeout, {}, actions, mutation_actions);
return finish_publication(batch, actions, mutation_actions, std::move(*candidate),
std::move(published));
}
std::expected<void, CellularPublishFailure>
CellularRegionHost::begin_publish(CellularBatch batch, std::vector<CellularActionRecord> actions,
std::vector<CellularMutationActionRecord> mutation_actions,
std::unique_ptr<CellularLives> candidate) {
const auto grant = authority_.grant();
if (in_flight_ || !candidate || !status_.ready || mutations_uncertain_ || !state_ ||
!committed_ || !grant || population_mode() ||
!authority_.may_begin_commit(LeaseClock::now()))
return std::unexpected{CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}}};
const auto& head = committed_->head;
if (candidate->tick() != batch.header.end_tick || batch.header.begin_tick != head.end_tick ||
batch.header.predecessor != head.commit || batch.header.predecessor_digest != head.digest)
return std::unexpected{
CellularPublishFailure{CellularHostError::invalid_batch, {}, {}, {}}};
auto trace = encode_cellular_batch(batch);
auto action_block =
encode_cellular_action_block(actions, batch.header.begin_tick, batch.header.end_tick);
const auto commit =
trace && action_block ? commit_id(*committed_, *trace, *action_block) : std::nullopt;
if (!commit)
return std::unexpected{
CellularPublishFailure{CellularHostError::invalid_batch, {}, {}, {}}};
if (!commit_db_) {
auto connection = connect_();
if (!connection) {
++status_.store_failures;
return std::unexpected{
CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}}};
}
commit_db_ = std::move(*connection);
}
auto flight = std::make_unique<InFlight>();
flight->batch = std::move(batch);
flight->actions = std::move(actions);
flight->mutation_actions = std::move(mutation_actions);
flight->candidate = std::move(candidate);
flight->predecessor = *committed_;
flight->grant = *grant;
flight->commit = *commit;
auto* work = flight.get();
auto* db = commit_db_.get();
const auto* objects = objects_.get();
const auto build = config_.build;
const auto timeout = config_.lease.commit_timeout;
work->worker = std::thread([work, db, objects, build, timeout] {
work->result = persistence::publish_cellular_batch(
*db, *objects, work->grant, work->predecessor, work->batch, *work->candidate,
work->commit, build, timeout, {}, work->actions, work->mutation_actions);
work->done.store(true, std::memory_order_release);
});
in_flight_ = std::move(flight);
return {};
}
std::optional<std::expected<persistence::PublishedCellularBatch, CellularPublishFailure>>
CellularRegionHost::poll_publish() {
if (!in_flight_ || !in_flight_->done.load(std::memory_order_acquire))
return std::nullopt;
in_flight_->worker.join();
const auto flight = std::move(in_flight_);
auto& result = flight->result;
if (!result.has_value() || !result->has_value())
commit_db_.reset(); // A failed connection is never reused.
// Authority lost while the commit ran: recovery reloads whatever the database holds.
if (!state_ || !committed_ || committed_->head != flight->predecessor.head)
return std::unexpected{CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}}};
if (!result.has_value())
return std::unexpected{CellularPublishFailure{CellularHostError::publication, {}, {}, {}}};
return finish_publication(flight->batch, flight->actions, flight->mutation_actions,
std::move(flight->candidate), std::move(*result));
}
std::expected<persistence::PublishedCellularBatch, CellularPublishFailure>
CellularRegionHost::finish_publication(
const CellularBatch& batch, std::span<const CellularActionRecord> actions,
std::span<const CellularMutationActionRecord> mutation_actions,
std::unique_ptr<CellularLives> candidate,
std::expected<persistence::PublishedCellularBatch, persistence::CellularPublicationFailure>
published) {
if (!published) {
if (published.error().kind == persistence::PublicationError::unknown) {
pending_ = std::make_unique<Pending>(
Pending{published.error().prepared,
std::move(*candidate),
std::move(candidate),
{actions.begin(), actions.end()},
births_in(batch),
{mutation_actions.begin(), mutation_actions.end()}});
@@ -917,8 +1026,8 @@ CellularRegionHost::publish(const CellularBatch& batch,
CellularPublishFailure{CellularHostError::publication, published.error().kind,
published.error().store, published.error().object}};
}
note_endings(*state_, **candidate, published->committed.head.commit);
state_ = std::move(*candidate);
note_endings(*state_, *candidate, published->committed.head.commit);
state_ = std::move(candidate);
committed_ = published->committed;
flush_endings();
for (const auto& action : actions)
@@ -1093,6 +1202,11 @@ void CellularRegionHost::drain() {
status_.ready = false;
status_.phase = HostPhase::draining;
pending_.reset();
// A running commit finishes (or times out) before the lease is released.
if (in_flight_ && in_flight_->worker.joinable())
in_flight_->worker.join();
in_flight_.reset();
commit_db_.reset();
const auto grant = authority_.grant();
if (grant && authority_.may_originate(LeaseClock::now()) && database()) {
// Best effort during shutdown; the lease expires if release cannot be confirmed.
@@ -1166,9 +1280,10 @@ CellularRegionHost::accept_population_publication(
}
std::expected<void, CellularPublishFailure> CellularRegionHost::set_population_mode(bool enabled) {
if (!status_.ready || pending_ || mutations_uncertain_ || admitted_interval_.has_value() ||
interval_request_.has_value() || interval_reload_ || !state_ || !committed_.has_value() ||
!authority_.may_begin_commit(LeaseClock::now()) || !database())
if (in_flight_ || !status_.ready || pending_ || mutations_uncertain_ ||
admitted_interval_.has_value() || interval_request_.has_value() || interval_reload_ ||
!state_ || !committed_.has_value() || !authority_.may_begin_commit(LeaseClock::now()) ||
!database())
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
if (population_mode() == enabled)
return {};
+137 -1
View File
@@ -4,8 +4,12 @@
#include "evo/persistence/life_reservation_store.hpp"
#include "evo/persistence/sha256.hpp"
#include "evo/server/cellular_region_host.hpp"
#ifdef EVO_POSTGRES_M10
#include "evo/server/cellular_session_batch.hpp"
#endif
#include <algorithm>
#include <chrono>
#include <cstdio>
#include <cstdlib>
#include <optional>
@@ -13,6 +17,7 @@
#include <stdexcept>
#include <string>
#include <string_view>
#include <thread>
void run_cellular_publication_tests();
namespace {
@@ -927,6 +932,134 @@ void cellular_bud_publishes_and_recovers() {
(*restarted)->drain();
}
#ifdef EVO_POSTGRES_M10
// Write-behind commits (evo-server's rules-12 loop): each batch commits on the host's worker
// thread while the next one builds on its candidate under a provisional head, rebased when
// the commit lands; recovery reloads exactly the chained state.
void cellular_write_behind_commits_chain_and_recover() {
using namespace evo::biology;
constexpr RegionKey key{9778, 1};
auto owner = connect_as(env("PGUSER"));
auto runtime = connect_as(env("EVO_PG_RUNTIME_USER"));
const auto build = filled<32>(0xDB);
CHECK(owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
"(decode(repeat('db',32),'hex')) ON CONFLICT DO NOTHING",
{}, 5s));
auto endowed = primitive_endowment();
CHECK(endowed);
auto funding = endowed->body;
funding.energy.value = endowed->reserve.energy.value + endowed->construction.value;
auto primitive = PrimitiveLives::create({key.world, key.region, 1, 17, funding, {{0, 0}}});
CHECK(primitive);
constexpr std::uint64_t stock = 10'000'000'000;
auto habitat = CellularHabitat::create(
{key.world,
key.region,
1,
3'000,
2'000,
{{{0, 0}, 29, {{stock}, {{{stock}, {stock}, {stock}, {stock}}}}}}});
CHECK(habitat);
auto opening =
CellularLives::create(**primitive, std::move(*habitat), {{3'000, 2'000}, 0, 293'150},
CellularLivesProfile::budding);
CHECK(opening);
auto objects = FilesystemObjectStore::open(env("PGHOST"), key, max_cellular_lives_bytes);
CHECK(objects);
CHECK(provision_cellular_region(*owner, **objects, key, 1, **opening, distinct_commit(0x9E),
build, 5s));
CHECK(owner->query("INSERT INTO evolution.life_allocators VALUES(9778,1000) "
"ON CONFLICT DO NOTHING",
{}, 5s));
const auto connect = [&]() -> std::expected<std::unique_ptr<Postgres>, PgFailure> {
return Postgres::connect({env("PGHOST"),
{},
env("PGPORT"),
env("PGDATABASE"),
env("EVO_PG_RUNTIME_USER"),
{},
{}},
5s);
};
const auto config = [&] { return CellularHostConfig{key, 1, {}, build, {build}}; };
auto made = CellularRegionHost::create(config(), connect, std::move(*objects));
CHECK(made);
auto host = std::move(*made);
CHECK(host->step().ready);
const auto account = filled<16>(0x85);
const auto control = claim_account_control(*runtime, key, checked_value(host->authority()),
account, 1, 1, 60'000ms, 5s);
CHECK(control);
CHECK(host->create_primitive_life(account, control->generation, 11).result ==
evo::network::PrimitiveLifeResult::pending);
const auto& committed = checked_value(host->committed());
const auto digest = sha256(committed.manifest);
CHECK(digest);
CellularBatch birth;
birth.header = {key.world,
key.region,
1,
committed.head.commit,
checked_value(digest),
host->state()->tick(),
host->state()->tick() + 1};
birth.ticks.push_back({{}, {host->queued_births().front()}, {}});
CHECK(host->publish(birth));
CellularStepIntent feed{};
feed.intake_target = CellularPatchAddress{0, 0};
const std::array baseline{
CellularLifeIntent{host->state()->lives().front().identity.life, feed}};
auto created =
CellularSessionBatchBuilder::create(*host->state(), checked_value(host->committed()).head);
CHECK(created);
std::optional<CellularSessionBatchBuilder> open{std::move(*created)};
const auto fill = [&](int ticks) {
for (int i = 0; i < ticks; ++i)
CHECK(open->append(open->candidate().tick() + 1, baseline, {}));
};
fill(20);
for (int round = 0; round < 3; ++round) {
const auto& candidate = open->candidate();
const auto end = candidate.tick();
RegionHead provisional{
candidate.frame(), checked_value(host->committed()).head.sequence + 1, {}, {}, end};
provisional.commit.fill(0xFF);
provisional.digest.fill(0xFF);
auto next = CellularSessionBatchBuilder::create(candidate, provisional);
CHECK(next);
const auto batch = open->batch();
CHECK(host->begin_publish(batch, {}, {}, open->take_candidate()));
// One commit at a time; the synchronous path waits too.
CHECK(host->commit_in_flight() && !host->publish(batch) &&
!host->begin_publish(batch, {}, {}, nullptr));
open.emplace(std::move(*next));
fill(20); // Simulation continues while the commit runs.
std::optional<std::expected<PublishedCellularBatch, CellularPublishFailure>> landed;
for (int wait = 0; wait < 1'000 && !landed; ++wait) {
landed = host->poll_publish();
if (!landed)
std::this_thread::sleep_for(10ms);
}
CHECK(landed && *landed && !host->commit_in_flight() && host->state()->tick() == end &&
checked_value(host->committed()).head.sequence == provisional.sequence);
CHECK(open->rebase(checked_value(host->committed()).head) && host->step().ready);
}
const auto snapshot = host->state()->snapshot();
const auto tick = host->state()->tick();
host->drain();
host.reset();
auto reopened = FilesystemObjectStore::open(env("PGHOST"), key, max_cellular_lives_bytes);
CHECK(reopened);
auto restarted = CellularRegionHost::create(config(), connect, std::move(*reopened));
CHECK(restarted);
CHECK((*restarted)->step().ready && (*restarted)->state()->tick() == tick &&
(*restarted)->state()->snapshot() == snapshot);
(*restarted)->drain();
}
#endif
// ADR-0127 (schema v37): a dead life stays dead, its corpse stays, and its account starts
// a new primitive life with a new identity; the first life's history is unchanged.
void cellular_death_is_final_and_account_restarts() {
@@ -956,7 +1089,7 @@ void cellular_death_is_final_and_account_restarts() {
CHECK(opening);
auto objects = FilesystemObjectStore::open(env("PGHOST"), key, max_cellular_lives_bytes);
CHECK(objects);
CHECK(provision_cellular_region(*owner, **objects, key, 1, **opening, distinct_commit(0x9E),
CHECK(provision_cellular_region(*owner, **objects, key, 1, **opening, distinct_commit(0x9F),
build, 5s));
CHECK(owner->query("INSERT INTO evolution.life_allocators VALUES(9768,1000) "
"ON CONFLICT DO NOTHING",
@@ -1078,6 +1211,9 @@ void run_cellular_publication_tests() {
cellular_mutation_publication_binds_results();
cellular_host_admits_mutation_choices();
cellular_bud_publishes_and_recovers();
#ifdef EVO_POSTGRES_M10
cellular_write_behind_commits_chain_and_recover();
#endif
cellular_balance_region_founds_and_recovers({{9771, 1},
CellularLivesProfile::balance,
evo::biology::primitive_founder_v2,