From ff333d97c2c4c2bc02a91dd12b1916ece80eb319 Mon Sep 17 00:00:00 2001 From: dingo Date: Thu, 22 Oct 2026 08:41:56 -0400 Subject: [PATCH] feat(server): commit cellular batches on a background thread The rules-12 loop blocked for ~411 ms per 20-tick commit, so players saw one-second bursts. The host now commits a batch on a worker thread with its own connection (begin_publish / poll_publish), using the builder's already-simulated candidate. The next batch builds on that candidate under a provisional head and is rebased when the commit lands. One commit is in flight at a time; births and mutation choices wait for a batch that starts on the durable head, and host reconciliation pauses meanwhile. Co-Authored-By: Claude Opus 5.5 --- apps/evo-server/main.cpp | 128 ++++++++++++---- .../region/dev-session-route/listener.yaml | 2 +- include/evo/server/cellular_region_host.hpp | 26 ++++ include/evo/server/cellular_session_batch.hpp | 13 ++ src/server/cellular_region_host.cpp | 133 +++++++++++++++-- tests/cellular_publication_tests.cpp | 138 +++++++++++++++++- 6 files changed, 403 insertions(+), 37 deletions(-) diff --git a/apps/evo-server/main.cpp b/apps/evo-server/main.cpp index 4863833..a279719 100644 --- a/apps/evo-server/main.cpp +++ b/apps/evo-server/main.cpp @@ -942,6 +942,20 @@ int serve_cellular(Options& o) { std::vector batch_actions; std::uint64_t pace_epoch = 0, pace_tick = 0, flushes_logged = 0; std::vector awaiting_outcomes; + // The batch being committed on the host's worker thread while the next one builds on its + // candidate: its carried actions, birth refusals and operator timing. + std::vector flight_actions; + std::vector flight_refusals; + std::size_t flight_ticks = 0; + long long flight_handoff_ms = 0; + auto flight_start = std::chrono::steady_clock::now(); + const auto drop_carried = [&](std::vector& carried) { + bridge->dropped(carried, true); + std::erase_if(awaiting_outcomes, [&](const auto& action) { + return std::ranges::find(carried, action) != carried.end(); + }); + carried.clear(); + }; // A tick that cannot be built is retried every 50 ms; name each distinct cause once. std::optional> stalled; const auto report_stall = [&](std::string_view where, server::CellularSessionBatchError error, @@ -1008,6 +1022,35 @@ int serve_cellular(Options& o) { host.drain(); return finish(0); } + // A finished background commit lands first, so this step sees its head. + if (auto landed = host.poll_publish(); landed) { + const auto commit_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - flight_start) + .count(); + // Operator evidence: the loop is blocked only for the hand-off, not the commit. + if (flight_handoff_ms > 50 || !*landed || ++flushes_logged % 30 == 0) + std::printf("cellular commit ticks=%zu ms=%lld handoff_ms=%lld ok=%d\n", + flight_ticks, static_cast(commit_ms), flight_handoff_ms, + *landed ? 1 : 0); + if (*landed) { + host.refuse_births(flight_refusals); + flight_actions.clear(); + if (batch_builder && !batch_builder->rebase(host.committed()->head)) { + drop_carried(batch_actions); + batch_builder.reset(); + } + } else { + // An unresolved outcome keeps its actions for the host to finalize. + if (host.status().phase == server::HostPhase::resolving) + flight_actions.clear(); + else + drop_carried(flight_actions); + // The open batch continued from a state that is not durable. + drop_carried(batch_actions); + batch_builder.reset(); + } + flight_refusals.clear(); + } const auto& status = host.step(); const auto wall = static_cast(std::chrono::duration_cast( @@ -1115,34 +1158,60 @@ int serve_cellular(Options& o) { } const auto flush_ticks = std::clamp( static_cast(o.host.flush_interval.count()) / 50, 1, 60); + // Births and mutation choices wait for a batch that starts on the durable head. + const bool admissions_waiting = !held_births.empty() || !host.queued_mutations().empty(); + // Write-behind commit: the open batch goes to the host's worker thread and the next + // batch continues from its candidate under a provisional head, rebased when the commit + // lands. One commit is in flight; meanwhile the open batch simply keeps growing. const auto flush = [&] { if (!batch_builder || batch_builder->batch().ticks.empty()) { - batch_builder.reset(); + if (!host.commit_in_flight()) + batch_builder.reset(); return; } - const auto flush_start = std::chrono::steady_clock::now(); - const auto flush_ticks_held = batch_builder->batch().ticks.size(); - auto published = host.publish(batch_builder->batch(), batch_builder->actions(), - batch_builder->mutation_actions()); - // Operator evidence for commit stalls: every slow commit, and one in 60 otherwise. - const auto flush_ms = std::chrono::duration_cast( - std::chrono::steady_clock::now() - flush_start) - .count(); - if (flush_ms > 50 || ++flushes_logged % 60 == 0) - std::printf("cellular commit ticks=%zu ms=%lld ok=%d\n", flush_ticks_held, - static_cast(flush_ms), published ? 1 : 0); - if (published) - host.refuse_births(batch_builder->refused_births()); - if (!published && host.status().phase != server::HostPhase::resolving) { - bridge->dropped(batch_actions, true); - std::erase_if(awaiting_outcomes, [&](const auto& action) { - return std::ranges::find(batch_actions, action) != batch_actions.end(); - }); + if (host.commit_in_flight()) + return; + const auto handoff_start = std::chrono::steady_clock::now(); + const auto& candidate = batch_builder->candidate(); + std::optional next; + if (!admissions_waiting && candidate.tick() != UINT64_MAX) { + persistence::RegionHead provisional{candidate.frame(), + host.committed()->head.sequence + 1, + {}, + {}, + candidate.tick()}; + provisional.commit.fill(0xFF); + provisional.digest.fill(0xFF); + if (auto created = + server::CellularSessionBatchBuilder::create(candidate, provisional); + created) + next.emplace(std::move(*created)); } + flight_ticks = batch_builder->batch().ticks.size(); + const auto refusals = batch_builder->refused_births(); + flight_refusals.assign(refusals.begin(), refusals.end()); + const auto actions = batch_builder->actions(); + const auto mutation_actions = batch_builder->mutation_actions(); + auto begun = + host.begin_publish(batch_builder->batch(), {actions.begin(), actions.end()}, + {mutation_actions.begin(), mutation_actions.end()}, + batch_builder->take_candidate()); batch_builder.reset(); + flight_actions = std::move(batch_actions); batch_actions.clear(); - deliver_finalized(); - state = host.state(); // Publication replaced the committed state. + flight_start = std::chrono::steady_clock::now(); + flight_handoff_ms = + std::chrono::duration_cast(flight_start - handoff_start) + .count(); + if (!begun) { + std::printf("cellular commit refused handoff_ms=%lld error=%u\n", flight_handoff_ms, + static_cast(begun.error().kind)); + drop_carried(flight_actions); + flight_refusals.clear(); + return; + } + if (next) + batch_builder = std::move(next); }; // A batch cannot outlive the authority or the raw mode it was built under; when demand // ends, what it holds is committed. @@ -1168,11 +1237,16 @@ int serve_cellular(Options& o) { // large: commit every flush interval (ADR-0089: 1 s), not every tick. Players // see the batch's working state at once; outcomes become final at its commit. // Births and mutation choices commit promptly, in their own fresh batch. - if (batch_builder && (!held_births.empty() || !host.queued_mutations().empty())) + if (batch_builder && admissions_waiting) flush(); const auto* current = batch_builder ? &batch_builder->candidate() : state; - // A failed commit can leave the host without state until it recovers. - if (current && host.committed() && paced->due && needed && + // A failed commit can leave the host without state until it recovers. While a + // commit is in flight the open batch is the only place to simulate, and it is + // bounded below the batch limit. + const bool can_build = batch_builder ? batch_builder->batch().ticks.size() + 4 < + server::max_cellular_batch_ticks + : !host.commit_in_flight(); + if (current && host.committed() && paced->due && needed && can_build && current->tick() != UINT64_MAX) { const auto tick = current->tick() + 1; std::vector baseline; @@ -1227,8 +1301,10 @@ int serve_cellular(Options& o) { if (!batch_builder) { bridge->dropped(actions, true); } else { - // Births and mutation choices only ever enter a fresh batch. - const bool fresh = batch_builder->batch().ticks.empty(); + // Births and mutation choices only ever enter a fresh batch on the + // durable head. + const bool fresh = + batch_builder->batch().ticks.empty() && !host.commit_in_flight(); const std::span births = fresh ? std::span{held_births} : std::span{}; diff --git a/deploy/kubernetes/region/dev-session-route/listener.yaml b/deploy/kubernetes/region/dev-session-route/listener.yaml index 0f640f8..72c85de 100644 --- a/deploy/kubernetes/region/dev-session-route/listener.yaml +++ b/deploy/kubernetes/region/dev-session-route/listener.yaml @@ -12,7 +12,7 @@ spec: env: - {name: REGION_SESSION, value: "dev"} - {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"} - - {name: REGION_SESSION_WINDOW, value: "dev-20261022T120212-e2b4030c:1792684932"} + - {name: REGION_SESSION_WINDOW, value: "dev-20261022T123452-6a08d01f:1792686892"} - {name: REGION_M10_MAX_GRANT_LIFETIME, value: "14400"} ports: # QUIC (ADR-0090): UDP only; nothing listens on TCP 7400. diff --git a/include/evo/server/cellular_region_host.hpp b/include/evo/server/cellular_region_host.hpp index 8de3230..1d41cf7 100644 --- a/include/evo/server/cellular_region_host.hpp +++ b/include/evo/server/cellular_region_host.hpp @@ -114,6 +114,22 @@ class CellularRegionHost { [[nodiscard]] std::expected, CellularPublishFailure> advance_unattended(std::uint64_t due_ticks); + // Write-behind commit (rules-12 regions, Cellular_Region_Movement_Plan.md): commits a raw + // batch on a worker thread with its own connection while the caller keeps simulating. + // `candidate` is the state the batch's builder already computed, so nothing is replayed. + // At most one commit is in flight; its result is applied on this thread by poll_publish. + [[nodiscard]] std::expected + begin_publish(CellularBatch batch, std::vector actions, + std::vector mutation_actions, + std::unique_ptr candidate); + [[nodiscard]] bool commit_in_flight() const noexcept { + return in_flight_ != nullptr; + } + // The finished background commit's outcome, applied exactly as publish applies its own; + // nullopt while none has finished. + [[nodiscard]] std::optional< + std::expected> + poll_publish(); [[nodiscard]] std::expected publish(const CellularBatch& batch, std::span actions = {}, std::span mutation_actions = {}); @@ -160,6 +176,13 @@ class CellularRegionHost { private: struct Pending; + struct InFlight; + [[nodiscard]] std::expected + finish_publication(const CellularBatch& batch, std::span actions, + std::span mutation_actions, + std::unique_ptr candidate, + std::expected published); CellularRegionHost(CellularHostConfig config, CellularConnect connect, std::unique_ptr objects); bool database(); @@ -190,6 +213,9 @@ class CellularRegionHost { CellularConnect connect_; std::unique_ptr objects_; std::unique_ptr db_; + // The write-behind commit and its own connection (libpq connections are not shared). + std::unique_ptr in_flight_; + std::unique_ptr commit_db_; persistence::LeaseAuthority authority_; // Exactly one owner. The common interface is a const biological view, never // a second independently spendable cellular copy alongside a population. diff --git a/include/evo/server/cellular_session_batch.hpp b/include/evo/server/cellular_session_batch.hpp index 2a5c2be..7728a69 100644 --- a/include/evo/server/cellular_session_batch.hpp +++ b/include/evo/server/cellular_session_batch.hpp @@ -82,6 +82,19 @@ class CellularSessionBatchBuilder { [[nodiscard]] const CellularLives& candidate() const noexcept { return *working_; } + // Hands the working state to a commit; the builder is spent afterwards. + [[nodiscard]] std::unique_ptr take_candidate() noexcept { + return std::move(working_); + } + // A batch started on an uncommitted predecessor (a commit still in flight) takes that + // predecessor's real head once it is known. The head must end where this batch begins. + [[nodiscard]] bool rebase(const persistence::RegionHead& head) noexcept { + if (head.frame != batch_.header.frame || head.end_tick != batch_.header.begin_tick) + return false; + batch_.header.predecessor = head.commit; + batch_.header.predecessor_digest = head.digest; + return true; + } // The owner's own refusal behind the last `biology` error, for operator logs. [[nodiscard]] std::optional last_lives_error() const noexcept { return last_lives_error_; diff --git a/src/server/cellular_region_host.cpp b/src/server/cellular_region_host.cpp index 0305331..ce4b3b9 100644 --- a/src/server/cellular_region_host.cpp +++ b/src/server/cellular_region_host.cpp @@ -5,8 +5,10 @@ #include #include +#include #include #include +#include #include namespace evo::server { @@ -18,6 +20,22 @@ struct CellularRegionHost::Pending { std::vector mutation_actions; std::optional population_prepared{}; }; +// A write-behind commit: everything the worker reads is owned here and untouched by the host +// thread until `done`, and the worker touches nothing else. +struct CellularRegionHost::InFlight { + CellularBatch batch; + std::vector actions; + std::vector mutation_actions; + std::unique_ptr candidate; + persistence::CommittedCheckpoint predecessor; + persistence::LeaseGrant grant; + persistence::Uuid commit{}; + std::optional< + std::expected> + result; + std::atomic done{false}; + std::thread worker; +}; namespace { using persistence::LeaseClock; @@ -155,7 +173,10 @@ CellularRegionHost::CellularRegionHost(CellularHostConfig config, CellularConnec : config_{std::move(config)}, connect_{std::move(connect)}, objects_{std::move(objects)}, authority_{config_.lease} {} -CellularRegionHost::~CellularRegionHost() = default; +CellularRegionHost::~CellularRegionHost() { + if (in_flight_ && in_flight_->worker.joinable()) + in_flight_->worker.join(); +} std::expected, CellularHostError> CellularRegionHost::create(CellularHostConfig config, CellularConnect connect, @@ -676,8 +697,13 @@ const HostStatus& CellularRegionHost::step() { upkeep(); if (pending_) resolve(); - else if (!state_) + else if (!state_ && !in_flight_) acquire(); + // A write-behind commit owns the next head: nothing else commits or reloads meanwhile. + if (in_flight_) { + refresh(); + return status_; + } if (!pending_ && state_ && status_.phase == HostPhase::running) reconcile_migration_freeze(); if (migration_reload_ || migration_frozen_ || migration_request_ || migration_stage_request_ || @@ -867,7 +893,7 @@ CellularRegionHost::publish(const CellularBatch& batch, std::span actions, std::span mutation_actions) { const auto grant = authority_.grant(); - if (!status_.ready || mutations_uncertain_ || !state_ || !committed_ || !grant || + if (in_flight_ || !status_.ready || mutations_uncertain_ || !state_ || !committed_ || !grant || !authority_.may_begin_commit(LeaseClock::now())) return std::unexpected{CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}}}; if (population_mode()) { @@ -891,11 +917,94 @@ CellularRegionHost::publish(const CellularBatch& batch, auto published = persistence::publish_cellular_batch( *db_, *objects_, grant.value_or(persistence::LeaseGrant{}), *committed_, batch, **candidate, *commit, config_.build, config_.lease.commit_timeout, {}, actions, mutation_actions); + return finish_publication(batch, actions, mutation_actions, std::move(*candidate), + std::move(published)); +} + +std::expected +CellularRegionHost::begin_publish(CellularBatch batch, std::vector actions, + std::vector mutation_actions, + std::unique_ptr candidate) { + const auto grant = authority_.grant(); + if (in_flight_ || !candidate || !status_.ready || mutations_uncertain_ || !state_ || + !committed_ || !grant || population_mode() || + !authority_.may_begin_commit(LeaseClock::now())) + return std::unexpected{CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}}}; + const auto& head = committed_->head; + if (candidate->tick() != batch.header.end_tick || batch.header.begin_tick != head.end_tick || + batch.header.predecessor != head.commit || batch.header.predecessor_digest != head.digest) + return std::unexpected{ + CellularPublishFailure{CellularHostError::invalid_batch, {}, {}, {}}}; + auto trace = encode_cellular_batch(batch); + auto action_block = + encode_cellular_action_block(actions, batch.header.begin_tick, batch.header.end_tick); + const auto commit = + trace && action_block ? commit_id(*committed_, *trace, *action_block) : std::nullopt; + if (!commit) + return std::unexpected{ + CellularPublishFailure{CellularHostError::invalid_batch, {}, {}, {}}}; + if (!commit_db_) { + auto connection = connect_(); + if (!connection) { + ++status_.store_failures; + return std::unexpected{ + CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}}}; + } + commit_db_ = std::move(*connection); + } + auto flight = std::make_unique(); + flight->batch = std::move(batch); + flight->actions = std::move(actions); + flight->mutation_actions = std::move(mutation_actions); + flight->candidate = std::move(candidate); + flight->predecessor = *committed_; + flight->grant = *grant; + flight->commit = *commit; + auto* work = flight.get(); + auto* db = commit_db_.get(); + const auto* objects = objects_.get(); + const auto build = config_.build; + const auto timeout = config_.lease.commit_timeout; + work->worker = std::thread([work, db, objects, build, timeout] { + work->result = persistence::publish_cellular_batch( + *db, *objects, work->grant, work->predecessor, work->batch, *work->candidate, + work->commit, build, timeout, {}, work->actions, work->mutation_actions); + work->done.store(true, std::memory_order_release); + }); + in_flight_ = std::move(flight); + return {}; +} + +std::optional> +CellularRegionHost::poll_publish() { + if (!in_flight_ || !in_flight_->done.load(std::memory_order_acquire)) + return std::nullopt; + in_flight_->worker.join(); + const auto flight = std::move(in_flight_); + auto& result = flight->result; + if (!result.has_value() || !result->has_value()) + commit_db_.reset(); // A failed connection is never reused. + // Authority lost while the commit ran: recovery reloads whatever the database holds. + if (!state_ || !committed_ || committed_->head != flight->predecessor.head) + return std::unexpected{CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}}}; + if (!result.has_value()) + return std::unexpected{CellularPublishFailure{CellularHostError::publication, {}, {}, {}}}; + return finish_publication(flight->batch, flight->actions, flight->mutation_actions, + std::move(flight->candidate), std::move(*result)); +} + +std::expected +CellularRegionHost::finish_publication( + const CellularBatch& batch, std::span actions, + std::span mutation_actions, + std::unique_ptr candidate, + std::expected + published) { if (!published) { if (published.error().kind == persistence::PublicationError::unknown) { pending_ = std::make_unique( Pending{published.error().prepared, - std::move(*candidate), + std::move(candidate), {actions.begin(), actions.end()}, births_in(batch), {mutation_actions.begin(), mutation_actions.end()}}); @@ -917,8 +1026,8 @@ CellularRegionHost::publish(const CellularBatch& batch, CellularPublishFailure{CellularHostError::publication, published.error().kind, published.error().store, published.error().object}}; } - note_endings(*state_, **candidate, published->committed.head.commit); - state_ = std::move(*candidate); + note_endings(*state_, *candidate, published->committed.head.commit); + state_ = std::move(candidate); committed_ = published->committed; flush_endings(); for (const auto& action : actions) @@ -1093,6 +1202,11 @@ void CellularRegionHost::drain() { status_.ready = false; status_.phase = HostPhase::draining; pending_.reset(); + // A running commit finishes (or times out) before the lease is released. + if (in_flight_ && in_flight_->worker.joinable()) + in_flight_->worker.join(); + in_flight_.reset(); + commit_db_.reset(); const auto grant = authority_.grant(); if (grant && authority_.may_originate(LeaseClock::now()) && database()) { // Best effort during shutdown; the lease expires if release cannot be confirmed. @@ -1166,9 +1280,10 @@ CellularRegionHost::accept_population_publication( } std::expected CellularRegionHost::set_population_mode(bool enabled) { - if (!status_.ready || pending_ || mutations_uncertain_ || admitted_interval_.has_value() || - interval_request_.has_value() || interval_reload_ || !state_ || !committed_.has_value() || - !authority_.may_begin_commit(LeaseClock::now()) || !database()) + if (in_flight_ || !status_.ready || pending_ || mutations_uncertain_ || + admitted_interval_.has_value() || interval_request_.has_value() || interval_reload_ || + !state_ || !committed_.has_value() || !authority_.may_begin_commit(LeaseClock::now()) || + !database()) return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}}); if (population_mode() == enabled) return {}; diff --git a/tests/cellular_publication_tests.cpp b/tests/cellular_publication_tests.cpp index 837b1ca..9769460 100644 --- a/tests/cellular_publication_tests.cpp +++ b/tests/cellular_publication_tests.cpp @@ -4,8 +4,12 @@ #include "evo/persistence/life_reservation_store.hpp" #include "evo/persistence/sha256.hpp" #include "evo/server/cellular_region_host.hpp" +#ifdef EVO_POSTGRES_M10 +#include "evo/server/cellular_session_batch.hpp" +#endif #include +#include #include #include #include @@ -13,6 +17,7 @@ #include #include #include +#include void run_cellular_publication_tests(); namespace { @@ -927,6 +932,134 @@ void cellular_bud_publishes_and_recovers() { (*restarted)->drain(); } +#ifdef EVO_POSTGRES_M10 +// Write-behind commits (evo-server's rules-12 loop): each batch commits on the host's worker +// thread while the next one builds on its candidate under a provisional head, rebased when +// the commit lands; recovery reloads exactly the chained state. +void cellular_write_behind_commits_chain_and_recover() { + using namespace evo::biology; + constexpr RegionKey key{9778, 1}; + auto owner = connect_as(env("PGUSER")); + auto runtime = connect_as(env("EVO_PG_RUNTIME_USER")); + const auto build = filled<32>(0xDB); + CHECK(owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES " + "(decode(repeat('db',32),'hex')) ON CONFLICT DO NOTHING", + {}, 5s)); + auto endowed = primitive_endowment(); + CHECK(endowed); + auto funding = endowed->body; + funding.energy.value = endowed->reserve.energy.value + endowed->construction.value; + auto primitive = PrimitiveLives::create({key.world, key.region, 1, 17, funding, {{0, 0}}}); + CHECK(primitive); + constexpr std::uint64_t stock = 10'000'000'000; + auto habitat = CellularHabitat::create( + {key.world, + key.region, + 1, + 3'000, + 2'000, + {{{0, 0}, 29, {{stock}, {{{stock}, {stock}, {stock}, {stock}}}}}}}); + CHECK(habitat); + auto opening = + CellularLives::create(**primitive, std::move(*habitat), {{3'000, 2'000}, 0, 293'150}, + CellularLivesProfile::budding); + CHECK(opening); + auto objects = FilesystemObjectStore::open(env("PGHOST"), key, max_cellular_lives_bytes); + CHECK(objects); + CHECK(provision_cellular_region(*owner, **objects, key, 1, **opening, distinct_commit(0x9E), + build, 5s)); + CHECK(owner->query("INSERT INTO evolution.life_allocators VALUES(9778,1000) " + "ON CONFLICT DO NOTHING", + {}, 5s)); + const auto connect = [&]() -> std::expected, PgFailure> { + return Postgres::connect({env("PGHOST"), + {}, + env("PGPORT"), + env("PGDATABASE"), + env("EVO_PG_RUNTIME_USER"), + {}, + {}}, + 5s); + }; + const auto config = [&] { return CellularHostConfig{key, 1, {}, build, {build}}; }; + auto made = CellularRegionHost::create(config(), connect, std::move(*objects)); + CHECK(made); + auto host = std::move(*made); + CHECK(host->step().ready); + const auto account = filled<16>(0x85); + const auto control = claim_account_control(*runtime, key, checked_value(host->authority()), + account, 1, 1, 60'000ms, 5s); + CHECK(control); + CHECK(host->create_primitive_life(account, control->generation, 11).result == + evo::network::PrimitiveLifeResult::pending); + const auto& committed = checked_value(host->committed()); + const auto digest = sha256(committed.manifest); + CHECK(digest); + CellularBatch birth; + birth.header = {key.world, + key.region, + 1, + committed.head.commit, + checked_value(digest), + host->state()->tick(), + host->state()->tick() + 1}; + birth.ticks.push_back({{}, {host->queued_births().front()}, {}}); + CHECK(host->publish(birth)); + + CellularStepIntent feed{}; + feed.intake_target = CellularPatchAddress{0, 0}; + const std::array baseline{ + CellularLifeIntent{host->state()->lives().front().identity.life, feed}}; + auto created = + CellularSessionBatchBuilder::create(*host->state(), checked_value(host->committed()).head); + CHECK(created); + std::optional open{std::move(*created)}; + const auto fill = [&](int ticks) { + for (int i = 0; i < ticks; ++i) + CHECK(open->append(open->candidate().tick() + 1, baseline, {})); + }; + fill(20); + for (int round = 0; round < 3; ++round) { + const auto& candidate = open->candidate(); + const auto end = candidate.tick(); + RegionHead provisional{ + candidate.frame(), checked_value(host->committed()).head.sequence + 1, {}, {}, end}; + provisional.commit.fill(0xFF); + provisional.digest.fill(0xFF); + auto next = CellularSessionBatchBuilder::create(candidate, provisional); + CHECK(next); + const auto batch = open->batch(); + CHECK(host->begin_publish(batch, {}, {}, open->take_candidate())); + // One commit at a time; the synchronous path waits too. + CHECK(host->commit_in_flight() && !host->publish(batch) && + !host->begin_publish(batch, {}, {}, nullptr)); + open.emplace(std::move(*next)); + fill(20); // Simulation continues while the commit runs. + std::optional> landed; + for (int wait = 0; wait < 1'000 && !landed; ++wait) { + landed = host->poll_publish(); + if (!landed) + std::this_thread::sleep_for(10ms); + } + CHECK(landed && *landed && !host->commit_in_flight() && host->state()->tick() == end && + checked_value(host->committed()).head.sequence == provisional.sequence); + CHECK(open->rebase(checked_value(host->committed()).head) && host->step().ready); + } + const auto snapshot = host->state()->snapshot(); + const auto tick = host->state()->tick(); + host->drain(); + host.reset(); + + auto reopened = FilesystemObjectStore::open(env("PGHOST"), key, max_cellular_lives_bytes); + CHECK(reopened); + auto restarted = CellularRegionHost::create(config(), connect, std::move(*reopened)); + CHECK(restarted); + CHECK((*restarted)->step().ready && (*restarted)->state()->tick() == tick && + (*restarted)->state()->snapshot() == snapshot); + (*restarted)->drain(); +} +#endif + // ADR-0127 (schema v37): a dead life stays dead, its corpse stays, and its account starts // a new primitive life with a new identity; the first life's history is unchanged. void cellular_death_is_final_and_account_restarts() { @@ -956,7 +1089,7 @@ void cellular_death_is_final_and_account_restarts() { CHECK(opening); auto objects = FilesystemObjectStore::open(env("PGHOST"), key, max_cellular_lives_bytes); CHECK(objects); - CHECK(provision_cellular_region(*owner, **objects, key, 1, **opening, distinct_commit(0x9E), + CHECK(provision_cellular_region(*owner, **objects, key, 1, **opening, distinct_commit(0x9F), build, 5s)); CHECK(owner->query("INSERT INTO evolution.life_allocators VALUES(9768,1000) " "ON CONFLICT DO NOTHING", @@ -1078,6 +1211,9 @@ void run_cellular_publication_tests() { cellular_mutation_publication_binds_results(); cellular_host_admits_mutation_choices(); cellular_bud_publishes_and_recovers(); +#ifdef EVO_POSTGRES_M10 + cellular_write_behind_commits_chain_and_recover(); +#endif cellular_balance_region_founds_and_recovers({{9771, 1}, CellularLivesProfile::balance, evo::biology::primitive_founder_v2,