Recover cellular checkpoints with action evidence
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (checks) (push) Failing after 41s
Foundation / checks (postgres) (push) Canceled after 0s
Foundation / checks (tsan) (push) Canceled after 0s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (client) (push) Canceled after 1m6s
Foundation / checks (network) (push) Canceled after 29s
Foundation / checks (ecs) (push) Canceled after 58s

This commit is contained in:
dingo
2026-10-10 23:23:17 -04:00
parent ad6e53c2e9
commit 13273b46f4
7 changed files with 256 additions and 0 deletions
+1
View File
@@ -271,6 +271,7 @@ if(EVO_BUILD_POSTGRES)
if(BUILD_TESTING)
# Contention tests use one thread and one connection per contender.
add_executable(evo-postgres-tests tests/postgres_tests.cpp tests/region_lease_store_tests.cpp
tests/cellular_publication_tests.cpp
tests/account_control_store_tests.cpp tests/probe_handoff_store_tests.cpp
tests/life_reservation_store_tests.cpp
tests/primitive_publication_tests.cpp
+6
View File
@@ -243,6 +243,12 @@ scoped change before starting the next segment.
manifest validator now applies ADR-0113's variable cellular trace bounds,
matching migration 21. Result-row reconciliation and full restart recovery,
host integration, and CI execution remain open.
- 11.3d-6f: the cellular recovery adapter now verifies the committed payload,
current tail trace and predecessor link, loads the immutable action block under
the lease, replays the batch and checks every action's persisted kind, result,
tick, sequence and canonical input bytes. A PostgreSQL integration test covers
empty-cell genesis through publish and restart-style recovery. CI execution is
pending; populated-life action-result cases and process fault recovery remain.
## Exit boundary
+14
View File
@@ -6270,3 +6270,17 @@ manifest validator, and boundary tests pass. Full CMake build/CTest and PostgreS
integration are pending CI. The documented debug preset currently selects the
cached MSVC 19.44 compiler and fails CMake's C++23 feature check; this environment
also has no PostgreSQL runtime on `PATH`.
## 2026-10-10 — M11.3d-6f cellular action result recovery (CI pending)
Added lease-fenced cellular checkpoint recovery. It verifies the head payload,
trace/header and predecessor digest, replays the complete batch, and checks each
action row against the action block's account/action identity, cellular kind,
committed result, intended tick, head sequence and exact canonical input. The
PostgreSQL integration suite now exercises profile-4 genesis, a two-tick
publication, and recovery of that committed head.
`git diff --check` and GCC 13.2 C++23 syntax checks with warnings as errors pass
for the changed adapter and integration test. Full build, PostgreSQL execution,
and sanitizer checks are pending CI; the local CMake preset remains unable to
configure with its cached MSVC compiler selection.
@@ -115,6 +115,12 @@ struct CellularPublicationFailure {
struct PublishedCellularBatch {
CommittedCheckpoint committed;
};
struct RecoveredCellularCheckpoint {
CommittedCheckpoint committed;
CheckpointManifest manifest;
std::unique_ptr<server::CellularLives> state;
std::vector<server::CellularActionRecord> actions;
};
// Owner/operator connection only: uploads a tick-0 state and records genesis.
[[nodiscard]] std::expected<CommittedCheckpoint, PublicationFailure> provision_environmental_region(
@@ -204,6 +210,11 @@ retry_cellular_prepared(Postgres& db, const CellularPreparedCommit& prepared,
[[nodiscard]] std::expected<ResolvedCommit, LeaseStoreFailure>
resolve_cellular_prepared(Postgres& db, const CellularPreparedCommit& prepared,
std::chrono::milliseconds timeout);
[[nodiscard]] std::expected<RecoveredCellularCheckpoint, RecoveryFailure>
recover_cellular_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
const LeaseGrant& lease,
std::span<const Sha256Digest> retained_builds,
std::chrono::milliseconds timeout);
// After an `unknown` publication: resolve exactly the prepared bytes. A committed
// result yields the final batch; absent may be retried with the same prepared bytes
@@ -8,6 +8,13 @@ namespace evo::persistence {
namespace {
using std::chrono::milliseconds;
std::uint64_t input_tick(const server::CellularActionRecord& action) {
std::uint64_t tick{};
for (std::size_t i = 0; i < 8; ++i)
tick |= static_cast<std::uint64_t>(action.input[i]) << (i * 8U);
return tick;
}
CellularPublicationFailure failure(PublicationError kind, ObjectError object = {},
LeaseStoreFailure store = {},
CellularPreparedCommit prepared = {}) {
@@ -150,4 +157,115 @@ publish_cellular_batch(Postgres& db, const FilesystemObjectStore& objects,
return std::unexpected{committed.error()};
return PublishedCellularBatch{std::move(*committed)};
}
std::expected<RecoveredCellularCheckpoint, RecoveryFailure>
recover_cellular_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
const LeaseGrant& lease,
std::span<const Sha256Digest> retained_builds,
milliseconds timeout) {
const auto fail_with = [](RecoveryError kind) {
return std::unexpected{RecoveryFailure{kind, {}}};
};
auto committed = load_region_checkpoint(db, key, timeout);
if (!committed)
return std::unexpected{RecoveryFailure{
committed.error().kind == LeaseStoreError::not_provisioned
? RecoveryError::not_provisioned
: RecoveryError::store,
committed.error()}};
const auto digest = sha256(committed->manifest);
auto manifest = decode_manifest(committed->manifest);
if (!digest || *digest != committed->head.digest || !manifest || manifest->key != key ||
manifest->sequence != committed->head.sequence ||
manifest->commit != committed->head.commit || manifest->end_tick != committed->head.end_tick ||
manifest->frame != committed->head.frame)
return fail_with(RecoveryError::manifest);
if (!supported(*manifest) || manifest->profile != CheckpointProfile::cellular_lives_v1 ||
std::ranges::find(retained_builds, manifest->build) == retained_builds.end())
return fail_with(RecoveryError::unsupported);
if (lease.frame != manifest->frame)
return fail_with(RecoveryError::unsupported);
auto payload = objects.get({manifest->payload_sha256, manifest->payload_bytes});
if (!payload)
return fail_with(payload.error() == ObjectError::io ? RecoveryError::store
: RecoveryError::corrupt_payload);
auto state = restore_cellular_payload(*payload, *manifest);
if (!state)
return fail_with(RecoveryError::payload);
std::vector<server::CellularActionRecord> actions;
if (manifest->sequence != 0) {
auto action_bytes = load_cellular_publication_actions(db, key, lease, manifest->commit, timeout);
if (!action_bytes)
return std::unexpected{RecoveryFailure{RecoveryError::store, action_bytes.error()}};
if (!*action_bytes)
return fail_with(RecoveryError::history);
auto trace_bytes = objects.get({manifest->batch.trace_sha256, manifest->batch.trace_bytes});
if (!trace_bytes)
return fail_with(trace_bytes.error() == ObjectError::io ? RecoveryError::store
: RecoveryError::tail);
auto trace = server::decode_cellular_batch(*trace_bytes);
if (!trace)
return fail_with(RecoveryError::tail);
if (trace->header.world != key.world || trace->header.region != key.region ||
trace->header.frame != manifest->frame ||
trace->header.predecessor != manifest->predecessor_commit ||
trace->header.predecessor_digest != manifest->predecessor_digest ||
trace->header.begin_tick != manifest->begin_tick ||
trace->header.end_tick != manifest->end_tick)
return fail_with(RecoveryError::tail);
auto predecessor = load_region_commit(db, key, manifest->sequence - 1, timeout);
if (!predecessor)
return std::unexpected{RecoveryFailure{RecoveryError::tail, predecessor.error()}};
const auto predecessor_digest = sha256(predecessor->manifest);
auto previous = decode_manifest(predecessor->manifest);
if (!predecessor_digest || *predecessor_digest != manifest->predecessor_digest ||
predecessor->head.digest != manifest->predecessor_digest ||
predecessor->head.commit != manifest->predecessor_commit ||
predecessor->head.sequence != manifest->sequence - 1 || !previous ||
previous->profile != CheckpointProfile::cellular_lives_v1 ||
previous->end_tick != manifest->begin_tick)
return fail_with(RecoveryError::tail);
auto base_bytes = objects.get({previous->payload_sha256, previous->payload_bytes});
if (!base_bytes)
return fail_with(base_bytes.error() == ObjectError::io ? RecoveryError::store
: RecoveryError::tail);
auto base = restore_cellular_payload(*base_bytes, *previous);
if (!base)
return fail_with(RecoveryError::tail);
auto batch_actions = server::decode_cellular_action_block(
**action_bytes, manifest->begin_tick, manifest->end_tick);
if (!batch_actions || !server::match_cellular_actions_to_trace(**base, *trace, *batch_actions))
return fail_with(RecoveryError::history);
auto replay = server::replay_cellular_batch(**base, *trace);
if (!replay || (*replay)->snapshot() != *payload)
return fail_with(RecoveryError::tail);
std::vector<ActionRef> refs;
refs.reserve(batch_actions->size());
for (const auto& action : *batch_actions)
refs.push_back({action.account, action.action_id});
auto records = find_region_action_records(db, key, lease, refs, timeout);
if (!records)
return std::unexpected{RecoveryFailure{RecoveryError::store, records.error()}};
for (std::size_t i = 0; i < batch_actions->size(); ++i) {
if (!(*records)[i])
return fail_with(RecoveryError::history);
const auto& record = *(*records)[i];
const auto& action = (*batch_actions)[i];
if (record.kind != 2 || record.result != 5 || !record.cellular_input ||
*record.cellular_input !=
std::vector<std::uint8_t>(action.input.begin(), action.input.end()) ||
record.action.region != key.region || record.action.tick != input_tick(action) ||
record.action.commit_sequence != manifest->sequence)
return fail_with(RecoveryError::history);
}
actions = std::move(*batch_actions);
}
return RecoveredCellularCheckpoint{std::move(*committed), *manifest, std::move(*state),
std::move(actions)};
}
} // namespace evo::persistence
+104
View File
@@ -0,0 +1,104 @@
// Profile-4 publication and recovery against the migration-backed PostgreSQL adapter.
#include "evo/persistence/checkpoint_store.hpp"
#include "evo/persistence/sha256.hpp"
#include <cstdlib>
#include <source_location>
#include <stdexcept>
void run_cellular_publication_tests();
namespace {
using namespace evo::persistence;
using namespace evo::server;
using namespace std::chrono_literals;
void check(bool value, std::source_location at = std::source_location::current()) {
if (!value)
throw std::runtime_error("cellular publication test line " + std::to_string(at.line()));
}
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
std::string env(const char* name) {
const auto* value = std::getenv(name);
if (!value)
throw std::runtime_error("missing PostgreSQL test environment");
return value;
}
auto connect_as(const std::string& user) {
auto db = Postgres::connect(
{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), user, {}, {}}, 5s);
CHECK(db);
return std::move(*db);
}
template <std::size_t N> std::array<std::uint8_t, N> filled(std::uint8_t value) {
std::array<std::uint8_t, N> out{};
out.fill(value);
return out;
}
std::string hex(std::span<const std::uint8_t> bytes) {
constexpr char digits[] = "0123456789abcdef";
std::string out = "\\x";
for (const auto byte : bytes) {
out.push_back(digits[byte >> 4U]);
out.push_back(digits[byte & 15U]);
}
return out;
}
} // namespace
void run_cellular_publication_tests() {
auto owner = connect_as(env("PGUSER"));
auto db = connect_as(env("EVO_PG_RUNTIME_USER"));
constexpr RegionKey key{9756, 1};
const auto build = filled<32>(0xD9);
CHECK(owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
"(decode(repeat('d9',32),'hex')) ON CONFLICT DO NOTHING",
{}, 5s));
auto endowed = evo::biology::primitive_endowment();
CHECK(endowed);
auto funding = endowed->body;
funding.energy.value = endowed->reserve.energy.value + endowed->construction.value;
auto primitive = PrimitiveLives::create({key.world, key.region, 1, 1, funding, {{0, 0}}});
CHECK(primitive);
auto habitat = evo::biology::CellularHabitat::create(
{key.world, key.region, 1, 1'000, 1'000, {}});
CHECK(habitat);
auto opening = CellularLives::create(**primitive, std::move(*habitat),
{{1'000, 1'000}, 0, 293'150});
CHECK(opening && (*opening)->lives().empty());
const auto payload = (*opening)->snapshot();
const auto payload_digest = sha256(payload);
CHECK(payload_digest);
const auto genesis = genesis_manifest(key, 1, filled<16>(0x21), 50'000, build,
payload.size(), *payload_digest,
CheckpointProfile::cellular_lives_v1);
CHECK(genesis);
const auto genesis_bytes = encode_manifest(*genesis);
CHECK(genesis_bytes);
auto objects = FilesystemObjectStore::open(env("PGHOST"), key, max_cellular_lives_bytes);
CHECK(objects && (*objects)->put(payload));
CHECK(owner->query("SELECT evolution.provision_region_checkpoint($1::bytea)",
std::array{hex(*genesis_bytes)}, 5s));
auto predecessor = load_region_checkpoint(*owner, key, 5s);
CHECK(predecessor);
const auto lease = acquire_region_lease(*db, key, build, 20'000ms, 5s);
CHECK(lease);
const auto predecessor_digest = sha256(predecessor->manifest);
CHECK(predecessor_digest);
CellularBatch batch;
batch.header = {key.world, key.region, 1, predecessor->head.commit, *predecessor_digest, 0, 2};
batch.ticks.resize(2); // The funded genesis has no player-controlled lives.
const auto replay = replay_cellular_batch(**opening, batch);
CHECK(replay);
auto published = publish_cellular_batch(*db, **objects, *lease, *predecessor, batch,
**replay, filled<16>(0x22), build, 5s);
CHECK(published && published->committed.head.sequence == 1 &&
published->committed.head.end_tick == 2);
auto recovered = recover_cellular_checkpoint(*db, **objects, key, *lease,
std::span{&build, 1}, 5s);
CHECK(recovered && recovered->manifest.sequence == 1 && recovered->state->tick() == 2 &&
recovered->state->snapshot() == (**replay).snapshot() && recovered->actions.empty());
CHECK(release_region_lease(*db, key, *lease, 5s));
}
+2
View File
@@ -8,6 +8,7 @@ void run_region_lease_store_tests();
void run_account_control_store_tests();
void run_life_reservation_store_tests();
void run_primitive_publication_tests();
void run_cellular_publication_tests();
void run_probe_handoff_store_tests();
void run_region_action_tests();
void run_region_checkpoint_store_tests();
@@ -137,6 +138,7 @@ int main(int argc, char** argv) {
run_account_control_store_tests();
run_life_reservation_store_tests();
run_primitive_publication_tests();
run_cellular_publication_tests();
run_probe_handoff_store_tests();
run_region_action_tests();
std::puts("region lease fencing integration passed");