Foundation / checks (client) (push) Successful in 5s
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (network) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m2s
Foundation / checks (tsan) (push) Successful in 8m28s
Foundation / checks (postgres) (push) Failing after 32m52s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 34m47s
448 lines
21 KiB
C++
448 lines
21 KiB
C++
// Profile-4 publication and recovery against the migration-backed PostgreSQL adapter.
|
|
#include "evo/persistence/account_control_store.hpp"
|
|
#include "evo/persistence/checkpoint_store.hpp"
|
|
#include "evo/persistence/life_reservation_store.hpp"
|
|
#include "evo/persistence/sha256.hpp"
|
|
#include "evo/server/cellular_region_host.hpp"
|
|
|
|
#include <cstdlib>
|
|
#include <optional>
|
|
#include <source_location>
|
|
#include <stdexcept>
|
|
|
|
void run_cellular_publication_tests();
|
|
namespace {
|
|
using namespace evo::persistence;
|
|
using namespace evo::server;
|
|
using namespace std::chrono_literals;
|
|
void check(bool value, std::source_location at = std::source_location::current()) {
|
|
if (!value)
|
|
throw std::runtime_error("cellular publication test line " + std::to_string(at.line()));
|
|
}
|
|
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
|
|
template <typename T> const T& checked_value(const std::optional<T>& value) {
|
|
if (!value.has_value())
|
|
throw std::runtime_error("cellular publication test expected an optional value");
|
|
return *value;
|
|
}
|
|
template <typename T, typename E> const T& checked_value(const std::expected<T, E>& value) {
|
|
if (!value.has_value())
|
|
throw std::runtime_error("cellular publication test expected a successful result");
|
|
return *value;
|
|
}
|
|
std::string env(const char* name) {
|
|
const auto* value = std::getenv(name);
|
|
if (!value)
|
|
throw std::runtime_error("missing PostgreSQL test environment");
|
|
return value;
|
|
}
|
|
auto connect_as(const std::string& user) {
|
|
auto db =
|
|
Postgres::connect({env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), user, {}, {}}, 5s);
|
|
CHECK(db);
|
|
return std::move(*db);
|
|
}
|
|
template <std::size_t N> std::array<std::uint8_t, N> filled(std::uint8_t value) {
|
|
std::array<std::uint8_t, N> out{};
|
|
out.fill(value);
|
|
return out;
|
|
}
|
|
std::string hex(std::span<const std::uint8_t> bytes) {
|
|
constexpr char digits[] = "0123456789abcdef";
|
|
std::string out = "\\x";
|
|
for (const auto byte : bytes) {
|
|
out.push_back(digits[byte >> 4U]);
|
|
out.push_back(digits[byte & 15U]);
|
|
}
|
|
return out;
|
|
}
|
|
void refresh_cellular_checksum(std::vector<std::uint8_t>& bytes) {
|
|
std::uint64_t value = 14695981039346656037ULL;
|
|
for (const auto byte : std::span{bytes}.first(bytes.size() - 8))
|
|
value = (value ^ byte) * 1099511628211ULL;
|
|
for (unsigned index = 0; index < 8; ++index)
|
|
bytes[bytes.size() - 8 + index] = static_cast<std::uint8_t>(value >> (index * 8U));
|
|
}
|
|
} // namespace
|
|
|
|
namespace {
|
|
void cellular_birth_publication_is_durable() {
|
|
using namespace evo::biology;
|
|
constexpr RegionKey key{9758, 1};
|
|
auto owner = connect_as(env("PGUSER"));
|
|
auto runtime = connect_as(env("EVO_PG_RUNTIME_USER"));
|
|
const auto build = filled<32>(0xD9);
|
|
CHECK(owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
|
|
"(decode(repeat('d9',32),'hex')) ON CONFLICT DO NOTHING",
|
|
{}, 5s));
|
|
auto endowed = primitive_endowment();
|
|
CHECK(endowed);
|
|
auto funding = endowed->body;
|
|
funding.energy.value = endowed->reserve.energy.value + endowed->construction.value;
|
|
auto primitive = PrimitiveLives::create({key.world, key.region, 1, 17, funding, {{0, 0}}});
|
|
CHECK(primitive);
|
|
auto habitat = CellularHabitat::create({key.world, key.region, 1, 1'000, 1'000, {}});
|
|
CHECK(habitat);
|
|
auto opening =
|
|
CellularLives::create(**primitive, std::move(*habitat), {{1'000, 1'000}, 0, 293'150});
|
|
CHECK(opening && (*opening)->lives().empty());
|
|
auto objects = FilesystemObjectStore::open(env("PGHOST"), key, max_cellular_lives_bytes);
|
|
CHECK(objects);
|
|
const auto genesis = provision_cellular_region(*owner, **objects, key, 1, **opening,
|
|
filled<16>(0x30), build, 5s);
|
|
CHECK(genesis);
|
|
CHECK(owner->query("INSERT INTO evolution.life_allocators VALUES(9758,1000)", {}, 5s));
|
|
const auto connect = [&]() -> std::expected<std::unique_ptr<Postgres>, PgFailure> {
|
|
return Postgres::connect({env("PGHOST"),
|
|
{},
|
|
env("PGPORT"),
|
|
env("PGDATABASE"),
|
|
env("EVO_PG_RUNTIME_USER"),
|
|
{},
|
|
{}},
|
|
5s);
|
|
};
|
|
CellularHostConfig config{key, 1, {}, build, {build}};
|
|
auto made = CellularRegionHost::create(config, connect, std::move(*objects));
|
|
CHECK(made);
|
|
auto host = std::move(*made);
|
|
CHECK(host->step().ready);
|
|
const auto authority = host->authority();
|
|
CHECK(authority);
|
|
const auto& authority_value = checked_value(authority);
|
|
const auto committed = host->committed();
|
|
CHECK(committed);
|
|
const auto& committed_value = checked_value(committed);
|
|
const auto account = filled<16>(0x47);
|
|
const auto control =
|
|
claim_account_control(*runtime, key, authority_value, account, 1, 1, 20'000ms, 5s);
|
|
CHECK(control);
|
|
const auto requested = host->create_primitive_life(account, control->generation, 7);
|
|
CHECK(requested.result == evo::network::PrimitiveLifeResult::pending &&
|
|
requested.life_identity != 0 && host->queued_births().size() == 1);
|
|
const auto retried = host->create_primitive_life(account, control->generation, 7);
|
|
CHECK(retried == requested && host->queued_births().size() == 1);
|
|
const auto reservation = host->queued_births().front();
|
|
|
|
CellularBatch batch;
|
|
const auto predecessor_digest = sha256(committed_value.manifest);
|
|
CHECK(predecessor_digest);
|
|
batch.header = {
|
|
key.world, key.region, 1, committed_value.head.commit, checked_value(predecessor_digest),
|
|
0, 1};
|
|
CellularTickIntents tick;
|
|
tick.births.push_back(reservation);
|
|
batch.ticks.push_back(tick);
|
|
auto candidate = replay_cellular_batch(*host->state(), batch);
|
|
CHECK(candidate && (*candidate)->lives().size() == 1 &&
|
|
(*candidate)->lives()[0].identity.life == reservation.identity);
|
|
auto published = host->publish(batch);
|
|
if (!published) {
|
|
const auto& failure = published.error();
|
|
auto message = "cellular host publish failed, error=" +
|
|
std::to_string(static_cast<unsigned>(failure.kind));
|
|
if (failure.publication)
|
|
message +=
|
|
", publication=" + std::to_string(static_cast<unsigned>(*failure.publication));
|
|
if (failure.store) {
|
|
message += ", store=" + std::to_string(static_cast<unsigned>(failure.store->kind));
|
|
const auto& state = failure.store->cause.sqlstate;
|
|
if (state[0] != '\0')
|
|
message += ", sqlstate=" + std::string(state.data(), 5);
|
|
}
|
|
if (failure.object)
|
|
message += ", object=" + std::to_string(static_cast<unsigned>(*failure.object));
|
|
throw std::runtime_error(message);
|
|
}
|
|
CHECK(published->committed.head.sequence == 1);
|
|
CHECK(host->state()->snapshot() == (*candidate)->snapshot());
|
|
CHECK(host->queued_births().empty());
|
|
const auto host_outcome = host->query_primitive_life(account, control->generation, 7);
|
|
CHECK(host_outcome.result == evo::network::PrimitiveLifeResult::born &&
|
|
host_outcome.life_identity == reservation.identity && host_outcome.commit_sequence == 1);
|
|
const auto outcome =
|
|
query_primitive_life(*runtime, key, authority_value, account, control->generation, 7, 5s);
|
|
CHECK(outcome && outcome.value().has_value());
|
|
const auto& outcome_value = checked_value(outcome.value());
|
|
CHECK(outcome_value.published.has_value());
|
|
const auto& published_value = checked_value(outcome_value.published);
|
|
CHECK(published_value.identity == reservation.identity && published_value.born == 1);
|
|
host->drain();
|
|
host.reset();
|
|
|
|
auto restarted_objects =
|
|
FilesystemObjectStore::open(env("PGHOST"), key, max_cellular_lives_bytes);
|
|
CHECK(restarted_objects);
|
|
auto restarted_made =
|
|
CellularRegionHost::create(config, connect, std::move(*restarted_objects));
|
|
CHECK(restarted_made);
|
|
auto restarted = std::move(*restarted_made);
|
|
CHECK(restarted->step().ready && restarted->state()->tick() == 1 &&
|
|
restarted->state()->snapshot() == (*candidate)->snapshot());
|
|
restarted->drain();
|
|
}
|
|
|
|
void cellular_profile_migration_is_durable_and_recovers_without_a_tick() {
|
|
using namespace evo::biology;
|
|
constexpr RegionKey key{9759, 1};
|
|
auto owner = connect_as(env("PGUSER"));
|
|
const auto build = filled<32>(0xD9);
|
|
CHECK(owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
|
|
"(decode(repeat('d9',32),'hex')) ON CONFLICT DO NOTHING",
|
|
{}, 5s));
|
|
auto endowment = primitive_endowment();
|
|
CHECK(endowment);
|
|
auto funding = endowment->body;
|
|
funding.energy.value += endowment->reserve.energy.value + endowment->construction.value;
|
|
for (std::size_t index = 0; index < funding.matter.size(); ++index)
|
|
funding.matter[index].value += endowment->reserve.matter[index].value;
|
|
auto primitive = PrimitiveLives::create({key.world, key.region, 1, 17, funding, {{0, 0}}});
|
|
CHECK(primitive);
|
|
auto habitat = CellularHabitat::create({key.world, key.region, 1, 1'000, 1'000, {}});
|
|
CHECK(habitat);
|
|
auto current =
|
|
CellularLives::create(**primitive, std::move(*habitat), {{1'000, 1'000}, 0, 293'150});
|
|
CHECK(current && (*current)->lives().empty());
|
|
auto legacy_bytes = (*current)->snapshot();
|
|
constexpr std::uint64_t legacy_magic = 0x3130564C43564541ULL;
|
|
for (unsigned index = 0; index < 8; ++index)
|
|
legacy_bytes[index] = static_cast<std::uint8_t>(legacy_magic >> (index * 8U));
|
|
legacy_bytes[8] = 1;
|
|
legacy_bytes[9] = legacy_bytes[10] = legacy_bytes[11] = 0;
|
|
legacy_bytes[12] = 1;
|
|
legacy_bytes[13] = legacy_bytes[14] = legacy_bytes[15] = 0;
|
|
refresh_cellular_checksum(legacy_bytes);
|
|
auto legacy = CellularLives::restore(legacy_bytes, key.world, key.region, 1, 0);
|
|
CHECK(legacy && (*legacy)->format_version() == cellular_lives_legacy_format);
|
|
|
|
auto objects = FilesystemObjectStore::open(env("PGHOST"), key, max_cellular_lives_bytes);
|
|
CHECK(objects);
|
|
auto provisioned =
|
|
provision_cellular_region(*owner, **objects, key, 1, **legacy, filled<16>(0x50), build, 5s);
|
|
CHECK(provisioned);
|
|
CHECK(owner->query("INSERT INTO evolution.life_allocators VALUES(9759,1000) "
|
|
"ON CONFLICT DO NOTHING",
|
|
{}, 5s));
|
|
const auto connect = [&]() -> std::expected<std::unique_ptr<Postgres>, PgFailure> {
|
|
return Postgres::connect({env("PGHOST"),
|
|
{},
|
|
env("PGPORT"),
|
|
env("PGDATABASE"),
|
|
env("EVO_PG_RUNTIME_USER"),
|
|
{},
|
|
{}},
|
|
5s);
|
|
};
|
|
CellularHostConfig config{key, 1, {}, build, {build}};
|
|
auto migration_objects =
|
|
FilesystemObjectStore::open(env("PGHOST"), key, max_cellular_lives_bytes);
|
|
CHECK(migration_objects);
|
|
auto made = CellularRegionHost::create(config, connect, std::move(*objects));
|
|
CHECK(made);
|
|
auto host = std::move(*made);
|
|
CHECK(host->step().ready);
|
|
const auto authority = host->authority();
|
|
CHECK(authority);
|
|
const auto& authority_value = checked_value(authority);
|
|
const auto predecessor = host->committed();
|
|
CHECK(predecessor);
|
|
const auto& predecessor_value = checked_value(predecessor);
|
|
CHECK(predecessor_value.head.sequence == 0);
|
|
|
|
auto runtime = connect_as(env("EVO_PG_RUNTIME_USER"));
|
|
const auto account = filled<16>(0x5A);
|
|
const auto control =
|
|
claim_account_control(*runtime, key, authority_value, account, 1, 1, 20'000ms, 5s);
|
|
CHECK(control);
|
|
const auto requested = host->create_primitive_life(account, control->generation, 9);
|
|
CHECK(requested.result == evo::network::PrimitiveLifeResult::pending &&
|
|
requested.life_identity != 0 && host->queued_births().size() == 1);
|
|
|
|
CellularBatch batch;
|
|
const auto predecessor_digest = sha256(predecessor_value.manifest);
|
|
CHECK(predecessor_digest);
|
|
batch.header = {
|
|
key.world, key.region, 1, predecessor_value.head.commit, checked_value(predecessor_digest),
|
|
0, 1};
|
|
CellularTickIntents tick;
|
|
tick.births.push_back(host->queued_births().front());
|
|
batch.ticks.push_back(tick);
|
|
const auto published = host->publish(batch);
|
|
CHECK(published && published->committed.head.sequence == 1);
|
|
CHECK(host->state()->lives().size() == 1 &&
|
|
host->state()->lives().front().identity.born.value == 1);
|
|
auto migrated =
|
|
migrate_cellular_profile_to_v2(*owner, **migration_objects, authority_value,
|
|
published->committed, filled<16>(0x51), build, 5s);
|
|
CHECK(migrated && migrated->head.sequence == 2 && migrated->head.end_tick == 1);
|
|
CHECK(
|
|
retry_cellular_profile_migration(*owner, {key, filled<16>(0x51), migrated->manifest}, 5s));
|
|
auto revision_migrated = migrate_cellular_profile_to_v3(
|
|
*owner, **migration_objects, authority_value, *migrated, filled<16>(0x52), build, 5s);
|
|
CHECK(revision_migrated && revision_migrated->head.sequence == 3 &&
|
|
revision_migrated->head.end_tick == 1);
|
|
CHECK(retry_cellular_profile_migration(
|
|
*owner, {key, filled<16>(0x52), revision_migrated->manifest}, 5s));
|
|
host->drain();
|
|
host.reset();
|
|
|
|
auto recovery_objects =
|
|
FilesystemObjectStore::open(env("PGHOST"), key, max_cellular_lives_bytes);
|
|
CHECK(recovery_objects);
|
|
const auto target_manifest = decode_manifest(revision_migrated->manifest);
|
|
CHECK(target_manifest);
|
|
const auto target_payload =
|
|
(*recovery_objects)->get({target_manifest->payload_sha256, target_manifest->payload_bytes});
|
|
CHECK(target_payload);
|
|
auto restarted_made = CellularRegionHost::create(config, connect, std::move(*recovery_objects));
|
|
CHECK(restarted_made);
|
|
auto restarted = std::move(*restarted_made);
|
|
const auto restarted_committed = restarted->committed();
|
|
CHECK(restarted_committed);
|
|
const auto& restarted_committed_value = checked_value(restarted_committed);
|
|
CHECK(restarted->step().ready && restarted_committed_value.head.sequence == 3 &&
|
|
restarted_committed_value.head.end_tick == 1 && restarted->state()->tick() == 1 &&
|
|
restarted->state()->format_version() == cellular_lives_revision_format &&
|
|
restarted->state()->snapshot() == *target_payload);
|
|
restarted->drain();
|
|
}
|
|
} // namespace
|
|
|
|
void run_cellular_publication_tests() {
|
|
cellular_birth_publication_is_durable();
|
|
cellular_profile_migration_is_durable_and_recovers_without_a_tick();
|
|
auto owner = connect_as(env("PGUSER"));
|
|
const auto build = filled<32>(0xD9);
|
|
CHECK(owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
|
|
"(decode(repeat('d9',32),'hex')) ON CONFLICT DO NOTHING",
|
|
{}, 5s));
|
|
|
|
constexpr RegionKey empty_key{9757, 1};
|
|
auto empty_nursery =
|
|
PrimitiveLives::create({empty_key.world, empty_key.region, 1, 17, {}, {{0, 0}}});
|
|
CHECK(empty_nursery);
|
|
auto empty_habitat = evo::biology::CellularHabitat::create(
|
|
{empty_key.world,
|
|
empty_key.region,
|
|
1,
|
|
1'000,
|
|
1'000,
|
|
{{{0, 0}, 19, {{12'000}, {{{10}, {20}, {30}, {40}}}}}}});
|
|
CHECK(empty_habitat);
|
|
auto empty_opening = CellularLives::create(**empty_nursery, std::move(*empty_habitat),
|
|
{{1'000, 1'000}, 0, 293'150});
|
|
CHECK(empty_opening && (*empty_opening)->lives().empty());
|
|
auto empty_objects =
|
|
FilesystemObjectStore::open(env("PGHOST"), empty_key, max_cellular_lives_bytes);
|
|
CHECK(empty_objects);
|
|
const auto empty_commit = filled<16>(0x20);
|
|
auto provisioned = provision_cellular_region(*owner, **empty_objects, empty_key, 1,
|
|
**empty_opening, empty_commit, build, 5s);
|
|
CHECK(provisioned && provisioned->head.sequence == 0 && provisioned->head.end_tick == 0);
|
|
auto empty_manifest = decode_manifest(provisioned->manifest);
|
|
CHECK(empty_manifest && empty_manifest->profile == CheckpointProfile::cellular_lives_v1 &&
|
|
empty_manifest->cellular_versions == cellular_v2_versions &&
|
|
empty_manifest->payload_bytes == (*empty_opening)->snapshot().size());
|
|
auto genesis_payload =
|
|
(*empty_objects)->get({empty_manifest->payload_sha256, empty_manifest->payload_bytes});
|
|
CHECK(genesis_payload);
|
|
auto restored_genesis = restore_cellular_payload(*genesis_payload, *empty_manifest);
|
|
CHECK(restored_genesis && (*restored_genesis)->lives().empty() &&
|
|
(*restored_genesis)->nursery() == (*empty_opening)->nursery() &&
|
|
(*restored_genesis)->habitat().snapshot() == (*empty_opening)->habitat().snapshot());
|
|
auto exact_retry = provision_cellular_region(*owner, **empty_objects, empty_key, 1,
|
|
**empty_opening, empty_commit, build, 5s);
|
|
CHECK(exact_retry && exact_retry->manifest == provisioned->manifest);
|
|
CHECK(!provision_cellular_region(*owner, **empty_objects, empty_key, 1, **empty_opening,
|
|
filled<16>(0x22), build, 5s));
|
|
|
|
constexpr RegionKey key{9756, 1};
|
|
|
|
auto endowed = evo::biology::primitive_endowment();
|
|
CHECK(endowed);
|
|
auto funding = endowed->body;
|
|
funding.energy.value = endowed->reserve.energy.value + endowed->construction.value;
|
|
auto primitive = PrimitiveLives::create({key.world, key.region, 1, 17, funding, {{0, 0}}});
|
|
CHECK(primitive);
|
|
LifeRequest founder;
|
|
founder.account = filled<16>(0x43);
|
|
founder.action = 7;
|
|
CHECK((*primitive)->spawn(founder, {1, 101, 201, 301, {0}}));
|
|
auto habitat =
|
|
evo::biology::CellularHabitat::create({key.world, key.region, 1, 1'000, 1'000, {}});
|
|
CHECK(habitat);
|
|
auto opening =
|
|
CellularLives::create(**primitive, std::move(*habitat), {{1'000, 1'000}, 0, 293'150});
|
|
CHECK(opening && (*opening)->lives().size() == 1);
|
|
|
|
const auto payload = (*opening)->snapshot();
|
|
const auto payload_digest = sha256(payload);
|
|
CHECK(payload_digest);
|
|
auto genesis = genesis_manifest(key, 1, filled<16>(0x21), 50'000, build, payload.size(),
|
|
*payload_digest, CheckpointProfile::cellular_lives_v1);
|
|
CHECK(genesis);
|
|
genesis->cellular_versions = cellular_v2_versions;
|
|
const auto genesis_bytes = encode_manifest(*genesis);
|
|
CHECK(genesis_bytes);
|
|
auto objects = FilesystemObjectStore::open(env("PGHOST"), key, max_cellular_lives_bytes);
|
|
CHECK(objects && (*objects)->put(payload));
|
|
CHECK(owner->query("SELECT evolution.provision_region_checkpoint($1::bytea)",
|
|
std::array{hex(*genesis_bytes)}, 5s));
|
|
const auto connect = [&]() -> std::expected<std::unique_ptr<Postgres>, PgFailure> {
|
|
return Postgres::connect({env("PGHOST"),
|
|
{},
|
|
env("PGPORT"),
|
|
env("PGDATABASE"),
|
|
env("EVO_PG_RUNTIME_USER"),
|
|
{},
|
|
{}},
|
|
5s);
|
|
};
|
|
CellularHostConfig config{key, 1, {}, build, {build}};
|
|
auto made_host = CellularRegionHost::create(config, connect, std::move(*objects));
|
|
CHECK(made_host);
|
|
auto host = std::move(*made_host);
|
|
CHECK(host->step().ready);
|
|
const auto committed = host->committed();
|
|
CHECK(committed);
|
|
const auto& committed_value = checked_value(committed);
|
|
CellularBatch batch;
|
|
const auto predecessor_digest = sha256(committed_value.manifest);
|
|
CHECK(predecessor_digest);
|
|
batch.header = {
|
|
key.world, key.region, 1, committed_value.head.commit, checked_value(predecessor_digest),
|
|
0, 2};
|
|
batch.ticks.resize(2);
|
|
for (auto& tick : batch.ticks)
|
|
tick.lives.push_back({(*opening)->lives()[0].identity.life, {}});
|
|
const auto replay = replay_cellular_batch(*host->state(), batch);
|
|
CHECK(replay);
|
|
CellularActionRecord action;
|
|
action.account = founder.account;
|
|
action.action_id = 77;
|
|
action.input[0] = 1; // Tick 1, neutral movement, no patch target.
|
|
const std::array actions{action};
|
|
auto published = host->publish(batch, actions);
|
|
CHECK(published && published->committed.head.sequence == 1 &&
|
|
published->committed.head.end_tick == 2 &&
|
|
host->state()->snapshot() == (**replay).snapshot());
|
|
CHECK(host->take_finalized_actions() == std::vector<FinalCellularAction>{{action, 1}});
|
|
host->drain();
|
|
host.reset();
|
|
|
|
auto restarted_objects =
|
|
FilesystemObjectStore::open(env("PGHOST"), key, max_cellular_lives_bytes);
|
|
CHECK(restarted_objects);
|
|
auto made_restarted =
|
|
CellularRegionHost::create(config, connect, std::move(*restarted_objects));
|
|
CHECK(made_restarted);
|
|
auto restarted = std::move(*made_restarted);
|
|
const auto restarted_committed = restarted->committed();
|
|
CHECK(restarted_committed);
|
|
const auto& restarted_committed_value = checked_value(restarted_committed);
|
|
CHECK(restarted->step().ready && restarted->state()->tick() == 2 &&
|
|
restarted->state()->snapshot() == (**replay).snapshot() &&
|
|
restarted_committed_value.head.sequence == 1);
|
|
restarted->drain();
|
|
}
|