M12.4: retain migration source freezes and explicit aborts
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m24s
Foundation / checks (network) (push) Successful in 20m44s
Foundation / checks (client) (push) Successful in 22m3s
Foundation / checks (tsan) (push) Successful in 23m1s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 24m7s
Foundation / checks (postgres) (push) Canceled after 24m3s
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m24s
Foundation / checks (network) (push) Successful in 20m44s
Foundation / checks (client) (push) Successful in 22m3s
Foundation / checks (tsan) (push) Successful in 23m1s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 24m7s
Foundation / checks (postgres) (push) Canceled after 24m3s
This commit is contained in:
@@ -0,0 +1,255 @@
|
||||
#include "evo/persistence/cellular_migration_store.hpp"
|
||||
#include "evo/persistence/sha256.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <charconv>
|
||||
#include <string_view>
|
||||
|
||||
namespace evo::persistence {
|
||||
namespace {
|
||||
using std::chrono::milliseconds;
|
||||
LeaseStoreFailure failure(const PgFailure& cause) {
|
||||
const std::string_view state(cause.sqlstate.data(), 5);
|
||||
auto kind = LeaseStoreError::store;
|
||||
if (cause.kind == PgError::unknown)
|
||||
kind = LeaseStoreError::unknown;
|
||||
else if (state == "22023" || cause.kind == PgError::bounds || cause.kind == PgError::config)
|
||||
kind = LeaseStoreError::invalid;
|
||||
else if (state == "EVL02")
|
||||
kind = LeaseStoreError::fenced;
|
||||
else if (state == "EVL03")
|
||||
kind = LeaseStoreError::head_mismatch;
|
||||
else if (state == "EVL04")
|
||||
kind = LeaseStoreError::not_provisioned;
|
||||
else if (state == "EVL06")
|
||||
kind = LeaseStoreError::unsupported;
|
||||
else if (state == "EVL07" || state == "EVL18")
|
||||
kind = LeaseStoreError::commit_conflict;
|
||||
return {kind, cause};
|
||||
}
|
||||
std::string hex(std::span<const std::uint8_t> bytes) {
|
||||
constexpr std::string_view digits = "0123456789abcdef";
|
||||
std::string out;
|
||||
for (const auto byte : bytes) {
|
||||
out += digits[byte >> 4U];
|
||||
out += digits[byte & 15U];
|
||||
}
|
||||
return out;
|
||||
}
|
||||
std::string uuid(const Uuid& id) {
|
||||
const auto h = hex(id);
|
||||
return h.substr(0, 8) + '-' + h.substr(8, 4) + '-' + h.substr(12, 4) + '-' + h.substr(16, 4) +
|
||||
'-' + h.substr(20);
|
||||
}
|
||||
std::optional<std::vector<std::uint8_t>> unhex(std::string_view value, std::size_t cap) {
|
||||
if (value.size() % 2 != 0 || value.size() / 2 > cap)
|
||||
return std::nullopt;
|
||||
std::vector<std::uint8_t> out;
|
||||
out.reserve(value.size() / 2);
|
||||
for (std::size_t at = 0; at < value.size(); at += 2) {
|
||||
unsigned byte{};
|
||||
const auto [end, error] =
|
||||
std::from_chars(value.data() + at, value.data() + at + 2, byte, 16);
|
||||
if (error != std::errc{} || end != value.data() + at + 2 || byte > 255)
|
||||
return std::nullopt;
|
||||
out.push_back(static_cast<std::uint8_t>(byte));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
struct Record {
|
||||
std::vector<std::uint8_t> trace, source, destination;
|
||||
bool aborted{};
|
||||
};
|
||||
std::optional<Record> parse(const PgRows::value_type& row) {
|
||||
if (row.size() != 4 || (row[3].value_or("") != "true" && row[3].value_or("") != "false"))
|
||||
return std::nullopt;
|
||||
auto trace = unhex(row[0].value_or(""), server::max_migration_trace_bytes);
|
||||
auto source = unhex(row[1].value_or(""), checkpoint_manifest_bytes);
|
||||
auto destination = unhex(row[2].value_or(""), checkpoint_manifest_bytes);
|
||||
if (!trace || !source || !destination)
|
||||
return std::nullopt;
|
||||
return Record{std::move(*trace), std::move(*source), std::move(*destination),
|
||||
row[3].value_or("") == "true"};
|
||||
}
|
||||
bool bound(const CommittedCheckpoint& c, const CheckpointManifest& m) {
|
||||
const auto digest = sha256(c.manifest);
|
||||
return digest && *digest == c.head.digest && m.commit == c.head.commit &&
|
||||
m.frame == c.head.frame && m.sequence == c.head.sequence &&
|
||||
m.end_tick == c.head.end_tick;
|
||||
}
|
||||
bool cuts(const CheckpointManifest& s, const CheckpointManifest& d,
|
||||
const Sha256Digest& source_digest, const Sha256Digest& destination_digest,
|
||||
const server::CellularMigrationTrace& t) {
|
||||
const auto profile = t.representation == server::CellularMigrationRepresentation::individual
|
||||
? CheckpointProfile::cellular_lives_v1
|
||||
: CheckpointProfile::cellular_population_v1;
|
||||
return supported(s) && supported(d) && encode_manifest(s) && encode_manifest(d) &&
|
||||
s.cellular_versions == cellular_v10_versions &&
|
||||
d.cellular_versions == cellular_v10_versions && s.profile == profile &&
|
||||
d.profile == profile && s.sequence > 0 && d.sequence > 0 &&
|
||||
s.key == RegionKey{t.world, t.request.source.region} &&
|
||||
d.key == RegionKey{t.world, t.request.destination.region} && s.frame == t.frame &&
|
||||
d.frame == t.frame && s.end_tick == t.tick && d.end_tick == t.tick &&
|
||||
s.commit == t.request.source.predecessor &&
|
||||
d.commit == t.request.destination.predecessor &&
|
||||
source_digest == t.request.source.predecessor_digest &&
|
||||
destination_digest == t.request.destination.predecessor_digest;
|
||||
}
|
||||
std::expected<server::CellularMigrationTrace, CheckpointError>
|
||||
derive(const CheckpointManifest& s, const CheckpointManifest& d,
|
||||
std::span<const std::uint8_t> source, std::span<const std::uint8_t> destination,
|
||||
const server::CellularMigrationRequest& request) {
|
||||
if (s.profile == CheckpointProfile::cellular_lives_v1) {
|
||||
auto src = restore_cellular_payload(source, s),
|
||||
dst = restore_cellular_payload(destination, d);
|
||||
if (!src || !dst)
|
||||
return std::unexpected(CheckpointError::payload);
|
||||
auto pair = server::prepare_migration_trace(**src, **dst, request);
|
||||
if (!pair)
|
||||
return std::unexpected(CheckpointError::payload);
|
||||
return std::move(pair->trace);
|
||||
}
|
||||
auto src = restore_population_payload(source, s),
|
||||
dst = restore_population_payload(destination, d);
|
||||
if (!src || !dst)
|
||||
return std::unexpected(CheckpointError::payload);
|
||||
auto pair = server::prepare_migration_trace(**src, **dst, request);
|
||||
if (!pair)
|
||||
return std::unexpected(CheckpointError::payload);
|
||||
return std::move(pair->trace);
|
||||
}
|
||||
std::optional<CellularMigrationFreeze> checked_record(const Record& record, RegionKey key,
|
||||
const LeaseGrant& lease) {
|
||||
auto trace = server::decode_migration_trace(record.trace);
|
||||
const auto source = decode_manifest(record.source),
|
||||
destination = decode_manifest(record.destination);
|
||||
const auto s = sha256(record.source), d = sha256(record.destination);
|
||||
if (!trace || !source || !destination || !s || !d ||
|
||||
key != RegionKey{trace->world, trace->request.source.region} ||
|
||||
lease.frame != trace->frame || !cuts(*source, *destination, *s, *d, *trace))
|
||||
return std::nullopt;
|
||||
return CellularMigrationFreeze{std::move(*trace), record.aborted};
|
||||
}
|
||||
constexpr auto columns = "encode(trace,'hex'),encode(source_manifest,'hex'),"
|
||||
"encode(destination_manifest,'hex'),aborted::text";
|
||||
CellularMigrationFailure failed(PublicationError kind, LeaseStoreFailure store = {},
|
||||
ObjectError object = {}, CellularMigrationPrepared prepared = {}) {
|
||||
return {kind, store, object, std::move(prepared)};
|
||||
}
|
||||
} // namespace
|
||||
|
||||
std::expected<CellularMigrationFreeze, CellularMigrationFailure>
|
||||
retry_cellular_migration_freeze(Postgres& db, const LeaseGrant& lease,
|
||||
const CellularMigrationPrepared& prepared, milliseconds timeout) {
|
||||
const auto trace = server::decode_migration_trace(prepared.trace);
|
||||
if (!trace || prepared.key != RegionKey{trace->world, trace->request.source.region} ||
|
||||
lease.frame != trace->frame)
|
||||
return std::unexpected(failed(PublicationError::candidate));
|
||||
const std::array<std::string, 5> args{std::to_string(prepared.key.world),
|
||||
std::to_string(prepared.key.region), uuid(lease.holder),
|
||||
std::to_string(lease.epoch), "\\x" + hex(prepared.trace)};
|
||||
auto rows =
|
||||
db.query(std::string("SELECT ") + columns +
|
||||
" FROM evolution.prepare_cellular_migration_freeze("
|
||||
"$1::evolution.u64,$2::evolution.u64,$3::uuid,$4::evolution.u64,$5::bytea)",
|
||||
args, timeout);
|
||||
if (!rows) {
|
||||
const auto store = failure(rows.error());
|
||||
return std::unexpected(failed(store.kind == LeaseStoreError::unknown
|
||||
? PublicationError::unknown
|
||||
: PublicationError::rejected,
|
||||
store, {}, prepared));
|
||||
}
|
||||
const auto record = rows->size() == 1 ? parse(rows->front()) : std::nullopt;
|
||||
const auto result = record ? checked_record(*record, prepared.key, lease) : std::nullopt;
|
||||
if (!result || record->trace != prepared.trace)
|
||||
return std::unexpected(failed(PublicationError::unknown, {}, {}, prepared));
|
||||
return *result;
|
||||
}
|
||||
|
||||
std::expected<CellularMigrationFreeze, CellularMigrationFailure>
|
||||
prepare_cellular_migration_freeze(Postgres& db, const FilesystemObjectStore& source_objects,
|
||||
const FilesystemObjectStore& destination_objects,
|
||||
const LeaseGrant& lease, const CommittedCheckpoint& source,
|
||||
const CommittedCheckpoint& destination,
|
||||
const server::CellularMigrationRequest& request,
|
||||
milliseconds timeout, const PublishHooks& hooks) {
|
||||
const auto s = decode_manifest(source.manifest), d = decode_manifest(destination.manifest);
|
||||
if (!s || !d || !bound(source, *s) || !bound(destination, *d))
|
||||
return std::unexpected(failed(PublicationError::predecessor));
|
||||
auto src = source_objects.get({s->payload_sha256, s->payload_bytes});
|
||||
auto dst = destination_objects.get({d->payload_sha256, d->payload_bytes});
|
||||
if (!src || !dst)
|
||||
return std::unexpected(
|
||||
failed(PublicationError::predecessor, {}, !src ? src.error() : dst.error()));
|
||||
auto trace = derive(*s, *d, *src, *dst, request);
|
||||
if (!trace || !cuts(*s, *d, source.head.digest, destination.head.digest, *trace) ||
|
||||
lease.frame != s->frame)
|
||||
return std::unexpected(failed(PublicationError::candidate));
|
||||
auto encoded = server::encode_migration_trace(*trace);
|
||||
if (!encoded)
|
||||
return std::unexpected(failed(PublicationError::candidate));
|
||||
auto capsule = source_objects.put(*dst);
|
||||
if (!capsule)
|
||||
return std::unexpected(failed(PublicationError::upload, {}, capsule.error()));
|
||||
auto readback = source_objects.get(*capsule);
|
||||
if (!readback || *readback != *dst ||
|
||||
*capsule != VerifiedObject{d->payload_sha256, d->payload_bytes})
|
||||
return std::unexpected(failed(PublicationError::verify, {},
|
||||
readback ? ObjectError::corrupt : readback.error()));
|
||||
if (hooks.before_transaction)
|
||||
hooks.before_transaction();
|
||||
return retry_cellular_migration_freeze(db, lease, {s->key, std::move(*encoded)}, timeout);
|
||||
}
|
||||
|
||||
std::expected<CellularMigrationFreeze, LeaseStoreFailure>
|
||||
abort_cellular_migration_freeze(Postgres& db, RegionKey key, const LeaseGrant& lease,
|
||||
const Uuid& request, milliseconds timeout) {
|
||||
const std::array<std::string, 5> args{std::to_string(key.world), std::to_string(key.region),
|
||||
uuid(lease.holder), std::to_string(lease.epoch),
|
||||
uuid(request)};
|
||||
auto rows =
|
||||
db.query(std::string("SELECT ") + columns +
|
||||
" FROM evolution.abort_cellular_migration_freeze("
|
||||
"$1::evolution.u64,$2::evolution.u64,$3::uuid,$4::evolution.u64,$5::uuid)",
|
||||
args, timeout);
|
||||
if (!rows)
|
||||
return std::unexpected(failure(rows.error()));
|
||||
const auto record = rows->size() == 1 ? parse(rows->front()) : std::nullopt;
|
||||
auto result = record ? checked_record(*record, key, lease) : std::nullopt;
|
||||
if (!result || !result->aborted || result->trace.request.request != request)
|
||||
return std::unexpected(LeaseStoreFailure{LeaseStoreError::unknown, {}});
|
||||
return *result;
|
||||
}
|
||||
|
||||
std::expected<std::optional<CellularMigrationFreeze>, RecoveryFailure>
|
||||
load_cellular_migration_freeze(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
const LeaseGrant& lease, milliseconds timeout) {
|
||||
const auto fail = [](RecoveryError kind) { return std::unexpected(RecoveryFailure{kind, {}}); };
|
||||
const std::array<std::string, 4> args{std::to_string(key.world), std::to_string(key.region),
|
||||
uuid(lease.holder), std::to_string(lease.epoch)};
|
||||
auto rows = db.query(std::string("SELECT ") + columns +
|
||||
" FROM evolution.load_cellular_migration_freeze("
|
||||
"$1::evolution.u64,$2::evolution.u64,$3::uuid,$4::evolution.u64)",
|
||||
args, timeout);
|
||||
if (!rows)
|
||||
return std::unexpected(RecoveryFailure{RecoveryError::store, failure(rows.error())});
|
||||
if (rows->empty())
|
||||
return std::optional<CellularMigrationFreeze>{};
|
||||
const auto record = rows->size() == 1 ? parse(rows->front()) : std::nullopt;
|
||||
auto result = record ? checked_record(*record, key, lease) : std::nullopt;
|
||||
if (!result || result->aborted)
|
||||
return fail(RecoveryError::history);
|
||||
const auto s = decode_manifest(record->source), d = decode_manifest(record->destination);
|
||||
if (!s || !d)
|
||||
return fail(RecoveryError::manifest);
|
||||
auto src = objects.get({s->payload_sha256, s->payload_bytes});
|
||||
auto dst = objects.get({d->payload_sha256, d->payload_bytes});
|
||||
if (!src || !dst)
|
||||
return fail(RecoveryError::tail);
|
||||
const auto replay = derive(*s, *d, *src, *dst, result->trace.request);
|
||||
if (!replay || *replay != result->trace)
|
||||
return fail(RecoveryError::history);
|
||||
return result;
|
||||
}
|
||||
} // namespace evo::persistence
|
||||
Reference in New Issue
Block a user