Fix object reuse durability and refuse unsafe pre-v5 upgrades
Review P1 (object durability): a name installed by a put whose directory fsync failed was later reused as success without syncing again, so a publication could acknowledge an object that might vanish on power loss. Reuse (existing and lost-link-race paths) now fsyncs the verified file and the directory before succeeding, and namespace opens always sync their parent. A test-only ObjectStoreFaults hook injects directory sync failures; tests cover failed sync then retries, namespace creation syncs, and publication refusing to proceed until durability holds. Review P1 (pre-v5 history): v5 recovery requires one durable decision per terminal actor, but v3/v4 published checkpoints carry none, so an accepted upgrade could make regions unrecoverable. pre_v5_guard.sql now runs before 0005 in every upgrade path and refuses (EVL10) when any published pre-v5 checkpoint exists; genesis-only regions pass. 0005 is unchanged. CI publishes a real v4 checkpoint and proves the upgrade is refused with nothing changed; a committed death without a decision is shown to recover as quarantined history. Validated: PostgreSQL suites Release x3, clang-tidy, ASan/UBSan x2; reverting each fix is caught by its test. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -48,15 +48,26 @@ std::string key_name(const Sha256Digest& d) {
|
||||
}
|
||||
return out;
|
||||
}
|
||||
// Opens or creates a private subdirectory without following symlinks.
|
||||
int subdirectory(int parent, const std::string& name) {
|
||||
bool created = false;
|
||||
if (::mkdirat(parent, name.c_str(), 0700) == 0)
|
||||
created = true;
|
||||
else if (errno != EEXIST)
|
||||
// Every directory fsync goes through here so tests can inject failures.
|
||||
bool sync_directory(int fd, ObjectStoreFaults* faults) {
|
||||
if (faults && faults->fail_directory_syncs > 0) {
|
||||
--faults->fail_directory_syncs;
|
||||
return false;
|
||||
}
|
||||
if (::fsync(fd) != 0)
|
||||
return false;
|
||||
if (faults)
|
||||
++faults->directory_syncs;
|
||||
return true;
|
||||
}
|
||||
// Opens or creates a private subdirectory without following symlinks. The parent
|
||||
// is synced even when the entry already existed: an earlier creation may have
|
||||
// failed its sync, and an unsynced entry must not be trusted as durable.
|
||||
int subdirectory(int parent, const std::string& name, ObjectStoreFaults* faults) {
|
||||
if (::mkdirat(parent, name.c_str(), 0700) != 0 && errno != EEXIST)
|
||||
return -1;
|
||||
const int fd = ::openat(parent, name.c_str(), O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||
if (fd >= 0 && created && ::fsync(parent) != 0) {
|
||||
if (fd >= 0 && !sync_directory(parent, faults)) {
|
||||
static_cast<void>(::close(fd));
|
||||
return -1;
|
||||
}
|
||||
@@ -74,8 +85,9 @@ bool write_all(int fd, std::span<const std::uint8_t> bytes) {
|
||||
return true;
|
||||
}
|
||||
// Exact-length regular file whose content has the expected digest.
|
||||
std::expected<std::vector<std::uint8_t>, ObjectError> read_verified(int directory,
|
||||
const VerifiedObject& object) {
|
||||
// With sync_file, the verified file is also fsynced before it is trusted as durable.
|
||||
std::expected<std::vector<std::uint8_t>, ObjectError>
|
||||
read_verified(int directory, const VerifiedObject& object, bool sync_file = false) {
|
||||
const auto name = key_name(object.sha256);
|
||||
const Descriptor fd{::openat(directory, name.c_str(), O_RDONLY | O_NOFOLLOW | O_CLOEXEC)};
|
||||
if (fd.get() < 0)
|
||||
@@ -109,13 +121,16 @@ std::expected<std::vector<std::uint8_t>, ObjectError> read_verified(int director
|
||||
return std::unexpected{ObjectError::io};
|
||||
if (*digest != object.sha256)
|
||||
return std::unexpected{ObjectError::corrupt};
|
||||
if (sync_file && ::fsync(fd.get()) != 0)
|
||||
return std::unexpected{ObjectError::io};
|
||||
return bytes;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
FilesystemObjectStore::FilesystemObjectStore(std::filesystem::path directory, int descriptor,
|
||||
std::size_t maximum)
|
||||
: directory_(std::move(directory)), descriptor_(descriptor), maximum_(maximum) {}
|
||||
std::size_t maximum, ObjectStoreFaults* faults)
|
||||
: directory_(std::move(directory)), descriptor_(descriptor), maximum_(maximum),
|
||||
faults_(faults) {}
|
||||
|
||||
FilesystemObjectStore::~FilesystemObjectStore() {
|
||||
static_cast<void>(::close(descriptor_));
|
||||
@@ -123,7 +138,7 @@ FilesystemObjectStore::~FilesystemObjectStore() {
|
||||
|
||||
std::expected<std::unique_ptr<FilesystemObjectStore>, ObjectError>
|
||||
FilesystemObjectStore::open(const std::filesystem::path& trusted_root, RegionKey key,
|
||||
std::size_t max_object_bytes) {
|
||||
std::size_t max_object_bytes, ObjectStoreFaults* faults) {
|
||||
if (key.world == 0 || key.region == 0 || max_object_bytes == 0 ||
|
||||
max_object_bytes > maximum_object_limit || trusted_root.empty())
|
||||
return std::unexpected{ObjectError::invalid};
|
||||
@@ -133,14 +148,14 @@ FilesystemObjectStore::open(const std::filesystem::path& trusted_root, RegionKey
|
||||
return std::unexpected{ObjectError::invalid};
|
||||
const auto world = "w" + std::to_string(key.world);
|
||||
const auto region = "r" + std::to_string(key.region);
|
||||
const Descriptor world_fd{subdirectory(root.get(), world)};
|
||||
const Descriptor world_fd{subdirectory(root.get(), world, faults)};
|
||||
if (world_fd.get() < 0)
|
||||
return std::unexpected{ObjectError::io};
|
||||
Descriptor region_fd{subdirectory(world_fd.get(), region)};
|
||||
Descriptor region_fd{subdirectory(world_fd.get(), region, faults)};
|
||||
if (region_fd.get() < 0)
|
||||
return std::unexpected{ObjectError::io};
|
||||
return std::unique_ptr<FilesystemObjectStore>(new FilesystemObjectStore(
|
||||
trusted_root / world / region, region_fd.release(), max_object_bytes));
|
||||
trusted_root / world / region, region_fd.release(), max_object_bytes, faults));
|
||||
}
|
||||
|
||||
std::expected<VerifiedObject, ObjectError>
|
||||
@@ -153,14 +168,8 @@ FilesystemObjectStore::put(std::span<const std::uint8_t> bytes) const {
|
||||
if (!digest)
|
||||
return std::unexpected{ObjectError::io};
|
||||
const VerifiedObject object{*digest, bytes.size()};
|
||||
const auto existing = [&]() -> std::expected<VerifiedObject, ObjectError> {
|
||||
auto stored = read_verified(descriptor_, object);
|
||||
if (!stored)
|
||||
return std::unexpected{stored.error()};
|
||||
return object;
|
||||
};
|
||||
// Immutable: identical content is reused; anything else under the key is corrupt.
|
||||
if (auto found = existing(); found || found.error() != ObjectError::missing)
|
||||
// Immutable: identical content is reused (durably); anything else is corrupt.
|
||||
if (auto found = durable_existing(object); found || found.error() != ObjectError::missing)
|
||||
return found;
|
||||
|
||||
const auto name = key_name(*digest);
|
||||
@@ -180,11 +189,23 @@ FilesystemObjectStore::put(std::span<const std::uint8_t> bytes) const {
|
||||
const bool raced = errno == EEXIST;
|
||||
discard();
|
||||
if (raced)
|
||||
return existing();
|
||||
return durable_existing(object);
|
||||
return std::unexpected{ObjectError::io};
|
||||
}
|
||||
discard();
|
||||
if (::fsync(descriptor_) != 0)
|
||||
if (!sync_directory(descriptor_, faults_))
|
||||
return std::unexpected{ObjectError::io};
|
||||
return object;
|
||||
}
|
||||
|
||||
// A visible name is not proof of durability: its installer's directory sync may
|
||||
// have failed. Re-sync the verified file and the directory before reusing it.
|
||||
std::expected<VerifiedObject, ObjectError>
|
||||
FilesystemObjectStore::durable_existing(const VerifiedObject& object) const {
|
||||
auto stored = read_verified(descriptor_, object, true);
|
||||
if (!stored)
|
||||
return std::unexpected{stored.error()};
|
||||
if (!sync_directory(descriptor_, faults_))
|
||||
return std::unexpected{ObjectError::io};
|
||||
return object;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user