Keep the filesystem object store on a node-local Retain-class PVC per environment (ADR-0070), with the database on the external PostgreSQL VM. Because local-path does not enforce sizes, the object store now refuses new objects below a free-space reserve with a distinct full error, so a full volume publishes and acknowledges nothing. evo-world-clone gains check-store to qualify volume retention, reserve, database version/TLS and the runtime privilege boundary from inside the cluster; it now ships in the server image. Add the storage class, object claims, hand-applied store Jobs, policy rules forbidding password env vars, and the runbook. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
239 lines
9.5 KiB
C++
239 lines
9.5 KiB
C++
#include "evo/persistence/object_store.hpp"
|
|
#include "evo/persistence/sha256.hpp"
|
|
#include <atomic>
|
|
#include <cerrno>
|
|
#include <fcntl.h>
|
|
#include <string>
|
|
#include <sys/stat.h>
|
|
#include <sys/statvfs.h>
|
|
#include <unistd.h>
|
|
|
|
namespace evo::persistence {
|
|
namespace {
|
|
constexpr std::size_t maximum_object_limit = std::size_t{64} * 1024 * 1024;
|
|
std::atomic<std::uint64_t> temporary_counter{0};
|
|
|
|
class Descriptor {
|
|
public:
|
|
explicit Descriptor(int fd) noexcept : fd_(fd) {}
|
|
Descriptor(const Descriptor&) = delete;
|
|
Descriptor& operator=(const Descriptor&) = delete;
|
|
~Descriptor() {
|
|
if (fd_ >= 0)
|
|
static_cast<void>(::close(fd_));
|
|
}
|
|
[[nodiscard]] int get() const noexcept {
|
|
return fd_;
|
|
}
|
|
[[nodiscard]] int release() noexcept {
|
|
const int fd = fd_;
|
|
fd_ = -1;
|
|
return fd;
|
|
}
|
|
// Explicit close so a deferred write error is observed.
|
|
[[nodiscard]] bool close() noexcept {
|
|
const int fd = release();
|
|
return fd < 0 || ::close(fd) == 0;
|
|
}
|
|
|
|
private:
|
|
int fd_;
|
|
};
|
|
std::string key_name(const Sha256Digest& d) {
|
|
constexpr char digits[] = "0123456789abcdef";
|
|
std::string out;
|
|
out.reserve(64);
|
|
for (const auto b : d) {
|
|
out.push_back(digits[b >> 4U]);
|
|
out.push_back(digits[b & 0x0FU]);
|
|
}
|
|
return out;
|
|
}
|
|
// Every directory fsync goes through here so tests can inject failures.
|
|
bool sync_directory(int fd, ObjectStoreFaults* faults) {
|
|
if (faults && faults->fail_directory_syncs > 0) {
|
|
--faults->fail_directory_syncs;
|
|
return false;
|
|
}
|
|
if (::fsync(fd) != 0)
|
|
return false;
|
|
if (faults)
|
|
++faults->directory_syncs;
|
|
return true;
|
|
}
|
|
// Opens or creates a private subdirectory without following symlinks. The parent
|
|
// is synced even when the entry already existed: an earlier creation may have
|
|
// failed its sync, and an unsynced entry must not be trusted as durable.
|
|
int subdirectory(int parent, const std::string& name, ObjectStoreFaults* faults) {
|
|
if (::mkdirat(parent, name.c_str(), 0700) != 0 && errno != EEXIST)
|
|
return -1;
|
|
const int fd = ::openat(parent, name.c_str(), O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
|
if (fd >= 0 && !sync_directory(parent, faults)) {
|
|
static_cast<void>(::close(fd));
|
|
return -1;
|
|
}
|
|
return fd;
|
|
}
|
|
bool write_all(int fd, std::span<const std::uint8_t> bytes) {
|
|
while (!bytes.empty()) {
|
|
const auto n = ::write(fd, bytes.data(), bytes.size());
|
|
if (n < 0 && errno == EINTR)
|
|
continue;
|
|
if (n <= 0)
|
|
return false;
|
|
bytes = bytes.subspan(static_cast<std::size_t>(n));
|
|
}
|
|
return true;
|
|
}
|
|
// Exact-length regular file whose content has the expected digest.
|
|
// With sync_file, the verified file is also fsynced before it is trusted as durable.
|
|
std::expected<std::vector<std::uint8_t>, ObjectError>
|
|
read_verified(int directory, const VerifiedObject& object, bool sync_file = false) {
|
|
const auto name = key_name(object.sha256);
|
|
const Descriptor fd{::openat(directory, name.c_str(), O_RDONLY | O_NOFOLLOW | O_CLOEXEC)};
|
|
if (fd.get() < 0)
|
|
return std::unexpected{errno == ENOENT ? ObjectError::missing
|
|
: errno == ELOOP ? ObjectError::corrupt
|
|
: ObjectError::io};
|
|
struct stat info{};
|
|
if (::fstat(fd.get(), &info) != 0)
|
|
return std::unexpected{ObjectError::io};
|
|
// Size is checked before any allocation.
|
|
if (!S_ISREG(info.st_mode) || info.st_size < 0 ||
|
|
static_cast<std::uint64_t>(info.st_size) != object.bytes)
|
|
return std::unexpected{ObjectError::corrupt};
|
|
std::vector<std::uint8_t> bytes(static_cast<std::size_t>(object.bytes));
|
|
std::size_t done = 0;
|
|
while (done < bytes.size()) {
|
|
const auto n = ::read(fd.get(), bytes.data() + done, bytes.size() - done);
|
|
if (n < 0 && errno == EINTR)
|
|
continue;
|
|
if (n < 0)
|
|
return std::unexpected{ObjectError::io};
|
|
if (n == 0)
|
|
return std::unexpected{ObjectError::corrupt}; // Shrunk after fstat.
|
|
done += static_cast<std::size_t>(n);
|
|
}
|
|
std::uint8_t extra{};
|
|
if (::read(fd.get(), &extra, 1) != 0)
|
|
return std::unexpected{ObjectError::corrupt}; // Grew after fstat.
|
|
const auto digest = sha256(bytes);
|
|
if (!digest)
|
|
return std::unexpected{ObjectError::io};
|
|
if (*digest != object.sha256)
|
|
return std::unexpected{ObjectError::corrupt};
|
|
if (sync_file && ::fsync(fd.get()) != 0)
|
|
return std::unexpected{ObjectError::io};
|
|
return bytes;
|
|
}
|
|
} // namespace
|
|
|
|
FilesystemObjectStore::FilesystemObjectStore(std::filesystem::path directory, int descriptor,
|
|
std::size_t maximum, ObjectStoreFaults* faults,
|
|
std::uint64_t reserve)
|
|
: directory_(std::move(directory)), descriptor_(descriptor), maximum_(maximum), faults_(faults),
|
|
reserve_(reserve) {}
|
|
|
|
std::optional<std::uint64_t> FilesystemObjectStore::available_bytes() const noexcept {
|
|
struct statvfs info{};
|
|
if (::fstatvfs(descriptor_, &info) != 0)
|
|
return std::nullopt;
|
|
return static_cast<std::uint64_t>(info.f_bavail) * static_cast<std::uint64_t>(info.f_frsize);
|
|
}
|
|
|
|
FilesystemObjectStore::~FilesystemObjectStore() {
|
|
static_cast<void>(::close(descriptor_));
|
|
}
|
|
|
|
std::expected<std::unique_ptr<FilesystemObjectStore>, ObjectError>
|
|
FilesystemObjectStore::open(const std::filesystem::path& trusted_root, RegionKey key,
|
|
std::size_t max_object_bytes, ObjectStoreFaults* faults,
|
|
std::uint64_t reserve_bytes) {
|
|
if (key.world == 0 || key.region == 0 || max_object_bytes == 0 ||
|
|
max_object_bytes > maximum_object_limit || trusted_root.empty())
|
|
return std::unexpected{ObjectError::invalid};
|
|
const Descriptor root{
|
|
::open(trusted_root.c_str(), O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC)};
|
|
if (root.get() < 0)
|
|
return std::unexpected{ObjectError::invalid};
|
|
const auto world = "w" + std::to_string(key.world);
|
|
const auto region = "r" + std::to_string(key.region);
|
|
const Descriptor world_fd{subdirectory(root.get(), world, faults)};
|
|
if (world_fd.get() < 0)
|
|
return std::unexpected{ObjectError::io};
|
|
Descriptor region_fd{subdirectory(world_fd.get(), region, faults)};
|
|
if (region_fd.get() < 0)
|
|
return std::unexpected{ObjectError::io};
|
|
return std::unique_ptr<FilesystemObjectStore>(
|
|
new FilesystemObjectStore(trusted_root / world / region, region_fd.release(),
|
|
max_object_bytes, faults, reserve_bytes));
|
|
}
|
|
|
|
std::expected<VerifiedObject, ObjectError>
|
|
FilesystemObjectStore::put(std::span<const std::uint8_t> bytes) const {
|
|
if (bytes.empty())
|
|
return std::unexpected{ObjectError::invalid};
|
|
if (bytes.size() > maximum_)
|
|
return std::unexpected{ObjectError::bounds};
|
|
const auto digest = sha256(bytes);
|
|
if (!digest)
|
|
return std::unexpected{ObjectError::io};
|
|
const VerifiedObject object{*digest, bytes.size()};
|
|
// Immutable: identical content is reused (durably); anything else is corrupt.
|
|
if (auto found = durable_existing(object); found || found.error() != ObjectError::missing)
|
|
return found;
|
|
// Only new bytes need space; checked before anything is written.
|
|
const auto available = available_bytes();
|
|
if (!available)
|
|
return std::unexpected{ObjectError::io};
|
|
if (*available < reserve_ || *available - reserve_ < bytes.size())
|
|
return std::unexpected{ObjectError::full};
|
|
|
|
const auto name = key_name(*digest);
|
|
const auto temporary = ".tmp-" + name + "-" + std::to_string(::getpid()) + "-" +
|
|
std::to_string(temporary_counter.fetch_add(1));
|
|
Descriptor fd{::openat(descriptor_, temporary.c_str(),
|
|
O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW | O_CLOEXEC, 0600)};
|
|
if (fd.get() < 0)
|
|
return std::unexpected{ObjectError::io};
|
|
const auto discard = [&] { static_cast<void>(::unlinkat(descriptor_, temporary.c_str(), 0)); };
|
|
if (!write_all(fd.get(), bytes) || ::fsync(fd.get()) != 0 || !fd.close()) {
|
|
discard();
|
|
return std::unexpected{ObjectError::io};
|
|
}
|
|
// link() never replaces an existing name, unlike rename().
|
|
if (::linkat(descriptor_, temporary.c_str(), descriptor_, name.c_str(), 0) != 0) {
|
|
const bool raced = errno == EEXIST;
|
|
discard();
|
|
if (raced)
|
|
return durable_existing(object);
|
|
return std::unexpected{ObjectError::io};
|
|
}
|
|
discard();
|
|
if (!sync_directory(descriptor_, faults_))
|
|
return std::unexpected{ObjectError::io};
|
|
return object;
|
|
}
|
|
|
|
// A visible name is not proof of durability: its installer's directory sync may
|
|
// have failed. Re-sync the verified file and the directory before reusing it.
|
|
std::expected<VerifiedObject, ObjectError>
|
|
FilesystemObjectStore::durable_existing(const VerifiedObject& object) const {
|
|
auto stored = read_verified(descriptor_, object, true);
|
|
if (!stored)
|
|
return std::unexpected{stored.error()};
|
|
if (!sync_directory(descriptor_, faults_))
|
|
return std::unexpected{ObjectError::io};
|
|
return object;
|
|
}
|
|
|
|
std::expected<std::vector<std::uint8_t>, ObjectError>
|
|
FilesystemObjectStore::get(const VerifiedObject& object) const {
|
|
if (object.bytes == 0)
|
|
return std::unexpected{ObjectError::invalid};
|
|
if (object.bytes > maximum_)
|
|
return std::unexpected{ObjectError::bounds};
|
|
return read_verified(descriptor_, object);
|
|
}
|
|
} // namespace evo::persistence
|