Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m20s
Foundation / checks (network) (push) Successful in 21m3s
Foundation / checks (client) (push) Successful in 22m14s
Foundation / checks (tsan) (push) Successful in 24m52s
Foundation / checks (postgres) (push) Failing after 36m19s
Foundation / checks (checks) (push) Successful in 1h10m25s
Foundation / postgres-tools (push) Skipped
Foundation / server-image (push) Skipped
Foundation / image (push) Skipped
334 lines
13 KiB
C++
334 lines
13 KiB
C++
#include "evo/persistence/postgres.hpp"
|
|
#include <algorithm>
|
|
#include <arpa/inet.h>
|
|
#include <cerrno>
|
|
#include <climits>
|
|
#include <cstdlib>
|
|
#include <fstream>
|
|
#include <libpq-fe.h>
|
|
#include <linux/memfd.h>
|
|
#include <poll.h>
|
|
#include <sys/stat.h>
|
|
#include <sys/syscall.h>
|
|
#include <unistd.h>
|
|
#include <utility>
|
|
#include <vector>
|
|
|
|
namespace evo::persistence {
|
|
namespace {
|
|
std::string_view trim(std::string_view s) {
|
|
while (!s.empty() && (s.front() == ' ' || s.front() == '\t' || s.front() == '\r'))
|
|
s.remove_prefix(1);
|
|
while (!s.empty() && (s.back() == ' ' || s.back() == '\t' || s.back() == '\r'))
|
|
s.remove_suffix(1);
|
|
return s;
|
|
}
|
|
using Clock = std::chrono::steady_clock;
|
|
struct Finish {
|
|
void operator()(PGconn* p) const {
|
|
PQfinish(p);
|
|
}
|
|
};
|
|
struct Clear {
|
|
void operator()(PGresult* p) const {
|
|
PQclear(p);
|
|
}
|
|
};
|
|
using Connection = std::unique_ptr<PGconn, Finish>;
|
|
using Result = std::unique_ptr<PGresult, Clear>;
|
|
struct PrivatePassfile {
|
|
int fd{-1};
|
|
explicit PrivatePassfile(int value) : fd(value) {}
|
|
~PrivatePassfile() {
|
|
if (fd >= 0)
|
|
::close(fd);
|
|
}
|
|
PrivatePassfile(const PrivatePassfile&) = delete;
|
|
PrivatePassfile& operator=(const PrivatePassfile&) = delete;
|
|
PrivatePassfile(PrivatePassfile&& other) noexcept : fd(std::exchange(other.fd, -1)) {}
|
|
PrivatePassfile& operator=(PrivatePassfile&& other) noexcept {
|
|
if (this != &other) {
|
|
if (fd >= 0)
|
|
::close(fd);
|
|
fd = std::exchange(other.fd, -1);
|
|
}
|
|
return *this;
|
|
}
|
|
[[nodiscard]] std::string path() const {
|
|
return "/proc/self/fd/" + std::to_string(fd);
|
|
}
|
|
};
|
|
std::expected<PrivatePassfile, PgError> private_passfile(const std::string& source) {
|
|
std::ifstream in(source, std::ios::binary);
|
|
if (!in)
|
|
return std::unexpected{PgError::config};
|
|
std::string contents(4097, '\0');
|
|
in.read(contents.data(), static_cast<std::streamsize>(contents.size()));
|
|
contents.resize(static_cast<std::size_t>(in.gcount()));
|
|
if (contents.empty() || contents.size() > 4096 || contents.find('\0') != std::string::npos)
|
|
return std::unexpected{PgError::config};
|
|
const int fd = static_cast<int>(::syscall(SYS_memfd_create, "evo-pgpass", MFD_CLOEXEC));
|
|
if (fd < 0 || ::fchmod(fd, S_IRUSR | S_IWUSR) != 0) {
|
|
if (fd >= 0)
|
|
::close(fd);
|
|
return std::unexpected{PgError::config};
|
|
}
|
|
std::size_t written{};
|
|
while (written < contents.size()) {
|
|
const auto count = ::write(fd, contents.data() + written, contents.size() - written);
|
|
if (count <= 0) {
|
|
::close(fd);
|
|
return std::unexpected{PgError::config};
|
|
}
|
|
written += static_cast<std::size_t>(count);
|
|
}
|
|
if (::lseek(fd, 0, SEEK_SET) < 0)
|
|
return std::unexpected{PgError::config};
|
|
return PrivatePassfile{fd};
|
|
}
|
|
bool wait(PGconn* c, short events, Clock::time_point deadline) {
|
|
for (;;) {
|
|
const auto left =
|
|
std::chrono::duration_cast<std::chrono::milliseconds>(deadline - Clock::now());
|
|
if (left.count() <= 0 || PQsocket(c) < 0)
|
|
return false;
|
|
pollfd descriptor{PQsocket(c), events, 0};
|
|
const auto ready = ::poll(
|
|
&descriptor, 1, static_cast<int>(std::min(left.count(), static_cast<long>(INT_MAX))));
|
|
if (ready > 0)
|
|
return (descriptor.revents & events) != 0;
|
|
if (ready == 0 || errno != EINTR)
|
|
return false;
|
|
}
|
|
}
|
|
// ADR-0126: a parameter holds at most 4 MiB of text, enough for the largest cellular trace
|
|
// (64 ticks of 256 lives) as a hex bytea; it was 4 KiB before profile-4 traces.
|
|
constexpr std::size_t max_parameter_bytes = std::size_t{4} * 1024 * 1024;
|
|
bool bounded(const std::string& s) {
|
|
return s.size() <= max_parameter_bytes && s.find('\0') == std::string::npos;
|
|
}
|
|
bool duration(std::chrono::milliseconds t) {
|
|
return t.count() > 0 && t.count() <= 30000;
|
|
}
|
|
PgFailure fail(PgError e) {
|
|
return {e, {}};
|
|
}
|
|
} // namespace
|
|
struct Postgres::Impl {
|
|
Connection connection;
|
|
};
|
|
Postgres::Postgres(std::unique_ptr<Impl> impl) : impl_(std::move(impl)) {}
|
|
Postgres::~Postgres() = default;
|
|
void Postgres::disconnect() noexcept {
|
|
impl_->connection.reset();
|
|
}
|
|
bool Postgres::transaction_idle() const noexcept {
|
|
return impl_->connection && PQstatus(impl_->connection.get()) == CONNECTION_OK &&
|
|
PQtransactionStatus(impl_->connection.get()) == PQTRANS_IDLE;
|
|
}
|
|
std::expected<PgOptions, PgError> read_pg_service(const std::string& file,
|
|
std::string_view section) {
|
|
constexpr std::size_t limit = 4096;
|
|
std::ifstream in(file, std::ios::binary);
|
|
if (!in)
|
|
return std::unexpected{PgError::config};
|
|
std::string text(limit + 1, '\0');
|
|
in.read(text.data(), static_cast<std::streamsize>(text.size()));
|
|
text.resize(static_cast<std::size_t>(in.gcount()));
|
|
if (text.size() > limit || section.empty())
|
|
return std::unexpected{PgError::config};
|
|
PgOptions o;
|
|
std::string sslmode;
|
|
bool inside = false, found = false;
|
|
std::vector<std::string> seen;
|
|
std::string_view rest = text;
|
|
while (!rest.empty()) {
|
|
const auto end = rest.find('\n');
|
|
const auto line = trim(rest.substr(0, end));
|
|
rest.remove_prefix(end == std::string_view::npos ? rest.size() : end + 1);
|
|
if (line.empty() || line.front() == '#')
|
|
continue;
|
|
if (line.front() == '[') {
|
|
if (line.back() != ']')
|
|
return std::unexpected{PgError::config};
|
|
inside = line.substr(1, line.size() - 2) == section;
|
|
if (inside && found)
|
|
return std::unexpected{PgError::config}; // Repeated section.
|
|
found = found || inside;
|
|
continue;
|
|
}
|
|
if (!inside)
|
|
continue;
|
|
const auto equals = line.find('=');
|
|
if (equals == std::string_view::npos)
|
|
return std::unexpected{PgError::config};
|
|
const std::string key{trim(line.substr(0, equals))};
|
|
const std::string value{trim(line.substr(equals + 1))};
|
|
if (std::ranges::find(seen, key) != seen.end())
|
|
return std::unexpected{PgError::config};
|
|
seen.push_back(key);
|
|
if (key == "host")
|
|
o.host = value;
|
|
else if (key == "hostaddr")
|
|
o.hostaddr = value;
|
|
else if (key == "port")
|
|
o.port = value;
|
|
else if (key == "dbname")
|
|
o.database = value;
|
|
else if (key == "user")
|
|
o.user = value;
|
|
else if (key == "sslrootcert")
|
|
o.ca_file = value;
|
|
else if (key == "sslmode")
|
|
sslmode = value;
|
|
else if (key == "connect_timeout") {
|
|
if (value.empty() ||
|
|
!std::ranges::all_of(value, [](char ch) { return ch >= '0' && ch <= '9'; }))
|
|
return std::unexpected{PgError::config};
|
|
} else {
|
|
return std::unexpected{PgError::config}; // password, passfile, options, ...
|
|
}
|
|
}
|
|
const bool local = !o.host.empty() && o.host.front() == '/';
|
|
if (!found ||
|
|
(!sslmode.empty() && sslmode != "verify-full" && !(local && sslmode == "disable")))
|
|
return std::unexpected{PgError::config};
|
|
return o;
|
|
}
|
|
|
|
std::expected<std::unique_ptr<Postgres>, PgFailure>
|
|
Postgres::connect(const PgOptions& o, std::chrono::milliseconds timeout) {
|
|
if (!duration(timeout) || o.host.empty() || o.database.empty() || o.user.empty() ||
|
|
o.port.empty())
|
|
return std::unexpected{fail(PgError::config)};
|
|
for (const auto* value :
|
|
{&o.host, &o.hostaddr, &o.port, &o.database, &o.user, &o.password_file, &o.ca_file})
|
|
if (!bounded(*value))
|
|
return std::unexpected{fail(PgError::config)};
|
|
const bool local = o.host.front() == '/';
|
|
if (o.host.find(',') != std::string::npos ||
|
|
(local && (std::getenv("PGHOSTADDR") || std::getenv("PGSERVICE"))))
|
|
return std::unexpected{fail(PgError::config)};
|
|
std::array<unsigned char, 16> address{};
|
|
if ((!local &&
|
|
(o.ca_file.empty() || (inet_pton(AF_INET, o.hostaddr.c_str(), address.data()) != 1 &&
|
|
inet_pton(AF_INET6, o.hostaddr.c_str(), address.data()) != 1))) ||
|
|
(local && !o.hostaddr.empty()))
|
|
return std::unexpected{fail(PgError::config)};
|
|
std::optional<PrivatePassfile> private_file;
|
|
if (!o.password_file.empty()) {
|
|
auto made = private_passfile(o.password_file);
|
|
if (!made)
|
|
return std::unexpected{fail(made.error())};
|
|
private_file.emplace(std::move(*made));
|
|
}
|
|
const std::string passfile = private_file ? private_file->path() : std::string{};
|
|
const char* keys[] = {"host", "hostaddr", "port", "dbname", "user", "passfile",
|
|
"sslrootcert", "sslmode", "gssencmode", "options", nullptr};
|
|
const char* values[] = {o.host.c_str(),
|
|
o.hostaddr.c_str(),
|
|
o.port.c_str(),
|
|
o.database.c_str(),
|
|
o.user.c_str(),
|
|
passfile.c_str(),
|
|
o.ca_file.c_str(),
|
|
local ? "disable" : "verify-full",
|
|
"disable",
|
|
"-csearch_path= -cstatement_timeout=10000 -clock_timeout=5000",
|
|
nullptr};
|
|
const auto deadline = Clock::now() + timeout;
|
|
Connection c{PQconnectStartParams(keys, values, 0)};
|
|
if (!c)
|
|
return std::unexpected{fail(PgError::connect)};
|
|
auto status = PGRES_POLLING_WRITING;
|
|
while (status != PGRES_POLLING_OK) {
|
|
if (status == PGRES_POLLING_FAILED ||
|
|
!wait(c.get(), status == PGRES_POLLING_READING ? POLLIN : POLLOUT, deadline))
|
|
return std::unexpected{fail(PgError::connect)};
|
|
status = PQconnectPoll(c.get());
|
|
}
|
|
if (PQlibVersion() != 180006 || PQserverVersion(c.get()) != 180006)
|
|
return std::unexpected{fail(PgError::version)};
|
|
if (PQsetnonblocking(c.get(), 1) != 0)
|
|
return std::unexpected{fail(PgError::connect)};
|
|
auto impl = std::make_unique<Impl>();
|
|
impl->connection = std::move(c);
|
|
return std::unique_ptr<Postgres>(new Postgres(std::move(impl)));
|
|
}
|
|
std::expected<PgRows, PgFailure> Postgres::query(const std::string& sql,
|
|
std::span<const std::string> parameters,
|
|
std::chrono::milliseconds timeout) {
|
|
auto& owned = impl_->connection;
|
|
if (!owned)
|
|
return std::unexpected{fail(PgError::closed)};
|
|
if (!duration(timeout) || sql.size() > 16384 || sql.find('\0') != std::string::npos ||
|
|
parameters.size() > 16)
|
|
return std::unexpected{fail(PgError::config)};
|
|
std::array<const char*, 16> values{};
|
|
for (std::size_t i = 0; i < parameters.size(); ++i) {
|
|
if (!bounded(parameters[i]))
|
|
return std::unexpected{fail(PgError::bounds)};
|
|
values[i] = parameters[i].c_str();
|
|
}
|
|
auto* c = owned.get();
|
|
const auto deadline = Clock::now() + timeout;
|
|
const auto unknown = [&]() -> std::expected<PgRows, PgFailure> {
|
|
owned.reset();
|
|
return std::unexpected{fail(PgError::unknown)};
|
|
};
|
|
if (PQsendQueryParams(c, sql.c_str(), static_cast<int>(parameters.size()), nullptr,
|
|
values.data(), nullptr, nullptr, 0) != 1 ||
|
|
PQsetSingleRowMode(c) != 1)
|
|
return unknown();
|
|
int flush{};
|
|
while ((flush = PQflush(c)) == 1)
|
|
if (!wait(c, POLLOUT, deadline))
|
|
return unknown();
|
|
if (flush < 0)
|
|
return unknown();
|
|
PgRows rows;
|
|
std::size_t bytes{};
|
|
std::optional<PgFailure> error;
|
|
for (;;) {
|
|
if (Clock::now() >= deadline)
|
|
return unknown();
|
|
while (PQisBusy(c))
|
|
if (!wait(c, POLLIN, deadline) || PQconsumeInput(c) != 1)
|
|
return unknown();
|
|
Result r{PQgetResult(c)};
|
|
if (!r)
|
|
break;
|
|
const auto status = PQresultStatus(r.get());
|
|
if (status == PGRES_SINGLE_TUPLE) {
|
|
if (rows.size() == 128 || PQnfields(r.get()) > 16)
|
|
return unknown();
|
|
std::vector<std::optional<std::string>> row;
|
|
for (int j = 0; j < PQnfields(r.get()); ++j) {
|
|
const auto length = static_cast<std::size_t>(PQgetlength(r.get(), 0, j));
|
|
if (length > 65536 - bytes)
|
|
return unknown();
|
|
bytes += length;
|
|
if (PQgetisnull(r.get(), 0, j))
|
|
row.emplace_back(std::nullopt);
|
|
else
|
|
row.emplace_back(std::string(PQgetvalue(r.get(), 0, j), length));
|
|
}
|
|
rows.push_back(std::move(row));
|
|
} else if (status != PGRES_COMMAND_OK && status != PGRES_TUPLES_OK) {
|
|
if (status != PGRES_FATAL_ERROR)
|
|
return unknown();
|
|
PgFailure f = fail(PgError::sql);
|
|
const auto* state = PQresultErrorField(r.get(), PG_DIAG_SQLSTATE);
|
|
if (state)
|
|
std::copy_n(state, 5, f.sqlstate.begin());
|
|
error = f;
|
|
}
|
|
}
|
|
if (PQstatus(c) != CONNECTION_OK)
|
|
return unknown();
|
|
if (error)
|
|
return std::unexpected{*error};
|
|
return rows;
|
|
}
|
|
} // namespace evo::persistence
|