Complete M7.2 PostgreSQL adapter and migration bootstrap

This commit is contained in:
dingo
2026-09-28 07:55:18 -04:00
parent 0b9ed18503
commit 60bb492123
24 changed files with 784 additions and 4 deletions
+1
View File
@@ -1,5 +1,6 @@
* text=auto
*.sh text eol=lf
*.sql text eol=lf
*.yaml text eol=lf
*.yml text eol=lf
Dockerfile text eol=lf
+30 -1
View File
@@ -19,7 +19,7 @@ jobs:
strategy:
fail-fast: false
matrix:
suite: [checks, tsan, ecs]
suite: [checks, tsan, ecs, postgres]
steps:
- name: Checkout
uses: https://github.com/actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -47,6 +47,35 @@ jobs:
--tmpfs /tmp:rw,noexec,nosuid,size=16m,mode=1777 "$ci_image"
fi
postgres-tools:
needs: [checks]
runs-on: ${{ vars.CI_RUNNER_LABEL || 'ubuntu-22.04' }}
timeout-minutes: 40
steps:
- name: Checkout
uses: https://github.com/actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
with:
persist-credentials: false
- name: Build and smoke-test migration tooling
run: |
set -eu
docker build --file deploy/docker/Dockerfile --target postgres-tools \
--build-arg "UBUNTU_IMAGE=$UBUNTU_IMAGE" \
--tag "aevolution-postgres-tools:${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" .
docker run --rm --network none --read-only --cap-drop ALL \
--entrypoint /opt/postgres/bin/psql \
"aevolution-postgres-tools:${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" --version
- name: Publish configured migration tools
if: ${{ gitea.ref == 'refs/heads/main' && gitea.event_name != 'pull_request' && vars.HARBOR_POSTGRES_REPOSITORY != '' }}
env:
HARBOR_PROJECT: ${{ vars.HARBOR_PROJECT }}
HARBOR_REPOSITORY: ${{ vars.HARBOR_POSTGRES_REPOSITORY }}
HARBOR_USERNAME: ${{ secrets.HARBOR_USERNAME }}
HARBOR_PASSWORD: ${{ secrets.HARBOR_PASSWORD }}
run: |
export SOURCE_IMAGE="aevolution-postgres-tools:${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
bash tools/ci/publish-image.sh
image:
needs: [checks]
runs-on: ${{ vars.CI_RUNNER_LABEL || 'ubuntu-22.04' }}
+18
View File
@@ -23,6 +23,7 @@ option(EVO_WARNINGS_AS_ERRORS "Treat owned-code warnings as errors" ON)
option(EVO_ENABLE_TIDY "Run clang-tidy during compilation" OFF)
option(EVO_BUILD_FUZZER "Build the Clang libFuzzer snapshot target" OFF)
option(EVO_BUILD_ECS_EXPERIMENT "Build the isolated EnTT comparison (explicit preparation required)" OFF)
option(EVO_BUILD_POSTGRES "Build the isolated Linux PostgreSQL persistence adapter" OFF)
set(EVO_SANITIZER "none" CACHE STRING "none, address (ASan/UBSan), or thread")
set_property(CACHE EVO_SANITIZER PROPERTY STRINGS none address thread)
include(cmake/ProjectOptions.cmake)
@@ -71,6 +72,23 @@ target_link_libraries(evo-sim PRIVATE evo::simulation evo::processes)
evo_owned_target(evo-sim)
include(CTest)
if(EVO_BUILD_POSTGRES)
if(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux")
message(FATAL_ERROR "The initial PostgreSQL adapter is qualified on Linux only")
endif()
find_package(PostgreSQL 18.6 EXACT REQUIRED)
add_library(evo_postgres STATIC src/persistence/postgres.cpp)
add_library(evo::postgres ALIAS evo_postgres)
target_include_directories(evo_postgres PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include")
target_link_libraries(evo_postgres PRIVATE PostgreSQL::PostgreSQL)
evo_owned_target(evo_postgres)
if(BUILD_TESTING)
add_executable(evo-postgres-tests tests/postgres_tests.cpp)
target_link_libraries(evo-postgres-tests PRIVATE evo::postgres)
evo_owned_target(evo-postgres-tests)
# Intentionally not an ordinary CTest: tools/ci/postgres-check.sh owns its private DB.
endif()
endif()
if(BUILD_TESTING)
add_executable(evo-environmental-qualification tests/environmental_qualification.cpp)
target_link_libraries(evo-environmental-qualification PRIVATE evo::processes)
+6
View File
@@ -157,3 +157,9 @@ For the M6.5 boundary/seeded suite and local cost report, run
`build/release/evo-environmental-qualification --long` (add `.exe` on Windows).
See [environmental qualification](docs/engineering/ENVIRONMENTAL_QUALIFICATION.md)
for parameters, coverage and limitations. The shorter suite runs through CTest.
## PostgreSQL bootstrap (M7.2)
See [PostgreSQL setup](docs/infrastructure/POSTGRESQL.md) for the optional Linux
adapter, isolated integration checks and Kubernetes Secret/migration instructions.
The default simulation build does not require PostgreSQL.
+15
View File
@@ -118,3 +118,18 @@ bounded to 16 MiB in the workflow. The world restart test retains generated snap
journals and JSON reports beneath its own random run directory; it cannot create
that directory on the read-only image filesystem. Any new file-producing CTest
must declare its scratch directory in this container invocation as well.
## PostgreSQL migration tooling
The `postgres` CI matrix builds and tests the isolated adapter in Release,
clang-tidy and ASan/UBSan against private PostgreSQL clusters. The `postgres-tools`
Docker target contains pinned psql plus checked-in migrations, not a database server.
Set Gitea variable `HARBOR_POSTGRES_REPOSITORY=aevolution-postgres-tools` to enable
publishing that image using existing HARBOR_PROJECT and robot secrets. Leave it
unset for build/smoke only. Use the reported immutable digest in the Job example.
Do not reuse the simulation image repository for this entrypoint.
[Secret and role setup](../docs/infrastructure/POSTGRESQL.md) describes the separate
migrator/runtime credentials. The example is outside active Argo resources until
an operator supplies the database endpoint, role, Secret and egress rules. No region
server or world-state migration is deployed by M7.2.
+36
View File
@@ -45,6 +45,42 @@ RUN cmake --preset analysis -B build/ecs-analysis -DCMAKE_CXX_COMPILER=clang++-1
&& cmake --build build/ecs-analysis --target evo-ecs-experiment --parallel 2 \
&& ctest --test-dir build/ecs-analysis -R ecs-experiment --output-on-failure --no-tests=error
FROM ci-toolchain AS postgres-build
RUN apt-get update && apt-get install -y --no-install-recommends \
curl bzip2 make flex bison libssl-dev zlib1g-dev pkg-config \
&& rm -rf /var/lib/apt/lists/*
COPY migrations/ migrations/
RUN bash tools/prepare-postgres.sh
RUN cmake --preset release -B build/postgres -DCMAKE_CXX_COMPILER=g++-14 \
-DEVO_BUILD_POSTGRES=ON -DPostgreSQL_ROOT=/source/.cache/postgres/install \
&& cmake --build build/postgres --target evo-postgres-tests --parallel 2
RUN cmake --preset analysis -B build/postgres-analysis -DCMAKE_CXX_COMPILER=clang++-19 \
-DEVO_BUILD_POSTGRES=ON -DPostgreSQL_ROOT=/source/.cache/postgres/install \
&& cmake --build build/postgres-analysis --target evo-postgres-tests --parallel 2
RUN cmake --preset asan -B build/postgres-asan -DCMAKE_CXX_COMPILER=clang++-19 \
-DEVO_BUILD_POSTGRES=ON -DPostgreSQL_ROOT=/source/.cache/postgres/install \
&& cmake --build build/postgres-asan --target evo-postgres-tests --parallel 2
FROM postgres-build AS ci-postgres
RUN useradd --uid 10001 --no-create-home evo
USER 10001:10001
RUN bash tools/ci/postgres-check.sh /source/build/postgres/evo-postgres-tests \
&& bash tools/ci/postgres-check.sh /source/build/postgres-analysis/evo-postgres-tests \
&& bash tools/ci/postgres-check.sh /source/build/postgres-asan/evo-postgres-tests
FROM ${UBUNTU_IMAGE} AS postgres-tools
RUN apt-get update && apt-get install -y --no-install-recommends libssl3t64 zlib1g ca-certificates \
&& rm -rf /var/lib/apt/lists/* \
&& groupadd --gid 10001 evo && useradd --uid 10001 --gid 10001 --no-create-home evo
COPY --from=postgres-build /source/.cache/postgres/install/bin/psql /opt/postgres/bin/psql
COPY --from=postgres-build /source/.cache/postgres/install/lib/libpq.so* /opt/postgres/lib/
COPY --from=postgres-build /source/.cache/postgres/postgresql-18.6/COPYRIGHT /usr/share/doc/evo-postgres/COPYRIGHT
COPY tools/migrate-postgres.sh /opt/evo/tools/migrate-postgres.sh
COPY migrations/ /opt/evo/migrations/
ENV PATH="/opt/postgres/bin:${PATH}" LD_LIBRARY_PATH=/opt/postgres/lib
USER 10001:10001
ENTRYPOINT ["bash", "/opt/evo/tools/migrate-postgres.sh"]
FROM toolchain AS build
RUN cmake --preset release -B build/container \
-DCMAKE_CXX_COMPILER=g++-14 -DCMAKE_RUNTIME_OUTPUT_DIRECTORY=/opt/evo/bin \
@@ -0,0 +1,50 @@
# Review and pin the postgres-tools image digest before applying to evolution-dev.
# Not included in the active Argo kustomization; never target World0 routinely.
apiVersion: batch/v1
kind: Job
metadata:
name: evolution-db-migrate-v1
namespace: evolution-dev
spec:
backoffLimit: 0
activeDeadlineSeconds: 120
template:
spec:
restartPolicy: Never
automountServiceAccountToken: false
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
seccompProfile:
type: RuntimeDefault
containers:
- name: migrate
image: registry.aeternalisgames.com/aeternalis/aevolution-postgres-tools:replace-with-immutable-digest
securityContext:
readOnlyRootFilesystem: true
allowPrivilegeEscalation: false
capabilities:
drop: [ALL]
resources:
requests: {cpu: 100m, memory: 64Mi}
limits: {cpu: 500m, memory: 128Mi}
env:
- {name: PGSERVICE, value: evolution-migrator}
- {name: PGSERVICEFILE, value: /run/postgres/pg_service.conf}
- {name: EVO_PGPASS_SOURCE, value: /run/postgres/pgpass}
volumeMounts:
- {name: connection, mountPath: /run/postgres, readOnly: true}
- {name: scratch, mountPath: /tmp}
volumes:
- name: connection
secret:
secretName: evolution-postgres-migrator
defaultMode: 0440
items:
- {key: pg_service.conf, path: pg_service.conf}
- {key: pgpass, path: pgpass}
- {key: ca.crt, path: ca.crt}
- name: scratch
emptyDir: {medium: Memory, sizeLimit: 8Mi}
+6
View File
@@ -495,3 +495,9 @@ not a running database/server. M6 environmental state is complete only within it
profile; M7.4 must add archive serialization and organism/genome binding before
M7.9 can claim retained ancestry. The M7 exit criteria remain unchanged.
Next: M7.2 evaluate/pin storage dependencies and implement/test migrations.
M7.2 implemented and locally qualified (2026-09-28). ADR-0059 adopts the pinned
server/libpq pair, optional persistence adapter and transactional schema v1.
Private PostgreSQL integration checks passed in Release, analysis and ASan/UBSan;
container/cluster verification remains separate. Secret provisioning instructions
are in infrastructure/POSTGRESQL.md. Next: M7.3 fenced region ownership.
+9
View File
@@ -746,3 +746,12 @@ atomic permanent outcomes and exact reconciliation after ambiguous commits.
REGION_DURABILITY.md inventories included/missing state and the crash-point tests.
No PostgreSQL connection, persistence adapter or region pod exists yet. M7.2 is
next; lineage closure remains an explicit M7.4/M7.9 prerequisite, not waived scope.
M7.2 complete (2026-09-28): optional Linux libpq adapter and PostgreSQL 18.6
source pin, transactional checksum-checked bootstrap, bounded parameterized I/O,
private database integration tests and Secret-mounted migration Job example.
Release, clang-tidy and ASan/UBSan PostgreSQL checks passed in WSL; Windows
headless suite passed 54/54. Docker/remote CI and cluster deployment are not
claimed. See infrastructure/POSTGRESQL.md and engineering/VALIDATION.md.
Next: M7.3 fenced ownership and narrowly granted database functions. The bootstrap
lease table alone does not establish authority or durable region publication.
+4
View File
@@ -32,3 +32,7 @@ Mutable tags, ranges, stale caches, transitive licenses and fictitious generated
Record revision/hash, owner, license, build options and removal boundary; mirror release artifacts. No empty manifests/fake pins. [vcpkg](https://learn.microsoft.com/en-us/vcpkg/concepts/manifest-mode), [Conan](https://docs.conan.io/2/tutorial/versioning/lockfiles.html).
M7.2's [ADR-0059](0059-postgresql-bootstrap.md) accepts a scoped checksum-pinned
source build for the paired PostgreSQL server/client. The general cross-platform
package-manager comparison remains open.
+2 -2
View File
@@ -2,7 +2,8 @@
## Status
Proposed for M7; no database dependency in proof. PostgreSQL preference follows brief.
Superseded for M7.2 by [ADR-0059](0059-postgresql-bootstrap.md), which adopts
direct libpq for bounded asynchronous I/O. Historical recommendation follows.
## Context
@@ -31,4 +32,3 @@ Ambiguous commit, blocked requests, injection, retry duplication and schema mism
## Consequences
Parameterized SQL; unique constraints/idempotency for retries. Timeout does not prove rollback. Inject failure after commit before ACK. Pin supported PostgreSQL/libpqxx pair and measure throughput/size. [libpqxx BSD-3-Clause](https://github.com/jtv/libpqxx), [libpq](https://www.postgresql.org/docs/current/libpq.html).
+71
View File
@@ -0,0 +1,71 @@
# ADR-0059: PostgreSQL adapter and migration bootstrap
## Status
Accepted for M7.2, 2026-09-28. No region authority or world deployment yet.
Supersedes ADR-0013's libpqxx recommendation for this scoped adapter.
## Context
M7 needs parameterized SQL, bounded I/O and explicit ambiguous failure handling.
Kubernetes connection details will be supplied through Secrets, per maintainer.
The headless library must remain dependency-free with respect to PostgreSQL.
## Decision
Pin PostgreSQL server and libpq to 18.6, built from the same official source:
https://ftp.postgresql.org/pub/source/v18.6/postgresql-18.6.tar.bz2
SHA-256: 555610c24d53e4316da5b7d3fc25c279d96856d5e0e23ee308c328c5fa881d9f.
Use direct libpq behind a C++ RAII adapter, optional Linux-only build target.
Windows headless targets remain unchanged. No public PGconn/PGresult types.
One I/O owner, one outstanding command, no hidden threads or connection pool.
Use nonblocking connect/send/receive with a steady-clock deadline. Require a Unix
socket or numeric hostaddr plus TLS hostname to avoid blocking DNS inside connect.
Remote connections require verify-full. Drop a timed-out/broken connection and
return unknown outcome, never retry mutations automatically. Bound parameters and
stream rows; expose categories and SQLSTATE, never server messages/credentials.
Migrations run separately under an operator-provided owner login, using psql with
ON_ERROR_STOP and transactional SQL. Initial schema is only migration metadata
and region lease rows needed by M7.3; no speculative 30-table domain schema.
Runtime login is not owner and receives no table mutation rights. Functions and
grants follow with fencing in M7.3. No automatic migrations on server startup.
Use a checksum-pinned source preparation script, not a package manager for this
single server/client artifact. It performs explicit downloads outside configure.
Record system build prerequisites separately; a source pin is not a fully hermetic
OS toolchain. vcpkg/Conan remain candidates for future client dependencies; neither
would remove the need to provision/test the matching PostgreSQL server here.
## Alternatives Considered
libpqxx 8.0.2 provides mature BSD-3-Clause C++ transactions but adds an API/exception
layer over libpq without removing the need for bounded asynchronous I/O. Direct
libpq has more RAII/result code but one official server/client release. ORM mapping
and connection pools add no value to this single-owner bootstrap. Distro unpinned
libpq is easier but does not establish a repeatable evaluated pair.
## Benefits
Official maintained client, PostgreSQL permissive license, no extra runtime library
beyond libpq and its declared TLS/system dependencies. Parameter separation and
typed unknown outcomes are straightforward to inspect and test.
## Costs
Linux polling adapter; explicit resource wrappers; source build/test time. Native
Windows storage support and cross-platform package-manager cost comparison are
not claimed. No performance advantage over libpqxx is claimed without measurement.
## Risks
Timeout does not prove rollback. Query bounds protect the client, not arbitrary
SQL resource use; callers are trusted persistence code and server statement limits
remain necessary. Secrets require cluster RBAC/encryption policy, not just YAML.
Minor release updates require a new checksum, security review and integration run.
## Consequences
Real isolated PostgreSQL tests must cover parameters, rollback, deadlines, clean
and repeated migrations, unknown migration versions, constraints and runtime-role
denial. Deployment templates contain only Secret references; no credentials, no
automatic World0 changes. SHA-256 provider for object manifests remains M7.4 work.
## Primary evidence
- [Official releases](https://www.postgresql.org/ftp/source/)
- [PostgreSQL license](https://www.postgresql.org/about/licence/)
- [Connection polling and DNS restrictions](https://www.postgresql.org/docs/18/libpq-connect.html)
- [Async parameterized queries](https://www.postgresql.org/docs/18/libpq-async.html)
- [libpqxx releases and license](https://pqxx.org/libpqxx/)
+2
View File
@@ -68,3 +68,5 @@ design. No unadopted candidate is a link dependency of the proof.
- [ADR-0053: Atomic environmental transfers](0053-atomic-environmental-transfers.md) - accepted for M6.4 tagged resource ownership, atomic local batches and bounded exact retries.
- [ADR-0054: M6.5 source persistence prerequisite](0054-m65-source-persistence-prerequisite.md) - accepted: coupled codec brought forward into M6.5; M6.5a implemented, rate/biology envelope follows.
- [ADR-0055: Environmental process contract](0055-environmental-process-contract.md) - accepted M6.5b synthetic conversions, source interval policy and process checkpoint boundary; implementation follows in M6.5c.
- [ADR-0059: PostgreSQL bootstrap](0059-postgresql-bootstrap.md) - accepted for M7.2 optional adapter, pinned server/client, migrations and Secret boundary.
+24
View File
@@ -2017,3 +2017,27 @@ No database, object storage, Kubernetes access or deployment was performed.
M7.1 design gate is complete. M7.2 must evaluate/pin dependencies and exercise real
migrations; M7.4 must implement additional ancestry state closure before M7.9 can
claim lineage recovery. No retained-history or World0 requirement was relaxed.
## M7.2 PostgreSQL bootstrap - 2026-09-28
- Windows Release configure/build/CTest passed 54/54 (24.17s); format passed.
- Built checksum-verified PostgreSQL/libpq 18.6 from source in WSL Ubuntu.
- GCC14 Release, clang19 analysis (clang-tidy warnings-as-errors), and clang19
ASan/UBSan optional adapter builds and real private-database integration passed.
Tests cover parameter separation/NULL, rollback, duplicate SQLSTATE, u64 range
and fractional rejection, runtime-role denial, deadline closure, result bounds,
backend termination, clean/repeated migrations, failed migration rollback,
unknown versions and changed checksums. Expected SQL errors are negative tests.
- Fixed analysis enum-size warning and migration rejection exit status during
qualification; reran all three final configurations successfully.
- clang-format19 dry-run passed for all three new C++ files; bash -n passed for
preparation, migration, integration and publish scripts. ldd found all psql
dependencies: libpq, libc/libm and OpenSSL (matching tooling image packages).
- Docker Linux engine is unavailable locally. Container build/smoke and Gitea
remote checks are configured, not claimed as executed. No cluster, external DB,
TLS endpoint or Kubernetes Secret was accessed. TSan was not rerun: adapter has
one caller-owned I/O thread and introduces no concurrent execution.
The adapter is separate from the headless kernel. Region fencing, publication,
ambiguous durable commit reconciliation and crash recovery are later M7 gates.
Only clean-to-v1 and exact-v1 restart are supported migration paths at this gate.
+92
View File
@@ -0,0 +1,92 @@
# PostgreSQL M7.2 bootstrap
ADR-0059 pins server/libpq 18.6. `evo::postgres` is optional, Linux-only and separate
from all simulation targets. Preparation explicitly downloads a verified source
archive; ordinary CMake configure does not fetch anything. PostgreSQL license is
permissive; source COPYRIGHT and system TLS license information must accompany
distributed tooling. No custom hashing/cryptography is introduced.
## Local isolated verification
Linux prerequisites: GCC14, CMake>=3.28, Ninja, make, bison, flex, curl, bzip2,
OpenSSL development headers and zlib development headers. Optional clang19 tools
support analysis/ASan. Run as a normal user, not root:
```sh
bash tools/prepare-postgres.sh
cmake --preset release -B build/postgres -DCMAKE_CXX_COMPILER=g++-14 \
-DEVO_BUILD_POSTGRES=ON -DPostgreSQL_ROOT="$PWD/.cache/postgres/install"
cmake --build build/postgres --target evo-postgres-tests --parallel 2
bash tools/ci/postgres-check.sh "$PWD/build/postgres/evo-postgres-tests"
```
The test script overrides all connection coordinates and creates a new private
0700 /tmp directory, private Unix socket and disposable local cluster. TCP is off;
local trust applies only inside that private directory. Cluster stops on exit;
logs/data are retained there for diagnosis. It never uses production PG credentials
or deletes a pre-existing database. Do not point this script at a shared server.
## Adapter contract
One connection belongs to one I/O owner; one outstanding parameterized statement.
No hidden threads, pool, transaction replay or automatic reconnect. Connections
and queries take 1..30000ms deadlines on a steady clock. Remote hostaddr must be
numeric, host supplies TLS identity, CA file is explicit, TLS verify-full required.
Local sockets require no PGHOSTADDR/PGSERVICE override. GSS negotiation is disabled.
Query SQL <=16KiB, <=16 parameters of <=4KiB, response <=128 rows/16 columns/64KiB
decoded values; single-row mode prevents accumulating a whole result set. A single
server field is allocated by libpq before its length can be checked: this is not
an arbitrary-query memory sandbox. Only trusted bounded persistence SQL is admitted.
Server statement/lock timeouts default to 10s/5s; callers choose shorter I/O deadlines.
Admission config/bounds errors send nothing. SQL errors include only SQLSTATE and
require caller rollback when inside a transaction. Receive bounds, network errors
and deadlines close the connection and report unknown: a mutation may have
committed. Resolve durable request IDs at M7.5; never treat disconnect as rollback.
Allocation exceptions propagate to the host boundary; do not catch and reuse an
incomplete session. No PostgreSQL diagnostic strings or connection values are logged.
## Migrations and privilege boundary
`tools/migrate-postgres.sh` uses psql environment/service files, no URI arguments.
It requires exactly server18.6, serializes migrations with a transaction advisory
lock, hashes the checked-in SQL, and admits clean->v1 or exact v1->v1 only.
Unknown versions/checksum changes/partial foreign schemas fail closed. No down
migration or destructive repair exists. The u64 domain rejects fractional,
negative and above-u64 values without signed truncation.
Use a dedicated owner/migrator login with CREATE on the database. Runtime login
must be separate, not a member of the owner and without database CREATE or public
schema CREATE. Bootstrap revokes PUBLIC access to its schema/tables and default
function execution. Runtime has no direct mutation rights; M7.3 adds fenced
functions and narrowly scoped EXECUTE. Operators own database/role provisioning;
the migration never creates login passwords or grants superuser.
## Kubernetes Secrets
Create Secret **evolution-postgres-migrator** in evolution-dev outside Git. Keys:
`pg_service.conf`, `pgpass`, `ca.crt`. The service file has section
`[evolution-migrator]` with host, numeric hostaddr, port, dbname, user,
sslmode=verify-full, sslrootcert=/run/postgres/ca.crt and connect_timeout=5.
pgpass uses libpq's escaped host:port:database:user:password format. Do not paste
values into chat, manifests or image arguments. Configure cluster Secret RBAC and
encryption at rest through the existing platform policy.
The example Job mounts the Secret read-only; the entrypoint copies pgpass into a
private 0600 temporary file because libpq rejects group-readable password files.
The copy is removed on exit. Use a distinct **evolution-postgres-runtime** Secret
with a restricted login for the later region pod; never mount migrator credentials
into it. Rotation requires a new connection/Job; no hot reload is claimed.
`deploy/kubernetes/postgres-migration/job.yaml.example` is intentionally outside the
active Argo overlay: select a built immutable postgres-tools digest and verify DB
target/role before adding it. Existing default-deny egress needs a specific DB and
DNS allowance chosen for the actual endpoint. No live database/Secret was created
and no World0 resources are touched by this bootstrap.
CI builds the `postgres-tools` Docker target and publishes it when Gitea variable
`HARBOR_POSTGRES_REPOSITORY` is set (for example `aevolution-postgres-tools`). It uses
the existing HARBOR_PROJECT and Harbor robot secrets. Pin the reported registry
digest in the example Job; do not use the simulation image. See deploy/README.md.
Local Unix-socket integration is qualified; remote TLS/authentication, actual Secret
mounts and container execution still require environment-specific verification.
+40
View File
@@ -0,0 +1,40 @@
#pragma once
#include <array>
#include <chrono>
#include <cstdint>
#include <expected>
#include <memory>
#include <optional>
#include <span>
#include <string>
#include <vector>
namespace evo::persistence {
enum class PgError : std::uint8_t { config, connect, version, sql, unknown, bounds, closed };
struct PgFailure {
PgError kind{};
std::array<char, 6> sqlstate{}; // Sanitized SQLSTATE only, never server text.
};
struct PgOptions {
std::string host, hostaddr, port, database, user, password_file, ca_file;
};
using PgRows = std::vector<std::vector<std::optional<std::string>>>;
// Linux I/O-owner adapter. Query text is trusted application SQL, values are parameters.
// No automatic retry. unknown means the command may have executed; connection is closed.
class Postgres {
public:
~Postgres();
Postgres(const Postgres&) = delete;
Postgres& operator=(const Postgres&) = delete;
[[nodiscard]] static std::expected<std::unique_ptr<Postgres>, PgFailure>
connect(const PgOptions& options, std::chrono::milliseconds timeout);
[[nodiscard]] std::expected<PgRows, PgFailure> query(const std::string& sql,
std::span<const std::string> parameters,
std::chrono::milliseconds timeout);
private:
struct Impl;
explicit Postgres(std::unique_ptr<Impl> impl);
std::unique_ptr<Impl> impl_;
};
} // namespace evo::persistence
+27
View File
@@ -0,0 +1,27 @@
-- Owner-only bootstrap. No runtime mutation grants until fenced functions exist.
CREATE SCHEMA evolution;
REVOKE ALL ON SCHEMA evolution FROM PUBLIC;
CREATE DOMAIN evolution.u64 AS numeric
CHECK (VALUE >= 0 AND VALUE <= 18446744073709551615 AND VALUE = trunc(VALUE));
CREATE TABLE evolution.schema_migrations (
version integer PRIMARY KEY CHECK (version > 0),
sha256 text NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'),
applied_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE evolution.region_leases (
world_id evolution.u64 NOT NULL CHECK (world_id > 0),
region_id evolution.u64 NOT NULL CHECK (region_id > 0),
frame_id evolution.u64 NOT NULL CHECK (frame_id > 0),
epoch evolution.u64 NOT NULL DEFAULT 0,
holder uuid,
expires_at timestamptz,
PRIMARY KEY (world_id, region_id),
CHECK ((holder IS NULL AND expires_at IS NULL) OR
(holder IS NOT NULL AND expires_at IS NOT NULL AND epoch > 0))
);
REVOKE ALL ON ALL TABLES IN SCHEMA evolution FROM PUBLIC;
ALTER DEFAULT PRIVILEGES IN SCHEMA evolution REVOKE ALL ON TABLES FROM PUBLIC;
-- PostgreSQL's global PUBLIC execute default cannot be revoked per schema.
-- This is the dedicated migration owner's default, not other owners' defaults.
ALTER DEFAULT PRIVILEGES REVOKE EXECUTE ON FUNCTIONS FROM PUBLIC;
INSERT INTO evolution.schema_migrations(version, sha256) VALUES (1, :'migration_sha');
+26
View File
@@ -0,0 +1,26 @@
\set ON_ERROR_STOP on
BEGIN;
SET LOCAL search_path = pg_catalog;
SET LOCAL statement_timeout = '10s';
SET LOCAL lock_timeout = '5s';
SELECT pg_catalog.pg_advisory_xact_lock(71420001);
SELECT current_setting('server_version_num')::int = 180006 AS supported \gset
\if :supported
\else
\echo 'Unsupported PostgreSQL version; expected 18.6'
DO $$ BEGIN RAISE EXCEPTION 'unsupported PostgreSQL version'; END $$;
\endif
SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') AS exists \gset
\if :exists
-- Unknown/partial schema fails closed; no IF NOT EXISTS repair of foreign data.
SELECT count(*) = 1 AND min(version) = 1 AND min(sha256) = :'migration_sha'
AS compatible FROM evolution.schema_migrations \gset
\if :compatible
\else
\echo 'Unknown or changed migration history'
DO $$ BEGIN RAISE EXCEPTION 'unknown or changed migration history'; END $$;
\endif
\else
\ir 0001_region_bootstrap.sql
\endif
COMMIT;
+181
View File
@@ -0,0 +1,181 @@
#include "evo/persistence/postgres.hpp"
#include <algorithm>
#include <arpa/inet.h>
#include <cerrno>
#include <climits>
#include <cstdlib>
#include <libpq-fe.h>
#include <poll.h>
namespace evo::persistence {
namespace {
using Clock = std::chrono::steady_clock;
struct Finish {
void operator()(PGconn* p) const {
PQfinish(p);
}
};
struct Clear {
void operator()(PGresult* p) const {
PQclear(p);
}
};
using Connection = std::unique_ptr<PGconn, Finish>;
using Result = std::unique_ptr<PGresult, Clear>;
bool wait(PGconn* c, short events, Clock::time_point deadline) {
for (;;) {
const auto left =
std::chrono::duration_cast<std::chrono::milliseconds>(deadline - Clock::now());
if (left.count() <= 0 || PQsocket(c) < 0)
return false;
pollfd descriptor{PQsocket(c), events, 0};
const auto ready = ::poll(
&descriptor, 1, static_cast<int>(std::min(left.count(), static_cast<long>(INT_MAX))));
if (ready > 0)
return (descriptor.revents & events) != 0;
if (ready == 0 || errno != EINTR)
return false;
}
}
bool bounded(const std::string& s) {
return s.size() <= 4096 && s.find('\0') == std::string::npos;
}
bool duration(std::chrono::milliseconds t) {
return t.count() > 0 && t.count() <= 30000;
}
PgFailure fail(PgError e) {
return {e, {}};
}
} // namespace
struct Postgres::Impl {
Connection connection;
};
Postgres::Postgres(std::unique_ptr<Impl> impl) : impl_(std::move(impl)) {}
Postgres::~Postgres() = default;
std::expected<std::unique_ptr<Postgres>, PgFailure>
Postgres::connect(const PgOptions& o, std::chrono::milliseconds timeout) {
if (!duration(timeout) || o.host.empty() || o.database.empty() || o.user.empty() ||
o.port.empty())
return std::unexpected{fail(PgError::config)};
for (const auto* value :
{&o.host, &o.hostaddr, &o.port, &o.database, &o.user, &o.password_file, &o.ca_file})
if (!bounded(*value))
return std::unexpected{fail(PgError::config)};
const bool local = o.host.front() == '/';
if (o.host.find(',') != std::string::npos ||
(local && (std::getenv("PGHOSTADDR") || std::getenv("PGSERVICE"))))
return std::unexpected{fail(PgError::config)};
std::array<unsigned char, 16> address{};
if ((!local &&
(o.ca_file.empty() || (inet_pton(AF_INET, o.hostaddr.c_str(), address.data()) != 1 &&
inet_pton(AF_INET6, o.hostaddr.c_str(), address.data()) != 1))) ||
(local && !o.hostaddr.empty()))
return std::unexpected{fail(PgError::config)};
const char* keys[] = {"host", "hostaddr", "port", "dbname", "user", "passfile",
"sslrootcert", "sslmode", "gssencmode", "options", nullptr};
const char* values[] = {o.host.c_str(),
o.hostaddr.c_str(),
o.port.c_str(),
o.database.c_str(),
o.user.c_str(),
o.password_file.c_str(),
o.ca_file.c_str(),
local ? "disable" : "verify-full",
"disable",
"-csearch_path= -cstatement_timeout=10000 -clock_timeout=5000",
nullptr};
const auto deadline = Clock::now() + timeout;
Connection c{PQconnectStartParams(keys, values, 0)};
if (!c)
return std::unexpected{fail(PgError::connect)};
auto status = PGRES_POLLING_WRITING;
while (status != PGRES_POLLING_OK) {
if (status == PGRES_POLLING_FAILED ||
!wait(c.get(), status == PGRES_POLLING_READING ? POLLIN : POLLOUT, deadline))
return std::unexpected{fail(PgError::connect)};
status = PQconnectPoll(c.get());
}
if (PQlibVersion() != 180006 || PQserverVersion(c.get()) != 180006)
return std::unexpected{fail(PgError::version)};
if (PQsetnonblocking(c.get(), 1) != 0)
return std::unexpected{fail(PgError::connect)};
auto impl = std::make_unique<Impl>();
impl->connection = std::move(c);
return std::unique_ptr<Postgres>(new Postgres(std::move(impl)));
}
std::expected<PgRows, PgFailure> Postgres::query(const std::string& sql,
std::span<const std::string> parameters,
std::chrono::milliseconds timeout) {
auto& owned = impl_->connection;
if (!owned)
return std::unexpected{fail(PgError::closed)};
if (!duration(timeout) || sql.size() > 16384 || sql.find('\0') != std::string::npos ||
parameters.size() > 16)
return std::unexpected{fail(PgError::config)};
std::array<const char*, 16> values{};
for (std::size_t i = 0; i < parameters.size(); ++i) {
if (!bounded(parameters[i]))
return std::unexpected{fail(PgError::bounds)};
values[i] = parameters[i].c_str();
}
auto* c = owned.get();
const auto deadline = Clock::now() + timeout;
const auto unknown = [&]() -> std::expected<PgRows, PgFailure> {
owned.reset();
return std::unexpected{fail(PgError::unknown)};
};
if (PQsendQueryParams(c, sql.c_str(), static_cast<int>(parameters.size()), nullptr,
values.data(), nullptr, nullptr, 0) != 1 ||
PQsetSingleRowMode(c) != 1)
return unknown();
int flush{};
while ((flush = PQflush(c)) == 1)
if (!wait(c, POLLOUT, deadline))
return unknown();
if (flush < 0)
return unknown();
PgRows rows;
std::size_t bytes{};
std::optional<PgFailure> error;
for (;;) {
if (Clock::now() >= deadline)
return unknown();
while (PQisBusy(c))
if (!wait(c, POLLIN, deadline) || PQconsumeInput(c) != 1)
return unknown();
Result r{PQgetResult(c)};
if (!r)
break;
const auto status = PQresultStatus(r.get());
if (status == PGRES_SINGLE_TUPLE) {
if (rows.size() == 128 || PQnfields(r.get()) > 16)
return unknown();
std::vector<std::optional<std::string>> row;
for (int j = 0; j < PQnfields(r.get()); ++j) {
const auto length = static_cast<std::size_t>(PQgetlength(r.get(), 0, j));
if (length > 65536 - bytes)
return unknown();
bytes += length;
if (PQgetisnull(r.get(), 0, j))
row.emplace_back(std::nullopt);
else
row.emplace_back(std::string(PQgetvalue(r.get(), 0, j), length));
}
rows.push_back(std::move(row));
} else if (status != PGRES_COMMAND_OK && status != PGRES_TUPLES_OK) {
if (status != PGRES_FATAL_ERROR)
return unknown();
PgFailure f = fail(PgError::sql);
const auto* state = PQresultErrorField(r.get(), PG_DIAG_SQLSTATE);
if (state)
std::copy_n(state, 5, f.sqlstate.begin());
error = f;
}
}
if (PQstatus(c) != CONNECTION_OK)
return unknown();
if (error)
return std::unexpected{*error};
return rows;
}
} // namespace evo::persistence
+77
View File
@@ -0,0 +1,77 @@
#include "evo/persistence/postgres.hpp"
#include <cstdio>
#include <cstdlib>
#include <source_location>
#include <stdexcept>
namespace {
using namespace evo::persistence;
using namespace std::chrono_literals;
void check(bool value, std::source_location at = std::source_location::current()) {
if (!value)
throw std::runtime_error("PostgreSQL test line " + std::to_string(at.line()));
}
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
std::string env(const char* key) {
const auto* value = std::getenv(key);
if (!value)
throw std::runtime_error("missing test environment");
return value;
}
void run() {
PgOptions options{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), env("PGUSER"), {}, {}};
auto made = Postgres::connect(options, 5s);
CHECK(made);
auto db = std::move(*made);
const std::array<std::string, 1> hostile{"a'); DROP SCHEMA evolution CASCADE; --"};
auto rows = db->query("SELECT $1::text, NULL::text", hostile, 5s);
CHECK(rows && rows->size() == 1 && (*rows)[0].size() == 2);
CHECK((*rows)[0][0] == hostile[0] && !(*rows)[0][1]);
CHECK(db->query("BEGIN", {}, 5s));
CHECK(db->query(
"INSERT INTO evolution.region_leases(world_id,region_id,frame_id) VALUES (1,1,1)", {}, 5s));
auto duplicate = db->query(
"INSERT INTO evolution.region_leases(world_id,region_id,frame_id) VALUES (1,1,1)", {}, 5s);
CHECK(!duplicate && duplicate.error().kind == PgError::sql);
CHECK(std::string(duplicate.error().sqlstate.data()) == "23505");
CHECK(db->query("ROLLBACK", {}, 5s));
rows = db->query("SELECT count(*) FROM evolution.region_leases", {}, 5s);
CHECK(rows && (*rows)[0][0] == "0");
for (const auto& value :
{std::string{"-1"}, std::string{"18446744073709551616"}, std::string{"1.5"}}) {
const std::array<std::string, 1> parameter{value};
CHECK(!db->query("SELECT $1::evolution.u64", parameter, 5s));
}
rows = db->query("SELECT 18446744073709551615::evolution.u64", {}, 5s);
CHECK(rows && (*rows)[0][0] == "18446744073709551615");
CHECK(db->query("SET ROLE evo_runtime_test", {}, 5s));
auto denied = db->query(
"INSERT INTO evolution.region_leases(world_id,region_id,frame_id) VALUES (2,2,2)", {}, 5s);
CHECK(!denied && std::string(denied.error().sqlstate.data()) == "42501");
CHECK(db->query("RESET ROLE", {}, 5s));
auto timeout = db->query("SELECT pg_catalog.pg_sleep(2)", {}, 50ms);
CHECK(!timeout && timeout.error().kind == PgError::unknown);
auto closed = db->query("SELECT 1", {}, 5s);
CHECK(!closed && closed.error().kind == PgError::closed);
made = Postgres::connect(options, 5s);
CHECK(made);
auto oversized = (*made)->query("SELECT pg_catalog.generate_series(1,129)", {}, 5s);
CHECK(!oversized && oversized.error().kind == PgError::unknown);
made = Postgres::connect(options, 5s);
CHECK(made);
auto disconnected = (*made)->query(
"SELECT pg_catalog.pg_terminate_backend(pg_catalog.pg_backend_pid())", {}, 5s);
CHECK(!disconnected && disconnected.error().kind == PgError::unknown);
options.user = "no_such_evo_user";
CHECK(!Postgres::connect(options, 1s));
}
} // namespace
int main() {
try {
run();
std::puts("postgres adapter integration passed");
return 0;
} catch (const std::exception& e) {
std::fprintf(stderr, "%s\n", e.what());
return 1;
}
}
+30
View File
@@ -0,0 +1,30 @@
#!/usr/bin/env bash
set -euo pipefail
root="$(cd "$(dirname "$0")/../.." && pwd)"
prefix="$root/.cache/postgres/install"
export PATH="$prefix/bin:$PATH"
export LD_LIBRARY_PATH="$prefix/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
test_dir="$(mktemp -d /tmp/evo-pg-test.XXXXXXXX)"
chmod 700 "$test_dir"
export PGHOST="$test_dir" PGPORT=55432 PGDATABASE=postgres PGUSER="$(id -un)"
unset PGHOSTADDR PGPASSWORD PGSERVICE PGSERVICEFILE
initdb -D "$test_dir/data" --no-locale --encoding=UTF8 --auth-local=trust --auth-host=reject > "$test_dir/init.log"
trap 'pg_ctl -D "$test_dir/data" -m immediate stop > /dev/null || true' EXIT
pg_ctl -D "$test_dir/data" -l "$test_dir/server.log" -o "-k $test_dir -p 55432 -c listen_addresses=''" -w start
psql -X -v ON_ERROR_STOP=1 -c 'CREATE ROLE evo_runtime_test NOLOGIN'
bash "$root/tools/migrate-postgres.sh"
bash "$root/tools/migrate-postgres.sh" # exact supported upgrade/restart is a no-op
"${1:?path to evo-postgres-tests}"
# Migration failure must not publish a partial schema.
createdb migration_rollback
if (export PGDATABASE=migration_rollback; psql -X -v ON_ERROR_STOP=1 -1 \
-v migration_sha=invalid -f "$root/migrations/0001_region_bootstrap.sql"); then
echo 'Expected failed migration' >&2; exit 1
fi
test "$(PGDATABASE=migration_rollback psql -XAtc "SELECT count(*) FROM pg_namespace WHERE nspname='evolution'")" = 0
# Unknown future versions and checksum drift must not be repaired silently.
psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=2'
if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=1,sha256=repeat('0',64)"
if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi
echo 'postgres migrations, rollback, constraints and permissions passed'
+1 -1
View File
@@ -19,7 +19,7 @@ DOCKER_CONFIG="$(mktemp -d "$RUNNER_TEMP/aevolution-docker.XXXXXX")"
trap 'rm -rf -- "$DOCKER_CONFIG"' EXIT
printf '%s' "$HARBOR_PASSWORD" | docker login registry.aeternalisgames.com \
--username "$HARBOR_USERNAME" --password-stdin
source_image="aevolution-proof:${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
source_image="${SOURCE_IMAGE:-aevolution-proof:${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}}"
target_image="registry.aeternalisgames.com/${HARBOR_PROJECT}/${HARBOR_REPOSITORY}:sha-${GITHUB_SHA}-run-${GITHUB_RUN_NUMBER}-${GITHUB_RUN_ATTEMPT}"
docker tag "$source_image" "$target_image"
docker push "$target_image"
+16
View File
@@ -0,0 +1,16 @@
#!/usr/bin/env bash
set -euo pipefail
root="$(cd "$(dirname "$0")/.." && pwd)"
if [ -n "${EVO_PGPASS_SOURCE:-}" ]; then
umask 077
export PGPASSFILE
PGPASSFILE="$(mktemp /tmp/evo-pgpass.XXXXXXXX)"
trap 'rm -f -- "$PGPASSFILE"' EXIT
cat "$EVO_PGPASS_SOURCE" > "$PGPASSFILE"
fi
# Connection comes from PG* environment / mounted service and password files.
# Never print the connection or enable shell tracing here.
digest="$(sha256sum "$root/migrations/0001_region_bootstrap.sql")"
digest="${digest%% *}"
psql -X --no-password --set=ON_ERROR_STOP=1 --set="migration_sha=$digest" \
--file="$root/migrations/apply.sql"
+20
View File
@@ -0,0 +1,20 @@
#!/usr/bin/env bash
set -euo pipefail
# Explicit preparation only; never downloaded by CMake configure.
root="$(cd "$(dirname "$0")/.." && pwd)"
cache="$root/.cache/postgres"
mkdir -p "$cache"
cd "$cache"
archive=postgresql-18.6.tar.bz2
if [ ! -f "$archive" ]; then
curl --fail --location --proto '=https' --tlsv1.2 \
"https://ftp.postgresql.org/pub/source/v18.6/$archive" -o "$archive"
fi
echo "555610c24d53e4316da5b7d3fc25c279d96856d5e0e23ee308c328c5fa881d9f $archive" | sha256sum --check -
if [ ! -d postgresql-18.6 ]; then tar -xjf "$archive"; fi
mkdir -p build
cd build
../postgresql-18.6/configure --prefix="$cache/install" --without-readline --without-icu --with-ssl=openssl
make -j2
make install
"$cache/install/bin/pg_config" --version