diff --git a/.gitea/workflows/build.yaml b/.gitea/workflows/build.yaml index c48f490..c9d7bc1 100644 --- a/.gitea/workflows/build.yaml +++ b/.gitea/workflows/build.yaml @@ -53,6 +53,14 @@ jobs: exit 0 fi docker version + if [ "$CHECK_SUITE" = postgres ]; then + # The PostgreSQL stages compile with PG_BUILD_JOBS=6 (deploy/docker/Dockerfile); + # each clang ASan/analysis job can take 1-2 GiB. Record the runner's real CPU, + # memory and disk headroom so an eviction or cache loss is attributable. + echo "runner: nproc=$(nproc) memory.max=$(cat /sys/fs/cgroup/memory.max 2>/dev/null || echo unknown)" + df -h "$(docker info --format '{{.DockerRootDir}}')" "${RUNNER_TEMP:-/tmp}" || true + docker buildx du 2>/dev/null | tail -n 3 || true + fi ci_image="aevolution-ci:${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-$CHECK_SUITE" if [[ "${GITHUB_REF:-}" == refs/heads/main && "${GITHUB_EVENT_NAME:-}" != pull_request && -n "${HARBOR_USERNAME:-}" ]]; then export CACHE_WRITE=1 diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 6856976..903f20d 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -111,12 +111,16 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* # The toolchain installs versioned compilers only; there is no bare cc/gcc. COPY tools/prepare-postgres.sh tools/prepare-postgres.sh -RUN CC=gcc-14 bash tools/prepare-postgres.sh +# Compile parallelism for the PostgreSQL stages. Each clang ASan/analysis job can take +# 1-2 GiB, so raising this needs matching runner memory (see .gitea/workflows/build.yaml). +ARG PG_BUILD_JOBS=6 +RUN CC=gcc-14 PG_BUILD_JOBS=${PG_BUILD_JOBS} bash tools/prepare-postgres.sh # PostgreSQL itself is prepared once. Source/test changes invalidate only the # layers below, while postgres-tools can reuse this image without compiling the # full C++ test matrix. FROM postgres-toolchain AS postgres-build +ARG PG_BUILD_JOBS=6 COPY CMakeLists.txt CMakePresets.json .clang-format .clang-tidy ./ COPY cmake/ cmake/ COPY include/ include/ @@ -135,25 +139,29 @@ RUN cmake --preset release -B build/postgres -DCMAKE_CXX_COMPILER=g++-14 \ -DCMAKE_C_COMPILER=gcc-14 -DEVO_BUILD_NETWORK=ON \ -DEVO_BUILD_QUIC=ON -DEVO_BUILD_POSTGRES=ON \ -DPostgreSQL_ROOT=/source/.cache/postgres/install \ - && cmake --build build/postgres --target evo-server evo-session-probe evo-world-clone --parallel 2 + && cmake --build build/postgres --target evo-server evo-session-probe evo-world-clone --parallel "${PG_BUILD_JOBS}" FROM postgres-build AS postgres-test-build -RUN cmake --build build/postgres --target evo-postgres-tests --parallel 2 +ARG PG_BUILD_JOBS=6 +RUN cmake --build build/postgres --target evo-postgres-tests --parallel "${PG_BUILD_JOBS}" RUN cmake --preset analysis -B build/postgres-analysis -DCMAKE_CXX_COMPILER=clang++-19 \ -DCMAKE_C_COMPILER=clang-19 -DEVO_BUILD_NETWORK=ON \ -DEVO_BUILD_POSTGRES=ON -DPostgreSQL_ROOT=/source/.cache/postgres/install \ - && cmake --build build/postgres-analysis --target evo-postgres-tests --parallel 2 + && cmake --build build/postgres-analysis --target evo-postgres-tests --parallel "${PG_BUILD_JOBS}" RUN cmake --preset asan -B build/postgres-asan -DCMAKE_CXX_COMPILER=clang++-19 \ -DCMAKE_C_COMPILER=clang-19 -DEVO_BUILD_NETWORK=ON \ -DEVO_BUILD_POSTGRES=ON -DPostgreSQL_ROOT=/source/.cache/postgres/install \ - && cmake --build build/postgres-asan --target evo-postgres-tests --parallel 2 + && cmake --build build/postgres-asan --target evo-postgres-tests --parallel "${PG_BUILD_JOBS}" FROM postgres-test-build AS ci-postgres RUN useradd --uid 10001 --no-create-home evo USER 10001:10001 -RUN bash tools/ci/postgres-check.sh /source/build/postgres/evo-postgres-tests \ - && bash tools/ci/postgres-check.sh /source/build/postgres-analysis/evo-postgres-tests \ - && bash tools/ci/postgres-check.sh /source/build/postgres-asan/evo-postgres-tests +# Full qualification once on the release build. ASan/UBSan reruns everything that +# executes its instrumented binaries; the uninstrumented clang-tidy build reruns only +# the C++ suite. SQL-only checks do not change with compiler flags. +RUN bash tools/ci/postgres-check.sh --full /source/build/postgres/evo-postgres-tests \ + && bash tools/ci/postgres-check.sh --suite-only /source/build/postgres-analysis/evo-postgres-tests \ + && bash tools/ci/postgres-check.sh --sanitizer /source/build/postgres-asan/evo-postgres-tests FROM postgres-toolchain AS postgres-tools RUN apt-get update && apt-get install -y --no-install-recommends libssl3t64 zlib1g ca-certificates \ diff --git a/deploy/kubernetes/region/dev-session-route/listener.yaml b/deploy/kubernetes/region/dev-session-route/listener.yaml index 395af62..f6fdda6 100644 --- a/deploy/kubernetes/region/dev-session-route/listener.yaml +++ b/deploy/kubernetes/region/dev-session-route/listener.yaml @@ -12,7 +12,7 @@ spec: env: - {name: REGION_SESSION, value: "dev"} - {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"} - - {name: REGION_SESSION_WINDOW, value: "dev-20261013T111133-17978:1791904293"} + - {name: REGION_SESSION_WINDOW, value: "dev-20261013T123806-14512:1791909486"} - {name: REGION_M10_MAX_GRANT_LIFETIME, value: "14400"} ports: # QUIC (ADR-0090): UDP only; nothing listens on TCP 7400. diff --git a/docs/adr/0074-world-backup-and-restore.md b/docs/adr/0074-world-backup-and-restore.md index bea2697..ae2cf48 100644 --- a/docs/adr/0074-world-backup-and-restore.md +++ b/docs/adr/0074-world-backup-and-restore.md @@ -136,7 +136,8 @@ runs. It works as follows: A scheduled CronJob, synced by Argo, follows once the destination exists. **Qualification.** `tools/ci/backup-check.sh` runs inside `postgres-check.sh` for -every PostgreSQL build: +the release and ASan/UBSan PostgreSQL builds (the uninstrumented clang-tidy build runs +only the C++ suite): - It backs up a real `evo-server` while the server commits under synthetic load. The backup head covers every acknowledgement made before the backup, and the source then moves past it. diff --git a/tools/ci/postgres-check.sh b/tools/ci/postgres-check.sh index 7e6d9f7..fb63aa9 100644 --- a/tools/ci/postgres-check.sh +++ b/tools/ci/postgres-check.sh @@ -1,6 +1,37 @@ #!/usr/bin/env bash set -euo pipefail +# Usage: postgres-check.sh [--full|--sanitizer|--suite-only] +# postgres-check.sh --life-reservations-only [] +# --full (default) is the complete qualification: the C++ suite, the backup, DR and +# upgrade process checks, then the SQL-only migration, permission and checksum checks. +# --sanitizer keeps everything that runs the binaries built beside the test binary +# (the C++ suite and the backup/DR/upgrade checks, which spawn that evo-server and +# evo-world-clone) and drops the SQL-only checks, which do not change with compiler +# flags. --suite-only runs just the C++ suite (for the uninstrumented analysis build). root="$(cd "$(dirname "$0")/../.." && pwd)" +mode=full +case "${1:-}" in + --full) shift ;; + --sanitizer) mode=sanitizer; shift ;; + --suite-only) mode=suite-only; shift ;; + --life-reservations-only) mode=life-reservations ;; +esac +# Per-phase wall-clock timing; the summary is printed on success and on failure. +phase_names=() phase_secs=() phase_name=setup phase_start=$SECONDS run_start=$SECONDS +phase() { + phase_names+=("$phase_name") phase_secs+=($((SECONDS - phase_start))) + phase_name=$1 phase_start=$SECONDS + echo "postgres-check[$mode]: phase $phase_name" +} +timing_summary() { + local i + phase_names+=("$phase_name") phase_secs+=($((SECONDS - phase_start))) + echo "postgres-check[$mode] timing ($1):" + for i in "${!phase_names[@]}"; do + printf ' %-28s %5ds\n' "${phase_names[$i]}" "${phase_secs[$i]}" + done + printf ' %-28s %5ds\n' total $((SECONDS - run_start)) +} prefix="$root/.cache/postgres/install" export PATH="$prefix/bin:$PATH" export LD_LIBRARY_PATH="$prefix/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" @@ -9,15 +40,17 @@ chmod 700 "$test_dir" export PGHOST="$test_dir" PGPORT=55432 PGDATABASE=postgres PGUSER="$(id -un)" unset PGHOSTADDR PGPASSWORD PGSERVICE PGSERVICEFILE initdb -D "$test_dir/data" --no-locale --encoding=UTF8 --auth-local=trust --auth-host=reject > "$test_dir/init.log" -trap 'pg_ctl -D "$test_dir/data" -m immediate stop > /dev/null || true' EXIT +trap 'rc=$?; timing_summary "exit $rc" >&2; pg_ctl -D "$test_dir/data" -m immediate stop > /dev/null || true' EXIT pg_ctl -D "$test_dir/data" -l "$test_dir/server.log" -o "-k $test_dir -p 55432 -c listen_addresses=''" -w start # Runtime group role receives grants; a separate login inherits only that role. psql -X -v ON_ERROR_STOP=1 -c 'CREATE ROLE evo_runtime_test NOLOGIN' \ -c 'CREATE ROLE evo_runtime_login LOGIN IN ROLE evo_runtime_test' export EVO_PG_RUNTIME_ROLE=evo_runtime_test EVO_PG_RUNTIME_USER=evo_runtime_login +phase migrate bash "$root/tools/migrate-postgres.sh" bash "$root/tools/migrate-postgres.sh" # exact supported upgrade/restart is a no-op -if [ "${1:-}" = --life-reservations-only ]; then +if [ "$mode" = life-reservations ]; then + phase life-reservations if [ "$#" = 2 ]; then "$2" --life-reservations-only; fi psql -X -v ON_ERROR_STOP=1 -f "$root/tools/ci/life-reservations-check.sql" > /dev/null bash "$root/tools/ci/life-reservations-concurrency.sh" @@ -35,19 +68,35 @@ export EVO_SOURCE_ROOT="$root" export EVO_SERVER_BIN EVO_WORLD_CLONE_BIN EVO_SERVER_BIN="$(dirname "${1:?path to evo-postgres-tests}")/evo-server" EVO_WORLD_CLONE_BIN="$(dirname "$1")/evo-world-clone" +phase cxx-suite if ! "$1"; then echo 'PostgreSQL integration tests failed; recent disposable-cluster server log:' >&2 tail -n 80 "$test_dir/server.log" >&2 || true exit 1 fi -psql -X -v ON_ERROR_STOP=1 -f "$root/tools/ci/life-reservations-check.sql" > /dev/null -bash "$root/tools/ci/life-reservations-concurrency.sh" +if [ "$mode" = suite-only ]; then + echo "postgres-check[suite-only]: $1 passed; process and SQL checks run in --full" + exit 0 +fi +if [ "$mode" = full ]; then + phase life-reservations + psql -X -v ON_ERROR_STOP=1 -f "$root/tools/ci/life-reservations-check.sql" > /dev/null + bash "$root/tools/ci/life-reservations-concurrency.sh" +fi +phase backup-check # M8.7: online backup under load, restore on a separate cluster, isolated rebuild. bash "$root/tools/ci/backup-check.sh" +phase dr-check # M8.8: host-loss rehearsal: in-place restore, fenced old environment, measured rewind. bash "$root/tools/ci/dr-check.sh" +phase upgrade-check # M8.9: failed migration under load and the version barrier with real hosts. bash "$root/tools/ci/upgrade-check.sh" +if [ "$mode" = sanitizer ]; then + echo "postgres-check[sanitizer]: suite and process checks passed; SQL-only checks run in --full" + exit 0 +fi +phase migration-history v1_sha="$(sha256sum "$root/migrations/0001_region_bootstrap.sql")" v1_sha="${v1_sha%% *}" # Migration failure must not publish a partial schema. @@ -151,6 +200,7 @@ PGDATABASE=v6_populated bash "$root/tools/migrate-postgres.sh" test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33 test "$(PGDATABASE=v6_populated psql -XAtc "SELECT count(*) FROM evolution.region_commits c, LATERAL evolution.parse_checkpoint_manifest(c.manifest) m WHERE m.commit_sequence = 0 AND m.predecessor_commit IS NULL")" = 1 test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(conname, ',') FROM pg_constraint WHERE conrelid = 'evolution.region_heads'::regclass AND contype = 'c' AND pg_get_constraintdef(oid) LIKE '%end_tick%'")" = region_heads_published_tick +phase permission-audit # Inherited privileges bypass fencing, so every migrate run audits effective rights. psql -X -v ON_ERROR_STOP=1 -c 'CREATE ROLE evo_bad_runtime NOLOGIN IN ROLE pg_write_all_data' if (export PGDATABASE=migration_rollback EVO_PG_RUNTIME_ROLE=evo_bad_runtime; \ @@ -198,6 +248,7 @@ expect_audit_failure 'GRANT SELECT ON evolution.cellular_mutation_action_publica expect_audit_failure 'GRANT SELECT ON evolution.account_controls TO evo_runtime_test' 'REVOKE SELECT ON evolution.account_controls FROM evo_runtime_test' expect_audit_failure 'REVOKE EXECUTE ON FUNCTION evolution.claim_account_control(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, evolution.u64, evolution.u64, integer) FROM evo_runtime_test' 'GRANT EXECUTE ON FUNCTION evolution.claim_account_control(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, evolution.u64, evolution.u64, integer) TO evo_runtime_test' bash "$root/tools/migrate-postgres.sh" # Restored configuration passes the audit again. +phase checksum-drift # Unknown future versions and checksum drift must not be repaired silently. psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=34 WHERE version=33' if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi diff --git a/tools/prepare-postgres.sh b/tools/prepare-postgres.sh index 7884d55..7a8fcf6 100644 --- a/tools/prepare-postgres.sh +++ b/tools/prepare-postgres.sh @@ -15,6 +15,6 @@ if [ ! -d postgresql-18.6 ]; then tar -xjf "$archive"; fi mkdir -p build cd build ../postgresql-18.6/configure --prefix="$cache/install" --without-readline --without-icu --with-ssl=openssl -make -j2 +make -j"${PG_BUILD_JOBS:-6}" make install "$cache/install/bin/pg_config" --version