Cut PostgreSQL CI time: parallel builds and split qualification
Foundation / checks (ecs) (push) Successful in 5m55s
Foundation / checks (deploy) (push) Successful in 6m56s
Foundation / checks (tsan) (push) Successful in 9m35s
Foundation / checks (client) (push) Successful in 20m0s
Foundation / checks (network) (push) Successful in 22m38s
Foundation / checks (checks) (push) Successful in 44m57s
Foundation / checks (postgres) (push) Successful in 48m59s
Foundation / server-image (push) Successful in 40s
Foundation / image (push) Successful in 3m11s
Foundation / postgres-tools (push) Successful in 4m26s

Compile the PostgreSQL stages with PG_BUILD_JOBS=6 (was 2), including
PostgreSQL itself. postgres-check.sh gains --full (release),
--sanitizer (ASan: C++ suite plus backup/DR/upgrade process checks that
run the instrumented binaries) and --suite-only (clang-tidy build), so
SQL-only checks run once. Each run prints per-phase timings, and the
postgres job logs runner CPU, memory and disk headroom.

Also refresh the dev REGION_SESSION_WINDOW.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-10-13 08:38:26 -04:00
co-authored by Claude Opus 5.5
parent 08b4e843c1
commit a33755c60d
6 changed files with 83 additions and 15 deletions
+8
View File
@@ -53,6 +53,14 @@ jobs:
exit 0
fi
docker version
if [ "$CHECK_SUITE" = postgres ]; then
# The PostgreSQL stages compile with PG_BUILD_JOBS=6 (deploy/docker/Dockerfile);
# each clang ASan/analysis job can take 1-2 GiB. Record the runner's real CPU,
# memory and disk headroom so an eviction or cache loss is attributable.
echo "runner: nproc=$(nproc) memory.max=$(cat /sys/fs/cgroup/memory.max 2>/dev/null || echo unknown)"
df -h "$(docker info --format '{{.DockerRootDir}}')" "${RUNNER_TEMP:-/tmp}" || true
docker buildx du 2>/dev/null | tail -n 3 || true
fi
ci_image="aevolution-ci:${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-$CHECK_SUITE"
if [[ "${GITHUB_REF:-}" == refs/heads/main && "${GITHUB_EVENT_NAME:-}" != pull_request && -n "${HARBOR_USERNAME:-}" ]]; then
export CACHE_WRITE=1
+16 -8
View File
@@ -111,12 +111,16 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
&& rm -rf /var/lib/apt/lists/*
# The toolchain installs versioned compilers only; there is no bare cc/gcc.
COPY tools/prepare-postgres.sh tools/prepare-postgres.sh
RUN CC=gcc-14 bash tools/prepare-postgres.sh
# Compile parallelism for the PostgreSQL stages. Each clang ASan/analysis job can take
# 1-2 GiB, so raising this needs matching runner memory (see .gitea/workflows/build.yaml).
ARG PG_BUILD_JOBS=6
RUN CC=gcc-14 PG_BUILD_JOBS=${PG_BUILD_JOBS} bash tools/prepare-postgres.sh
# PostgreSQL itself is prepared once. Source/test changes invalidate only the
# layers below, while postgres-tools can reuse this image without compiling the
# full C++ test matrix.
FROM postgres-toolchain AS postgres-build
ARG PG_BUILD_JOBS=6
COPY CMakeLists.txt CMakePresets.json .clang-format .clang-tidy ./
COPY cmake/ cmake/
COPY include/ include/
@@ -135,25 +139,29 @@ RUN cmake --preset release -B build/postgres -DCMAKE_CXX_COMPILER=g++-14 \
-DCMAKE_C_COMPILER=gcc-14 -DEVO_BUILD_NETWORK=ON \
-DEVO_BUILD_QUIC=ON -DEVO_BUILD_POSTGRES=ON \
-DPostgreSQL_ROOT=/source/.cache/postgres/install \
&& cmake --build build/postgres --target evo-server evo-session-probe evo-world-clone --parallel 2
&& cmake --build build/postgres --target evo-server evo-session-probe evo-world-clone --parallel "${PG_BUILD_JOBS}"
FROM postgres-build AS postgres-test-build
RUN cmake --build build/postgres --target evo-postgres-tests --parallel 2
ARG PG_BUILD_JOBS=6
RUN cmake --build build/postgres --target evo-postgres-tests --parallel "${PG_BUILD_JOBS}"
RUN cmake --preset analysis -B build/postgres-analysis -DCMAKE_CXX_COMPILER=clang++-19 \
-DCMAKE_C_COMPILER=clang-19 -DEVO_BUILD_NETWORK=ON \
-DEVO_BUILD_POSTGRES=ON -DPostgreSQL_ROOT=/source/.cache/postgres/install \
&& cmake --build build/postgres-analysis --target evo-postgres-tests --parallel 2
&& cmake --build build/postgres-analysis --target evo-postgres-tests --parallel "${PG_BUILD_JOBS}"
RUN cmake --preset asan -B build/postgres-asan -DCMAKE_CXX_COMPILER=clang++-19 \
-DCMAKE_C_COMPILER=clang-19 -DEVO_BUILD_NETWORK=ON \
-DEVO_BUILD_POSTGRES=ON -DPostgreSQL_ROOT=/source/.cache/postgres/install \
&& cmake --build build/postgres-asan --target evo-postgres-tests --parallel 2
&& cmake --build build/postgres-asan --target evo-postgres-tests --parallel "${PG_BUILD_JOBS}"
FROM postgres-test-build AS ci-postgres
RUN useradd --uid 10001 --no-create-home evo
USER 10001:10001
RUN bash tools/ci/postgres-check.sh /source/build/postgres/evo-postgres-tests \
&& bash tools/ci/postgres-check.sh /source/build/postgres-analysis/evo-postgres-tests \
&& bash tools/ci/postgres-check.sh /source/build/postgres-asan/evo-postgres-tests
# Full qualification once on the release build. ASan/UBSan reruns everything that
# executes its instrumented binaries; the uninstrumented clang-tidy build reruns only
# the C++ suite. SQL-only checks do not change with compiler flags.
RUN bash tools/ci/postgres-check.sh --full /source/build/postgres/evo-postgres-tests \
&& bash tools/ci/postgres-check.sh --suite-only /source/build/postgres-analysis/evo-postgres-tests \
&& bash tools/ci/postgres-check.sh --sanitizer /source/build/postgres-asan/evo-postgres-tests
FROM postgres-toolchain AS postgres-tools
RUN apt-get update && apt-get install -y --no-install-recommends libssl3t64 zlib1g ca-certificates \
@@ -12,7 +12,7 @@ spec:
env:
- {name: REGION_SESSION, value: "dev"}
- {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"}
- {name: REGION_SESSION_WINDOW, value: "dev-20261013T111133-17978:1791904293"}
- {name: REGION_SESSION_WINDOW, value: "dev-20261013T123806-14512:1791909486"}
- {name: REGION_M10_MAX_GRANT_LIFETIME, value: "14400"}
ports:
# QUIC (ADR-0090): UDP only; nothing listens on TCP 7400.
+2 -1
View File
@@ -136,7 +136,8 @@ runs. It works as follows:
A scheduled CronJob, synced by Argo, follows once the destination exists.
**Qualification.** `tools/ci/backup-check.sh` runs inside `postgres-check.sh` for
every PostgreSQL build:
the release and ASan/UBSan PostgreSQL builds (the uninstrumented clang-tidy build runs
only the C++ suite):
- It backs up a real `evo-server` while the server commits under synthetic load.
The backup head covers every acknowledgement made before the backup, and the
source then moves past it.
+55 -4
View File
@@ -1,6 +1,37 @@
#!/usr/bin/env bash
set -euo pipefail
# Usage: postgres-check.sh [--full|--sanitizer|--suite-only] <evo-postgres-tests>
# postgres-check.sh --life-reservations-only [<evo-postgres-tests>]
# --full (default) is the complete qualification: the C++ suite, the backup, DR and
# upgrade process checks, then the SQL-only migration, permission and checksum checks.
# --sanitizer keeps everything that runs the binaries built beside the test binary
# (the C++ suite and the backup/DR/upgrade checks, which spawn that evo-server and
# evo-world-clone) and drops the SQL-only checks, which do not change with compiler
# flags. --suite-only runs just the C++ suite (for the uninstrumented analysis build).
root="$(cd "$(dirname "$0")/../.." && pwd)"
mode=full
case "${1:-}" in
--full) shift ;;
--sanitizer) mode=sanitizer; shift ;;
--suite-only) mode=suite-only; shift ;;
--life-reservations-only) mode=life-reservations ;;
esac
# Per-phase wall-clock timing; the summary is printed on success and on failure.
phase_names=() phase_secs=() phase_name=setup phase_start=$SECONDS run_start=$SECONDS
phase() {
phase_names+=("$phase_name") phase_secs+=($((SECONDS - phase_start)))
phase_name=$1 phase_start=$SECONDS
echo "postgres-check[$mode]: phase $phase_name"
}
timing_summary() {
local i
phase_names+=("$phase_name") phase_secs+=($((SECONDS - phase_start)))
echo "postgres-check[$mode] timing ($1):"
for i in "${!phase_names[@]}"; do
printf ' %-28s %5ds\n' "${phase_names[$i]}" "${phase_secs[$i]}"
done
printf ' %-28s %5ds\n' total $((SECONDS - run_start))
}
prefix="$root/.cache/postgres/install"
export PATH="$prefix/bin:$PATH"
export LD_LIBRARY_PATH="$prefix/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
@@ -9,15 +40,17 @@ chmod 700 "$test_dir"
export PGHOST="$test_dir" PGPORT=55432 PGDATABASE=postgres PGUSER="$(id -un)"
unset PGHOSTADDR PGPASSWORD PGSERVICE PGSERVICEFILE
initdb -D "$test_dir/data" --no-locale --encoding=UTF8 --auth-local=trust --auth-host=reject > "$test_dir/init.log"
trap 'pg_ctl -D "$test_dir/data" -m immediate stop > /dev/null || true' EXIT
trap 'rc=$?; timing_summary "exit $rc" >&2; pg_ctl -D "$test_dir/data" -m immediate stop > /dev/null || true' EXIT
pg_ctl -D "$test_dir/data" -l "$test_dir/server.log" -o "-k $test_dir -p 55432 -c listen_addresses=''" -w start
# Runtime group role receives grants; a separate login inherits only that role.
psql -X -v ON_ERROR_STOP=1 -c 'CREATE ROLE evo_runtime_test NOLOGIN' \
-c 'CREATE ROLE evo_runtime_login LOGIN IN ROLE evo_runtime_test'
export EVO_PG_RUNTIME_ROLE=evo_runtime_test EVO_PG_RUNTIME_USER=evo_runtime_login
phase migrate
bash "$root/tools/migrate-postgres.sh"
bash "$root/tools/migrate-postgres.sh" # exact supported upgrade/restart is a no-op
if [ "${1:-}" = --life-reservations-only ]; then
if [ "$mode" = life-reservations ]; then
phase life-reservations
if [ "$#" = 2 ]; then "$2" --life-reservations-only; fi
psql -X -v ON_ERROR_STOP=1 -f "$root/tools/ci/life-reservations-check.sql" > /dev/null
bash "$root/tools/ci/life-reservations-concurrency.sh"
@@ -35,19 +68,35 @@ export EVO_SOURCE_ROOT="$root"
export EVO_SERVER_BIN EVO_WORLD_CLONE_BIN
EVO_SERVER_BIN="$(dirname "${1:?path to evo-postgres-tests}")/evo-server"
EVO_WORLD_CLONE_BIN="$(dirname "$1")/evo-world-clone"
phase cxx-suite
if ! "$1"; then
echo 'PostgreSQL integration tests failed; recent disposable-cluster server log:' >&2
tail -n 80 "$test_dir/server.log" >&2 || true
exit 1
fi
psql -X -v ON_ERROR_STOP=1 -f "$root/tools/ci/life-reservations-check.sql" > /dev/null
bash "$root/tools/ci/life-reservations-concurrency.sh"
if [ "$mode" = suite-only ]; then
echo "postgres-check[suite-only]: $1 passed; process and SQL checks run in --full"
exit 0
fi
if [ "$mode" = full ]; then
phase life-reservations
psql -X -v ON_ERROR_STOP=1 -f "$root/tools/ci/life-reservations-check.sql" > /dev/null
bash "$root/tools/ci/life-reservations-concurrency.sh"
fi
phase backup-check
# M8.7: online backup under load, restore on a separate cluster, isolated rebuild.
bash "$root/tools/ci/backup-check.sh"
phase dr-check
# M8.8: host-loss rehearsal: in-place restore, fenced old environment, measured rewind.
bash "$root/tools/ci/dr-check.sh"
phase upgrade-check
# M8.9: failed migration under load and the version barrier with real hosts.
bash "$root/tools/ci/upgrade-check.sh"
if [ "$mode" = sanitizer ]; then
echo "postgres-check[sanitizer]: suite and process checks passed; SQL-only checks run in --full"
exit 0
fi
phase migration-history
v1_sha="$(sha256sum "$root/migrations/0001_region_bootstrap.sql")"
v1_sha="${v1_sha%% *}"
# Migration failure must not publish a partial schema.
@@ -151,6 +200,7 @@ PGDATABASE=v6_populated bash "$root/tools/migrate-postgres.sh"
test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33
test "$(PGDATABASE=v6_populated psql -XAtc "SELECT count(*) FROM evolution.region_commits c, LATERAL evolution.parse_checkpoint_manifest(c.manifest) m WHERE m.commit_sequence = 0 AND m.predecessor_commit IS NULL")" = 1
test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(conname, ',') FROM pg_constraint WHERE conrelid = 'evolution.region_heads'::regclass AND contype = 'c' AND pg_get_constraintdef(oid) LIKE '%end_tick%'")" = region_heads_published_tick
phase permission-audit
# Inherited privileges bypass fencing, so every migrate run audits effective rights.
psql -X -v ON_ERROR_STOP=1 -c 'CREATE ROLE evo_bad_runtime NOLOGIN IN ROLE pg_write_all_data'
if (export PGDATABASE=migration_rollback EVO_PG_RUNTIME_ROLE=evo_bad_runtime; \
@@ -198,6 +248,7 @@ expect_audit_failure 'GRANT SELECT ON evolution.cellular_mutation_action_publica
expect_audit_failure 'GRANT SELECT ON evolution.account_controls TO evo_runtime_test' 'REVOKE SELECT ON evolution.account_controls FROM evo_runtime_test'
expect_audit_failure 'REVOKE EXECUTE ON FUNCTION evolution.claim_account_control(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, evolution.u64, evolution.u64, integer) FROM evo_runtime_test' 'GRANT EXECUTE ON FUNCTION evolution.claim_account_control(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, evolution.u64, evolution.u64, integer) TO evo_runtime_test'
bash "$root/tools/migrate-postgres.sh" # Restored configuration passes the audit again.
phase checksum-drift
# Unknown future versions and checksum drift must not be repaired silently.
psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=34 WHERE version=33'
if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi
+1 -1
View File
@@ -15,6 +15,6 @@ if [ ! -d postgresql-18.6 ]; then tar -xjf "$archive"; fi
mkdir -p build
cd build
../postgresql-18.6/configure --prefix="$cache/install" --without-readline --without-icu --with-ssl=openssl
make -j2
make -j"${PG_BUILD_JOBS:-6}"
make install
"$cache/install/bin/pg_config" --version