Cut PostgreSQL CI time: parallel builds and split qualification
Foundation / checks (ecs) (push) Successful in 5m55s
Foundation / checks (deploy) (push) Successful in 6m56s
Foundation / checks (tsan) (push) Successful in 9m35s
Foundation / checks (client) (push) Successful in 20m0s
Foundation / checks (network) (push) Successful in 22m38s
Foundation / checks (checks) (push) Successful in 44m57s
Foundation / checks (postgres) (push) Successful in 48m59s
Foundation / server-image (push) Successful in 40s
Foundation / image (push) Successful in 3m11s
Foundation / postgres-tools (push) Successful in 4m26s
Foundation / checks (ecs) (push) Successful in 5m55s
Foundation / checks (deploy) (push) Successful in 6m56s
Foundation / checks (tsan) (push) Successful in 9m35s
Foundation / checks (client) (push) Successful in 20m0s
Foundation / checks (network) (push) Successful in 22m38s
Foundation / checks (checks) (push) Successful in 44m57s
Foundation / checks (postgres) (push) Successful in 48m59s
Foundation / server-image (push) Successful in 40s
Foundation / image (push) Successful in 3m11s
Foundation / postgres-tools (push) Successful in 4m26s
Compile the PostgreSQL stages with PG_BUILD_JOBS=6 (was 2), including PostgreSQL itself. postgres-check.sh gains --full (release), --sanitizer (ASan: C++ suite plus backup/DR/upgrade process checks that run the instrumented binaries) and --suite-only (clang-tidy build), so SQL-only checks run once. Each run prints per-phase timings, and the postgres job logs runner CPU, memory and disk headroom. Also refresh the dev REGION_SESSION_WINDOW. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -53,6 +53,14 @@ jobs:
|
||||
exit 0
|
||||
fi
|
||||
docker version
|
||||
if [ "$CHECK_SUITE" = postgres ]; then
|
||||
# The PostgreSQL stages compile with PG_BUILD_JOBS=6 (deploy/docker/Dockerfile);
|
||||
# each clang ASan/analysis job can take 1-2 GiB. Record the runner's real CPU,
|
||||
# memory and disk headroom so an eviction or cache loss is attributable.
|
||||
echo "runner: nproc=$(nproc) memory.max=$(cat /sys/fs/cgroup/memory.max 2>/dev/null || echo unknown)"
|
||||
df -h "$(docker info --format '{{.DockerRootDir}}')" "${RUNNER_TEMP:-/tmp}" || true
|
||||
docker buildx du 2>/dev/null | tail -n 3 || true
|
||||
fi
|
||||
ci_image="aevolution-ci:${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-$CHECK_SUITE"
|
||||
if [[ "${GITHUB_REF:-}" == refs/heads/main && "${GITHUB_EVENT_NAME:-}" != pull_request && -n "${HARBOR_USERNAME:-}" ]]; then
|
||||
export CACHE_WRITE=1
|
||||
|
||||
@@ -111,12 +111,16 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
# The toolchain installs versioned compilers only; there is no bare cc/gcc.
|
||||
COPY tools/prepare-postgres.sh tools/prepare-postgres.sh
|
||||
RUN CC=gcc-14 bash tools/prepare-postgres.sh
|
||||
# Compile parallelism for the PostgreSQL stages. Each clang ASan/analysis job can take
|
||||
# 1-2 GiB, so raising this needs matching runner memory (see .gitea/workflows/build.yaml).
|
||||
ARG PG_BUILD_JOBS=6
|
||||
RUN CC=gcc-14 PG_BUILD_JOBS=${PG_BUILD_JOBS} bash tools/prepare-postgres.sh
|
||||
|
||||
# PostgreSQL itself is prepared once. Source/test changes invalidate only the
|
||||
# layers below, while postgres-tools can reuse this image without compiling the
|
||||
# full C++ test matrix.
|
||||
FROM postgres-toolchain AS postgres-build
|
||||
ARG PG_BUILD_JOBS=6
|
||||
COPY CMakeLists.txt CMakePresets.json .clang-format .clang-tidy ./
|
||||
COPY cmake/ cmake/
|
||||
COPY include/ include/
|
||||
@@ -135,25 +139,29 @@ RUN cmake --preset release -B build/postgres -DCMAKE_CXX_COMPILER=g++-14 \
|
||||
-DCMAKE_C_COMPILER=gcc-14 -DEVO_BUILD_NETWORK=ON \
|
||||
-DEVO_BUILD_QUIC=ON -DEVO_BUILD_POSTGRES=ON \
|
||||
-DPostgreSQL_ROOT=/source/.cache/postgres/install \
|
||||
&& cmake --build build/postgres --target evo-server evo-session-probe evo-world-clone --parallel 2
|
||||
&& cmake --build build/postgres --target evo-server evo-session-probe evo-world-clone --parallel "${PG_BUILD_JOBS}"
|
||||
|
||||
FROM postgres-build AS postgres-test-build
|
||||
RUN cmake --build build/postgres --target evo-postgres-tests --parallel 2
|
||||
ARG PG_BUILD_JOBS=6
|
||||
RUN cmake --build build/postgres --target evo-postgres-tests --parallel "${PG_BUILD_JOBS}"
|
||||
RUN cmake --preset analysis -B build/postgres-analysis -DCMAKE_CXX_COMPILER=clang++-19 \
|
||||
-DCMAKE_C_COMPILER=clang-19 -DEVO_BUILD_NETWORK=ON \
|
||||
-DEVO_BUILD_POSTGRES=ON -DPostgreSQL_ROOT=/source/.cache/postgres/install \
|
||||
&& cmake --build build/postgres-analysis --target evo-postgres-tests --parallel 2
|
||||
&& cmake --build build/postgres-analysis --target evo-postgres-tests --parallel "${PG_BUILD_JOBS}"
|
||||
RUN cmake --preset asan -B build/postgres-asan -DCMAKE_CXX_COMPILER=clang++-19 \
|
||||
-DCMAKE_C_COMPILER=clang-19 -DEVO_BUILD_NETWORK=ON \
|
||||
-DEVO_BUILD_POSTGRES=ON -DPostgreSQL_ROOT=/source/.cache/postgres/install \
|
||||
&& cmake --build build/postgres-asan --target evo-postgres-tests --parallel 2
|
||||
&& cmake --build build/postgres-asan --target evo-postgres-tests --parallel "${PG_BUILD_JOBS}"
|
||||
|
||||
FROM postgres-test-build AS ci-postgres
|
||||
RUN useradd --uid 10001 --no-create-home evo
|
||||
USER 10001:10001
|
||||
RUN bash tools/ci/postgres-check.sh /source/build/postgres/evo-postgres-tests \
|
||||
&& bash tools/ci/postgres-check.sh /source/build/postgres-analysis/evo-postgres-tests \
|
||||
&& bash tools/ci/postgres-check.sh /source/build/postgres-asan/evo-postgres-tests
|
||||
# Full qualification once on the release build. ASan/UBSan reruns everything that
|
||||
# executes its instrumented binaries; the uninstrumented clang-tidy build reruns only
|
||||
# the C++ suite. SQL-only checks do not change with compiler flags.
|
||||
RUN bash tools/ci/postgres-check.sh --full /source/build/postgres/evo-postgres-tests \
|
||||
&& bash tools/ci/postgres-check.sh --suite-only /source/build/postgres-analysis/evo-postgres-tests \
|
||||
&& bash tools/ci/postgres-check.sh --sanitizer /source/build/postgres-asan/evo-postgres-tests
|
||||
|
||||
FROM postgres-toolchain AS postgres-tools
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends libssl3t64 zlib1g ca-certificates \
|
||||
|
||||
@@ -12,7 +12,7 @@ spec:
|
||||
env:
|
||||
- {name: REGION_SESSION, value: "dev"}
|
||||
- {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"}
|
||||
- {name: REGION_SESSION_WINDOW, value: "dev-20261013T111133-17978:1791904293"}
|
||||
- {name: REGION_SESSION_WINDOW, value: "dev-20261013T123806-14512:1791909486"}
|
||||
- {name: REGION_M10_MAX_GRANT_LIFETIME, value: "14400"}
|
||||
ports:
|
||||
# QUIC (ADR-0090): UDP only; nothing listens on TCP 7400.
|
||||
|
||||
@@ -136,7 +136,8 @@ runs. It works as follows:
|
||||
A scheduled CronJob, synced by Argo, follows once the destination exists.
|
||||
|
||||
**Qualification.** `tools/ci/backup-check.sh` runs inside `postgres-check.sh` for
|
||||
every PostgreSQL build:
|
||||
the release and ASan/UBSan PostgreSQL builds (the uninstrumented clang-tidy build runs
|
||||
only the C++ suite):
|
||||
- It backs up a real `evo-server` while the server commits under synthetic load.
|
||||
The backup head covers every acknowledgement made before the backup, and the
|
||||
source then moves past it.
|
||||
|
||||
@@ -1,6 +1,37 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
# Usage: postgres-check.sh [--full|--sanitizer|--suite-only] <evo-postgres-tests>
|
||||
# postgres-check.sh --life-reservations-only [<evo-postgres-tests>]
|
||||
# --full (default) is the complete qualification: the C++ suite, the backup, DR and
|
||||
# upgrade process checks, then the SQL-only migration, permission and checksum checks.
|
||||
# --sanitizer keeps everything that runs the binaries built beside the test binary
|
||||
# (the C++ suite and the backup/DR/upgrade checks, which spawn that evo-server and
|
||||
# evo-world-clone) and drops the SQL-only checks, which do not change with compiler
|
||||
# flags. --suite-only runs just the C++ suite (for the uninstrumented analysis build).
|
||||
root="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
mode=full
|
||||
case "${1:-}" in
|
||||
--full) shift ;;
|
||||
--sanitizer) mode=sanitizer; shift ;;
|
||||
--suite-only) mode=suite-only; shift ;;
|
||||
--life-reservations-only) mode=life-reservations ;;
|
||||
esac
|
||||
# Per-phase wall-clock timing; the summary is printed on success and on failure.
|
||||
phase_names=() phase_secs=() phase_name=setup phase_start=$SECONDS run_start=$SECONDS
|
||||
phase() {
|
||||
phase_names+=("$phase_name") phase_secs+=($((SECONDS - phase_start)))
|
||||
phase_name=$1 phase_start=$SECONDS
|
||||
echo "postgres-check[$mode]: phase $phase_name"
|
||||
}
|
||||
timing_summary() {
|
||||
local i
|
||||
phase_names+=("$phase_name") phase_secs+=($((SECONDS - phase_start)))
|
||||
echo "postgres-check[$mode] timing ($1):"
|
||||
for i in "${!phase_names[@]}"; do
|
||||
printf ' %-28s %5ds\n' "${phase_names[$i]}" "${phase_secs[$i]}"
|
||||
done
|
||||
printf ' %-28s %5ds\n' total $((SECONDS - run_start))
|
||||
}
|
||||
prefix="$root/.cache/postgres/install"
|
||||
export PATH="$prefix/bin:$PATH"
|
||||
export LD_LIBRARY_PATH="$prefix/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
|
||||
@@ -9,15 +40,17 @@ chmod 700 "$test_dir"
|
||||
export PGHOST="$test_dir" PGPORT=55432 PGDATABASE=postgres PGUSER="$(id -un)"
|
||||
unset PGHOSTADDR PGPASSWORD PGSERVICE PGSERVICEFILE
|
||||
initdb -D "$test_dir/data" --no-locale --encoding=UTF8 --auth-local=trust --auth-host=reject > "$test_dir/init.log"
|
||||
trap 'pg_ctl -D "$test_dir/data" -m immediate stop > /dev/null || true' EXIT
|
||||
trap 'rc=$?; timing_summary "exit $rc" >&2; pg_ctl -D "$test_dir/data" -m immediate stop > /dev/null || true' EXIT
|
||||
pg_ctl -D "$test_dir/data" -l "$test_dir/server.log" -o "-k $test_dir -p 55432 -c listen_addresses=''" -w start
|
||||
# Runtime group role receives grants; a separate login inherits only that role.
|
||||
psql -X -v ON_ERROR_STOP=1 -c 'CREATE ROLE evo_runtime_test NOLOGIN' \
|
||||
-c 'CREATE ROLE evo_runtime_login LOGIN IN ROLE evo_runtime_test'
|
||||
export EVO_PG_RUNTIME_ROLE=evo_runtime_test EVO_PG_RUNTIME_USER=evo_runtime_login
|
||||
phase migrate
|
||||
bash "$root/tools/migrate-postgres.sh"
|
||||
bash "$root/tools/migrate-postgres.sh" # exact supported upgrade/restart is a no-op
|
||||
if [ "${1:-}" = --life-reservations-only ]; then
|
||||
if [ "$mode" = life-reservations ]; then
|
||||
phase life-reservations
|
||||
if [ "$#" = 2 ]; then "$2" --life-reservations-only; fi
|
||||
psql -X -v ON_ERROR_STOP=1 -f "$root/tools/ci/life-reservations-check.sql" > /dev/null
|
||||
bash "$root/tools/ci/life-reservations-concurrency.sh"
|
||||
@@ -35,19 +68,35 @@ export EVO_SOURCE_ROOT="$root"
|
||||
export EVO_SERVER_BIN EVO_WORLD_CLONE_BIN
|
||||
EVO_SERVER_BIN="$(dirname "${1:?path to evo-postgres-tests}")/evo-server"
|
||||
EVO_WORLD_CLONE_BIN="$(dirname "$1")/evo-world-clone"
|
||||
phase cxx-suite
|
||||
if ! "$1"; then
|
||||
echo 'PostgreSQL integration tests failed; recent disposable-cluster server log:' >&2
|
||||
tail -n 80 "$test_dir/server.log" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
psql -X -v ON_ERROR_STOP=1 -f "$root/tools/ci/life-reservations-check.sql" > /dev/null
|
||||
bash "$root/tools/ci/life-reservations-concurrency.sh"
|
||||
if [ "$mode" = suite-only ]; then
|
||||
echo "postgres-check[suite-only]: $1 passed; process and SQL checks run in --full"
|
||||
exit 0
|
||||
fi
|
||||
if [ "$mode" = full ]; then
|
||||
phase life-reservations
|
||||
psql -X -v ON_ERROR_STOP=1 -f "$root/tools/ci/life-reservations-check.sql" > /dev/null
|
||||
bash "$root/tools/ci/life-reservations-concurrency.sh"
|
||||
fi
|
||||
phase backup-check
|
||||
# M8.7: online backup under load, restore on a separate cluster, isolated rebuild.
|
||||
bash "$root/tools/ci/backup-check.sh"
|
||||
phase dr-check
|
||||
# M8.8: host-loss rehearsal: in-place restore, fenced old environment, measured rewind.
|
||||
bash "$root/tools/ci/dr-check.sh"
|
||||
phase upgrade-check
|
||||
# M8.9: failed migration under load and the version barrier with real hosts.
|
||||
bash "$root/tools/ci/upgrade-check.sh"
|
||||
if [ "$mode" = sanitizer ]; then
|
||||
echo "postgres-check[sanitizer]: suite and process checks passed; SQL-only checks run in --full"
|
||||
exit 0
|
||||
fi
|
||||
phase migration-history
|
||||
v1_sha="$(sha256sum "$root/migrations/0001_region_bootstrap.sql")"
|
||||
v1_sha="${v1_sha%% *}"
|
||||
# Migration failure must not publish a partial schema.
|
||||
@@ -151,6 +200,7 @@ PGDATABASE=v6_populated bash "$root/tools/migrate-postgres.sh"
|
||||
test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33
|
||||
test "$(PGDATABASE=v6_populated psql -XAtc "SELECT count(*) FROM evolution.region_commits c, LATERAL evolution.parse_checkpoint_manifest(c.manifest) m WHERE m.commit_sequence = 0 AND m.predecessor_commit IS NULL")" = 1
|
||||
test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(conname, ',') FROM pg_constraint WHERE conrelid = 'evolution.region_heads'::regclass AND contype = 'c' AND pg_get_constraintdef(oid) LIKE '%end_tick%'")" = region_heads_published_tick
|
||||
phase permission-audit
|
||||
# Inherited privileges bypass fencing, so every migrate run audits effective rights.
|
||||
psql -X -v ON_ERROR_STOP=1 -c 'CREATE ROLE evo_bad_runtime NOLOGIN IN ROLE pg_write_all_data'
|
||||
if (export PGDATABASE=migration_rollback EVO_PG_RUNTIME_ROLE=evo_bad_runtime; \
|
||||
@@ -198,6 +248,7 @@ expect_audit_failure 'GRANT SELECT ON evolution.cellular_mutation_action_publica
|
||||
expect_audit_failure 'GRANT SELECT ON evolution.account_controls TO evo_runtime_test' 'REVOKE SELECT ON evolution.account_controls FROM evo_runtime_test'
|
||||
expect_audit_failure 'REVOKE EXECUTE ON FUNCTION evolution.claim_account_control(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, evolution.u64, evolution.u64, integer) FROM evo_runtime_test' 'GRANT EXECUTE ON FUNCTION evolution.claim_account_control(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, evolution.u64, evolution.u64, integer) TO evo_runtime_test'
|
||||
bash "$root/tools/migrate-postgres.sh" # Restored configuration passes the audit again.
|
||||
phase checksum-drift
|
||||
# Unknown future versions and checksum drift must not be repaired silently.
|
||||
psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=34 WHERE version=33'
|
||||
if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi
|
||||
|
||||
@@ -15,6 +15,6 @@ if [ ! -d postgresql-18.6 ]; then tar -xjf "$archive"; fi
|
||||
mkdir -p build
|
||||
cd build
|
||||
../postgresql-18.6/configure --prefix="$cache/install" --without-readline --without-icu --with-ssl=openssl
|
||||
make -j2
|
||||
make -j"${PG_BUILD_JOBS:-6}"
|
||||
make install
|
||||
"$cache/install/bin/pg_config" --version
|
||||
|
||||
Reference in New Issue
Block a user