ci: cache and narrow foundation builds
This commit is contained in:
+58
-22
@@ -6,11 +6,20 @@ on:
|
||||
pull_request:
|
||||
workflow_dispatch:
|
||||
|
||||
# Do not spend a full build on obsolete commits when pushes arrive quickly.
|
||||
concurrency:
|
||||
group: foundation-${{ gitea.ref || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
code: read
|
||||
|
||||
env:
|
||||
UBUNTU_IMAGE: ${{ vars.UBUNTU_IMAGE || 'ubuntu:24.04' }}
|
||||
HARBOR_PROJECT: ${{ vars.HARBOR_PROJECT || 'aeternalis' }}
|
||||
HARBOR_USERNAME: ${{ secrets.HARBOR_USERNAME }}
|
||||
HARBOR_PASSWORD: ${{ secrets.HARBOR_PASSWORD }}
|
||||
CACHE_WRITE: '0'
|
||||
|
||||
jobs:
|
||||
checks:
|
||||
@@ -30,11 +39,16 @@ jobs:
|
||||
CHECK_SUITE: ${{ matrix.suite }}
|
||||
run: |
|
||||
set -eu
|
||||
if ! bash tools/ci/ci-paths.sh "$CHECK_SUITE"; then
|
||||
echo "No files relevant to $CHECK_SUITE changed; gate skipped."
|
||||
exit 0
|
||||
fi
|
||||
docker version
|
||||
ci_image="aevolution-ci:${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-$CHECK_SUITE"
|
||||
docker build --progress=plain --file deploy/docker/Dockerfile \
|
||||
--build-arg "UBUNTU_IMAGE=$UBUNTU_IMAGE" \
|
||||
--target "ci-$CHECK_SUITE" --tag "$ci_image" .
|
||||
if [[ "${GITHUB_REF:-}" == refs/heads/main && "${GITHUB_EVENT_NAME:-}" != pull_request && -n "${HARBOR_USERNAME:-}" ]]; then
|
||||
export CACHE_WRITE=1
|
||||
fi
|
||||
bash tools/ci/docker-build.sh "ci-$CHECK_SUITE" "$ci_image" aevolution-ci-cache
|
||||
if [ "$CHECK_SUITE" = tsan ]; then
|
||||
profile="$RUNNER_TEMP/aevolution-tsan-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
|
||||
node tools/ci/tsan-profile.mjs "$profile"
|
||||
@@ -59,21 +73,27 @@ jobs:
|
||||
- name: Build and smoke-test migration tooling
|
||||
run: |
|
||||
set -eu
|
||||
docker build --file deploy/docker/Dockerfile --target postgres-tools \
|
||||
--build-arg "UBUNTU_IMAGE=$UBUNTU_IMAGE" \
|
||||
--tag "aevolution-postgres-tools:${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" .
|
||||
if ! bash tools/ci/ci-paths.sh postgres-tools; then
|
||||
echo 'No migration-tool files changed; gate skipped.'
|
||||
exit 0
|
||||
fi
|
||||
image="aevolution-postgres-tools:${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
if [[ "${GITHUB_REF:-}" == refs/heads/main && "${GITHUB_EVENT_NAME:-}" != pull_request && -n "${HARBOR_USERNAME:-}" ]]; then
|
||||
export CACHE_WRITE=1
|
||||
fi
|
||||
bash tools/ci/docker-build.sh postgres-tools "$image" aevolution-postgres-cache
|
||||
docker run --rm --network none --read-only --cap-drop ALL \
|
||||
--entrypoint /opt/postgres/bin/psql \
|
||||
"aevolution-postgres-tools:${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" --version
|
||||
--entrypoint /opt/postgres/bin/psql "$image" --version
|
||||
- name: Publish configured migration tools
|
||||
if: ${{ gitea.ref == 'refs/heads/main' && gitea.event_name != 'pull_request' && vars.HARBOR_POSTGRES_REPOSITORY != '' }}
|
||||
env:
|
||||
HARBOR_PROJECT: ${{ vars.HARBOR_PROJECT }}
|
||||
HARBOR_REPOSITORY: ${{ vars.HARBOR_POSTGRES_REPOSITORY }}
|
||||
HARBOR_USERNAME: ${{ secrets.HARBOR_USERNAME }}
|
||||
HARBOR_PASSWORD: ${{ secrets.HARBOR_PASSWORD }}
|
||||
run: |
|
||||
export SOURCE_IMAGE="aevolution-postgres-tools:${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
if ! docker image inspect "$SOURCE_IMAGE" >/dev/null 2>&1; then
|
||||
echo 'Migration-tools build was skipped; nothing to publish.'
|
||||
exit 0
|
||||
fi
|
||||
bash tools/ci/publish-image.sh
|
||||
|
||||
server-image:
|
||||
@@ -88,18 +108,26 @@ jobs:
|
||||
- name: Build and smoke-test the region server image
|
||||
run: |
|
||||
set -eu
|
||||
if ! bash tools/ci/ci-paths.sh server-image; then
|
||||
echo 'No server-image files changed; gate skipped.'
|
||||
exit 0
|
||||
fi
|
||||
image="aevolution-server:${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
docker build --file deploy/docker/Dockerfile --target server-runtime --build-arg "UBUNTU_IMAGE=$UBUNTU_IMAGE" --tag "$image" .
|
||||
if [[ "${GITHUB_REF:-}" == refs/heads/main && "${GITHUB_EVENT_NAME:-}" != pull_request && -n "${HARBOR_USERNAME:-}" ]]; then
|
||||
export CACHE_WRITE=1
|
||||
fi
|
||||
bash tools/ci/docker-build.sh server-runtime "$image" aevolution-server-cache
|
||||
bash tools/ci/server-image-smoke.sh "$image"
|
||||
- name: Publish configured region server image
|
||||
if: ${{ gitea.ref == 'refs/heads/main' && gitea.event_name != 'pull_request' && vars.HARBOR_SERVER_REPOSITORY != '' }}
|
||||
env:
|
||||
HARBOR_PROJECT: ${{ vars.HARBOR_PROJECT }}
|
||||
HARBOR_REPOSITORY: ${{ vars.HARBOR_SERVER_REPOSITORY }}
|
||||
HARBOR_USERNAME: ${{ secrets.HARBOR_USERNAME }}
|
||||
HARBOR_PASSWORD: ${{ secrets.HARBOR_PASSWORD }}
|
||||
run: |
|
||||
export SOURCE_IMAGE="aevolution-server:${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
if ! docker image inspect "$SOURCE_IMAGE" >/dev/null 2>&1; then
|
||||
echo 'Server-image build was skipped; nothing to publish.'
|
||||
exit 0
|
||||
fi
|
||||
bash tools/ci/publish-image.sh
|
||||
|
||||
image:
|
||||
@@ -114,23 +142,31 @@ jobs:
|
||||
- name: Build and smoke-test runtime
|
||||
run: |
|
||||
set -eu
|
||||
if ! bash tools/ci/ci-paths.sh image; then
|
||||
echo 'No runtime-image files changed; gate skipped.'
|
||||
exit 0
|
||||
fi
|
||||
image="aevolution-proof:${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
docker build --progress=plain --file deploy/docker/Dockerfile \
|
||||
if [[ "${GITHUB_REF:-}" == refs/heads/main && "${GITHUB_EVENT_NAME:-}" != pull_request && -n "${HARBOR_USERNAME:-}" ]]; then
|
||||
export CACHE_WRITE=1
|
||||
fi
|
||||
bash tools/ci/docker-build.sh runtime "$image" aevolution-image-cache \
|
||||
--build-arg "UBUNTU_IMAGE=$UBUNTU_IMAGE" \
|
||||
--label "org.opencontainers.image.revision=$GITHUB_SHA" \
|
||||
--label "org.opencontainers.image.source=https://git.aeternalisgames.com/dingo/AEvolution" \
|
||||
--tag "$image" .
|
||||
--label "org.opencontainers.image.source=https://git.aeternalisgames.com/dingo/AEvolution"
|
||||
docker run --rm --network none --read-only --cap-drop ALL \
|
||||
--security-opt no-new-privileges "$image" \
|
||||
--seed 123456 --entities 8 --ticks 1000 --verify-replay
|
||||
- name: Publish configured main build to Harbor
|
||||
if: ${{ gitea.ref == 'refs/heads/main' && gitea.event_name != 'pull_request' && vars.HARBOR_PROJECT != '' }}
|
||||
env:
|
||||
HARBOR_PROJECT: ${{ vars.HARBOR_PROJECT }}
|
||||
HARBOR_REPOSITORY: ${{ vars.HARBOR_REPOSITORY || 'aevolution' }}
|
||||
HARBOR_USERNAME: ${{ secrets.HARBOR_USERNAME }}
|
||||
HARBOR_PASSWORD: ${{ secrets.HARBOR_PASSWORD }}
|
||||
run: bash tools/ci/publish-image.sh
|
||||
run: |
|
||||
if ! docker image inspect "aevolution-proof:${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >/dev/null 2>&1; then
|
||||
echo 'Runtime-image build was skipped; nothing to publish.'
|
||||
exit 0
|
||||
fi
|
||||
bash tools/ci/publish-image.sh
|
||||
- name: Explain unconfigured publishing
|
||||
if: ${{ vars.HARBOR_PROJECT == '' }}
|
||||
run: echo 'Image build/smoke only. Set HARBOR_PROJECT and Harbor robot secrets to enable main publishing. See deploy/README.md.'
|
||||
|
||||
+32
-16
@@ -6,6 +6,16 @@ RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends ca-certificates cmake ninja-build g++-14 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
WORKDIR /source
|
||||
|
||||
FROM toolchain AS ci-toolchain
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends clang-19 clang-format-19 \
|
||||
clang-tidy-19 libclang-rt-19-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
RUN dpkg-query -W
|
||||
|
||||
# Keep expensive compiler/package layers independent from source changes.
|
||||
FROM ci-toolchain AS ci-source
|
||||
COPY CMakeLists.txt CMakePresets.json .clang-format .clang-tidy ./
|
||||
COPY cmake/ cmake/
|
||||
COPY include/ include/
|
||||
@@ -15,22 +25,15 @@ COPY tests/ tests/
|
||||
COPY experiments/ experiments/
|
||||
COPY tools/ tools/
|
||||
|
||||
FROM toolchain AS ci-toolchain
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends clang-19 clang-format-19 \
|
||||
clang-tidy-19 libclang-rt-19-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
RUN dpkg-query -W
|
||||
|
||||
FROM ci-toolchain AS ci-checks
|
||||
FROM ci-source AS ci-checks
|
||||
RUN bash tools/ci/linux-check.sh checks
|
||||
|
||||
FROM ci-toolchain AS ci-tsan
|
||||
FROM ci-source AS ci-tsan
|
||||
RUN cmake --preset tsan -DCMAKE_CXX_COMPILER=clang++-19 \
|
||||
&& cmake --build --preset tsan --parallel 2
|
||||
CMD ["ctest", "--preset", "tsan", "--no-tests=error", "--timeout", "120"]
|
||||
|
||||
FROM ci-toolchain AS ci-ecs
|
||||
FROM ci-source AS ci-ecs
|
||||
# Explicit experiment preparation; normal configure/runtime remains dependency-free.
|
||||
RUN cmake -P tools/prepare-entt.cmake \
|
||||
&& cmake --preset release -B build/ecs -DCMAKE_CXX_COMPILER=g++-14 -DEVO_BUILD_ECS_EXPERIMENT=ON \
|
||||
@@ -63,13 +66,26 @@ COPY --from=quay.io/prometheus/alertmanager@sha256:e9733bafb1bdef9b00e25a21f8f99
|
||||
COPY tools/ci/monitoring-check.sh tools/ci/
|
||||
RUN bash tools/ci/monitoring-check.sh
|
||||
|
||||
FROM ci-toolchain AS postgres-build
|
||||
FROM ci-toolchain AS postgres-toolchain
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
curl bzip2 make flex bison perl libssl-dev zlib1g-dev pkg-config \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
COPY migrations/ migrations/
|
||||
# The toolchain installs versioned compilers only; there is no bare cc/gcc.
|
||||
COPY tools/prepare-postgres.sh tools/prepare-postgres.sh
|
||||
RUN CC=gcc-14 bash tools/prepare-postgres.sh
|
||||
|
||||
# PostgreSQL itself is prepared once. Source/test changes invalidate only the
|
||||
# layers below, while postgres-tools can reuse this image without compiling the
|
||||
# full C++ test matrix.
|
||||
FROM postgres-toolchain AS postgres-build
|
||||
COPY CMakeLists.txt CMakePresets.json .clang-format .clang-tidy ./
|
||||
COPY cmake/ cmake/
|
||||
COPY include/ include/
|
||||
COPY src/ src/
|
||||
COPY apps/ apps/
|
||||
COPY tests/ tests/
|
||||
COPY tools/ tools/
|
||||
COPY migrations/ migrations/
|
||||
RUN cmake --preset release -B build/postgres -DCMAKE_CXX_COMPILER=g++-14 \
|
||||
-DEVO_BUILD_POSTGRES=ON -DPostgreSQL_ROOT=/source/.cache/postgres/install \
|
||||
&& cmake --build build/postgres --target evo-postgres-tests --parallel 2
|
||||
@@ -87,13 +103,13 @@ RUN bash tools/ci/postgres-check.sh /source/build/postgres/evo-postgres-tests \
|
||||
&& bash tools/ci/postgres-check.sh /source/build/postgres-analysis/evo-postgres-tests \
|
||||
&& bash tools/ci/postgres-check.sh /source/build/postgres-asan/evo-postgres-tests
|
||||
|
||||
FROM ${UBUNTU_IMAGE} AS postgres-tools
|
||||
FROM postgres-toolchain AS postgres-tools
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends libssl3t64 zlib1g ca-certificates \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& groupadd --gid 10001 evo && useradd --uid 10001 --gid 10001 --no-create-home evo
|
||||
COPY --from=postgres-build /source/.cache/postgres/install/bin/psql /opt/postgres/bin/psql
|
||||
COPY --from=postgres-build /source/.cache/postgres/install/lib/libpq.so* /opt/postgres/lib/
|
||||
COPY --from=postgres-build /source/.cache/postgres/postgresql-18.6/COPYRIGHT /usr/share/doc/evo-postgres/COPYRIGHT
|
||||
COPY --from=postgres-toolchain /source/.cache/postgres/install/bin/psql /opt/postgres/bin/psql
|
||||
COPY --from=postgres-toolchain /source/.cache/postgres/install/lib/libpq.so* /opt/postgres/lib/
|
||||
COPY --from=postgres-toolchain /source/.cache/postgres/postgresql-18.6/COPYRIGHT /usr/share/doc/evo-postgres/COPYRIGHT
|
||||
COPY tools/migrate-postgres.sh /opt/evo/tools/migrate-postgres.sh
|
||||
COPY migrations/ /opt/evo/migrations/
|
||||
ENV PATH="/opt/postgres/bin:${PATH}" LD_LIBRARY_PATH=/opt/postgres/lib
|
||||
|
||||
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
suite=${1:?suite required}
|
||||
base=${GITHUB_EVENT_BEFORE:-}
|
||||
if [[ -z "$base" || "$base" == 0000000000000000000000000000000000000000 ]]; then
|
||||
base=HEAD^
|
||||
fi
|
||||
|
||||
# A single-parent diff is sufficient for push and the merge commit generated for
|
||||
# pull requests. If history is unavailable, fail open and run the gate.
|
||||
if ! files=$(git diff --name-only "$base" HEAD 2>/dev/null); then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
case "$suite" in
|
||||
checks|tsan|ecs)
|
||||
grep -Eq '^(CMakeLists.txt|CMakePresets.txt|CMakePresets.json|cmake/|include/|src/|apps/|tests/|experiments/|tools/ci/linux-check.sh|tools/ci/tsan-profile.mjs|deploy/docker/Dockerfile|\.gitea/workflows/)' <<<"$files"
|
||||
;;
|
||||
postgres|postgres-tools)
|
||||
grep -Eq '^(CMakeLists.txt|CMakePresets.json|cmake/|include/|src/|apps/|tests/|migrations/|tools/prepare-postgres.sh|tools/ci/postgres-check.sh|tools/migrate-postgres.sh|deploy/docker/Dockerfile|\.gitea/workflows/)' <<<"$files"
|
||||
;;
|
||||
deploy)
|
||||
grep -Eq '^(deploy/|tools/ci/deploy-policy-check.py|\.gitea/workflows/)' <<<"$files"
|
||||
;;
|
||||
server-image)
|
||||
grep -Eq '^(CMakeLists.txt|CMakePresets.json|cmake/|include/|src/|apps/|migrations/|tools/prepare-postgres.sh|tools/ci/server-image-smoke.sh|deploy/docker/Dockerfile|\.gitea/workflows/)' <<<"$files"
|
||||
;;
|
||||
image)
|
||||
grep -Eq '^(CMakeLists.txt|CMakePresets.json|cmake/|include/|src/|apps/|tests/|deploy/docker/Dockerfile|\.gitea/workflows/)' <<<"$files"
|
||||
;;
|
||||
*)
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
Executable
+51
@@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Build a target with a reusable Harbor-backed BuildKit cache. The cache is
|
||||
# readable on trusted runners and only written by main pushes.
|
||||
: "${1:?target required}"
|
||||
: "${2:?image tag required}"
|
||||
: "${3:?cache repository required}"
|
||||
target=$1
|
||||
image=$2
|
||||
cache_repository=$3
|
||||
shift 3
|
||||
|
||||
registry=${HARBOR_REGISTRY:-registry.aeternalisgames.com}
|
||||
project=${HARBOR_PROJECT:-aeternalis}
|
||||
cache_ref="${registry}/${project}/${cache_repository}:${CACHE_TAG:-main}-${target}"
|
||||
|
||||
builder=${BUILDX_BUILDER:-aevolution}
|
||||
if docker buildx inspect "$builder" >/dev/null 2>&1; then
|
||||
docker buildx use "$builder"
|
||||
else
|
||||
docker buildx create --name "$builder" --driver docker-container --use
|
||||
fi
|
||||
|
||||
# PRs may not receive registry credentials. Main and trusted runs do.
|
||||
if [[ -n "${HARBOR_USERNAME:-}" && -n "${HARBOR_PASSWORD:-}" ]]; then
|
||||
export DOCKER_CONFIG
|
||||
DOCKER_CONFIG="$(mktemp -d "${RUNNER_TEMP:-/tmp}/aevolution-buildx-docker.XXXXXX")"
|
||||
trap 'rm -rf -- "$DOCKER_CONFIG"' EXIT
|
||||
printf '%s' "$HARBOR_PASSWORD" | docker login "$registry" \
|
||||
--username "$HARBOR_USERNAME" --password-stdin >/dev/null
|
||||
cache_from=(--cache-from "type=registry,ref=${cache_ref}")
|
||||
else
|
||||
cache_from=()
|
||||
fi
|
||||
|
||||
cache_to=()
|
||||
if [[ "${CACHE_WRITE:-0}" == 1 ]]; then
|
||||
cache_to=(--cache-to "type=registry,ref=${cache_ref},mode=max")
|
||||
fi
|
||||
|
||||
docker buildx build \
|
||||
--progress=plain \
|
||||
--file deploy/docker/Dockerfile \
|
||||
--target "$target" \
|
||||
--tag "$image" \
|
||||
--build-arg "UBUNTU_IMAGE=${UBUNTU_IMAGE:-ubuntu:24.04}" \
|
||||
--load \
|
||||
"${cache_from[@]}" \
|
||||
"${cache_to[@]}" \
|
||||
"$@" .
|
||||
Reference in New Issue
Block a user