CI: repair sanitizer fixtures and client execution gates
Foundation / checks (ecs) (push) Successful in 2m33s
Foundation / checks (deploy) (push) Successful in 4m36s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (tsan) (push) Canceled after 8m8s
Foundation / checks (checks) (push) Canceled after 10m47s
Foundation / checks (client) (push) Canceled after 10m49s
Foundation / checks (network) (push) Canceled after 10m48s
Foundation / checks (postgres) (push) Canceled after 10m49s

This commit is contained in:
dingo
2026-10-17 22:22:34 -04:00
parent 89f29fef3a
commit 1e23dc3799
12 changed files with 162 additions and 8 deletions
+4 -1
View File
@@ -29,7 +29,9 @@ env:
jobs:
checks:
runs-on: ${{ vars.CI_RUNNER_LABEL || 'ubuntu-22.04' }}
timeout-minutes: 60
# Captured cold/shared-runner checks reached 59 minutes before the ASan
# cohort timeout, with the remaining fuzz/benchmark steps still pending.
timeout-minutes: 90
strategy:
fail-fast: false
matrix:
@@ -70,6 +72,7 @@ jobs:
profile="$RUNNER_TEMP/aevolution-tsan-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
node tools/ci/tsan-profile.mjs "$profile"
docker run --rm --network none --read-only --user 10001:10001 \
--env EVO_TEST_TMPDIR=/tmp \
--cap-drop ALL --security-opt no-new-privileges \
--security-opt "seccomp=$profile" \
--tmpfs /source/build/tsan/Testing:rw,noexec,nosuid,size=16m,mode=1777 \
+13 -2
View File
@@ -663,8 +663,19 @@ add_executable(evo-cellular-cohorts experiments/resolution/cellular_cohorts.cpp)
target_link_libraries(evo-cellular-cohorts PRIVATE evo::lifecycle)
evo_owned_target(evo-cellular-cohorts)
if(BUILD_TESTING)
# Sanitizer instrumentation and shared CI runner contention are not product
# performance gates. Keep the full matrix, with bounded instrumented budgets.
set(evo_cohort_timeout 180)
set(evo_migration_timeout 180)
if(EVO_SANITIZER STREQUAL "address")
set(evo_cohort_timeout 600)
set(evo_migration_timeout 360)
elseif(EVO_SANITIZER STREQUAL "thread")
set(evo_cohort_timeout 900)
set(evo_migration_timeout 600)
endif()
add_test(NAME cellular-cohort-qualification COMMAND evo-cellular-cohorts --verify)
set_tests_properties(cellular-cohort-qualification PROPERTIES TIMEOUT 180)
set_tests_properties(cellular-cohort-qualification PROPERTIES TIMEOUT ${evo_cohort_timeout})
endif()
if(BUILD_TESTING)
@@ -686,7 +697,7 @@ if(BUILD_TESTING)
target_link_libraries(evo-cellular-migration-tests PRIVATE evo::lifecycle)
evo_owned_target(evo-cellular-migration-tests)
add_test(NAME cellular-migration-contract COMMAND evo-cellular-migration-tests)
set_tests_properties(cellular-migration-contract PROPERTIES TIMEOUT 180)
set_tests_properties(cellular-migration-contract PROPERTIES TIMEOUT ${evo_migration_timeout})
add_test(NAME cellular-migration-process-restart COMMAND "${CMAKE_COMMAND}"
"-DMIGRATION_TEST=$<TARGET_FILE:evo-cellular-migration-tests>"
"-DTEST_DIR=${CMAKE_CURRENT_BINARY_DIR}/migration-restart"
+4 -2
View File
@@ -39,6 +39,8 @@ FROM ci-source AS ci-client
RUN apt-get update && apt-get install -y --no-install-recommends \
libx11-dev libxext-dev libxrandr-dev libxcursor-dev libxi-dev libxss-dev libxtst-dev xvfb xauth \
&& rm -rf /var/lib/apt/lists/*
# Lifecycle tests deliberately quit/reopen SDL: retain the virtual X server with
# -noreset through those cycles, matching a persistent desktop display server.
RUN cmake -P tools/prepare-sdl3.cmake \
&& cmake -P tools/prepare-imgui.cmake \
&& cmake --preset client-probe -DCMAKE_CXX_COMPILER=clang++-19 -DCMAKE_C_COMPILER=clang-19 \
@@ -47,12 +49,12 @@ RUN cmake -P tools/prepare-sdl3.cmake \
&& ctest --preset client-probe --no-tests=error \
&& xvfb-run -a env SDL_VIDEO_DRIVER=x11 ./build/client-probe/evo-client-probe --renderer software \
&& for cycle in 1 2 3; do xvfb-run -a env SDL_VIDEO_DRIVER=x11 ./build/client-probe/evo-ui-probe --renderer software --cycles 1; done \
&& xvfb-run -a env SDL_VIDEO_DRIVER=x11 ./build/client-probe/evo-client-lifecycle-tests \
&& xvfb-run -a -s "-screen 0 1280x1024x24 -noreset" env SDL_VIDEO_DRIVER=x11 ./build/client-probe/evo-client-lifecycle-tests \
&& cmake --preset client-probe -DEVO_BUILD_CLIENT=ON -DEVO_SANITIZER=address -DEVO_ENABLE_TIDY=OFF \
&& cmake --build build/client-probe --target evo-client-state-tests evo-ui-probe evo-client evo-client-lifecycle-tests evo-client-environment-tests evo-client-organism-tests evo-client-ui-model-tests evo-client-panel-tests evo-client-bindings-tests evo-client-auth-tests --parallel 2 \
&& ctest --preset client-probe --no-tests=error \
&& for cycle in 1 2 3; do xvfb-run -a env SDL_VIDEO_DRIVER=x11 ./build/client-probe/evo-ui-probe --renderer software --cycles 1; done \
&& xvfb-run -a env SDL_VIDEO_DRIVER=x11 ./build/client-probe/evo-client-lifecycle-tests
&& xvfb-run -a -s "-screen 0 1280x1024x24 -noreset" env SDL_VIDEO_DRIVER=x11 ./build/client-probe/evo-client-lifecycle-tests
FROM ci-source AS ci-network
# Explicit pinned Mbed TLS preparation (ADR-0082); codec, admission, dev grants and
+13
View File
@@ -1,5 +1,18 @@
# Project status and next steps
## M12 CI execution repair - 2026-10-17
The captured sanitizer timeout and read-only fixture failures now have scoped
test-runner fixes: full instrumented matrices keep their cases with bounded
budgets, and restart fixtures use the TSan container's existing `/tmp` tmpfs.
Read-only root, non-root user, network isolation and seccomp policy remain intact.
Real-X11 lifecycle checks keep a persistent Xvfb server and report the typed open
error; the original intermittent reopen failure was not reproduced locally.
GCC warning fixes preserve consume-once authentication and panel-test behavior.
[Evidence and limitations](engineering/M12_CI_EXECUTION_REPAIR.md) distinguish
local qualification from the remote CI/container rerun still required. M12's
remaining integration/biology gates are unchanged; no deployment occurred.
## M12.4e continued population replay after transfer - 2026-10-17
Portable population batch 2 binds envelope 2/owner 7 and preserves boundary flows
@@ -0,0 +1,46 @@
# M12 CI execution repair
The maintainer supplied `Build_Errors.txt`, retained as local evidence rather than
committed build output. Its repeated checks/TSan logs show the unchanged full
cohort verification exceeding 180 seconds. The read-only TSan container also
rejects the new population/migration fixture directories under `/source/build`.
The client gate passes its 14 CTests, then intermittently fails while opening the
rebinding test's SDL application under a real Xvfb display.
## Changes
- Keep every cohort case, seed, capacity sample and sanitizer enabled. Its CTest
budget is 180/600/900 seconds for normal/ASan/TSan. The larger migration contract
gets 180/360/600 seconds. These are execution bounds, not performance acceptance.
- Population/migration restart scripts accept runtime `EVO_TEST_TMPDIR`, using a
distinct fixture directory for each invocation. The TSan container sets it to
the existing bounded `/tmp` tmpfs. Source/build mounts stay read-only; user,
capabilities, network isolation and the pinned seccomp policy are unchanged.
- Allow 90 minutes for the checks matrix job: the captured cold/shared-runner run
reached 59 minutes before the timeout, before remaining fuzz/benchmark steps.
- Xvfb lifecycle invocations use `-noreset` to retain their display server across
deliberate SDL quit/reopen cycles. If opening still fails, the test now records
the typed Application error category instead of only an uninformative line.
The server-reset explanation is an inference: the original reopen failure was
not reproduced in twenty local runs of the old invocation. CI must confirm it.
- Full GCC client builds exposed two additional warnings-as-errors: extraction
through `std::exchange(optional, {})` and a copied panel-fixture range value.
The grant is now checked/moved/reset under the same mutex, preserving its
consume-once public contract; the fixture uses a const reference. No warning
suppression, authentication policy, protocol, UI appearance or biology changes.
## Evidence and limits
Executed compiler/test results are in [VALIDATION](VALIDATION.md). Local Linux
TSan runs the full cohort matrix and both process-restart contracts; ASan runs
the full cohort verification. Protected configured output fails without the
runtime override and both restart scripts pass using `/tmp` with that output
still inaccessible. CTest metadata confirms the normal and instrumented budgets.
GCC/MSVC client contracts, repeated real-X11 lifecycle runs and sanitizer checks
exercise the affected client paths without an external identity provider.
The local Docker executable cannot reach its Linux engine. No full Docker image,
actual read-only/seccomp container, remote CI success or deployment is claimed.
The captured client reopen failure remains unconfirmed locally; the next CI run
has an explicit display lifetime and an informative failure category.
World0, live regions and the biological/persistence contracts were not changed.
+14
View File
@@ -33,3 +33,17 @@ needs error envelopes and ensemble comparisons in addition to exact fixtures.
Unexecuted configurations are not validation. Record exact tools, commands,
results and limitations in VALIDATION.md at the end of each foundation change.
The cohort qualification runs its complete matrix under sanitizers; its wall
timeout is a bounded CI execution guard, not a portable performance claim.
Budgets are 180 s normally, 600 s with ASan/UBSan and 900 s with TSan. The larger
migration contract gets 180/360/600 s respectively. Cases, seeds, checks and
instrumentation remain identical. The checks job allows 90 minutes for the full
cold/shared-runner build, instrumented suites and remaining fuzz/benchmark steps.
Population and migration restart scripts honor the runtime `EVO_TEST_TMPDIR`
override and create distinct random fixture subdirectories there. Without the
override they retain the configured build-directory output paths. The non-root,
networkless, read-only TSan container sets the override to its existing bounded
`/tmp` tmpfs; no source/build mount is made writable and no sandbox permission is
broadened. Fixture path randomness has no biological or test-result meaning.
+50
View File
@@ -1,5 +1,55 @@
# Validation report
## 2026-10-17 - captured M12 CI execution failures
Reviewed the maintainer's local `Build_Errors.txt`. Repeated checks/TSan runs hit
the 180-second cohort limit; the read-only TSan container rejects both new
restart fixture paths. The client gate passes 14 CTests before an intermittent
real-X11 application reopen failure. [Repair scope](M12_CI_EXECUTION_REPAIR.md)
keeps these environmental observations separate from biological qualification.
Changes preserve the complete test matrices and instrumentation, set bounded
sanitizer-specific timeouts, and route restart fixtures to the existing `/tmp`
tmpfs via runtime `EVO_TEST_TMPDIR`. The job gets 90 minutes for its full cold/
shared-runner sequence. Xvfb retains its display server across SDL reopen cycles;
the failing open now reports its typed error category. Two additional GCC client
warnings are fixed through checked grant extraction and a non-copying fixture
loop. No warning suppression, public API, authentication policy, biology, version,
live state, dependency or deployment changed.
Executed locally:
- Native MSVC Release **78/78 passed**, 13.97 s.
- Linux Clang 19 **TSan 4/4 passed**, 134.24 s: full cohort qualification (77.27 s),
population restart, migration contract and migration restart; no reports.
- Linux Clang 19 **ASan/UBSan full cohort qualification passed**, 63.58 s;
no case/seed/capacity sample was removed.
- Protected configured output fails without the override; both restart scripts
pass with output still protected and their fixture root redirected to `/tmp`.
Effective CTest budgets were inspected for normal, ASan and TSan configurations.
- Workflow YAML parses; the existing offline CI path-selector regression and
final repository format gate pass. Explicit clang-tidy 19 on affected client
authentication/lifecycle/panel test sources passes, with no owned warnings.
- GCC 14 Release client subset **12/12 passed**, 16.54 s; final stable-reference
grant extraction additionally passes its authentication test (0.11 s).
- Twenty real-Xvfb lifecycle invocations pass using the old invocation, and
twenty pass with `-noreset`. A real-Xvfb lifecycle invocation also passes with
ASan/UBSan; final sanitizer authentication check passes (0.19 s), no reports.
- Native client-package suite **18/18 passed**, 127.59 s. After the final guarded
extraction/panel-fixture edits, the affected native auth/panels/lifecycle subset
is rebuilt and passes **3/3**, 19.80 s. Tests remain active under NDEBUG.
The first broad local client invocation could not run five unbuilt targets;
building the full selected target set exposed and then resolved the GCC warnings.
Final pass counts above refer to the rebuilt targets. No golden was replaced.
Limits: the local Docker executable cannot connect to its Linux engine, so the
full image/read-only/seccomp container and remote CI have not run locally. The
captured SDL reopen failure was not reproduced in twenty baseline runs; server
reset is an inference, and CI must confirm the explicit display-lifetime change.
All failures still block the pipeline. `Build_Errors.txt` and unrelated local work
are preserved outside the commit. World0 and live regions were not touched.
## 2026-10-17 - M12.4e versioned population continuation
Portable population batch 2 is bound to envelope 2/owner 7. Batch 1 keeps its
+6 -1
View File
@@ -217,7 +217,12 @@ AuthStatus AuthClient::status() const {
}
std::optional<AuthGrant> AuthClient::take_grant() {
std::lock_guard lock{impl_->mutex};
return std::exchange(impl_->grant, {});
auto& pending = impl_->grant;
if (!pending.has_value())
return {};
auto grant = std::move(*pending);
pending.reset();
return std::optional<AuthGrant>{std::move(grant)};
}
#ifndef _WIN32
std::unique_ptr<AuthPlatform> native_auth_platform(std::string_view) {
+4
View File
@@ -1,3 +1,7 @@
if(DEFINED ENV{EVO_TEST_TMPDIR} AND NOT "$ENV{EVO_TEST_TMPDIR}" STREQUAL "")
string(RANDOM LENGTH 16 ALPHABET 0123456789abcdef run_id)
set(TEST_DIR "$ENV{EVO_TEST_TMPDIR}/migration-restart-${run_id}")
endif()
file(MAKE_DIRECTORY "${TEST_DIR}")
set(prefix "${TEST_DIR}/pair")
execute_process(COMMAND "${MIGRATION_TEST}" --record "${prefix}" RESULT_VARIABLE recorded)
+4
View File
@@ -1,3 +1,7 @@
if(DEFINED ENV{EVO_TEST_TMPDIR} AND NOT "$ENV{EVO_TEST_TMPDIR}" STREQUAL "")
string(RANDOM LENGTH 16 ALPHABET 0123456789abcdef run_id)
set(TEST_DIR "$ENV{EVO_TEST_TMPDIR}/population-restart-${run_id}")
endif()
file(MAKE_DIRECTORY "${TEST_DIR}")
execute_process(COMMAND "${POPULATION_TEST}" --checkpoint "${TEST_DIR}/checkpoint.pop"
"${TEST_DIR}/expected.pop" RESULT_VARIABLE prepared)
+3 -1
View File
@@ -469,7 +469,9 @@ void toolkit_capture() {
void rebinding() {
const auto frame = fixture();
auto result = Application::open({320, 240, false, "software"});
CHECK(result);
if (!result)
throw std::runtime_error("rebinding Application::open error " +
std::to_string(static_cast<unsigned>(result.error())));
auto app = std::move(result.value());
CHECK(app->poll_events());
const auto id = SDL_GetWindowID(window());
+1 -1
View File
@@ -357,7 +357,7 @@ void apply_save_restart() {
std::filesystem::remove_all(dir);
}
void cellular_panels(const std::filesystem::path& review) {
for (const auto size : {std::pair{960, 640}, std::pair{1280, 800}, std::pair{640, 480}}) {
for (const auto& size : {std::pair{960, 640}, std::pair{1280, 800}, std::pair{640, 480}}) {
const auto [width, height] = size;
auto opened = Application::open({width, height, false, "software"});
CHECK(opened);