From 1e23dc3799288e3b1117bb17817ca1d5144205c9 Mon Sep 17 00:00:00 2001 From: dingo Date: Sat, 17 Oct 2026 22:22:34 -0400 Subject: [PATCH] CI: repair sanitizer fixtures and client execution gates --- .gitea/workflows/build.yaml | 5 ++- CMakeLists.txt | 15 ++++++- deploy/docker/Dockerfile | 6 ++- docs/STATUS.md | 13 ++++++ docs/engineering/M12_CI_EXECUTION_REPAIR.md | 46 +++++++++++++++++++ docs/engineering/TEST_STRATEGY.md | 14 ++++++ docs/engineering/VALIDATION.md | 50 +++++++++++++++++++++ src/client/auth/client.cpp | 7 ++- tests/cellular_migration_restart.cmake | 4 ++ tests/cellular_population_restart.cmake | 4 ++ tests/client_lifecycle.cpp | 4 +- tests/client_panels.cpp | 2 +- 12 files changed, 162 insertions(+), 8 deletions(-) create mode 100644 docs/engineering/M12_CI_EXECUTION_REPAIR.md diff --git a/.gitea/workflows/build.yaml b/.gitea/workflows/build.yaml index c9d7bc1..cf7cd5b 100644 --- a/.gitea/workflows/build.yaml +++ b/.gitea/workflows/build.yaml @@ -29,7 +29,9 @@ env: jobs: checks: runs-on: ${{ vars.CI_RUNNER_LABEL || 'ubuntu-22.04' }} - timeout-minutes: 60 + # Captured cold/shared-runner checks reached 59 minutes before the ASan + # cohort timeout, with the remaining fuzz/benchmark steps still pending. + timeout-minutes: 90 strategy: fail-fast: false matrix: @@ -70,6 +72,7 @@ jobs: profile="$RUNNER_TEMP/aevolution-tsan-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json" node tools/ci/tsan-profile.mjs "$profile" docker run --rm --network none --read-only --user 10001:10001 \ + --env EVO_TEST_TMPDIR=/tmp \ --cap-drop ALL --security-opt no-new-privileges \ --security-opt "seccomp=$profile" \ --tmpfs /source/build/tsan/Testing:rw,noexec,nosuid,size=16m,mode=1777 \ diff --git a/CMakeLists.txt b/CMakeLists.txt index bcf7072..29791fa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -663,8 +663,19 @@ add_executable(evo-cellular-cohorts experiments/resolution/cellular_cohorts.cpp) target_link_libraries(evo-cellular-cohorts PRIVATE evo::lifecycle) evo_owned_target(evo-cellular-cohorts) if(BUILD_TESTING) + # Sanitizer instrumentation and shared CI runner contention are not product + # performance gates. Keep the full matrix, with bounded instrumented budgets. + set(evo_cohort_timeout 180) + set(evo_migration_timeout 180) + if(EVO_SANITIZER STREQUAL "address") + set(evo_cohort_timeout 600) + set(evo_migration_timeout 360) + elseif(EVO_SANITIZER STREQUAL "thread") + set(evo_cohort_timeout 900) + set(evo_migration_timeout 600) + endif() add_test(NAME cellular-cohort-qualification COMMAND evo-cellular-cohorts --verify) - set_tests_properties(cellular-cohort-qualification PROPERTIES TIMEOUT 180) + set_tests_properties(cellular-cohort-qualification PROPERTIES TIMEOUT ${evo_cohort_timeout}) endif() if(BUILD_TESTING) @@ -686,7 +697,7 @@ if(BUILD_TESTING) target_link_libraries(evo-cellular-migration-tests PRIVATE evo::lifecycle) evo_owned_target(evo-cellular-migration-tests) add_test(NAME cellular-migration-contract COMMAND evo-cellular-migration-tests) - set_tests_properties(cellular-migration-contract PROPERTIES TIMEOUT 180) + set_tests_properties(cellular-migration-contract PROPERTIES TIMEOUT ${evo_migration_timeout}) add_test(NAME cellular-migration-process-restart COMMAND "${CMAKE_COMMAND}" "-DMIGRATION_TEST=$" "-DTEST_DIR=${CMAKE_CURRENT_BINARY_DIR}/migration-restart" diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index f39e51f..e9ac2fe 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -39,6 +39,8 @@ FROM ci-source AS ci-client RUN apt-get update && apt-get install -y --no-install-recommends \ libx11-dev libxext-dev libxrandr-dev libxcursor-dev libxi-dev libxss-dev libxtst-dev xvfb xauth \ && rm -rf /var/lib/apt/lists/* +# Lifecycle tests deliberately quit/reopen SDL: retain the virtual X server with +# -noreset through those cycles, matching a persistent desktop display server. RUN cmake -P tools/prepare-sdl3.cmake \ && cmake -P tools/prepare-imgui.cmake \ && cmake --preset client-probe -DCMAKE_CXX_COMPILER=clang++-19 -DCMAKE_C_COMPILER=clang-19 \ @@ -47,12 +49,12 @@ RUN cmake -P tools/prepare-sdl3.cmake \ && ctest --preset client-probe --no-tests=error \ && xvfb-run -a env SDL_VIDEO_DRIVER=x11 ./build/client-probe/evo-client-probe --renderer software \ && for cycle in 1 2 3; do xvfb-run -a env SDL_VIDEO_DRIVER=x11 ./build/client-probe/evo-ui-probe --renderer software --cycles 1; done \ - && xvfb-run -a env SDL_VIDEO_DRIVER=x11 ./build/client-probe/evo-client-lifecycle-tests \ + && xvfb-run -a -s "-screen 0 1280x1024x24 -noreset" env SDL_VIDEO_DRIVER=x11 ./build/client-probe/evo-client-lifecycle-tests \ && cmake --preset client-probe -DEVO_BUILD_CLIENT=ON -DEVO_SANITIZER=address -DEVO_ENABLE_TIDY=OFF \ && cmake --build build/client-probe --target evo-client-state-tests evo-ui-probe evo-client evo-client-lifecycle-tests evo-client-environment-tests evo-client-organism-tests evo-client-ui-model-tests evo-client-panel-tests evo-client-bindings-tests evo-client-auth-tests --parallel 2 \ && ctest --preset client-probe --no-tests=error \ && for cycle in 1 2 3; do xvfb-run -a env SDL_VIDEO_DRIVER=x11 ./build/client-probe/evo-ui-probe --renderer software --cycles 1; done \ - && xvfb-run -a env SDL_VIDEO_DRIVER=x11 ./build/client-probe/evo-client-lifecycle-tests + && xvfb-run -a -s "-screen 0 1280x1024x24 -noreset" env SDL_VIDEO_DRIVER=x11 ./build/client-probe/evo-client-lifecycle-tests FROM ci-source AS ci-network # Explicit pinned Mbed TLS preparation (ADR-0082); codec, admission, dev grants and diff --git a/docs/STATUS.md b/docs/STATUS.md index 73f531a..ce453ee 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -1,5 +1,18 @@ # Project status and next steps +## M12 CI execution repair - 2026-10-17 + +The captured sanitizer timeout and read-only fixture failures now have scoped +test-runner fixes: full instrumented matrices keep their cases with bounded +budgets, and restart fixtures use the TSan container's existing `/tmp` tmpfs. +Read-only root, non-root user, network isolation and seccomp policy remain intact. +Real-X11 lifecycle checks keep a persistent Xvfb server and report the typed open +error; the original intermittent reopen failure was not reproduced locally. +GCC warning fixes preserve consume-once authentication and panel-test behavior. +[Evidence and limitations](engineering/M12_CI_EXECUTION_REPAIR.md) distinguish +local qualification from the remote CI/container rerun still required. M12's +remaining integration/biology gates are unchanged; no deployment occurred. + ## M12.4e continued population replay after transfer - 2026-10-17 Portable population batch 2 binds envelope 2/owner 7 and preserves boundary flows diff --git a/docs/engineering/M12_CI_EXECUTION_REPAIR.md b/docs/engineering/M12_CI_EXECUTION_REPAIR.md new file mode 100644 index 0000000..acf01d3 --- /dev/null +++ b/docs/engineering/M12_CI_EXECUTION_REPAIR.md @@ -0,0 +1,46 @@ +# M12 CI execution repair + +The maintainer supplied `Build_Errors.txt`, retained as local evidence rather than +committed build output. Its repeated checks/TSan logs show the unchanged full +cohort verification exceeding 180 seconds. The read-only TSan container also +rejects the new population/migration fixture directories under `/source/build`. +The client gate passes its 14 CTests, then intermittently fails while opening the +rebinding test's SDL application under a real Xvfb display. + +## Changes + +- Keep every cohort case, seed, capacity sample and sanitizer enabled. Its CTest + budget is 180/600/900 seconds for normal/ASan/TSan. The larger migration contract + gets 180/360/600 seconds. These are execution bounds, not performance acceptance. +- Population/migration restart scripts accept runtime `EVO_TEST_TMPDIR`, using a + distinct fixture directory for each invocation. The TSan container sets it to + the existing bounded `/tmp` tmpfs. Source/build mounts stay read-only; user, + capabilities, network isolation and the pinned seccomp policy are unchanged. +- Allow 90 minutes for the checks matrix job: the captured cold/shared-runner run + reached 59 minutes before the timeout, before remaining fuzz/benchmark steps. +- Xvfb lifecycle invocations use `-noreset` to retain their display server across + deliberate SDL quit/reopen cycles. If opening still fails, the test now records + the typed Application error category instead of only an uninformative line. + The server-reset explanation is an inference: the original reopen failure was + not reproduced in twenty local runs of the old invocation. CI must confirm it. +- Full GCC client builds exposed two additional warnings-as-errors: extraction + through `std::exchange(optional, {})` and a copied panel-fixture range value. + The grant is now checked/moved/reset under the same mutex, preserving its + consume-once public contract; the fixture uses a const reference. No warning + suppression, authentication policy, protocol, UI appearance or biology changes. + +## Evidence and limits + +Executed compiler/test results are in [VALIDATION](VALIDATION.md). Local Linux +TSan runs the full cohort matrix and both process-restart contracts; ASan runs +the full cohort verification. Protected configured output fails without the +runtime override and both restart scripts pass using `/tmp` with that output +still inaccessible. CTest metadata confirms the normal and instrumented budgets. +GCC/MSVC client contracts, repeated real-X11 lifecycle runs and sanitizer checks +exercise the affected client paths without an external identity provider. + +The local Docker executable cannot reach its Linux engine. No full Docker image, +actual read-only/seccomp container, remote CI success or deployment is claimed. +The captured client reopen failure remains unconfirmed locally; the next CI run +has an explicit display lifetime and an informative failure category. +World0, live regions and the biological/persistence contracts were not changed. diff --git a/docs/engineering/TEST_STRATEGY.md b/docs/engineering/TEST_STRATEGY.md index 4de4029..5b93b60 100644 --- a/docs/engineering/TEST_STRATEGY.md +++ b/docs/engineering/TEST_STRATEGY.md @@ -33,3 +33,17 @@ needs error envelopes and ensemble comparisons in addition to exact fixtures. Unexecuted configurations are not validation. Record exact tools, commands, results and limitations in VALIDATION.md at the end of each foundation change. + +The cohort qualification runs its complete matrix under sanitizers; its wall +timeout is a bounded CI execution guard, not a portable performance claim. +Budgets are 180 s normally, 600 s with ASan/UBSan and 900 s with TSan. The larger +migration contract gets 180/360/600 s respectively. Cases, seeds, checks and +instrumentation remain identical. The checks job allows 90 minutes for the full +cold/shared-runner build, instrumented suites and remaining fuzz/benchmark steps. + +Population and migration restart scripts honor the runtime `EVO_TEST_TMPDIR` +override and create distinct random fixture subdirectories there. Without the +override they retain the configured build-directory output paths. The non-root, +networkless, read-only TSan container sets the override to its existing bounded +`/tmp` tmpfs; no source/build mount is made writable and no sandbox permission is +broadened. Fixture path randomness has no biological or test-result meaning. diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index f1e89fd..2ddbbb8 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -1,5 +1,55 @@ # Validation report +## 2026-10-17 - captured M12 CI execution failures + +Reviewed the maintainer's local `Build_Errors.txt`. Repeated checks/TSan runs hit +the 180-second cohort limit; the read-only TSan container rejects both new +restart fixture paths. The client gate passes 14 CTests before an intermittent +real-X11 application reopen failure. [Repair scope](M12_CI_EXECUTION_REPAIR.md) +keeps these environmental observations separate from biological qualification. + +Changes preserve the complete test matrices and instrumentation, set bounded +sanitizer-specific timeouts, and route restart fixtures to the existing `/tmp` +tmpfs via runtime `EVO_TEST_TMPDIR`. The job gets 90 minutes for its full cold/ +shared-runner sequence. Xvfb retains its display server across SDL reopen cycles; +the failing open now reports its typed error category. Two additional GCC client +warnings are fixed through checked grant extraction and a non-copying fixture +loop. No warning suppression, public API, authentication policy, biology, version, +live state, dependency or deployment changed. + +Executed locally: + +- Native MSVC Release **78/78 passed**, 13.97 s. +- Linux Clang 19 **TSan 4/4 passed**, 134.24 s: full cohort qualification (77.27 s), + population restart, migration contract and migration restart; no reports. +- Linux Clang 19 **ASan/UBSan full cohort qualification passed**, 63.58 s; + no case/seed/capacity sample was removed. +- Protected configured output fails without the override; both restart scripts + pass with output still protected and their fixture root redirected to `/tmp`. + Effective CTest budgets were inspected for normal, ASan and TSan configurations. +- Workflow YAML parses; the existing offline CI path-selector regression and + final repository format gate pass. Explicit clang-tidy 19 on affected client + authentication/lifecycle/panel test sources passes, with no owned warnings. +- GCC 14 Release client subset **12/12 passed**, 16.54 s; final stable-reference + grant extraction additionally passes its authentication test (0.11 s). +- Twenty real-Xvfb lifecycle invocations pass using the old invocation, and + twenty pass with `-noreset`. A real-Xvfb lifecycle invocation also passes with + ASan/UBSan; final sanitizer authentication check passes (0.19 s), no reports. +- Native client-package suite **18/18 passed**, 127.59 s. After the final guarded + extraction/panel-fixture edits, the affected native auth/panels/lifecycle subset + is rebuilt and passes **3/3**, 19.80 s. Tests remain active under NDEBUG. + +The first broad local client invocation could not run five unbuilt targets; +building the full selected target set exposed and then resolved the GCC warnings. +Final pass counts above refer to the rebuilt targets. No golden was replaced. + +Limits: the local Docker executable cannot connect to its Linux engine, so the +full image/read-only/seccomp container and remote CI have not run locally. The +captured SDL reopen failure was not reproduced in twenty baseline runs; server +reset is an inference, and CI must confirm the explicit display-lifetime change. +All failures still block the pipeline. `Build_Errors.txt` and unrelated local work +are preserved outside the commit. World0 and live regions were not touched. + ## 2026-10-17 - M12.4e versioned population continuation Portable population batch 2 is bound to envelope 2/owner 7. Batch 1 keeps its diff --git a/src/client/auth/client.cpp b/src/client/auth/client.cpp index 4f4d95b..495e04c 100644 --- a/src/client/auth/client.cpp +++ b/src/client/auth/client.cpp @@ -217,7 +217,12 @@ AuthStatus AuthClient::status() const { } std::optional AuthClient::take_grant() { std::lock_guard lock{impl_->mutex}; - return std::exchange(impl_->grant, {}); + auto& pending = impl_->grant; + if (!pending.has_value()) + return {}; + auto grant = std::move(*pending); + pending.reset(); + return std::optional{std::move(grant)}; } #ifndef _WIN32 std::unique_ptr native_auth_platform(std::string_view) { diff --git a/tests/cellular_migration_restart.cmake b/tests/cellular_migration_restart.cmake index 01815e6..6967811 100644 --- a/tests/cellular_migration_restart.cmake +++ b/tests/cellular_migration_restart.cmake @@ -1,3 +1,7 @@ +if(DEFINED ENV{EVO_TEST_TMPDIR} AND NOT "$ENV{EVO_TEST_TMPDIR}" STREQUAL "") + string(RANDOM LENGTH 16 ALPHABET 0123456789abcdef run_id) + set(TEST_DIR "$ENV{EVO_TEST_TMPDIR}/migration-restart-${run_id}") +endif() file(MAKE_DIRECTORY "${TEST_DIR}") set(prefix "${TEST_DIR}/pair") execute_process(COMMAND "${MIGRATION_TEST}" --record "${prefix}" RESULT_VARIABLE recorded) diff --git a/tests/cellular_population_restart.cmake b/tests/cellular_population_restart.cmake index 86b394b..849ac52 100644 --- a/tests/cellular_population_restart.cmake +++ b/tests/cellular_population_restart.cmake @@ -1,3 +1,7 @@ +if(DEFINED ENV{EVO_TEST_TMPDIR} AND NOT "$ENV{EVO_TEST_TMPDIR}" STREQUAL "") + string(RANDOM LENGTH 16 ALPHABET 0123456789abcdef run_id) + set(TEST_DIR "$ENV{EVO_TEST_TMPDIR}/population-restart-${run_id}") +endif() file(MAKE_DIRECTORY "${TEST_DIR}") execute_process(COMMAND "${POPULATION_TEST}" --checkpoint "${TEST_DIR}/checkpoint.pop" "${TEST_DIR}/expected.pop" RESULT_VARIABLE prepared) diff --git a/tests/client_lifecycle.cpp b/tests/client_lifecycle.cpp index f21f9eb..276a638 100644 --- a/tests/client_lifecycle.cpp +++ b/tests/client_lifecycle.cpp @@ -469,7 +469,9 @@ void toolkit_capture() { void rebinding() { const auto frame = fixture(); auto result = Application::open({320, 240, false, "software"}); - CHECK(result); + if (!result) + throw std::runtime_error("rebinding Application::open error " + + std::to_string(static_cast(result.error()))); auto app = std::move(result.value()); CHECK(app->poll_events()); const auto id = SDL_GetWindowID(window()); diff --git a/tests/client_panels.cpp b/tests/client_panels.cpp index a09884f..08960c6 100644 --- a/tests/client_panels.cpp +++ b/tests/client_panels.cpp @@ -357,7 +357,7 @@ void apply_save_restart() { std::filesystem::remove_all(dir); } void cellular_panels(const std::filesystem::path& review) { - for (const auto size : {std::pair{960, 640}, std::pair{1280, 800}, std::pair{640, 480}}) { + for (const auto& size : {std::pair{960, 640}, std::pair{1280, 800}, std::pair{640, 480}}) { const auto [width, height] = size; auto opened = Application::open({width, height, false, "software"}); CHECK(opened);