CI: repair sanitizer fixtures and client execution gates
Foundation / checks (ecs) (push) Successful in 2m33s
Foundation / checks (deploy) (push) Successful in 4m36s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (tsan) (push) Canceled after 8m8s
Foundation / checks (checks) (push) Canceled after 10m47s
Foundation / checks (client) (push) Canceled after 10m49s
Foundation / checks (network) (push) Canceled after 10m48s
Foundation / checks (postgres) (push) Canceled after 10m49s
Foundation / checks (ecs) (push) Successful in 2m33s
Foundation / checks (deploy) (push) Successful in 4m36s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (tsan) (push) Canceled after 8m8s
Foundation / checks (checks) (push) Canceled after 10m47s
Foundation / checks (client) (push) Canceled after 10m49s
Foundation / checks (network) (push) Canceled after 10m48s
Foundation / checks (postgres) (push) Canceled after 10m49s
This commit is contained in:
@@ -29,7 +29,9 @@ env:
|
||||
jobs:
|
||||
checks:
|
||||
runs-on: ${{ vars.CI_RUNNER_LABEL || 'ubuntu-22.04' }}
|
||||
timeout-minutes: 60
|
||||
# Captured cold/shared-runner checks reached 59 minutes before the ASan
|
||||
# cohort timeout, with the remaining fuzz/benchmark steps still pending.
|
||||
timeout-minutes: 90
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -70,6 +72,7 @@ jobs:
|
||||
profile="$RUNNER_TEMP/aevolution-tsan-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
|
||||
node tools/ci/tsan-profile.mjs "$profile"
|
||||
docker run --rm --network none --read-only --user 10001:10001 \
|
||||
--env EVO_TEST_TMPDIR=/tmp \
|
||||
--cap-drop ALL --security-opt no-new-privileges \
|
||||
--security-opt "seccomp=$profile" \
|
||||
--tmpfs /source/build/tsan/Testing:rw,noexec,nosuid,size=16m,mode=1777 \
|
||||
|
||||
+13
-2
@@ -663,8 +663,19 @@ add_executable(evo-cellular-cohorts experiments/resolution/cellular_cohorts.cpp)
|
||||
target_link_libraries(evo-cellular-cohorts PRIVATE evo::lifecycle)
|
||||
evo_owned_target(evo-cellular-cohorts)
|
||||
if(BUILD_TESTING)
|
||||
# Sanitizer instrumentation and shared CI runner contention are not product
|
||||
# performance gates. Keep the full matrix, with bounded instrumented budgets.
|
||||
set(evo_cohort_timeout 180)
|
||||
set(evo_migration_timeout 180)
|
||||
if(EVO_SANITIZER STREQUAL "address")
|
||||
set(evo_cohort_timeout 600)
|
||||
set(evo_migration_timeout 360)
|
||||
elseif(EVO_SANITIZER STREQUAL "thread")
|
||||
set(evo_cohort_timeout 900)
|
||||
set(evo_migration_timeout 600)
|
||||
endif()
|
||||
add_test(NAME cellular-cohort-qualification COMMAND evo-cellular-cohorts --verify)
|
||||
set_tests_properties(cellular-cohort-qualification PROPERTIES TIMEOUT 180)
|
||||
set_tests_properties(cellular-cohort-qualification PROPERTIES TIMEOUT ${evo_cohort_timeout})
|
||||
endif()
|
||||
|
||||
if(BUILD_TESTING)
|
||||
@@ -686,7 +697,7 @@ if(BUILD_TESTING)
|
||||
target_link_libraries(evo-cellular-migration-tests PRIVATE evo::lifecycle)
|
||||
evo_owned_target(evo-cellular-migration-tests)
|
||||
add_test(NAME cellular-migration-contract COMMAND evo-cellular-migration-tests)
|
||||
set_tests_properties(cellular-migration-contract PROPERTIES TIMEOUT 180)
|
||||
set_tests_properties(cellular-migration-contract PROPERTIES TIMEOUT ${evo_migration_timeout})
|
||||
add_test(NAME cellular-migration-process-restart COMMAND "${CMAKE_COMMAND}"
|
||||
"-DMIGRATION_TEST=$<TARGET_FILE:evo-cellular-migration-tests>"
|
||||
"-DTEST_DIR=${CMAKE_CURRENT_BINARY_DIR}/migration-restart"
|
||||
|
||||
@@ -39,6 +39,8 @@ FROM ci-source AS ci-client
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
libx11-dev libxext-dev libxrandr-dev libxcursor-dev libxi-dev libxss-dev libxtst-dev xvfb xauth \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
# Lifecycle tests deliberately quit/reopen SDL: retain the virtual X server with
|
||||
# -noreset through those cycles, matching a persistent desktop display server.
|
||||
RUN cmake -P tools/prepare-sdl3.cmake \
|
||||
&& cmake -P tools/prepare-imgui.cmake \
|
||||
&& cmake --preset client-probe -DCMAKE_CXX_COMPILER=clang++-19 -DCMAKE_C_COMPILER=clang-19 \
|
||||
@@ -47,12 +49,12 @@ RUN cmake -P tools/prepare-sdl3.cmake \
|
||||
&& ctest --preset client-probe --no-tests=error \
|
||||
&& xvfb-run -a env SDL_VIDEO_DRIVER=x11 ./build/client-probe/evo-client-probe --renderer software \
|
||||
&& for cycle in 1 2 3; do xvfb-run -a env SDL_VIDEO_DRIVER=x11 ./build/client-probe/evo-ui-probe --renderer software --cycles 1; done \
|
||||
&& xvfb-run -a env SDL_VIDEO_DRIVER=x11 ./build/client-probe/evo-client-lifecycle-tests \
|
||||
&& xvfb-run -a -s "-screen 0 1280x1024x24 -noreset" env SDL_VIDEO_DRIVER=x11 ./build/client-probe/evo-client-lifecycle-tests \
|
||||
&& cmake --preset client-probe -DEVO_BUILD_CLIENT=ON -DEVO_SANITIZER=address -DEVO_ENABLE_TIDY=OFF \
|
||||
&& cmake --build build/client-probe --target evo-client-state-tests evo-ui-probe evo-client evo-client-lifecycle-tests evo-client-environment-tests evo-client-organism-tests evo-client-ui-model-tests evo-client-panel-tests evo-client-bindings-tests evo-client-auth-tests --parallel 2 \
|
||||
&& ctest --preset client-probe --no-tests=error \
|
||||
&& for cycle in 1 2 3; do xvfb-run -a env SDL_VIDEO_DRIVER=x11 ./build/client-probe/evo-ui-probe --renderer software --cycles 1; done \
|
||||
&& xvfb-run -a env SDL_VIDEO_DRIVER=x11 ./build/client-probe/evo-client-lifecycle-tests
|
||||
&& xvfb-run -a -s "-screen 0 1280x1024x24 -noreset" env SDL_VIDEO_DRIVER=x11 ./build/client-probe/evo-client-lifecycle-tests
|
||||
|
||||
FROM ci-source AS ci-network
|
||||
# Explicit pinned Mbed TLS preparation (ADR-0082); codec, admission, dev grants and
|
||||
|
||||
@@ -1,5 +1,18 @@
|
||||
# Project status and next steps
|
||||
|
||||
## M12 CI execution repair - 2026-10-17
|
||||
|
||||
The captured sanitizer timeout and read-only fixture failures now have scoped
|
||||
test-runner fixes: full instrumented matrices keep their cases with bounded
|
||||
budgets, and restart fixtures use the TSan container's existing `/tmp` tmpfs.
|
||||
Read-only root, non-root user, network isolation and seccomp policy remain intact.
|
||||
Real-X11 lifecycle checks keep a persistent Xvfb server and report the typed open
|
||||
error; the original intermittent reopen failure was not reproduced locally.
|
||||
GCC warning fixes preserve consume-once authentication and panel-test behavior.
|
||||
[Evidence and limitations](engineering/M12_CI_EXECUTION_REPAIR.md) distinguish
|
||||
local qualification from the remote CI/container rerun still required. M12's
|
||||
remaining integration/biology gates are unchanged; no deployment occurred.
|
||||
|
||||
## M12.4e continued population replay after transfer - 2026-10-17
|
||||
|
||||
Portable population batch 2 binds envelope 2/owner 7 and preserves boundary flows
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
# M12 CI execution repair
|
||||
|
||||
The maintainer supplied `Build_Errors.txt`, retained as local evidence rather than
|
||||
committed build output. Its repeated checks/TSan logs show the unchanged full
|
||||
cohort verification exceeding 180 seconds. The read-only TSan container also
|
||||
rejects the new population/migration fixture directories under `/source/build`.
|
||||
The client gate passes its 14 CTests, then intermittently fails while opening the
|
||||
rebinding test's SDL application under a real Xvfb display.
|
||||
|
||||
## Changes
|
||||
|
||||
- Keep every cohort case, seed, capacity sample and sanitizer enabled. Its CTest
|
||||
budget is 180/600/900 seconds for normal/ASan/TSan. The larger migration contract
|
||||
gets 180/360/600 seconds. These are execution bounds, not performance acceptance.
|
||||
- Population/migration restart scripts accept runtime `EVO_TEST_TMPDIR`, using a
|
||||
distinct fixture directory for each invocation. The TSan container sets it to
|
||||
the existing bounded `/tmp` tmpfs. Source/build mounts stay read-only; user,
|
||||
capabilities, network isolation and the pinned seccomp policy are unchanged.
|
||||
- Allow 90 minutes for the checks matrix job: the captured cold/shared-runner run
|
||||
reached 59 minutes before the timeout, before remaining fuzz/benchmark steps.
|
||||
- Xvfb lifecycle invocations use `-noreset` to retain their display server across
|
||||
deliberate SDL quit/reopen cycles. If opening still fails, the test now records
|
||||
the typed Application error category instead of only an uninformative line.
|
||||
The server-reset explanation is an inference: the original reopen failure was
|
||||
not reproduced in twenty local runs of the old invocation. CI must confirm it.
|
||||
- Full GCC client builds exposed two additional warnings-as-errors: extraction
|
||||
through `std::exchange(optional, {})` and a copied panel-fixture range value.
|
||||
The grant is now checked/moved/reset under the same mutex, preserving its
|
||||
consume-once public contract; the fixture uses a const reference. No warning
|
||||
suppression, authentication policy, protocol, UI appearance or biology changes.
|
||||
|
||||
## Evidence and limits
|
||||
|
||||
Executed compiler/test results are in [VALIDATION](VALIDATION.md). Local Linux
|
||||
TSan runs the full cohort matrix and both process-restart contracts; ASan runs
|
||||
the full cohort verification. Protected configured output fails without the
|
||||
runtime override and both restart scripts pass using `/tmp` with that output
|
||||
still inaccessible. CTest metadata confirms the normal and instrumented budgets.
|
||||
GCC/MSVC client contracts, repeated real-X11 lifecycle runs and sanitizer checks
|
||||
exercise the affected client paths without an external identity provider.
|
||||
|
||||
The local Docker executable cannot reach its Linux engine. No full Docker image,
|
||||
actual read-only/seccomp container, remote CI success or deployment is claimed.
|
||||
The captured client reopen failure remains unconfirmed locally; the next CI run
|
||||
has an explicit display lifetime and an informative failure category.
|
||||
World0, live regions and the biological/persistence contracts were not changed.
|
||||
@@ -33,3 +33,17 @@ needs error envelopes and ensemble comparisons in addition to exact fixtures.
|
||||
|
||||
Unexecuted configurations are not validation. Record exact tools, commands,
|
||||
results and limitations in VALIDATION.md at the end of each foundation change.
|
||||
|
||||
The cohort qualification runs its complete matrix under sanitizers; its wall
|
||||
timeout is a bounded CI execution guard, not a portable performance claim.
|
||||
Budgets are 180 s normally, 600 s with ASan/UBSan and 900 s with TSan. The larger
|
||||
migration contract gets 180/360/600 s respectively. Cases, seeds, checks and
|
||||
instrumentation remain identical. The checks job allows 90 minutes for the full
|
||||
cold/shared-runner build, instrumented suites and remaining fuzz/benchmark steps.
|
||||
|
||||
Population and migration restart scripts honor the runtime `EVO_TEST_TMPDIR`
|
||||
override and create distinct random fixture subdirectories there. Without the
|
||||
override they retain the configured build-directory output paths. The non-root,
|
||||
networkless, read-only TSan container sets the override to its existing bounded
|
||||
`/tmp` tmpfs; no source/build mount is made writable and no sandbox permission is
|
||||
broadened. Fixture path randomness has no biological or test-result meaning.
|
||||
|
||||
@@ -1,5 +1,55 @@
|
||||
# Validation report
|
||||
|
||||
## 2026-10-17 - captured M12 CI execution failures
|
||||
|
||||
Reviewed the maintainer's local `Build_Errors.txt`. Repeated checks/TSan runs hit
|
||||
the 180-second cohort limit; the read-only TSan container rejects both new
|
||||
restart fixture paths. The client gate passes 14 CTests before an intermittent
|
||||
real-X11 application reopen failure. [Repair scope](M12_CI_EXECUTION_REPAIR.md)
|
||||
keeps these environmental observations separate from biological qualification.
|
||||
|
||||
Changes preserve the complete test matrices and instrumentation, set bounded
|
||||
sanitizer-specific timeouts, and route restart fixtures to the existing `/tmp`
|
||||
tmpfs via runtime `EVO_TEST_TMPDIR`. The job gets 90 minutes for its full cold/
|
||||
shared-runner sequence. Xvfb retains its display server across SDL reopen cycles;
|
||||
the failing open now reports its typed error category. Two additional GCC client
|
||||
warnings are fixed through checked grant extraction and a non-copying fixture
|
||||
loop. No warning suppression, public API, authentication policy, biology, version,
|
||||
live state, dependency or deployment changed.
|
||||
|
||||
Executed locally:
|
||||
|
||||
- Native MSVC Release **78/78 passed**, 13.97 s.
|
||||
- Linux Clang 19 **TSan 4/4 passed**, 134.24 s: full cohort qualification (77.27 s),
|
||||
population restart, migration contract and migration restart; no reports.
|
||||
- Linux Clang 19 **ASan/UBSan full cohort qualification passed**, 63.58 s;
|
||||
no case/seed/capacity sample was removed.
|
||||
- Protected configured output fails without the override; both restart scripts
|
||||
pass with output still protected and their fixture root redirected to `/tmp`.
|
||||
Effective CTest budgets were inspected for normal, ASan and TSan configurations.
|
||||
- Workflow YAML parses; the existing offline CI path-selector regression and
|
||||
final repository format gate pass. Explicit clang-tidy 19 on affected client
|
||||
authentication/lifecycle/panel test sources passes, with no owned warnings.
|
||||
- GCC 14 Release client subset **12/12 passed**, 16.54 s; final stable-reference
|
||||
grant extraction additionally passes its authentication test (0.11 s).
|
||||
- Twenty real-Xvfb lifecycle invocations pass using the old invocation, and
|
||||
twenty pass with `-noreset`. A real-Xvfb lifecycle invocation also passes with
|
||||
ASan/UBSan; final sanitizer authentication check passes (0.19 s), no reports.
|
||||
- Native client-package suite **18/18 passed**, 127.59 s. After the final guarded
|
||||
extraction/panel-fixture edits, the affected native auth/panels/lifecycle subset
|
||||
is rebuilt and passes **3/3**, 19.80 s. Tests remain active under NDEBUG.
|
||||
|
||||
The first broad local client invocation could not run five unbuilt targets;
|
||||
building the full selected target set exposed and then resolved the GCC warnings.
|
||||
Final pass counts above refer to the rebuilt targets. No golden was replaced.
|
||||
|
||||
Limits: the local Docker executable cannot connect to its Linux engine, so the
|
||||
full image/read-only/seccomp container and remote CI have not run locally. The
|
||||
captured SDL reopen failure was not reproduced in twenty baseline runs; server
|
||||
reset is an inference, and CI must confirm the explicit display-lifetime change.
|
||||
All failures still block the pipeline. `Build_Errors.txt` and unrelated local work
|
||||
are preserved outside the commit. World0 and live regions were not touched.
|
||||
|
||||
## 2026-10-17 - M12.4e versioned population continuation
|
||||
|
||||
Portable population batch 2 is bound to envelope 2/owner 7. Batch 1 keeps its
|
||||
|
||||
@@ -217,7 +217,12 @@ AuthStatus AuthClient::status() const {
|
||||
}
|
||||
std::optional<AuthGrant> AuthClient::take_grant() {
|
||||
std::lock_guard lock{impl_->mutex};
|
||||
return std::exchange(impl_->grant, {});
|
||||
auto& pending = impl_->grant;
|
||||
if (!pending.has_value())
|
||||
return {};
|
||||
auto grant = std::move(*pending);
|
||||
pending.reset();
|
||||
return std::optional<AuthGrant>{std::move(grant)};
|
||||
}
|
||||
#ifndef _WIN32
|
||||
std::unique_ptr<AuthPlatform> native_auth_platform(std::string_view) {
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
if(DEFINED ENV{EVO_TEST_TMPDIR} AND NOT "$ENV{EVO_TEST_TMPDIR}" STREQUAL "")
|
||||
string(RANDOM LENGTH 16 ALPHABET 0123456789abcdef run_id)
|
||||
set(TEST_DIR "$ENV{EVO_TEST_TMPDIR}/migration-restart-${run_id}")
|
||||
endif()
|
||||
file(MAKE_DIRECTORY "${TEST_DIR}")
|
||||
set(prefix "${TEST_DIR}/pair")
|
||||
execute_process(COMMAND "${MIGRATION_TEST}" --record "${prefix}" RESULT_VARIABLE recorded)
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
if(DEFINED ENV{EVO_TEST_TMPDIR} AND NOT "$ENV{EVO_TEST_TMPDIR}" STREQUAL "")
|
||||
string(RANDOM LENGTH 16 ALPHABET 0123456789abcdef run_id)
|
||||
set(TEST_DIR "$ENV{EVO_TEST_TMPDIR}/population-restart-${run_id}")
|
||||
endif()
|
||||
file(MAKE_DIRECTORY "${TEST_DIR}")
|
||||
execute_process(COMMAND "${POPULATION_TEST}" --checkpoint "${TEST_DIR}/checkpoint.pop"
|
||||
"${TEST_DIR}/expected.pop" RESULT_VARIABLE prepared)
|
||||
|
||||
@@ -469,7 +469,9 @@ void toolkit_capture() {
|
||||
void rebinding() {
|
||||
const auto frame = fixture();
|
||||
auto result = Application::open({320, 240, false, "software"});
|
||||
CHECK(result);
|
||||
if (!result)
|
||||
throw std::runtime_error("rebinding Application::open error " +
|
||||
std::to_string(static_cast<unsigned>(result.error())));
|
||||
auto app = std::move(result.value());
|
||||
CHECK(app->poll_events());
|
||||
const auto id = SDL_GetWindowID(window());
|
||||
|
||||
@@ -357,7 +357,7 @@ void apply_save_restart() {
|
||||
std::filesystem::remove_all(dir);
|
||||
}
|
||||
void cellular_panels(const std::filesystem::path& review) {
|
||||
for (const auto size : {std::pair{960, 640}, std::pair{1280, 800}, std::pair{640, 480}}) {
|
||||
for (const auto& size : {std::pair{960, 640}, std::pair{1280, 800}, std::pair{640, 480}}) {
|
||||
const auto [width, height] = size;
|
||||
auto opened = Application::open({width, height, false, "software"});
|
||||
CHECK(opened);
|
||||
|
||||
Reference in New Issue
Block a user