From b371ad9217eb8e0becf1bb6f338fd15d4c025f8f Mon Sep 17 00:00:00 2001 From: Katarzyna Kaczmarska Date: Wed, 30 Sep 2026 11:59:58 +0200 Subject: [PATCH 01/19] Apply UR test summary/stats/artifacts tooling to E2E tests --- .github/workflows/sycl-linux-run-tests.yml | 1 + .github/workflows/sycl-windows-run-tests.yml | 1 + devops/actions/run-tests/linux/e2e/action.yml | 49 +++++++++++-------- devops/actions/run-tests/ur/action.yml | 32 +++++++++--- .../actions/run-tests/windows/e2e/action.yml | 49 +++++++++++-------- devops/scripts/ur_test_tools/cli.py | 2 + devops/scripts/ur_test_tools/constants.py | 1 + devops/scripts/ur_test_tools/models/config.py | 3 ++ devops/scripts/ur_test_tools/test_runner.py | 34 ++++++++++--- 9 files changed, 116 insertions(+), 56 deletions(-) diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml index 35badd106674c..0901e06c91eec 100644 --- a/.github/workflows/sycl-linux-run-tests.yml +++ b/.github/workflows/sycl-linux-run-tests.yml @@ -366,6 +366,7 @@ jobs: target_devices: ${{ inputs.target_devices }} extra_lit_opts: ${{ inputs.extra_lit_opts }} retention-days: ${{ inputs.retention-days }} + job_name: ${{ inputs.name }} - name: Run SYCL CTS Tests if: inputs.tests_selector == 'cts' diff --git a/.github/workflows/sycl-windows-run-tests.yml b/.github/workflows/sycl-windows-run-tests.yml index a9ebcaf58fbe6..4c0134f39113e 100644 --- a/.github/workflows/sycl-windows-run-tests.yml +++ b/.github/workflows/sycl-windows-run-tests.yml @@ -152,6 +152,7 @@ jobs: extra_lit_opts: ${{ inputs.extra_lit_opts }} retention-days: ${{ inputs.artifact_retention_days }} cxx: ${{ inputs.cxx }} + job_name: ${{ inputs.name }} - name: Run SYCL CTS Tests if: inputs.tests_selector == 'cts' diff --git a/devops/actions/run-tests/linux/e2e/action.yml b/devops/actions/run-tests/linux/e2e/action.yml index c9f11c5324694..4d1b953e0e797 100644 --- a/devops/actions/run-tests/linux/e2e/action.yml +++ b/devops/actions/run-tests/linux/e2e/action.yml @@ -18,6 +18,10 @@ inputs: required: false sycl_compiler: required: false + job_name: + description: 'Name of the calling job, used to build a unique artifact name' + required: false + default: '' runs: @@ -67,32 +71,35 @@ runs: shell: bash run: | cmake -GNinja -B./build-e2e -S./llvm/sycl/test-e2e -DCMAKE_CXX_COMPILER="${{ inputs.sycl_compiler || '$(which dpclang++)'}}" -DLLVM_LIT="$PWD/llvm/llvm/utils/lit/lit.py" ${{ steps.cmake_opts.outputs.opts }} - - name: SYCL End-to-end tests - id: run_e2e - continue-on-error: true + + # Composed here, not in ur-test, since these depend on caller-only workflow inputs. + - name: Compose E2E LIT options + id: lit_opts shell: bash env: - LIT_OPTS: -v --no-progress-bar --show-unsupported --show-pass --show-xfail --max-time 3600 --time-tests --param print_features=True --param test-mode=${{ inputs.testing_mode }} --param sycl_devices=${{ inputs.target_devices }} ${{ inputs.extra_lit_opts }} - run: | - ninja -C build-e2e check-sycl-e2e > e2e.log 2>&1 - # Two steps below are duplicated between Lin/Win actions, updates must change both - - name: E2E logs - if: ${{ always() }} - shell: bash + TESTING_MODE: ${{ inputs.testing_mode }} + TARGET_DEVICES: ${{ inputs.target_devices }} + USER_LIT_OPTS: ${{ inputs.extra_lit_opts }} run: | - echo "::group::Show Full E2E Log" - cat e2e.log - echo "::endgroup::" - - name: Report E2E Failures - if: steps.run_e2e.outcome != 'success' + echo "opts=--max-time 3600 --param print_features=True --param test-mode=${TESTING_MODE} --param sycl_devices=${TARGET_DEVICES} ${USER_LIT_OPTS}" >> "$GITHUB_OUTPUT" + + - name: Build artifact name + if: ${{ always() && inputs.job_name != '' }} + id: artifact_name shell: bash - # For some reason Github uses the first line from the `run: |` section for - # the folded entry when displaying instead of this step's name. + env: + JOB_NAME: ${{ inputs.job_name }} run: | - # Report E2E Failures - awk '/^Failed Tests|Unexpectedly Passed Tests|Unresolved tests|Timed Out Tests|Testing Time/{flag=1}/FAILED: CMakeFiles/{flag=0}flag' e2e.log >> $GITHUB_STEP_SUMMARY - awk '/^Failed Tests|Unexpectedly Passed Tests|Unresolved tests|Timed Out Tests|Testing Time/{flag=1}/FAILED: CMakeFiles/{flag=0}flag' e2e.log - exit 1 + safe_name=$(echo "$JOB_NAME" | tr -c '[:alnum:]._-' '-' | sed 's/-\+/-/g; s/^-//; s/-$//') + echo "name=e2e-${safe_name}" >> "$GITHUB_OUTPUT" + + - name: Run E2E tests + uses: ./devops/actions/run-tests/ur + with: + test_type: 'e2e' + build_dir: 'build-e2e' + extra_lit_opts: ${{ steps.lit_opts.outputs.opts }} + artifact_name: ${{ steps.artifact_name.outputs.name }} - name: Pack E2E binaries if: ${{ !cancelled() && inputs.binaries_artifact != '' && inputs.testing_mode != 'run-only' }} diff --git a/devops/actions/run-tests/ur/action.yml b/devops/actions/run-tests/ur/action.yml index 0e161f6970846..8893a3ea5beb5 100644 --- a/devops/actions/run-tests/ur/action.yml +++ b/devops/actions/run-tests/ur/action.yml @@ -2,7 +2,7 @@ name: 'Run UR Tests' inputs: test_type: - description: 'Type of tests to run (adapter-specific or conformance)' + description: 'Type of tests to run (adapter-specific, conformance or e2e)' required: true build_dir: required: false @@ -11,10 +11,25 @@ inputs: description: 'Name for uploaded test artifacts (logs, XML)' required: false default: '' + extra_lit_opts: + description: 'Extra options appended to LIT_OPTS (e.g. E2E --param/--max-time flags)' + required: false + default: '' runs: using: "composite" steps: + # `python3` may not be on PATH on Windows runners; resolve it once for reuse below. + - name: Resolve Python interpreter + id: python + shell: bash + run: | + if command -v python3 >/dev/null 2>&1; then + echo "exe=python3" >> "$GITHUB_OUTPUT" + else + echo "exe=python" >> "$GITHUB_OUTPUT" + fi + - name: Run ${{ inputs.test_type }} tests id: run_tests continue-on-error: true @@ -22,14 +37,17 @@ runs: env: BUILD_DIR: ${{ inputs.build_dir }} TEST_TYPE: ${{ inputs.test_type }} + EXTRA_LIT_OPTS: ${{ inputs.extra_lit_opts }} + PY: ${{ steps.python.outputs.exe }} run: | - devops/scripts/ur-test run "$TEST_TYPE" "$BUILD_DIR" "$GITHUB_WORKSPACE" + "$PY" devops/scripts/ur-test run "$TEST_TYPE" "$BUILD_DIR" "$GITHUB_WORKSPACE" - name: Filter test log if: ${{ always() && steps.run_tests.outputs['skip-artifacts'] != '1' }} shell: bash env: LOG_FILE: ${{ steps.run_tests.outputs['log-file'] }} + PY: ${{ steps.python.outputs.exe }} run: | if [ -z "$LOG_FILE" ]; then echo "::warning::Test log path not provided by runner; skipping log filtering." @@ -42,7 +60,7 @@ runs: fi echo "::group::Test Log" - devops/scripts/ur-test filter-log "$LOG_FILE" || { echo "::error::Filter failed"; head -1000 "$LOG_FILE"; } + "$PY" devops/scripts/ur-test filter-log "$LOG_FILE" || { echo "::error::Filter failed"; head -1000 "$LOG_FILE"; } echo "::endgroup::" - name: Report test failures @@ -50,13 +68,14 @@ runs: shell: bash env: LOG_FILE: ${{ steps.run_tests.outputs['log-file'] }} + PY: ${{ steps.python.outputs.exe }} run: | if [ -z "$LOG_FILE" ] || [ ! -f "$LOG_FILE" ]; then echo "::warning::Test execution failed, but no test log is available." exit 1 fi - devops/scripts/ur-test extract-errors "$LOG_FILE" + "$PY" devops/scripts/ur-test extract-errors "$LOG_FILE" exit 1 - name: Test summary @@ -68,6 +87,7 @@ runs: ARTIFACT_NAME: ${{ inputs.artifact_name }} TEST_TYPE: ${{ inputs.test_type }} WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + PY: ${{ steps.python.outputs.exe }} run: | if [ -z "$LOG_FILE" ]; then echo "::warning::Test log path not provided by runner; skipping summary." @@ -84,7 +104,7 @@ runs: exit 0 fi - devops/scripts/ur-test summary "$LOG_FILE" "${XML_FILE:-}" + "$PY" devops/scripts/ur-test summary "$LOG_FILE" "${XML_FILE:-}" if [ -n "$ARTIFACT_NAME" ]; then echo "" @@ -96,7 +116,7 @@ runs: if [ -n "$GITHUB_STEP_SUMMARY" ]; then echo "### ${TEST_TYPE} tests" >> "$GITHUB_STEP_SUMMARY" - devops/scripts/ur-test step-summary "$LOG_FILE" "${XML_FILE:-}" >> "$GITHUB_STEP_SUMMARY" + "$PY" devops/scripts/ur-test step-summary "$LOG_FILE" "${XML_FILE:-}" >> "$GITHUB_STEP_SUMMARY" fi - name: Copy artifacts diff --git a/devops/actions/run-tests/windows/e2e/action.yml b/devops/actions/run-tests/windows/e2e/action.yml index f22bb0f65185c..1cc4e60fec90c 100644 --- a/devops/actions/run-tests/windows/e2e/action.yml +++ b/devops/actions/run-tests/windows/e2e/action.yml @@ -21,6 +21,10 @@ inputs: cxx: required: false default: "cl" + job_name: + description: 'Name of the calling job, used to build a unique artifact name' + required: false + default: '' runs: using: "composite" @@ -71,32 +75,35 @@ runs: shell: bash run: ls build-e2e > e2econf_files.txt - - name: Run End-to-End tests - id: run_e2e - continue-on-error: true + # Composed here, not in ur-test, since these depend on caller-only workflow inputs. + - name: Compose E2E LIT options + id: lit_opts shell: bash env: - LIT_OPTS: -v --no-progress-bar --show-unsupported --show-pass --show-xfail --max-time ${{ inputs.testing_mode == 'run-only' && 1200 || 3600 }} --time-tests --param print_features=True --param test-mode=${{ inputs.testing_mode }} --param sycl_devices=${{ inputs.target_devices }} ${{ inputs.extra_lit_opts }} - run: | - cmake --build build-e2e --target check-sycl-e2e > e2e.log 2>&1 - # Two steps below are duplicated between Lin/Win actions, updates must change both - - name: E2E logs - if: ${{ always() }} - shell: bash + TESTING_MODE: ${{ inputs.testing_mode }} + TARGET_DEVICES: ${{ inputs.target_devices }} + USER_LIT_OPTS: ${{ inputs.extra_lit_opts }} + MAX_TIME: ${{ inputs.testing_mode == 'run-only' && 1200 || 3600 }} run: | - echo "::group::Show Full E2E Log" - cat e2e.log - echo "::endgroup::" - - name: Report E2E Failures - if: steps.run_e2e.outcome != 'success' + echo "opts=--max-time ${MAX_TIME} --param print_features=True --param test-mode=${TESTING_MODE} --param sycl_devices=${TARGET_DEVICES} ${USER_LIT_OPTS}" >> "$GITHUB_OUTPUT" + + - name: Build artifact name + if: ${{ always() && inputs.job_name != '' }} + id: artifact_name shell: bash - # For some reason Github uses the first line from the `run: |` section for - # the folded entry when displaying instead of this step's name. + env: + JOB_NAME: ${{ inputs.job_name }} run: | - # Report E2E Failures - awk '/^Failed Tests|Unexpectedly Passed Tests|Unresolved tests|Timed Out Tests|Testing Time/{flag=1}/FAILED: CMakeFiles/{flag=0}flag' e2e.log >> $GITHUB_STEP_SUMMARY - awk '/^Failed Tests|Unexpectedly Passed Tests|Unresolved tests|Timed Out Tests|Testing Time/{flag=1}/FAILED: CMakeFiles/{flag=0}flag' e2e.log - exit 1 + safe_name=$(echo "$JOB_NAME" | tr -c '[:alnum:]._-' '-' | sed 's/-\+/-/g; s/^-//; s/-$//') + echo "name=e2e-${safe_name}" >> "$GITHUB_OUTPUT" + + - name: Run E2E tests + uses: ./devops/actions/run-tests/ur + with: + test_type: 'e2e' + build_dir: 'build-e2e' + extra_lit_opts: ${{ steps.lit_opts.outputs.opts }} + artifact_name: ${{ steps.artifact_name.outputs.name }} # Github CI doesn't support containers on Windows, so we cannot guarantee # that paths are the same between building and running systems. To avoid diff --git a/devops/scripts/ur_test_tools/cli.py b/devops/scripts/ur_test_tools/cli.py index 53ffcfc943526..c94c6531ae721 100644 --- a/devops/scripts/ur_test_tools/cli.py +++ b/devops/scripts/ur_test_tools/cli.py @@ -120,6 +120,8 @@ def _run_tests_command() -> int: github_actions.print_error(str(e)) return 1 + config.extra_lit_opts = os.environ.get("EXTRA_LIT_OPTS", "") + workspace_path = Path(workspace).resolve() build_dir_path = workspace_path / build_dir diff --git a/devops/scripts/ur_test_tools/constants.py b/devops/scripts/ur_test_tools/constants.py index 396574d62516c..6e695fb888827 100644 --- a/devops/scripts/ur_test_tools/constants.py +++ b/devops/scripts/ur_test_tools/constants.py @@ -29,6 +29,7 @@ TEST_TYPE_ADAPTER_SPECIFIC = "adapter-specific" TEST_TYPE_CONFORMANCE = "conformance" +TEST_TYPE_E2E = "e2e" # These tests cause timeouts on CI and are excluded from adapter-specific runs LIT_FILTER_OUT_ADAPTER_SPECIFIC = ( diff --git a/devops/scripts/ur_test_tools/models/config.py b/devops/scripts/ur_test_tools/models/config.py index 0c640fa06751c..4bc038d9fac65 100644 --- a/devops/scripts/ur_test_tools/models/config.py +++ b/devops/scripts/ur_test_tools/models/config.py @@ -8,6 +8,9 @@ class TestConfig: target: str log_file: str lit_filter_out: Optional[str] = None + lit_timeout: Optional[int] = None + lit_jobs: Optional[int] = None + extra_lit_opts: str = "" def __post_init__(self): if not all([self.target, self.log_file]): diff --git a/devops/scripts/ur_test_tools/test_runner.py b/devops/scripts/ur_test_tools/test_runner.py index f46f3101b0998..4ea3c7f70b9f2 100644 --- a/devops/scripts/ur_test_tools/test_runner.py +++ b/devops/scripts/ur_test_tools/test_runner.py @@ -13,6 +13,7 @@ LIT_CI_OPTIONS, TEST_TYPE_ADAPTER_SPECIFIC, TEST_TYPE_CONFORMANCE, + TEST_TYPE_E2E, LIT_FILTER_OUT_ADAPTER_SPECIFIC, MAX_LINES_TO_SCAN, ) @@ -27,11 +28,21 @@ def get_test_config(test_type: str) -> TestConfig: target="check-unified-runtime-adapter", log_file="adapter_tests.log", lit_filter_out=LIT_FILTER_OUT_ADAPTER_SPECIFIC, + lit_timeout=DEFAULT_LIT_TIMEOUT, + lit_jobs=DEFAULT_LIT_JOBS, ) elif test_type == TEST_TYPE_CONFORMANCE: return TestConfig( target="check-unified-runtime-conformance", log_file="conformance_tests.log", + lit_timeout=DEFAULT_LIT_TIMEOUT, + lit_jobs=DEFAULT_LIT_JOBS, + ) + elif test_type == TEST_TYPE_E2E: + # Timeout/parallelism come from the caller via extra_lit_opts (e.g. --max-time). + return TestConfig( + target="check-sycl-e2e", + log_file="e2e_tests.log", ) else: raise ValueError(f"Invalid test_type: {test_type}") @@ -81,20 +92,27 @@ def run(self) -> int: return result.returncode def _setup_environment(self) -> None: + config = self.context.config lit_opts_parts = [ *LIT_COMMON_REPORTING_OPTIONS, *LIT_CI_OPTIONS, - "--timeout", - str(DEFAULT_LIT_TIMEOUT), - "-j", - str(DEFAULT_LIT_JOBS), - "--xunit-xml-output", - str(self.context.xml_output_path), ] + + if config.lit_timeout is not None: + lit_opts_parts += ["--timeout", str(config.lit_timeout)] + + if config.lit_jobs is not None: + lit_opts_parts += ["-j", str(config.lit_jobs)] + + lit_opts_parts += ["--xunit-xml-output", str(self.context.xml_output_path)] + + if config.extra_lit_opts: + lit_opts_parts.append(config.extra_lit_opts) + self.context.env["LIT_OPTS"] = " ".join(lit_opts_parts) - if self.context.config.lit_filter_out: - self.context.env["LIT_FILTER_OUT"] = self.context.config.lit_filter_out + if config.lit_filter_out: + self.context.env["LIT_FILTER_OUT"] = config.lit_filter_out def _build_cmake_command(self) -> List[str]: return [ From 529861c6c75ec3b47cdac6d5601f838d763dad4d Mon Sep 17 00:00:00 2001 From: Katarzyna Kaczmarska Date: Wed, 30 Sep 2026 12:27:13 +0200 Subject: [PATCH 02/19] Install defusedxml on demand in run-tests/ur action --- devops/actions/run-tests/ur/action.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/devops/actions/run-tests/ur/action.yml b/devops/actions/run-tests/ur/action.yml index 8893a3ea5beb5..58a89219dacaa 100644 --- a/devops/actions/run-tests/ur/action.yml +++ b/devops/actions/run-tests/ur/action.yml @@ -30,6 +30,16 @@ runs: echo "exe=python" >> "$GITHUB_OUTPUT" fi + - name: Ensure ur-test Python dependencies + shell: bash + env: + PY: ${{ steps.python.outputs.exe }} + run: | + "$PY" -c "import defusedxml" 2>/dev/null && exit 0 + # --break-system-packages: nightly images have no venv and broken apt, same as devops/actions/run-tests/benchmark. + "$PY" -m pip install --disable-pip-version-check --quiet --user --break-system-packages defusedxml==0.7.1 \ + || "$PY" -m pip install --disable-pip-version-check --quiet --user defusedxml==0.7.1 + - name: Run ${{ inputs.test_type }} tests id: run_tests continue-on-error: true From 88303844026edcfe8113b8164f02f6c6ec68c8f1 Mon Sep 17 00:00:00 2001 From: Katarzyna Kaczmarska Date: Wed, 30 Sep 2026 13:30:26 +0200 Subject: [PATCH 03/19] Include devops/scripts in Windows E2E sparse checkout --- .github/workflows/sycl-windows-run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sycl-windows-run-tests.yml b/.github/workflows/sycl-windows-run-tests.yml index 4c0134f39113e..89c7da642e2b3 100644 --- a/.github/workflows/sycl-windows-run-tests.yml +++ b/.github/workflows/sycl-windows-run-tests.yml @@ -97,7 +97,7 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: sparse-checkout: | - devops/actions + devops sycl/cts_exclude_filter - uses: TheMrMilchmann/setup-msvc-dev@368ef7d1ee4d1171b31d4a7f67f4d954f903f5a9 # v4.1.0 with: From 7564b800bad57c9395c1644ace103057e3d699bc Mon Sep 17 00:00:00 2001 From: Katarzyna Kaczmarska Date: Tue, 6 Oct 2026 10:34:55 +0200 Subject: [PATCH 04/19] Include devops/scripts in build-job sparse checkouts sycl-linux-build.yml and sycl-windows-build.yml only checked out devops/actions, so devops/scripts/ur-test (invoked by the E2E build-only pack step via run-tests/linux(windows)/e2e -> run-tests/ur) was missing, making these jobs fail immediately and upload an incomplete/empty E2E binaries artifact to downstream run-only jobs. --- .github/workflows/sycl-linux-build.yml | 2 +- .github/workflows/sycl-windows-build.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sycl-linux-build.yml b/.github/workflows/sycl-linux-build.yml index 54c5bcbaf9cea..527f5a937e99b 100644 --- a/.github/workflows/sycl-linux-build.yml +++ b/.github/workflows/sycl-linux-build.yml @@ -175,7 +175,7 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: sparse-checkout: | - devops/actions + devops # Cleanup will be run after all actions are completed. - name: Register cleanup after job is finished uses: ./devops/actions/cleanup diff --git a/.github/workflows/sycl-windows-build.yml b/.github/workflows/sycl-windows-build.yml index 5d8edfde7bf45..042992623c621 100644 --- a/.github/workflows/sycl-windows-build.yml +++ b/.github/workflows/sycl-windows-build.yml @@ -117,7 +117,7 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: sparse-checkout: | - devops/actions + devops - uses: TheMrMilchmann/setup-msvc-dev@368ef7d1ee4d1171b31d4a7f67f4d954f903f5a9 # v4.1.0 with: arch: x64 From eebb84b0ef81bcb9b07166350977dd5c98e175f1 Mon Sep 17 00:00:00 2001 From: Katarzyna Kaczmarska Date: Tue, 6 Oct 2026 11:37:27 +0200 Subject: [PATCH 05/19] Install ur-test Python deps into workspace dir, not $HOME --- devops/actions/run-tests/ur/action.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/devops/actions/run-tests/ur/action.yml b/devops/actions/run-tests/ur/action.yml index 58a89219dacaa..894bf0d418823 100644 --- a/devops/actions/run-tests/ur/action.yml +++ b/devops/actions/run-tests/ur/action.yml @@ -36,9 +36,14 @@ runs: PY: ${{ steps.python.outputs.exe }} run: | "$PY" -c "import defusedxml" 2>/dev/null && exit 0 + # Install into a workspace-local dir rather than --user: some containers run + # as a UID that cannot write to $HOME (where --user installs always land). + deps_dir="$GITHUB_WORKSPACE/.ur-test-deps" + mkdir -p "$deps_dir" # --break-system-packages: nightly images have no venv and broken apt, same as devops/actions/run-tests/benchmark. - "$PY" -m pip install --disable-pip-version-check --quiet --user --break-system-packages defusedxml==0.7.1 \ - || "$PY" -m pip install --disable-pip-version-check --quiet --user defusedxml==0.7.1 + "$PY" -m pip install --disable-pip-version-check --quiet --break-system-packages --target "$deps_dir" defusedxml==0.7.1 \ + || "$PY" -m pip install --disable-pip-version-check --quiet --target "$deps_dir" defusedxml==0.7.1 + echo "PYTHONPATH=$deps_dir:$PYTHONPATH" >> "$GITHUB_ENV" - name: Run ${{ inputs.test_type }} tests id: run_tests From 32ca47ef86ecaa6168a96757374afdc401e11280 Mon Sep 17 00:00:00 2001 From: Katarzyna Kaczmarska Date: Tue, 6 Oct 2026 15:10:25 +0200 Subject: [PATCH 06/19] Avoid writing to $GITHUB_ENV, use step output instead --- devops/actions/run-tests/ur/action.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/devops/actions/run-tests/ur/action.yml b/devops/actions/run-tests/ur/action.yml index 894bf0d418823..7edf87bd71cc8 100644 --- a/devops/actions/run-tests/ur/action.yml +++ b/devops/actions/run-tests/ur/action.yml @@ -31,6 +31,7 @@ runs: fi - name: Ensure ur-test Python dependencies + id: deps shell: bash env: PY: ${{ steps.python.outputs.exe }} @@ -43,7 +44,7 @@ runs: # --break-system-packages: nightly images have no venv and broken apt, same as devops/actions/run-tests/benchmark. "$PY" -m pip install --disable-pip-version-check --quiet --break-system-packages --target "$deps_dir" defusedxml==0.7.1 \ || "$PY" -m pip install --disable-pip-version-check --quiet --target "$deps_dir" defusedxml==0.7.1 - echo "PYTHONPATH=$deps_dir:$PYTHONPATH" >> "$GITHUB_ENV" + echo "deps-dir=$deps_dir" >> "$GITHUB_OUTPUT" - name: Run ${{ inputs.test_type }} tests id: run_tests @@ -54,6 +55,7 @@ runs: TEST_TYPE: ${{ inputs.test_type }} EXTRA_LIT_OPTS: ${{ inputs.extra_lit_opts }} PY: ${{ steps.python.outputs.exe }} + PYTHONPATH: ${{ steps.deps.outputs['deps-dir'] }} run: | "$PY" devops/scripts/ur-test run "$TEST_TYPE" "$BUILD_DIR" "$GITHUB_WORKSPACE" From 245876182126dd27e0cbe41566354aae2aa4080f Mon Sep 17 00:00:00 2001 From: Katarzyna Kaczmarska Date: Tue, 6 Oct 2026 15:38:48 +0200 Subject: [PATCH 07/19] Add per-test --timeout to E2E lit invocations A single hung test (e.g. a GPU driver hang/device-lost) could previously block the whole suite until the job's external timeout-minutes fired, cascading a timeout failure into every later composite-action step (filter-log, report, pack/upload artifacts, cleanup). lit's --timeout kills the hung test itself and lets the run finish normally. This requires psutil, so proactively install it alongside defusedxml. --- devops/actions/run-tests/linux/e2e/action.yml | 4 +++- devops/actions/run-tests/ur/action.yml | 12 +++++++++--- devops/actions/run-tests/windows/e2e/action.yml | 4 +++- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/devops/actions/run-tests/linux/e2e/action.yml b/devops/actions/run-tests/linux/e2e/action.yml index 4d1b953e0e797..8edebb44e9020 100644 --- a/devops/actions/run-tests/linux/e2e/action.yml +++ b/devops/actions/run-tests/linux/e2e/action.yml @@ -81,7 +81,9 @@ runs: TARGET_DEVICES: ${{ inputs.target_devices }} USER_LIT_OPTS: ${{ inputs.extra_lit_opts }} run: | - echo "opts=--max-time 3600 --param print_features=True --param test-mode=${TESTING_MODE} --param sycl_devices=${TARGET_DEVICES} ${USER_LIT_OPTS}" >> "$GITHUB_OUTPUT" + # --timeout kills an individual hung test (e.g. a GPU driver hang) instead of + # letting it block the whole suite until the job's own timeout-minutes fires. + echo "opts=--max-time 3600 --timeout 600 --param print_features=True --param test-mode=${TESTING_MODE} --param sycl_devices=${TARGET_DEVICES} ${USER_LIT_OPTS}" >> "$GITHUB_OUTPUT" - name: Build artifact name if: ${{ always() && inputs.job_name != '' }} diff --git a/devops/actions/run-tests/ur/action.yml b/devops/actions/run-tests/ur/action.yml index 7edf87bd71cc8..05a8e3a714886 100644 --- a/devops/actions/run-tests/ur/action.yml +++ b/devops/actions/run-tests/ur/action.yml @@ -36,14 +36,20 @@ runs: env: PY: ${{ steps.python.outputs.exe }} run: | - "$PY" -c "import defusedxml" 2>/dev/null && exit 0 + missing="" + "$PY" -c "import defusedxml" 2>/dev/null || missing="$missing defusedxml==0.7.1" + # psutil is required by lit's --timeout (kills hung test processes); without + # it lit aborts the whole run at startup instead of just skipping the option. + "$PY" -c "import psutil" 2>/dev/null || missing="$missing psutil>=7.0.0" + [ -z "$missing" ] && exit 0 + # Install into a workspace-local dir rather than --user: some containers run # as a UID that cannot write to $HOME (where --user installs always land). deps_dir="$GITHUB_WORKSPACE/.ur-test-deps" mkdir -p "$deps_dir" # --break-system-packages: nightly images have no venv and broken apt, same as devops/actions/run-tests/benchmark. - "$PY" -m pip install --disable-pip-version-check --quiet --break-system-packages --target "$deps_dir" defusedxml==0.7.1 \ - || "$PY" -m pip install --disable-pip-version-check --quiet --target "$deps_dir" defusedxml==0.7.1 + "$PY" -m pip install --disable-pip-version-check --quiet --break-system-packages --target "$deps_dir" $missing \ + || "$PY" -m pip install --disable-pip-version-check --quiet --target "$deps_dir" $missing echo "deps-dir=$deps_dir" >> "$GITHUB_OUTPUT" - name: Run ${{ inputs.test_type }} tests diff --git a/devops/actions/run-tests/windows/e2e/action.yml b/devops/actions/run-tests/windows/e2e/action.yml index 1cc4e60fec90c..2152f18597a59 100644 --- a/devops/actions/run-tests/windows/e2e/action.yml +++ b/devops/actions/run-tests/windows/e2e/action.yml @@ -85,7 +85,9 @@ runs: USER_LIT_OPTS: ${{ inputs.extra_lit_opts }} MAX_TIME: ${{ inputs.testing_mode == 'run-only' && 1200 || 3600 }} run: | - echo "opts=--max-time ${MAX_TIME} --param print_features=True --param test-mode=${TESTING_MODE} --param sycl_devices=${TARGET_DEVICES} ${USER_LIT_OPTS}" >> "$GITHUB_OUTPUT" + # --timeout kills an individual hung test (e.g. a GPU driver hang) instead of + # letting it block the whole suite until the job's own timeout-minutes fires. + echo "opts=--max-time ${MAX_TIME} --timeout 600 --param print_features=True --param test-mode=${TESTING_MODE} --param sycl_devices=${TARGET_DEVICES} ${USER_LIT_OPTS}" >> "$GITHUB_OUTPUT" - name: Build artifact name if: ${{ always() && inputs.job_name != '' }} From c63e963439522ff9fc9bf3ab748a346a3e5c5c20 Mon Sep 17 00:00:00 2001 From: Katarzyna Kaczmarska Date: Tue, 6 Oct 2026 15:41:39 +0200 Subject: [PATCH 08/19] Fail fast when E2E binaries artifact is empty/broken Previously a failed build-only job would still pack and upload whatever was in build-e2e (even just the bare CMake configure skeleton), and every downstream run-only job would then burn CI time running thousands of tests doomed to fail with 'command not found'. Now both sides check for at least one compiled binary under */Output/ and fail immediately with a clear diagnostic instead of propagating broken state silently. --- devops/actions/run-tests/linux/e2e/action.yml | 17 +++++++++++++++++ devops/actions/run-tests/windows/e2e/action.yml | 13 +++++++++++++ 2 files changed, 30 insertions(+) diff --git a/devops/actions/run-tests/linux/e2e/action.yml b/devops/actions/run-tests/linux/e2e/action.yml index 8edebb44e9020..25794899203f6 100644 --- a/devops/actions/run-tests/linux/e2e/action.yml +++ b/devops/actions/run-tests/linux/e2e/action.yml @@ -47,6 +47,10 @@ runs: run: | mkdir build-e2e tar -I 'zstd' -xf e2e_binaries.tar.zst -C build-e2e + if [ -z "$(find build-e2e -path '*/Output/*' -type f -print -quit)" ]; then + echo "::error::Downloaded E2E binaries artifact '${{ inputs.binaries_artifact }}' has no compiled test binaries under */Output/ (likely produced by a failed upstream build job). Failing fast instead of running thousands of doomed tests." + exit 1 + fi - name: Extract E2E tests from container image if: ${{ inputs.testing_mode == 'run-only' && inputs.binaries_artifact == 'in-container' }} @@ -55,6 +59,10 @@ runs: mkdir build-e2e llvm tar -I 'zstd' -xf /sycl-prebuilt/e2e_binaries.tar.zst -C build-e2e tar -I 'zstd' -xf /sycl-prebuilt/e2e_sources.tar.zst -C llvm + if [ -z "$(find build-e2e -path '*/Output/*' -type f -print -quit)" ]; then + echo "::error::Container image's prebuilt E2E binaries have no compiled test binaries under */Output/. Failing fast instead of running thousands of doomed tests." + exit 1 + fi - name: Deduce E2E CMake options if: inputs.testing_mode != 'run-only' @@ -103,6 +111,15 @@ runs: extra_lit_opts: ${{ steps.lit_opts.outputs.opts }} artifact_name: ${{ steps.artifact_name.outputs.name }} + - name: Verify E2E binaries were built + if: ${{ !cancelled() && inputs.binaries_artifact != '' && inputs.testing_mode != 'run-only' }} + shell: bash + run: | + if [ -z "$(find build-e2e -path '*/Output/*' -type f -print -quit)" ]; then + echo "::error::No compiled E2E test binaries found under build-e2e/**/Output/ after the build step. Refusing to upload a broken artifact." + exit 1 + fi + - name: Pack E2E binaries if: ${{ !cancelled() && inputs.binaries_artifact != '' && inputs.testing_mode != 'run-only' }} shell: bash diff --git a/devops/actions/run-tests/windows/e2e/action.yml b/devops/actions/run-tests/windows/e2e/action.yml index 2152f18597a59..c2ab706d47d2d 100644 --- a/devops/actions/run-tests/windows/e2e/action.yml +++ b/devops/actions/run-tests/windows/e2e/action.yml @@ -48,6 +48,10 @@ runs: run: | mkdir build-e2e tar -xf e2e_bin.tar.gz -C build-e2e + if [ -z "$(find build-e2e -path '*/Output/*' -type f -print -quit)" ]; then + echo "::error::Downloaded E2E binaries artifact '${{ inputs.binaries_artifact }}' has no compiled test binaries under */Output/ (likely produced by a failed upstream build job). Failing fast instead of running thousands of doomed tests." + exit 1 + fi - name: Deduce E2E CMake options id: cmake_opts @@ -118,6 +122,15 @@ runs: for FILE in $(cat e2econf_files.txt); do rm -r build-e2e/$FILE; done rm e2econf_files.txt + - name: Verify E2E binaries were built + if: ${{ !cancelled() && inputs.binaries_artifact != '' && inputs.testing_mode != 'run-only' }} + shell: bash + run: | + if [ -z "$(find build-e2e -path '*/Output/*' -type f -print -quit)" ]; then + echo "::error::No compiled E2E test binaries found under build-e2e/**/Output/ after the build step. Refusing to upload a broken artifact." + exit 1 + fi + - name: Pack E2E test binaries if: ${{ !cancelled() && inputs.binaries_artifact != '' && inputs.testing_mode != 'run-only' }} shell: bash From 823dcbd4132c9ceef39ac337962e833e9c3ef7be Mon Sep 17 00:00:00 2001 From: Katarzyna Kaczmarska Date: Tue, 6 Oct 2026 15:49:56 +0200 Subject: [PATCH 09/19] Fix Slowest Tests/Test Times header matching for lit's count suffix --- devops/scripts/ur_test_tools/constants.py | 5 ++- .../scripts/ur_test_tools/outputs/console.py | 4 +- .../ur_test_tools/parsers/log_parser.py | 8 ++-- .../tests/test_result_reconciliation.py | 37 +++++++++++++++++++ 4 files changed, 47 insertions(+), 7 deletions(-) diff --git a/devops/scripts/ur_test_tools/constants.py b/devops/scripts/ur_test_tools/constants.py index 6e695fb888827..f7d831c6e8d69 100644 --- a/devops/scripts/ur_test_tools/constants.py +++ b/devops/scripts/ur_test_tools/constants.py @@ -38,8 +38,9 @@ ) TEST_NOT_SELECTED_MSG = "Test not selected" -SLOWEST_TESTS_HEADER = "Slowest Tests:" -TEST_TIMES_HEADERS = ("Tests Times:", "Test Times:") +# No trailing colon: lit appends a dynamic "(N of M):"/"(N):" suffix to these. +SLOWEST_TESTS_HEADER = "Slowest Tests" +TEST_TIMES_HEADERS = ("Tests Times", "Test Times") FAIL_TIMEOUT_PATTERN = re.compile(r"^(FAIL|TIMEOUT):") TEST_CATEGORY_PATTERN = re.compile(r"^([A-Za-z]+(?: [A-Za-z]+)*) Tests \((\d+)\):") diff --git a/devops/scripts/ur_test_tools/outputs/console.py b/devops/scripts/ur_test_tools/outputs/console.py index 2d88e3a29ca0c..70e124c92a2bf 100644 --- a/devops/scripts/ur_test_tools/outputs/console.py +++ b/devops/scripts/ur_test_tools/outputs/console.py @@ -27,7 +27,9 @@ def filter_log_for_display(lines: List[str]) -> List[str]: skip_until_empty = True continue - if stripped == SLOWEST_TESTS_HEADER or stripped in TEST_TIMES_HEADERS: + if stripped.startswith(SLOWEST_TESTS_HEADER) or any( + stripped.startswith(h) for h in TEST_TIMES_HEADERS + ): in_timing = True continue diff --git a/devops/scripts/ur_test_tools/parsers/log_parser.py b/devops/scripts/ur_test_tools/parsers/log_parser.py index d312c0438b4ff..227c5cb63b690 100644 --- a/devops/scripts/ur_test_tools/parsers/log_parser.py +++ b/devops/scripts/ur_test_tools/parsers/log_parser.py @@ -67,8 +67,8 @@ def extract_error_details(self) -> List[str]: if in_error and ( TEST_CATEGORY_PATTERN.match(line) - or line.strip() == SLOWEST_TESTS_HEADER - or line.strip() in TEST_TIMES_HEADERS + or line.strip().startswith(SLOWEST_TESTS_HEADER) + or any(line.strip().startswith(h) for h in TEST_TIMES_HEADERS) ): break @@ -101,11 +101,11 @@ def extract_time_summary(self) -> Tuple[List[str], List[str]]: for line in self.lines: stripped = line.strip() - if stripped == SLOWEST_TESTS_HEADER: + if stripped.startswith(SLOWEST_TESTS_HEADER): current_section = "slowest" skip_next_hr = True continue - elif stripped in TEST_TIMES_HEADERS: + elif any(stripped.startswith(h) for h in TEST_TIMES_HEADERS): current_section = "histogram" skip_next_hr = True continue diff --git a/devops/scripts/ur_test_tools/tests/test_result_reconciliation.py b/devops/scripts/ur_test_tools/tests/test_result_reconciliation.py index 65326233c091c..d0a27a18357e1 100644 --- a/devops/scripts/ur_test_tools/tests/test_result_reconciliation.py +++ b/devops/scripts/ur_test_tools/tests/test_result_reconciliation.py @@ -179,6 +179,43 @@ def test_sums_repeated_statistics_labels(self): self.assertEqual(result["Passed"], 5) + def test_extracts_time_summary_with_lit_count_suffix(self): + # lit appends a dynamic "(N of M)"/"(N)" suffix to these headers. + lines = [ + "Slowest Tests (20 of 2022):\n", + "-" * 70 + "\n", + "41.86s: SYCL :: WorkGroupMemory/basic_usage.cpp\n", + "\n", + "Test Times (2022):\n", + "-" * 70 + "\n", + "[40.0s,42.0s) :: [] :: [ 1/2022]\n", + "-" * 70 + "\n", + ] + + slowest, histogram = LITLogParser(lines).extract_time_summary() + + self.assertEqual( + slowest, ["41.86s: SYCL :: WorkGroupMemory/basic_usage.cpp"] + ) + self.assertEqual( + histogram, + ["[40.0s,42.0s) :: [] :: [ 1/2022]", "-" * 70], + ) + + def test_error_details_stop_at_lit_count_suffixed_slowest_header(self): + lines = [ + "FAIL: Suite :: test.cpp (1 of 1)\n", + "some failure output\n", + "Slowest Tests (1 of 1):\n", + "0.10s: Suite :: test.cpp\n", + ] + + details = LITLogParser(lines).extract_error_details() + + self.assertEqual( + details, ["FAIL: Suite :: test.cpp (1 of 1)\n", "some failure output\n"] + ) + def test_sums_repeated_category_counts(self): lines = [ "Passed Tests (1):\n", From 7e62dd04214d13fd087b14bd408e169969c02891 Mon Sep 17 00:00:00 2001 From: Katarzyna Kaczmarska Date: Tue, 6 Oct 2026 15:52:43 +0200 Subject: [PATCH 10/19] Fix zizmor code-injection-via-template-expansion finding --- devops/actions/run-tests/linux/e2e/action.yml | 4 +++- devops/actions/run-tests/windows/e2e/action.yml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/devops/actions/run-tests/linux/e2e/action.yml b/devops/actions/run-tests/linux/e2e/action.yml index 25794899203f6..3476b10469972 100644 --- a/devops/actions/run-tests/linux/e2e/action.yml +++ b/devops/actions/run-tests/linux/e2e/action.yml @@ -44,11 +44,13 @@ runs: - name: Extract E2E Binaries if: ${{ inputs.testing_mode == 'run-only' && inputs.binaries_artifact != 'in-container' }} shell: bash + env: + BINARIES_ARTIFACT: ${{ inputs.binaries_artifact }} run: | mkdir build-e2e tar -I 'zstd' -xf e2e_binaries.tar.zst -C build-e2e if [ -z "$(find build-e2e -path '*/Output/*' -type f -print -quit)" ]; then - echo "::error::Downloaded E2E binaries artifact '${{ inputs.binaries_artifact }}' has no compiled test binaries under */Output/ (likely produced by a failed upstream build job). Failing fast instead of running thousands of doomed tests." + echo "::error::Downloaded E2E binaries artifact '$BINARIES_ARTIFACT' has no compiled test binaries under */Output/ (likely produced by a failed upstream build job). Failing fast instead of running thousands of doomed tests." exit 1 fi diff --git a/devops/actions/run-tests/windows/e2e/action.yml b/devops/actions/run-tests/windows/e2e/action.yml index c2ab706d47d2d..95fae4291e0f8 100644 --- a/devops/actions/run-tests/windows/e2e/action.yml +++ b/devops/actions/run-tests/windows/e2e/action.yml @@ -45,11 +45,13 @@ runs: - name: Extract E2E Binaries if: inputs.testing_mode == 'run-only' shell: bash + env: + BINARIES_ARTIFACT: ${{ inputs.binaries_artifact }} run: | mkdir build-e2e tar -xf e2e_bin.tar.gz -C build-e2e if [ -z "$(find build-e2e -path '*/Output/*' -type f -print -quit)" ]; then - echo "::error::Downloaded E2E binaries artifact '${{ inputs.binaries_artifact }}' has no compiled test binaries under */Output/ (likely produced by a failed upstream build job). Failing fast instead of running thousands of doomed tests." + echo "::error::Downloaded E2E binaries artifact '$BINARIES_ARTIFACT' has no compiled test binaries under */Output/ (likely produced by a failed upstream build job). Failing fast instead of running thousands of doomed tests." exit 1 fi From 0709e643ae99f99ddfc524fa7aba262440d90691 Mon Sep 17 00:00:00 2001 From: Katarzyna Kaczmarska Date: Tue, 6 Oct 2026 16:12:22 +0200 Subject: [PATCH 11/19] Propagate PYTHONPATH to all ur-test invoking steps --- devops/actions/run-tests/ur/action.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/devops/actions/run-tests/ur/action.yml b/devops/actions/run-tests/ur/action.yml index 05a8e3a714886..2a451e467bb1d 100644 --- a/devops/actions/run-tests/ur/action.yml +++ b/devops/actions/run-tests/ur/action.yml @@ -71,6 +71,7 @@ runs: env: LOG_FILE: ${{ steps.run_tests.outputs['log-file'] }} PY: ${{ steps.python.outputs.exe }} + PYTHONPATH: ${{ steps.deps.outputs['deps-dir'] }} run: | if [ -z "$LOG_FILE" ]; then echo "::warning::Test log path not provided by runner; skipping log filtering." @@ -92,6 +93,7 @@ runs: env: LOG_FILE: ${{ steps.run_tests.outputs['log-file'] }} PY: ${{ steps.python.outputs.exe }} + PYTHONPATH: ${{ steps.deps.outputs['deps-dir'] }} run: | if [ -z "$LOG_FILE" ] || [ ! -f "$LOG_FILE" ]; then echo "::warning::Test execution failed, but no test log is available." @@ -111,6 +113,7 @@ runs: TEST_TYPE: ${{ inputs.test_type }} WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} PY: ${{ steps.python.outputs.exe }} + PYTHONPATH: ${{ steps.deps.outputs['deps-dir'] }} run: | if [ -z "$LOG_FILE" ]; then echo "::warning::Test log path not provided by runner; skipping summary." From fd5683947c78e1bd622ccfa809c94192d5912f79 Mon Sep 17 00:00:00 2001 From: Katarzyna Kaczmarska Date: Tue, 6 Oct 2026 17:40:25 +0200 Subject: [PATCH 12/19] Fix darker formatting in test_result_reconciliation.py --- .../scripts/ur_test_tools/tests/test_result_reconciliation.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/devops/scripts/ur_test_tools/tests/test_result_reconciliation.py b/devops/scripts/ur_test_tools/tests/test_result_reconciliation.py index d0a27a18357e1..dbfba00610f6b 100644 --- a/devops/scripts/ur_test_tools/tests/test_result_reconciliation.py +++ b/devops/scripts/ur_test_tools/tests/test_result_reconciliation.py @@ -194,9 +194,7 @@ def test_extracts_time_summary_with_lit_count_suffix(self): slowest, histogram = LITLogParser(lines).extract_time_summary() - self.assertEqual( - slowest, ["41.86s: SYCL :: WorkGroupMemory/basic_usage.cpp"] - ) + self.assertEqual(slowest, ["41.86s: SYCL :: WorkGroupMemory/basic_usage.cpp"]) self.assertEqual( histogram, ["[40.0s,42.0s) :: [] :: [ 1/2022]", "-" * 70], From 21033a8a5fdbf06621ce175621eca6f5f60c89ca Mon Sep 17 00:00:00 2001 From: Katarzyna Kaczmarska Date: Tue, 6 Oct 2026 18:10:45 +0200 Subject: [PATCH 13/19] Move ur-test Python dependency management to a dedicated setup action --- .github/workflows/sycl-linux-build.yml | 3 +- .github/workflows/sycl-linux-precommit.yml | 3 +- .github/workflows/sycl-linux-run-tests.yml | 3 +- .../workflows/sycl-prebuilt-e2e-container.yml | 3 +- .github/workflows/sycl-windows-build.yml | 3 +- .github/workflows/sycl-windows-run-tests.yml | 3 +- devops/actions/run-tests/ur/action.yml | 50 ++++------------- devops/actions/setup-ur-test/action.yml | 54 +++++++++++++++++++ devops/scripts/ur_test_tools/requirements.txt | 3 ++ 9 files changed, 79 insertions(+), 46 deletions(-) create mode 100644 devops/actions/setup-ur-test/action.yml create mode 100644 devops/scripts/ur_test_tools/requirements.txt diff --git a/.github/workflows/sycl-linux-build.yml b/.github/workflows/sycl-linux-build.yml index 527f5a937e99b..e8440947ed009 100644 --- a/.github/workflows/sycl-linux-build.yml +++ b/.github/workflows/sycl-linux-build.yml @@ -175,7 +175,8 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: sparse-checkout: | - devops + devops/actions + devops/scripts # Cleanup will be run after all actions are completed. - name: Register cleanup after job is finished uses: ./devops/actions/cleanup diff --git a/.github/workflows/sycl-linux-precommit.yml b/.github/workflows/sycl-linux-precommit.yml index 68a6ce9f939ee..ec7616e60e59b 100644 --- a/.github/workflows/sycl-linux-precommit.yml +++ b/.github/workflows/sycl-linux-precommit.yml @@ -81,7 +81,8 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: sparse-checkout: | - devops/ + devops/actions + devops/scripts # download build artefact - name: Download toolchain diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml index 0901e06c91eec..fcfd32c4f8ec9 100644 --- a/.github/workflows/sycl-linux-run-tests.yml +++ b/.github/workflows/sycl-linux-run-tests.yml @@ -261,7 +261,8 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: sparse-checkout: | - devops + devops/actions + devops/scripts sycl/cts_exclude_filter - name: Register cleanup after job is finished uses: ./devops/actions/cleanup diff --git a/.github/workflows/sycl-prebuilt-e2e-container.yml b/.github/workflows/sycl-prebuilt-e2e-container.yml index 15891fc59f6f2..734eeae5d840d 100644 --- a/.github/workflows/sycl-prebuilt-e2e-container.yml +++ b/.github/workflows/sycl-prebuilt-e2e-container.yml @@ -102,7 +102,8 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: sparse-checkout: | - devops + devops/actions + devops/scripts - run: | mkdir toolchain tar -I 'zstd' -xf /sycl-prebuilt/toolchain.tar.zst -C toolchain diff --git a/.github/workflows/sycl-windows-build.yml b/.github/workflows/sycl-windows-build.yml index 042992623c621..ca91860646962 100644 --- a/.github/workflows/sycl-windows-build.yml +++ b/.github/workflows/sycl-windows-build.yml @@ -117,7 +117,8 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: sparse-checkout: | - devops + devops/actions + devops/scripts - uses: TheMrMilchmann/setup-msvc-dev@368ef7d1ee4d1171b31d4a7f67f4d954f903f5a9 # v4.1.0 with: arch: x64 diff --git a/.github/workflows/sycl-windows-run-tests.yml b/.github/workflows/sycl-windows-run-tests.yml index 89c7da642e2b3..5b683be76c596 100644 --- a/.github/workflows/sycl-windows-run-tests.yml +++ b/.github/workflows/sycl-windows-run-tests.yml @@ -97,7 +97,8 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: sparse-checkout: | - devops + devops/actions + devops/scripts sycl/cts_exclude_filter - uses: TheMrMilchmann/setup-msvc-dev@368ef7d1ee4d1171b31d4a7f67f4d954f903f5a9 # v4.1.0 with: diff --git a/devops/actions/run-tests/ur/action.yml b/devops/actions/run-tests/ur/action.yml index 2a451e467bb1d..ec4e2bf719600 100644 --- a/devops/actions/run-tests/ur/action.yml +++ b/devops/actions/run-tests/ur/action.yml @@ -19,38 +19,9 @@ inputs: runs: using: "composite" steps: - # `python3` may not be on PATH on Windows runners; resolve it once for reuse below. - - name: Resolve Python interpreter - id: python - shell: bash - run: | - if command -v python3 >/dev/null 2>&1; then - echo "exe=python3" >> "$GITHUB_OUTPUT" - else - echo "exe=python" >> "$GITHUB_OUTPUT" - fi - - - name: Ensure ur-test Python dependencies - id: deps - shell: bash - env: - PY: ${{ steps.python.outputs.exe }} - run: | - missing="" - "$PY" -c "import defusedxml" 2>/dev/null || missing="$missing defusedxml==0.7.1" - # psutil is required by lit's --timeout (kills hung test processes); without - # it lit aborts the whole run at startup instead of just skipping the option. - "$PY" -c "import psutil" 2>/dev/null || missing="$missing psutil>=7.0.0" - [ -z "$missing" ] && exit 0 - - # Install into a workspace-local dir rather than --user: some containers run - # as a UID that cannot write to $HOME (where --user installs always land). - deps_dir="$GITHUB_WORKSPACE/.ur-test-deps" - mkdir -p "$deps_dir" - # --break-system-packages: nightly images have no venv and broken apt, same as devops/actions/run-tests/benchmark. - "$PY" -m pip install --disable-pip-version-check --quiet --break-system-packages --target "$deps_dir" $missing \ - || "$PY" -m pip install --disable-pip-version-check --quiet --target "$deps_dir" $missing - echo "deps-dir=$deps_dir" >> "$GITHUB_OUTPUT" + - name: Set up ur-test + id: ur_test + uses: ./devops/actions/setup-ur-test - name: Run ${{ inputs.test_type }} tests id: run_tests @@ -60,8 +31,10 @@ runs: BUILD_DIR: ${{ inputs.build_dir }} TEST_TYPE: ${{ inputs.test_type }} EXTRA_LIT_OPTS: ${{ inputs.extra_lit_opts }} - PY: ${{ steps.python.outputs.exe }} - PYTHONPATH: ${{ steps.deps.outputs['deps-dir'] }} + PY: ${{ steps.ur_test.outputs.python }} + # lit is spawned by cmake/ninja using the system Python, not the venv above, + # so it needs this on PYTHONPATH to see psutil (required by --timeout). + PYTHONPATH: ${{ steps.ur_test.outputs['site-packages'] }} run: | "$PY" devops/scripts/ur-test run "$TEST_TYPE" "$BUILD_DIR" "$GITHUB_WORKSPACE" @@ -70,8 +43,7 @@ runs: shell: bash env: LOG_FILE: ${{ steps.run_tests.outputs['log-file'] }} - PY: ${{ steps.python.outputs.exe }} - PYTHONPATH: ${{ steps.deps.outputs['deps-dir'] }} + PY: ${{ steps.ur_test.outputs.python }} run: | if [ -z "$LOG_FILE" ]; then echo "::warning::Test log path not provided by runner; skipping log filtering." @@ -92,8 +64,7 @@ runs: shell: bash env: LOG_FILE: ${{ steps.run_tests.outputs['log-file'] }} - PY: ${{ steps.python.outputs.exe }} - PYTHONPATH: ${{ steps.deps.outputs['deps-dir'] }} + PY: ${{ steps.ur_test.outputs.python }} run: | if [ -z "$LOG_FILE" ] || [ ! -f "$LOG_FILE" ]; then echo "::warning::Test execution failed, but no test log is available." @@ -112,8 +83,7 @@ runs: ARTIFACT_NAME: ${{ inputs.artifact_name }} TEST_TYPE: ${{ inputs.test_type }} WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - PY: ${{ steps.python.outputs.exe }} - PYTHONPATH: ${{ steps.deps.outputs['deps-dir'] }} + PY: ${{ steps.ur_test.outputs.python }} run: | if [ -z "$LOG_FILE" ]; then echo "::warning::Test log path not provided by runner; skipping summary." diff --git a/devops/actions/setup-ur-test/action.yml b/devops/actions/setup-ur-test/action.yml new file mode 100644 index 0000000000000..44a798d1ca8d1 --- /dev/null +++ b/devops/actions/setup-ur-test/action.yml @@ -0,0 +1,54 @@ +name: 'Set up ur-test' +description: > + Provision an isolated Python environment with ur_test_tools' own + dependencies (devops/scripts/ur_test_tools/requirements.txt) installed, + without touching the system/base Python installation. + +outputs: + python: + description: 'Path to the Python interpreter with ur-test dependencies installed.' + value: ${{ steps.setup.outputs.python }} + site-packages: + description: > + Path to the directory containing ur-test's dependencies. Needed on + PYTHONPATH only by other processes (e.g. lit, run via a different, + system Python) that must see these packages but do not use the + interpreter above directly. + value: ${{ steps.setup.outputs.site-packages }} + +runs: + using: "composite" + steps: + - name: Provision ur-test Python environment + id: setup + shell: bash + run: | + # `python3` may not be on PATH on Windows runners. + if command -v python3 >/dev/null 2>&1; then + base_py=python3 + else + base_py=python + fi + + venv_dir="${RUNNER_TEMP:-$GITHUB_WORKSPACE}/ur-test-venv" + if "$base_py" -m venv "$venv_dir" >/dev/null 2>&1; then + venv_py="$venv_dir/bin/python" + [ -f "$venv_py" ] || venv_py="$venv_dir/Scripts/python.exe" + "$venv_py" -m pip install --disable-pip-version-check --quiet \ + -r devops/scripts/ur_test_tools/requirements.txt + site_packages="$("$venv_py" -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')" + echo "python=$venv_py" >> "$GITHUB_OUTPUT" + echo "site-packages=$site_packages" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # Some containers (nightly images) have no venv/ensurepip and broken apt; + # fall back to a workspace-local --target install on the base interpreter. + echo "::warning::'python -m venv' unavailable; falling back to a --target install." + deps_dir="${RUNNER_TEMP:-$GITHUB_WORKSPACE}/ur-test-deps" + mkdir -p "$deps_dir" + # --break-system-packages: nightly images have no venv and broken apt, same as devops/actions/run-tests/benchmark. + "$base_py" -m pip install --disable-pip-version-check --quiet --break-system-packages --target "$deps_dir" -r devops/scripts/ur_test_tools/requirements.txt \ + || "$base_py" -m pip install --disable-pip-version-check --quiet --target "$deps_dir" -r devops/scripts/ur_test_tools/requirements.txt + echo "python=$base_py" >> "$GITHUB_OUTPUT" + echo "site-packages=$deps_dir" >> "$GITHUB_OUTPUT" diff --git a/devops/scripts/ur_test_tools/requirements.txt b/devops/scripts/ur_test_tools/requirements.txt new file mode 100644 index 0000000000000..6c631f784380b --- /dev/null +++ b/devops/scripts/ur_test_tools/requirements.txt @@ -0,0 +1,3 @@ +defusedxml==0.7.1 +# Required by lit's --timeout (kills hung test processes). +psutil>=7.0.0 From 5e9209eaf96ed944d3cfc7b5d0dd0f133427f72b Mon Sep 17 00:00:00 2001 From: Katarzyna Kaczmarska Date: Tue, 6 Oct 2026 18:44:28 +0200 Subject: [PATCH 14/19] idempotent provisioning, clearer description --- devops/actions/setup-ur-test/action.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/devops/actions/setup-ur-test/action.yml b/devops/actions/setup-ur-test/action.yml index 44a798d1ca8d1..1be2a10c05d8e 100644 --- a/devops/actions/setup-ur-test/action.yml +++ b/devops/actions/setup-ur-test/action.yml @@ -1,8 +1,8 @@ name: 'Set up ur-test' description: > - Provision an isolated Python environment with ur_test_tools' own - dependencies (devops/scripts/ur_test_tools/requirements.txt) installed, - without touching the system/base Python installation. + Provision the Python dependencies required by UR test reporting (ur-test) + and LIT execution (devops/scripts/ur_test_tools/requirements.txt), without + touching the system/base Python installation. outputs: python: @@ -31,6 +31,7 @@ runs: fi venv_dir="${RUNNER_TEMP:-$GITHUB_WORKSPACE}/ur-test-venv" + rm -rf "$venv_dir" if "$base_py" -m venv "$venv_dir" >/dev/null 2>&1; then venv_py="$venv_dir/bin/python" [ -f "$venv_py" ] || venv_py="$venv_dir/Scripts/python.exe" @@ -46,8 +47,9 @@ runs: # fall back to a workspace-local --target install on the base interpreter. echo "::warning::'python -m venv' unavailable; falling back to a --target install." deps_dir="${RUNNER_TEMP:-$GITHUB_WORKSPACE}/ur-test-deps" + rm -rf "$deps_dir" mkdir -p "$deps_dir" - # --break-system-packages: nightly images have no venv and broken apt, same as devops/actions/run-tests/benchmark. + # Same --break-system-packages fallback as run-tests/benchmark for nightly images. "$base_py" -m pip install --disable-pip-version-check --quiet --break-system-packages --target "$deps_dir" -r devops/scripts/ur_test_tools/requirements.txt \ || "$base_py" -m pip install --disable-pip-version-check --quiet --target "$deps_dir" -r devops/scripts/ur_test_tools/requirements.txt echo "python=$base_py" >> "$GITHUB_OUTPUT" From 0633b6983699bf3be6b4edf6fd80c76170559245 Mon Sep 17 00:00:00 2001 From: Katarzyna Kaczmarska Date: Wed, 7 Oct 2026 12:04:12 +0200 Subject: [PATCH 15/19] Fix PYTHONPATH propagation gap and missing E2E artifact names --- .github/workflows/sycl-linux-build.yml | 4 ++++ .github/workflows/sycl-linux-precommit.yml | 1 + .github/workflows/sycl-prebuilt-e2e-container.yml | 1 + .github/workflows/sycl-windows-build.yml | 2 ++ devops/actions/run-tests/ur/action.yml | 7 +++++-- 5 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sycl-linux-build.yml b/.github/workflows/sycl-linux-build.yml index e8440947ed009..90746c90611b7 100644 --- a/.github/workflows/sycl-linux-build.yml +++ b/.github/workflows/sycl-linux-build.yml @@ -330,6 +330,7 @@ jobs: testing_mode: build-only target_devices: all binaries_artifact: ${{ inputs.e2e_binaries_artifact }} + job_name: ${{ inputs.e2e_binaries_artifact }} sycl_compiler: $GITHUB_WORKSPACE/toolchain/bin/dpclang++ extra_lit_opts: --param sycl_build_targets="spir;nvidia" @@ -345,6 +346,7 @@ jobs: testing_mode: build-only target_devices: all binaries_artifact: ${{ inputs.e2e_binaries_new_offload_model_artifact }} + job_name: ${{ inputs.e2e_binaries_new_offload_model_artifact }} sycl_compiler: $GITHUB_WORKSPACE/toolchain/bin/dpclang++ extra_lit_opts: --param sycl_build_targets="spir;nvidia" --param enable_new_offload_model=True @@ -356,6 +358,7 @@ jobs: testing_mode: build-only target_devices: all binaries_artifact: ${{ inputs.e2e_binaries_spirv_backend_artifact }} + job_name: ${{ inputs.e2e_binaries_spirv_backend_artifact }} sycl_compiler: $GITHUB_WORKSPACE/toolchain/bin/dpclang++ extra_lit_opts: --param spirv-backend=True @@ -367,5 +370,6 @@ jobs: testing_mode: build-only target_devices: all binaries_artifact: ${{ inputs.e2e_binaries_preview_artifact }} + job_name: ${{ inputs.e2e_binaries_preview_artifact }} sycl_compiler: $GITHUB_WORKSPACE/toolchain/bin/dpclang++ extra_lit_opts: --param test-preview-mode=True diff --git a/.github/workflows/sycl-linux-precommit.yml b/.github/workflows/sycl-linux-precommit.yml index ec7616e60e59b..c28d0723702bd 100644 --- a/.github/workflows/sycl-linux-precommit.yml +++ b/.github/workflows/sycl-linux-precommit.yml @@ -101,6 +101,7 @@ jobs: ref: ${{ github.ref || github.sha }} testing_mode: full target_devices: native_cpu:cpu + job_name: ${{ github.job }} sycl_compiler: $GITHUB_WORKSPACE/toolchain/bin/dpclang++ extra_lit_opts: --param sycl_build_targets="native_cpu" extra_cmake_args: -DSYCL_TEST_E2E_TARGETS="native_cpu:cpu" -DSYCL_TEST_E2E_STANDALONE=ON diff --git a/.github/workflows/sycl-prebuilt-e2e-container.yml b/.github/workflows/sycl-prebuilt-e2e-container.yml index 734eeae5d840d..b5fdfe3210eb0 100644 --- a/.github/workflows/sycl-prebuilt-e2e-container.yml +++ b/.github/workflows/sycl-prebuilt-e2e-container.yml @@ -117,3 +117,4 @@ jobs: with: testing_mode: run-only target_devices: level_zero:gpu + job_name: ${{ github.job }} diff --git a/.github/workflows/sycl-windows-build.yml b/.github/workflows/sycl-windows-build.yml index ca91860646962..307db0338c4b2 100644 --- a/.github/workflows/sycl-windows-build.yml +++ b/.github/workflows/sycl-windows-build.yml @@ -258,6 +258,7 @@ jobs: testing_mode: build-only target_devices: all binaries_artifact: ${{ inputs.e2e_binaries_artifact }} + job_name: ${{ inputs.e2e_binaries_artifact }} extra_lit_opts: --param sycl_build_targets="spir" cxx: ${{ inputs.cxx }} @@ -273,6 +274,7 @@ jobs: testing_mode: build-only target_devices: all binaries_artifact: ${{ inputs.e2e_binaries_new_offload_model_artifact }} + job_name: ${{ inputs.e2e_binaries_new_offload_model_artifact }} extra_lit_opts: --param sycl_build_targets="spir" --param enable_new_offload_model=True cxx: ${{ inputs.cxx }} diff --git a/devops/actions/run-tests/ur/action.yml b/devops/actions/run-tests/ur/action.yml index ec4e2bf719600..20ae3b6de525f 100644 --- a/devops/actions/run-tests/ur/action.yml +++ b/devops/actions/run-tests/ur/action.yml @@ -32,8 +32,8 @@ runs: TEST_TYPE: ${{ inputs.test_type }} EXTRA_LIT_OPTS: ${{ inputs.extra_lit_opts }} PY: ${{ steps.ur_test.outputs.python }} - # lit is spawned by cmake/ninja using the system Python, not the venv above, - # so it needs this on PYTHONPATH to see psutil (required by --timeout). + # Needed by lit (spawned via cmake/ninja's system Python) to see psutil, + # and by ur-test itself under the --target fallback; set on every step below too. PYTHONPATH: ${{ steps.ur_test.outputs['site-packages'] }} run: | "$PY" devops/scripts/ur-test run "$TEST_TYPE" "$BUILD_DIR" "$GITHUB_WORKSPACE" @@ -44,6 +44,7 @@ runs: env: LOG_FILE: ${{ steps.run_tests.outputs['log-file'] }} PY: ${{ steps.ur_test.outputs.python }} + PYTHONPATH: ${{ steps.ur_test.outputs['site-packages'] }} run: | if [ -z "$LOG_FILE" ]; then echo "::warning::Test log path not provided by runner; skipping log filtering." @@ -65,6 +66,7 @@ runs: env: LOG_FILE: ${{ steps.run_tests.outputs['log-file'] }} PY: ${{ steps.ur_test.outputs.python }} + PYTHONPATH: ${{ steps.ur_test.outputs['site-packages'] }} run: | if [ -z "$LOG_FILE" ] || [ ! -f "$LOG_FILE" ]; then echo "::warning::Test execution failed, but no test log is available." @@ -84,6 +86,7 @@ runs: TEST_TYPE: ${{ inputs.test_type }} WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} PY: ${{ steps.ur_test.outputs.python }} + PYTHONPATH: ${{ steps.ur_test.outputs['site-packages'] }} run: | if [ -z "$LOG_FILE" ]; then echo "::warning::Test log path not provided by runner; skipping summary." From 57a2a346b938cf876948a1bb1f4d0c33dfa6d3be Mon Sep 17 00:00:00 2001 From: Katarzyna Kaczmarska Date: Wed, 7 Oct 2026 14:46:08 +0200 Subject: [PATCH 16/19] Fix Windows XML output path being mangled by shlex in LIT_OPTS --- devops/scripts/ur_test_tools/test_runner.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/devops/scripts/ur_test_tools/test_runner.py b/devops/scripts/ur_test_tools/test_runner.py index 4ea3c7f70b9f2..b6a7953874c3b 100644 --- a/devops/scripts/ur_test_tools/test_runner.py +++ b/devops/scripts/ur_test_tools/test_runner.py @@ -104,7 +104,13 @@ def _setup_environment(self) -> None: if config.lit_jobs is not None: lit_opts_parts += ["-j", str(config.lit_jobs)] - lit_opts_parts += ["--xunit-xml-output", str(self.context.xml_output_path)] + # Use forward slashes: LIT_OPTS is re-split with shlex (POSIX mode) on + # the lit side, which eats backslashes as escapes and mangles Windows + # paths (e.g. "D:\...\e2e_results.xml" -> "D:...e2e_resultsxml"). + lit_opts_parts += [ + "--xunit-xml-output", + self.context.xml_output_path.as_posix(), + ] if config.extra_lit_opts: lit_opts_parts.append(config.extra_lit_opts) From 8a5f4ea8aa2dcb1c85e0dbcc3cdee82e51263c70 Mon Sep 17 00:00:00 2001 From: Katarzyna Kaczmarska Date: Thu, 8 Oct 2026 15:45:30 +0200 Subject: [PATCH 17/19] Skip redundant venv when ur-test deps are already importable --- devops/actions/setup-ur-test/action.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/devops/actions/setup-ur-test/action.yml b/devops/actions/setup-ur-test/action.yml index 1be2a10c05d8e..fadb94f343e6b 100644 --- a/devops/actions/setup-ur-test/action.yml +++ b/devops/actions/setup-ur-test/action.yml @@ -30,6 +30,14 @@ runs: base_py=python fi + # Already satisfied (e.g. a caller-provisioned venv already on PATH)? + # Avoid provisioning a redundant nested environment. + if "$base_py" -c "import defusedxml, psutil" >/dev/null 2>&1; then + echo "python=$base_py" >> "$GITHUB_OUTPUT" + echo "site-packages=" >> "$GITHUB_OUTPUT" + exit 0 + fi + venv_dir="${RUNNER_TEMP:-$GITHUB_WORKSPACE}/ur-test-venv" rm -rf "$venv_dir" if "$base_py" -m venv "$venv_dir" >/dev/null 2>&1; then From 3129c215c863a83fdd45cfb678a83bd3aa833571 Mon Sep 17 00:00:00 2001 From: Katarzyna Kaczmarska Date: Fri, 23 Oct 2026 09:59:58 +0200 Subject: [PATCH 18/19] Never truncate test name lists in summaries/step summaries --- .../ur_test_tools/summary_generator.py | 43 ++++++------------- .../tests/test_result_reconciliation.py | 17 +++++--- 2 files changed, 24 insertions(+), 36 deletions(-) diff --git a/devops/scripts/ur_test_tools/summary_generator.py b/devops/scripts/ur_test_tools/summary_generator.py index ed02323e3e23c..f4ccbb6852341 100644 --- a/devops/scripts/ur_test_tools/summary_generator.py +++ b/devops/scripts/ur_test_tools/summary_generator.py @@ -19,16 +19,6 @@ TestStatus.PASS, ] -# Statuses worth listing by name in the compact GitHub step summary. -_STEP_SUMMARY_LISTED_STATUSES = [ - TestStatus.FAIL, - TestStatus.XPASS, - TestStatus.TIMEOUT, - TestStatus.UNRESOLVED, -] - -_STEP_SUMMARY_MAX_LISTED_TESTS = 50 - class SummaryReporter: def __init__(self, result: TestRunResult): @@ -59,12 +49,9 @@ def _display_test_groups(self) -> None: def _print_test_group(self, status: TestStatus, tests: List[TestResult]) -> None: print(f"::group::{status.display_label} ({len(tests)})") - for test in tests[:100]: + for test in tests: print(test.name) - if len(tests) > 100: - print(f"... and {len(tests) - 100} more") - print("::endgroup::") def _display_timing(self) -> None: @@ -96,29 +83,26 @@ def _display_timing(self) -> None: print("::endgroup::") def generate_github_step_summary(self) -> None: - """Print a compact summary for $GITHUB_STEP_SUMMARY. + """Print a full summary for $GITHUB_STEP_SUMMARY. - Unlike generate(), this lists only statuses that need attention - (Failed, Unexpectedly Passed, Timed Out, Unresolved) and shows - all other statuses as counts only, to keep the workflow run page - readable regardless of how many tests were discovered. + Every status with at least one test gets its own collapsible section + listing every one of its test names, with no truncation. """ result = self.result grouped = result.group_by_status() - print("```") - - for status in _STEP_SUMMARY_LISTED_STATUSES: + for status in _STATUS_DISPLAY_ORDER: tests = grouped.get(status, []) if not tests: continue - print(f"{status.display_label} Tests ({len(tests)}):") - for test in tests[:_STEP_SUMMARY_MAX_LISTED_TESTS]: + print(f"
{status.display_label} ({len(tests)})") + print() + print("```") + for test in tests: print(test.name) - if len(tests) > _STEP_SUMMARY_MAX_LISTED_TESTS: - remaining = len(tests) - _STEP_SUMMARY_MAX_LISTED_TESTS - print(f"... and {remaining} more") + print("```") + print("
") print() if result.testing_time_ms is not None: @@ -129,6 +113,8 @@ def generate_github_step_summary(self) -> None: counted_statuses = [s for s in _STATUS_DISPLAY_ORDER if grouped.get(s)] if total and counted_statuses: print(f"Total Discovered Tests: {total}") + print() + print("```") label_width = max(len(s.display_label) for s in counted_statuses) for status in counted_statuses: count = len(grouped[status]) @@ -137,5 +123,4 @@ def generate_github_step_summary(self) -> None: f"{status.display_label:<{label_width}} : " f"{count} ({percentage:.2f}%)" ) - - print("```") + print("```") diff --git a/devops/scripts/ur_test_tools/tests/test_result_reconciliation.py b/devops/scripts/ur_test_tools/tests/test_result_reconciliation.py index dbfba00610f6b..79efbade49c29 100644 --- a/devops/scripts/ur_test_tools/tests/test_result_reconciliation.py +++ b/devops/scripts/ur_test_tools/tests/test_result_reconciliation.py @@ -415,7 +415,7 @@ def test_displays_skipped_and_unsupported_separately(self): self.assertIn(" Unsupported: 3", output) self.assertIn("Testing Time: 1.25s", output) - def test_step_summary_lists_only_statuses_needing_attention(self): + def test_step_summary_lists_every_status_in_full(self): lines = [ "Failed Tests (1):\n", "suite :: fail.cpp\n", @@ -440,16 +440,19 @@ def test_step_summary_lists_only_statuses_needing_attention(self): SummaryReporter(result).generate_github_step_summary() output = stdout.getvalue() - self.assertIn("Failed Tests (1):", output) + self.assertIn("Failed (1)", output) self.assertIn("suite :: fail.cpp", output) - self.assertNotIn("suite :: pass-1.cpp", output) - self.assertNotIn("suite :: unsupported.cpp", output) + self.assertIn("Passed (2)", output) + self.assertIn("suite :: pass-1.cpp", output) + self.assertIn("suite :: pass-2.cpp", output) + self.assertIn("Unsupported (1)", output) + self.assertIn("suite :: unsupported.cpp", output) self.assertIn("Testing Time: 1.50s", output) self.assertIn("Total Discovered Tests: 4", output) self.assertIn("(25.00%)", output) self.assertIn("(50.00%)", output) - def test_step_summary_caps_listed_tests(self): + def test_step_summary_does_not_truncate_long_lists(self): lines = ["Failed Tests (60):\n"] lines.extend(f"suite :: fail-{i}.cpp\n" for i in range(60)) lines.append("\n") @@ -462,8 +465,8 @@ def test_step_summary_caps_listed_tests(self): output = stdout.getvalue() self.assertIn("suite :: fail-49.cpp", output) - self.assertNotIn("suite :: fail-50.cpp", output) - self.assertIn("... and 10 more", output) + self.assertIn("suite :: fail-59.cpp", output) + self.assertNotIn("more", output) class PathValidatorTest(unittest.TestCase): From 2ecdd0dd75eed2705e81de4f1ef102e6e08fde8e Mon Sep 17 00:00:00 2001 From: Katarzyna Kaczmarska Date: Mon, 26 Oct 2026 17:04:13 +0100 Subject: [PATCH 19/19] Rename run-tests/ur to run-tests/lit, extract binary validation script --- .github/workflows/ur-build-hw.yml | 4 +-- devops/actions/run-tests/linux/e2e/action.yml | 32 +++---------------- .../actions/run-tests/{ur => lit}/action.yml | 31 ++++++++++++++---- .../actions/run-tests/windows/e2e/action.yml | 27 +++------------- devops/scripts/validate-e2e-binaries | 9 ++++++ 5 files changed, 45 insertions(+), 58 deletions(-) rename devops/actions/run-tests/{ur => lit}/action.yml (79%) create mode 100755 devops/scripts/validate-e2e-binaries diff --git a/.github/workflows/ur-build-hw.yml b/.github/workflows/ur-build-hw.yml index 7870cb85f1b34..d6cf13633ffd2 100644 --- a/.github/workflows/ur-build-hw.yml +++ b/.github/workflows/ur-build-hw.yml @@ -184,7 +184,7 @@ jobs: env: # Enable Level Zero loader debug trace for L0 adapters only ZE_ENABLE_LOADER_DEBUG_TRACE: ${{ startsWith(matrix.adapter.name, 'L0') && '1' || '' }} - uses: ./devops/actions/run-tests/ur + uses: ./devops/actions/run-tests/lit with: test_type: 'adapter-specific' build_dir: build @@ -194,7 +194,7 @@ jobs: env: # Enable Level Zero loader debug trace for L0 adapters only ZE_ENABLE_LOADER_DEBUG_TRACE: ${{ startsWith(matrix.adapter.name, 'L0') && '1' || '' }} - uses: ./devops/actions/run-tests/ur + uses: ./devops/actions/run-tests/lit with: test_type: 'conformance' build_dir: build diff --git a/devops/actions/run-tests/linux/e2e/action.yml b/devops/actions/run-tests/linux/e2e/action.yml index 3476b10469972..d8d40b3560ce9 100644 --- a/devops/actions/run-tests/linux/e2e/action.yml +++ b/devops/actions/run-tests/linux/e2e/action.yml @@ -44,15 +44,10 @@ runs: - name: Extract E2E Binaries if: ${{ inputs.testing_mode == 'run-only' && inputs.binaries_artifact != 'in-container' }} shell: bash - env: - BINARIES_ARTIFACT: ${{ inputs.binaries_artifact }} run: | mkdir build-e2e tar -I 'zstd' -xf e2e_binaries.tar.zst -C build-e2e - if [ -z "$(find build-e2e -path '*/Output/*' -type f -print -quit)" ]; then - echo "::error::Downloaded E2E binaries artifact '$BINARIES_ARTIFACT' has no compiled test binaries under */Output/ (likely produced by a failed upstream build job). Failing fast instead of running thousands of doomed tests." - exit 1 - fi + devops/scripts/validate-e2e-binaries build-e2e - name: Extract E2E tests from container image if: ${{ inputs.testing_mode == 'run-only' && inputs.binaries_artifact == 'in-container' }} @@ -61,10 +56,7 @@ runs: mkdir build-e2e llvm tar -I 'zstd' -xf /sycl-prebuilt/e2e_binaries.tar.zst -C build-e2e tar -I 'zstd' -xf /sycl-prebuilt/e2e_sources.tar.zst -C llvm - if [ -z "$(find build-e2e -path '*/Output/*' -type f -print -quit)" ]; then - echo "::error::Container image's prebuilt E2E binaries have no compiled test binaries under */Output/. Failing fast instead of running thousands of doomed tests." - exit 1 - fi + devops/scripts/validate-e2e-binaries build-e2e - name: Deduce E2E CMake options if: inputs.testing_mode != 'run-only' @@ -95,32 +87,18 @@ runs: # letting it block the whole suite until the job's own timeout-minutes fires. echo "opts=--max-time 3600 --timeout 600 --param print_features=True --param test-mode=${TESTING_MODE} --param sycl_devices=${TARGET_DEVICES} ${USER_LIT_OPTS}" >> "$GITHUB_OUTPUT" - - name: Build artifact name - if: ${{ always() && inputs.job_name != '' }} - id: artifact_name - shell: bash - env: - JOB_NAME: ${{ inputs.job_name }} - run: | - safe_name=$(echo "$JOB_NAME" | tr -c '[:alnum:]._-' '-' | sed 's/-\+/-/g; s/^-//; s/-$//') - echo "name=e2e-${safe_name}" >> "$GITHUB_OUTPUT" - - name: Run E2E tests - uses: ./devops/actions/run-tests/ur + uses: ./devops/actions/run-tests/lit with: test_type: 'e2e' build_dir: 'build-e2e' extra_lit_opts: ${{ steps.lit_opts.outputs.opts }} - artifact_name: ${{ steps.artifact_name.outputs.name }} + job_name: ${{ inputs.job_name }} - name: Verify E2E binaries were built if: ${{ !cancelled() && inputs.binaries_artifact != '' && inputs.testing_mode != 'run-only' }} shell: bash - run: | - if [ -z "$(find build-e2e -path '*/Output/*' -type f -print -quit)" ]; then - echo "::error::No compiled E2E test binaries found under build-e2e/**/Output/ after the build step. Refusing to upload a broken artifact." - exit 1 - fi + run: devops/scripts/validate-e2e-binaries build-e2e - name: Pack E2E binaries if: ${{ !cancelled() && inputs.binaries_artifact != '' && inputs.testing_mode != 'run-only' }} diff --git a/devops/actions/run-tests/ur/action.yml b/devops/actions/run-tests/lit/action.yml similarity index 79% rename from devops/actions/run-tests/ur/action.yml rename to devops/actions/run-tests/lit/action.yml index 20ae3b6de525f..eee68160d3bb9 100644 --- a/devops/actions/run-tests/ur/action.yml +++ b/devops/actions/run-tests/lit/action.yml @@ -1,4 +1,4 @@ -name: 'Run UR Tests' +name: 'Run LIT Tests' inputs: test_type: @@ -8,7 +8,11 @@ inputs: required: false default: 'build' artifact_name: - description: 'Name for uploaded test artifacts (logs, XML)' + description: 'Name for uploaded test artifacts (logs, XML). If empty and job_name is set, a name is derived from test_type and job_name.' + required: false + default: '' + job_name: + description: 'Human-readable job identifier used to derive artifact_name when it is not given explicitly.' required: false default: '' extra_lit_opts: @@ -23,6 +27,21 @@ runs: id: ur_test uses: ./devops/actions/setup-ur-test + - name: Resolve artifact name + id: artifact + shell: bash + env: + ARTIFACT_NAME: ${{ inputs.artifact_name }} + JOB_NAME: ${{ inputs.job_name }} + TEST_TYPE: ${{ inputs.test_type }} + run: | + name="$ARTIFACT_NAME" + if [ -z "$name" ] && [ -n "$JOB_NAME" ]; then + safe_name=$(echo "$JOB_NAME" | tr -c '[:alnum:]._-' '-' | sed 's/-\+/-/g; s/^-//; s/-$//') + name="${TEST_TYPE}-${safe_name}" + fi + echo "name=$name" >> "$GITHUB_OUTPUT" + - name: Run ${{ inputs.test_type }} tests id: run_tests continue-on-error: true @@ -82,7 +101,7 @@ runs: env: LOG_FILE: ${{ steps.run_tests.outputs['log-file'] }} XML_FILE: ${{ steps.run_tests.outputs['xml-file'] }} - ARTIFACT_NAME: ${{ inputs.artifact_name }} + ARTIFACT_NAME: ${{ steps.artifact.outputs.name }} TEST_TYPE: ${{ inputs.test_type }} WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} PY: ${{ steps.ur_test.outputs.python }} @@ -119,7 +138,7 @@ runs: fi - name: Copy artifacts - if: ${{ always() && inputs.artifact_name != '' && (inputs.test_type != 'adapter-specific' || steps.run_tests.outputs['skip-artifacts'] != '1') }} + if: ${{ always() && steps.artifact.outputs.name != '' && (inputs.test_type != 'adapter-specific' || steps.run_tests.outputs['skip-artifacts'] != '1') }} shell: bash env: LOG_FILE: ${{ steps.run_tests.outputs['log-file'] }} @@ -139,10 +158,10 @@ runs: fi - name: Upload test artifacts - if: ${{ always() && inputs.artifact_name != '' && (inputs.test_type != 'adapter-specific' || steps.run_tests.outputs['skip-artifacts'] != '1') }} + if: ${{ always() && steps.artifact.outputs.name != '' && (inputs.test_type != 'adapter-specific' || steps.run_tests.outputs['skip-artifacts'] != '1') }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: ${{ inputs.artifact_name }} + name: ${{ steps.artifact.outputs.name }} path: test_artifacts/ retention-days: 7 if-no-files-found: warn diff --git a/devops/actions/run-tests/windows/e2e/action.yml b/devops/actions/run-tests/windows/e2e/action.yml index 95fae4291e0f8..3304253dc6e3b 100644 --- a/devops/actions/run-tests/windows/e2e/action.yml +++ b/devops/actions/run-tests/windows/e2e/action.yml @@ -45,15 +45,10 @@ runs: - name: Extract E2E Binaries if: inputs.testing_mode == 'run-only' shell: bash - env: - BINARIES_ARTIFACT: ${{ inputs.binaries_artifact }} run: | mkdir build-e2e tar -xf e2e_bin.tar.gz -C build-e2e - if [ -z "$(find build-e2e -path '*/Output/*' -type f -print -quit)" ]; then - echo "::error::Downloaded E2E binaries artifact '$BINARIES_ARTIFACT' has no compiled test binaries under */Output/ (likely produced by a failed upstream build job). Failing fast instead of running thousands of doomed tests." - exit 1 - fi + devops/scripts/validate-e2e-binaries build-e2e - name: Deduce E2E CMake options id: cmake_opts @@ -95,23 +90,13 @@ runs: # letting it block the whole suite until the job's own timeout-minutes fires. echo "opts=--max-time ${MAX_TIME} --timeout 600 --param print_features=True --param test-mode=${TESTING_MODE} --param sycl_devices=${TARGET_DEVICES} ${USER_LIT_OPTS}" >> "$GITHUB_OUTPUT" - - name: Build artifact name - if: ${{ always() && inputs.job_name != '' }} - id: artifact_name - shell: bash - env: - JOB_NAME: ${{ inputs.job_name }} - run: | - safe_name=$(echo "$JOB_NAME" | tr -c '[:alnum:]._-' '-' | sed 's/-\+/-/g; s/^-//; s/-$//') - echo "name=e2e-${safe_name}" >> "$GITHUB_OUTPUT" - - name: Run E2E tests - uses: ./devops/actions/run-tests/ur + uses: ./devops/actions/run-tests/lit with: test_type: 'e2e' build_dir: 'build-e2e' extra_lit_opts: ${{ steps.lit_opts.outputs.opts }} - artifact_name: ${{ steps.artifact_name.outputs.name }} + job_name: ${{ inputs.job_name }} # Github CI doesn't support containers on Windows, so we cannot guarantee # that paths are the same between building and running systems. To avoid @@ -127,11 +112,7 @@ runs: - name: Verify E2E binaries were built if: ${{ !cancelled() && inputs.binaries_artifact != '' && inputs.testing_mode != 'run-only' }} shell: bash - run: | - if [ -z "$(find build-e2e -path '*/Output/*' -type f -print -quit)" ]; then - echo "::error::No compiled E2E test binaries found under build-e2e/**/Output/ after the build step. Refusing to upload a broken artifact." - exit 1 - fi + run: devops/scripts/validate-e2e-binaries build-e2e - name: Pack E2E test binaries if: ${{ !cancelled() && inputs.binaries_artifact != '' && inputs.testing_mode != 'run-only' }} diff --git a/devops/scripts/validate-e2e-binaries b/devops/scripts/validate-e2e-binaries new file mode 100755 index 0000000000000..4162ea3339a91 --- /dev/null +++ b/devops/scripts/validate-e2e-binaries @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +build_dir="${1:?Usage: validate-e2e-binaries }" + +if [ -z "$(find "$build_dir" -path '*/Output/*' -type f -print -quit)" ]; then + echo "::error::No E2E test binaries found under $build_dir/**/Output/" + exit 1 +fi