diff --git a/.github/workflows/sycl-linux-build.yml b/.github/workflows/sycl-linux-build.yml
index 19a80f14d5ecd..45612131b7869 100644
--- a/.github/workflows/sycl-linux-build.yml
+++ b/.github/workflows/sycl-linux-build.yml
@@ -179,6 +179,7 @@ jobs:
with:
sparse-checkout: |
devops/actions
+ devops/scripts
# Cleanup will be run after all actions are completed.
- name: Register cleanup after job is finished
uses: ./devops/actions/cleanup
@@ -332,6 +333,7 @@ jobs:
testing_mode: build-only
target_devices: all
binaries_artifact: ${{ inputs.e2e_binaries_artifact }}
+ job_name: ${{ inputs.e2e_binaries_artifact }}
sycl_compiler: $GITHUB_WORKSPACE/toolchain/bin/dpclang++
extra_lit_opts: --param sycl_build_targets="spir;nvidia" --param gpu_aot_target=${{ env.GPU_AOT_TARGET }}
@@ -347,6 +349,7 @@ jobs:
testing_mode: build-only
target_devices: all
binaries_artifact: ${{ inputs.e2e_binaries_new_offload_model_artifact }}
+ job_name: ${{ inputs.e2e_binaries_new_offload_model_artifact }}
sycl_compiler: $GITHUB_WORKSPACE/toolchain/bin/dpclang++
extra_lit_opts: --param sycl_build_targets="spir;nvidia" --param enable_new_offload_model=True --param gpu_aot_target=${{ env.GPU_AOT_TARGET }}
@@ -358,6 +361,7 @@ jobs:
testing_mode: build-only
target_devices: all
binaries_artifact: ${{ inputs.e2e_binaries_spirv_backend_artifact }}
+ job_name: ${{ inputs.e2e_binaries_spirv_backend_artifact }}
sycl_compiler: $GITHUB_WORKSPACE/toolchain/bin/dpclang++
extra_lit_opts: --param spirv-backend=True --param gpu_aot_target=${{ env.GPU_AOT_TARGET }}
@@ -369,5 +373,6 @@ jobs:
testing_mode: build-only
target_devices: all
binaries_artifact: ${{ inputs.e2e_binaries_preview_artifact }}
+ job_name: ${{ inputs.e2e_binaries_preview_artifact }}
sycl_compiler: $GITHUB_WORKSPACE/toolchain/bin/dpclang++
extra_lit_opts: --param test-preview-mode=True --param gpu_aot_target=${{ env.GPU_AOT_TARGET }}
diff --git a/.github/workflows/sycl-linux-precommit.yml b/.github/workflows/sycl-linux-precommit.yml
index c473b3830140d..b676b79272595 100644
--- a/.github/workflows/sycl-linux-precommit.yml
+++ b/.github/workflows/sycl-linux-precommit.yml
@@ -81,7 +81,8 @@ jobs:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
sparse-checkout: |
- devops/
+ devops/actions
+ devops/scripts
# download build artefact
- name: Download toolchain
@@ -100,6 +101,7 @@ jobs:
ref: ${{ github.ref || github.sha }}
testing_mode: full
target_devices: native_cpu:cpu
+ job_name: ${{ github.job }}
sycl_compiler: $GITHUB_WORKSPACE/toolchain/bin/dpclang++
extra_lit_opts: --param sycl_build_targets="native_cpu"
extra_cmake_args: -DSYCL_TEST_E2E_TARGETS="native_cpu:cpu" -DSYCL_TEST_E2E_STANDALONE=ON
diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml
index 35badd106674c..fcfd32c4f8ec9 100644
--- a/.github/workflows/sycl-linux-run-tests.yml
+++ b/.github/workflows/sycl-linux-run-tests.yml
@@ -261,7 +261,8 @@ jobs:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
sparse-checkout: |
- devops
+ devops/actions
+ devops/scripts
sycl/cts_exclude_filter
- name: Register cleanup after job is finished
uses: ./devops/actions/cleanup
@@ -366,6 +367,7 @@ jobs:
target_devices: ${{ inputs.target_devices }}
extra_lit_opts: ${{ inputs.extra_lit_opts }}
retention-days: ${{ inputs.retention-days }}
+ job_name: ${{ inputs.name }}
- name: Run SYCL CTS Tests
if: inputs.tests_selector == 'cts'
diff --git a/.github/workflows/sycl-prebuilt-e2e-container.yml b/.github/workflows/sycl-prebuilt-e2e-container.yml
index 15891fc59f6f2..b5fdfe3210eb0 100644
--- a/.github/workflows/sycl-prebuilt-e2e-container.yml
+++ b/.github/workflows/sycl-prebuilt-e2e-container.yml
@@ -102,7 +102,8 @@ jobs:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
sparse-checkout: |
- devops
+ devops/actions
+ devops/scripts
- run: |
mkdir toolchain
tar -I 'zstd' -xf /sycl-prebuilt/toolchain.tar.zst -C toolchain
@@ -116,3 +117,4 @@ jobs:
with:
testing_mode: run-only
target_devices: level_zero:gpu
+ job_name: ${{ github.job }}
diff --git a/.github/workflows/sycl-windows-build.yml b/.github/workflows/sycl-windows-build.yml
index da370d946fe8a..44cc212935dcb 100644
--- a/.github/workflows/sycl-windows-build.yml
+++ b/.github/workflows/sycl-windows-build.yml
@@ -123,6 +123,7 @@ jobs:
with:
sparse-checkout: |
devops/actions
+ devops/scripts
- uses: TheMrMilchmann/setup-msvc-dev@368ef7d1ee4d1171b31d4a7f67f4d954f903f5a9 # v4.1.0
with:
arch: x64
@@ -262,6 +263,7 @@ jobs:
testing_mode: build-only
target_devices: all
binaries_artifact: ${{ inputs.e2e_binaries_artifact }}
+ job_name: ${{ inputs.e2e_binaries_artifact }}
extra_lit_opts: --param sycl_build_targets="spir" --param gpu_aot_target=${{ env.GPU_AOT_TARGET }}
cxx: ${{ inputs.cxx }}
@@ -277,6 +279,7 @@ jobs:
testing_mode: build-only
target_devices: all
binaries_artifact: ${{ inputs.e2e_binaries_new_offload_model_artifact }}
+ job_name: ${{ inputs.e2e_binaries_new_offload_model_artifact }}
extra_lit_opts: --param sycl_build_targets="spir" --param enable_new_offload_model=True --param gpu_aot_target=${{ env.GPU_AOT_TARGET }}
cxx: ${{ inputs.cxx }}
diff --git a/.github/workflows/sycl-windows-run-tests.yml b/.github/workflows/sycl-windows-run-tests.yml
index a9ebcaf58fbe6..5b683be76c596 100644
--- a/.github/workflows/sycl-windows-run-tests.yml
+++ b/.github/workflows/sycl-windows-run-tests.yml
@@ -98,6 +98,7 @@ jobs:
with:
sparse-checkout: |
devops/actions
+ devops/scripts
sycl/cts_exclude_filter
- uses: TheMrMilchmann/setup-msvc-dev@368ef7d1ee4d1171b31d4a7f67f4d954f903f5a9 # v4.1.0
with:
@@ -152,6 +153,7 @@ jobs:
extra_lit_opts: ${{ inputs.extra_lit_opts }}
retention-days: ${{ inputs.artifact_retention_days }}
cxx: ${{ inputs.cxx }}
+ job_name: ${{ inputs.name }}
- name: Run SYCL CTS Tests
if: inputs.tests_selector == 'cts'
diff --git a/.github/workflows/ur-build-hw.yml b/.github/workflows/ur-build-hw.yml
index 7870cb85f1b34..d6cf13633ffd2 100644
--- a/.github/workflows/ur-build-hw.yml
+++ b/.github/workflows/ur-build-hw.yml
@@ -184,7 +184,7 @@ jobs:
env:
# Enable Level Zero loader debug trace for L0 adapters only
ZE_ENABLE_LOADER_DEBUG_TRACE: ${{ startsWith(matrix.adapter.name, 'L0') && '1' || '' }}
- uses: ./devops/actions/run-tests/ur
+ uses: ./devops/actions/run-tests/lit
with:
test_type: 'adapter-specific'
build_dir: build
@@ -194,7 +194,7 @@ jobs:
env:
# Enable Level Zero loader debug trace for L0 adapters only
ZE_ENABLE_LOADER_DEBUG_TRACE: ${{ startsWith(matrix.adapter.name, 'L0') && '1' || '' }}
- uses: ./devops/actions/run-tests/ur
+ uses: ./devops/actions/run-tests/lit
with:
test_type: 'conformance'
build_dir: build
diff --git a/devops/actions/run-tests/linux/e2e/action.yml b/devops/actions/run-tests/linux/e2e/action.yml
index c9f11c5324694..d8d40b3560ce9 100644
--- a/devops/actions/run-tests/linux/e2e/action.yml
+++ b/devops/actions/run-tests/linux/e2e/action.yml
@@ -18,6 +18,10 @@ inputs:
required: false
sycl_compiler:
required: false
+ job_name:
+ description: 'Name of the calling job, used to build a unique artifact name'
+ required: false
+ default: ''
runs:
@@ -43,6 +47,7 @@ runs:
run: |
mkdir build-e2e
tar -I 'zstd' -xf e2e_binaries.tar.zst -C build-e2e
+ devops/scripts/validate-e2e-binaries build-e2e
- name: Extract E2E tests from container image
if: ${{ inputs.testing_mode == 'run-only' && inputs.binaries_artifact == 'in-container' }}
@@ -51,6 +56,7 @@ runs:
mkdir build-e2e llvm
tar -I 'zstd' -xf /sycl-prebuilt/e2e_binaries.tar.zst -C build-e2e
tar -I 'zstd' -xf /sycl-prebuilt/e2e_sources.tar.zst -C llvm
+ devops/scripts/validate-e2e-binaries build-e2e
- name: Deduce E2E CMake options
if: inputs.testing_mode != 'run-only'
@@ -67,32 +73,32 @@ runs:
shell: bash
run: |
cmake -GNinja -B./build-e2e -S./llvm/sycl/test-e2e -DCMAKE_CXX_COMPILER="${{ inputs.sycl_compiler || '$(which dpclang++)'}}" -DLLVM_LIT="$PWD/llvm/llvm/utils/lit/lit.py" ${{ steps.cmake_opts.outputs.opts }}
- - name: SYCL End-to-end tests
- id: run_e2e
- continue-on-error: true
+
+ # Composed here, not in ur-test, since these depend on caller-only workflow inputs.
+ - name: Compose E2E LIT options
+ id: lit_opts
shell: bash
env:
- LIT_OPTS: -v --no-progress-bar --show-unsupported --show-pass --show-xfail --max-time 3600 --time-tests --param print_features=True --param test-mode=${{ inputs.testing_mode }} --param sycl_devices=${{ inputs.target_devices }} ${{ inputs.extra_lit_opts }}
+ TESTING_MODE: ${{ inputs.testing_mode }}
+ TARGET_DEVICES: ${{ inputs.target_devices }}
+ USER_LIT_OPTS: ${{ inputs.extra_lit_opts }}
run: |
- ninja -C build-e2e check-sycl-e2e > e2e.log 2>&1
- # Two steps below are duplicated between Lin/Win actions, updates must change both
- - name: E2E logs
- if: ${{ always() }}
- shell: bash
- run: |
- echo "::group::Show Full E2E Log"
- cat e2e.log
- echo "::endgroup::"
- - name: Report E2E Failures
- if: steps.run_e2e.outcome != 'success'
+ # --timeout kills an individual hung test (e.g. a GPU driver hang) instead of
+ # letting it block the whole suite until the job's own timeout-minutes fires.
+ echo "opts=--max-time 3600 --timeout 600 --param print_features=True --param test-mode=${TESTING_MODE} --param sycl_devices=${TARGET_DEVICES} ${USER_LIT_OPTS}" >> "$GITHUB_OUTPUT"
+
+ - name: Run E2E tests
+ uses: ./devops/actions/run-tests/lit
+ with:
+ test_type: 'e2e'
+ build_dir: 'build-e2e'
+ extra_lit_opts: ${{ steps.lit_opts.outputs.opts }}
+ job_name: ${{ inputs.job_name }}
+
+ - name: Verify E2E binaries were built
+ if: ${{ !cancelled() && inputs.binaries_artifact != '' && inputs.testing_mode != 'run-only' }}
shell: bash
- # For some reason Github uses the first line from the `run: |` section for
- # the folded entry when displaying instead of this step's name.
- run: |
- # Report E2E Failures
- awk '/^Failed Tests|Unexpectedly Passed Tests|Unresolved tests|Timed Out Tests|Testing Time/{flag=1}/FAILED: CMakeFiles/{flag=0}flag' e2e.log >> $GITHUB_STEP_SUMMARY
- awk '/^Failed Tests|Unexpectedly Passed Tests|Unresolved tests|Timed Out Tests|Testing Time/{flag=1}/FAILED: CMakeFiles/{flag=0}flag' e2e.log
- exit 1
+ run: devops/scripts/validate-e2e-binaries build-e2e
- name: Pack E2E binaries
if: ${{ !cancelled() && inputs.binaries_artifact != '' && inputs.testing_mode != 'run-only' }}
diff --git a/devops/actions/run-tests/ur/action.yml b/devops/actions/run-tests/lit/action.yml
similarity index 54%
rename from devops/actions/run-tests/ur/action.yml
rename to devops/actions/run-tests/lit/action.yml
index 0e161f6970846..eee68160d3bb9 100644
--- a/devops/actions/run-tests/ur/action.yml
+++ b/devops/actions/run-tests/lit/action.yml
@@ -1,20 +1,47 @@
-name: 'Run UR Tests'
+name: 'Run LIT Tests'
inputs:
test_type:
- description: 'Type of tests to run (adapter-specific or conformance)'
+ description: 'Type of tests to run (adapter-specific, conformance or e2e)'
required: true
build_dir:
required: false
default: 'build'
artifact_name:
- description: 'Name for uploaded test artifacts (logs, XML)'
+ description: 'Name for uploaded test artifacts (logs, XML). If empty and job_name is set, a name is derived from test_type and job_name.'
+ required: false
+ default: ''
+ job_name:
+ description: 'Human-readable job identifier used to derive artifact_name when it is not given explicitly.'
+ required: false
+ default: ''
+ extra_lit_opts:
+ description: 'Extra options appended to LIT_OPTS (e.g. E2E --param/--max-time flags)'
required: false
default: ''
runs:
using: "composite"
steps:
+ - name: Set up ur-test
+ id: ur_test
+ uses: ./devops/actions/setup-ur-test
+
+ - name: Resolve artifact name
+ id: artifact
+ shell: bash
+ env:
+ ARTIFACT_NAME: ${{ inputs.artifact_name }}
+ JOB_NAME: ${{ inputs.job_name }}
+ TEST_TYPE: ${{ inputs.test_type }}
+ run: |
+ name="$ARTIFACT_NAME"
+ if [ -z "$name" ] && [ -n "$JOB_NAME" ]; then
+ safe_name=$(echo "$JOB_NAME" | tr -c '[:alnum:]._-' '-' | sed 's/-\+/-/g; s/^-//; s/-$//')
+ name="${TEST_TYPE}-${safe_name}"
+ fi
+ echo "name=$name" >> "$GITHUB_OUTPUT"
+
- name: Run ${{ inputs.test_type }} tests
id: run_tests
continue-on-error: true
@@ -22,14 +49,21 @@ runs:
env:
BUILD_DIR: ${{ inputs.build_dir }}
TEST_TYPE: ${{ inputs.test_type }}
+ EXTRA_LIT_OPTS: ${{ inputs.extra_lit_opts }}
+ PY: ${{ steps.ur_test.outputs.python }}
+ # Needed by lit (spawned via cmake/ninja's system Python) to see psutil,
+ # and by ur-test itself under the --target fallback; set on every step below too.
+ PYTHONPATH: ${{ steps.ur_test.outputs['site-packages'] }}
run: |
- devops/scripts/ur-test run "$TEST_TYPE" "$BUILD_DIR" "$GITHUB_WORKSPACE"
+ "$PY" devops/scripts/ur-test run "$TEST_TYPE" "$BUILD_DIR" "$GITHUB_WORKSPACE"
- name: Filter test log
if: ${{ always() && steps.run_tests.outputs['skip-artifacts'] != '1' }}
shell: bash
env:
LOG_FILE: ${{ steps.run_tests.outputs['log-file'] }}
+ PY: ${{ steps.ur_test.outputs.python }}
+ PYTHONPATH: ${{ steps.ur_test.outputs['site-packages'] }}
run: |
if [ -z "$LOG_FILE" ]; then
echo "::warning::Test log path not provided by runner; skipping log filtering."
@@ -42,7 +76,7 @@ runs:
fi
echo "::group::Test Log"
- devops/scripts/ur-test filter-log "$LOG_FILE" || { echo "::error::Filter failed"; head -1000 "$LOG_FILE"; }
+ "$PY" devops/scripts/ur-test filter-log "$LOG_FILE" || { echo "::error::Filter failed"; head -1000 "$LOG_FILE"; }
echo "::endgroup::"
- name: Report test failures
@@ -50,13 +84,15 @@ runs:
shell: bash
env:
LOG_FILE: ${{ steps.run_tests.outputs['log-file'] }}
+ PY: ${{ steps.ur_test.outputs.python }}
+ PYTHONPATH: ${{ steps.ur_test.outputs['site-packages'] }}
run: |
if [ -z "$LOG_FILE" ] || [ ! -f "$LOG_FILE" ]; then
echo "::warning::Test execution failed, but no test log is available."
exit 1
fi
- devops/scripts/ur-test extract-errors "$LOG_FILE"
+ "$PY" devops/scripts/ur-test extract-errors "$LOG_FILE"
exit 1
- name: Test summary
@@ -65,9 +101,11 @@ runs:
env:
LOG_FILE: ${{ steps.run_tests.outputs['log-file'] }}
XML_FILE: ${{ steps.run_tests.outputs['xml-file'] }}
- ARTIFACT_NAME: ${{ inputs.artifact_name }}
+ ARTIFACT_NAME: ${{ steps.artifact.outputs.name }}
TEST_TYPE: ${{ inputs.test_type }}
WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+ PY: ${{ steps.ur_test.outputs.python }}
+ PYTHONPATH: ${{ steps.ur_test.outputs['site-packages'] }}
run: |
if [ -z "$LOG_FILE" ]; then
echo "::warning::Test log path not provided by runner; skipping summary."
@@ -84,7 +122,7 @@ runs:
exit 0
fi
- devops/scripts/ur-test summary "$LOG_FILE" "${XML_FILE:-}"
+ "$PY" devops/scripts/ur-test summary "$LOG_FILE" "${XML_FILE:-}"
if [ -n "$ARTIFACT_NAME" ]; then
echo ""
@@ -96,11 +134,11 @@ runs:
if [ -n "$GITHUB_STEP_SUMMARY" ]; then
echo "### ${TEST_TYPE} tests" >> "$GITHUB_STEP_SUMMARY"
- devops/scripts/ur-test step-summary "$LOG_FILE" "${XML_FILE:-}" >> "$GITHUB_STEP_SUMMARY"
+ "$PY" devops/scripts/ur-test step-summary "$LOG_FILE" "${XML_FILE:-}" >> "$GITHUB_STEP_SUMMARY"
fi
- name: Copy artifacts
- if: ${{ always() && inputs.artifact_name != '' && (inputs.test_type != 'adapter-specific' || steps.run_tests.outputs['skip-artifacts'] != '1') }}
+ if: ${{ always() && steps.artifact.outputs.name != '' && (inputs.test_type != 'adapter-specific' || steps.run_tests.outputs['skip-artifacts'] != '1') }}
shell: bash
env:
LOG_FILE: ${{ steps.run_tests.outputs['log-file'] }}
@@ -120,10 +158,10 @@ runs:
fi
- name: Upload test artifacts
- if: ${{ always() && inputs.artifact_name != '' && (inputs.test_type != 'adapter-specific' || steps.run_tests.outputs['skip-artifacts'] != '1') }}
+ if: ${{ always() && steps.artifact.outputs.name != '' && (inputs.test_type != 'adapter-specific' || steps.run_tests.outputs['skip-artifacts'] != '1') }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
- name: ${{ inputs.artifact_name }}
+ name: ${{ steps.artifact.outputs.name }}
path: test_artifacts/
retention-days: 7
if-no-files-found: warn
diff --git a/devops/actions/run-tests/windows/e2e/action.yml b/devops/actions/run-tests/windows/e2e/action.yml
index f22bb0f65185c..3304253dc6e3b 100644
--- a/devops/actions/run-tests/windows/e2e/action.yml
+++ b/devops/actions/run-tests/windows/e2e/action.yml
@@ -21,6 +21,10 @@ inputs:
cxx:
required: false
default: "cl"
+ job_name:
+ description: 'Name of the calling job, used to build a unique artifact name'
+ required: false
+ default: ''
runs:
using: "composite"
@@ -44,6 +48,7 @@ runs:
run: |
mkdir build-e2e
tar -xf e2e_bin.tar.gz -C build-e2e
+ devops/scripts/validate-e2e-binaries build-e2e
- name: Deduce E2E CMake options
id: cmake_opts
@@ -71,32 +76,27 @@ runs:
shell: bash
run: ls build-e2e > e2econf_files.txt
- - name: Run End-to-End tests
- id: run_e2e
- continue-on-error: true
+ # Composed here, not in ur-test, since these depend on caller-only workflow inputs.
+ - name: Compose E2E LIT options
+ id: lit_opts
shell: bash
env:
- LIT_OPTS: -v --no-progress-bar --show-unsupported --show-pass --show-xfail --max-time ${{ inputs.testing_mode == 'run-only' && 1200 || 3600 }} --time-tests --param print_features=True --param test-mode=${{ inputs.testing_mode }} --param sycl_devices=${{ inputs.target_devices }} ${{ inputs.extra_lit_opts }}
- run: |
- cmake --build build-e2e --target check-sycl-e2e > e2e.log 2>&1
- # Two steps below are duplicated between Lin/Win actions, updates must change both
- - name: E2E logs
- if: ${{ always() }}
- shell: bash
- run: |
- echo "::group::Show Full E2E Log"
- cat e2e.log
- echo "::endgroup::"
- - name: Report E2E Failures
- if: steps.run_e2e.outcome != 'success'
- shell: bash
- # For some reason Github uses the first line from the `run: |` section for
- # the folded entry when displaying instead of this step's name.
+ TESTING_MODE: ${{ inputs.testing_mode }}
+ TARGET_DEVICES: ${{ inputs.target_devices }}
+ USER_LIT_OPTS: ${{ inputs.extra_lit_opts }}
+ MAX_TIME: ${{ inputs.testing_mode == 'run-only' && 1200 || 3600 }}
run: |
- # Report E2E Failures
- awk '/^Failed Tests|Unexpectedly Passed Tests|Unresolved tests|Timed Out Tests|Testing Time/{flag=1}/FAILED: CMakeFiles/{flag=0}flag' e2e.log >> $GITHUB_STEP_SUMMARY
- awk '/^Failed Tests|Unexpectedly Passed Tests|Unresolved tests|Timed Out Tests|Testing Time/{flag=1}/FAILED: CMakeFiles/{flag=0}flag' e2e.log
- exit 1
+ # --timeout kills an individual hung test (e.g. a GPU driver hang) instead of
+ # letting it block the whole suite until the job's own timeout-minutes fires.
+ echo "opts=--max-time ${MAX_TIME} --timeout 600 --param print_features=True --param test-mode=${TESTING_MODE} --param sycl_devices=${TARGET_DEVICES} ${USER_LIT_OPTS}" >> "$GITHUB_OUTPUT"
+
+ - name: Run E2E tests
+ uses: ./devops/actions/run-tests/lit
+ with:
+ test_type: 'e2e'
+ build_dir: 'build-e2e'
+ extra_lit_opts: ${{ steps.lit_opts.outputs.opts }}
+ job_name: ${{ inputs.job_name }}
# Github CI doesn't support containers on Windows, so we cannot guarantee
# that paths are the same between building and running systems. To avoid
@@ -109,6 +109,11 @@ runs:
for FILE in $(cat e2econf_files.txt); do rm -r build-e2e/$FILE; done
rm e2econf_files.txt
+ - name: Verify E2E binaries were built
+ if: ${{ !cancelled() && inputs.binaries_artifact != '' && inputs.testing_mode != 'run-only' }}
+ shell: bash
+ run: devops/scripts/validate-e2e-binaries build-e2e
+
- name: Pack E2E test binaries
if: ${{ !cancelled() && inputs.binaries_artifact != '' && inputs.testing_mode != 'run-only' }}
shell: bash
diff --git a/devops/actions/setup-ur-test/action.yml b/devops/actions/setup-ur-test/action.yml
new file mode 100644
index 0000000000000..fadb94f343e6b
--- /dev/null
+++ b/devops/actions/setup-ur-test/action.yml
@@ -0,0 +1,64 @@
+name: 'Set up ur-test'
+description: >
+ Provision the Python dependencies required by UR test reporting (ur-test)
+ and LIT execution (devops/scripts/ur_test_tools/requirements.txt), without
+ touching the system/base Python installation.
+
+outputs:
+ python:
+ description: 'Path to the Python interpreter with ur-test dependencies installed.'
+ value: ${{ steps.setup.outputs.python }}
+ site-packages:
+ description: >
+ Path to the directory containing ur-test's dependencies. Needed on
+ PYTHONPATH only by other processes (e.g. lit, run via a different,
+ system Python) that must see these packages but do not use the
+ interpreter above directly.
+ value: ${{ steps.setup.outputs.site-packages }}
+
+runs:
+ using: "composite"
+ steps:
+ - name: Provision ur-test Python environment
+ id: setup
+ shell: bash
+ run: |
+ # `python3` may not be on PATH on Windows runners.
+ if command -v python3 >/dev/null 2>&1; then
+ base_py=python3
+ else
+ base_py=python
+ fi
+
+ # Already satisfied (e.g. a caller-provisioned venv already on PATH)?
+ # Avoid provisioning a redundant nested environment.
+ if "$base_py" -c "import defusedxml, psutil" >/dev/null 2>&1; then
+ echo "python=$base_py" >> "$GITHUB_OUTPUT"
+ echo "site-packages=" >> "$GITHUB_OUTPUT"
+ exit 0
+ fi
+
+ venv_dir="${RUNNER_TEMP:-$GITHUB_WORKSPACE}/ur-test-venv"
+ rm -rf "$venv_dir"
+ if "$base_py" -m venv "$venv_dir" >/dev/null 2>&1; then
+ venv_py="$venv_dir/bin/python"
+ [ -f "$venv_py" ] || venv_py="$venv_dir/Scripts/python.exe"
+ "$venv_py" -m pip install --disable-pip-version-check --quiet \
+ -r devops/scripts/ur_test_tools/requirements.txt
+ site_packages="$("$venv_py" -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"
+ echo "python=$venv_py" >> "$GITHUB_OUTPUT"
+ echo "site-packages=$site_packages" >> "$GITHUB_OUTPUT"
+ exit 0
+ fi
+
+ # Some containers (nightly images) have no venv/ensurepip and broken apt;
+ # fall back to a workspace-local --target install on the base interpreter.
+ echo "::warning::'python -m venv' unavailable; falling back to a --target install."
+ deps_dir="${RUNNER_TEMP:-$GITHUB_WORKSPACE}/ur-test-deps"
+ rm -rf "$deps_dir"
+ mkdir -p "$deps_dir"
+ # Same --break-system-packages fallback as run-tests/benchmark for nightly images.
+ "$base_py" -m pip install --disable-pip-version-check --quiet --break-system-packages --target "$deps_dir" -r devops/scripts/ur_test_tools/requirements.txt \
+ || "$base_py" -m pip install --disable-pip-version-check --quiet --target "$deps_dir" -r devops/scripts/ur_test_tools/requirements.txt
+ echo "python=$base_py" >> "$GITHUB_OUTPUT"
+ echo "site-packages=$deps_dir" >> "$GITHUB_OUTPUT"
diff --git a/devops/scripts/ur_test_tools/cli.py b/devops/scripts/ur_test_tools/cli.py
index 53ffcfc943526..c94c6531ae721 100644
--- a/devops/scripts/ur_test_tools/cli.py
+++ b/devops/scripts/ur_test_tools/cli.py
@@ -120,6 +120,8 @@ def _run_tests_command() -> int:
github_actions.print_error(str(e))
return 1
+ config.extra_lit_opts = os.environ.get("EXTRA_LIT_OPTS", "")
+
workspace_path = Path(workspace).resolve()
build_dir_path = workspace_path / build_dir
diff --git a/devops/scripts/ur_test_tools/constants.py b/devops/scripts/ur_test_tools/constants.py
index 396574d62516c..f7d831c6e8d69 100644
--- a/devops/scripts/ur_test_tools/constants.py
+++ b/devops/scripts/ur_test_tools/constants.py
@@ -29,6 +29,7 @@
TEST_TYPE_ADAPTER_SPECIFIC = "adapter-specific"
TEST_TYPE_CONFORMANCE = "conformance"
+TEST_TYPE_E2E = "e2e"
# These tests cause timeouts on CI and are excluded from adapter-specific runs
LIT_FILTER_OUT_ADAPTER_SPECIFIC = (
@@ -37,8 +38,9 @@
)
TEST_NOT_SELECTED_MSG = "Test not selected"
-SLOWEST_TESTS_HEADER = "Slowest Tests:"
-TEST_TIMES_HEADERS = ("Tests Times:", "Test Times:")
+# No trailing colon: lit appends a dynamic "(N of M):"/"(N):" suffix to these.
+SLOWEST_TESTS_HEADER = "Slowest Tests"
+TEST_TIMES_HEADERS = ("Tests Times", "Test Times")
FAIL_TIMEOUT_PATTERN = re.compile(r"^(FAIL|TIMEOUT):")
TEST_CATEGORY_PATTERN = re.compile(r"^([A-Za-z]+(?: [A-Za-z]+)*) Tests \((\d+)\):")
diff --git a/devops/scripts/ur_test_tools/models/config.py b/devops/scripts/ur_test_tools/models/config.py
index 0c640fa06751c..4bc038d9fac65 100644
--- a/devops/scripts/ur_test_tools/models/config.py
+++ b/devops/scripts/ur_test_tools/models/config.py
@@ -8,6 +8,9 @@ class TestConfig:
target: str
log_file: str
lit_filter_out: Optional[str] = None
+ lit_timeout: Optional[int] = None
+ lit_jobs: Optional[int] = None
+ extra_lit_opts: str = ""
def __post_init__(self):
if not all([self.target, self.log_file]):
diff --git a/devops/scripts/ur_test_tools/outputs/console.py b/devops/scripts/ur_test_tools/outputs/console.py
index 2d88e3a29ca0c..70e124c92a2bf 100644
--- a/devops/scripts/ur_test_tools/outputs/console.py
+++ b/devops/scripts/ur_test_tools/outputs/console.py
@@ -27,7 +27,9 @@ def filter_log_for_display(lines: List[str]) -> List[str]:
skip_until_empty = True
continue
- if stripped == SLOWEST_TESTS_HEADER or stripped in TEST_TIMES_HEADERS:
+ if stripped.startswith(SLOWEST_TESTS_HEADER) or any(
+ stripped.startswith(h) for h in TEST_TIMES_HEADERS
+ ):
in_timing = True
continue
diff --git a/devops/scripts/ur_test_tools/parsers/log_parser.py b/devops/scripts/ur_test_tools/parsers/log_parser.py
index d312c0438b4ff..227c5cb63b690 100644
--- a/devops/scripts/ur_test_tools/parsers/log_parser.py
+++ b/devops/scripts/ur_test_tools/parsers/log_parser.py
@@ -67,8 +67,8 @@ def extract_error_details(self) -> List[str]:
if in_error and (
TEST_CATEGORY_PATTERN.match(line)
- or line.strip() == SLOWEST_TESTS_HEADER
- or line.strip() in TEST_TIMES_HEADERS
+ or line.strip().startswith(SLOWEST_TESTS_HEADER)
+ or any(line.strip().startswith(h) for h in TEST_TIMES_HEADERS)
):
break
@@ -101,11 +101,11 @@ def extract_time_summary(self) -> Tuple[List[str], List[str]]:
for line in self.lines:
stripped = line.strip()
- if stripped == SLOWEST_TESTS_HEADER:
+ if stripped.startswith(SLOWEST_TESTS_HEADER):
current_section = "slowest"
skip_next_hr = True
continue
- elif stripped in TEST_TIMES_HEADERS:
+ elif any(stripped.startswith(h) for h in TEST_TIMES_HEADERS):
current_section = "histogram"
skip_next_hr = True
continue
diff --git a/devops/scripts/ur_test_tools/requirements.txt b/devops/scripts/ur_test_tools/requirements.txt
new file mode 100644
index 0000000000000..6c631f784380b
--- /dev/null
+++ b/devops/scripts/ur_test_tools/requirements.txt
@@ -0,0 +1,3 @@
+defusedxml==0.7.1
+# Required by lit's --timeout (kills hung test processes).
+psutil>=7.0.0
diff --git a/devops/scripts/ur_test_tools/summary_generator.py b/devops/scripts/ur_test_tools/summary_generator.py
index ed02323e3e23c..f4ccbb6852341 100644
--- a/devops/scripts/ur_test_tools/summary_generator.py
+++ b/devops/scripts/ur_test_tools/summary_generator.py
@@ -19,16 +19,6 @@
TestStatus.PASS,
]
-# Statuses worth listing by name in the compact GitHub step summary.
-_STEP_SUMMARY_LISTED_STATUSES = [
- TestStatus.FAIL,
- TestStatus.XPASS,
- TestStatus.TIMEOUT,
- TestStatus.UNRESOLVED,
-]
-
-_STEP_SUMMARY_MAX_LISTED_TESTS = 50
-
class SummaryReporter:
def __init__(self, result: TestRunResult):
@@ -59,12 +49,9 @@ def _display_test_groups(self) -> None:
def _print_test_group(self, status: TestStatus, tests: List[TestResult]) -> None:
print(f"::group::{status.display_label} ({len(tests)})")
- for test in tests[:100]:
+ for test in tests:
print(test.name)
- if len(tests) > 100:
- print(f"... and {len(tests) - 100} more")
-
print("::endgroup::")
def _display_timing(self) -> None:
@@ -96,29 +83,26 @@ def _display_timing(self) -> None:
print("::endgroup::")
def generate_github_step_summary(self) -> None:
- """Print a compact summary for $GITHUB_STEP_SUMMARY.
+ """Print a full summary for $GITHUB_STEP_SUMMARY.
- Unlike generate(), this lists only statuses that need attention
- (Failed, Unexpectedly Passed, Timed Out, Unresolved) and shows
- all other statuses as counts only, to keep the workflow run page
- readable regardless of how many tests were discovered.
+ Every status with at least one test gets its own collapsible section
+ listing every one of its test names, with no truncation.
"""
result = self.result
grouped = result.group_by_status()
- print("```")
-
- for status in _STEP_SUMMARY_LISTED_STATUSES:
+ for status in _STATUS_DISPLAY_ORDER:
tests = grouped.get(status, [])
if not tests:
continue
- print(f"{status.display_label} Tests ({len(tests)}):")
- for test in tests[:_STEP_SUMMARY_MAX_LISTED_TESTS]:
+ print(f"{status.display_label} ({len(tests)})
")
+ print()
+ print("```")
+ for test in tests:
print(test.name)
- if len(tests) > _STEP_SUMMARY_MAX_LISTED_TESTS:
- remaining = len(tests) - _STEP_SUMMARY_MAX_LISTED_TESTS
- print(f"... and {remaining} more")
+ print("```")
+ print("