From 02dd64cc0a034965be991fbd44b07082639a3371 Mon Sep 17 00:00:00 2001 From: Shengliang Xu Date: Sat, 17 Oct 2026 06:34:02 +0000 Subject: [PATCH 01/12] Run the required checks in GitHub's merge queue The merge queue builds a temporary branch of the target branch plus the queued pull requests and waits for the required status checks on it. The unit, code quality and docs workflows only ran on pull_request events, and the GPU, example and regression workflows only on pushes to the copy-pr-bot branches (pull-request/N) and on schedule, so no required check would ever report on a merge group. Add the merge_group trigger (checks_requested) to all six workflows and treat a merge group like a PR run: - The gate of the GPU, example and regression workflows and the unit tests' changed-files check diff the group against the commit it was built on (the pr-merge-base action reads it from the merge_group payload), so the same lanes run as on the PR and the wait for the unit `linux` check stays. - The *-pr-required-check aggregators, the single-GPU runners, the example timeouts, hf_ptq's -n 4 and the left-out `manual` unit tests all key off "a PR run", which now includes a merge group. - TruffleHog scans base..head of the group, like a PR. - The Codecov uploads are skipped: a queue commit never lands on main. Nightly, on-demand and PR runs behave as before. The queue itself is a ruleset setting (see the PR description); CONTRIBUTING.md describes it. Signed-off-by: Shengliang Xu --- .github/actions/pr-merge-base/action.yml | 22 +++++++++++++---- .github/workflows/_example_tests_runner.yml | 3 ++- .github/workflows/_pr_gate.yml | 13 +++++----- .github/workflows/code_quality.yml | 11 +++++++-- .github/workflows/example_tests.yml | 27 ++++++++++++--------- .github/workflows/gpu_tests.yml | 8 ++++-- .github/workflows/pages.yml | 3 +++ .github/workflows/regression_tests.yml | 8 ++++-- .github/workflows/unit_tests.yml | 17 +++++++++---- CONTRIBUTING.md | 6 ++++- 10 files changed, 82 insertions(+), 36 deletions(-) diff --git a/.github/actions/pr-merge-base/action.yml b/.github/actions/pr-merge-base/action.yml index cd4c1bc55df..d30d7436718 100644 --- a/.github/actions/pr-merge-base/action.yml +++ b/.github/actions/pr-merge-base/action.yml @@ -1,16 +1,17 @@ name: PR merge base description: > - Resolve the commit to diff a copied PR branch against: the merge base of the PR's head and its - target branch. Outputs are empty for non-PR triggers, which have no diff to inspect. + Resolve the commit to diff a PR run against. For a copied PR branch it is the merge base of the + PR's head and its target branch; for a merge queue group it is the commit the group was built + on. Outputs are empty for non-PR triggers, which have no diff to inspect. Requires a prior actions/checkout with fetch-depth 0. outputs: merge_base: description: Commit to use as the base for changed-file comparisons. - value: ${{ steps.calculate-merge-base.outputs.merge-base }} + value: ${{ steps.calculate-merge-base.outputs.merge-base || steps.merge-group.outputs.base_sha }} head_sha: - description: The PR's head commit. - value: ${{ steps.pr-shas.outputs.head_sha }} + description: The PR's head commit, or the merge queue group's head commit. + value: ${{ steps.pr-shas.outputs.head_sha || steps.merge-group.outputs.head_sha }} runs: using: composite @@ -36,3 +37,14 @@ runs: # status and emit an empty base, silently changing which lanes run. merge_base=$(git merge-base "${{ steps.pr-shas.outputs.base_sha }}" "${{ steps.pr-shas.outputs.head_sha }}") echo "merge-base=$merge_base" | tee --append "${GITHUB_OUTPUT}" + # A merge queue group is the target branch plus the queued PRs on top of it, so its diff + # against the commit it was built on is what the queue is about to merge. + - if: github.event_name == 'merge_group' + id: merge-group + shell: bash + env: + BASE_SHA: ${{ github.event.merge_group.base_sha }} + HEAD_SHA: ${{ github.event.merge_group.head_sha }} + run: | + echo "base_sha=$BASE_SHA" | tee --append "${GITHUB_OUTPUT}" + echo "head_sha=$HEAD_SHA" | tee --append "${GITHUB_OUTPUT}" diff --git a/.github/workflows/_example_tests_runner.yml b/.github/workflows/_example_tests_runner.yml index e3378865b65..3044ea66471 100644 --- a/.github/workflows/_example_tests_runner.yml +++ b/.github/workflows/_example_tests_runner.yml @@ -107,7 +107,7 @@ jobs: # Keep legacy ONNX/TRT tests available manually during deprecation. pytest_args+=(--ignore=tests/examples/diffusers/test_diffusers_onnx_trt.py) fi - if [[ "${{ inputs.example }}" == "hf_ptq" && "${{ github.ref }}" == refs/heads/pull-request/* ]]; then + if [[ "${{ inputs.example }}" == "hf_ptq" && ( "${{ github.ref }}" == refs/heads/pull-request/* || "${{ github.event_name }}" == "merge_group" ) ]]; then # One GPU on PR runners: small-model tests share it and the rest run alone on it # (tests/examples/hf_ptq/conftest.py). The 2-GPU nightly stays serial. pytest_args+=(-n 4) @@ -118,6 +118,7 @@ jobs: run: | echo "::warning title=Allowed example failure::'${{ inputs.example }}' failed but is in the allow-failure list (vars.ALLOW_FAILURE_EXAMPLE_TESTS); not blocking. Remove it from the variable once fixed." - name: Upload coverage to Codecov + if: github.event_name != 'merge_group' uses: codecov/codecov-action@v7 with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/_pr_gate.yml b/.github/workflows/_pr_gate.yml index e3605202f7c..2e2de28d657 100644 --- a/.github/workflows/_pr_gate.yml +++ b/.github/workflows/_pr_gate.yml @@ -27,18 +27,19 @@ jobs: outputs: any_changed: ${{ steps.changed-tests.outputs.any_modified || steps.non-pr.outputs.any_changed }} steps: - # For non-PR triggers (schedule, workflow_dispatch), always run tests + # For non-PR triggers (schedule, workflow_dispatch), always run tests. A PR run is a push to a + # copy-pr-bot branch (pull-request/N) or a merge queue group, which both diff against the base. - id: non-pr - if: "!startsWith(github.ref, 'refs/heads/pull-request/')" + if: "!(startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group')" run: echo "any_changed=true" >> $GITHUB_OUTPUT - - if: startsWith(github.ref, 'refs/heads/pull-request/') + - if: startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group' uses: actions/checkout@v6 with: fetch-depth: 0 - - if: startsWith(github.ref, 'refs/heads/pull-request/') + - if: startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group' id: base uses: ./.github/actions/pr-merge-base - - if: startsWith(github.ref, 'refs/heads/pull-request/') + - if: startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group' name: Check for changes in test-relevant directories id: changed-tests uses: step-security/changed-files@v47.0.5 @@ -51,7 +52,7 @@ jobs: wait-checks: needs: [check-file-changes] if: >- - startsWith(github.ref, 'refs/heads/pull-request/') && + (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && needs.check-file-changes.outputs.any_changed == 'true' runs-on: ubuntu-latest permissions: diff --git a/.github/workflows/code_quality.yml b/.github/workflows/code_quality.yml index 4743fef960b..abf82233195 100644 --- a/.github/workflows/code_quality.yml +++ b/.github/workflows/code_quality.yml @@ -3,6 +3,9 @@ name: Code Quality on: pull_request: branches: [main, release/*, feature/*] + # The merge queue re-runs the required checks on the PR merged with the target branch. + merge_group: + types: [checks_requested] schedule: - cron: "0 0 * * *" # Nightly workflow_dispatch: @@ -21,12 +24,16 @@ jobs: steps: - uses: actions/checkout@v6 with: - # PRs: TruffleHog needs the base commit, else it errors, scans nothing, and passes. - fetch-depth: ${{ github.event_name == 'pull_request' && '0' || '1' }} + # PRs and merge queue groups: TruffleHog needs the base commit, else it errors, scans nothing, and passes. + fetch-depth: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && '0' || '1' }} - uses: ./.github/actions/ubuntu-setup - name: Secret Scanning uses: trufflesecurity/trufflehog@v3.90.5 with: + # Only set in a merge queue group, where the diff to scan is base..head (empty for the other events, + # which the action handles itself) + base: ${{ github.event.merge_group.base_sha }} + head: ${{ github.event.merge_group.head_sha }} # Exclude lob: its `(live|test)_[a-zA-Z0-9_]{35}` pattern matches any pytest function # whose name happens to be exactly 35 characters after `test_`, and it reports them as # verified. Only nightly sees them, since it scans all history while PRs scan their diff. diff --git a/.github/workflows/example_tests.yml b/.github/workflows/example_tests.yml index 8cc432a22ba..2d87ad0496a 100644 --- a/.github/workflows/example_tests.yml +++ b/.github/workflows/example_tests.yml @@ -4,6 +4,9 @@ on: push: branches: ["pull-request/[0-9]+"] # NOTE: paths cannot be used since push happens to copied PR and only latest commit to PR is used + # The merge queue re-runs the required checks on the PR merged with the target branch. + merge_group: + types: [checks_requested] schedule: - cron: "0 0 * * *" # Nightly workflow_dispatch: @@ -37,10 +40,10 @@ jobs: - uses: actions/checkout@v6 with: fetch-depth: 0 - - if: startsWith(github.ref, 'refs/heads/pull-request/') + - if: startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group' id: base uses: ./.github/actions/pr-merge-base - - if: startsWith(github.ref, 'refs/heads/pull-request/') + - if: startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group' id: changed uses: step-security/changed-files@v47.0.5 with: @@ -87,7 +90,7 @@ jobs: - id: lanes env: # Nightly and on-demand runs have no diff to inspect, so they run everything. - NON_PR: ${{ !startsWith(github.ref, 'refs/heads/pull-request/') }} + NON_PR: ${{ !(startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') }} COMMON: ${{ steps.changed.outputs.common_any_modified }} TORCH: ${{ steps.changed.outputs.torch_any_modified }} VLLM: ${{ steps.changed.outputs.vllm_any_modified }} @@ -108,7 +111,7 @@ jobs: done echo "any_changed=$any" | tee --append "$GITHUB_OUTPUT" - name: Wait for unit tests before spending GPU runners - if: ${{ steps.lanes.outputs.any_changed == 'true' && startsWith(github.ref, 'refs/heads/pull-request/') }} + if: ${{ steps.lanes.outputs.any_changed == 'true' && (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') }} uses: poseidon/wait-for-status-checks@v0.7.0 with: token: ${{ secrets.GITHUB_TOKEN }} @@ -132,7 +135,7 @@ jobs: example: ${{ matrix.example }} timeout_minutes: 30 pip_install_extras: "[hf,dev-test]" - runner: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + runner: ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} allow_failure: ${{ contains(format(',{0},', vars.ALLOW_FAILURE_EXAMPLE_TESTS), format(',{0},', matrix.example)) }} ##### vLLM Example Tests ##### @@ -153,7 +156,7 @@ jobs: example: ${{ matrix.example }} timeout_minutes: 30 pip_install_extras: "[hf,dev-test]" - runner: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + runner: ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} allow_failure: ${{ contains(format(',{0},', vars.ALLOW_FAILURE_EXAMPLE_TESTS), format(',{0},', matrix.example)) }} ##### TensorRT-LLM Example Tests ##### @@ -172,7 +175,7 @@ jobs: docker_image: "nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc20" example: ${{ matrix.example }} pip_install_extras: "[hf,dev-test]" - runner: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + runner: ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} allow_failure: ${{ contains(format(',{0},', vars.ALLOW_FAILURE_EXAMPLE_TESTS), format(',{0},', matrix.example)) }} ##### Megatron Example Tests ##### @@ -187,9 +190,9 @@ jobs: docker_image: "nvcr.io/nvidia/nemo:26.08" example: megatron_bridge # PR runs single-GPU (~11 min); the nightly runs every test multi-GPU (~58 min). - timeout_minutes: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 30 || 75 }} + timeout_minutes: ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && 30 || 75 }} pip_install_extras: "[hf,puzzletron,dev-test]" - runner: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + runner: ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} allow_failure: ${{ contains(format(',{0},', vars.ALLOW_FAILURE_EXAMPLE_TESTS), ',megatron_bridge,') }} ##### Diffusers Example Tests ##### @@ -206,7 +209,7 @@ jobs: example: diffusers timeout_minutes: 60 pip_install_extras: "[onnx,hf,dev-test]" - runner: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + runner: ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} allow_failure: ${{ contains(format(',{0},', vars.ALLOW_FAILURE_EXAMPLE_TESTS), ',diffusers,') }} ##### ONNX/TensorRT Example Tests ##### @@ -229,13 +232,13 @@ jobs: example: ${{ matrix.example }} timeout_minutes: 45 pip_install_extras: "[onnx,hf,dev-test]" - runner: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + runner: ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} allow_failure: ${{ contains(format(',{0},', vars.ALLOW_FAILURE_EXAMPLE_TESTS), format(',{0},', matrix.example)) }} ##### Required Check for PR ##### example-pr-required-check: # Run even if example tests are skipped - if: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && always() }} + if: ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && always() }} needs: [pr-gate, torch, vllm, trtllm, megatron, diffusers, onnx] runs-on: ubuntu-latest steps: diff --git a/.github/workflows/gpu_tests.yml b/.github/workflows/gpu_tests.yml index a0389da7b62..906f467ddd2 100644 --- a/.github/workflows/gpu_tests.yml +++ b/.github/workflows/gpu_tests.yml @@ -4,6 +4,9 @@ on: push: branches: ["pull-request/[0-9]+"] # NOTE: paths cannot be used since push happens to copied PR and only latest commit to PR is used + # The merge queue re-runs the required checks on the PR merged with the target branch. + merge_group: + types: [checks_requested] schedule: - cron: "0 0 * * *" # Nightly workflow_dispatch: @@ -60,7 +63,7 @@ jobs: - example: gpu_vllm timeout: 15 container_image: docker.io/vllm/vllm-openai:v0.20.0 - runs-on: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + runs-on: ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} timeout-minutes: ${{ matrix.timeout }} container: image: ${{ matrix.container_image }} @@ -92,6 +95,7 @@ jobs: # Use `python3` (the vllm image has no `python` on PATH) python3 -m pip install nox && nox -s ${{ matrix.example }} - name: Upload GPU coverage to Codecov + if: github.event_name != 'merge_group' uses: codecov/codecov-action@v7 with: token: ${{ secrets.CODECOV_TOKEN }} @@ -102,7 +106,7 @@ jobs: gpu-pr-required-check: # Run even if gpu-tests is skipped - if: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && always() }} + if: ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && always() }} needs: [pr-gate, gpu-tests] runs-on: ubuntu-latest steps: diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 2132cdd302e..8e6720a6759 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -4,6 +4,9 @@ on: pull_request: types: [opened, synchronize, reopened, closed] branches: [main, release/*, feature/*] + # The merge queue re-runs the required checks on the PR merged with the target branch. + merge_group: + types: [checks_requested] push: branches: [main] schedule: diff --git a/.github/workflows/regression_tests.yml b/.github/workflows/regression_tests.yml index 001c4e6af4a..9dfb686c910 100644 --- a/.github/workflows/regression_tests.yml +++ b/.github/workflows/regression_tests.yml @@ -4,6 +4,9 @@ on: push: branches: ["pull-request/[0-9]+"] # NOTE: paths cannot be used since push happens to copied PR and only latest commit to PR is used + # The merge queue re-runs the required checks on the PR merged with the target branch. + merge_group: + types: [checks_requested] schedule: - cron: "0 0 * * *" # Nightly workflow_dispatch: @@ -40,7 +43,7 @@ jobs: regression-tests: needs: [pr-gate] if: needs.pr-gate.outputs.any_changed == 'true' - runs-on: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + runs-on: ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} timeout-minutes: 15 container: image: nvcr.io/nvidia/pytorch:26.07-py3 @@ -62,6 +65,7 @@ jobs: COVERAGE_FILE: ${{ github.workspace }}/.coverage run: python -m pip install nox && nox -s regression - name: Upload regression coverage to Codecov + if: github.event_name != 'merge_group' uses: codecov/codecov-action@v7 with: token: ${{ secrets.CODECOV_TOKEN }} @@ -72,7 +76,7 @@ jobs: regression-pr-required-check: # Run even if regression-tests is skipped - if: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && always() }} + if: ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && always() }} needs: [pr-gate, regression-tests] runs-on: ubuntu-latest steps: diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 6fbca185d9f..f33f7de53a5 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -3,6 +3,9 @@ name: Unit tests on: pull_request: branches: [main, release/*, feature/*] + # The merge queue re-runs the required checks on the PR merged with the target branch. + merge_group: + types: [checks_requested] push: branches: [main, release/*, feature/*] paths: @@ -35,15 +38,18 @@ jobs: - id: non-pr if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' run: echo "any_changed=true" >> $GITHUB_OUTPUT - - if: github.event_name == 'pull_request' + - if: github.event_name == 'pull_request' || github.event_name == 'merge_group' uses: actions/checkout@v6 with: fetch-depth: 0 - - if: github.event_name == 'pull_request' + - if: github.event_name == 'pull_request' || github.event_name == 'merge_group' name: Check for changes in test-relevant paths id: changed uses: step-security/changed-files@v47.0.5 with: + # Only set in a merge queue group (empty for PRs, where the action finds the base itself) + base_sha: ${{ github.event.merge_group.base_sha }} + sha: ${{ github.event.merge_group.head_sha }} files: | .github/workflows/unit_tests.yml modelopt/** @@ -67,10 +73,11 @@ jobs: env: COVERAGE_PROCESS_START: ${{ github.workspace }}/pyproject.toml COVERAGE_FILE: ${{ github.workspace }}/.coverage - # Everything but PRs also runs the slow tests marked ``manual`` (e.g. AutoNAS). - PYTEST_EXTRA: ${{ github.event_name != 'pull_request' && '--run-manual' || '' }} + # Everything but PRs and merge queue groups also runs the slow tests marked ``manual`` (e.g. AutoNAS). + PYTEST_EXTRA: ${{ github.event_name != 'pull_request' && github.event_name != 'merge_group' && '--run-manual' || '' }} run: pip install nox uv && nox -s "unit-3.12(torch_214, tf_latest)" -- $PYTEST_EXTRA - name: Upload coverage reports to Codecov + if: github.event_name != 'merge_group' uses: codecov/codecov-action@v7 with: token: ${{ secrets.CODECOV_TOKEN }} @@ -220,7 +227,7 @@ jobs: python -m pytest plugins/modelopt/skills/ -o addopts="" -p no:cacheprovider -v unit-pr-required-check: # Run even if some jobs are skipped - if: ${{ github.event_name == 'pull_request' && always() }} + if: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && always() }} needs: [check-file-changes, linux, windows, multi-version, partial-install, launcher, mcp, skills] runs-on: ubuntu-latest steps: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 45b6ffbf9f2..7c071db098f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -282,7 +282,11 @@ nox -s "unit-3.12(torch_211, tf_latest)" - **Contributors without write access:** When GitHub requests workflow approval, a reviewer with write permission must select **Approve and run workflows** to start GitHub Actions CI. - If any CI/CD checks fail, fix the issues and push again. -- Once your PR is approved and all checks pass, one of the reviewers will merge the PR. +- Once your PR is approved and all checks pass, one of the reviewers will merge the PR. With the merge queue + enabled on `main`, merging adds the PR to the queue: GitHub builds a temporary branch of `main` plus the + queued PRs, re-runs the required checks on it (the same checks, path gating and runners as a PR run, without + the nightly-only tests) and merges the PR only if they pass. A PR that fails in the queue is removed from it; + fix it and queue it again. - If your PR is a bug fix that should also land in an ongoing release branch, add the `cherry-pick-` label (e.g. `cherry-pick-0.47.0`) matching that release. Labeled PRs are cherry-picked into the release branch so the fix is included in the next release candidate. From 4665a0d7be39597963a4e98a70a8182676875932 Mon Sep 17 00:00:00 2001 From: Shengliang Xu Date: Sun, 18 Oct 2026 04:09:15 +0000 Subject: [PATCH 02/12] Run the merge queue checks only for main, release/* and feature/* The merge_group triggers had no branch filter, so a merge queue enabled on any branch would start the required checks there. Give the six triggers the branches of their pull_request triggers (main, release/*, feature/*), which are the branches that have the required checks, and say so in CONTRIBUTING.md. Signed-off-by: Shengliang Xu --- .github/workflows/code_quality.yml | 3 ++- .github/workflows/example_tests.yml | 3 ++- .github/workflows/gpu_tests.yml | 3 ++- .github/workflows/pages.yml | 3 ++- .github/workflows/regression_tests.yml | 3 ++- .github/workflows/unit_tests.yml | 3 ++- CONTRIBUTING.md | 9 +++++---- 7 files changed, 17 insertions(+), 10 deletions(-) diff --git a/.github/workflows/code_quality.yml b/.github/workflows/code_quality.yml index abf82233195..054aadb7f43 100644 --- a/.github/workflows/code_quality.yml +++ b/.github/workflows/code_quality.yml @@ -3,9 +3,10 @@ name: Code Quality on: pull_request: branches: [main, release/*, feature/*] - # The merge queue re-runs the required checks on the PR merged with the target branch. + # The merge queue re-runs the required checks on the PR merged with the target branch merge_group: types: [checks_requested] + branches: [main, release/*, feature/*] schedule: - cron: "0 0 * * *" # Nightly workflow_dispatch: diff --git a/.github/workflows/example_tests.yml b/.github/workflows/example_tests.yml index 2d87ad0496a..7761fc56181 100644 --- a/.github/workflows/example_tests.yml +++ b/.github/workflows/example_tests.yml @@ -4,9 +4,10 @@ on: push: branches: ["pull-request/[0-9]+"] # NOTE: paths cannot be used since push happens to copied PR and only latest commit to PR is used - # The merge queue re-runs the required checks on the PR merged with the target branch. + # The merge queue re-runs the required checks on the PR merged with the target branch merge_group: types: [checks_requested] + branches: [main, release/*, feature/*] schedule: - cron: "0 0 * * *" # Nightly workflow_dispatch: diff --git a/.github/workflows/gpu_tests.yml b/.github/workflows/gpu_tests.yml index 906f467ddd2..bd9e39451f4 100644 --- a/.github/workflows/gpu_tests.yml +++ b/.github/workflows/gpu_tests.yml @@ -4,9 +4,10 @@ on: push: branches: ["pull-request/[0-9]+"] # NOTE: paths cannot be used since push happens to copied PR and only latest commit to PR is used - # The merge queue re-runs the required checks on the PR merged with the target branch. + # The merge queue re-runs the required checks on the PR merged with the target branch merge_group: types: [checks_requested] + branches: [main, release/*, feature/*] schedule: - cron: "0 0 * * *" # Nightly workflow_dispatch: diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 8e6720a6759..91177043c8f 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -4,9 +4,10 @@ on: pull_request: types: [opened, synchronize, reopened, closed] branches: [main, release/*, feature/*] - # The merge queue re-runs the required checks on the PR merged with the target branch. + # The merge queue re-runs the required checks on the PR merged with the target branch merge_group: types: [checks_requested] + branches: [main, release/*, feature/*] push: branches: [main] schedule: diff --git a/.github/workflows/regression_tests.yml b/.github/workflows/regression_tests.yml index 9dfb686c910..e8c5b0d3db2 100644 --- a/.github/workflows/regression_tests.yml +++ b/.github/workflows/regression_tests.yml @@ -4,9 +4,10 @@ on: push: branches: ["pull-request/[0-9]+"] # NOTE: paths cannot be used since push happens to copied PR and only latest commit to PR is used - # The merge queue re-runs the required checks on the PR merged with the target branch. + # The merge queue re-runs the required checks on the PR merged with the target branch merge_group: types: [checks_requested] + branches: [main, release/*, feature/*] schedule: - cron: "0 0 * * *" # Nightly workflow_dispatch: diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index f33f7de53a5..3e2a0524da0 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -3,9 +3,10 @@ name: Unit tests on: pull_request: branches: [main, release/*, feature/*] - # The merge queue re-runs the required checks on the PR merged with the target branch. + # The merge queue re-runs the required checks on the PR merged with the target branch merge_group: types: [checks_requested] + branches: [main, release/*, feature/*] push: branches: [main, release/*, feature/*] paths: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7c071db098f..8e1ef903ba3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -283,10 +283,11 @@ nox -s "unit-3.12(torch_211, tf_latest)" must select **Approve and run workflows** to start GitHub Actions CI. - If any CI/CD checks fail, fix the issues and push again. - Once your PR is approved and all checks pass, one of the reviewers will merge the PR. With the merge queue - enabled on `main`, merging adds the PR to the queue: GitHub builds a temporary branch of `main` plus the - queued PRs, re-runs the required checks on it (the same checks, path gating and runners as a PR run, without - the nightly-only tests) and merges the PR only if they pass. A PR that fails in the queue is removed from it; - fix it and queue it again. + enabled on its target branch (the queue runs the checks for `main`, `release/*` and `feature/*`), merging adds + the PR to the queue: GitHub builds a temporary branch of the target branch plus the queued PRs ahead of it, + re-runs the required checks on it (the same checks, path gating and runners as a PR run, without the + nightly-only tests) and merges the PR only if they pass. Each PR still lands on its own. A PR that fails in + the queue is removed from it and the PRs behind it are re-tested without it; fix it and queue it again. - If your PR is a bug fix that should also land in an ongoing release branch, add the `cherry-pick-` label (e.g. `cherry-pick-0.47.0`) matching that release. Labeled PRs are cherry-picked into the release branch so the fix is included in the next release candidate. From e3e2b32f53cc2f6d165bdc49a6d9c866f5821e07 Mon Sep 17 00:00:00 2001 From: Shengliang Xu Date: Sat, 17 Oct 2026 07:33:04 +0000 Subject: [PATCH 03/12] Run the GPU unit tests on every change and the GPU system tests in the merge queue A pull request ran up to five GPU lanes and thirteen example lanes on every push, although most of that time goes to tests that start several processes, build engines or compile for tens of seconds. Split the GPU tests by what they do: - GPU unit tests run in the pytest process on one GPU and take seconds. They run on every push to a pull request, as before. - GPU system tests carry the new pytest marker `system` and run in the merge queue and on the nightly. That is every test of tests/gpu_megatron, gpu_trtllm, gpu_vllm, regression and examples, and in tests/gpu the tests that request `dist_workers` or `need_N_gpus` (marked by tests/conftest.py) and the modules marked by hand: puzzletron, the unified HF export, the extension JIT builds, the TensorRT engine and trtexec tests, and the tests that spawn processes or run a subprocess. The pr-merge-base action now also reports `run_system`: always true in a merge queue group and on nightly and on-demand runs, and on a pull request only when it carries the `run-system-tests` label. When it is false the GPU workflow runs the `gpu` lane with `-m "not system"` and skips the system lanes (the lane steps move to a reusable _gpu_tests_runner.yml, like the example runner), and the regression and example workflows skip their jobs. The *-pr-required-check jobs keep their names and count skipped lanes as passed, with a notice on the run. A unit-only run does not upload GPU coverage, so carryforward keeps the last full report. CONTRIBUTING.md describes the split. Signed-off-by: Shengliang Xu --- .github/actions/pr-merge-base/action.yml | 9 ++- .github/workflows/_gpu_tests_runner.yml | 77 +++++++++++++++++++ .github/workflows/_pr_gate.yml | 5 ++ .github/workflows/example_tests.yml | 8 +- .github/workflows/gpu_tests.yml | 77 ++++++++----------- .github/workflows/regression_tests.yml | 9 ++- CONTRIBUTING.md | 25 ++++-- pyproject.toml | 1 + tests/conftest.py | 24 ++++-- .../gpu/_extensions/test_torch_extensions.py | 2 +- tests/gpu/_extensions/test_torch_kernels.py | 2 +- .../quantization/autotune/test_benchmark.py | 2 + .../quantization/autotune/test_workflow.py | 2 + .../test_autotune_quantization_integration.py | 3 + .../torch/deploy/_runtime/test_trt_client.py | 2 + .../export/test_qwen_vision_recipe_export.py | 2 + ...unified_hf_export_and_check_safetensors.py | 2 + .../test_fla_chunk_gated_delta_rule.py | 2 + .../attention/test_triton_fa_calibrate.py | 1 + .../models/test_modeling_ptq_registration.py | 2 + .../gpu/torch/puzzletron/test_nas_convert.py | 3 + tests/gpu/torch/puzzletron/test_nas_search.py | 3 + tests/gpu/torch/puzzletron/test_puzzletron.py | 2 + .../tools/test_save_ckpt_from_shards.py | 2 + .../utils/test_distributed_worker_pool.py | 2 + .../utils/test_hf_checkpoint_utils_dist.py | 2 + 26 files changed, 207 insertions(+), 64 deletions(-) create mode 100644 .github/workflows/_gpu_tests_runner.yml diff --git a/.github/actions/pr-merge-base/action.yml b/.github/actions/pr-merge-base/action.yml index d30d7436718..a1e15be5761 100644 --- a/.github/actions/pr-merge-base/action.yml +++ b/.github/actions/pr-merge-base/action.yml @@ -2,7 +2,9 @@ name: PR merge base description: > Resolve the commit to diff a PR run against. For a copied PR branch it is the merge base of the PR's head and its target branch; for a merge queue group it is the commit the group was built - on. Outputs are empty for non-PR triggers, which have no diff to inspect. + on. Outputs are empty for non-PR triggers, which have no diff to inspect. Also says whether the + GPU system tests run: always in a merge queue group, and on a copied PR branch only when the PR + carries the `run-system-tests` label (the other PR runs get the unit tests only). Requires a prior actions/checkout with fetch-depth 0. outputs: @@ -12,6 +14,9 @@ outputs: head_sha: description: The PR's head commit, or the merge queue group's head commit. value: ${{ steps.pr-shas.outputs.head_sha || steps.merge-group.outputs.head_sha }} + run_system: + description: '"true" if the GPU system tests run in this PR run, else "false". Empty for non-PR triggers.' + value: ${{ steps.pr-shas.outputs.run_system || steps.merge-group.outputs.run_system }} runs: using: composite @@ -28,6 +33,7 @@ runs: run: | echo "head_sha=$(echo "$PR_INFO" | jq -r '.head.sha')" >> $GITHUB_OUTPUT echo "base_sha=$(echo "$PR_INFO" | jq -r '.base.sha')" >> $GITHUB_OUTPUT + echo "run_system=$(echo "$PR_INFO" | jq -r '[(.labels // [])[].name] | index("run-system-tests") != null')" >> $GITHUB_OUTPUT # Get commit from the target branch that is present in the PR to use as base for changed files - if: startsWith(github.ref, 'refs/heads/pull-request/') id: calculate-merge-base @@ -48,3 +54,4 @@ runs: run: | echo "base_sha=$BASE_SHA" | tee --append "${GITHUB_OUTPUT}" echo "head_sha=$HEAD_SHA" | tee --append "${GITHUB_OUTPUT}" + echo "run_system=true" | tee --append "${GITHUB_OUTPUT}" diff --git a/.github/workflows/_gpu_tests_runner.yml b/.github/workflows/_gpu_tests_runner.yml new file mode 100644 index 00000000000..f378d135fc6 --- /dev/null +++ b/.github/workflows/_gpu_tests_runner.yml @@ -0,0 +1,77 @@ +# Reusable workflow for running one lane of the GPU tests +name: GPU Tests Runner + +on: + workflow_call: + inputs: + example: + description: "nox session of the lane (gpu, gpu_megatron, gpu_trtllm or gpu_vllm)" + required: true + type: string + container_image: + description: "Docker image to use for the lane" + required: true + type: string + timeout_minutes: + description: "Timeout in minutes for the job" + required: false + type: number + default: 60 + runner: + description: "GitHub runner to use" + required: true + type: string + unit_only: + description: "Run only the GPU unit tests (pytest -m 'not system'), as a PR without the run-system-tests label does" + required: false + type: boolean + default: false + +jobs: + run-test: + runs-on: ${{ inputs.runner }} + timeout-minutes: ${{ inputs.timeout_minutes }} + permissions: + contents: read + container: + image: ${{ inputs.container_image }} + env: + GIT_DEPTH: 1000 # For correct version + PIP_CONSTRAINT: "" # Disable pip constraint for upgrading packages + HF_TOKEN: ${{ secrets.HF_TOKEN }} + # Build CUDA kernels only for the runner's RTX PRO 6000 (sm_120), not the image's ~6 archs. + TORCH_CUDA_ARCH_LIST: "12.0" + steps: + - name: Install git + # The vllm container ships without git; needed for a real checkout (correct + # setuptools-scm version) and for the Codecov upload below. + if: inputs.example == 'gpu_vllm' + run: apt-get update && apt-get install -y git + - uses: actions/checkout@v6 + - uses: nv-gha-runners/setup-proxy-cache@main + - uses: ./.github/actions/cache-extensions + with: + cache-key: rtxpro6000-${{ inputs.container_image }} + - name: Setup environment variables + run: | + echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/include:/usr/lib/x86_64-linux-gnu" >> $GITHUB_ENV + - name: Run gpu tests + env: + COVERAGE_PROCESS_START: ${{ github.workspace }}/pyproject.toml + COVERAGE_FILE: ${{ github.workspace }}/.coverage + # The system tests (see tests/conftest.py) are left out of a PR run without the label + PYTEST_ADDOPTS: ${{ inputs.unit_only && '-m "not system"' || '' }} + run: | + # Use `python3` (the vllm image has no `python` on PATH) + python3 -m pip install nox && nox -s ${{ inputs.example }} + - name: Upload GPU coverage to Codecov + # A run of the unit tests only would upload a partial `gpu` report and skew the comparison + # with main; carryforward keeps the last full report instead. + if: ${{ !inputs.unit_only && github.event_name != 'merge_group' }} + uses: codecov/codecov-action@v7 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: coverage.xml + flags: gpu + fail_ci_if_error: false # test may be skipped if relevant file changes are not detected + verbose: true diff --git a/.github/workflows/_pr_gate.yml b/.github/workflows/_pr_gate.yml index 2e2de28d657..9c932762664 100644 --- a/.github/workflows/_pr_gate.yml +++ b/.github/workflows/_pr_gate.yml @@ -20,12 +20,17 @@ on: any_changed: description: "Whether any relevant files changed" value: ${{ jobs.check-file-changes.outputs.any_changed }} + run_system: + description: "Whether the GPU system tests run: always except on a PR run without the run-system-tests label" + value: ${{ jobs.check-file-changes.outputs.run_system }} jobs: check-file-changes: runs-on: ubuntu-latest outputs: any_changed: ${{ steps.changed-tests.outputs.any_modified || steps.non-pr.outputs.any_changed }} + # Empty outside PR runs (nightly, on demand), which run everything + run_system: ${{ steps.base.outputs.run_system || 'true' }} steps: # For non-PR triggers (schedule, workflow_dispatch), always run tests. A PR run is a push to a # copy-pr-bot branch (pull-request/N) or a merge queue group, which both diff against the base. diff --git a/.github/workflows/example_tests.yml b/.github/workflows/example_tests.yml index 7761fc56181..9a3574101be 100644 --- a/.github/workflows/example_tests.yml +++ b/.github/workflows/example_tests.yml @@ -31,6 +31,7 @@ jobs: contents: read outputs: any_changed: ${{ steps.lanes.outputs.any_changed }} + run_system: ${{ steps.base.outputs.run_system || 'true' }} torch: ${{ steps.lanes.outputs.torch }} vllm: ${{ steps.lanes.outputs.vllm }} trtllm: ${{ steps.lanes.outputs.trtllm }} @@ -92,6 +93,8 @@ jobs: env: # Nightly and on-demand runs have no diff to inspect, so they run everything. NON_PR: ${{ !(startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') }} + # The example tests are system tests: a PR run without the run-system-tests label skips them + RUN_SYSTEM: ${{ steps.base.outputs.run_system || 'true' }} COMMON: ${{ steps.changed.outputs.common_any_modified }} TORCH: ${{ steps.changed.outputs.torch_any_modified }} VLLM: ${{ steps.changed.outputs.vllm_any_modified }} @@ -102,7 +105,7 @@ jobs: run: | any=false for lane in TORCH VLLM TRTLLM MEGATRON DIFFUSERS ONNX; do - if [[ "$NON_PR" == "true" || "$COMMON" == "true" || "${!lane}" == "true" ]]; then + if [[ "$RUN_SYSTEM" == "true" && ("$NON_PR" == "true" || "$COMMON" == "true" || "${!lane}" == "true") ]]; then run=true any=true else @@ -243,6 +246,9 @@ jobs: needs: [pr-gate, torch, vllm, trtllm, megatron, diffusers, onnx] runs-on: ubuntu-latest steps: + - name: Example tests are left to the merge queue + if: ${{ needs.pr-gate.outputs.run_system != 'true' }} + run: echo "::notice::The example tests are system tests and run in the merge queue. Add the run-system-tests label and re-run this workflow to run them on this PR." - name: Required example tests did not succeed if: | needs.pr-gate.result != 'success' || diff --git a/.github/workflows/gpu_tests.yml b/.github/workflows/gpu_tests.yml index bd9e39451f4..623b9ed41ee 100644 --- a/.github/workflows/gpu_tests.yml +++ b/.github/workflows/gpu_tests.yml @@ -40,16 +40,31 @@ jobs: tests/gpu_trtllm/** tests/gpu_vllm/** + # The GPU unit tests run on every change. The system tests (pytest marker `system`, see + # tests/conftest.py) run in the merge queue, on nightly and on-demand runs, and on a PR that + # carries the `run-system-tests` label: the gate says which (`run_system`). gpu-tests: needs: [pr-gate] if: needs.pr-gate.outputs.any_changed == 'true' + uses: ./.github/workflows/_gpu_tests_runner.yml + permissions: + contents: read + secrets: inherit + with: + example: gpu + timeout_minutes: 60 + container_image: nvcr.io/nvidia/pytorch:26.07-py3 + runner: ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + unit_only: ${{ needs.pr-gate.outputs.run_system != 'true' }} + + # Lanes whose tests are all system tests + gpu-system-tests: + needs: [pr-gate] + if: needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_system == 'true' strategy: fail-fast: false matrix: include: - - example: gpu - timeout: 60 - container_image: nvcr.io/nvidia/pytorch:26.07-py3 - example: gpu_megatron timeout: 60 container_image: nvcr.io/nvidia/nemo:26.08 @@ -64,53 +79,25 @@ jobs: - example: gpu_vllm timeout: 15 container_image: docker.io/vllm/vllm-openai:v0.20.0 - runs-on: ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} - timeout-minutes: ${{ matrix.timeout }} - container: - image: ${{ matrix.container_image }} - env: - GIT_DEPTH: 1000 # For correct version - PIP_CONSTRAINT: "" # Disable pip constraint for upgrading packages - HF_TOKEN: ${{ secrets.HF_TOKEN }} - # Build CUDA kernels only for the runner's RTX PRO 6000 (sm_120), not the image's ~6 archs. - TORCH_CUDA_ARCH_LIST: "12.0" - steps: - - name: Install git - # The vllm container ships without git; needed for a real checkout (correct - # setuptools-scm version) and for the Codecov upload below. - if: matrix.example == 'gpu_vllm' - run: apt-get update && apt-get install -y git - - uses: actions/checkout@v6 - - uses: nv-gha-runners/setup-proxy-cache@main - - uses: ./.github/actions/cache-extensions - with: - cache-key: rtxpro6000-${{ matrix.container_image }} - - name: Setup environment variables - run: | - echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/include:/usr/lib/x86_64-linux-gnu" >> $GITHUB_ENV - - name: Run gpu tests - env: - COVERAGE_PROCESS_START: ${{ github.workspace }}/pyproject.toml - COVERAGE_FILE: ${{ github.workspace }}/.coverage - run: | - # Use `python3` (the vllm image has no `python` on PATH) - python3 -m pip install nox && nox -s ${{ matrix.example }} - - name: Upload GPU coverage to Codecov - if: github.event_name != 'merge_group' - uses: codecov/codecov-action@v7 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: coverage.xml - flags: gpu - fail_ci_if_error: false # test may be skipped if relevant file changes are not detected - verbose: true + uses: ./.github/workflows/_gpu_tests_runner.yml + permissions: + contents: read + secrets: inherit + with: + example: ${{ matrix.example }} + timeout_minutes: ${{ matrix.timeout }} + container_image: ${{ matrix.container_image }} + runner: ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} gpu-pr-required-check: # Run even if gpu-tests is skipped if: ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && always() }} - needs: [pr-gate, gpu-tests] + needs: [pr-gate, gpu-tests, gpu-system-tests] runs-on: ubuntu-latest steps: + - name: GPU system tests are left to the merge queue + if: ${{ needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_system != 'true' }} + run: echo "::notice::Only the GPU unit tests ran. The system tests run in the merge queue; add the run-system-tests label and re-run this workflow to run them on this PR." - name: Required GPU tests did not succeed - if: ${{ needs.pr-gate.result != 'success' || (needs.pr-gate.outputs.any_changed == 'true' && needs.gpu-tests.result != 'success') }} + if: ${{ needs.pr-gate.result != 'success' || (needs.pr-gate.outputs.any_changed == 'true' && (needs.gpu-tests.result != 'success' || (needs.pr-gate.outputs.run_system == 'true' && needs.gpu-system-tests.result != 'success'))) }} run: exit 1 diff --git a/.github/workflows/regression_tests.yml b/.github/workflows/regression_tests.yml index e8c5b0d3db2..2e0003e40ec 100644 --- a/.github/workflows/regression_tests.yml +++ b/.github/workflows/regression_tests.yml @@ -41,9 +41,11 @@ jobs: modelopt_recipes/general/speculative_decoding/** tools/launcher/** + # System tests: they run in the merge queue, on nightly and on-demand runs, and on a PR that + # carries the `run-system-tests` label. regression-tests: needs: [pr-gate] - if: needs.pr-gate.outputs.any_changed == 'true' + if: needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_system == 'true' runs-on: ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} timeout-minutes: 15 container: @@ -81,8 +83,11 @@ jobs: needs: [pr-gate, regression-tests] runs-on: ubuntu-latest steps: + - name: Regression tests are left to the merge queue + if: ${{ needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_system != 'true' }} + run: echo "::notice::The regression tests are system tests and run in the merge queue. Add the run-system-tests label and re-run this workflow to run them on this PR." - name: Required regression tests did not succeed if: | needs.pr-gate.result != 'success' || - (needs.pr-gate.outputs.any_changed == 'true' && needs.regression-tests.result != 'success') + (needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_system == 'true' && needs.regression-tests.result != 'success') run: exit 1 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8e1ef903ba3..71e8b8b713a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -150,11 +150,10 @@ We use [pytest](https://docs.pytest.org/) for all tests. For any new features / Test filenames must mirror the production module they primarily cover: tests for `foo.py` belong in `test_foo.py`. When a production module is renamed, split, or removed, rename or split its tests in the same change so test filenames do not preserve obsolete module names. A cross-module integration test may use a feature-oriented name only when no single production module owns the behavior. - `tests/unit`: Fast cpu-based unit tests for the core ModelOpt library. They should not take more than a few seconds to run. -- `tests/gpu`: Fast GPU-based unit tests for the core ModelOpt library. In most cases, they should not take more than a few seconds to run. -- `tests/gpu_megatron`: Fast GPU-based unit tests for the core ModelOpt library for Megatron-Core features. In most cases, they should not take more than a few seconds to run. -- `tests/gpu_trtllm`: Fast GPU-based unit tests for the core ModelOpt library for TensorRT-LLM features. In most cases, they should not take more than a few seconds to run. -- `tests/gpu_vllm`: Fast GPU-based unit tests for the core ModelOpt library for vLLM features. In most cases, they should not take more than a few seconds to run. -- `tests/examples`: Integration tests for ModelOpt examples. They should not take more than a few minutes to run. Please refer to [example test README](./tests/examples/README.md) for more details. +- `tests/gpu`: GPU-based tests for the core ModelOpt library: fast GPU unit tests, and the GPU system tests marked `system` (see [GPU unit and system tests](#gpu-unit-and-system-tests)). +- `tests/gpu_megatron`, `tests/gpu_trtllm`, `tests/gpu_vllm`: GPU system tests for the Megatron-Core, TensorRT-LLM and vLLM features of the core ModelOpt library. +- `tests/regression`: Training regression tests (GPU system tests). +- `tests/examples`: Integration tests for ModelOpt examples (system tests). They should not take more than a few minutes to run. Please refer to [example test README](./tests/examples/README.md) for more details. For lightweight focused local validation, run `pytest` directly on the relevant test path. For example: @@ -168,6 +167,15 @@ For broader repo validation and dependency setup, use [noxfile.py](./noxfile.py) nox -s "unit-3.12(torch_211, tf_latest)" ``` +### GPU unit and system tests + +GPU tests come in two kinds, and CI runs them at different times: + +- **GPU unit tests** run in the pytest process on one GPU, on tiny in-memory models, and take seconds. They run on every push to a pull request. +- **GPU system tests** carry the pytest marker `system`. A test is one if it uses several processes or GPUs (the `dist_workers` and `need_N_gpus` fixtures, `spawn_multiprocess_job`), launches a subprocess (`hf_ptq`, `trtexec`, an example script), builds or runs an inference engine, trains or evaluates for several steps, or compiles for tens of seconds. They run in the merge queue and on the nightly, not on every pull request push. To run them on a pull request, add the `run-system-tests` label and re-run the workflows. + +Everything in `tests/gpu_megatron`, `tests/gpu_trtllm`, `tests/gpu_vllm`, `tests/regression` and `tests/examples` is a system test. In `tests/gpu`, a test that requests `dist_workers` or `need_N_gpus` is one automatically; mark any other with `pytestmark = pytest.mark.system` (a whole module) or `@pytest.mark.system`. Locally, `pytest tests/gpu -m "not system"` runs the GPU unit tests and `pytest tests/gpu -m system` the system tests. + ### Test design principles - **Prefer the highest-level test that runs the real code path.** Choose an end-to-end test over a mock-heavy one @@ -285,9 +293,10 @@ nox -s "unit-3.12(torch_211, tf_latest)" - Once your PR is approved and all checks pass, one of the reviewers will merge the PR. With the merge queue enabled on its target branch (the queue runs the checks for `main`, `release/*` and `feature/*`), merging adds the PR to the queue: GitHub builds a temporary branch of the target branch plus the queued PRs ahead of it, - re-runs the required checks on it (the same checks, path gating and runners as a PR run, without the - nightly-only tests) and merges the PR only if they pass. Each PR still lands on its own. A PR that fails in - the queue is removed from it and the PRs behind it are re-tested without it; fix it and queue it again. + re-runs the required checks on it (path gating and runners as in a PR run, but with the GPU system tests, + which a PR run leaves out unless it carries the `run-system-tests` label) and merges the PR only if they + pass. Each PR still lands on its own. A PR that fails in the queue is removed from it and the PRs behind it + are re-tested without it; fix it and queue it again. - If your PR is a bug fix that should also land in an ongoing release branch, add the `cherry-pick-` label (e.g. `cherry-pick-0.47.0`) matching that release. Labeled PRs are cherry-picked into the release branch so the fix is included in the next release candidate. diff --git a/pyproject.toml b/pyproject.toml index 8843bcfc23e..078eb8dd7ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -371,6 +371,7 @@ markers = [ "integration: Tests that require external services or other non-hermetic dependencies", "manual: Only run when --run-manual is given", "release: Regression tests that should be run before every release", + "system: GPU system tests (several processes or GPUs, subprocesses, inference engines, heavy compilation); run by the merge queue and nightly, not on every pull request push", ] diff --git a/tests/conftest.py b/tests/conftest.py index af77546a244..d8129673e39 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -62,8 +62,18 @@ def pytest_addoption(parser): } +# GPU system tests need several processes or GPUs, spawn subprocesses, run an inference engine or +# compile heavy kernels (see CONTRIBUTING.md). Every test of these groups is one, and in `gpu` so is a +# test that requests one of these fixtures; other `gpu` tests opt in with `pytest.mark.system`. +# Pull requests run the rest with `-m "not system"`; the merge queue and the nightly run everything. +_SYSTEM_GROUPS = {"examples", "gpu_megatron", "gpu_trtllm", "gpu_vllm", "regression"} +_SYSTEM_FIXTURES = ("dist_workers", "need_2_gpus", "need_4_gpus", "need_8_gpus") + + +# tryfirst: the `system` marks must exist before pytest applies `-m` +@pytest.hookimpl(tryfirst=True) def pytest_collection_modifyitems(config, items): - """Skip flag-gated tests and apply a default per-test timeout based on the test directory.""" + """Mark system tests, skip flag-gated tests and apply a default per-test timeout by directory.""" skip_marks = [ ("manual", "--run-manual"), ("release", "--run-release"), @@ -78,14 +88,18 @@ def pytest_collection_modifyitems(config, items): tests_root = Path(__file__).parent for item in items: - if item.get_closest_marker("timeout") is not None or not item.path.is_relative_to( - tests_root - ): + if not item.path.is_relative_to(tests_root): continue # First path component under tests/ is the group dir (unit, gpu, examples, ...). + group = item.path.relative_to(tests_root).parts[0] + if group in _SYSTEM_GROUPS or ( + group == "gpu" and any(name.startswith(_SYSTEM_FIXTURES) for name in item.fixturenames) + ): + item.add_marker(pytest.mark.system) + if item.get_closest_marker("timeout") is not None: + continue # Crash loudly (rather than silently skip) if a group has no configured default, so a # newly added tests// must be given an explicit timeout in the mapping above. - group = item.path.relative_to(tests_root).parts[0] if group not in _DEFAULT_TIMEOUT: raise pytest.UsageError( f"tests/{group}/ has no default timeout; add '{group}' to " diff --git a/tests/gpu/_extensions/test_torch_extensions.py b/tests/gpu/_extensions/test_torch_extensions.py index 8c10a81ad4f..a83e85b49fd 100644 --- a/tests/gpu/_extensions/test_torch_extensions.py +++ b/tests/gpu/_extensions/test_torch_extensions.py @@ -22,7 +22,7 @@ import modelopt.torch.quantization.extensions as ext # Override default timeout as these tests JIT-compile the CUDA extensions, which is slow -pytestmark = pytest.mark.timeout(240) +pytestmark = [pytest.mark.system, pytest.mark.timeout(240)] # Compile extensions first so it does not count towards time used to run a test that needs it diff --git a/tests/gpu/_extensions/test_torch_kernels.py b/tests/gpu/_extensions/test_torch_kernels.py index 22c6c584a19..0e9354fb1da 100644 --- a/tests/gpu/_extensions/test_torch_kernels.py +++ b/tests/gpu/_extensions/test_torch_kernels.py @@ -22,7 +22,7 @@ import pytest # Override default timeout as these tests JIT-compile the CUDA extensions, which is slow -pytestmark = pytest.mark.timeout(180) +pytestmark = [pytest.mark.system, pytest.mark.timeout(180)] def test_conv3d_implicit_gemm(): diff --git a/tests/gpu/onnx/quantization/autotune/test_benchmark.py b/tests/gpu/onnx/quantization/autotune/test_benchmark.py index 012bf495ab6..44758f84aa2 100644 --- a/tests/gpu/onnx/quantization/autotune/test_benchmark.py +++ b/tests/gpu/onnx/quantization/autotune/test_benchmark.py @@ -342,6 +342,7 @@ def test_run_timing_passes_stream_handle_to_trt(): # --- TensorRTPyBenchmark (integration) --- +@pytest.mark.system class TestTensorRTPyBenchmark: """End-to-end tests for TensorRTPyBenchmark. Requires TensorRT and CUDA.""" @@ -418,6 +419,7 @@ def test_callable(self, simple_conv_model_bytes): # --- TrtExecBenchmark (integration) --- +@pytest.mark.system class TestTrtExecBenchmark: """End-to-end tests for TrtExecBenchmark. Requires trtexec in PATH.""" diff --git a/tests/gpu/onnx/quantization/autotune/test_workflow.py b/tests/gpu/onnx/quantization/autotune/test_workflow.py index eebc42f6c33..09dd9f61347 100644 --- a/tests/gpu/onnx/quantization/autotune/test_workflow.py +++ b/tests/gpu/onnx/quantization/autotune/test_workflow.py @@ -28,6 +28,8 @@ region_pattern_autotuning_workflow, ) +pytestmark = pytest.mark.system + @pytest.fixture def simple_conv_model(): diff --git a/tests/gpu/onnx/quantization/test_autotune_quantization_integration.py b/tests/gpu/onnx/quantization/test_autotune_quantization_integration.py index 829eebb553b..b4eac53308f 100644 --- a/tests/gpu/onnx/quantization/test_autotune_quantization_integration.py +++ b/tests/gpu/onnx/quantization/test_autotune_quantization_integration.py @@ -18,6 +18,7 @@ import onnx import onnx_graphsurgeon as gs +import pytest from _test_utils.import_helper import skip_if_no_tensorrt from _test_utils.onnx.lib_test_models import export_as_onnx from _test_utils.onnx.quantization.autotune.models import _create_simple_resnet18_model @@ -30,6 +31,8 @@ skip_if_no_tensorrt() +pytestmark = pytest.mark.system + def _quantized_tensor_indices(model: onnx.ModelProto) -> set[tuple[str, int]]: """Return (node_name, input_index) for every DQ-fed input slot in the model.""" diff --git a/tests/gpu/torch/deploy/_runtime/test_trt_client.py b/tests/gpu/torch/deploy/_runtime/test_trt_client.py index 72709336a6b..a35131977f4 100644 --- a/tests/gpu/torch/deploy/_runtime/test_trt_client.py +++ b/tests/gpu/torch/deploy/_runtime/test_trt_client.py @@ -31,6 +31,8 @@ from modelopt.torch._deploy._runtime.trt_client import TRTLocalClient from modelopt.torch._deploy.compilation import compile +pytestmark = pytest.mark.system + deployment = { "runtime": "TRT", "accelerator": "GPU", diff --git a/tests/gpu/torch/export/test_qwen_vision_recipe_export.py b/tests/gpu/torch/export/test_qwen_vision_recipe_export.py index 389375428b1..d473e118b9a 100644 --- a/tests/gpu/torch/export/test_qwen_vision_recipe_export.py +++ b/tests/gpu/torch/export/test_qwen_vision_recipe_export.py @@ -25,6 +25,8 @@ from modelopt.recipe import load_recipe from modelopt.torch.export import export_hf_checkpoint +pytestmark = pytest.mark.system + _VISION_RECIPE = "fp8_vision-kv_none" _JOINT_RECIPE = "fp8_vision_lm-kv_fp8_cast" diff --git a/tests/gpu/torch/export/test_unified_hf_export_and_check_safetensors.py b/tests/gpu/torch/export/test_unified_hf_export_and_check_safetensors.py index b45943e6e5e..bc9b0410873 100644 --- a/tests/gpu/torch/export/test_unified_hf_export_and_check_safetensors.py +++ b/tests/gpu/torch/export/test_unified_hf_export_and_check_safetensors.py @@ -25,6 +25,8 @@ ) from safetensors import safe_open +pytestmark = pytest.mark.system + # Here we map each qformat -> the suffix we expect in the generated safetensors directory @pytest.mark.parametrize( diff --git a/tests/gpu/torch/kernels/quantization/linear_attention/test_fla_chunk_gated_delta_rule.py b/tests/gpu/torch/kernels/quantization/linear_attention/test_fla_chunk_gated_delta_rule.py index defe05aa5d5..cb4cab2fe7b 100644 --- a/tests/gpu/torch/kernels/quantization/linear_attention/test_fla_chunk_gated_delta_rule.py +++ b/tests/gpu/torch/kernels/quantization/linear_attention/test_fla_chunk_gated_delta_rule.py @@ -29,6 +29,8 @@ chunk_gated_delta_rule, ) +pytestmark = pytest.mark.system + def make_inputs(): torch.manual_seed(123) diff --git a/tests/gpu/torch/kernels/sparsity/attention/test_triton_fa_calibrate.py b/tests/gpu/torch/kernels/sparsity/attention/test_triton_fa_calibrate.py index 598f97a3fda..6df3aac8046 100644 --- a/tests/gpu/torch/kernels/sparsity/attention/test_triton_fa_calibrate.py +++ b/tests/gpu/torch/kernels/sparsity/attention/test_triton_fa_calibrate.py @@ -272,6 +272,7 @@ def test_measure_sparsity_returns_counts(self): assert out._sparsity_total > 0 assert out._sparsity_skipped <= out._sparsity_total + @pytest.mark.system def test_first_measured_call_has_real_tile_count_with_autotune(self): """Counters from the first measured call should not include autotune trials.""" script = r""" diff --git a/tests/gpu/torch/models/test_modeling_ptq_registration.py b/tests/gpu/torch/models/test_modeling_ptq_registration.py index 6782b64c344..cf8a77f984b 100644 --- a/tests/gpu/torch/models/test_modeling_ptq_registration.py +++ b/tests/gpu/torch/models/test_modeling_ptq_registration.py @@ -30,6 +30,8 @@ pytest.importorskip("transformers") +pytestmark = pytest.mark.system + FIRST_IMPORTS = [ "modelopt.torch.models.falcon.modeling_ptq", "modelopt.torch.models.gpt_oss.modeling_ptq", diff --git a/tests/gpu/torch/puzzletron/test_nas_convert.py b/tests/gpu/torch/puzzletron/test_nas_convert.py index 57239f1c0e4..656ae39b9ec 100644 --- a/tests/gpu/torch/puzzletron/test_nas_convert.py +++ b/tests/gpu/torch/puzzletron/test_nas_convert.py @@ -18,6 +18,7 @@ from functools import partial from pathlib import Path +import pytest import torch from _test_utils.torch.distributed.utils import spawn_multiprocess_job from _test_utils.torch.puzzletron.utils import setup_test_model_and_data @@ -26,6 +27,8 @@ import modelopt.torch.puzzletron as mtpz import modelopt.torch.utils.distributed as dist +pytestmark = pytest.mark.system + def test_nas_convert_ffn_pruning(project_root_path: Path, tmp_path: Path): spawn_multiprocess_job( diff --git a/tests/gpu/torch/puzzletron/test_nas_search.py b/tests/gpu/torch/puzzletron/test_nas_search.py index ae0dff47be5..f5916088e1f 100644 --- a/tests/gpu/torch/puzzletron/test_nas_search.py +++ b/tests/gpu/torch/puzzletron/test_nas_search.py @@ -17,6 +17,7 @@ from functools import partial from pathlib import Path +import pytest import torch from _test_utils.torch.distributed.utils import spawn_multiprocess_job from _test_utils.torch.puzzletron.utils import setup_test_model_and_data @@ -25,6 +26,8 @@ import modelopt.torch.puzzletron as mtpz import modelopt.torch.utils.distributed as dist +pytestmark = pytest.mark.system + def test_nas_search(project_root_path: Path, tmp_path: Path): spawn_multiprocess_job( diff --git a/tests/gpu/torch/puzzletron/test_puzzletron.py b/tests/gpu/torch/puzzletron/test_puzzletron.py index 4953db2c737..6b0fe7eed9d 100644 --- a/tests/gpu/torch/puzzletron/test_puzzletron.py +++ b/tests/gpu/torch/puzzletron/test_puzzletron.py @@ -25,6 +25,8 @@ import modelopt.torch.puzzletron as mtpz import modelopt.torch.utils.distributed as dist +pytestmark = pytest.mark.system + # The e2e test to compress a model based on Local Neural Architecture Search (Mixed Integer Programing NAS search) # using a one-click command. # diff --git a/tests/gpu/torch/puzzletron/tools/test_save_ckpt_from_shards.py b/tests/gpu/torch/puzzletron/tools/test_save_ckpt_from_shards.py index a31c687cc1e..0480df10c16 100644 --- a/tests/gpu/torch/puzzletron/tools/test_save_ckpt_from_shards.py +++ b/tests/gpu/torch/puzzletron/tools/test_save_ckpt_from_shards.py @@ -33,6 +33,8 @@ save_checkpoint_from_shards, ) +pytestmark = pytest.mark.system + class TestSaveCheckpointFromShardsSingleProcess: """Tests that run without torch.distributed (world_size=1 path).""" diff --git a/tests/gpu/torch/utils/test_distributed_worker_pool.py b/tests/gpu/torch/utils/test_distributed_worker_pool.py index 7ae3a966278..96cbb9fa671 100644 --- a/tests/gpu/torch/utils/test_distributed_worker_pool.py +++ b/tests/gpu/torch/utils/test_distributed_worker_pool.py @@ -23,6 +23,8 @@ import torch.distributed as dist from _test_utils.torch.distributed.utils import DistributedWorkerPool, get_live_pool +pytestmark = pytest.mark.system + def _hang(rank, size): time.sleep(600) diff --git a/tests/gpu/torch/utils/test_hf_checkpoint_utils_dist.py b/tests/gpu/torch/utils/test_hf_checkpoint_utils_dist.py index 72322ad50b4..6665405c836 100644 --- a/tests/gpu/torch/utils/test_hf_checkpoint_utils_dist.py +++ b/tests/gpu/torch/utils/test_hf_checkpoint_utils_dist.py @@ -26,6 +26,8 @@ pytest.importorskip("huggingface_hub") +pytestmark = pytest.mark.system + class _FakeHfApi: """``HfApi`` whose ``repo_info`` succeeds, so the Hub counts as reachable.""" From 337790e6f6b84eeb71531ad5a376b7f119a78fa5 Mon Sep 17 00:00:00 2001 From: Shengliang Xu Date: Sat, 17 Oct 2026 07:35:18 +0000 Subject: [PATCH 04/12] Run the GPU lanes when the reusable GPU lane workflow changes The gate of the GPU workflow lists the files whose change runs the lanes; the new _gpu_tests_runner.yml belongs there like _pr_gate.yml does. Signed-off-by: Shengliang Xu --- .github/workflows/gpu_tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/gpu_tests.yml b/.github/workflows/gpu_tests.yml index 623b9ed41ee..97be1b02c63 100644 --- a/.github/workflows/gpu_tests.yml +++ b/.github/workflows/gpu_tests.yml @@ -27,6 +27,7 @@ jobs: with: files: | .github/actions/** + .github/workflows/_gpu_tests_runner.yml .github/workflows/_pr_gate.yml .github/workflows/gpu_tests.yml modelopt/** From ed7fe8587d8409790b73f2b580946e52de76027a Mon Sep 17 00:00:00 2001 From: Shengliang Xu Date: Sun, 18 Oct 2026 04:57:53 +0000 Subject: [PATCH 05/12] Wrap the long lines the merge queue change added to the workflows The PR-like predicate made several runner, if and env values up to 193 columns wide. Fold them into `>-` scalars broken after `&&` / `||` (the parsed YAML value is byte-identical) and wrap two shell conditions and two comments, so no line this PR adds is longer than 110 columns. Signed-off-by: Shengliang Xu --- .github/workflows/_example_tests_runner.yml | 4 ++- .github/workflows/code_quality.yml | 6 ++-- .github/workflows/example_tests.yml | 39 +++++++++++++++------ .github/workflows/gpu_tests.yml | 8 +++-- .github/workflows/regression_tests.yml | 8 +++-- .github/workflows/unit_tests.yml | 7 ++-- 6 files changed, 53 insertions(+), 19 deletions(-) diff --git a/.github/workflows/_example_tests_runner.yml b/.github/workflows/_example_tests_runner.yml index 3044ea66471..e920db48ba8 100644 --- a/.github/workflows/_example_tests_runner.yml +++ b/.github/workflows/_example_tests_runner.yml @@ -107,7 +107,9 @@ jobs: # Keep legacy ONNX/TRT tests available manually during deprecation. pytest_args+=(--ignore=tests/examples/diffusers/test_diffusers_onnx_trt.py) fi - if [[ "${{ inputs.example }}" == "hf_ptq" && ( "${{ github.ref }}" == refs/heads/pull-request/* || "${{ github.event_name }}" == "merge_group" ) ]]; then + if [[ "${{ inputs.example }}" == "hf_ptq" && + ( "${{ github.ref }}" == refs/heads/pull-request/* || + "${{ github.event_name }}" == "merge_group" ) ]]; then # One GPU on PR runners: small-model tests share it and the rest run alone on it # (tests/examples/hf_ptq/conftest.py). The 2-GPU nightly stays serial. pytest_args+=(-n 4) diff --git a/.github/workflows/code_quality.yml b/.github/workflows/code_quality.yml index 054aadb7f43..98898778e3d 100644 --- a/.github/workflows/code_quality.yml +++ b/.github/workflows/code_quality.yml @@ -25,8 +25,10 @@ jobs: steps: - uses: actions/checkout@v6 with: - # PRs and merge queue groups: TruffleHog needs the base commit, else it errors, scans nothing, and passes. - fetch-depth: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && '0' || '1' }} + # PRs and merge queue groups: TruffleHog needs the base commit, else it errors, + # scans nothing, and passes. + fetch-depth: >- + ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && '0' || '1' }} - uses: ./.github/actions/ubuntu-setup - name: Secret Scanning uses: trufflesecurity/trufflehog@v3.90.5 diff --git a/.github/workflows/example_tests.yml b/.github/workflows/example_tests.yml index 7761fc56181..0d64e692721 100644 --- a/.github/workflows/example_tests.yml +++ b/.github/workflows/example_tests.yml @@ -91,7 +91,8 @@ jobs: - id: lanes env: # Nightly and on-demand runs have no diff to inspect, so they run everything. - NON_PR: ${{ !(startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') }} + NON_PR: >- + ${{ !(startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') }} COMMON: ${{ steps.changed.outputs.common_any_modified }} TORCH: ${{ steps.changed.outputs.torch_any_modified }} VLLM: ${{ steps.changed.outputs.vllm_any_modified }} @@ -112,7 +113,9 @@ jobs: done echo "any_changed=$any" | tee --append "$GITHUB_OUTPUT" - name: Wait for unit tests before spending GPU runners - if: ${{ steps.lanes.outputs.any_changed == 'true' && (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') }} + if: >- + ${{ steps.lanes.outputs.any_changed == 'true' && + (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') }} uses: poseidon/wait-for-status-checks@v0.7.0 with: token: ${{ secrets.GITHUB_TOKEN }} @@ -136,7 +139,9 @@ jobs: example: ${{ matrix.example }} timeout_minutes: 30 pip_install_extras: "[hf,dev-test]" - runner: ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + runner: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && + 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} allow_failure: ${{ contains(format(',{0},', vars.ALLOW_FAILURE_EXAMPLE_TESTS), format(',{0},', matrix.example)) }} ##### vLLM Example Tests ##### @@ -157,7 +162,9 @@ jobs: example: ${{ matrix.example }} timeout_minutes: 30 pip_install_extras: "[hf,dev-test]" - runner: ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + runner: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && + 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} allow_failure: ${{ contains(format(',{0},', vars.ALLOW_FAILURE_EXAMPLE_TESTS), format(',{0},', matrix.example)) }} ##### TensorRT-LLM Example Tests ##### @@ -176,7 +183,9 @@ jobs: docker_image: "nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc20" example: ${{ matrix.example }} pip_install_extras: "[hf,dev-test]" - runner: ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + runner: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && + 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} allow_failure: ${{ contains(format(',{0},', vars.ALLOW_FAILURE_EXAMPLE_TESTS), format(',{0},', matrix.example)) }} ##### Megatron Example Tests ##### @@ -191,9 +200,13 @@ jobs: docker_image: "nvcr.io/nvidia/nemo:26.08" example: megatron_bridge # PR runs single-GPU (~11 min); the nightly runs every test multi-GPU (~58 min). - timeout_minutes: ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && 30 || 75 }} + timeout_minutes: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || + github.event_name == 'merge_group') && 30 || 75 }} pip_install_extras: "[hf,puzzletron,dev-test]" - runner: ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + runner: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && + 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} allow_failure: ${{ contains(format(',{0},', vars.ALLOW_FAILURE_EXAMPLE_TESTS), ',megatron_bridge,') }} ##### Diffusers Example Tests ##### @@ -210,7 +223,9 @@ jobs: example: diffusers timeout_minutes: 60 pip_install_extras: "[onnx,hf,dev-test]" - runner: ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + runner: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && + 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} allow_failure: ${{ contains(format(',{0},', vars.ALLOW_FAILURE_EXAMPLE_TESTS), ',diffusers,') }} ##### ONNX/TensorRT Example Tests ##### @@ -233,13 +248,17 @@ jobs: example: ${{ matrix.example }} timeout_minutes: 45 pip_install_extras: "[onnx,hf,dev-test]" - runner: ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + runner: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && + 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} allow_failure: ${{ contains(format(',{0},', vars.ALLOW_FAILURE_EXAMPLE_TESTS), format(',{0},', matrix.example)) }} ##### Required Check for PR ##### example-pr-required-check: # Run even if example tests are skipped - if: ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && always() }} + if: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || + github.event_name == 'merge_group') && always() }} needs: [pr-gate, torch, vllm, trtllm, megatron, diffusers, onnx] runs-on: ubuntu-latest steps: diff --git a/.github/workflows/gpu_tests.yml b/.github/workflows/gpu_tests.yml index bd9e39451f4..0d315f2af56 100644 --- a/.github/workflows/gpu_tests.yml +++ b/.github/workflows/gpu_tests.yml @@ -64,7 +64,9 @@ jobs: - example: gpu_vllm timeout: 15 container_image: docker.io/vllm/vllm-openai:v0.20.0 - runs-on: ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + runs-on: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && + 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} timeout-minutes: ${{ matrix.timeout }} container: image: ${{ matrix.container_image }} @@ -107,7 +109,9 @@ jobs: gpu-pr-required-check: # Run even if gpu-tests is skipped - if: ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && always() }} + if: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || + github.event_name == 'merge_group') && always() }} needs: [pr-gate, gpu-tests] runs-on: ubuntu-latest steps: diff --git a/.github/workflows/regression_tests.yml b/.github/workflows/regression_tests.yml index e8c5b0d3db2..b76f819bc17 100644 --- a/.github/workflows/regression_tests.yml +++ b/.github/workflows/regression_tests.yml @@ -44,7 +44,9 @@ jobs: regression-tests: needs: [pr-gate] if: needs.pr-gate.outputs.any_changed == 'true' - runs-on: ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + runs-on: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && + 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} timeout-minutes: 15 container: image: nvcr.io/nvidia/pytorch:26.07-py3 @@ -77,7 +79,9 @@ jobs: regression-pr-required-check: # Run even if regression-tests is skipped - if: ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && always() }} + if: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || + github.event_name == 'merge_group') && always() }} needs: [pr-gate, regression-tests] runs-on: ubuntu-latest steps: diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 3e2a0524da0..ad5371c799d 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -74,8 +74,11 @@ jobs: env: COVERAGE_PROCESS_START: ${{ github.workspace }}/pyproject.toml COVERAGE_FILE: ${{ github.workspace }}/.coverage - # Everything but PRs and merge queue groups also runs the slow tests marked ``manual`` (e.g. AutoNAS). - PYTEST_EXTRA: ${{ github.event_name != 'pull_request' && github.event_name != 'merge_group' && '--run-manual' || '' }} + # Everything but PRs and merge queue groups also runs the slow tests marked ``manual`` + # (e.g. AutoNAS). + PYTEST_EXTRA: >- + ${{ github.event_name != 'pull_request' && + github.event_name != 'merge_group' && '--run-manual' || '' }} run: pip install nox uv && nox -s "unit-3.12(torch_214, tf_latest)" -- $PYTEST_EXTRA - name: Upload coverage reports to Codecov if: github.event_name != 'merge_group' From 1f42c84b91e1a0fefb11fe89a0741670ec9d672d Mon Sep 17 00:00:00 2001 From: Shengliang Xu Date: Sun, 18 Oct 2026 05:00:36 +0000 Subject: [PATCH 06/12] Wrap the long lines the GPU system tests change added to the workflows Fold the runner, if, description and notice values this PR added into `>-` scalars (the parsed YAML value is identical) and split the lane condition, the run_system echo and one `if` continuation, so no line either PR of the stack adds is longer than 110 columns. Signed-off-by: Shengliang Xu --- .github/actions/pr-merge-base/action.yml | 3 ++- .github/workflows/_gpu_tests_runner.yml | 4 +++- .github/workflows/_pr_gate.yml | 3 ++- .github/workflows/example_tests.yml | 10 +++++++--- .github/workflows/gpu_tests.yml | 21 ++++++++++++++++----- .github/workflows/regression_tests.yml | 11 ++++++++--- 6 files changed, 38 insertions(+), 14 deletions(-) diff --git a/.github/actions/pr-merge-base/action.yml b/.github/actions/pr-merge-base/action.yml index a1e15be5761..a80a929776b 100644 --- a/.github/actions/pr-merge-base/action.yml +++ b/.github/actions/pr-merge-base/action.yml @@ -33,7 +33,8 @@ runs: run: | echo "head_sha=$(echo "$PR_INFO" | jq -r '.head.sha')" >> $GITHUB_OUTPUT echo "base_sha=$(echo "$PR_INFO" | jq -r '.base.sha')" >> $GITHUB_OUTPUT - echo "run_system=$(echo "$PR_INFO" | jq -r '[(.labels // [])[].name] | index("run-system-tests") != null')" >> $GITHUB_OUTPUT + echo "run_system=$(echo "$PR_INFO" | + jq -r '[(.labels // [])[].name] | index("run-system-tests") != null')" >> $GITHUB_OUTPUT # Get commit from the target branch that is present in the PR to use as base for changed files - if: startsWith(github.ref, 'refs/heads/pull-request/') id: calculate-merge-base diff --git a/.github/workflows/_gpu_tests_runner.yml b/.github/workflows/_gpu_tests_runner.yml index f378d135fc6..d8d57cee64a 100644 --- a/.github/workflows/_gpu_tests_runner.yml +++ b/.github/workflows/_gpu_tests_runner.yml @@ -22,7 +22,9 @@ on: required: true type: string unit_only: - description: "Run only the GPU unit tests (pytest -m 'not system'), as a PR without the run-system-tests label does" + description: >- + Run only the GPU unit tests (pytest -m 'not system'), as a PR without the + run-system-tests label does required: false type: boolean default: false diff --git a/.github/workflows/_pr_gate.yml b/.github/workflows/_pr_gate.yml index 9c932762664..49ceabe0471 100644 --- a/.github/workflows/_pr_gate.yml +++ b/.github/workflows/_pr_gate.yml @@ -21,7 +21,8 @@ on: description: "Whether any relevant files changed" value: ${{ jobs.check-file-changes.outputs.any_changed }} run_system: - description: "Whether the GPU system tests run: always except on a PR run without the run-system-tests label" + description: >- + Whether the GPU system tests run: always except on a PR run without the run-system-tests label value: ${{ jobs.check-file-changes.outputs.run_system }} jobs: diff --git a/.github/workflows/example_tests.yml b/.github/workflows/example_tests.yml index fcacc258a6b..4fbe796a31d 100644 --- a/.github/workflows/example_tests.yml +++ b/.github/workflows/example_tests.yml @@ -92,7 +92,8 @@ jobs: - id: lanes env: # Nightly and on-demand runs have no diff to inspect, so they run everything. - NON_PR: ${{ !(startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') }} + NON_PR: >- + ${{ !(startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') }} # The example tests are system tests: a PR run without the run-system-tests label skips them RUN_SYSTEM: ${{ steps.base.outputs.run_system || 'true' }} COMMON: ${{ steps.changed.outputs.common_any_modified }} @@ -105,7 +106,8 @@ jobs: run: | any=false for lane in TORCH VLLM TRTLLM MEGATRON DIFFUSERS ONNX; do - if [[ "$RUN_SYSTEM" == "true" && ("$NON_PR" == "true" || "$COMMON" == "true" || "${!lane}" == "true") ]]; then + if [[ "$RUN_SYSTEM" == "true" && + ("$NON_PR" == "true" || "$COMMON" == "true" || "${!lane}" == "true") ]]; then run=true any=true else @@ -266,7 +268,9 @@ jobs: steps: - name: Example tests are left to the merge queue if: ${{ needs.pr-gate.outputs.run_system != 'true' }} - run: echo "::notice::The example tests are system tests and run in the merge queue. Add the run-system-tests label and re-run this workflow to run them on this PR." + run: >- + echo "::notice::The example tests are system tests and run in the merge queue. Add the + run-system-tests label and re-run this workflow to run them on this PR." - name: Required example tests did not succeed if: | needs.pr-gate.result != 'success' || diff --git a/.github/workflows/gpu_tests.yml b/.github/workflows/gpu_tests.yml index 97be1b02c63..5f47fc9155e 100644 --- a/.github/workflows/gpu_tests.yml +++ b/.github/workflows/gpu_tests.yml @@ -55,7 +55,9 @@ jobs: example: gpu timeout_minutes: 60 container_image: nvcr.io/nvidia/pytorch:26.07-py3 - runner: ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + runner: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && + 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} unit_only: ${{ needs.pr-gate.outputs.run_system != 'true' }} # Lanes whose tests are all system tests @@ -88,17 +90,26 @@ jobs: example: ${{ matrix.example }} timeout_minutes: ${{ matrix.timeout }} container_image: ${{ matrix.container_image }} - runner: ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + runner: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && + 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} gpu-pr-required-check: # Run even if gpu-tests is skipped - if: ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && always() }} + if: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || + github.event_name == 'merge_group') && always() }} needs: [pr-gate, gpu-tests, gpu-system-tests] runs-on: ubuntu-latest steps: - name: GPU system tests are left to the merge queue if: ${{ needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_system != 'true' }} - run: echo "::notice::Only the GPU unit tests ran. The system tests run in the merge queue; add the run-system-tests label and re-run this workflow to run them on this PR." + run: >- + echo "::notice::Only the GPU unit tests ran. The system tests run in the merge queue; add the + run-system-tests label and re-run this workflow to run them on this PR." - name: Required GPU tests did not succeed - if: ${{ needs.pr-gate.result != 'success' || (needs.pr-gate.outputs.any_changed == 'true' && (needs.gpu-tests.result != 'success' || (needs.pr-gate.outputs.run_system == 'true' && needs.gpu-system-tests.result != 'success'))) }} + if: >- + ${{ needs.pr-gate.result != 'success' || (needs.pr-gate.outputs.any_changed == 'true' && + (needs.gpu-tests.result != 'success' || (needs.pr-gate.outputs.run_system == 'true' && + needs.gpu-system-tests.result != 'success'))) }} run: exit 1 diff --git a/.github/workflows/regression_tests.yml b/.github/workflows/regression_tests.yml index 909fcc14c29..32e81b1b25a 100644 --- a/.github/workflows/regression_tests.yml +++ b/.github/workflows/regression_tests.yml @@ -46,7 +46,9 @@ jobs: regression-tests: needs: [pr-gate] if: needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_system == 'true' - runs-on: ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + runs-on: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && + 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} timeout-minutes: 15 container: image: nvcr.io/nvidia/pytorch:26.07-py3 @@ -87,9 +89,12 @@ jobs: steps: - name: Regression tests are left to the merge queue if: ${{ needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_system != 'true' }} - run: echo "::notice::The regression tests are system tests and run in the merge queue. Add the run-system-tests label and re-run this workflow to run them on this PR." + run: >- + echo "::notice::The regression tests are system tests and run in the merge queue. Add the + run-system-tests label and re-run this workflow to run them on this PR." - name: Required regression tests did not succeed if: | needs.pr-gate.result != 'success' || - (needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_system == 'true' && needs.regression-tests.result != 'success') + (needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_system == 'true' && + needs.regression-tests.result != 'success') run: exit 1 From 19a65c52f6347edc16b036657d041498acc15985 Mon Sep 17 00:00:00 2001 From: Shengliang Xu Date: Sun, 18 Oct 2026 05:38:06 +0000 Subject: [PATCH 07/12] Rename the GPU system tests to integration tests `integration` is the pytest convention next to `unit`, and pyproject.toml already declared a marker of that name that nothing uses, so reuse it instead of adding `system`: the marker, the `run_integration` gate output, the `run-integration-tests` label, the `gpu-integration-tests` job and the docs. Its description now covers what the set holds: other tools or engines, several processes or GPUs, subprocesses and heavy compilation. The same 114 tests of tests/gpu and tests/regression are marked before and after (collected on CPU). Signed-off-by: Shengliang Xu --- .github/actions/pr-merge-base/action.yml | 17 +++++------ .github/workflows/_gpu_tests_runner.yml | 8 +++--- .github/workflows/_pr_gate.yml | 9 +++--- .github/workflows/example_tests.yml | 14 +++++----- .github/workflows/gpu_tests.yml | 28 ++++++++++--------- .github/workflows/regression_tests.yml | 16 ++++++----- CONTRIBUTING.md | 18 ++++++------ pyproject.toml | 3 +- tests/conftest.py | 24 ++++++++-------- .../gpu/_extensions/test_torch_extensions.py | 2 +- tests/gpu/_extensions/test_torch_kernels.py | 2 +- .../quantization/autotune/test_benchmark.py | 4 +-- .../quantization/autotune/test_workflow.py | 2 +- .../test_autotune_quantization_integration.py | 2 +- .../torch/deploy/_runtime/test_trt_client.py | 2 +- .../export/test_qwen_vision_recipe_export.py | 2 +- ...unified_hf_export_and_check_safetensors.py | 2 +- .../test_fla_chunk_gated_delta_rule.py | 2 +- .../attention/test_triton_fa_calibrate.py | 2 +- .../models/test_modeling_ptq_registration.py | 2 +- .../gpu/torch/puzzletron/test_nas_convert.py | 2 +- tests/gpu/torch/puzzletron/test_nas_search.py | 2 +- tests/gpu/torch/puzzletron/test_puzzletron.py | 2 +- .../tools/test_save_ckpt_from_shards.py | 2 +- .../utils/test_distributed_worker_pool.py | 2 +- .../utils/test_hf_checkpoint_utils_dist.py | 2 +- 26 files changed, 90 insertions(+), 83 deletions(-) diff --git a/.github/actions/pr-merge-base/action.yml b/.github/actions/pr-merge-base/action.yml index a80a929776b..1a449df40ea 100644 --- a/.github/actions/pr-merge-base/action.yml +++ b/.github/actions/pr-merge-base/action.yml @@ -3,8 +3,8 @@ description: > Resolve the commit to diff a PR run against. For a copied PR branch it is the merge base of the PR's head and its target branch; for a merge queue group it is the commit the group was built on. Outputs are empty for non-PR triggers, which have no diff to inspect. Also says whether the - GPU system tests run: always in a merge queue group, and on a copied PR branch only when the PR - carries the `run-system-tests` label (the other PR runs get the unit tests only). + GPU integration tests run: always in a merge queue group, and on a copied PR branch only when the PR + carries the `run-integration-tests` label (the other PR runs get the unit tests only). Requires a prior actions/checkout with fetch-depth 0. outputs: @@ -14,9 +14,10 @@ outputs: head_sha: description: The PR's head commit, or the merge queue group's head commit. value: ${{ steps.pr-shas.outputs.head_sha || steps.merge-group.outputs.head_sha }} - run_system: - description: '"true" if the GPU system tests run in this PR run, else "false". Empty for non-PR triggers.' - value: ${{ steps.pr-shas.outputs.run_system || steps.merge-group.outputs.run_system }} + run_integration: + description: >- + "true" if the GPU integration tests run in this PR run, else "false". Empty for non-PR triggers. + value: ${{ steps.pr-shas.outputs.run_integration || steps.merge-group.outputs.run_integration }} runs: using: composite @@ -33,8 +34,8 @@ runs: run: | echo "head_sha=$(echo "$PR_INFO" | jq -r '.head.sha')" >> $GITHUB_OUTPUT echo "base_sha=$(echo "$PR_INFO" | jq -r '.base.sha')" >> $GITHUB_OUTPUT - echo "run_system=$(echo "$PR_INFO" | - jq -r '[(.labels // [])[].name] | index("run-system-tests") != null')" >> $GITHUB_OUTPUT + echo "run_integration=$(echo "$PR_INFO" | + jq -r '[(.labels // [])[].name] | index("run-integration-tests") != null')" >> $GITHUB_OUTPUT # Get commit from the target branch that is present in the PR to use as base for changed files - if: startsWith(github.ref, 'refs/heads/pull-request/') id: calculate-merge-base @@ -55,4 +56,4 @@ runs: run: | echo "base_sha=$BASE_SHA" | tee --append "${GITHUB_OUTPUT}" echo "head_sha=$HEAD_SHA" | tee --append "${GITHUB_OUTPUT}" - echo "run_system=true" | tee --append "${GITHUB_OUTPUT}" + echo "run_integration=true" | tee --append "${GITHUB_OUTPUT}" diff --git a/.github/workflows/_gpu_tests_runner.yml b/.github/workflows/_gpu_tests_runner.yml index d8d57cee64a..e0867d10b70 100644 --- a/.github/workflows/_gpu_tests_runner.yml +++ b/.github/workflows/_gpu_tests_runner.yml @@ -23,8 +23,8 @@ on: type: string unit_only: description: >- - Run only the GPU unit tests (pytest -m 'not system'), as a PR without the - run-system-tests label does + Run only the GPU unit tests (pytest -m 'not integration'), as a PR without the + run-integration-tests label does required: false type: boolean default: false @@ -61,8 +61,8 @@ jobs: env: COVERAGE_PROCESS_START: ${{ github.workspace }}/pyproject.toml COVERAGE_FILE: ${{ github.workspace }}/.coverage - # The system tests (see tests/conftest.py) are left out of a PR run without the label - PYTEST_ADDOPTS: ${{ inputs.unit_only && '-m "not system"' || '' }} + # The integration tests (see tests/conftest.py) are left out of a PR run without the label + PYTEST_ADDOPTS: ${{ inputs.unit_only && '-m "not integration"' || '' }} run: | # Use `python3` (the vllm image has no `python` on PATH) python3 -m pip install nox && nox -s ${{ inputs.example }} diff --git a/.github/workflows/_pr_gate.yml b/.github/workflows/_pr_gate.yml index 49ceabe0471..2e5380b05e9 100644 --- a/.github/workflows/_pr_gate.yml +++ b/.github/workflows/_pr_gate.yml @@ -20,10 +20,11 @@ on: any_changed: description: "Whether any relevant files changed" value: ${{ jobs.check-file-changes.outputs.any_changed }} - run_system: + run_integration: description: >- - Whether the GPU system tests run: always except on a PR run without the run-system-tests label - value: ${{ jobs.check-file-changes.outputs.run_system }} + Whether the GPU integration tests run: always except on a PR run without the + run-integration-tests label + value: ${{ jobs.check-file-changes.outputs.run_integration }} jobs: check-file-changes: @@ -31,7 +32,7 @@ jobs: outputs: any_changed: ${{ steps.changed-tests.outputs.any_modified || steps.non-pr.outputs.any_changed }} # Empty outside PR runs (nightly, on demand), which run everything - run_system: ${{ steps.base.outputs.run_system || 'true' }} + run_integration: ${{ steps.base.outputs.run_integration || 'true' }} steps: # For non-PR triggers (schedule, workflow_dispatch), always run tests. A PR run is a push to a # copy-pr-bot branch (pull-request/N) or a merge queue group, which both diff against the base. diff --git a/.github/workflows/example_tests.yml b/.github/workflows/example_tests.yml index 4fbe796a31d..4fd52b7503d 100644 --- a/.github/workflows/example_tests.yml +++ b/.github/workflows/example_tests.yml @@ -31,7 +31,7 @@ jobs: contents: read outputs: any_changed: ${{ steps.lanes.outputs.any_changed }} - run_system: ${{ steps.base.outputs.run_system || 'true' }} + run_integration: ${{ steps.base.outputs.run_integration || 'true' }} torch: ${{ steps.lanes.outputs.torch }} vllm: ${{ steps.lanes.outputs.vllm }} trtllm: ${{ steps.lanes.outputs.trtllm }} @@ -94,8 +94,8 @@ jobs: # Nightly and on-demand runs have no diff to inspect, so they run everything. NON_PR: >- ${{ !(startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') }} - # The example tests are system tests: a PR run without the run-system-tests label skips them - RUN_SYSTEM: ${{ steps.base.outputs.run_system || 'true' }} + # Example tests are integration tests: a PR run without the run-integration-tests label skips them + RUN_INTEGRATION: ${{ steps.base.outputs.run_integration || 'true' }} COMMON: ${{ steps.changed.outputs.common_any_modified }} TORCH: ${{ steps.changed.outputs.torch_any_modified }} VLLM: ${{ steps.changed.outputs.vllm_any_modified }} @@ -106,7 +106,7 @@ jobs: run: | any=false for lane in TORCH VLLM TRTLLM MEGATRON DIFFUSERS ONNX; do - if [[ "$RUN_SYSTEM" == "true" && + if [[ "$RUN_INTEGRATION" == "true" && ("$NON_PR" == "true" || "$COMMON" == "true" || "${!lane}" == "true") ]]; then run=true any=true @@ -267,10 +267,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Example tests are left to the merge queue - if: ${{ needs.pr-gate.outputs.run_system != 'true' }} + if: ${{ needs.pr-gate.outputs.run_integration != 'true' }} run: >- - echo "::notice::The example tests are system tests and run in the merge queue. Add the - run-system-tests label and re-run this workflow to run them on this PR." + echo "::notice::The example tests are integration tests and run in the merge queue. Add the + run-integration-tests label and re-run this workflow to run them on this PR." - name: Required example tests did not succeed if: | needs.pr-gate.result != 'success' || diff --git a/.github/workflows/gpu_tests.yml b/.github/workflows/gpu_tests.yml index 5f47fc9155e..5e365e674ab 100644 --- a/.github/workflows/gpu_tests.yml +++ b/.github/workflows/gpu_tests.yml @@ -41,9 +41,9 @@ jobs: tests/gpu_trtllm/** tests/gpu_vllm/** - # The GPU unit tests run on every change. The system tests (pytest marker `system`, see + # The GPU unit tests run on every change. The integration tests (pytest marker `integration`, see # tests/conftest.py) run in the merge queue, on nightly and on-demand runs, and on a PR that - # carries the `run-system-tests` label: the gate says which (`run_system`). + # carries the `run-integration-tests` label: the gate says which (`run_integration`). gpu-tests: needs: [pr-gate] if: needs.pr-gate.outputs.any_changed == 'true' @@ -58,12 +58,12 @@ jobs: runner: >- ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} - unit_only: ${{ needs.pr-gate.outputs.run_system != 'true' }} + unit_only: ${{ needs.pr-gate.outputs.run_integration != 'true' }} - # Lanes whose tests are all system tests - gpu-system-tests: + # Lanes whose tests are all integration tests + gpu-integration-tests: needs: [pr-gate] - if: needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_system == 'true' + if: needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_integration == 'true' strategy: fail-fast: false matrix: @@ -99,17 +99,19 @@ jobs: if: >- ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && always() }} - needs: [pr-gate, gpu-tests, gpu-system-tests] + needs: [pr-gate, gpu-tests, gpu-integration-tests] runs-on: ubuntu-latest steps: - - name: GPU system tests are left to the merge queue - if: ${{ needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_system != 'true' }} + - name: GPU integration tests are left to the merge queue + if: >- + ${{ needs.pr-gate.outputs.any_changed == 'true' && + needs.pr-gate.outputs.run_integration != 'true' }} run: >- - echo "::notice::Only the GPU unit tests ran. The system tests run in the merge queue; add the - run-system-tests label and re-run this workflow to run them on this PR." + echo "::notice::Only the GPU unit tests ran. The integration tests run in the merge queue; add the + run-integration-tests label and re-run this workflow to run them on this PR." - name: Required GPU tests did not succeed if: >- ${{ needs.pr-gate.result != 'success' || (needs.pr-gate.outputs.any_changed == 'true' && - (needs.gpu-tests.result != 'success' || (needs.pr-gate.outputs.run_system == 'true' && - needs.gpu-system-tests.result != 'success'))) }} + (needs.gpu-tests.result != 'success' || (needs.pr-gate.outputs.run_integration == 'true' && + needs.gpu-integration-tests.result != 'success'))) }} run: exit 1 diff --git a/.github/workflows/regression_tests.yml b/.github/workflows/regression_tests.yml index 32e81b1b25a..76fe97205b0 100644 --- a/.github/workflows/regression_tests.yml +++ b/.github/workflows/regression_tests.yml @@ -41,11 +41,11 @@ jobs: modelopt_recipes/general/speculative_decoding/** tools/launcher/** - # System tests: they run in the merge queue, on nightly and on-demand runs, and on a PR that - # carries the `run-system-tests` label. + # Integration tests: they run in the merge queue, on nightly and on-demand runs, and on a PR that + # carries the `run-integration-tests` label. regression-tests: needs: [pr-gate] - if: needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_system == 'true' + if: needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_integration == 'true' runs-on: >- ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} @@ -88,13 +88,15 @@ jobs: runs-on: ubuntu-latest steps: - name: Regression tests are left to the merge queue - if: ${{ needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_system != 'true' }} + if: >- + ${{ needs.pr-gate.outputs.any_changed == 'true' && + needs.pr-gate.outputs.run_integration != 'true' }} run: >- - echo "::notice::The regression tests are system tests and run in the merge queue. Add the - run-system-tests label and re-run this workflow to run them on this PR." + echo "::notice::The regression tests are integration tests and run in the merge queue. Add the + run-integration-tests label and re-run this workflow to run them on this PR." - name: Required regression tests did not succeed if: | needs.pr-gate.result != 'success' || - (needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_system == 'true' && + (needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_integration == 'true' && needs.regression-tests.result != 'success') run: exit 1 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 71e8b8b713a..bf702af44c5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -150,10 +150,10 @@ We use [pytest](https://docs.pytest.org/) for all tests. For any new features / Test filenames must mirror the production module they primarily cover: tests for `foo.py` belong in `test_foo.py`. When a production module is renamed, split, or removed, rename or split its tests in the same change so test filenames do not preserve obsolete module names. A cross-module integration test may use a feature-oriented name only when no single production module owns the behavior. - `tests/unit`: Fast cpu-based unit tests for the core ModelOpt library. They should not take more than a few seconds to run. -- `tests/gpu`: GPU-based tests for the core ModelOpt library: fast GPU unit tests, and the GPU system tests marked `system` (see [GPU unit and system tests](#gpu-unit-and-system-tests)). -- `tests/gpu_megatron`, `tests/gpu_trtllm`, `tests/gpu_vllm`: GPU system tests for the Megatron-Core, TensorRT-LLM and vLLM features of the core ModelOpt library. -- `tests/regression`: Training regression tests (GPU system tests). -- `tests/examples`: Integration tests for ModelOpt examples (system tests). They should not take more than a few minutes to run. Please refer to [example test README](./tests/examples/README.md) for more details. +- `tests/gpu`: GPU-based tests for the core ModelOpt library: fast GPU unit tests, and the GPU integration tests marked `integration` (see [GPU unit and integration tests](#gpu-unit-and-integration-tests)). +- `tests/gpu_megatron`, `tests/gpu_trtllm`, `tests/gpu_vllm`: GPU integration tests for the Megatron-Core, TensorRT-LLM and vLLM features of the core ModelOpt library. +- `tests/regression`: Training regression tests (GPU integration tests). +- `tests/examples`: Integration tests for ModelOpt examples. They should not take more than a few minutes to run. Please refer to [example test README](./tests/examples/README.md) for more details. For lightweight focused local validation, run `pytest` directly on the relevant test path. For example: @@ -167,14 +167,14 @@ For broader repo validation and dependency setup, use [noxfile.py](./noxfile.py) nox -s "unit-3.12(torch_211, tf_latest)" ``` -### GPU unit and system tests +### GPU unit and integration tests GPU tests come in two kinds, and CI runs them at different times: - **GPU unit tests** run in the pytest process on one GPU, on tiny in-memory models, and take seconds. They run on every push to a pull request. -- **GPU system tests** carry the pytest marker `system`. A test is one if it uses several processes or GPUs (the `dist_workers` and `need_N_gpus` fixtures, `spawn_multiprocess_job`), launches a subprocess (`hf_ptq`, `trtexec`, an example script), builds or runs an inference engine, trains or evaluates for several steps, or compiles for tens of seconds. They run in the merge queue and on the nightly, not on every pull request push. To run them on a pull request, add the `run-system-tests` label and re-run the workflows. +- **GPU integration tests** carry the pytest marker `integration`. A test is one if it uses several processes or GPUs (the `dist_workers` and `need_N_gpus` fixtures, `spawn_multiprocess_job`), launches a subprocess (`hf_ptq`, `trtexec`, an example script), builds or runs an inference engine, trains or evaluates for several steps, or compiles for tens of seconds. They run in the merge queue and on the nightly, not on every pull request push. To run them on a pull request, add the `run-integration-tests` label and re-run the workflows. -Everything in `tests/gpu_megatron`, `tests/gpu_trtllm`, `tests/gpu_vllm`, `tests/regression` and `tests/examples` is a system test. In `tests/gpu`, a test that requests `dist_workers` or `need_N_gpus` is one automatically; mark any other with `pytestmark = pytest.mark.system` (a whole module) or `@pytest.mark.system`. Locally, `pytest tests/gpu -m "not system"` runs the GPU unit tests and `pytest tests/gpu -m system` the system tests. +Everything in `tests/gpu_megatron`, `tests/gpu_trtllm`, `tests/gpu_vllm`, `tests/regression` and `tests/examples` is an integration test. In `tests/gpu`, a test that requests `dist_workers` or `need_N_gpus` is one automatically; mark any other with `pytestmark = pytest.mark.integration` (a whole module) or `@pytest.mark.integration`. Locally, `pytest tests/gpu -m "not integration"` runs the GPU unit tests and `pytest tests/gpu -m integration` the integration tests. ### Test design principles @@ -293,8 +293,8 @@ Everything in `tests/gpu_megatron`, `tests/gpu_trtllm`, `tests/gpu_vllm`, `tests - Once your PR is approved and all checks pass, one of the reviewers will merge the PR. With the merge queue enabled on its target branch (the queue runs the checks for `main`, `release/*` and `feature/*`), merging adds the PR to the queue: GitHub builds a temporary branch of the target branch plus the queued PRs ahead of it, - re-runs the required checks on it (path gating and runners as in a PR run, but with the GPU system tests, - which a PR run leaves out unless it carries the `run-system-tests` label) and merges the PR only if they + re-runs the required checks on it (path gating and runners as in a PR run, but with the GPU integration tests, + which a PR run leaves out unless it carries the `run-integration-tests` label) and merges the PR only if they pass. Each PR still lands on its own. A PR that fails in the queue is removed from it and the PRs behind it are re-tested without it; fix it and queue it again. - If your PR is a bug fix that should also land in an ongoing release branch, add the diff --git a/pyproject.toml b/pyproject.toml index 078eb8dd7ba..39c605ad112 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -368,10 +368,9 @@ pythonpath = ["tests/"] # Apply per-test timeouts (see tests/conftest.py) to the test call only, not fixture setup/teardown timeout_func_only = true markers = [ - "integration: Tests that require external services or other non-hermetic dependencies", + "integration: Tests that require external services or other non-hermetic dependencies, several processes or GPUs, subprocesses, inference engines or heavy compilation; run by the merge queue and nightly, not on every pull request push", "manual: Only run when --run-manual is given", "release: Regression tests that should be run before every release", - "system: GPU system tests (several processes or GPUs, subprocesses, inference engines, heavy compilation); run by the merge queue and nightly, not on every pull request push", ] diff --git a/tests/conftest.py b/tests/conftest.py index d8129673e39..f846d5b2f49 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -62,18 +62,19 @@ def pytest_addoption(parser): } -# GPU system tests need several processes or GPUs, spawn subprocesses, run an inference engine or -# compile heavy kernels (see CONTRIBUTING.md). Every test of these groups is one, and in `gpu` so is a -# test that requests one of these fixtures; other `gpu` tests opt in with `pytest.mark.system`. -# Pull requests run the rest with `-m "not system"`; the merge queue and the nightly run everything. -_SYSTEM_GROUPS = {"examples", "gpu_megatron", "gpu_trtllm", "gpu_vllm", "regression"} -_SYSTEM_FIXTURES = ("dist_workers", "need_2_gpus", "need_4_gpus", "need_8_gpus") +# GPU integration tests need several processes or GPUs, spawn subprocesses, run an inference engine +# or compile heavy kernels (see CONTRIBUTING.md). Every test of these groups is one, and in `gpu` so +# is a test that requests one of these fixtures; other `gpu` tests opt in with +# `pytest.mark.integration`. Pull requests run the rest with `-m "not integration"`; the merge queue +# and the nightly run everything. +_INTEGRATION_GROUPS = {"examples", "gpu_megatron", "gpu_trtllm", "gpu_vllm", "regression"} +_INTEGRATION_FIXTURES = ("dist_workers", "need_2_gpus", "need_4_gpus", "need_8_gpus") -# tryfirst: the `system` marks must exist before pytest applies `-m` +# tryfirst: the `integration` marks must exist before pytest applies `-m` @pytest.hookimpl(tryfirst=True) def pytest_collection_modifyitems(config, items): - """Mark system tests, skip flag-gated tests and apply a default per-test timeout by directory.""" + """Mark integration tests, skip flag-gated tests and set per-test timeouts by directory.""" skip_marks = [ ("manual", "--run-manual"), ("release", "--run-release"), @@ -92,10 +93,11 @@ def pytest_collection_modifyitems(config, items): continue # First path component under tests/ is the group dir (unit, gpu, examples, ...). group = item.path.relative_to(tests_root).parts[0] - if group in _SYSTEM_GROUPS or ( - group == "gpu" and any(name.startswith(_SYSTEM_FIXTURES) for name in item.fixturenames) + if group in _INTEGRATION_GROUPS or ( + group == "gpu" + and any(name.startswith(_INTEGRATION_FIXTURES) for name in item.fixturenames) ): - item.add_marker(pytest.mark.system) + item.add_marker(pytest.mark.integration) if item.get_closest_marker("timeout") is not None: continue # Crash loudly (rather than silently skip) if a group has no configured default, so a diff --git a/tests/gpu/_extensions/test_torch_extensions.py b/tests/gpu/_extensions/test_torch_extensions.py index a83e85b49fd..3725fc5d0f0 100644 --- a/tests/gpu/_extensions/test_torch_extensions.py +++ b/tests/gpu/_extensions/test_torch_extensions.py @@ -22,7 +22,7 @@ import modelopt.torch.quantization.extensions as ext # Override default timeout as these tests JIT-compile the CUDA extensions, which is slow -pytestmark = [pytest.mark.system, pytest.mark.timeout(240)] +pytestmark = [pytest.mark.integration, pytest.mark.timeout(240)] # Compile extensions first so it does not count towards time used to run a test that needs it diff --git a/tests/gpu/_extensions/test_torch_kernels.py b/tests/gpu/_extensions/test_torch_kernels.py index 0e9354fb1da..1bbee5e9d62 100644 --- a/tests/gpu/_extensions/test_torch_kernels.py +++ b/tests/gpu/_extensions/test_torch_kernels.py @@ -22,7 +22,7 @@ import pytest # Override default timeout as these tests JIT-compile the CUDA extensions, which is slow -pytestmark = [pytest.mark.system, pytest.mark.timeout(180)] +pytestmark = [pytest.mark.integration, pytest.mark.timeout(180)] def test_conv3d_implicit_gemm(): diff --git a/tests/gpu/onnx/quantization/autotune/test_benchmark.py b/tests/gpu/onnx/quantization/autotune/test_benchmark.py index 44758f84aa2..06cce63572b 100644 --- a/tests/gpu/onnx/quantization/autotune/test_benchmark.py +++ b/tests/gpu/onnx/quantization/autotune/test_benchmark.py @@ -342,7 +342,7 @@ def test_run_timing_passes_stream_handle_to_trt(): # --- TensorRTPyBenchmark (integration) --- -@pytest.mark.system +@pytest.mark.integration class TestTensorRTPyBenchmark: """End-to-end tests for TensorRTPyBenchmark. Requires TensorRT and CUDA.""" @@ -419,7 +419,7 @@ def test_callable(self, simple_conv_model_bytes): # --- TrtExecBenchmark (integration) --- -@pytest.mark.system +@pytest.mark.integration class TestTrtExecBenchmark: """End-to-end tests for TrtExecBenchmark. Requires trtexec in PATH.""" diff --git a/tests/gpu/onnx/quantization/autotune/test_workflow.py b/tests/gpu/onnx/quantization/autotune/test_workflow.py index 09dd9f61347..5b610976582 100644 --- a/tests/gpu/onnx/quantization/autotune/test_workflow.py +++ b/tests/gpu/onnx/quantization/autotune/test_workflow.py @@ -28,7 +28,7 @@ region_pattern_autotuning_workflow, ) -pytestmark = pytest.mark.system +pytestmark = pytest.mark.integration @pytest.fixture diff --git a/tests/gpu/onnx/quantization/test_autotune_quantization_integration.py b/tests/gpu/onnx/quantization/test_autotune_quantization_integration.py index b4eac53308f..8d266ed2068 100644 --- a/tests/gpu/onnx/quantization/test_autotune_quantization_integration.py +++ b/tests/gpu/onnx/quantization/test_autotune_quantization_integration.py @@ -31,7 +31,7 @@ skip_if_no_tensorrt() -pytestmark = pytest.mark.system +pytestmark = pytest.mark.integration def _quantized_tensor_indices(model: onnx.ModelProto) -> set[tuple[str, int]]: diff --git a/tests/gpu/torch/deploy/_runtime/test_trt_client.py b/tests/gpu/torch/deploy/_runtime/test_trt_client.py index a35131977f4..4b5832b18d8 100644 --- a/tests/gpu/torch/deploy/_runtime/test_trt_client.py +++ b/tests/gpu/torch/deploy/_runtime/test_trt_client.py @@ -31,7 +31,7 @@ from modelopt.torch._deploy._runtime.trt_client import TRTLocalClient from modelopt.torch._deploy.compilation import compile -pytestmark = pytest.mark.system +pytestmark = pytest.mark.integration deployment = { "runtime": "TRT", diff --git a/tests/gpu/torch/export/test_qwen_vision_recipe_export.py b/tests/gpu/torch/export/test_qwen_vision_recipe_export.py index d473e118b9a..d015611f879 100644 --- a/tests/gpu/torch/export/test_qwen_vision_recipe_export.py +++ b/tests/gpu/torch/export/test_qwen_vision_recipe_export.py @@ -25,7 +25,7 @@ from modelopt.recipe import load_recipe from modelopt.torch.export import export_hf_checkpoint -pytestmark = pytest.mark.system +pytestmark = pytest.mark.integration _VISION_RECIPE = "fp8_vision-kv_none" _JOINT_RECIPE = "fp8_vision_lm-kv_fp8_cast" diff --git a/tests/gpu/torch/export/test_unified_hf_export_and_check_safetensors.py b/tests/gpu/torch/export/test_unified_hf_export_and_check_safetensors.py index bc9b0410873..247e64f905a 100644 --- a/tests/gpu/torch/export/test_unified_hf_export_and_check_safetensors.py +++ b/tests/gpu/torch/export/test_unified_hf_export_and_check_safetensors.py @@ -25,7 +25,7 @@ ) from safetensors import safe_open -pytestmark = pytest.mark.system +pytestmark = pytest.mark.integration # Here we map each qformat -> the suffix we expect in the generated safetensors directory diff --git a/tests/gpu/torch/kernels/quantization/linear_attention/test_fla_chunk_gated_delta_rule.py b/tests/gpu/torch/kernels/quantization/linear_attention/test_fla_chunk_gated_delta_rule.py index cb4cab2fe7b..be13df0146f 100644 --- a/tests/gpu/torch/kernels/quantization/linear_attention/test_fla_chunk_gated_delta_rule.py +++ b/tests/gpu/torch/kernels/quantization/linear_attention/test_fla_chunk_gated_delta_rule.py @@ -29,7 +29,7 @@ chunk_gated_delta_rule, ) -pytestmark = pytest.mark.system +pytestmark = pytest.mark.integration def make_inputs(): diff --git a/tests/gpu/torch/kernels/sparsity/attention/test_triton_fa_calibrate.py b/tests/gpu/torch/kernels/sparsity/attention/test_triton_fa_calibrate.py index 6df3aac8046..c78dcb1ba8c 100644 --- a/tests/gpu/torch/kernels/sparsity/attention/test_triton_fa_calibrate.py +++ b/tests/gpu/torch/kernels/sparsity/attention/test_triton_fa_calibrate.py @@ -272,7 +272,7 @@ def test_measure_sparsity_returns_counts(self): assert out._sparsity_total > 0 assert out._sparsity_skipped <= out._sparsity_total - @pytest.mark.system + @pytest.mark.integration def test_first_measured_call_has_real_tile_count_with_autotune(self): """Counters from the first measured call should not include autotune trials.""" script = r""" diff --git a/tests/gpu/torch/models/test_modeling_ptq_registration.py b/tests/gpu/torch/models/test_modeling_ptq_registration.py index cf8a77f984b..0ba27611e6c 100644 --- a/tests/gpu/torch/models/test_modeling_ptq_registration.py +++ b/tests/gpu/torch/models/test_modeling_ptq_registration.py @@ -30,7 +30,7 @@ pytest.importorskip("transformers") -pytestmark = pytest.mark.system +pytestmark = pytest.mark.integration FIRST_IMPORTS = [ "modelopt.torch.models.falcon.modeling_ptq", diff --git a/tests/gpu/torch/puzzletron/test_nas_convert.py b/tests/gpu/torch/puzzletron/test_nas_convert.py index 656ae39b9ec..53cf23c5f90 100644 --- a/tests/gpu/torch/puzzletron/test_nas_convert.py +++ b/tests/gpu/torch/puzzletron/test_nas_convert.py @@ -27,7 +27,7 @@ import modelopt.torch.puzzletron as mtpz import modelopt.torch.utils.distributed as dist -pytestmark = pytest.mark.system +pytestmark = pytest.mark.integration def test_nas_convert_ffn_pruning(project_root_path: Path, tmp_path: Path): diff --git a/tests/gpu/torch/puzzletron/test_nas_search.py b/tests/gpu/torch/puzzletron/test_nas_search.py index f5916088e1f..768d10a027b 100644 --- a/tests/gpu/torch/puzzletron/test_nas_search.py +++ b/tests/gpu/torch/puzzletron/test_nas_search.py @@ -26,7 +26,7 @@ import modelopt.torch.puzzletron as mtpz import modelopt.torch.utils.distributed as dist -pytestmark = pytest.mark.system +pytestmark = pytest.mark.integration def test_nas_search(project_root_path: Path, tmp_path: Path): diff --git a/tests/gpu/torch/puzzletron/test_puzzletron.py b/tests/gpu/torch/puzzletron/test_puzzletron.py index 6b0fe7eed9d..24dfea9902e 100644 --- a/tests/gpu/torch/puzzletron/test_puzzletron.py +++ b/tests/gpu/torch/puzzletron/test_puzzletron.py @@ -25,7 +25,7 @@ import modelopt.torch.puzzletron as mtpz import modelopt.torch.utils.distributed as dist -pytestmark = pytest.mark.system +pytestmark = pytest.mark.integration # The e2e test to compress a model based on Local Neural Architecture Search (Mixed Integer Programing NAS search) # using a one-click command. diff --git a/tests/gpu/torch/puzzletron/tools/test_save_ckpt_from_shards.py b/tests/gpu/torch/puzzletron/tools/test_save_ckpt_from_shards.py index 0480df10c16..935663ac887 100644 --- a/tests/gpu/torch/puzzletron/tools/test_save_ckpt_from_shards.py +++ b/tests/gpu/torch/puzzletron/tools/test_save_ckpt_from_shards.py @@ -33,7 +33,7 @@ save_checkpoint_from_shards, ) -pytestmark = pytest.mark.system +pytestmark = pytest.mark.integration class TestSaveCheckpointFromShardsSingleProcess: diff --git a/tests/gpu/torch/utils/test_distributed_worker_pool.py b/tests/gpu/torch/utils/test_distributed_worker_pool.py index 96cbb9fa671..276e6cbfa82 100644 --- a/tests/gpu/torch/utils/test_distributed_worker_pool.py +++ b/tests/gpu/torch/utils/test_distributed_worker_pool.py @@ -23,7 +23,7 @@ import torch.distributed as dist from _test_utils.torch.distributed.utils import DistributedWorkerPool, get_live_pool -pytestmark = pytest.mark.system +pytestmark = pytest.mark.integration def _hang(rank, size): diff --git a/tests/gpu/torch/utils/test_hf_checkpoint_utils_dist.py b/tests/gpu/torch/utils/test_hf_checkpoint_utils_dist.py index 6665405c836..ecf698855f2 100644 --- a/tests/gpu/torch/utils/test_hf_checkpoint_utils_dist.py +++ b/tests/gpu/torch/utils/test_hf_checkpoint_utils_dist.py @@ -26,7 +26,7 @@ pytest.importorskip("huggingface_hub") -pytestmark = pytest.mark.system +pytestmark = pytest.mark.integration class _FakeHfApi: From da2595d134d3ac37057ff892f6874948251de9c0 Mon Sep 17 00:00:00 2001 From: Shengliang Xu Date: Sun, 18 Oct 2026 05:43:04 +0000 Subject: [PATCH 08/12] Run the whole tests/gpu on every push and split the GPU tests by lane only A plain PR push runs the `gpu` lane in full again, as before the split, instead of its unit-marked subset: +9-20 min per push today, about nothing once the speed-up series lands. Without a marker split inside tests/gpu the pytest hook, the hand marks, the `unit_only` input and the Codecov carve-out are gone. The merge queue only adds the lanes that integrate with other frameworks (Megatron-Core, TensorRT-LLM, vLLM) and the example and regression tests. Signed-off-by: Shengliang Xu --- .github/actions/pr-merge-base/action.yml | 5 ++-- .github/workflows/_gpu_tests_runner.yml | 13 +--------- .github/workflows/gpu_tests.yml | 9 +++---- CONTRIBUTING.md | 26 +++++++------------ pyproject.toml | 2 +- tests/conftest.py | 26 ++++--------------- .../gpu/_extensions/test_torch_extensions.py | 2 +- tests/gpu/_extensions/test_torch_kernels.py | 2 +- .../quantization/autotune/test_benchmark.py | 2 -- .../quantization/autotune/test_workflow.py | 2 -- .../test_autotune_quantization_integration.py | 3 --- .../torch/deploy/_runtime/test_trt_client.py | 2 -- .../export/test_qwen_vision_recipe_export.py | 2 -- ...unified_hf_export_and_check_safetensors.py | 2 -- .../test_fla_chunk_gated_delta_rule.py | 2 -- .../attention/test_triton_fa_calibrate.py | 1 - .../models/test_modeling_ptq_registration.py | 2 -- .../gpu/torch/puzzletron/test_nas_convert.py | 3 --- tests/gpu/torch/puzzletron/test_nas_search.py | 3 --- tests/gpu/torch/puzzletron/test_puzzletron.py | 2 -- .../tools/test_save_ckpt_from_shards.py | 2 -- .../utils/test_distributed_worker_pool.py | 2 -- .../utils/test_hf_checkpoint_utils_dist.py | 2 -- 23 files changed, 26 insertions(+), 91 deletions(-) diff --git a/.github/actions/pr-merge-base/action.yml b/.github/actions/pr-merge-base/action.yml index 1a449df40ea..4cc5457b679 100644 --- a/.github/actions/pr-merge-base/action.yml +++ b/.github/actions/pr-merge-base/action.yml @@ -3,8 +3,9 @@ description: > Resolve the commit to diff a PR run against. For a copied PR branch it is the merge base of the PR's head and its target branch; for a merge queue group it is the commit the group was built on. Outputs are empty for non-PR triggers, which have no diff to inspect. Also says whether the - GPU integration tests run: always in a merge queue group, and on a copied PR branch only when the PR - carries the `run-integration-tests` label (the other PR runs get the unit tests only). + integration tests (the Megatron, TensorRT-LLM and vLLM lanes, the example and regression tests) + run: always in a merge queue group, and on a copied PR branch only when the PR carries the + `run-integration-tests` label (the other PR runs skip them). Requires a prior actions/checkout with fetch-depth 0. outputs: diff --git a/.github/workflows/_gpu_tests_runner.yml b/.github/workflows/_gpu_tests_runner.yml index e0867d10b70..4175bb3e6c7 100644 --- a/.github/workflows/_gpu_tests_runner.yml +++ b/.github/workflows/_gpu_tests_runner.yml @@ -21,13 +21,6 @@ on: description: "GitHub runner to use" required: true type: string - unit_only: - description: >- - Run only the GPU unit tests (pytest -m 'not integration'), as a PR without the - run-integration-tests label does - required: false - type: boolean - default: false jobs: run-test: @@ -61,15 +54,11 @@ jobs: env: COVERAGE_PROCESS_START: ${{ github.workspace }}/pyproject.toml COVERAGE_FILE: ${{ github.workspace }}/.coverage - # The integration tests (see tests/conftest.py) are left out of a PR run without the label - PYTEST_ADDOPTS: ${{ inputs.unit_only && '-m "not integration"' || '' }} run: | # Use `python3` (the vllm image has no `python` on PATH) python3 -m pip install nox && nox -s ${{ inputs.example }} - name: Upload GPU coverage to Codecov - # A run of the unit tests only would upload a partial `gpu` report and skew the comparison - # with main; carryforward keeps the last full report instead. - if: ${{ !inputs.unit_only && github.event_name != 'merge_group' }} + if: github.event_name != 'merge_group' uses: codecov/codecov-action@v7 with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/gpu_tests.yml b/.github/workflows/gpu_tests.yml index 5e365e674ab..6e94f68fa04 100644 --- a/.github/workflows/gpu_tests.yml +++ b/.github/workflows/gpu_tests.yml @@ -41,8 +41,8 @@ jobs: tests/gpu_trtllm/** tests/gpu_vllm/** - # The GPU unit tests run on every change. The integration tests (pytest marker `integration`, see - # tests/conftest.py) run in the merge queue, on nightly and on-demand runs, and on a PR that + # The `gpu` lane (tests/gpu) runs on every change. The lanes that integrate with Megatron-Core, + # TensorRT-LLM and vLLM run in the merge queue, on nightly and on-demand runs, and on a PR that # carries the `run-integration-tests` label: the gate says which (`run_integration`). gpu-tests: needs: [pr-gate] @@ -58,9 +58,8 @@ jobs: runner: >- ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} - unit_only: ${{ needs.pr-gate.outputs.run_integration != 'true' }} - # Lanes whose tests are all integration tests + # Lanes that integrate with other frameworks gpu-integration-tests: needs: [pr-gate] if: needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_integration == 'true' @@ -107,7 +106,7 @@ jobs: ${{ needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_integration != 'true' }} run: >- - echo "::notice::Only the GPU unit tests ran. The integration tests run in the merge queue; add the + echo "::notice::The Megatron, TensorRT-LLM and vLLM lanes run in the merge queue; add the run-integration-tests label and re-run this workflow to run them on this PR." - name: Required GPU tests did not succeed if: >- diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bf702af44c5..28933729325 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -150,11 +150,14 @@ We use [pytest](https://docs.pytest.org/) for all tests. For any new features / Test filenames must mirror the production module they primarily cover: tests for `foo.py` belong in `test_foo.py`. When a production module is renamed, split, or removed, rename or split its tests in the same change so test filenames do not preserve obsolete module names. A cross-module integration test may use a feature-oriented name only when no single production module owns the behavior. - `tests/unit`: Fast cpu-based unit tests for the core ModelOpt library. They should not take more than a few seconds to run. -- `tests/gpu`: GPU-based tests for the core ModelOpt library: fast GPU unit tests, and the GPU integration tests marked `integration` (see [GPU unit and integration tests](#gpu-unit-and-integration-tests)). -- `tests/gpu_megatron`, `tests/gpu_trtllm`, `tests/gpu_vllm`: GPU integration tests for the Megatron-Core, TensorRT-LLM and vLLM features of the core ModelOpt library. -- `tests/regression`: Training regression tests (GPU integration tests). +- `tests/gpu`: Fast GPU-based unit tests for the core ModelOpt library. In most cases, they should not take more than a few seconds to run. +- `tests/gpu_megatron`: GPU-based integration tests for the core ModelOpt library for Megatron-Core features. +- `tests/gpu_trtllm`: GPU-based integration tests for the core ModelOpt library for TensorRT-LLM features. +- `tests/gpu_vllm`: GPU-based integration tests for the core ModelOpt library for vLLM features. - `tests/examples`: Integration tests for ModelOpt examples. They should not take more than a few minutes to run. Please refer to [example test README](./tests/examples/README.md) for more details. +CI runs the integration tests (`tests/gpu_megatron`, `tests/gpu_trtllm`, `tests/gpu_vllm`, `tests/regression` and `tests/examples`) in the merge queue and on the nightly, not on every pull request push. To run them on a pull request, add the `run-integration-tests` label and re-run the workflows. + For lightweight focused local validation, run `pytest` directly on the relevant test path. For example: ```bash @@ -167,15 +170,6 @@ For broader repo validation and dependency setup, use [noxfile.py](./noxfile.py) nox -s "unit-3.12(torch_211, tf_latest)" ``` -### GPU unit and integration tests - -GPU tests come in two kinds, and CI runs them at different times: - -- **GPU unit tests** run in the pytest process on one GPU, on tiny in-memory models, and take seconds. They run on every push to a pull request. -- **GPU integration tests** carry the pytest marker `integration`. A test is one if it uses several processes or GPUs (the `dist_workers` and `need_N_gpus` fixtures, `spawn_multiprocess_job`), launches a subprocess (`hf_ptq`, `trtexec`, an example script), builds or runs an inference engine, trains or evaluates for several steps, or compiles for tens of seconds. They run in the merge queue and on the nightly, not on every pull request push. To run them on a pull request, add the `run-integration-tests` label and re-run the workflows. - -Everything in `tests/gpu_megatron`, `tests/gpu_trtllm`, `tests/gpu_vllm`, `tests/regression` and `tests/examples` is an integration test. In `tests/gpu`, a test that requests `dist_workers` or `need_N_gpus` is one automatically; mark any other with `pytestmark = pytest.mark.integration` (a whole module) or `@pytest.mark.integration`. Locally, `pytest tests/gpu -m "not integration"` runs the GPU unit tests and `pytest tests/gpu -m integration` the integration tests. - ### Test design principles - **Prefer the highest-level test that runs the real code path.** Choose an end-to-end test over a mock-heavy one @@ -293,10 +287,10 @@ Everything in `tests/gpu_megatron`, `tests/gpu_trtllm`, `tests/gpu_vllm`, `tests - Once your PR is approved and all checks pass, one of the reviewers will merge the PR. With the merge queue enabled on its target branch (the queue runs the checks for `main`, `release/*` and `feature/*`), merging adds the PR to the queue: GitHub builds a temporary branch of the target branch plus the queued PRs ahead of it, - re-runs the required checks on it (path gating and runners as in a PR run, but with the GPU integration tests, - which a PR run leaves out unless it carries the `run-integration-tests` label) and merges the PR only if they - pass. Each PR still lands on its own. A PR that fails in the queue is removed from it and the PRs behind it - are re-tested without it; fix it and queue it again. + re-runs the required checks on it (path gating and runners as in a PR run, but with the integration tests, + which a PR run leaves out unless it carries the `run-integration-tests` label, and without the nightly-only + tests) and merges the PR only if they pass. Each PR still lands on its own. A PR that fails in the queue is + removed from it and the PRs behind it are re-tested without it; fix it and queue it again. - If your PR is a bug fix that should also land in an ongoing release branch, add the `cherry-pick-` label (e.g. `cherry-pick-0.47.0`) matching that release. Labeled PRs are cherry-picked into the release branch so the fix is included in the next release candidate. diff --git a/pyproject.toml b/pyproject.toml index 39c605ad112..8843bcfc23e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -368,7 +368,7 @@ pythonpath = ["tests/"] # Apply per-test timeouts (see tests/conftest.py) to the test call only, not fixture setup/teardown timeout_func_only = true markers = [ - "integration: Tests that require external services or other non-hermetic dependencies, several processes or GPUs, subprocesses, inference engines or heavy compilation; run by the merge queue and nightly, not on every pull request push", + "integration: Tests that require external services or other non-hermetic dependencies", "manual: Only run when --run-manual is given", "release: Regression tests that should be run before every release", ] diff --git a/tests/conftest.py b/tests/conftest.py index f846d5b2f49..af77546a244 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -62,19 +62,8 @@ def pytest_addoption(parser): } -# GPU integration tests need several processes or GPUs, spawn subprocesses, run an inference engine -# or compile heavy kernels (see CONTRIBUTING.md). Every test of these groups is one, and in `gpu` so -# is a test that requests one of these fixtures; other `gpu` tests opt in with -# `pytest.mark.integration`. Pull requests run the rest with `-m "not integration"`; the merge queue -# and the nightly run everything. -_INTEGRATION_GROUPS = {"examples", "gpu_megatron", "gpu_trtllm", "gpu_vllm", "regression"} -_INTEGRATION_FIXTURES = ("dist_workers", "need_2_gpus", "need_4_gpus", "need_8_gpus") - - -# tryfirst: the `integration` marks must exist before pytest applies `-m` -@pytest.hookimpl(tryfirst=True) def pytest_collection_modifyitems(config, items): - """Mark integration tests, skip flag-gated tests and set per-test timeouts by directory.""" + """Skip flag-gated tests and apply a default per-test timeout based on the test directory.""" skip_marks = [ ("manual", "--run-manual"), ("release", "--run-release"), @@ -89,19 +78,14 @@ def pytest_collection_modifyitems(config, items): tests_root = Path(__file__).parent for item in items: - if not item.path.is_relative_to(tests_root): - continue - # First path component under tests/ is the group dir (unit, gpu, examples, ...). - group = item.path.relative_to(tests_root).parts[0] - if group in _INTEGRATION_GROUPS or ( - group == "gpu" - and any(name.startswith(_INTEGRATION_FIXTURES) for name in item.fixturenames) + if item.get_closest_marker("timeout") is not None or not item.path.is_relative_to( + tests_root ): - item.add_marker(pytest.mark.integration) - if item.get_closest_marker("timeout") is not None: continue + # First path component under tests/ is the group dir (unit, gpu, examples, ...). # Crash loudly (rather than silently skip) if a group has no configured default, so a # newly added tests// must be given an explicit timeout in the mapping above. + group = item.path.relative_to(tests_root).parts[0] if group not in _DEFAULT_TIMEOUT: raise pytest.UsageError( f"tests/{group}/ has no default timeout; add '{group}' to " diff --git a/tests/gpu/_extensions/test_torch_extensions.py b/tests/gpu/_extensions/test_torch_extensions.py index 3725fc5d0f0..8c10a81ad4f 100644 --- a/tests/gpu/_extensions/test_torch_extensions.py +++ b/tests/gpu/_extensions/test_torch_extensions.py @@ -22,7 +22,7 @@ import modelopt.torch.quantization.extensions as ext # Override default timeout as these tests JIT-compile the CUDA extensions, which is slow -pytestmark = [pytest.mark.integration, pytest.mark.timeout(240)] +pytestmark = pytest.mark.timeout(240) # Compile extensions first so it does not count towards time used to run a test that needs it diff --git a/tests/gpu/_extensions/test_torch_kernels.py b/tests/gpu/_extensions/test_torch_kernels.py index 1bbee5e9d62..22c6c584a19 100644 --- a/tests/gpu/_extensions/test_torch_kernels.py +++ b/tests/gpu/_extensions/test_torch_kernels.py @@ -22,7 +22,7 @@ import pytest # Override default timeout as these tests JIT-compile the CUDA extensions, which is slow -pytestmark = [pytest.mark.integration, pytest.mark.timeout(180)] +pytestmark = pytest.mark.timeout(180) def test_conv3d_implicit_gemm(): diff --git a/tests/gpu/onnx/quantization/autotune/test_benchmark.py b/tests/gpu/onnx/quantization/autotune/test_benchmark.py index 06cce63572b..012bf495ab6 100644 --- a/tests/gpu/onnx/quantization/autotune/test_benchmark.py +++ b/tests/gpu/onnx/quantization/autotune/test_benchmark.py @@ -342,7 +342,6 @@ def test_run_timing_passes_stream_handle_to_trt(): # --- TensorRTPyBenchmark (integration) --- -@pytest.mark.integration class TestTensorRTPyBenchmark: """End-to-end tests for TensorRTPyBenchmark. Requires TensorRT and CUDA.""" @@ -419,7 +418,6 @@ def test_callable(self, simple_conv_model_bytes): # --- TrtExecBenchmark (integration) --- -@pytest.mark.integration class TestTrtExecBenchmark: """End-to-end tests for TrtExecBenchmark. Requires trtexec in PATH.""" diff --git a/tests/gpu/onnx/quantization/autotune/test_workflow.py b/tests/gpu/onnx/quantization/autotune/test_workflow.py index 5b610976582..eebc42f6c33 100644 --- a/tests/gpu/onnx/quantization/autotune/test_workflow.py +++ b/tests/gpu/onnx/quantization/autotune/test_workflow.py @@ -28,8 +28,6 @@ region_pattern_autotuning_workflow, ) -pytestmark = pytest.mark.integration - @pytest.fixture def simple_conv_model(): diff --git a/tests/gpu/onnx/quantization/test_autotune_quantization_integration.py b/tests/gpu/onnx/quantization/test_autotune_quantization_integration.py index 8d266ed2068..829eebb553b 100644 --- a/tests/gpu/onnx/quantization/test_autotune_quantization_integration.py +++ b/tests/gpu/onnx/quantization/test_autotune_quantization_integration.py @@ -18,7 +18,6 @@ import onnx import onnx_graphsurgeon as gs -import pytest from _test_utils.import_helper import skip_if_no_tensorrt from _test_utils.onnx.lib_test_models import export_as_onnx from _test_utils.onnx.quantization.autotune.models import _create_simple_resnet18_model @@ -31,8 +30,6 @@ skip_if_no_tensorrt() -pytestmark = pytest.mark.integration - def _quantized_tensor_indices(model: onnx.ModelProto) -> set[tuple[str, int]]: """Return (node_name, input_index) for every DQ-fed input slot in the model.""" diff --git a/tests/gpu/torch/deploy/_runtime/test_trt_client.py b/tests/gpu/torch/deploy/_runtime/test_trt_client.py index 4b5832b18d8..72709336a6b 100644 --- a/tests/gpu/torch/deploy/_runtime/test_trt_client.py +++ b/tests/gpu/torch/deploy/_runtime/test_trt_client.py @@ -31,8 +31,6 @@ from modelopt.torch._deploy._runtime.trt_client import TRTLocalClient from modelopt.torch._deploy.compilation import compile -pytestmark = pytest.mark.integration - deployment = { "runtime": "TRT", "accelerator": "GPU", diff --git a/tests/gpu/torch/export/test_qwen_vision_recipe_export.py b/tests/gpu/torch/export/test_qwen_vision_recipe_export.py index d015611f879..389375428b1 100644 --- a/tests/gpu/torch/export/test_qwen_vision_recipe_export.py +++ b/tests/gpu/torch/export/test_qwen_vision_recipe_export.py @@ -25,8 +25,6 @@ from modelopt.recipe import load_recipe from modelopt.torch.export import export_hf_checkpoint -pytestmark = pytest.mark.integration - _VISION_RECIPE = "fp8_vision-kv_none" _JOINT_RECIPE = "fp8_vision_lm-kv_fp8_cast" diff --git a/tests/gpu/torch/export/test_unified_hf_export_and_check_safetensors.py b/tests/gpu/torch/export/test_unified_hf_export_and_check_safetensors.py index 247e64f905a..b45943e6e5e 100644 --- a/tests/gpu/torch/export/test_unified_hf_export_and_check_safetensors.py +++ b/tests/gpu/torch/export/test_unified_hf_export_and_check_safetensors.py @@ -25,8 +25,6 @@ ) from safetensors import safe_open -pytestmark = pytest.mark.integration - # Here we map each qformat -> the suffix we expect in the generated safetensors directory @pytest.mark.parametrize( diff --git a/tests/gpu/torch/kernels/quantization/linear_attention/test_fla_chunk_gated_delta_rule.py b/tests/gpu/torch/kernels/quantization/linear_attention/test_fla_chunk_gated_delta_rule.py index be13df0146f..defe05aa5d5 100644 --- a/tests/gpu/torch/kernels/quantization/linear_attention/test_fla_chunk_gated_delta_rule.py +++ b/tests/gpu/torch/kernels/quantization/linear_attention/test_fla_chunk_gated_delta_rule.py @@ -29,8 +29,6 @@ chunk_gated_delta_rule, ) -pytestmark = pytest.mark.integration - def make_inputs(): torch.manual_seed(123) diff --git a/tests/gpu/torch/kernels/sparsity/attention/test_triton_fa_calibrate.py b/tests/gpu/torch/kernels/sparsity/attention/test_triton_fa_calibrate.py index c78dcb1ba8c..598f97a3fda 100644 --- a/tests/gpu/torch/kernels/sparsity/attention/test_triton_fa_calibrate.py +++ b/tests/gpu/torch/kernels/sparsity/attention/test_triton_fa_calibrate.py @@ -272,7 +272,6 @@ def test_measure_sparsity_returns_counts(self): assert out._sparsity_total > 0 assert out._sparsity_skipped <= out._sparsity_total - @pytest.mark.integration def test_first_measured_call_has_real_tile_count_with_autotune(self): """Counters from the first measured call should not include autotune trials.""" script = r""" diff --git a/tests/gpu/torch/models/test_modeling_ptq_registration.py b/tests/gpu/torch/models/test_modeling_ptq_registration.py index 0ba27611e6c..6782b64c344 100644 --- a/tests/gpu/torch/models/test_modeling_ptq_registration.py +++ b/tests/gpu/torch/models/test_modeling_ptq_registration.py @@ -30,8 +30,6 @@ pytest.importorskip("transformers") -pytestmark = pytest.mark.integration - FIRST_IMPORTS = [ "modelopt.torch.models.falcon.modeling_ptq", "modelopt.torch.models.gpt_oss.modeling_ptq", diff --git a/tests/gpu/torch/puzzletron/test_nas_convert.py b/tests/gpu/torch/puzzletron/test_nas_convert.py index 53cf23c5f90..57239f1c0e4 100644 --- a/tests/gpu/torch/puzzletron/test_nas_convert.py +++ b/tests/gpu/torch/puzzletron/test_nas_convert.py @@ -18,7 +18,6 @@ from functools import partial from pathlib import Path -import pytest import torch from _test_utils.torch.distributed.utils import spawn_multiprocess_job from _test_utils.torch.puzzletron.utils import setup_test_model_and_data @@ -27,8 +26,6 @@ import modelopt.torch.puzzletron as mtpz import modelopt.torch.utils.distributed as dist -pytestmark = pytest.mark.integration - def test_nas_convert_ffn_pruning(project_root_path: Path, tmp_path: Path): spawn_multiprocess_job( diff --git a/tests/gpu/torch/puzzletron/test_nas_search.py b/tests/gpu/torch/puzzletron/test_nas_search.py index 768d10a027b..ae0dff47be5 100644 --- a/tests/gpu/torch/puzzletron/test_nas_search.py +++ b/tests/gpu/torch/puzzletron/test_nas_search.py @@ -17,7 +17,6 @@ from functools import partial from pathlib import Path -import pytest import torch from _test_utils.torch.distributed.utils import spawn_multiprocess_job from _test_utils.torch.puzzletron.utils import setup_test_model_and_data @@ -26,8 +25,6 @@ import modelopt.torch.puzzletron as mtpz import modelopt.torch.utils.distributed as dist -pytestmark = pytest.mark.integration - def test_nas_search(project_root_path: Path, tmp_path: Path): spawn_multiprocess_job( diff --git a/tests/gpu/torch/puzzletron/test_puzzletron.py b/tests/gpu/torch/puzzletron/test_puzzletron.py index 24dfea9902e..4953db2c737 100644 --- a/tests/gpu/torch/puzzletron/test_puzzletron.py +++ b/tests/gpu/torch/puzzletron/test_puzzletron.py @@ -25,8 +25,6 @@ import modelopt.torch.puzzletron as mtpz import modelopt.torch.utils.distributed as dist -pytestmark = pytest.mark.integration - # The e2e test to compress a model based on Local Neural Architecture Search (Mixed Integer Programing NAS search) # using a one-click command. # diff --git a/tests/gpu/torch/puzzletron/tools/test_save_ckpt_from_shards.py b/tests/gpu/torch/puzzletron/tools/test_save_ckpt_from_shards.py index 935663ac887..a31c687cc1e 100644 --- a/tests/gpu/torch/puzzletron/tools/test_save_ckpt_from_shards.py +++ b/tests/gpu/torch/puzzletron/tools/test_save_ckpt_from_shards.py @@ -33,8 +33,6 @@ save_checkpoint_from_shards, ) -pytestmark = pytest.mark.integration - class TestSaveCheckpointFromShardsSingleProcess: """Tests that run without torch.distributed (world_size=1 path).""" diff --git a/tests/gpu/torch/utils/test_distributed_worker_pool.py b/tests/gpu/torch/utils/test_distributed_worker_pool.py index 276e6cbfa82..7ae3a966278 100644 --- a/tests/gpu/torch/utils/test_distributed_worker_pool.py +++ b/tests/gpu/torch/utils/test_distributed_worker_pool.py @@ -23,8 +23,6 @@ import torch.distributed as dist from _test_utils.torch.distributed.utils import DistributedWorkerPool, get_live_pool -pytestmark = pytest.mark.integration - def _hang(rank, size): time.sleep(600) diff --git a/tests/gpu/torch/utils/test_hf_checkpoint_utils_dist.py b/tests/gpu/torch/utils/test_hf_checkpoint_utils_dist.py index ecf698855f2..72322ad50b4 100644 --- a/tests/gpu/torch/utils/test_hf_checkpoint_utils_dist.py +++ b/tests/gpu/torch/utils/test_hf_checkpoint_utils_dist.py @@ -26,8 +26,6 @@ pytest.importorskip("huggingface_hub") -pytestmark = pytest.mark.integration - class _FakeHfApi: """``HfApi`` whose ``repo_info`` succeeds, so the Hub counts as reachable.""" From e6f9bb18f470f3796c1f1b4234aebc45c513cade Mon Sep 17 00:00:00 2001 From: Shengliang Xu Date: Sun, 18 Oct 2026 06:05:34 +0000 Subject: [PATCH 09/12] Run the GPU tests in the merge queue instead of on every push A full tests/gpu run takes 39-49 min, so a pull request push no longer runs any GPU test: the GPU, example and regression workflows run in the merge queue, on the nightly and on demand, and on a PR that carries the `run-gpu-tests` label. One gate output, `run_gpu`, switches all three, so the lane split, the pytest marker and the reusable runner workflow of the earlier revisions are gone and the GPU job is the matrix job of main again. The gate does not wait for the unit tests when nothing will run, and the shared `gpu` Codecov flag is either complete or absent, which carryforward already handles. Signed-off-by: Shengliang Xu --- .github/actions/pr-merge-base/action.yml | 17 ++--- .github/workflows/_gpu_tests_runner.yml | 68 ----------------- .github/workflows/_pr_gate.yml | 13 ++-- .github/workflows/example_tests.yml | 14 ++-- .github/workflows/gpu_tests.yml | 93 ++++++++++++++---------- .github/workflows/regression_tests.yml | 14 ++-- CONTRIBUTING.md | 16 ++-- 7 files changed, 91 insertions(+), 144 deletions(-) delete mode 100644 .github/workflows/_gpu_tests_runner.yml diff --git a/.github/actions/pr-merge-base/action.yml b/.github/actions/pr-merge-base/action.yml index 4cc5457b679..da526a00fdd 100644 --- a/.github/actions/pr-merge-base/action.yml +++ b/.github/actions/pr-merge-base/action.yml @@ -3,9 +3,8 @@ description: > Resolve the commit to diff a PR run against. For a copied PR branch it is the merge base of the PR's head and its target branch; for a merge queue group it is the commit the group was built on. Outputs are empty for non-PR triggers, which have no diff to inspect. Also says whether the - integration tests (the Megatron, TensorRT-LLM and vLLM lanes, the example and regression tests) - run: always in a merge queue group, and on a copied PR branch only when the PR carries the - `run-integration-tests` label (the other PR runs skip them). + GPU tests (the GPU, example and regression workflows) run: always in a merge queue group, and on + a copied PR branch only when the PR carries the `run-gpu-tests` label (the other PR runs skip them). Requires a prior actions/checkout with fetch-depth 0. outputs: @@ -15,10 +14,10 @@ outputs: head_sha: description: The PR's head commit, or the merge queue group's head commit. value: ${{ steps.pr-shas.outputs.head_sha || steps.merge-group.outputs.head_sha }} - run_integration: + run_gpu: description: >- - "true" if the GPU integration tests run in this PR run, else "false". Empty for non-PR triggers. - value: ${{ steps.pr-shas.outputs.run_integration || steps.merge-group.outputs.run_integration }} + "true" if the GPU tests run in this PR run, else "false". Empty for non-PR triggers. + value: ${{ steps.pr-shas.outputs.run_gpu || steps.merge-group.outputs.run_gpu }} runs: using: composite @@ -35,8 +34,8 @@ runs: run: | echo "head_sha=$(echo "$PR_INFO" | jq -r '.head.sha')" >> $GITHUB_OUTPUT echo "base_sha=$(echo "$PR_INFO" | jq -r '.base.sha')" >> $GITHUB_OUTPUT - echo "run_integration=$(echo "$PR_INFO" | - jq -r '[(.labels // [])[].name] | index("run-integration-tests") != null')" >> $GITHUB_OUTPUT + echo "run_gpu=$(echo "$PR_INFO" | + jq -r '[(.labels // [])[].name] | index("run-gpu-tests") != null')" >> $GITHUB_OUTPUT # Get commit from the target branch that is present in the PR to use as base for changed files - if: startsWith(github.ref, 'refs/heads/pull-request/') id: calculate-merge-base @@ -57,4 +56,4 @@ runs: run: | echo "base_sha=$BASE_SHA" | tee --append "${GITHUB_OUTPUT}" echo "head_sha=$HEAD_SHA" | tee --append "${GITHUB_OUTPUT}" - echo "run_integration=true" | tee --append "${GITHUB_OUTPUT}" + echo "run_gpu=true" | tee --append "${GITHUB_OUTPUT}" diff --git a/.github/workflows/_gpu_tests_runner.yml b/.github/workflows/_gpu_tests_runner.yml deleted file mode 100644 index 4175bb3e6c7..00000000000 --- a/.github/workflows/_gpu_tests_runner.yml +++ /dev/null @@ -1,68 +0,0 @@ -# Reusable workflow for running one lane of the GPU tests -name: GPU Tests Runner - -on: - workflow_call: - inputs: - example: - description: "nox session of the lane (gpu, gpu_megatron, gpu_trtllm or gpu_vllm)" - required: true - type: string - container_image: - description: "Docker image to use for the lane" - required: true - type: string - timeout_minutes: - description: "Timeout in minutes for the job" - required: false - type: number - default: 60 - runner: - description: "GitHub runner to use" - required: true - type: string - -jobs: - run-test: - runs-on: ${{ inputs.runner }} - timeout-minutes: ${{ inputs.timeout_minutes }} - permissions: - contents: read - container: - image: ${{ inputs.container_image }} - env: - GIT_DEPTH: 1000 # For correct version - PIP_CONSTRAINT: "" # Disable pip constraint for upgrading packages - HF_TOKEN: ${{ secrets.HF_TOKEN }} - # Build CUDA kernels only for the runner's RTX PRO 6000 (sm_120), not the image's ~6 archs. - TORCH_CUDA_ARCH_LIST: "12.0" - steps: - - name: Install git - # The vllm container ships without git; needed for a real checkout (correct - # setuptools-scm version) and for the Codecov upload below. - if: inputs.example == 'gpu_vllm' - run: apt-get update && apt-get install -y git - - uses: actions/checkout@v6 - - uses: nv-gha-runners/setup-proxy-cache@main - - uses: ./.github/actions/cache-extensions - with: - cache-key: rtxpro6000-${{ inputs.container_image }} - - name: Setup environment variables - run: | - echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/include:/usr/lib/x86_64-linux-gnu" >> $GITHUB_ENV - - name: Run gpu tests - env: - COVERAGE_PROCESS_START: ${{ github.workspace }}/pyproject.toml - COVERAGE_FILE: ${{ github.workspace }}/.coverage - run: | - # Use `python3` (the vllm image has no `python` on PATH) - python3 -m pip install nox && nox -s ${{ inputs.example }} - - name: Upload GPU coverage to Codecov - if: github.event_name != 'merge_group' - uses: codecov/codecov-action@v7 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: coverage.xml - flags: gpu - fail_ci_if_error: false # test may be skipped if relevant file changes are not detected - verbose: true diff --git a/.github/workflows/_pr_gate.yml b/.github/workflows/_pr_gate.yml index 2e5380b05e9..f3dc047153d 100644 --- a/.github/workflows/_pr_gate.yml +++ b/.github/workflows/_pr_gate.yml @@ -20,11 +20,10 @@ on: any_changed: description: "Whether any relevant files changed" value: ${{ jobs.check-file-changes.outputs.any_changed }} - run_integration: + run_gpu: description: >- - Whether the GPU integration tests run: always except on a PR run without the - run-integration-tests label - value: ${{ jobs.check-file-changes.outputs.run_integration }} + Whether the GPU tests run: always except on a PR run without the run-gpu-tests label + value: ${{ jobs.check-file-changes.outputs.run_gpu }} jobs: check-file-changes: @@ -32,7 +31,7 @@ jobs: outputs: any_changed: ${{ steps.changed-tests.outputs.any_modified || steps.non-pr.outputs.any_changed }} # Empty outside PR runs (nightly, on demand), which run everything - run_integration: ${{ steps.base.outputs.run_integration || 'true' }} + run_gpu: ${{ steps.base.outputs.run_gpu || 'true' }} steps: # For non-PR triggers (schedule, workflow_dispatch), always run tests. A PR run is a push to a # copy-pr-bot branch (pull-request/N) or a merge queue group, which both diff against the base. @@ -58,9 +57,11 @@ jobs: fail_on_initial_diff_error: true wait-checks: needs: [check-file-changes] + # Nothing to wait for when the GPU tests are left to the merge queue if: >- (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && - needs.check-file-changes.outputs.any_changed == 'true' + needs.check-file-changes.outputs.any_changed == 'true' && + needs.check-file-changes.outputs.run_gpu == 'true' runs-on: ubuntu-latest permissions: checks: read diff --git a/.github/workflows/example_tests.yml b/.github/workflows/example_tests.yml index 4fd52b7503d..4e2401f9321 100644 --- a/.github/workflows/example_tests.yml +++ b/.github/workflows/example_tests.yml @@ -31,7 +31,7 @@ jobs: contents: read outputs: any_changed: ${{ steps.lanes.outputs.any_changed }} - run_integration: ${{ steps.base.outputs.run_integration || 'true' }} + run_gpu: ${{ steps.base.outputs.run_gpu || 'true' }} torch: ${{ steps.lanes.outputs.torch }} vllm: ${{ steps.lanes.outputs.vllm }} trtllm: ${{ steps.lanes.outputs.trtllm }} @@ -94,8 +94,8 @@ jobs: # Nightly and on-demand runs have no diff to inspect, so they run everything. NON_PR: >- ${{ !(startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') }} - # Example tests are integration tests: a PR run without the run-integration-tests label skips them - RUN_INTEGRATION: ${{ steps.base.outputs.run_integration || 'true' }} + # The example tests run on GPUs: a PR run without the run-gpu-tests label skips them + RUN_GPU: ${{ steps.base.outputs.run_gpu || 'true' }} COMMON: ${{ steps.changed.outputs.common_any_modified }} TORCH: ${{ steps.changed.outputs.torch_any_modified }} VLLM: ${{ steps.changed.outputs.vllm_any_modified }} @@ -106,7 +106,7 @@ jobs: run: | any=false for lane in TORCH VLLM TRTLLM MEGATRON DIFFUSERS ONNX; do - if [[ "$RUN_INTEGRATION" == "true" && + if [[ "$RUN_GPU" == "true" && ("$NON_PR" == "true" || "$COMMON" == "true" || "${!lane}" == "true") ]]; then run=true any=true @@ -267,10 +267,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Example tests are left to the merge queue - if: ${{ needs.pr-gate.outputs.run_integration != 'true' }} + if: ${{ needs.pr-gate.outputs.run_gpu != 'true' }} run: >- - echo "::notice::The example tests are integration tests and run in the merge queue. Add the - run-integration-tests label and re-run this workflow to run them on this PR." + echo "::notice::The example tests run in the merge queue. Add the run-gpu-tests label and + re-run this workflow to run them on this PR." - name: Required example tests did not succeed if: | needs.pr-gate.result != 'success' || diff --git a/.github/workflows/gpu_tests.yml b/.github/workflows/gpu_tests.yml index 6e94f68fa04..9b2a4462383 100644 --- a/.github/workflows/gpu_tests.yml +++ b/.github/workflows/gpu_tests.yml @@ -27,7 +27,6 @@ jobs: with: files: | .github/actions/** - .github/workflows/_gpu_tests_runner.yml .github/workflows/_pr_gate.yml .github/workflows/gpu_tests.yml modelopt/** @@ -41,32 +40,18 @@ jobs: tests/gpu_trtllm/** tests/gpu_vllm/** - # The `gpu` lane (tests/gpu) runs on every change. The lanes that integrate with Megatron-Core, - # TensorRT-LLM and vLLM run in the merge queue, on nightly and on-demand runs, and on a PR that - # carries the `run-integration-tests` label: the gate says which (`run_integration`). + # The GPU tests run in the merge queue, on nightly and on-demand runs, and on a PR that carries the + # `run-gpu-tests` label: the gate says which (`run_gpu`). gpu-tests: needs: [pr-gate] - if: needs.pr-gate.outputs.any_changed == 'true' - uses: ./.github/workflows/_gpu_tests_runner.yml - permissions: - contents: read - secrets: inherit - with: - example: gpu - timeout_minutes: 60 - container_image: nvcr.io/nvidia/pytorch:26.07-py3 - runner: >- - ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && - 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} - - # Lanes that integrate with other frameworks - gpu-integration-tests: - needs: [pr-gate] - if: needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_integration == 'true' + if: needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_gpu == 'true' strategy: fail-fast: false matrix: include: + - example: gpu + timeout: 60 + container_image: nvcr.io/nvidia/pytorch:26.07-py3 - example: gpu_megatron timeout: 60 container_image: nvcr.io/nvidia/nemo:26.08 @@ -81,36 +66,66 @@ jobs: - example: gpu_vllm timeout: 15 container_image: docker.io/vllm/vllm-openai:v0.20.0 - uses: ./.github/workflows/_gpu_tests_runner.yml - permissions: - contents: read - secrets: inherit - with: - example: ${{ matrix.example }} - timeout_minutes: ${{ matrix.timeout }} - container_image: ${{ matrix.container_image }} - runner: >- - ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && - 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + runs-on: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && + 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + timeout-minutes: ${{ matrix.timeout }} + container: + image: ${{ matrix.container_image }} + env: + GIT_DEPTH: 1000 # For correct version + PIP_CONSTRAINT: "" # Disable pip constraint for upgrading packages + HF_TOKEN: ${{ secrets.HF_TOKEN }} + # Build CUDA kernels only for the runner's RTX PRO 6000 (sm_120), not the image's ~6 archs. + TORCH_CUDA_ARCH_LIST: "12.0" + steps: + - name: Install git + # The vllm container ships without git; needed for a real checkout (correct + # setuptools-scm version) and for the Codecov upload below. + if: matrix.example == 'gpu_vllm' + run: apt-get update && apt-get install -y git + - uses: actions/checkout@v6 + - uses: nv-gha-runners/setup-proxy-cache@main + - uses: ./.github/actions/cache-extensions + with: + cache-key: rtxpro6000-${{ matrix.container_image }} + - name: Setup environment variables + run: | + echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/include:/usr/lib/x86_64-linux-gnu" >> $GITHUB_ENV + - name: Run gpu tests + env: + COVERAGE_PROCESS_START: ${{ github.workspace }}/pyproject.toml + COVERAGE_FILE: ${{ github.workspace }}/.coverage + run: | + # Use `python3` (the vllm image has no `python` on PATH) + python3 -m pip install nox && nox -s ${{ matrix.example }} + - name: Upload GPU coverage to Codecov + if: github.event_name != 'merge_group' + uses: codecov/codecov-action@v7 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: coverage.xml + flags: gpu + fail_ci_if_error: false # test may be skipped if relevant file changes are not detected + verbose: true gpu-pr-required-check: # Run even if gpu-tests is skipped if: >- ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && always() }} - needs: [pr-gate, gpu-tests, gpu-integration-tests] + needs: [pr-gate, gpu-tests] runs-on: ubuntu-latest steps: - - name: GPU integration tests are left to the merge queue + - name: GPU tests are left to the merge queue if: >- ${{ needs.pr-gate.outputs.any_changed == 'true' && - needs.pr-gate.outputs.run_integration != 'true' }} + needs.pr-gate.outputs.run_gpu != 'true' }} run: >- - echo "::notice::The Megatron, TensorRT-LLM and vLLM lanes run in the merge queue; add the - run-integration-tests label and re-run this workflow to run them on this PR." + echo "::notice::The GPU tests run in the merge queue. Add the run-gpu-tests label and + re-run this workflow to run them on this PR." - name: Required GPU tests did not succeed if: >- ${{ needs.pr-gate.result != 'success' || (needs.pr-gate.outputs.any_changed == 'true' && - (needs.gpu-tests.result != 'success' || (needs.pr-gate.outputs.run_integration == 'true' && - needs.gpu-integration-tests.result != 'success'))) }} + needs.pr-gate.outputs.run_gpu == 'true' && needs.gpu-tests.result != 'success') }} run: exit 1 diff --git a/.github/workflows/regression_tests.yml b/.github/workflows/regression_tests.yml index 76fe97205b0..d0a3c5d7712 100644 --- a/.github/workflows/regression_tests.yml +++ b/.github/workflows/regression_tests.yml @@ -41,11 +41,11 @@ jobs: modelopt_recipes/general/speculative_decoding/** tools/launcher/** - # Integration tests: they run in the merge queue, on nightly and on-demand runs, and on a PR that - # carries the `run-integration-tests` label. + # Runs in the merge queue, on nightly and on-demand runs, and on a PR that carries the + # `run-gpu-tests` label. regression-tests: needs: [pr-gate] - if: needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_integration == 'true' + if: needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_gpu == 'true' runs-on: >- ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} @@ -90,13 +90,13 @@ jobs: - name: Regression tests are left to the merge queue if: >- ${{ needs.pr-gate.outputs.any_changed == 'true' && - needs.pr-gate.outputs.run_integration != 'true' }} + needs.pr-gate.outputs.run_gpu != 'true' }} run: >- - echo "::notice::The regression tests are integration tests and run in the merge queue. Add the - run-integration-tests label and re-run this workflow to run them on this PR." + echo "::notice::The regression tests run in the merge queue. Add the run-gpu-tests label and + re-run this workflow to run them on this PR." - name: Required regression tests did not succeed if: | needs.pr-gate.result != 'success' || - (needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_integration == 'true' && + (needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_gpu == 'true' && needs.regression-tests.result != 'success') run: exit 1 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 28933729325..b3b079266d3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -151,12 +151,12 @@ Test filenames must mirror the production module they primarily cover: tests for - `tests/unit`: Fast cpu-based unit tests for the core ModelOpt library. They should not take more than a few seconds to run. - `tests/gpu`: Fast GPU-based unit tests for the core ModelOpt library. In most cases, they should not take more than a few seconds to run. -- `tests/gpu_megatron`: GPU-based integration tests for the core ModelOpt library for Megatron-Core features. -- `tests/gpu_trtllm`: GPU-based integration tests for the core ModelOpt library for TensorRT-LLM features. -- `tests/gpu_vllm`: GPU-based integration tests for the core ModelOpt library for vLLM features. +- `tests/gpu_megatron`: Fast GPU-based unit tests for the core ModelOpt library for Megatron-Core features. In most cases, they should not take more than a few seconds to run. +- `tests/gpu_trtllm`: Fast GPU-based unit tests for the core ModelOpt library for TensorRT-LLM features. In most cases, they should not take more than a few seconds to run. +- `tests/gpu_vllm`: Fast GPU-based unit tests for the core ModelOpt library for vLLM features. In most cases, they should not take more than a few seconds to run. - `tests/examples`: Integration tests for ModelOpt examples. They should not take more than a few minutes to run. Please refer to [example test README](./tests/examples/README.md) for more details. -CI runs the integration tests (`tests/gpu_megatron`, `tests/gpu_trtllm`, `tests/gpu_vllm`, `tests/regression` and `tests/examples`) in the merge queue and on the nightly, not on every pull request push. To run them on a pull request, add the `run-integration-tests` label and re-run the workflows. +CI runs the GPU tests (`tests/gpu`, `tests/gpu_megatron`, `tests/gpu_trtllm`, `tests/gpu_vllm`, `tests/regression` and `tests/examples`) in the merge queue and on the nightly, not on every pull request push. To run them on a pull request, add the `run-gpu-tests` label and re-run the workflows. For lightweight focused local validation, run `pytest` directly on the relevant test path. For example: @@ -287,10 +287,10 @@ nox -s "unit-3.12(torch_211, tf_latest)" - Once your PR is approved and all checks pass, one of the reviewers will merge the PR. With the merge queue enabled on its target branch (the queue runs the checks for `main`, `release/*` and `feature/*`), merging adds the PR to the queue: GitHub builds a temporary branch of the target branch plus the queued PRs ahead of it, - re-runs the required checks on it (path gating and runners as in a PR run, but with the integration tests, - which a PR run leaves out unless it carries the `run-integration-tests` label, and without the nightly-only - tests) and merges the PR only if they pass. Each PR still lands on its own. A PR that fails in the queue is - removed from it and the PRs behind it are re-tested without it; fix it and queue it again. + re-runs the required checks on it (path gating and runners as in a PR run, but with the GPU tests, which a + PR run leaves out unless it carries the `run-gpu-tests` label, and without the nightly-only tests) and + merges the PR only if they pass. Each PR still lands on its own. A PR that fails in the queue is removed + from it and the PRs behind it are re-tested without it; fix it and queue it again. - If your PR is a bug fix that should also land in an ongoing release branch, add the `cherry-pick-` label (e.g. `cherry-pick-0.47.0`) matching that release. Labeled PRs are cherry-picked into the release branch so the fix is included in the next release candidate. From da04173f21bdade147f6ea35530c4c34d73da51d Mon Sep 17 00:00:00 2001 From: Shengliang Xu Date: Sun, 18 Oct 2026 07:45:20 +0000 Subject: [PATCH 10/12] Remove the run-gpu-tests label when a round of runs finishes The label now applies to one round: the aggregate job of each of the GPU, example and regression workflows deletes it once that workflow has run its tests, pass or fail, so later pushes go back to leaving the GPU tests to the merge queue. A cancelled run (superseded by a newer push) leaves it, and so does a workflow that skipped its tests, which keeps the label from disappearing before the other workflows' gates have read it. Signed-off-by: Shengliang Xu --- .github/workflows/example_tests.yml | 14 ++++++++++++++ .github/workflows/gpu_tests.yml | 14 ++++++++++++++ .github/workflows/regression_tests.yml | 14 ++++++++++++++ CONTRIBUTING.md | 2 +- 4 files changed, 43 insertions(+), 1 deletion(-) diff --git a/.github/workflows/example_tests.yml b/.github/workflows/example_tests.yml index 4e2401f9321..f1fb0af9392 100644 --- a/.github/workflows/example_tests.yml +++ b/.github/workflows/example_tests.yml @@ -265,6 +265,8 @@ jobs: github.event_name == 'merge_group') && always() }} needs: [pr-gate, torch, vllm, trtllm, megatron, diffusers, onnx] runs-on: ubuntu-latest + permissions: + pull-requests: write steps: - name: Example tests are left to the merge queue if: ${{ needs.pr-gate.outputs.run_gpu != 'true' }} @@ -283,3 +285,15 @@ jobs: !contains(fromJSON('["success", "skipped"]'), needs.onnx.result) )) run: exit 1 + - name: Remove the run-gpu-tests label, it applies to one round of runs + if: >- + ${{ !cancelled() && startsWith(github.ref, 'refs/heads/pull-request/') && + needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_gpu == 'true' }} + env: + GH_TOKEN: ${{ github.token }} + run: | + pr="${GITHUB_REF_NAME#pull-request/}" + url="repos/${GITHUB_REPOSITORY}/issues/${pr}/labels/run-gpu-tests" + if gh api -X DELETE "$url" --silent; then + echo "::notice::Removed the run-gpu-tests label after this round; add it again for the next push." + fi diff --git a/.github/workflows/gpu_tests.yml b/.github/workflows/gpu_tests.yml index 9b2a4462383..1bbb1fd6df6 100644 --- a/.github/workflows/gpu_tests.yml +++ b/.github/workflows/gpu_tests.yml @@ -116,6 +116,8 @@ jobs: github.event_name == 'merge_group') && always() }} needs: [pr-gate, gpu-tests] runs-on: ubuntu-latest + permissions: + pull-requests: write steps: - name: GPU tests are left to the merge queue if: >- @@ -129,3 +131,15 @@ jobs: ${{ needs.pr-gate.result != 'success' || (needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_gpu == 'true' && needs.gpu-tests.result != 'success') }} run: exit 1 + - name: Remove the run-gpu-tests label, it applies to one round of runs + if: >- + ${{ !cancelled() && startsWith(github.ref, 'refs/heads/pull-request/') && + needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_gpu == 'true' }} + env: + GH_TOKEN: ${{ github.token }} + run: | + pr="${GITHUB_REF_NAME#pull-request/}" + url="repos/${GITHUB_REPOSITORY}/issues/${pr}/labels/run-gpu-tests" + if gh api -X DELETE "$url" --silent; then + echo "::notice::Removed the run-gpu-tests label after this round; add it again for the next push." + fi diff --git a/.github/workflows/regression_tests.yml b/.github/workflows/regression_tests.yml index d0a3c5d7712..1a9a89cb3f6 100644 --- a/.github/workflows/regression_tests.yml +++ b/.github/workflows/regression_tests.yml @@ -86,6 +86,8 @@ jobs: github.event_name == 'merge_group') && always() }} needs: [pr-gate, regression-tests] runs-on: ubuntu-latest + permissions: + pull-requests: write steps: - name: Regression tests are left to the merge queue if: >- @@ -100,3 +102,15 @@ jobs: (needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_gpu == 'true' && needs.regression-tests.result != 'success') run: exit 1 + - name: Remove the run-gpu-tests label, it applies to one round of runs + if: >- + ${{ !cancelled() && startsWith(github.ref, 'refs/heads/pull-request/') && + needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_gpu == 'true' }} + env: + GH_TOKEN: ${{ github.token }} + run: | + pr="${GITHUB_REF_NAME#pull-request/}" + url="repos/${GITHUB_REPOSITORY}/issues/${pr}/labels/run-gpu-tests" + if gh api -X DELETE "$url" --silent; then + echo "::notice::Removed the run-gpu-tests label after this round; add it again for the next push." + fi diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b3b079266d3..f411e7ed018 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -156,7 +156,7 @@ Test filenames must mirror the production module they primarily cover: tests for - `tests/gpu_vllm`: Fast GPU-based unit tests for the core ModelOpt library for vLLM features. In most cases, they should not take more than a few seconds to run. - `tests/examples`: Integration tests for ModelOpt examples. They should not take more than a few minutes to run. Please refer to [example test README](./tests/examples/README.md) for more details. -CI runs the GPU tests (`tests/gpu`, `tests/gpu_megatron`, `tests/gpu_trtllm`, `tests/gpu_vllm`, `tests/regression` and `tests/examples`) in the merge queue and on the nightly, not on every pull request push. To run them on a pull request, add the `run-gpu-tests` label and re-run the workflows. +CI runs the GPU tests (`tests/gpu`, `tests/gpu_megatron`, `tests/gpu_trtllm`, `tests/gpu_vllm`, `tests/regression` and `tests/examples`) in the merge queue and on the nightly, not on every pull request push. To run them on a pull request, add the `run-gpu-tests` label and re-run the workflows; CI removes the label again when that round of runs finishes. For lightweight focused local validation, run `pytest` directly on the relevant test path. For example: From 5f91c3ce033ef9d5245d7a648fba7d1275d8f250 Mon Sep 17 00:00:00 2001 From: Shengliang Xu Date: Sun, 18 Oct 2026 18:58:36 +0000 Subject: [PATCH 11/12] Rename the run-gpu-tests label to run-merge-tests The label makes a pull request push run what the merge queue runs on top of a PR run (the GPU, example and regression tests), not only GPU tests, and the name stays right if other queue-only tests are added later. The gate output follows: run_gpu becomes run_merge_tests. Signed-off-by: Shengliang Xu --- .github/actions/pr-merge-base/action.yml | 12 ++++++------ .github/workflows/_pr_gate.yml | 10 +++++----- .github/workflows/example_tests.yml | 20 ++++++++++---------- .github/workflows/gpu_tests.yml | 18 +++++++++--------- .github/workflows/regression_tests.yml | 18 +++++++++--------- CONTRIBUTING.md | 4 ++-- 6 files changed, 41 insertions(+), 41 deletions(-) diff --git a/.github/actions/pr-merge-base/action.yml b/.github/actions/pr-merge-base/action.yml index da526a00fdd..429ad06027e 100644 --- a/.github/actions/pr-merge-base/action.yml +++ b/.github/actions/pr-merge-base/action.yml @@ -4,7 +4,7 @@ description: > PR's head and its target branch; for a merge queue group it is the commit the group was built on. Outputs are empty for non-PR triggers, which have no diff to inspect. Also says whether the GPU tests (the GPU, example and regression workflows) run: always in a merge queue group, and on - a copied PR branch only when the PR carries the `run-gpu-tests` label (the other PR runs skip them). + a copied PR branch only when the PR carries the `run-merge-tests` label (the other PR runs skip them). Requires a prior actions/checkout with fetch-depth 0. outputs: @@ -14,10 +14,10 @@ outputs: head_sha: description: The PR's head commit, or the merge queue group's head commit. value: ${{ steps.pr-shas.outputs.head_sha || steps.merge-group.outputs.head_sha }} - run_gpu: + run_merge_tests: description: >- "true" if the GPU tests run in this PR run, else "false". Empty for non-PR triggers. - value: ${{ steps.pr-shas.outputs.run_gpu || steps.merge-group.outputs.run_gpu }} + value: ${{ steps.pr-shas.outputs.run_merge_tests || steps.merge-group.outputs.run_merge_tests }} runs: using: composite @@ -34,8 +34,8 @@ runs: run: | echo "head_sha=$(echo "$PR_INFO" | jq -r '.head.sha')" >> $GITHUB_OUTPUT echo "base_sha=$(echo "$PR_INFO" | jq -r '.base.sha')" >> $GITHUB_OUTPUT - echo "run_gpu=$(echo "$PR_INFO" | - jq -r '[(.labels // [])[].name] | index("run-gpu-tests") != null')" >> $GITHUB_OUTPUT + echo "run_merge_tests=$(echo "$PR_INFO" | + jq -r '[(.labels // [])[].name] | index("run-merge-tests") != null')" >> $GITHUB_OUTPUT # Get commit from the target branch that is present in the PR to use as base for changed files - if: startsWith(github.ref, 'refs/heads/pull-request/') id: calculate-merge-base @@ -56,4 +56,4 @@ runs: run: | echo "base_sha=$BASE_SHA" | tee --append "${GITHUB_OUTPUT}" echo "head_sha=$HEAD_SHA" | tee --append "${GITHUB_OUTPUT}" - echo "run_gpu=true" | tee --append "${GITHUB_OUTPUT}" + echo "run_merge_tests=true" | tee --append "${GITHUB_OUTPUT}" diff --git a/.github/workflows/_pr_gate.yml b/.github/workflows/_pr_gate.yml index f3dc047153d..d9d761bc012 100644 --- a/.github/workflows/_pr_gate.yml +++ b/.github/workflows/_pr_gate.yml @@ -20,10 +20,10 @@ on: any_changed: description: "Whether any relevant files changed" value: ${{ jobs.check-file-changes.outputs.any_changed }} - run_gpu: + run_merge_tests: description: >- - Whether the GPU tests run: always except on a PR run without the run-gpu-tests label - value: ${{ jobs.check-file-changes.outputs.run_gpu }} + Whether the GPU tests run: always except on a PR run without the run-merge-tests label + value: ${{ jobs.check-file-changes.outputs.run_merge_tests }} jobs: check-file-changes: @@ -31,7 +31,7 @@ jobs: outputs: any_changed: ${{ steps.changed-tests.outputs.any_modified || steps.non-pr.outputs.any_changed }} # Empty outside PR runs (nightly, on demand), which run everything - run_gpu: ${{ steps.base.outputs.run_gpu || 'true' }} + run_merge_tests: ${{ steps.base.outputs.run_merge_tests || 'true' }} steps: # For non-PR triggers (schedule, workflow_dispatch), always run tests. A PR run is a push to a # copy-pr-bot branch (pull-request/N) or a merge queue group, which both diff against the base. @@ -61,7 +61,7 @@ jobs: if: >- (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && needs.check-file-changes.outputs.any_changed == 'true' && - needs.check-file-changes.outputs.run_gpu == 'true' + needs.check-file-changes.outputs.run_merge_tests == 'true' runs-on: ubuntu-latest permissions: checks: read diff --git a/.github/workflows/example_tests.yml b/.github/workflows/example_tests.yml index f1fb0af9392..8d7101d07cd 100644 --- a/.github/workflows/example_tests.yml +++ b/.github/workflows/example_tests.yml @@ -31,7 +31,7 @@ jobs: contents: read outputs: any_changed: ${{ steps.lanes.outputs.any_changed }} - run_gpu: ${{ steps.base.outputs.run_gpu || 'true' }} + run_merge_tests: ${{ steps.base.outputs.run_merge_tests || 'true' }} torch: ${{ steps.lanes.outputs.torch }} vllm: ${{ steps.lanes.outputs.vllm }} trtllm: ${{ steps.lanes.outputs.trtllm }} @@ -94,8 +94,8 @@ jobs: # Nightly and on-demand runs have no diff to inspect, so they run everything. NON_PR: >- ${{ !(startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') }} - # The example tests run on GPUs: a PR run without the run-gpu-tests label skips them - RUN_GPU: ${{ steps.base.outputs.run_gpu || 'true' }} + # The example tests run on GPUs: a PR run without the run-merge-tests label skips them + RUN_MERGE_TESTS: ${{ steps.base.outputs.run_merge_tests || 'true' }} COMMON: ${{ steps.changed.outputs.common_any_modified }} TORCH: ${{ steps.changed.outputs.torch_any_modified }} VLLM: ${{ steps.changed.outputs.vllm_any_modified }} @@ -106,7 +106,7 @@ jobs: run: | any=false for lane in TORCH VLLM TRTLLM MEGATRON DIFFUSERS ONNX; do - if [[ "$RUN_GPU" == "true" && + if [[ "$RUN_MERGE_TESTS" == "true" && ("$NON_PR" == "true" || "$COMMON" == "true" || "${!lane}" == "true") ]]; then run=true any=true @@ -269,9 +269,9 @@ jobs: pull-requests: write steps: - name: Example tests are left to the merge queue - if: ${{ needs.pr-gate.outputs.run_gpu != 'true' }} + if: ${{ needs.pr-gate.outputs.run_merge_tests != 'true' }} run: >- - echo "::notice::The example tests run in the merge queue. Add the run-gpu-tests label and + echo "::notice::The example tests run in the merge queue. Add the run-merge-tests label and re-run this workflow to run them on this PR." - name: Required example tests did not succeed if: | @@ -285,15 +285,15 @@ jobs: !contains(fromJSON('["success", "skipped"]'), needs.onnx.result) )) run: exit 1 - - name: Remove the run-gpu-tests label, it applies to one round of runs + - name: Remove the run-merge-tests label, it applies to one round of runs if: >- ${{ !cancelled() && startsWith(github.ref, 'refs/heads/pull-request/') && - needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_gpu == 'true' }} + needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_merge_tests == 'true' }} env: GH_TOKEN: ${{ github.token }} run: | pr="${GITHUB_REF_NAME#pull-request/}" - url="repos/${GITHUB_REPOSITORY}/issues/${pr}/labels/run-gpu-tests" + url="repos/${GITHUB_REPOSITORY}/issues/${pr}/labels/run-merge-tests" if gh api -X DELETE "$url" --silent; then - echo "::notice::Removed the run-gpu-tests label after this round; add it again for the next push." + echo "::notice::Removed the run-merge-tests label after this round; re-add it for the next push." fi diff --git a/.github/workflows/gpu_tests.yml b/.github/workflows/gpu_tests.yml index 1bbb1fd6df6..1f2c1d8edb6 100644 --- a/.github/workflows/gpu_tests.yml +++ b/.github/workflows/gpu_tests.yml @@ -41,10 +41,10 @@ jobs: tests/gpu_vllm/** # The GPU tests run in the merge queue, on nightly and on-demand runs, and on a PR that carries the - # `run-gpu-tests` label: the gate says which (`run_gpu`). + # `run-merge-tests` label: the gate says which (`run_merge_tests`). gpu-tests: needs: [pr-gate] - if: needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_gpu == 'true' + if: needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_merge_tests == 'true' strategy: fail-fast: false matrix: @@ -122,24 +122,24 @@ jobs: - name: GPU tests are left to the merge queue if: >- ${{ needs.pr-gate.outputs.any_changed == 'true' && - needs.pr-gate.outputs.run_gpu != 'true' }} + needs.pr-gate.outputs.run_merge_tests != 'true' }} run: >- - echo "::notice::The GPU tests run in the merge queue. Add the run-gpu-tests label and + echo "::notice::The GPU tests run in the merge queue. Add the run-merge-tests label and re-run this workflow to run them on this PR." - name: Required GPU tests did not succeed if: >- ${{ needs.pr-gate.result != 'success' || (needs.pr-gate.outputs.any_changed == 'true' && - needs.pr-gate.outputs.run_gpu == 'true' && needs.gpu-tests.result != 'success') }} + needs.pr-gate.outputs.run_merge_tests == 'true' && needs.gpu-tests.result != 'success') }} run: exit 1 - - name: Remove the run-gpu-tests label, it applies to one round of runs + - name: Remove the run-merge-tests label, it applies to one round of runs if: >- ${{ !cancelled() && startsWith(github.ref, 'refs/heads/pull-request/') && - needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_gpu == 'true' }} + needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_merge_tests == 'true' }} env: GH_TOKEN: ${{ github.token }} run: | pr="${GITHUB_REF_NAME#pull-request/}" - url="repos/${GITHUB_REPOSITORY}/issues/${pr}/labels/run-gpu-tests" + url="repos/${GITHUB_REPOSITORY}/issues/${pr}/labels/run-merge-tests" if gh api -X DELETE "$url" --silent; then - echo "::notice::Removed the run-gpu-tests label after this round; add it again for the next push." + echo "::notice::Removed the run-merge-tests label after this round; re-add it for the next push." fi diff --git a/.github/workflows/regression_tests.yml b/.github/workflows/regression_tests.yml index 1a9a89cb3f6..c86fdf5d420 100644 --- a/.github/workflows/regression_tests.yml +++ b/.github/workflows/regression_tests.yml @@ -42,10 +42,10 @@ jobs: tools/launcher/** # Runs in the merge queue, on nightly and on-demand runs, and on a PR that carries the - # `run-gpu-tests` label. + # `run-merge-tests` label. regression-tests: needs: [pr-gate] - if: needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_gpu == 'true' + if: needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_merge_tests == 'true' runs-on: >- ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} @@ -92,25 +92,25 @@ jobs: - name: Regression tests are left to the merge queue if: >- ${{ needs.pr-gate.outputs.any_changed == 'true' && - needs.pr-gate.outputs.run_gpu != 'true' }} + needs.pr-gate.outputs.run_merge_tests != 'true' }} run: >- - echo "::notice::The regression tests run in the merge queue. Add the run-gpu-tests label and + echo "::notice::The regression tests run in the merge queue. Add the run-merge-tests label and re-run this workflow to run them on this PR." - name: Required regression tests did not succeed if: | needs.pr-gate.result != 'success' || - (needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_gpu == 'true' && + (needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_merge_tests == 'true' && needs.regression-tests.result != 'success') run: exit 1 - - name: Remove the run-gpu-tests label, it applies to one round of runs + - name: Remove the run-merge-tests label, it applies to one round of runs if: >- ${{ !cancelled() && startsWith(github.ref, 'refs/heads/pull-request/') && - needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_gpu == 'true' }} + needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_merge_tests == 'true' }} env: GH_TOKEN: ${{ github.token }} run: | pr="${GITHUB_REF_NAME#pull-request/}" - url="repos/${GITHUB_REPOSITORY}/issues/${pr}/labels/run-gpu-tests" + url="repos/${GITHUB_REPOSITORY}/issues/${pr}/labels/run-merge-tests" if gh api -X DELETE "$url" --silent; then - echo "::notice::Removed the run-gpu-tests label after this round; add it again for the next push." + echo "::notice::Removed the run-merge-tests label after this round; re-add it for the next push." fi diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f411e7ed018..f3fca2c7223 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -156,7 +156,7 @@ Test filenames must mirror the production module they primarily cover: tests for - `tests/gpu_vllm`: Fast GPU-based unit tests for the core ModelOpt library for vLLM features. In most cases, they should not take more than a few seconds to run. - `tests/examples`: Integration tests for ModelOpt examples. They should not take more than a few minutes to run. Please refer to [example test README](./tests/examples/README.md) for more details. -CI runs the GPU tests (`tests/gpu`, `tests/gpu_megatron`, `tests/gpu_trtllm`, `tests/gpu_vllm`, `tests/regression` and `tests/examples`) in the merge queue and on the nightly, not on every pull request push. To run them on a pull request, add the `run-gpu-tests` label and re-run the workflows; CI removes the label again when that round of runs finishes. +CI runs the GPU tests (`tests/gpu`, `tests/gpu_megatron`, `tests/gpu_trtllm`, `tests/gpu_vllm`, `tests/regression` and `tests/examples`) in the merge queue and on the nightly, not on every pull request push. To run them on a pull request, add the `run-merge-tests` label and re-run the workflows; CI removes the label again when that round of runs finishes. For lightweight focused local validation, run `pytest` directly on the relevant test path. For example: @@ -288,7 +288,7 @@ nox -s "unit-3.12(torch_211, tf_latest)" enabled on its target branch (the queue runs the checks for `main`, `release/*` and `feature/*`), merging adds the PR to the queue: GitHub builds a temporary branch of the target branch plus the queued PRs ahead of it, re-runs the required checks on it (path gating and runners as in a PR run, but with the GPU tests, which a - PR run leaves out unless it carries the `run-gpu-tests` label, and without the nightly-only tests) and + PR run leaves out unless it carries the `run-merge-tests` label, and without the nightly-only tests) and merges the PR only if they pass. Each PR still lands on its own. A PR that fails in the queue is removed from it and the PRs behind it are re-tested without it; fix it and queue it again. - If your PR is a bug fix that should also land in an ongoing release branch, add the From 3a309d3837e71a679ee71bb5fcd6f1c6790ef63a Mon Sep 17 00:00:00 2001 From: Shengliang Xu Date: Sun, 18 Oct 2026 19:17:35 +0000 Subject: [PATCH 12/12] Call what the run-merge-tests label switches the merge tests The action and gate descriptions, the comments and the notices still said "GPU tests" for the tests that the label and run_merge_tests switch, which also cover the example and regression tests. Name them merge tests, say which tests that is where the gate or the action describes it, and let each workflow's notice say that its tests are merge tests. Text only: the parsed YAML differs in those six strings and nowhere else. Signed-off-by: Shengliang Xu --- .github/actions/pr-merge-base/action.yml | 6 +++--- .github/workflows/_pr_gate.yml | 5 +++-- .github/workflows/example_tests.yml | 6 +++--- .github/workflows/gpu_tests.yml | 8 ++++---- .github/workflows/regression_tests.yml | 8 ++++---- CONTRIBUTING.md | 4 ++-- 6 files changed, 19 insertions(+), 18 deletions(-) diff --git a/.github/actions/pr-merge-base/action.yml b/.github/actions/pr-merge-base/action.yml index 429ad06027e..c592c08d832 100644 --- a/.github/actions/pr-merge-base/action.yml +++ b/.github/actions/pr-merge-base/action.yml @@ -3,8 +3,8 @@ description: > Resolve the commit to diff a PR run against. For a copied PR branch it is the merge base of the PR's head and its target branch; for a merge queue group it is the commit the group was built on. Outputs are empty for non-PR triggers, which have no diff to inspect. Also says whether the - GPU tests (the GPU, example and regression workflows) run: always in a merge queue group, and on - a copied PR branch only when the PR carries the `run-merge-tests` label (the other PR runs skip them). + merge tests (the GPU, example and regression tests, which a PR run skips) run: always in a merge + queue group, and on a copied PR branch only when the PR carries the `run-merge-tests` label. Requires a prior actions/checkout with fetch-depth 0. outputs: @@ -16,7 +16,7 @@ outputs: value: ${{ steps.pr-shas.outputs.head_sha || steps.merge-group.outputs.head_sha }} run_merge_tests: description: >- - "true" if the GPU tests run in this PR run, else "false". Empty for non-PR triggers. + "true" if the merge tests run in this PR run, else "false". Empty for non-PR triggers. value: ${{ steps.pr-shas.outputs.run_merge_tests || steps.merge-group.outputs.run_merge_tests }} runs: diff --git a/.github/workflows/_pr_gate.yml b/.github/workflows/_pr_gate.yml index d9d761bc012..5a76ba64a79 100644 --- a/.github/workflows/_pr_gate.yml +++ b/.github/workflows/_pr_gate.yml @@ -22,7 +22,8 @@ on: value: ${{ jobs.check-file-changes.outputs.any_changed }} run_merge_tests: description: >- - Whether the GPU tests run: always except on a PR run without the run-merge-tests label + Whether the merge tests (the GPU, example and regression tests) run: always except on a PR + run without the run-merge-tests label value: ${{ jobs.check-file-changes.outputs.run_merge_tests }} jobs: @@ -57,7 +58,7 @@ jobs: fail_on_initial_diff_error: true wait-checks: needs: [check-file-changes] - # Nothing to wait for when the GPU tests are left to the merge queue + # Nothing to wait for when the merge tests are skipped if: >- (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && needs.check-file-changes.outputs.any_changed == 'true' && diff --git a/.github/workflows/example_tests.yml b/.github/workflows/example_tests.yml index 8d7101d07cd..5a470d02b0e 100644 --- a/.github/workflows/example_tests.yml +++ b/.github/workflows/example_tests.yml @@ -94,7 +94,7 @@ jobs: # Nightly and on-demand runs have no diff to inspect, so they run everything. NON_PR: >- ${{ !(startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') }} - # The example tests run on GPUs: a PR run without the run-merge-tests label skips them + # The example tests are merge tests: a PR run without the run-merge-tests label skips them RUN_MERGE_TESTS: ${{ steps.base.outputs.run_merge_tests || 'true' }} COMMON: ${{ steps.changed.outputs.common_any_modified }} TORCH: ${{ steps.changed.outputs.torch_any_modified }} @@ -271,8 +271,8 @@ jobs: - name: Example tests are left to the merge queue if: ${{ needs.pr-gate.outputs.run_merge_tests != 'true' }} run: >- - echo "::notice::The example tests run in the merge queue. Add the run-merge-tests label and - re-run this workflow to run them on this PR." + echo "::notice::The example tests are merge tests and run in the merge queue. Add the + run-merge-tests label and re-run this workflow to run them on this PR." - name: Required example tests did not succeed if: | needs.pr-gate.result != 'success' || diff --git a/.github/workflows/gpu_tests.yml b/.github/workflows/gpu_tests.yml index 1f2c1d8edb6..7eef2a61501 100644 --- a/.github/workflows/gpu_tests.yml +++ b/.github/workflows/gpu_tests.yml @@ -40,8 +40,8 @@ jobs: tests/gpu_trtllm/** tests/gpu_vllm/** - # The GPU tests run in the merge queue, on nightly and on-demand runs, and on a PR that carries the - # `run-merge-tests` label: the gate says which (`run_merge_tests`). + # The GPU tests are merge tests: they run in the merge queue, on nightly and on-demand runs, and + # on a PR that carries the `run-merge-tests` label (the gate says which: `run_merge_tests`). gpu-tests: needs: [pr-gate] if: needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_merge_tests == 'true' @@ -124,8 +124,8 @@ jobs: ${{ needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_merge_tests != 'true' }} run: >- - echo "::notice::The GPU tests run in the merge queue. Add the run-merge-tests label and - re-run this workflow to run them on this PR." + echo "::notice::The GPU tests are merge tests and run in the merge queue. Add the + run-merge-tests label and re-run this workflow to run them on this PR." - name: Required GPU tests did not succeed if: >- ${{ needs.pr-gate.result != 'success' || (needs.pr-gate.outputs.any_changed == 'true' && diff --git a/.github/workflows/regression_tests.yml b/.github/workflows/regression_tests.yml index c86fdf5d420..95368f63b30 100644 --- a/.github/workflows/regression_tests.yml +++ b/.github/workflows/regression_tests.yml @@ -41,8 +41,8 @@ jobs: modelopt_recipes/general/speculative_decoding/** tools/launcher/** - # Runs in the merge queue, on nightly and on-demand runs, and on a PR that carries the - # `run-merge-tests` label. + # The regression tests are merge tests: they run in the merge queue, on nightly and on-demand runs, + # and on a PR that carries the `run-merge-tests` label. regression-tests: needs: [pr-gate] if: needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_merge_tests == 'true' @@ -94,8 +94,8 @@ jobs: ${{ needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_merge_tests != 'true' }} run: >- - echo "::notice::The regression tests run in the merge queue. Add the run-merge-tests label and - re-run this workflow to run them on this PR." + echo "::notice::The regression tests are merge tests and run in the merge queue. Add the + run-merge-tests label and re-run this workflow to run them on this PR." - name: Required regression tests did not succeed if: | needs.pr-gate.result != 'success' || diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f3fca2c7223..003b16277a4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -156,7 +156,7 @@ Test filenames must mirror the production module they primarily cover: tests for - `tests/gpu_vllm`: Fast GPU-based unit tests for the core ModelOpt library for vLLM features. In most cases, they should not take more than a few seconds to run. - `tests/examples`: Integration tests for ModelOpt examples. They should not take more than a few minutes to run. Please refer to [example test README](./tests/examples/README.md) for more details. -CI runs the GPU tests (`tests/gpu`, `tests/gpu_megatron`, `tests/gpu_trtllm`, `tests/gpu_vllm`, `tests/regression` and `tests/examples`) in the merge queue and on the nightly, not on every pull request push. To run them on a pull request, add the `run-merge-tests` label and re-run the workflows; CI removes the label again when that round of runs finishes. +CI runs the merge tests (the GPU, example and regression tests: `tests/gpu`, `tests/gpu_megatron`, `tests/gpu_trtllm`, `tests/gpu_vllm`, `tests/regression` and `tests/examples`) in the merge queue and on the nightly, not on every pull request push. To run them on a pull request, add the `run-merge-tests` label and re-run the workflows; CI removes the label again when that round of runs finishes. For lightweight focused local validation, run `pytest` directly on the relevant test path. For example: @@ -287,7 +287,7 @@ nox -s "unit-3.12(torch_211, tf_latest)" - Once your PR is approved and all checks pass, one of the reviewers will merge the PR. With the merge queue enabled on its target branch (the queue runs the checks for `main`, `release/*` and `feature/*`), merging adds the PR to the queue: GitHub builds a temporary branch of the target branch plus the queued PRs ahead of it, - re-runs the required checks on it (path gating and runners as in a PR run, but with the GPU tests, which a + re-runs the required checks on it (path gating and runners as in a PR run, but with the merge tests, which a PR run leaves out unless it carries the `run-merge-tests` label, and without the nightly-only tests) and merges the PR only if they pass. Each PR still lands on its own. A PR that fails in the queue is removed from it and the PRs behind it are re-tested without it; fix it and queue it again.