diff --git a/.github/actions/pr-merge-base/action.yml b/.github/actions/pr-merge-base/action.yml index cd4c1bc55df..d30d7436718 100644 --- a/.github/actions/pr-merge-base/action.yml +++ b/.github/actions/pr-merge-base/action.yml @@ -1,16 +1,17 @@ name: PR merge base description: > - Resolve the commit to diff a copied PR branch against: the merge base of the PR's head and its - target branch. Outputs are empty for non-PR triggers, which have no diff to inspect. + Resolve the commit to diff a PR run against. For a copied PR branch it is the merge base of the + PR's head and its target branch; for a merge queue group it is the commit the group was built + on. Outputs are empty for non-PR triggers, which have no diff to inspect. Requires a prior actions/checkout with fetch-depth 0. outputs: merge_base: description: Commit to use as the base for changed-file comparisons. - value: ${{ steps.calculate-merge-base.outputs.merge-base }} + value: ${{ steps.calculate-merge-base.outputs.merge-base || steps.merge-group.outputs.base_sha }} head_sha: - description: The PR's head commit. - value: ${{ steps.pr-shas.outputs.head_sha }} + description: The PR's head commit, or the merge queue group's head commit. + value: ${{ steps.pr-shas.outputs.head_sha || steps.merge-group.outputs.head_sha }} runs: using: composite @@ -36,3 +37,14 @@ runs: # status and emit an empty base, silently changing which lanes run. merge_base=$(git merge-base "${{ steps.pr-shas.outputs.base_sha }}" "${{ steps.pr-shas.outputs.head_sha }}") echo "merge-base=$merge_base" | tee --append "${GITHUB_OUTPUT}" + # A merge queue group is the target branch plus the queued PRs on top of it, so its diff + # against the commit it was built on is what the queue is about to merge. + - if: github.event_name == 'merge_group' + id: merge-group + shell: bash + env: + BASE_SHA: ${{ github.event.merge_group.base_sha }} + HEAD_SHA: ${{ github.event.merge_group.head_sha }} + run: | + echo "base_sha=$BASE_SHA" | tee --append "${GITHUB_OUTPUT}" + echo "head_sha=$HEAD_SHA" | tee --append "${GITHUB_OUTPUT}" diff --git a/.github/workflows/_example_tests_runner.yml b/.github/workflows/_example_tests_runner.yml index e3378865b65..e920db48ba8 100644 --- a/.github/workflows/_example_tests_runner.yml +++ b/.github/workflows/_example_tests_runner.yml @@ -107,7 +107,9 @@ jobs: # Keep legacy ONNX/TRT tests available manually during deprecation. pytest_args+=(--ignore=tests/examples/diffusers/test_diffusers_onnx_trt.py) fi - if [[ "${{ inputs.example }}" == "hf_ptq" && "${{ github.ref }}" == refs/heads/pull-request/* ]]; then + if [[ "${{ inputs.example }}" == "hf_ptq" && + ( "${{ github.ref }}" == refs/heads/pull-request/* || + "${{ github.event_name }}" == "merge_group" ) ]]; then # One GPU on PR runners: small-model tests share it and the rest run alone on it # (tests/examples/hf_ptq/conftest.py). The 2-GPU nightly stays serial. pytest_args+=(-n 4) @@ -118,6 +120,7 @@ jobs: run: | echo "::warning title=Allowed example failure::'${{ inputs.example }}' failed but is in the allow-failure list (vars.ALLOW_FAILURE_EXAMPLE_TESTS); not blocking. Remove it from the variable once fixed." - name: Upload coverage to Codecov + if: github.event_name != 'merge_group' uses: codecov/codecov-action@v7 with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/_pr_gate.yml b/.github/workflows/_pr_gate.yml index e3605202f7c..2e2de28d657 100644 --- a/.github/workflows/_pr_gate.yml +++ b/.github/workflows/_pr_gate.yml @@ -27,18 +27,19 @@ jobs: outputs: any_changed: ${{ steps.changed-tests.outputs.any_modified || steps.non-pr.outputs.any_changed }} steps: - # For non-PR triggers (schedule, workflow_dispatch), always run tests + # For non-PR triggers (schedule, workflow_dispatch), always run tests. A PR run is a push to a + # copy-pr-bot branch (pull-request/N) or a merge queue group, which both diff against the base. - id: non-pr - if: "!startsWith(github.ref, 'refs/heads/pull-request/')" + if: "!(startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group')" run: echo "any_changed=true" >> $GITHUB_OUTPUT - - if: startsWith(github.ref, 'refs/heads/pull-request/') + - if: startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group' uses: actions/checkout@v6 with: fetch-depth: 0 - - if: startsWith(github.ref, 'refs/heads/pull-request/') + - if: startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group' id: base uses: ./.github/actions/pr-merge-base - - if: startsWith(github.ref, 'refs/heads/pull-request/') + - if: startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group' name: Check for changes in test-relevant directories id: changed-tests uses: step-security/changed-files@v47.0.5 @@ -51,7 +52,7 @@ jobs: wait-checks: needs: [check-file-changes] if: >- - startsWith(github.ref, 'refs/heads/pull-request/') && + (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && needs.check-file-changes.outputs.any_changed == 'true' runs-on: ubuntu-latest permissions: diff --git a/.github/workflows/code_quality.yml b/.github/workflows/code_quality.yml index 4743fef960b..98898778e3d 100644 --- a/.github/workflows/code_quality.yml +++ b/.github/workflows/code_quality.yml @@ -3,6 +3,10 @@ name: Code Quality on: pull_request: branches: [main, release/*, feature/*] + # The merge queue re-runs the required checks on the PR merged with the target branch + merge_group: + types: [checks_requested] + branches: [main, release/*, feature/*] schedule: - cron: "0 0 * * *" # Nightly workflow_dispatch: @@ -21,12 +25,18 @@ jobs: steps: - uses: actions/checkout@v6 with: - # PRs: TruffleHog needs the base commit, else it errors, scans nothing, and passes. - fetch-depth: ${{ github.event_name == 'pull_request' && '0' || '1' }} + # PRs and merge queue groups: TruffleHog needs the base commit, else it errors, + # scans nothing, and passes. + fetch-depth: >- + ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && '0' || '1' }} - uses: ./.github/actions/ubuntu-setup - name: Secret Scanning uses: trufflesecurity/trufflehog@v3.90.5 with: + # Only set in a merge queue group, where the diff to scan is base..head (empty for the other events, + # which the action handles itself) + base: ${{ github.event.merge_group.base_sha }} + head: ${{ github.event.merge_group.head_sha }} # Exclude lob: its `(live|test)_[a-zA-Z0-9_]{35}` pattern matches any pytest function # whose name happens to be exactly 35 characters after `test_`, and it reports them as # verified. Only nightly sees them, since it scans all history while PRs scan their diff. diff --git a/.github/workflows/example_tests.yml b/.github/workflows/example_tests.yml index 8cc432a22ba..0d64e692721 100644 --- a/.github/workflows/example_tests.yml +++ b/.github/workflows/example_tests.yml @@ -4,6 +4,10 @@ on: push: branches: ["pull-request/[0-9]+"] # NOTE: paths cannot be used since push happens to copied PR and only latest commit to PR is used + # The merge queue re-runs the required checks on the PR merged with the target branch + merge_group: + types: [checks_requested] + branches: [main, release/*, feature/*] schedule: - cron: "0 0 * * *" # Nightly workflow_dispatch: @@ -37,10 +41,10 @@ jobs: - uses: actions/checkout@v6 with: fetch-depth: 0 - - if: startsWith(github.ref, 'refs/heads/pull-request/') + - if: startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group' id: base uses: ./.github/actions/pr-merge-base - - if: startsWith(github.ref, 'refs/heads/pull-request/') + - if: startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group' id: changed uses: step-security/changed-files@v47.0.5 with: @@ -87,7 +91,8 @@ jobs: - id: lanes env: # Nightly and on-demand runs have no diff to inspect, so they run everything. - NON_PR: ${{ !startsWith(github.ref, 'refs/heads/pull-request/') }} + NON_PR: >- + ${{ !(startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') }} COMMON: ${{ steps.changed.outputs.common_any_modified }} TORCH: ${{ steps.changed.outputs.torch_any_modified }} VLLM: ${{ steps.changed.outputs.vllm_any_modified }} @@ -108,7 +113,9 @@ jobs: done echo "any_changed=$any" | tee --append "$GITHUB_OUTPUT" - name: Wait for unit tests before spending GPU runners - if: ${{ steps.lanes.outputs.any_changed == 'true' && startsWith(github.ref, 'refs/heads/pull-request/') }} + if: >- + ${{ steps.lanes.outputs.any_changed == 'true' && + (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') }} uses: poseidon/wait-for-status-checks@v0.7.0 with: token: ${{ secrets.GITHUB_TOKEN }} @@ -132,7 +139,9 @@ jobs: example: ${{ matrix.example }} timeout_minutes: 30 pip_install_extras: "[hf,dev-test]" - runner: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + runner: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && + 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} allow_failure: ${{ contains(format(',{0},', vars.ALLOW_FAILURE_EXAMPLE_TESTS), format(',{0},', matrix.example)) }} ##### vLLM Example Tests ##### @@ -153,7 +162,9 @@ jobs: example: ${{ matrix.example }} timeout_minutes: 30 pip_install_extras: "[hf,dev-test]" - runner: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + runner: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && + 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} allow_failure: ${{ contains(format(',{0},', vars.ALLOW_FAILURE_EXAMPLE_TESTS), format(',{0},', matrix.example)) }} ##### TensorRT-LLM Example Tests ##### @@ -172,7 +183,9 @@ jobs: docker_image: "nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc20" example: ${{ matrix.example }} pip_install_extras: "[hf,dev-test]" - runner: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + runner: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && + 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} allow_failure: ${{ contains(format(',{0},', vars.ALLOW_FAILURE_EXAMPLE_TESTS), format(',{0},', matrix.example)) }} ##### Megatron Example Tests ##### @@ -187,9 +200,13 @@ jobs: docker_image: "nvcr.io/nvidia/nemo:26.08" example: megatron_bridge # PR runs single-GPU (~11 min); the nightly runs every test multi-GPU (~58 min). - timeout_minutes: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 30 || 75 }} + timeout_minutes: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || + github.event_name == 'merge_group') && 30 || 75 }} pip_install_extras: "[hf,puzzletron,dev-test]" - runner: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + runner: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && + 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} allow_failure: ${{ contains(format(',{0},', vars.ALLOW_FAILURE_EXAMPLE_TESTS), ',megatron_bridge,') }} ##### Diffusers Example Tests ##### @@ -206,7 +223,9 @@ jobs: example: diffusers timeout_minutes: 60 pip_install_extras: "[onnx,hf,dev-test]" - runner: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + runner: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && + 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} allow_failure: ${{ contains(format(',{0},', vars.ALLOW_FAILURE_EXAMPLE_TESTS), ',diffusers,') }} ##### ONNX/TensorRT Example Tests ##### @@ -229,13 +248,17 @@ jobs: example: ${{ matrix.example }} timeout_minutes: 45 pip_install_extras: "[onnx,hf,dev-test]" - runner: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + runner: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && + 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} allow_failure: ${{ contains(format(',{0},', vars.ALLOW_FAILURE_EXAMPLE_TESTS), format(',{0},', matrix.example)) }} ##### Required Check for PR ##### example-pr-required-check: # Run even if example tests are skipped - if: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && always() }} + if: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || + github.event_name == 'merge_group') && always() }} needs: [pr-gate, torch, vllm, trtllm, megatron, diffusers, onnx] runs-on: ubuntu-latest steps: diff --git a/.github/workflows/gpu_tests.yml b/.github/workflows/gpu_tests.yml index a0389da7b62..0d315f2af56 100644 --- a/.github/workflows/gpu_tests.yml +++ b/.github/workflows/gpu_tests.yml @@ -4,6 +4,10 @@ on: push: branches: ["pull-request/[0-9]+"] # NOTE: paths cannot be used since push happens to copied PR and only latest commit to PR is used + # The merge queue re-runs the required checks on the PR merged with the target branch + merge_group: + types: [checks_requested] + branches: [main, release/*, feature/*] schedule: - cron: "0 0 * * *" # Nightly workflow_dispatch: @@ -60,7 +64,9 @@ jobs: - example: gpu_vllm timeout: 15 container_image: docker.io/vllm/vllm-openai:v0.20.0 - runs-on: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + runs-on: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && + 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} timeout-minutes: ${{ matrix.timeout }} container: image: ${{ matrix.container_image }} @@ -92,6 +98,7 @@ jobs: # Use `python3` (the vllm image has no `python` on PATH) python3 -m pip install nox && nox -s ${{ matrix.example }} - name: Upload GPU coverage to Codecov + if: github.event_name != 'merge_group' uses: codecov/codecov-action@v7 with: token: ${{ secrets.CODECOV_TOKEN }} @@ -102,7 +109,9 @@ jobs: gpu-pr-required-check: # Run even if gpu-tests is skipped - if: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && always() }} + if: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || + github.event_name == 'merge_group') && always() }} needs: [pr-gate, gpu-tests] runs-on: ubuntu-latest steps: diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 2132cdd302e..91177043c8f 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -4,6 +4,10 @@ on: pull_request: types: [opened, synchronize, reopened, closed] branches: [main, release/*, feature/*] + # The merge queue re-runs the required checks on the PR merged with the target branch + merge_group: + types: [checks_requested] + branches: [main, release/*, feature/*] push: branches: [main] schedule: diff --git a/.github/workflows/regression_tests.yml b/.github/workflows/regression_tests.yml index 001c4e6af4a..b76f819bc17 100644 --- a/.github/workflows/regression_tests.yml +++ b/.github/workflows/regression_tests.yml @@ -4,6 +4,10 @@ on: push: branches: ["pull-request/[0-9]+"] # NOTE: paths cannot be used since push happens to copied PR and only latest commit to PR is used + # The merge queue re-runs the required checks on the PR merged with the target branch + merge_group: + types: [checks_requested] + branches: [main, release/*, feature/*] schedule: - cron: "0 0 * * *" # Nightly workflow_dispatch: @@ -40,7 +44,9 @@ jobs: regression-tests: needs: [pr-gate] if: needs.pr-gate.outputs.any_changed == 'true' - runs-on: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + runs-on: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && + 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} timeout-minutes: 15 container: image: nvcr.io/nvidia/pytorch:26.07-py3 @@ -62,6 +68,7 @@ jobs: COVERAGE_FILE: ${{ github.workspace }}/.coverage run: python -m pip install nox && nox -s regression - name: Upload regression coverage to Codecov + if: github.event_name != 'merge_group' uses: codecov/codecov-action@v7 with: token: ${{ secrets.CODECOV_TOKEN }} @@ -72,7 +79,9 @@ jobs: regression-pr-required-check: # Run even if regression-tests is skipped - if: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && always() }} + if: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || + github.event_name == 'merge_group') && always() }} needs: [pr-gate, regression-tests] runs-on: ubuntu-latest steps: diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 6fbca185d9f..ad5371c799d 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -3,6 +3,10 @@ name: Unit tests on: pull_request: branches: [main, release/*, feature/*] + # The merge queue re-runs the required checks on the PR merged with the target branch + merge_group: + types: [checks_requested] + branches: [main, release/*, feature/*] push: branches: [main, release/*, feature/*] paths: @@ -35,15 +39,18 @@ jobs: - id: non-pr if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' run: echo "any_changed=true" >> $GITHUB_OUTPUT - - if: github.event_name == 'pull_request' + - if: github.event_name == 'pull_request' || github.event_name == 'merge_group' uses: actions/checkout@v6 with: fetch-depth: 0 - - if: github.event_name == 'pull_request' + - if: github.event_name == 'pull_request' || github.event_name == 'merge_group' name: Check for changes in test-relevant paths id: changed uses: step-security/changed-files@v47.0.5 with: + # Only set in a merge queue group (empty for PRs, where the action finds the base itself) + base_sha: ${{ github.event.merge_group.base_sha }} + sha: ${{ github.event.merge_group.head_sha }} files: | .github/workflows/unit_tests.yml modelopt/** @@ -67,10 +74,14 @@ jobs: env: COVERAGE_PROCESS_START: ${{ github.workspace }}/pyproject.toml COVERAGE_FILE: ${{ github.workspace }}/.coverage - # Everything but PRs also runs the slow tests marked ``manual`` (e.g. AutoNAS). - PYTEST_EXTRA: ${{ github.event_name != 'pull_request' && '--run-manual' || '' }} + # Everything but PRs and merge queue groups also runs the slow tests marked ``manual`` + # (e.g. AutoNAS). + PYTEST_EXTRA: >- + ${{ github.event_name != 'pull_request' && + github.event_name != 'merge_group' && '--run-manual' || '' }} run: pip install nox uv && nox -s "unit-3.12(torch_214, tf_latest)" -- $PYTEST_EXTRA - name: Upload coverage reports to Codecov + if: github.event_name != 'merge_group' uses: codecov/codecov-action@v7 with: token: ${{ secrets.CODECOV_TOKEN }} @@ -220,7 +231,7 @@ jobs: python -m pytest plugins/modelopt/skills/ -o addopts="" -p no:cacheprovider -v unit-pr-required-check: # Run even if some jobs are skipped - if: ${{ github.event_name == 'pull_request' && always() }} + if: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && always() }} needs: [check-file-changes, linux, windows, multi-version, partial-install, launcher, mcp, skills] runs-on: ubuntu-latest steps: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 45b6ffbf9f2..8e1ef903ba3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -282,7 +282,12 @@ nox -s "unit-3.12(torch_211, tf_latest)" - **Contributors without write access:** When GitHub requests workflow approval, a reviewer with write permission must select **Approve and run workflows** to start GitHub Actions CI. - If any CI/CD checks fail, fix the issues and push again. -- Once your PR is approved and all checks pass, one of the reviewers will merge the PR. +- Once your PR is approved and all checks pass, one of the reviewers will merge the PR. With the merge queue + enabled on its target branch (the queue runs the checks for `main`, `release/*` and `feature/*`), merging adds + the PR to the queue: GitHub builds a temporary branch of the target branch plus the queued PRs ahead of it, + re-runs the required checks on it (the same checks, path gating and runners as a PR run, without the + nightly-only tests) and merges the PR only if they pass. Each PR still lands on its own. A PR that fails in + the queue is removed from it and the PRs behind it are re-tested without it; fix it and queue it again. - If your PR is a bug fix that should also land in an ongoing release branch, add the `cherry-pick-` label (e.g. `cherry-pick-0.47.0`) matching that release. Labeled PRs are cherry-picked into the release branch so the fix is included in the next release candidate.