diff --git a/.github/actions/pr-merge-base/action.yml b/.github/actions/pr-merge-base/action.yml index cd4c1bc55df..c592c08d832 100644 --- a/.github/actions/pr-merge-base/action.yml +++ b/.github/actions/pr-merge-base/action.yml @@ -1,16 +1,23 @@ name: PR merge base description: > - Resolve the commit to diff a copied PR branch against: the merge base of the PR's head and its - target branch. Outputs are empty for non-PR triggers, which have no diff to inspect. + Resolve the commit to diff a PR run against. For a copied PR branch it is the merge base of the + PR's head and its target branch; for a merge queue group it is the commit the group was built + on. Outputs are empty for non-PR triggers, which have no diff to inspect. Also says whether the + merge tests (the GPU, example and regression tests, which a PR run skips) run: always in a merge + queue group, and on a copied PR branch only when the PR carries the `run-merge-tests` label. Requires a prior actions/checkout with fetch-depth 0. outputs: merge_base: description: Commit to use as the base for changed-file comparisons. - value: ${{ steps.calculate-merge-base.outputs.merge-base }} + value: ${{ steps.calculate-merge-base.outputs.merge-base || steps.merge-group.outputs.base_sha }} head_sha: - description: The PR's head commit. - value: ${{ steps.pr-shas.outputs.head_sha }} + description: The PR's head commit, or the merge queue group's head commit. + value: ${{ steps.pr-shas.outputs.head_sha || steps.merge-group.outputs.head_sha }} + run_merge_tests: + description: >- + "true" if the merge tests run in this PR run, else "false". Empty for non-PR triggers. + value: ${{ steps.pr-shas.outputs.run_merge_tests || steps.merge-group.outputs.run_merge_tests }} runs: using: composite @@ -27,6 +34,8 @@ runs: run: | echo "head_sha=$(echo "$PR_INFO" | jq -r '.head.sha')" >> $GITHUB_OUTPUT echo "base_sha=$(echo "$PR_INFO" | jq -r '.base.sha')" >> $GITHUB_OUTPUT + echo "run_merge_tests=$(echo "$PR_INFO" | + jq -r '[(.labels // [])[].name] | index("run-merge-tests") != null')" >> $GITHUB_OUTPUT # Get commit from the target branch that is present in the PR to use as base for changed files - if: startsWith(github.ref, 'refs/heads/pull-request/') id: calculate-merge-base @@ -36,3 +45,15 @@ runs: # status and emit an empty base, silently changing which lanes run. merge_base=$(git merge-base "${{ steps.pr-shas.outputs.base_sha }}" "${{ steps.pr-shas.outputs.head_sha }}") echo "merge-base=$merge_base" | tee --append "${GITHUB_OUTPUT}" + # A merge queue group is the target branch plus the queued PRs on top of it, so its diff + # against the commit it was built on is what the queue is about to merge. + - if: github.event_name == 'merge_group' + id: merge-group + shell: bash + env: + BASE_SHA: ${{ github.event.merge_group.base_sha }} + HEAD_SHA: ${{ github.event.merge_group.head_sha }} + run: | + echo "base_sha=$BASE_SHA" | tee --append "${GITHUB_OUTPUT}" + echo "head_sha=$HEAD_SHA" | tee --append "${GITHUB_OUTPUT}" + echo "run_merge_tests=true" | tee --append "${GITHUB_OUTPUT}" diff --git a/.github/workflows/_example_tests_runner.yml b/.github/workflows/_example_tests_runner.yml index e3378865b65..e920db48ba8 100644 --- a/.github/workflows/_example_tests_runner.yml +++ b/.github/workflows/_example_tests_runner.yml @@ -107,7 +107,9 @@ jobs: # Keep legacy ONNX/TRT tests available manually during deprecation. pytest_args+=(--ignore=tests/examples/diffusers/test_diffusers_onnx_trt.py) fi - if [[ "${{ inputs.example }}" == "hf_ptq" && "${{ github.ref }}" == refs/heads/pull-request/* ]]; then + if [[ "${{ inputs.example }}" == "hf_ptq" && + ( "${{ github.ref }}" == refs/heads/pull-request/* || + "${{ github.event_name }}" == "merge_group" ) ]]; then # One GPU on PR runners: small-model tests share it and the rest run alone on it # (tests/examples/hf_ptq/conftest.py). The 2-GPU nightly stays serial. pytest_args+=(-n 4) @@ -118,6 +120,7 @@ jobs: run: | echo "::warning title=Allowed example failure::'${{ inputs.example }}' failed but is in the allow-failure list (vars.ALLOW_FAILURE_EXAMPLE_TESTS); not blocking. Remove it from the variable once fixed." - name: Upload coverage to Codecov + if: github.event_name != 'merge_group' uses: codecov/codecov-action@v7 with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/_pr_gate.yml b/.github/workflows/_pr_gate.yml index e3605202f7c..5a76ba64a79 100644 --- a/.github/workflows/_pr_gate.yml +++ b/.github/workflows/_pr_gate.yml @@ -20,25 +20,33 @@ on: any_changed: description: "Whether any relevant files changed" value: ${{ jobs.check-file-changes.outputs.any_changed }} + run_merge_tests: + description: >- + Whether the merge tests (the GPU, example and regression tests) run: always except on a PR + run without the run-merge-tests label + value: ${{ jobs.check-file-changes.outputs.run_merge_tests }} jobs: check-file-changes: runs-on: ubuntu-latest outputs: any_changed: ${{ steps.changed-tests.outputs.any_modified || steps.non-pr.outputs.any_changed }} + # Empty outside PR runs (nightly, on demand), which run everything + run_merge_tests: ${{ steps.base.outputs.run_merge_tests || 'true' }} steps: - # For non-PR triggers (schedule, workflow_dispatch), always run tests + # For non-PR triggers (schedule, workflow_dispatch), always run tests. A PR run is a push to a + # copy-pr-bot branch (pull-request/N) or a merge queue group, which both diff against the base. - id: non-pr - if: "!startsWith(github.ref, 'refs/heads/pull-request/')" + if: "!(startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group')" run: echo "any_changed=true" >> $GITHUB_OUTPUT - - if: startsWith(github.ref, 'refs/heads/pull-request/') + - if: startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group' uses: actions/checkout@v6 with: fetch-depth: 0 - - if: startsWith(github.ref, 'refs/heads/pull-request/') + - if: startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group' id: base uses: ./.github/actions/pr-merge-base - - if: startsWith(github.ref, 'refs/heads/pull-request/') + - if: startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group' name: Check for changes in test-relevant directories id: changed-tests uses: step-security/changed-files@v47.0.5 @@ -50,9 +58,11 @@ jobs: fail_on_initial_diff_error: true wait-checks: needs: [check-file-changes] + # Nothing to wait for when the merge tests are skipped if: >- - startsWith(github.ref, 'refs/heads/pull-request/') && - needs.check-file-changes.outputs.any_changed == 'true' + (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && + needs.check-file-changes.outputs.any_changed == 'true' && + needs.check-file-changes.outputs.run_merge_tests == 'true' runs-on: ubuntu-latest permissions: checks: read diff --git a/.github/workflows/code_quality.yml b/.github/workflows/code_quality.yml index 4743fef960b..98898778e3d 100644 --- a/.github/workflows/code_quality.yml +++ b/.github/workflows/code_quality.yml @@ -3,6 +3,10 @@ name: Code Quality on: pull_request: branches: [main, release/*, feature/*] + # The merge queue re-runs the required checks on the PR merged with the target branch + merge_group: + types: [checks_requested] + branches: [main, release/*, feature/*] schedule: - cron: "0 0 * * *" # Nightly workflow_dispatch: @@ -21,12 +25,18 @@ jobs: steps: - uses: actions/checkout@v6 with: - # PRs: TruffleHog needs the base commit, else it errors, scans nothing, and passes. - fetch-depth: ${{ github.event_name == 'pull_request' && '0' || '1' }} + # PRs and merge queue groups: TruffleHog needs the base commit, else it errors, + # scans nothing, and passes. + fetch-depth: >- + ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && '0' || '1' }} - uses: ./.github/actions/ubuntu-setup - name: Secret Scanning uses: trufflesecurity/trufflehog@v3.90.5 with: + # Only set in a merge queue group, where the diff to scan is base..head (empty for the other events, + # which the action handles itself) + base: ${{ github.event.merge_group.base_sha }} + head: ${{ github.event.merge_group.head_sha }} # Exclude lob: its `(live|test)_[a-zA-Z0-9_]{35}` pattern matches any pytest function # whose name happens to be exactly 35 characters after `test_`, and it reports them as # verified. Only nightly sees them, since it scans all history while PRs scan their diff. diff --git a/.github/workflows/example_tests.yml b/.github/workflows/example_tests.yml index 8cc432a22ba..5a470d02b0e 100644 --- a/.github/workflows/example_tests.yml +++ b/.github/workflows/example_tests.yml @@ -4,6 +4,10 @@ on: push: branches: ["pull-request/[0-9]+"] # NOTE: paths cannot be used since push happens to copied PR and only latest commit to PR is used + # The merge queue re-runs the required checks on the PR merged with the target branch + merge_group: + types: [checks_requested] + branches: [main, release/*, feature/*] schedule: - cron: "0 0 * * *" # Nightly workflow_dispatch: @@ -27,6 +31,7 @@ jobs: contents: read outputs: any_changed: ${{ steps.lanes.outputs.any_changed }} + run_merge_tests: ${{ steps.base.outputs.run_merge_tests || 'true' }} torch: ${{ steps.lanes.outputs.torch }} vllm: ${{ steps.lanes.outputs.vllm }} trtllm: ${{ steps.lanes.outputs.trtllm }} @@ -37,10 +42,10 @@ jobs: - uses: actions/checkout@v6 with: fetch-depth: 0 - - if: startsWith(github.ref, 'refs/heads/pull-request/') + - if: startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group' id: base uses: ./.github/actions/pr-merge-base - - if: startsWith(github.ref, 'refs/heads/pull-request/') + - if: startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group' id: changed uses: step-security/changed-files@v47.0.5 with: @@ -87,7 +92,10 @@ jobs: - id: lanes env: # Nightly and on-demand runs have no diff to inspect, so they run everything. - NON_PR: ${{ !startsWith(github.ref, 'refs/heads/pull-request/') }} + NON_PR: >- + ${{ !(startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') }} + # The example tests are merge tests: a PR run without the run-merge-tests label skips them + RUN_MERGE_TESTS: ${{ steps.base.outputs.run_merge_tests || 'true' }} COMMON: ${{ steps.changed.outputs.common_any_modified }} TORCH: ${{ steps.changed.outputs.torch_any_modified }} VLLM: ${{ steps.changed.outputs.vllm_any_modified }} @@ -98,7 +106,8 @@ jobs: run: | any=false for lane in TORCH VLLM TRTLLM MEGATRON DIFFUSERS ONNX; do - if [[ "$NON_PR" == "true" || "$COMMON" == "true" || "${!lane}" == "true" ]]; then + if [[ "$RUN_MERGE_TESTS" == "true" && + ("$NON_PR" == "true" || "$COMMON" == "true" || "${!lane}" == "true") ]]; then run=true any=true else @@ -108,7 +117,9 @@ jobs: done echo "any_changed=$any" | tee --append "$GITHUB_OUTPUT" - name: Wait for unit tests before spending GPU runners - if: ${{ steps.lanes.outputs.any_changed == 'true' && startsWith(github.ref, 'refs/heads/pull-request/') }} + if: >- + ${{ steps.lanes.outputs.any_changed == 'true' && + (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') }} uses: poseidon/wait-for-status-checks@v0.7.0 with: token: ${{ secrets.GITHUB_TOKEN }} @@ -132,7 +143,9 @@ jobs: example: ${{ matrix.example }} timeout_minutes: 30 pip_install_extras: "[hf,dev-test]" - runner: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + runner: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && + 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} allow_failure: ${{ contains(format(',{0},', vars.ALLOW_FAILURE_EXAMPLE_TESTS), format(',{0},', matrix.example)) }} ##### vLLM Example Tests ##### @@ -153,7 +166,9 @@ jobs: example: ${{ matrix.example }} timeout_minutes: 30 pip_install_extras: "[hf,dev-test]" - runner: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + runner: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && + 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} allow_failure: ${{ contains(format(',{0},', vars.ALLOW_FAILURE_EXAMPLE_TESTS), format(',{0},', matrix.example)) }} ##### TensorRT-LLM Example Tests ##### @@ -172,7 +187,9 @@ jobs: docker_image: "nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc20" example: ${{ matrix.example }} pip_install_extras: "[hf,dev-test]" - runner: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + runner: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && + 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} allow_failure: ${{ contains(format(',{0},', vars.ALLOW_FAILURE_EXAMPLE_TESTS), format(',{0},', matrix.example)) }} ##### Megatron Example Tests ##### @@ -187,9 +204,13 @@ jobs: docker_image: "nvcr.io/nvidia/nemo:26.08" example: megatron_bridge # PR runs single-GPU (~11 min); the nightly runs every test multi-GPU (~58 min). - timeout_minutes: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 30 || 75 }} + timeout_minutes: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || + github.event_name == 'merge_group') && 30 || 75 }} pip_install_extras: "[hf,puzzletron,dev-test]" - runner: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + runner: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && + 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} allow_failure: ${{ contains(format(',{0},', vars.ALLOW_FAILURE_EXAMPLE_TESTS), ',megatron_bridge,') }} ##### Diffusers Example Tests ##### @@ -206,7 +227,9 @@ jobs: example: diffusers timeout_minutes: 60 pip_install_extras: "[onnx,hf,dev-test]" - runner: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + runner: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && + 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} allow_failure: ${{ contains(format(',{0},', vars.ALLOW_FAILURE_EXAMPLE_TESTS), ',diffusers,') }} ##### ONNX/TensorRT Example Tests ##### @@ -229,16 +252,27 @@ jobs: example: ${{ matrix.example }} timeout_minutes: 45 pip_install_extras: "[onnx,hf,dev-test]" - runner: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + runner: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && + 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} allow_failure: ${{ contains(format(',{0},', vars.ALLOW_FAILURE_EXAMPLE_TESTS), format(',{0},', matrix.example)) }} ##### Required Check for PR ##### example-pr-required-check: # Run even if example tests are skipped - if: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && always() }} + if: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || + github.event_name == 'merge_group') && always() }} needs: [pr-gate, torch, vllm, trtllm, megatron, diffusers, onnx] runs-on: ubuntu-latest + permissions: + pull-requests: write steps: + - name: Example tests are left to the merge queue + if: ${{ needs.pr-gate.outputs.run_merge_tests != 'true' }} + run: >- + echo "::notice::The example tests are merge tests and run in the merge queue. Add the + run-merge-tests label and re-run this workflow to run them on this PR." - name: Required example tests did not succeed if: | needs.pr-gate.result != 'success' || @@ -251,3 +285,15 @@ jobs: !contains(fromJSON('["success", "skipped"]'), needs.onnx.result) )) run: exit 1 + - name: Remove the run-merge-tests label, it applies to one round of runs + if: >- + ${{ !cancelled() && startsWith(github.ref, 'refs/heads/pull-request/') && + needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_merge_tests == 'true' }} + env: + GH_TOKEN: ${{ github.token }} + run: | + pr="${GITHUB_REF_NAME#pull-request/}" + url="repos/${GITHUB_REPOSITORY}/issues/${pr}/labels/run-merge-tests" + if gh api -X DELETE "$url" --silent; then + echo "::notice::Removed the run-merge-tests label after this round; re-add it for the next push." + fi diff --git a/.github/workflows/gpu_tests.yml b/.github/workflows/gpu_tests.yml index a0389da7b62..7eef2a61501 100644 --- a/.github/workflows/gpu_tests.yml +++ b/.github/workflows/gpu_tests.yml @@ -4,6 +4,10 @@ on: push: branches: ["pull-request/[0-9]+"] # NOTE: paths cannot be used since push happens to copied PR and only latest commit to PR is used + # The merge queue re-runs the required checks on the PR merged with the target branch + merge_group: + types: [checks_requested] + branches: [main, release/*, feature/*] schedule: - cron: "0 0 * * *" # Nightly workflow_dispatch: @@ -36,9 +40,11 @@ jobs: tests/gpu_trtllm/** tests/gpu_vllm/** + # The GPU tests are merge tests: they run in the merge queue, on nightly and on-demand runs, and + # on a PR that carries the `run-merge-tests` label (the gate says which: `run_merge_tests`). gpu-tests: needs: [pr-gate] - if: needs.pr-gate.outputs.any_changed == 'true' + if: needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_merge_tests == 'true' strategy: fail-fast: false matrix: @@ -60,7 +66,9 @@ jobs: - example: gpu_vllm timeout: 15 container_image: docker.io/vllm/vllm-openai:v0.20.0 - runs-on: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + runs-on: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && + 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} timeout-minutes: ${{ matrix.timeout }} container: image: ${{ matrix.container_image }} @@ -92,6 +100,7 @@ jobs: # Use `python3` (the vllm image has no `python` on PATH) python3 -m pip install nox && nox -s ${{ matrix.example }} - name: Upload GPU coverage to Codecov + if: github.event_name != 'merge_group' uses: codecov/codecov-action@v7 with: token: ${{ secrets.CODECOV_TOKEN }} @@ -102,10 +111,35 @@ jobs: gpu-pr-required-check: # Run even if gpu-tests is skipped - if: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && always() }} + if: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || + github.event_name == 'merge_group') && always() }} needs: [pr-gate, gpu-tests] runs-on: ubuntu-latest + permissions: + pull-requests: write steps: + - name: GPU tests are left to the merge queue + if: >- + ${{ needs.pr-gate.outputs.any_changed == 'true' && + needs.pr-gate.outputs.run_merge_tests != 'true' }} + run: >- + echo "::notice::The GPU tests are merge tests and run in the merge queue. Add the + run-merge-tests label and re-run this workflow to run them on this PR." - name: Required GPU tests did not succeed - if: ${{ needs.pr-gate.result != 'success' || (needs.pr-gate.outputs.any_changed == 'true' && needs.gpu-tests.result != 'success') }} + if: >- + ${{ needs.pr-gate.result != 'success' || (needs.pr-gate.outputs.any_changed == 'true' && + needs.pr-gate.outputs.run_merge_tests == 'true' && needs.gpu-tests.result != 'success') }} run: exit 1 + - name: Remove the run-merge-tests label, it applies to one round of runs + if: >- + ${{ !cancelled() && startsWith(github.ref, 'refs/heads/pull-request/') && + needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_merge_tests == 'true' }} + env: + GH_TOKEN: ${{ github.token }} + run: | + pr="${GITHUB_REF_NAME#pull-request/}" + url="repos/${GITHUB_REPOSITORY}/issues/${pr}/labels/run-merge-tests" + if gh api -X DELETE "$url" --silent; then + echo "::notice::Removed the run-merge-tests label after this round; re-add it for the next push." + fi diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 2132cdd302e..91177043c8f 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -4,6 +4,10 @@ on: pull_request: types: [opened, synchronize, reopened, closed] branches: [main, release/*, feature/*] + # The merge queue re-runs the required checks on the PR merged with the target branch + merge_group: + types: [checks_requested] + branches: [main, release/*, feature/*] push: branches: [main] schedule: diff --git a/.github/workflows/regression_tests.yml b/.github/workflows/regression_tests.yml index 001c4e6af4a..95368f63b30 100644 --- a/.github/workflows/regression_tests.yml +++ b/.github/workflows/regression_tests.yml @@ -4,6 +4,10 @@ on: push: branches: ["pull-request/[0-9]+"] # NOTE: paths cannot be used since push happens to copied PR and only latest commit to PR is used + # The merge queue re-runs the required checks on the PR merged with the target branch + merge_group: + types: [checks_requested] + branches: [main, release/*, feature/*] schedule: - cron: "0 0 * * *" # Nightly workflow_dispatch: @@ -37,10 +41,14 @@ jobs: modelopt_recipes/general/speculative_decoding/** tools/launcher/** + # The regression tests are merge tests: they run in the merge queue, on nightly and on-demand runs, + # and on a PR that carries the `run-merge-tests` label. regression-tests: needs: [pr-gate] - if: needs.pr-gate.outputs.any_changed == 'true' - runs-on: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + if: needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_merge_tests == 'true' + runs-on: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') && + 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} timeout-minutes: 15 container: image: nvcr.io/nvidia/pytorch:26.07-py3 @@ -62,6 +70,7 @@ jobs: COVERAGE_FILE: ${{ github.workspace }}/.coverage run: python -m pip install nox && nox -s regression - name: Upload regression coverage to Codecov + if: github.event_name != 'merge_group' uses: codecov/codecov-action@v7 with: token: ${{ secrets.CODECOV_TOKEN }} @@ -72,12 +81,36 @@ jobs: regression-pr-required-check: # Run even if regression-tests is skipped - if: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && always() }} + if: >- + ${{ (startsWith(github.ref, 'refs/heads/pull-request/') || + github.event_name == 'merge_group') && always() }} needs: [pr-gate, regression-tests] runs-on: ubuntu-latest + permissions: + pull-requests: write steps: + - name: Regression tests are left to the merge queue + if: >- + ${{ needs.pr-gate.outputs.any_changed == 'true' && + needs.pr-gate.outputs.run_merge_tests != 'true' }} + run: >- + echo "::notice::The regression tests are merge tests and run in the merge queue. Add the + run-merge-tests label and re-run this workflow to run them on this PR." - name: Required regression tests did not succeed if: | needs.pr-gate.result != 'success' || - (needs.pr-gate.outputs.any_changed == 'true' && needs.regression-tests.result != 'success') + (needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_merge_tests == 'true' && + needs.regression-tests.result != 'success') run: exit 1 + - name: Remove the run-merge-tests label, it applies to one round of runs + if: >- + ${{ !cancelled() && startsWith(github.ref, 'refs/heads/pull-request/') && + needs.pr-gate.outputs.any_changed == 'true' && needs.pr-gate.outputs.run_merge_tests == 'true' }} + env: + GH_TOKEN: ${{ github.token }} + run: | + pr="${GITHUB_REF_NAME#pull-request/}" + url="repos/${GITHUB_REPOSITORY}/issues/${pr}/labels/run-merge-tests" + if gh api -X DELETE "$url" --silent; then + echo "::notice::Removed the run-merge-tests label after this round; re-add it for the next push." + fi diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 6fbca185d9f..ad5371c799d 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -3,6 +3,10 @@ name: Unit tests on: pull_request: branches: [main, release/*, feature/*] + # The merge queue re-runs the required checks on the PR merged with the target branch + merge_group: + types: [checks_requested] + branches: [main, release/*, feature/*] push: branches: [main, release/*, feature/*] paths: @@ -35,15 +39,18 @@ jobs: - id: non-pr if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' run: echo "any_changed=true" >> $GITHUB_OUTPUT - - if: github.event_name == 'pull_request' + - if: github.event_name == 'pull_request' || github.event_name == 'merge_group' uses: actions/checkout@v6 with: fetch-depth: 0 - - if: github.event_name == 'pull_request' + - if: github.event_name == 'pull_request' || github.event_name == 'merge_group' name: Check for changes in test-relevant paths id: changed uses: step-security/changed-files@v47.0.5 with: + # Only set in a merge queue group (empty for PRs, where the action finds the base itself) + base_sha: ${{ github.event.merge_group.base_sha }} + sha: ${{ github.event.merge_group.head_sha }} files: | .github/workflows/unit_tests.yml modelopt/** @@ -67,10 +74,14 @@ jobs: env: COVERAGE_PROCESS_START: ${{ github.workspace }}/pyproject.toml COVERAGE_FILE: ${{ github.workspace }}/.coverage - # Everything but PRs also runs the slow tests marked ``manual`` (e.g. AutoNAS). - PYTEST_EXTRA: ${{ github.event_name != 'pull_request' && '--run-manual' || '' }} + # Everything but PRs and merge queue groups also runs the slow tests marked ``manual`` + # (e.g. AutoNAS). + PYTEST_EXTRA: >- + ${{ github.event_name != 'pull_request' && + github.event_name != 'merge_group' && '--run-manual' || '' }} run: pip install nox uv && nox -s "unit-3.12(torch_214, tf_latest)" -- $PYTEST_EXTRA - name: Upload coverage reports to Codecov + if: github.event_name != 'merge_group' uses: codecov/codecov-action@v7 with: token: ${{ secrets.CODECOV_TOKEN }} @@ -220,7 +231,7 @@ jobs: python -m pytest plugins/modelopt/skills/ -o addopts="" -p no:cacheprovider -v unit-pr-required-check: # Run even if some jobs are skipped - if: ${{ github.event_name == 'pull_request' && always() }} + if: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && always() }} needs: [check-file-changes, linux, windows, multi-version, partial-install, launcher, mcp, skills] runs-on: ubuntu-latest steps: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 45b6ffbf9f2..003b16277a4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -156,6 +156,8 @@ Test filenames must mirror the production module they primarily cover: tests for - `tests/gpu_vllm`: Fast GPU-based unit tests for the core ModelOpt library for vLLM features. In most cases, they should not take more than a few seconds to run. - `tests/examples`: Integration tests for ModelOpt examples. They should not take more than a few minutes to run. Please refer to [example test README](./tests/examples/README.md) for more details. +CI runs the merge tests (the GPU, example and regression tests: `tests/gpu`, `tests/gpu_megatron`, `tests/gpu_trtllm`, `tests/gpu_vllm`, `tests/regression` and `tests/examples`) in the merge queue and on the nightly, not on every pull request push. To run them on a pull request, add the `run-merge-tests` label and re-run the workflows; CI removes the label again when that round of runs finishes. + For lightweight focused local validation, run `pytest` directly on the relevant test path. For example: ```bash @@ -282,7 +284,13 @@ nox -s "unit-3.12(torch_211, tf_latest)" - **Contributors without write access:** When GitHub requests workflow approval, a reviewer with write permission must select **Approve and run workflows** to start GitHub Actions CI. - If any CI/CD checks fail, fix the issues and push again. -- Once your PR is approved and all checks pass, one of the reviewers will merge the PR. +- Once your PR is approved and all checks pass, one of the reviewers will merge the PR. With the merge queue + enabled on its target branch (the queue runs the checks for `main`, `release/*` and `feature/*`), merging adds + the PR to the queue: GitHub builds a temporary branch of the target branch plus the queued PRs ahead of it, + re-runs the required checks on it (path gating and runners as in a PR run, but with the merge tests, which a + PR run leaves out unless it carries the `run-merge-tests` label, and without the nightly-only tests) and + merges the PR only if they pass. Each PR still lands on its own. A PR that fails in the queue is removed + from it and the PRs behind it are re-tested without it; fix it and queue it again. - If your PR is a bug fix that should also land in an ongoing release branch, add the `cherry-pick-` label (e.g. `cherry-pick-0.47.0`) matching that release. Labeled PRs are cherry-picked into the release branch so the fix is included in the next release candidate.