Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 17 additions & 5 deletions .github/actions/pr-merge-base/action.yml
Original file line number Diff line number Diff line change
@@ -1,16 +1,17 @@
name: PR merge base
description: >
Resolve the commit to diff a copied PR branch against: the merge base of the PR's head and its
target branch. Outputs are empty for non-PR triggers, which have no diff to inspect.
Resolve the commit to diff a PR run against. For a copied PR branch it is the merge base of the
PR's head and its target branch; for a merge queue group it is the commit the group was built
on. Outputs are empty for non-PR triggers, which have no diff to inspect.
Requires a prior actions/checkout with fetch-depth 0.

outputs:
merge_base:
description: Commit to use as the base for changed-file comparisons.
value: ${{ steps.calculate-merge-base.outputs.merge-base }}
value: ${{ steps.calculate-merge-base.outputs.merge-base || steps.merge-group.outputs.base_sha }}
head_sha:
description: The PR's head commit.
value: ${{ steps.pr-shas.outputs.head_sha }}
description: The PR's head commit, or the merge queue group's head commit.
value: ${{ steps.pr-shas.outputs.head_sha || steps.merge-group.outputs.head_sha }}

runs:
using: composite
Expand All @@ -36,3 +37,14 @@ runs:
# status and emit an empty base, silently changing which lanes run.
merge_base=$(git merge-base "${{ steps.pr-shas.outputs.base_sha }}" "${{ steps.pr-shas.outputs.head_sha }}")
echo "merge-base=$merge_base" | tee --append "${GITHUB_OUTPUT}"
# A merge queue group is the target branch plus the queued PRs on top of it, so its diff
# against the commit it was built on is what the queue is about to merge.
- if: github.event_name == 'merge_group'
id: merge-group
shell: bash
env:
BASE_SHA: ${{ github.event.merge_group.base_sha }}
HEAD_SHA: ${{ github.event.merge_group.head_sha }}
run: |
echo "base_sha=$BASE_SHA" | tee --append "${GITHUB_OUTPUT}"
echo "head_sha=$HEAD_SHA" | tee --append "${GITHUB_OUTPUT}"
5 changes: 4 additions & 1 deletion .github/workflows/_example_tests_runner.yml
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,9 @@ jobs:
# Keep legacy ONNX/TRT tests available manually during deprecation.
pytest_args+=(--ignore=tests/examples/diffusers/test_diffusers_onnx_trt.py)
fi
if [[ "${{ inputs.example }}" == "hf_ptq" && "${{ github.ref }}" == refs/heads/pull-request/* ]]; then
if [[ "${{ inputs.example }}" == "hf_ptq" &&
( "${{ github.ref }}" == refs/heads/pull-request/* ||
"${{ github.event_name }}" == "merge_group" ) ]]; then
# One GPU on PR runners: small-model tests share it and the rest run alone on it
# (tests/examples/hf_ptq/conftest.py). The 2-GPU nightly stays serial.
pytest_args+=(-n 4)
Expand All @@ -118,6 +120,7 @@ jobs:
run: |
echo "::warning title=Allowed example failure::'${{ inputs.example }}' failed but is in the allow-failure list (vars.ALLOW_FAILURE_EXAMPLE_TESTS); not blocking. Remove it from the variable once fixed."
- name: Upload coverage to Codecov
if: github.event_name != 'merge_group'
uses: codecov/codecov-action@v7
with:
token: ${{ secrets.CODECOV_TOKEN }}
Expand Down
13 changes: 7 additions & 6 deletions .github/workflows/_pr_gate.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,18 +27,19 @@ jobs:
outputs:
any_changed: ${{ steps.changed-tests.outputs.any_modified || steps.non-pr.outputs.any_changed }}
steps:
# For non-PR triggers (schedule, workflow_dispatch), always run tests
# For non-PR triggers (schedule, workflow_dispatch), always run tests. A PR run is a push to a
# copy-pr-bot branch (pull-request/N) or a merge queue group, which both diff against the base.
- id: non-pr
if: "!startsWith(github.ref, 'refs/heads/pull-request/')"
if: "!(startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group')"
run: echo "any_changed=true" >> $GITHUB_OUTPUT
- if: startsWith(github.ref, 'refs/heads/pull-request/')
- if: startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group'
uses: actions/checkout@v6
with:
fetch-depth: 0
- if: startsWith(github.ref, 'refs/heads/pull-request/')
- if: startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group'
id: base
uses: ./.github/actions/pr-merge-base
- if: startsWith(github.ref, 'refs/heads/pull-request/')
- if: startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group'
name: Check for changes in test-relevant directories
id: changed-tests
uses: step-security/changed-files@v47.0.5
Expand All @@ -51,7 +52,7 @@ jobs:
wait-checks:
needs: [check-file-changes]
if: >-
startsWith(github.ref, 'refs/heads/pull-request/') &&
(startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') &&
needs.check-file-changes.outputs.any_changed == 'true'
runs-on: ubuntu-latest
permissions:
Expand Down
14 changes: 12 additions & 2 deletions .github/workflows/code_quality.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,10 @@ name: Code Quality
on:
pull_request:
branches: [main, release/*, feature/*]
# The merge queue re-runs the required checks on the PR merged with the target branch
merge_group:
types: [checks_requested]
branches: [main, release/*, feature/*]
schedule:
- cron: "0 0 * * *" # Nightly
workflow_dispatch:
Expand All @@ -21,12 +25,18 @@ jobs:
steps:
- uses: actions/checkout@v6
with:
# PRs: TruffleHog needs the base commit, else it errors, scans nothing, and passes.
fetch-depth: ${{ github.event_name == 'pull_request' && '0' || '1' }}
# PRs and merge queue groups: TruffleHog needs the base commit, else it errors,
# scans nothing, and passes.
fetch-depth: >-
${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && '0' || '1' }}
- uses: ./.github/actions/ubuntu-setup
- name: Secret Scanning
uses: trufflesecurity/trufflehog@v3.90.5
with:
# Only set in a merge queue group, where the diff to scan is base..head (empty for the other events,
# which the action handles itself)
base: ${{ github.event.merge_group.base_sha }}
head: ${{ github.event.merge_group.head_sha }}
# Exclude lob: its `(live|test)_[a-zA-Z0-9_]{35}` pattern matches any pytest function
# whose name happens to be exactly 35 characters after `test_`, and it reports them as
# verified. Only nightly sees them, since it scans all history while PRs scan their diff.
Expand Down
47 changes: 35 additions & 12 deletions .github/workflows/example_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,10 @@ on:
push:
branches: ["pull-request/[0-9]+"]
# NOTE: paths cannot be used since push happens to copied PR and only latest commit to PR is used
# The merge queue re-runs the required checks on the PR merged with the target branch
merge_group:
types: [checks_requested]
branches: [main, release/*, feature/*]
schedule:
- cron: "0 0 * * *" # Nightly
workflow_dispatch:
Expand Down Expand Up @@ -37,10 +41,10 @@ jobs:
- uses: actions/checkout@v6
with:
fetch-depth: 0
- if: startsWith(github.ref, 'refs/heads/pull-request/')
- if: startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group'
id: base
uses: ./.github/actions/pr-merge-base
- if: startsWith(github.ref, 'refs/heads/pull-request/')
- if: startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group'
id: changed
uses: step-security/changed-files@v47.0.5
with:
Expand Down Expand Up @@ -87,7 +91,8 @@ jobs:
- id: lanes
env:
# Nightly and on-demand runs have no diff to inspect, so they run everything.
NON_PR: ${{ !startsWith(github.ref, 'refs/heads/pull-request/') }}
NON_PR: >-
${{ !(startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') }}
COMMON: ${{ steps.changed.outputs.common_any_modified }}
TORCH: ${{ steps.changed.outputs.torch_any_modified }}
VLLM: ${{ steps.changed.outputs.vllm_any_modified }}
Expand All @@ -108,7 +113,9 @@ jobs:
done
echo "any_changed=$any" | tee --append "$GITHUB_OUTPUT"
- name: Wait for unit tests before spending GPU runners
if: ${{ steps.lanes.outputs.any_changed == 'true' && startsWith(github.ref, 'refs/heads/pull-request/') }}
if: >-
${{ steps.lanes.outputs.any_changed == 'true' &&
(startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') }}
uses: poseidon/wait-for-status-checks@v0.7.0
with:
token: ${{ secrets.GITHUB_TOKEN }}
Expand All @@ -132,7 +139,9 @@ jobs:
example: ${{ matrix.example }}
timeout_minutes: 30
pip_install_extras: "[hf,dev-test]"
runner: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }}
runner: >-
${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') &&
'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }}
allow_failure: ${{ contains(format(',{0},', vars.ALLOW_FAILURE_EXAMPLE_TESTS), format(',{0},', matrix.example)) }}

##### vLLM Example Tests #####
Expand All @@ -153,7 +162,9 @@ jobs:
example: ${{ matrix.example }}
timeout_minutes: 30
pip_install_extras: "[hf,dev-test]"
runner: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }}
runner: >-
${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') &&
'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }}
allow_failure: ${{ contains(format(',{0},', vars.ALLOW_FAILURE_EXAMPLE_TESTS), format(',{0},', matrix.example)) }}

##### TensorRT-LLM Example Tests #####
Expand All @@ -172,7 +183,9 @@ jobs:
docker_image: "nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc20"
example: ${{ matrix.example }}
pip_install_extras: "[hf,dev-test]"
runner: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }}
runner: >-
${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') &&
'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }}
allow_failure: ${{ contains(format(',{0},', vars.ALLOW_FAILURE_EXAMPLE_TESTS), format(',{0},', matrix.example)) }}

##### Megatron Example Tests #####
Expand All @@ -187,9 +200,13 @@ jobs:
docker_image: "nvcr.io/nvidia/nemo:26.08"
example: megatron_bridge
# PR runs single-GPU (~11 min); the nightly runs every test multi-GPU (~58 min).
timeout_minutes: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 30 || 75 }}
timeout_minutes: >-
${{ (startsWith(github.ref, 'refs/heads/pull-request/') ||
github.event_name == 'merge_group') && 30 || 75 }}
pip_install_extras: "[hf,puzzletron,dev-test]"
runner: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }}
runner: >-
${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') &&
'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }}
allow_failure: ${{ contains(format(',{0},', vars.ALLOW_FAILURE_EXAMPLE_TESTS), ',megatron_bridge,') }}

##### Diffusers Example Tests #####
Expand All @@ -206,7 +223,9 @@ jobs:
example: diffusers
timeout_minutes: 60
pip_install_extras: "[onnx,hf,dev-test]"
runner: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }}
runner: >-
${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') &&
'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }}
allow_failure: ${{ contains(format(',{0},', vars.ALLOW_FAILURE_EXAMPLE_TESTS), ',diffusers,') }}

##### ONNX/TensorRT Example Tests #####
Expand All @@ -229,13 +248,17 @@ jobs:
example: ${{ matrix.example }}
timeout_minutes: 45
pip_install_extras: "[onnx,hf,dev-test]"
runner: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }}
runner: >-
${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') &&
'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }}
allow_failure: ${{ contains(format(',{0},', vars.ALLOW_FAILURE_EXAMPLE_TESTS), format(',{0},', matrix.example)) }}

##### Required Check for PR #####
example-pr-required-check:
# Run even if example tests are skipped
if: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && always() }}
if: >-
${{ (startsWith(github.ref, 'refs/heads/pull-request/') ||
github.event_name == 'merge_group') && always() }}
needs: [pr-gate, torch, vllm, trtllm, megatron, diffusers, onnx]
runs-on: ubuntu-latest
steps:
Expand Down
13 changes: 11 additions & 2 deletions .github/workflows/gpu_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,10 @@ on:
push:
branches: ["pull-request/[0-9]+"]
# NOTE: paths cannot be used since push happens to copied PR and only latest commit to PR is used
# The merge queue re-runs the required checks on the PR merged with the target branch
merge_group:
types: [checks_requested]
branches: [main, release/*, feature/*]
schedule:
- cron: "0 0 * * *" # Nightly
workflow_dispatch:
Expand Down Expand Up @@ -60,7 +64,9 @@ jobs:
- example: gpu_vllm
timeout: 15
container_image: docker.io/vllm/vllm-openai:v0.20.0
runs-on: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }}
runs-on: >-
${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') &&
'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }}
timeout-minutes: ${{ matrix.timeout }}
container:
image: ${{ matrix.container_image }}
Expand Down Expand Up @@ -92,6 +98,7 @@ jobs:
# Use `python3` (the vllm image has no `python` on PATH)
python3 -m pip install nox && nox -s ${{ matrix.example }}
- name: Upload GPU coverage to Codecov
if: github.event_name != 'merge_group'
uses: codecov/codecov-action@v7
with:
token: ${{ secrets.CODECOV_TOKEN }}
Expand All @@ -102,7 +109,9 @@ jobs:

gpu-pr-required-check:
# Run even if gpu-tests is skipped
if: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && always() }}
if: >-
${{ (startsWith(github.ref, 'refs/heads/pull-request/') ||
github.event_name == 'merge_group') && always() }}
needs: [pr-gate, gpu-tests]
runs-on: ubuntu-latest
steps:
Expand Down
4 changes: 4 additions & 0 deletions .github/workflows/pages.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,10 @@ on:
pull_request:
types: [opened, synchronize, reopened, closed]
branches: [main, release/*, feature/*]
# The merge queue re-runs the required checks on the PR merged with the target branch
merge_group:
types: [checks_requested]
branches: [main, release/*, feature/*]
push:
branches: [main]
schedule:
Expand Down
13 changes: 11 additions & 2 deletions .github/workflows/regression_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,10 @@ on:
push:
branches: ["pull-request/[0-9]+"]
# NOTE: paths cannot be used since push happens to copied PR and only latest commit to PR is used
# The merge queue re-runs the required checks on the PR merged with the target branch
merge_group:
types: [checks_requested]
branches: [main, release/*, feature/*]
schedule:
- cron: "0 0 * * *" # Nightly
workflow_dispatch:
Expand Down Expand Up @@ -40,7 +44,9 @@ jobs:
regression-tests:
needs: [pr-gate]
if: needs.pr-gate.outputs.any_changed == 'true'
runs-on: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }}
runs-on: >-
${{ (startsWith(github.ref, 'refs/heads/pull-request/') || github.event_name == 'merge_group') &&
'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }}
timeout-minutes: 15
container:
image: nvcr.io/nvidia/pytorch:26.07-py3
Expand All @@ -62,6 +68,7 @@ jobs:
COVERAGE_FILE: ${{ github.workspace }}/.coverage
run: python -m pip install nox && nox -s regression
- name: Upload regression coverage to Codecov
if: github.event_name != 'merge_group'
uses: codecov/codecov-action@v7
with:
token: ${{ secrets.CODECOV_TOKEN }}
Expand All @@ -72,7 +79,9 @@ jobs:

regression-pr-required-check:
# Run even if regression-tests is skipped
if: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && always() }}
if: >-
${{ (startsWith(github.ref, 'refs/heads/pull-request/') ||
github.event_name == 'merge_group') && always() }}
needs: [pr-gate, regression-tests]
runs-on: ubuntu-latest
steps:
Expand Down
21 changes: 16 additions & 5 deletions .github/workflows/unit_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,10 @@ name: Unit tests
on:
pull_request:
branches: [main, release/*, feature/*]
# The merge queue re-runs the required checks on the PR merged with the target branch
merge_group:
types: [checks_requested]
branches: [main, release/*, feature/*]
push:
branches: [main, release/*, feature/*]
paths:
Expand Down Expand Up @@ -35,15 +39,18 @@ jobs:
- id: non-pr
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
run: echo "any_changed=true" >> $GITHUB_OUTPUT
- if: github.event_name == 'pull_request'
- if: github.event_name == 'pull_request' || github.event_name == 'merge_group'
uses: actions/checkout@v6
with:
fetch-depth: 0
- if: github.event_name == 'pull_request'
- if: github.event_name == 'pull_request' || github.event_name == 'merge_group'
name: Check for changes in test-relevant paths
id: changed
uses: step-security/changed-files@v47.0.5
with:
# Only set in a merge queue group (empty for PRs, where the action finds the base itself)
base_sha: ${{ github.event.merge_group.base_sha }}
sha: ${{ github.event.merge_group.head_sha }}
files: |
.github/workflows/unit_tests.yml
modelopt/**
Expand All @@ -67,10 +74,14 @@ jobs:
env:
COVERAGE_PROCESS_START: ${{ github.workspace }}/pyproject.toml
COVERAGE_FILE: ${{ github.workspace }}/.coverage
# Everything but PRs also runs the slow tests marked ``manual`` (e.g. AutoNAS).
PYTEST_EXTRA: ${{ github.event_name != 'pull_request' && '--run-manual' || '' }}
# Everything but PRs and merge queue groups also runs the slow tests marked ``manual``
# (e.g. AutoNAS).
PYTEST_EXTRA: >-
${{ github.event_name != 'pull_request' &&
github.event_name != 'merge_group' && '--run-manual' || '' }}
run: pip install nox uv && nox -s "unit-3.12(torch_214, tf_latest)" -- $PYTEST_EXTRA
- name: Upload coverage reports to Codecov
if: github.event_name != 'merge_group'
uses: codecov/codecov-action@v7
with:
token: ${{ secrets.CODECOV_TOKEN }}
Expand Down Expand Up @@ -220,7 +231,7 @@ jobs:
python -m pytest plugins/modelopt/skills/ -o addopts="" -p no:cacheprovider -v
unit-pr-required-check:
# Run even if some jobs are skipped
if: ${{ github.event_name == 'pull_request' && always() }}
if: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && always() }}
needs: [check-file-changes, linux, windows, multi-version, partial-install, launcher, mcp, skills]
runs-on: ubuntu-latest
steps:
Expand Down
Loading
Loading