From eccea4832b7280ddaade2c1e1c971d768f5220cc Mon Sep 17 00:00:00 2001 From: Shengliang Xu Date: Thu, 22 Oct 2026 20:16:05 +0000 Subject: [PATCH] Add a nightly test marker and use it for the example tests that skip on PRs nightly tests are skipped unless --run-nightly is given, like manual and release. The unit, GPU and example workflows pass it off pull requests, and the llm_qat and torch_onnx example tests use it instead of checking GITHUB_REF themselves (the torch_onnx cases keep their test ids). Local runs now skip those cases unless --run-nightly is given. Signed-off-by: Shengliang Xu --- .github/actions/run-example/action.yml | 4 ++++ .github/workflows/gpu_tests.yml | 6 ++--- .github/workflows/unit_tests.yml | 4 ++-- CONTRIBUTING.md | 4 +++- pyproject.toml | 1 + tests/conftest.py | 7 ++++++ tests/examples/llm_qat/test_llm_qat.py | 15 ++++-------- .../torch_onnx/test_torch_quant_to_onnx.py | 23 ++++++++++++------- 8 files changed, 40 insertions(+), 24 deletions(-) diff --git a/.github/actions/run-example/action.yml b/.github/actions/run-example/action.yml index 339860e6860..09a07297a8f 100644 --- a/.github/actions/run-example/action.yml +++ b/.github/actions/run-example/action.yml @@ -77,6 +77,10 @@ runs: # (tests/examples/hf_ptq/conftest.py). The 2-GPU nightly stays serial. pytest_args+=(-n 4) fi + if [[ "${{ github.ref }}" != refs/heads/pull-request/* ]]; then + # The scheduled and on-demand runs also run the tests marked nightly (tests/conftest.py). + pytest_args+=(--run-nightly) + fi python3 -m pytest tests/examples/${{ inputs.example }} --cov "${pytest_args[@]}" || status=$? fi echo "status=$status" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/gpu_tests.yml b/.github/workflows/gpu_tests.yml index 6340788026f..80a1874f3eb 100644 --- a/.github/workflows/gpu_tests.yml +++ b/.github/workflows/gpu_tests.yml @@ -99,9 +99,9 @@ jobs: # sys.monitoring records the same lines as the default C tracer at a fraction of the # overhead on import-heavy worker processes. Needs Python 3.12+. COVERAGE_CORE: sysmon - # Tests marked `manual` (real-weight downloads and the like) run on nightly and on-demand - # runs, as in the unit tests, not on pull requests (copy-pr-bot runs those as `push`). - PYTEST_ADDOPTS: ${{ github.event_name != 'push' && '--run-manual' || '' }} + # Tests marked `manual` (real-weight downloads and the like) or `nightly` run on nightly and + # on-demand runs, as in the unit tests, not on pull requests (copy-pr-bot runs those as `push`). + PYTEST_ADDOPTS: ${{ github.event_name != 'push' && '--run-manual --run-nightly' || '' }} run: | # Use `python3` (the vllm image has no `python` on PATH) python3 -m pip install nox && nox -s ${{ matrix.example }} diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 5c4cd998c1b..5ef445a78f4 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -67,8 +67,8 @@ jobs: env: COVERAGE_PROCESS_START: ${{ github.workspace }}/pyproject.toml COVERAGE_FILE: ${{ github.workspace }}/.coverage - # Everything but PRs also runs the slow tests marked ``manual`` (e.g. AutoNAS). - PYTEST_EXTRA: ${{ github.event_name != 'pull_request' && '--run-manual' || '' }} + # Everything but PRs also runs the slow tests marked ``manual`` (e.g. AutoNAS) and ``nightly``. + PYTEST_EXTRA: ${{ github.event_name != 'pull_request' && '--run-manual --run-nightly' || '' }} # sys.monitoring records the same lines as the default C tracer at a fraction of the overhead COVERAGE_CORE: sysmon # One single-threaded xdist worker per vCPU (`-n auto` counts physical cores); torch's diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3c587b47201..d556b221748 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -198,7 +198,9 @@ nox -s "unit-3.12(torch_211, tf_latest)" subprocesses (`spawn_multiprocess_job`, `subprocess.run` importing torch), train for many steps, or build real-size models belong in `tests/gpu*`; batch several cases into one spawn rather than one spawn per case, or reuse the session-scoped `cpu_dist_workers` fixture for multi-rank CPU tests. - Slow tests for features without active maintenance (e.g. AutoNAS) should be `@pytest.mark.manual` or nightly-only. + Slow tests for features without active maintenance (e.g. AutoNAS) should be `@pytest.mark.manual` or `@pytest.mark.nightly`. + `nightly` tests are skipped on pull requests and run in the scheduled and on-demand CI; both marks are skipped unless + you pass `--run-nightly` / `--run-manual`. - **Exercise the behavior a test claims to validate.** Mocks are useful for focused interface and wiring coverage, but replacing the implementation under test does not validate its real behavior. Include an end-to-end test that runs the actual implementation whenever the test claims backend or runtime behavior. For example, a test of diff --git a/pyproject.toml b/pyproject.toml index 7d7acc80414..e585e7fb378 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -371,6 +371,7 @@ timeout_func_only = true markers = [ "integration: Tests that require external services or other non-hermetic dependencies", "manual: Only run when --run-manual is given", + "nightly: Only run when --run-nightly is given (the scheduled and on-demand CI)", "release: Regression tests that should be run before every release", ] diff --git a/tests/conftest.py b/tests/conftest.py index b1685e3fe63..7f606326b8d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -40,6 +40,12 @@ def pytest_addoption(parser): default=False, help="Run manual tests", ) + parser.addoption( + "--run-nightly", + action="store_true", + default=False, + help="Run nightly tests (skipped on pull requests)", + ) parser.addoption( "--run-release", action="store_true", @@ -69,6 +75,7 @@ def pytest_collection_modifyitems(config, items): """Skip flag-gated tests and apply a default per-test timeout based on the test directory.""" skip_marks = [ ("manual", "--run-manual"), + ("nightly", "--run-nightly"), ("release", "--run-release"), ] diff --git a/tests/examples/llm_qat/test_llm_qat.py b/tests/examples/llm_qat/test_llm_qat.py index 846fdb05eae..ef3983aeeb1 100644 --- a/tests/examples/llm_qat/test_llm_qat.py +++ b/tests/examples/llm_qat/test_llm_qat.py @@ -15,7 +15,6 @@ import json -import os import pytest import torch @@ -34,12 +33,8 @@ GRADIENT_CHECKPOINTING_BACKENDS = {"ddp", "deepspeed"} # PR runners have one GPU, so a backend case only exercises the wrapper and ModelOpt's patches, not -# sharding. PRs run fsdp2 (the default, with the most ModelOpt-specific code); the other backends run -# in the scheduled CI. QLoRA still trains on ddp on PRs. -_PR_CI = os.environ.get("GITHUB_REF", "").startswith("refs/heads/pull-request/") -_NIGHTLY_ONLY = pytest.mark.skipif( - _PR_CI, reason="Only fsdp2 runs on PRs; runs in the scheduled CI" -) +# sharding. PRs run fsdp2 (the default, with the most ModelOpt-specific code); the other backends are +# marked nightly. QLoRA still trains on ddp on PRs. # Fast training overrides (short runs with frequent eval) FAST_TRAIN_ARGS = [ @@ -161,8 +156,8 @@ def test_dataset_utils_pretokenize(tiny_qwen3_path, tmp_path): @pytest.mark.parametrize("backend", [ "fsdp2", - pytest.param("deepspeed", marks=_NIGHTLY_ONLY), - pytest.param("ddp", marks=_NIGHTLY_ONLY), + pytest.param("deepspeed", marks=pytest.mark.nightly), + pytest.param("ddp", marks=pytest.mark.nightly), ]) def test_qwen3_qat_nvfp4(nvfp4_ptq_dir, dataset_cache_dir, tmp_path, backend): qat_output_dir = tmp_path / "qat" @@ -236,7 +231,7 @@ def test_qwen3_lora_qat_nvfp4(tiny_qwen3_path, tmp_path): @pytest.mark.parametrize("backend", [ "fsdp2", - pytest.param("deepspeed", marks=_NIGHTLY_ONLY), + pytest.param("deepspeed", marks=pytest.mark.nightly), ]) def test_qwen3_qad_nvfp4(tiny_qwen3_path, nvfp4_ptq_dir, dataset_cache_dir, tmp_path, backend): qad_output_dir = tmp_path / "qad" diff --git a/tests/examples/torch_onnx/test_torch_quant_to_onnx.py b/tests/examples/torch_onnx/test_torch_quant_to_onnx.py index 9fa4aa958d1..65b4f8c0ce3 100644 --- a/tests/examples/torch_onnx/test_torch_quant_to_onnx.py +++ b/tests/examples/torch_onnx/test_torch_quant_to_onnx.py @@ -15,7 +15,6 @@ import importlib -import os from collections import defaultdict from pathlib import Path @@ -39,9 +38,8 @@ "resnet50": ("resnet50", None), } -# PRs run swinv2 for fp8 and nvfp4 only (swin_tiny covers every format); scheduled runs keep all. -_PR_CI = os.environ.get("GITHUB_REF", "").startswith("refs/heads/pull-request/") -_SKIPPED_ON_PR = {("swinv2_tiny", "int8"), ("swinv2_tiny", "mxfp8"), ("swinv2_tiny", "auto")} +# PRs run swinv2 for fp8 and nvfp4 only (swin_tiny covers every format); the other cases are nightly. +_NIGHTLY_ONLY = {("swinv2_tiny", "int8"), ("swinv2_tiny", "mxfp8"), ("swinv2_tiny", "auto")} def _assert_residual_inputs_are_quantized(onnx_save_path): @@ -74,13 +72,22 @@ def _assert_residual_inputs_are_quantized(onnx_save_path): ) -@pytest.mark.parametrize("qformat", _QFORMATS) -@pytest.mark.parametrize("model_key", list(_MODELS)) +@pytest.mark.parametrize( + ("model_key", "qformat"), + [ + pytest.param( + model_key, + qformat, + id=f"{model_key}-{qformat}", + marks=pytest.mark.nightly if (model_key, qformat) in _NIGHTLY_ONLY else (), + ) + for model_key in _MODELS + for qformat in _QFORMATS + ], +) def test_torch_onnx(tmp_path, model_key, qformat): if model_key == "resnet50" and qformat not in _RESNET_RECIPE_QFORMATS: pytest.skip("Only FP8 and INT8 quantization are supported for ResNet") - if _PR_CI and (model_key, qformat) in _SKIPPED_ON_PR: - pytest.skip("Covered by swin_tiny on PRs; runs in the scheduled CI") timm_model_name, model_kwargs = _MODELS[model_key] onnx_save_path = tmp_path / f"{model_key}.{qformat}.onnx"