[OMNIML-6180] Add a nightly test marker and use it for the example PR skips

.github/actions/run-example/action.yml

@@ -77,6 +77,10 @@ runs:
             # (tests/examples/hf_ptq/conftest.py). The 2-GPU nightly stays serial.
             pytest_args+=(-n 4)
           fi
+          if [[ "${{ github.ref }}" != refs/heads/pull-request/* ]]; then
+            # The scheduled and on-demand runs also run the tests marked nightly (tests/conftest.py).
+            pytest_args+=(--run-nightly)
+          fi
           python3 -m pytest tests/examples/${{ inputs.example }} --cov "${pytest_args[@]}" || status=$?
         fi
         echo "status=$status" >> "$GITHUB_OUTPUT"

.github/workflows/gpu_tests.yml

@@ -99,9 +99,9 @@ jobs:
           # sys.monitoring records the same lines as the default C tracer at a fraction of the
           # overhead on import-heavy worker processes. Needs Python 3.12+.
           COVERAGE_CORE: sysmon
-          # Tests marked `manual` (real-weight downloads and the like) run on nightly and on-demand
-          # runs, as in the unit tests, not on pull requests (copy-pr-bot runs those as `push`).
-          PYTEST_ADDOPTS: ${{ github.event_name != 'push' && '--run-manual' || '' }}
+          # Tests marked `manual` (real-weight downloads and the like) or `nightly` run on nightly and
+          # on-demand runs, as in the unit tests, not on pull requests (copy-pr-bot runs those as `push`).
+          PYTEST_ADDOPTS: ${{ github.event_name != 'push' && '--run-manual --run-nightly' || '' }}
         run: |
           # Use `python3` (the vllm image has no `python` on PATH)
           python3 -m pip install nox && nox -s ${{ matrix.example }}

.github/workflows/unit_tests.yml

@@ -67,8 +67,8 @@ jobs:
         env:
           COVERAGE_PROCESS_START: ${{ github.workspace }}/pyproject.toml
           COVERAGE_FILE: ${{ github.workspace }}/.coverage
-          # Everything but PRs also runs the slow tests marked ``manual`` (e.g. AutoNAS).
-          PYTEST_EXTRA: ${{ github.event_name != 'pull_request' && '--run-manual' || '' }}
+          # Everything but PRs also runs the slow tests marked ``manual`` (e.g. AutoNAS) and ``nightly``.
+          PYTEST_EXTRA: ${{ github.event_name != 'pull_request' && '--run-manual --run-nightly' || '' }}
           # sys.monitoring records the same lines as the default C tracer at a fraction of the overhead
           COVERAGE_CORE: sysmon
           # One single-threaded xdist worker per vCPU (`-n auto` counts physical cores); torch's

CONTRIBUTING.md

@@ -198,7 +198,9 @@ nox -s "unit-3.12(torch_211, tf_latest)"
   subprocesses (`spawn_multiprocess_job`, `subprocess.run` importing torch), train for many steps, or build
   real-size models belong in `tests/gpu*`; batch several cases into one spawn rather than one spawn per case,
   or reuse the session-scoped `cpu_dist_workers` fixture for multi-rank CPU tests.
-  Slow tests for features without active maintenance (e.g. AutoNAS) should be `@pytest.mark.manual` or nightly-only.
+  Slow tests for features without active maintenance (e.g. AutoNAS) should be `@pytest.mark.manual` or `@pytest.mark.nightly`.
+  `nightly` tests are skipped on pull requests and run in the scheduled and on-demand CI; both marks are skipped unless
+  you pass `--run-nightly` / `--run-manual`.
 - **Exercise the behavior a test claims to validate.** Mocks are useful for focused interface and wiring coverage, but
   replacing the implementation under test does not validate its real behavior. Include an end-to-end test that runs
   the actual implementation whenever the test claims backend or runtime behavior. For example, a test of

pyproject.toml

@@ -371,6 +371,7 @@ timeout_func_only = true
 markers = [
     "integration: Tests that require external services or other non-hermetic dependencies",
     "manual: Only run when --run-manual is given",
+    "nightly: Only run when --run-nightly is given (the scheduled and on-demand CI)",
     "release: Regression tests that should be run before every release",
 ]
 

tests/conftest.py

@@ -40,6 +40,12 @@ def pytest_addoption(parser):
         default=False,
         help="Run manual tests",
     )
+    parser.addoption(
+        "--run-nightly",
+        action="store_true",
+        default=False,
+        help="Run nightly tests (skipped on pull requests)",
+    )
     parser.addoption(
         "--run-release",
         action="store_true",
@@ -69,6 +75,7 @@ def pytest_collection_modifyitems(config, items):
     """Skip flag-gated tests and apply a default per-test timeout based on the test directory."""
     skip_marks = [
         ("manual", "--run-manual"),
+        ("nightly", "--run-nightly"),
         ("release", "--run-release"),
     ]
 

tests/examples/llm_qat/test_llm_qat.py

@@ -15,7 +15,6 @@
 
 
 import json
-import os
 
 import pytest
 import torch
@@ -34,12 +33,8 @@
 GRADIENT_CHECKPOINTING_BACKENDS = {"ddp", "deepspeed"}
 
 # PR runners have one GPU, so a backend case only exercises the wrapper and ModelOpt's patches, not
-# sharding. PRs run fsdp2 (the default, with the most ModelOpt-specific code); the other backends run
-# in the scheduled CI. QLoRA still trains on ddp on PRs.
-_PR_CI = os.environ.get("GITHUB_REF", "").startswith("refs/heads/pull-request/")
-_NIGHTLY_ONLY = pytest.mark.skipif(
-    _PR_CI, reason="Only fsdp2 runs on PRs; runs in the scheduled CI"
-)
+# sharding. PRs run fsdp2 (the default, with the most ModelOpt-specific code); the other backends are
+# marked nightly. QLoRA still trains on ddp on PRs.
 
 # Fast training overrides (short runs with frequent eval)
 FAST_TRAIN_ARGS = [
@@ -161,8 +156,8 @@ def test_dataset_utils_pretokenize(tiny_qwen3_path, tmp_path):
 
 @pytest.mark.parametrize("backend", [
     "fsdp2",
-    pytest.param("deepspeed", marks=_NIGHTLY_ONLY),
-    pytest.param("ddp", marks=_NIGHTLY_ONLY),
+    pytest.param("deepspeed", marks=pytest.mark.nightly),
+    pytest.param("ddp", marks=pytest.mark.nightly),
 ])
 def test_qwen3_qat_nvfp4(nvfp4_ptq_dir, dataset_cache_dir, tmp_path, backend):
     qat_output_dir = tmp_path / "qat"
@@ -236,7 +231,7 @@ def test_qwen3_lora_qat_nvfp4(tiny_qwen3_path, tmp_path):
 
 @pytest.mark.parametrize("backend", [
     "fsdp2",
-    pytest.param("deepspeed", marks=_NIGHTLY_ONLY),
+    pytest.param("deepspeed", marks=pytest.mark.nightly),
 ])
 def test_qwen3_qad_nvfp4(tiny_qwen3_path, nvfp4_ptq_dir, dataset_cache_dir, tmp_path, backend):
     qad_output_dir = tmp_path / "qad"

tests/examples/torch_onnx/test_torch_quant_to_onnx.py

@@ -15,7 +15,6 @@
 
 
 import importlib
-import os
 from collections import defaultdict
 from pathlib import Path
 
@@ -39,9 +38,8 @@
     "resnet50": ("resnet50", None),
 }
 
-# PRs run swinv2 for fp8 and nvfp4 only (swin_tiny covers every format); scheduled runs keep all.
-_PR_CI = os.environ.get("GITHUB_REF", "").startswith("refs/heads/pull-request/")
-_SKIPPED_ON_PR = {("swinv2_tiny", "int8"), ("swinv2_tiny", "mxfp8"), ("swinv2_tiny", "auto")}
+# PRs run swinv2 for fp8 and nvfp4 only (swin_tiny covers every format); the other cases are nightly.
+_NIGHTLY_ONLY = {("swinv2_tiny", "int8"), ("swinv2_tiny", "mxfp8"), ("swinv2_tiny", "auto")}
 
 
 def _assert_residual_inputs_are_quantized(onnx_save_path):
@@ -74,13 +72,22 @@ def _assert_residual_inputs_are_quantized(onnx_save_path):
         )
 
 
-@pytest.mark.parametrize("qformat", _QFORMATS)
-@pytest.mark.parametrize("model_key", list(_MODELS))
+@pytest.mark.parametrize(
+    ("model_key", "qformat"),
+    [
+        pytest.param(
+            model_key,
+            qformat,
+            id=f"{model_key}-{qformat}",
+            marks=pytest.mark.nightly if (model_key, qformat) in _NIGHTLY_ONLY else (),
+        )
+        for model_key in _MODELS
+        for qformat in _QFORMATS
+    ],
+)
 def test_torch_onnx(tmp_path, model_key, qformat):
     if model_key == "resnet50" and qformat not in _RESNET_RECIPE_QFORMATS:
         pytest.skip("Only FP8 and INT8 quantization are supported for ResNet")
-    if _PR_CI and (model_key, qformat) in _SKIPPED_ON_PR:
-        pytest.skip("Covered by swin_tiny on PRs; runs in the scheduled CI")
 
     timm_model_name, model_kwargs = _MODELS[model_key]
     onnx_save_path = tmp_path / f"{model_key}.{qformat}.onnx"