From 8904c1707907eaf815988463c16035273adb6c2d Mon Sep 17 00:00:00 2001 From: Shengliang Xu Date: Fri, 9 Oct 2026 23:59:08 +0000 Subject: [PATCH 1/4] Run the CPU-only ONNX quantization tests in the unit lane The three ONNX test files added with #2678 (test_dla_qdq_rules.py, test_dla_scalar_constants.py and test_quantization_initializer_names.py, 68 test cases) quantize with calibration_eps=["cpu"] and run the result on the CPUExecutionProvider, so they need no GPU. They lived under tests/gpu/onnx and ran in the `gpu` job, where they cost 26.8 s per job (1.5 + 5.3 + 20.0 s in the CI logs). The `gpu` job is the second-longest lane of a PR run; the unit job runs the same tests in a few seconds on its xdist workers. Move them to tests/unit/onnx/quantization/ without changing their content. Testing: ran the three files on CPU with CUDA_VISIBLE_DEVICES empty and the repository's own modelopt on the path. 68 passed in 7.1 s at the old location and 68 passed in 13.3 s from the new one (this includes the unit conftest's session setup) on Python 3.12 with onnx 1.21.0 and onnxruntime 1.24.4. On Python 3.10.19 with onnxruntime 1.22.0, the oldest runtime pyproject allows, 68 passed in 15.3 s. The whole tests/unit/onnx directory still passes (756 passed, 12 xfailed). Not run: the Windows unit job and the GPU lane; CI should show the `gpu` job losing about 27 s. Signed-off-by: Shengliang Xu --- tests/{gpu => unit}/onnx/quantization/test_dla_qdq_rules.py | 0 .../{gpu => unit}/onnx/quantization/test_dla_scalar_constants.py | 0 .../onnx/quantization/test_quantization_initializer_names.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename tests/{gpu => unit}/onnx/quantization/test_dla_qdq_rules.py (100%) rename tests/{gpu => unit}/onnx/quantization/test_dla_scalar_constants.py (100%) rename tests/{gpu => unit}/onnx/quantization/test_quantization_initializer_names.py (100%) diff --git a/tests/gpu/onnx/quantization/test_dla_qdq_rules.py b/tests/unit/onnx/quantization/test_dla_qdq_rules.py similarity index 100% rename from tests/gpu/onnx/quantization/test_dla_qdq_rules.py rename to tests/unit/onnx/quantization/test_dla_qdq_rules.py diff --git a/tests/gpu/onnx/quantization/test_dla_scalar_constants.py b/tests/unit/onnx/quantization/test_dla_scalar_constants.py similarity index 100% rename from tests/gpu/onnx/quantization/test_dla_scalar_constants.py rename to tests/unit/onnx/quantization/test_dla_scalar_constants.py diff --git a/tests/gpu/onnx/quantization/test_quantization_initializer_names.py b/tests/unit/onnx/quantization/test_quantization_initializer_names.py similarity index 100% rename from tests/gpu/onnx/quantization/test_quantization_initializer_names.py rename to tests/unit/onnx/quantization/test_quantization_initializer_names.py From 995c8a47c1e7dafb9b11a628e0e7e9c865b9b45a Mon Sep 17 00:00:00 2001 From: Shengliang Xu Date: Sat, 10 Oct 2026 00:00:15 +0000 Subject: [PATCH 2/4] Calibrate test_concat_elim on CPU with a small model test_concat_elim_int8 and test_concat_elim_fp8 take about 22 s each in the CI logs, and their assertions only look at where the concat_elimination pass places Q/DQ nodes. quantize() calibrated a (1, 128, 240, 64) three-conv model on the default [cpu, cuda:0, trt] providers. It now calibrates a 32-channel 16x16 model on the CPU. The Convs stay above the 16-channel limit that graph_selection.py applies to INT8 and FP8, and both modes produce the same 8 Q and 8 DQ nodes as with the default size. build_conv_concat_model takes optional channels, height and width arguments. Their defaults are the old values, and with a seeded RNG the default model serializes to the same bytes as before. Testing: ruff check and format with the pinned ruff 0.15.20. With the repository's own modelopt on CPU, the two tests pass in 1.3 s; quantizing the default and the small model in both modes gives Q=8, DQ=8 each time. Not run: the gpu job. From the CI timings this should save about 40 s per job. Signed-off-by: Shengliang Xu --- tests/_test_utils/onnx/lib_test_models.py | 22 +++++++++---------- .../gpu/onnx/quantization/test_concat_elim.py | 17 ++++++++++---- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/tests/_test_utils/onnx/lib_test_models.py b/tests/_test_utils/onnx/lib_test_models.py index 57a0621c5da..12dba85b20f 100644 --- a/tests/_test_utils/onnx/lib_test_models.py +++ b/tests/_test_utils/onnx/lib_test_models.py @@ -260,12 +260,12 @@ def find_init(onnx_model: onnx.ModelProto, init_name: str) -> np.ndarray: # conv_4 # | # -def build_conv_concat_model(): +def build_conv_concat_model(channels=128, height=240, width=64): # Define your model inputs and outputs input_names = ["input_0"] output_names = ["conv4_conv/Conv2D:0"] - input_shapes = [(1, 128, 240, 64)] - output_shapes = [(1, 256, 240, 64)] + input_shapes = [(1, channels, height, width)] + output_shapes = [(1, 2 * channels, height, width)] inputs = [ helper.make_tensor_value_info(input_name, onnx.TensorProto.FLOAT, input_shape) @@ -340,26 +340,26 @@ def build_conv_concat_model(): helper.make_tensor( name="ReadVariableOp:0", data_type=onnx.TensorProto.FLOAT, - dims=(128, 128, 3, 3), - vals=np.random.uniform(low=0.5, high=1.0, size=128 * 128 * 3 * 3), + dims=(channels, channels, 3, 3), + vals=np.random.uniform(low=0.5, high=1.0, size=channels * channels * 3 * 3), ), helper.make_tensor( name="ReadVariableOp:1", data_type=onnx.TensorProto.FLOAT, - dims=(128, 128, 3, 3), - vals=np.random.uniform(low=0.5, high=1.0, size=128 * 128 * 3 * 3), + dims=(channels, channels, 3, 3), + vals=np.random.uniform(low=0.5, high=1.0, size=channels * channels * 3 * 3), ), helper.make_tensor( name="ReadVariableOp:2", data_type=onnx.TensorProto.FLOAT, - dims=(128, 128, 3, 3), - vals=np.random.uniform(low=0.5, high=1.0, size=128 * 128 * 3 * 3), + dims=(channels, channels, 3, 3), + vals=np.random.uniform(low=0.5, high=1.0, size=channels * channels * 3 * 3), ), helper.make_tensor( name="ReadVariableOp:3", data_type=onnx.TensorProto.FLOAT, - dims=(256, 384, 1, 1), - vals=np.random.uniform(low=0.5, high=1.0, size=256 * 384 * 1 * 1), + dims=(2 * channels, 3 * channels, 1, 1), + vals=np.random.uniform(low=0.5, high=1.0, size=2 * channels * 3 * channels), ), ] diff --git a/tests/gpu/onnx/quantization/test_concat_elim.py b/tests/gpu/onnx/quantization/test_concat_elim.py index 68179051909..de9feafe7e7 100644 --- a/tests/gpu/onnx/quantization/test_concat_elim.py +++ b/tests/gpu/onnx/quantization/test_concat_elim.py @@ -23,10 +23,19 @@ from modelopt.onnx.quantization.quantize import quantize +# Keep every Conv above the 16-channel limit for INT8/FP8 quantization (graph_selection.py) so the +# same nodes are quantized as with the default 128-channel model, but calibrate a small input. +SMALL_MODEL_SHAPE = {"channels": 32, "height": 16, "width": 16} + def _check_concat_qdq_status(onnx_path, quantize_mode): - # Quantize the input model - quantize(onnx_path, quantize_mode=quantize_mode, passes="concat_elimination") + # Quantize the input model. Only the graph rewrite is checked, so calibrate on CPU. + quantize( + onnx_path, + quantize_mode=quantize_mode, + passes="concat_elimination", + calibration_eps=["cpu"], + ) # Output model should be produced in the same tmpdir output_onnx_path = onnx_path.replace(".onnx", ".quant.onnx") @@ -61,7 +70,7 @@ def _check_concat_qdq_status(onnx_path, quantize_mode): def test_concat_elim_int8(tmp_path): # Copy the test model to the tmp_path - model = build_conv_concat_model() + model = build_conv_concat_model(**SMALL_MODEL_SHAPE) this_function_name = sys._getframe().f_code.co_name onnx_path = os.path.join(tmp_path, f"{this_function_name}.onnx") onnx.save(model, onnx_path) @@ -70,7 +79,7 @@ def test_concat_elim_int8(tmp_path): def test_concat_elim_fp8(tmp_path): # Copy the test model to the tmp_path - model = build_conv_concat_model() + model = build_conv_concat_model(**SMALL_MODEL_SHAPE) this_function_name = sys._getframe().f_code.co_name onnx_path = os.path.join(tmp_path, f"{this_function_name}.onnx") onnx.save(model, onnx_path) From a6cf2cfb6de4c459e057986b1d3bb149b6ce5bc3 Mon Sep 17 00:00:00 2001 From: Shengliang Xu Date: Sat, 10 Oct 2026 00:01:17 +0000 Subject: [PATCH 3/4] Benchmark a small model in the autotune benchmark tests test_benchmark.py built the 64x32x224x224 autotune model for every integration test. For the Python-API benchmark that means filling 103 M random floats and copying about 1.2 GB per iteration, and the five tests that benchmark a model take 31 s in the PR run (7.9, 7.9, 5.9, 4.8 and 4.8 s in run 37703177445). None of them looks at the latency beyond "a positive float". The fixtures now build a 1x32x32x32 model. The two trtexec tests also pass --duration=0: trtexec measures for at least 3 s by default whatever the iteration count, and these tests ask for two iterations. _create_simple_conv_onnx_model takes optional batch_size and spatial_size arguments with the old values as defaults, so the unit autotuner tests that share it build the same model (their 161 tests pass). Testing: ruff check and format with the pinned ruff 0.15.20, and the unit autotune tests with the repository's own modelopt on CPU. Not run: the TensorRT tests, because there is no TensorRT here. Estimated saving is about 20 s per gpu job; the CI run will show it. Signed-off-by: Shengliang Xu --- .../onnx/quantization/autotune/models.py | 6 +++--- .../quantization/autotune/test_benchmark.py | 18 ++++++++++++++---- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/tests/_test_utils/onnx/quantization/autotune/models.py b/tests/_test_utils/onnx/quantization/autotune/models.py index 84a8b4ab82e..89721661b4a 100644 --- a/tests/_test_utils/onnx/quantization/autotune/models.py +++ b/tests/_test_utils/onnx/quantization/autotune/models.py @@ -25,13 +25,13 @@ from onnx import helper -def _create_simple_conv_onnx_model(): +def _create_simple_conv_onnx_model(batch_size=64, spatial_size=224): """Build ONNX model: Input -> Conv -> Relu -> Output (minimal for autotuner tests).""" input_tensor = helper.make_tensor_value_info( - "input", onnx.TensorProto.FLOAT, [64, 32, 224, 224] + "input", onnx.TensorProto.FLOAT, [batch_size, 32, spatial_size, spatial_size] ) output_tensor = helper.make_tensor_value_info( - "output", onnx.TensorProto.FLOAT, [64, 64, 224, 224] + "output", onnx.TensorProto.FLOAT, [batch_size, 64, spatial_size, spatial_size] ) conv_node = helper.make_node( "Conv", diff --git a/tests/gpu/onnx/quantization/autotune/test_benchmark.py b/tests/gpu/onnx/quantization/autotune/test_benchmark.py index 012bf495ab6..47311e25219 100644 --- a/tests/gpu/onnx/quantization/autotune/test_benchmark.py +++ b/tests/gpu/onnx/quantization/autotune/test_benchmark.py @@ -46,8 +46,8 @@ @pytest.fixture def simple_conv_model_bytes(): - """ONNX model bytes: Input -> Conv -> Relu -> Output (from _test_utils).""" - model = _create_simple_conv_onnx_model() + """ONNX model bytes: Input -> Conv -> Relu -> Output (from _test_utils), small input.""" + model = _create_simple_conv_onnx_model(batch_size=1, spatial_size=32) return model.SerializeToString() @@ -431,7 +431,12 @@ def test_run_with_path(self, simple_conv_model_path): with tempfile.NamedTemporaryFile(suffix=".cache", delete=False) as f: cache_path = f.name try: - benchmark = TrtExecBenchmark(timing_cache_file=cache_path, warmup_runs=1, timing_runs=2) + benchmark = TrtExecBenchmark( + timing_cache_file=cache_path, + warmup_runs=1, + timing_runs=2, + trtexec_args=["--duration=0"], + ) latency_ms = benchmark.run(simple_conv_model_path) assert isinstance(latency_ms, float) assert latency_ms > 0 @@ -445,7 +450,12 @@ def test_run_with_bytes(self, simple_conv_model_bytes): with tempfile.NamedTemporaryFile(suffix=".cache", delete=False) as f: cache_path = f.name try: - benchmark = TrtExecBenchmark(timing_cache_file=cache_path, warmup_runs=1, timing_runs=2) + benchmark = TrtExecBenchmark( + timing_cache_file=cache_path, + warmup_runs=1, + timing_runs=2, + trtexec_args=["--duration=0"], + ) latency_ms = benchmark.run(simple_conv_model_bytes) assert isinstance(latency_ms, float) assert latency_ms > 0 From 6a935ee86534ce1f1ee06501c4a3b64bb2df2829 Mon Sep 17 00:00:00 2001 From: Shengliang Xu Date: Sat, 10 Oct 2026 00:01:52 +0000 Subject: [PATCH 4/4] Skip trtexec's 3-second measurement soak in the autotune workflow test test_export_quantized_model[True] takes 44 s in the PR run (run 37703177445), mostly inside trtexec. trtexec measures for at least 3 s per run by default whatever --iterations says, and the workflow builds and measures the baseline plus a number of Q/DQ schemes with it. The test only needs a latency to rank the schemes by, so pass --duration=0 and let trtexec stop after the 100 requested iterations. That is still 100 iterations per measurement; only the extra iterations the soak added are gone. The Python-API parameter ignores trtexec_args. The model keeps its size: the test asserts that the best scheme carries Q/DQ nodes, which needs a model whose latency is well above launch noise, and timing_runs stays at 100 for the same reason. Testing: ruff check and format with the pinned ruff 0.15.20. Not run: the test, because there is no TensorRT here. Expected saving is 3 s per measured scheme, probably 10-20 s per gpu job; the CI run will show it. Signed-off-by: Shengliang Xu --- tests/gpu/onnx/quantization/autotune/test_workflow.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/gpu/onnx/quantization/autotune/test_workflow.py b/tests/gpu/onnx/quantization/autotune/test_workflow.py index eebc42f6c33..b323402f1eb 100644 --- a/tests/gpu/onnx/quantization/autotune/test_workflow.py +++ b/tests/gpu/onnx/quantization/autotune/test_workflow.py @@ -59,7 +59,10 @@ def test_export_quantized_model(use_trtexec, simple_conv_model): output_path = output_dir + ".quant.onnx" try: - init_benchmark_instance(use_trtexec=use_trtexec, timing_runs=100) + # trtexec measures for at least 3 s by default; --duration=0 stops after timing_runs iterations + init_benchmark_instance( + use_trtexec=use_trtexec, timing_runs=100, trtexec_args=["--duration=0"] + ) autotuner = region_pattern_autotuning_workflow(baseline_model_path, Path(output_dir)) # Export model with Q/DQ insertion