diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 66f4015c760..e71d601b16d 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -34,6 +34,10 @@ Changelog - The Megatron unified HF export (``export_mcore_gpt_to_hf``) now copies the source checkpoint's non-model files (tokenizer, processor, remote code, chat templates, ...) into the export unchanged, from local disk only. The ``examples/megatron_bridge`` scripts make a Hugging Face Hub model local first with ``modelopt.torch.export.ensure_local_checkpoint``. - Add an end-to-end W4A4 NVFP4 PTQ and QAD tutorial for NVIDIA Nemotron 3.5 Super VL, covering chat-data blend preparation, assistant-only distillation, Hugging Face export and vLLM serving. See `examples/megatron_bridge/tutorials/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16/README.md `_ for details. +*Sparsity* + +- ``mts.sparsify`` now also applies sparse attention: pass a sparse attention config such as ``mts.SKIP_SOFTMAX_CALIB`` in place of the mode, or combine it with weight sparsity as ``[("sparsegpt", {}), ("sparse_attention", cfg)]``. ``mts.export`` now also handles models with sparse attention, which it keeps. + *Misc* - The ``evaluation`` agent skill can stop looping ("doom loop") reasoning early: ask it to avoid doom loops and it adds vLLM's ``repetition_detection`` request field (vLLM 0.17.0 or later) to every task, ending a request once its output repeats. Use the same setting for baseline and candidate runs. @@ -48,6 +52,7 @@ Changelog **Deprecations** - ``modelopt.torch.export.copy_non_safetensor_files_from_ckpt`` is deprecated; use ``copy_non_model_files``. +- ``modelopt.torch.sparsity.attention_sparsity.sparsify`` (``mtsa.sparsify``) is deprecated; use ``mts.sparsify(model, config, forward_loop=forward_loop)``. **Bug Fixes** diff --git a/docs/source/guides/6_sparsity.rst b/docs/source/guides/6_sparsity.rst index 12a17b3c66c..ca759a70079 100644 --- a/docs/source/guides/6_sparsity.rst +++ b/docs/source/guides/6_sparsity.rst @@ -6,8 +6,8 @@ Introduction ============ ModelOpt's Sparsity module (:mod:`modelopt.torch.sparsity `) enables -you to sparsify the weights of your model. This can be useful for reducing the memory footprint of -your model and can also be used to speed up inference. +you to sparsify the weights and the attention of your model. This can be useful for reducing the +memory footprint of your model and can also be used to speed up inference. Follow the steps described below to obtain a model with sparse weights using ModelOpt's Sparsity @@ -69,6 +69,25 @@ Example usage: ``sparsegpt``. `sparse_magnitude` does not require `data_loader` as it uses magnitude-based method for thresholding. +Sparse attention +---------------- + +Given a sparse attention config instead of a mode, the same API applies sparse attention, e.g., +skip-softmax with a fixed threshold (``mts.SKIP_SOFTMAX_DEFAULT``) or with a threshold calibrated +on generated RULER data (``mts.SKIP_SOFTMAX_CALIB``): + +.. code-block:: python + + model = mts.sparsify(model, mts.SKIP_SOFTMAX_CALIB) + + # Weight and attention sparsity can be combined in one call + model = mts.sparsify( + model, [("sparsegpt", {}), ("sparse_attention", mts.SKIP_SOFTMAX_DEFAULT)], config=sparsity_config + ) + +See the `attention sparsity example `_ +for all methods and options. + Save and restore the sparse model --------------------------------- @@ -107,7 +126,8 @@ To restore the saved sparse model you can use :meth:`mts.export() ` will export the sparse model to a plain pytorch model. The sparse masks will be applied to model weights and all the sparse metadata will be removed. After exporting, sparsity will no longer be enforced during - subsequent fine-tuning. If you want to continue fine-tuning, do not export the model. + subsequent fine-tuning. If you want to continue fine-tuning, do not export the model. Sparse + attention needs no export and stays applied. .. note:: diff --git a/examples/diffusers/README.md b/examples/diffusers/README.md index b67d9630b8d..37fdff50689 100644 --- a/examples/diffusers/README.md +++ b/examples/diffusers/README.md @@ -385,7 +385,7 @@ Skip-softmax sparse attention skips KV tiles whose attention scores are negligib ### Getting Started ```python -import modelopt.torch.sparsity.attention_sparsity as mtsa +import modelopt.torch.sparsity as mts # 1. Define config with calibration config = { @@ -410,7 +410,7 @@ def forward_loop(model): pipeline(prompt="a cat", num_frames=81, num_inference_steps=40, ...) # 3. Sparsify + calibrate -mtsa.sparsify(transformer, config, forward_loop=forward_loop) +mts.sparsify(transformer, config, forward_loop=forward_loop) # 4. Generate as usual — sparsity is applied automatically output = pipeline(prompt="a dog on the beach", ...) diff --git a/examples/diffusers/sparsity/wan22_skip_softmax.py b/examples/diffusers/sparsity/wan22_skip_softmax.py index 890cec547ff..38d01c35a6e 100644 --- a/examples/diffusers/sparsity/wan22_skip_softmax.py +++ b/examples/diffusers/sparsity/wan22_skip_softmax.py @@ -58,7 +58,7 @@ from diffusers import AutoencoderKLWan, WanPipeline from diffusers.utils import export_to_video -import modelopt.torch.sparsity.attention_sparsity as mtsa +import modelopt.torch.sparsity as mts from modelopt.torch.export import export_hf_checkpoint from modelopt.torch.sparsity.attention_sparsity.sparse_attention import SparseAttentionModule @@ -426,7 +426,7 @@ def main() -> None: num_blocks = _get_num_blocks(transformer) print(f"Applying Triton backend to {name} ({num_blocks} blocks)...") config = build_sparse_config(args, num_blocks=num_blocks) - mtsa.sparsify(transformer, config, forward_loop=None) + mts.sparsify(transformer, config, forward_loop=None) else: # Build calibration forward loop if needed forward_loop = None @@ -461,7 +461,7 @@ def main() -> None: num_blocks = _get_num_blocks(transformer) print(f"Applying skip-softmax to {name} ({num_blocks} blocks)...") config = build_sparse_config(args, num_blocks=num_blocks) - mtsa.sparsify(transformer, config, forward_loop=forward_loop) + mts.sparsify(transformer, config, forward_loop=forward_loop) # ---- Free calibration memory before inference ---- if not args.baseline and not args.triton_baseline and forward_loop is not None: diff --git a/examples/llm_eval/sparse_attention_utils.py b/examples/llm_eval/sparse_attention_utils.py index 8dc560851f1..66530293055 100644 --- a/examples/llm_eval/sparse_attention_utils.py +++ b/examples/llm_eval/sparse_attention_utils.py @@ -15,7 +15,7 @@ """Utilities for sparse attention integration with llm_eval.""" -import modelopt.torch.sparsity.attention_sparsity as mtsa +import modelopt.torch.sparsity as mts def _extract_model(model_obj): @@ -52,8 +52,8 @@ def sparsify_model( # Resolve config if isinstance(sparse_cfg, str): - # Get config from mtsa module (e.g., SKIP_SOFTMAX_CALIB, SKIP_SOFTMAX_DEFAULT) - mtsa_cfg = getattr(mtsa, sparse_cfg, None) + # Get config from mts module (e.g., SKIP_SOFTMAX_CALIB, SKIP_SOFTMAX_DEFAULT) + mtsa_cfg = getattr(mts, sparse_cfg, None) if mtsa_cfg is None: raise ValueError(f"Unknown sparse_cfg: {sparse_cfg}.") else: @@ -72,7 +72,7 @@ def sparsify_model( # Apply sparsification print(f"\nApplying sparse attention with config: {sparse_cfg}") - mtsa.sparsify(net, mtsa_cfg) + mts.sparsify(net, mtsa_cfg) print("Sparse attention applied successfully!") return model diff --git a/examples/llm_sparsity/attention_sparsity/README.md b/examples/llm_sparsity/attention_sparsity/README.md index 00cf6e0ac21..e430a11f802 100644 --- a/examples/llm_sparsity/attention_sparsity/README.md +++ b/examples/llm_sparsity/attention_sparsity/README.md @@ -15,7 +15,7 @@ Two attention backends are available: ### Quick Example ```python -import modelopt.torch.sparsity.attention_sparsity as mtsa +import modelopt.torch.sparsity as mts from modelopt.torch.sparsity.attention_sparsity.config import SKIP_SOFTMAX_DEFAULT # Load your model @@ -26,7 +26,7 @@ model = AutoModelForCausalLM.from_pretrained( ) # Apply sparse attention -model = mtsa.sparsify(model, config=SKIP_SOFTMAX_DEFAULT) +model = mts.sparsify(model, SKIP_SOFTMAX_DEFAULT) ``` > [!Note] @@ -43,7 +43,7 @@ Uses a fixed threshold value. Simple but may not be optimal for all sequence len ```python from modelopt.torch.sparsity.attention_sparsity.config import SKIP_SOFTMAX_DEFAULT -model = mtsa.sparsify(model, config=SKIP_SOFTMAX_DEFAULT) +model = mts.sparsify(model, SKIP_SOFTMAX_DEFAULT) ``` #### 2. Calibrated Threshold (SKIP_SOFTMAX_CALIB) @@ -53,7 +53,7 @@ Uses RULER-based calibration to determine an optimal dynamic threshold that adap ```python from modelopt.torch.sparsity.attention_sparsity.config import SKIP_SOFTMAX_CALIB -model = mtsa.sparsify(model, config=SKIP_SOFTMAX_CALIB) +model = mts.sparsify(model, SKIP_SOFTMAX_CALIB) ``` ### N:M Sparse Softmax (SPARSE_SOFTMAX_DEFAULT) @@ -69,7 +69,7 @@ model = AutoModelForCausalLM.from_pretrained( device_map="cuda", ) -model = mtsa.sparsify(model, config=SPARSE_SOFTMAX_DEFAULT) +model = mts.sparsify(model, SPARSE_SOFTMAX_DEFAULT) ``` Custom N:M configuration: @@ -90,11 +90,11 @@ sparse_cfg = { }, } -model = mtsa.sparsify(model, config=sparse_cfg) +model = mts.sparsify(model, sparse_cfg) ``` > [!Note] -> N:M sparse softmax requires the Triton backend (`backend="triton"`). The `attn_implementation` is automatically set to `"modelopt_triton"` by `mtsa.sparsify()`. N:M sparsity is applied during prefill only — decode tokens are not sparsified. +> N:M sparse softmax requires the Triton backend (`backend="triton"`). The `attn_implementation` is automatically set to `"modelopt_triton"` by `mts.sparsify()`. N:M sparsity is applied during prefill only — decode tokens are not sparsified. ## Prerequisites @@ -233,7 +233,7 @@ custom_config = { }, } -model = mtsa.sparsify(model, config=custom_config) +model = mts.sparsify(model, custom_config) ``` ## References diff --git a/examples/llm_sparsity/attention_sparsity/hf_sa.py b/examples/llm_sparsity/attention_sparsity/hf_sa.py index b94238b8cb1..4fea29180af 100644 --- a/examples/llm_sparsity/attention_sparsity/hf_sa.py +++ b/examples/llm_sparsity/attention_sparsity/hf_sa.py @@ -26,7 +26,7 @@ from transformers import AutoModelForCausalLM, AutoTokenizer import modelopt.torch.opt as mto -import modelopt.torch.sparsity.attention_sparsity as mtsa +import modelopt.torch.sparsity as mts from modelopt.torch.export import ensure_local_checkpoint, export_hf_checkpoint from modelopt.torch.sparsity.attention_sparsity.config import ( SKIP_SOFTMAX_CALIB, @@ -152,7 +152,7 @@ def main(args): args.hub_model_id, args.local_checkpoint_path = ensure_local_checkpoint(args.pyt_ckpt_path) print(f"Loading model: {args.local_checkpoint_path}") - # No need to specify attn_implementation here — mtsa.sparsify() sets it + # No need to specify attn_implementation here — mts.sparsify() sets it # automatically ("eager" for pytorch backend, "modelopt_triton" for triton). model = AutoModelForCausalLM.from_pretrained( args.local_checkpoint_path, attn_implementation="eager", dtype="auto", device_map="auto" @@ -202,7 +202,7 @@ def main(args): else str(Path(__file__).parent / "data"), ) - model = mtsa.sparsify(model, config=sparse_config) + model = mts.sparsify(model, sparse_config) print("Sparse attention applied successfully!") # Generate sample output AFTER sparse attention diff --git a/modelopt/torch/sparsity/__init__.py b/modelopt/torch/sparsity/__init__.py index 2013fded1ae..f0789748d21 100644 --- a/modelopt/torch/sparsity/__init__.py +++ b/modelopt/torch/sparsity/__init__.py @@ -15,10 +15,13 @@ """API for sparsification algorithms. -This module provides access to both weight sparsity and attention sparsity algorithms. -For backward compatibility, weight sparsity APIs are re-exported at the module level. +This module provides access to both weight sparsity and attention sparsity algorithms through +:meth:`sparsify `. """ +from . import attention_sparsity +from .attention_sparsity.config import * +from .sparsification import * + # Import weight sparsity for backward compatibility from .weight_sparsity import mode, module, plugins -from .weight_sparsity.sparsification import * diff --git a/modelopt/torch/sparsity/attention_sparsity/conversion.py b/modelopt/torch/sparsity/attention_sparsity/conversion.py index 38944517740..a51d77dc263 100644 --- a/modelopt/torch/sparsity/attention_sparsity/conversion.py +++ b/modelopt/torch/sparsity/attention_sparsity/conversion.py @@ -74,7 +74,7 @@ def _set_attn_implementation(model: nn.Module, config: SparseAttentionConfig) -> directly in ``SparseAttentionModule.forward()``, so no ``attn_implementation`` change is needed. - This is called automatically during ``mtsa.sparsify()`` so users never need + This is called automatically during ``mts.sparsify()`` so users never need to manually set ``attn_implementation``. """ sparse_cfg = config.sparse_cfg if hasattr(config, "sparse_cfg") else {} @@ -354,6 +354,21 @@ def update_sparse_attention_metadata( ) +def get_sparse_attention_mode_state( + model: nn.Module, +) -> tuple[SparseAttentionConfig, MetadataDict] | None: + """Return the config and metadata of the latest ``sparse_attention`` mode applied to ``model``.""" + # Wrappers and containers of converted models have no mode state of their own. + if not hasattr(model, ModeloptStateManager._state_key): + return None + mode_states = [ + (config, metadata) + for mode, config, metadata in ModeloptStateManager(model).modes_with_states() + if str(mode) == "sparse_attention" + ] + return mode_states[-1] if mode_states else None + + def _get_sparse_softmax_export_config(module: SparseAttentionModule) -> dict[str, Any] | None: """Return N:M sparse-softmax export metadata for one sparse attention module.""" method_config = getattr(module, "_method_config", {}) @@ -523,8 +538,9 @@ def disable_sparse_attention(model: nn.Module, wildcard_or_filter_func: str | Ca For example: "*lm_head*", "*layer_0*", etc. Example: + >>> import modelopt.torch.sparsity as mts >>> import modelopt.torch.sparsity.attention_sparsity as sparse_attn - >>> model = sparse_attn.sparsify(model, config) + >>> model = mts.sparsify(model, config) >>> # Disable sparse attention for lm_head >>> sparse_attn.disable_sparse_attention(model, "*lm_head*") """ @@ -550,8 +566,9 @@ def enable_sparse_attention(model: nn.Module, wildcard_or_filter_func: str | Cal For example: "*attention*", "*attn*", etc. Example: + >>> import modelopt.torch.sparsity as mts >>> import modelopt.torch.sparsity.attention_sparsity as sparse_attn - >>> model = sparse_attn.sparsify(model, config) + >>> model = mts.sparsify(model, config) >>> # Re-enable sparse attention for all attention modules >>> sparse_attn.enable_sparse_attention(model, "*attention*") """ diff --git a/modelopt/torch/sparsity/attention_sparsity/methods/vsa.py b/modelopt/torch/sparsity/attention_sparsity/methods/vsa.py index 66acfb510ce..d050da52125 100644 --- a/modelopt/torch/sparsity/attention_sparsity/methods/vsa.py +++ b/modelopt/torch/sparsity/attention_sparsity/methods/vsa.py @@ -22,7 +22,7 @@ Uses the optimized Triton kernel from fastvideo_kernel. Integration: - After ``mtsa.sparsify(model, VSA_DEFAULT)``, each attention layer's + After ``mts.sparsify(model, VSA_DEFAULT)``, each attention layer's ``F.scaled_dot_product_attention`` call is intercepted and replaced by the VSA kernel. Cross-attention (Q/K have different seq_len) is automatically skipped. This works with HF transformers and diffusers. diff --git a/modelopt/torch/sparsity/attention_sparsity/mode.py b/modelopt/torch/sparsity/attention_sparsity/mode.py index f389509a592..06210d78cea 100644 --- a/modelopt/torch/sparsity/attention_sparsity/mode.py +++ b/modelopt/torch/sparsity/attention_sparsity/mode.py @@ -23,6 +23,7 @@ UpdateEntrypoint, _ModeRegistryCls, ) +from modelopt.torch.opt.searcher import BaseSearcher from .config import SparseAttentionConfig from .conversion import ( @@ -30,6 +31,7 @@ restore_sparse_attention_model, update_sparse_attention_metadata, ) +from .searcher import SparseAttentionSearcher # Create registry for sparse attention modes SparseAttentionModeRegistry = _ModeRegistryCls("sparse_attention") @@ -54,15 +56,9 @@ def config_class(self) -> type[ModeloptBaseConfig]: return SparseAttentionConfig @property - def next_prohibited_modes(self) -> set[str] | None: - """Modes that should not be applied after this mode.""" - # Can work with quantization but not with weight sparsity - return {"sparsity"} - - @property - def export_mode(self) -> str | None: - """The mode that corresponds to the export mode of this mode.""" - return "export_sparse_attention" + def search_algorithm(self) -> type[BaseSearcher]: + """Specifies the search algorithm for the mode.""" + return SparseAttentionSearcher @property def convert(self) -> ConvertEntrypoint: diff --git a/modelopt/torch/sparsity/attention_sparsity/model_sparsify.py b/modelopt/torch/sparsity/attention_sparsity/model_sparsify.py index a33938b0576..079750e67af 100644 --- a/modelopt/torch/sparsity/attention_sparsity/model_sparsify.py +++ b/modelopt/torch/sparsity/attention_sparsity/model_sparsify.py @@ -15,6 +15,7 @@ """Main API functions for sparse attention optimization.""" +import warnings from typing import Any import torch @@ -24,7 +25,9 @@ from .calibration import calibrate_sparse_attention from .config import SparseAttentionConfig +from .conversion import get_sparse_attention_mode_state, update_sparse_attention_metadata from .mode import SparseAttentionModeRegistry +from .searcher import SparseAttentionSearcher __all__ = [ "calibrate", @@ -39,6 +42,10 @@ def sparsify( ) -> torch.nn.Module: """Applies sparse attention optimization to the model in-place. + .. deprecated:: 0.49.0 + Use :meth:`mts.sparsify(model, config, forward_loop=forward_loop) + ` instead. + This method performs replacement of attention modules with their sparse counterparts. Args: @@ -101,7 +108,9 @@ def sparsify( This requires the model to be loaded with ``attn_implementation="eager"``. - forward_loop: Reserved for future use. + forward_loop: Callable that forwards calibration data through the model, used when + ``sparse_cfg`` contains a ``"calibration"`` entry. If None, a RULER calibration + dataset is generated from the model's tokenizer. Here are a few examples for correct ``forward_loop`` definitions: @@ -148,12 +157,16 @@ def forward_loop(model) -> float: Returns: A pytorch model which has sparse attention applied and optionally calibrated. """ + warnings.warn( + "mtsa.sparsify() is deprecated; use modelopt.torch.sparsity.sparsify() instead.", + DeprecationWarning, + stacklevel=2, + ) model = apply_mode( model, mode=[("sparse_attention", config)], registry=SparseAttentionModeRegistry ) - - # Calibrate the sparsity ratio of the attention modules - return calibrate(model, config, forward_loop=forward_loop) + SparseAttentionSearcher().search(model, {}, (), {"forward_loop": forward_loop}) + return model def calibrate( @@ -173,5 +186,9 @@ def calibrate( Returns: The calibrated model with optimized sparse attention thresholds. """ + mode_state = get_sparse_attention_mode_state(model) calibrate_sparse_attention(model, config, forward_loop=forward_loop) + # Saving refreshes only the last mode's metadata, which may not be sparse_attention. + if mode_state is not None: + update_sparse_attention_metadata(model, *mode_state) return model diff --git a/modelopt/torch/sparsity/attention_sparsity/searcher.py b/modelopt/torch/sparsity/attention_sparsity/searcher.py new file mode 100644 index 00000000000..ac623f2c238 --- /dev/null +++ b/modelopt/torch/sparsity/attention_sparsity/searcher.py @@ -0,0 +1,43 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Searcher that calibrates sparse attention thresholds.""" + +from modelopt.torch.opt.searcher import BaseSearcher, SearchStateDict + +from .calibration import calibrate_sparse_attention +from .conversion import get_sparse_attention_mode_state + +__all__ = ["SparseAttentionSearcher"] + + +class SparseAttentionSearcher(BaseSearcher): + """Calibrate sparse attention thresholds with the forward loop of the search config. + + Calibration is configured by ``sparse_cfg["calibration"]`` of the model's ``sparse_attention`` + mode config and is skipped without it. Without a ``forward_loop`` or ``data_loader``, a RULER + calibration dataset is generated from the model's tokenizer. + """ + + @property + def default_state_dict(self) -> SearchStateDict: + """Return default state dict.""" + return {} + + def run_search(self) -> None: + """Calibrate the sparse attention modules of the model.""" + mode_state = get_sparse_attention_mode_state(self.model) + assert mode_state is not None, "Model has no sparse_attention mode to calibrate!" + calibrate_sparse_attention(self.model, mode_state[0], forward_loop=self.forward_loop) diff --git a/modelopt/torch/sparsity/sparsification.py b/modelopt/torch/sparsity/sparsification.py new file mode 100644 index 00000000000..e6fd914fe90 --- /dev/null +++ b/modelopt/torch/sparsity/sparsification.py @@ -0,0 +1,112 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""High-level API to sparsify the weights and the attention of a model.""" + +from typing import Any + +from torch import nn + +from modelopt.torch.opt.conversion import apply_mode +from modelopt.torch.opt.mode import ModeLike, get_mode_config +from modelopt.torch.opt.searcher import ForwardLoop, SearchConfig + +from .attention_sparsity.config import SparseAttentionConfig +from .attention_sparsity.conversion import is_attn_sparsified +from .attention_sparsity.mode import SparseAttentionModeRegistry +from .attention_sparsity.model_sparsify import calibrate +from .weight_sparsity import sparsification as weight_sparsification +from .weight_sparsity.mode import SparsityModeRegistry +from .weight_sparsity.module import SparseModule + +__all__ = ["calibrate", "export", "sparsify"] + +_REGISTRIES = (SparsityModeRegistry, SparseAttentionModeRegistry) + + +def sparsify( + model: nn.Module, + mode: ModeLike | dict[str, Any] | SparseAttentionConfig, + config: SearchConfig | None = None, + *, + forward_loop: ForwardLoop | None = None, +) -> nn.Module: + """Sparsify the weights and/or the attention of a model in-place. + + Example:: + + model = mts.sparsify(model, "sparsegpt", config={"data_loader": data_loader}) + model = mts.sparsify(model, mts.SKIP_SOFTMAX_CALIB) # calibrates on RULER data + + Args: + model: The model to sparsify. + mode: A mode, a list of modes, or a list of ``(mode, mode_config)`` tuples. Modes are + applied and then searched or calibrated one at a time, in order: + + * ``"sparse_magnitude"``: 2:4 weight sparsity by weight magnitude. + * ``"sparsegpt"``: 2:4 weight sparsity with SparseGPT weight updates. + * ``"sparse_attention"``: sparse attention configured by a + :class:`SparseAttentionConfig ` + such as ``mts.SKIP_SOFTMAX_CALIB``, which may also be passed directly as ``mode``. + config: Search options such as ``forward_loop``, ``data_loader``, ``collect_func`` and + ``verbose``, plus algorithm-specific ones such as SparseGPT's ``device``. Each mode + receives the options its algorithm accepts. + forward_loop: Shorthand for ``config["forward_loop"]``. Sparse attention calibration + accepts a ``forward_loop`` or ``data_loader`` only with a decode target sparsity of 0; + without either, it calibrates on generated RULER data. + + Returns: + The sparsified model. + """ + if isinstance(mode, (dict, SparseAttentionConfig)): + mode = [("sparse_attention", dict(mode))] + config = dict(config or {}) + if forward_loop is not None: + if config.get("forward_loop") is not None: + raise ValueError("Pass `forward_loop` either as an argument or in `config`, not both.") + config["forward_loop"] = forward_loop + if config.get("forward_loop") is not None and config.get("data_loader") is not None: + raise ValueError("Pass either `forward_loop` or `data_loader`, not both.") + + # Validate all modes and config keys before modifying the model. + modes = get_mode_config(mode) + searchers = [] + for m, mode_config in modes: + registry = next((r for r in _REGISTRIES if m in r), None) + if registry is None: + raise ValueError(f"{m} is not a sparsity mode.") + registry[m].config_class(**mode_config) + searchers.append(registry[m].search_algorithm()) + supported_keys = set().union(*(s.default_search_config for s in searchers)) + if unsupported_keys := config.keys() - supported_keys: + raise ValueError(f"Unsupported config keys: {sorted(unsupported_keys)}") + + # The next mode freezes the metadata of the previous one, so each mode is searched right away. + for (m, mode_config), searcher in zip(modes, searchers): + model = apply_mode(model, [(m, mode_config)]) + search_keys = searcher.default_search_config.keys() + searcher.search(model, {}, (), {k: v for k, v in config.items() if k in search_keys}) + return model + + +def export(model: nn.Module) -> nn.Module: + """Export a sparse model to a regular model by folding the weight sparsity masks into the weights. + + Export after fine-tuning: the masks are no longer enforced afterwards. Sparse attention needs no + export and stays applied. + """ + if is_attn_sparsified(model) and not any(isinstance(m, SparseModule) for m in model.modules()): + return model + return weight_sparsification.export(model) diff --git a/modelopt/torch/sparsity/weight_sparsity/mode.py b/modelopt/torch/sparsity/weight_sparsity/mode.py index db7e1f33253..a963007d56c 100644 --- a/modelopt/torch/sparsity/weight_sparsity/mode.py +++ b/modelopt/torch/sparsity/weight_sparsity/mode.py @@ -117,7 +117,7 @@ def config_class(self) -> type[ModeloptBaseConfig]: @property def next_modes(self) -> set[str] | None: """Specifies the next modes for the mode.""" - return {"export_sparse", "kd_loss", "quantize"} + return {"export_sparse", "kd_loss", "quantize", "sparse_attention"} @property def export_mode(self) -> str | None: diff --git a/tests/_test_utils/torch/sparsity/sparse_attention_common.py b/tests/_test_utils/torch/sparsity/sparse_attention_common.py index 58d711affa2..9efa7a6d8fc 100644 --- a/tests/_test_utils/torch/sparsity/sparse_attention_common.py +++ b/tests/_test_utils/torch/sparsity/sparse_attention_common.py @@ -19,7 +19,7 @@ import torch.nn as nn import modelopt.torch.opt as mto -import modelopt.torch.sparsity.attention_sparsity as sparse_attn +import modelopt.torch.sparsity as mts from modelopt.torch.sparsity.attention_sparsity.sparse_attention import SparseAttentionModule @@ -141,7 +141,7 @@ def forward_loop(model): model(batch) # Apply sparse attention - model = sparse_attn.sparsify(model, config, forward_loop=forward_loop) + model = mts.sparsify(model, config, forward_loop=forward_loop) # Verify sparse attention modules were inserted assert any(isinstance(m, SparseAttentionModule) for m in model.modules()), ( diff --git a/tests/examples/diffusers/sparsity/test_sparsity.py b/tests/examples/diffusers/sparsity/test_sparsity.py index 1727ee39b7f..f296437d55f 100644 --- a/tests/examples/diffusers/sparsity/test_sparsity.py +++ b/tests/examples/diffusers/sparsity/test_sparsity.py @@ -30,7 +30,7 @@ from _test_utils.examples.run_command import run_example_command from diffusers import AutoencoderKLWan, WanPipeline -import modelopt.torch.sparsity.attention_sparsity as mtsa +import modelopt.torch.sparsity as mts from modelopt.torch.export import export_hf_checkpoint from modelopt.torch.sparsity.attention_sparsity.sparse_attention import SparseAttentionModule @@ -183,7 +183,7 @@ def test_wan22_calibrated_export(tiny_wan22_path, tmp_path): } config = {"sparse_cfg": sparse_cfg} for transformer in [pipe.transformer, pipe.transformer_2]: - mtsa.sparsify(transformer, config) + mts.sparsify(transformer, config) # Inject calibration params (simulating a successful log-space calibration). test_log_a = math.log(1.5) diff --git a/tests/gpu/torch/kernels/sparsity/attention/test_triton_fa_skip_softmax.py b/tests/gpu/torch/kernels/sparsity/attention/test_triton_fa_skip_softmax.py index 033952d34eb..133444ee24a 100644 --- a/tests/gpu/torch/kernels/sparsity/attention/test_triton_fa_skip_softmax.py +++ b/tests/gpu/torch/kernels/sparsity/attention/test_triton_fa_skip_softmax.py @@ -20,7 +20,7 @@ from conftest import make_varlen_meta from transformers import AutoModelForCausalLM, AutoTokenizer -import modelopt.torch.sparsity.attention_sparsity as mtsa +import modelopt.torch.sparsity as mts from modelopt.torch.kernels.common.attention import IS_AVAILABLE as TRITON_KERNEL_AVAILABLE from modelopt.torch.sparsity.attention_sparsity.methods.flash_skip_softmax import FlashSkipSoftmax @@ -246,10 +246,10 @@ def test_triton_matches_pytorch_reference(self): @pytest.mark.skipif(not TRITON_KERNEL_AVAILABLE, reason="Need CUDA + triton") class TestSkipSoftmaxHFIntegration: - """HF integration for skip-softmax via mtsa.sparsify().""" + """HF integration for skip-softmax via mts.sparsify().""" def test_skip_softmax_via_sparsify(self, tiny_llama_dir): - """mtsa.sparsify() with triton_skip_softmax produces finite logits.""" + """mts.sparsify() with triton_skip_softmax produces finite logits.""" tok = AutoTokenizer.from_pretrained(tiny_llama_dir) if tok.pad_token_id is None: tok.pad_token_id = tok.eos_token_id @@ -267,13 +267,13 @@ def test_skip_softmax_via_sparsify(self, tiny_llama_dir): logits_dense = model_dense(input_ids=ids).logits del model_dense - # Skip-softmax via mtsa.sparsify() + # Skip-softmax via mts.sparsify() model_skip = AutoModelForCausalLM.from_pretrained( tiny_llama_dir, torch_dtype=torch.bfloat16, device_map="cuda", ) - mtsa.sparsify(model_skip, mtsa.SKIP_SOFTMAX_TRITON_DEFAULT) + mts.sparsify(model_skip, mts.SKIP_SOFTMAX_TRITON_DEFAULT) model_skip.eval() with torch.no_grad(): logits_skip = model_skip(input_ids=ids).logits diff --git a/tests/gpu/torch/kernels/sparsity/attention/test_triton_fa_sparse_nm.py b/tests/gpu/torch/kernels/sparsity/attention/test_triton_fa_sparse_nm.py index 651aeab4116..538e6597def 100644 --- a/tests/gpu/torch/kernels/sparsity/attention/test_triton_fa_sparse_nm.py +++ b/tests/gpu/torch/kernels/sparsity/attention/test_triton_fa_sparse_nm.py @@ -22,7 +22,7 @@ from conftest import make_qkv, make_varlen_meta from transformers import AutoModelForCausalLM, AutoTokenizer -import modelopt.torch.sparsity.attention_sparsity as mtsa +import modelopt.torch.sparsity as mts from modelopt.torch.kernels.common.attention import IS_AVAILABLE as TRITON_KERNEL_AVAILABLE if TRITON_KERNEL_AVAILABLE: @@ -417,7 +417,7 @@ def test_sparse_disabled_matches_dense(self): assert torch.equal(out_dense, out_n0) def test_sparse_nm_via_sparsify(self, tiny_llama_dir): - """mtsa.sparsify() with N:M sparse softmax produces finite logits that differ from dense.""" + """mts.sparsify() with N:M sparse softmax produces finite logits that differ from dense.""" tok = AutoTokenizer.from_pretrained(tiny_llama_dir) if tok.pad_token_id is None: tok.pad_token_id = tok.eos_token_id @@ -435,7 +435,7 @@ def test_sparse_nm_via_sparsify(self, tiny_llama_dir): logits_dense = model_dense(input_ids=ids).logits del model_dense - # Sparse via mtsa.sparsify() with dense_recent_tokens=0 to force sparsity on all tiles + # Sparse via mts.sparsify() with dense_recent_tokens=0 to force sparsity on all tiles sparse_cfg = { "sparse_cfg": { "*attn*": { @@ -455,7 +455,7 @@ def test_sparse_nm_via_sparsify(self, tiny_llama_dir): torch_dtype=torch.bfloat16, device_map="cuda", ) - mtsa.sparsify(model_sparse, sparse_cfg) + mts.sparsify(model_sparse, sparse_cfg) assert model_sparse.config._attn_implementation == "modelopt_triton" model_sparse.eval() with torch.no_grad(): diff --git a/tests/gpu/torch/sparsity/attention_sparsity/test_attention_sparsity_gpu.py b/tests/gpu/torch/sparsity/attention_sparsity/test_attention_sparsity_gpu.py index ce762267a23..ab26a810d7e 100644 --- a/tests/gpu/torch/sparsity/attention_sparsity/test_attention_sparsity_gpu.py +++ b/tests/gpu/torch/sparsity/attention_sparsity/test_attention_sparsity_gpu.py @@ -27,7 +27,7 @@ sparsify_model_and_forward, ) -import modelopt.torch.sparsity.attention_sparsity as sparse_attn +import modelopt.torch.sparsity as mts class TestAttentionSparsityGPU: @@ -86,7 +86,7 @@ def test_different_dtypes(self, dtype): def test_backward_pass(self): """Test that gradients flow correctly through sparse attention.""" model = SimpleAttentionModel(hidden_size=128, num_heads=4).to(self.device) - model = sparse_attn.sparsify(model, FLASH_SKIP_SOFTMAX_DEFAULT_CFG) + model = mts.sparsify(model, FLASH_SKIP_SOFTMAX_DEFAULT_CFG) # Enable training mode model.train() @@ -114,7 +114,7 @@ def test_backward_pass(self): def test_various_sequence_lengths(self, seq_len): """Test sparse attention with various sequence lengths.""" model = SimpleAttentionModel(hidden_size=128, num_heads=4).to(self.device) - model = sparse_attn.sparsify(model, FLASH_SKIP_SOFTMAX_DEFAULT_CFG) + model = mts.sparsify(model, FLASH_SKIP_SOFTMAX_DEFAULT_CFG) x = model.get_input(hidden_size=128, seq_len=seq_len, batch_size=1).to(self.device) @@ -129,7 +129,7 @@ def test_various_sequence_lengths(self, seq_len): def test_various_batch_sizes(self, batch_size): """Test sparse attention with various batch sizes.""" model = SimpleTransformerEncoderLayer(d_model=128, nhead=4).to(self.device) - model = sparse_attn.sparsify(model, FLASH_SKIP_SOFTMAX_DEFAULT_CFG) + model = mts.sparsify(model, FLASH_SKIP_SOFTMAX_DEFAULT_CFG) x = model.get_input(d_model=128, seq_len=64, batch_size=batch_size).to(self.device) diff --git a/tests/gpu/torch/sparsity/attention_sparsity/test_calibration_gpu.py b/tests/gpu/torch/sparsity/attention_sparsity/test_calibration_gpu.py index 39f9837667a..b8edfe44c77 100644 --- a/tests/gpu/torch/sparsity/attention_sparsity/test_calibration_gpu.py +++ b/tests/gpu/torch/sparsity/attention_sparsity/test_calibration_gpu.py @@ -20,7 +20,7 @@ from _test_utils.torch.sparsity.sparse_attention_common import SimpleTransformerEncoderLayer import modelopt.torch.opt as mto -from modelopt.torch.sparsity.attention_sparsity import sparsify +from modelopt.torch.sparsity import sparsify from modelopt.torch.sparsity.attention_sparsity.calibration import RulerDatasetBuilder from modelopt.torch.sparsity.attention_sparsity.sparse_attention import SparseAttentionModule diff --git a/tests/gpu/torch/sparsity/attention_sparsity/test_integration_gpu.py b/tests/gpu/torch/sparsity/attention_sparsity/test_integration_gpu.py index 80408024a2e..55e03a1eb53 100644 --- a/tests/gpu/torch/sparsity/attention_sparsity/test_integration_gpu.py +++ b/tests/gpu/torch/sparsity/attention_sparsity/test_integration_gpu.py @@ -20,7 +20,7 @@ from _test_utils.torch.transformers_models import create_tiny_llama_dir from transformers import AutoModelForCausalLM, AutoTokenizer -import modelopt.torch.sparsity.attention_sparsity as sparse_attn +import modelopt.torch.sparsity as mts from modelopt.torch.sparsity.attention_sparsity import SparseAttentionConfig from modelopt.torch.sparsity.attention_sparsity.sparse_attention import SparseAttentionModule @@ -75,7 +75,7 @@ def test_load_and_sparsify(self, tinyllama_model): }, ) - sparse_model = sparse_attn.sparsify(model, config) + sparse_model = mts.sparsify(model, config) # Verify sparse attention modules were added sparse_count = sum( @@ -101,7 +101,7 @@ def test_forward_prefill(self, tinyllama_model, tinyllama_tokenizer): }, ) - sparse_model = sparse_attn.sparsify(model, config) + sparse_model = mts.sparsify(model, config) # Create prefill input (seq_len > 1) test_text = "Once upon a time in a land far away" @@ -134,7 +134,7 @@ def test_forward_decode(self, tinyllama_model): }, ) - sparse_model = sparse_attn.sparsify(model, config) + sparse_model = mts.sparsify(model, config) # Create decode input (seq_len = 1) vocab_size = model.config.vocab_size @@ -174,7 +174,7 @@ def test_gqa_attention(self, tinyllama_model): }, ) - sparse_model = sparse_attn.sparsify(model, sparse_config) + sparse_model = mts.sparsify(model, sparse_config) # Test forward pass with GQA input_ids = torch.randint(0, config.vocab_size, (1, 32), device="cuda") diff --git a/tests/gpu/torch/sparsity/attention_sparsity/test_triton_calibration_gpu.py b/tests/gpu/torch/sparsity/attention_sparsity/test_triton_calibration_gpu.py index 949e67b2cd8..c83a3aba707 100644 --- a/tests/gpu/torch/sparsity/attention_sparsity/test_triton_calibration_gpu.py +++ b/tests/gpu/torch/sparsity/attention_sparsity/test_triton_calibration_gpu.py @@ -29,7 +29,7 @@ from _test_utils.torch.transformers_models import create_tiny_llama_dir from transformers import AutoModelForCausalLM -import modelopt.torch.sparsity.attention_sparsity as mtsa +import modelopt.torch.sparsity as mts from modelopt.torch.kernels.common.attention import IS_AVAILABLE as TRITON_KERNEL_AVAILABLE from modelopt.torch.kernels.common.attention.hf_triton_attention import triton_attention_forward from modelopt.torch.sparsity.attention_sparsity.config import SKIP_SOFTMAX_TRITON_CALIB @@ -111,7 +111,7 @@ def test_calibrated_model_inference(self, tiny_llama_dir): config["sparse_cfg"]["calibration"]["target_sparse_ratio"] = {"prefill": 0.5} forward_loop = _make_forward_loop(model.config.vocab_size) - sparse_model = mtsa.sparsify(model, config, forward_loop=forward_loop) + sparse_model = mts.sparsify(model, config, forward_loop=forward_loop) assert sparse_model.config._attn_implementation == "modelopt_triton" sparse_model.eval() diff --git a/tests/gpu/torch/sparsity/attention_sparsity/test_wan22_skip_softmax.py b/tests/gpu/torch/sparsity/attention_sparsity/test_wan22_skip_softmax.py index 814a03ba564..841e12162e9 100644 --- a/tests/gpu/torch/sparsity/attention_sparsity/test_wan22_skip_softmax.py +++ b/tests/gpu/torch/sparsity/attention_sparsity/test_wan22_skip_softmax.py @@ -17,7 +17,7 @@ Uses ``create_tiny_wan22_pipeline_dir`` (matches diffusers' own tiny Wan 2.2 test config): dual 2-layer transformer, tiny VAE, tiny T5 text encoder. -The full ``WanPipeline`` is loaded, sparsified with ``mtsa.sparsify``, and +The full ``WanPipeline`` is loaded, sparsified with ``mts.sparsify``, and run end-to-end with a 2-step denoising loop — asserting no NaN/Inf in the pipeline output. """ @@ -39,7 +39,7 @@ from modelopt.torch.kernels.common.attention import IS_AVAILABLE as TRITON_KERNEL_AVAILABLE if TRITON_KERNEL_AVAILABLE: - import modelopt.torch.sparsity.attention_sparsity as mtsa + import modelopt.torch.sparsity as mts from modelopt.torch.sparsity.attention_sparsity.methods.triton_skip_softmax import ( TritonSkipSoftmaxMethod, ) @@ -92,8 +92,8 @@ def _skip_softmax_cfg(threshold=0.03125): def _sparsify_both_transformers(pipe, cfg): """Apply sparsify to both transformer and transformer_2 (14B-style dual).""" - mtsa.sparsify(pipe.transformer, cfg) - mtsa.sparsify(pipe.transformer_2, cfg) + mts.sparsify(pipe.transformer, cfg) + mts.sparsify(pipe.transformer_2, cfg) def _run_pipe(pipe, seed=0): diff --git a/tests/unit/torch/sparsity/attention_sparsity/test_calibrator_fitting.py b/tests/unit/torch/sparsity/attention_sparsity/test_calibrator_fitting.py index c7c6f56928b..f818f249082 100644 --- a/tests/unit/torch/sparsity/attention_sparsity/test_calibrator_fitting.py +++ b/tests/unit/torch/sparsity/attention_sparsity/test_calibrator_fitting.py @@ -27,7 +27,7 @@ from _test_utils.torch.sparsity.sparse_attention_common import SimpleAttentionModel -from modelopt.torch.sparsity.attention_sparsity import sparsify +from modelopt.torch.sparsity import sparsify from modelopt.torch.sparsity.attention_sparsity.calibration.calibrator import ( DynamicThresholdCalibrator, ) diff --git a/tests/unit/torch/sparsity/attention_sparsity/test_sparse_attention_calibration.py b/tests/unit/torch/sparsity/attention_sparsity/test_sparse_attention_calibration.py index e14d498a415..cc9228b900f 100644 --- a/tests/unit/torch/sparsity/attention_sparsity/test_sparse_attention_calibration.py +++ b/tests/unit/torch/sparsity/attention_sparsity/test_sparse_attention_calibration.py @@ -26,7 +26,7 @@ from _test_utils.torch.sparsity.sparse_attention_common import SimpleAttentionModel from pydantic import ValidationError -from modelopt.torch.sparsity.attention_sparsity import sparsify +from modelopt.torch.sparsity import sparsify from modelopt.torch.sparsity.attention_sparsity.calibration import ( DynamicThresholdCalibrator, RulerDatasetBuilder, diff --git a/tests/unit/torch/sparsity/attention_sparsity/test_sparse_attention_conversion.py b/tests/unit/torch/sparsity/attention_sparsity/test_sparse_attention_conversion.py index e70f2b7de65..c26482472ac 100644 --- a/tests/unit/torch/sparsity/attention_sparsity/test_sparse_attention_conversion.py +++ b/tests/unit/torch/sparsity/attention_sparsity/test_sparse_attention_conversion.py @@ -23,6 +23,7 @@ from unittest.mock import MagicMock, patch +import torch import torch.nn as nn from _test_utils.torch.sparsity.sparse_attention_common import ( FLASH_SKIP_SOFTMAX_DEFAULT_CFG, @@ -31,6 +32,7 @@ ) import modelopt.torch.opt as mto +import modelopt.torch.sparsity as mts import modelopt.torch.sparsity.attention_sparsity as sparse_attn from modelopt.torch.sparsity.attention_sparsity.config import SKIP_SOFTMAX_CALIB_SPARSE24 from modelopt.torch.sparsity.attention_sparsity.conversion import ( @@ -41,6 +43,7 @@ print_sparse_attention_summary, ) from modelopt.torch.sparsity.attention_sparsity.sparse_attention import SparseAttentionModule +from modelopt.torch.sparsity.weight_sparsity.module import SparseModule class TestSparseAttentionReplacement: @@ -57,7 +60,7 @@ def test_basic_replacement(self): assert original_attention_count > 0 # Apply sparse attention - sparse_model = sparse_attn.sparsify(model, FLASH_SKIP_SOFTMAX_DEFAULT_CFG) + sparse_model = mts.sparsify(model, FLASH_SKIP_SOFTMAX_DEFAULT_CFG) # Count sparse attention modules sparse_attention_count = sum( @@ -85,7 +88,7 @@ def test_pattern_based_replacement(self): }, } - sparse_model = sparse_attn.sparsify(model, config) + sparse_model = mts.sparsify(model, config) # Verify sparse modules exist has_sparse = any(isinstance(m, SparseAttentionModule) for m in sparse_model.modules()) @@ -113,7 +116,7 @@ def filter_func(name): }, } - sparse_model = sparse_attn.sparsify(model, config) + sparse_model = mts.sparsify(model, config) has_sparse = any(isinstance(m, SparseAttentionModule) for m in sparse_model.modules()) assert has_sparse @@ -132,13 +135,13 @@ def test_no_matching_modules(self): } # Should not error, even with no matches - sparse_attn.sparsify(model, config) + mts.sparsify(model, config) def test_disable_enable_functions(self): """Test disable/enable utility functions.""" model = SimpleAttentionModel() - model = sparse_attn.sparsify(model, FLASH_SKIP_SOFTMAX_DEFAULT_CFG) + model = mts.sparsify(model, FLASH_SKIP_SOFTMAX_DEFAULT_CFG) # Disable all disable_sparse_attention(model, "*") @@ -155,7 +158,7 @@ def test_disable_enable_functions(self): def test_print_sparse_attention_summary(self, capsys): """Test print_sparse_attention_summary function.""" model = SimpleAttentionModel() - model = sparse_attn.sparsify(model, FLASH_SKIP_SOFTMAX_DEFAULT_CFG) + model = mts.sparsify(model, FLASH_SKIP_SOFTMAX_DEFAULT_CFG) # Print summary print_sparse_attention_summary(model) @@ -169,7 +172,7 @@ def test_restore_sparse_attention_model(self): """Test save/restore via modelopt_state.""" # Create and sparsify original model model_orig = SimpleAttentionModel() - model_orig = sparse_attn.sparsify(model_orig, FLASH_SKIP_SOFTMAX_DEFAULT_CFG) + model_orig = mts.sparsify(model_orig, FLASH_SKIP_SOFTMAX_DEFAULT_CFG) # Save state state_dict = mto.modelopt_state(model_orig) @@ -193,7 +196,7 @@ def test_restore_keeps_calibration_params(self, tmp_path): calibration_params = {"prefill": {"a": 3.14, "b": 7.5}, "decode": {"a": 0.5, "b": 9.0}} target_sparse_ratio = {"prefill": 0.4, "decode": 0.6} - model = sparse_attn.sparsify(SimpleAttentionModel(), FLASH_SKIP_SOFTMAX_DEFAULT_CFG) + model = mts.sparsify(SimpleAttentionModel(), FLASH_SKIP_SOFTMAX_DEFAULT_CFG) for module in model.modules(): if isinstance(module, SparseAttentionModule): module._sparse_method_instance.calibration_params = calibration_params @@ -212,9 +215,94 @@ def test_restore_keeps_calibration_params(self, tmp_path): assert method.target_sparse_ratio == target_sparse_ratio assert module.get_threshold_info()["type"] == "dynamic_calibrated" + def test_calibrate_after_later_mode_survives_restore(self): + """calibrate() persists its results even when sparse_attention is not the last mode.""" + calibration_params = {"prefill": {"a": 3.14, "b": 7.5}} + + def fake_calibration(model, config, forward_loop=None): + for module in model.modules(): + if isinstance(module, SparseAttentionModule): + module._sparse_method_instance.calibration_params = calibration_params + + model = mts.sparsify(SimpleAttentionModel(), FLASH_SKIP_SOFTMAX_DEFAULT_CFG) + model = mts.sparsify(model, "sparse_magnitude") + with patch( + "modelopt.torch.sparsity.attention_sparsity.model_sparsify.calibrate_sparse_attention", + side_effect=fake_calibration, + ): + sparse_attn.calibrate(model, FLASH_SKIP_SOFTMAX_DEFAULT_CFG) + + model_restored = mto.restore_from_modelopt_state( + SimpleAttentionModel(), mto.modelopt_state(model) + ) + sparse_modules = [ + m for m in model_restored.modules() if isinstance(m, SparseAttentionModule) + ] + assert sparse_modules + for module in sparse_modules: + assert module._sparse_method_instance.calibration_params == calibration_params + + def test_calibrate_container_of_converted_models(self): + """calibrate() works on a module holding several separately converted models.""" + models = nn.ModuleList( + mts.sparsify(SimpleAttentionModel(), FLASH_SKIP_SOFTMAX_DEFAULT_CFG) for _ in range(2) + ) + sparse_attn.calibrate(models, FLASH_SKIP_SOFTMAX_DEFAULT_CFG) + + @pytest.mark.parametrize("weight_first", [True, False]) + def test_chain_with_weight_sparsity(self, weight_first): + """Weight and attention sparsity in one call, either order; export folds only weight masks.""" + modes = [("sparsegpt", {}), ("sparse_attention", FLASH_SKIP_SOFTMAX_DEFAULT_CFG)] + x = SimpleAttentionModel.get_input() + model = mts.sparsify( + SimpleAttentionModel().eval(), + modes if weight_first else modes[::-1], + config={"device": "cpu"}, # SparseGPT-only option + forward_loop=lambda m: m(x), + ) + assert (model.fc.weight == 0).float().mean().item() == 0.5 + with torch.no_grad(): + output = model(x) + + model_restored = mto.restore_from_modelopt_state( + SimpleAttentionModel(), mto.modelopt_state(model) + ).eval() + model_restored.load_state_dict(model.state_dict()) + with torch.no_grad(): + assert torch.allclose(model_restored(x), output) + + mts.export(model) + assert not any(isinstance(m, SparseModule) for m in model.modules()) + assert any(isinstance(m, SparseAttentionModule) for m in model.modules()) + assert mts.export(model) is model # only sparse attention is left, which needs no export + with torch.no_grad(): + assert torch.allclose(model(x), output) + + @pytest.mark.parametrize( + ("mode", "config", "forward_loop"), + [ + (FLASH_SKIP_SOFTMAX_DEFAULT_CFG, {"devcie": "cpu"}, None), + (FLASH_SKIP_SOFTMAX_DEFAULT_CFG, {"forward_loop": print}, print), + (FLASH_SKIP_SOFTMAX_DEFAULT_CFG, {"data_loader": [None]}, print), + ("quantize", None, None), + ], + ) + def test_sparsify_rejects_invalid_input_before_converting(self, mode, config, forward_loop): + model = SimpleAttentionModel() + with pytest.raises(ValueError): + mts.sparsify(model, mode, config, forward_loop=forward_loop) + assert not mto.ModeloptStateManager.is_converted(model) + + def test_deprecated_sparsify(self): + """The deprecated mtsa.sparsify warns and matches mts.sparsify.""" + with pytest.deprecated_call(): + model = sparse_attn.sparsify(SimpleAttentionModel(), FLASH_SKIP_SOFTMAX_DEFAULT_CFG) + model_ref = mts.sparsify(SimpleAttentionModel(), FLASH_SKIP_SOFTMAX_DEFAULT_CFG) + assert mto.modelopt_state(model) == mto.modelopt_state(model_ref) + def test_restore_keeps_disabled_modules(self): """Modules disabled after sparsify stay disabled after restore.""" - model = sparse_attn.sparsify(SimpleAttentionModel(), FLASH_SKIP_SOFTMAX_DEFAULT_CFG) + model = mts.sparsify(SimpleAttentionModel(), FLASH_SKIP_SOFTMAX_DEFAULT_CFG) disable_sparse_attention(model, "*attention*") model_restored = mto.restore_from_modelopt_state( @@ -229,7 +317,7 @@ def test_restore_keeps_disabled_modules(self): def test_restore_legacy_state_without_calibration_keys(self): """Metadata saved before calibration state was persisted still restores.""" - model = sparse_attn.sparsify(SimpleAttentionModel(), FLASH_SKIP_SOFTMAX_DEFAULT_CFG) + model = mts.sparsify(SimpleAttentionModel(), FLASH_SKIP_SOFTMAX_DEFAULT_CFG) modelopt_state = mto.modelopt_state(model) for _, mode_state in modelopt_state["modelopt_state_dict"]: for module_state in mode_state["metadata"]["sparse_attention_state"].values(): @@ -262,7 +350,7 @@ def test_get_stats_with_stats_manager(self): }, } - sparse_model = sparse_attn.sparsify(model, config) + sparse_model = mts.sparsify(model, config) # Find sparse module sparse_module = None @@ -298,7 +386,7 @@ def test_get_stats_without_stats_manager(self): }, } - sparse_model = sparse_attn.sparsify(model, config) + sparse_model = mts.sparsify(model, config) # Find sparse module for module in sparse_model.modules(): @@ -411,14 +499,14 @@ class TestExportSparseAttentionConfig: def test_returns_none_without_calibration(self): """When no module has calibration params, returns None.""" model = SimpleAttentionModel() - model = sparse_attn.sparsify(model, FLASH_SKIP_SOFTMAX_DEFAULT_CFG) + model = mts.sparsify(model, FLASH_SKIP_SOFTMAX_DEFAULT_CFG) out = export_sparse_attention_config(model) assert out is None def test_exports_when_calibration_present(self): """Calibration params are reflected in the exported config.""" model = SimpleAttentionModel() - model = sparse_attn.sparsify(model, FLASH_SKIP_SOFTMAX_DEFAULT_CFG) + model = mts.sparsify(model, FLASH_SKIP_SOFTMAX_DEFAULT_CFG) for module in model.modules(): if isinstance(module, SparseAttentionModule): @@ -472,7 +560,7 @@ def test_exports_sparse_softmax_metadata(self): "modelopt.torch.kernels.sparsity.attention.register_triton_attention", MagicMock(return_value=True), ): - model = sparse_attn.sparsify(model, config) + model = mts.sparsify(model, config) out = export_sparse_attention_config(model) @@ -491,7 +579,7 @@ def test_exports_calibrated_skip_softmax_with_sparse_softmax_overlay(self): config = copy.deepcopy(SKIP_SOFTMAX_CALIB_SPARSE24) config["sparse_cfg"].pop("calibration") config["sparse_cfg"]["*attention*"] = config["sparse_cfg"].pop("*attn*") - model = sparse_attn.sparsify(model, config) + model = mts.sparsify(model, config) for module in model.modules(): if isinstance(module, SparseAttentionModule): diff --git a/tests/unit/torch/sparsity/attention_sparsity/test_threshold_info.py b/tests/unit/torch/sparsity/attention_sparsity/test_threshold_info.py index 00622d365d4..78849e1652f 100644 --- a/tests/unit/torch/sparsity/attention_sparsity/test_threshold_info.py +++ b/tests/unit/torch/sparsity/attention_sparsity/test_threshold_info.py @@ -21,7 +21,7 @@ from _test_utils.torch.sparsity.sparse_attention_common import SimpleAttentionModel -from modelopt.torch.sparsity.attention_sparsity import sparsify +from modelopt.torch.sparsity import sparsify from modelopt.torch.sparsity.attention_sparsity.methods.flash_skip_softmax import FlashSkipSoftmax from modelopt.torch.sparsity.attention_sparsity.sparse_attention import SparseAttentionModule diff --git a/tests/unit/torch/sparsity/attention_sparsity/test_vsa.py b/tests/unit/torch/sparsity/attention_sparsity/test_vsa.py index 09621656e8c..af53ca5bc77 100644 --- a/tests/unit/torch/sparsity/attention_sparsity/test_vsa.py +++ b/tests/unit/torch/sparsity/attention_sparsity/test_vsa.py @@ -322,7 +322,7 @@ def test_vsa_config_defaults(self): from _test_utils.torch.sparsity.sparse_attention_common import SimpleAttentionModel import modelopt.torch.opt as mto -import modelopt.torch.sparsity.attention_sparsity as sparse_attn +import modelopt.torch.sparsity as mts from modelopt.torch.sparsity.attention_sparsity.sparse_attention import SparseAttentionModule VSA_TEST_CFG = { @@ -344,7 +344,7 @@ class TestVSASparsifyIntegration: def test_sparsify_creates_sparse_modules(self): """sparsify() with VSA config replaces attention modules.""" model = SimpleAttentionModel() - sparse_model = sparse_attn.sparsify(model, VSA_TEST_CFG) + sparse_model = mts.sparsify(model, VSA_TEST_CFG) sparse_modules = [m for m in sparse_model.modules() if isinstance(m, SparseAttentionModule)] assert len(sparse_modules) > 0 @@ -352,7 +352,7 @@ def test_sparsify_creates_sparse_modules(self): def test_sparse_module_has_vsa_method(self): """Replaced modules are configured with VSA method.""" model = SimpleAttentionModel() - sparse_model = sparse_attn.sparsify(model, VSA_TEST_CFG) + sparse_model = mts.sparsify(model, VSA_TEST_CFG) for module in sparse_model.modules(): if isinstance(module, SparseAttentionModule): @@ -364,7 +364,7 @@ def test_sparse_module_has_vsa_method(self): def test_enable_disable(self): """Enable/disable works on VSA sparse modules.""" model = SimpleAttentionModel() - sparse_model = sparse_attn.sparsify(model, VSA_TEST_CFG) + sparse_model = mts.sparsify(model, VSA_TEST_CFG) for module in sparse_model.modules(): if isinstance(module, SparseAttentionModule): @@ -377,7 +377,7 @@ def test_enable_disable(self): def test_threshold_info(self): """VSA sparse modules report correct threshold info.""" model = SimpleAttentionModel() - sparse_model = sparse_attn.sparsify(model, VSA_TEST_CFG) + sparse_model = mts.sparsify(model, VSA_TEST_CFG) for module in sparse_model.modules(): if isinstance(module, SparseAttentionModule): @@ -388,7 +388,7 @@ def test_threshold_info(self): def test_save_restore(self): """VSA modelopt_state can be saved and restored.""" model = SimpleAttentionModel() - sparse_model = sparse_attn.sparsify(model, VSA_TEST_CFG) + sparse_model = mts.sparsify(model, VSA_TEST_CFG) state = mto.modelopt_state(sparse_model) @@ -416,7 +416,7 @@ def test_pattern_matching(self): "default": {"enable": False}, }, } - sparse_model = sparse_attn.sparsify(model, config) + sparse_model = mts.sparsify(model, config) # No modules should have VSA enabled for module in sparse_model.modules(): @@ -431,7 +431,7 @@ def test_forward_patches_sdpa(self): ImportError — proving the interception works. """ model = SimpleAttentionModel() - sparse_model = sparse_attn.sparsify(model, VSA_TEST_CFG) + sparse_model = mts.sparsify(model, VSA_TEST_CFG) # Set video_shape so metadata can be computed. # seq_len=64, video_shape (4,4,4) -> T*H*W=64