From 3a04c0205f85d0f61721c4291b03d051ec93e34c Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Tue, 20 Oct 2026 19:48:01 +0000 Subject: [PATCH 1/7] Export GGML-quantized layers as INT8 compressed-tensors for vLLM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every GGML weight is an integer level of at most 8 bits times the step of its 16- or 32-weight sub-block (IQ1: ±1/±7/±9 eighths, IQ2: ±8/±25/±43, Q8_0: its quants). Storing those levels as INT8 with the steps as group scales reproduces the GGML decoding up to rounding the scales to the model dtype, and stock vLLM serves it with its integer kernels (Marlin or Humming) instead of a BF16 fold. - GGMLFormat gains int8_group_size, unpack_int8 and decode_int8. Each format's torch decoder now multiplies out its own unpack_int8, so the integer view and the decoding cannot disagree. - convert_ggml_export_to_int8 rewrites a GGML HF export in place as a pack-quantized compressed-tensors checkpoint, one config group per group size, targeting layers by name; it validates before rewriting. - hf_ptq.py --export_ggml_as_int8 runs it after the export. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Chenjie Luo --- CHANGELOG.rst | 1 + examples/hf_ptq/README.md | 2 +- examples/hf_ptq/hf_ptq.py | 16 ++ modelopt/torch/export/__init__.py | 1 + modelopt/torch/export/ggml_int8_export.py | 165 ++++++++++++++++++ modelopt/torch/quantization/ggml/common.py | 55 ++++++ modelopt/torch/quantization/ggml/iq1_m.py | 75 ++++---- modelopt/torch/quantization/ggml/iq1_s.py | 45 +++-- modelopt/torch/quantization/ggml/iq2_s.py | 53 +++--- modelopt/torch/quantization/ggml/iq2_xs.py | 67 +++---- modelopt/torch/quantization/ggml/iq2_xxs.py | 53 +++--- modelopt/torch/quantization/ggml/q8_0.py | 25 ++- .../torch/export/test_ggml_int8_export.py | 98 +++++++++++ .../torch/export/test_ggml_int8_export.py | 99 +++++++++++ .../torch/quantization/test_iq_formats.py | 21 +++ 15 files changed, 639 insertions(+), 137 deletions(-) create mode 100644 modelopt/torch/export/ggml_int8_export.py create mode 100644 tests/gpu_vllm/torch/export/test_ggml_int8_export.py create mode 100644 tests/unit/torch/export/test_ggml_int8_export.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f20a33f1c17..b756120d2c2 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,6 +11,7 @@ Changelog - On Linux x86-64 with Python 3.11 and later, the ONNX extra now selects ONNX Runtime 1.30.x and CUDA 13 packages by default. GPU calibration requires a compatible CUDA 13 runtime and driver; Python 3.10 and other platforms retain their existing dependency selections. - Add imatrix calibration (``algorithm: {method: imatrix}``) for the GGML formats with a weighted search: each weight is encoded once, its search weighted by the square root of its inputs' per-column ``E[x^2]`` (llama.cpp's imatrix statistic), and HF fused-MoE experts get one statistic per expert. A weight that already holds final codes is left alone, so ``[{method: gptq}, {method: imatrix}]`` runs GPTQ on the dense linears and imatrix on the experts GPTQ skips; the built-in IQ1 and IQ2 recipes now do this, and new ``_experts_only`` recipes apply imatrix to the routed MoE experts alone. - Add a checkpoint-specific NVIDIA Nemotron 3.5 Super VL AutoQuantize recipe with a 4.9-effective-bit NVFP4-MSE/FP8/BF16 search and fixed NVFP4 MTP experts. Select it under ``models/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16/auto_quantize/``. +- Add ``--export_ggml_as_int8`` to ``hf_ptq.py`` (and ``convert_ggml_export_to_int8`` for existing exports) to write GGML-quantized layers (IQ formats, ``q8_0``) as an INT8 weight-only compressed-tensors checkpoint. Stock vLLM serves it with its integer kernels at about half the BF16 weight memory, reproducing the GGML weights up to rounding their scales to the model dtype. *Speculative Decoding* diff --git a/examples/hf_ptq/README.md b/examples/hf_ptq/README.md index 02aae3c07eb..da52d4de1b6 100755 --- a/examples/hf_ptq/README.md +++ b/examples/hf_ptq/README.md @@ -675,7 +675,7 @@ with torch.inference_mode(): python hf_ptq.py --pyt_ckpt_path --qformat fp8 --export_path --trust_remote_code ``` -> *For exporting fake-quantized models for vLLM serving (e.g., for research or kernels not yet supported in real-quant), use the `--vllm_fakequant_export` flag. See [vllm_serve/README.md](../vllm_serve/README.md) for details. When every quantizer is a weight quantizer, as in the GGML recipes (IQ, Q8_0), the export is a plain BF16 checkpoint that a stock vLLM image serves directly.* +> *For exporting fake-quantized models for vLLM serving (e.g., for research or kernels not yet supported in real-quant), use the `--vllm_fakequant_export` flag. See [vllm_serve/README.md](../vllm_serve/README.md) for details. When every quantizer is a weight quantizer, as in the GGML recipes (IQ, Q8_0), the export is a plain BF16 checkpoint that a stock vLLM image serves directly. For GGML recipes, `--export_ggml_as_int8` instead writes each weight's GGML integer levels and sub-block scales as an INT8 weight-only compressed-tensors checkpoint: the same weights up to scale rounding, served by vLLM's integer kernels in about half the memory.* ### Hugging Face framework [Script](./scripts/huggingface_example.sh) diff --git a/examples/hf_ptq/hf_ptq.py b/examples/hf_ptq/hf_ptq.py index d7a14a68192..02802599657 100755 --- a/examples/hf_ptq/hf_ptq.py +++ b/examples/hf_ptq/hf_ptq.py @@ -74,6 +74,7 @@ RecipeSupersededAction, ) from modelopt.torch.export import ( + convert_ggml_export_to_int8, export_hf_checkpoint, export_hf_vllm_fq_checkpoint, export_speculative_decoding, @@ -728,6 +729,8 @@ def export_quantized( "vLLM deployment support is in progress." ) + if args.export_ggml_as_int8 and args.dist_state.is_main: + convert_ggml_export_to_int8(export_path) args.checkpoint_exported = True end_time = time.time() print_rank_0( @@ -1357,6 +1360,17 @@ def parse_args() -> argparse.Namespace: help="Export as vLLM fake-quant checkpoint (produces vllm_fq_modelopt_state.pth " "for use with vllm_serve_fakequant.py).", ) + parser.add_argument( + "--export_ggml_as_int8", + default=False, + action="store_true", + help=( + "Export GGML-quantized layers (IQ1/IQ2 formats, Q8_0) as an INT8 weight-only " + "compressed-tensors checkpoint that vLLM serves with its integer kernels. Each weight " + "keeps its GGML integer level and sub-block scale, so this reproduces the GGML " + "weights up to rounding the scales to the model dtype." + ), + ) parser.add_argument( "--cast_mxfp4_to_nvfp4", action="store_true", @@ -1437,6 +1451,8 @@ def parse_args() -> argparse.Namespace: parser.error("--cpu_offload requires --use_fsdp2") if args.use_fsdp2 and args.sparsity_fmt != "dense": parser.error(f"--use_fsdp2 does not support --sparsity_fmt {args.sparsity_fmt}.") + if args.export_ggml_as_int8 and args.vllm_fakequant_export: + parser.error("--export_ggml_as_int8 and --vllm_fakequant_export are separate exports.") if args.use_fsdp2 and args.vllm_fakequant_export: parser.error("--use_fsdp2 does not support --vllm_fakequant_export.") if args.use_fsdp2 and args.cast_mxfp4_to_nvfp4: diff --git a/modelopt/torch/export/__init__.py b/modelopt/torch/export/__init__.py index 44d1ad2557e..30c3b808f9b 100644 --- a/modelopt/torch/export/__init__.py +++ b/modelopt/torch/export/__init__.py @@ -34,6 +34,7 @@ ) from .convert_hf_config import * +from .ggml_int8_export import * from .model_utils import * from .moe_utils import * from .plugins import * diff --git a/modelopt/torch/export/ggml_int8_export.py b/modelopt/torch/export/ggml_int8_export.py new file mode 100644 index 00000000000..859cd7580e9 --- /dev/null +++ b/modelopt/torch/export/ggml_int8_export.py @@ -0,0 +1,165 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Rewrite a GGML Hugging Face export as an INT8 weight-only compressed-tensors checkpoint.""" + +import json +import os +from pathlib import Path + +import torch +from safetensors import safe_open +from safetensors.torch import save_file + +from modelopt.torch.quantization.ggml import GGML_FORMAT_REGISTRY, GGMLFormat +from modelopt.torch.utils.plugins.hf_checkpoint_utils import indexed_weight_map + +__all__ = ["convert_ggml_export_to_int8"] + + +def convert_ggml_export_to_int8(export_dir: str | os.PathLike) -> None: + """Rewrite a GGML export in place as INT8 weight-only compressed-tensors, which vLLM serves. + + Every GGML weight is an integer level of at most 8 bits times the step of its 16- or + 32-weight sub-block, so storing those levels with the steps as group scales reproduces the + GGML decoding up to rounding the steps to the model dtype. Unquantized tensors are kept as + they are. The checkpoint is validated before anything is rewritten. + + Args: + export_dir: A directory written by :func:`export_hf_checkpoint` whose quantized layers + are all GGML formats. + """ + export_dir = Path(export_dir) + config = json.loads((export_dir / "config.json").read_text()) + quant_config_path = export_dir / "hf_quant_config.json" + quantization = json.loads(quant_config_path.read_text())["quantization"] + scale_dtype = getattr(torch, config.get("dtype") or config["torch_dtype"]) + weight_map = indexed_weight_map(export_dir) + layer_formats = _ggml_layer_formats(export_dir, weight_map, quantization) + device = "cuda" if torch.cuda.is_available() else "cpu" + + for shard in sorted({weight_map[f"{layer}.weight"] for layer in layer_formats}): + _convert_shard(export_dir / shard, layer_formats, scale_dtype, device) + + for layer in layer_formats: + shard = weight_map.pop(f"{layer}.weight") + weight_map.update( + { + f"{layer}.{suffix}": shard + for suffix in ("weight_packed", "weight_scale", "weight_shape") + } + ) + index_path = export_dir / "model.safetensors.index.json" + if index_path.exists(): + index = json.loads(index_path.read_text()) + index["weight_map"] = dict(sorted(weight_map.items())) + index_path.write_text(json.dumps(index, indent=2)) + + config["quantization_config"] = _compressed_tensors_config(layer_formats) + (export_dir / "config.json").write_text(json.dumps(config, indent=4)) + # It describes the GGML payloads this replaced, and runtimes that read it would misload. + quant_config_path.unlink() + + +def _ggml_layer_formats( + export_dir: Path, weight_map: dict[str, str], quantization: dict +) -> dict[str, GGMLFormat]: + """The GGML format of every layer holding a packed payload, refusing anything else.""" + if quantization.get("kv_cache_quant_algo") is not None: + raise ValueError("INT8 conversion of a GGML export does not carry KV cache quantization.") + mixed = quantization.get("quant_algo") == "MIXED_PRECISION" + layer_configs = quantization.get("quantized_layers", {}) if mixed else {} + algos = ( + {cfg.get("quant_algo") for cfg in layer_configs.values()} + if mixed + else {quantization.get("quant_algo")} + ) + unsupported = sorted(str(a) for a in algos if str(a).lower() not in GGML_FORMAT_REGISTRY) + if unsupported: + raise ValueError(f"Only GGML layers convert to INT8, but the export has {unsupported}.") + + layer_formats = {} + for shard in sorted(set(weight_map.values())): + with safe_open(export_dir / shard, framework="pt") as f: + # GGML payloads are the only uint8 weights an export writes. + payloads = [ + name + for name in f.keys() # noqa: SIM118 -- a safe_open handle is not a dict + if name.endswith(".weight") and f.get_slice(name).get_dtype() == "U8" + ] + for name in payloads: + layer = name.removesuffix(".weight") + algo = layer_configs[layer]["quant_algo"] if mixed else quantization["quant_algo"] + layer_formats[layer] = GGML_FORMAT_REGISTRY[algo.lower()] + if not layer_formats: + raise ValueError(f"{export_dir} holds no GGML weights to convert.") + return layer_formats + + +def _convert_shard( + path: Path, layer_formats: dict[str, GGMLFormat], scale_dtype: torch.dtype, device: str +) -> None: + tensors = {} + with safe_open(path, framework="pt") as f: + metadata = f.metadata() + for name in f.keys(): # noqa: SIM118 -- a safe_open handle is not a dict + layer = name.removesuffix(".weight") + ggml_format = layer_formats.get(layer) if name.endswith(".weight") else None + if ggml_format is None: + tensors[name] = f.get_tensor(name) + continue + payload = f.get_tensor(name).to(device) + shape = torch.tensor((*payload.shape[:-2], payload.shape[-2] * ggml_format.block_size)) + levels, steps = ggml_format.decode_int8(payload, shape) + # compressed-tensors packs 8-bit values offset by 128, four little-endian per int32. + tensors[f"{layer}.weight_packed"] = ( + (levels.view(torch.uint8) ^ 0x80).view(torch.int32).cpu() + ) + tensors[f"{layer}.weight_scale"] = steps.to(scale_dtype).cpu() + tensors[f"{layer}.weight_shape"] = shape + tmp_path = path.with_suffix(".tmp") + save_file(tensors, tmp_path, metadata=metadata) + os.replace(tmp_path, path) + + +def _compressed_tensors_config(layer_formats: dict[str, GGMLFormat]) -> dict: + """One config group per INT8 group size, targeting its layers by name.""" + layers_by_group_size: dict[int, list[str]] = {} + for layer, ggml_format in sorted(layer_formats.items()): + layers_by_group_size.setdefault(ggml_format.int8_group_size, []).append(layer) + config_groups = { + f"group_{i}": { + "targets": layers, + "weights": { + "num_bits": 8, + "type": "int", + "symmetric": True, + "strategy": "group", + "group_size": group_size, + "dynamic": False, + }, + "input_activations": None, + "output_activations": None, + "format": "pack-quantized", + } + for i, (group_size, layers) in enumerate(sorted(layers_by_group_size.items())) + } + return { + "quant_method": "compressed-tensors", + "format": "pack-quantized", + "quantization_status": "compressed", + "config_groups": config_groups, + "ignore": [], + } diff --git a/modelopt/torch/quantization/ggml/common.py b/modelopt/torch/quantization/ggml/common.py index 83dfc2f00d5..59a24cbdb14 100644 --- a/modelopt/torch/quantization/ggml/common.py +++ b/modelopt/torch/quantization/ggml/common.py @@ -247,6 +247,11 @@ class GGMLFormat: # its search temporaries, decoding runs every forward and is bounded by kernel launches. block_chunk_size: int decode_chunk_size: int + # Every decoded weight is an int8 level times a float32 step shared by ``int8_group_size`` + # weights. ``unpack_int8`` returns both for a chunk of blocks, and the torch decoder + # multiplies them out, so the integer view and the decoding cannot disagree. + int8_group_size: int + unpack_int8: Callable[[torch.Tensor], tuple[torch.Tensor, torch.Tensor]] # Whether ``quantize`` takes a per-input-column ``importance`` that weights its search. weighted: bool = False @@ -255,6 +260,36 @@ def effective_bits(self) -> float: """Packed storage cost per weight.""" return self.block_bytes * 8 / self.block_size + def decode_int8( + self, packed_weights: torch.Tensor, weight_shape: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor]: + """Decode a payload into int8 levels and the float32 step of each ``int8_group_size``. + + ``levels * steps`` (each step repeated over its group) equals ``dequantize`` in float32 + exactly, so a weight-only integer kernel with these group scales reproduces the format. + + Returns: + ``levels`` shaped like the weight and ``steps`` shaped + ``[*weight_shape[:-1], weight_shape[-1] // int8_group_size]``. + """ + shape = validate_packed_weights( + packed_weights, + weight_shape, + block_bytes=self.block_bytes, + block_size=self.block_size, + format_name=self.name.upper(), + ) + blocks = packed_weights.contiguous().reshape(-1, self.block_bytes) + groups = self.block_size // self.int8_group_size + levels = torch.empty((len(blocks), self.block_size), dtype=torch.int8, device=blocks.device) + steps = torch.empty((len(blocks), groups), dtype=torch.float32, device=blocks.device) + for start in range(0, len(blocks), self.decode_chunk_size): + stop = start + self.decode_chunk_size + chunk_levels, chunk_steps = self.unpack_int8(blocks[start:stop]) + levels[start:stop] = chunk_levels + steps[start:stop] = chunk_steps + return levels.reshape(shape), steps.reshape(*shape[:-1], -1) + def fake_quant( self, inputs: torch.Tensor, @@ -354,6 +389,26 @@ def decoding(storage_dtype): return None +def dequantize_blocks( + blocks: torch.Tensor, + unpack_int8: Callable[[torch.Tensor], tuple[torch.Tensor, torch.Tensor]], + *, + block_size: int, + group_size: int, + dtype: torch.dtype, + block_chunk_size: int, +) -> torch.Tensor: + """Decode ``[blocks, block_bytes]`` payload rows a chunk at a time as ``levels * steps``.""" + decoded = torch.empty((len(blocks), block_size), dtype=dtype, device=blocks.device) + for start in range(0, len(blocks), block_chunk_size): + stop = start + block_chunk_size + levels, steps = unpack_int8(blocks[start:stop]) + decoded[start:stop] = ( + levels.reshape(len(levels), -1, group_size) * steps.unsqueeze(-1) + ).reshape(-1, block_size) + return decoded + + def narrow_to_float32(blocks: torch.Tensor) -> torch.Tensor: """Narrow ``blocks`` to float32 the way the CUDA ``load_float`` helper does. diff --git a/modelopt/torch/quantization/ggml/iq1_m.py b/modelopt/torch/quantization/ggml/iq1_m.py index e1e5907aaa4..b0cd994b771 100644 --- a/modelopt/torch/quantization/ggml/iq1_m.py +++ b/modelopt/torch/quantization/ggml/iq1_m.py @@ -45,6 +45,7 @@ GGML_BLOCK_SIZE, GGMLFormat, chunk_importance, + dequantize_blocks, importance_blocks, narrow_to_float32, validate_block_chunk_size, @@ -67,6 +68,7 @@ IQ1_M_BLOCK_BYTES = 56 IQ1_M_EFFECTIVE_BITS = IQ1_M_BLOCK_BYTES * 8 / IQ1_M_BLOCK_SIZE _IQ1_M_DELTA = 0.125 +_IQ1_M_INT8_GROUP_SIZE = 16 # Largest representable magnitude: (1 + delta) at local scale 7 -> 15 * 1.125. _IQ1_M_NATIVE_MAX = 16.875 # IQ1_M anchors differently from IQ1_S: the ratio rises with a block's peak-to-RMS @@ -213,45 +215,46 @@ def dequantize_iq1_m( if extension is not None: grid = iq1_s_grid(blocks.device) return extension.iq1_m_unpack(blocks, grid, dtype).reshape(shape) - grid = iq1_s_grid(blocks.device) + return dequantize_blocks( + blocks, + _unpack_int8, + block_size=IQ1_M_BLOCK_SIZE, + group_size=_IQ1_M_INT8_GROUP_SIZE, + dtype=dtype, + block_chunk_size=block_chunk_size, + ).reshape(shape) + + +def _unpack_int8(blocks: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """Levels in eighths of a grid step (±1, ±7, ±9) and the step of each 16-value half.""" + count = blocks.shape[0] scale_shifts = torch.tensor([0, 3, 6, 9], dtype=torch.int64, device=blocks.device) - decoded = torch.empty((blocks.shape[0], IQ1_M_BLOCK_SIZE), dtype=dtype, device=blocks.device) - for start in range(0, blocks.shape[0], block_chunk_size): - stop = min(start + block_chunk_size, blocks.shape[0]) - block_chunk = blocks[start:stop] - count = block_chunk.shape[0] - low = block_chunk[:, :32].to(torch.int64).reshape(count, _IQ1_M_SUBBLOCKS, 4) - qh = block_chunk[:, 32:48].to(torch.int64).reshape(count, _IQ1_M_SUBBLOCKS, 2) - words = block_chunk[:, 48:56:2].to(torch.int64) | ( - block_chunk[:, 49:56:2].to(torch.int64) << 8 - ) - - # The FP16 super-block scale is the four words' top nibbles, low word first. - d_bits = ( - (words[:, 0] >> 12) - | ((words[:, 1] >> 8) & 0x00F0) - | ((words[:, 2] >> 4) & 0x0F00) - | (words[:, 3] & 0xF000) - ) - d = d_bits.to(torch.int16).view(torch.float16).float() + low = blocks[:, :32].to(torch.int64).reshape(count, _IQ1_M_SUBBLOCKS, 4) + qh = blocks[:, 32:48].to(torch.int64).reshape(count, _IQ1_M_SUBBLOCKS, 2) + words = blocks[:, 48:56:2].to(torch.int64) | (blocks[:, 49:56:2].to(torch.int64) << 8) + + # The FP16 super-block scale is the four words' top nibbles, low word first. + d_bits = ( + (words[:, 0] >> 12) + | ((words[:, 1] >> 8) & 0x00F0) + | ((words[:, 2] >> 4) & 0x0F00) + | (words[:, 3] & 0xF000) + ) + d = d_bits.to(torch.int16).view(torch.float16).float() - # Each qh byte carries two groups, low nibble first: three high index bits and a delta sign. - nibbles = torch.stack((qh & 0xF, qh >> 4), dim=-1).reshape(count, _IQ1_M_SUBBLOCKS, 4) - entries = low | ((nibbles & 0x7) << 8) - deltas = torch.where((nibbles & 0x8).bool(), -_IQ1_M_DELTA, _IQ1_M_DELTA) + # Each qh byte carries two groups, low nibble first: three high index bits and a delta sign. + nibbles = torch.stack((qh & 0xF, qh >> 4), dim=-1).reshape(count, _IQ1_M_SUBBLOCKS, 4) + entries = low | ((nibbles & 0x7) << 8) + delta_sign = torch.where((nibbles & 0x8).bool(), -1.0, 1.0).unsqueeze(-1) - # Each word packs four 3-bit local scales below its nibble of d: two sub-blocks, two - # halves each, so word w, slot k is sub-block 2 * w + k // 2, half k % 2. - local = ((words.unsqueeze(-1) >> scale_shifts) & 0x7).reshape(count, _IQ1_M_SUBBLOCKS, 2) - scales = d.unsqueeze(-1).unsqueeze(-1) * (2 * local + 1).float() + # Each word packs four 3-bit local scales below its nibble of d: two sub-blocks, two + # halves each, so word w, slot k is sub-block 2 * w + k // 2, half k % 2. Groups 0,1 take + # the first half's scale and groups 2,3 the second, which is the order the steps flatten in. + local = ((words.unsqueeze(-1) >> scale_shifts) & 0x7).reshape(count, _IQ1_M_SUBBLOCKS * 2) + steps = d.unsqueeze(-1) * (2 * local + 1).float() * _IQ1_M_DELTA - values = grid[entries] + deltas.unsqueeze(-1) - # Groups 0,1 take the first local scale and groups 2,3 the second, so the repeat - # runs along the half axis: [h0, h0, h1, h1], not [h0, h1, h0, h1]. - group_scale = scales.repeat_interleave(2, dim=2) - chunk_decoded = values * group_scale.unsqueeze(-1) - decoded[start:stop] = chunk_decoded.reshape(-1, IQ1_M_BLOCK_SIZE) - return decoded.reshape(shape) + levels = iq1_s_grid(blocks.device)[entries] / _IQ1_M_DELTA + delta_sign + return levels.reshape(count, IQ1_M_BLOCK_SIZE), steps IQ1_M_FORMAT = GGMLFormat( @@ -262,6 +265,8 @@ def dequantize_iq1_m( dequantize=dequantize_iq1_m, block_chunk_size=_DEFAULT_BLOCK_CHUNK_SIZE, decode_chunk_size=_DEFAULT_DECODE_CHUNK_SIZE, + int8_group_size=_IQ1_M_INT8_GROUP_SIZE, + unpack_int8=_unpack_int8, weighted=True, ) diff --git a/modelopt/torch/quantization/ggml/iq1_s.py b/modelopt/torch/quantization/ggml/iq1_s.py index 57ee47cf7cb..f00d777c0d2 100644 --- a/modelopt/torch/quantization/ggml/iq1_s.py +++ b/modelopt/torch/quantization/ggml/iq1_s.py @@ -41,6 +41,7 @@ GGML_BLOCK_SIZE, GGMLFormat, chunk_importance, + dequantize_blocks, importance_blocks, narrow_to_float32, validate_block_chunk_size, @@ -62,6 +63,7 @@ IQ1_S_BLOCK_BYTES = 50 IQ1_S_EFFECTIVE_BITS = IQ1_S_BLOCK_BYTES * 8 / IQ1_S_BLOCK_SIZE _IQ1_S_DELTA = 0.125 +_IQ1_S_INT8_GROUP_SIZE = 32 _IQ1_S_NATIVE_MAX = 16.875 _IQ1_S_SCALE_ANCHOR = 0.61 # Bounds the torch encode fallback, whose codebook search holds the large temporaries: at @@ -273,26 +275,29 @@ def dequantize_iq1_s( if extension is not None: grid = iq1_s_grid(blocks.device) return extension.iq1_s_unpack(blocks, grid, dtype).reshape(shape) + return dequantize_blocks( + blocks, + _unpack_int8, + block_size=IQ1_S_BLOCK_SIZE, + group_size=_IQ1_S_INT8_GROUP_SIZE, + dtype=dtype, + block_chunk_size=block_chunk_size, + ).reshape(shape) + + +def _unpack_int8(blocks: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """Levels in eighths of a grid step (±1, ±7, ±9) and the step of each 32-value sub-block.""" shifts = torch.tensor([0, 3, 6, 9], dtype=torch.int64, device=blocks.device) - grid = iq1_s_grid(blocks.device) - decoded = torch.empty((blocks.shape[0], IQ1_S_BLOCK_SIZE), dtype=dtype, device=blocks.device) - for start in range(0, blocks.shape[0], block_chunk_size): - stop = min(start + block_chunk_size, blocks.shape[0]) - block_chunk = blocks[start:stop] - d = block_chunk[:, :2].contiguous().view(torch.float16).reshape(-1).float() - low = block_chunk[:, 2:34].to(torch.int64).reshape(-1, 8, 4) - qh = block_chunk[:, 34:50:2].to(torch.int64) | ( - block_chunk[:, 35:50:2].to(torch.int64) << 8 - ) - high = (qh.unsqueeze(-1) >> shifts) & 0x7 - entries = low | (high << 8) - local = (qh >> 12) & 0x7 - delta = torch.where((qh & 0x8000).bool(), -_IQ1_S_DELTA, _IQ1_S_DELTA) - values = grid[entries] + delta.unsqueeze(-1).unsqueeze(-1) - scales = d.unsqueeze(-1) * (2 * local + 1).float() - chunk_decoded = values * scales.unsqueeze(-1).unsqueeze(-1) - decoded[start:stop] = chunk_decoded.reshape(-1, IQ1_S_BLOCK_SIZE) - return decoded.reshape(shape) + d = blocks[:, :2].contiguous().view(torch.float16).reshape(-1).float() + low = blocks[:, 2:34].to(torch.int64).reshape(-1, 8, 4) + qh = blocks[:, 34:50:2].to(torch.int64) | (blocks[:, 35:50:2].to(torch.int64) << 8) + high = (qh.unsqueeze(-1) >> shifts) & 0x7 + entries = low | (high << 8) + local = (qh >> 12) & 0x7 + delta_sign = torch.where((qh & 0x8000).bool(), -1.0, 1.0).unsqueeze(-1).unsqueeze(-1) + levels = iq1_s_grid(blocks.device)[entries] / _IQ1_S_DELTA + delta_sign + steps = d.unsqueeze(-1) * (2 * local + 1).float() * _IQ1_S_DELTA + return levels.reshape(-1, IQ1_S_BLOCK_SIZE), steps IQ1_S_FORMAT = GGMLFormat( @@ -303,6 +308,8 @@ def dequantize_iq1_s( dequantize=dequantize_iq1_s, block_chunk_size=_DEFAULT_BLOCK_CHUNK_SIZE, decode_chunk_size=_DEFAULT_DECODE_CHUNK_SIZE, + int8_group_size=_IQ1_S_INT8_GROUP_SIZE, + unpack_int8=_unpack_int8, weighted=True, ) diff --git a/modelopt/torch/quantization/ggml/iq2_s.py b/modelopt/torch/quantization/ggml/iq2_s.py index c7c24c826ba..c64ae46fdd7 100644 --- a/modelopt/torch/quantization/ggml/iq2_s.py +++ b/modelopt/torch/quantization/ggml/iq2_s.py @@ -43,6 +43,7 @@ GGML_BLOCK_SIZE, GGMLFormat, chunk_importance, + dequantize_blocks, importance_blocks, iq2_tile_terms, narrow_to_float32, @@ -68,6 +69,7 @@ _IQ2_S_LOCAL_SCALES = 16 _IQ2_S_GROUPS = 32 _IQ2_S_SUBBLOCKS = 8 +_IQ2_S_INT8_GROUP_SIZE = 16 _IQ2_S_NATIVE_MAX = 43 * 31 / 8 _IQ2_S_SCALE_ANCHOR_MIN = 0.65 _IQ2_S_SCALE_ANCHOR_MAX = 0.92 @@ -227,30 +229,35 @@ def dequantize_iq2_s( if extension is not None: grid = iq2_s_grid(blocks.device) return extension.iq2_s_unpack(blocks, grid, dtype).reshape(shape) + return dequantize_blocks( + blocks, + _unpack_int8, + block_size=IQ2_S_BLOCK_SIZE, + group_size=_IQ2_S_INT8_GROUP_SIZE, + dtype=dtype, + block_chunk_size=block_chunk_size, + ).reshape(shape) + + +def _unpack_int8(blocks: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """Signed grid levels (±8, ±25, ±43) and the step of each 16-value half sub-block.""" + count = blocks.shape[0] bit_positions = torch.arange(8, dtype=torch.int64, device=blocks.device) high_shifts = torch.tensor([0, 2, 4, 6], dtype=torch.int64, device=blocks.device) - grid = iq2_s_grid(blocks.device) - decoded = torch.empty((blocks.shape[0], IQ2_S_BLOCK_SIZE), dtype=dtype, device=blocks.device) - for start in range(0, blocks.shape[0], block_chunk_size): - stop = min(start + block_chunk_size, blocks.shape[0]) - block_chunk = blocks[start:stop] - count = block_chunk.shape[0] - d = block_chunk[:, :2].contiguous().view(torch.float16).reshape(-1).float() - low = block_chunk[:, 2:34].to(torch.int64).reshape(count, _IQ2_S_SUBBLOCKS, 4) - sign_mask = block_chunk[:, 34:66].to(torch.int64).reshape(count, _IQ2_S_SUBBLOCKS, 4) - qh = block_chunk[:, 66:74].to(torch.int64).reshape(count, _IQ2_S_SUBBLOCKS) - scale_bytes = block_chunk[:, 74:].to(torch.int64) - - entries = low | (((qh.unsqueeze(-1) >> high_shifts) & 0x3) << 8) - local = torch.empty((count, 16), dtype=torch.int64, device=blocks.device) - local[:, 0::2] = scale_bytes & 0x0F - local[:, 1::2] = scale_bytes >> 4 - scales = d.unsqueeze(-1) * (0.5 + local.float()) * 0.25 - signs = 1.0 - 2.0 * ((sign_mask.unsqueeze(-1) >> bit_positions) & 1).float() - values = grid[entries] * signs - chunk_decoded = values.reshape(count, 16, 2, 8) * scales.unsqueeze(-1).unsqueeze(-1) - decoded[start:stop] = chunk_decoded.reshape(-1, IQ2_S_BLOCK_SIZE) - return decoded.reshape(shape) + d = blocks[:, :2].contiguous().view(torch.float16).reshape(-1).float() + low = blocks[:, 2:34].to(torch.int64).reshape(count, _IQ2_S_SUBBLOCKS, 4) + sign_mask = blocks[:, 34:66].to(torch.int64).reshape(count, _IQ2_S_SUBBLOCKS, 4) + qh = blocks[:, 66:74].to(torch.int64).reshape(count, _IQ2_S_SUBBLOCKS) + scale_bytes = blocks[:, 74:].to(torch.int64) + + entries = low | (((qh.unsqueeze(-1) >> high_shifts) & 0x3) << 8) + local = torch.empty((count, 16), dtype=torch.int64, device=blocks.device) + local[:, 0::2] = scale_bytes & 0x0F + local[:, 1::2] = scale_bytes >> 4 + steps = d.unsqueeze(-1) * (0.5 + local.float()) * 0.25 + signs = 1.0 - 2.0 * ((sign_mask.unsqueeze(-1) >> bit_positions) & 1).float() + levels = iq2_s_grid(blocks.device)[entries] * signs + return levels.reshape(count, IQ2_S_BLOCK_SIZE), steps IQ2_S_FORMAT = GGMLFormat( @@ -261,6 +268,8 @@ def dequantize_iq2_s( dequantize=dequantize_iq2_s, block_chunk_size=_DEFAULT_BLOCK_CHUNK_SIZE, decode_chunk_size=_DEFAULT_DECODE_CHUNK_SIZE, + int8_group_size=_IQ2_S_INT8_GROUP_SIZE, + unpack_int8=_unpack_int8, weighted=True, ) diff --git a/modelopt/torch/quantization/ggml/iq2_xs.py b/modelopt/torch/quantization/ggml/iq2_xs.py index 27e26a8fbe3..9f73c1f81b7 100644 --- a/modelopt/torch/quantization/ggml/iq2_xs.py +++ b/modelopt/torch/quantization/ggml/iq2_xs.py @@ -38,6 +38,7 @@ GGML_BLOCK_SIZE, GGMLFormat, chunk_importance, + dequantize_blocks, importance_blocks, iq2_tile_terms, narrow_to_float32, @@ -59,6 +60,7 @@ IQ2_XS_BLOCK_SIZE = GGML_BLOCK_SIZE IQ2_XS_BLOCK_BYTES = 74 IQ2_XS_EFFECTIVE_BITS = IQ2_XS_BLOCK_BYTES * 8 / IQ2_XS_BLOCK_SIZE +_IQ2_XS_INT8_GROUP_SIZE = 16 _IQ2_XS_NATIVE_MAX = 43 * 31 / 8 _IQ2_XS_SCALE_ANCHOR_MIN = 0.65 _IQ2_XS_SCALE_ANCHOR_MAX = 0.92 @@ -239,37 +241,40 @@ def dequantize_iq2_xs( if extension is not None: grid = iq2_xs_grid(blocks.device) return extension.iq2_xs_unpack(blocks, grid, dtype).reshape(shape) + return dequantize_blocks( + blocks, + _unpack_int8, + block_size=IQ2_XS_BLOCK_SIZE, + group_size=_IQ2_XS_INT8_GROUP_SIZE, + dtype=dtype, + block_chunk_size=block_chunk_size, + ).reshape(shape) + + +def _unpack_int8(blocks: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """Signed grid levels (±8, ±25, ±43) and the step of each 16-value half sub-block.""" bit_positions = torch.arange(8, dtype=torch.int64, device=blocks.device) - grid = iq2_xs_grid(blocks.device) - decoded = torch.empty((blocks.shape[0], IQ2_XS_BLOCK_SIZE), dtype=dtype, device=blocks.device) - for start in range(0, blocks.shape[0], block_chunk_size): - stop = min(start + block_chunk_size, blocks.shape[0]) - block_chunk = blocks[start:stop] - d = block_chunk[:, :2].contiguous().view(torch.float16).reshape(-1).float() - codes = block_chunk[:, 2:66:2].to(torch.int64) | ( - block_chunk[:, 3:66:2].to(torch.int64) << 8 - ) - entries = codes & 0x1FF - sign_index = codes >> 9 - # XOR-fold the seven payload bits down to bit 0 to recover the eighth sign bit. - # The loop this replaces cost seven elementwise passes per chunk, and unlike packing - # the decode is not cached -- it runs again on every forward. - folded = sign_index ^ (sign_index >> 4) - folded ^= folded >> 2 - folded ^= folded >> 1 - sign_mask = sign_index | ((folded & 1) << 7) - signs = 1.0 - 2.0 * ((sign_mask.unsqueeze(-1) >> bit_positions) & 1).float() - - scale_bytes = block_chunk[:, 66:].to(torch.int64) - local = torch.empty((block_chunk.shape[0], 16), dtype=torch.int64, device=blocks.device) - local[:, 0::2] = scale_bytes & 0x0F - local[:, 1::2] = scale_bytes >> 4 - # Pinned format rule: d * (0.5 + local) * 0.25 == d * (2 * local + 1) / 8. - scales = d.unsqueeze(-1) * (2 * local + 1).float() / 8.0 - values = grid[entries] * signs - chunk_decoded = values * scales.repeat_interleave(2, dim=1).unsqueeze(-1) - decoded[start:stop] = chunk_decoded.reshape(-1, IQ2_XS_BLOCK_SIZE) - return decoded.reshape(shape) + d = blocks[:, :2].contiguous().view(torch.float16).reshape(-1).float() + codes = blocks[:, 2:66:2].to(torch.int64) | (blocks[:, 3:66:2].to(torch.int64) << 8) + entries = codes & 0x1FF + sign_index = codes >> 9 + # XOR-fold the seven payload bits down to bit 0 to recover the eighth sign bit. + # The loop this replaces cost seven elementwise passes per chunk, and unlike packing + # the decode is not cached -- it runs again on every forward. + folded = sign_index ^ (sign_index >> 4) + folded ^= folded >> 2 + folded ^= folded >> 1 + sign_mask = sign_index | ((folded & 1) << 7) + signs = 1.0 - 2.0 * ((sign_mask.unsqueeze(-1) >> bit_positions) & 1).float() + + scale_bytes = blocks[:, 66:].to(torch.int64) + local = torch.empty((blocks.shape[0], 16), dtype=torch.int64, device=blocks.device) + local[:, 0::2] = scale_bytes & 0x0F + local[:, 1::2] = scale_bytes >> 4 + # Pinned format rule: d * (0.5 + local) * 0.25 == d * (2 * local + 1) / 8. + steps = d.unsqueeze(-1) * (2 * local + 1).float() / 8.0 + levels = iq2_xs_grid(blocks.device)[entries] * signs + return levels.reshape(-1, IQ2_XS_BLOCK_SIZE), steps IQ2_XS_FORMAT = GGMLFormat( @@ -280,6 +285,8 @@ def dequantize_iq2_xs( dequantize=dequantize_iq2_xs, block_chunk_size=_DEFAULT_BLOCK_CHUNK_SIZE, decode_chunk_size=_DEFAULT_DECODE_CHUNK_SIZE, + int8_group_size=_IQ2_XS_INT8_GROUP_SIZE, + unpack_int8=_unpack_int8, weighted=True, ) diff --git a/modelopt/torch/quantization/ggml/iq2_xxs.py b/modelopt/torch/quantization/ggml/iq2_xxs.py index 7ea803a1358..50ddf8c3b97 100644 --- a/modelopt/torch/quantization/ggml/iq2_xxs.py +++ b/modelopt/torch/quantization/ggml/iq2_xxs.py @@ -43,6 +43,7 @@ GGML_BLOCK_SIZE, GGMLFormat, chunk_importance, + dequantize_blocks, importance_blocks, iq2_tile_terms, narrow_to_float32, @@ -69,6 +70,7 @@ _IQ2_XXS_GROUPS = 32 _IQ2_XXS_GROUPS_PER_SUBBLOCK = 4 _IQ2_XXS_SUBBLOCKS = _IQ2_XXS_GROUPS // _IQ2_XXS_GROUPS_PER_SUBBLOCK +_IQ2_XXS_INT8_GROUP_SIZE = IQ2_XXS_BLOCK_SIZE // _IQ2_XXS_SUBBLOCKS # Largest representable magnitude: grid entry 43 at local scale 15 -> (0.5 + 15) * 0.25. _IQ2_XXS_NATIVE_MAX = 43 * 31 / 8 _IQ2_XXS_SCALE_ANCHOR_MIN = 0.65 @@ -258,32 +260,37 @@ def dequantize_iq2_xxs( if extension is not None: grid = iq2_xxs_grid(blocks.device) return extension.iq2_xxs_unpack(blocks, grid, dtype).reshape(shape) + return dequantize_blocks( + blocks, + _unpack_int8, + block_size=IQ2_XXS_BLOCK_SIZE, + group_size=_IQ2_XXS_INT8_GROUP_SIZE, + dtype=dtype, + block_chunk_size=block_chunk_size, + ).reshape(shape) + + +def _unpack_int8(blocks: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """Signed grid levels (±8, ±25, ±43) and the step of each 32-value sub-block.""" + count = blocks.shape[0] bit_positions = torch.arange(8, dtype=torch.int64, device=blocks.device) sign_shifts = 7 * torch.arange( _IQ2_XXS_GROUPS_PER_SUBBLOCK, dtype=torch.int64, device=blocks.device ) - grid = iq2_xxs_grid(blocks.device) - decoded = torch.empty((blocks.shape[0], IQ2_XXS_BLOCK_SIZE), dtype=dtype, device=blocks.device) - for start in range(0, blocks.shape[0], block_chunk_size): - stop = min(start + block_chunk_size, blocks.shape[0]) - block_chunk = blocks[start:stop] - count = block_chunk.shape[0] - d = block_chunk[:, :2].contiguous().view(torch.float16).reshape(-1).float() - body = block_chunk[:, 2:].reshape(count, _IQ2_XXS_SUBBLOCKS, 8).to(torch.int64) - entries = body[:, :, 0:4] - aux = body[:, :, 4] | (body[:, :, 5] << 8) | (body[:, :, 6] << 16) | (body[:, :, 7] << 24) - # Top nibble is the sub-block scale; the low 28 bits are four 7-bit sign indices. - scales = d.unsqueeze(-1) * (0.5 + ((aux >> 28) & 0xF).float()) * 0.25 - sign_index = (aux.unsqueeze(-1) >> sign_shifts) & 0x7F - folded = sign_index ^ (sign_index >> 4) - folded ^= folded >> 2 - folded ^= folded >> 1 - sign_mask = sign_index | ((folded & 1) << 7) - signs = 1.0 - 2.0 * ((sign_mask.unsqueeze(-1) >> bit_positions) & 1).float() - values = grid[entries] * signs - chunk_decoded = values * scales.unsqueeze(-1).unsqueeze(-1) - decoded[start:stop] = chunk_decoded.reshape(-1, IQ2_XXS_BLOCK_SIZE) - return decoded.reshape(shape) + d = blocks[:, :2].contiguous().view(torch.float16).reshape(-1).float() + body = blocks[:, 2:].reshape(count, _IQ2_XXS_SUBBLOCKS, 8).to(torch.int64) + entries = body[:, :, 0:4] + aux = body[:, :, 4] | (body[:, :, 5] << 8) | (body[:, :, 6] << 16) | (body[:, :, 7] << 24) + # Top nibble is the sub-block scale; the low 28 bits are four 7-bit sign indices. + steps = d.unsqueeze(-1) * (0.5 + ((aux >> 28) & 0xF).float()) * 0.25 + sign_index = (aux.unsqueeze(-1) >> sign_shifts) & 0x7F + folded = sign_index ^ (sign_index >> 4) + folded ^= folded >> 2 + folded ^= folded >> 1 + sign_mask = sign_index | ((folded & 1) << 7) + signs = 1.0 - 2.0 * ((sign_mask.unsqueeze(-1) >> bit_positions) & 1).float() + levels = iq2_xxs_grid(blocks.device)[entries] * signs + return levels.reshape(count, IQ2_XXS_BLOCK_SIZE), steps IQ2_XXS_FORMAT = GGMLFormat( @@ -294,6 +301,8 @@ def dequantize_iq2_xxs( dequantize=dequantize_iq2_xxs, block_chunk_size=_DEFAULT_BLOCK_CHUNK_SIZE, decode_chunk_size=_DEFAULT_DECODE_CHUNK_SIZE, + int8_group_size=_IQ2_XXS_INT8_GROUP_SIZE, + unpack_int8=_unpack_int8, weighted=True, ) diff --git a/modelopt/torch/quantization/ggml/q8_0.py b/modelopt/torch/quantization/ggml/q8_0.py index 242a823835d..8b3700e9d5d 100644 --- a/modelopt/torch/quantization/ggml/q8_0.py +++ b/modelopt/torch/quantization/ggml/q8_0.py @@ -24,6 +24,7 @@ from ..extensions import get_cuda_ext_ggml from .common import ( GGMLFormat, + dequantize_blocks, narrow_to_float32, validate_block_chunk_size, validate_packed_weights, @@ -127,14 +128,20 @@ def dequantize_q8_0( validate_block_chunk_size(block_chunk_size) blocks = packed_weights.contiguous().reshape(-1, Q8_0_BLOCK_BYTES) - decoded = torch.empty((blocks.shape[0], Q8_0_BLOCK_SIZE), dtype=dtype, device=blocks.device) - for start in range(0, blocks.shape[0], block_chunk_size): - stop = min(start + block_chunk_size, blocks.shape[0]) - block_chunk = blocks[start:stop] - d = block_chunk[:, :2].contiguous().view(torch.float16).reshape(-1).float() - quants = block_chunk[:, 2:].contiguous().view(torch.int8).float() - decoded[start:stop] = (d.unsqueeze(1) * quants).to(dtype) - return decoded.reshape(shape) + return dequantize_blocks( + blocks, + _unpack_int8, + block_size=Q8_0_BLOCK_SIZE, + group_size=Q8_0_BLOCK_SIZE, + dtype=dtype, + block_chunk_size=block_chunk_size, + ).reshape(shape) + + +def _unpack_int8(blocks: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """The stored quants and each block's scale.""" + d = blocks[:, :2].contiguous().view(torch.float16).float() + return blocks[:, 2:].contiguous().view(torch.int8).float(), d Q8_0_FORMAT = GGMLFormat( @@ -145,6 +152,8 @@ def dequantize_q8_0( dequantize=dequantize_q8_0, block_chunk_size=_DEFAULT_BLOCK_CHUNK_SIZE, decode_chunk_size=_DEFAULT_DECODE_CHUNK_SIZE, + int8_group_size=Q8_0_BLOCK_SIZE, + unpack_int8=_unpack_int8, ) # Keep the public per-format entry point on the same record used by backend dispatch. diff --git a/tests/gpu_vllm/torch/export/test_ggml_int8_export.py b/tests/gpu_vllm/torch/export/test_ggml_int8_export.py new file mode 100644 index 00000000000..81af0fed915 --- /dev/null +++ b/tests/gpu_vllm/torch/export/test_ggml_int8_export.py @@ -0,0 +1,98 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc + +import pytest +import torch +from _test_utils.torch.transformers_models import create_tiny_llama_dir +from transformers import AutoModelForCausalLM +from vllm import LLM, SamplingParams +from vllm.distributed import cleanup_dist_env_and_memory +from vllm.inputs import TokensPrompt + +import modelopt.torch.quantization as mtq +from modelopt.torch.export import convert_ggml_export_to_int8, export_hf_checkpoint + + +@pytest.fixture(scope="module") +def int8_export_in_vllm(tmp_path_factory): + """vLLM serving the INT8 export of a GGML model, the model's own logprobs and their tokens.""" + tmp_path = tmp_path_factory.mktemp("ggml_int8") + source = create_tiny_llama_dir( + tmp_path, + hidden_size=256, + intermediate_size=512, + num_attention_heads=4, + num_key_value_heads=2, + head_dim=64, + max_position_embeddings=64, + vocab_size=128, + initializer_range=0.05, + ) + # Fake quant runs on CPU: the reference needs no GPU, and it spares the GGML CUDA extension build. + model = AutoModelForCausalLM.from_pretrained(source, dtype=torch.bfloat16) + # Group 32 (IQ1_S) and group 16 (IQ2_XS) layers take different vLLM kernels. + quant_cfg = { + "quant_cfg": [ + {"quantizer_name": "*", "enable": False}, + *( + { + "quantizer_name": f"*layers.{i}.mlp*weight_quantizer", + "cfg": {"num_bits": num_bits, "block_sizes": {-1: 256}, "backend": "ggml"}, + } + for i, num_bits in enumerate(("iq1_s", "iq2_xs")) + ), + ], + "algorithm": "max", + } + token_ids = torch.randint(0, 128, (1, 48), generator=torch.Generator().manual_seed(0)) + mtq.quantize(model, quant_cfg, lambda m: m(token_ids)) + with torch.no_grad(): + logits = model(token_ids).logits[0, :-1].float() + expected = logits.log_softmax(-1).gather(-1, token_ids[0, 1:, None])[:, 0] + + export_dir = tmp_path / "export" + export_hf_checkpoint(model, export_dir=export_dir) + convert_ggml_export_to_int8(export_dir) + + llm = LLM( + model=str(export_dir), + enforce_eager=True, + gpu_memory_utilization=0.2, + max_model_len=64, + max_num_seqs=1, + dtype="bfloat16", + skip_tokenizer_init=True, + ) + yield llm, token_ids[0], expected + del llm + gc.collect() + cleanup_dist_env_and_memory(shutdown_ray=False) + + +def test_vllm_serves_the_int8_export_like_the_fake_quantized_model(int8_export_in_vllm): + llm, token_ids, expected = int8_export_in_vllm + (output,) = llm.generate( + TokensPrompt(prompt_token_ids=token_ids.tolist()), + SamplingParams(max_tokens=1, prompt_logprobs=0, detokenize=False), + ) + served = torch.tensor( + [lp[t].logprob for t, lp in zip(token_ids[1:].tolist(), output.prompt_logprobs[1:])] + ) + + # vLLM's kernels round differently from the fake-quant reference by up to ~0.02; giving each + # group its neighbour's scale moves these logprobs by over 1. + torch.testing.assert_close(served, expected, atol=0.05, rtol=0) diff --git a/tests/unit/torch/export/test_ggml_int8_export.py b/tests/unit/torch/export/test_ggml_int8_export.py new file mode 100644 index 00000000000..5ab8f3d6dc8 --- /dev/null +++ b/tests/unit/torch/export/test_ggml_int8_export.py @@ -0,0 +1,99 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +import pytest +import torch +from _test_utils.torch.transformers_models import create_tiny_llama_dir +from safetensors.torch import load_file +from transformers import AutoModelForCausalLM + +import modelopt.torch.quantization as mtq +from modelopt.torch.export import convert_ggml_export_to_int8, export_hf_checkpoint +from modelopt.torch.quantization.ggml import GGML_FORMAT_REGISTRY + +# Layer 0 takes a format with 32-weight sub-blocks and layer 1 one with 16, so the export needs +# one compressed-tensors config group per group size. +LAYER_FORMATS = {0: "iq1_s", 1: "iq2_xs"} + + +def _export(tmp_path, second_layer_cfg): + source = create_tiny_llama_dir(tmp_path, hidden_size=256, intermediate_size=512) + model = AutoModelForCausalLM.from_pretrained(source, dtype=torch.bfloat16) + layer_cfgs = [ + {"num_bits": "iq1_s", "block_sizes": {-1: 256}, "backend": "ggml"}, + second_layer_cfg, + ] + quant_cfg = { + "quant_cfg": [ + {"quantizer_name": "*", "enable": False}, + *( + {"quantizer_name": f"*layers.{i}.mlp*weight_quantizer", "cfg": cfg} + for i, cfg in enumerate(layer_cfgs) + ), + ], + "algorithm": "max", + } + mtq.quantize(model, quant_cfg, lambda m: m(torch.randint(0, 32, (1, 8)))) + export_dir = tmp_path / "export" + export_hf_checkpoint(model, export_dir=export_dir) + return export_dir + + +def test_ggml_export_converts_to_int8_levels_and_steps(tmp_path): + export_dir = _export( + tmp_path, {"num_bits": "iq2_xs", "block_sizes": {-1: 256}, "backend": "ggml"} + ) + before = load_file(export_dir / "model.safetensors") + + convert_ggml_export_to_int8(export_dir) + + config = json.loads((export_dir / "config.json").read_text())["quantization_config"] + assert config["quant_method"] == "compressed-tensors" + layers = { + i: [f"model.layers.{i}.mlp.{proj}" for proj in ("down_proj", "gate_proj", "up_proj")] + for i in LAYER_FORMATS + } + assert { + group["weights"]["group_size"]: group["targets"] + for group in config["config_groups"].values() + } == {32: layers[0], 16: layers[1]} + assert not (export_dir / "hf_quant_config.json").exists() + + after = load_file(export_dir / "model.safetensors") + for i, name in LAYER_FORMATS.items(): + for layer in layers[i]: + payload = before.pop(f"{layer}.weight") + shape = after[f"{layer}.weight_shape"] + levels, steps = GGML_FORMAT_REGISTRY[name].decode_int8(payload, shape) + # compressed-tensors' layout: offset by 128, four values per int32, lowest byte first. + packed = after[f"{layer}.weight_packed"] + unpacked = torch.stack([(packed >> (8 * k)) & 0xFF for k in range(4)], -1).flatten(1) + assert torch.equal(unpacked - 128, levels.long()) + assert torch.equal(after[f"{layer}.weight_scale"], steps.to(torch.bfloat16)) + assert shape.tolist() == list(levels.shape) + for name, tensor in before.items(): + assert torch.equal(after[name], tensor) + + +def test_export_with_a_non_ggml_layer_is_refused_unchanged(tmp_path): + export_dir = _export(tmp_path, {"num_bits": (4, 3), "axis": None}) + files = {path.name: path.read_bytes() for path in export_dir.iterdir()} + + with pytest.raises(ValueError, match="Only GGML layers"): + convert_ggml_export_to_int8(export_dir) + + assert {path.name: path.read_bytes() for path in export_dir.iterdir()} == files diff --git a/tests/unit/torch/quantization/test_iq_formats.py b/tests/unit/torch/quantization/test_iq_formats.py index 992ec981019..5402b78c9b9 100644 --- a/tests/unit/torch/quantization/test_iq_formats.py +++ b/tests/unit/torch/quantization/test_iq_formats.py @@ -52,6 +52,9 @@ DISPATCHED = sorted(IQ_FORMAT_REGISTRY) # IQ1 grids are ternary; IQ2 grids hold the magnitudes 8, 25 and 43. TERNARY = {"iq1_s", "iq1_m"} +# Weights sharing one sub-block scale: the formats with an extra 0.19-0.25 bits per weight spend +# them on scales twice as fine. +INT8_GROUP_SIZES = {"iq1_s": 32, "iq1_m": 16, "iq2_xxs": 32, "iq2_xs": 16, "iq2_s": 16} # name -> (largest representable magnitude, scale anchor at peak-to-RMS 1, 4, 8 and 16), where the # anchor is d * native max / amax. IQ1_S is flat; IQ1_M rises with peak-to-RMS and the IQ2 formats # fall with it, each clamped at both ends. Written out rather than read from the modules, so a @@ -295,6 +298,24 @@ def test_decoder_matches_llama_cpp_on_captured_blocks(name): assert np.array_equal(decoded.numpy(), expected_values(name)) +@pytest.mark.parametrize("name", NAMES) +def test_int8_view_matches_llama_cpp_on_captured_blocks(name): + """Integer levels times their group's step are llama.cpp's decoding, exactly.""" + record = IQ_FORMAT_REGISTRY[name] + blocks = torch.from_numpy(packed_blocks(name)) + count, group_size = blocks.shape[0], INT8_GROUP_SIZES[name] + + levels, steps = record.decode_int8(blocks.reshape(count, 1, -1), torch.tensor([count, 256])) + + assert record.int8_group_size == group_size + assert levels.dtype == torch.int8 + assert set(levels.abs().unique().tolist()) <= ( + {8, 25, 43} if name not in TERNARY else {1, 7, 9} + ) + decoded = levels.float().reshape(count, -1, group_size) * steps.unsqueeze(-1) + assert np.array_equal(decoded.reshape(count, 256).numpy(), expected_values(name)) + + def test_every_format_has_conformance_vectors(): """A new format must arrive with blocks captured from a real llama.cpp checkpoint.""" assert sorted(formats()) == NAMES From 2e7b207272512807f78501bfc497ae4b33cb7006 Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Tue, 20 Oct 2026 23:17:15 +0000 Subject: [PATCH 2/7] Export GGML INT8 from the in-memory model, beside the vLLM fake-quant export Replace the post-export converter, which rewrote a written GGML checkpoint, with export_hf_ggml_int8_checkpoint(model, export_dir). Like export_hf_vllm_fq_checkpoint, it builds the state dict from the model: each GGML layer's pack() payload (the codes fake quant decoded, including GPTQ's pinned ones) becomes INT8 levels and scales, then save_pretrained writes it and the compressed-tensors config is added with the saved layer names. This writes once instead of writing GGML and rewriting it, leaves the GGML exporter untouched, and no longer depends on the on-disk payload layout or deletes hf_quant_config.json. Every enabled quantizer must be a GGML weight quantizer, checked before anything is written. hf_ptq.py now branches if --vllm_fakequant_export / elif --export_ggml_as_int8 / else export_hf_checkpoint, and rejects the flag with FSDP2, disk offload and layerwise.export_dir before calibrating. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Chenjie Luo --- CHANGELOG.rst | 2 +- examples/hf_ptq/hf_ptq.py | 13 +- modelopt/torch/export/__init__.py | 1 - modelopt/torch/export/ggml_int8_export.py | 165 ------------------ modelopt/torch/export/plugins/__init__.py | 4 + modelopt/torch/export/plugins/ggml_int8_hf.py | 153 ++++++++++++++++ ...ml_int8_export.py => test_ggml_int8_hf.py} | 5 +- .../torch/export/test_ggml_int8_export.py | 99 ----------- tests/unit/torch/export/test_ggml_int8_hf.py | 96 ++++++++++ 9 files changed, 266 insertions(+), 272 deletions(-) delete mode 100644 modelopt/torch/export/ggml_int8_export.py create mode 100644 modelopt/torch/export/plugins/ggml_int8_hf.py rename tests/gpu_vllm/torch/export/{test_ggml_int8_export.py => test_ggml_int8_hf.py} (95%) delete mode 100644 tests/unit/torch/export/test_ggml_int8_export.py create mode 100644 tests/unit/torch/export/test_ggml_int8_hf.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b756120d2c2..55d8eece99a 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,7 +11,7 @@ Changelog - On Linux x86-64 with Python 3.11 and later, the ONNX extra now selects ONNX Runtime 1.30.x and CUDA 13 packages by default. GPU calibration requires a compatible CUDA 13 runtime and driver; Python 3.10 and other platforms retain their existing dependency selections. - Add imatrix calibration (``algorithm: {method: imatrix}``) for the GGML formats with a weighted search: each weight is encoded once, its search weighted by the square root of its inputs' per-column ``E[x^2]`` (llama.cpp's imatrix statistic), and HF fused-MoE experts get one statistic per expert. A weight that already holds final codes is left alone, so ``[{method: gptq}, {method: imatrix}]`` runs GPTQ on the dense linears and imatrix on the experts GPTQ skips; the built-in IQ1 and IQ2 recipes now do this, and new ``_experts_only`` recipes apply imatrix to the routed MoE experts alone. - Add a checkpoint-specific NVIDIA Nemotron 3.5 Super VL AutoQuantize recipe with a 4.9-effective-bit NVFP4-MSE/FP8/BF16 search and fixed NVFP4 MTP experts. Select it under ``models/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16/auto_quantize/``. -- Add ``--export_ggml_as_int8`` to ``hf_ptq.py`` (and ``convert_ggml_export_to_int8`` for existing exports) to write GGML-quantized layers (IQ formats, ``q8_0``) as an INT8 weight-only compressed-tensors checkpoint. Stock vLLM serves it with its integer kernels at about half the BF16 weight memory, reproducing the GGML weights up to rounding their scales to the model dtype. +- Add ``export_hf_ggml_int8_checkpoint`` and ``hf_ptq.py --export_ggml_as_int8`` to export GGML-quantized layers (IQ formats, ``q8_0``) as an INT8 weight-only compressed-tensors checkpoint. Stock vLLM serves it with its integer kernels at about half the BF16 weight memory, reproducing the GGML weights up to rounding their scales to the model dtype. *Speculative Decoding* diff --git a/examples/hf_ptq/hf_ptq.py b/examples/hf_ptq/hf_ptq.py index 02802599657..2562137a57c 100755 --- a/examples/hf_ptq/hf_ptq.py +++ b/examples/hf_ptq/hf_ptq.py @@ -74,8 +74,8 @@ RecipeSupersededAction, ) from modelopt.torch.export import ( - convert_ggml_export_to_int8, export_hf_checkpoint, + export_hf_ggml_int8_checkpoint, export_hf_vllm_fq_checkpoint, export_speculative_decoding, has_spec_opt, @@ -671,6 +671,8 @@ def export_quantized( or "int8_smoothquant" in args.qformat ) if is_tensorrt_llm_export: + if args.export_ggml_as_int8: + raise ValueError("--export_ggml_as_int8 writes a Hugging Face checkpoint only.") if ( args.inference_tensor_parallel != 1 or args.inference_pipeline_parallel != 1 ) and args.qformat == "nvfp4_svdquant": @@ -714,6 +716,8 @@ def export_quantized( export_hf_vllm_fq_checkpoint( full_model, export_dir=export_path, inplace_mem_efficient=True ) + elif args.export_ggml_as_int8: + export_hf_ggml_int8_checkpoint(full_model, export_dir=export_path) else: # Weights the loader could not place (an MTP head, an auxiliary tower) are # carried over by the exporter from the keys recorded at load time; nothing @@ -729,8 +733,6 @@ def export_quantized( "vLLM deployment support is in progress." ) - if args.export_ggml_as_int8 and args.dist_state.is_main: - convert_ggml_export_to_int8(export_path) args.checkpoint_exported = True end_time = time.time() print_rank_0( @@ -964,6 +966,9 @@ def quantize_main( "holds the full state dict in host memory." ) if args.layerwise_export: + if args.export_ggml_as_int8: + # The per-layer pass writes the GGML shards and leaves the weights packed. + raise ValueError("--export_ggml_as_int8 does not support layerwise.export_dir.") if isinstance(recipe, ModelOptAutoQuantizeRecipe): # Only the mono-quantize path retargets export_dir and runs the refusals; # auto_quantize would export to the placeholder and skip the real export. @@ -1453,6 +1458,8 @@ def parse_args() -> argparse.Namespace: parser.error(f"--use_fsdp2 does not support --sparsity_fmt {args.sparsity_fmt}.") if args.export_ggml_as_int8 and args.vllm_fakequant_export: parser.error("--export_ggml_as_int8 and --vllm_fakequant_export are separate exports.") + if args.export_ggml_as_int8 and (args.use_fsdp2 or args.offload_folder is not None): + parser.error("--export_ggml_as_int8 does not support --use_fsdp2 or --offload_folder.") if args.use_fsdp2 and args.vllm_fakequant_export: parser.error("--use_fsdp2 does not support --vllm_fakequant_export.") if args.use_fsdp2 and args.cast_mxfp4_to_nvfp4: diff --git a/modelopt/torch/export/__init__.py b/modelopt/torch/export/__init__.py index 30c3b808f9b..44d1ad2557e 100644 --- a/modelopt/torch/export/__init__.py +++ b/modelopt/torch/export/__init__.py @@ -34,7 +34,6 @@ ) from .convert_hf_config import * -from .ggml_int8_export import * from .model_utils import * from .moe_utils import * from .plugins import * diff --git a/modelopt/torch/export/ggml_int8_export.py b/modelopt/torch/export/ggml_int8_export.py deleted file mode 100644 index 859cd7580e9..00000000000 --- a/modelopt/torch/export/ggml_int8_export.py +++ /dev/null @@ -1,165 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Rewrite a GGML Hugging Face export as an INT8 weight-only compressed-tensors checkpoint.""" - -import json -import os -from pathlib import Path - -import torch -from safetensors import safe_open -from safetensors.torch import save_file - -from modelopt.torch.quantization.ggml import GGML_FORMAT_REGISTRY, GGMLFormat -from modelopt.torch.utils.plugins.hf_checkpoint_utils import indexed_weight_map - -__all__ = ["convert_ggml_export_to_int8"] - - -def convert_ggml_export_to_int8(export_dir: str | os.PathLike) -> None: - """Rewrite a GGML export in place as INT8 weight-only compressed-tensors, which vLLM serves. - - Every GGML weight is an integer level of at most 8 bits times the step of its 16- or - 32-weight sub-block, so storing those levels with the steps as group scales reproduces the - GGML decoding up to rounding the steps to the model dtype. Unquantized tensors are kept as - they are. The checkpoint is validated before anything is rewritten. - - Args: - export_dir: A directory written by :func:`export_hf_checkpoint` whose quantized layers - are all GGML formats. - """ - export_dir = Path(export_dir) - config = json.loads((export_dir / "config.json").read_text()) - quant_config_path = export_dir / "hf_quant_config.json" - quantization = json.loads(quant_config_path.read_text())["quantization"] - scale_dtype = getattr(torch, config.get("dtype") or config["torch_dtype"]) - weight_map = indexed_weight_map(export_dir) - layer_formats = _ggml_layer_formats(export_dir, weight_map, quantization) - device = "cuda" if torch.cuda.is_available() else "cpu" - - for shard in sorted({weight_map[f"{layer}.weight"] for layer in layer_formats}): - _convert_shard(export_dir / shard, layer_formats, scale_dtype, device) - - for layer in layer_formats: - shard = weight_map.pop(f"{layer}.weight") - weight_map.update( - { - f"{layer}.{suffix}": shard - for suffix in ("weight_packed", "weight_scale", "weight_shape") - } - ) - index_path = export_dir / "model.safetensors.index.json" - if index_path.exists(): - index = json.loads(index_path.read_text()) - index["weight_map"] = dict(sorted(weight_map.items())) - index_path.write_text(json.dumps(index, indent=2)) - - config["quantization_config"] = _compressed_tensors_config(layer_formats) - (export_dir / "config.json").write_text(json.dumps(config, indent=4)) - # It describes the GGML payloads this replaced, and runtimes that read it would misload. - quant_config_path.unlink() - - -def _ggml_layer_formats( - export_dir: Path, weight_map: dict[str, str], quantization: dict -) -> dict[str, GGMLFormat]: - """The GGML format of every layer holding a packed payload, refusing anything else.""" - if quantization.get("kv_cache_quant_algo") is not None: - raise ValueError("INT8 conversion of a GGML export does not carry KV cache quantization.") - mixed = quantization.get("quant_algo") == "MIXED_PRECISION" - layer_configs = quantization.get("quantized_layers", {}) if mixed else {} - algos = ( - {cfg.get("quant_algo") for cfg in layer_configs.values()} - if mixed - else {quantization.get("quant_algo")} - ) - unsupported = sorted(str(a) for a in algos if str(a).lower() not in GGML_FORMAT_REGISTRY) - if unsupported: - raise ValueError(f"Only GGML layers convert to INT8, but the export has {unsupported}.") - - layer_formats = {} - for shard in sorted(set(weight_map.values())): - with safe_open(export_dir / shard, framework="pt") as f: - # GGML payloads are the only uint8 weights an export writes. - payloads = [ - name - for name in f.keys() # noqa: SIM118 -- a safe_open handle is not a dict - if name.endswith(".weight") and f.get_slice(name).get_dtype() == "U8" - ] - for name in payloads: - layer = name.removesuffix(".weight") - algo = layer_configs[layer]["quant_algo"] if mixed else quantization["quant_algo"] - layer_formats[layer] = GGML_FORMAT_REGISTRY[algo.lower()] - if not layer_formats: - raise ValueError(f"{export_dir} holds no GGML weights to convert.") - return layer_formats - - -def _convert_shard( - path: Path, layer_formats: dict[str, GGMLFormat], scale_dtype: torch.dtype, device: str -) -> None: - tensors = {} - with safe_open(path, framework="pt") as f: - metadata = f.metadata() - for name in f.keys(): # noqa: SIM118 -- a safe_open handle is not a dict - layer = name.removesuffix(".weight") - ggml_format = layer_formats.get(layer) if name.endswith(".weight") else None - if ggml_format is None: - tensors[name] = f.get_tensor(name) - continue - payload = f.get_tensor(name).to(device) - shape = torch.tensor((*payload.shape[:-2], payload.shape[-2] * ggml_format.block_size)) - levels, steps = ggml_format.decode_int8(payload, shape) - # compressed-tensors packs 8-bit values offset by 128, four little-endian per int32. - tensors[f"{layer}.weight_packed"] = ( - (levels.view(torch.uint8) ^ 0x80).view(torch.int32).cpu() - ) - tensors[f"{layer}.weight_scale"] = steps.to(scale_dtype).cpu() - tensors[f"{layer}.weight_shape"] = shape - tmp_path = path.with_suffix(".tmp") - save_file(tensors, tmp_path, metadata=metadata) - os.replace(tmp_path, path) - - -def _compressed_tensors_config(layer_formats: dict[str, GGMLFormat]) -> dict: - """One config group per INT8 group size, targeting its layers by name.""" - layers_by_group_size: dict[int, list[str]] = {} - for layer, ggml_format in sorted(layer_formats.items()): - layers_by_group_size.setdefault(ggml_format.int8_group_size, []).append(layer) - config_groups = { - f"group_{i}": { - "targets": layers, - "weights": { - "num_bits": 8, - "type": "int", - "symmetric": True, - "strategy": "group", - "group_size": group_size, - "dynamic": False, - }, - "input_activations": None, - "output_activations": None, - "format": "pack-quantized", - } - for i, (group_size, layers) in enumerate(sorted(layers_by_group_size.items())) - } - return { - "quant_method": "compressed-tensors", - "format": "pack-quantized", - "quantization_status": "compressed", - "config_groups": config_groups, - "ignore": [], - } diff --git a/modelopt/torch/export/plugins/__init__.py b/modelopt/torch/export/plugins/__init__.py index d54d423f38b..8f9f05e034f 100644 --- a/modelopt/torch/export/plugins/__init__.py +++ b/modelopt/torch/export/plugins/__init__.py @@ -23,5 +23,9 @@ from .hf_spec_export import * from .vllm_fakequant_hf import * +# isort: split +# Imports unified_export_hf, which imports hf_spec_export, so it must come after it. +from .ggml_int8_hf import * + with import_plugin("vllm_fakequant_megatron"): from .vllm_fakequant_megatron import * diff --git a/modelopt/torch/export/plugins/ggml_int8_hf.py b/modelopt/torch/export/plugins/ggml_int8_hf.py new file mode 100644 index 00000000000..5c8e92afcad --- /dev/null +++ b/modelopt/torch/export/plugins/ggml_int8_hf.py @@ -0,0 +1,153 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Export GGML-quantized HF models as INT8 weight-only compressed-tensors checkpoints for vLLM.""" + +import json +from pathlib import Path + +import torch +import torch.nn as nn +from safetensors import safe_open + +from modelopt.torch.quantization.ggml import GGML_FORMAT_REGISTRY, GGMLFormat +from modelopt.torch.quantization.nn import TensorQuantizer +from modelopt.torch.quantization.utils.core_utils import has_accelerate_offload +from modelopt.torch.utils.distributed import is_fsdp2_model +from modelopt.torch.utils.plugins.hf_checkpoint_utils import ( + copy_off_index_safetensors, + indexed_weight_map, +) + +from ..unified_export_hf import _copy_non_model_files_from_source, _source_checkpoint +from .vllm_fakequant_hf import _carry_over_unplaced_weights, _copy_source_config + +__all__ = ["export_hf_ggml_int8_checkpoint"] + + +def export_hf_ggml_int8_checkpoint(model: nn.Module, export_dir: Path | str) -> None: + """Export a GGML-quantized model as an INT8 weight-only compressed-tensors checkpoint. + + Every GGML weight is an integer level of at most 8 bits times the step of its 16- or + 32-weight sub-block, so storing those levels with the steps as group scales reproduces the + GGML weights up to rounding the steps to the model dtype, and vLLM serves the checkpoint + with its integer weight-only kernels. Unquantized weights are written as they are. Every + enabled quantizer must be a GGML weight quantizer, which is checked before anything is + written. + + Args: + model: In-memory model quantized with GGML weight formats. + export_dir: Output directory. + """ + if has_accelerate_offload(model) or is_fsdp2_model(model): + raise NotImplementedError( + "GGML INT8 export needs every weight resident; offloaded and FSDP2 models are not " + "supported." + ) + ggml_layers = _ggml_layers(model) + export_dir = Path(export_dir) + export_dir.mkdir(parents=True, exist_ok=True) + + state_dict = {k: v for k, v in model.state_dict().items() if "quantizer" not in k} + with torch.inference_mode(): + for name, (weight, quantizer, ggml_format) in ggml_layers.items(): + shape = torch.tensor(weight.shape) + # pack() returns the payload fake quant decoded, including codes GPTQ pinned. + levels, steps = ggml_format.decode_int8(ggml_format.pack(weight, quantizer), shape) + del state_dict[f"{name}.weight"] + # compressed-tensors packs 8-bit values offset by 128, four little-endian per int32. + state_dict[f"{name}.weight_packed"] = ( + (levels.view(torch.uint8) ^ 0x80).view(torch.int32).cpu() + ) + state_dict[f"{name}.weight_scale"] = steps.to(weight.dtype).cpu() + state_dict[f"{name}.weight_shape"] = shape + model.save_pretrained(export_dir, state_dict=state_dict, save_modelopt_state=False) + _carry_over_unplaced_weights(export_dir, model) + copy_off_index_safetensors(_source_checkpoint(model), export_dir) + _copy_non_model_files_from_source(model, export_dir) + _copy_source_config(model, export_dir) + + config_path = export_dir / "config.json" + config = json.loads(config_path.read_text()) + config["quantization_config"] = _compressed_tensors_config(export_dir) + config_path.write_text(json.dumps(config, indent=4)) + + +def _ggml_layers(model: nn.Module) -> dict[str, tuple[torch.Tensor, TensorQuantizer, GGMLFormat]]: + """Each GGML-quantized layer's weight, quantizer and format, refusing any other quantizer.""" + layers, unsupported = {}, [] + for name, quantizer in model.named_modules(): + if not (isinstance(quantizer, TensorQuantizer) and quantizer.is_enabled): + continue + layer, _, attr = name.rpartition(".") + weight = getattr(model.get_submodule(layer), "weight", None) + num_bits = quantizer.num_bits + ggml_format = GGML_FORMAT_REGISTRY.get(num_bits) if isinstance(num_bits, str) else None + if attr != "weight_quantizer" or ggml_format is None or weight is None or weight.dim() != 2: + unsupported.append(name) + continue + layers[layer] = (weight, quantizer, ggml_format) + if unsupported: + raise ValueError( + "GGML INT8 export writes only GGML weight quantizers of 2-D weights, but these " + f"quantizers are enabled too: {unsupported[:8]}" + ) + if not layers: + raise ValueError("The model has no GGML-quantized weights to export.") + return layers + + +def _compressed_tensors_config(export_dir: Path) -> dict: + """One config group per INT8 group size, targeting the layers by their saved names. + + The names are read back from the checkpoint because saving can rename modules; vLLM maps + explicit layer names onto its own modules, but not ``re:`` patterns. + """ + shapes = {} + for shard in sorted(set(indexed_weight_map(export_dir).values())): + with safe_open(export_dir / shard, framework="pt") as f: + for key in f.keys(): # noqa: SIM118 -- a safe_open handle is not a dict + if key.endswith((".weight_packed", ".weight_scale")): + shapes[key] = f.get_slice(key).get_shape() + layers_by_group_size: dict[int, list[str]] = {} + for key, shape in sorted(shapes.items()): + if key.endswith(".weight_packed"): + layer = key.removesuffix(".weight_packed") + group_size = shape[-1] * 4 // shapes[f"{layer}.weight_scale"][-1] + layers_by_group_size.setdefault(group_size, []).append(layer) + config_groups = { + f"group_{i}": { + "targets": layers, + "weights": { + "num_bits": 8, + "type": "int", + "symmetric": True, + "strategy": "group", + "group_size": group_size, + "dynamic": False, + }, + "input_activations": None, + "output_activations": None, + "format": "pack-quantized", + } + for i, (group_size, layers) in enumerate(sorted(layers_by_group_size.items())) + } + return { + "quant_method": "compressed-tensors", + "format": "pack-quantized", + "quantization_status": "compressed", + "config_groups": config_groups, + "ignore": [], + } diff --git a/tests/gpu_vllm/torch/export/test_ggml_int8_export.py b/tests/gpu_vllm/torch/export/test_ggml_int8_hf.py similarity index 95% rename from tests/gpu_vllm/torch/export/test_ggml_int8_export.py rename to tests/gpu_vllm/torch/export/test_ggml_int8_hf.py index 81af0fed915..dca196ad876 100644 --- a/tests/gpu_vllm/torch/export/test_ggml_int8_export.py +++ b/tests/gpu_vllm/torch/export/test_ggml_int8_hf.py @@ -24,7 +24,7 @@ from vllm.inputs import TokensPrompt import modelopt.torch.quantization as mtq -from modelopt.torch.export import convert_ggml_export_to_int8, export_hf_checkpoint +from modelopt.torch.export import export_hf_ggml_int8_checkpoint @pytest.fixture(scope="module") @@ -65,8 +65,7 @@ def int8_export_in_vllm(tmp_path_factory): expected = logits.log_softmax(-1).gather(-1, token_ids[0, 1:, None])[:, 0] export_dir = tmp_path / "export" - export_hf_checkpoint(model, export_dir=export_dir) - convert_ggml_export_to_int8(export_dir) + export_hf_ggml_int8_checkpoint(model, export_dir) llm = LLM( model=str(export_dir), diff --git a/tests/unit/torch/export/test_ggml_int8_export.py b/tests/unit/torch/export/test_ggml_int8_export.py deleted file mode 100644 index 5ab8f3d6dc8..00000000000 --- a/tests/unit/torch/export/test_ggml_int8_export.py +++ /dev/null @@ -1,99 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json - -import pytest -import torch -from _test_utils.torch.transformers_models import create_tiny_llama_dir -from safetensors.torch import load_file -from transformers import AutoModelForCausalLM - -import modelopt.torch.quantization as mtq -from modelopt.torch.export import convert_ggml_export_to_int8, export_hf_checkpoint -from modelopt.torch.quantization.ggml import GGML_FORMAT_REGISTRY - -# Layer 0 takes a format with 32-weight sub-blocks and layer 1 one with 16, so the export needs -# one compressed-tensors config group per group size. -LAYER_FORMATS = {0: "iq1_s", 1: "iq2_xs"} - - -def _export(tmp_path, second_layer_cfg): - source = create_tiny_llama_dir(tmp_path, hidden_size=256, intermediate_size=512) - model = AutoModelForCausalLM.from_pretrained(source, dtype=torch.bfloat16) - layer_cfgs = [ - {"num_bits": "iq1_s", "block_sizes": {-1: 256}, "backend": "ggml"}, - second_layer_cfg, - ] - quant_cfg = { - "quant_cfg": [ - {"quantizer_name": "*", "enable": False}, - *( - {"quantizer_name": f"*layers.{i}.mlp*weight_quantizer", "cfg": cfg} - for i, cfg in enumerate(layer_cfgs) - ), - ], - "algorithm": "max", - } - mtq.quantize(model, quant_cfg, lambda m: m(torch.randint(0, 32, (1, 8)))) - export_dir = tmp_path / "export" - export_hf_checkpoint(model, export_dir=export_dir) - return export_dir - - -def test_ggml_export_converts_to_int8_levels_and_steps(tmp_path): - export_dir = _export( - tmp_path, {"num_bits": "iq2_xs", "block_sizes": {-1: 256}, "backend": "ggml"} - ) - before = load_file(export_dir / "model.safetensors") - - convert_ggml_export_to_int8(export_dir) - - config = json.loads((export_dir / "config.json").read_text())["quantization_config"] - assert config["quant_method"] == "compressed-tensors" - layers = { - i: [f"model.layers.{i}.mlp.{proj}" for proj in ("down_proj", "gate_proj", "up_proj")] - for i in LAYER_FORMATS - } - assert { - group["weights"]["group_size"]: group["targets"] - for group in config["config_groups"].values() - } == {32: layers[0], 16: layers[1]} - assert not (export_dir / "hf_quant_config.json").exists() - - after = load_file(export_dir / "model.safetensors") - for i, name in LAYER_FORMATS.items(): - for layer in layers[i]: - payload = before.pop(f"{layer}.weight") - shape = after[f"{layer}.weight_shape"] - levels, steps = GGML_FORMAT_REGISTRY[name].decode_int8(payload, shape) - # compressed-tensors' layout: offset by 128, four values per int32, lowest byte first. - packed = after[f"{layer}.weight_packed"] - unpacked = torch.stack([(packed >> (8 * k)) & 0xFF for k in range(4)], -1).flatten(1) - assert torch.equal(unpacked - 128, levels.long()) - assert torch.equal(after[f"{layer}.weight_scale"], steps.to(torch.bfloat16)) - assert shape.tolist() == list(levels.shape) - for name, tensor in before.items(): - assert torch.equal(after[name], tensor) - - -def test_export_with_a_non_ggml_layer_is_refused_unchanged(tmp_path): - export_dir = _export(tmp_path, {"num_bits": (4, 3), "axis": None}) - files = {path.name: path.read_bytes() for path in export_dir.iterdir()} - - with pytest.raises(ValueError, match="Only GGML layers"): - convert_ggml_export_to_int8(export_dir) - - assert {path.name: path.read_bytes() for path in export_dir.iterdir()} == files diff --git a/tests/unit/torch/export/test_ggml_int8_hf.py b/tests/unit/torch/export/test_ggml_int8_hf.py new file mode 100644 index 00000000000..e84b55e3e60 --- /dev/null +++ b/tests/unit/torch/export/test_ggml_int8_hf.py @@ -0,0 +1,96 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +import pytest +import torch +from _test_utils.torch.transformers_models import create_tiny_llama_dir +from safetensors.torch import load_file +from transformers import AutoModelForCausalLM + +import modelopt.torch.quantization as mtq +from modelopt.torch.export import export_hf_ggml_int8_checkpoint +from modelopt.torch.quantization.ggml import GGML_FORMAT_REGISTRY + +IQ1_S = {"num_bits": "iq1_s", "block_sizes": {-1: 256}, "backend": "ggml"} +IQ2_XS = {"num_bits": "iq2_xs", "block_sizes": {-1: 256}, "backend": "ggml"} +PROJECTIONS = ("down_proj", "gate_proj", "up_proj") + + +def _quantized_tiny_llama(tmp_path, layer_cfgs): + source = create_tiny_llama_dir(tmp_path, hidden_size=256, intermediate_size=256) + model = AutoModelForCausalLM.from_pretrained(source, dtype=torch.bfloat16) + quant_cfg = { + "quant_cfg": [ + {"quantizer_name": "*", "enable": False}, + *( + {"quantizer_name": f"*layers.{i}.mlp*weight_quantizer", "cfg": cfg} + for i, cfg in enumerate(layer_cfgs) + ), + ], + "algorithm": "max", + } + token_ids = torch.randint(0, 32, (1, 8)) + mtq.quantize(model, quant_cfg, lambda m: m(token_ids)) + # A fake-quant forward caches each payload, so the test and the export encode it once. + model(token_ids) + return model + + +def test_ggml_layers_export_as_their_int8_levels_and_steps(tmp_path): + # A format with 32-weight sub-blocks and one with 16 need one config group each. + model = _quantized_tiny_llama(tmp_path, [IQ1_S, IQ2_XS]) + layers = {i: [f"model.layers.{i}.mlp.{proj}" for proj in PROJECTIONS] for i in (0, 1)} + expected = {} + for layer in layers[0] + layers[1]: + module = model.get_submodule(layer) + ggml_format = GGML_FORMAT_REGISTRY[module.weight_quantizer.num_bits] + payload = ggml_format.pack(module.weight, module.weight_quantizer) + expected[layer] = ggml_format.decode_int8(payload, torch.tensor(module.weight.shape)) + unquantized = { + k: v.clone() + for k, v in model.state_dict().items() + if "quantizer" not in k and k.removesuffix(".weight") not in expected + } + + export_hf_ggml_int8_checkpoint(model, tmp_path / "export") + + config = json.loads((tmp_path / "export" / "config.json").read_text())["quantization_config"] + assert config["quant_method"] == "compressed-tensors" + assert { + group["weights"]["group_size"]: group["targets"] + for group in config["config_groups"].values() + } == {32: layers[0], 16: layers[1]} + saved = load_file(tmp_path / "export" / "model.safetensors") + for layer, (levels, steps) in expected.items(): + # compressed-tensors' layout: offset by 128, four values per int32, lowest byte first. + packed = saved.pop(f"{layer}.weight_packed") + unpacked = torch.stack([(packed >> (8 * k)) & 0xFF for k in range(4)], -1).flatten(1) + assert torch.equal(unpacked - 128, levels.long()) + assert torch.equal(saved.pop(f"{layer}.weight_scale"), steps.to(torch.bfloat16)) + assert saved.pop(f"{layer}.weight_shape").tolist() == list(levels.shape) + assert saved.keys() <= unquantized.keys() + for name, tensor in saved.items(): + assert torch.equal(tensor, unquantized[name]) + + +def test_a_non_ggml_quantizer_is_refused_before_writing(tmp_path): + model = _quantized_tiny_llama(tmp_path, [IQ1_S, {"num_bits": (4, 3), "axis": None}]) + + with pytest.raises(ValueError, match="only GGML weight quantizers"): + export_hf_ggml_int8_checkpoint(model, tmp_path / "export") + + assert not (tmp_path / "export").exists() From 94a42beffbf02aae399fea5535dff174d8736c9d Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Wed, 21 Oct 2026 06:20:42 +0000 Subject: [PATCH 3/7] Convert GGML exports to INT8 with a script beside hf_ptq.py, not an export flag Drop --export_ggml_as_int8 and the in-memory exporter. Like the vLLM fake-quant in-place export, it saved through save_pretrained, which fails on offloaded quantized models ('SequentialHook' object has no attribute 'weights_map'), so it could not serve the large models INT8 is for. The regular GGML export already handles offloading, FSDP2 and layerwise export. examples/hf_ptq/convert_ggml_to_int8.py, a thin wrapper over the new convert_ggml_export_to_int8(ggml_dir, output_dir), expands that export into a new directory as INT8 compressed-tensors, streaming tensors into output shards of bounded size. hf_ptq.py is back to main, and the README documents the two steps. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Chenjie Luo --- CHANGELOG.rst | 2 +- examples/hf_ptq/README.md | 16 +- examples/hf_ptq/convert_ggml_to_int8.py | 45 ++++ examples/hf_ptq/hf_ptq.py | 23 -- modelopt/torch/export/__init__.py | 1 + modelopt/torch/export/ggml_int8_export.py | 209 ++++++++++++++++++ modelopt/torch/export/plugins/__init__.py | 4 - modelopt/torch/export/plugins/ggml_int8_hf.py | 153 ------------- ...ml_int8_hf.py => test_ggml_int8_export.py} | 7 +- .../torch/export/test_ggml_int8_export.py | 108 +++++++++ tests/unit/torch/export/test_ggml_int8_hf.py | 96 -------- 11 files changed, 383 insertions(+), 281 deletions(-) create mode 100644 examples/hf_ptq/convert_ggml_to_int8.py create mode 100644 modelopt/torch/export/ggml_int8_export.py delete mode 100644 modelopt/torch/export/plugins/ggml_int8_hf.py rename tests/gpu_vllm/torch/export/{test_ggml_int8_hf.py => test_ggml_int8_export.py} (93%) create mode 100644 tests/unit/torch/export/test_ggml_int8_export.py delete mode 100644 tests/unit/torch/export/test_ggml_int8_hf.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 55d8eece99a..ede632aa3ab 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,7 +11,7 @@ Changelog - On Linux x86-64 with Python 3.11 and later, the ONNX extra now selects ONNX Runtime 1.30.x and CUDA 13 packages by default. GPU calibration requires a compatible CUDA 13 runtime and driver; Python 3.10 and other platforms retain their existing dependency selections. - Add imatrix calibration (``algorithm: {method: imatrix}``) for the GGML formats with a weighted search: each weight is encoded once, its search weighted by the square root of its inputs' per-column ``E[x^2]`` (llama.cpp's imatrix statistic), and HF fused-MoE experts get one statistic per expert. A weight that already holds final codes is left alone, so ``[{method: gptq}, {method: imatrix}]`` runs GPTQ on the dense linears and imatrix on the experts GPTQ skips; the built-in IQ1 and IQ2 recipes now do this, and new ``_experts_only`` recipes apply imatrix to the routed MoE experts alone. - Add a checkpoint-specific NVIDIA Nemotron 3.5 Super VL AutoQuantize recipe with a 4.9-effective-bit NVFP4-MSE/FP8/BF16 search and fixed NVFP4 MTP experts. Select it under ``models/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16/auto_quantize/``. -- Add ``export_hf_ggml_int8_checkpoint`` and ``hf_ptq.py --export_ggml_as_int8`` to export GGML-quantized layers (IQ formats, ``q8_0``) as an INT8 weight-only compressed-tensors checkpoint. Stock vLLM serves it with its integer kernels at about half the BF16 weight memory, reproducing the GGML weights up to rounding their scales to the model dtype. +- Add ``examples/hf_ptq/convert_ggml_to_int8.py`` (``convert_ggml_export_to_int8``) to expand a checkpoint exported with a GGML recipe (IQ formats, ``q8_0``) into an INT8 weight-only compressed-tensors checkpoint. Stock vLLM serves it with its integer kernels at about half the BF16 weight memory, reproducing the GGML weights up to rounding their scales to the model dtype. *Speculative Decoding* diff --git a/examples/hf_ptq/README.md b/examples/hf_ptq/README.md index da52d4de1b6..0815c2301d0 100755 --- a/examples/hf_ptq/README.md +++ b/examples/hf_ptq/README.md @@ -675,7 +675,7 @@ with torch.inference_mode(): python hf_ptq.py --pyt_ckpt_path --qformat fp8 --export_path --trust_remote_code ``` -> *For exporting fake-quantized models for vLLM serving (e.g., for research or kernels not yet supported in real-quant), use the `--vllm_fakequant_export` flag. See [vllm_serve/README.md](../vllm_serve/README.md) for details. When every quantizer is a weight quantizer, as in the GGML recipes (IQ, Q8_0), the export is a plain BF16 checkpoint that a stock vLLM image serves directly. For GGML recipes, `--export_ggml_as_int8` instead writes each weight's GGML integer levels and sub-block scales as an INT8 weight-only compressed-tensors checkpoint: the same weights up to scale rounding, served by vLLM's integer kernels in about half the memory.* +> *For exporting fake-quantized models for vLLM serving (e.g., for research or kernels not yet supported in real-quant), use the `--vllm_fakequant_export` flag. See [vllm_serve/README.md](../vllm_serve/README.md) for details. When every quantizer is a weight quantizer, as in the GGML recipes (IQ, Q8_0), the export is a plain BF16 checkpoint that a stock vLLM image serves directly.* ### Hugging Face framework [Script](./scripts/huggingface_example.sh) @@ -707,6 +707,20 @@ llm_fp8 = LLM(model="", quantization="modelopt") print(llm_fp8.generate(["What's the age of the earth? "])) ``` +#### vLLM with GGML recipes (IQ, Q8_0) + +vLLM has no GGML kernels. To serve a checkpoint exported with a GGML recipe on vLLM's integer kernels, expand it into an INT8 weight-only compressed-tensors checkpoint with [convert_ggml_to_int8.py](./convert_ggml_to_int8.py): + +```bash +python hf_ptq.py --pyt_ckpt_path --recipe general/ptq/iq1_s --export_path +python convert_ggml_to_int8.py +vllm serve +``` + +Each weight keeps its GGML integer level, and the step of its 16- or 32-weight sub-block becomes the INT8 group scale. The INT8 checkpoint therefore holds the same weights up to rounding the scales to the model dtype, at about half the BF16 weight memory. The GGML checkpoint is left unchanged. + +vLLM serves group-32 layers (IQ1_S, IQ2_XXS, Q8_0) with Marlin and group-16 layers (IQ1_M, IQ2_XS, IQ2_S) with Humming. On Blackwell GPUs, Marlin is much slower at large batch sizes, so set `VLLM_DISABLED_KERNELS=MarlinLinearKernel` to use Humming for group 32 as well. + #### SGLang ```python diff --git a/examples/hf_ptq/convert_ggml_to_int8.py b/examples/hf_ptq/convert_ggml_to_int8.py new file mode 100644 index 00000000000..2a0ecd8b2fb --- /dev/null +++ b/examples/hf_ptq/convert_ggml_to_int8.py @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Expand a GGML checkpoint exported by hf_ptq.py into an INT8 checkpoint that vLLM serves. + +python hf_ptq.py --pyt_ckpt_path --recipe general/ptq/iq1_s --export_path +python convert_ggml_to_int8.py +vllm serve +""" + +import argparse + +from modelopt.torch.export import convert_ggml_export_to_int8 + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("ggml_dir", help="Checkpoint exported with a GGML recipe (IQ, q8_0).") + parser.add_argument("output_dir", help="Where to write the INT8 checkpoint; must be empty.") + parser.add_argument( + "--max_shard_size_gb", type=float, default=10, help="Size of each output shard in GiB." + ) + args = parser.parse_args() + convert_ggml_export_to_int8( + args.ggml_dir, args.output_dir, max_shard_size=int(args.max_shard_size_gb * 2**30) + ) + print(f"INT8 checkpoint written to {args.output_dir}") + + +if __name__ == "__main__": + main() diff --git a/examples/hf_ptq/hf_ptq.py b/examples/hf_ptq/hf_ptq.py index 2562137a57c..d7a14a68192 100755 --- a/examples/hf_ptq/hf_ptq.py +++ b/examples/hf_ptq/hf_ptq.py @@ -75,7 +75,6 @@ ) from modelopt.torch.export import ( export_hf_checkpoint, - export_hf_ggml_int8_checkpoint, export_hf_vllm_fq_checkpoint, export_speculative_decoding, has_spec_opt, @@ -671,8 +670,6 @@ def export_quantized( or "int8_smoothquant" in args.qformat ) if is_tensorrt_llm_export: - if args.export_ggml_as_int8: - raise ValueError("--export_ggml_as_int8 writes a Hugging Face checkpoint only.") if ( args.inference_tensor_parallel != 1 or args.inference_pipeline_parallel != 1 ) and args.qformat == "nvfp4_svdquant": @@ -716,8 +713,6 @@ def export_quantized( export_hf_vllm_fq_checkpoint( full_model, export_dir=export_path, inplace_mem_efficient=True ) - elif args.export_ggml_as_int8: - export_hf_ggml_int8_checkpoint(full_model, export_dir=export_path) else: # Weights the loader could not place (an MTP head, an auxiliary tower) are # carried over by the exporter from the keys recorded at load time; nothing @@ -966,9 +961,6 @@ def quantize_main( "holds the full state dict in host memory." ) if args.layerwise_export: - if args.export_ggml_as_int8: - # The per-layer pass writes the GGML shards and leaves the weights packed. - raise ValueError("--export_ggml_as_int8 does not support layerwise.export_dir.") if isinstance(recipe, ModelOptAutoQuantizeRecipe): # Only the mono-quantize path retargets export_dir and runs the refusals; # auto_quantize would export to the placeholder and skip the real export. @@ -1365,17 +1357,6 @@ def parse_args() -> argparse.Namespace: help="Export as vLLM fake-quant checkpoint (produces vllm_fq_modelopt_state.pth " "for use with vllm_serve_fakequant.py).", ) - parser.add_argument( - "--export_ggml_as_int8", - default=False, - action="store_true", - help=( - "Export GGML-quantized layers (IQ1/IQ2 formats, Q8_0) as an INT8 weight-only " - "compressed-tensors checkpoint that vLLM serves with its integer kernels. Each weight " - "keeps its GGML integer level and sub-block scale, so this reproduces the GGML " - "weights up to rounding the scales to the model dtype." - ), - ) parser.add_argument( "--cast_mxfp4_to_nvfp4", action="store_true", @@ -1456,10 +1437,6 @@ def parse_args() -> argparse.Namespace: parser.error("--cpu_offload requires --use_fsdp2") if args.use_fsdp2 and args.sparsity_fmt != "dense": parser.error(f"--use_fsdp2 does not support --sparsity_fmt {args.sparsity_fmt}.") - if args.export_ggml_as_int8 and args.vllm_fakequant_export: - parser.error("--export_ggml_as_int8 and --vllm_fakequant_export are separate exports.") - if args.export_ggml_as_int8 and (args.use_fsdp2 or args.offload_folder is not None): - parser.error("--export_ggml_as_int8 does not support --use_fsdp2 or --offload_folder.") if args.use_fsdp2 and args.vllm_fakequant_export: parser.error("--use_fsdp2 does not support --vllm_fakequant_export.") if args.use_fsdp2 and args.cast_mxfp4_to_nvfp4: diff --git a/modelopt/torch/export/__init__.py b/modelopt/torch/export/__init__.py index 44d1ad2557e..30c3b808f9b 100644 --- a/modelopt/torch/export/__init__.py +++ b/modelopt/torch/export/__init__.py @@ -34,6 +34,7 @@ ) from .convert_hf_config import * +from .ggml_int8_export import * from .model_utils import * from .moe_utils import * from .plugins import * diff --git a/modelopt/torch/export/ggml_int8_export.py b/modelopt/torch/export/ggml_int8_export.py new file mode 100644 index 00000000000..eb309740bd0 --- /dev/null +++ b/modelopt/torch/export/ggml_int8_export.py @@ -0,0 +1,209 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Expand a GGML Hugging Face export into an INT8 weight-only compressed-tensors checkpoint.""" + +import json +import os +import shutil +from pathlib import Path + +import torch +from safetensors import safe_open +from safetensors.torch import save_file + +from modelopt.torch.quantization.ggml import GGML_FORMAT_REGISTRY, GGMLFormat +from modelopt.torch.utils.plugins.hf_checkpoint_utils import indexed_weight_map + +__all__ = ["convert_ggml_export_to_int8"] + +_INDEX = "model.safetensors.index.json" + + +def convert_ggml_export_to_int8( + ggml_dir: str | os.PathLike, + output_dir: str | os.PathLike, + max_shard_size: int = 10 * 2**30, +) -> None: + """Write a GGML export as an INT8 weight-only compressed-tensors checkpoint, which vLLM serves. + + Every GGML weight is an integer level of at most 8 bits times the step of its 16- or + 32-weight sub-block, so storing those levels with the steps as group scales reproduces the + GGML weights up to rounding the steps to the model dtype. Unquantized tensors and the other + files are copied as they are; the GGML export is left unchanged. + + Args: + ggml_dir: A Hugging Face export whose quantized layers are all GGML formats, as written + by :func:`export_hf_checkpoint`. + output_dir: Where to write the INT8 checkpoint. Must be empty or not exist yet. + max_shard_size: Bytes per output safetensors shard, which bounds the memory conversion uses. + """ + ggml_dir, output_dir = Path(ggml_dir), Path(output_dir) + config = json.loads((ggml_dir / "config.json").read_text()) + quantization = json.loads((ggml_dir / "hf_quant_config.json").read_text())["quantization"] + scale_dtype = getattr(torch, config.get("dtype") or config["torch_dtype"]) + weight_map = indexed_weight_map(ggml_dir) + layer_formats = _ggml_layer_formats(ggml_dir, weight_map, quantization) + if output_dir.exists() and any(output_dir.iterdir()): + raise ValueError(f"{output_dir} is not empty.") + output_dir.mkdir(parents=True, exist_ok=True) + + shards = sorted(set(weight_map.values())) + for path in ggml_dir.iterdir(): + # The shards are rewritten below, and the GGML metadata does not describe the result. + if path.is_file() and path.name not in {*shards, _INDEX, "hf_quant_config.json"}: + shutil.copy2(path, output_dir / path.name) + new_weight_map = _write_int8_shards( + ggml_dir, shards, output_dir, layer_formats, scale_dtype, max_shard_size + ) + total_size = sum((output_dir / shard).stat().st_size for shard in set(new_weight_map.values())) + index = {"metadata": {"total_size": total_size}, "weight_map": new_weight_map} + (output_dir / _INDEX).write_text(json.dumps(index, indent=2)) + + config["quantization_config"] = _compressed_tensors_config(layer_formats) + (output_dir / "config.json").write_text(json.dumps(config, indent=4)) + + +def _ggml_layer_formats( + ggml_dir: Path, weight_map: dict[str, str], quantization: dict +) -> dict[str, GGMLFormat]: + """The GGML format of every layer holding a packed payload, refusing anything else.""" + if quantization.get("kv_cache_quant_algo") is not None: + raise ValueError("INT8 conversion of a GGML export does not carry KV cache quantization.") + mixed = quantization.get("quant_algo") == "MIXED_PRECISION" + layer_configs = quantization.get("quantized_layers", {}) if mixed else {} + algos = ( + {cfg.get("quant_algo") for cfg in layer_configs.values()} + if mixed + else {quantization.get("quant_algo")} + ) + unsupported = sorted(str(a) for a in algos if str(a).lower() not in GGML_FORMAT_REGISTRY) + if unsupported: + raise ValueError(f"Only GGML layers convert to INT8, but the export has {unsupported}.") + + layer_formats = {} + for shard in sorted(set(weight_map.values())): + with safe_open(ggml_dir / shard, framework="pt") as f: + # GGML payloads are the only uint8 weights an export writes. + payloads = [ + name + for name in f.keys() # noqa: SIM118 -- a safe_open handle is not a dict + if name.endswith(".weight") and f.get_slice(name).get_dtype() == "U8" + ] + for name in payloads: + layer = name.removesuffix(".weight") + algo = layer_configs[layer]["quant_algo"] if mixed else quantization["quant_algo"] + layer_formats[layer] = GGML_FORMAT_REGISTRY[algo.lower()] + if not layer_formats: + raise ValueError(f"{ggml_dir} holds no GGML weights to convert.") + return layer_formats + + +def _write_int8_shards( + ggml_dir: Path, + shards: list[str], + output_dir: Path, + layer_formats: dict[str, GGMLFormat], + scale_dtype: torch.dtype, + max_shard_size: int, +) -> dict[str, str]: + """Rewrite every shard's tensors into size-bounded output shards; return the weight map.""" + device = "cuda" if torch.cuda.is_available() else "cpu" + pending: dict[str, torch.Tensor] = {} + written: list[list[str]] = [] + + def flush(): + save_file(pending, output_dir / f"part-{len(written):05d}.safetensors", {"format": "pt"}) + written.append(list(pending)) + pending.clear() + + for shard in shards: + with safe_open(ggml_dir / shard, framework="pt") as f: + for name in f.keys(): # noqa: SIM118 -- a safe_open handle is not a dict + layer = name.removesuffix(".weight") + ggml_format = layer_formats.get(layer) if name.endswith(".weight") else None + if ggml_format is None: + tensors = {name: f.get_tensor(name)} + else: + tensors = _int8_tensors( + layer, f.get_tensor(name), ggml_format, scale_dtype, device + ) + if pending and _nbytes(pending) + _nbytes(tensors) > max_shard_size: + flush() + pending.update(tensors) + flush() + + weight_map = {} + for i, names in enumerate(written): + shard_name = f"model-{i + 1:05d}-of-{len(written):05d}.safetensors" + os.replace(output_dir / f"part-{i:05d}.safetensors", output_dir / shard_name) + weight_map.update(dict.fromkeys(names, shard_name)) + return dict(sorted(weight_map.items())) + + +def _int8_tensors( + layer: str, + payload: torch.Tensor, + ggml_format: GGMLFormat, + scale_dtype: torch.dtype, + device: str, +) -> dict[str, torch.Tensor]: + """A layer's GGML payload as compressed-tensors ``weight_packed``, ``weight_scale`` and shape.""" + shape = torch.tensor((*payload.shape[:-2], payload.shape[-2] * ggml_format.block_size)) + levels, steps = ggml_format.decode_int8(payload.to(device), shape) + return { + # compressed-tensors packs 8-bit values offset by 128, four little-endian per int32. + f"{layer}.weight_packed": (levels.view(torch.uint8) ^ 0x80).view(torch.int32).cpu(), + f"{layer}.weight_scale": steps.to(scale_dtype).cpu(), + f"{layer}.weight_shape": shape, + } + + +def _nbytes(tensors: dict[str, torch.Tensor]) -> int: + return sum(t.numel() * t.element_size() for t in tensors.values()) + + +def _compressed_tensors_config(layer_formats: dict[str, GGMLFormat]) -> dict: + """One config group per INT8 group size, targeting its layers by name. + + vLLM maps explicit layer names onto its own modules, but not ``re:`` patterns. + """ + layers_by_group_size: dict[int, list[str]] = {} + for layer, ggml_format in sorted(layer_formats.items()): + layers_by_group_size.setdefault(ggml_format.int8_group_size, []).append(layer) + config_groups = { + f"group_{i}": { + "targets": layers, + "weights": { + "num_bits": 8, + "type": "int", + "symmetric": True, + "strategy": "group", + "group_size": group_size, + "dynamic": False, + }, + "input_activations": None, + "output_activations": None, + "format": "pack-quantized", + } + for i, (group_size, layers) in enumerate(sorted(layers_by_group_size.items())) + } + return { + "quant_method": "compressed-tensors", + "format": "pack-quantized", + "quantization_status": "compressed", + "config_groups": config_groups, + "ignore": [], + } diff --git a/modelopt/torch/export/plugins/__init__.py b/modelopt/torch/export/plugins/__init__.py index 8f9f05e034f..d54d423f38b 100644 --- a/modelopt/torch/export/plugins/__init__.py +++ b/modelopt/torch/export/plugins/__init__.py @@ -23,9 +23,5 @@ from .hf_spec_export import * from .vllm_fakequant_hf import * -# isort: split -# Imports unified_export_hf, which imports hf_spec_export, so it must come after it. -from .ggml_int8_hf import * - with import_plugin("vllm_fakequant_megatron"): from .vllm_fakequant_megatron import * diff --git a/modelopt/torch/export/plugins/ggml_int8_hf.py b/modelopt/torch/export/plugins/ggml_int8_hf.py deleted file mode 100644 index 5c8e92afcad..00000000000 --- a/modelopt/torch/export/plugins/ggml_int8_hf.py +++ /dev/null @@ -1,153 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Export GGML-quantized HF models as INT8 weight-only compressed-tensors checkpoints for vLLM.""" - -import json -from pathlib import Path - -import torch -import torch.nn as nn -from safetensors import safe_open - -from modelopt.torch.quantization.ggml import GGML_FORMAT_REGISTRY, GGMLFormat -from modelopt.torch.quantization.nn import TensorQuantizer -from modelopt.torch.quantization.utils.core_utils import has_accelerate_offload -from modelopt.torch.utils.distributed import is_fsdp2_model -from modelopt.torch.utils.plugins.hf_checkpoint_utils import ( - copy_off_index_safetensors, - indexed_weight_map, -) - -from ..unified_export_hf import _copy_non_model_files_from_source, _source_checkpoint -from .vllm_fakequant_hf import _carry_over_unplaced_weights, _copy_source_config - -__all__ = ["export_hf_ggml_int8_checkpoint"] - - -def export_hf_ggml_int8_checkpoint(model: nn.Module, export_dir: Path | str) -> None: - """Export a GGML-quantized model as an INT8 weight-only compressed-tensors checkpoint. - - Every GGML weight is an integer level of at most 8 bits times the step of its 16- or - 32-weight sub-block, so storing those levels with the steps as group scales reproduces the - GGML weights up to rounding the steps to the model dtype, and vLLM serves the checkpoint - with its integer weight-only kernels. Unquantized weights are written as they are. Every - enabled quantizer must be a GGML weight quantizer, which is checked before anything is - written. - - Args: - model: In-memory model quantized with GGML weight formats. - export_dir: Output directory. - """ - if has_accelerate_offload(model) or is_fsdp2_model(model): - raise NotImplementedError( - "GGML INT8 export needs every weight resident; offloaded and FSDP2 models are not " - "supported." - ) - ggml_layers = _ggml_layers(model) - export_dir = Path(export_dir) - export_dir.mkdir(parents=True, exist_ok=True) - - state_dict = {k: v for k, v in model.state_dict().items() if "quantizer" not in k} - with torch.inference_mode(): - for name, (weight, quantizer, ggml_format) in ggml_layers.items(): - shape = torch.tensor(weight.shape) - # pack() returns the payload fake quant decoded, including codes GPTQ pinned. - levels, steps = ggml_format.decode_int8(ggml_format.pack(weight, quantizer), shape) - del state_dict[f"{name}.weight"] - # compressed-tensors packs 8-bit values offset by 128, four little-endian per int32. - state_dict[f"{name}.weight_packed"] = ( - (levels.view(torch.uint8) ^ 0x80).view(torch.int32).cpu() - ) - state_dict[f"{name}.weight_scale"] = steps.to(weight.dtype).cpu() - state_dict[f"{name}.weight_shape"] = shape - model.save_pretrained(export_dir, state_dict=state_dict, save_modelopt_state=False) - _carry_over_unplaced_weights(export_dir, model) - copy_off_index_safetensors(_source_checkpoint(model), export_dir) - _copy_non_model_files_from_source(model, export_dir) - _copy_source_config(model, export_dir) - - config_path = export_dir / "config.json" - config = json.loads(config_path.read_text()) - config["quantization_config"] = _compressed_tensors_config(export_dir) - config_path.write_text(json.dumps(config, indent=4)) - - -def _ggml_layers(model: nn.Module) -> dict[str, tuple[torch.Tensor, TensorQuantizer, GGMLFormat]]: - """Each GGML-quantized layer's weight, quantizer and format, refusing any other quantizer.""" - layers, unsupported = {}, [] - for name, quantizer in model.named_modules(): - if not (isinstance(quantizer, TensorQuantizer) and quantizer.is_enabled): - continue - layer, _, attr = name.rpartition(".") - weight = getattr(model.get_submodule(layer), "weight", None) - num_bits = quantizer.num_bits - ggml_format = GGML_FORMAT_REGISTRY.get(num_bits) if isinstance(num_bits, str) else None - if attr != "weight_quantizer" or ggml_format is None or weight is None or weight.dim() != 2: - unsupported.append(name) - continue - layers[layer] = (weight, quantizer, ggml_format) - if unsupported: - raise ValueError( - "GGML INT8 export writes only GGML weight quantizers of 2-D weights, but these " - f"quantizers are enabled too: {unsupported[:8]}" - ) - if not layers: - raise ValueError("The model has no GGML-quantized weights to export.") - return layers - - -def _compressed_tensors_config(export_dir: Path) -> dict: - """One config group per INT8 group size, targeting the layers by their saved names. - - The names are read back from the checkpoint because saving can rename modules; vLLM maps - explicit layer names onto its own modules, but not ``re:`` patterns. - """ - shapes = {} - for shard in sorted(set(indexed_weight_map(export_dir).values())): - with safe_open(export_dir / shard, framework="pt") as f: - for key in f.keys(): # noqa: SIM118 -- a safe_open handle is not a dict - if key.endswith((".weight_packed", ".weight_scale")): - shapes[key] = f.get_slice(key).get_shape() - layers_by_group_size: dict[int, list[str]] = {} - for key, shape in sorted(shapes.items()): - if key.endswith(".weight_packed"): - layer = key.removesuffix(".weight_packed") - group_size = shape[-1] * 4 // shapes[f"{layer}.weight_scale"][-1] - layers_by_group_size.setdefault(group_size, []).append(layer) - config_groups = { - f"group_{i}": { - "targets": layers, - "weights": { - "num_bits": 8, - "type": "int", - "symmetric": True, - "strategy": "group", - "group_size": group_size, - "dynamic": False, - }, - "input_activations": None, - "output_activations": None, - "format": "pack-quantized", - } - for i, (group_size, layers) in enumerate(sorted(layers_by_group_size.items())) - } - return { - "quant_method": "compressed-tensors", - "format": "pack-quantized", - "quantization_status": "compressed", - "config_groups": config_groups, - "ignore": [], - } diff --git a/tests/gpu_vllm/torch/export/test_ggml_int8_hf.py b/tests/gpu_vllm/torch/export/test_ggml_int8_export.py similarity index 93% rename from tests/gpu_vllm/torch/export/test_ggml_int8_hf.py rename to tests/gpu_vllm/torch/export/test_ggml_int8_export.py index dca196ad876..d2f131581d1 100644 --- a/tests/gpu_vllm/torch/export/test_ggml_int8_hf.py +++ b/tests/gpu_vllm/torch/export/test_ggml_int8_export.py @@ -24,7 +24,7 @@ from vllm.inputs import TokensPrompt import modelopt.torch.quantization as mtq -from modelopt.torch.export import export_hf_ggml_int8_checkpoint +from modelopt.torch.export import convert_ggml_export_to_int8, export_hf_checkpoint @pytest.fixture(scope="module") @@ -64,8 +64,9 @@ def int8_export_in_vllm(tmp_path_factory): logits = model(token_ids).logits[0, :-1].float() expected = logits.log_softmax(-1).gather(-1, token_ids[0, 1:, None])[:, 0] - export_dir = tmp_path / "export" - export_hf_ggml_int8_checkpoint(model, export_dir) + export_hf_checkpoint(model, export_dir=tmp_path / "ggml") + export_dir = tmp_path / "int8" + convert_ggml_export_to_int8(tmp_path / "ggml", export_dir) llm = LLM( model=str(export_dir), diff --git a/tests/unit/torch/export/test_ggml_int8_export.py b/tests/unit/torch/export/test_ggml_int8_export.py new file mode 100644 index 00000000000..d7949db3664 --- /dev/null +++ b/tests/unit/torch/export/test_ggml_int8_export.py @@ -0,0 +1,108 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +import pytest +import torch +from _test_utils.torch.transformers_models import create_tiny_llama_dir +from safetensors.torch import load_file +from transformers import AutoModelForCausalLM + +import modelopt.torch.quantization as mtq +from modelopt.torch.export import convert_ggml_export_to_int8, export_hf_checkpoint +from modelopt.torch.quantization.ggml import GGML_FORMAT_REGISTRY + +# Layer 0 takes a format with 32-weight sub-blocks and layer 1 one with 16, so the export needs +# one compressed-tensors config group per group size. +LAYER_FORMATS = {0: "iq1_s", 1: "iq2_xs"} + + +def _export(tmp_path, second_layer_cfg): + source = create_tiny_llama_dir(tmp_path, hidden_size=256, intermediate_size=256) + model = AutoModelForCausalLM.from_pretrained(source, dtype=torch.bfloat16) + layer_cfgs = [ + {"num_bits": "iq1_s", "block_sizes": {-1: 256}, "backend": "ggml"}, + second_layer_cfg, + ] + quant_cfg = { + "quant_cfg": [ + {"quantizer_name": "*", "enable": False}, + *( + {"quantizer_name": f"*layers.{i}.mlp*weight_quantizer", "cfg": cfg} + for i, cfg in enumerate(layer_cfgs) + ), + ], + "algorithm": "max", + } + mtq.quantize(model, quant_cfg, lambda m: m(torch.randint(0, 32, (1, 8)))) + export_dir = tmp_path / "export" + export_hf_checkpoint(model, export_dir=export_dir) + return export_dir + + +def test_ggml_export_converts_to_int8_levels_and_steps(tmp_path): + export_dir = _export( + tmp_path, {"num_bits": "iq2_xs", "block_sizes": {-1: 256}, "backend": "ggml"} + ) + ggml_files = {path.name: path.read_bytes() for path in export_dir.iterdir()} + before = load_file(export_dir / "model.safetensors") + int8_dir = tmp_path / "int8" + + # A small shard size splits the output, so the index has to stitch the shards together. + convert_ggml_export_to_int8(export_dir, int8_dir, max_shard_size=2**18) + + assert {path.name: path.read_bytes() for path in export_dir.iterdir()} == ggml_files + assert not (int8_dir / "hf_quant_config.json").exists() + config = json.loads((int8_dir / "config.json").read_text())["quantization_config"] + assert config["quant_method"] == "compressed-tensors" + layers = { + i: [f"model.layers.{i}.mlp.{proj}" for proj in ("down_proj", "gate_proj", "up_proj")] + for i in LAYER_FORMATS + } + assert { + group["weights"]["group_size"]: group["targets"] + for group in config["config_groups"].values() + } == {32: layers[0], 16: layers[1]} + + weight_map = json.loads((int8_dir / "model.safetensors.index.json").read_text())["weight_map"] + assert len(set(weight_map.values())) > 1 + after = {} + for shard in set(weight_map.values()): + after.update(load_file(int8_dir / shard)) + assert after.keys() == weight_map.keys() + for i, name in LAYER_FORMATS.items(): + for layer in layers[i]: + payload = before.pop(f"{layer}.weight") + shape = after.pop(f"{layer}.weight_shape") + levels, steps = GGML_FORMAT_REGISTRY[name].decode_int8(payload, shape) + # compressed-tensors' layout: offset by 128, four values per int32, lowest byte first. + packed = after.pop(f"{layer}.weight_packed") + unpacked = torch.stack([(packed >> (8 * k)) & 0xFF for k in range(4)], -1).flatten(1) + assert torch.equal(unpacked - 128, levels.long()) + assert torch.equal(after.pop(f"{layer}.weight_scale"), steps.to(torch.bfloat16)) + assert shape.tolist() == list(levels.shape) + assert after.keys() == before.keys() + for name, tensor in before.items(): + assert torch.equal(after[name], tensor) + + +def test_export_with_a_non_ggml_layer_is_refused_before_writing(tmp_path): + export_dir = _export(tmp_path, {"num_bits": (4, 3), "axis": None}) + + with pytest.raises(ValueError, match="Only GGML layers"): + convert_ggml_export_to_int8(export_dir, tmp_path / "int8") + + assert not (tmp_path / "int8").exists() diff --git a/tests/unit/torch/export/test_ggml_int8_hf.py b/tests/unit/torch/export/test_ggml_int8_hf.py deleted file mode 100644 index e84b55e3e60..00000000000 --- a/tests/unit/torch/export/test_ggml_int8_hf.py +++ /dev/null @@ -1,96 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json - -import pytest -import torch -from _test_utils.torch.transformers_models import create_tiny_llama_dir -from safetensors.torch import load_file -from transformers import AutoModelForCausalLM - -import modelopt.torch.quantization as mtq -from modelopt.torch.export import export_hf_ggml_int8_checkpoint -from modelopt.torch.quantization.ggml import GGML_FORMAT_REGISTRY - -IQ1_S = {"num_bits": "iq1_s", "block_sizes": {-1: 256}, "backend": "ggml"} -IQ2_XS = {"num_bits": "iq2_xs", "block_sizes": {-1: 256}, "backend": "ggml"} -PROJECTIONS = ("down_proj", "gate_proj", "up_proj") - - -def _quantized_tiny_llama(tmp_path, layer_cfgs): - source = create_tiny_llama_dir(tmp_path, hidden_size=256, intermediate_size=256) - model = AutoModelForCausalLM.from_pretrained(source, dtype=torch.bfloat16) - quant_cfg = { - "quant_cfg": [ - {"quantizer_name": "*", "enable": False}, - *( - {"quantizer_name": f"*layers.{i}.mlp*weight_quantizer", "cfg": cfg} - for i, cfg in enumerate(layer_cfgs) - ), - ], - "algorithm": "max", - } - token_ids = torch.randint(0, 32, (1, 8)) - mtq.quantize(model, quant_cfg, lambda m: m(token_ids)) - # A fake-quant forward caches each payload, so the test and the export encode it once. - model(token_ids) - return model - - -def test_ggml_layers_export_as_their_int8_levels_and_steps(tmp_path): - # A format with 32-weight sub-blocks and one with 16 need one config group each. - model = _quantized_tiny_llama(tmp_path, [IQ1_S, IQ2_XS]) - layers = {i: [f"model.layers.{i}.mlp.{proj}" for proj in PROJECTIONS] for i in (0, 1)} - expected = {} - for layer in layers[0] + layers[1]: - module = model.get_submodule(layer) - ggml_format = GGML_FORMAT_REGISTRY[module.weight_quantizer.num_bits] - payload = ggml_format.pack(module.weight, module.weight_quantizer) - expected[layer] = ggml_format.decode_int8(payload, torch.tensor(module.weight.shape)) - unquantized = { - k: v.clone() - for k, v in model.state_dict().items() - if "quantizer" not in k and k.removesuffix(".weight") not in expected - } - - export_hf_ggml_int8_checkpoint(model, tmp_path / "export") - - config = json.loads((tmp_path / "export" / "config.json").read_text())["quantization_config"] - assert config["quant_method"] == "compressed-tensors" - assert { - group["weights"]["group_size"]: group["targets"] - for group in config["config_groups"].values() - } == {32: layers[0], 16: layers[1]} - saved = load_file(tmp_path / "export" / "model.safetensors") - for layer, (levels, steps) in expected.items(): - # compressed-tensors' layout: offset by 128, four values per int32, lowest byte first. - packed = saved.pop(f"{layer}.weight_packed") - unpacked = torch.stack([(packed >> (8 * k)) & 0xFF for k in range(4)], -1).flatten(1) - assert torch.equal(unpacked - 128, levels.long()) - assert torch.equal(saved.pop(f"{layer}.weight_scale"), steps.to(torch.bfloat16)) - assert saved.pop(f"{layer}.weight_shape").tolist() == list(levels.shape) - assert saved.keys() <= unquantized.keys() - for name, tensor in saved.items(): - assert torch.equal(tensor, unquantized[name]) - - -def test_a_non_ggml_quantizer_is_refused_before_writing(tmp_path): - model = _quantized_tiny_llama(tmp_path, [IQ1_S, {"num_bits": (4, 3), "axis": None}]) - - with pytest.raises(ValueError, match="only GGML weight quantizers"): - export_hf_ggml_int8_checkpoint(model, tmp_path / "export") - - assert not (tmp_path / "export").exists() From 04843740384ae543febc88871ea38bfb69990be9 Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Wed, 21 Oct 2026 06:28:16 +0000 Subject: [PATCH 4/7] Fold the GGML-to-INT8 converter into convert_ggml_to_int8.py The conversion has one caller, the example script, so it lives there instead of in modelopt.torch.export. Its tests move with it and load the script by path, like the other example-script tests. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Chenjie Luo --- CHANGELOG.rst | 2 +- examples/hf_ptq/convert_ggml_to_int8.py | 190 +++++++++++++++- modelopt/torch/export/__init__.py | 1 - modelopt/torch/export/ggml_int8_export.py | 209 ------------------ .../test_convert_ggml_to_int8.py} | 12 +- .../test_convert_ggml_to_int8.py} | 14 +- 6 files changed, 211 insertions(+), 217 deletions(-) delete mode 100644 modelopt/torch/export/ggml_int8_export.py rename tests/gpu_vllm/{torch/export/test_ggml_int8_export.py => examples/test_convert_ggml_to_int8.py} (87%) rename tests/unit/{torch/export/test_ggml_int8_export.py => examples/test_convert_ggml_to_int8.py} (87%) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ede632aa3ab..e7abe4ac733 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,7 +11,7 @@ Changelog - On Linux x86-64 with Python 3.11 and later, the ONNX extra now selects ONNX Runtime 1.30.x and CUDA 13 packages by default. GPU calibration requires a compatible CUDA 13 runtime and driver; Python 3.10 and other platforms retain their existing dependency selections. - Add imatrix calibration (``algorithm: {method: imatrix}``) for the GGML formats with a weighted search: each weight is encoded once, its search weighted by the square root of its inputs' per-column ``E[x^2]`` (llama.cpp's imatrix statistic), and HF fused-MoE experts get one statistic per expert. A weight that already holds final codes is left alone, so ``[{method: gptq}, {method: imatrix}]`` runs GPTQ on the dense linears and imatrix on the experts GPTQ skips; the built-in IQ1 and IQ2 recipes now do this, and new ``_experts_only`` recipes apply imatrix to the routed MoE experts alone. - Add a checkpoint-specific NVIDIA Nemotron 3.5 Super VL AutoQuantize recipe with a 4.9-effective-bit NVFP4-MSE/FP8/BF16 search and fixed NVFP4 MTP experts. Select it under ``models/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16/auto_quantize/``. -- Add ``examples/hf_ptq/convert_ggml_to_int8.py`` (``convert_ggml_export_to_int8``) to expand a checkpoint exported with a GGML recipe (IQ formats, ``q8_0``) into an INT8 weight-only compressed-tensors checkpoint. Stock vLLM serves it with its integer kernels at about half the BF16 weight memory, reproducing the GGML weights up to rounding their scales to the model dtype. +- Add ``examples/hf_ptq/convert_ggml_to_int8.py`` to expand a checkpoint exported with a GGML recipe (IQ formats, ``q8_0``) into an INT8 weight-only compressed-tensors checkpoint. Stock vLLM serves it with its integer kernels at about half the BF16 weight memory, reproducing the GGML weights up to rounding their scales to the model dtype. *Speculative Decoding* diff --git a/examples/hf_ptq/convert_ggml_to_int8.py b/examples/hf_ptq/convert_ggml_to_int8.py index 2a0ecd8b2fb..4d0d2139cc6 100644 --- a/examples/hf_ptq/convert_ggml_to_int8.py +++ b/examples/hf_ptq/convert_ggml_to_int8.py @@ -21,8 +21,196 @@ """ import argparse +import json +import os +import shutil +from pathlib import Path -from modelopt.torch.export import convert_ggml_export_to_int8 +import torch +from safetensors import safe_open +from safetensors.torch import save_file + +from modelopt.torch.quantization.ggml import GGML_FORMAT_REGISTRY, GGMLFormat +from modelopt.torch.utils.plugins.hf_checkpoint_utils import indexed_weight_map + +_INDEX = "model.safetensors.index.json" + + +def convert_ggml_export_to_int8( + ggml_dir: str | os.PathLike, + output_dir: str | os.PathLike, + max_shard_size: int = 10 * 2**30, +) -> None: + """Write a GGML export as an INT8 weight-only compressed-tensors checkpoint, which vLLM serves. + + Every GGML weight is an integer level of at most 8 bits times the step of its 16- or + 32-weight sub-block, so storing those levels with the steps as group scales reproduces the + GGML weights up to rounding the steps to the model dtype. Unquantized tensors and the other + files are copied as they are; the GGML export is left unchanged. + + Args: + ggml_dir: A Hugging Face export whose quantized layers are all GGML formats, as written + by hf_ptq.py with a GGML recipe. + output_dir: Where to write the INT8 checkpoint. Must be empty or not exist yet. + max_shard_size: Bytes per output safetensors shard, which bounds the memory conversion uses. + """ + ggml_dir, output_dir = Path(ggml_dir), Path(output_dir) + config = json.loads((ggml_dir / "config.json").read_text()) + quantization = json.loads((ggml_dir / "hf_quant_config.json").read_text())["quantization"] + scale_dtype = getattr(torch, config.get("dtype") or config["torch_dtype"]) + weight_map = indexed_weight_map(ggml_dir) + layer_formats = _ggml_layer_formats(ggml_dir, weight_map, quantization) + if output_dir.exists() and any(output_dir.iterdir()): + raise ValueError(f"{output_dir} is not empty.") + output_dir.mkdir(parents=True, exist_ok=True) + + shards = sorted(set(weight_map.values())) + for path in ggml_dir.iterdir(): + # The shards are rewritten below, and the GGML metadata does not describe the result. + if path.is_file() and path.name not in {*shards, _INDEX, "hf_quant_config.json"}: + shutil.copy2(path, output_dir / path.name) + new_weight_map = _write_int8_shards( + ggml_dir, shards, output_dir, layer_formats, scale_dtype, max_shard_size + ) + total_size = sum((output_dir / shard).stat().st_size for shard in set(new_weight_map.values())) + index = {"metadata": {"total_size": total_size}, "weight_map": new_weight_map} + (output_dir / _INDEX).write_text(json.dumps(index, indent=2)) + + config["quantization_config"] = _compressed_tensors_config(layer_formats) + (output_dir / "config.json").write_text(json.dumps(config, indent=4)) + + +def _ggml_layer_formats( + ggml_dir: Path, weight_map: dict[str, str], quantization: dict +) -> dict[str, GGMLFormat]: + """The GGML format of every layer holding a packed payload, refusing anything else.""" + if quantization.get("kv_cache_quant_algo") is not None: + raise ValueError("INT8 conversion of a GGML export does not carry KV cache quantization.") + mixed = quantization.get("quant_algo") == "MIXED_PRECISION" + layer_configs = quantization.get("quantized_layers", {}) if mixed else {} + algos = ( + {cfg.get("quant_algo") for cfg in layer_configs.values()} + if mixed + else {quantization.get("quant_algo")} + ) + unsupported = sorted(str(a) for a in algos if str(a).lower() not in GGML_FORMAT_REGISTRY) + if unsupported: + raise ValueError(f"Only GGML layers convert to INT8, but the export has {unsupported}.") + + layer_formats = {} + for shard in sorted(set(weight_map.values())): + with safe_open(ggml_dir / shard, framework="pt") as f: + # GGML payloads are the only uint8 weights an export writes. + payloads = [ + name + for name in f.keys() # noqa: SIM118 -- a safe_open handle is not a dict + if name.endswith(".weight") and f.get_slice(name).get_dtype() == "U8" + ] + for name in payloads: + layer = name.removesuffix(".weight") + algo = layer_configs[layer]["quant_algo"] if mixed else quantization["quant_algo"] + layer_formats[layer] = GGML_FORMAT_REGISTRY[algo.lower()] + if not layer_formats: + raise ValueError(f"{ggml_dir} holds no GGML weights to convert.") + return layer_formats + + +def _write_int8_shards( + ggml_dir: Path, + shards: list[str], + output_dir: Path, + layer_formats: dict[str, GGMLFormat], + scale_dtype: torch.dtype, + max_shard_size: int, +) -> dict[str, str]: + """Rewrite every shard's tensors into size-bounded output shards; return the weight map.""" + device = "cuda" if torch.cuda.is_available() else "cpu" + pending: dict[str, torch.Tensor] = {} + written: list[list[str]] = [] + + def flush(): + save_file(pending, output_dir / f"part-{len(written):05d}.safetensors", {"format": "pt"}) + written.append(list(pending)) + pending.clear() + + for shard in shards: + with safe_open(ggml_dir / shard, framework="pt") as f: + for name in f.keys(): # noqa: SIM118 -- a safe_open handle is not a dict + layer = name.removesuffix(".weight") + ggml_format = layer_formats.get(layer) if name.endswith(".weight") else None + if ggml_format is None: + tensors = {name: f.get_tensor(name)} + else: + tensors = _int8_tensors( + layer, f.get_tensor(name), ggml_format, scale_dtype, device + ) + if pending and _nbytes(pending) + _nbytes(tensors) > max_shard_size: + flush() + pending.update(tensors) + flush() + + weight_map = {} + for i, names in enumerate(written): + shard_name = f"model-{i + 1:05d}-of-{len(written):05d}.safetensors" + os.replace(output_dir / f"part-{i:05d}.safetensors", output_dir / shard_name) + weight_map.update(dict.fromkeys(names, shard_name)) + return dict(sorted(weight_map.items())) + + +def _int8_tensors( + layer: str, + payload: torch.Tensor, + ggml_format: GGMLFormat, + scale_dtype: torch.dtype, + device: str, +) -> dict[str, torch.Tensor]: + """A layer's GGML payload as compressed-tensors ``weight_packed``, ``weight_scale`` and shape.""" + shape = torch.tensor((*payload.shape[:-2], payload.shape[-2] * ggml_format.block_size)) + levels, steps = ggml_format.decode_int8(payload.to(device), shape) + return { + # compressed-tensors packs 8-bit values offset by 128, four little-endian per int32. + f"{layer}.weight_packed": (levels.view(torch.uint8) ^ 0x80).view(torch.int32).cpu(), + f"{layer}.weight_scale": steps.to(scale_dtype).cpu(), + f"{layer}.weight_shape": shape, + } + + +def _nbytes(tensors: dict[str, torch.Tensor]) -> int: + return sum(t.numel() * t.element_size() for t in tensors.values()) + + +def _compressed_tensors_config(layer_formats: dict[str, GGMLFormat]) -> dict: + """One config group per INT8 group size, targeting its layers by name. + + vLLM maps explicit layer names onto its own modules, but not ``re:`` patterns. + """ + layers_by_group_size: dict[int, list[str]] = {} + for layer, ggml_format in sorted(layer_formats.items()): + layers_by_group_size.setdefault(ggml_format.int8_group_size, []).append(layer) + config_groups = { + f"group_{i}": { + "targets": layers, + "weights": { + "num_bits": 8, + "type": "int", + "symmetric": True, + "strategy": "group", + "group_size": group_size, + "dynamic": False, + }, + "input_activations": None, + "output_activations": None, + "format": "pack-quantized", + } + for i, (group_size, layers) in enumerate(sorted(layers_by_group_size.items())) + } + return { + "quant_method": "compressed-tensors", + "format": "pack-quantized", + "quantization_status": "compressed", + "config_groups": config_groups, + "ignore": [], + } def main(): diff --git a/modelopt/torch/export/__init__.py b/modelopt/torch/export/__init__.py index 30c3b808f9b..44d1ad2557e 100644 --- a/modelopt/torch/export/__init__.py +++ b/modelopt/torch/export/__init__.py @@ -34,7 +34,6 @@ ) from .convert_hf_config import * -from .ggml_int8_export import * from .model_utils import * from .moe_utils import * from .plugins import * diff --git a/modelopt/torch/export/ggml_int8_export.py b/modelopt/torch/export/ggml_int8_export.py deleted file mode 100644 index eb309740bd0..00000000000 --- a/modelopt/torch/export/ggml_int8_export.py +++ /dev/null @@ -1,209 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Expand a GGML Hugging Face export into an INT8 weight-only compressed-tensors checkpoint.""" - -import json -import os -import shutil -from pathlib import Path - -import torch -from safetensors import safe_open -from safetensors.torch import save_file - -from modelopt.torch.quantization.ggml import GGML_FORMAT_REGISTRY, GGMLFormat -from modelopt.torch.utils.plugins.hf_checkpoint_utils import indexed_weight_map - -__all__ = ["convert_ggml_export_to_int8"] - -_INDEX = "model.safetensors.index.json" - - -def convert_ggml_export_to_int8( - ggml_dir: str | os.PathLike, - output_dir: str | os.PathLike, - max_shard_size: int = 10 * 2**30, -) -> None: - """Write a GGML export as an INT8 weight-only compressed-tensors checkpoint, which vLLM serves. - - Every GGML weight is an integer level of at most 8 bits times the step of its 16- or - 32-weight sub-block, so storing those levels with the steps as group scales reproduces the - GGML weights up to rounding the steps to the model dtype. Unquantized tensors and the other - files are copied as they are; the GGML export is left unchanged. - - Args: - ggml_dir: A Hugging Face export whose quantized layers are all GGML formats, as written - by :func:`export_hf_checkpoint`. - output_dir: Where to write the INT8 checkpoint. Must be empty or not exist yet. - max_shard_size: Bytes per output safetensors shard, which bounds the memory conversion uses. - """ - ggml_dir, output_dir = Path(ggml_dir), Path(output_dir) - config = json.loads((ggml_dir / "config.json").read_text()) - quantization = json.loads((ggml_dir / "hf_quant_config.json").read_text())["quantization"] - scale_dtype = getattr(torch, config.get("dtype") or config["torch_dtype"]) - weight_map = indexed_weight_map(ggml_dir) - layer_formats = _ggml_layer_formats(ggml_dir, weight_map, quantization) - if output_dir.exists() and any(output_dir.iterdir()): - raise ValueError(f"{output_dir} is not empty.") - output_dir.mkdir(parents=True, exist_ok=True) - - shards = sorted(set(weight_map.values())) - for path in ggml_dir.iterdir(): - # The shards are rewritten below, and the GGML metadata does not describe the result. - if path.is_file() and path.name not in {*shards, _INDEX, "hf_quant_config.json"}: - shutil.copy2(path, output_dir / path.name) - new_weight_map = _write_int8_shards( - ggml_dir, shards, output_dir, layer_formats, scale_dtype, max_shard_size - ) - total_size = sum((output_dir / shard).stat().st_size for shard in set(new_weight_map.values())) - index = {"metadata": {"total_size": total_size}, "weight_map": new_weight_map} - (output_dir / _INDEX).write_text(json.dumps(index, indent=2)) - - config["quantization_config"] = _compressed_tensors_config(layer_formats) - (output_dir / "config.json").write_text(json.dumps(config, indent=4)) - - -def _ggml_layer_formats( - ggml_dir: Path, weight_map: dict[str, str], quantization: dict -) -> dict[str, GGMLFormat]: - """The GGML format of every layer holding a packed payload, refusing anything else.""" - if quantization.get("kv_cache_quant_algo") is not None: - raise ValueError("INT8 conversion of a GGML export does not carry KV cache quantization.") - mixed = quantization.get("quant_algo") == "MIXED_PRECISION" - layer_configs = quantization.get("quantized_layers", {}) if mixed else {} - algos = ( - {cfg.get("quant_algo") for cfg in layer_configs.values()} - if mixed - else {quantization.get("quant_algo")} - ) - unsupported = sorted(str(a) for a in algos if str(a).lower() not in GGML_FORMAT_REGISTRY) - if unsupported: - raise ValueError(f"Only GGML layers convert to INT8, but the export has {unsupported}.") - - layer_formats = {} - for shard in sorted(set(weight_map.values())): - with safe_open(ggml_dir / shard, framework="pt") as f: - # GGML payloads are the only uint8 weights an export writes. - payloads = [ - name - for name in f.keys() # noqa: SIM118 -- a safe_open handle is not a dict - if name.endswith(".weight") and f.get_slice(name).get_dtype() == "U8" - ] - for name in payloads: - layer = name.removesuffix(".weight") - algo = layer_configs[layer]["quant_algo"] if mixed else quantization["quant_algo"] - layer_formats[layer] = GGML_FORMAT_REGISTRY[algo.lower()] - if not layer_formats: - raise ValueError(f"{ggml_dir} holds no GGML weights to convert.") - return layer_formats - - -def _write_int8_shards( - ggml_dir: Path, - shards: list[str], - output_dir: Path, - layer_formats: dict[str, GGMLFormat], - scale_dtype: torch.dtype, - max_shard_size: int, -) -> dict[str, str]: - """Rewrite every shard's tensors into size-bounded output shards; return the weight map.""" - device = "cuda" if torch.cuda.is_available() else "cpu" - pending: dict[str, torch.Tensor] = {} - written: list[list[str]] = [] - - def flush(): - save_file(pending, output_dir / f"part-{len(written):05d}.safetensors", {"format": "pt"}) - written.append(list(pending)) - pending.clear() - - for shard in shards: - with safe_open(ggml_dir / shard, framework="pt") as f: - for name in f.keys(): # noqa: SIM118 -- a safe_open handle is not a dict - layer = name.removesuffix(".weight") - ggml_format = layer_formats.get(layer) if name.endswith(".weight") else None - if ggml_format is None: - tensors = {name: f.get_tensor(name)} - else: - tensors = _int8_tensors( - layer, f.get_tensor(name), ggml_format, scale_dtype, device - ) - if pending and _nbytes(pending) + _nbytes(tensors) > max_shard_size: - flush() - pending.update(tensors) - flush() - - weight_map = {} - for i, names in enumerate(written): - shard_name = f"model-{i + 1:05d}-of-{len(written):05d}.safetensors" - os.replace(output_dir / f"part-{i:05d}.safetensors", output_dir / shard_name) - weight_map.update(dict.fromkeys(names, shard_name)) - return dict(sorted(weight_map.items())) - - -def _int8_tensors( - layer: str, - payload: torch.Tensor, - ggml_format: GGMLFormat, - scale_dtype: torch.dtype, - device: str, -) -> dict[str, torch.Tensor]: - """A layer's GGML payload as compressed-tensors ``weight_packed``, ``weight_scale`` and shape.""" - shape = torch.tensor((*payload.shape[:-2], payload.shape[-2] * ggml_format.block_size)) - levels, steps = ggml_format.decode_int8(payload.to(device), shape) - return { - # compressed-tensors packs 8-bit values offset by 128, four little-endian per int32. - f"{layer}.weight_packed": (levels.view(torch.uint8) ^ 0x80).view(torch.int32).cpu(), - f"{layer}.weight_scale": steps.to(scale_dtype).cpu(), - f"{layer}.weight_shape": shape, - } - - -def _nbytes(tensors: dict[str, torch.Tensor]) -> int: - return sum(t.numel() * t.element_size() for t in tensors.values()) - - -def _compressed_tensors_config(layer_formats: dict[str, GGMLFormat]) -> dict: - """One config group per INT8 group size, targeting its layers by name. - - vLLM maps explicit layer names onto its own modules, but not ``re:`` patterns. - """ - layers_by_group_size: dict[int, list[str]] = {} - for layer, ggml_format in sorted(layer_formats.items()): - layers_by_group_size.setdefault(ggml_format.int8_group_size, []).append(layer) - config_groups = { - f"group_{i}": { - "targets": layers, - "weights": { - "num_bits": 8, - "type": "int", - "symmetric": True, - "strategy": "group", - "group_size": group_size, - "dynamic": False, - }, - "input_activations": None, - "output_activations": None, - "format": "pack-quantized", - } - for i, (group_size, layers) in enumerate(sorted(layers_by_group_size.items())) - } - return { - "quant_method": "compressed-tensors", - "format": "pack-quantized", - "quantization_status": "compressed", - "config_groups": config_groups, - "ignore": [], - } diff --git a/tests/gpu_vllm/torch/export/test_ggml_int8_export.py b/tests/gpu_vllm/examples/test_convert_ggml_to_int8.py similarity index 87% rename from tests/gpu_vllm/torch/export/test_ggml_int8_export.py rename to tests/gpu_vllm/examples/test_convert_ggml_to_int8.py index d2f131581d1..9a64a065a40 100644 --- a/tests/gpu_vllm/torch/export/test_ggml_int8_export.py +++ b/tests/gpu_vllm/examples/test_convert_ggml_to_int8.py @@ -14,6 +14,8 @@ # limitations under the License. import gc +import importlib.util +from pathlib import Path import pytest import torch @@ -24,7 +26,13 @@ from vllm.inputs import TokensPrompt import modelopt.torch.quantization as mtq -from modelopt.torch.export import convert_ggml_export_to_int8, export_hf_checkpoint +from modelopt.torch.export import export_hf_checkpoint + +_SCRIPT = Path(__file__).resolve().parents[3] / "examples" / "hf_ptq" / "convert_ggml_to_int8.py" +_SPEC = importlib.util.spec_from_file_location("convert_ggml_to_int8", _SCRIPT) +assert _SPEC is not None and _SPEC.loader is not None +convert_ggml_to_int8 = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(convert_ggml_to_int8) @pytest.fixture(scope="module") @@ -66,7 +74,7 @@ def int8_export_in_vllm(tmp_path_factory): export_hf_checkpoint(model, export_dir=tmp_path / "ggml") export_dir = tmp_path / "int8" - convert_ggml_export_to_int8(tmp_path / "ggml", export_dir) + convert_ggml_to_int8.convert_ggml_export_to_int8(tmp_path / "ggml", export_dir) llm = LLM( model=str(export_dir), diff --git a/tests/unit/torch/export/test_ggml_int8_export.py b/tests/unit/examples/test_convert_ggml_to_int8.py similarity index 87% rename from tests/unit/torch/export/test_ggml_int8_export.py rename to tests/unit/examples/test_convert_ggml_to_int8.py index d7949db3664..9ea7e553d16 100644 --- a/tests/unit/torch/export/test_ggml_int8_export.py +++ b/tests/unit/examples/test_convert_ggml_to_int8.py @@ -13,7 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import importlib.util import json +from pathlib import Path import pytest import torch @@ -22,9 +24,15 @@ from transformers import AutoModelForCausalLM import modelopt.torch.quantization as mtq -from modelopt.torch.export import convert_ggml_export_to_int8, export_hf_checkpoint +from modelopt.torch.export import export_hf_checkpoint from modelopt.torch.quantization.ggml import GGML_FORMAT_REGISTRY +_SCRIPT = Path(__file__).resolve().parents[3] / "examples" / "hf_ptq" / "convert_ggml_to_int8.py" +_SPEC = importlib.util.spec_from_file_location("convert_ggml_to_int8", _SCRIPT) +assert _SPEC is not None and _SPEC.loader is not None +convert_ggml_to_int8 = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(convert_ggml_to_int8) + # Layer 0 takes a format with 32-weight sub-blocks and layer 1 one with 16, so the export needs # one compressed-tensors config group per group size. LAYER_FORMATS = {0: "iq1_s", 1: "iq2_xs"} @@ -62,7 +70,7 @@ def test_ggml_export_converts_to_int8_levels_and_steps(tmp_path): int8_dir = tmp_path / "int8" # A small shard size splits the output, so the index has to stitch the shards together. - convert_ggml_export_to_int8(export_dir, int8_dir, max_shard_size=2**18) + convert_ggml_to_int8.convert_ggml_export_to_int8(export_dir, int8_dir, max_shard_size=2**18) assert {path.name: path.read_bytes() for path in export_dir.iterdir()} == ggml_files assert not (int8_dir / "hf_quant_config.json").exists() @@ -103,6 +111,6 @@ def test_export_with_a_non_ggml_layer_is_refused_before_writing(tmp_path): export_dir = _export(tmp_path, {"num_bits": (4, 3), "axis": None}) with pytest.raises(ValueError, match="Only GGML layers"): - convert_ggml_export_to_int8(export_dir, tmp_path / "int8") + convert_ggml_to_int8.convert_ggml_export_to_int8(export_dir, tmp_path / "int8") assert not (tmp_path / "int8").exists() From 2244ea47e220fbb07865b510ddfd605045fd5257 Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Wed, 21 Oct 2026 16:25:42 +0000 Subject: [PATCH 5/7] Keep nested sidecars and clean up after a failed GGML-to-INT8 conversion - Copy the GGML export's other files with copy_non_model_files and copy_off_index_safetensors, as the exporters do. The top-level loop dropped files in subdirectories such as chat_templates/, and bypassed the helpers' checks on links that point outside the checkpoint. - Empty the output directory if conversion fails, so a rerun is not refused over a partial checkpoint. - Document max_shard_size as a target: a larger tensor gets its own shard. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Chenjie Luo --- examples/hf_ptq/convert_ggml_to_int8.py | 50 ++++++++++++------- .../examples/test_convert_ggml_to_int8.py | 50 +++++++++++++++---- 2 files changed, 74 insertions(+), 26 deletions(-) diff --git a/examples/hf_ptq/convert_ggml_to_int8.py b/examples/hf_ptq/convert_ggml_to_int8.py index 4d0d2139cc6..377bf77ea20 100644 --- a/examples/hf_ptq/convert_ggml_to_int8.py +++ b/examples/hf_ptq/convert_ggml_to_int8.py @@ -31,7 +31,11 @@ from safetensors.torch import save_file from modelopt.torch.quantization.ggml import GGML_FORMAT_REGISTRY, GGMLFormat -from modelopt.torch.utils.plugins.hf_checkpoint_utils import indexed_weight_map +from modelopt.torch.utils.plugins.hf_checkpoint_utils import ( + copy_non_model_files, + copy_off_index_safetensors, + indexed_weight_map, +) _INDEX = "model.safetensors.index.json" @@ -52,7 +56,8 @@ def convert_ggml_export_to_int8( ggml_dir: A Hugging Face export whose quantized layers are all GGML formats, as written by hf_ptq.py with a GGML recipe. output_dir: Where to write the INT8 checkpoint. Must be empty or not exist yet. - max_shard_size: Bytes per output safetensors shard, which bounds the memory conversion uses. + max_shard_size: Target bytes per output safetensors shard; a larger tensor gets a shard + of its own. Conversion holds one output shard in memory at a time. """ ggml_dir, output_dir = Path(ggml_dir), Path(output_dir) config = json.loads((ggml_dir / "config.json").read_text()) @@ -64,20 +69,28 @@ def convert_ggml_export_to_int8( raise ValueError(f"{output_dir} is not empty.") output_dir.mkdir(parents=True, exist_ok=True) - shards = sorted(set(weight_map.values())) - for path in ggml_dir.iterdir(): - # The shards are rewritten below, and the GGML metadata does not describe the result. - if path.is_file() and path.name not in {*shards, _INDEX, "hf_quant_config.json"}: - shutil.copy2(path, output_dir / path.name) - new_weight_map = _write_int8_shards( - ggml_dir, shards, output_dir, layer_formats, scale_dtype, max_shard_size - ) - total_size = sum((output_dir / shard).stat().st_size for shard in set(new_weight_map.values())) - index = {"metadata": {"total_size": total_size}, "weight_map": new_weight_map} - (output_dir / _INDEX).write_text(json.dumps(index, indent=2)) - - config["quantization_config"] = _compressed_tensors_config(layer_formats) - (output_dir / "config.json").write_text(json.dumps(config, indent=4)) + try: + shards = sorted(set(weight_map.values())) + new_weight_map = _write_int8_shards( + ggml_dir, shards, output_dir, layer_formats, scale_dtype, max_shard_size + ) + total_size = sum((output_dir / s).stat().st_size for s in set(new_weight_map.values())) + index = {"metadata": {"total_size": total_size}, "weight_map": new_weight_map} + (output_dir / _INDEX).write_text(json.dumps(index, indent=2)) + config["quantization_config"] = _compressed_tensors_config(layer_formats) + (output_dir / "config.json").write_text(json.dumps(config, indent=4)) + # Both skip files already written, and copy_non_model_files skips hf_quant_config.json, + # which describes the GGML payloads this replaced. + copy_off_index_safetensors(ggml_dir, output_dir) + copy_non_model_files(ggml_dir, output_dir) + except BaseException: + # Empty the directory again, so a rerun is not refused over a partial checkpoint. + for path in output_dir.iterdir(): + if path.is_dir(): + shutil.rmtree(path) + else: + path.unlink() + raise def _ggml_layer_formats( @@ -220,7 +233,10 @@ def main(): parser.add_argument("ggml_dir", help="Checkpoint exported with a GGML recipe (IQ, q8_0).") parser.add_argument("output_dir", help="Where to write the INT8 checkpoint; must be empty.") parser.add_argument( - "--max_shard_size_gb", type=float, default=10, help="Size of each output shard in GiB." + "--max_shard_size_gb", + type=float, + default=10, + help="Target size of each output shard in GiB; a larger tensor gets a shard of its own.", ) args = parser.parse_args() convert_ggml_export_to_int8( diff --git a/tests/unit/examples/test_convert_ggml_to_int8.py b/tests/unit/examples/test_convert_ggml_to_int8.py index 9ea7e553d16..df7f9ddb0f8 100644 --- a/tests/unit/examples/test_convert_ggml_to_int8.py +++ b/tests/unit/examples/test_convert_ggml_to_int8.py @@ -20,7 +20,7 @@ import pytest import torch from _test_utils.torch.transformers_models import create_tiny_llama_dir -from safetensors.torch import load_file +from safetensors.torch import load_file, save_file from transformers import AutoModelForCausalLM import modelopt.torch.quantization as mtq @@ -36,6 +36,7 @@ # Layer 0 takes a format with 32-weight sub-blocks and layer 1 one with 16, so the export needs # one compressed-tensors config group per group size. LAYER_FORMATS = {0: "iq1_s", 1: "iq2_xs"} +IQ2_XS = {"num_bits": "iq2_xs", "block_sizes": {-1: 256}, "backend": "ggml"} def _export(tmp_path, second_layer_cfg): @@ -61,19 +62,37 @@ def _export(tmp_path, second_layer_cfg): return export_dir -def test_ggml_export_converts_to_int8_levels_and_steps(tmp_path): - export_dir = _export( - tmp_path, {"num_bits": "iq2_xs", "block_sizes": {-1: 256}, "backend": "ggml"} - ) - ggml_files = {path.name: path.read_bytes() for path in export_dir.iterdir()} - before = load_file(export_dir / "model.safetensors") +def _files(directory): + return { + path.relative_to(directory): path.read_bytes() + for path in directory.rglob("*") + if path.is_file() + } + + +@pytest.fixture(scope="module") +def ggml_export(tmp_path_factory): + export_dir = _export(tmp_path_factory.mktemp("ggml"), IQ2_XS) + # Files a checkpoint can ship beside its indexed weights: a nested sidecar and an off-index + # weight file such as an MTP head. + (export_dir / "chat_templates").mkdir() + (export_dir / "chat_templates" / "tool_use.jinja").write_text("{{ messages }}") + save_file({"mtp.fc.weight": torch.ones(2, 2)}, export_dir / "mtp.safetensors") + return export_dir + + +def test_ggml_export_converts_to_int8_levels_and_steps(ggml_export, tmp_path): + ggml_files = _files(ggml_export) + before = load_file(ggml_export / "model.safetensors") int8_dir = tmp_path / "int8" # A small shard size splits the output, so the index has to stitch the shards together. - convert_ggml_to_int8.convert_ggml_export_to_int8(export_dir, int8_dir, max_shard_size=2**18) + convert_ggml_to_int8.convert_ggml_export_to_int8(ggml_export, int8_dir, max_shard_size=2**18) - assert {path.name: path.read_bytes() for path in export_dir.iterdir()} == ggml_files + assert _files(ggml_export) == ggml_files assert not (int8_dir / "hf_quant_config.json").exists() + for sidecar in (Path("chat_templates/tool_use.jinja"), Path("mtp.safetensors")): + assert (int8_dir / sidecar).read_bytes() == ggml_files[sidecar] config = json.loads((int8_dir / "config.json").read_text())["quantization_config"] assert config["quant_method"] == "compressed-tensors" layers = { @@ -107,6 +126,19 @@ def test_ggml_export_converts_to_int8_levels_and_steps(tmp_path): assert torch.equal(after[name], tensor) +def test_a_failed_conversion_leaves_the_output_directory_empty(ggml_export, tmp_path, monkeypatch): + def fail(*args, **kwargs): + raise OSError("disk full") + + # The last step: by then the shards, index and config are all written. + monkeypatch.setattr(convert_ggml_to_int8, "copy_non_model_files", fail) + + with pytest.raises(OSError, match="disk full"): + convert_ggml_to_int8.convert_ggml_export_to_int8(ggml_export, tmp_path / "int8") + + assert not any((tmp_path / "int8").iterdir()) + + def test_export_with_a_non_ggml_layer_is_refused_before_writing(tmp_path): export_dir = _export(tmp_path, {"num_bits": (4, 3), "axis": None}) From 76076787fe86c03a98eade1e9c51c13d96a40527 Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Wed, 21 Oct 2026 18:44:24 +0000 Subject: [PATCH 6/7] Move the GGML-to-INT8 tests under tests/examples and refuse unconvertible payloads The CPU tests now run convert_ggml_to_int8.py as a CLI from tests/examples/hf_ptq, and the vLLM test lives in tests/examples/vllm_serve, whose CI filter now includes the script. Both build their model with a shared helper in _test_utils. The converter refuses a payload that isn't a 2-D weight, or a uint8 weight missing from quantized_layers, with a clear error before writing anything. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Chenjie Luo --- .github/workflows/example_tests.yml | 1 + examples/hf_ptq/README.md | 4 +- examples/hf_ptq/convert_ggml_to_int8.py | 14 ++-- tests/_test_utils/examples/ggml_export.py | 57 +++++++++++++++ .../examples/hf_ptq_example_utils.py | 3 +- .../hf_ptq}/test_convert_ggml_to_int8.py | 71 ++++++------------- .../vllm_serve}/test_convert_ggml_to_int8.py | 52 +++----------- 7 files changed, 104 insertions(+), 98 deletions(-) create mode 100644 tests/_test_utils/examples/ggml_export.py rename tests/{unit/examples => examples/hf_ptq}/test_convert_ggml_to_int8.py (65%) rename tests/{gpu_vllm/examples => examples/vllm_serve}/test_convert_ggml_to_int8.py (55%) diff --git a/.github/workflows/example_tests.yml b/.github/workflows/example_tests.yml index 67b98cbd230..08e8b699d48 100644 --- a/.github/workflows/example_tests.yml +++ b/.github/workflows/example_tests.yml @@ -70,6 +70,7 @@ jobs: - "!**/*.{md,rst,png,ipynb}" vllm: - "{examples,tests/examples}/vllm_serve/**" + - examples/hf_ptq/convert_ggml_to_int8.py # tests/examples/vllm_serve serves its output - "!**/*.{md,rst,png,ipynb}" trtllm: - "{examples,tests/examples}/{gpt-oss,hf_ptq,llm_eval}/**" diff --git a/examples/hf_ptq/README.md b/examples/hf_ptq/README.md index 0815c2301d0..58394dc7a84 100755 --- a/examples/hf_ptq/README.md +++ b/examples/hf_ptq/README.md @@ -709,7 +709,7 @@ print(llm_fp8.generate(["What's the age of the earth? "])) #### vLLM with GGML recipes (IQ, Q8_0) -vLLM has no GGML kernels. To serve a checkpoint exported with a GGML recipe on vLLM's integer kernels, expand it into an INT8 weight-only compressed-tensors checkpoint with [convert_ggml_to_int8.py](./convert_ggml_to_int8.py): +vLLM cannot load the GGML checkpoints ModelOpt exports, which are Hugging Face checkpoints holding packed GGML blocks. To serve one on vLLM's integer kernels, expand it into an INT8 weight-only compressed-tensors checkpoint with [convert_ggml_to_int8.py](./convert_ggml_to_int8.py): ```bash python hf_ptq.py --pyt_ckpt_path --recipe general/ptq/iq1_s --export_path @@ -717,7 +717,7 @@ python convert_ggml_to_int8.py vllm serve ``` -Each weight keeps its GGML integer level, and the step of its 16- or 32-weight sub-block becomes the INT8 group scale. The INT8 checkpoint therefore holds the same weights up to rounding the scales to the model dtype, at about half the BF16 weight memory. The GGML checkpoint is left unchanged. +Each weight keeps its GGML integer level, and the step of its 16- or 32-weight sub-block becomes the INT8 group scale. The INT8 checkpoint therefore holds the same weights up to rounding the scales to the model dtype, at about half the BF16 weight memory. Its config names each converted layer explicitly, so every other layer loads unquantized. The GGML checkpoint is left unchanged. vLLM serves group-32 layers (IQ1_S, IQ2_XXS, Q8_0) with Marlin and group-16 layers (IQ1_M, IQ2_XS, IQ2_S) with Humming. On Blackwell GPUs, Marlin is much slower at large batch sizes, so set `VLLM_DISABLED_KERNELS=MarlinLinearKernel` to use Humming for group 32 as well. diff --git a/examples/hf_ptq/convert_ggml_to_int8.py b/examples/hf_ptq/convert_ggml_to_int8.py index 377bf77ea20..5ca496aa5fd 100644 --- a/examples/hf_ptq/convert_ggml_to_int8.py +++ b/examples/hf_ptq/convert_ggml_to_int8.py @@ -114,13 +114,19 @@ def _ggml_layer_formats( for shard in sorted(set(weight_map.values())): with safe_open(ggml_dir / shard, framework="pt") as f: # GGML payloads are the only uint8 weights an export writes. - payloads = [ - name + payloads = { + name: f.get_slice(name).get_shape() for name in f.keys() # noqa: SIM118 -- a safe_open handle is not a dict if name.endswith(".weight") and f.get_slice(name).get_dtype() == "U8" - ] - for name in payloads: + } + for name, shape in payloads.items(): layer = name.removesuffix(".weight") + # A 2-D weight's payload is [rows, blocks, block bytes]; weight_shape and the int32 + # packing assume that layout. + if len(shape) != 3: + raise ValueError(f"{layer} is not a 2-D weight; only 2-D weights convert to INT8.") + if mixed and layer not in layer_configs: + raise ValueError(f"{layer} holds a uint8 weight but is not in quantized_layers.") algo = layer_configs[layer]["quant_algo"] if mixed else quantization["quant_algo"] layer_formats[layer] = GGML_FORMAT_REGISTRY[algo.lower()] if not layer_formats: diff --git a/tests/_test_utils/examples/ggml_export.py b/tests/_test_utils/examples/ggml_export.py new file mode 100644 index 00000000000..cc4ef9bda5e --- /dev/null +++ b/tests/_test_utils/examples/ggml_export.py @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""A tiny LLaMA with GGML-quantized MLPs, for the ``convert_ggml_to_int8.py`` example tests.""" + +import torch +from _test_utils.torch.transformers_models import create_tiny_llama_dir +from transformers import AutoModelForCausalLM + +import modelopt.torch.quantization as mtq + +IQ1_S = {"num_bits": "iq1_s", "block_sizes": {-1: 256}, "backend": "ggml"} +IQ2_XS = {"num_bits": "iq2_xs", "block_sizes": {-1: 256}, "backend": "ggml"} + + +def quantize_tiny_llama(tmp_path, layer_cfgs=(IQ1_S, IQ2_XS)): + """Return a two-layer tiny LLaMA whose layer ``i`` MLP takes ``layer_cfgs[i]``, and its tokens. + + The default formats have 32- and 16-weight sub-blocks, so the INT8 checkpoint needs a config + group for each. The shapes suit vLLM's kernels, and fake quant runs on CPU. + """ + source = create_tiny_llama_dir( + tmp_path, + hidden_size=256, + intermediate_size=512, + num_attention_heads=4, + num_key_value_heads=2, + head_dim=64, + max_position_embeddings=64, + vocab_size=128, + initializer_range=0.05, + ) + model = AutoModelForCausalLM.from_pretrained(source, dtype=torch.bfloat16) + quant_cfg = { + "quant_cfg": [ + {"quantizer_name": "*", "enable": False}, + *( + {"quantizer_name": f"*layers.{i}.mlp*weight_quantizer", "cfg": cfg} + for i, cfg in enumerate(layer_cfgs) + ), + ], + "algorithm": "max", + } + token_ids = torch.randint(0, 128, (1, 48), generator=torch.Generator().manual_seed(0)) + mtq.quantize(model, quant_cfg, lambda m: m(token_ids)) + return model, token_ids diff --git a/tests/_test_utils/examples/hf_ptq_example_utils.py b/tests/_test_utils/examples/hf_ptq_example_utils.py index 0a1b70503ec..21adae3227b 100644 --- a/tests/_test_utils/examples/hf_ptq_example_utils.py +++ b/tests/_test_utils/examples/hf_ptq_example_utils.py @@ -25,7 +25,8 @@ if str(_HF_PTQ_DIR) not in sys.path: sys.path.insert(0, str(_HF_PTQ_DIR)) +import convert_ggml_to_int8 import example_utils import nemotron_vl_calib -__all__ = ["example_utils", "nemotron_vl_calib"] +__all__ = ["convert_ggml_to_int8", "example_utils", "nemotron_vl_calib"] diff --git a/tests/unit/examples/test_convert_ggml_to_int8.py b/tests/examples/hf_ptq/test_convert_ggml_to_int8.py similarity index 65% rename from tests/unit/examples/test_convert_ggml_to_int8.py rename to tests/examples/hf_ptq/test_convert_ggml_to_int8.py index df7f9ddb0f8..9e1d9a5e4c8 100644 --- a/tests/unit/examples/test_convert_ggml_to_int8.py +++ b/tests/examples/hf_ptq/test_convert_ggml_to_int8.py @@ -13,66 +13,40 @@ # See the License for the specific language governing permissions and # limitations under the License. -import importlib.util import json +import subprocess from pathlib import Path import pytest import torch -from _test_utils.torch.transformers_models import create_tiny_llama_dir +from _test_utils.examples.ggml_export import IQ1_S, quantize_tiny_llama +from _test_utils.examples.hf_ptq_example_utils import convert_ggml_to_int8 +from _test_utils.examples.run_command import run_example_command from safetensors.torch import load_file, save_file -from transformers import AutoModelForCausalLM -import modelopt.torch.quantization as mtq from modelopt.torch.export import export_hf_checkpoint from modelopt.torch.quantization.ggml import GGML_FORMAT_REGISTRY -_SCRIPT = Path(__file__).resolve().parents[3] / "examples" / "hf_ptq" / "convert_ggml_to_int8.py" -_SPEC = importlib.util.spec_from_file_location("convert_ggml_to_int8", _SCRIPT) -assert _SPEC is not None and _SPEC.loader is not None -convert_ggml_to_int8 = importlib.util.module_from_spec(_SPEC) -_SPEC.loader.exec_module(convert_ggml_to_int8) - -# Layer 0 takes a format with 32-weight sub-blocks and layer 1 one with 16, so the export needs -# one compressed-tensors config group per group size. +# quantize_tiny_llama's layer 0 is IQ1_S (group 32) and layer 1 IQ2_XS (group 16). LAYER_FORMATS = {0: "iq1_s", 1: "iq2_xs"} -IQ2_XS = {"num_bits": "iq2_xs", "block_sizes": {-1: 256}, "backend": "ggml"} - - -def _export(tmp_path, second_layer_cfg): - source = create_tiny_llama_dir(tmp_path, hidden_size=256, intermediate_size=256) - model = AutoModelForCausalLM.from_pretrained(source, dtype=torch.bfloat16) - layer_cfgs = [ - {"num_bits": "iq1_s", "block_sizes": {-1: 256}, "backend": "ggml"}, - second_layer_cfg, - ] - quant_cfg = { - "quant_cfg": [ - {"quantizer_name": "*", "enable": False}, - *( - {"quantizer_name": f"*layers.{i}.mlp*weight_quantizer", "cfg": cfg} - for i, cfg in enumerate(layer_cfgs) - ), - ], - "algorithm": "max", - } - mtq.quantize(model, quant_cfg, lambda m: m(torch.randint(0, 32, (1, 8)))) - export_dir = tmp_path / "export" - export_hf_checkpoint(model, export_dir=export_dir) - return export_dir + + +def _convert(ggml_dir, int8_dir, *args): + run_example_command( + ["python", "convert_ggml_to_int8.py", str(ggml_dir), str(int8_dir), *args], "hf_ptq" + ) def _files(directory): - return { - path.relative_to(directory): path.read_bytes() - for path in directory.rglob("*") - if path.is_file() - } + return {p.relative_to(directory): p.read_bytes() for p in directory.rglob("*") if p.is_file()} @pytest.fixture(scope="module") def ggml_export(tmp_path_factory): - export_dir = _export(tmp_path_factory.mktemp("ggml"), IQ2_XS) + tmp_path = tmp_path_factory.mktemp("ggml") + model, _ = quantize_tiny_llama(tmp_path) + export_dir = tmp_path / "export" + export_hf_checkpoint(model, export_dir=export_dir) # Files a checkpoint can ship beside its indexed weights: a nested sidecar and an off-index # weight file such as an MTP head. (export_dir / "chat_templates").mkdir() @@ -86,8 +60,8 @@ def test_ggml_export_converts_to_int8_levels_and_steps(ggml_export, tmp_path): before = load_file(ggml_export / "model.safetensors") int8_dir = tmp_path / "int8" - # A small shard size splits the output, so the index has to stitch the shards together. - convert_ggml_to_int8.convert_ggml_export_to_int8(ggml_export, int8_dir, max_shard_size=2**18) + # Shards of about 200 KB split the output, so the index has to stitch them together. + _convert(ggml_export, int8_dir, "--max_shard_size_gb", "0.0002") assert _files(ggml_export) == ggml_files assert not (int8_dir / "hf_quant_config.json").exists() @@ -130,7 +104,7 @@ def test_a_failed_conversion_leaves_the_output_directory_empty(ggml_export, tmp_ def fail(*args, **kwargs): raise OSError("disk full") - # The last step: by then the shards, index and config are all written. + # Called in-process to fail the last step: by then the shards, index and config are written. monkeypatch.setattr(convert_ggml_to_int8, "copy_non_model_files", fail) with pytest.raises(OSError, match="disk full"): @@ -140,9 +114,10 @@ def fail(*args, **kwargs): def test_export_with_a_non_ggml_layer_is_refused_before_writing(tmp_path): - export_dir = _export(tmp_path, {"num_bits": (4, 3), "axis": None}) + model, _ = quantize_tiny_llama(tmp_path, (IQ1_S, {"num_bits": (4, 3), "axis": None})) + export_hf_checkpoint(model, export_dir=tmp_path / "ggml") - with pytest.raises(ValueError, match="Only GGML layers"): - convert_ggml_to_int8.convert_ggml_export_to_int8(export_dir, tmp_path / "int8") + with pytest.raises(subprocess.CalledProcessError): + _convert(tmp_path / "ggml", tmp_path / "int8") assert not (tmp_path / "int8").exists() diff --git a/tests/gpu_vllm/examples/test_convert_ggml_to_int8.py b/tests/examples/vllm_serve/test_convert_ggml_to_int8.py similarity index 55% rename from tests/gpu_vllm/examples/test_convert_ggml_to_int8.py rename to tests/examples/vllm_serve/test_convert_ggml_to_int8.py index 9a64a065a40..18642dd1324 100644 --- a/tests/gpu_vllm/examples/test_convert_ggml_to_int8.py +++ b/tests/examples/vllm_serve/test_convert_ggml_to_int8.py @@ -14,70 +14,36 @@ # limitations under the License. import gc -import importlib.util -from pathlib import Path import pytest import torch -from _test_utils.torch.transformers_models import create_tiny_llama_dir -from transformers import AutoModelForCausalLM +from _test_utils.examples.ggml_export import quantize_tiny_llama +from _test_utils.examples.run_command import run_example_command from vllm import LLM, SamplingParams from vllm.distributed import cleanup_dist_env_and_memory from vllm.inputs import TokensPrompt -import modelopt.torch.quantization as mtq from modelopt.torch.export import export_hf_checkpoint -_SCRIPT = Path(__file__).resolve().parents[3] / "examples" / "hf_ptq" / "convert_ggml_to_int8.py" -_SPEC = importlib.util.spec_from_file_location("convert_ggml_to_int8", _SCRIPT) -assert _SPEC is not None and _SPEC.loader is not None -convert_ggml_to_int8 = importlib.util.module_from_spec(_SPEC) -_SPEC.loader.exec_module(convert_ggml_to_int8) - @pytest.fixture(scope="module") def int8_export_in_vllm(tmp_path_factory): - """vLLM serving the INT8 export of a GGML model, the model's own logprobs and their tokens.""" + """vLLM serving the INT8 conversion of a GGML export, the model's own logprobs and its tokens.""" tmp_path = tmp_path_factory.mktemp("ggml_int8") - source = create_tiny_llama_dir( - tmp_path, - hidden_size=256, - intermediate_size=512, - num_attention_heads=4, - num_key_value_heads=2, - head_dim=64, - max_position_embeddings=64, - vocab_size=128, - initializer_range=0.05, - ) - # Fake quant runs on CPU: the reference needs no GPU, and it spares the GGML CUDA extension build. - model = AutoModelForCausalLM.from_pretrained(source, dtype=torch.bfloat16) # Group 32 (IQ1_S) and group 16 (IQ2_XS) layers take different vLLM kernels. - quant_cfg = { - "quant_cfg": [ - {"quantizer_name": "*", "enable": False}, - *( - { - "quantizer_name": f"*layers.{i}.mlp*weight_quantizer", - "cfg": {"num_bits": num_bits, "block_sizes": {-1: 256}, "backend": "ggml"}, - } - for i, num_bits in enumerate(("iq1_s", "iq2_xs")) - ), - ], - "algorithm": "max", - } - token_ids = torch.randint(0, 128, (1, 48), generator=torch.Generator().manual_seed(0)) - mtq.quantize(model, quant_cfg, lambda m: m(token_ids)) + model, token_ids = quantize_tiny_llama(tmp_path) with torch.no_grad(): logits = model(token_ids).logits[0, :-1].float() expected = logits.log_softmax(-1).gather(-1, token_ids[0, 1:, None])[:, 0] export_hf_checkpoint(model, export_dir=tmp_path / "ggml") - export_dir = tmp_path / "int8" - convert_ggml_to_int8.convert_ggml_export_to_int8(tmp_path / "ggml", export_dir) + run_example_command( + ["python", "convert_ggml_to_int8.py", str(tmp_path / "ggml"), str(tmp_path / "int8")], + "hf_ptq", + ) llm = LLM( - model=str(export_dir), + model=str(tmp_path / "int8"), enforce_eager=True, gpu_memory_utilization=0.2, max_model_len=64, From d1dad230dcc07bd2c984d2032a43e74c92e0502e Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Wed, 21 Oct 2026 20:57:12 +0000 Subject: [PATCH 7/7] Run the converter with sys.executable in the vLLM example test The vLLM image has no python binary, so the vllm_serve lane failed with FileNotFoundError before the test reached vLLM. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Chenjie Luo --- tests/examples/vllm_serve/test_convert_ggml_to_int8.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/examples/vllm_serve/test_convert_ggml_to_int8.py b/tests/examples/vllm_serve/test_convert_ggml_to_int8.py index 18642dd1324..252350e788d 100644 --- a/tests/examples/vllm_serve/test_convert_ggml_to_int8.py +++ b/tests/examples/vllm_serve/test_convert_ggml_to_int8.py @@ -14,6 +14,7 @@ # limitations under the License. import gc +import sys import pytest import torch @@ -37,8 +38,9 @@ def int8_export_in_vllm(tmp_path_factory): expected = logits.log_softmax(-1).gather(-1, token_ids[0, 1:, None])[:, 0] export_hf_checkpoint(model, export_dir=tmp_path / "ggml") + # The vLLM image has no `python` binary. run_example_command( - ["python", "convert_ggml_to_int8.py", str(tmp_path / "ggml"), str(tmp_path / "int8")], + [sys.executable, "convert_ggml_to_int8.py", str(tmp_path / "ggml"), str(tmp_path / "int8")], "hf_ptq", )