Read the dtype from text_config in convert_ggml_to_int8.py

examples/hf_ptq/convert_ggml_to_int8.py

@@ -62,7 +62,15 @@ def convert_ggml_export_to_int8(
     ggml_dir, output_dir = Path(ggml_dir), Path(output_dir)
     config = json.loads((ggml_dir / "config.json").read_text())
     quantization = json.loads((ggml_dir / "hf_quant_config.json").read_text())["quantization"]
-    scale_dtype = getattr(torch, config.get("dtype") or config["torch_dtype"])
+    # Multimodal configs such as GLM-5.3-Flash's keep the dtype in text_config only.
+    text_config = config.get("text_config", {})
+    dtype = (
+        config.get("dtype")
+        or config.get("torch_dtype")
+        or text_config.get("dtype")
+        or text_config["torch_dtype"]
+    )
+    scale_dtype = getattr(torch, dtype)
     weight_map = indexed_weight_map(ggml_dir)
     layer_formats = _ggml_layer_formats(ggml_dir, weight_map, quantization)
     if output_dir.exists() and any(output_dir.iterdir()):

tests/examples/hf_ptq/test_convert_ggml_to_int8.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 import json
+import shutil
 import subprocess
 from pathlib import Path
 
@@ -100,6 +101,24 @@ def test_ggml_export_converts_to_int8_levels_and_steps(ggml_export, tmp_path):
         assert torch.equal(after[name], tensor)
 
 
+def test_dtype_is_read_from_text_config(ggml_export, tmp_path):
+    # Multimodal configs keep the dtype in text_config only.
+    ggml_dir = tmp_path / "ggml"
+    shutil.copytree(ggml_export, ggml_dir)
+    config = json.loads((ggml_dir / "config.json").read_text())
+    dtype = config.pop("dtype", None) or config.pop("torch_dtype")
+    config.pop("torch_dtype", None)
+    config["text_config"] = {"dtype": dtype}
+    (ggml_dir / "config.json").write_text(json.dumps(config))
+
+    int8_dir = tmp_path / "int8"
+    convert_ggml_to_int8.convert_ggml_export_to_int8(ggml_dir, int8_dir)
+
+    name = "model.layers.0.mlp.up_proj.weight_scale"
+    shard = json.loads((int8_dir / "model.safetensors.index.json").read_text())["weight_map"][name]
+    assert load_file(int8_dir / shard)[name].dtype == torch.bfloat16
+
+
 def test_a_failed_conversion_leaves_the_output_directory_empty(ggml_export, tmp_path, monkeypatch):
     def fail(*args, **kwargs):
         raise OSError("disk full")