From 482c5ec6cfac6a666821ef37f32c6be919ad8f46 Mon Sep 17 00:00:00 2001 From: allanmeng Date: Wed, 2 Sep 2026 12:50:42 +0800 Subject: [PATCH] Fix Qwen3-VL (qwen3vl) mmproj not loaded: Krea2 CLIP falls back to QWEN3_4B gguf_clip_loader only called gguf_mmproj_loader() when arch == "qwen2vl", so a qwen3vl encoder never loaded its vision tower. Without the vision keys, comfy.sd.detect_te_model() cannot find "model.visual.deepstack_merger_list.0.norm.weight", falls back to QWEN3_4B and the text encoder emits a single 2560-dim layer instead of the 12 tapped layers that Krea2 requires ("expects conditioning with 12x2560=30720 features but got 2560"). Affects all backends, not XPU-specific. - add CLIP_VISION_SD_MAP_QWEN3VL: Qwen3-VL differs from Qwen2-VL -- merger is {norm, linear_fc1, linear_fc2} (not {ln_q, mlp.0, mlp.2}), blocks use a fused attn.qkv (not split attn_q/k/v) and mlp.linear_fc1/linear_fc2 (not mlp.up_proj/down_proj). Keys are emitted in HF "model.visual.*" layout so detect_te_model() can identify Qwen3-VL before the KREA2 branch rewrites the prefix. - add qwen3vl_deepstack_remap(): GGUF indexes DeepStack mergers by source vision layer (4B: [5, 11, 17]; 8B/32B: [8, 16, 24]) while ComfyUI expects a sequential list 0..N-1. Indices are derived by sorting the layer ids found in the file, so nothing is hardcoded per model size. - gate on arch in ("qwen2vl", "qwen3vl") and pass arch through to the loader. qwen2vl behaviour is unchanged (arch defaults to None). Verified against the authoritative qwen3vl_4b_fp8_scaled.safetensors layout (315/315 keys, missing=0 / unexpected=0 / shape mismatch=0, cosine ~1.000) and on hardware: Intel Arc B580, Krea2 sampling completes at 2.85 s/it with Krea2RMSNorm shape=[400, 12, 2560]. Supersedes the Qwen3-VL mmproj part of #473 (see comment there: "v.deepstast." typo and layer-index remap bound to a non-existent "ck." prefix). Related: #464. --- loader.py | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/loader.py b/loader.py index 7cefb11..67cae29 100644 --- a/loader.py +++ b/loader.py @@ -219,6 +219,50 @@ def gguf_sd_loader(path, handle_prefix="model.diffusion_model.", is_text_model=F "ln2.": "norm2.", } +# Qwen3-VL vision tower (comfy/text_encoders/qwen35.py: Qwen35VisionModel). +# Differs from Qwen2-VL: merger is {norm, linear_fc1, linear_fc2} (not {ln_q, mlp.0, mlp.2}), +# blocks use a fused attn.qkv (not split attn_q/attn_k/attn_v) and mlp.linear_fc1/linear_fc2 +# (not mlp.up_proj/down_proj). Keys stay in HF `model.visual.*` layout so that +# comfy.sd.detect_te_model() can identify Qwen3-VL via `model.visual.deepstack_merger_list.0.norm.weight` +# before the KREA2/Qwen3-VL branch rewrites `model.visual.` -> `visual.`. +CLIP_VISION_SD_MAP_QWEN3VL = { + "mm.0.": "model.visual.merger.linear_fc1.", + "mm.2.": "model.visual.merger.linear_fc2.", + "v.post_ln.": "model.visual.merger.norm.", + "v.patch_embd": "model.visual.patch_embed.proj", + "v.position_embd": "model.visual.pos_embed", + "v.blk.": "model.visual.blocks.", + "attn_qkv.": "attn.qkv.", + "attn_out.": "attn.proj.", + "ffn_up": "mlp.linear_fc1", + "ffn_down": "mlp.linear_fc2", + "ln1.": "norm1.", + "ln2.": "norm2.", +} + +# GGUF indexes DeepStack mergers by their source vision layer number +# (4B: [5, 11, 17]; 8B / 32B: [8, 16, 24]), ComfyUI expects a sequential list 0..N-1. +QWEN3VL_DEEPSTACK_PROJ = { + "norm": "norm", + "fc1": "linear_fc1", + "fc2": "linear_fc2", +} + +def qwen3vl_deepstack_remap(vsd): + ds_keys = [k for k in vsd if k.startswith("v.deepstack.")] + if not ds_keys: + logging.warning("Qwen3-VL mmproj has no DeepStack tensors - vision features will be degraded.") + return vsd + layer_ids = sorted({int(k.split(".")[2]) for k in ds_keys}) + idx_of = {lid: i for i, lid in enumerate(layer_ids)} + for k in ds_keys: + # v.deepstack.{layer}.{norm|fc1|fc2}.{weight|bias} + _, _, layer, proj, tensor = k.split(".") + new_key = f"model.visual.deepstack_merger_list.{idx_of[int(layer)]}.{QWEN3VL_DEEPSTACK_PROJ[proj]}.{tensor}" + vsd[new_key] = vsd.pop(k) + logging.info(f"Qwen3-VL DeepStack: remapped vision layers {layer_ids} -> list index {list(range(len(layer_ids)))}.") + return vsd + def sd_map_replace(raw_sd, key_map): sd = {} for k,v in raw_sd.items(): @@ -268,7 +312,7 @@ def strip_quant_suffix(name): name = name[:match.start()] return name -def gguf_mmproj_loader(path): +def gguf_mmproj_loader(path, arch=None): # Reverse version of Qwen2VLVisionModel.modify_tensors logging.info("Attenpting to find mmproj file for text encoder...") @@ -305,6 +349,11 @@ def gguf_mmproj_loader(path): w2 = dequantize_tensor(vsd.pop("v.patch_embd.weight.1"), dtype=torch.float32) vsd["v.patch_embd.weight"] = torch.stack([w1, w2], dim=2) + if arch == "qwen3vl": + # Qwen3-VL: fused qkv + DeepStack, and a different merger layout than Qwen2-VL. + vsd = qwen3vl_deepstack_remap(vsd) + return sd_map_replace(vsd, CLIP_VISION_SD_MAP_QWEN3VL) + # run main replacement vsd = sd_map_replace(vsd, CLIP_VISION_SD_MAP) @@ -498,8 +547,8 @@ def gguf_clip_loader(path): sd = sd_map_replace(sd, LLAMA_SD_MAP) if arch == "llama": sd = llama_permute(sd, 32, 8) # L3 / Mistral - if arch == "qwen2vl": - vsd = gguf_mmproj_loader(path) + if arch in ("qwen2vl", "qwen3vl"): + vsd = gguf_mmproj_loader(path, arch=arch) sd.update(vsd) else: pass