From a24dcb308861e725d0ed8b1eca2ae590f68599f7 Mon Sep 17 00:00:00 2001 From: Salim Alkharsa Date: Thu, 15 Oct 2026 22:47:51 +0000 Subject: [PATCH 1/4] Add Muse Glimmer 30B through the chat wrapper Muse Glimmer has no thinking-off switch, so MuseGlimmerVLM prefills the template's answer channel (" to=user<|message|>") with reasoning strength low, pins the template's current date, splits the bf16 weights across both GPUs (device_map="auto"), and resizes images with PIL LANCZOS to the processor's target size, since the processor falls back to BICUBIC with torchvision < 0.27. The generic wrapper gains a device_map option, a _build_inputs hook, and per-GPU peak memory logging. --- README.md | 2 +- docs/hpc.mdx | 4 +- docs/metrics-and-models.mdx | 3 ++ models/hf_chat_vlm.py | 80 ++++++++++++++++++++++++++----- predict/factory.py | 2 + tests/test_muse_glimmer_inputs.py | 66 +++++++++++++++++++++++++ 6 files changed, 143 insertions(+), 14 deletions(-) create mode 100644 tests/test_muse_glimmer_inputs.py diff --git a/README.md b/README.md index 8dba6c2..f7cf19c 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Prepare the dataset files and model weights locally, then run: python demo.py --dataset coco --metric chair --model paligemma --samples 5 --output results/chair-coco.json ``` -Mirage supports `paligemma`, `qwen2vl`, `gemma4-e4b`, `gemma4-12b`, `ministral3-8b`, `ministral3-14b`, and `aya-vision-8b`, plus `-think` variants of the Gemma 4 keys for a thinking ablation. See the [documentation](https://mirage-f432d87b.mintlify.app) for dataset setup, metrics, model configuration, and HPC instructions. +Mirage supports `paligemma`, `qwen2vl`, `gemma4-e4b`, `gemma4-12b`, `ministral3-8b`, `ministral3-14b`, `aya-vision-8b`, and `muse-glimmer-30b`, plus `-think` variants of the Gemma 4 keys for a thinking ablation. See the [documentation](https://mirage-f432d87b.mintlify.app) for dataset setup, metrics, model configuration, and HPC instructions. ## Reference results diff --git a/docs/hpc.mdx b/docs/hpc.mdx index bbae2d0..d50d088 100644 --- a/docs/hpc.mdx +++ b/docs/hpc.mdx @@ -13,7 +13,7 @@ This guide covers a first GPU run and repeatable model comparisons on a SLURM cl - The model weights already available in the Hugging Face cache or on cluster storage. Nodes without internet access cannot download weights during the job. - A GPU allocation sized for the selected checkpoint and evaluation. The sample SLURM request is a starting point, not a guarantee that a model fits. -The repository exposes `paligemma`, `qwen2vl`, `gemma4-e4b`, `gemma4-12b`, `ministral3-8b`, `ministral3-14b`, and `aya-vision-8b` to `demo.py`, plus `gemma4-e4b-think` and `gemma4-12b-think` for a thinking ablation. See [Metrics and models](/metrics-and-models) for checkpoint defaults and overrides. The Gemma 4, Ministral 3, and Aya Vision keys were tested on A100 40 GB GPUs with `transformers==5.19.0`, `torch==2.9.1` (CUDA 12.8), and `mistral-common==1.12.0`, as pinned in `requirements.txt`. DeepSeek-VL is not currently implemented as a Mirage model backend, so it cannot be selected with `--model` or `MIRAGE_VL_MODEL`. +The repository exposes `paligemma`, `qwen2vl`, `gemma4-e4b`, `gemma4-12b`, `ministral3-8b`, `ministral3-14b`, `aya-vision-8b`, and `muse-glimmer-30b` to `demo.py`, plus `gemma4-e4b-think` and `gemma4-12b-think` for a thinking ablation. See [Metrics and models](/metrics-and-models) for checkpoint defaults and overrides. The Gemma 4, Ministral 3, and Aya Vision keys were tested on A100 40 GB GPUs with `transformers==5.19.0`, `torch==2.9.1` (CUDA 12.8), and `mistral-common==1.12.0`, as pinned in `requirements.txt`. `muse-glimmer-30b` was tested with the same packages at checkpoint revision `a4e59da52a7bc87ae7251dd5545c0dd437c44b68`; its bf16 weights (about 60 GB) need two A100 40 GB GPUs on one node, with about 30 GB used on each. DeepSeek-VL is not currently implemented as a Mirage model backend, so it cannot be selected with `--model` or `MIRAGE_VL_MODEL`. ## Clone and prepare the environment @@ -116,4 +116,4 @@ MIRAGE_CACHE_DIR=$PWD/cache/qwen2vl_coco_captions \ sbatch --time=01:30:00 --job-name=qwen2vl_coco_chair slurm/eval_cell.slurm ``` -Give each cell its own cache directory. To score THRONE on the same captions without regenerating them, submit a second cell with the captions cell's `MIRAGE_CACHE_DIR`, `--dependency=afterok:`, and `MIRAGE_EXTRA_ARGS="--cache-only --throne-evaluator --throne-distractor-ratio 38"`. The THRONE evaluator answers are cached in `$MIRAGE_CACHE_DIR/evaluator.sqlite`. To replay a fully cached cell on CPU, add `--partition=short --gres=none`. +Give each cell its own cache directory. To score THRONE on the same captions without regenerating them, submit a second cell with the captions cell's `MIRAGE_CACHE_DIR`, `--dependency=afterok:`, and `MIRAGE_EXTRA_ARGS="--cache-only --throne-evaluator --throne-distractor-ratio 38"`. The THRONE evaluator answers are cached in `$MIRAGE_CACHE_DIR/evaluator.sqlite`. To replay a fully cached cell on CPU, add `--partition=short --gres=none`. For `muse-glimmer-30b`, add `--gres=gpu:a100:2 --cpus-per-task=8 --mem=96G`; the wrapper spreads the weights across both GPUs. diff --git a/docs/metrics-and-models.mdx b/docs/metrics-and-models.mdx index fcbfba5..5a8d90d 100644 --- a/docs/metrics-and-models.mdx +++ b/docs/metrics-and-models.mdx @@ -31,10 +31,13 @@ The prediction factory supports these local vision-language models: | `ministral3-8b` | `mistralai/Ministral-3-8B-Instruct-2512-BF16` | `MIRAGE_MINISTRAL3_8B_MODEL_PATH` | | `ministral3-14b` | `mistralai/Ministral-3-14B-Instruct-2512-BF16` | `MIRAGE_MINISTRAL3_14B_MODEL_PATH` | | `aya-vision-8b` | `CohereLabs/aya-vision-8b` | `MIRAGE_AYA_VISION_8B_MODEL_PATH` | +| `muse-glimmer-30b` | `meta-models/Muse-Glimmer-30B` | `MIRAGE_MUSE_GLIMMER_30B_MODEL_PATH` | | `gemma4-e4b-think`, `gemma4-12b-think` | Same as the Gemma 4 keys, thinking on | Same as the Gemma 4 keys | The Gemma 4, Ministral 3, and Aya Vision keys share one wrapper, `models/hf_chat_vlm.py`. It sends one image and one text turn through the processor's chat template, decodes only the new tokens, and runs with thinking disabled (`enable_thinking=False`). The `gemma4-e4b-think` and `gemma4-12b-think` keys load the same Gemma 4 checkpoints with thinking enabled. They add 1,024 tokens to each metric's `max_new_tokens`, shared by the thought and the answer, and drop the thought before scoring. If the thought does not close within that budget, the wrapper logs a warning and returns the truncated thought as the response, and the metric parses it like any other answer. Aya Vision is a gated, non-commercial (CC-BY-NC) checkpoint; accept its license on Hugging Face before downloading it. +`muse-glimmer-30b` uses the same wrapper through `MuseGlimmerVLM`. Muse Glimmer has no switch that turns thinking off: its chat template only sets a reasoning strength (low, medium, high, or xhigh), and after `<|start|>assistant` the model chooses a reasoning channel (`to=self`) or an answer channel (`to=user`). Mirage keeps `Reasoning strength: low` in the system turn and prefills the answer channel (` to=user<|message|>`), the same turn the template itself writes for an answer, so the model answers without reasoning. The template's current date is pinned to 2026-10-15 so prompts do not change from day to day. Images are resized with PIL LANCZOS to the size the image processor would choose, because the processor falls back to BICUBIC with torchvision older than 0.27; its own resize then leaves the pixels unchanged. + These are the model keys currently supported by `demo.py`. A different checkpoint can be used through the model-path override only when it is compatible with that wrapper. DeepSeek-VL is not currently supported; adding it requires a model wrapper and factory registration before it can be selected as a separate model key. See [Maintain the docs](/maintaining-docs) for the implementation handoff requirements. Set `MIRAGE_VL_MODEL` to choose the default model when `--model` is omitted. Models are loaded for inference at runtime; model weights are not included in the repository. diff --git a/models/hf_chat_vlm.py b/models/hf_chat_vlm.py index ab43bcb..e9b0f35 100644 --- a/models/hf_chat_vlm.py +++ b/models/hf_chat_vlm.py @@ -1,7 +1,7 @@ """Generic wrapper for Hugging Face chat vision-language models. Covers models whose processor ships a chat template that accepts one image -and one text turn (Gemma 4, Ministral 3, Aya Vision). Decoding is greedy by +and one text turn (Gemma 4, Ministral 3, Aya Vision, Muse Glimmer). Decoding is greedy by default and only the newly generated tokens are returned. """ @@ -34,6 +34,8 @@ class HFChatVLM(BaseModel): thinking_tokens: With thinking on, extra new tokens added to ``max_new_tokens``; thought and answer share the combined budget. device: Device for inference (default: CUDA when available). + device_map: ``from_pretrained`` device map; ``"auto"`` splits the + weights across all visible GPUs (default: ``device``). torch_dtype: Weight dtype (default: bfloat16). """ @@ -43,6 +45,7 @@ def __init__( enable_thinking: bool = False, thinking_tokens: int = 1024, device: Optional[str] = None, + device_map: Optional[str] = None, torch_dtype: Optional[torch.dtype] = None, **kwargs, ): @@ -51,6 +54,7 @@ def __init__( self.enable_thinking = enable_thinking self.thinking_tokens = thinking_tokens if enable_thinking else 0 self.device = device or ("cuda" if torch.cuda.is_available() else "cpu") + self.device_map = device_map self.torch_dtype = torch_dtype or torch.bfloat16 self.model = None self.processor = None @@ -63,7 +67,7 @@ def load_model(self) -> None: self.processor = AutoProcessor.from_pretrained(self.model_name) self.model = AutoModelForImageTextToText.from_pretrained( - self.model_name, dtype=self.torch_dtype, device_map=self.device, + self.model_name, dtype=self.torch_dtype, device_map=self.device_map or self.device, ).eval() self._raw_logged = False vocab = getattr(self.processor.tokenizer, "get_vocab", dict)() @@ -75,7 +79,8 @@ def load_model(self) -> None: def unload_model(self) -> None: if torch.cuda.is_available() and self.model is not None: - logger.info("Peak CUDA memory: %.2f GB", torch.cuda.max_memory_allocated() / 1e9) + for index in range(torch.cuda.device_count()): + logger.info("Peak CUDA memory (cuda:%d): %.2f GB", index, torch.cuda.max_memory_allocated(index) / 1e9) self.model = None self.processor = None if torch.cuda.is_available(): @@ -93,14 +98,7 @@ def generate( content = [{"type": "text", "text": prompt}] if image is not None: content.insert(0, {"type": "image", "image": self.preprocess_image(image)}) - inputs = self.processor.apply_chat_template( - [{"role": "user", "content": content}], - add_generation_prompt=True, - tokenize=True, - return_dict=True, - return_tensors="pt", - enable_thinking=self.enable_thinking, - ).to(self.model.device) + inputs = self._build_inputs(content).to(self.model.device) for key, value in inputs.items(): if torch.is_floating_point(value): inputs[key] = value.to(self.torch_dtype) @@ -124,6 +122,16 @@ def generate( text = self.processor.decode(new_tokens, skip_special_tokens=True) return _THINKING_RE.sub("", text).strip() + def _build_inputs(self, content): + return self.processor.apply_chat_template( + [{"role": "user", "content": content}], + add_generation_prompt=True, + tokenize=True, + return_dict=True, + return_tensors="pt", + enable_thinking=self.enable_thinking, + ) + def batch_generate( self, prompts: List[str], @@ -139,3 +147,53 @@ class HFChatThinkingVLM(HFChatVLM): def __init__(self, model_name: str, **kwargs): super().__init__(model_name, enable_thinking=True, **kwargs) + + +class MuseGlimmerVLM(HFChatVLM): + """Muse Glimmer: reasoning skipped, weights split across all visible GPUs. + + The chat template has no switch to turn reasoning off; after + ``<|start|>assistant`` the model chooses the reasoning channel + (``to=self``) or the answer channel (``to=user``). The prompt is prefilled + with the answer channel, so the response is the answer alone. The date the + template writes into the system turn is pinned so prompts do not change + from day to day. + + The image processor asks for LANCZOS resizing but falls back to BICUBIC + with torchvision < 0.27. Images are therefore resized here with PIL + LANCZOS to the size the processor would pick, which makes its own resize a + same-size pass (pixel values identical to ``do_resize=False``). + """ + + ANSWER_PREFIX = " to=user<|message|>" + TEMPLATE_KWARGS = {"reasoning_strength": "low", "current_date": "2026-10-15"} + + def __init__(self, model_name: str, **kwargs): + kwargs.setdefault("device_map", "auto") + super().__init__(model_name, **kwargs) + + def preprocess_image(self, image): + from transformers.models.muse_glimmer.image_processing_muse_glimmer import smart_resize + + image = super().preprocess_image(image) + processor = self.processor.image_processor + height, width = smart_resize(image.height, image.width, + processor.patch_size * processor.merge_size, processor.max_image_tokens) + if (width, height) == image.size: + return image + return image.resize((width, height), Image.LANCZOS) + + def _build_inputs(self, content): + images = [part.pop("image") for part in content if part["type"] == "image"] + text = self.processor.apply_chat_template( + [{"role": "user", "content": content}], + add_generation_prompt=True, + tokenize=False, + **self.TEMPLATE_KWARGS, + ) + return self.processor( + text=[text + self.ANSWER_PREFIX], + images=images or None, + add_special_tokens=False, + return_tensors="pt", + ) diff --git a/predict/factory.py b/predict/factory.py index 2fe422f..3ea94ff 100644 --- a/predict/factory.py +++ b/predict/factory.py @@ -27,6 +27,8 @@ "MIRAGE_MINISTRAL3_14B_MODEL_PATH", "mistralai/Ministral-3-14B-Instruct-2512-BF16"), "aya-vision-8b": ("models.hf_chat_vlm", "HFChatVLM", "MIRAGE_AYA_VISION_8B_MODEL_PATH", "CohereLabs/aya-vision-8b"), + "muse-glimmer-30b": ("models.hf_chat_vlm", "MuseGlimmerVLM", + "MIRAGE_MUSE_GLIMMER_30B_MODEL_PATH", "meta-models/Muse-Glimmer-30B"), # Thinking-on variants for the ablation (same checkpoints). "gemma4-e4b-think": ("models.hf_chat_vlm", "HFChatThinkingVLM", "MIRAGE_GEMMA4_E4B_MODEL_PATH", "google/gemma-4-E4B-it"), diff --git a/tests/test_muse_glimmer_inputs.py b/tests/test_muse_glimmer_inputs.py new file mode 100644 index 0000000..6fd9a91 --- /dev/null +++ b/tests/test_muse_glimmer_inputs.py @@ -0,0 +1,66 @@ +"""Muse Glimmer prompt construction: answer channel prefilled, template kwargs pinned. + +Uses a fake processor, so no checkpoint or GPU is needed (torch is imported). + +Run: python3 -m pytest tests/test_muse_glimmer_inputs.py -q +""" +import sys +from pathlib import Path + +from PIL import Image + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT)) + +from models.hf_chat_vlm import MuseGlimmerVLM +from predict.factory import _REGISTRY + + +class FakeImageProcessor: + patch_size, merge_size, max_image_tokens = 14, 2, 4096 + + +class FakeProcessor: + image_processor = FakeImageProcessor() + + def __init__(self): + self.template_calls, self.calls = [], [] + + def apply_chat_template(self, messages, **kwargs): + self.template_calls.append((messages, kwargs)) + return "<|start|>user<|message|><|patch|>Q<|eot|><|start|>assistant" + + def __call__(self, **kwargs): + self.calls.append(kwargs) + return kwargs + + +def test_answer_channel_is_prefilled(): + model = MuseGlimmerVLM("unused", device="cpu") + model.processor = FakeProcessor() + image = Image.new("RGB", (28, 28)) + model._build_inputs([{"type": "image", "image": image}, {"type": "text", "text": "Q"}]) + + messages, kwargs = model.processor.template_calls[0] + assert kwargs["tokenize"] is False and kwargs["add_generation_prompt"] is True + assert kwargs["reasoning_strength"] == "low" and kwargs["current_date"] == "2026-10-15" + assert messages[0]["content"] == [{"type": "image"}, {"type": "text", "text": "Q"}] + + call = model.processor.calls[0] + assert call["text"] == ["<|start|>user<|message|><|patch|>Q<|eot|><|start|>assistant to=user<|message|>"] + assert call["images"] == [image] + assert call["add_special_tokens"] is False + + +def test_weights_split_across_gpus_by_default(): + assert MuseGlimmerVLM("unused", device="cpu").device_map == "auto" + assert _REGISTRY["muse-glimmer-30b"][1] == "MuseGlimmerVLM" + + +def test_image_is_lanczos_resized_to_the_processor_grid(): + model = MuseGlimmerVLM("unused", device="cpu") + model.processor = FakeProcessor() + # Same size the processor picks for a 375x500 COCO image (checked on a GPU node). + assert model.preprocess_image(Image.new("RGB", (375, 500))).size == (364, 476) + already = Image.new("RGB", (364, 476)) + assert model.preprocess_image(already).size == (364, 476) From f6e1aaaf5c2c838b101568af376fc3d6b116f7b2 Mon Sep 17 00:00:00 2001 From: Salim Alkharsa Date: Fri, 16 Oct 2026 01:03:01 +0000 Subject: [PATCH 2/4] Pin the HFChatVLM chat-template call across the _build_inputs hook Drives generate() with a fake processor and model and checks the exact messages and apply_chat_template kwargs, thinking off and on. The same assertions pass against the wrapper before the hook was added. Also re-wraps the module docstring. --- models/hf_chat_vlm.py | 4 +-- tests/test_hf_chat_inputs.py | 70 ++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 tests/test_hf_chat_inputs.py diff --git a/models/hf_chat_vlm.py b/models/hf_chat_vlm.py index e9b0f35..fd0297e 100644 --- a/models/hf_chat_vlm.py +++ b/models/hf_chat_vlm.py @@ -1,8 +1,8 @@ """Generic wrapper for Hugging Face chat vision-language models. Covers models whose processor ships a chat template that accepts one image -and one text turn (Gemma 4, Ministral 3, Aya Vision, Muse Glimmer). Decoding is greedy by -default and only the newly generated tokens are returned. +and one text turn (Gemma 4, Ministral 3, Aya Vision, Muse Glimmer). Decoding +is greedy by default and only the newly generated tokens are returned. """ import logging diff --git a/tests/test_hf_chat_inputs.py b/tests/test_hf_chat_inputs.py new file mode 100644 index 0000000..36a001c --- /dev/null +++ b/tests/test_hf_chat_inputs.py @@ -0,0 +1,70 @@ +"""HFChatVLM prompt construction is unchanged by the ``_build_inputs`` hook. + +Drives ``generate`` with a fake processor and model and checks that the chat +template gets exactly the messages and kwargs the wrapper passed before the +hook existed. No checkpoint or GPU is needed (torch is imported). + +Run: python3 -m pytest tests/test_hf_chat_inputs.py -q +""" +import sys +from pathlib import Path + +import pytest +import torch +from PIL import Image + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT)) + +from models.hf_chat_vlm import HFChatThinkingVLM, HFChatVLM + + +class FakeInputs(dict): + def to(self, device): + self.device = device + return self + + +class FakeProcessor: + def __init__(self): + self.template_calls = [] + + def apply_chat_template(self, messages, **kwargs): + self.template_calls.append((messages, kwargs)) + return FakeInputs(input_ids=torch.tensor([[1, 2, 3]]), pixel_values=torch.zeros(1, 3, dtype=torch.float32)) + + def decode(self, tokens, skip_special_tokens=True): + return "yes" + + +class FakeModel: + device = "cpu" + + def generate(self, input_ids, **kwargs): + return torch.cat([input_ids, torch.tensor([[7]])], dim=1) + + +def _run(model, image): + model.processor, model.model = FakeProcessor(), FakeModel() + model.is_loaded, model._raw_logged, model._thinking_end_ids = True, False, set() + assert model.generate("Is there a dog?", image=image) == "yes" + return model.processor.template_calls + + +@pytest.mark.parametrize("cls, thinking", [(HFChatVLM, False), (HFChatThinkingVLM, True)]) +def test_template_call_matches_pre_hook_wrapper(cls, thinking): + image = Image.new("RGB", (28, 28)) + (messages, kwargs), = _run(cls("unused", device="cpu"), image) + assert messages == [{"role": "user", "content": [ + {"type": "image", "image": image}, {"type": "text", "text": "Is there a dog?"}]}] + assert kwargs == {"add_generation_prompt": True, "tokenize": True, "return_dict": True, + "return_tensors": "pt", "enable_thinking": thinking} + + +def test_text_only_prompt_has_no_image_part(): + (messages, _), = _run(HFChatVLM("unused", device="cpu"), None) + assert messages[0]["content"] == [{"type": "text", "text": "Is there a dog?"}] + + +def test_default_device_map_is_the_device(): + assert HFChatVLM("unused", device="cpu").device_map is None From fe1cfc6833e744ce8efad9e2ec32c806aff643c2 Mon Sep 17 00:00:00 2001 From: Salim Alkharsa Date: Sat, 17 Oct 2026 04:35:22 +0000 Subject: [PATCH 3/4] Move MuseGlimmerVLM into models/muse_glimmer.py Class body unchanged (cache_name stays MuseGlimmerVLM); the registry row and the lazy models exports point at the new module. --- models/__init__.py | 1 + models/hf_chat.py | 51 --------------------------------------- models/muse_glimmer.py | 55 ++++++++++++++++++++++++++++++++++++++++++ predict/factory.py | 2 +- 4 files changed, 57 insertions(+), 52 deletions(-) create mode 100644 models/muse_glimmer.py diff --git a/models/__init__.py b/models/__init__.py index 64e5f26..ed2b0b2 100644 --- a/models/__init__.py +++ b/models/__init__.py @@ -13,6 +13,7 @@ "Gemma4Thinking": "gemma4", "Ministral3": "ministral3", "AyaVision": "aya_vision", + "MuseGlimmerVLM": "muse_glimmer", } __all__ = list(_MODULES) diff --git a/models/hf_chat.py b/models/hf_chat.py index e3db1ce..4212829 100644 --- a/models/hf_chat.py +++ b/models/hf_chat.py @@ -133,54 +133,3 @@ def _build_inputs(self, content): return_tensors="pt", enable_thinking=self.enable_thinking, ) - - -class MuseGlimmerVLM(HFChatVLM): - """Muse Glimmer: reasoning skipped, weights split across all visible GPUs. - - The chat template has no switch to turn reasoning off; after - ``<|start|>assistant`` the model chooses the reasoning channel - (``to=self``) or the answer channel (``to=user``). The prompt is prefilled - with the answer channel, so the response is the answer alone. The date the - template writes into the system turn is pinned so prompts do not change - from day to day. - - The image processor asks for LANCZOS resizing but falls back to BICUBIC - with torchvision < 0.27. Images are therefore resized here with PIL - LANCZOS to the size the processor would pick, which makes its own resize a - same-size pass (pixel values identical to ``do_resize=False``). - """ - - cache_name = "MuseGlimmerVLM" - ANSWER_PREFIX = " to=user<|message|>" - TEMPLATE_KWARGS = {"reasoning_strength": "low", "current_date": "2026-10-15"} - - def __init__(self, model_name: str, **kwargs): - kwargs.setdefault("device_map", "auto") - super().__init__(model_name, **kwargs) - - def preprocess_image(self, image): - from transformers.models.muse_glimmer.image_processing_muse_glimmer import smart_resize - - image = super().preprocess_image(image) - processor = self.processor.image_processor - height, width = smart_resize(image.height, image.width, - processor.patch_size * processor.merge_size, processor.max_image_tokens) - if (width, height) == image.size: - return image - return image.resize((width, height), Image.LANCZOS) - - def _build_inputs(self, content): - images = [part.pop("image") for part in content if part["type"] == "image"] - text = self.processor.apply_chat_template( - [{"role": "user", "content": content}], - add_generation_prompt=True, - tokenize=False, - **self.TEMPLATE_KWARGS, - ) - return self.processor( - text=[text + self.ANSWER_PREFIX], - images=images or None, - add_special_tokens=False, - return_tensors="pt", - ) diff --git a/models/muse_glimmer.py b/models/muse_glimmer.py new file mode 100644 index 0000000..f622675 --- /dev/null +++ b/models/muse_glimmer.py @@ -0,0 +1,55 @@ +"""Muse Glimmer (meta-models/Muse-Glimmer-*) through the generic chat wrapper.""" +from PIL import Image + +from .hf_chat import HFChatVLM + + +class MuseGlimmerVLM(HFChatVLM): + """Muse Glimmer: reasoning skipped, weights split across all visible GPUs. + + The chat template has no switch to turn reasoning off; after + ``<|start|>assistant`` the model chooses the reasoning channel + (``to=self``) or the answer channel (``to=user``). The prompt is prefilled + with the answer channel, so the response is the answer alone. The date the + template writes into the system turn is pinned so prompts do not change + from day to day. + + The image processor asks for LANCZOS resizing but falls back to BICUBIC + with torchvision < 0.27. Images are therefore resized here with PIL + LANCZOS to the size the processor would pick, which makes its own resize a + same-size pass (pixel values identical to ``do_resize=False``). + """ + + cache_name = "MuseGlimmerVLM" + ANSWER_PREFIX = " to=user<|message|>" + TEMPLATE_KWARGS = {"reasoning_strength": "low", "current_date": "2026-10-15"} + + def __init__(self, model_name: str, **kwargs): + kwargs.setdefault("device_map", "auto") + super().__init__(model_name, **kwargs) + + def preprocess_image(self, image): + from transformers.models.muse_glimmer.image_processing_muse_glimmer import smart_resize + + image = super().preprocess_image(image) + processor = self.processor.image_processor + height, width = smart_resize(image.height, image.width, + processor.patch_size * processor.merge_size, processor.max_image_tokens) + if (width, height) == image.size: + return image + return image.resize((width, height), Image.LANCZOS) + + def _build_inputs(self, content): + images = [part.pop("image") for part in content if part["type"] == "image"] + text = self.processor.apply_chat_template( + [{"role": "user", "content": content}], + add_generation_prompt=True, + tokenize=False, + **self.TEMPLATE_KWARGS, + ) + return self.processor( + text=[text + self.ANSWER_PREFIX], + images=images or None, + add_special_tokens=False, + return_tensors="pt", + ) diff --git a/predict/factory.py b/predict/factory.py index 4736136..573c87b 100644 --- a/predict/factory.py +++ b/predict/factory.py @@ -44,7 +44,7 @@ class Entry(NamedTuple): "mistralai/Ministral-3-14B-Instruct-2512-BF16", "HFChatVLM", _BF16), "aya-vision-8b": Entry("models.aya_vision:AyaVision", "MIRAGE_AYA_VISION_8B_MODEL_PATH", "CohereLabs/aya-vision-8b", "HFChatVLM", _BF16), - "muse-glimmer-30b": Entry("models.hf_chat:MuseGlimmerVLM", "MIRAGE_MUSE_GLIMMER_30B_MODEL_PATH", + "muse-glimmer-30b": Entry("models.muse_glimmer:MuseGlimmerVLM", "MIRAGE_MUSE_GLIMMER_30B_MODEL_PATH", "meta-models/Muse-Glimmer-30B", "MuseGlimmerVLM", _BF16), # Thinking-on variants for the ablation (same checkpoints). "gemma4-e4b-think": Entry("models.gemma4:Gemma4Thinking", "MIRAGE_GEMMA4_E4B_MODEL_PATH", From cc9a3bf97274a5919ff7ad721f3344350cca75a4 Mon Sep 17 00:00:00 2001 From: Salim Alkharsa Date: Sat, 17 Oct 2026 04:51:08 +0000 Subject: [PATCH 4/4] Name the Muse Glimmer class MuseGlimmer, like the other families cache_name stays MuseGlimmerVLM, so cache keys do not change. --- models/__init__.py | 2 +- models/muse_glimmer.py | 2 +- predict/factory.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/models/__init__.py b/models/__init__.py index ed2b0b2..9fa422f 100644 --- a/models/__init__.py +++ b/models/__init__.py @@ -13,7 +13,7 @@ "Gemma4Thinking": "gemma4", "Ministral3": "ministral3", "AyaVision": "aya_vision", - "MuseGlimmerVLM": "muse_glimmer", + "MuseGlimmer": "muse_glimmer", } __all__ = list(_MODULES) diff --git a/models/muse_glimmer.py b/models/muse_glimmer.py index f622675..084bea1 100644 --- a/models/muse_glimmer.py +++ b/models/muse_glimmer.py @@ -4,7 +4,7 @@ from .hf_chat import HFChatVLM -class MuseGlimmerVLM(HFChatVLM): +class MuseGlimmer(HFChatVLM): """Muse Glimmer: reasoning skipped, weights split across all visible GPUs. The chat template has no switch to turn reasoning off; after diff --git a/predict/factory.py b/predict/factory.py index 573c87b..4fc283e 100644 --- a/predict/factory.py +++ b/predict/factory.py @@ -44,7 +44,7 @@ class Entry(NamedTuple): "mistralai/Ministral-3-14B-Instruct-2512-BF16", "HFChatVLM", _BF16), "aya-vision-8b": Entry("models.aya_vision:AyaVision", "MIRAGE_AYA_VISION_8B_MODEL_PATH", "CohereLabs/aya-vision-8b", "HFChatVLM", _BF16), - "muse-glimmer-30b": Entry("models.muse_glimmer:MuseGlimmerVLM", "MIRAGE_MUSE_GLIMMER_30B_MODEL_PATH", + "muse-glimmer-30b": Entry("models.muse_glimmer:MuseGlimmer", "MIRAGE_MUSE_GLIMMER_30B_MODEL_PATH", "meta-models/Muse-Glimmer-30B", "MuseGlimmerVLM", _BF16), # Thinking-on variants for the ablation (same checkpoints). "gemma4-e4b-think": Entry("models.gemma4:Gemma4Thinking", "MIRAGE_GEMMA4_E4B_MODEL_PATH",