diff --git a/README.md b/README.md index d7acf4e..67b9b75 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Prepare the dataset files and model weights locally, then run: python demo.py --dataset coco --metric chair --model paligemma --samples 5 --output results/chair-coco.json ``` -Mirage supports `paligemma`, `qwen2vl`, `gemma4-e4b`, `gemma4-12b`, `ministral3-8b`, `ministral3-14b`, and `aya-vision-8b`. See the [documentation](https://mirage-f432d87b.mintlify.app) for dataset setup, metrics, model configuration, and HPC instructions. +Mirage supports `paligemma`, `qwen2vl`, `gemma4-e4b`, `gemma4-12b`, `ministral3-8b`, `ministral3-14b`, `aya-vision-8b`, and `muse-glimmer-30b`. See the [documentation](https://mirage-f432d87b.mintlify.app) for dataset setup, metrics, model configuration, and HPC instructions. ## Reference results diff --git a/models/__init__.py b/models/__init__.py index 64e5f26..9fa422f 100644 --- a/models/__init__.py +++ b/models/__init__.py @@ -13,6 +13,7 @@ "Gemma4Thinking": "gemma4", "Ministral3": "ministral3", "AyaVision": "aya_vision", + "MuseGlimmer": "muse_glimmer", } __all__ = list(_MODULES) diff --git a/models/hf_chat.py b/models/hf_chat.py index acf60be..4212829 100644 --- a/models/hf_chat.py +++ b/models/hf_chat.py @@ -1,8 +1,8 @@ """Generic wrapper for Hugging Face chat vision-language models. Covers models whose processor ships a chat template that accepts one image -and one text turn (Gemma 4, Ministral 3, Aya Vision). Decoding is greedy by -default and only the newly generated tokens are returned. +and one text turn (Gemma 4, Ministral 3, Aya Vision, Muse Glimmer). Decoding +is greedy by default and only the newly generated tokens are returned. """ import logging @@ -34,6 +34,8 @@ class HFChatVLM(BaseModel): thinking_tokens: With thinking on, extra new tokens added to ``max_new_tokens``; thought and answer share the combined budget. device: Device for inference (default: CUDA when available). + device_map: ``from_pretrained`` device map; ``"auto"`` splits the + weights across all visible GPUs (default: ``device``). torch_dtype: Weight dtype (default: bfloat16). """ @@ -45,6 +47,7 @@ def __init__( enable_thinking: bool = False, thinking_tokens: int = 1024, device: Optional[str] = None, + device_map: Optional[str] = None, torch_dtype: Optional[torch.dtype] = None, **kwargs, ): @@ -53,6 +56,7 @@ def __init__( self.enable_thinking = enable_thinking self.thinking_tokens = thinking_tokens if enable_thinking else 0 self.device = device or ("cuda" if torch.cuda.is_available() else "cpu") + self.device_map = device_map self.torch_dtype = torch_dtype or torch.bfloat16 self.model = None self.processor = None @@ -65,7 +69,7 @@ def load_model(self) -> None: self.processor = AutoProcessor.from_pretrained(self.model_name) self.model = AutoModelForImageTextToText.from_pretrained( - self.model_name, dtype=self.torch_dtype, device_map=self.device, + self.model_name, dtype=self.torch_dtype, device_map=self.device_map or self.device, ).eval() self._raw_logged = False vocab = getattr(self.processor.tokenizer, "get_vocab", dict)() @@ -77,7 +81,8 @@ def load_model(self) -> None: def unload_model(self) -> None: if torch.cuda.is_available() and self.model is not None: - logger.info("Peak CUDA memory: %.2f GB", torch.cuda.max_memory_allocated() / 1e9) + for index in range(torch.cuda.device_count()): + logger.info("Peak CUDA memory (cuda:%d): %.2f GB", index, torch.cuda.max_memory_allocated(index) / 1e9) self.model = None self.processor = None if torch.cuda.is_available(): @@ -95,14 +100,7 @@ def generate( content = [{"type": "text", "text": prompt}] if image is not None: content.insert(0, {"type": "image", "image": self.preprocess_image(image)}) - inputs = self.processor.apply_chat_template( - [{"role": "user", "content": content}], - add_generation_prompt=True, - tokenize=True, - return_dict=True, - return_tensors="pt", - enable_thinking=self.enable_thinking, - ).to(self.model.device) + inputs = self._build_inputs(content).to(self.model.device) for key, value in inputs.items(): if torch.is_floating_point(value): inputs[key] = value.to(self.torch_dtype) @@ -125,3 +123,13 @@ def generate( logger.warning("Thinking block not closed within %d tokens", gen_kwargs["max_new_tokens"]) text = self.processor.decode(new_tokens, skip_special_tokens=True) return _THINKING_RE.sub("", text).strip() + + def _build_inputs(self, content): + return self.processor.apply_chat_template( + [{"role": "user", "content": content}], + add_generation_prompt=True, + tokenize=True, + return_dict=True, + return_tensors="pt", + enable_thinking=self.enable_thinking, + ) diff --git a/models/muse_glimmer.py b/models/muse_glimmer.py new file mode 100644 index 0000000..084bea1 --- /dev/null +++ b/models/muse_glimmer.py @@ -0,0 +1,55 @@ +"""Muse Glimmer (meta-models/Muse-Glimmer-*) through the generic chat wrapper.""" +from PIL import Image + +from .hf_chat import HFChatVLM + + +class MuseGlimmer(HFChatVLM): + """Muse Glimmer: reasoning skipped, weights split across all visible GPUs. + + The chat template has no switch to turn reasoning off; after + ``<|start|>assistant`` the model chooses the reasoning channel + (``to=self``) or the answer channel (``to=user``). The prompt is prefilled + with the answer channel, so the response is the answer alone. The date the + template writes into the system turn is pinned so prompts do not change + from day to day. + + The image processor asks for LANCZOS resizing but falls back to BICUBIC + with torchvision < 0.27. Images are therefore resized here with PIL + LANCZOS to the size the processor would pick, which makes its own resize a + same-size pass (pixel values identical to ``do_resize=False``). + """ + + cache_name = "MuseGlimmerVLM" + ANSWER_PREFIX = " to=user<|message|>" + TEMPLATE_KWARGS = {"reasoning_strength": "low", "current_date": "2026-10-15"} + + def __init__(self, model_name: str, **kwargs): + kwargs.setdefault("device_map", "auto") + super().__init__(model_name, **kwargs) + + def preprocess_image(self, image): + from transformers.models.muse_glimmer.image_processing_muse_glimmer import smart_resize + + image = super().preprocess_image(image) + processor = self.processor.image_processor + height, width = smart_resize(image.height, image.width, + processor.patch_size * processor.merge_size, processor.max_image_tokens) + if (width, height) == image.size: + return image + return image.resize((width, height), Image.LANCZOS) + + def _build_inputs(self, content): + images = [part.pop("image") for part in content if part["type"] == "image"] + text = self.processor.apply_chat_template( + [{"role": "user", "content": content}], + add_generation_prompt=True, + tokenize=False, + **self.TEMPLATE_KWARGS, + ) + return self.processor( + text=[text + self.ANSWER_PREFIX], + images=images or None, + add_special_tokens=False, + return_tensors="pt", + ) diff --git a/predict/factory.py b/predict/factory.py index e467d1d..dcdf628 100644 --- a/predict/factory.py +++ b/predict/factory.py @@ -46,6 +46,8 @@ class Entry(NamedTuple): "mistralai/Ministral-3-14B-Instruct-2512-BF16", "HFChatVLM", _BF16), "aya-vision-8b": Entry("models.aya_vision:AyaVision", "MIRAGE_AYA_VISION_8B_MODEL_PATH", "CohereLabs/aya-vision-8b", "HFChatVLM", _BF16), + "muse-glimmer-30b": Entry("models.muse_glimmer:MuseGlimmer", "MIRAGE_MUSE_GLIMMER_30B_MODEL_PATH", + "meta-models/Muse-Glimmer-30B", "MuseGlimmerVLM", _BF16), # Thinking-on variants for the ablation (same checkpoints). "gemma4-e4b-think": Entry("models.gemma4:Gemma4Thinking", "MIRAGE_GEMMA4_E4B_MODEL_PATH", "google/gemma-4-E4B-it", "HFChatThinkingVLM", _BF16),