Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ Prepare the dataset files and model weights locally, then run:
python demo.py --dataset coco --metric chair --model paligemma --samples 5 --output results/chair-coco.json
```

Mirage supports `paligemma`, `qwen2vl`, `gemma4-e4b`, `gemma4-12b`, `ministral3-8b`, `ministral3-14b`, and `aya-vision-8b`. See the [documentation](https://mirage-f432d87b.mintlify.app) for dataset setup, metrics, model configuration, and HPC instructions.
Mirage supports `paligemma`, `qwen2vl`, `gemma4-e4b`, `gemma4-12b`, `ministral3-8b`, `ministral3-14b`, `aya-vision-8b`, and `muse-glimmer-30b`. See the [documentation](https://mirage-f432d87b.mintlify.app) for dataset setup, metrics, model configuration, and HPC instructions.

## Reference results

Expand Down
1 change: 1 addition & 0 deletions models/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
"Gemma4Thinking": "gemma4",
"Ministral3": "ministral3",
"AyaVision": "aya_vision",
"MuseGlimmer": "muse_glimmer",
}

__all__ = list(_MODULES)
Expand Down
32 changes: 20 additions & 12 deletions models/hf_chat.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
"""Generic wrapper for Hugging Face chat vision-language models.

Covers models whose processor ships a chat template that accepts one image
and one text turn (Gemma 4, Ministral 3, Aya Vision). Decoding is greedy by
default and only the newly generated tokens are returned.
and one text turn (Gemma 4, Ministral 3, Aya Vision, Muse Glimmer). Decoding
is greedy by default and only the newly generated tokens are returned.
"""

import logging
Expand Down Expand Up @@ -34,6 +34,8 @@ class HFChatVLM(BaseModel):
thinking_tokens: With thinking on, extra new tokens added to
``max_new_tokens``; thought and answer share the combined budget.
device: Device for inference (default: CUDA when available).
device_map: ``from_pretrained`` device map; ``"auto"`` splits the
weights across all visible GPUs (default: ``device``).
torch_dtype: Weight dtype (default: bfloat16).
"""

Expand All @@ -45,6 +47,7 @@ def __init__(
enable_thinking: bool = False,
thinking_tokens: int = 1024,
device: Optional[str] = None,
device_map: Optional[str] = None,
torch_dtype: Optional[torch.dtype] = None,
**kwargs,
):
Expand All @@ -53,6 +56,7 @@ def __init__(
self.enable_thinking = enable_thinking
self.thinking_tokens = thinking_tokens if enable_thinking else 0
self.device = device or ("cuda" if torch.cuda.is_available() else "cpu")
self.device_map = device_map
self.torch_dtype = torch_dtype or torch.bfloat16
self.model = None
self.processor = None
Expand All @@ -65,7 +69,7 @@ def load_model(self) -> None:

self.processor = AutoProcessor.from_pretrained(self.model_name)
self.model = AutoModelForImageTextToText.from_pretrained(
self.model_name, dtype=self.torch_dtype, device_map=self.device,
self.model_name, dtype=self.torch_dtype, device_map=self.device_map or self.device,
).eval()
self._raw_logged = False
vocab = getattr(self.processor.tokenizer, "get_vocab", dict)()
Expand All @@ -77,7 +81,8 @@ def load_model(self) -> None:

def unload_model(self) -> None:
if torch.cuda.is_available() and self.model is not None:
logger.info("Peak CUDA memory: %.2f GB", torch.cuda.max_memory_allocated() / 1e9)
for index in range(torch.cuda.device_count()):
logger.info("Peak CUDA memory (cuda:%d): %.2f GB", index, torch.cuda.max_memory_allocated(index) / 1e9)
self.model = None
self.processor = None
if torch.cuda.is_available():
Expand All @@ -95,14 +100,7 @@ def generate(
content = [{"type": "text", "text": prompt}]
if image is not None:
content.insert(0, {"type": "image", "image": self.preprocess_image(image)})
inputs = self.processor.apply_chat_template(
[{"role": "user", "content": content}],
add_generation_prompt=True,
tokenize=True,
return_dict=True,
return_tensors="pt",
enable_thinking=self.enable_thinking,
).to(self.model.device)
inputs = self._build_inputs(content).to(self.model.device)
for key, value in inputs.items():
if torch.is_floating_point(value):
inputs[key] = value.to(self.torch_dtype)
Expand All @@ -125,3 +123,13 @@ def generate(
logger.warning("Thinking block not closed within %d tokens", gen_kwargs["max_new_tokens"])
text = self.processor.decode(new_tokens, skip_special_tokens=True)
return _THINKING_RE.sub("", text).strip()

def _build_inputs(self, content):
return self.processor.apply_chat_template(
[{"role": "user", "content": content}],
add_generation_prompt=True,
tokenize=True,
return_dict=True,
return_tensors="pt",
enable_thinking=self.enable_thinking,
)
55 changes: 55 additions & 0 deletions models/muse_glimmer.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
"""Muse Glimmer (meta-models/Muse-Glimmer-*) through the generic chat wrapper."""
from PIL import Image

from .hf_chat import HFChatVLM


class MuseGlimmer(HFChatVLM):
"""Muse Glimmer: reasoning skipped, weights split across all visible GPUs.

The chat template has no switch to turn reasoning off; after
``<|start|>assistant`` the model chooses the reasoning channel
(``to=self``) or the answer channel (``to=user``). The prompt is prefilled
with the answer channel, so the response is the answer alone. The date the
template writes into the system turn is pinned so prompts do not change
from day to day.

The image processor asks for LANCZOS resizing but falls back to BICUBIC
with torchvision < 0.27. Images are therefore resized here with PIL
LANCZOS to the size the processor would pick, which makes its own resize a
same-size pass (pixel values identical to ``do_resize=False``).
"""

cache_name = "MuseGlimmerVLM"
ANSWER_PREFIX = " to=user<|message|>"
TEMPLATE_KWARGS = {"reasoning_strength": "low", "current_date": "2026-10-15"}

def __init__(self, model_name: str, **kwargs):
kwargs.setdefault("device_map", "auto")
super().__init__(model_name, **kwargs)

def preprocess_image(self, image):
from transformers.models.muse_glimmer.image_processing_muse_glimmer import smart_resize

image = super().preprocess_image(image)
processor = self.processor.image_processor
height, width = smart_resize(image.height, image.width,
processor.patch_size * processor.merge_size, processor.max_image_tokens)
if (width, height) == image.size:
return image
return image.resize((width, height), Image.LANCZOS)

def _build_inputs(self, content):
images = [part.pop("image") for part in content if part["type"] == "image"]
text = self.processor.apply_chat_template(
[{"role": "user", "content": content}],
add_generation_prompt=True,
tokenize=False,
**self.TEMPLATE_KWARGS,
)
return self.processor(
text=[text + self.ANSWER_PREFIX],
images=images or None,
add_special_tokens=False,
return_tensors="pt",
)
2 changes: 2 additions & 0 deletions predict/factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ class Entry(NamedTuple):
"mistralai/Ministral-3-14B-Instruct-2512-BF16", "HFChatVLM", _BF16),
"aya-vision-8b": Entry("models.aya_vision:AyaVision", "MIRAGE_AYA_VISION_8B_MODEL_PATH",
"CohereLabs/aya-vision-8b", "HFChatVLM", _BF16),
"muse-glimmer-30b": Entry("models.muse_glimmer:MuseGlimmer", "MIRAGE_MUSE_GLIMMER_30B_MODEL_PATH",
"meta-models/Muse-Glimmer-30B", "MuseGlimmerVLM", _BF16),
# Thinking-on variants for the ablation (same checkpoints).
"gemma4-e4b-think": Entry("models.gemma4:Gemma4Thinking", "MIRAGE_GEMMA4_E4B_MODEL_PATH",
"google/gemma-4-E4B-it", "HFChatThinkingVLM", _BF16),
Expand Down
Loading