Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ Prepare the dataset files and model weights locally, then run:
python demo.py --dataset coco --metric chair --model paligemma --samples 5 --output results/chair-coco.json
```

Mirage currently supports `paligemma` and `qwen2vl`. See the [documentation](https://mirage-f432d87b.mintlify.app) for dataset setup, metrics, model configuration, and HPC instructions.
Mirage supports `paligemma`, `qwen2vl`, `gemma4-e4b`, `gemma4-12b`, `ministral3-8b`, `ministral3-14b`, and `aya-vision-8b`. See the [documentation](https://mirage-f432d87b.mintlify.app) for dataset setup, metrics, model configuration, and HPC instructions.

## Reference results

Expand Down
2 changes: 1 addition & 1 deletion demo.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ def coco(metric):
def main():
if sys.argv[1:]:
raise SystemExit("demo.py takes no flags; use run.py (python run.py --help)")
model = build_predictor("paligemma") # any key in predict.factory._REGISTRY
model = build_predictor("paligemma") # any key in predict.factory.REGISTRY
model.load_model() # lazy: weights load only on a cache miss

pope = POPEMetric() # 6 yes/no questions per image, seed 42
Expand Down
30 changes: 21 additions & 9 deletions models/__init__.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,25 @@
"""
Models module for the Mirage package.
"""Model wrappers, one file per family; imported on first use so ``import models`` needs no torch.

This module provides a flexible architecture for loading and using various
Vision-Language Models (VLMs) and Large Language Models (LLMs) for
hallucination testing and evaluation.
The registry in ``predict.factory`` names each model as ``module:Class``.
"""
import importlib

_MODULES = {
"BaseModel": "base",
"PaligemmaModel": "paligemma",
"Qwen2VLModel": "qwen2vl",
"HFChatVLM": "hf_chat",
"Gemma4": "gemma4",
"Ministral3": "ministral3",
"AyaVision": "aya_vision",
}

__all__ = list(_MODULES)

from .base import BaseModel
from .paligemma import PaligemmaModel
from .qwen2vl import Qwen2VLModel

__all__ = ['BaseModel', 'PaligemmaModel', 'Qwen2VLModel']
def __getattr__(name):
if name not in _MODULES:
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
value = getattr(importlib.import_module(f".{_MODULES[name]}", __name__), name)
globals()[name] = value
return value
6 changes: 6 additions & 0 deletions models/aya_vision.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
"""Aya Vision (CohereLabs/aya-vision-*) through the generic chat wrapper."""
from .hf_chat import HFChatVLM


class AyaVision(HFChatVLM):
"""Aya Vision; keys its cache as ``HFChatVLM``."""
128 changes: 7 additions & 121 deletions models/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,10 @@
"""

from abc import ABC, abstractmethod
from typing import Any, Dict, List, Optional, Union, Tuple, Literal
from typing import Any, ClassVar, Dict, Optional, Union, Tuple, Literal
from pathlib import Path
from PIL import Image
import logging
import torch


class BaseModel(ABC):
Expand All @@ -38,17 +37,19 @@ class BaseModel(ABC):
supports_text (bool): Whether the model can process text
max_tokens (Optional[int]): Maximum tokens the model can handle
quantization (Optional[str]): Quantization method ('int4', 'int8', 'none')
use_true_batching (bool): Whether to use true batch processing
_logger (logging.Logger): Logger instance for the model
"""

# Model name in every prediction cache key. Pinned per family so moving or
# renaming a class never changes keys; None falls back to the class name.
cache_name: ClassVar[Optional[str]] = None

def __init__(
self,
model_name: str,
model_type: str = "local",
config: Optional[Dict[str, Any]] = None,
quantization: Optional[Literal["int4", "int8", "none"]] = None,
use_true_batching: bool = False,
**kwargs
):
"""
Expand All @@ -59,15 +60,13 @@ def __init__(
model_type: Type of model deployment ('local' or 'api')
config: Dictionary containing model-specific configuration
quantization: Quantization method to use ('int4', 'int8', or 'none' for full precision)
use_true_batching: Whether to use true batch processing (default: False for sequential)
**kwargs: Additional keyword arguments for model initialization
"""
self.model_name = model_name
self.model_type = model_type.lower()
self.config = config or {}
self.is_loaded = False
self.quantization = quantization
self.use_true_batching = use_true_batching

# Model capabilities - to be set by subclasses
self.supports_images = False
Expand Down Expand Up @@ -146,35 +145,6 @@ def generate(
"""
pass

@abstractmethod
def batch_generate(
self,
prompts: List[str],
images: Optional[List[Union[Image.Image, str, Path, None]]] = None,
**kwargs
) -> List[Optional[str]]:
"""
Generate responses for a batch of inputs.

This method should efficiently process multiple inputs, potentially
using batching optimizations for local models or parallel API calls.

Args:
prompts: List of text prompts
images: Optional list of images (can contain None for text-only prompts)
**kwargs: Additional generation parameters

Returns:
List[Optional[str]]: One entry per prompt; None where generation
failed, so callers can exclude it instead of scoring it.

Raises:
NotImplementedError: If not implemented by subclass
ModelNotLoadedError: If model is not loaded
InvalidInputError: If inputs are invalid
"""
pass

def validate_input(
self,
prompt: str,
Expand Down Expand Up @@ -249,45 +219,6 @@ def preprocess_image(self, image: Union[Image.Image, str, Path]) -> Image.Image:

return image

def get_model_info(self) -> Dict[str, Any]:
"""
Get comprehensive information about the model.

Returns:
Dict containing model metadata and current state
"""
return {
'model_name': self.model_name,
'model_type': self.model_type,
'is_loaded': self.is_loaded,
'supports_images': self.supports_images,
'supports_text': self.supports_text,
'max_tokens': self.max_tokens,
'config': self.config.copy(),
'class_name': self.__class__.__name__
}

def update_config(self, **kwargs) -> None:
"""
Update model configuration.

Args:
**kwargs: Configuration parameters to update
"""
self.config.update(kwargs)
self._logger.info(f"Updated config: {kwargs}")

def __enter__(self):
"""Context manager entry - load model if not already loaded."""
if not self.is_loaded:
self.load_model()
return self

def __exit__(self, exc_type, exc_val, exc_tb):
"""Context manager exit - unload model."""
if self.is_loaded:
self.unload_model()

def __repr__(self) -> str:
"""String representation of the model."""
status = "loaded" if self.is_loaded else "not loaded"
Expand Down Expand Up @@ -337,6 +268,8 @@ def get_quantization_config(self, quantization_backend: str = "torchao") -> Opti

elif quantization_backend == "bitsandbytes":
if self.quantization == "int4":
import torch

try:
from transformers import BitsAndBytesConfig
self._logger.info("Using BitsAndBytes 4-bit quantization")
Expand Down Expand Up @@ -369,53 +302,6 @@ def get_quantization_config(self, quantization_backend: str = "torchao") -> Opti
f"BitsAndBytes backend only supports int4/int8, got: {self.quantization}"
)

def batch_inputs(
self,
prompts: List[str],
images: Optional[List[Union[Image.Image, str, Path, None]]] = None,
batch_size: Optional[int] = None
) -> List[Tuple[List[str], Optional[List[Union[Image.Image, str, Path, None]]]]]:
"""
Split inputs into batches for processing.

Utility method to help subclasses implement batch processing. Groups prompts
and images into batches of specified size.

Args:
prompts: List of text prompts
images: Optional list of images (same length as prompts or None)
batch_size: Number of items per batch (None = process all at once)

Returns:
List of tuples, each containing (batch_prompts, batch_images)

Example:
>>> batches = self.batch_inputs(prompts, images, batch_size=4)
>>> for batch_prompts, batch_images in batches:
>>> # Process batch
>>> results = self._process_batch(batch_prompts, batch_images)
"""
if images is None:
images = [None] * len(prompts)

if len(prompts) != len(images):
raise ValueError(
f"Length mismatch: {len(prompts)} prompts but {len(images)} images"
)

# If no batch size specified, return everything as one batch
if batch_size is None:
return [(prompts, images)]

# Split into batches
batches = []
for i in range(0, len(prompts), batch_size):
batch_prompts = prompts[i:i + batch_size]
batch_images = images[i:i + batch_size]
batches.append((batch_prompts, batch_images))

return batches


class ModelError(Exception):
"""Base exception class for model-related errors."""
Expand Down
6 changes: 6 additions & 0 deletions models/gemma4.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
"""Gemma 4 (google/gemma-4-*-it) through the generic chat wrapper."""
from .hf_chat import HFChatVLM


class Gemma4(HFChatVLM):
"""Gemma 4; keys its cache as ``HFChatVLM``."""
120 changes: 120 additions & 0 deletions models/hf_chat.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
"""Generic wrapper for Hugging Face chat vision-language models.

Covers models whose processor ships a chat template that accepts one image
and one text turn (Gemma 4, Ministral 3, Aya Vision). Decoding is greedy by
default and only the newly generated tokens are returned.
"""

import logging
import re
import time
from pathlib import Path
from typing import Optional, Union

import torch
from PIL import Image

from .base import BaseModel, ModelLoadError, ModelNotLoadedError

logger = logging.getLogger(__name__)

# Tokens that close a reasoning block (Gemma 4 uses a "thought" channel). They are
# special tokens, so the answer is cut at token level before decoding.
_THINKING_END_TOKENS = ("<channel|>", "</think>")
# Reasoning blocks written as plain text.
_THINKING_RE = re.compile(r"<think>.*?</think>", re.DOTALL)


class HFChatVLM(BaseModel):
"""Chat-template vision-language model loaded with ``AutoModelForImageTextToText``.

Args:
model_name: HF model id or local snapshot path.
enable_thinking: Forwarded to the chat template; off for evaluation.
device: Device for inference (default: CUDA when available).
torch_dtype: Weight dtype (default: bfloat16).
"""

cache_name = "HFChatVLM"

def __init__(
self,
model_name: str,
enable_thinking: bool = False,
device: Optional[str] = None,
torch_dtype: Optional[torch.dtype] = None,
**kwargs,
):
super().__init__(model_name, model_type="local", **kwargs)
self.supports_images = True
self.enable_thinking = enable_thinking
self.device = device or ("cuda" if torch.cuda.is_available() else "cpu")
self.torch_dtype = torch_dtype or torch.bfloat16
self.model = None
self.processor = None
self.config = {"max_new_tokens": 512, "do_sample": False, **self.config}

def load_model(self) -> None:
start = time.monotonic()
try:
from transformers import AutoModelForImageTextToText, AutoProcessor

self.processor = AutoProcessor.from_pretrained(self.model_name)
self.model = AutoModelForImageTextToText.from_pretrained(
self.model_name, dtype=self.torch_dtype, device_map=self.device,
).eval()
self._raw_logged = False
vocab = getattr(self.processor.tokenizer, "get_vocab", dict)()
self._thinking_end_ids = {vocab[t] for t in _THINKING_END_TOKENS if t in vocab}
except Exception as e:
raise ModelLoadError(f"Failed to load {self.model_name}: {e}") from e
self.is_loaded = True
logger.info("Loaded %s in %.1f s", self.model_name, time.monotonic() - start)

def unload_model(self) -> None:
if torch.cuda.is_available() and self.model is not None:
logger.info("Peak CUDA memory: %.2f GB", torch.cuda.max_memory_allocated() / 1e9)
self.model = None
self.processor = None
if torch.cuda.is_available():
torch.cuda.empty_cache()
self.is_loaded = False

def generate(
self,
prompt: str,
image: Optional[Union[Image.Image, str, Path]] = None,
**kwargs,
) -> str:
if not self.is_loaded:
raise ModelNotLoadedError("Call load_model() before generate().")
content = [{"type": "text", "text": prompt}]
if image is not None:
content.insert(0, {"type": "image", "image": self.preprocess_image(image)})
inputs = self.processor.apply_chat_template(
[{"role": "user", "content": content}],
add_generation_prompt=True,
tokenize=True,
return_dict=True,
return_tensors="pt",
enable_thinking=self.enable_thinking,
).to(self.model.device)
for key, value in inputs.items():
if torch.is_floating_point(value):
inputs[key] = value.to(self.torch_dtype)
gen = {**self.config, **kwargs}
gen_kwargs = {"max_new_tokens": gen["max_new_tokens"], "do_sample": bool(gen.get("do_sample"))}
if gen_kwargs["do_sample"]:
gen_kwargs.update({k: gen[k] for k in ("temperature", "top_p", "top_k") if gen.get(k) is not None})
with torch.inference_mode():
output = self.model.generate(**inputs, **gen_kwargs)
new_tokens = output[0, inputs["input_ids"].shape[1]:].tolist()
if not self._raw_logged:
# One raw decode per load, so a run's log shows any thinking markers.
logger.info("Raw decode: %r", self.processor.decode(new_tokens, skip_special_tokens=False))
self._raw_logged = True
ends = [i for i, token in enumerate(new_tokens) if token in self._thinking_end_ids]
if ends:
new_tokens = new_tokens[ends[-1] + 1:]
text = self.processor.decode(new_tokens, skip_special_tokens=True)
return _THINKING_RE.sub("", text).strip()
6 changes: 6 additions & 0 deletions models/ministral3.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
"""Ministral 3 (mistralai/Ministral-3-*-Instruct-2512-BF16) through the generic chat wrapper."""
from .hf_chat import HFChatVLM


class Ministral3(HFChatVLM):
"""Ministral 3; keys its cache as ``HFChatVLM``."""
Loading
Loading