diff --git a/README.md b/README.md index fa702e2..2db9500 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Prepare the dataset files and model weights locally, then run: python demo.py --dataset coco --metric chair --model paligemma --samples 5 --output results/chair-coco.json ``` -Mirage currently supports `paligemma` and `qwen2vl`. See the [documentation](https://mirage-f432d87b.mintlify.app) for dataset setup, metrics, model configuration, and HPC instructions. +Mirage supports `paligemma`, `qwen2vl`, `gemma4-e4b`, `gemma4-12b`, `ministral3-8b`, `ministral3-14b`, and `aya-vision-8b`. See the [documentation](https://mirage-f432d87b.mintlify.app) for dataset setup, metrics, model configuration, and HPC instructions. ## Reference results diff --git a/demo.py b/demo.py index f6673ff..91400b0 100644 --- a/demo.py +++ b/demo.py @@ -24,7 +24,7 @@ def coco(metric): def main(): if sys.argv[1:]: raise SystemExit("demo.py takes no flags; use run.py (python run.py --help)") - model = build_predictor("paligemma") # any key in predict.factory._REGISTRY + model = build_predictor("paligemma") # any key in predict.factory.REGISTRY model.load_model() # lazy: weights load only on a cache miss pope = POPEMetric() # 6 yes/no questions per image, seed 42 diff --git a/models/__init__.py b/models/__init__.py index 7fa6553..da339c3 100644 --- a/models/__init__.py +++ b/models/__init__.py @@ -1,13 +1,25 @@ -""" -Models module for the Mirage package. +"""Model wrappers, one file per family; imported on first use so ``import models`` needs no torch. -This module provides a flexible architecture for loading and using various -Vision-Language Models (VLMs) and Large Language Models (LLMs) for -hallucination testing and evaluation. +The registry in ``predict.factory`` names each model as ``module:Class``. """ +import importlib + +_MODULES = { + "BaseModel": "base", + "PaligemmaModel": "paligemma", + "Qwen2VLModel": "qwen2vl", + "HFChatVLM": "hf_chat", + "Gemma4": "gemma4", + "Ministral3": "ministral3", + "AyaVision": "aya_vision", +} + +__all__ = list(_MODULES) -from .base import BaseModel -from .paligemma import PaligemmaModel -from .qwen2vl import Qwen2VLModel -__all__ = ['BaseModel', 'PaligemmaModel', 'Qwen2VLModel'] +def __getattr__(name): + if name not in _MODULES: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + value = getattr(importlib.import_module(f".{_MODULES[name]}", __name__), name) + globals()[name] = value + return value diff --git a/models/aya_vision.py b/models/aya_vision.py new file mode 100644 index 0000000..8407385 --- /dev/null +++ b/models/aya_vision.py @@ -0,0 +1,6 @@ +"""Aya Vision (CohereLabs/aya-vision-*) through the generic chat wrapper.""" +from .hf_chat import HFChatVLM + + +class AyaVision(HFChatVLM): + """Aya Vision; keys its cache as ``HFChatVLM``.""" diff --git a/models/base.py b/models/base.py index 4bb73af..2a851dc 100644 --- a/models/base.py +++ b/models/base.py @@ -7,11 +7,10 @@ """ from abc import ABC, abstractmethod -from typing import Any, Dict, List, Optional, Union, Tuple, Literal +from typing import Any, ClassVar, Dict, Optional, Union, Tuple, Literal from pathlib import Path from PIL import Image import logging -import torch class BaseModel(ABC): @@ -38,17 +37,19 @@ class BaseModel(ABC): supports_text (bool): Whether the model can process text max_tokens (Optional[int]): Maximum tokens the model can handle quantization (Optional[str]): Quantization method ('int4', 'int8', 'none') - use_true_batching (bool): Whether to use true batch processing _logger (logging.Logger): Logger instance for the model """ + # Model name in every prediction cache key. Pinned per family so moving or + # renaming a class never changes keys; None falls back to the class name. + cache_name: ClassVar[Optional[str]] = None + def __init__( self, model_name: str, model_type: str = "local", config: Optional[Dict[str, Any]] = None, quantization: Optional[Literal["int4", "int8", "none"]] = None, - use_true_batching: bool = False, **kwargs ): """ @@ -59,7 +60,6 @@ def __init__( model_type: Type of model deployment ('local' or 'api') config: Dictionary containing model-specific configuration quantization: Quantization method to use ('int4', 'int8', or 'none' for full precision) - use_true_batching: Whether to use true batch processing (default: False for sequential) **kwargs: Additional keyword arguments for model initialization """ self.model_name = model_name @@ -67,7 +67,6 @@ def __init__( self.config = config or {} self.is_loaded = False self.quantization = quantization - self.use_true_batching = use_true_batching # Model capabilities - to be set by subclasses self.supports_images = False @@ -146,35 +145,6 @@ def generate( """ pass - @abstractmethod - def batch_generate( - self, - prompts: List[str], - images: Optional[List[Union[Image.Image, str, Path, None]]] = None, - **kwargs - ) -> List[Optional[str]]: - """ - Generate responses for a batch of inputs. - - This method should efficiently process multiple inputs, potentially - using batching optimizations for local models or parallel API calls. - - Args: - prompts: List of text prompts - images: Optional list of images (can contain None for text-only prompts) - **kwargs: Additional generation parameters - - Returns: - List[Optional[str]]: One entry per prompt; None where generation - failed, so callers can exclude it instead of scoring it. - - Raises: - NotImplementedError: If not implemented by subclass - ModelNotLoadedError: If model is not loaded - InvalidInputError: If inputs are invalid - """ - pass - def validate_input( self, prompt: str, @@ -249,45 +219,6 @@ def preprocess_image(self, image: Union[Image.Image, str, Path]) -> Image.Image: return image - def get_model_info(self) -> Dict[str, Any]: - """ - Get comprehensive information about the model. - - Returns: - Dict containing model metadata and current state - """ - return { - 'model_name': self.model_name, - 'model_type': self.model_type, - 'is_loaded': self.is_loaded, - 'supports_images': self.supports_images, - 'supports_text': self.supports_text, - 'max_tokens': self.max_tokens, - 'config': self.config.copy(), - 'class_name': self.__class__.__name__ - } - - def update_config(self, **kwargs) -> None: - """ - Update model configuration. - - Args: - **kwargs: Configuration parameters to update - """ - self.config.update(kwargs) - self._logger.info(f"Updated config: {kwargs}") - - def __enter__(self): - """Context manager entry - load model if not already loaded.""" - if not self.is_loaded: - self.load_model() - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - """Context manager exit - unload model.""" - if self.is_loaded: - self.unload_model() - def __repr__(self) -> str: """String representation of the model.""" status = "loaded" if self.is_loaded else "not loaded" @@ -337,6 +268,8 @@ def get_quantization_config(self, quantization_backend: str = "torchao") -> Opti elif quantization_backend == "bitsandbytes": if self.quantization == "int4": + import torch + try: from transformers import BitsAndBytesConfig self._logger.info("Using BitsAndBytes 4-bit quantization") @@ -369,53 +302,6 @@ def get_quantization_config(self, quantization_backend: str = "torchao") -> Opti f"BitsAndBytes backend only supports int4/int8, got: {self.quantization}" ) - def batch_inputs( - self, - prompts: List[str], - images: Optional[List[Union[Image.Image, str, Path, None]]] = None, - batch_size: Optional[int] = None - ) -> List[Tuple[List[str], Optional[List[Union[Image.Image, str, Path, None]]]]]: - """ - Split inputs into batches for processing. - - Utility method to help subclasses implement batch processing. Groups prompts - and images into batches of specified size. - - Args: - prompts: List of text prompts - images: Optional list of images (same length as prompts or None) - batch_size: Number of items per batch (None = process all at once) - - Returns: - List of tuples, each containing (batch_prompts, batch_images) - - Example: - >>> batches = self.batch_inputs(prompts, images, batch_size=4) - >>> for batch_prompts, batch_images in batches: - >>> # Process batch - >>> results = self._process_batch(batch_prompts, batch_images) - """ - if images is None: - images = [None] * len(prompts) - - if len(prompts) != len(images): - raise ValueError( - f"Length mismatch: {len(prompts)} prompts but {len(images)} images" - ) - - # If no batch size specified, return everything as one batch - if batch_size is None: - return [(prompts, images)] - - # Split into batches - batches = [] - for i in range(0, len(prompts), batch_size): - batch_prompts = prompts[i:i + batch_size] - batch_images = images[i:i + batch_size] - batches.append((batch_prompts, batch_images)) - - return batches - class ModelError(Exception): """Base exception class for model-related errors.""" diff --git a/models/gemma4.py b/models/gemma4.py new file mode 100644 index 0000000..0951f25 --- /dev/null +++ b/models/gemma4.py @@ -0,0 +1,6 @@ +"""Gemma 4 (google/gemma-4-*-it) through the generic chat wrapper.""" +from .hf_chat import HFChatVLM + + +class Gemma4(HFChatVLM): + """Gemma 4; keys its cache as ``HFChatVLM``.""" diff --git a/models/hf_chat.py b/models/hf_chat.py new file mode 100644 index 0000000..0adf446 --- /dev/null +++ b/models/hf_chat.py @@ -0,0 +1,120 @@ +"""Generic wrapper for Hugging Face chat vision-language models. + +Covers models whose processor ships a chat template that accepts one image +and one text turn (Gemma 4, Ministral 3, Aya Vision). Decoding is greedy by +default and only the newly generated tokens are returned. +""" + +import logging +import re +import time +from pathlib import Path +from typing import Optional, Union + +import torch +from PIL import Image + +from .base import BaseModel, ModelLoadError, ModelNotLoadedError + +logger = logging.getLogger(__name__) + +# Tokens that close a reasoning block (Gemma 4 uses a "thought" channel). They are +# special tokens, so the answer is cut at token level before decoding. +_THINKING_END_TOKENS = ("", "") +# Reasoning blocks written as plain text. +_THINKING_RE = re.compile(r".*?", re.DOTALL) + + +class HFChatVLM(BaseModel): + """Chat-template vision-language model loaded with ``AutoModelForImageTextToText``. + + Args: + model_name: HF model id or local snapshot path. + enable_thinking: Forwarded to the chat template; off for evaluation. + device: Device for inference (default: CUDA when available). + torch_dtype: Weight dtype (default: bfloat16). + """ + + cache_name = "HFChatVLM" + + def __init__( + self, + model_name: str, + enable_thinking: bool = False, + device: Optional[str] = None, + torch_dtype: Optional[torch.dtype] = None, + **kwargs, + ): + super().__init__(model_name, model_type="local", **kwargs) + self.supports_images = True + self.enable_thinking = enable_thinking + self.device = device or ("cuda" if torch.cuda.is_available() else "cpu") + self.torch_dtype = torch_dtype or torch.bfloat16 + self.model = None + self.processor = None + self.config = {"max_new_tokens": 512, "do_sample": False, **self.config} + + def load_model(self) -> None: + start = time.monotonic() + try: + from transformers import AutoModelForImageTextToText, AutoProcessor + + self.processor = AutoProcessor.from_pretrained(self.model_name) + self.model = AutoModelForImageTextToText.from_pretrained( + self.model_name, dtype=self.torch_dtype, device_map=self.device, + ).eval() + self._raw_logged = False + vocab = getattr(self.processor.tokenizer, "get_vocab", dict)() + self._thinking_end_ids = {vocab[t] for t in _THINKING_END_TOKENS if t in vocab} + except Exception as e: + raise ModelLoadError(f"Failed to load {self.model_name}: {e}") from e + self.is_loaded = True + logger.info("Loaded %s in %.1f s", self.model_name, time.monotonic() - start) + + def unload_model(self) -> None: + if torch.cuda.is_available() and self.model is not None: + logger.info("Peak CUDA memory: %.2f GB", torch.cuda.max_memory_allocated() / 1e9) + self.model = None + self.processor = None + if torch.cuda.is_available(): + torch.cuda.empty_cache() + self.is_loaded = False + + def generate( + self, + prompt: str, + image: Optional[Union[Image.Image, str, Path]] = None, + **kwargs, + ) -> str: + if not self.is_loaded: + raise ModelNotLoadedError("Call load_model() before generate().") + content = [{"type": "text", "text": prompt}] + if image is not None: + content.insert(0, {"type": "image", "image": self.preprocess_image(image)}) + inputs = self.processor.apply_chat_template( + [{"role": "user", "content": content}], + add_generation_prompt=True, + tokenize=True, + return_dict=True, + return_tensors="pt", + enable_thinking=self.enable_thinking, + ).to(self.model.device) + for key, value in inputs.items(): + if torch.is_floating_point(value): + inputs[key] = value.to(self.torch_dtype) + gen = {**self.config, **kwargs} + gen_kwargs = {"max_new_tokens": gen["max_new_tokens"], "do_sample": bool(gen.get("do_sample"))} + if gen_kwargs["do_sample"]: + gen_kwargs.update({k: gen[k] for k in ("temperature", "top_p", "top_k") if gen.get(k) is not None}) + with torch.inference_mode(): + output = self.model.generate(**inputs, **gen_kwargs) + new_tokens = output[0, inputs["input_ids"].shape[1]:].tolist() + if not self._raw_logged: + # One raw decode per load, so a run's log shows any thinking markers. + logger.info("Raw decode: %r", self.processor.decode(new_tokens, skip_special_tokens=False)) + self._raw_logged = True + ends = [i for i, token in enumerate(new_tokens) if token in self._thinking_end_ids] + if ends: + new_tokens = new_tokens[ends[-1] + 1:] + text = self.processor.decode(new_tokens, skip_special_tokens=True) + return _THINKING_RE.sub("", text).strip() diff --git a/models/ministral3.py b/models/ministral3.py new file mode 100644 index 0000000..39033d4 --- /dev/null +++ b/models/ministral3.py @@ -0,0 +1,6 @@ +"""Ministral 3 (mistralai/Ministral-3-*-Instruct-2512-BF16) through the generic chat wrapper.""" +from .hf_chat import HFChatVLM + + +class Ministral3(HFChatVLM): + """Ministral 3; keys its cache as ``HFChatVLM``.""" diff --git a/models/paligemma.py b/models/paligemma.py index 28e8d13..2da3441 100644 --- a/models/paligemma.py +++ b/models/paligemma.py @@ -7,11 +7,11 @@ """ import torch -from typing import Any, Dict, List, Optional, Union, Literal +from typing import Any, Dict, Optional, Union, Literal from pathlib import Path from PIL import Image -from .base import BaseModel, ModelLoadError, ModelNotLoadedError, InvalidInputError +from .base import BaseModel, ModelLoadError, InvalidInputError class PaligemmaModel(BaseModel): @@ -34,6 +34,8 @@ class PaligemmaModel(BaseModel): device: PyTorch device for model execution """ + cache_name = "PaligemmaModel" + def __init__( self, model_name: str = "google/paligemma-3b-mix-224", @@ -41,7 +43,6 @@ def __init__( torch_dtype: Optional[torch.dtype] = None, quantization: Optional[Literal["int4", "int8", "none"]] = None, quantization_backend: str = "torchao", - use_true_batching: bool = False, **kwargs ): """ @@ -53,14 +54,12 @@ def __init__( torch_dtype: PyTorch data type for model weights (default: bfloat16) quantization: Quantization method ('int4', 'int8', 'none', or None) quantization_backend: Backend for quantization ('torchao' or 'bitsandbytes') - use_true_batching: Whether to use true batch processing (default: False) **kwargs: Additional arguments passed to base class """ super().__init__( model_name, model_type="local", quantization=quantization, - use_true_batching=use_true_batching, **kwargs ) @@ -226,57 +225,6 @@ def generate( self._logger.error(f"Generation failed: {e}") raise InvalidInputError(f"Failed to generate response: {e}") - def batch_generate( - self, - prompts: List[str], - images: Optional[List[Union[Image.Image, str, Path, None]]] = None, - batch_size: Optional[int] = None, - **kwargs - ) -> List[Optional[str]]: - """ - Generate responses for a batch of inputs. - - Supports both sequential processing (default) and true batch processing - based on the `use_true_batching` setting. When true batching is enabled, - processes multiple samples in parallel for improved efficiency. - - Args: - prompts: List of text prompts - images: Optional list of images (can contain None values) - batch_size: Size of batches for true batching (only used if use_true_batching=True) - **kwargs: Generation parameters - - Returns: - List[Optional[str]]: One entry per prompt; None where generation failed - - Raises: - ModelNotLoadedError: If model is not loaded - InvalidInputError: If inputs are invalid - """ - if not self.is_loaded: - raise ModelNotLoadedError("Model is not loaded") - - if images is None: - images = [None] * len(prompts) - - if len(prompts) != len(images): - raise InvalidInputError("Number of prompts and images must match") - - # Sequential processing (original behavior) - if not self.use_true_batching: - responses = [] - for prompt, image in zip(prompts, images): - try: - response = self.generate(prompt, image, **kwargs) - responses.append(response) - except Exception as e: - self._logger.warning(f"Failed to generate for prompt '{prompt[:50]}...': {e}") - responses.append(None) - return responses - - # True batch processing - return self._batch_generate_parallel(prompts, images, batch_size, **kwargs) - def _generate_with_config( self, inputs: Dict[str, torch.Tensor], @@ -285,9 +233,6 @@ def _generate_with_config( """ Generate outputs using model.generate with the given config. - This helper method centralizes the generation logic to avoid duplication - between single and batch generation methods. - Args: inputs: Prepared model inputs generation_config: Generation configuration parameters @@ -307,104 +252,6 @@ def _generate_with_config( ) return outputs - def _batch_generate_parallel( - self, - prompts: List[str], - images: Optional[List[Union[Image.Image, str, Path, None]]], - batch_size: Optional[int] = None, - **kwargs - ) -> List[Optional[str]]: - """ - Generate responses using true batch processing. - - This method processes multiple inputs in parallel batches for improved - efficiency compared to sequential processing. - - Args: - prompts: List of text prompts - images: List of images (or None values) - batch_size: Size of batches (None = all at once) - **kwargs: Generation parameters - - Returns: - List[Optional[str]]: One entry per prompt; None where generation failed - """ - all_responses = [] - - # Use base class helper to split into batches - batches = self.batch_inputs(prompts, images, batch_size) - - for batch_prompts, batch_images in batches: - try: - # Prepare batch inputs - batch_inputs = self._prepare_batch_inputs(batch_prompts, batch_images) - - # Merge generation config - generation_config = self.config.copy() - generation_config.update(kwargs) - - # Generate for batch using helper - outputs = self._generate_with_config(batch_inputs, generation_config) - - # Decode all outputs in batch - input_len = batch_inputs["input_ids"].shape[1] - for output in outputs: - response = self.processor.decode( - output[input_len:], - skip_special_tokens=True - ) - all_responses.append(response.strip()) - - except Exception as e: - self._logger.error(f"Batch generation failed: {e}") - all_responses.extend([None] * len(batch_prompts)) - - return all_responses - - def _prepare_batch_inputs( - self, - prompts: List[str], - images: Optional[List[Union[Image.Image, str, Path, None]]] - ) -> Dict[str, torch.Tensor]: - """ - Prepare inputs for batch processing. - - Args: - prompts: List of text prompts - images: List of images (or None values) - - Returns: - Dict containing processed batch inputs for the model - """ - # Preprocess all images and add tokens to prompts - processed_images = [] - processed_prompts = [] - - for prompt, image in zip(prompts, images): - if image is not None: - processed_images.append(self.preprocess_image(image)) - # Add token prefix if not already present - if not prompt.startswith(""): - processed_prompts.append(f"{prompt}") - else: - processed_prompts.append(prompt) - else: - processed_images.append(None) - processed_prompts.append(prompt) - - # Use processor to prepare batch inputs - inputs = self.processor( - text=processed_prompts, - images=processed_images if any(img is not None for img in processed_images) else None, - return_tensors="pt", - padding=True - ) - - # Move inputs to device - inputs = {k: v.to(self.device) for k, v in inputs.items()} - - return inputs - def _prepare_inputs( self, prompt: str, @@ -439,20 +286,3 @@ def _prepare_inputs( inputs = {k: v.to(self.device) for k, v in inputs.items()} return inputs - - def get_model_info(self) -> Dict[str, Any]: - """ - Get Paligemma-specific model information. - - Returns: - Dict containing model metadata and current state - """ - info = super().get_model_info() - info.update({ - 'device': self.device, - 'torch_dtype': str(self.torch_dtype), - 'model_architecture': 'PaliGemma', - 'input_resolution': '224x224', - 'parameter_count': '3B' - }) - return info diff --git a/models/qwen2vl.py b/models/qwen2vl.py index 346c744..8f24adb 100644 --- a/models/qwen2vl.py +++ b/models/qwen2vl.py @@ -7,11 +7,11 @@ """ import torch -from typing import Any, Dict, List, Optional, Union, Literal +from typing import Any, Dict, Optional, Union, Literal from pathlib import Path from PIL import Image -from .base import BaseModel, ModelLoadError, ModelNotLoadedError, InvalidInputError +from .base import BaseModel, ModelLoadError, InvalidInputError class Qwen2VLModel(BaseModel): @@ -35,6 +35,8 @@ class Qwen2VLModel(BaseModel): device: PyTorch device for model execution """ + cache_name = "Qwen2VLModel" + def __init__( self, model_name: str = "Qwen/Qwen2-VL-2B-Instruct", @@ -42,7 +44,6 @@ def __init__( torch_dtype: Optional[torch.dtype] = None, quantization: Optional[Literal["int4", "int8", "none"]] = None, quantization_backend: str = "torchao", - use_true_batching: bool = False, **kwargs ): """ @@ -54,14 +55,12 @@ def __init__( torch_dtype: PyTorch data type for model weights (default: bfloat16) quantization: Quantization method ('int4', 'int8', 'none', or None) quantization_backend: Backend for quantization ('torchao' or 'bitsandbytes') - use_true_batching: Whether to use true batch processing (default: False) **kwargs: Additional arguments passed to base class """ super().__init__( model_name, model_type="local", quantization=quantization, - use_true_batching=use_true_batching, **kwargs ) @@ -233,57 +232,6 @@ def generate( self._logger.error(f"Generation failed: {e}") raise InvalidInputError(f"Failed to generate response: {e}") - def batch_generate( - self, - prompts: List[str], - images: Optional[List[Union[Image.Image, str, Path, None]]] = None, - batch_size: Optional[int] = None, - **kwargs - ) -> List[Optional[str]]: - """ - Generate responses for a batch of inputs. - - Supports both sequential processing (default) and true batch processing - based on the `use_true_batching` setting. When true batching is enabled, - processes multiple samples in parallel for improved efficiency. - - Args: - prompts: List of text prompts - images: Optional list of images (can contain None values) - batch_size: Size of batches for true batching (only used if use_true_batching=True) - **kwargs: Generation parameters - - Returns: - List[Optional[str]]: One entry per prompt; None where generation failed - - Raises: - ModelNotLoadedError: If model is not loaded - InvalidInputError: If inputs are invalid - """ - if not self.is_loaded: - raise ModelNotLoadedError("Model is not loaded") - - if images is None: - images = [None] * len(prompts) - - if len(prompts) != len(images): - raise InvalidInputError("Number of prompts and images must match") - - # Sequential processing (original behavior) - if not self.use_true_batching: - responses = [] - for prompt, image in zip(prompts, images): - try: - response = self.generate(prompt, image, **kwargs) - responses.append(response) - except Exception as e: - self._logger.warning(f"Failed to generate for prompt '{prompt[:50]}...': {e}") - responses.append(None) - return responses - - # True batch processing - return self._batch_generate_parallel(prompts, images, batch_size, **kwargs) - def _generate_with_config( self, inputs: Dict[str, torch.Tensor], @@ -292,9 +240,6 @@ def _generate_with_config( """ Generate outputs using model.generate with the given config. - This helper method centralizes the generation logic to avoid duplication - between single and batch generation methods. - Args: inputs: Prepared model inputs generation_config: Generation configuration parameters @@ -314,60 +259,6 @@ def _generate_with_config( ) return outputs - def _batch_generate_parallel( - self, - prompts: List[str], - images: Optional[List[Union[Image.Image, str, Path, None]]], - batch_size: Optional[int] = None, - **kwargs - ) -> List[Optional[str]]: - """ - Generate responses using true batch processing. - - This method processes multiple inputs in parallel batches for improved - efficiency compared to sequential processing. - - Args: - prompts: List of text prompts - images: List of images (or None values) - batch_size: Size of batches (None = all at once) - **kwargs: Generation parameters - - Returns: - List[Optional[str]]: One entry per prompt; None where generation failed - """ - all_responses = [] - - # Use base class helper to split into batches - batches = self.batch_inputs(prompts, images, batch_size) - - for batch_prompts, batch_images in batches: - try: - # Prepare batch inputs - batch_inputs = self._prepare_batch_inputs(batch_prompts, batch_images) - - # Merge generation config - generation_config = self.config.copy() - generation_config.update(kwargs) - - # Generate for batch using helper - outputs = self._generate_with_config(batch_inputs, generation_config) - - # Decode all outputs in batch - input_len = batch_inputs["input_ids"].shape[1] - for output in outputs: - response = self.processor.decode( - output[input_len:], - skip_special_tokens=True - ) - all_responses.append(response.strip()) - - except Exception as e: - self._logger.error(f"Batch generation failed: {e}") - all_responses.extend([None] * len(batch_prompts)) - - return all_responses - def _prepare_inputs( self, prompt: str, @@ -429,90 +320,3 @@ def _prepare_inputs( inputs = {k: v.to(self.device) for k, v in inputs.items()} return inputs - - def _prepare_batch_inputs( - self, - prompts: List[str], - images: Optional[List[Union[Image.Image, str, Path, None]]] - ) -> Dict[str, torch.Tensor]: - """ - Prepare inputs for batch processing with Qwen2-VL's conversation format. - - Args: - prompts: List of text prompts - images: List of images (or None values) - - Returns: - Dict containing processed batch inputs for the model - """ - # Preprocess all images - processed_images = [] - for image in images: - if image is not None: - processed_images.append(self.preprocess_image(image)) - else: - processed_images.append(None) - - # Prepare messages for each sample - all_texts = [] - batch_images = [] - - for prompt, image in zip(prompts, processed_images): - messages = [ - { - "role": "user", - "content": [] - } - ] - - # Add image to content if provided - if image is not None: - messages[0]["content"].append({ - "type": "image", - "image": image - }) - batch_images.append(image) - - # Add text to content - messages[0]["content"].append({ - "type": "text", - "text": prompt - }) - - # Apply chat template - text = self.processor.apply_chat_template( - messages, - tokenize=False, - add_generation_prompt=True - ) - all_texts.append(text) - - # Process batch with processor - inputs = self.processor( - text=all_texts, - images=batch_images if batch_images else None, - return_tensors="pt", - padding=True - ) - - # Move inputs to device - inputs = {k: v.to(self.device) for k, v in inputs.items()} - - return inputs - - def get_model_info(self) -> Dict[str, Any]: - """ - Get Qwen2-VL-specific model information. - - Returns: - Dict containing model metadata and current state - """ - info = super().get_model_info() - info.update({ - 'device': self.device, - 'torch_dtype': str(self.torch_dtype), - 'model_architecture': 'Qwen2-VL', - 'context_length': self.max_tokens, - 'supports_multilingual': True - }) - return info diff --git a/predict/backends.py b/predict/backends.py index aa47c05..928066f 100644 --- a/predict/backends.py +++ b/predict/backends.py @@ -2,7 +2,7 @@ Routing is decided by which backend the Predictor is constructed with — never by branching inside the Predictor and never inside ``BaseModel``. -A backend provides: ``generate``, ``batch_generate``, ``identity()``, +A backend provides: ``generate``, ``identity()``, ``merged_generation_config(kwargs)``, ``load``/``unload``, ``is_loaded``, and ``model_name``. A future API backend implements the same surface with an identity like ``{"provider": "gemini", "model_id": "models/gemini-2.5-flash"}``; @@ -18,9 +18,9 @@ class LocalModelBackend: """Adapts a ``BaseModel``-shaped object to the backend surface. Args: - model: Any object exposing ``generate``, ``batch_generate``, - ``load_model``, ``unload_model``, ``model_name``, ``config``, - and ``is_loaded`` (duck-typed; no torch import here). + model: Any object exposing ``generate``, ``load_model``, + ``unload_model``, ``model_name``, ``config``, and ``is_loaded`` + (duck-typed; no torch import here). """ def __init__(self, model: Any): @@ -30,15 +30,14 @@ def __init__(self, model: Any): def generate(self, prompt, image=None, **kwargs): return self.model.generate(prompt=prompt, image=image, **kwargs) - def batch_generate(self, prompts, images=None, **kwargs): - return self.model.batch_generate(prompts, images=images, **kwargs) - def identity(self) -> Dict[str, Any]: if self._identity is None: model_id, revision = parse_model_name(self.model.model_name) self._identity = { "provider": "local", - "class_name": type(self.model).__name__, + # Pinned by the model class (BaseModel.cache_name) so the key + # survives moves and renames; plain objects use their class name. + "class_name": getattr(self.model, "cache_name", None) or type(self.model).__name__, "model_id": model_id, "revision": revision, "quantization": getattr(self.model, "quantization", None) or "none", @@ -105,12 +104,6 @@ def generate(self, prompt, image=None, **kwargs): self._reject_conflicting_kwargs(kwargs) return self._fn([prompt])[0] - def batch_generate(self, prompts, images=None, **kwargs): - if images is not None and any(im is not None for im in images): - raise ValueError("CallableBackend is text-only") - self._reject_conflicting_kwargs(kwargs) - return self._fn(list(prompts)) - def _reject_conflicting_kwargs(self, kwargs): # fn owns its generation config; kwargs it cannot honor must not be # silently recorded in cache keys/metadata as if they were applied. diff --git a/predict/factory.py b/predict/factory.py index dee3e1c..bdcec49 100644 --- a/predict/factory.py +++ b/predict/factory.py @@ -1,4 +1,4 @@ -"""Factory helpers replacing the per-driver model-selection blocks. +"""Model registry and the factory helpers drivers use to build models. Model classes are imported lazily inside :func:`build_model` so this module (and the whole ``predict`` package) stays importable without torch. @@ -6,32 +6,90 @@ import importlib import os -from typing import Any, Optional +from types import MappingProxyType +from typing import Any, Mapping, NamedTuple, Optional, Tuple, Union from predict.backends import LocalModelBackend from predict.cache import SQLiteResponseCache +from predict.identity import parse_model_name from predict.predictor import Predictor -_REGISTRY = { - "paligemma": ("models.paligemma", "PaligemmaModel", - "MIRAGE_PALIGEMMA_MODEL_PATH", "google/paligemma-3b-mix-224"), - "qwen2vl": ("models.qwen2vl", "Qwen2VLModel", - "MIRAGE_QWEN_MODEL_PATH", "Qwen/Qwen2-VL-2B-Instruct"), -} + +class Entry(NamedTuple): + target: str # "module:Class" + env: Union[str, Tuple[str, ...]] # model path variable(s), first set one wins + default: str # HF id used when no variable is set + cache_name: str # mirror of the class's cache_name, so identity needs no torch + dtype: str # mirror of str(model.torch_dtype) + + +_BF16 = "torch.bfloat16" +_QWEN = "MIRAGE_QWEN_MODEL_PATH" + +# A new model needs a row here (and, for a new class, a models/__init__.py _MODULES line); +# build_model checks each row against its class. +REGISTRY: Mapping[str, Entry] = MappingProxyType({ + "paligemma": Entry("models.paligemma:PaligemmaModel", "MIRAGE_PALIGEMMA_MODEL_PATH", + "google/paligemma-3b-mix-224", "PaligemmaModel", _BF16), + "qwen2vl": Entry("models.qwen2vl:Qwen2VLModel", _QWEN, "Qwen/Qwen2-VL-2B-Instruct", "Qwen2VLModel", _BF16), + "qwen2vl-2b": Entry("models.qwen2vl:Qwen2VLModel", ("MIRAGE_QWEN2VL_2B_MODEL_PATH", _QWEN), + "Qwen/Qwen2-VL-2B-Instruct", "Qwen2VLModel", _BF16), + "qwen2vl-7b": Entry("models.qwen2vl:Qwen2VLModel", ("MIRAGE_QWEN2VL_7B_MODEL_PATH", _QWEN), + "Qwen/Qwen2-VL-7B-Instruct", "Qwen2VLModel", _BF16), + "gemma4-e4b": Entry("models.gemma4:Gemma4", "MIRAGE_GEMMA4_E4B_MODEL_PATH", + "google/gemma-4-E4B-it", "HFChatVLM", _BF16), + "gemma4-12b": Entry("models.gemma4:Gemma4", "MIRAGE_GEMMA4_12B_MODEL_PATH", + "google/gemma-4-12B-it", "HFChatVLM", _BF16), + "ministral3-8b": Entry("models.ministral3:Ministral3", "MIRAGE_MINISTRAL3_8B_MODEL_PATH", + "mistralai/Ministral-3-8B-Instruct-2512-BF16", "HFChatVLM", _BF16), + "ministral3-14b": Entry("models.ministral3:Ministral3", "MIRAGE_MINISTRAL3_14B_MODEL_PATH", + "mistralai/Ministral-3-14B-Instruct-2512-BF16", "HFChatVLM", _BF16), + "aya-vision-8b": Entry("models.aya_vision:AyaVision", "MIRAGE_AYA_VISION_8B_MODEL_PATH", + "CohereLabs/aya-vision-8b", "HFChatVLM", _BF16), +}) def _resolve_key(model_key: Optional[str]) -> str: key = (model_key or os.getenv("MIRAGE_VL_MODEL", "paligemma")).lower() - if key not in _REGISTRY: - raise ValueError(f"unknown model key {key!r}; expected one of {sorted(_REGISTRY)}") + if key not in REGISTRY: + raise ValueError(f"unknown model key {key!r}; add a row to predict.factory.REGISTRY " + f"or use one of {sorted(REGISTRY)}") return key +def _model_name(entry: Entry, env: Mapping[str, str]) -> str: + names = (entry.env,) if isinstance(entry.env, str) else entry.env + for i, var in enumerate(names): + if var in env: + # A shared fallback variable (e.g. MIRAGE_QWEN_MODEL_PATH for both Qwen sizes) must + # point at this row's checkpoint; a dedicated variable may name any local dir. + if i and parse_model_name(env[var])[0].rsplit("/", 1)[-1] != entry.default.rsplit("/", 1)[-1]: + raise ValueError(f"${var}={env[var]!r} is not {entry.default}; set ${names[0]} instead") + return env[var] + return entry.default + + +def identity_for(key: str, env: Mapping[str, str] = os.environ) -> dict: + """Cache identity of a registry model as :func:`build_model` would build it, without torch. + + Args: + key: Registry key. + env: Environment holding the ``MIRAGE_*_MODEL_PATH`` variables. + + Returns: + The record ``LocalModelBackend.identity()`` returns for that model. + """ + entry = REGISTRY[key] + model_id, revision = parse_model_name(_model_name(entry, env)) + return {"provider": "local", "class_name": entry.cache_name, "model_id": model_id, + "revision": revision, "quantization": "none", "dtype": entry.dtype} + + def build_model(model_key: Optional[str] = None, **overrides) -> Any: """Construct the model selected by ``model_key`` or ``$MIRAGE_VL_MODEL``. Args: - model_key: Registry key (``"paligemma"`` or ``"qwen2vl"``); None + model_key: Registry key (see ``REGISTRY``); None reads ``MIRAGE_VL_MODEL`` (default ``"paligemma"``). **overrides: Constructor kwargs; ``model_name`` defaults to the model's ``MIRAGE_*_MODEL_PATH`` environment variable. @@ -39,28 +97,49 @@ def build_model(model_key: Optional[str] = None, **overrides) -> Any: Returns: A constructed (not yet loaded) model instance. """ - key = _resolve_key(model_key) - module_name, class_name, env_var, default_name = _REGISTRY[key] + entry = REGISTRY[_resolve_key(model_key)] + module_name, class_name = entry.target.split(":") model_cls = getattr(importlib.import_module(module_name), class_name) - overrides.setdefault("model_name", os.getenv(env_var, default_name)) - return model_cls(**overrides) + # The registry mirror must match the class pin, or identity_for and the cache disagree. + if getattr(model_cls, "cache_name", None) != entry.cache_name: + raise RuntimeError(f"{entry.target}.cache_name != registry cache_name {entry.cache_name!r}; " + "update the row in predict.factory.REGISTRY") + overrides.setdefault("model_name", _model_name(entry, os.environ)) + model = model_cls(**overrides) + if "torch_dtype" not in overrides and str(getattr(model, "torch_dtype", None)) != entry.dtype: + raise RuntimeError(f"{entry.target} dtype {getattr(model, 'torch_dtype', None)} != registry dtype " + f"{entry.dtype}; update the row in predict.factory.REGISTRY") + return model -def build_predictor(model_key: Optional[str] = None, +def build_predictor(model: Any = None, key: Optional[str] = None, cache: Optional[SQLiteResponseCache] = None, - **overrides) -> Predictor: - """Construct a cached :class:`Predictor` around :func:`build_model`. + cache_only: bool = False, **overrides) -> Predictor: + """Construct a cached :class:`Predictor` around a registry model or a model instance. Args: - model_key: See :func:`build_model`. + model: Registry key (see :func:`build_model`), None for + ``key``/``$MIRAGE_VL_MODEL``, or a constructed model instance. + key: Registry key when ``model`` is None. Required with an + instance: it names the model in results and verdicts. cache: Cache instance; None uses the default SQLite cache. - **overrides: Forwarded to :func:`build_model`. + cache_only: Open the default cache read-only (ignored when + ``cache`` is given). + **overrides: Forwarded to :func:`build_model` (registry models only). Returns: A Predictor that is a drop-in replacement for the model object; - its ``model_key`` attribute records which registry entry was used. + its ``model_key`` attribute records the key. """ - key = _resolve_key(model_key) - predictor = Predictor(LocalModelBackend(build_model(key, **overrides)), cache=cache) + if model is None or isinstance(model, str): + if model is not None and key is not None and key.lower() != model.lower(): + raise ValueError(f"model {model!r} and key {key!r} disagree") + key = _resolve_key(model or key) + model = build_model(key, **overrides) + elif key is None or overrides: + raise ValueError("a model instance needs key= and takes no constructor overrides") + if cache is None and cache_only: + cache = SQLiteResponseCache(readonly=True) + predictor = Predictor(LocalModelBackend(model), cache=cache) predictor.model_key = key return predictor diff --git a/predict/predictor.py b/predict/predictor.py index a7fb990..6028a64 100644 --- a/predict/predictor.py +++ b/predict/predictor.py @@ -2,15 +2,14 @@ Sits between the experiment driver and the model. On a cache hit the model is never called; on a miss the request is forwarded to the backend, -saved, and returned. ``generate``/``batch_generate`` are aliases of -``predict``/``batch_predict``, so a Predictor drops in wherever a -``BaseModel`` object is used today. +saved, and returned. ``generate`` is an alias of ``predict``, so a +Predictor drops in wherever a ``BaseModel`` object is used today. """ import json import logging import os -from typing import Any, List, Optional +from typing import Any, Optional from predict.backends import LocalModelBackend from predict.cache import SQLiteResponseCache @@ -79,51 +78,8 @@ def predict(self, prompt: str, image: Any = None, **generation_kwargs) -> str: self.cache.put(key, response, meta=self._meta(identity, prompt, image_hash, gen_json)) return response - def batch_predict(self, prompts: List[str], images: Any = None, - **generation_kwargs) -> List[Optional[str]]: - """Batch variant of :meth:`predict`; only misses reach the backend. - - Returns: - One entry per prompt; None where generation failed. - """ - gen = self.backend.merged_generation_config(generation_kwargs) - if self.cache is None or gen.get("do_sample"): - self.stats["bypassed"] += len(prompts) - self._ensure_backend_loaded() - return self.backend.batch_generate(prompts, images=images, **generation_kwargs) - - images = list(images) if images is not None else [None] * len(prompts) - if len(images) != len(prompts): - raise ValueError(f"got {len(prompts)} prompts but {len(images)} images") - identity = self.backend.identity() - hashes = [hash_image(im) for im in images] - keys = [cache_key(identity, p, h, gen) for p, h in zip(prompts, hashes)] - found = self.cache.get_many(keys) - results: List[Optional[str]] = [found.get(k) for k in keys] - miss_idx = [i for i, r in enumerate(results) if r is None] - self.stats["hits"] += len(prompts) - len(miss_idx) - - if miss_idx: - self._ensure_backend_loaded() - outputs = self.backend.batch_generate( - [prompts[i] for i in miss_idx], - images=[images[i] for i in miss_idx], - **generation_kwargs, - ) - self.stats["misses"] += len(miss_idx) - for i, out in zip(miss_idx, outputs): - results[i] = out - if not self.cache.readonly: - gen_json = json.dumps(gen, sort_keys=True) - self.cache.put_many([ - (keys[i], out, self._meta(identity, prompts[i], hashes[i], gen_json)) - for i, out in zip(miss_idx, outputs) if out is not None - ]) - return results - - # BaseModel-compatible names for the same operations. + # BaseModel-compatible name for the same operation. generate = predict - batch_generate = batch_predict def load_model(self) -> None: """Request backend loading; the real load happens on the first miss.""" diff --git a/requirements.txt b/requirements.txt index 67b40b6..21bacab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,11 @@ # Direct runtime dependencies. Pip resolves their transitive dependencies. accelerate==1.12.0 inflect==7.5.0 +mistral-common==1.12.0 nltk==3.9.4 numpy==2.2.6 pillow==11.3.0 torch==2.9.1 torchvision==0.24.1 tqdm==4.67.1 -transformers==4.57.3 +transformers==5.19.0 diff --git a/run.py b/run.py index be63db5..d082797 100644 --- a/run.py +++ b/run.py @@ -15,6 +15,7 @@ from metrics.base_metric import CAPTION_GEN, SAMPLES, BaseMetric, object_names, pool_objects from metrics.pope import QUESTIONS_PER_IMAGE +from predict.factory import REGISTRY as MODEL_REGISTRY logger = logging.getLogger("mirage.run") DS = {"coco": "coco", "visual_genome": "vg"} # --dataset -> the key metrics use (POOL, CAPTION_PROMPTS) @@ -24,7 +25,7 @@ def _parse_args(argv=None): p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) p.add_argument("--dataset", choices=("coco", "visual_genome"), default="coco") p.add_argument("--metric", choices=("pope", "chair", "rope", "throne"), default="chair") - p.add_argument("--model", choices=("paligemma", "qwen2vl"), default=None) + p.add_argument("--model", choices=tuple(MODEL_REGISTRY), default=None) p.add_argument("--samples", type=int, default=SAMPLES, help="evaluated images") p.add_argument("--pool-size", type=int, default=None, help="candidate images loaded before filtering (default 2000; 3000 for COCO ROPE)") @@ -63,14 +64,13 @@ def _load_dataset(args): def _run_dataset_metric(args): - from predict.cache import SQLiteResponseCache from predict.factory import build_predictor loader, all_objects = _load_dataset(args) if not len(loader): raise RuntimeError("No images were loaded. Check the annotation and image paths.") # --cache-only opens the prediction cache read-only: a replay never writes to it, and a miss raises. - model = build_predictor(args.model, cache=SQLiteResponseCache(readonly=True) if args.cache_only else None) + model = build_predictor(args.model, cache_only=args.cache_only) if not args.cache_only: model.load_model() try: