From 38eb9726b328d51e8840589894597c4bf8d52e89 Mon Sep 17 00:00:00 2001 From: Salim Alkharsa Date: Thu, 15 Oct 2026 06:08:46 +0000 Subject: [PATCH 01/10] Add Gemma 4, Ministral 3 and Aya Vision through one chat-template wrapper models/hf_chat_vlm.py loads any processor/auto-class pair whose chat template takes one image and one text turn, decodes only new tokens, strips reasoning blocks and keeps thinking off by default. Register gemma4-e4b, gemma4-12b, ministral3-8b, ministral3-14b and aya-vision-8b with MIRAGE__MODEL_PATH overrides. --- README.md | 2 +- demo.py | 4 +- docs/hpc.mdx | 2 +- docs/metrics-and-models.mdx | 15 ++++- docs/running-evaluations.mdx | 4 +- models/hf_chat_vlm.py | 127 +++++++++++++++++++++++++++++++++++ predict/factory.py | 12 +++- requirements.txt | 3 +- 8 files changed, 159 insertions(+), 10 deletions(-) create mode 100644 models/hf_chat_vlm.py diff --git a/README.md b/README.md index fa702e2..2db9500 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Prepare the dataset files and model weights locally, then run: python demo.py --dataset coco --metric chair --model paligemma --samples 5 --output results/chair-coco.json ``` -Mirage currently supports `paligemma` and `qwen2vl`. See the [documentation](https://mirage-f432d87b.mintlify.app) for dataset setup, metrics, model configuration, and HPC instructions. +Mirage supports `paligemma`, `qwen2vl`, `gemma4-e4b`, `gemma4-12b`, `ministral3-8b`, `ministral3-14b`, and `aya-vision-8b`. See the [documentation](https://mirage-f432d87b.mintlify.app) for dataset setup, metrics, model configuration, and HPC instructions. ## Reference results diff --git a/demo.py b/demo.py index 9a3b701..eebf4d0 100644 --- a/demo.py +++ b/demo.py @@ -19,6 +19,8 @@ from PIL import Image +from predict.factory import _REGISTRY as MODEL_REGISTRY + logger = logging.getLogger("mirage.demo") CAPTION_PROMPT = ( "Describe this image in detail. List every object you can see, their colors, " @@ -54,7 +56,7 @@ def _parse_args() -> argparse.Namespace: choices=("pope", "chair", "rope", "throne", "faithscore", "nope"), default="chair", ) - parser.add_argument("--model", choices=("paligemma", "qwen2vl"), default=None) + parser.add_argument("--model", choices=tuple(MODEL_REGISTRY), default=None) parser.add_argument("--samples", type=int, default=5, help="Number of evaluated images") parser.add_argument( "--pool-size", type=int, default=None, diff --git a/docs/hpc.mdx b/docs/hpc.mdx index 30b797e..351f1d0 100644 --- a/docs/hpc.mdx +++ b/docs/hpc.mdx @@ -13,7 +13,7 @@ This guide covers a first GPU run and repeatable model comparisons on a SLURM cl - The model weights already available in the Hugging Face cache or on cluster storage. Nodes without internet access cannot download weights during the job. - A GPU allocation sized for the selected checkpoint and evaluation. The sample SLURM request is a starting point, not a guarantee that a model fits. -The repository currently exposes `paligemma` and `qwen2vl` to `demo.py`. See [Metrics and models](/metrics-and-models) for checkpoint defaults and overrides. DeepSeek-VL is not currently implemented as a Mirage model backend, so it cannot be selected with `--model` or `MIRAGE_VL_MODEL`. +The repository exposes `paligemma`, `qwen2vl`, `gemma4-e4b`, `gemma4-12b`, `ministral3-8b`, `ministral3-14b`, and `aya-vision-8b` to `demo.py`. See [Metrics and models](/metrics-and-models) for checkpoint defaults and overrides. The Gemma 4, Ministral 3, and Aya Vision keys were tested on A100 40 GB GPUs with `transformers==5.19.0`, `torch==2.9.1` (CUDA 12.8), and `mistral-common==1.12.0`, as pinned in `requirements.txt`. DeepSeek-VL is not currently implemented as a Mirage model backend, so it cannot be selected with `--model` or `MIRAGE_VL_MODEL`. ## Clone and prepare the environment diff --git a/docs/metrics-and-models.mdx b/docs/metrics-and-models.mdx index 8c45624..1acfb2d 100644 --- a/docs/metrics-and-models.mdx +++ b/docs/metrics-and-models.mdx @@ -20,10 +20,19 @@ Metric availability can depend on the dataset path and local files. For example, ## Models -The prediction factory supports two local vision-language model wrappers: +The prediction factory supports these local vision-language models: -- `paligemma` uses `google/paligemma-3b-mix-224` by default. Override the model identifier or local path with `MIRAGE_PALIGEMMA_MODEL_PATH`. -- `qwen2vl` uses `Qwen/Qwen2-VL-2B-Instruct` by default. Override it with `MIRAGE_QWEN_MODEL_PATH`. +| Key | Default checkpoint | Path override | +| --- | --- | --- | +| `paligemma` | `google/paligemma-3b-mix-224` | `MIRAGE_PALIGEMMA_MODEL_PATH` | +| `qwen2vl` | `Qwen/Qwen2-VL-2B-Instruct` | `MIRAGE_QWEN_MODEL_PATH` | +| `gemma4-e4b` | `google/gemma-4-E4B-it` | `MIRAGE_GEMMA4_E4B_MODEL_PATH` | +| `gemma4-12b` | `google/gemma-4-12B-it` | `MIRAGE_GEMMA4_12B_MODEL_PATH` | +| `ministral3-8b` | `mistralai/Ministral-3-8B-Instruct-2512-BF16` | `MIRAGE_MINISTRAL3_8B_MODEL_PATH` | +| `ministral3-14b` | `mistralai/Ministral-3-14B-Instruct-2512-BF16` | `MIRAGE_MINISTRAL3_14B_MODEL_PATH` | +| `aya-vision-8b` | `CohereLabs/aya-vision-8b` | `MIRAGE_AYA_VISION_8B_MODEL_PATH` | + +The Gemma 4, Ministral 3, and Aya Vision keys share one wrapper, `models/hf_chat_vlm.py`. It sends one image and one text turn through the processor's chat template, decodes only the new tokens, and runs with thinking disabled (`enable_thinking=False`). Aya Vision is a gated, non-commercial (CC-BY-NC) checkpoint; accept its license on Hugging Face before downloading it. These are the model keys currently supported by `demo.py`. A different checkpoint can be used through the model-path override only when it is compatible with that wrapper. DeepSeek-VL is not currently supported; adding it requires a model wrapper and factory registration before it can be selected as a separate model key. See [Maintain the docs](/maintaining-docs) for the implementation handoff requirements. diff --git a/docs/running-evaluations.mdx b/docs/running-evaluations.mdx index 0163cec..c0a785d 100644 --- a/docs/running-evaluations.mdx +++ b/docs/running-evaluations.mdx @@ -16,7 +16,7 @@ python demo.py \ --output results/chair-coco.json ``` -`--dataset` accepts `coco` or `visual_genome`. `--model` accepts `paligemma` or `qwen2vl`; if omitted, Mirage reads `MIRAGE_VL_MODEL` and otherwise defaults to `paligemma`. `--samples` sets the number of evaluated images for the selected path. +`--dataset` accepts `coco` or `visual_genome`. `--model` accepts any model key in [Metrics and models](/metrics-and-models#models); if omitted, Mirage reads `MIRAGE_VL_MODEL` and otherwise defaults to `paligemma`. `--samples` sets the number of evaluated images for the selected path. ## Useful options @@ -24,7 +24,7 @@ python demo.py \ | --- | --- | | `--dataset` | Select `coco` or `visual_genome`. | | `--metric` | Select `pope`, `chair`, `rope`, `throne`, `faithscore`, or `nope`. | -| `--model` | Select `paligemma` or `qwen2vl`. | +| `--model` | Select a model key, such as `paligemma`, `qwen2vl`, or `gemma4-e4b`. | | `--samples` | Set the requested evaluation sample count. | | `--pool-size` | Set how many dataset entries are loaded before evaluation. Defaults to 2000 (3000 for COCO ROPE). The pool also fixes the Visual Genome THRONE categories and ROPE prompt classes, so keep it fixed when comparing runs with different `--samples`. | | `--annotations`, `--images` | Override default annotation and image paths. | diff --git a/models/hf_chat_vlm.py b/models/hf_chat_vlm.py new file mode 100644 index 0000000..b7db76f --- /dev/null +++ b/models/hf_chat_vlm.py @@ -0,0 +1,127 @@ +"""Generic wrapper for Hugging Face chat vision-language models. + +Covers models whose processor ships a chat template that accepts one image +and one text turn (Gemma 4, Ministral 3, Aya Vision). Decoding is greedy by +default and only the newly generated tokens are returned. +""" + +import logging +import re +import time +from pathlib import Path +from typing import List, Optional, Union + +import torch +from PIL import Image + +from .base import BaseModel, ModelLoadError, ModelNotLoadedError + +logger = logging.getLogger(__name__) + +# Tokens that close a reasoning block (Gemma 4 uses a "thought" channel). They are +# special tokens, so the answer is cut at token level before decoding. +_THINKING_END_TOKENS = ("", "") +# Reasoning blocks written as plain text. +_THINKING_RE = re.compile(r".*?", re.DOTALL) + + +class HFChatVLM(BaseModel): + """Chat-template vision-language model loaded with ``AutoModelForImageTextToText``. + + Args: + model_name: HF model id or local snapshot path. + enable_thinking: Forwarded to the chat template; off for evaluation. + device: Device for inference (default: CUDA when available). + torch_dtype: Weight dtype (default: bfloat16). + """ + + def __init__( + self, + model_name: str, + enable_thinking: bool = False, + device: Optional[str] = None, + torch_dtype: Optional[torch.dtype] = None, + **kwargs, + ): + super().__init__(model_name, model_type="local", **kwargs) + self.supports_images = True + self.enable_thinking = enable_thinking + self.device = device or ("cuda" if torch.cuda.is_available() else "cpu") + self.torch_dtype = torch_dtype or torch.bfloat16 + self.model = None + self.processor = None + self.config = {"max_new_tokens": 512, "do_sample": False, **self.config} + + def load_model(self) -> None: + start = time.monotonic() + try: + from transformers import AutoModelForImageTextToText, AutoProcessor + + self.processor = AutoProcessor.from_pretrained(self.model_name) + self.model = AutoModelForImageTextToText.from_pretrained( + self.model_name, dtype=self.torch_dtype, device_map=self.device, + ).eval() + self._raw_logged = False + vocab = getattr(self.processor.tokenizer, "get_vocab", dict)() + self._thinking_end_ids = {vocab[t] for t in _THINKING_END_TOKENS if t in vocab} + except Exception as e: + raise ModelLoadError(f"Failed to load {self.model_name}: {e}") from e + self.is_loaded = True + logger.info("Loaded %s in %.1f s", self.model_name, time.monotonic() - start) + + def unload_model(self) -> None: + if torch.cuda.is_available() and self.model is not None: + logger.info("Peak CUDA memory: %.2f GB", torch.cuda.max_memory_allocated() / 1e9) + self.model = None + self.processor = None + if torch.cuda.is_available(): + torch.cuda.empty_cache() + self.is_loaded = False + + def generate( + self, + prompt: str, + image: Optional[Union[Image.Image, str, Path]] = None, + **kwargs, + ) -> str: + if not self.is_loaded: + raise ModelNotLoadedError("Call load_model() before generate().") + content = [{"type": "text", "text": prompt}] + if image is not None: + content.insert(0, {"type": "image", "image": self.preprocess_image(image)}) + inputs = self.processor.apply_chat_template( + [{"role": "user", "content": content}], + add_generation_prompt=True, + tokenize=True, + return_dict=True, + return_tensors="pt", + enable_thinking=self.enable_thinking, + ).to(self.model.device) + for key, value in inputs.items(): + if torch.is_floating_point(value): + inputs[key] = value.to(self.torch_dtype) + gen = {**self.config, **kwargs} + gen_kwargs = {"max_new_tokens": gen["max_new_tokens"], "do_sample": bool(gen.get("do_sample"))} + if gen_kwargs["do_sample"]: + gen_kwargs.update({k: gen[k] for k in ("temperature", "top_p", "top_k") if gen.get(k) is not None}) + with torch.inference_mode(): + output = self.model.generate(**inputs, **gen_kwargs) + new_tokens = output[0, inputs["input_ids"].shape[1]:].tolist() + if not self._raw_logged: + # One raw decode per load, so a run's log shows any thinking markers. + logger.info("Raw decode: %r", self.processor.decode(new_tokens, skip_special_tokens=False)) + self._raw_logged = True + ends = [i for i, token in enumerate(new_tokens) if token in self._thinking_end_ids] + if ends: + new_tokens = new_tokens[ends[-1] + 1:] + text = self.processor.decode(new_tokens, skip_special_tokens=True) + return _THINKING_RE.sub("", text).strip() + + def batch_generate( + self, + prompts: List[str], + images: Optional[List[Union[Image.Image, str, Path, None]]] = None, + **kwargs, + ) -> List[Optional[str]]: + images = images or [None] * len(prompts) + return [self.generate(prompt, image=image, **kwargs) for prompt, image in zip(prompts, images)] diff --git a/predict/factory.py b/predict/factory.py index dee3e1c..da06e1e 100644 --- a/predict/factory.py +++ b/predict/factory.py @@ -17,6 +17,16 @@ "MIRAGE_PALIGEMMA_MODEL_PATH", "google/paligemma-3b-mix-224"), "qwen2vl": ("models.qwen2vl", "Qwen2VLModel", "MIRAGE_QWEN_MODEL_PATH", "Qwen/Qwen2-VL-2B-Instruct"), + "gemma4-e4b": ("models.hf_chat_vlm", "HFChatVLM", + "MIRAGE_GEMMA4_E4B_MODEL_PATH", "google/gemma-4-E4B-it"), + "gemma4-12b": ("models.hf_chat_vlm", "HFChatVLM", + "MIRAGE_GEMMA4_12B_MODEL_PATH", "google/gemma-4-12B-it"), + "ministral3-8b": ("models.hf_chat_vlm", "HFChatVLM", + "MIRAGE_MINISTRAL3_8B_MODEL_PATH", "mistralai/Ministral-3-8B-Instruct-2512-BF16"), + "ministral3-14b": ("models.hf_chat_vlm", "HFChatVLM", + "MIRAGE_MINISTRAL3_14B_MODEL_PATH", "mistralai/Ministral-3-14B-Instruct-2512-BF16"), + "aya-vision-8b": ("models.hf_chat_vlm", "HFChatVLM", + "MIRAGE_AYA_VISION_8B_MODEL_PATH", "CohereLabs/aya-vision-8b"), } @@ -31,7 +41,7 @@ def build_model(model_key: Optional[str] = None, **overrides) -> Any: """Construct the model selected by ``model_key`` or ``$MIRAGE_VL_MODEL``. Args: - model_key: Registry key (``"paligemma"`` or ``"qwen2vl"``); None + model_key: Registry key (see ``_REGISTRY``); None reads ``MIRAGE_VL_MODEL`` (default ``"paligemma"``). **overrides: Constructor kwargs; ``model_name`` defaults to the model's ``MIRAGE_*_MODEL_PATH`` environment variable. diff --git a/requirements.txt b/requirements.txt index 67b40b6..21bacab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,11 @@ # Direct runtime dependencies. Pip resolves their transitive dependencies. accelerate==1.12.0 inflect==7.5.0 +mistral-common==1.12.0 nltk==3.9.4 numpy==2.2.6 pillow==11.3.0 torch==2.9.1 torchvision==0.24.1 tqdm==4.67.1 -transformers==4.57.3 +transformers==5.19.0 From 8c92e456943e6ce4a0d850d05a0b02f0e9c71917 Mon Sep 17 00:00:00 2001 From: Salim Alkharsa Date: Sat, 17 Oct 2026 03:50:34 +0000 Subject: [PATCH 02/10] Rename models/hf_chat_vlm.py to models/hf_chat.py The generic chat wrapper keeps its body; the registry points at the new module path. --- models/{hf_chat_vlm.py => hf_chat.py} | 0 predict/factory.py | 10 +++++----- 2 files changed, 5 insertions(+), 5 deletions(-) rename models/{hf_chat_vlm.py => hf_chat.py} (100%) diff --git a/models/hf_chat_vlm.py b/models/hf_chat.py similarity index 100% rename from models/hf_chat_vlm.py rename to models/hf_chat.py diff --git a/predict/factory.py b/predict/factory.py index da06e1e..35964cb 100644 --- a/predict/factory.py +++ b/predict/factory.py @@ -17,15 +17,15 @@ "MIRAGE_PALIGEMMA_MODEL_PATH", "google/paligemma-3b-mix-224"), "qwen2vl": ("models.qwen2vl", "Qwen2VLModel", "MIRAGE_QWEN_MODEL_PATH", "Qwen/Qwen2-VL-2B-Instruct"), - "gemma4-e4b": ("models.hf_chat_vlm", "HFChatVLM", + "gemma4-e4b": ("models.hf_chat", "HFChatVLM", "MIRAGE_GEMMA4_E4B_MODEL_PATH", "google/gemma-4-E4B-it"), - "gemma4-12b": ("models.hf_chat_vlm", "HFChatVLM", + "gemma4-12b": ("models.hf_chat", "HFChatVLM", "MIRAGE_GEMMA4_12B_MODEL_PATH", "google/gemma-4-12B-it"), - "ministral3-8b": ("models.hf_chat_vlm", "HFChatVLM", + "ministral3-8b": ("models.hf_chat", "HFChatVLM", "MIRAGE_MINISTRAL3_8B_MODEL_PATH", "mistralai/Ministral-3-8B-Instruct-2512-BF16"), - "ministral3-14b": ("models.hf_chat_vlm", "HFChatVLM", + "ministral3-14b": ("models.hf_chat", "HFChatVLM", "MIRAGE_MINISTRAL3_14B_MODEL_PATH", "mistralai/Ministral-3-14B-Instruct-2512-BF16"), - "aya-vision-8b": ("models.hf_chat_vlm", "HFChatVLM", + "aya-vision-8b": ("models.hf_chat", "HFChatVLM", "MIRAGE_AYA_VISION_8B_MODEL_PATH", "CohereLabs/aya-vision-8b"), } From 4aa7c62ba811b4bae9d22eb00dae706de4d8aeb6 Mon Sep 17 00:00:00 2001 From: Salim Alkharsa Date: Sat, 17 Oct 2026 03:51:23 +0000 Subject: [PATCH 03/10] Remove batch_generate everywhere No driver called it: every published cache row came from single-item Predictor.predict, and padded batches can differ from single-item outputs while sharing their cache keys. Removed BaseModel.batch_generate and batch_inputs, use_true_batching, the PaliGemma and Qwen2-VL _batch_generate_parallel/_prepare_batch_inputs, HFChatVLM.batch_generate, Predictor.batch_predict and its alias, and both backends' batch_generate. The THRONE evaluator keeps its own batching (it calls Predictor.predict). generate() is unchanged. --- models/base.py | 82 +------------------ models/hf_chat.py | 11 +-- models/paligemma.py | 157 +------------------------------------ models/qwen2vl.py | 183 +------------------------------------------ predict/backends.py | 17 +--- predict/predictor.py | 52 +----------- 6 files changed, 12 insertions(+), 490 deletions(-) diff --git a/models/base.py b/models/base.py index 4bb73af..191af0c 100644 --- a/models/base.py +++ b/models/base.py @@ -7,7 +7,7 @@ """ from abc import ABC, abstractmethod -from typing import Any, Dict, List, Optional, Union, Tuple, Literal +from typing import Any, Dict, Optional, Union, Tuple, Literal from pathlib import Path from PIL import Image import logging @@ -38,7 +38,6 @@ class BaseModel(ABC): supports_text (bool): Whether the model can process text max_tokens (Optional[int]): Maximum tokens the model can handle quantization (Optional[str]): Quantization method ('int4', 'int8', 'none') - use_true_batching (bool): Whether to use true batch processing _logger (logging.Logger): Logger instance for the model """ @@ -48,7 +47,6 @@ def __init__( model_type: str = "local", config: Optional[Dict[str, Any]] = None, quantization: Optional[Literal["int4", "int8", "none"]] = None, - use_true_batching: bool = False, **kwargs ): """ @@ -59,7 +57,6 @@ def __init__( model_type: Type of model deployment ('local' or 'api') config: Dictionary containing model-specific configuration quantization: Quantization method to use ('int4', 'int8', or 'none' for full precision) - use_true_batching: Whether to use true batch processing (default: False for sequential) **kwargs: Additional keyword arguments for model initialization """ self.model_name = model_name @@ -67,7 +64,6 @@ def __init__( self.config = config or {} self.is_loaded = False self.quantization = quantization - self.use_true_batching = use_true_batching # Model capabilities - to be set by subclasses self.supports_images = False @@ -146,35 +142,6 @@ def generate( """ pass - @abstractmethod - def batch_generate( - self, - prompts: List[str], - images: Optional[List[Union[Image.Image, str, Path, None]]] = None, - **kwargs - ) -> List[Optional[str]]: - """ - Generate responses for a batch of inputs. - - This method should efficiently process multiple inputs, potentially - using batching optimizations for local models or parallel API calls. - - Args: - prompts: List of text prompts - images: Optional list of images (can contain None for text-only prompts) - **kwargs: Additional generation parameters - - Returns: - List[Optional[str]]: One entry per prompt; None where generation - failed, so callers can exclude it instead of scoring it. - - Raises: - NotImplementedError: If not implemented by subclass - ModelNotLoadedError: If model is not loaded - InvalidInputError: If inputs are invalid - """ - pass - def validate_input( self, prompt: str, @@ -369,53 +336,6 @@ def get_quantization_config(self, quantization_backend: str = "torchao") -> Opti f"BitsAndBytes backend only supports int4/int8, got: {self.quantization}" ) - def batch_inputs( - self, - prompts: List[str], - images: Optional[List[Union[Image.Image, str, Path, None]]] = None, - batch_size: Optional[int] = None - ) -> List[Tuple[List[str], Optional[List[Union[Image.Image, str, Path, None]]]]]: - """ - Split inputs into batches for processing. - - Utility method to help subclasses implement batch processing. Groups prompts - and images into batches of specified size. - - Args: - prompts: List of text prompts - images: Optional list of images (same length as prompts or None) - batch_size: Number of items per batch (None = process all at once) - - Returns: - List of tuples, each containing (batch_prompts, batch_images) - - Example: - >>> batches = self.batch_inputs(prompts, images, batch_size=4) - >>> for batch_prompts, batch_images in batches: - >>> # Process batch - >>> results = self._process_batch(batch_prompts, batch_images) - """ - if images is None: - images = [None] * len(prompts) - - if len(prompts) != len(images): - raise ValueError( - f"Length mismatch: {len(prompts)} prompts but {len(images)} images" - ) - - # If no batch size specified, return everything as one batch - if batch_size is None: - return [(prompts, images)] - - # Split into batches - batches = [] - for i in range(0, len(prompts), batch_size): - batch_prompts = prompts[i:i + batch_size] - batch_images = images[i:i + batch_size] - batches.append((batch_prompts, batch_images)) - - return batches - class ModelError(Exception): """Base exception class for model-related errors.""" diff --git a/models/hf_chat.py b/models/hf_chat.py index b7db76f..b59c19c 100644 --- a/models/hf_chat.py +++ b/models/hf_chat.py @@ -9,7 +9,7 @@ import re import time from pathlib import Path -from typing import List, Optional, Union +from typing import Optional, Union import torch from PIL import Image @@ -116,12 +116,3 @@ def generate( new_tokens = new_tokens[ends[-1] + 1:] text = self.processor.decode(new_tokens, skip_special_tokens=True) return _THINKING_RE.sub("", text).strip() - - def batch_generate( - self, - prompts: List[str], - images: Optional[List[Union[Image.Image, str, Path, None]]] = None, - **kwargs, - ) -> List[Optional[str]]: - images = images or [None] * len(prompts) - return [self.generate(prompt, image=image, **kwargs) for prompt, image in zip(prompts, images)] diff --git a/models/paligemma.py b/models/paligemma.py index 28e8d13..cb2651d 100644 --- a/models/paligemma.py +++ b/models/paligemma.py @@ -7,7 +7,7 @@ """ import torch -from typing import Any, Dict, List, Optional, Union, Literal +from typing import Any, Dict, Optional, Union, Literal from pathlib import Path from PIL import Image @@ -41,7 +41,6 @@ def __init__( torch_dtype: Optional[torch.dtype] = None, quantization: Optional[Literal["int4", "int8", "none"]] = None, quantization_backend: str = "torchao", - use_true_batching: bool = False, **kwargs ): """ @@ -53,14 +52,12 @@ def __init__( torch_dtype: PyTorch data type for model weights (default: bfloat16) quantization: Quantization method ('int4', 'int8', 'none', or None) quantization_backend: Backend for quantization ('torchao' or 'bitsandbytes') - use_true_batching: Whether to use true batch processing (default: False) **kwargs: Additional arguments passed to base class """ super().__init__( model_name, model_type="local", quantization=quantization, - use_true_batching=use_true_batching, **kwargs ) @@ -226,57 +223,6 @@ def generate( self._logger.error(f"Generation failed: {e}") raise InvalidInputError(f"Failed to generate response: {e}") - def batch_generate( - self, - prompts: List[str], - images: Optional[List[Union[Image.Image, str, Path, None]]] = None, - batch_size: Optional[int] = None, - **kwargs - ) -> List[Optional[str]]: - """ - Generate responses for a batch of inputs. - - Supports both sequential processing (default) and true batch processing - based on the `use_true_batching` setting. When true batching is enabled, - processes multiple samples in parallel for improved efficiency. - - Args: - prompts: List of text prompts - images: Optional list of images (can contain None values) - batch_size: Size of batches for true batching (only used if use_true_batching=True) - **kwargs: Generation parameters - - Returns: - List[Optional[str]]: One entry per prompt; None where generation failed - - Raises: - ModelNotLoadedError: If model is not loaded - InvalidInputError: If inputs are invalid - """ - if not self.is_loaded: - raise ModelNotLoadedError("Model is not loaded") - - if images is None: - images = [None] * len(prompts) - - if len(prompts) != len(images): - raise InvalidInputError("Number of prompts and images must match") - - # Sequential processing (original behavior) - if not self.use_true_batching: - responses = [] - for prompt, image in zip(prompts, images): - try: - response = self.generate(prompt, image, **kwargs) - responses.append(response) - except Exception as e: - self._logger.warning(f"Failed to generate for prompt '{prompt[:50]}...': {e}") - responses.append(None) - return responses - - # True batch processing - return self._batch_generate_parallel(prompts, images, batch_size, **kwargs) - def _generate_with_config( self, inputs: Dict[str, torch.Tensor], @@ -285,9 +231,6 @@ def _generate_with_config( """ Generate outputs using model.generate with the given config. - This helper method centralizes the generation logic to avoid duplication - between single and batch generation methods. - Args: inputs: Prepared model inputs generation_config: Generation configuration parameters @@ -307,104 +250,6 @@ def _generate_with_config( ) return outputs - def _batch_generate_parallel( - self, - prompts: List[str], - images: Optional[List[Union[Image.Image, str, Path, None]]], - batch_size: Optional[int] = None, - **kwargs - ) -> List[Optional[str]]: - """ - Generate responses using true batch processing. - - This method processes multiple inputs in parallel batches for improved - efficiency compared to sequential processing. - - Args: - prompts: List of text prompts - images: List of images (or None values) - batch_size: Size of batches (None = all at once) - **kwargs: Generation parameters - - Returns: - List[Optional[str]]: One entry per prompt; None where generation failed - """ - all_responses = [] - - # Use base class helper to split into batches - batches = self.batch_inputs(prompts, images, batch_size) - - for batch_prompts, batch_images in batches: - try: - # Prepare batch inputs - batch_inputs = self._prepare_batch_inputs(batch_prompts, batch_images) - - # Merge generation config - generation_config = self.config.copy() - generation_config.update(kwargs) - - # Generate for batch using helper - outputs = self._generate_with_config(batch_inputs, generation_config) - - # Decode all outputs in batch - input_len = batch_inputs["input_ids"].shape[1] - for output in outputs: - response = self.processor.decode( - output[input_len:], - skip_special_tokens=True - ) - all_responses.append(response.strip()) - - except Exception as e: - self._logger.error(f"Batch generation failed: {e}") - all_responses.extend([None] * len(batch_prompts)) - - return all_responses - - def _prepare_batch_inputs( - self, - prompts: List[str], - images: Optional[List[Union[Image.Image, str, Path, None]]] - ) -> Dict[str, torch.Tensor]: - """ - Prepare inputs for batch processing. - - Args: - prompts: List of text prompts - images: List of images (or None values) - - Returns: - Dict containing processed batch inputs for the model - """ - # Preprocess all images and add tokens to prompts - processed_images = [] - processed_prompts = [] - - for prompt, image in zip(prompts, images): - if image is not None: - processed_images.append(self.preprocess_image(image)) - # Add token prefix if not already present - if not prompt.startswith(""): - processed_prompts.append(f"{prompt}") - else: - processed_prompts.append(prompt) - else: - processed_images.append(None) - processed_prompts.append(prompt) - - # Use processor to prepare batch inputs - inputs = self.processor( - text=processed_prompts, - images=processed_images if any(img is not None for img in processed_images) else None, - return_tensors="pt", - padding=True - ) - - # Move inputs to device - inputs = {k: v.to(self.device) for k, v in inputs.items()} - - return inputs - def _prepare_inputs( self, prompt: str, diff --git a/models/qwen2vl.py b/models/qwen2vl.py index 346c744..428d8c1 100644 --- a/models/qwen2vl.py +++ b/models/qwen2vl.py @@ -7,7 +7,7 @@ """ import torch -from typing import Any, Dict, List, Optional, Union, Literal +from typing import Any, Dict, Optional, Union, Literal from pathlib import Path from PIL import Image @@ -42,7 +42,6 @@ def __init__( torch_dtype: Optional[torch.dtype] = None, quantization: Optional[Literal["int4", "int8", "none"]] = None, quantization_backend: str = "torchao", - use_true_batching: bool = False, **kwargs ): """ @@ -54,14 +53,12 @@ def __init__( torch_dtype: PyTorch data type for model weights (default: bfloat16) quantization: Quantization method ('int4', 'int8', 'none', or None) quantization_backend: Backend for quantization ('torchao' or 'bitsandbytes') - use_true_batching: Whether to use true batch processing (default: False) **kwargs: Additional arguments passed to base class """ super().__init__( model_name, model_type="local", quantization=quantization, - use_true_batching=use_true_batching, **kwargs ) @@ -233,57 +230,6 @@ def generate( self._logger.error(f"Generation failed: {e}") raise InvalidInputError(f"Failed to generate response: {e}") - def batch_generate( - self, - prompts: List[str], - images: Optional[List[Union[Image.Image, str, Path, None]]] = None, - batch_size: Optional[int] = None, - **kwargs - ) -> List[Optional[str]]: - """ - Generate responses for a batch of inputs. - - Supports both sequential processing (default) and true batch processing - based on the `use_true_batching` setting. When true batching is enabled, - processes multiple samples in parallel for improved efficiency. - - Args: - prompts: List of text prompts - images: Optional list of images (can contain None values) - batch_size: Size of batches for true batching (only used if use_true_batching=True) - **kwargs: Generation parameters - - Returns: - List[Optional[str]]: One entry per prompt; None where generation failed - - Raises: - ModelNotLoadedError: If model is not loaded - InvalidInputError: If inputs are invalid - """ - if not self.is_loaded: - raise ModelNotLoadedError("Model is not loaded") - - if images is None: - images = [None] * len(prompts) - - if len(prompts) != len(images): - raise InvalidInputError("Number of prompts and images must match") - - # Sequential processing (original behavior) - if not self.use_true_batching: - responses = [] - for prompt, image in zip(prompts, images): - try: - response = self.generate(prompt, image, **kwargs) - responses.append(response) - except Exception as e: - self._logger.warning(f"Failed to generate for prompt '{prompt[:50]}...': {e}") - responses.append(None) - return responses - - # True batch processing - return self._batch_generate_parallel(prompts, images, batch_size, **kwargs) - def _generate_with_config( self, inputs: Dict[str, torch.Tensor], @@ -292,9 +238,6 @@ def _generate_with_config( """ Generate outputs using model.generate with the given config. - This helper method centralizes the generation logic to avoid duplication - between single and batch generation methods. - Args: inputs: Prepared model inputs generation_config: Generation configuration parameters @@ -314,60 +257,6 @@ def _generate_with_config( ) return outputs - def _batch_generate_parallel( - self, - prompts: List[str], - images: Optional[List[Union[Image.Image, str, Path, None]]], - batch_size: Optional[int] = None, - **kwargs - ) -> List[Optional[str]]: - """ - Generate responses using true batch processing. - - This method processes multiple inputs in parallel batches for improved - efficiency compared to sequential processing. - - Args: - prompts: List of text prompts - images: List of images (or None values) - batch_size: Size of batches (None = all at once) - **kwargs: Generation parameters - - Returns: - List[Optional[str]]: One entry per prompt; None where generation failed - """ - all_responses = [] - - # Use base class helper to split into batches - batches = self.batch_inputs(prompts, images, batch_size) - - for batch_prompts, batch_images in batches: - try: - # Prepare batch inputs - batch_inputs = self._prepare_batch_inputs(batch_prompts, batch_images) - - # Merge generation config - generation_config = self.config.copy() - generation_config.update(kwargs) - - # Generate for batch using helper - outputs = self._generate_with_config(batch_inputs, generation_config) - - # Decode all outputs in batch - input_len = batch_inputs["input_ids"].shape[1] - for output in outputs: - response = self.processor.decode( - output[input_len:], - skip_special_tokens=True - ) - all_responses.append(response.strip()) - - except Exception as e: - self._logger.error(f"Batch generation failed: {e}") - all_responses.extend([None] * len(batch_prompts)) - - return all_responses - def _prepare_inputs( self, prompt: str, @@ -430,76 +319,6 @@ def _prepare_inputs( return inputs - def _prepare_batch_inputs( - self, - prompts: List[str], - images: Optional[List[Union[Image.Image, str, Path, None]]] - ) -> Dict[str, torch.Tensor]: - """ - Prepare inputs for batch processing with Qwen2-VL's conversation format. - - Args: - prompts: List of text prompts - images: List of images (or None values) - - Returns: - Dict containing processed batch inputs for the model - """ - # Preprocess all images - processed_images = [] - for image in images: - if image is not None: - processed_images.append(self.preprocess_image(image)) - else: - processed_images.append(None) - - # Prepare messages for each sample - all_texts = [] - batch_images = [] - - for prompt, image in zip(prompts, processed_images): - messages = [ - { - "role": "user", - "content": [] - } - ] - - # Add image to content if provided - if image is not None: - messages[0]["content"].append({ - "type": "image", - "image": image - }) - batch_images.append(image) - - # Add text to content - messages[0]["content"].append({ - "type": "text", - "text": prompt - }) - - # Apply chat template - text = self.processor.apply_chat_template( - messages, - tokenize=False, - add_generation_prompt=True - ) - all_texts.append(text) - - # Process batch with processor - inputs = self.processor( - text=all_texts, - images=batch_images if batch_images else None, - return_tensors="pt", - padding=True - ) - - # Move inputs to device - inputs = {k: v.to(self.device) for k, v in inputs.items()} - - return inputs - def get_model_info(self) -> Dict[str, Any]: """ Get Qwen2-VL-specific model information. diff --git a/predict/backends.py b/predict/backends.py index aa47c05..f6c9843 100644 --- a/predict/backends.py +++ b/predict/backends.py @@ -2,7 +2,7 @@ Routing is decided by which backend the Predictor is constructed with — never by branching inside the Predictor and never inside ``BaseModel``. -A backend provides: ``generate``, ``batch_generate``, ``identity()``, +A backend provides: ``generate``, ``identity()``, ``merged_generation_config(kwargs)``, ``load``/``unload``, ``is_loaded``, and ``model_name``. A future API backend implements the same surface with an identity like ``{"provider": "gemini", "model_id": "models/gemini-2.5-flash"}``; @@ -18,9 +18,9 @@ class LocalModelBackend: """Adapts a ``BaseModel``-shaped object to the backend surface. Args: - model: Any object exposing ``generate``, ``batch_generate``, - ``load_model``, ``unload_model``, ``model_name``, ``config``, - and ``is_loaded`` (duck-typed; no torch import here). + model: Any object exposing ``generate``, ``load_model``, + ``unload_model``, ``model_name``, ``config``, and ``is_loaded`` + (duck-typed; no torch import here). """ def __init__(self, model: Any): @@ -30,9 +30,6 @@ def __init__(self, model: Any): def generate(self, prompt, image=None, **kwargs): return self.model.generate(prompt=prompt, image=image, **kwargs) - def batch_generate(self, prompts, images=None, **kwargs): - return self.model.batch_generate(prompts, images=images, **kwargs) - def identity(self) -> Dict[str, Any]: if self._identity is None: model_id, revision = parse_model_name(self.model.model_name) @@ -105,12 +102,6 @@ def generate(self, prompt, image=None, **kwargs): self._reject_conflicting_kwargs(kwargs) return self._fn([prompt])[0] - def batch_generate(self, prompts, images=None, **kwargs): - if images is not None and any(im is not None for im in images): - raise ValueError("CallableBackend is text-only") - self._reject_conflicting_kwargs(kwargs) - return self._fn(list(prompts)) - def _reject_conflicting_kwargs(self, kwargs): # fn owns its generation config; kwargs it cannot honor must not be # silently recorded in cache keys/metadata as if they were applied. diff --git a/predict/predictor.py b/predict/predictor.py index a7fb990..6028a64 100644 --- a/predict/predictor.py +++ b/predict/predictor.py @@ -2,15 +2,14 @@ Sits between the experiment driver and the model. On a cache hit the model is never called; on a miss the request is forwarded to the backend, -saved, and returned. ``generate``/``batch_generate`` are aliases of -``predict``/``batch_predict``, so a Predictor drops in wherever a -``BaseModel`` object is used today. +saved, and returned. ``generate`` is an alias of ``predict``, so a +Predictor drops in wherever a ``BaseModel`` object is used today. """ import json import logging import os -from typing import Any, List, Optional +from typing import Any, Optional from predict.backends import LocalModelBackend from predict.cache import SQLiteResponseCache @@ -79,51 +78,8 @@ def predict(self, prompt: str, image: Any = None, **generation_kwargs) -> str: self.cache.put(key, response, meta=self._meta(identity, prompt, image_hash, gen_json)) return response - def batch_predict(self, prompts: List[str], images: Any = None, - **generation_kwargs) -> List[Optional[str]]: - """Batch variant of :meth:`predict`; only misses reach the backend. - - Returns: - One entry per prompt; None where generation failed. - """ - gen = self.backend.merged_generation_config(generation_kwargs) - if self.cache is None or gen.get("do_sample"): - self.stats["bypassed"] += len(prompts) - self._ensure_backend_loaded() - return self.backend.batch_generate(prompts, images=images, **generation_kwargs) - - images = list(images) if images is not None else [None] * len(prompts) - if len(images) != len(prompts): - raise ValueError(f"got {len(prompts)} prompts but {len(images)} images") - identity = self.backend.identity() - hashes = [hash_image(im) for im in images] - keys = [cache_key(identity, p, h, gen) for p, h in zip(prompts, hashes)] - found = self.cache.get_many(keys) - results: List[Optional[str]] = [found.get(k) for k in keys] - miss_idx = [i for i, r in enumerate(results) if r is None] - self.stats["hits"] += len(prompts) - len(miss_idx) - - if miss_idx: - self._ensure_backend_loaded() - outputs = self.backend.batch_generate( - [prompts[i] for i in miss_idx], - images=[images[i] for i in miss_idx], - **generation_kwargs, - ) - self.stats["misses"] += len(miss_idx) - for i, out in zip(miss_idx, outputs): - results[i] = out - if not self.cache.readonly: - gen_json = json.dumps(gen, sort_keys=True) - self.cache.put_many([ - (keys[i], out, self._meta(identity, prompts[i], hashes[i], gen_json)) - for i, out in zip(miss_idx, outputs) if out is not None - ]) - return results - - # BaseModel-compatible names for the same operations. + # BaseModel-compatible name for the same operation. generate = predict - batch_generate = batch_predict def load_model(self) -> None: """Request backend loading; the real load happens on the first miss.""" From 1b93e8679f48a373ae43669a820d3ba30224b4c6 Mon Sep 17 00:00:00 2001 From: Salim Alkharsa Date: Sat, 17 Oct 2026 03:52:02 +0000 Subject: [PATCH 04/10] models: drop dead BaseModel helpers; import families lazily get_model_info (and the PaliGemma/Qwen2-VL overrides), update_config and the context manager had no callers. base.py imports torch only inside the bitsandbytes branch, and models/__init__ resolves classes on first use, so importing models needs no torch. get_quantization_config stays: both load_model methods call it. --- models/__init__.py | 27 ++++++++++++++++++--------- models/base.py | 42 ++---------------------------------------- models/paligemma.py | 17 ----------------- models/qwen2vl.py | 17 ----------------- 4 files changed, 20 insertions(+), 83 deletions(-) diff --git a/models/__init__.py b/models/__init__.py index 7fa6553..f3cebda 100644 --- a/models/__init__.py +++ b/models/__init__.py @@ -1,13 +1,22 @@ -""" -Models module for the Mirage package. +"""Model wrappers, one file per family; imported on first use so ``import models`` needs no torch. -This module provides a flexible architecture for loading and using various -Vision-Language Models (VLMs) and Large Language Models (LLMs) for -hallucination testing and evaluation. +The registry in ``predict.factory`` names each model as ``module:Class``. """ +import importlib + +_MODULES = { + "BaseModel": "base", + "PaligemmaModel": "paligemma", + "Qwen2VLModel": "qwen2vl", + "HFChatVLM": "hf_chat", +} + +__all__ = list(_MODULES) -from .base import BaseModel -from .paligemma import PaligemmaModel -from .qwen2vl import Qwen2VLModel -__all__ = ['BaseModel', 'PaligemmaModel', 'Qwen2VLModel'] +def __getattr__(name): + if name not in _MODULES: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + value = getattr(importlib.import_module(f".{_MODULES[name]}", __name__), name) + globals()[name] = value + return value diff --git a/models/base.py b/models/base.py index 191af0c..aeba927 100644 --- a/models/base.py +++ b/models/base.py @@ -11,7 +11,6 @@ from pathlib import Path from PIL import Image import logging -import torch class BaseModel(ABC): @@ -216,45 +215,6 @@ def preprocess_image(self, image: Union[Image.Image, str, Path]) -> Image.Image: return image - def get_model_info(self) -> Dict[str, Any]: - """ - Get comprehensive information about the model. - - Returns: - Dict containing model metadata and current state - """ - return { - 'model_name': self.model_name, - 'model_type': self.model_type, - 'is_loaded': self.is_loaded, - 'supports_images': self.supports_images, - 'supports_text': self.supports_text, - 'max_tokens': self.max_tokens, - 'config': self.config.copy(), - 'class_name': self.__class__.__name__ - } - - def update_config(self, **kwargs) -> None: - """ - Update model configuration. - - Args: - **kwargs: Configuration parameters to update - """ - self.config.update(kwargs) - self._logger.info(f"Updated config: {kwargs}") - - def __enter__(self): - """Context manager entry - load model if not already loaded.""" - if not self.is_loaded: - self.load_model() - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - """Context manager exit - unload model.""" - if self.is_loaded: - self.unload_model() - def __repr__(self) -> str: """String representation of the model.""" status = "loaded" if self.is_loaded else "not loaded" @@ -304,6 +264,8 @@ def get_quantization_config(self, quantization_backend: str = "torchao") -> Opti elif quantization_backend == "bitsandbytes": if self.quantization == "int4": + import torch + try: from transformers import BitsAndBytesConfig self._logger.info("Using BitsAndBytes 4-bit quantization") diff --git a/models/paligemma.py b/models/paligemma.py index cb2651d..c70a73d 100644 --- a/models/paligemma.py +++ b/models/paligemma.py @@ -284,20 +284,3 @@ def _prepare_inputs( inputs = {k: v.to(self.device) for k, v in inputs.items()} return inputs - - def get_model_info(self) -> Dict[str, Any]: - """ - Get Paligemma-specific model information. - - Returns: - Dict containing model metadata and current state - """ - info = super().get_model_info() - info.update({ - 'device': self.device, - 'torch_dtype': str(self.torch_dtype), - 'model_architecture': 'PaliGemma', - 'input_resolution': '224x224', - 'parameter_count': '3B' - }) - return info diff --git a/models/qwen2vl.py b/models/qwen2vl.py index 428d8c1..d1aab7e 100644 --- a/models/qwen2vl.py +++ b/models/qwen2vl.py @@ -318,20 +318,3 @@ def _prepare_inputs( inputs = {k: v.to(self.device) for k, v in inputs.items()} return inputs - - def get_model_info(self) -> Dict[str, Any]: - """ - Get Qwen2-VL-specific model information. - - Returns: - Dict containing model metadata and current state - """ - info = super().get_model_info() - info.update({ - 'device': self.device, - 'torch_dtype': str(self.torch_dtype), - 'model_architecture': 'Qwen2-VL', - 'context_length': self.max_tokens, - 'supports_multilingual': True - }) - return info From cdc85f5062b0a04409312c20d535be9d1bead6bd Mon Sep 17 00:00:00 2001 From: Salim Alkharsa Date: Sat, 17 Oct 2026 03:52:32 +0000 Subject: [PATCH 05/10] Pin each model family's cache name on its class BaseModel.cache_name is the model name that goes into prediction cache keys; LocalModelBackend uses it and falls back to the class name for objects without one. PaligemmaModel, Qwen2VLModel and HFChatVLM pin the names their caches were written under, so splitting or renaming the wrapper files cannot change a key. --- models/base.py | 6 +++++- models/hf_chat.py | 2 ++ models/paligemma.py | 2 ++ models/qwen2vl.py | 2 ++ predict/backends.py | 4 +++- 5 files changed, 14 insertions(+), 2 deletions(-) diff --git a/models/base.py b/models/base.py index aeba927..2a851dc 100644 --- a/models/base.py +++ b/models/base.py @@ -7,7 +7,7 @@ """ from abc import ABC, abstractmethod -from typing import Any, Dict, Optional, Union, Tuple, Literal +from typing import Any, ClassVar, Dict, Optional, Union, Tuple, Literal from pathlib import Path from PIL import Image import logging @@ -40,6 +40,10 @@ class BaseModel(ABC): _logger (logging.Logger): Logger instance for the model """ + # Model name in every prediction cache key. Pinned per family so moving or + # renaming a class never changes keys; None falls back to the class name. + cache_name: ClassVar[Optional[str]] = None + def __init__( self, model_name: str, diff --git a/models/hf_chat.py b/models/hf_chat.py index b59c19c..0adf446 100644 --- a/models/hf_chat.py +++ b/models/hf_chat.py @@ -35,6 +35,8 @@ class HFChatVLM(BaseModel): torch_dtype: Weight dtype (default: bfloat16). """ + cache_name = "HFChatVLM" + def __init__( self, model_name: str, diff --git a/models/paligemma.py b/models/paligemma.py index c70a73d..4c08555 100644 --- a/models/paligemma.py +++ b/models/paligemma.py @@ -34,6 +34,8 @@ class PaligemmaModel(BaseModel): device: PyTorch device for model execution """ + cache_name = "PaligemmaModel" + def __init__( self, model_name: str = "google/paligemma-3b-mix-224", diff --git a/models/qwen2vl.py b/models/qwen2vl.py index d1aab7e..f256ccc 100644 --- a/models/qwen2vl.py +++ b/models/qwen2vl.py @@ -35,6 +35,8 @@ class Qwen2VLModel(BaseModel): device: PyTorch device for model execution """ + cache_name = "Qwen2VLModel" + def __init__( self, model_name: str = "Qwen/Qwen2-VL-2B-Instruct", diff --git a/predict/backends.py b/predict/backends.py index f6c9843..928066f 100644 --- a/predict/backends.py +++ b/predict/backends.py @@ -35,7 +35,9 @@ def identity(self) -> Dict[str, Any]: model_id, revision = parse_model_name(self.model.model_name) self._identity = { "provider": "local", - "class_name": type(self.model).__name__, + # Pinned by the model class (BaseModel.cache_name) so the key + # survives moves and renames; plain objects use their class name. + "class_name": getattr(self.model, "cache_name", None) or type(self.model).__name__, "model_id": model_id, "revision": revision, "quantization": getattr(self.model, "quantization", None) or "none", From 2bb65fffba9784e7157d205abd75ca200c9b3eca Mon Sep 17 00:00:00 2001 From: Salim Alkharsa Date: Sat, 17 Oct 2026 03:54:01 +0000 Subject: [PATCH 06/10] One file per model family and a public, torch-free model registry gemma4.py, ministral3.py and aya_vision.py hold thin HFChatVLM subclasses that inherit its cache name, so their keys do not change. predict.factory.REGISTRY (read-only) maps each key to module:Class, its model path variable, default HF id, and mirrors of the cache name and dtype; identity_for(key) computes a model's cache identity from those without importing torch. qwen2vl-2b and qwen2vl-7b get their own rows (MIRAGE_QWEN2VL_{2B,7B}_MODEL_PATH, falling back to MIRAGE_QWEN_MODEL_PATH) so results and verdicts are keyed by size; qwen2vl stays as it was. Every other key, variable and id is unchanged. run.py and demo.py use the public name. --- demo.py | 2 +- models/__init__.py | 3 ++ models/aya_vision.py | 6 +++ models/gemma4.py | 6 +++ models/ministral3.py | 6 +++ predict/factory.py | 88 ++++++++++++++++++++++++++++++++------------ run.py | 2 +- 7 files changed, 87 insertions(+), 26 deletions(-) create mode 100644 models/aya_vision.py create mode 100644 models/gemma4.py create mode 100644 models/ministral3.py diff --git a/demo.py b/demo.py index f6673ff..91400b0 100644 --- a/demo.py +++ b/demo.py @@ -24,7 +24,7 @@ def coco(metric): def main(): if sys.argv[1:]: raise SystemExit("demo.py takes no flags; use run.py (python run.py --help)") - model = build_predictor("paligemma") # any key in predict.factory._REGISTRY + model = build_predictor("paligemma") # any key in predict.factory.REGISTRY model.load_model() # lazy: weights load only on a cache miss pope = POPEMetric() # 6 yes/no questions per image, seed 42 diff --git a/models/__init__.py b/models/__init__.py index f3cebda..da339c3 100644 --- a/models/__init__.py +++ b/models/__init__.py @@ -9,6 +9,9 @@ "PaligemmaModel": "paligemma", "Qwen2VLModel": "qwen2vl", "HFChatVLM": "hf_chat", + "Gemma4": "gemma4", + "Ministral3": "ministral3", + "AyaVision": "aya_vision", } __all__ = list(_MODULES) diff --git a/models/aya_vision.py b/models/aya_vision.py new file mode 100644 index 0000000..8407385 --- /dev/null +++ b/models/aya_vision.py @@ -0,0 +1,6 @@ +"""Aya Vision (CohereLabs/aya-vision-*) through the generic chat wrapper.""" +from .hf_chat import HFChatVLM + + +class AyaVision(HFChatVLM): + """Aya Vision; keys its cache as ``HFChatVLM``.""" diff --git a/models/gemma4.py b/models/gemma4.py new file mode 100644 index 0000000..0951f25 --- /dev/null +++ b/models/gemma4.py @@ -0,0 +1,6 @@ +"""Gemma 4 (google/gemma-4-*-it) through the generic chat wrapper.""" +from .hf_chat import HFChatVLM + + +class Gemma4(HFChatVLM): + """Gemma 4; keys its cache as ``HFChatVLM``.""" diff --git a/models/ministral3.py b/models/ministral3.py new file mode 100644 index 0000000..39033d4 --- /dev/null +++ b/models/ministral3.py @@ -0,0 +1,6 @@ +"""Ministral 3 (mistralai/Ministral-3-*-Instruct-2512-BF16) through the generic chat wrapper.""" +from .hf_chat import HFChatVLM + + +class Ministral3(HFChatVLM): + """Ministral 3; keys its cache as ``HFChatVLM``.""" diff --git a/predict/factory.py b/predict/factory.py index 35964cb..46a6d82 100644 --- a/predict/factory.py +++ b/predict/factory.py @@ -1,4 +1,4 @@ -"""Factory helpers replacing the per-driver model-selection blocks. +"""Model registry and the factory helpers drivers use to build models. Model classes are imported lazily inside :func:`build_model` so this module (and the whole ``predict`` package) stays importable without torch. @@ -6,42 +6,82 @@ import importlib import os -from typing import Any, Optional +from types import MappingProxyType +from typing import Any, Mapping, NamedTuple, Optional, Tuple, Union from predict.backends import LocalModelBackend from predict.cache import SQLiteResponseCache +from predict.identity import parse_model_name from predict.predictor import Predictor -_REGISTRY = { - "paligemma": ("models.paligemma", "PaligemmaModel", - "MIRAGE_PALIGEMMA_MODEL_PATH", "google/paligemma-3b-mix-224"), - "qwen2vl": ("models.qwen2vl", "Qwen2VLModel", - "MIRAGE_QWEN_MODEL_PATH", "Qwen/Qwen2-VL-2B-Instruct"), - "gemma4-e4b": ("models.hf_chat", "HFChatVLM", - "MIRAGE_GEMMA4_E4B_MODEL_PATH", "google/gemma-4-E4B-it"), - "gemma4-12b": ("models.hf_chat", "HFChatVLM", - "MIRAGE_GEMMA4_12B_MODEL_PATH", "google/gemma-4-12B-it"), - "ministral3-8b": ("models.hf_chat", "HFChatVLM", - "MIRAGE_MINISTRAL3_8B_MODEL_PATH", "mistralai/Ministral-3-8B-Instruct-2512-BF16"), - "ministral3-14b": ("models.hf_chat", "HFChatVLM", - "MIRAGE_MINISTRAL3_14B_MODEL_PATH", "mistralai/Ministral-3-14B-Instruct-2512-BF16"), - "aya-vision-8b": ("models.hf_chat", "HFChatVLM", - "MIRAGE_AYA_VISION_8B_MODEL_PATH", "CohereLabs/aya-vision-8b"), -} + +class Entry(NamedTuple): + target: str # "module:Class" + env: Union[str, Tuple[str, ...]] # model path variable(s), first set one wins + default: str # HF id used when no variable is set + cache_name: str # mirror of the class's cache_name, so identity needs no torch + dtype: str # mirror of str(model.torch_dtype) + + +_BF16 = "torch.bfloat16" +_QWEN = "MIRAGE_QWEN_MODEL_PATH" + +REGISTRY: Mapping[str, Entry] = MappingProxyType({ + "paligemma": Entry("models.paligemma:PaligemmaModel", "MIRAGE_PALIGEMMA_MODEL_PATH", + "google/paligemma-3b-mix-224", "PaligemmaModel", _BF16), + "qwen2vl": Entry("models.qwen2vl:Qwen2VLModel", _QWEN, "Qwen/Qwen2-VL-2B-Instruct", "Qwen2VLModel", _BF16), + "qwen2vl-2b": Entry("models.qwen2vl:Qwen2VLModel", ("MIRAGE_QWEN2VL_2B_MODEL_PATH", _QWEN), + "Qwen/Qwen2-VL-2B-Instruct", "Qwen2VLModel", _BF16), + "qwen2vl-7b": Entry("models.qwen2vl:Qwen2VLModel", ("MIRAGE_QWEN2VL_7B_MODEL_PATH", _QWEN), + "Qwen/Qwen2-VL-7B-Instruct", "Qwen2VLModel", _BF16), + "gemma4-e4b": Entry("models.gemma4:Gemma4", "MIRAGE_GEMMA4_E4B_MODEL_PATH", + "google/gemma-4-E4B-it", "HFChatVLM", _BF16), + "gemma4-12b": Entry("models.gemma4:Gemma4", "MIRAGE_GEMMA4_12B_MODEL_PATH", + "google/gemma-4-12B-it", "HFChatVLM", _BF16), + "ministral3-8b": Entry("models.ministral3:Ministral3", "MIRAGE_MINISTRAL3_8B_MODEL_PATH", + "mistralai/Ministral-3-8B-Instruct-2512-BF16", "HFChatVLM", _BF16), + "ministral3-14b": Entry("models.ministral3:Ministral3", "MIRAGE_MINISTRAL3_14B_MODEL_PATH", + "mistralai/Ministral-3-14B-Instruct-2512-BF16", "HFChatVLM", _BF16), + "aya-vision-8b": Entry("models.aya_vision:AyaVision", "MIRAGE_AYA_VISION_8B_MODEL_PATH", + "CohereLabs/aya-vision-8b", "HFChatVLM", _BF16), +}) def _resolve_key(model_key: Optional[str]) -> str: key = (model_key or os.getenv("MIRAGE_VL_MODEL", "paligemma")).lower() - if key not in _REGISTRY: - raise ValueError(f"unknown model key {key!r}; expected one of {sorted(_REGISTRY)}") + if key not in REGISTRY: + raise ValueError(f"unknown model key {key!r}; expected one of {sorted(REGISTRY)}") return key +def _model_name(entry: Entry, env: Mapping[str, str]) -> str: + for var in (entry.env,) if isinstance(entry.env, str) else entry.env: + if var in env: + return env[var] + return entry.default + + +def identity_for(key: str, env: Mapping[str, str] = os.environ) -> dict: + """Cache identity of a registry model as :func:`build_model` would build it, without torch. + + Args: + key: Registry key. + env: Environment holding the ``MIRAGE_*_MODEL_PATH`` variables. + + Returns: + The record ``LocalModelBackend.identity()`` returns for that model. + """ + entry = REGISTRY[key] + model_id, revision = parse_model_name(_model_name(entry, env)) + return {"provider": "local", "class_name": entry.cache_name, "model_id": model_id, + "revision": revision, "quantization": "none", "dtype": entry.dtype} + + def build_model(model_key: Optional[str] = None, **overrides) -> Any: """Construct the model selected by ``model_key`` or ``$MIRAGE_VL_MODEL``. Args: - model_key: Registry key (see ``_REGISTRY``); None + model_key: Registry key (see ``REGISTRY``); None reads ``MIRAGE_VL_MODEL`` (default ``"paligemma"``). **overrides: Constructor kwargs; ``model_name`` defaults to the model's ``MIRAGE_*_MODEL_PATH`` environment variable. @@ -49,10 +89,10 @@ def build_model(model_key: Optional[str] = None, **overrides) -> Any: Returns: A constructed (not yet loaded) model instance. """ - key = _resolve_key(model_key) - module_name, class_name, env_var, default_name = _REGISTRY[key] + entry = REGISTRY[_resolve_key(model_key)] + module_name, class_name = entry.target.split(":") model_cls = getattr(importlib.import_module(module_name), class_name) - overrides.setdefault("model_name", os.getenv(env_var, default_name)) + overrides.setdefault("model_name", _model_name(entry, os.environ)) return model_cls(**overrides) diff --git a/run.py b/run.py index 36306f0..953c04a 100644 --- a/run.py +++ b/run.py @@ -17,7 +17,7 @@ from metrics.base_metric import CAPTION_GEN, SAMPLES, BaseMetric, caption_images, object_names, pool_objects from metrics.pope import QUESTIONS_PER_IMAGE -from predict.factory import _REGISTRY as MODEL_REGISTRY +from predict.factory import REGISTRY as MODEL_REGISTRY logger = logging.getLogger("mirage.run") DS = {"coco": "coco", "visual_genome": "vg"} # --dataset -> the key metrics use (POOL, CAPTION_PROMPTS) From d3f133b3b8e056a44af78019fbd66166fd4c69a0 Mon Sep 17 00:00:00 2001 From: Salim Alkharsa Date: Sat, 17 Oct 2026 03:54:07 +0000 Subject: [PATCH 07/10] build_predictor accepts a model instance build_predictor(model, key=...) wraps an already constructed model (for tests or a model outside the registry) at the same entry point the CLI uses; key is required because it names the model in results and verdicts. A string or None still selects a registry model as before. cache_only=True opens the default cache read-only. --- predict/factory.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/predict/factory.py b/predict/factory.py index 46a6d82..0f8d95e 100644 --- a/predict/factory.py +++ b/predict/factory.py @@ -96,21 +96,32 @@ def build_model(model_key: Optional[str] = None, **overrides) -> Any: return model_cls(**overrides) -def build_predictor(model_key: Optional[str] = None, +def build_predictor(model: Any = None, key: Optional[str] = None, cache: Optional[SQLiteResponseCache] = None, - **overrides) -> Predictor: - """Construct a cached :class:`Predictor` around :func:`build_model`. + cache_only: bool = False, **overrides) -> Predictor: + """Construct a cached :class:`Predictor` around a registry model or a model instance. Args: - model_key: See :func:`build_model`. + model: Registry key (see :func:`build_model`), None for + ``key``/``$MIRAGE_VL_MODEL``, or a constructed model instance. + key: Registry key when ``model`` is None. Required with an + instance: it names the model in results and verdicts. cache: Cache instance; None uses the default SQLite cache. - **overrides: Forwarded to :func:`build_model`. + cache_only: Open the default cache read-only (ignored when + ``cache`` is given). + **overrides: Forwarded to :func:`build_model` (registry models only). Returns: A Predictor that is a drop-in replacement for the model object; - its ``model_key`` attribute records which registry entry was used. + its ``model_key`` attribute records the key. """ - key = _resolve_key(model_key) - predictor = Predictor(LocalModelBackend(build_model(key, **overrides)), cache=cache) + if model is None or isinstance(model, str): + key = _resolve_key(model or key) + model = build_model(key, **overrides) + elif key is None or overrides: + raise ValueError("a model instance needs key= and takes no constructor overrides") + if cache is None and cache_only: + cache = SQLiteResponseCache(readonly=True) + predictor = Predictor(LocalModelBackend(model), cache=cache) predictor.model_key = key return predictor From 35e64408926f7e77b8edc897097b7fc06e9d1aa9 Mon Sep 17 00:00:00 2001 From: Salim Alkharsa Date: Sat, 17 Oct 2026 04:12:12 +0000 Subject: [PATCH 08/10] Guard the model registry against drift and mismatched keys build_model checks the registry cache_name/dtype mirror against the class, a shared Qwen fallback path must match the row's checkpoint, and build_predictor rejects a model key that disagrees with key=. run.py uses build_predictor(cache_only=...) and builds the FaithScore verifier from the registry. Drop the unused ModelNotLoadedError imports. --- models/paligemma.py | 2 +- models/qwen2vl.py | 2 +- predict/factory.py | 17 +++++++++++++++-- run.py | 7 +++---- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/models/paligemma.py b/models/paligemma.py index 4c08555..2da3441 100644 --- a/models/paligemma.py +++ b/models/paligemma.py @@ -11,7 +11,7 @@ from pathlib import Path from PIL import Image -from .base import BaseModel, ModelLoadError, ModelNotLoadedError, InvalidInputError +from .base import BaseModel, ModelLoadError, InvalidInputError class PaligemmaModel(BaseModel): diff --git a/models/qwen2vl.py b/models/qwen2vl.py index f256ccc..8f24adb 100644 --- a/models/qwen2vl.py +++ b/models/qwen2vl.py @@ -11,7 +11,7 @@ from pathlib import Path from PIL import Image -from .base import BaseModel, ModelLoadError, ModelNotLoadedError, InvalidInputError +from .base import BaseModel, ModelLoadError, InvalidInputError class Qwen2VLModel(BaseModel): diff --git a/predict/factory.py b/predict/factory.py index 0f8d95e..a26b035 100644 --- a/predict/factory.py +++ b/predict/factory.py @@ -55,8 +55,13 @@ def _resolve_key(model_key: Optional[str]) -> str: def _model_name(entry: Entry, env: Mapping[str, str]) -> str: - for var in (entry.env,) if isinstance(entry.env, str) else entry.env: + names = (entry.env,) if isinstance(entry.env, str) else entry.env + for i, var in enumerate(names): if var in env: + # A shared fallback variable (e.g. MIRAGE_QWEN_MODEL_PATH for both Qwen sizes) must + # point at this row's checkpoint; a dedicated variable may name any local dir. + if i and parse_model_name(env[var])[0].rsplit("/", 1)[-1] != entry.default.rsplit("/", 1)[-1]: + raise ValueError(f"${var}={env[var]!r} is not {entry.default}; set ${names[0]} instead") return env[var] return entry.default @@ -92,8 +97,14 @@ def build_model(model_key: Optional[str] = None, **overrides) -> Any: entry = REGISTRY[_resolve_key(model_key)] module_name, class_name = entry.target.split(":") model_cls = getattr(importlib.import_module(module_name), class_name) + # The registry mirror must match the class pin, or identity_for and the cache disagree. + if getattr(model_cls, "cache_name", None) != entry.cache_name: + raise RuntimeError(f"{entry.target}.cache_name != registry cache_name {entry.cache_name!r}") overrides.setdefault("model_name", _model_name(entry, os.environ)) - return model_cls(**overrides) + model = model_cls(**overrides) + if "torch_dtype" not in overrides and str(getattr(model, "torch_dtype", None)) != entry.dtype: + raise RuntimeError(f"{entry.target} dtype {getattr(model, 'torch_dtype', None)} != registry dtype {entry.dtype}") + return model def build_predictor(model: Any = None, key: Optional[str] = None, @@ -116,6 +127,8 @@ def build_predictor(model: Any = None, key: Optional[str] = None, its ``model_key`` attribute records the key. """ if model is None or isinstance(model, str): + if model is not None and key is not None and key.lower() != model.lower(): + raise ValueError(f"model {model!r} and key {key!r} disagree") key = _resolve_key(model or key) model = build_model(key, **overrides) elif key is None or overrides: diff --git a/run.py b/run.py index 953c04a..b7b61ce 100644 --- a/run.py +++ b/run.py @@ -69,14 +69,13 @@ def _load_dataset(args): def _run_dataset_metric(args): - from predict.cache import SQLiteResponseCache from predict.factory import build_predictor loader, all_objects = _load_dataset(args) if not len(loader): raise RuntimeError("No images were loaded. Check the annotation and image paths.") # --cache-only opens the prediction cache read-only: a replay never writes to it, and a miss raises. - model = build_predictor(args.model, cache=SQLiteResponseCache(readonly=True) if args.cache_only else None) + model = build_predictor(args.model, cache_only=args.cache_only) if not args.cache_only: model.load_model() try: @@ -124,11 +123,11 @@ def _evaluate(args, model, loader, all_objects): if not args.llama_model: raise ValueError("FaithScore needs --llama-model or MIRAGE_LLAMA_MODEL_PATH.") - from models.paligemma import PaligemmaModel + from predict.factory import build_model scored, captions = caption_images(loader, model, args.samples, gen) image_paths = [str(s.get("image_path") or loader.images_path / s["image_filename"]) for s in scored] - verifier = PaligemmaModel(model_name=os.getenv("MIRAGE_PALIGEMMA_MODEL_PATH", "google/paligemma-3b-mix-224")) + verifier = build_model("paligemma") verifier.load_model() try: metric = FaithScoreMetric(llama_path=args.llama_model, verification_model=verifier, From 60867481b9fdbc589a077b927dfde4fdbda2081f Mon Sep 17 00:00:00 2001 From: Salim Alkharsa Date: Sat, 17 Oct 2026 07:11:18 +0000 Subject: [PATCH 09/10] run.py: NOPE opens its cache through build_predictor(cache_only=...) --- run.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/run.py b/run.py index 0a75bda..3a182a5 100644 --- a/run.py +++ b/run.py @@ -142,7 +142,6 @@ def _evaluate(args, model, loader, all_objects): def _run_nope(args): from metrics.nope import NOPEMetric - from predict.cache import SQLiteResponseCache from predict.factory import build_predictor if not args.nope_turn2: @@ -151,8 +150,7 @@ def _run_nope(args): metric = NOPEMetric(turn2_path=str(args.nope_turn2), image_dir=str(image_dir), turn1_path=str(args.nope_turn1) if args.nope_turn1 else None) records = metric.records[:args.samples] - model = build_predictor(args.model or "paligemma", - cache=SQLiteResponseCache(readonly=True) if args.cache_only else None) + model = build_predictor(args.model or "paligemma", cache_only=args.cache_only) if not args.cache_only: model.load_model() outputs = [] From 50512e7330f54762c2aa18a779aae30d30ef04e7 Mon Sep 17 00:00:00 2001 From: Salim Alkharsa Date: Sat, 17 Oct 2026 18:22:08 +0000 Subject: [PATCH 10/10] Registry errors say to add or update the row in predict.factory.REGISTRY --- predict/factory.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/predict/factory.py b/predict/factory.py index a26b035..bdcec49 100644 --- a/predict/factory.py +++ b/predict/factory.py @@ -26,6 +26,8 @@ class Entry(NamedTuple): _BF16 = "torch.bfloat16" _QWEN = "MIRAGE_QWEN_MODEL_PATH" +# A new model needs a row here (and, for a new class, a models/__init__.py _MODULES line); +# build_model checks each row against its class. REGISTRY: Mapping[str, Entry] = MappingProxyType({ "paligemma": Entry("models.paligemma:PaligemmaModel", "MIRAGE_PALIGEMMA_MODEL_PATH", "google/paligemma-3b-mix-224", "PaligemmaModel", _BF16), @@ -50,7 +52,8 @@ class Entry(NamedTuple): def _resolve_key(model_key: Optional[str]) -> str: key = (model_key or os.getenv("MIRAGE_VL_MODEL", "paligemma")).lower() if key not in REGISTRY: - raise ValueError(f"unknown model key {key!r}; expected one of {sorted(REGISTRY)}") + raise ValueError(f"unknown model key {key!r}; add a row to predict.factory.REGISTRY " + f"or use one of {sorted(REGISTRY)}") return key @@ -99,11 +102,13 @@ def build_model(model_key: Optional[str] = None, **overrides) -> Any: model_cls = getattr(importlib.import_module(module_name), class_name) # The registry mirror must match the class pin, or identity_for and the cache disagree. if getattr(model_cls, "cache_name", None) != entry.cache_name: - raise RuntimeError(f"{entry.target}.cache_name != registry cache_name {entry.cache_name!r}") + raise RuntimeError(f"{entry.target}.cache_name != registry cache_name {entry.cache_name!r}; " + "update the row in predict.factory.REGISTRY") overrides.setdefault("model_name", _model_name(entry, os.environ)) model = model_cls(**overrides) if "torch_dtype" not in overrides and str(getattr(model, "torch_dtype", None)) != entry.dtype: - raise RuntimeError(f"{entry.target} dtype {getattr(model, 'torch_dtype', None)} != registry dtype {entry.dtype}") + raise RuntimeError(f"{entry.target} dtype {getattr(model, 'torch_dtype', None)} != registry dtype " + f"{entry.dtype}; update the row in predict.factory.REGISTRY") return model