diff --git a/README.md b/README.md index 2db9500..d7acf4e 100644 --- a/README.md +++ b/README.md @@ -25,15 +25,25 @@ Mirage supports `paligemma`, `qwen2vl`, `gemma4-e4b`, `gemma4-12b`, `ministral3- ## Reference results -Scores are percentages from 1,000 images per model and dataset. Higher is better for POPE, ROPE, and THRONE; lower is better for CHAIR-I. +Scores are percentages from 1,000 images per model and dataset (500 for Gemma 4, Ministral 3 and Aya Vision); THRONE F1 uses a FLAN-T5-XL evaluator. Higher is better for POPE, ROPE, and THRONE; lower is better for CHAIR-I. | Dataset | Model | POPE accuracy ↑ | ROPE strict accuracy ↑ | CHAIR-I hallucination rate ↓ | THRONE F1 ↑ | | --- | --- | ---: | ---: | ---: | ---: | -| COCO | PaliGemma-3B | 94.2% | 21.6% | 9.1% | 72.9% | -| COCO | Qwen2-VL-2B | 93.6% | 3.6% | 12.8% | 73.4% | -| COCO | Qwen2-VL-7B | 93.5% | 13.1% | 12.7% | 72.9% | -| Visual Genome | PaliGemma-3B | 88.0% | 8.2% | 67.8% | 21.5% | -| Visual Genome | Qwen2-VL-2B | 88.7% | 8.6% | 75.6% | 24.0% | -| Visual Genome | Qwen2-VL-7B | 89.5% | 10.3% | 69.3% | 23.6% | +| COCO | PaliGemma-3B | 94.2% | 21.6% | 9.1% | 80.9% | +| COCO | Qwen2-VL-2B | 93.6% | 3.7% | 12.8% | 83.8% | +| COCO | Qwen2-VL-7B | 93.5% | 13.8% | 12.7% | 84.4% | +| COCO | Gemma-4-E4B | 93.5% | 30.9% | 12.5% | 77.0% | +| COCO | Ministral-3-8B | 88.1% | 11.0% | 13.7% | 80.8% | +| COCO | Aya-Vision-8B | 95.5% | 25.0% | 16.9% | 82.2% | +| COCO | Gemma-4-12B | 93.0% | 44.5% | 15.2% | 78.4% | +| COCO | Ministral-3-14B | 88.2% | 12.7% | 14.1% | 81.6% | +| Visual Genome | PaliGemma-3B | 88.0% | 8.2% | 67.8% | 27.6% | +| Visual Genome | Qwen2-VL-2B | 88.7% | 8.6% | 75.6% | 27.4% | +| Visual Genome | Qwen2-VL-7B | 89.5% | 11.1% | 69.3% | 27.2% | +| Visual Genome | Gemma-4-E4B | 89.6% | 9.5% | 84.1% | 25.5% | +| Visual Genome | Ministral-3-8B | 84.6% | 3.6% | 87.7% | 26.4% | +| Visual Genome | Aya-Vision-8B | 88.1% | 9.1% | 85.6% | 26.8% | +| Visual Genome | Gemma-4-12B | 90.4% | 21.4% | 76.8% | 27.1% | +| Visual Genome | Ministral-3-14B | 83.9% | 4.2% | 86.6% | 26.6% | The [results CSV](docs/model-results.csv) is the source for these values. diff --git a/docs/index.mdx b/docs/index.mdx index e776909..94c2077 100644 --- a/docs/index.mdx +++ b/docs/index.mdx @@ -49,24 +49,34 @@ Mirage turns dataset examples into model responses, scores those responses, and ## Reference model results -These reference results compare three model configurations across COCO and Visual Genome. Values are percentages from 1,000 images per model and dataset. Higher is better for POPE accuracy, ROPE strict accuracy, and THRONE F1. Lower is better for CHAIR-I hallucination rate. The tables use Mintlify's native tabs and tables so the values remain readable on every screen size. +These reference results compare eight model configurations across COCO and Visual Genome. Values are percentages from 1,000 images per model and dataset (500 for Gemma 4, Ministral 3 and Aya Vision); THRONE F1 uses a FLAN-T5-XL evaluator. Higher is better for POPE accuracy, ROPE strict accuracy, and THRONE F1. Lower is better for CHAIR-I hallucination rate. The tables use Mintlify's native tabs and tables so the values remain readable on every screen size. | Model | POPE accuracy ↑ | ROPE strict accuracy ↑ | CHAIR-I hallucination rate ↓ | THRONE F1 ↑ | | --- | ---: | ---: | ---: | ---: | -| PaliGemma-3B | 94.2% | 21.6% | 9.1% | 72.9% | -| Qwen2-VL-2B | 93.6% | 3.6% | 12.8% | 73.4% | -| Qwen2-VL-7B | 93.5% | 13.1% | 12.7% | 72.9% | +| PaliGemma-3B | 94.2% | 21.6% | 9.1% | 80.9% | +| Qwen2-VL-2B | 93.6% | 3.7% | 12.8% | 83.8% | +| Qwen2-VL-7B | 93.5% | 13.8% | 12.7% | 84.4% | +| Gemma-4-E4B | 93.5% | 30.9% | 12.5% | 77.0% | +| Ministral-3-8B | 88.1% | 11.0% | 13.7% | 80.8% | +| Aya-Vision-8B | 95.5% | 25.0% | 16.9% | 82.2% | +| Gemma-4-12B | 93.0% | 44.5% | 15.2% | 78.4% | +| Ministral-3-14B | 88.2% | 12.7% | 14.1% | 81.6% | | Model | POPE accuracy ↑ | ROPE strict accuracy ↑ | CHAIR-I hallucination rate ↓ | THRONE F1 ↑ | | --- | ---: | ---: | ---: | ---: | -| PaliGemma-3B | 88.0% | 8.2% | 67.8% | 21.5% | -| Qwen2-VL-2B | 88.7% | 8.6% | 75.6% | 24.0% | -| Qwen2-VL-7B | 89.5% | 10.3% | 69.3% | 23.6% | +| PaliGemma-3B | 88.0% | 8.2% | 67.8% | 27.6% | +| Qwen2-VL-2B | 88.7% | 8.6% | 75.6% | 27.4% | +| Qwen2-VL-7B | 89.5% | 11.1% | 69.3% | 27.2% | +| Gemma-4-E4B | 89.6% | 9.5% | 84.1% | 25.5% | +| Ministral-3-8B | 84.6% | 3.6% | 87.7% | 26.4% | +| Aya-Vision-8B | 88.1% | 9.1% | 85.6% | 26.8% | +| Gemma-4-12B | 90.4% | 21.4% | 76.8% | 27.1% | +| Ministral-3-14B | 83.9% | 4.2% | 86.6% | 26.6% | diff --git a/docs/model-results.csv b/docs/model-results.csv index c44d8d0..747eb66 100644 --- a/docs/model-results.csv +++ b/docs/model-results.csv @@ -1,7 +1,17 @@ model,dataset,pope_accuracy,rope_strict_accuracy,chair_instance_hallucination_rate,throne_f1 -PaliGemma-3B,COCO,0.942,0.216,0.091,0.729 -PaliGemma-3B,Visual Genome,0.880,0.082,0.678,0.215 -Qwen2-VL-2B,COCO,0.936,0.036,0.128,0.734 -Qwen2-VL-2B,Visual Genome,0.887,0.086,0.756,0.240 -Qwen2-VL-7B,COCO,0.935,0.131,0.127,0.729 -Qwen2-VL-7B,Visual Genome,0.895,0.103,0.693,0.236 +PaliGemma-3B,COCO,0.942,0.216,0.091,0.809 +PaliGemma-3B,Visual Genome,0.880,0.082,0.678,0.276 +Qwen2-VL-2B,COCO,0.936,0.037,0.128,0.838 +Qwen2-VL-2B,Visual Genome,0.887,0.086,0.756,0.274 +Qwen2-VL-7B,COCO,0.935,0.138,0.127,0.844 +Qwen2-VL-7B,Visual Genome,0.895,0.111,0.693,0.272 +Gemma-4-E4B,COCO,0.935,0.309,0.125,0.770 +Gemma-4-E4B,Visual Genome,0.896,0.095,0.841,0.255 +Ministral-3-8B,COCO,0.881,0.110,0.137,0.808 +Ministral-3-8B,Visual Genome,0.846,0.036,0.877,0.264 +Aya-Vision-8B,COCO,0.955,0.250,0.169,0.822 +Aya-Vision-8B,Visual Genome,0.881,0.091,0.856,0.268 +Gemma-4-12B,COCO,0.930,0.445,0.152,0.784 +Gemma-4-12B,Visual Genome,0.904,0.214,0.768,0.271 +Ministral-3-14B,COCO,0.882,0.127,0.141,0.816 +Ministral-3-14B,Visual Genome,0.839,0.042,0.866,0.266 diff --git a/matching/__init__.py b/matching/__init__.py deleted file mode 100644 index 6777c0f..0000000 --- a/matching/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Pluggable object-name matchers for grading VLM predictions against GT strings. - -This package lives at the top level (not under ``metrics/``) on purpose: -``metrics/__init__`` eagerly imports torch (via throne/faithscore), so anything -that needs to run offline — the ROPE log-rescore, the synonym regrade — must be -importable without torch/PIL/GPU. Nothing here imports torch at module load; -the optional LLM judge lazy-imports it only when actually used. - -Public objects: - Matcher, MatchResult, KEY_FOR, PER_POS_KEY_FOR, normalize (base) - ExactMatcher, SubstringMatcher (exact) - WordNetMatcher (wordnet) - SynonymMatcher, MissingPairError (synonym) - PairCache (cache) - LLMJudge, FlanJudge, DEFAULT_PROMPTS, PROMPT_VERSION (judge, dormant) -""" -from .base import Matcher, MatchResult, KEY_FOR, PER_POS_KEY_FOR, normalize -from .exact import ExactMatcher, SubstringMatcher -from .wordnet import WordNetMatcher -from .synonym import SynonymMatcher, MissingPairError -from .cache import PairCache - -__all__ = [ - "Matcher", "MatchResult", "KEY_FOR", "PER_POS_KEY_FOR", "normalize", - "ExactMatcher", "SubstringMatcher", "WordNetMatcher", - "SynonymMatcher", "MissingPairError", "PairCache", -] diff --git a/matching/base.py b/matching/base.py deleted file mode 100644 index 9f01eb5..0000000 --- a/matching/base.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Matcher protocol, result type, and the name->result-key maps. - -A ``Matcher`` decides whether a model prediction should count as correct for a -given ground-truth class string. Metrics accept a list of matchers and emit one -accuracy column per matcher, keyed by ``KEY_FOR[matcher.name]``. -""" -from __future__ import annotations - -from dataclasses import dataclass -from typing import Protocol, runtime_checkable - - -def normalize(s: str) -> str: - """Canonical form used by every matcher: strip + lowercase.""" - return s.strip().lower() - - -# Overall-accuracy JSON key each named matcher writes. Existing ROPE keys -# (overall_accuracy = lenient headline, overall_accuracy_strict) are preserved -# verbatim so previously-committed results stay byte-identical. -KEY_FOR = { - "strict": "overall_accuracy_strict", - "lenient": "overall_accuracy", - "synonym": "overall_accuracy_synonym", -} - -PER_POS_KEY_FOR = { - "strict": "per_position_accuracy_strict", - "lenient": "per_position_accuracy", - "synonym": "per_position_accuracy_synonym", -} - - -@runtime_checkable -class Matcher(Protocol): - name: str - - def match(self, pred: str, gt: str) -> bool: # pragma: no cover - protocol - ... - - -@dataclass -class MatchResult: - """Outcome of a single match, with the tier that decided it (flip-audit).""" - matched: bool - tier: str - votes: int = 0 diff --git a/matching/cache.py b/matching/cache.py deleted file mode 100644 index 1e7363a..0000000 --- a/matching/cache.py +++ /dev/null @@ -1,57 +0,0 @@ -"""Order-independent verdict cache for the (dormant) LLM judge tier. - -Only the judge path uses this — the default WordNet regrade is deterministic and -needs no cache. Kept so a future ``--build-cache`` run (issue #27) can judge the -residual once on GPU, commit the verdicts, and then re-grade fully offline. - -The cache invalidates itself if the prompt version or judge model changes. -""" -from __future__ import annotations - -import hashlib -import json -from pathlib import Path - - -class PairCache: - def __init__(self, path, prompt_version: str, judge_model: str): - self.path = Path(path) - self.prompt_version = prompt_version - self.judge_model = judge_model - self._store: dict = {} - - @staticmethod - def _norm(s: str) -> str: - return s.strip().lower() - - def _key(self, a: str, b: str) -> str: - x, y = sorted((self._norm(a), self._norm(b))) - return hashlib.sha1((x + "␟" + y).encode("utf-8")).hexdigest() - - def get(self, a: str, b: str): - return self._store.get(self._key(a, b)) - - def put(self, a: str, b: str, verdict: dict) -> None: - self._store[self._key(a, b)] = verdict - - def missing(self, pairs): - return [(a, b) for (a, b) in pairs if self._key(a, b) not in self._store] - - def load(self) -> None: - if not self.path.exists(): - return - data = json.loads(self.path.read_text()) - if (data.get("prompt_version") != self.prompt_version - or data.get("judge_model") != self.judge_model): - self._store = {} # stale -> invalidate - return - self._store = data.get("pairs", {}) - - def save(self) -> None: - self.path.parent.mkdir(parents=True, exist_ok=True) - payload = { - "prompt_version": self.prompt_version, - "judge_model": self.judge_model, - "pairs": self._store, - } - self.path.write_text(json.dumps(payload, indent=2, sort_keys=True)) diff --git a/matching/exact.py b/matching/exact.py deleted file mode 100644 index 1394855..0000000 --- a/matching/exact.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Exact and substring matchers — bodies moved verbatim from metrics/rope.py. - -ExactMatcher == the old ``_rope_match_strict`` (canonical ROPE, exact equality). -SubstringMatcher == the old ``_rope_match`` (lenient whole-word substring). - -Keeping the logic identical is what guarantees the no-regression gate: the -default matcher set [SubstringMatcher, ExactMatcher] reproduces the committed -``overall_accuracy`` / ``overall_accuracy_strict`` byte-for-byte. -""" -from __future__ import annotations - -import re - -from .base import normalize - - -class ExactMatcher: - """Strict ROPE match: exact (case-insensitive) equality.""" - name = "strict" - - def match(self, pred: str, gt: str) -> bool: - return normalize(pred) == normalize(gt) - - -class SubstringMatcher: - """Lenient ROPE match: GT class appears as a whole-word substring of the - prediction (or vice versa). Tolerates "wooden cabinet" vs GT "cabinet".""" - name = "lenient" - - def match(self, pred: str, gt: str) -> bool: - p = normalize(pred) - t = normalize(gt) - if not p or not t: - return False - if p == t: - return True - t_pattern = r"\b" + re.escape(t) + r"\b" - p_pattern = r"\b" + re.escape(p) + r"\b" - return bool(re.search(t_pattern, p)) or bool(re.search(p_pattern, t)) diff --git a/matching/judge.py b/matching/judge.py deleted file mode 100644 index 79c6757..0000000 --- a/matching/judge.py +++ /dev/null @@ -1,100 +0,0 @@ -"""LLM synonym judge — DORMANT tier (not used by the default WordNet regrade). - -Kept present-but-inactive so a stronger judge can be plugged in later without a -redesign. The issue #24 Phase-0 spike found flan-t5-base insufficient; issue #27 -tracks ablating larger judges. Supports both architectures that ablation needs: - - arch="seq2seq" -> AutoModelForSeq2SeqLM (google/flan-t5-base, -xl) - arch="causal" -> AutoModelForCausalLM (meta-llama/Llama-2-7b-chat-hf) - -torch/transformers are lazy-imported inside ``_load`` so importing this module -(and therefore the whole ``matching`` package) stays torch-free. -""" -from __future__ import annotations - -from typing import Dict, List, Optional, Tuple - -DEFAULT_PROMPTS = [ - 'Do the phrases "{a}" and "{b}" refer to the same kind of object? Answer yes or no.', - 'Is a "{a}" the same thing as a "{b}"? Answer yes or no.', - 'Can "{a}" and "{b}" describe the same object in an image? Answer yes or no.', -] -# Bump when prompts/vote logic change so PairCache invalidates stale verdicts. -PROMPT_VERSION = "v1-3prompt-majority-greedy" - - -class LLMJudge: - def __init__( - self, - model_name: str = "google/flan-t5-base", - arch: str = "seq2seq", - prompts: Optional[List[str]] = None, - device: Optional[str] = None, - batch_size: int = 32, - max_new_tokens: int = 3, - ): - if arch not in ("seq2seq", "causal"): - raise ValueError(f"arch must be 'seq2seq' or 'causal', got {arch!r}") - self.model_name = model_name - self.arch = arch - self.prompts = prompts or DEFAULT_PROMPTS - self.device = device - self.batch_size = batch_size - self.max_new_tokens = max_new_tokens - self._model = None - self._tok = None - - def _load(self) -> None: - import torch - from transformers import AutoTokenizer - if self.device is None: - self.device = "cuda" if torch.cuda.is_available() else "cpu" - self._tok = AutoTokenizer.from_pretrained(self.model_name) - dtype = torch.bfloat16 if self.device == "cuda" else torch.float32 - if self.arch == "seq2seq": - from transformers import AutoModelForSeq2SeqLM - self._model = AutoModelForSeq2SeqLM.from_pretrained( - self.model_name, torch_dtype=dtype).to(self.device).eval() - else: - from transformers import AutoModelForCausalLM - if self._tok.pad_token is None: - self._tok.pad_token = self._tok.eos_token - self._model = AutoModelForCausalLM.from_pretrained( - self.model_name, torch_dtype=dtype).to(self.device).eval() - - @staticmethod - def _is_yes(text: str) -> bool: - return text.strip().lower().startswith("y") - - def judge_batch(self, pairs: List[Tuple[str, str]]) -> List[Dict]: - """Return [{"yes": bool, "votes": int}] — majority vote over prompts, greedy.""" - import torch - if self._model is None: - self._load() - flat = [p.format(a=a, b=b) for (a, b) in pairs for p in self.prompts] - outs: List[str] = [] - with torch.no_grad(): - for i in range(0, len(flat), self.batch_size): - batch = flat[i:i + self.batch_size] - enc = self._tok(batch, return_tensors="pt", padding=True, - truncation=True).to(self.device) - gen = self._model.generate(**enc, max_new_tokens=self.max_new_tokens, - do_sample=False) - if self.arch == "causal": - gen = gen[:, enc["input_ids"].shape[1]:] # strip the prompt - outs.extend(self._tok.batch_decode(gen, skip_special_tokens=True)) - np_ = len(self.prompts) - results = [] - for idx in range(len(pairs)): - ans = outs[idx * np_:(idx + 1) * np_] - votes = sum(self._is_yes(a) for a in ans) - results.append({"yes": votes > np_ // 2, "votes": votes}) - return results - - -class FlanJudge(LLMJudge): - """Convenience alias matching the issue #24 naming (seq2seq flan-t5-*).""" - - def __init__(self, model_name: str = "google/flan-t5-base", **kwargs): - kwargs.pop("arch", None) - super().__init__(model_name=model_name, arch="seq2seq", **kwargs) diff --git a/matching/synonym.py b/matching/synonym.py deleted file mode 100644 index a9e20fb..0000000 --- a/matching/synonym.py +++ /dev/null @@ -1,74 +0,0 @@ -"""SynonymMatcher — tiered, add-on matcher for the ``overall_accuracy_synonym`` column. - -Tiers (first hit wins), so ``synonym`` is a strict superset of ``lenient`` and the -``synonym >= lenient`` invariant holds by construction: - - 1. base (SubstringMatcher) — free lenient check - 2. wordnet (WordNetMatcher) — active default tier (offline, deterministic) - 3. judge (LLMJudge|cache) — DORMANT; only consulted if a judge or cache - is provided (issue #27 ablation / --build-cache) - -Default construction — ``SynonymMatcher()`` — uses Substring + WordNet only, which -is exactly the shipped, precision-1.0 config from the Phase-0 spike. -""" -from __future__ import annotations - -from typing import Optional - -from .base import MatchResult, Matcher -from .exact import SubstringMatcher -from .wordnet import WordNetMatcher - - -class MissingPairError(KeyError): - """Offline judge mode: a residual pair is absent from the cache.""" - - -class SynonymMatcher: - name = "synonym" - - def __init__( - self, - base: Optional[Matcher] = None, - wordnet: Optional[WordNetMatcher] = None, - cache=None, - judge=None, - mode: str = "wordnet", - ): - # mode in {"wordnet", "trusting"}. "wordnet" (default) = base+WordNet, - # no LLM. "trusting" additionally consults the judge tier when present. - if mode not in ("wordnet", "trusting"): - raise ValueError(f"unknown mode {mode!r}") - self.base = base if base is not None else SubstringMatcher() - self.wordnet = wordnet if wordnet is not None else WordNetMatcher() - self.cache = cache - self.judge = judge - self.mode = mode - - def match(self, pred: str, gt: str) -> bool: - return self.explain(pred, gt).matched - - def explain(self, pred: str, gt: str) -> MatchResult: - if self.base.match(pred, gt): - return MatchResult(True, "base") - if self.wordnet is not None and self.wordnet.match(pred, gt): - return MatchResult(True, "wordnet") - # Dormant unless a judge or cache was supplied. - if self.mode == "trusting" and (self.judge is not None or self.cache is not None): - verdict = self._judge_verdict(pred, gt) - if verdict is not None and verdict.get("yes"): - return MatchResult(True, "judge", verdict.get("votes", 0)) - return MatchResult(False, "none") - - def _judge_verdict(self, pred: str, gt: str): - if self.cache is not None: - hit = self.cache.get(pred, gt) - if hit is not None: - return hit - if self.judge is not None: # build mode: compute now, cache if we have one - verdict = self.judge.judge_batch([(pred, gt)])[0] - if self.cache is not None: - self.cache.put(pred, gt, verdict) - return verdict - # offline mode: cache present but pair unknown, no judge to fall back on - raise MissingPairError(f"{pred!r} vs {gt!r} not in cache") diff --git a/matching/wordnet.py b/matching/wordnet.py deleted file mode 100644 index a47b635..0000000 --- a/matching/wordnet.py +++ /dev/null @@ -1,65 +0,0 @@ -"""WordNet synonym matcher — the active synonym tier shipped in issue #24. - -Phase-0 spike (tests/data/synonym_gold.jsonl, 200 pairs) measured this at -**precision 1.00 / recall 0.46 / 0 false positives** with ``max_hops=0`` — the -shipped default. It is deterministic, offline, and needs no GPU/SU. - -Algorithm (head-noun, shared-synset): - head = last alphabetic token of each phrase - match if heads are equal (catches "soccer ball" / "sports ball" via "ball"), - else if the two heads share a WordNet NOUN synset (catches bike/bicycle, - rock/rocks, egg/eggs, sofa/couch — NLTK Morphy handles the plural lemmas). - -``max_hops > 0`` additionally walks hypernyms, but that also merges hyponym -pairs the grading protocol wants kept apart (e.g. mug/cup, woman/person), which -measured lower precision (0.857 at hops=1). Left off by default on purpose. -""" -from __future__ import annotations - - -class WordNetMatcher: - name = "wordnet" - - def __init__(self, max_hops: int = 0): - self.max_hops = max_hops - self._wn = None - - def _ensure(self): - """Lazy-load the WordNet corpus. Raises LookupError if not downloaded - (run: python -m nltk.downloader wordnet omw-1.4).""" - if self._wn is None: - from nltk.corpus import wordnet as wn - wn.ensure_loaded() - self._wn = wn - return self._wn - - @staticmethod - def _head(s: str) -> str: - toks = [t for t in "".join(c if c.isalpha() or c == " " else " " - for c in s.lower()).split()] - return toks[-1] if toks else "" - - def _synsets(self, word: str): - wn = self._wn - return set(wn.synsets(word, pos=wn.NOUN)) - - def match(self, pred: str, gt: str) -> bool: - wn = self._ensure() - ha, hb = self._head(pred), self._head(gt) - if not ha or not hb: - return False - if ha == hb: - return True - sa, sb = self._synsets(ha), self._synsets(hb) - if not sa or not sb: - return False - if sa & sb: - return True - if self.max_hops > 0: - for src, dst in ((sa, sb), (sb, sa)): - frontier = set(src) - for _ in range(self.max_hops): - frontier = {h for s in frontier for h in s.hypernyms()} | frontier - if frontier & dst: - return True - return False diff --git a/metrics/base_metric.py b/metrics/base_metric.py index b87cd5b..26ebd5c 100644 --- a/metrics/base_metric.py +++ b/metrics/base_metric.py @@ -1,4 +1,3 @@ -from abc import ABC, abstractmethod from types import MappingProxyType from typing import Dict, List, Any, Mapping, Tuple, Union, Optional import numpy as np @@ -58,12 +57,12 @@ def caption_images(loader, model, n: int = SAMPLES, return samples, captions -class BaseMetric(ABC): +class BaseMetric: """ - Abstract base class for all evaluation metrics in Mirage. + Base class for all evaluation metrics in Mirage. - All metric implementations must inherit from this class and implement - the required abstract methods. + Subclasses override the methods their protocol uses: ``run`` or + ``evaluate``, and ``format_results``. """ POOL = MappingProxyType({"coco": 2000, "vg": 2000}) # candidate images loaded before filtering @@ -78,7 +77,6 @@ def __init__(self, name: str): self.name = name self.results = {} - @abstractmethod def evaluate(self, model_outputs: List[Any], ground_truth: List[Any], **kwargs) -> Dict[str, float]: """ Evaluate model outputs against ground truth. @@ -91,9 +89,8 @@ def evaluate(self, model_outputs: List[Any], ground_truth: List[Any], **kwargs) Returns: Dictionary of metric scores """ - pass + raise NotImplementedError(f"{type(self).__name__} has no evaluate()") - @abstractmethod def format_results(self) -> Dict[str, Any]: """ Format the evaluation results for reporting. @@ -101,7 +98,7 @@ def format_results(self) -> Dict[str, Any]: Returns: Dictionary containing formatted results """ - pass + raise NotImplementedError(f"{type(self).__name__} has no format_results()") def run(self, loader, model, n: int = SAMPLES, gen: Mapping[str, Any] = CAPTION_GEN) -> List[Tuple[Any, str]]: """Caption the first ``n`` annotated images, then evaluate the captions. @@ -127,16 +124,7 @@ def reset(self) -> None: self.results = {} def compute_accuracy(self, predictions: List[bool], labels: List[bool]) -> float: - """ - Compute accuracy from boolean predictions and labels. - - Args: - predictions: List of boolean predictions - labels: List of boolean labels - - Returns: - Accuracy score - """ + """Fraction of ``predictions`` equal to ``labels``; 0.0 when empty.""" if len(predictions) == 0: return 0.0 @@ -144,30 +132,7 @@ def compute_accuracy(self, predictions: List[bool], labels: List[bool]) -> float return correct / len(predictions) def compute_confusion_matrix(self, predictions: List[bool], labels: List[bool]) -> Dict[str, int]: - """ - Compute confusion matrix components from boolean predictions and labels. - - Calculates the four components of a binary confusion matrix: - - True Positives (TP): Correctly predicted positive cases - - True Negatives (TN): Correctly predicted negative cases - - False Positives (FP): Incorrectly predicted positive cases - - False Negatives (FN): Incorrectly predicted negative cases - - Args: - predictions: List of boolean predictions (True = positive, False = negative) - labels: List of boolean ground truth labels (True = positive, False = negative) - - Returns: - Dictionary containing confusion matrix components with keys: - 'true_positives', 'true_negatives', 'false_positives', 'false_negatives' - - Example: - >>> predictions = [True, False, True, False] - >>> labels = [True, False, False, False] - >>> cm = metric.compute_confusion_matrix(predictions, labels) - >>> print(cm) - {'true_positives': 1, 'true_negatives': 2, 'false_positives': 1, 'false_negatives': 0} - """ + """``true_positives``, ``true_negatives``, ``false_positives`` and ``false_negatives`` counts.""" if len(predictions) != len(labels): raise ValueError("Predictions and labels must have the same length") @@ -184,71 +149,19 @@ def compute_confusion_matrix(self, predictions: List[bool], labels: List[bool]) } def compute_precision(self, tp: int, fp: int) -> float: - """ - Compute precision from confusion matrix components. - - Precision measures the proportion of positive predictions that were correct. - Formula: precision = TP / (TP + FP) - - Args: - tp: Number of true positives - fp: Number of false positives - - Returns: - Precision score between 0 and 1. Returns 0.0 if no positive predictions were made. - - Example: - >>> precision = metric.compute_precision(tp=80, fp=20) - >>> print(precision) - 0.8 - """ + """TP / (TP + FP); 0.0 when nothing was predicted positive.""" if tp + fp == 0: return 0.0 return tp / (tp + fp) def compute_recall(self, tp: int, fn: int) -> float: - """ - Compute recall (sensitivity) from confusion matrix components. - - Recall measures the proportion of actual positive cases that were correctly identified. - Formula: recall = TP / (TP + FN) - - Args: - tp: Number of true positives - fn: Number of false negatives - - Returns: - Recall score between 0 and 1. Returns 0.0 if no positive cases exist. - - Example: - >>> recall = metric.compute_recall(tp=80, fn=20) - >>> print(recall) - 0.8 - """ + """TP / (TP + FN); 0.0 when there are no positives.""" if tp + fn == 0: return 0.0 return tp / (tp + fn) def compute_f1(self, precision: float, recall: float) -> float: - """ - Compute F1 score from precision and recall. - - F1 score is the harmonic mean of precision and recall, providing a balanced - measure that accounts for both false positives and false negatives. - Formula: F1 = 2 * (precision * recall) / (precision + recall) - - Args: - precision: Precision score (between 0 and 1) - recall: Recall score (between 0 and 1) - - Returns: - F1 score between 0 and 1. Returns 0.0 if both precision and recall are 0. - - Example: - >>> f1 = metric.compute_f1(precision=0.8, recall=0.9) - >>> print(f1) - 0.8470588235294118 - """ + """Harmonic mean of precision and recall; 0.0 when both are 0.""" if precision + recall == 0: return 0.0 return 2 * (precision * recall) / (precision + recall) diff --git a/metrics/rope.py b/metrics/rope.py index 5096d5c..b9ac212 100644 --- a/metrics/rope.py +++ b/metrics/rope.py @@ -5,10 +5,10 @@ https://github.com/sled-group/moh ROPE evaluates multi-object hallucination in Vision-Language Models by drawing -red bounding boxes around N objects in an image and asking the model to identify -all N objects in a single prompt. This contrasts with POPE's binary yes/no -questioning and tests whether VLMs can correctly ground and classify multiple -objects simultaneously. +red bounding boxes around N objects in an image and asking the model to name +the object in each numbered box, one question per box. This contrasts with +POPE's binary yes/no questioning and tests whether VLMs can correctly ground +and classify multiple objects in the same image. Reference: Zou et al., "Look, Compare, Decide: Alleviating Hallucination in Large @@ -20,29 +20,26 @@ import logging from copy import deepcopy from types import MappingProxyType -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Tuple from PIL import Image, ImageDraw, ImageFont -from .base_metric import SAMPLES, BaseMetric, dataset_key, object_names - -# Matchers live in the top-level ``matching`` package (torch-free) so offline -# rescoring can import them without pulling in metrics/__init__ (torch). Ensure -# the repo root is importable regardless of the current working directory. -import os as _os -import sys as _sys -_ROOT = _os.path.dirname(_os.path.dirname(_os.path.abspath(__file__))) -if _ROOT not in _sys.path: - _sys.path.insert(0, _ROOT) -from matching.exact import ExactMatcher, SubstringMatcher # noqa: E402 -from matching.base import KEY_FOR, PER_POS_KEY_FOR # noqa: E402 +from .base_metric import SAMPLES, BaseMetric logger = logging.getLogger(__name__) -# Shared singletons; the module-level helpers delegate so behavior is identical -# to the previous inline implementations (no-regression on lenient/strict). -_SUBSTRING = SubstringMatcher() -_EXACT = ExactMatcher() + +# parse_single_response normalizes answer formatting before exact matching: +# markdown marks, a leading header or list marker, and surrounding quotes. +_MARKUP_RE = re.compile(r"[*`]+") +_LINE_PREFIX_RE = re.compile(r"^(?:#+\s*|[-\u2022]\s+|\d+[.)]\s+)") +_QUOTES = "\"'\u2018\u2019\u201c\u201d" +# "... is a/an/the X", "... appears to be (a) X", "... is pointing to (the) X"; +# not an echoed question ("I am not sure what is the object inside box 1"). +_ANSWER_RE = re.compile( + r"\b(? bool: @@ -50,27 +47,13 @@ def _rects_overlap(a: Tuple[int, int, int, int], b: Tuple[int, int, int, int]) - return not (a[2] <= b[0] or b[2] <= a[0] or a[3] <= b[1] or b[3] <= a[1]) -def _rope_match(pred: str, truth: str) -> bool: - """Lenient ROPE class-name match (delegates to matching.SubstringMatcher). - - Counts a prediction as correct if the ground-truth class appears as a - whole-word substring of the prediction (or vice versa). - """ - return _SUBSTRING.match(pred, truth) - - -def _rope_match_strict(pred: str, truth: str) -> bool: - """Strict ROPE class-name match (delegates to matching.ExactMatcher).""" - return _EXACT.match(pred, truth) - - class ROPEMetric(BaseMetric): """ Recognition-based Object Probing Evaluation (ROPE) metric. Presents the model with an image annotated with red bounding boxes and asks - it to classify each boxed object from a provided class list in one prompt. - Tracks per-position and overall classification accuracy. + it to name the object in each box, one question per box. Tracks + per-position and overall exact-match accuracy (plain ROPE). Adapted from: https://github.com/sled-group/moh """ @@ -81,32 +64,23 @@ class ROPEMetric(BaseMetric): def __init__( self, num_objects: int = 5, - class_list: Optional[List[str]] = None, bbox_color: str = "red", bbox_width: int = 3, - matchers: Optional[List[Any]] = None, ): """ Initialize the ROPE metric. Args: num_objects: Number of bounding boxes / objects per image. - class_list: List of possible class names to identify from. bbox_color: Color of drawn bounding boxes (default red). bbox_width: Line width for bounding box outlines. - matchers: Optional list of matching.Matcher objects. Defaults to - [SubstringMatcher (lenient), ExactMatcher (strict)] which - reproduces the committed headline keys exactly. Add a - SynonymMatcher to also emit ``overall_accuracy_synonym``. """ super().__init__("rope") self.num_objects = num_objects - self.class_list = class_list or [] self.bbox_color = bbox_color self.bbox_width = bbox_width - self.matchers = matchers or [SubstringMatcher(), ExactMatcher()] - # Accumulated state across evaluate() calls + # Accumulated state across evaluate_predictions() calls self._all_predictions: List[List[str]] = [] self._all_ground_truth: List[List[str]] = [] @@ -118,8 +92,8 @@ def draw_bounding_boxes( """ Draw numbered red bounding boxes on a copy of the image. - Each box is labeled with its 1-based index (1, 2, 3, ...) so the model - can match the visual box number to the obj1/obj2/... output format. + Each box is labeled with its 1-based index (1, 2, 3, ...), matching + the "box i" in the per-box prompts. Adapted from bbox rendering logic in sled-group/moh evaluation pipeline. @@ -243,99 +217,26 @@ def _place_label( return lx, ly return fallback - def generate_prompt_for_box( - self, - box_index: int, - class_list: Optional[List[str]] = None, - ) -> str: + def generate_prompt_for_box(self, box_index: int) -> str: """ - Generate a single-box identification prompt for PaliGemma with few-shot examples. - - Keeps the prompt short and direct. PaliGemma performs better with a - simple, focused question than with long class lists or few-shot examples - that can anchor completions to the example answers. + The single-box ROPE question; its bytes are part of the cache key. Args: box_index: 1-based index of the bounding box to ask about. - class_list: Unused (kept for API compatibility). Returns: Prompt string for a single box. """ return f"What is the object inside box {box_index}?" - def generate_prompt( - self, - class_list: Optional[List[str]] = None, - num_objects: Optional[int] = None, - ) -> str: - """ - Generate the multi-box ROPE prompt (kept for reference / non-PaliGemma models). - - For PaliGemma use generate_prompt_for_box() instead, which queries - one box at a time to match PaliGemma's pretrained completion style. - - Prompt template adapted from sled-group/moh evaluation.py. - - Args: - class_list: Class names the model must choose from. - num_objects: Number of bounding boxes in the image. - - Returns: - Formatted prompt string. - """ - classes = class_list if class_list is not None else self.class_list - n = num_objects if num_objects is not None else self.num_objects - - classes_str = ", ".join(sorted(classes)) - obj_format = ", ".join(f"obj{i+1}: " for i in range(n)) - - return ( - f"There are {n} numbered red bounding boxes in this image, labeled 1 through {n}. " - f"Choose the class of each numbered object from this list: {classes_str}. " - f"Respond using ONLY this exact format with no extra words: {obj_format}." - ) - - def parse_response( - self, - response: str, - num_objects: Optional[int] = None, - ) -> List[str]: - """ - Parse a multi-box model response into per-position class predictions. - - Expects the format: 'obj1: class1, obj2: class2, ...' - Returns a list of length num_objects. Positions that cannot be parsed - are filled with an empty string. - - For single-box responses (from generate_prompt_for_box), use - parse_single_response() instead. - - Args: - response: Raw model output string. - num_objects: Expected number of objects. Falls back to self.num_objects. - - Returns: - List of predicted class names, one per bounding box position. - """ - n = num_objects if num_objects is not None else self.num_objects - predictions = [""] * n - - # Match patterns like "obj1: cat" or "obj1:cat" - matches = re.findall(r"obj(\d+)\s*:\s*([^,\n]+)", response, re.IGNORECASE) - for idx_str, class_name in matches: - idx = int(idx_str) - 1 # convert to 0-based - if 0 <= idx < n: - predictions[idx] = class_name.strip().lower() - - return predictions - def parse_single_response(self, response: str) -> str: """ Parse a single-box model response into a class name. Handles both terse outputs (PaliGemma: "spoon") and verbose - outputs (Qwen: "The object inside box 1 is a wooden cabinet"). + outputs (Qwen: "The object inside box 1 is a wooden cabinet"), with + markdown, quotes, and "appears to be" / "is pointing to" lead-ins + removed ("Based on the image, **box 1** is pointing to a **cup**"). Args: response: Raw model output string for a single-box query. @@ -344,14 +245,19 @@ def parse_single_response(self, response: str) -> str: Predicted class name string. """ text = response.strip().lower().split("\n")[0].strip() - # Extract the noun phrase after "is a/an" (Qwen-style verbose output). - m = re.search(r"\bis\s+an?\s+(.+?)(?:[.,;:]|$)", text) + # Formatting only: drop markdown emphasis/code marks and a leading + # header or list marker. Matching itself stays exact. + text = _LINE_PREFIX_RE.sub("", _MARKUP_RE.sub("", text)).strip() + # Extract the noun phrase after echoed boilerplate ("is a/an/the", + # "appears to be", "is pointing to"); a parenthetical aside ends it. + m = _ANSWER_RE.search(text) if m: phrase = m.group(1).strip() phrase = re.sub(r"\s+(of|on|in|at|with|that|which|for)(\b.*)?$", "", phrase).strip() + phrase = phrase.strip(_QUOTES) if phrase: return phrase - return text.strip(" .,;:()") + return text.strip(" .,;:()" + _QUOTES) def run(self, loader, model, n: int = SAMPLES) -> List[Tuple[Any, int, str, str]]: """Ask about each numbered box of the first ``n`` images that have enough boxes, then evaluate. @@ -363,14 +269,11 @@ def run(self, loader, model, n: int = SAMPLES) -> List[Tuple[Any, int, str, str] ``(image_id, box, response, ground_truth)`` per box (box is 1-based). """ boxes = loader.extract_bboxes_for_rope(num_objects=self.num_objects) - # The VG vocabulary is too large for one prompt: list the classes in the pool's boxes. - classes = object_names(loader) if dataset_key(loader) == "coco" else sorted( - {b["name"] for image_boxes in boxes.values() for b in image_boxes if b["name"]}) predictions, ground_truth, records = [], [], [] for sample in [s for s in loader if str(s["image_id"]) in boxes][:n]: image_boxes = boxes[str(sample["image_id"])] image = self.draw_bounding_boxes(loader.load_image(sample), image_boxes) - responses = [model.generate(image=image, prompt=self.generate_prompt_for_box(i, class_list=classes), + responses = [model.generate(image=image, prompt=self.generate_prompt_for_box(i), **self.GEN) for i in range(1, self.num_objects + 1)] predictions.append([self.parse_single_response(r) for r in responses]) ground_truth.append([b["name"] for b in image_boxes]) @@ -380,6 +283,11 @@ def run(self, loader, model, n: int = SAMPLES) -> List[Tuple[Any, int, str, str] self.evaluate_predictions(predictions, ground_truth) return records + @staticmethod + def strict(pred: str, gt: str) -> bool: + """Plain ROPE: the parsed answer equals the label, ignoring case and surrounding whitespace.""" + return pred.strip().lower() == gt.strip().lower() + def evaluate_predictions( self, predictions: List[List[str]], @@ -387,11 +295,7 @@ def evaluate_predictions( **kwargs, ) -> Dict[str, float]: """ - Evaluate pre-parsed predictions against ground truth. - - Use this when querying boxes one at a time (generate_prompt_for_box + - parse_single_response), where you already have a list of predicted class - names and don't need to re-parse a raw model string. + Score parsed answers (from parse_single_response) against ground truth with :meth:`strict`. Args: predictions: List of predicted class name lists, one per image. @@ -399,7 +303,7 @@ def evaluate_predictions( ground_truth: List of ground truth class name lists, one per image. Returns: - Dict with 'overall_accuracy' and per-position accuracies. + Dict with 'overall_accuracy_strict' for this batch. """ self._all_predictions.extend(predictions) self._all_ground_truth.extend(ground_truth) @@ -409,172 +313,50 @@ def evaluate_predictions( for preds, truths in zip(predictions, ground_truth) for p, t in zip(preds, truths) ] - correct = sum(_rope_match(p, t) for p, t in pairs) - correct_strict = sum(_rope_match_strict(p, t) for p, t in pairs) + correct_strict = sum(self.strict(p, t) for p, t in pairs) total = sum(len(p) for p in predictions) - accuracy = correct / total if total > 0 else 0.0 - accuracy_strict = correct_strict / total if total > 0 else 0.0 - self.results = { - "overall_accuracy": accuracy, - "overall_accuracy_strict": accuracy_strict, - } - return self.results - - def evaluate( - self, - model_outputs: List[str], - ground_truth: List[List[str]], - **kwargs, - ) -> Dict[str, float]: - """ - Evaluate a batch of model responses against ground truth class lists. - - Parses each response string from the multi-box format (obj1: x, obj2: y, ...). - For single-box querying use evaluate_predictions() instead. - - Args: - model_outputs: List of raw model response strings, one per image. - ground_truth: List of ground truth class name lists, one per image. - Each inner list has length num_objects. - - Returns: - Dict with 'overall_accuracy' and per-position accuracies. - """ - batch_predictions = [self.parse_response(r) for r in model_outputs] - - self._all_predictions.extend(batch_predictions) - self._all_ground_truth.extend(ground_truth) - - # Compute batch-level accuracy (lenient headline + strict alongside) - correct = 0 - correct_strict = 0 - total = 0 - for preds, truths in zip(batch_predictions, ground_truth): - for p, t in zip(preds, truths): - if _rope_match(p, t): - correct += 1 - if _rope_match_strict(p, t): - correct_strict += 1 - total += 1 - - accuracy = correct / total if total > 0 else 0.0 - accuracy_strict = correct_strict / total if total > 0 else 0.0 - self.results = { - "overall_accuracy": accuracy, - "overall_accuracy_strict": accuracy_strict, - } + self.results = {"overall_accuracy_strict": correct_strict / total if total > 0 else 0.0} return self.results def format_results(self) -> Dict[str, Any]: """ - Compute and return comprehensive ROPE evaluation results. + Compute and return the plain ROPE results over every evaluated image. Returns: Dict containing: - - overall_accuracy: Accuracy across all positions and images. - - per_position_accuracy: Dict mapping 'obj1' ... 'objN' to accuracy. - - precision: Macro-averaged precision (correct / predicted non-empty). - - recall: Macro-averaged recall (correct / total ground truth). - - f1_score: Harmonic mean of precision and recall. + - overall_accuracy_strict: Exact-match accuracy across all positions and images. + - per_position_accuracy_strict: Dict mapping 'obj1' ... 'objN' to accuracy. - num_images: Number of images evaluated. - num_objects_per_image: Configured num_objects. """ + n = self.num_objects if not self._all_predictions: return { - "overall_accuracy": 0.0, - "per_position_accuracy": {}, - "precision": 0.0, - "recall": 0.0, - "f1_score": 0.0, + "overall_accuracy_strict": 0.0, + "per_position_accuracy_strict": {}, "num_images": 0, - "num_objects_per_image": self.num_objects, + "num_objects_per_image": n, } - n = self.num_objects - per_position_correct = [0] * n per_position_correct_strict = [0] * n per_position_total = [0] * n - total_correct = 0 - total_correct_strict = 0 - total_predicted_nonempty = 0 - total_gt = 0 - for preds, truths in zip(self._all_predictions, self._all_ground_truth): for pos, (p, t) in enumerate(zip(preds, truths)): if pos < n: - match = _rope_match(p, t) - match_strict = _rope_match_strict(p, t) - per_position_correct[pos] += int(match) - per_position_correct_strict[pos] += int(match_strict) + per_position_correct_strict[pos] += int(self.strict(p, t)) per_position_total[pos] += 1 - total_correct += int(match) - total_correct_strict += int(match_strict) - total_gt += 1 - if p.strip(): - total_predicted_nonempty += 1 - - overall_accuracy = total_correct / total_gt if total_gt > 0 else 0.0 - overall_accuracy_strict = total_correct_strict / total_gt if total_gt > 0 else 0.0 - precision = total_correct / total_predicted_nonempty if total_predicted_nonempty > 0 else 0.0 - recall = total_correct / total_gt if total_gt > 0 else 0.0 - f1 = self.compute_f1(precision, recall) - - per_position_accuracy = {} - per_position_accuracy_strict = {} - for pos in range(n): - key = f"obj{pos + 1}" - denom = per_position_total[pos] - per_position_accuracy[key] = ( - per_position_correct[pos] / denom if denom > 0 else 0.0 - ) - per_position_accuracy_strict[key] = ( - per_position_correct_strict[pos] / denom if denom > 0 else 0.0 - ) - - out = { - # Headline metric uses lenient matching (substring-tolerant); - # overall_accuracy_strict reports the canonical exact-match ROPE - # number alongside it. See _rope_match / _rope_match_strict. - "match_mode": "lenient", - "overall_accuracy": overall_accuracy, - "overall_accuracy_strict": overall_accuracy_strict, - "per_position_accuracy": per_position_accuracy, - "per_position_accuracy_strict": per_position_accuracy_strict, - "precision": precision, - "recall": recall, - "f1_score": f1, + total_gt = sum(per_position_total) + + return { + "overall_accuracy_strict": sum(per_position_correct_strict) / total_gt if total_gt > 0 else 0.0, + "per_position_accuracy_strict": { + f"obj{pos + 1}": per_position_correct_strict[pos] / denom if denom > 0 else 0.0 + for pos, denom in enumerate(per_position_total) + }, "num_images": len(self._all_predictions), - "num_objects_per_image": self.num_objects, + "num_objects_per_image": n, } - # Emit one extra accuracy column per additional matcher (e.g. a - # SynonymMatcher). lenient/strict are already reported above, so only - # matchers beyond those add keys; the default set adds nothing, keeping - # the committed output byte-identical. - extra = [m for m in getattr(self, "matchers", []) - if getattr(m, "name", None) not in ("lenient", "strict")] - for m in extra: - m_correct = [0] * n - m_total = [0] * n - m_overall_correct = 0 - m_overall_total = 0 - for preds, truths in zip(self._all_predictions, self._all_ground_truth): - for pos, (p, t) in enumerate(zip(preds, truths)): - if pos < n: - hit = int(m.match(p, t)) - m_correct[pos] += hit - m_total[pos] += 1 - m_overall_correct += hit - m_overall_total += 1 - out[KEY_FOR.get(m.name, f"overall_accuracy_{m.name}")] = ( - m_overall_correct / m_overall_total if m_overall_total else 0.0 - ) - out[PER_POS_KEY_FOR.get(m.name, f"per_position_accuracy_{m.name}")] = { - f"obj{i + 1}": (m_correct[i] / m_total[i] if m_total[i] else 0.0) - for i in range(n) - } - return out - def reset(self) -> None: """Reset accumulated prediction state.""" super().reset() diff --git a/models/__init__.py b/models/__init__.py index da339c3..64e5f26 100644 --- a/models/__init__.py +++ b/models/__init__.py @@ -10,6 +10,7 @@ "Qwen2VLModel": "qwen2vl", "HFChatVLM": "hf_chat", "Gemma4": "gemma4", + "Gemma4Thinking": "gemma4", "Ministral3": "ministral3", "AyaVision": "aya_vision", } diff --git a/models/gemma4.py b/models/gemma4.py index 0951f25..540025b 100644 --- a/models/gemma4.py +++ b/models/gemma4.py @@ -4,3 +4,12 @@ class Gemma4(HFChatVLM): """Gemma 4; keys its cache as ``HFChatVLM``.""" + + +class Gemma4Thinking(Gemma4): + """Gemma 4 with thinking on; a separate class so cache keys never mix modes.""" + + cache_name = "HFChatThinkingVLM" + + def __init__(self, model_name: str, **kwargs): + super().__init__(model_name, enable_thinking=True, **kwargs) diff --git a/models/hf_chat.py b/models/hf_chat.py index 0adf446..acf60be 100644 --- a/models/hf_chat.py +++ b/models/hf_chat.py @@ -31,6 +31,8 @@ class HFChatVLM(BaseModel): Args: model_name: HF model id or local snapshot path. enable_thinking: Forwarded to the chat template; off for evaluation. + thinking_tokens: With thinking on, extra new tokens added to + ``max_new_tokens``; thought and answer share the combined budget. device: Device for inference (default: CUDA when available). torch_dtype: Weight dtype (default: bfloat16). """ @@ -41,6 +43,7 @@ def __init__( self, model_name: str, enable_thinking: bool = False, + thinking_tokens: int = 1024, device: Optional[str] = None, torch_dtype: Optional[torch.dtype] = None, **kwargs, @@ -48,6 +51,7 @@ def __init__( super().__init__(model_name, model_type="local", **kwargs) self.supports_images = True self.enable_thinking = enable_thinking + self.thinking_tokens = thinking_tokens if enable_thinking else 0 self.device = device or ("cuda" if torch.cuda.is_available() else "cpu") self.torch_dtype = torch_dtype or torch.bfloat16 self.model = None @@ -103,7 +107,8 @@ def generate( if torch.is_floating_point(value): inputs[key] = value.to(self.torch_dtype) gen = {**self.config, **kwargs} - gen_kwargs = {"max_new_tokens": gen["max_new_tokens"], "do_sample": bool(gen.get("do_sample"))} + gen_kwargs = {"max_new_tokens": gen["max_new_tokens"] + self.thinking_tokens, + "do_sample": bool(gen.get("do_sample"))} if gen_kwargs["do_sample"]: gen_kwargs.update({k: gen[k] for k in ("temperature", "top_p", "top_k") if gen.get(k) is not None}) with torch.inference_mode(): @@ -116,5 +121,7 @@ def generate( ends = [i for i, token in enumerate(new_tokens) if token in self._thinking_end_ids] if ends: new_tokens = new_tokens[ends[-1] + 1:] + elif self.enable_thinking and len(new_tokens) >= gen_kwargs["max_new_tokens"]: + logger.warning("Thinking block not closed within %d tokens", gen_kwargs["max_new_tokens"]) text = self.processor.decode(new_tokens, skip_special_tokens=True) return _THINKING_RE.sub("", text).strip() diff --git a/predict/factory.py b/predict/factory.py index bdcec49..e467d1d 100644 --- a/predict/factory.py +++ b/predict/factory.py @@ -46,6 +46,11 @@ class Entry(NamedTuple): "mistralai/Ministral-3-14B-Instruct-2512-BF16", "HFChatVLM", _BF16), "aya-vision-8b": Entry("models.aya_vision:AyaVision", "MIRAGE_AYA_VISION_8B_MODEL_PATH", "CohereLabs/aya-vision-8b", "HFChatVLM", _BF16), + # Thinking-on variants for the ablation (same checkpoints). + "gemma4-e4b-think": Entry("models.gemma4:Gemma4Thinking", "MIRAGE_GEMMA4_E4B_MODEL_PATH", + "google/gemma-4-E4B-it", "HFChatThinkingVLM", _BF16), + "gemma4-12b-think": Entry("models.gemma4:Gemma4Thinking", "MIRAGE_GEMMA4_12B_MODEL_PATH", + "google/gemma-4-12B-it", "HFChatThinkingVLM", _BF16), }) diff --git a/predict/identity.py b/predict/identity.py index ac8216d..17aae82 100644 --- a/predict/identity.py +++ b/predict/identity.py @@ -3,8 +3,7 @@ Cache keys are SHA-256 hashes over a sorted-keys JSON payload of the model identity, the input image, the prompt, and the generation config. Bump ``CACHE_KEY_VERSION`` whenever model post-processing or key -semantics change in a way that should invalidate existing cache entries -(same policy as ``matching.judge.PROMPT_VERSION``). +semantics change in a way that should invalidate existing cache entries. """ import hashlib diff --git a/run.py b/run.py index d082797..dce5f2e 100644 --- a/run.py +++ b/run.py @@ -93,7 +93,7 @@ def _evaluate(args, model, loader, all_objects): if args.metric == "rope": from metrics.rope import ROPEMetric - metric = ROPEMetric(num_objects=5, class_list=all_objects) + metric = ROPEMetric(num_objects=5) metric.run(loader, model, n=args.samples) return metric.format_results() diff --git a/tests/test_synonym_match.py b/tests/test_synonym_match.py deleted file mode 100644 index 7338d03..0000000 --- a/tests/test_synonym_match.py +++ /dev/null @@ -1,130 +0,0 @@ -"""Acceptance tests for the synonym-matching layer (issue #24). - -Runs fully offline (WordNet corpus only, no GPU/torch). Encodes the *measured* -Phase-0 behavior of the shipped WordNet hops=0 matcher — including the -deliberate rejections (mug/cup, woman/person) that a hypernym hop would wrongly -merge. See Type1_Experiments/synonym_regrade/phase0_spike.md. - -Run: python3 -m pytest tests/test_synonym_match.py -q - (or plain `python3 tests/test_synonym_match.py` for a no-pytest smoke run) -""" -import sys -import types -from pathlib import Path - -ROOT = Path(__file__).resolve().parent.parent -sys.path.insert(0, str(ROOT)) - -from matching import ExactMatcher, SubstringMatcher, WordNetMatcher, SynonymMatcher - - -def _import_rope_metric(): - # metrics/__init__ imports torch; load the torch-free metrics.rope without it. - if "metrics" not in sys.modules: - pkg = types.ModuleType("metrics") - pkg.__path__ = [str(ROOT / "metrics")] - sys.modules["metrics"] = pkg - from metrics.rope import ROPEMetric - return ROPEMetric - -# Pairs the grader SHOULD merge (true synonyms / number variants / same head noun). -MERGE = [ - ("sofa", "couch"), - ("couch", "sofa"), - ("bicycle", "bike"), - ("small rock", "rocks"), - ("sliced hard-boiled egg", "eggs"), - ("car", "automobile"), - ("soccer ball", "sports ball"), - ("white line", "lines"), -] - -# Pairs the grader must NOT merge. Includes hyponym pairs (mug/cup, woman/person) -# that hops=0 rejects on purpose to keep precision at 1.0. -REJECT = [ - ("cat", "dog"), - ("mug", "cup"), - ("towel", "couch"), - ("monitor", "tv"), - ("woman", "person"), - ("man", "person"), - ("bottle", "wine"), - ("ski boot", "skis"), -] - - -def test_wordnet_merges_true_synonyms(): - wn = WordNetMatcher() # hops=0 - for a, b in MERGE: - assert wn.match(a, b), f"WordNet should merge {a!r}/{b!r}" - - -def test_wordnet_rejects_hard_negatives(): - wn = WordNetMatcher() # hops=0 - for a, b in REJECT: - assert not wn.match(a, b), f"WordNet should NOT merge {a!r}/{b!r}" - - -def test_synonym_matcher_default_is_offline_and_deterministic(): - syn = SynonymMatcher() # base + wordnet, no judge/cache - # Never raises (no MissingPairError) and is repeatable. - for a, b in MERGE + REJECT: - assert syn.match(a, b) == syn.match(a, b) - - -def test_synonym_superset_of_lenient_invariant(): - """overall_accuracy_synonym >= overall_accuracy: synonym matches wherever - lenient does (synonym includes SubstringMatcher as its base tier).""" - sub = SubstringMatcher() - syn = SynonymMatcher() - cases = MERGE + REJECT + [ - ("wooden cabinet", "cabinet"), # lenient True -> synonym must be True - ("red fire hydrant", "hydrant"), - ("person", "person"), - ] - for a, b in cases: - if sub.match(a, b): - assert syn.match(a, b), f"synonym must hold wherever lenient does: {a!r}/{b!r}" - - -def test_exact_and_substring_no_regression(): - """ExactMatcher/SubstringMatcher reproduce the old _rope_match* semantics.""" - ex, sub = ExactMatcher(), SubstringMatcher() - # exact: case-insensitive equality only - assert ex.match("Cabinet", "cabinet") - assert not ex.match("wooden cabinet", "cabinet") - # lenient: whole-word substring either direction, but respects word boundaries - assert sub.match("wooden cabinet", "cabinet") - assert sub.match("cabinet", "wooden cabinet") - assert not sub.match("cabinet", "cabinets") # no word boundary -> not a match - assert not sub.match("", "cabinet") # empty -> False - - -def test_rope_metric_inline_synonym_matcher(): - """ROPEMetric(matchers=[..., SynonymMatcher()]) emits the synonym column - while the lenient/strict headline columns stay untouched.""" - ROPEMetric = _import_rope_metric() - metric = ROPEMetric( - num_objects=2, - matchers=[SubstringMatcher(), ExactMatcher(), SynonymMatcher()], - ) - # obj1: WordNet-only credit; obj2: lenient (substring) credit. - metric.evaluate_predictions([["bike", "wooden cabinet"]], [["bicycle", "cabinet"]]) - results = metric.format_results() - assert results["overall_accuracy_synonym"] == 1.0 - assert results["per_position_accuracy_synonym"] == {"obj1": 1.0, "obj2": 1.0} - assert results["overall_accuracy"] == 0.5 - assert results["per_position_accuracy"] == {"obj1": 0.0, "obj2": 1.0} - assert results["overall_accuracy_strict"] == 0.0 - - -def _run(): - fns = [v for k, v in sorted(globals().items()) if k.startswith("test_")] - for fn in fns: - fn() - print(f"PASS {fn.__name__}") - print(f"\nAll {len(fns)} tests passed.") - - -if __name__ == "__main__": - _run()