From a795e8b596d4933752befce20f531aa498e85bf8 Mon Sep 17 00:00:00 2001 From: Salim Alkharsa Date: Thu, 15 Oct 2026 12:53:54 +0000 Subject: [PATCH 01/12] Add thinking-on Gemma 4 keys for the thinking ablation gemma4-e4b-think and gemma4-12b-think load the same checkpoints through HFChatThinkingVLM, a separate class so cache keys never mix modes. They add a 1,024-token thinking budget on top of each metric's max_new_tokens and cut the thought at the end-of-thinking token before scoring. --- docs/hpc.mdx | 2 +- docs/metrics-and-models.mdx | 3 ++- models/hf_chat_vlm.py | 16 +++++++++++++++- predict/factory.py | 5 +++++ 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/docs/hpc.mdx b/docs/hpc.mdx index 351f1d0..bbae2d0 100644 --- a/docs/hpc.mdx +++ b/docs/hpc.mdx @@ -13,7 +13,7 @@ This guide covers a first GPU run and repeatable model comparisons on a SLURM cl - The model weights already available in the Hugging Face cache or on cluster storage. Nodes without internet access cannot download weights during the job. - A GPU allocation sized for the selected checkpoint and evaluation. The sample SLURM request is a starting point, not a guarantee that a model fits. -The repository exposes `paligemma`, `qwen2vl`, `gemma4-e4b`, `gemma4-12b`, `ministral3-8b`, `ministral3-14b`, and `aya-vision-8b` to `demo.py`. See [Metrics and models](/metrics-and-models) for checkpoint defaults and overrides. The Gemma 4, Ministral 3, and Aya Vision keys were tested on A100 40 GB GPUs with `transformers==5.19.0`, `torch==2.9.1` (CUDA 12.8), and `mistral-common==1.12.0`, as pinned in `requirements.txt`. DeepSeek-VL is not currently implemented as a Mirage model backend, so it cannot be selected with `--model` or `MIRAGE_VL_MODEL`. +The repository exposes `paligemma`, `qwen2vl`, `gemma4-e4b`, `gemma4-12b`, `ministral3-8b`, `ministral3-14b`, and `aya-vision-8b` to `demo.py`, plus `gemma4-e4b-think` and `gemma4-12b-think` for a thinking ablation. See [Metrics and models](/metrics-and-models) for checkpoint defaults and overrides. The Gemma 4, Ministral 3, and Aya Vision keys were tested on A100 40 GB GPUs with `transformers==5.19.0`, `torch==2.9.1` (CUDA 12.8), and `mistral-common==1.12.0`, as pinned in `requirements.txt`. DeepSeek-VL is not currently implemented as a Mirage model backend, so it cannot be selected with `--model` or `MIRAGE_VL_MODEL`. ## Clone and prepare the environment diff --git a/docs/metrics-and-models.mdx b/docs/metrics-and-models.mdx index 1acfb2d..06ea37e 100644 --- a/docs/metrics-and-models.mdx +++ b/docs/metrics-and-models.mdx @@ -31,8 +31,9 @@ The prediction factory supports these local vision-language models: | `ministral3-8b` | `mistralai/Ministral-3-8B-Instruct-2512-BF16` | `MIRAGE_MINISTRAL3_8B_MODEL_PATH` | | `ministral3-14b` | `mistralai/Ministral-3-14B-Instruct-2512-BF16` | `MIRAGE_MINISTRAL3_14B_MODEL_PATH` | | `aya-vision-8b` | `CohereLabs/aya-vision-8b` | `MIRAGE_AYA_VISION_8B_MODEL_PATH` | +| `gemma4-e4b-think`, `gemma4-12b-think` | Same as the Gemma 4 keys, thinking on | Same as the Gemma 4 keys | -The Gemma 4, Ministral 3, and Aya Vision keys share one wrapper, `models/hf_chat_vlm.py`. It sends one image and one text turn through the processor's chat template, decodes only the new tokens, and runs with thinking disabled (`enable_thinking=False`). Aya Vision is a gated, non-commercial (CC-BY-NC) checkpoint; accept its license on Hugging Face before downloading it. +The Gemma 4, Ministral 3, and Aya Vision keys share one wrapper, `models/hf_chat_vlm.py`. It sends one image and one text turn through the processor's chat template, decodes only the new tokens, and runs with thinking disabled (`enable_thinking=False`). The `gemma4-e4b-think` and `gemma4-12b-think` keys load the same Gemma 4 checkpoints with thinking enabled. They add 1,024 tokens to each metric's `max_new_tokens`, shared by the thought and the answer, and drop the thought before scoring. Aya Vision is a gated, non-commercial (CC-BY-NC) checkpoint; accept its license on Hugging Face before downloading it. These are the model keys currently supported by `demo.py`. A different checkpoint can be used through the model-path override only when it is compatible with that wrapper. DeepSeek-VL is not currently supported; adding it requires a model wrapper and factory registration before it can be selected as a separate model key. See [Maintain the docs](/maintaining-docs) for the implementation handoff requirements. diff --git a/models/hf_chat_vlm.py b/models/hf_chat_vlm.py index b7db76f..ab43bcb 100644 --- a/models/hf_chat_vlm.py +++ b/models/hf_chat_vlm.py @@ -31,6 +31,8 @@ class HFChatVLM(BaseModel): Args: model_name: HF model id or local snapshot path. enable_thinking: Forwarded to the chat template; off for evaluation. + thinking_tokens: With thinking on, extra new tokens added to + ``max_new_tokens``; thought and answer share the combined budget. device: Device for inference (default: CUDA when available). torch_dtype: Weight dtype (default: bfloat16). """ @@ -39,6 +41,7 @@ def __init__( self, model_name: str, enable_thinking: bool = False, + thinking_tokens: int = 1024, device: Optional[str] = None, torch_dtype: Optional[torch.dtype] = None, **kwargs, @@ -46,6 +49,7 @@ def __init__( super().__init__(model_name, model_type="local", **kwargs) self.supports_images = True self.enable_thinking = enable_thinking + self.thinking_tokens = thinking_tokens if enable_thinking else 0 self.device = device or ("cuda" if torch.cuda.is_available() else "cpu") self.torch_dtype = torch_dtype or torch.bfloat16 self.model = None @@ -101,7 +105,8 @@ def generate( if torch.is_floating_point(value): inputs[key] = value.to(self.torch_dtype) gen = {**self.config, **kwargs} - gen_kwargs = {"max_new_tokens": gen["max_new_tokens"], "do_sample": bool(gen.get("do_sample"))} + gen_kwargs = {"max_new_tokens": gen["max_new_tokens"] + self.thinking_tokens, + "do_sample": bool(gen.get("do_sample"))} if gen_kwargs["do_sample"]: gen_kwargs.update({k: gen[k] for k in ("temperature", "top_p", "top_k") if gen.get(k) is not None}) with torch.inference_mode(): @@ -114,6 +119,8 @@ def generate( ends = [i for i, token in enumerate(new_tokens) if token in self._thinking_end_ids] if ends: new_tokens = new_tokens[ends[-1] + 1:] + elif self.enable_thinking and len(new_tokens) >= gen_kwargs["max_new_tokens"]: + logger.warning("Thinking block not closed within %d tokens", gen_kwargs["max_new_tokens"]) text = self.processor.decode(new_tokens, skip_special_tokens=True) return _THINKING_RE.sub("", text).strip() @@ -125,3 +132,10 @@ def batch_generate( ) -> List[Optional[str]]: images = images or [None] * len(prompts) return [self.generate(prompt, image=image, **kwargs) for prompt, image in zip(prompts, images)] + + +class HFChatThinkingVLM(HFChatVLM): + """``HFChatVLM`` with thinking on; a separate class so cache keys never mix modes.""" + + def __init__(self, model_name: str, **kwargs): + super().__init__(model_name, enable_thinking=True, **kwargs) diff --git a/predict/factory.py b/predict/factory.py index da06e1e..2fe422f 100644 --- a/predict/factory.py +++ b/predict/factory.py @@ -27,6 +27,11 @@ "MIRAGE_MINISTRAL3_14B_MODEL_PATH", "mistralai/Ministral-3-14B-Instruct-2512-BF16"), "aya-vision-8b": ("models.hf_chat_vlm", "HFChatVLM", "MIRAGE_AYA_VISION_8B_MODEL_PATH", "CohereLabs/aya-vision-8b"), + # Thinking-on variants for the ablation (same checkpoints). + "gemma4-e4b-think": ("models.hf_chat_vlm", "HFChatThinkingVLM", + "MIRAGE_GEMMA4_E4B_MODEL_PATH", "google/gemma-4-E4B-it"), + "gemma4-12b-think": ("models.hf_chat_vlm", "HFChatThinkingVLM", + "MIRAGE_GEMMA4_12B_MODEL_PATH", "google/gemma-4-12B-it"), } From 2a9453e49dd398695866cf8ee995bb20d29e2502 Mon Sep 17 00:00:00 2001 From: Salim Alkharsa Date: Thu, 15 Oct 2026 12:53:54 +0000 Subject: [PATCH 02/12] Report Gemma 4, Ministral 3 and Aya Vision results; rescore THRONE with FLAN-T5-XL Add COCO and Visual Genome rows for gemma4-e4b, gemma4-12b, ministral3-8b, ministral3-14b and aya-vision-8b (500 images each). THRONE F1 for every row now uses a FLAN-T5-XL evaluator with 38 distractors per ground-truth object; the PaliGemma and Qwen2-VL values are rescored from their cached 1,000-image captions and supersede the FLAN-T5-base values. --- README.md | 26 ++++++++++++++++++-------- docs/index.mdx | 26 ++++++++++++++++++-------- docs/model-results.csv | 22 ++++++++++++++++------ 3 files changed, 52 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 2db9500..d4a46ac 100644 --- a/README.md +++ b/README.md @@ -21,19 +21,29 @@ Prepare the dataset files and model weights locally, then run: python demo.py --dataset coco --metric chair --model paligemma --samples 5 --output results/chair-coco.json ``` -Mirage supports `paligemma`, `qwen2vl`, `gemma4-e4b`, `gemma4-12b`, `ministral3-8b`, `ministral3-14b`, and `aya-vision-8b`. See the [documentation](https://mirage-f432d87b.mintlify.app) for dataset setup, metrics, model configuration, and HPC instructions. +Mirage supports `paligemma`, `qwen2vl`, `gemma4-e4b`, `gemma4-12b`, `ministral3-8b`, `ministral3-14b`, and `aya-vision-8b`, plus `-think` variants of the Gemma 4 keys for a thinking ablation. See the [documentation](https://mirage-f432d87b.mintlify.app) for dataset setup, metrics, model configuration, and HPC instructions. ## Reference results -Scores are percentages from 1,000 images per model and dataset. Higher is better for POPE, ROPE, and THRONE; lower is better for CHAIR-I. +Scores are percentages. PaliGemma-3B and Qwen2-VL rows use 1,000 images per dataset; Gemma 4, Ministral 3, and Aya Vision rows use 500 images, a prefix of the same candidate pool. THRONE F1 uses a FLAN-T5-XL evaluator with 38 distractors per ground-truth object for every row and supersedes the earlier FLAN-T5-base values. ROPE strict accuracy depends on answer format; see the [documentation](https://mirage-f432d87b.mintlify.app) for notes on individual models. Higher is better for POPE, ROPE, and THRONE; lower is better for CHAIR-I. | Dataset | Model | POPE accuracy ↑ | ROPE strict accuracy ↑ | CHAIR-I hallucination rate ↓ | THRONE F1 ↑ | | --- | --- | ---: | ---: | ---: | ---: | -| COCO | PaliGemma-3B | 94.2% | 21.6% | 9.1% | 72.9% | -| COCO | Qwen2-VL-2B | 93.6% | 3.6% | 12.8% | 73.4% | -| COCO | Qwen2-VL-7B | 93.5% | 13.1% | 12.7% | 72.9% | -| Visual Genome | PaliGemma-3B | 88.0% | 8.2% | 67.8% | 21.5% | -| Visual Genome | Qwen2-VL-2B | 88.7% | 8.6% | 75.6% | 24.0% | -| Visual Genome | Qwen2-VL-7B | 89.5% | 10.3% | 69.3% | 23.6% | +| COCO | PaliGemma-3B | 94.2% | 21.6% | 9.1% | 80.9% | +| COCO | Qwen2-VL-2B | 93.6% | 3.6% | 12.8% | 83.8% | +| COCO | Qwen2-VL-7B | 93.5% | 13.1% | 12.7% | 84.4% | +| COCO | Gemma-4-E4B | 93.5% | 0.0% | 12.5% | 77.0% | +| COCO | Ministral-3-8B | 88.1% | 5.1% | 13.7% | 80.8% | +| COCO | Aya-Vision-8B | 95.5% | 24.5% | 16.9% | 82.2% | +| COCO | Gemma-4-12B | 93.0% | 44.4% | 15.2% | 78.4% | +| COCO | Ministral-3-14B | 88.2% | 3.0% | 14.1% | 81.6% | +| Visual Genome | PaliGemma-3B | 88.0% | 8.2% | 67.8% | 27.6% | +| Visual Genome | Qwen2-VL-2B | 88.7% | 8.6% | 75.6% | 27.4% | +| Visual Genome | Qwen2-VL-7B | 89.5% | 10.3% | 69.3% | 27.2% | +| Visual Genome | Gemma-4-E4B | 89.6% | 0.0% | 84.1% | 25.5% | +| Visual Genome | Ministral-3-8B | 84.6% | 0.3% | 87.7% | 26.4% | +| Visual Genome | Aya-Vision-8B | 88.1% | 8.9% | 85.6% | 26.8% | +| Visual Genome | Gemma-4-12B | 90.4% | 21.1% | 76.8% | 27.1% | +| Visual Genome | Ministral-3-14B | 83.9% | 0.1% | 86.6% | 26.6% | The [results CSV](docs/model-results.csv) is the source for these values. diff --git a/docs/index.mdx b/docs/index.mdx index e776909..cf0aab7 100644 --- a/docs/index.mdx +++ b/docs/index.mdx @@ -49,25 +49,35 @@ Mirage turns dataset examples into model responses, scores those responses, and ## Reference model results -These reference results compare three model configurations across COCO and Visual Genome. Values are percentages from 1,000 images per model and dataset. Higher is better for POPE accuracy, ROPE strict accuracy, and THRONE F1. Lower is better for CHAIR-I hallucination rate. The tables use Mintlify's native tabs and tables so the values remain readable on every screen size. +These reference results compare eight model configurations across COCO and Visual Genome. Values are percentages. PaliGemma-3B and Qwen2-VL rows use 1,000 images per dataset; Gemma 4, Ministral 3, and Aya Vision rows use 500 images, a prefix of the same fixed candidate pool. Higher is better for POPE accuracy, ROPE strict accuracy, and THRONE F1. Lower is better for CHAIR-I hallucination rate. The tables use Mintlify's native tabs and tables so the values remain readable on every screen size. | Model | POPE accuracy ↑ | ROPE strict accuracy ↑ | CHAIR-I hallucination rate ↓ | THRONE F1 ↑ | | --- | ---: | ---: | ---: | ---: | -| PaliGemma-3B | 94.2% | 21.6% | 9.1% | 72.9% | -| Qwen2-VL-2B | 93.6% | 3.6% | 12.8% | 73.4% | -| Qwen2-VL-7B | 93.5% | 13.1% | 12.7% | 72.9% | +| PaliGemma-3B | 94.2% | 21.6% | 9.1% | 80.9% | +| Qwen2-VL-2B | 93.6% | 3.6% | 12.8% | 83.8% | +| Qwen2-VL-7B | 93.5% | 13.1% | 12.7% | 84.4% | +| Gemma-4-E4B | 93.5% | 0.0% | 12.5% | 77.0% | +| Ministral-3-8B | 88.1% | 5.1% | 13.7% | 80.8% | +| Aya-Vision-8B | 95.5% | 24.5% | 16.9% | 82.2% | +| Gemma-4-12B | 93.0% | 44.4% | 15.2% | 78.4% | +| Ministral-3-14B | 88.2% | 3.0% | 14.1% | 81.6% | | Model | POPE accuracy ↑ | ROPE strict accuracy ↑ | CHAIR-I hallucination rate ↓ | THRONE F1 ↑ | | --- | ---: | ---: | ---: | ---: | -| PaliGemma-3B | 88.0% | 8.2% | 67.8% | 21.5% | -| Qwen2-VL-2B | 88.7% | 8.6% | 75.6% | 24.0% | -| Qwen2-VL-7B | 89.5% | 10.3% | 69.3% | 23.6% | +| PaliGemma-3B | 88.0% | 8.2% | 67.8% | 27.6% | +| Qwen2-VL-2B | 88.7% | 8.6% | 75.6% | 27.4% | +| Qwen2-VL-7B | 89.5% | 10.3% | 69.3% | 27.2% | +| Gemma-4-E4B | 89.6% | 0.0% | 84.1% | 25.5% | +| Ministral-3-8B | 84.6% | 0.3% | 87.7% | 26.4% | +| Aya-Vision-8B | 88.1% | 8.9% | 85.6% | 26.8% | +| Gemma-4-12B | 90.4% | 21.1% | 76.8% | 27.1% | +| Ministral-3-14B | 83.9% | 0.1% | 86.6% | 26.6% | -ROPE strict accuracy requires an exact label match. THRONE F1 is normalized to 38 distractor categories per ground-truth object. Metrics measure different behaviors and should be compared within each metric, not combined into a single model ranking. The source values are in the [model results CSV](https://github.com/Advanced-Vision-and-Learning-Lab/mirage/blob/main/docs/model-results.csv). A new model belongs in these tables only after its results have been measured and reviewed. +ROPE strict accuracy requires the extracted answer to match the label exactly, so it also reflects answer format. Gemma-4-E4B answers in markdown ("Based on the image, **box 1** is pointing to…"), which the strict extractor cannot parse, and Ministral 3 answers with long descriptive phrases; their lenient ROPE accuracy is 32–40% on COCO. Ministral 3 answers POPE with "No" about twice as often as "Yes" on balanced questions, which lowers its POPE accuracy. THRONE F1 uses a FLAN-T5-XL evaluator and 38 distractor categories per ground-truth object for every row (1,000 images for the PaliGemma-3B and Qwen2-VL rows, rescored from their cached captions); it supersedes the earlier FLAN-T5-base THRONE values. To reproduce it, pass `--throne-evaluator google/flan-t5-xl --throne-distractor-ratio 38`. Metrics measure different behaviors and should be compared within each metric, not combined into a single model ranking. The source values are in the [model results CSV](https://github.com/Advanced-Vision-and-Learning-Lab/mirage/blob/main/docs/model-results.csv). A new model belongs in these tables only after its results have been measured and reviewed. diff --git a/docs/model-results.csv b/docs/model-results.csv index c44d8d0..86ef37c 100644 --- a/docs/model-results.csv +++ b/docs/model-results.csv @@ -1,7 +1,17 @@ model,dataset,pope_accuracy,rope_strict_accuracy,chair_instance_hallucination_rate,throne_f1 -PaliGemma-3B,COCO,0.942,0.216,0.091,0.729 -PaliGemma-3B,Visual Genome,0.880,0.082,0.678,0.215 -Qwen2-VL-2B,COCO,0.936,0.036,0.128,0.734 -Qwen2-VL-2B,Visual Genome,0.887,0.086,0.756,0.240 -Qwen2-VL-7B,COCO,0.935,0.131,0.127,0.729 -Qwen2-VL-7B,Visual Genome,0.895,0.103,0.693,0.236 +PaliGemma-3B,COCO,0.942,0.216,0.091,0.809 +PaliGemma-3B,Visual Genome,0.880,0.082,0.678,0.276 +Qwen2-VL-2B,COCO,0.936,0.036,0.128,0.838 +Qwen2-VL-2B,Visual Genome,0.887,0.086,0.756,0.274 +Qwen2-VL-7B,COCO,0.935,0.131,0.127,0.844 +Qwen2-VL-7B,Visual Genome,0.895,0.103,0.693,0.272 +Gemma-4-E4B,COCO,0.935,0.000,0.125,0.770 +Gemma-4-E4B,Visual Genome,0.896,0.000,0.841,0.255 +Ministral-3-8B,COCO,0.881,0.051,0.137,0.808 +Ministral-3-8B,Visual Genome,0.846,0.003,0.877,0.264 +Aya-Vision-8B,COCO,0.955,0.245,0.169,0.822 +Aya-Vision-8B,Visual Genome,0.881,0.089,0.856,0.268 +Gemma-4-12B,COCO,0.930,0.444,0.152,0.784 +Gemma-4-12B,Visual Genome,0.904,0.211,0.768,0.271 +Ministral-3-14B,COCO,0.882,0.030,0.141,0.816 +Ministral-3-14B,Visual Genome,0.839,0.001,0.866,0.266 From 3b3c35019d42d83aeaeb97444829ba91f4ee6830 Mon Sep 17 00:00:00 2001 From: Salim Alkharsa Date: Thu, 15 Oct 2026 19:45:56 +0000 Subject: [PATCH 03/12] Document how an unclosed thinking block is scored --- docs/metrics-and-models.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/metrics-and-models.mdx b/docs/metrics-and-models.mdx index 06ea37e..97cd244 100644 --- a/docs/metrics-and-models.mdx +++ b/docs/metrics-and-models.mdx @@ -33,7 +33,7 @@ The prediction factory supports these local vision-language models: | `aya-vision-8b` | `CohereLabs/aya-vision-8b` | `MIRAGE_AYA_VISION_8B_MODEL_PATH` | | `gemma4-e4b-think`, `gemma4-12b-think` | Same as the Gemma 4 keys, thinking on | Same as the Gemma 4 keys | -The Gemma 4, Ministral 3, and Aya Vision keys share one wrapper, `models/hf_chat_vlm.py`. It sends one image and one text turn through the processor's chat template, decodes only the new tokens, and runs with thinking disabled (`enable_thinking=False`). The `gemma4-e4b-think` and `gemma4-12b-think` keys load the same Gemma 4 checkpoints with thinking enabled. They add 1,024 tokens to each metric's `max_new_tokens`, shared by the thought and the answer, and drop the thought before scoring. Aya Vision is a gated, non-commercial (CC-BY-NC) checkpoint; accept its license on Hugging Face before downloading it. +The Gemma 4, Ministral 3, and Aya Vision keys share one wrapper, `models/hf_chat_vlm.py`. It sends one image and one text turn through the processor's chat template, decodes only the new tokens, and runs with thinking disabled (`enable_thinking=False`). The `gemma4-e4b-think` and `gemma4-12b-think` keys load the same Gemma 4 checkpoints with thinking enabled. They add 1,024 tokens to each metric's `max_new_tokens`, shared by the thought and the answer, and drop the thought before scoring. If the thought does not close within that budget, the wrapper logs a warning and returns the truncated thought as the response, and the metric parses it like any other answer. Aya Vision is a gated, non-commercial (CC-BY-NC) checkpoint; accept its license on Hugging Face before downloading it. These are the model keys currently supported by `demo.py`. A different checkpoint can be used through the model-path override only when it is compatible with that wrapper. DeepSeek-VL is not currently supported; adding it requires a model wrapper and factory registration before it can be selected as a separate model key. See [Maintain the docs](/maintaining-docs) for the implementation handoff requirements. From 96a2561431e54ab248e3b3fdb5cf704f5badd758 Mon Sep 17 00:00:00 2001 From: Salim Alkharsa Date: Thu, 15 Oct 2026 20:18:03 +0000 Subject: [PATCH 04/12] Normalize answer formatting in the ROPE single-box extractor Strip markdown emphasis and code marks, a leading header or list marker, and surrounding quotes, and read the answer after "appears to be", "is pointing to" and "is the" as well as "is a/an". A parenthetical aside ends the answer. Matching stays exact: no synonyms and no picking one of several alternatives. Tests use real cached answers from every model. --- metrics/rope.py | 28 +++++++++++-- tests/test_rope_parse.py | 85 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+), 4 deletions(-) create mode 100644 tests/test_rope_parse.py diff --git a/metrics/rope.py b/metrics/rope.py index ea44df8..043f1ba 100644 --- a/metrics/rope.py +++ b/metrics/rope.py @@ -44,6 +44,19 @@ _EXACT = ExactMatcher() +# parse_single_response normalizes answer formatting before exact matching: +# markdown marks, a leading header or list marker, and surrounding quotes. +_MARKUP_RE = re.compile(r"[*`]+") +_LINE_PREFIX_RE = re.compile(r"^(?:#+\s*|[-\u2022]\s+|\d+[.)]\s+)") +_QUOTES = "\"'\u2018\u2019\u201c\u201d" +# "... is a/an/the X", "... appears to be (a) X", "... is pointing to (the) X"; +# not an echoed question ("I am not sure what is the object inside box 1"). +_ANSWER_RE = re.compile( + r"\b(? bool: """True when two (x0, y0, x1, y1) rectangles share any area.""" return not (a[2] <= b[0] or b[2] <= a[0] or a[3] <= b[1] or b[3] <= a[1]) @@ -331,7 +344,9 @@ def parse_single_response(self, response: str) -> str: Parse a single-box model response into a class name. Handles both terse outputs (PaliGemma: "spoon") and verbose - outputs (Qwen: "The object inside box 1 is a wooden cabinet"). + outputs (Qwen: "The object inside box 1 is a wooden cabinet"), with + markdown, quotes, and "appears to be" / "is pointing to" lead-ins + removed ("Based on the image, **box 1** is pointing to a **cup**"). Args: response: Raw model output string for a single-box query. @@ -340,14 +355,19 @@ def parse_single_response(self, response: str) -> str: Predicted class name string. """ text = response.strip().lower().split("\n")[0].strip() - # Extract the noun phrase after "is a/an" (Qwen-style verbose output). - m = re.search(r"\bis\s+an?\s+(.+?)(?:[.,;:]|$)", text) + # Formatting only: drop markdown emphasis/code marks and a leading + # header or list marker. Matching itself stays exact. + text = _LINE_PREFIX_RE.sub("", _MARKUP_RE.sub("", text)).strip() + # Extract the noun phrase after echoed boilerplate ("is a/an/the", + # "appears to be", "is pointing to"); a parenthetical aside ends it. + m = _ANSWER_RE.search(text) if m: phrase = m.group(1).strip() phrase = re.sub(r"\s+(of|on|in|at|with|that|which|for)(\b.*)?$", "", phrase).strip() + phrase = phrase.strip(_QUOTES) if phrase: return phrase - return text.strip(" .,;:()") + return text.strip(" .,;:()" + _QUOTES) def evaluate_predictions( self, diff --git a/tests/test_rope_parse.py b/tests/test_rope_parse.py new file mode 100644 index 0000000..baf100d --- /dev/null +++ b/tests/test_rope_parse.py @@ -0,0 +1,85 @@ +"""ROPE single-box answer extraction: formatting is normalized, matching stays exact. + +Every input below is a real cached model answer. Runs offline (no GPU, no torch). + +Run: python3 -m pytest tests/test_rope_parse.py -q +""" +import sys +import types +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT)) + +# metrics/__init__ imports torch; load the torch-free metrics.rope without it. +if "metrics" not in sys.modules: + pkg = types.ModuleType("metrics") + pkg.__path__ = [str(ROOT / "metrics")] + sys.modules["metrics"] = pkg + +from metrics.rope import ROPEMetric, _rope_match_strict + + +@pytest.fixture(scope="module") +def parse(): + return ROPEMetric(num_objects=5, class_list=["person"]).parse_single_response + + +@pytest.mark.parametrize("response, expected", [ + # Gemma-4-E4B: markdown emphasis and parenthetical asides. + ("The object inside box 4 is a **zebra**.", "zebra"), + ("The object inside box 3 appears to be a **person** (a man).", "person"), + ("The object inside box 3 appears to be a **person (a man)**.", "person"), + ("The object inside box 5 is a **vehicle** (specifically, a light-colored van or small truck).", "vehicle"), + ("The object inside **box 2** is a **boat** (specifically, a tour or excursion boat).", "boat"), + ("The object inside box 1 appears to be a **dining table** with **chairs** around it.", "dining table"), + ("Based on the image, the object inside **box 3** appears to be **durian**.", "durian"), + ("Based on the image, **box 1** is pointing to the **swimming pool**.", "swimming pool"), + # Ministral 3: "appears to be" before a later "It is a ..." sentence. + ("The object inside **box 1** appears to be a **rolling pin** (also known as a dough roller). " + "It is a cylindrical tool typically", "rolling pin"), + ("The object inside **box 1** appears to be a **person**.", "person"), + # Gemma-4-12B and Qwen2-VL: "is the". + ("The object inside box 2 is the sky.", "sky"), + ("The object inside box 5 is the water.", "water"), + # Qwen2-VL-7B: "appears to be" without markdown. + ("The object inside box 4 appears to be a book.", "book"), + # Markdown list, header, code, and quoted answers. + ("- **cup**", "cup"), + ("### person", "person"), + ("`laptop`", "laptop"), + ('The object inside box 1 is a "bus".', "bus"), +]) +def test_formatting_is_normalized(parse, response, expected): + assert parse(response) == expected + + +@pytest.mark.parametrize("response, expected", [ + # Answers that already parsed cleanly parse identically. + ("flour", "flour"), + ("fireplace", "fireplace"), + ("The object inside box 2 is a soccer ball.", "soccer ball"), + ("The object inside box 1 is a wooden cabinet.", "wooden cabinet"), + ("The object inside box 2 is a toaster oven. It is located on the countertop", "toaster oven"), + ("The object inside box 1 is a person wearing a red shirt.", "person wearing a red shirt"), + ("The object inside box 3 is a piece of lettuce.", "piece"), + # Qwen2-VL-7B hedge: the echoed question is not an answer. + ("I am not sure what is the object inside box 1. It can be a suitcase, a box, a bag", + "i am not sure what is the object inside box 1. it can be a suitcase, a box, a bag"), +]) +def test_clean_answers_unchanged(parse, response, expected): + assert parse(response) == expected + + +@pytest.mark.parametrize("response, truth", [ + # Normalization is about format only: no synonyms, no picking one of + # several alternatives, no dropping modifiers. + ("The object inside box 5 is a **tent** or **canopy**.", "tent"), + ("The object inside box 2 is a **black cow**.", "cow"), + ("The object inside box 1 is a piece of **meat** (likely ham).", "meat"), + ("Based on the image, **box 1** contains **bread** or **baked goods**.", "bread"), +]) +def test_strict_stays_exact(parse, response, truth): + assert not _rope_match_strict(parse(response), truth) From b4cfb7bd1d4b462f765442b350f69f7357b6e63c Mon Sep 17 00:00:00 2001 From: Salim Alkharsa Date: Thu, 15 Oct 2026 20:35:02 +0000 Subject: [PATCH 05/12] Rescore ROPE strict for every row with the formatting-robust extractor All 16 rows were rescored from their cached answers (0 cache misses). Gemma-4-E4B rises from 0.0% to 30.9% (COCO) and 9.5% (VG); Ministral 3 gains 3-10 points; Qwen2-VL-7B gains 0.8 points on each dataset from "appears to be" answers. PaliGemma-3B is unchanged. The homepage and README notes now describe the extractor instead of a per-model caveat. --- README.md | 28 ++++++++++++++-------------- docs/index.mdx | 28 ++++++++++++++-------------- docs/metrics-and-models.mdx | 2 +- docs/model-results.csv | 26 +++++++++++++------------- 4 files changed, 42 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index d4a46ac..8dba6c2 100644 --- a/README.md +++ b/README.md @@ -25,25 +25,25 @@ Mirage supports `paligemma`, `qwen2vl`, `gemma4-e4b`, `gemma4-12b`, `ministral3- ## Reference results -Scores are percentages. PaliGemma-3B and Qwen2-VL rows use 1,000 images per dataset; Gemma 4, Ministral 3, and Aya Vision rows use 500 images, a prefix of the same candidate pool. THRONE F1 uses a FLAN-T5-XL evaluator with 38 distractors per ground-truth object for every row and supersedes the earlier FLAN-T5-base values. ROPE strict accuracy depends on answer format; see the [documentation](https://mirage-f432d87b.mintlify.app) for notes on individual models. Higher is better for POPE, ROPE, and THRONE; lower is better for CHAIR-I. +Scores are percentages. PaliGemma-3B and Qwen2-VL rows use 1,000 images per dataset; Gemma 4, Ministral 3, and Aya Vision rows use 500 images, a prefix of the same candidate pool. THRONE F1 uses a FLAN-T5-XL evaluator with 38 distractors per ground-truth object for every row and supersedes the earlier FLAN-T5-base values. ROPE strict accuracy is an exact label match after the extractor removes answer formatting (markdown, quotes, "appears to be" lead-ins); every row is scored this way from cached answers. See the [documentation](https://mirage-f432d87b.mintlify.app) for notes on individual models. Higher is better for POPE, ROPE, and THRONE; lower is better for CHAIR-I. | Dataset | Model | POPE accuracy ↑ | ROPE strict accuracy ↑ | CHAIR-I hallucination rate ↓ | THRONE F1 ↑ | | --- | --- | ---: | ---: | ---: | ---: | | COCO | PaliGemma-3B | 94.2% | 21.6% | 9.1% | 80.9% | -| COCO | Qwen2-VL-2B | 93.6% | 3.6% | 12.8% | 83.8% | -| COCO | Qwen2-VL-7B | 93.5% | 13.1% | 12.7% | 84.4% | -| COCO | Gemma-4-E4B | 93.5% | 0.0% | 12.5% | 77.0% | -| COCO | Ministral-3-8B | 88.1% | 5.1% | 13.7% | 80.8% | -| COCO | Aya-Vision-8B | 95.5% | 24.5% | 16.9% | 82.2% | -| COCO | Gemma-4-12B | 93.0% | 44.4% | 15.2% | 78.4% | -| COCO | Ministral-3-14B | 88.2% | 3.0% | 14.1% | 81.6% | +| COCO | Qwen2-VL-2B | 93.6% | 3.7% | 12.8% | 83.8% | +| COCO | Qwen2-VL-7B | 93.5% | 13.8% | 12.7% | 84.4% | +| COCO | Gemma-4-E4B | 93.5% | 30.9% | 12.5% | 77.0% | +| COCO | Ministral-3-8B | 88.1% | 11.0% | 13.7% | 80.8% | +| COCO | Aya-Vision-8B | 95.5% | 25.0% | 16.9% | 82.2% | +| COCO | Gemma-4-12B | 93.0% | 44.5% | 15.2% | 78.4% | +| COCO | Ministral-3-14B | 88.2% | 12.7% | 14.1% | 81.6% | | Visual Genome | PaliGemma-3B | 88.0% | 8.2% | 67.8% | 27.6% | | Visual Genome | Qwen2-VL-2B | 88.7% | 8.6% | 75.6% | 27.4% | -| Visual Genome | Qwen2-VL-7B | 89.5% | 10.3% | 69.3% | 27.2% | -| Visual Genome | Gemma-4-E4B | 89.6% | 0.0% | 84.1% | 25.5% | -| Visual Genome | Ministral-3-8B | 84.6% | 0.3% | 87.7% | 26.4% | -| Visual Genome | Aya-Vision-8B | 88.1% | 8.9% | 85.6% | 26.8% | -| Visual Genome | Gemma-4-12B | 90.4% | 21.1% | 76.8% | 27.1% | -| Visual Genome | Ministral-3-14B | 83.9% | 0.1% | 86.6% | 26.6% | +| Visual Genome | Qwen2-VL-7B | 89.5% | 11.1% | 69.3% | 27.2% | +| Visual Genome | Gemma-4-E4B | 89.6% | 9.5% | 84.1% | 25.5% | +| Visual Genome | Ministral-3-8B | 84.6% | 3.6% | 87.7% | 26.4% | +| Visual Genome | Aya-Vision-8B | 88.1% | 9.1% | 85.6% | 26.8% | +| Visual Genome | Gemma-4-12B | 90.4% | 21.4% | 76.8% | 27.1% | +| Visual Genome | Ministral-3-14B | 83.9% | 4.2% | 86.6% | 26.6% | The [results CSV](docs/model-results.csv) is the source for these values. diff --git a/docs/index.mdx b/docs/index.mdx index cf0aab7..6e415c3 100644 --- a/docs/index.mdx +++ b/docs/index.mdx @@ -57,13 +57,13 @@ These reference results compare eight model configurations across COCO and Visua | Model | POPE accuracy ↑ | ROPE strict accuracy ↑ | CHAIR-I hallucination rate ↓ | THRONE F1 ↑ | | --- | ---: | ---: | ---: | ---: | | PaliGemma-3B | 94.2% | 21.6% | 9.1% | 80.9% | -| Qwen2-VL-2B | 93.6% | 3.6% | 12.8% | 83.8% | -| Qwen2-VL-7B | 93.5% | 13.1% | 12.7% | 84.4% | -| Gemma-4-E4B | 93.5% | 0.0% | 12.5% | 77.0% | -| Ministral-3-8B | 88.1% | 5.1% | 13.7% | 80.8% | -| Aya-Vision-8B | 95.5% | 24.5% | 16.9% | 82.2% | -| Gemma-4-12B | 93.0% | 44.4% | 15.2% | 78.4% | -| Ministral-3-14B | 88.2% | 3.0% | 14.1% | 81.6% | +| Qwen2-VL-2B | 93.6% | 3.7% | 12.8% | 83.8% | +| Qwen2-VL-7B | 93.5% | 13.8% | 12.7% | 84.4% | +| Gemma-4-E4B | 93.5% | 30.9% | 12.5% | 77.0% | +| Ministral-3-8B | 88.1% | 11.0% | 13.7% | 80.8% | +| Aya-Vision-8B | 95.5% | 25.0% | 16.9% | 82.2% | +| Gemma-4-12B | 93.0% | 44.5% | 15.2% | 78.4% | +| Ministral-3-14B | 88.2% | 12.7% | 14.1% | 81.6% | @@ -71,13 +71,13 @@ These reference results compare eight model configurations across COCO and Visua | --- | ---: | ---: | ---: | ---: | | PaliGemma-3B | 88.0% | 8.2% | 67.8% | 27.6% | | Qwen2-VL-2B | 88.7% | 8.6% | 75.6% | 27.4% | -| Qwen2-VL-7B | 89.5% | 10.3% | 69.3% | 27.2% | -| Gemma-4-E4B | 89.6% | 0.0% | 84.1% | 25.5% | -| Ministral-3-8B | 84.6% | 0.3% | 87.7% | 26.4% | -| Aya-Vision-8B | 88.1% | 8.9% | 85.6% | 26.8% | -| Gemma-4-12B | 90.4% | 21.1% | 76.8% | 27.1% | -| Ministral-3-14B | 83.9% | 0.1% | 86.6% | 26.6% | +| Qwen2-VL-7B | 89.5% | 11.1% | 69.3% | 27.2% | +| Gemma-4-E4B | 89.6% | 9.5% | 84.1% | 25.5% | +| Ministral-3-8B | 84.6% | 3.6% | 87.7% | 26.4% | +| Aya-Vision-8B | 88.1% | 9.1% | 85.6% | 26.8% | +| Gemma-4-12B | 90.4% | 21.4% | 76.8% | 27.1% | +| Ministral-3-14B | 83.9% | 4.2% | 86.6% | 26.6% | -ROPE strict accuracy requires the extracted answer to match the label exactly, so it also reflects answer format. Gemma-4-E4B answers in markdown ("Based on the image, **box 1** is pointing to…"), which the strict extractor cannot parse, and Ministral 3 answers with long descriptive phrases; their lenient ROPE accuracy is 32–40% on COCO. Ministral 3 answers POPE with "No" about twice as often as "Yes" on balanced questions, which lowers its POPE accuracy. THRONE F1 uses a FLAN-T5-XL evaluator and 38 distractor categories per ground-truth object for every row (1,000 images for the PaliGemma-3B and Qwen2-VL rows, rescored from their cached captions); it supersedes the earlier FLAN-T5-base THRONE values. To reproduce it, pass `--throne-evaluator google/flan-t5-xl --throne-distractor-ratio 38`. Metrics measure different behaviors and should be compared within each metric, not combined into a single model ranking. The source values are in the [model results CSV](https://github.com/Advanced-Vision-and-Learning-Lab/mirage/blob/main/docs/model-results.csv). A new model belongs in these tables only after its results have been measured and reviewed. +ROPE strict accuracy requires the extracted answer to match the label exactly. Before matching, the extractor removes answer formatting: markdown, surrounding quotes, a parenthetical aside, and lead-ins such as "appears to be" or "is pointing to". Every row, including PaliGemma-3B and Qwen2-VL, is scored this way from its cached answers. This replaces the earlier extractor, which scored Gemma-4-E4B's markdown answers ("The object inside box 4 is a **zebra**.") as 0.0%; Qwen2-VL-7B gains 0.8 points on each dataset from "appears to be" answers. Answers that name a part or a broader category ("a piece of broccoli", "a vehicle") still count as wrong. Ministral 3 answers POPE with "No" about twice as often as "Yes" on balanced questions, which lowers its POPE accuracy. THRONE F1 uses a FLAN-T5-XL evaluator and 38 distractor categories per ground-truth object for every row (1,000 images for the PaliGemma-3B and Qwen2-VL rows, rescored from their cached captions); it supersedes the earlier FLAN-T5-base THRONE values. To reproduce it, pass `--throne-evaluator google/flan-t5-xl --throne-distractor-ratio 38`. Metrics measure different behaviors and should be compared within each metric, not combined into a single model ranking. The source values are in the [model results CSV](https://github.com/Advanced-Vision-and-Learning-Lab/mirage/blob/main/docs/model-results.csv). A new model belongs in these tables only after its results have been measured and reviewed. diff --git a/docs/metrics-and-models.mdx b/docs/metrics-and-models.mdx index 97cd244..fcbfba5 100644 --- a/docs/metrics-and-models.mdx +++ b/docs/metrics-and-models.mdx @@ -11,7 +11,7 @@ The demo exposes these metric choices: | --- | --- | --- | | CHAIR | Object mentions in generated image captions against annotated objects. | Uses COCO synonym handling for COCO; Visual Genome uses its object names without stopwords, colours, and other non-object strings. | | POPE | Yes/no answers to questions about objects present or absent in an image. | The demo uses random negative-object sampling by default. | -| ROPE | Object recognition for numbered, boxed image regions. | The demo uses five boxes and asks the model to identify each region. | +| ROPE | Object recognition for numbered, boxed image regions. | The demo uses five boxes and asks the model to identify each region. Before the strict exact-label match, the extractor removes markdown, quotes, parenthetical asides, and lead-ins such as "The object inside box 1 appears to be". | | THRONE | Caption object predictions against the dataset's object categories. | Uses the loaded dataset's category or object-name list. Choose the evaluator with `--throne-evaluator` and limit questions to ground-truth objects plus sampled distractors with `--throne-distractor-ratio`. | | FaithScore | Faithfulness checks on generated captions. | Requires a local LLaMA model path via `--llama-model` or `MIRAGE_LLAMA_MODEL_PATH`, plus a PaliGemma verification model. | | NOPE | Scores responses to the NOPE question set. | Requires pre-generated Turn 2 responses via `--nope-turn2`; Turn 1 input is optional. | diff --git a/docs/model-results.csv b/docs/model-results.csv index 86ef37c..747eb66 100644 --- a/docs/model-results.csv +++ b/docs/model-results.csv @@ -1,17 +1,17 @@ model,dataset,pope_accuracy,rope_strict_accuracy,chair_instance_hallucination_rate,throne_f1 PaliGemma-3B,COCO,0.942,0.216,0.091,0.809 PaliGemma-3B,Visual Genome,0.880,0.082,0.678,0.276 -Qwen2-VL-2B,COCO,0.936,0.036,0.128,0.838 +Qwen2-VL-2B,COCO,0.936,0.037,0.128,0.838 Qwen2-VL-2B,Visual Genome,0.887,0.086,0.756,0.274 -Qwen2-VL-7B,COCO,0.935,0.131,0.127,0.844 -Qwen2-VL-7B,Visual Genome,0.895,0.103,0.693,0.272 -Gemma-4-E4B,COCO,0.935,0.000,0.125,0.770 -Gemma-4-E4B,Visual Genome,0.896,0.000,0.841,0.255 -Ministral-3-8B,COCO,0.881,0.051,0.137,0.808 -Ministral-3-8B,Visual Genome,0.846,0.003,0.877,0.264 -Aya-Vision-8B,COCO,0.955,0.245,0.169,0.822 -Aya-Vision-8B,Visual Genome,0.881,0.089,0.856,0.268 -Gemma-4-12B,COCO,0.930,0.444,0.152,0.784 -Gemma-4-12B,Visual Genome,0.904,0.211,0.768,0.271 -Ministral-3-14B,COCO,0.882,0.030,0.141,0.816 -Ministral-3-14B,Visual Genome,0.839,0.001,0.866,0.266 +Qwen2-VL-7B,COCO,0.935,0.138,0.127,0.844 +Qwen2-VL-7B,Visual Genome,0.895,0.111,0.693,0.272 +Gemma-4-E4B,COCO,0.935,0.309,0.125,0.770 +Gemma-4-E4B,Visual Genome,0.896,0.095,0.841,0.255 +Ministral-3-8B,COCO,0.881,0.110,0.137,0.808 +Ministral-3-8B,Visual Genome,0.846,0.036,0.877,0.264 +Aya-Vision-8B,COCO,0.955,0.250,0.169,0.822 +Aya-Vision-8B,Visual Genome,0.881,0.091,0.856,0.268 +Gemma-4-12B,COCO,0.930,0.445,0.152,0.784 +Gemma-4-12B,Visual Genome,0.904,0.214,0.768,0.271 +Ministral-3-14B,COCO,0.882,0.127,0.141,0.816 +Ministral-3-14B,Visual Genome,0.839,0.042,0.866,0.266 From 26f1f5d7481b93c8bcfc3bd5b1649fee00146ade Mon Sep 17 00:00:00 2001 From: Salim Alkharsa Date: Sat, 17 Oct 2026 04:22:19 +0000 Subject: [PATCH 06/12] Make BaseMetric.evaluate and format_results non-abstract They raise NotImplementedError instead, so a metric that scores through its own entry point (ROPE's evaluate_predictions) still constructs. --- metrics/base_metric.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/metrics/base_metric.py b/metrics/base_metric.py index b87cd5b..e9d1b1b 100644 --- a/metrics/base_metric.py +++ b/metrics/base_metric.py @@ -1,4 +1,4 @@ -from abc import ABC, abstractmethod +from abc import ABC from types import MappingProxyType from typing import Dict, List, Any, Mapping, Tuple, Union, Optional import numpy as np @@ -60,10 +60,10 @@ def caption_images(loader, model, n: int = SAMPLES, class BaseMetric(ABC): """ - Abstract base class for all evaluation metrics in Mirage. + Base class for all evaluation metrics in Mirage. - All metric implementations must inherit from this class and implement - the required abstract methods. + Subclasses override the methods their protocol uses: ``run`` or + ``evaluate``, and ``format_results``. """ POOL = MappingProxyType({"coco": 2000, "vg": 2000}) # candidate images loaded before filtering @@ -78,7 +78,6 @@ def __init__(self, name: str): self.name = name self.results = {} - @abstractmethod def evaluate(self, model_outputs: List[Any], ground_truth: List[Any], **kwargs) -> Dict[str, float]: """ Evaluate model outputs against ground truth. @@ -91,9 +90,8 @@ def evaluate(self, model_outputs: List[Any], ground_truth: List[Any], **kwargs) Returns: Dictionary of metric scores """ - pass + raise NotImplementedError(f"{type(self).__name__} has no evaluate()") - @abstractmethod def format_results(self) -> Dict[str, Any]: """ Format the evaluation results for reporting. @@ -101,7 +99,7 @@ def format_results(self) -> Dict[str, Any]: Returns: Dictionary containing formatted results """ - pass + raise NotImplementedError(f"{type(self).__name__} has no format_results()") def run(self, loader, model, n: int = SAMPLES, gen: Mapping[str, Any] = CAPTION_GEN) -> List[Tuple[Any, str]]: """Caption the first ``n`` annotated images, then evaluate the captions. From abb839afac75ee9df9fa56d18a4b30d260d6ca8b Mon Sep 17 00:00:00 2001 From: Salim Alkharsa Date: Sat, 17 Oct 2026 04:23:24 +0000 Subject: [PATCH 07/12] ROPE: plain ROPE is one exact-match comparison ROPEMetric.strict(pred, gt) is pred.strip().lower() == gt.strip().lower(), the body of matching's ExactMatcher. evaluate_predictions and format_results score with it only; the result JSON keeps overall_accuracy_strict, per_position_accuracy_strict, num_images and num_objects_per_image, and drops the lenient fields (match_mode, overall_accuracy, per_position_accuracy, precision, recall, f1_score). Also removed: the matching imports and sys.path hack, the matchers and unused class_list parameters, the multi-box generate_prompt, parse_response and evaluate. Prompts and box drawing are unchanged. --- metrics/rope.py | 298 ++++++------------------------------------------ run.py | 2 +- 2 files changed, 33 insertions(+), 267 deletions(-) diff --git a/metrics/rope.py b/metrics/rope.py index 8d9f4df..2c5b0f7 100644 --- a/metrics/rope.py +++ b/metrics/rope.py @@ -20,30 +20,14 @@ import logging from copy import deepcopy from types import MappingProxyType -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Tuple from PIL import Image, ImageDraw, ImageFont -from .base_metric import SAMPLES, BaseMetric, dataset_key, object_names - -# Matchers live in the top-level ``matching`` package (torch-free) so offline -# rescoring can import them without pulling in metrics/__init__ (torch). Ensure -# the repo root is importable regardless of the current working directory. -import os as _os -import sys as _sys -_ROOT = _os.path.dirname(_os.path.dirname(_os.path.abspath(__file__))) -if _ROOT not in _sys.path: - _sys.path.insert(0, _ROOT) -from matching.exact import ExactMatcher, SubstringMatcher # noqa: E402 -from matching.base import KEY_FOR, PER_POS_KEY_FOR # noqa: E402 +from .base_metric import SAMPLES, BaseMetric logger = logging.getLogger(__name__) -# Shared singletons; the module-level helpers delegate so behavior is identical -# to the previous inline implementations (no-regression on lenient/strict). -_SUBSTRING = SubstringMatcher() -_EXACT = ExactMatcher() - # parse_single_response normalizes answer formatting before exact matching: # markdown marks, a leading header or list marker, and surrounding quotes. @@ -63,27 +47,13 @@ def _rects_overlap(a: Tuple[int, int, int, int], b: Tuple[int, int, int, int]) - return not (a[2] <= b[0] or b[2] <= a[0] or a[3] <= b[1] or b[3] <= a[1]) -def _rope_match(pred: str, truth: str) -> bool: - """Lenient ROPE class-name match (delegates to matching.SubstringMatcher). - - Counts a prediction as correct if the ground-truth class appears as a - whole-word substring of the prediction (or vice versa). - """ - return _SUBSTRING.match(pred, truth) - - -def _rope_match_strict(pred: str, truth: str) -> bool: - """Strict ROPE class-name match (delegates to matching.ExactMatcher).""" - return _EXACT.match(pred, truth) - - class ROPEMetric(BaseMetric): """ Recognition-based Object Probing Evaluation (ROPE) metric. Presents the model with an image annotated with red bounding boxes and asks - it to classify each boxed object from a provided class list in one prompt. - Tracks per-position and overall classification accuracy. + it to name the object in each box, one question per box. Tracks + per-position and overall exact-match accuracy (plain ROPE). Adapted from: https://github.com/sled-group/moh """ @@ -94,30 +64,21 @@ class ROPEMetric(BaseMetric): def __init__( self, num_objects: int = 5, - class_list: Optional[List[str]] = None, bbox_color: str = "red", bbox_width: int = 3, - matchers: Optional[List[Any]] = None, ): """ Initialize the ROPE metric. Args: num_objects: Number of bounding boxes / objects per image. - class_list: List of possible class names to identify from. bbox_color: Color of drawn bounding boxes (default red). bbox_width: Line width for bounding box outlines. - matchers: Optional list of matching.Matcher objects. Defaults to - [SubstringMatcher (lenient), ExactMatcher (strict)] which - reproduces the committed headline keys exactly. Add a - SynonymMatcher to also emit ``overall_accuracy_synonym``. """ super().__init__("rope") self.num_objects = num_objects - self.class_list = class_list or [] self.bbox_color = bbox_color self.bbox_width = bbox_width - self.matchers = matchers or [SubstringMatcher(), ExactMatcher()] # Accumulated state across evaluate() calls self._all_predictions: List[List[str]] = [] @@ -256,11 +217,7 @@ def _place_label( return lx, ly return fallback - def generate_prompt_for_box( - self, - box_index: int, - class_list: Optional[List[str]] = None, - ) -> str: + def generate_prompt_for_box(self, box_index: int) -> str: """ Generate a single-box identification prompt for PaliGemma with few-shot examples. @@ -270,79 +227,12 @@ def generate_prompt_for_box( Args: box_index: 1-based index of the bounding box to ask about. - class_list: Unused (kept for API compatibility). Returns: Prompt string for a single box. """ return f"What is the object inside box {box_index}?" - def generate_prompt( - self, - class_list: Optional[List[str]] = None, - num_objects: Optional[int] = None, - ) -> str: - """ - Generate the multi-box ROPE prompt (kept for reference / non-PaliGemma models). - - For PaliGemma use generate_prompt_for_box() instead, which queries - one box at a time to match PaliGemma's pretrained completion style. - - Prompt template adapted from sled-group/moh evaluation.py. - - Args: - class_list: Class names the model must choose from. - num_objects: Number of bounding boxes in the image. - - Returns: - Formatted prompt string. - """ - classes = class_list if class_list is not None else self.class_list - n = num_objects if num_objects is not None else self.num_objects - - classes_str = ", ".join(sorted(classes)) - obj_format = ", ".join(f"obj{i+1}: " for i in range(n)) - - return ( - f"There are {n} numbered red bounding boxes in this image, labeled 1 through {n}. " - f"Choose the class of each numbered object from this list: {classes_str}. " - f"Respond using ONLY this exact format with no extra words: {obj_format}." - ) - - def parse_response( - self, - response: str, - num_objects: Optional[int] = None, - ) -> List[str]: - """ - Parse a multi-box model response into per-position class predictions. - - Expects the format: 'obj1: class1, obj2: class2, ...' - Returns a list of length num_objects. Positions that cannot be parsed - are filled with an empty string. - - For single-box responses (from generate_prompt_for_box), use - parse_single_response() instead. - - Args: - response: Raw model output string. - num_objects: Expected number of objects. Falls back to self.num_objects. - - Returns: - List of predicted class names, one per bounding box position. - """ - n = num_objects if num_objects is not None else self.num_objects - predictions = [""] * n - - # Match patterns like "obj1: cat" or "obj1:cat" - matches = re.findall(r"obj(\d+)\s*:\s*([^,\n]+)", response, re.IGNORECASE) - for idx_str, class_name in matches: - idx = int(idx_str) - 1 # convert to 0-based - if 0 <= idx < n: - predictions[idx] = class_name.strip().lower() - - return predictions - def parse_single_response(self, response: str) -> str: """ Parse a single-box model response into a class name. @@ -383,14 +273,11 @@ def run(self, loader, model, n: int = SAMPLES) -> List[Tuple[Any, int, str, str] ``(image_id, box, response, ground_truth)`` per box (box is 1-based). """ boxes = loader.extract_bboxes_for_rope(num_objects=self.num_objects) - # The VG vocabulary is too large for one prompt: list the classes in the pool's boxes. - classes = object_names(loader) if dataset_key(loader) == "coco" else sorted( - {b["name"] for image_boxes in boxes.values() for b in image_boxes if b["name"]}) predictions, ground_truth, records = [], [], [] for sample in [s for s in loader if str(s["image_id"]) in boxes][:n]: image_boxes = boxes[str(sample["image_id"])] image = self.draw_bounding_boxes(loader.load_image(sample), image_boxes) - responses = [model.generate(image=image, prompt=self.generate_prompt_for_box(i, class_list=classes), + responses = [model.generate(image=image, prompt=self.generate_prompt_for_box(i), **self.GEN) for i in range(1, self.num_objects + 1)] predictions.append([self.parse_single_response(r) for r in responses]) ground_truth.append([b["name"] for b in image_boxes]) @@ -400,6 +287,11 @@ def run(self, loader, model, n: int = SAMPLES) -> List[Tuple[Any, int, str, str] self.evaluate_predictions(predictions, ground_truth) return records + @staticmethod + def strict(pred: str, gt: str) -> bool: + """Plain ROPE: the parsed answer equals the label, ignoring case and surrounding whitespace.""" + return pred.strip().lower() == gt.strip().lower() + def evaluate_predictions( self, predictions: List[List[str]], @@ -407,11 +299,7 @@ def evaluate_predictions( **kwargs, ) -> Dict[str, float]: """ - Evaluate pre-parsed predictions against ground truth. - - Use this when querying boxes one at a time (generate_prompt_for_box + - parse_single_response), where you already have a list of predicted class - names and don't need to re-parse a raw model string. + Score parsed answers (from parse_single_response) against ground truth with :meth:`strict`. Args: predictions: List of predicted class name lists, one per image. @@ -419,7 +307,7 @@ def evaluate_predictions( ground_truth: List of ground truth class name lists, one per image. Returns: - Dict with 'overall_accuracy' and per-position accuracies. + Dict with 'overall_accuracy_strict' for this batch. """ self._all_predictions.extend(predictions) self._all_ground_truth.extend(ground_truth) @@ -429,172 +317,50 @@ def evaluate_predictions( for preds, truths in zip(predictions, ground_truth) for p, t in zip(preds, truths) ] - correct = sum(_rope_match(p, t) for p, t in pairs) - correct_strict = sum(_rope_match_strict(p, t) for p, t in pairs) + correct_strict = sum(self.strict(p, t) for p, t in pairs) total = sum(len(p) for p in predictions) - accuracy = correct / total if total > 0 else 0.0 - accuracy_strict = correct_strict / total if total > 0 else 0.0 - self.results = { - "overall_accuracy": accuracy, - "overall_accuracy_strict": accuracy_strict, - } - return self.results - - def evaluate( - self, - model_outputs: List[str], - ground_truth: List[List[str]], - **kwargs, - ) -> Dict[str, float]: - """ - Evaluate a batch of model responses against ground truth class lists. - - Parses each response string from the multi-box format (obj1: x, obj2: y, ...). - For single-box querying use evaluate_predictions() instead. - - Args: - model_outputs: List of raw model response strings, one per image. - ground_truth: List of ground truth class name lists, one per image. - Each inner list has length num_objects. - - Returns: - Dict with 'overall_accuracy' and per-position accuracies. - """ - batch_predictions = [self.parse_response(r) for r in model_outputs] - - self._all_predictions.extend(batch_predictions) - self._all_ground_truth.extend(ground_truth) - - # Compute batch-level accuracy (lenient headline + strict alongside) - correct = 0 - correct_strict = 0 - total = 0 - for preds, truths in zip(batch_predictions, ground_truth): - for p, t in zip(preds, truths): - if _rope_match(p, t): - correct += 1 - if _rope_match_strict(p, t): - correct_strict += 1 - total += 1 - - accuracy = correct / total if total > 0 else 0.0 - accuracy_strict = correct_strict / total if total > 0 else 0.0 - self.results = { - "overall_accuracy": accuracy, - "overall_accuracy_strict": accuracy_strict, - } + self.results = {"overall_accuracy_strict": correct_strict / total if total > 0 else 0.0} return self.results def format_results(self) -> Dict[str, Any]: """ - Compute and return comprehensive ROPE evaluation results. + Compute and return the plain ROPE results over every evaluated image. Returns: Dict containing: - - overall_accuracy: Accuracy across all positions and images. - - per_position_accuracy: Dict mapping 'obj1' ... 'objN' to accuracy. - - precision: Macro-averaged precision (correct / predicted non-empty). - - recall: Macro-averaged recall (correct / total ground truth). - - f1_score: Harmonic mean of precision and recall. + - overall_accuracy_strict: Exact-match accuracy across all positions and images. + - per_position_accuracy_strict: Dict mapping 'obj1' ... 'objN' to accuracy. - num_images: Number of images evaluated. - num_objects_per_image: Configured num_objects. """ + n = self.num_objects if not self._all_predictions: return { - "overall_accuracy": 0.0, - "per_position_accuracy": {}, - "precision": 0.0, - "recall": 0.0, - "f1_score": 0.0, + "overall_accuracy_strict": 0.0, + "per_position_accuracy_strict": {}, "num_images": 0, - "num_objects_per_image": self.num_objects, + "num_objects_per_image": n, } - n = self.num_objects - per_position_correct = [0] * n per_position_correct_strict = [0] * n per_position_total = [0] * n - total_correct = 0 - total_correct_strict = 0 - total_predicted_nonempty = 0 - total_gt = 0 - for preds, truths in zip(self._all_predictions, self._all_ground_truth): for pos, (p, t) in enumerate(zip(preds, truths)): if pos < n: - match = _rope_match(p, t) - match_strict = _rope_match_strict(p, t) - per_position_correct[pos] += int(match) - per_position_correct_strict[pos] += int(match_strict) + per_position_correct_strict[pos] += int(self.strict(p, t)) per_position_total[pos] += 1 - total_correct += int(match) - total_correct_strict += int(match_strict) - total_gt += 1 - if p.strip(): - total_predicted_nonempty += 1 - - overall_accuracy = total_correct / total_gt if total_gt > 0 else 0.0 - overall_accuracy_strict = total_correct_strict / total_gt if total_gt > 0 else 0.0 - precision = total_correct / total_predicted_nonempty if total_predicted_nonempty > 0 else 0.0 - recall = total_correct / total_gt if total_gt > 0 else 0.0 - f1 = self.compute_f1(precision, recall) - - per_position_accuracy = {} - per_position_accuracy_strict = {} - for pos in range(n): - key = f"obj{pos + 1}" - denom = per_position_total[pos] - per_position_accuracy[key] = ( - per_position_correct[pos] / denom if denom > 0 else 0.0 - ) - per_position_accuracy_strict[key] = ( - per_position_correct_strict[pos] / denom if denom > 0 else 0.0 - ) - - out = { - # Headline metric uses lenient matching (substring-tolerant); - # overall_accuracy_strict reports the canonical exact-match ROPE - # number alongside it. See _rope_match / _rope_match_strict. - "match_mode": "lenient", - "overall_accuracy": overall_accuracy, - "overall_accuracy_strict": overall_accuracy_strict, - "per_position_accuracy": per_position_accuracy, - "per_position_accuracy_strict": per_position_accuracy_strict, - "precision": precision, - "recall": recall, - "f1_score": f1, + total_gt = sum(per_position_total) + + return { + "overall_accuracy_strict": sum(per_position_correct_strict) / total_gt if total_gt > 0 else 0.0, + "per_position_accuracy_strict": { + f"obj{pos + 1}": per_position_correct_strict[pos] / denom if denom > 0 else 0.0 + for pos, denom in enumerate(per_position_total) + }, "num_images": len(self._all_predictions), - "num_objects_per_image": self.num_objects, + "num_objects_per_image": n, } - # Emit one extra accuracy column per additional matcher (e.g. a - # SynonymMatcher). lenient/strict are already reported above, so only - # matchers beyond those add keys; the default set adds nothing, keeping - # the committed output byte-identical. - extra = [m for m in getattr(self, "matchers", []) - if getattr(m, "name", None) not in ("lenient", "strict")] - for m in extra: - m_correct = [0] * n - m_total = [0] * n - m_overall_correct = 0 - m_overall_total = 0 - for preds, truths in zip(self._all_predictions, self._all_ground_truth): - for pos, (p, t) in enumerate(zip(preds, truths)): - if pos < n: - hit = int(m.match(p, t)) - m_correct[pos] += hit - m_total[pos] += 1 - m_overall_correct += hit - m_overall_total += 1 - out[KEY_FOR.get(m.name, f"overall_accuracy_{m.name}")] = ( - m_overall_correct / m_overall_total if m_overall_total else 0.0 - ) - out[PER_POS_KEY_FOR.get(m.name, f"per_position_accuracy_{m.name}")] = { - f"obj{i + 1}": (m_correct[i] / m_total[i] if m_total[i] else 0.0) - for i in range(n) - } - return out - def reset(self) -> None: """Reset accumulated prediction state.""" super().reset() diff --git a/run.py b/run.py index b7b61ce..2961b28 100644 --- a/run.py +++ b/run.py @@ -98,7 +98,7 @@ def _evaluate(args, model, loader, all_objects): if args.metric == "rope": from metrics.rope import ROPEMetric - metric = ROPEMetric(num_objects=5, class_list=all_objects) + metric = ROPEMetric(num_objects=5) metric.run(loader, model, n=args.samples) return metric.format_results() From 176a7496c93e7fc8f95e16e3426103b7fc45107c Mon Sep 17 00:00:00 2001 From: Salim Alkharsa Date: Sat, 17 Oct 2026 04:23:29 +0000 Subject: [PATCH 08/12] Delete matching/ and its test Nothing imports it any more: the strict check lives in ROPEMetric.strict, and the lenient, synonym, WordNet and judge layers never produced a published number. --- matching/__init__.py | 27 -------- matching/base.py | 47 ------------- matching/cache.py | 57 ---------------- matching/exact.py | 39 ----------- matching/judge.py | 100 --------------------------- matching/synonym.py | 74 -------------------- matching/wordnet.py | 65 ------------------ tests/test_synonym_match.py | 130 ------------------------------------ 8 files changed, 539 deletions(-) delete mode 100644 matching/__init__.py delete mode 100644 matching/base.py delete mode 100644 matching/cache.py delete mode 100644 matching/exact.py delete mode 100644 matching/judge.py delete mode 100644 matching/synonym.py delete mode 100644 matching/wordnet.py delete mode 100644 tests/test_synonym_match.py diff --git a/matching/__init__.py b/matching/__init__.py deleted file mode 100644 index 6777c0f..0000000 --- a/matching/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Pluggable object-name matchers for grading VLM predictions against GT strings. - -This package lives at the top level (not under ``metrics/``) on purpose: -``metrics/__init__`` eagerly imports torch (via throne/faithscore), so anything -that needs to run offline — the ROPE log-rescore, the synonym regrade — must be -importable without torch/PIL/GPU. Nothing here imports torch at module load; -the optional LLM judge lazy-imports it only when actually used. - -Public objects: - Matcher, MatchResult, KEY_FOR, PER_POS_KEY_FOR, normalize (base) - ExactMatcher, SubstringMatcher (exact) - WordNetMatcher (wordnet) - SynonymMatcher, MissingPairError (synonym) - PairCache (cache) - LLMJudge, FlanJudge, DEFAULT_PROMPTS, PROMPT_VERSION (judge, dormant) -""" -from .base import Matcher, MatchResult, KEY_FOR, PER_POS_KEY_FOR, normalize -from .exact import ExactMatcher, SubstringMatcher -from .wordnet import WordNetMatcher -from .synonym import SynonymMatcher, MissingPairError -from .cache import PairCache - -__all__ = [ - "Matcher", "MatchResult", "KEY_FOR", "PER_POS_KEY_FOR", "normalize", - "ExactMatcher", "SubstringMatcher", "WordNetMatcher", - "SynonymMatcher", "MissingPairError", "PairCache", -] diff --git a/matching/base.py b/matching/base.py deleted file mode 100644 index 9f01eb5..0000000 --- a/matching/base.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Matcher protocol, result type, and the name->result-key maps. - -A ``Matcher`` decides whether a model prediction should count as correct for a -given ground-truth class string. Metrics accept a list of matchers and emit one -accuracy column per matcher, keyed by ``KEY_FOR[matcher.name]``. -""" -from __future__ import annotations - -from dataclasses import dataclass -from typing import Protocol, runtime_checkable - - -def normalize(s: str) -> str: - """Canonical form used by every matcher: strip + lowercase.""" - return s.strip().lower() - - -# Overall-accuracy JSON key each named matcher writes. Existing ROPE keys -# (overall_accuracy = lenient headline, overall_accuracy_strict) are preserved -# verbatim so previously-committed results stay byte-identical. -KEY_FOR = { - "strict": "overall_accuracy_strict", - "lenient": "overall_accuracy", - "synonym": "overall_accuracy_synonym", -} - -PER_POS_KEY_FOR = { - "strict": "per_position_accuracy_strict", - "lenient": "per_position_accuracy", - "synonym": "per_position_accuracy_synonym", -} - - -@runtime_checkable -class Matcher(Protocol): - name: str - - def match(self, pred: str, gt: str) -> bool: # pragma: no cover - protocol - ... - - -@dataclass -class MatchResult: - """Outcome of a single match, with the tier that decided it (flip-audit).""" - matched: bool - tier: str - votes: int = 0 diff --git a/matching/cache.py b/matching/cache.py deleted file mode 100644 index 1e7363a..0000000 --- a/matching/cache.py +++ /dev/null @@ -1,57 +0,0 @@ -"""Order-independent verdict cache for the (dormant) LLM judge tier. - -Only the judge path uses this — the default WordNet regrade is deterministic and -needs no cache. Kept so a future ``--build-cache`` run (issue #27) can judge the -residual once on GPU, commit the verdicts, and then re-grade fully offline. - -The cache invalidates itself if the prompt version or judge model changes. -""" -from __future__ import annotations - -import hashlib -import json -from pathlib import Path - - -class PairCache: - def __init__(self, path, prompt_version: str, judge_model: str): - self.path = Path(path) - self.prompt_version = prompt_version - self.judge_model = judge_model - self._store: dict = {} - - @staticmethod - def _norm(s: str) -> str: - return s.strip().lower() - - def _key(self, a: str, b: str) -> str: - x, y = sorted((self._norm(a), self._norm(b))) - return hashlib.sha1((x + "␟" + y).encode("utf-8")).hexdigest() - - def get(self, a: str, b: str): - return self._store.get(self._key(a, b)) - - def put(self, a: str, b: str, verdict: dict) -> None: - self._store[self._key(a, b)] = verdict - - def missing(self, pairs): - return [(a, b) for (a, b) in pairs if self._key(a, b) not in self._store] - - def load(self) -> None: - if not self.path.exists(): - return - data = json.loads(self.path.read_text()) - if (data.get("prompt_version") != self.prompt_version - or data.get("judge_model") != self.judge_model): - self._store = {} # stale -> invalidate - return - self._store = data.get("pairs", {}) - - def save(self) -> None: - self.path.parent.mkdir(parents=True, exist_ok=True) - payload = { - "prompt_version": self.prompt_version, - "judge_model": self.judge_model, - "pairs": self._store, - } - self.path.write_text(json.dumps(payload, indent=2, sort_keys=True)) diff --git a/matching/exact.py b/matching/exact.py deleted file mode 100644 index 1394855..0000000 --- a/matching/exact.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Exact and substring matchers — bodies moved verbatim from metrics/rope.py. - -ExactMatcher == the old ``_rope_match_strict`` (canonical ROPE, exact equality). -SubstringMatcher == the old ``_rope_match`` (lenient whole-word substring). - -Keeping the logic identical is what guarantees the no-regression gate: the -default matcher set [SubstringMatcher, ExactMatcher] reproduces the committed -``overall_accuracy`` / ``overall_accuracy_strict`` byte-for-byte. -""" -from __future__ import annotations - -import re - -from .base import normalize - - -class ExactMatcher: - """Strict ROPE match: exact (case-insensitive) equality.""" - name = "strict" - - def match(self, pred: str, gt: str) -> bool: - return normalize(pred) == normalize(gt) - - -class SubstringMatcher: - """Lenient ROPE match: GT class appears as a whole-word substring of the - prediction (or vice versa). Tolerates "wooden cabinet" vs GT "cabinet".""" - name = "lenient" - - def match(self, pred: str, gt: str) -> bool: - p = normalize(pred) - t = normalize(gt) - if not p or not t: - return False - if p == t: - return True - t_pattern = r"\b" + re.escape(t) + r"\b" - p_pattern = r"\b" + re.escape(p) + r"\b" - return bool(re.search(t_pattern, p)) or bool(re.search(p_pattern, t)) diff --git a/matching/judge.py b/matching/judge.py deleted file mode 100644 index 79c6757..0000000 --- a/matching/judge.py +++ /dev/null @@ -1,100 +0,0 @@ -"""LLM synonym judge — DORMANT tier (not used by the default WordNet regrade). - -Kept present-but-inactive so a stronger judge can be plugged in later without a -redesign. The issue #24 Phase-0 spike found flan-t5-base insufficient; issue #27 -tracks ablating larger judges. Supports both architectures that ablation needs: - - arch="seq2seq" -> AutoModelForSeq2SeqLM (google/flan-t5-base, -xl) - arch="causal" -> AutoModelForCausalLM (meta-llama/Llama-2-7b-chat-hf) - -torch/transformers are lazy-imported inside ``_load`` so importing this module -(and therefore the whole ``matching`` package) stays torch-free. -""" -from __future__ import annotations - -from typing import Dict, List, Optional, Tuple - -DEFAULT_PROMPTS = [ - 'Do the phrases "{a}" and "{b}" refer to the same kind of object? Answer yes or no.', - 'Is a "{a}" the same thing as a "{b}"? Answer yes or no.', - 'Can "{a}" and "{b}" describe the same object in an image? Answer yes or no.', -] -# Bump when prompts/vote logic change so PairCache invalidates stale verdicts. -PROMPT_VERSION = "v1-3prompt-majority-greedy" - - -class LLMJudge: - def __init__( - self, - model_name: str = "google/flan-t5-base", - arch: str = "seq2seq", - prompts: Optional[List[str]] = None, - device: Optional[str] = None, - batch_size: int = 32, - max_new_tokens: int = 3, - ): - if arch not in ("seq2seq", "causal"): - raise ValueError(f"arch must be 'seq2seq' or 'causal', got {arch!r}") - self.model_name = model_name - self.arch = arch - self.prompts = prompts or DEFAULT_PROMPTS - self.device = device - self.batch_size = batch_size - self.max_new_tokens = max_new_tokens - self._model = None - self._tok = None - - def _load(self) -> None: - import torch - from transformers import AutoTokenizer - if self.device is None: - self.device = "cuda" if torch.cuda.is_available() else "cpu" - self._tok = AutoTokenizer.from_pretrained(self.model_name) - dtype = torch.bfloat16 if self.device == "cuda" else torch.float32 - if self.arch == "seq2seq": - from transformers import AutoModelForSeq2SeqLM - self._model = AutoModelForSeq2SeqLM.from_pretrained( - self.model_name, torch_dtype=dtype).to(self.device).eval() - else: - from transformers import AutoModelForCausalLM - if self._tok.pad_token is None: - self._tok.pad_token = self._tok.eos_token - self._model = AutoModelForCausalLM.from_pretrained( - self.model_name, torch_dtype=dtype).to(self.device).eval() - - @staticmethod - def _is_yes(text: str) -> bool: - return text.strip().lower().startswith("y") - - def judge_batch(self, pairs: List[Tuple[str, str]]) -> List[Dict]: - """Return [{"yes": bool, "votes": int}] — majority vote over prompts, greedy.""" - import torch - if self._model is None: - self._load() - flat = [p.format(a=a, b=b) for (a, b) in pairs for p in self.prompts] - outs: List[str] = [] - with torch.no_grad(): - for i in range(0, len(flat), self.batch_size): - batch = flat[i:i + self.batch_size] - enc = self._tok(batch, return_tensors="pt", padding=True, - truncation=True).to(self.device) - gen = self._model.generate(**enc, max_new_tokens=self.max_new_tokens, - do_sample=False) - if self.arch == "causal": - gen = gen[:, enc["input_ids"].shape[1]:] # strip the prompt - outs.extend(self._tok.batch_decode(gen, skip_special_tokens=True)) - np_ = len(self.prompts) - results = [] - for idx in range(len(pairs)): - ans = outs[idx * np_:(idx + 1) * np_] - votes = sum(self._is_yes(a) for a in ans) - results.append({"yes": votes > np_ // 2, "votes": votes}) - return results - - -class FlanJudge(LLMJudge): - """Convenience alias matching the issue #24 naming (seq2seq flan-t5-*).""" - - def __init__(self, model_name: str = "google/flan-t5-base", **kwargs): - kwargs.pop("arch", None) - super().__init__(model_name=model_name, arch="seq2seq", **kwargs) diff --git a/matching/synonym.py b/matching/synonym.py deleted file mode 100644 index a9e20fb..0000000 --- a/matching/synonym.py +++ /dev/null @@ -1,74 +0,0 @@ -"""SynonymMatcher — tiered, add-on matcher for the ``overall_accuracy_synonym`` column. - -Tiers (first hit wins), so ``synonym`` is a strict superset of ``lenient`` and the -``synonym >= lenient`` invariant holds by construction: - - 1. base (SubstringMatcher) — free lenient check - 2. wordnet (WordNetMatcher) — active default tier (offline, deterministic) - 3. judge (LLMJudge|cache) — DORMANT; only consulted if a judge or cache - is provided (issue #27 ablation / --build-cache) - -Default construction — ``SynonymMatcher()`` — uses Substring + WordNet only, which -is exactly the shipped, precision-1.0 config from the Phase-0 spike. -""" -from __future__ import annotations - -from typing import Optional - -from .base import MatchResult, Matcher -from .exact import SubstringMatcher -from .wordnet import WordNetMatcher - - -class MissingPairError(KeyError): - """Offline judge mode: a residual pair is absent from the cache.""" - - -class SynonymMatcher: - name = "synonym" - - def __init__( - self, - base: Optional[Matcher] = None, - wordnet: Optional[WordNetMatcher] = None, - cache=None, - judge=None, - mode: str = "wordnet", - ): - # mode in {"wordnet", "trusting"}. "wordnet" (default) = base+WordNet, - # no LLM. "trusting" additionally consults the judge tier when present. - if mode not in ("wordnet", "trusting"): - raise ValueError(f"unknown mode {mode!r}") - self.base = base if base is not None else SubstringMatcher() - self.wordnet = wordnet if wordnet is not None else WordNetMatcher() - self.cache = cache - self.judge = judge - self.mode = mode - - def match(self, pred: str, gt: str) -> bool: - return self.explain(pred, gt).matched - - def explain(self, pred: str, gt: str) -> MatchResult: - if self.base.match(pred, gt): - return MatchResult(True, "base") - if self.wordnet is not None and self.wordnet.match(pred, gt): - return MatchResult(True, "wordnet") - # Dormant unless a judge or cache was supplied. - if self.mode == "trusting" and (self.judge is not None or self.cache is not None): - verdict = self._judge_verdict(pred, gt) - if verdict is not None and verdict.get("yes"): - return MatchResult(True, "judge", verdict.get("votes", 0)) - return MatchResult(False, "none") - - def _judge_verdict(self, pred: str, gt: str): - if self.cache is not None: - hit = self.cache.get(pred, gt) - if hit is not None: - return hit - if self.judge is not None: # build mode: compute now, cache if we have one - verdict = self.judge.judge_batch([(pred, gt)])[0] - if self.cache is not None: - self.cache.put(pred, gt, verdict) - return verdict - # offline mode: cache present but pair unknown, no judge to fall back on - raise MissingPairError(f"{pred!r} vs {gt!r} not in cache") diff --git a/matching/wordnet.py b/matching/wordnet.py deleted file mode 100644 index a47b635..0000000 --- a/matching/wordnet.py +++ /dev/null @@ -1,65 +0,0 @@ -"""WordNet synonym matcher — the active synonym tier shipped in issue #24. - -Phase-0 spike (tests/data/synonym_gold.jsonl, 200 pairs) measured this at -**precision 1.00 / recall 0.46 / 0 false positives** with ``max_hops=0`` — the -shipped default. It is deterministic, offline, and needs no GPU/SU. - -Algorithm (head-noun, shared-synset): - head = last alphabetic token of each phrase - match if heads are equal (catches "soccer ball" / "sports ball" via "ball"), - else if the two heads share a WordNet NOUN synset (catches bike/bicycle, - rock/rocks, egg/eggs, sofa/couch — NLTK Morphy handles the plural lemmas). - -``max_hops > 0`` additionally walks hypernyms, but that also merges hyponym -pairs the grading protocol wants kept apart (e.g. mug/cup, woman/person), which -measured lower precision (0.857 at hops=1). Left off by default on purpose. -""" -from __future__ import annotations - - -class WordNetMatcher: - name = "wordnet" - - def __init__(self, max_hops: int = 0): - self.max_hops = max_hops - self._wn = None - - def _ensure(self): - """Lazy-load the WordNet corpus. Raises LookupError if not downloaded - (run: python -m nltk.downloader wordnet omw-1.4).""" - if self._wn is None: - from nltk.corpus import wordnet as wn - wn.ensure_loaded() - self._wn = wn - return self._wn - - @staticmethod - def _head(s: str) -> str: - toks = [t for t in "".join(c if c.isalpha() or c == " " else " " - for c in s.lower()).split()] - return toks[-1] if toks else "" - - def _synsets(self, word: str): - wn = self._wn - return set(wn.synsets(word, pos=wn.NOUN)) - - def match(self, pred: str, gt: str) -> bool: - wn = self._ensure() - ha, hb = self._head(pred), self._head(gt) - if not ha or not hb: - return False - if ha == hb: - return True - sa, sb = self._synsets(ha), self._synsets(hb) - if not sa or not sb: - return False - if sa & sb: - return True - if self.max_hops > 0: - for src, dst in ((sa, sb), (sb, sa)): - frontier = set(src) - for _ in range(self.max_hops): - frontier = {h for s in frontier for h in s.hypernyms()} | frontier - if frontier & dst: - return True - return False diff --git a/tests/test_synonym_match.py b/tests/test_synonym_match.py deleted file mode 100644 index 7338d03..0000000 --- a/tests/test_synonym_match.py +++ /dev/null @@ -1,130 +0,0 @@ -"""Acceptance tests for the synonym-matching layer (issue #24). - -Runs fully offline (WordNet corpus only, no GPU/torch). Encodes the *measured* -Phase-0 behavior of the shipped WordNet hops=0 matcher — including the -deliberate rejections (mug/cup, woman/person) that a hypernym hop would wrongly -merge. See Type1_Experiments/synonym_regrade/phase0_spike.md. - -Run: python3 -m pytest tests/test_synonym_match.py -q - (or plain `python3 tests/test_synonym_match.py` for a no-pytest smoke run) -""" -import sys -import types -from pathlib import Path - -ROOT = Path(__file__).resolve().parent.parent -sys.path.insert(0, str(ROOT)) - -from matching import ExactMatcher, SubstringMatcher, WordNetMatcher, SynonymMatcher - - -def _import_rope_metric(): - # metrics/__init__ imports torch; load the torch-free metrics.rope without it. - if "metrics" not in sys.modules: - pkg = types.ModuleType("metrics") - pkg.__path__ = [str(ROOT / "metrics")] - sys.modules["metrics"] = pkg - from metrics.rope import ROPEMetric - return ROPEMetric - -# Pairs the grader SHOULD merge (true synonyms / number variants / same head noun). -MERGE = [ - ("sofa", "couch"), - ("couch", "sofa"), - ("bicycle", "bike"), - ("small rock", "rocks"), - ("sliced hard-boiled egg", "eggs"), - ("car", "automobile"), - ("soccer ball", "sports ball"), - ("white line", "lines"), -] - -# Pairs the grader must NOT merge. Includes hyponym pairs (mug/cup, woman/person) -# that hops=0 rejects on purpose to keep precision at 1.0. -REJECT = [ - ("cat", "dog"), - ("mug", "cup"), - ("towel", "couch"), - ("monitor", "tv"), - ("woman", "person"), - ("man", "person"), - ("bottle", "wine"), - ("ski boot", "skis"), -] - - -def test_wordnet_merges_true_synonyms(): - wn = WordNetMatcher() # hops=0 - for a, b in MERGE: - assert wn.match(a, b), f"WordNet should merge {a!r}/{b!r}" - - -def test_wordnet_rejects_hard_negatives(): - wn = WordNetMatcher() # hops=0 - for a, b in REJECT: - assert not wn.match(a, b), f"WordNet should NOT merge {a!r}/{b!r}" - - -def test_synonym_matcher_default_is_offline_and_deterministic(): - syn = SynonymMatcher() # base + wordnet, no judge/cache - # Never raises (no MissingPairError) and is repeatable. - for a, b in MERGE + REJECT: - assert syn.match(a, b) == syn.match(a, b) - - -def test_synonym_superset_of_lenient_invariant(): - """overall_accuracy_synonym >= overall_accuracy: synonym matches wherever - lenient does (synonym includes SubstringMatcher as its base tier).""" - sub = SubstringMatcher() - syn = SynonymMatcher() - cases = MERGE + REJECT + [ - ("wooden cabinet", "cabinet"), # lenient True -> synonym must be True - ("red fire hydrant", "hydrant"), - ("person", "person"), - ] - for a, b in cases: - if sub.match(a, b): - assert syn.match(a, b), f"synonym must hold wherever lenient does: {a!r}/{b!r}" - - -def test_exact_and_substring_no_regression(): - """ExactMatcher/SubstringMatcher reproduce the old _rope_match* semantics.""" - ex, sub = ExactMatcher(), SubstringMatcher() - # exact: case-insensitive equality only - assert ex.match("Cabinet", "cabinet") - assert not ex.match("wooden cabinet", "cabinet") - # lenient: whole-word substring either direction, but respects word boundaries - assert sub.match("wooden cabinet", "cabinet") - assert sub.match("cabinet", "wooden cabinet") - assert not sub.match("cabinet", "cabinets") # no word boundary -> not a match - assert not sub.match("", "cabinet") # empty -> False - - -def test_rope_metric_inline_synonym_matcher(): - """ROPEMetric(matchers=[..., SynonymMatcher()]) emits the synonym column - while the lenient/strict headline columns stay untouched.""" - ROPEMetric = _import_rope_metric() - metric = ROPEMetric( - num_objects=2, - matchers=[SubstringMatcher(), ExactMatcher(), SynonymMatcher()], - ) - # obj1: WordNet-only credit; obj2: lenient (substring) credit. - metric.evaluate_predictions([["bike", "wooden cabinet"]], [["bicycle", "cabinet"]]) - results = metric.format_results() - assert results["overall_accuracy_synonym"] == 1.0 - assert results["per_position_accuracy_synonym"] == {"obj1": 1.0, "obj2": 1.0} - assert results["overall_accuracy"] == 0.5 - assert results["per_position_accuracy"] == {"obj1": 0.0, "obj2": 1.0} - assert results["overall_accuracy_strict"] == 0.0 - - -def _run(): - fns = [v for k, v in sorted(globals().items()) if k.startswith("test_")] - for fn in fns: - fn() - print(f"PASS {fn.__name__}") - print(f"\nAll {len(fns)} tests passed.") - - -if __name__ == "__main__": - _run() From 81fc4a7aa5c8f2cae3c1b78f667323377eeda43c Mon Sep 17 00:00:00 2001 From: Salim Alkharsa Date: Sat, 17 Oct 2026 04:23:34 +0000 Subject: [PATCH 09/12] predict/identity: drop the reference to the deleted matching package --- predict/identity.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/predict/identity.py b/predict/identity.py index ac8216d..17aae82 100644 --- a/predict/identity.py +++ b/predict/identity.py @@ -3,8 +3,7 @@ Cache keys are SHA-256 hashes over a sorted-keys JSON payload of the model identity, the input image, the prompt, and the generation config. Bump ``CACHE_KEY_VERSION`` whenever model post-processing or key -semantics change in a way that should invalidate existing cache entries -(same policy as ``matching.judge.PROMPT_VERSION``). +semantics change in a way that should invalidate existing cache entries. """ import hashlib From 69e335612c27ad26dd203d0487d0a9615b6181af Mon Sep 17 00:00:00 2001 From: Salim Alkharsa Date: Sat, 17 Oct 2026 04:23:49 +0000 Subject: [PATCH 10/12] Move the Gemma 4 thinking variant into gemma4.py as Gemma4Thinking Same body as HFChatThinkingVLM; cache_name stays "HFChatThinkingVLM", so the gemma4-*-think cache keys do not change. The registry rows point at models.gemma4:Gemma4Thinking. --- models/__init__.py | 1 + models/gemma4.py | 9 +++++++++ models/hf_chat.py | 9 --------- predict/factory.py | 4 ++-- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/models/__init__.py b/models/__init__.py index da339c3..64e5f26 100644 --- a/models/__init__.py +++ b/models/__init__.py @@ -10,6 +10,7 @@ "Qwen2VLModel": "qwen2vl", "HFChatVLM": "hf_chat", "Gemma4": "gemma4", + "Gemma4Thinking": "gemma4", "Ministral3": "ministral3", "AyaVision": "aya_vision", } diff --git a/models/gemma4.py b/models/gemma4.py index 0951f25..540025b 100644 --- a/models/gemma4.py +++ b/models/gemma4.py @@ -4,3 +4,12 @@ class Gemma4(HFChatVLM): """Gemma 4; keys its cache as ``HFChatVLM``.""" + + +class Gemma4Thinking(Gemma4): + """Gemma 4 with thinking on; a separate class so cache keys never mix modes.""" + + cache_name = "HFChatThinkingVLM" + + def __init__(self, model_name: str, **kwargs): + super().__init__(model_name, enable_thinking=True, **kwargs) diff --git a/models/hf_chat.py b/models/hf_chat.py index c57c8e8..acf60be 100644 --- a/models/hf_chat.py +++ b/models/hf_chat.py @@ -125,12 +125,3 @@ def generate( logger.warning("Thinking block not closed within %d tokens", gen_kwargs["max_new_tokens"]) text = self.processor.decode(new_tokens, skip_special_tokens=True) return _THINKING_RE.sub("", text).strip() - - -class HFChatThinkingVLM(HFChatVLM): - """``HFChatVLM`` with thinking on; a separate class so cache keys never mix modes.""" - - cache_name = "HFChatThinkingVLM" - - def __init__(self, model_name: str, **kwargs): - super().__init__(model_name, enable_thinking=True, **kwargs) diff --git a/predict/factory.py b/predict/factory.py index 625de48..e3e4eef 100644 --- a/predict/factory.py +++ b/predict/factory.py @@ -45,9 +45,9 @@ class Entry(NamedTuple): "aya-vision-8b": Entry("models.aya_vision:AyaVision", "MIRAGE_AYA_VISION_8B_MODEL_PATH", "CohereLabs/aya-vision-8b", "HFChatVLM", _BF16), # Thinking-on variants for the ablation (same checkpoints). - "gemma4-e4b-think": Entry("models.hf_chat:HFChatThinkingVLM", "MIRAGE_GEMMA4_E4B_MODEL_PATH", + "gemma4-e4b-think": Entry("models.gemma4:Gemma4Thinking", "MIRAGE_GEMMA4_E4B_MODEL_PATH", "google/gemma-4-E4B-it", "HFChatThinkingVLM", _BF16), - "gemma4-12b-think": Entry("models.hf_chat:HFChatThinkingVLM", "MIRAGE_GEMMA4_12B_MODEL_PATH", + "gemma4-12b-think": Entry("models.gemma4:Gemma4Thinking", "MIRAGE_GEMMA4_12B_MODEL_PATH", "google/gemma-4-12B-it", "HFChatThinkingVLM", _BF16), }) From 975ea59b4abc3a2c9f2fcbd75ba8687924cdb037 Mon Sep 17 00:00:00 2001 From: Salim Alkharsa Date: Sat, 17 Oct 2026 04:34:58 +0000 Subject: [PATCH 11/12] ROPE: docstrings describe the single-box protocol --- metrics/rope.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/metrics/rope.py b/metrics/rope.py index 2c5b0f7..b9ac212 100644 --- a/metrics/rope.py +++ b/metrics/rope.py @@ -5,10 +5,10 @@ https://github.com/sled-group/moh ROPE evaluates multi-object hallucination in Vision-Language Models by drawing -red bounding boxes around N objects in an image and asking the model to identify -all N objects in a single prompt. This contrasts with POPE's binary yes/no -questioning and tests whether VLMs can correctly ground and classify multiple -objects simultaneously. +red bounding boxes around N objects in an image and asking the model to name +the object in each numbered box, one question per box. This contrasts with +POPE's binary yes/no questioning and tests whether VLMs can correctly ground +and classify multiple objects in the same image. Reference: Zou et al., "Look, Compare, Decide: Alleviating Hallucination in Large @@ -80,7 +80,7 @@ def __init__( self.bbox_color = bbox_color self.bbox_width = bbox_width - # Accumulated state across evaluate() calls + # Accumulated state across evaluate_predictions() calls self._all_predictions: List[List[str]] = [] self._all_ground_truth: List[List[str]] = [] @@ -92,8 +92,8 @@ def draw_bounding_boxes( """ Draw numbered red bounding boxes on a copy of the image. - Each box is labeled with its 1-based index (1, 2, 3, ...) so the model - can match the visual box number to the obj1/obj2/... output format. + Each box is labeled with its 1-based index (1, 2, 3, ...), matching + the "box i" in the per-box prompts. Adapted from bbox rendering logic in sled-group/moh evaluation pipeline. @@ -219,11 +219,7 @@ def _place_label( def generate_prompt_for_box(self, box_index: int) -> str: """ - Generate a single-box identification prompt for PaliGemma with few-shot examples. - - Keeps the prompt short and direct. PaliGemma performs better with a - simple, focused question than with long class lists or few-shot examples - that can anchor completions to the example answers. + The single-box ROPE question; its bytes are part of the cache key. Args: box_index: 1-based index of the bounding box to ask about. From b317f2497b19400f34c9ae5934791d3a1c0f85ef Mon Sep 17 00:00:00 2001 From: Salim Alkharsa Date: Sat, 17 Oct 2026 07:11:53 +0000 Subject: [PATCH 12/12] base_metric: BaseMetric is a plain base class; one-line compute_* docstrings Nothing in BaseMetric is abstract any more, so it no longer subclasses ABC. The compute_* helpers keep their bodies; their docstrings shrink to one line. --- metrics/base_metric.py | 97 +++--------------------------------------- 1 file changed, 6 insertions(+), 91 deletions(-) diff --git a/metrics/base_metric.py b/metrics/base_metric.py index e9d1b1b..26ebd5c 100644 --- a/metrics/base_metric.py +++ b/metrics/base_metric.py @@ -1,4 +1,3 @@ -from abc import ABC from types import MappingProxyType from typing import Dict, List, Any, Mapping, Tuple, Union, Optional import numpy as np @@ -58,7 +57,7 @@ def caption_images(loader, model, n: int = SAMPLES, return samples, captions -class BaseMetric(ABC): +class BaseMetric: """ Base class for all evaluation metrics in Mirage. @@ -125,16 +124,7 @@ def reset(self) -> None: self.results = {} def compute_accuracy(self, predictions: List[bool], labels: List[bool]) -> float: - """ - Compute accuracy from boolean predictions and labels. - - Args: - predictions: List of boolean predictions - labels: List of boolean labels - - Returns: - Accuracy score - """ + """Fraction of ``predictions`` equal to ``labels``; 0.0 when empty.""" if len(predictions) == 0: return 0.0 @@ -142,30 +132,7 @@ def compute_accuracy(self, predictions: List[bool], labels: List[bool]) -> float return correct / len(predictions) def compute_confusion_matrix(self, predictions: List[bool], labels: List[bool]) -> Dict[str, int]: - """ - Compute confusion matrix components from boolean predictions and labels. - - Calculates the four components of a binary confusion matrix: - - True Positives (TP): Correctly predicted positive cases - - True Negatives (TN): Correctly predicted negative cases - - False Positives (FP): Incorrectly predicted positive cases - - False Negatives (FN): Incorrectly predicted negative cases - - Args: - predictions: List of boolean predictions (True = positive, False = negative) - labels: List of boolean ground truth labels (True = positive, False = negative) - - Returns: - Dictionary containing confusion matrix components with keys: - 'true_positives', 'true_negatives', 'false_positives', 'false_negatives' - - Example: - >>> predictions = [True, False, True, False] - >>> labels = [True, False, False, False] - >>> cm = metric.compute_confusion_matrix(predictions, labels) - >>> print(cm) - {'true_positives': 1, 'true_negatives': 2, 'false_positives': 1, 'false_negatives': 0} - """ + """``true_positives``, ``true_negatives``, ``false_positives`` and ``false_negatives`` counts.""" if len(predictions) != len(labels): raise ValueError("Predictions and labels must have the same length") @@ -182,71 +149,19 @@ def compute_confusion_matrix(self, predictions: List[bool], labels: List[bool]) } def compute_precision(self, tp: int, fp: int) -> float: - """ - Compute precision from confusion matrix components. - - Precision measures the proportion of positive predictions that were correct. - Formula: precision = TP / (TP + FP) - - Args: - tp: Number of true positives - fp: Number of false positives - - Returns: - Precision score between 0 and 1. Returns 0.0 if no positive predictions were made. - - Example: - >>> precision = metric.compute_precision(tp=80, fp=20) - >>> print(precision) - 0.8 - """ + """TP / (TP + FP); 0.0 when nothing was predicted positive.""" if tp + fp == 0: return 0.0 return tp / (tp + fp) def compute_recall(self, tp: int, fn: int) -> float: - """ - Compute recall (sensitivity) from confusion matrix components. - - Recall measures the proportion of actual positive cases that were correctly identified. - Formula: recall = TP / (TP + FN) - - Args: - tp: Number of true positives - fn: Number of false negatives - - Returns: - Recall score between 0 and 1. Returns 0.0 if no positive cases exist. - - Example: - >>> recall = metric.compute_recall(tp=80, fn=20) - >>> print(recall) - 0.8 - """ + """TP / (TP + FN); 0.0 when there are no positives.""" if tp + fn == 0: return 0.0 return tp / (tp + fn) def compute_f1(self, precision: float, recall: float) -> float: - """ - Compute F1 score from precision and recall. - - F1 score is the harmonic mean of precision and recall, providing a balanced - measure that accounts for both false positives and false negatives. - Formula: F1 = 2 * (precision * recall) / (precision + recall) - - Args: - precision: Precision score (between 0 and 1) - recall: Recall score (between 0 and 1) - - Returns: - F1 score between 0 and 1. Returns 0.0 if both precision and recall are 0. - - Example: - >>> f1 = metric.compute_f1(precision=0.8, recall=0.9) - >>> print(f1) - 0.8470588235294118 - """ + """Harmonic mean of precision and recall; 0.0 when both are 0.""" if precision + recall == 0: return 0.0 return 2 * (precision * recall) / (precision + recall)