From a42457f233e71e9f436845c17416a7fa4681037b Mon Sep 17 00:00:00 2001 From: Xabi Ezpeleta Date: Tue, 26 May 2026 23:12:58 +0200 Subject: [PATCH] feat: add Belebele EU reading comprehension benchmark MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds facebook/belebele (eus_Latn) — a 900-item 4-way multiple choice reading comprehension benchmark with FLORES-sourced passages. - Plugin (_belebele_eu.py) assembles mc_answer1..4 into candidates list and converts 1-indexed correct_answer_num to 0-indexed gold. - Generic MC scorer handles answer extraction (no score_item needed). - Registered in site data under LatxaEvalSuite family and reading skill. Closes #8 --- TODO.md | 30 +++++++---- eval/benchmarks/belebele_eu.json | 18 +++++++ eval/benchmarks/plugins/_belebele_eu.py | 67 +++++++++++++++++++++++++ site/build_site_data.py | 10 +++- 4 files changed, 113 insertions(+), 12 deletions(-) create mode 100644 eval/benchmarks/belebele_eu.json create mode 100644 eval/benchmarks/plugins/_belebele_eu.py diff --git a/TODO.md b/TODO.md index fcc9c93..6361271 100644 --- a/TODO.md +++ b/TODO.md @@ -8,26 +8,33 @@ - `alexandrainst/m_mmlu` - `jon-tow/okapi_mmlu` -## Additional Basque Benchmarks to Add (pending) +## Already Implemented + +- [x] **MMLU EU** — `orai-nlp/MMLU_HT_eu_sample` +- [x] **Math reasoning** — `MGSM_eu` (HiTZ/MGSM-eu, 3-shot CoT) +- [x] **Basque QA** — `BertaQA_eu` (combined local + global, HiTZ/BertaQA) +- [x] **Exam / proficiency / trivia** + - [x] `EusTrivia` (HiTZ/EusTrivia) + - [x] `LatxaEval_eusexams` (HiTZ/EusExams) + - [x] `LatxaEval_eusproficiency` (HiTZ/EusProficiency) +- [x] **BasqueGLUE** — `BasqueGLUE_qnli`, `BasqueGLUE_bec`, `BasqueGLUE_wic`, `BasqueGLUE_intent` +- [x] **XNLI** — `XNLIeu` +- [x] **LatxaEval reading** — `LatxaEval_eusreading` +- [x] **Flores translation** — `eu↔en`, `eu↔es` (4 directions) + +## Still Pending -- [x] **Math reasoning in Basque** - - [x] `mgsm_native_cot_eu` → implemented as `MGSM_eu` (HiTZ/MGSM-eu, 3-shot CoT) - [ ] **Reading comprehension in Basque** + - [x] `Belebele_eu` (facebook/belebele, eus_Latn, 4-way MC reading comprehension) - Candidate: `xstorycloze_eu` - - Candidate: `belebele_eus_Latn` - [ ] **Science / commonsense QA in Basque** - Candidate: `arc_eu_easy_mc` - Candidate: `arc_eu_challenge_mc` - Candidate: `piqa_eu_mc` - Candidate: `siqa_eu_mc` -- [ ] **Exam / proficiency / trivia coverage alignment** - - Candidate: `eus_exams_eu` - - Candidate: `eus_proficiency` - - Candidate: `eus_trivia` -- [x] **Basque QA variants** - - [x] `bertaqa_eu` (combined local + global, HiTZ/BertaQA) -- [ ] **Other candidate benchmark from discussion** +- [ ] **Other candidates** - Candidate: `bl2mp` + - Multilingual alternatives: `alexandrainst/m_mmlu`, `jon-tow/okapi_mmlu` ## Integration Planning (no implementation yet) @@ -35,3 +42,4 @@ - [ ] Define evaluation order for incremental rollout (start with MMLU EU, then high-impact tasks). - [ ] Add acceptance criteria for each new benchmark before publishing to site. - [ ] Decide which benchmarks are shown in public leaderboard vs experimental section. +- [ ] Decide and document whether `orai-nlp/MMLU_HT_eu_sample` is temporary (pilot) or long-term benchmark source. diff --git a/eval/benchmarks/belebele_eu.json b/eval/benchmarks/belebele_eu.json new file mode 100644 index 0000000..485915c --- /dev/null +++ b/eval/benchmarks/belebele_eu.json @@ -0,0 +1,18 @@ +{ + "id": "Belebele_eu", + "description": "Belebele — Basque reading comprehension (facebook/belebele, eus_Latn)", + "dataset": { + "source": "custom" + }, + "prompt": { + "type": "multiple_choice", + "template": "Aukeratu aukera zuzena (A/B/C/D bakarrik).\nTestua: {context}\nGaldera: {question}\n{options}\nErantzuna:" + }, + "scoring": { + "type": "multiple_choice", + "extraction": "letter_or_index", + "fallback": "candidate_text_match" + }, + "plugin": "belebele_eu", + "default_limit": 100 +} diff --git a/eval/benchmarks/plugins/_belebele_eu.py b/eval/benchmarks/plugins/_belebele_eu.py new file mode 100644 index 0000000..cc81ff5 --- /dev/null +++ b/eval/benchmarks/plugins/_belebele_eu.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Builder plugin for Belebele EU — Basque reading comprehension. + +Loads facebook/belebele (config eus_Latn), assembles candidates from the four +individual mc_answer fields, and converts the 1-indexed correct_answer_num to +0-indexed gold. No score_item override needed — the generic MC scorer handles +letter/index extraction and candidate text matching. +""" + +from __future__ import annotations + +import random +from typing import Any + +from datasets import load_dataset + + +def build_items(limit: int, seed: int, spec: dict | None = None) -> list[dict[str, Any]]: + bench_id = spec["id"] if (spec and "id" in spec) else "Belebele_eu" + prompt_template = ( + spec["prompt"]["template"] + if (spec and isinstance(spec.get("prompt"), dict) and "template" in spec["prompt"]) + else "Aukeratu aukera zuzena (A/B/C/D bakarrik).\nTestua: {context}\nGaldera: {question}\n{options}\nErantzuna:" + ) + + ds = load_dataset("facebook/belebele", "eus_Latn", split="test") + + rng = random.Random(seed) + idxs = list(range(len(ds))) + rng.shuffle(idxs) + idxs = idxs[:limit] + + items: list[dict[str, Any]] = [] + for i in idxs: + row = ds[int(i)] + + candidates = [ + row["mc_answer1"], + row["mc_answer2"], + row["mc_answer3"], + row["mc_answer4"], + ] + if not any(candidates): + continue + + # correct_answer_num is 1-indexed string ("1".."4") → 0-indexed int + gold = int(row["correct_answer_num"]) - 1 + if not (0 <= gold < len(candidates)): + continue + + letters = [chr(ord("A") + j) for j in range(len(candidates))] + opts = "\n".join(f"{letters[j]}) {candidates[j]}" for j in range(len(candidates))) + context = row.get("flores_passage", "") + prompt = prompt_template.format( + question=row["question"], options=opts, context=context + ) + + items.append({ + "bench": bench_id, + "id": f"{bench_id.lower()}_{i}", + "prompt": prompt, + "gold": gold, + "label_names": letters, + "meta": {"candidates": candidates}, + }) + + return items diff --git a/site/build_site_data.py b/site/build_site_data.py index b1c6be6..7ba855a 100644 --- a/site/build_site_data.py +++ b/site/build_site_data.py @@ -74,6 +74,13 @@ def load_model_cards(root: Path): "metric": "Accuracy", "labels": "index of correct choice", }, + { + "id": "Belebele_eu", + "family": "LatxaEvalSuite", + "task": "Cross-lingual reading comprehension in Basque (Belebele)", + "metric": "Accuracy", + "labels": "4 options (A/B/C/D)", + }, { "id": "FloresTranslation_eu_en", "family": "FLORES", @@ -133,6 +140,7 @@ def load_model_cards(root: Path): "LatxaEval_eusexams": "LatxaEval-EusExams", "LatxaEval_eusproficiency": "LatxaEval-EusProficiency", "LatxaEval_eusreading": "LatxaEval-EusReading", + "Belebele_eu": "Belebele EU", "FloresTranslation_eu_en": "FLORES EU→EN", "FloresTranslation_en_eu": "FLORES EN→EU", "FloresTranslation_eu_es": "FLORES EU→ES", @@ -154,7 +162,7 @@ def load_model_cards(root: Path): "id": "reading", "label": "READING", "description": "How well the model understands text", - "benchmarks": ["XNLIeu", "LatxaEval_eusreading", "BasqueGLUE_qnli"], + "benchmarks": ["XNLIeu", "LatxaEval_eusreading", "Belebele_eu", "BasqueGLUE_qnli"], }, { "id": "writing",