diff --git a/judgearena/battles.py b/judgearena/battles.py index 78ef99bc..f07c5cbf 100644 --- a/judgearena/battles.py +++ b/judgearena/battles.py @@ -76,7 +76,7 @@ class RatingEntry: class Leaderboard: """Per-model ratings (mean + bootstrap CI) plus the run metadata that produced them. - Named distinctly from :class:`judgearena.benchmarks.elo.runner.EloReport`, + Named distinctly from :class:`judgearena.reports.EloReport`, which is the console/``results-*.json`` run report; this is the narrower ``elo_ratings.json`` leaderboard artifact with per-model CIs. """ diff --git a/judgearena/benchmarks/elo/calibration.py b/judgearena/benchmarks/elo/calibration.py new file mode 100644 index 00000000..cd43464e --- /dev/null +++ b/judgearena/benchmarks/elo/calibration.py @@ -0,0 +1,153 @@ +"""PairScore temperature calibration against human arena preferences.""" + +from __future__ import annotations + +from collections.abc import Mapping + +import numpy as np +import pandas as pd +from scipy.optimize import minimize_scalar + +from judgearena.arenas_utils import _extract_instruction_text +from judgearena.benchmarks.elo.rating import winner_to_pref +from judgearena.evaluate import judge_and_parse_prefs +from judgearena.log import get_logger +from judgearena.models import make_model +from judgearena.prompts.parsing import PairScore +from judgearena.prompts.registry import ResolvedJudgePrompt + +logger = get_logger(__name__) + + +def fit_temperature( + delta_s: np.ndarray, + y: np.ndarray, + bounds: tuple[float, float] = (-10.0, 10.0), +) -> float: + """Fit ``T`` in ``P(A>B) = sigmoid(T * (score_A - score_B))``.""" + delta_s = np.asarray(delta_s, dtype=float) + y = np.asarray(y, dtype=float) + non_tie = y != 0.5 + delta_s = delta_s[non_tie] + y = y[non_tie] + if len(delta_s) == 0: + raise ValueError( + "No non-tie observations available for temperature calibration." + ) + + agreement = (2 * y - 1) * delta_s + + def negative_log_likelihood(temperature: float) -> float: + return float(np.sum(np.logaddexp(0.0, -temperature * agreement))) + + result = minimize_scalar( + negative_log_likelihood, + bounds=bounds, + method="bounded", + ) + return float(result.x) + + +def calibrate_pairscore_temperature( + arena_battles: pd.DataFrame, + source_battles: pd.DataFrame, + *, + enabled: bool, + soft_elo: bool, + sample_size: int | None, + rng: np.random.Generator, + judge_model: str, + judge_model_kwargs: Mapping[str, object], + swap_mode: str, + prompt: ResolvedJudgePrompt, + truncate_input_chars: int | None, + default_temperature: float, +) -> float | None: + """Judge sampled human battles and return a fitted PairScore temperature.""" + if not enabled: + return None + if not soft_elo: + logger.warning( + "--calibrate-temperature has no effect with --no-soft-elo; skipping." + ) + return None + if not isinstance(prompt.parser, PairScore): + parser_name = getattr(prompt.parser, "name", type(prompt.parser).__name__) + logger.warning( + "PairScore temperature calibration does not apply to parser %r; " + "using its preferences unchanged.", + parser_name, + ) + return None + + logger.info("Calibrating PairScore temperature against human annotations.") + n_samples = ( + min(sample_size, len(arena_battles)) + if sample_size is not None + else len(arena_battles) + ) + calibration_battles = arena_battles.sample( + n=n_samples, + random_state=int(rng.integers(0, 2**31)), + ) + instructions = [ + _extract_instruction_text(source_battles.loc[index, "conversation_a"][0]) + for index in calibration_battles.index + ] + completions_a = [ + _extract_instruction_text(source_battles.loc[index, "conversation_a"][1]) + for index in calibration_battles.index + ] + completions_b = [ + _extract_instruction_text(source_battles.loc[index, "conversation_b"][1]) + for index in calibration_battles.index + ] + + calibration_judge = make_model(model=judge_model, **dict(judge_model_kwargs)) + annotations, _, _ = judge_and_parse_prefs( + judge_chat_model=calibration_judge, + instructions=instructions, + completions_A=completions_a, + completions_B=completions_b, + swap_mode=swap_mode, + system_prompt=prompt.system_prompt, + user_prompt_template=prompt.user_prompt_template, + prompt_preset=prompt.preset_name, + parse=prompt.parser, + truncate_input_chars=truncate_input_chars, + ) + + score_differences: list[float] = [] + outcomes: list[float] = [] + for annotation, human_winner in zip( + annotations, calibration_battles["winner"].tolist(), strict=True + ): + scores = {} if annotation.parsed is None else annotation.parsed.scores + score_a = scores.get("A") + score_b = scores.get("B") + if score_a is None or score_b is None: + continue + human_preference = winner_to_pref(human_winner) + if human_preference is None or human_preference == 0.5: + continue + score_differences.append(score_a - score_b) + outcomes.append(1.0 - human_preference) + + if len(score_differences) < 10: + logger.warning( + "Only %d valid calibration pairs (need ≥10); keeping default temperature.", + len(score_differences), + ) + return None + + temperature = fit_temperature( + np.array(score_differences), + np.array(outcomes), + ) + logger.info( + "Calibration pairs: %d T* = %.4f (default was %s)", + len(score_differences), + temperature, + default_temperature, + ) + return temperature diff --git a/judgearena/benchmarks/elo/rating.py b/judgearena/benchmarks/elo/rating.py index 5c1cba13..0965d510 100644 --- a/judgearena/benchmarks/elo/rating.py +++ b/judgearena/benchmarks/elo/rating.py @@ -142,7 +142,9 @@ def prefs_to_battle_results( for pref, is_pos_a, opponent in zip( prefs, our_model_is_position_a, opponent_models, strict=True ): - if _is_nan_pref(pref) or pref == 0.5: + if _is_nan_pref(pref): + winner = None + elif pref == 0.5: winner = "tie" elif pref < 0.5: winner = "model_a" diff --git a/judgearena/benchmarks/elo/runner.py b/judgearena/benchmarks/elo/runner.py index 1dcfb1b0..bc31bf6c 100644 --- a/judgearena/benchmarks/elo/runner.py +++ b/judgearena/benchmarks/elo/runner.py @@ -13,20 +13,19 @@ safe_filename, write_run_metadata_safely, ) -from judgearena.battles import Leaderboard, summarize_bootstrap, write_battles +from judgearena.battles import Leaderboard, RatingEntry, write_battles +from judgearena.benchmarks.elo.calibration import calibrate_pairscore_temperature from judgearena.benchmarks.elo.rating import ( arena_anchor_battles, prefs_to_battle_results, sampling_cache_token, select_seeded_random_arena_battles, - winner_to_pref, ) -from judgearena.benchmarks.elo.scoring import ELO_SCORERS from judgearena.benchmarks.execution import build_generation_kwargs +from judgearena.benchmarks.scoring import build_metrics, calculate_metrics from judgearena.datasets import load_battles from judgearena.evaluate import ( PairScore, - calibrate_temperature, combine_swapped_prefs, judge_and_parse_prefs, resolve_run_judge_prompt, @@ -34,9 +33,9 @@ from judgearena.generate import generate_instructions from judgearena.log import get_logger from judgearena.models import build_default_judge_model_kwargs, make_model +from judgearena.reports import EloReport from judgearena.tasks.schema import EloProtocol, ResolvedTaskSpec -from judgearena.utils import cache_function_dataframe, compute_pref_summary -from judgearena.utils.eval import PrefSummary, Report +from judgearena.utils import cache_function_dataframe if TYPE_CHECKING: from judgearena.config import RunConfig @@ -44,99 +43,6 @@ logger = get_logger(__name__) -class EloReport(Report): - """Bradley-Terry / Soft-ELO ratings for one focal model against an arena. - - This is the console/``results-*.json`` run report. The narrower per-model - leaderboard with bootstrap CIs is persisted separately as - ``elo_ratings.json`` via :class:`judgearena.battles.Leaderboard`. - """ - - arena: str - """Arena/benchmark the focal model is rated against.""" - judge_model: str - """LLM judge that scored the battles.""" - summary: PrefSummary - """Win/loss/tie stats for the focal model's LLM-judged battles.""" - num_battles: int - """Total battles (LLM-judged + human-anchor).""" - llm_judged_battles: int - """Battles the LLM judged for the focal model.""" - human_anchor_battles: int - """Human-annotated battles anchoring the other arena models.""" - elo_mean: float - """Focal model's mean ELO across bootstrap samples.""" - elo_std: float - """Std of the focal model's ELO across bootstrap samples.""" - elo_n_bootstraps: int - """Bootstrap samples that rated the focal model (≤ n_bootstraps); the n behind elo_mean/elo_std.""" - mae_vs_human: float - """Mean absolute error of estimated vs human ELO over overlapping models.""" - method: str - """Rating method label (e.g. "Soft-ELO").""" - n_bootstraps: int - """Total bootstrap iterations run.""" - model_name: str - """Focal model under evaluation.""" - mean_ratings: dict[str, float] - """Per-model mean ELO across bootstraps.""" - battle_counts: dict[str, int] - """Per-model battle count.""" - human_elo: dict[str, float] - """Per-model human-derived ELO (anchors).""" - bootstrap_ratings: list[dict[str, float]] - """One model→ELO dict per bootstrap sample.""" - sampling_metadata: dict[str, object] - """Instruction-sampling parameters for the run.""" - - def render(self) -> None: - s = self.summary - print(f"\n=== Results for {self.model_name} ===") - print( - f"Battles: {self.llm_judged_battles} | Wins: {s.num_wins} | " - f"Losses: {s.num_losses} | Ties: {s.num_ties}" - ) - print(f"Win rate: {s.winrate:.2%}") - - print( - f"\n=== {self.method} Ratings (Bradley-Terry, " - f"{self.n_bootstraps} bootstraps) ===" - ) - print( - f"Estimating {self.method} Ratings with {self.llm_judged_battles} " - f"LLM-judges for model {self.model_name} and {self.human_anchor_battles} " - "human annotations for other models. Number of battles is indicated in " - "parenthesis and confidence intervals are reported by computing ELO on " - f"{self.n_bootstraps} samples of instructions." - ) - - if not self.mean_ratings: - print(" Not enough data to compute ELO ratings.") - return - - # Percentile CIs (not mean ± std): matches the bounds persisted in - # elo_ratings.json so the console and the saved leaderboard never disagree. - for e in summarize_bootstrap( - self.bootstrap_ratings, self.battle_counts, self.model_name - ): - suffix = " <-----" if e.model == self.model_name else "" - print( - f" {e.model} ({e.n_battles}){suffix}: " - f"{e.rating:.1f} [{e.ci_low:.1f}, {e.ci_high:.1f}]" - ) - - overlap = [ - m for m in self.mean_ratings if m in self.human_elo and m != self.model_name - ] - if overlap: - print( - f"\n MAE vs Human-ELO ({len(overlap)} arena models): " - f"{self.mae_vs_human:.1f}" - ) - else: - print("\n No overlapping arena models to compute MAE.") - - def run_elo(cfg: "RunConfig", task: ResolvedTaskSpec | None = None) -> dict: """Rate one model against the human battles defined by an ELO task.""" protocol = task.spec.protocol if task is not None else None @@ -145,7 +51,6 @@ def run_elo(cfg: "RunConfig", task: ResolvedTaskSpec | None = None) -> dict: if cfg.elo is None: raise ValueError(f"Task {cfg.task!r} requires ELO runtime settings.") arena = protocol.arena - scorer = ELO_SCORERS[protocol.scoring.adapter] run_started_at = datetime.now(UTC) rng = np.random.default_rng(cfg.run.seed) @@ -395,124 +300,26 @@ def run_judge() -> pd.DataFrame: logger.debug("First judge output:\n%s", df_judge["judge_completion"].iloc[0][:500]) - # Map preferences back to model-name-level battle results. model_name = cfg.model.name - df_llm_judge = prefs_to_battle_results( - prefs, - our_model_is_position_a, - opponent_models, - model_name, - judge_model=cfg.judge.model, - question_ids=question_ids, - ) - - # Normalize prefs so pref < 0.5 always means our model wins, then summarise - prefs_normalized = pd.Series( - [ - p if (p is None or is_pos_a) else (1 - p) - for p, is_pos_a in zip(prefs, our_model_is_position_a, strict=True) - ] - ) - summary = compute_pref_summary(prefs_normalized) - # Anchor the llm-judge battles against the human arena battles. These are # rebuilt from the (revision-pinned) arena, not persisted per run. df_arena = arena_anchor_battles(df_arena_all) - df_results = pd.concat([df_llm_judge, df_arena], ignore_index=True) - - # Compute human-only BT ratings as ground-truth reference - human_elo = scorer.fit( - df_arena, pref_col="pref_hard", baseline_model=cfg.elo.baseline_model + calibrated_temperature = calibrate_pairscore_temperature( + df_arena, + df_arena_all, + enabled=cfg.elo.calibrate_temperature, + soft_elo=cfg.elo.soft_elo, + sample_size=cfg.elo.calibration_size, + rng=rng, + judge_model=cfg.judge.model, + judge_model_kwargs=judge_extra_kwargs, + swap_mode=cfg.judge.swap_mode, + prompt=resolved_prompt, + truncate_input_chars=cfg.generation.truncate_judge_input_chars, + default_temperature=cfg.elo.soft_elo_temperature, ) - # --- Temperature calibration (optional) --- - # Run the judge on a random subset of human arena battles that already - # have ground-truth winner labels so we can fit T* via MLE. - calibrated_temperature: float | None = None - if cfg.elo.calibrate_temperature: - if not cfg.elo.soft_elo: - logger.warning( - "--calibrate-temperature has no effect with --no-soft-elo; skipping." - ) - else: - logger.info("Calibrating PairScore temperature against human annotations.") - # Sample calibration battles from the already-loaded arena battles. - # Use the same judge to score them so scores and labels are comparable. - _cal_n = ( - min(cfg.elo.calibration_size, len(df_arena)) - if cfg.elo.calibration_size is not None - else len(df_arena) - ) - # Keep the original df_arena_all index so we can look up the full - # conversation rows below; reset_index would point at non-existent - # 0..N labels in df_arena_all. - cal_battles = df_arena.sample( - n=_cal_n, random_state=int(rng.integers(0, 2**31)) - ) - - cal_instructions = [ - _extract_instruction_text(df_arena_all.loc[i, "conversation_a"][0]) - for i in cal_battles.index - ] - cal_completions_a = [ - _extract_instruction_text(df_arena_all.loc[i, "conversation_a"][1]) - for i in cal_battles.index - ] - cal_completions_b = [ - _extract_instruction_text(df_arena_all.loc[i, "conversation_b"][1]) - for i in cal_battles.index - ] - - judge_chat_model_cal = make_model( - model=cfg.judge.model, - **judge_extra_kwargs, - ) - cal_annotations, _, cal_prefs = judge_and_parse_prefs( - judge_chat_model=judge_chat_model_cal, - instructions=cal_instructions, - completions_A=cal_completions_a, - completions_B=cal_completions_b, - swap_mode=cfg.judge.swap_mode, - system_prompt=resolved_prompt.system_prompt, - user_prompt_template=resolved_prompt.user_prompt_template, - prompt_preset=resolved_prompt.preset_name, - parse=resolved_prompt.parser, - truncate_input_chars=cfg.generation.truncate_judge_input_chars, - ) - - # Build (delta_s, y) pairs from calibration battles. - # delta_s = score_A - score_B, extracted exactly as the main run does. - delta_s_cal = [] - y_cal = [] - for ann, human_winner in zip( - cal_annotations, cal_battles["winner"].tolist(), strict=True - ): - sa, sb = PairScore.parse_raw_scores(ann.judge_completion) - if sa is None or sb is None: - continue - human_pref = winner_to_pref(human_winner) - if human_pref is None or human_pref == 0.5: - continue # skip ties and missing - delta_s_cal.append(sa - sb) - y_cal.append(1.0 - human_pref) # pref=0 → A wins → y=1 - - if len(delta_s_cal) < 10: - logger.warning( - "Only %d valid calibration pairs (need ≥10); keeping default temperature.", - len(delta_s_cal), - ) - else: - calibrated_temperature = calibrate_temperature( - np.array(delta_s_cal), np.array(y_cal) - ) - logger.info( - "Calibration pairs: %d T* = %.4f (default was %s)", - len(delta_s_cal), - calibrated_temperature, - cfg.elo.soft_elo_temperature, - ) - # Build the score parser used for the main evaluation run. score_parser = PairScore( temperature=calibrated_temperature @@ -525,7 +332,7 @@ def run_judge() -> pd.DataFrame: # --soft-elo-temperature (or a calibrated T*). Re-parse from the stored # judge completions with this run's score_parser so the soft-ELO bootstrap # uses the requested temperature. - if cfg.elo.soft_elo: + if cfg.elo.soft_elo and isinstance(resolved_prompt.parser, PairScore): new_prefs_ab = pd.Series( [score_parser.parse_model_raw(c) for c in df_judge["judge_completion"]] ).apply(lambda x: float("nan") if x is None else x) @@ -540,83 +347,62 @@ def run_judge() -> pd.DataFrame: else: prefs = new_prefs_ab.tolist() - # Rebuild battle results with the re-parsed prefs. - df_llm_judge = prefs_to_battle_results( - prefs, - our_model_is_position_a, - opponent_models, - model_name, - judge_model=cfg.judge.model, - question_ids=question_ids, - ) - df_results = pd.concat([df_llm_judge, df_arena], ignore_index=True) - - n_bootstraps = cfg.elo.n_bootstraps - use_soft = cfg.elo.soft_elo - - n_llm = len(df_llm_judge) - n_human = len(df_arena) - method_label = "Soft-ELO" if use_soft else "ELO" - - # Count battles per model across the combined results - battle_counts: dict[str, int] = {} - for _, row in df_results.iterrows(): - battle_counts[row["model_a"]] = battle_counts.get(row["model_a"], 0) + 1 - battle_counts[row["model_b"]] = battle_counts.get(row["model_b"], 0) + 1 - - pref_col = "pref" if use_soft else "pref_hard" - bootstrap_ratings: list[dict[str, float]] = [] - for _ in range(n_bootstraps): - df_sample = df_results.sample( - n=len(df_results), replace=True, random_state=int(rng.integers(0, 2**31)) - ) - ratings = scorer.fit( - df_sample, pref_col=pref_col, baseline_model=cfg.elo.baseline_model - ) - bootstrap_ratings.append(ratings) - - # One percentile-CI summary, reused for the console report, the MAE - # calculation below, and the persisted elo_ratings.json leaderboard so - # none of the three can disagree. - entries: list = [] - mean_ratings: dict[str, float] = {} - mae = np.nan - if bootstrap_ratings: - entries = summarize_bootstrap(bootstrap_ratings, battle_counts, model_name) - mean_ratings = {e.model: e.rating for e in entries} - overlap = [m for m in mean_ratings if m in human_elo and m != model_name] - if overlap: - abs_errors = [abs(mean_ratings[m] - human_elo[m]) for m in overlap] - mae = np.mean(abs_errors) - - model_rating_values = [ - rating[model_name] for rating in bootstrap_ratings if model_name in rating + # Map the final canonical preferences to model-name-level battle results. + df_llm_judge = prefs_to_battle_results( + prefs, + our_model_is_position_a, + opponent_models, + model_name, + judge_model=cfg.judge.model, + question_ids=question_ids, + ) + + # Mark and enrich the rows created for the model under evaluation. Human + # anchors keep these columns null. Focal metrics can therefore consume the + # same combined battle table as Bradley-Terry without knowing this runner. + repeats = max(1, len(df_llm_judge) // max(1, len(our_completions))) + row_our_completions = (list(our_completions) * repeats)[: len(df_llm_judge)] + row_opponent_completions = (list(opponent_completions) * repeats)[ + : len(df_llm_judge) ] - elo_mean = ( - float(np.mean(model_rating_values)) if model_rating_values else float("nan") + focal_is_a = pd.Series(our_model_is_position_a, dtype="bool") + df_llm_judge["evaluation_model"] = model_name + df_llm_judge["completion_a"] = pd.Series(row_our_completions).where( + focal_is_a, row_opponent_completions ) - elo_std = ( - float(np.std(model_rating_values)) if model_rating_values else float("nan") + df_llm_judge["completion_b"] = pd.Series(row_opponent_completions).where( + focal_is_a, row_our_completions ) + df_llm_judge["instruction_index"] = question_ids + if cfg.judge.swap_mode == "both": + half = len(df_llm_judge) // 2 + df_llm_judge["orientation"] = ["direct"] * half + ["reversed"] * half + else: + df_llm_judge["orientation"] = "single" + df_results = pd.concat([df_llm_judge, df_arena], ignore_index=True) + + metrics = build_metrics( + protocol.scoring.metrics, + parameter_overrides_by_metric={ + "bradley_terry": { + "n_bootstraps": cfg.elo.n_bootstraps, + "baseline_model": cfg.elo.baseline_model, + "soft": cfg.elo.soft_elo, + }, + }, + ) + metric_results = calculate_metrics( + df_results, + metrics, + runtime_by_metric={"bradley_terry": {"rng": rng}}, + ) report = EloReport( arena=arena, judge_model=cfg.judge.model, - summary=summary, + metrics=metric_results, num_battles=n, - llm_judged_battles=n_llm, - human_anchor_battles=n_human, - elo_mean=elo_mean, - elo_std=elo_std, - elo_n_bootstraps=len(model_rating_values), - mae_vs_human=mae, - method=method_label, - n_bootstraps=n_bootstraps, model_name=model_name, - mean_ratings=mean_ratings, - battle_counts=battle_counts, - human_elo=human_elo, - bootstrap_ratings=bootstrap_ratings, sampling_metadata=sampling_metadata, ) results = report.to_dict() @@ -652,15 +438,17 @@ def run_judge() -> pd.DataFrame: res_dir / "battles.parquet", df_llm_judge[[c for c in battle_cols if c in df_llm_judge.columns]], ) - if bootstrap_ratings: - pd.DataFrame(bootstrap_ratings).to_csv( + rating_result = metric_results.get("bradley_terry") + if rating_result is not None and rating_result["bootstrap_ratings"]: + pd.DataFrame(rating_result["bootstrap_ratings"]).to_csv( res_dir / "bootstrap_ratings.csv", index=False ) + entries = [RatingEntry(**entry) for entry in rating_result["rating_entries"]] Leaderboard( arena=arena, model=model_name, judge_model=cfg.judge.model, - n_bootstraps=n_bootstraps, + n_bootstraps=rating_result["n_bootstraps"], seed=cfg.run.seed, ratings=entries, ).write(res_dir / "elo_ratings.json") diff --git a/judgearena/benchmarks/elo/scoring.py b/judgearena/benchmarks/elo/scoring.py index af6c01d9..71e8705c 100644 --- a/judgearena/benchmarks/elo/scoring.py +++ b/judgearena/benchmarks/elo/scoring.py @@ -1,28 +1,197 @@ -"""Runtime scoring adapters for ELO rating tasks. - -A scorer turns judged battles into the metric a benchmark reports. Each -protocol defines its own, so scorers are named components selected by task -YAML rather than logic in the runner: the runner produces preferences, the -scorer owns the metric math. -""" +"""Battle-table scoring functions for Elo tasks.""" from __future__ import annotations -from collections.abc import Callable -from dataclasses import dataclass +from dataclasses import asdict, dataclass + +import numpy as np +import pandas as pd +from judgearena.battles import summarize_bootstrap from judgearena.benchmarks.elo.rating import fit_bradley_terry -RatingFunction = Callable[..., dict[str, float]] +@dataclass(frozen=True, kw_only=True) +class BradleyTerryMetric: + """Configured arena-anchored Bradley-Terry calculation.""" + + n_bootstraps: int = 0 + baseline_model: str | None = None + soft: bool = True + + def __post_init__(self) -> None: + if type(self.n_bootstraps) is not int or self.n_bootstraps < 0: + raise ValueError("n_bootstraps must be a non-negative integer") + if self.baseline_model is not None and not isinstance(self.baseline_model, str): + raise TypeError("baseline_model must be a string or None") + if type(self.soft) is not bool: + raise TypeError("soft must be a boolean") + + def calculate( + self, + battles: pd.DataFrame, + *, + rng: np.random.Generator | None = None, + ) -> dict[str, object]: + """Calculate Bradley-Terry ratings from battle rows.""" + required = {"model_a", "model_b", "pref"} + if not self.soft: + required.add("pref_hard") + missing = sorted(required - set(battles.columns)) + if missing: + raise ValueError(f"Bradley-Terry battles are missing columns: {missing}.") + scoring_battles = battles.copy() + if not self.soft: + scoring_battles["pref"] = scoring_battles["pref_hard"] + point_ratings = fit_bradley_terry( + scoring_battles, pref_col="pref", baseline_model=self.baseline_model + ) + lifecycle_columns = {"pref_hard", "source", "evaluation_model"} + missing_lifecycle = sorted(lifecycle_columns - set(battles.columns)) + if missing_lifecycle: + if self.n_bootstraps > 0: + raise ValueError( + "Bootstrapped Bradley-Terry battles are missing columns: " + f"{missing_lifecycle}." + ) + return {"ratings": point_ratings} + + evaluation_models = battles["evaluation_model"].dropna().unique() + if len(evaluation_models) != 1: + raise ValueError( + "Bradley-Terry requires exactly one model under evaluation." + ) + evaluation_model = str(evaluation_models[0]) + if self.n_bootstraps > 0 and rng is None: + raise ValueError("Bootstrapped Bradley-Terry requires an RNG.") + + human_battles = scoring_battles.loc[scoring_battles["source"] == "human"].copy() + human_battles["pref"] = human_battles["pref_hard"] + human_ratings = fit_bradley_terry( + human_battles, pref_col="pref", baseline_model=self.baseline_model + ) + + battle_counts: dict[str, int] = {} + for model in pd.concat( + [scoring_battles["model_a"], scoring_battles["model_b"]] + ): + battle_counts[model] = battle_counts.get(model, 0) + 1 -@dataclass(frozen=True) -class EloScorer: - """Rating implementation selected by an ELO task's scoring adapter.""" + bootstrap_ratings: list[dict[str, float]] = [] + for _ in range(self.n_bootstraps): + assert rng is not None + sample = scoring_battles.sample( + n=len(scoring_battles), + replace=True, + random_state=int(rng.integers(0, 2**31)), + ) + bootstrap_ratings.append( + fit_bradley_terry( + sample, pref_col="pref", baseline_model=self.baseline_model + ) + ) - fit: RatingFunction + rating_entries = ( + summarize_bootstrap(bootstrap_ratings, battle_counts, evaluation_model) + if bootstrap_ratings + else [] + ) + mean_ratings = {entry.model: entry.rating for entry in rating_entries} + overlap = [ + model + for model in mean_ratings + if model in human_ratings and model != evaluation_model + ] + mae_vs_human = ( + float( + np.mean( + [ + abs(mean_ratings[model] - human_ratings[model]) + for model in overlap + ] + ) + ) + if overlap + else float("nan") + ) + model_rating_values = [ + ratings[evaluation_model] + for ratings in bootstrap_ratings + if evaluation_model in ratings + ] + return { + "ratings": point_ratings, + "rating": float(np.mean(model_rating_values)) + if model_rating_values + else float("nan"), + "rating_std": float(np.std(model_rating_values)) + if model_rating_values + else float("nan"), + "rating_n_bootstraps": len(model_rating_values), + "mean_ratings": mean_ratings, + "human_ratings": human_ratings, + "bootstrap_ratings": bootstrap_ratings, + "rating_entries": [asdict(entry) for entry in rating_entries], + "battle_counts": battle_counts, + "mae_vs_human": mae_vs_human, + "mae_num_models": len(overlap), + "n_bootstraps": self.n_bootstraps, + "method": "Soft-ELO" if self.soft else "ELO", + "evaluation_model": evaluation_model, + "llm_judged_battles": int((battles["source"] == "llm-judge").sum()), + "human_anchor_battles": len(human_battles), + } -ELO_SCORERS = { - "bradley_terry": EloScorer(fit=fit_bradley_terry), -} + @staticmethod + def render(values: dict[str, object]) -> str: + """Render point or arena-anchored Bradley-Terry values.""" + if "method" not in values: + lines = ["bradley_terry ratings:"] + ratings = values["ratings"] + if not ratings: + lines.append(" Not enough data to compute ratings.") + else: + lines.extend( + f" {model}: {rating:.1f}" + for model, rating in sorted( + ratings.items(), key=lambda item: -item[1] + ) + ) + return "\n".join(lines) + lines = [ + f"bradley_terry: {values['method']} ratings " + f"({values['n_bootstraps']} bootstraps)", + f"{values['llm_judged_battles']} judged battles and " + f"{values['human_anchor_battles']} human anchor battles", + ] + entries = values["rating_entries"] + if not entries: + ratings = values["ratings"] + if ratings: + lines.append("Point ratings:") + lines.extend( + f" {model}: {rating:.1f}" + for model, rating in sorted( + ratings.items(), key=lambda item: -item[1] + ) + ) + else: + lines.append(" Not enough data to compute ratings.") + return "\n".join(lines) + evaluation_model = values["evaluation_model"] + for entry in entries: + suffix = " <-----" if entry["model"] == evaluation_model else "" + lines.append( + f" {entry['model']} ({entry['n_battles']}){suffix}: " + f"{entry['rating']:.1f} " + f"[{entry['ci_low']:.1f}, {entry['ci_high']:.1f}]" + ) + if values["mae_num_models"]: + lines.append( + f"MAE vs Human-ELO ({values['mae_num_models']} arena models): " + f"{values['mae_vs_human']:.1f}" + ) + else: + lines.append("No overlapping arena models to compute MAE.") + return "\n".join(lines) diff --git a/judgearena/benchmarks/mt_bench/preset_judging.py b/judgearena/benchmarks/mt_bench/preset_judging.py index 5a8154db..fc0de802 100644 --- a/judgearena/benchmarks/mt_bench/preset_judging.py +++ b/judgearena/benchmarks/mt_bench/preset_judging.py @@ -224,6 +224,13 @@ def _append_results( "question_id": item.question_id, "category": item.category, "turn": item.turn, + "orientation": ( + "single" + if swap_mode == "fixed" + else "reversed" + if swapped + else "direct" + ), } ) preferences.append(normalized_preference) diff --git a/judgearena/benchmarks/mt_bench/runner.py b/judgearena/benchmarks/mt_bench/runner.py index e031d5d2..71499d60 100644 --- a/judgearena/benchmarks/mt_bench/runner.py +++ b/judgearena/benchmarks/mt_bench/runner.py @@ -11,6 +11,7 @@ from pathlib import Path from typing import TYPE_CHECKING +import numpy as np import pandas as pd from judgearena.artifacts import prepare_run_directory, write_run_metadata_safely @@ -19,7 +20,7 @@ ) from judgearena.benchmarks.mt_bench.preset_judging import judge_mt_bench_with_preset from judgearena.benchmarks.pairwise.baselines import native_pairwise_baseline -from judgearena.benchmarks.pairwise.scoring import PAIRWISE_SCORERS +from judgearena.benchmarks.scoring import build_metrics, calculate_metrics from judgearena.datasets import load_instructions from judgearena.datasets.mt_bench import ( load_mt_bench_model_answers, @@ -28,12 +29,12 @@ from judgearena.log import get_logger from judgearena.models import is_thinking_model, make_model from judgearena.prompts.registry import ResolvedJudgePrompt, resolve_run_judge_prompt +from judgearena.reports import BattleReport from judgearena.tasks.schema import MTBenchProtocol from judgearena.utils import ( cache_function_dataframe, generation_cache_token, ) -from judgearena.utils.eval import BattleReport, _compute_grouped_stats logger = get_logger(__name__) @@ -188,6 +189,50 @@ def _save_mt_bench_results( ) +def _build_mt_bench_battles( + *, + cfg: RunConfig, + prefs: pd.Series, + combined_metadata: list[dict[str, object]], + completions_a: pd.DataFrame, + completions_b: pd.DataFrame, +) -> pd.DataFrame: + """Build the canonical metric table for judged MT-Bench turns.""" + rows: list[dict[str, object]] = [] + for metadata, pref in zip(combined_metadata, prefs, strict=True): + question_id = metadata["question_id"] + turn = int(metadata["turn"]) + completion_column = f"completion_turn_{turn}" + pref_hard = pref + if pref < 0.5: + pref_hard = 0.0 + elif pref > 0.5: + pref_hard = 1.0 + rows.append( + { + **metadata, + "instruction_index": f"{question_id}:turn-{turn}", + "model": cfg.model.name, + "baseline": cfg.model.baseline, + "completion_model": completions_a.loc[question_id, completion_column], + "completion_baseline": completions_b.loc[ + question_id, completion_column + ], + "model_a": cfg.model.name, + "model_b": cfg.model.baseline, + "completion_a": completions_a.loc[question_id, completion_column], + "completion_b": completions_b.loc[question_id, completion_column], + "evaluation_model": cfg.model.name, + "source": "llm-judge", + "orientation": metadata.get("orientation", "single"), + "judge": cfg.judge.model, + "pref": pref, + "pref_hard": pref_hard, + } + ) + return pd.DataFrame(rows) + + def _finalize_mt_bench_run( *, cfg: RunConfig, @@ -204,18 +249,27 @@ def _finalize_mt_bench_run( started_at_utc: datetime, extra_result_fields: dict[str, object] | None = None, ) -> pd.Series: - scorer = PAIRWISE_SCORERS[protocol.scoring.adapter] - # MT-Bench battles carry per-turn prefs only; the win-rate scorer reads - # just the pref column of the canonical battles frame. - stats = scorer(pd.DataFrame({"pref": pd.Series(prefs, dtype="float64")})) + battles = _build_mt_bench_battles( + cfg=cfg, + prefs=pd.Series(prefs, dtype="float64"), + combined_metadata=combined_metadata, + completions_a=completions_a, + completions_b=completions_b, + ) + metrics = build_metrics(protocol.scoring.metrics) + metric_results = calculate_metrics( + battles, + metrics, + runtime_by_metric={ + "bradley_terry": {"rng": np.random.default_rng(cfg.run.seed)}, + }, + ) report = BattleReport( task=cfg.task, model_a=cfg.model.name, model_b=cfg.model.baseline, judge_model=cfg.judge.model, - summary=stats, - per_category=_compute_grouped_stats(prefs, combined_metadata, "category"), - per_turn=_compute_grouped_stats(prefs, combined_metadata, "turn"), + metrics=metric_results, preferences=prefs.tolist(), metadata={ **resolved_prompt.metadata(), diff --git a/judgearena/benchmarks/pairwise/runner.py b/judgearena/benchmarks/pairwise/runner.py index ec7daa3e..3868f50c 100644 --- a/judgearena/benchmarks/pairwise/runner.py +++ b/judgearena/benchmarks/pairwise/runner.py @@ -8,20 +8,21 @@ from pathlib import Path from typing import TYPE_CHECKING +import numpy as np import pandas as pd from judgearena.artifacts import prepare_run_directory, write_run_metadata_safely from judgearena.benchmarks.execution import build_generation_kwargs, build_judge from judgearena.benchmarks.pairwise.baselines import resolve_baseline_plan -from judgearena.benchmarks.pairwise.scoring import PAIRWISE_SCORERS +from judgearena.benchmarks.scoring import build_metrics, calculate_metrics from judgearena.datasets.pairwise import load_pairwise_task_data from judgearena.evaluate import judge_and_parse_prefs, resolve_run_judge_prompt from judgearena.generate import generate_base, generate_instructions from judgearena.log import get_logger +from judgearena.reports import BattleReport from judgearena.tasks.registry import get_packaged_task from judgearena.tasks.schema import ResolvedTaskSpec from judgearena.utils import cache_function_dataframe, generation_cache_token -from judgearena.utils.eval import BattleReport if TYPE_CHECKING: from judgearena.config import RunConfig @@ -102,13 +103,45 @@ def run_pairwise(cfg: "RunConfig", resolved_task: ResolvedTaskSpec | None = None resolved_task = resolved_task or get_packaged_task(cfg.task) if resolved_task is None: raise ValueError(f"Unknown task {cfg.task!r}.") - scorer = PAIRWISE_SCORERS[resolved_task.spec.protocol.scoring.adapter] task_data = load_pairwise_task_data( resolved_task, n_instructions=cfg.generation.n_instructions, ) instructions_df = task_data.instructions instructions = instructions_df.loc[:, "instruction"] + metric_columns = { + "instruction_index", + "model", + "baseline", + "completion_model", + "completion_baseline", + "pref", + "orientation", + "judge", + "judge_prompt_preset", + "judge_temperature", + "judge_max_out_tokens", + "model_a", + "model_b", + "completion_a", + "completion_b", + "evaluation_model", + "source", + "pref_hard", + *instructions_df.columns, + } + missing_groups = sorted( + { + field + for request in resolved_task.spec.protocol.scoring.metrics + for field in request.breakdown_by + } + - metric_columns + ) + if missing_groups: + raise ValueError( + f"Metric breakdown_by columns are unavailable: {missing_groups}." + ) n_instructions = ( cfg.generation.n_instructions @@ -273,6 +306,11 @@ def _load_or_generate_completions(model_spec: str, *, role: str) -> pd.Series: eval_instruction_index = [ index for _, group_index in prompt_groups for index in group_index ] + eval_prompt_presets = [ + group_prompt.preset_name + for group_prompt, group_index in prompt_groups + for _ in group_index + ] baseline_per_eval = baseline_per_index.loc[eval_instruction_index] df = pd.DataFrame(annotations) df["instruction_index"] = eval_instruction_index @@ -305,29 +343,67 @@ def _load_or_generate_completions(model_spec: str, *, role: str) -> pd.Series: df.to_csv(res_folder / f"{name}-annotations.csv", index=False) - # Scorers see one canonically-oriented row per judged battle; under - # swap_mode="both" every instruction contributes two rows. + # Metrics see one canonically-oriented row per judgment. Under + # swap_mode="both", every physical battle contributes both answer orders. repeats = 2 if cfg.judge.swap_mode == "both" else 1 - battles = pd.DataFrame( - { - "instruction_index": list(eval_instruction_index) * repeats, - "model": cfg.model.name, - "baseline": baseline_per_eval.tolist() * repeats, - "completion_model": completions_A.loc[eval_instruction_index].tolist() - * repeats, - "completion_baseline": completions_B.loc[eval_instruction_index].tolist() - * repeats, - "pref": pd.Series(prefs, dtype="float64").to_numpy(), - } + battle_data = { + "instruction_index": list(eval_instruction_index) * repeats, + "model": cfg.model.name, + "baseline": baseline_per_eval.tolist() * repeats, + "completion_model": completions_A.loc[eval_instruction_index].tolist() + * repeats, + "completion_baseline": completions_B.loc[eval_instruction_index].tolist() + * repeats, + "model_a": cfg.model.name, + "model_b": baseline_per_eval.tolist() * repeats, + "completion_a": completions_A.loc[eval_instruction_index].tolist() * repeats, + "completion_b": completions_B.loc[eval_instruction_index].tolist() * repeats, + "evaluation_model": cfg.model.name, + "source": "llm-judge", + "pref": pd.Series(prefs, dtype="float64").to_numpy(), + "orientation": ( + ["direct"] * len(eval_instruction_index) + + ["reversed"] * len(eval_instruction_index) + ) + if repeats == 2 + else ["single"] * len(eval_instruction_index), + "judge": cfg.judge.model, + "judge_prompt_preset": eval_prompt_presets * repeats, + "judge_temperature": cfg.judge.temperature, + "judge_max_out_tokens": cfg.judge.max_out_tokens, + } + for column in instructions_df.columns: + if column not in battle_data: + battle_data[column] = ( + instructions_df.loc[eval_instruction_index, column].tolist() * repeats + ) + battles = pd.DataFrame(battle_data) + battles["pref_hard"] = battles["pref"].map( + lambda pref: ( + float("nan") + if pd.isna(pref) + else 0.0 + if pref < 0.5 + else 1.0 + if pref > 0.5 + else 0.5 + ) + ) + metrics = build_metrics(resolved_task.spec.protocol.scoring.metrics) + metric_results = calculate_metrics( + battles, + metrics, + runtime_by_metric={ + "bradley_terry": {"rng": np.random.default_rng(cfg.run.seed)}, + }, ) - summary = scorer(battles) report = BattleReport( task=cfg.task, model_a=cfg.model.name, model_b=baseline_plan.display_name, judge_model=cfg.judge.model, - summary=summary, + metrics=metric_results, swap_mode=cfg.judge.swap_mode, result_folder=str(res_folder), preferences=prefs.tolist(), diff --git a/judgearena/benchmarks/pairwise/scoring.py b/judgearena/benchmarks/pairwise/scoring.py deleted file mode 100644 index 485f7a54..00000000 --- a/judgearena/benchmarks/pairwise/scoring.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Runtime scoring adapters for pairwise preference tasks.""" - -from __future__ import annotations - -from collections.abc import Callable - -import pandas as pd - -from judgearena.utils.eval import PrefSummary, compute_pref_summary - -PairwiseScoreFn = Callable[[pd.DataFrame], PrefSummary] - - -def _score_win_rate(battles: pd.DataFrame) -> PrefSummary: - """Summarize canonical pairwise preferences.""" - return compute_pref_summary(battles["pref"]) - - -PAIRWISE_SCORERS: dict[str, PairwiseScoreFn] = { - "pairwise_win_rate": _score_win_rate, -} diff --git a/judgearena/benchmarks/pairwise/scoring/__init__.py b/judgearena/benchmarks/pairwise/scoring/__init__.py new file mode 100644 index 00000000..bf4ac38e --- /dev/null +++ b/judgearena/benchmarks/pairwise/scoring/__init__.py @@ -0,0 +1,13 @@ +"""Pairwise battle metrics.""" + +from judgearena.benchmarks.pairwise.scoring.metrics import ( + LengthControlledWinrateMetric, + PairwiseWinRateMetric, + collapse_pairwise_battles, +) + +__all__ = [ + "LengthControlledWinrateMetric", + "PairwiseWinRateMetric", + "collapse_pairwise_battles", +] diff --git a/judgearena/benchmarks/pairwise/scoring/metrics.py b/judgearena/benchmarks/pairwise/scoring/metrics.py new file mode 100644 index 00000000..99102f02 --- /dev/null +++ b/judgearena/benchmarks/pairwise/scoring/metrics.py @@ -0,0 +1,284 @@ +"""Reusable metric functions for canonical pairwise battles.""" + +from __future__ import annotations + +import math +import warnings +from dataclasses import dataclass + +import numpy as np +import pandas as pd +from sklearn.exceptions import ConvergenceWarning +from sklearn.linear_model import LogisticRegression + +from judgearena.utils.eval import compute_pref_summary + +BOOTSTRAP_ROUNDS = 1000 +BOOTSTRAP_SEED = 0 +CONFIDENCE_LEVEL = 0.95 + + +def _preferences(battles: pd.DataFrame, column: str = "pref") -> pd.Series: + if column not in battles: + raise ValueError(f"Pairwise metrics require a {column!r} column.") + try: + preferences = pd.Series(battles[column], dtype="float64") + except (TypeError, ValueError) as exc: + raise ValueError("Pairwise preferences must be numeric.") from exc + parsed = preferences.dropna() + invalid = ~np.isfinite(parsed) | ~parsed.between(0, 1) + if invalid.any(): + raise ValueError( + "Pairwise preferences must be missing or finite values in [0, 1]." + ) + return preferences + + +def _pairwise_view(battles: pd.DataFrame) -> pd.DataFrame: + """Normalize rows carrying an evaluation model to the focal/opponent view.""" + if "evaluation_model" not in battles: + return battles + frame = battles.loc[battles["evaluation_model"].notna()].copy() + required = {"model_a", "model_b", "pref"} + missing = sorted(required - set(frame.columns)) + if missing: + raise ValueError(f"Evaluation battles are missing columns: {missing}.") + focal_is_a = frame["model_a"] == frame["evaluation_model"] + focal_is_b = frame["model_b"] == frame["evaluation_model"] + if not (focal_is_a ^ focal_is_b).all(): + raise ValueError( + "Each evaluation battle must contain its evaluation model exactly once." + ) + frame["model"] = frame["evaluation_model"] + frame["baseline"] = frame["model_b"].where(focal_is_a, frame["model_a"]) + frame["pref"] = frame["pref"].where(focal_is_a, 1 - frame["pref"]) + if {"completion_a", "completion_b"} <= set(frame.columns): + frame["completion_model"] = frame["completion_a"].where( + focal_is_a, frame["completion_b"] + ) + frame["completion_baseline"] = frame["completion_b"].where( + focal_is_a, frame["completion_a"] + ) + return frame + + +def _format_winrate(value: object) -> str: + if value is None or pd.isna(value): + return "unavailable" + return f"{float(value):.2%}" + + +@dataclass(frozen=True, kw_only=True) +class PairwiseWinRateMetric: + """Configured candidate win-rate calculation.""" + + def calculate(self, battles: pd.DataFrame) -> dict[str, object]: + frame = _pairwise_view(battles) + preferences = _preferences(frame) + return compute_pref_summary(preferences).to_dict() + + @staticmethod + def render(result: dict[str, object]) -> str: + return ( + f"pairwise_win_rate: {_format_winrate(result['winrate'])} " + f"({result['num_wins']} wins, {result['num_losses']} losses, " + f"{result['num_ties']} ties, {result['num_missing']} missing)" + ) + + +def collapse_pairwise_battles(battles: pd.DataFrame) -> pd.DataFrame: + """Collapse answer-order judgments into one row per physical battle.""" + frame = _pairwise_view(battles).copy() + required = { + "instruction_index", + "model", + "baseline", + "completion_model", + "completion_baseline", + "pref", + "orientation", + } + missing = sorted(required - set(frame.columns)) + if missing: + raise ValueError(f"Length control requires battle columns: {missing}.") + + frame["pref"] = _preferences(frame) + if frame.empty: + frame["n_judgments"] = pd.Series(dtype="int64") + frame["n_parsed"] = pd.Series(dtype="int64") + return frame + keys = ["instruction_index", "model", "baseline"] + if "category" in frame: + keys.append("category") + grouped = frame.groupby(keys, sort=False, dropna=False) + if any(group["orientation"].duplicated().any() for _, group in grouped): + raise ValueError("A physical battle contains duplicate orientations.") + + orientations = set(frame["orientation"]) + allowed = {"single", "direct", "reversed"} + if not orientations <= allowed: + raise ValueError( + f"Unknown battle orientations: {sorted(orientations - allowed)}." + ) + if orientations == {"single"}: + expected = {"single"} + elif orientations == {"direct", "reversed"}: + expected = {"direct", "reversed"} + else: + raise ValueError( + "Battles require one single orientation or complete direct/reversed pairs." + ) + rows: list[dict[str, object]] = [] + for _, group in frame.groupby(keys, sort=False, dropna=False): + group_orientations = group["orientation"].tolist() + if set(group_orientations) != expected: + raise ValueError( + "Every physical battle must contain the expected orientations." + ) + for column in ("completion_model", "completion_baseline"): + values = group[column].tolist() + if any(not isinstance(value, str) for value in values): + raise ValueError("Length control requires string completions.") + if len(set(values)) != 1: + raise ValueError( + "A physical battle has different completions across orientations." + ) + + parsed = group["pref"].dropna() + row = group.iloc[0].to_dict() + row["pref"] = float(parsed.mean()) if len(parsed) else float("nan") + row["n_judgments"] = len(group) + row["n_parsed"] = len(parsed) + rows.append(row) + + return pd.DataFrame(rows) + + +def _has_separation(x: np.ndarray, outcomes: np.ndarray) -> bool: + positive = x[outcomes > 0] + negative = x[outcomes < 1] + if not len(positive) or not len(negative): + return True + return bool(positive.max() <= negative.min() or negative.max() <= positive.min()) + + +def _fit_length_model( + length_difference: np.ndarray, outcomes: np.ndarray +) -> tuple[float, float]: + scale = float(np.std(length_difference, ddof=1)) + if not math.isfinite(scale) or scale <= 0: + raise ValueError("Response length differences have no sample variance.") + x = np.asarray(length_difference / scale, dtype="float64") + y = np.asarray(outcomes, dtype="float64") + if _has_separation(x, y): + raise ValueError("Length-controlled logistic regression is separated.") + + design = np.repeat(x.reshape(-1, 1), 2, axis=0) + labels = np.tile([0, 1], len(y)) + weights = np.column_stack((1 - y, y)).ravel() + model = LogisticRegression( + fit_intercept=True, C=np.inf, solver="lbfgs", max_iter=1000, tol=1e-10 + ) + with warnings.catch_warnings(): + warnings.simplefilter("error", ConvergenceWarning) + model.fit(design, labels, sample_weight=weights) + intercept = float(model.intercept_[0]) + coefficient = float(model.coef_[0, 0]) + if not math.isfinite(intercept) or not math.isfinite(coefficient): + raise ValueError("Length-controlled logistic regression did not converge.") + return intercept, coefficient + + +def _sigmoid(value: float) -> float: + return 1 / (1 + math.exp(-max(min(value, 709), -709))) + + +def _bootstrap_interval( + length_difference: np.ndarray, outcomes: np.ndarray +) -> list[float] | None: + rng = np.random.default_rng(BOOTSTRAP_SEED) + scores: list[float] = [] + for _ in range(BOOTSTRAP_ROUNDS): + sample = rng.integers(0, len(outcomes), size=len(outcomes)) + try: + intercept, _ = _fit_length_model( + length_difference[sample], outcomes[sample] + ) + except (ValueError, ConvergenceWarning): + continue + scores.append(_sigmoid(intercept)) + if len(scores) < 0.95 * BOOTSTRAP_ROUNDS: + return None + alpha = (1 - CONFIDENCE_LEVEL) / 2 + low, high = np.quantile(scores, [alpha, 1 - alpha]) + return [float(low), float(high)] + + +@dataclass(frozen=True, kw_only=True) +class LengthControlledWinrateMetric: + """Configured equal-length candidate win-rate calculation.""" + + def calculate(self, battles: pd.DataFrame) -> dict[str, object]: + collapsed = collapse_pairwise_battles(battles) + result: dict[str, object] = { + "num_pairs": len(collapsed), + "num_scored": int( + ( + collapsed["pref"].notna() + & (collapsed["n_parsed"] == collapsed["n_judgments"]) + ).sum() + ), + } + if collapsed.empty: + return {**result, "winrate": None} + if collapsed["model"].nunique(dropna=False) != 1: + raise ValueError("Length control requires exactly one evaluation model.") + if collapsed["baseline"].nunique(dropna=False) != 1: + raise ValueError("Length control requires exactly one baseline model.") + + complete = collapsed.loc[ + collapsed["pref"].notna() + & (collapsed["n_parsed"] == collapsed["n_judgments"]) + ] + if len(complete) < 3: + return {**result, "winrate": None} + + length_difference = np.array( + [ + len(candidate) - len(baseline) + for candidate, baseline in zip( + complete["completion_model"], + complete["completion_baseline"], + strict=True, + ) + ], + dtype="float64", + ) + outcomes = 1 - complete["pref"].to_numpy(dtype="float64") + result["equal_length_extrapolation"] = not ( + length_difference.min() <= 0 <= length_difference.max() + ) + try: + intercept, _ = _fit_length_model(length_difference, outcomes) + except (ValueError, ConvergenceWarning): + return {**result, "winrate": None} + + result.update( + { + "winrate": _sigmoid(intercept), + "confidence_interval": _bootstrap_interval(length_difference, outcomes), + } + ) + return result + + @staticmethod + def render(result: dict[str, object]) -> str: + line = ( + "length_controlled_winrate: " + f"{_format_winrate(result['winrate'])} " + f"({result['num_scored']}/{result['num_pairs']} scored pairs)" + ) + interval = result.get("confidence_interval") + if interval is not None: + line += f" [{interval[0]:.2%}, {interval[1]:.2%}]" + return line diff --git a/judgearena/benchmarks/scoring.py b/judgearena/benchmarks/scoring.py new file mode 100644 index 00000000..86c36f4d --- /dev/null +++ b/judgearena/benchmarks/scoring.py @@ -0,0 +1,136 @@ +"""Construction and execution of configured battle metrics.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from typing import Protocol + +import pandas as pd + +from judgearena.benchmarks.elo.scoring import BradleyTerryMetric +from judgearena.benchmarks.pairwise.scoring.metrics import ( + LengthControlledWinrateMetric, + PairwiseWinRateMetric, +) + + +class _Metric(Protocol): + """The calculation and renderer supplied by every metric implementation.""" + + def calculate( + self, battles: pd.DataFrame, **runtime: object + ) -> dict[str, object]: ... + + @staticmethod + def render(result: dict[str, object]) -> str: ... + + +class MetricRequest(Protocol): + """The request fields needed to construct and group one metric.""" + + metric: str + breakdown_by: tuple[str, ...] + parameters: Mapping[str, object] + + +_METRIC_TYPES: dict[str, type[_Metric]] = { + "pairwise_win_rate": PairwiseWinRateMetric, + "length_controlled_winrate": LengthControlledWinrateMetric, + "bradley_terry": BradleyTerryMetric, +} + +ConfiguredMetrics = tuple[tuple[MetricRequest, _Metric], ...] + + +def available_metrics() -> tuple[str, ...]: + """Return the stable set of supported metric identifiers.""" + return tuple(sorted(_METRIC_TYPES)) + + +def _metric_type(name: str) -> type[_Metric]: + try: + return _METRIC_TYPES[name] + except KeyError as exc: + choices = ", ".join(available_metrics()) + raise ValueError( + f"Unknown metric {name!r}; available metrics: {choices}." + ) from exc + + +def build_metric(name: str, parameters: Mapping[str, object] | None = None) -> _Metric: + """Build one fresh, configured metric instance.""" + try: + return _metric_type(name)(**dict(parameters or {})) + except (TypeError, ValueError) as exc: + raise ValueError(f"Invalid parameters for metric {name!r}: {exc}") from exc + + +def build_metrics( + requests: Sequence[MetricRequest], + *, + parameter_overrides_by_metric: Mapping[str, Mapping[str, object]] | None = None, +) -> ConfiguredMetrics: + """Build requested metrics in order, applying temporary runtime overrides.""" + overrides = parameter_overrides_by_metric or {} + configured: list[tuple[MetricRequest, _Metric]] = [] + for request in requests: + parameters = dict(request.parameters) + parameters.update(overrides.get(request.metric, {})) + configured.append((request, build_metric(request.metric, parameters))) + return tuple(configured) + + +def _group_value(value: object) -> object: + if pd.isna(value): + return None + return value.item() if hasattr(value, "item") else value + + +def calculate_metrics( + battles: pd.DataFrame, + metrics: ConfiguredMetrics, + *, + runtime_by_metric: Mapping[str, Mapping[str, object]] | None = None, +) -> dict[str, dict[str, object]]: + """Run configured metrics over one battle table.""" + runtime_by_metric = runtime_by_metric or {} + calculated: dict[str, dict[str, object]] = {} + for request, metric in metrics: + runtime = dict(runtime_by_metric.get(request.metric, {})) + values = dict(metric.calculate(battles, **runtime)) + if request.breakdown_by: + groups: dict[str, list[dict[str, object]]] = {} + for field in request.breakdown_by: + if field not in battles: + raise ValueError( + f"Metric {request.metric!r} cannot group by missing column " + f"{field!r}." + ) + groups[field] = [ + { + "group": _group_value(key), + "values": metric.calculate(group, **runtime), + } + for key, group in battles.groupby(field, sort=False, dropna=False) + ] + values["groups"] = groups + calculated[request.metric] = values + return calculated + + +def _indent(text: str, prefix: str = " ") -> str: + return "\n".join(f"{prefix}{line}" for line in text.splitlines()) + + +def render_metrics(results: Mapping[str, dict[str, object]]) -> str: + """Render metric results in their configured dictionary order.""" + sections: list[str] = [] + for name, result in results.items(): + metric_type = _metric_type(name) + overall = {key: value for key, value in result.items() if key != "groups"} + sections.append(metric_type.render(overall)) + for field, groups in result.get("groups", {}).items(): + for group in groups: + rendered = metric_type.render(group["values"]) + sections.append(f"{field}={group['group']}:\n{_indent(rendered)}") + return "\n\n".join(sections) diff --git a/judgearena/evaluate.py b/judgearena/evaluate.py index 4ec52493..50e7c57b 100644 --- a/judgearena/evaluate.py +++ b/judgearena/evaluate.py @@ -1,9 +1,7 @@ from dataclasses import dataclass -import numpy as np import pandas as pd from langchain_core.prompts import ChatPromptTemplate -from scipy.optimize import minimize_scalar from judgearena.log import get_logger from judgearena.models import InferenceResult, do_inference @@ -28,58 +26,6 @@ ) -def calibrate_temperature( - delta_s: np.ndarray, - y: np.ndarray, - bounds: tuple[float, float] = (-10.0, 10.0), -) -> float: - """Find the MLE temperature T* for the model P(A>B) = σ(T·Δs). - - The log-likelihood is: - - L(T) = Σ_i [ y_i·log σ(T·Δs_i) + (1−y_i)·log σ(−T·Δs_i) ] - = Σ_i log σ(T · (2y_i − 1) · Δs_i) - - This is concave in T (single global maximum) so ``minimize_scalar`` with - the 'bounded' method is guaranteed to converge. - - Args: - delta_s: Score differences ``s_A − s_B`` for each battle, shape (N,). - y: Observed hard labels (1 = A was preferred, 0 = B was preferred, - 0.5 = tie). Ties contribute zero gradient and are skipped. - bounds: Search interval for T (default −10 to +10). - - Returns: - The calibrated temperature T*. - """ - delta_s = np.asarray(delta_s, dtype=float) - y = np.asarray(y, dtype=float) - - # Skip ties (y == 0.5) — they carry no directional information. - non_tie = y != 0.5 - delta_s = delta_s[non_tie] - y = y[non_tie] - - if len(delta_s) == 0: - raise ValueError( - "No non-tie observations available for temperature calibration." - ) - - # z_i = (2y_i − 1) · Δs_i (positive when the score difference agrees with the outcome) - z = (2 * y - 1) * delta_s - - def neg_log_likelihood(T: float) -> float: - # log σ(T·z) = −log(1 + exp(−T·z)) = −logaddexp(0, −T·z) - return float(np.sum(np.logaddexp(0.0, -T * z))) - - result = minimize_scalar( - neg_log_likelihood, - bounds=bounds, - method="bounded", - ) - return float(result.x) - - def load_judge_system_and_user_prompt( multi_turn: bool = False, ) -> tuple[str, str]: diff --git a/judgearena/reports.py b/judgearena/reports.py new file mode 100644 index 00000000..e57be4ea --- /dev/null +++ b/judgearena/reports.py @@ -0,0 +1,90 @@ +"""Serializable reports for configured battle metrics.""" + +from __future__ import annotations + +import abc +import json +from pathlib import Path + +from pydantic import BaseModel, ConfigDict, Field, computed_field + + +class Report(BaseModel, abc.ABC): + """A metric report that renders, serializes, and saves itself.""" + + model_config = ConfigDict( + arbitrary_types_allowed=True, + populate_by_name=True, + protected_namespaces=(), + use_attribute_docstrings=True, + ) + + @computed_field + @property + def schema_version(self) -> str: + return "1" + + @computed_field + @property + def report_type(self) -> str: + return type(self).__name__ + + @abc.abstractmethod + def render(self) -> None: ... + + def to_dict(self) -> dict: + return self.model_dump(by_alias=True, exclude_none=True) + + def save(self, path: str | Path) -> Path: + from judgearena.artifacts import to_jsonable + + output = Path(path) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(to_jsonable(self.to_dict()), indent=2) + "\n") + return output + + +class BattleReport(Report): + """Metric results for pairwise and MT-Bench evaluations.""" + + task: str + model_a: str = Field(serialization_alias="model_A") + model_b: str = Field(serialization_alias="model_B") + judge_model: str + metrics: dict[str, dict[str, object]] + swap_mode: str | None = None + result_folder: str | None = None + preferences: list = Field(default_factory=list) + metadata: dict = Field(default_factory=dict) + + def render(self) -> None: + from judgearena.benchmarks.scoring import render_metrics + + print("\n" + "=" * 60) + print("🏆 MODEL BATTLE RESULTS 🏆".center(60)) + print(f"📊 Task: {self.task}") + print(f"🤖 Competitors: Model A: {self.model_a} vs Model B: {self.model_b}") + print(f"⚖️ Judge: {self.judge_model}") + print("📈 Metrics:") + print(render_metrics(self.metrics)) + if self.result_folder: + print(f"📁 Results: {self.result_folder}") + print("=" * 60 + "\n") + + +class EloReport(Report): + """Configured battle metrics for one model against an arena.""" + + arena: str + judge_model: str + metrics: dict[str, dict[str, object]] + num_battles: int + model_name: str + sampling_metadata: dict[str, object] + + def render(self) -> None: + from judgearena.benchmarks.scoring import render_metrics + + print(f"\n=== Results for {self.model_name} ===") + print(f"Arena: {self.arena} | Judge: {self.judge_model}") + print(render_metrics(self.metrics)) diff --git a/judgearena/tasks/README.md b/judgearena/tasks/README.md index e30c93cf..235d20a4 100644 --- a/judgearena/tasks/README.md +++ b/judgearena/tasks/README.md @@ -60,7 +60,9 @@ protocol: judge: default_prompt_preset: default scoring: - adapter: pairwise_win_rate + metrics: + - metric: pairwise_win_rate + - metric: length_controlled_winrate ``` Reuse a private `_base.yaml` with `extends: _base.yaml` when several versions @@ -83,9 +85,22 @@ runner code. Keep task YAML boring: declarative facts belong in YAML, while downloading, format conversion, and scoring algorithms belong in Python. -The judge prompt preset owns the expected output format and its parser. The -scoring adapter owns the calculation, primary metric, and metric direction; -task YAML only selects those components by ID. +The judge prompt preset owns the expected output format and its parser. Scoring +metrics consume battle dataframes and return result dictionaries. Task YAML +selects metrics in order and may provide parameters or grouped breakdowns: + +```yaml +scoring: + metrics: + - metric: pairwise_win_rate + breakdown_by: [category] +``` + +Each `breakdown_by` field produces a separate breakdown. For example, +`[category, turn]` gives results per category and per turn, not per combination. + +Each metric owns its calculation and rendering. Runners only build battle data +and invoke the shared metric executor. If an upstream dataset has a new format, implement a dataset adapter under `judgearena/datasets/` and register it in the dataset registry. Task validation diff --git a/judgearena/tasks/definitions/alpaca_eval/alpaca-eval.yaml b/judgearena/tasks/definitions/alpaca_eval/alpaca-eval.yaml index b25324a8..1c3b06b7 100644 --- a/judgearena/tasks/definitions/alpaca_eval/alpaca-eval.yaml +++ b/judgearena/tasks/definitions/alpaca_eval/alpaca-eval.yaml @@ -27,7 +27,8 @@ protocol: default_prompt_preset: default default_swap_mode: fixed scoring: - adapter: pairwise_win_rate + metrics: + - metric: pairwise_win_rate metadata: reference_implementation: https://github.com/tatsu-lab/alpaca_eval diff --git a/judgearena/tasks/definitions/arena_hard/_base.yaml b/judgearena/tasks/definitions/arena_hard/_base.yaml index bc980ecd..c45fdcb0 100644 --- a/judgearena/tasks/definitions/arena_hard/_base.yaml +++ b/judgearena/tasks/definitions/arena_hard/_base.yaml @@ -21,7 +21,8 @@ protocol: default_prompt_preset: default default_swap_mode: fixed scoring: - adapter: pairwise_win_rate + metrics: + - metric: pairwise_win_rate metadata: reference_implementation: https://github.com/lmarena/arena-hard-auto diff --git a/judgearena/tasks/definitions/elo/_base.yaml b/judgearena/tasks/definitions/elo/_base.yaml index bd92ac32..a99f9a42 100644 --- a/judgearena/tasks/definitions/elo/_base.yaml +++ b/judgearena/tasks/definitions/elo/_base.yaml @@ -19,6 +19,8 @@ protocol: default_prompt_preset: default default_swap_mode: fixed scoring: - adapter: bradley_terry + metrics: + - metric: pairwise_win_rate + - metric: bradley_terry default_soft: true default_temperature: 0.3 diff --git a/judgearena/tasks/definitions/fluency/fluency.yaml b/judgearena/tasks/definitions/fluency/fluency.yaml index cd125c48..c47adf04 100644 --- a/judgearena/tasks/definitions/fluency/fluency.yaml +++ b/judgearena/tasks/definitions/fluency/fluency.yaml @@ -72,7 +72,8 @@ protocol: default_prompt_preset: fluency default_swap_mode: fixed scoring: - adapter: pairwise_win_rate + metrics: + - metric: pairwise_win_rate metadata: reference_implementation: https://huggingface.co/datasets/geoalgo/multilingual-fluency diff --git a/judgearena/tasks/definitions/m_arena_hard/_base.yaml b/judgearena/tasks/definitions/m_arena_hard/_base.yaml index f0370500..5d73d828 100644 --- a/judgearena/tasks/definitions/m_arena_hard/_base.yaml +++ b/judgearena/tasks/definitions/m_arena_hard/_base.yaml @@ -21,7 +21,8 @@ protocol: default_prompt_preset: default default_swap_mode: fixed scoring: - adapter: pairwise_win_rate + metrics: + - metric: pairwise_win_rate variants: selector: language diff --git a/judgearena/tasks/definitions/mt_bench/mt-bench.yaml b/judgearena/tasks/definitions/mt_bench/mt-bench.yaml index 32361c48..e529d22b 100644 --- a/judgearena/tasks/definitions/mt_bench/mt-bench.yaml +++ b/judgearena/tasks/definitions/mt_bench/mt-bench.yaml @@ -49,7 +49,9 @@ protocol: fastchat_temperature: 0.0 reference_categories: [math, reasoning, coding, arena-hard-200] scoring: - adapter: pairwise_win_rate + metrics: + - metric: pairwise_win_rate + breakdown_by: [category, turn] metadata: reference_implementation: https://github.com/lm-sys/FastChat/tree/master/fastchat/llm_judge diff --git a/judgearena/tasks/registry.py b/judgearena/tasks/registry.py index 5ee7f8d5..87c69260 100644 --- a/judgearena/tasks/registry.py +++ b/judgearena/tasks/registry.py @@ -16,12 +16,12 @@ import yaml from pydantic import ValidationError -from judgearena.benchmarks.elo.scoring import ELO_SCORERS -from judgearena.benchmarks.pairwise.scoring import PAIRWISE_SCORERS +from judgearena.benchmarks.scoring import available_metrics, build_metric from judgearena.log import get_logger from judgearena.prompts.registry import JUDGE_PROMPT_PRESETS from judgearena.tasks.schema import ( EloProtocol, + MTBenchProtocol, ResolvedTaskSpec, ResourceDigest, TaskProvenance, @@ -255,8 +255,7 @@ class AdapterCatalog: ) battle_datasets: frozenset[str] = frozenset({"arena_battles"}) prompts: frozenset[str] = frozenset(JUDGE_PROMPT_PRESETS) - pairwise_scorers: frozenset[str] = frozenset(PAIRWISE_SCORERS) - elo_scorers: frozenset[str] = frozenset(ELO_SCORERS) + metrics: frozenset[str] = frozenset(available_metrics()) def load_tasks( @@ -302,15 +301,33 @@ def _discover_tasks( def _validate_adapter_ids(resolved: ResolvedTaskSpec, adapters: AdapterCatalog) -> None: spec = resolved.spec is_elo = isinstance(spec.protocol, EloProtocol) - scorer_names = adapters.elo_scorers if is_elo else adapters.pairwise_scorers dataset_names = ( adapters.battle_datasets if is_elo else adapters.instruction_datasets ) references = { "runner": (spec.protocol.runner, adapters.runners), "dataset adapter": (spec.dataset.adapter, dataset_names), - "scorer": (spec.protocol.scoring.adapter, scorer_names), } + metric_names = adapters.metrics + for request in spec.protocol.scoring.metrics: + if request.metric not in metric_names: + raise TaskDefinitionError( + f"{resolved.provenance.source_path}: unknown metric {request.metric!r}" + ) + try: + build_metric(request.metric, request.parameters) + except ValueError as exc: + raise TaskDefinitionError( + f"{resolved.provenance.source_path}: invalid metric " + f"{request.metric!r}: {exc}" + ) from exc + if isinstance(spec.protocol, MTBenchProtocol): + unsupported = sorted(set(request.breakdown_by) - {"category", "turn"}) + if unsupported: + raise TaskDefinitionError( + f"{resolved.provenance.source_path}: MT-Bench cannot group " + f"metrics by {unsupported}" + ) judge = spec.protocol.judge references["prompt"] = (judge.default_prompt_preset, adapters.prompts) for kind, (adapter_id, available) in references.items(): diff --git a/judgearena/tasks/schema/__init__.py b/judgearena/tasks/schema/__init__.py index 89b00a69..97711943 100644 --- a/judgearena/tasks/schema/__init__.py +++ b/judgearena/tasks/schema/__init__.py @@ -15,6 +15,7 @@ ) from judgearena.tasks.schema.dataset import DatasetFields, DatasetSpec from judgearena.tasks.schema.elo import EloProtocol, EloScoringSpec +from judgearena.tasks.schema.metrics import MetricSpec, ScoringSpec from judgearena.tasks.schema.mt_bench import ( MTBenchJudgeSpec, MTBenchProtocol, @@ -23,7 +24,6 @@ from judgearena.tasks.schema.pairwise import ( PairwiseJudgeSpec, PairwiseProtocol, - ScoringSpec, SingleTurnGeneration, SwapMode, ) @@ -58,6 +58,7 @@ "HuggingFaceSpaceSource", "MTBenchJudgeSpec", "MTBenchProtocol", + "MetricSpec", "MultiTurnGeneration", "NoBaseline", "OfficialOutputsBaseline", diff --git a/judgearena/tasks/schema/elo.py b/judgearena/tasks/schema/elo.py index 08a179bc..c095de9d 100644 --- a/judgearena/tasks/schema/elo.py +++ b/judgearena/tasks/schema/elo.py @@ -8,13 +8,13 @@ from judgearena.tasks.schema.base import StrictFrozenModel from judgearena.tasks.schema.baselines import NoBaseline +from judgearena.tasks.schema.metrics import ScoringSpec from judgearena.tasks.schema.pairwise import PairwiseJudgeSpec, SingleTurnGeneration -class EloScoringSpec(StrictFrozenModel): +class EloScoringSpec(ScoringSpec): """Task-owned defaults for fitting arena-anchored ratings.""" - adapter: str = Field(min_length=1) default_soft: bool = True default_temperature: float = Field(default=0.3, gt=0) diff --git a/judgearena/tasks/schema/metrics.py b/judgearena/tasks/schema/metrics.py new file mode 100644 index 00000000..75aaa42a --- /dev/null +++ b/judgearena/tasks/schema/metrics.py @@ -0,0 +1,37 @@ +"""Declarative requests for battle-dataframe metrics.""" + +from __future__ import annotations + +from pydantic import Field, model_validator + +from judgearena.tasks.schema.base import StrictFrozenModel + + +class MetricSpec(StrictFrozenModel): + """One named calculation over a battle dataframe.""" + + metric: str = Field(min_length=1) + breakdown_by: tuple[str, ...] = () + """Each field produces a separate breakdown; fields are not combined.""" + parameters: dict[str, object] = Field(default_factory=dict) + + @model_validator(mode="after") + def _validate_breakdown_by(self) -> MetricSpec: + if any(not field for field in self.breakdown_by): + raise ValueError("metric breakdown_by fields must not be empty") + if len(set(self.breakdown_by)) != len(self.breakdown_by): + raise ValueError("metric breakdown_by fields must not contain duplicates") + return self + + +class ScoringSpec(StrictFrozenModel): + """Ordered metric calculations for one battle-producing protocol.""" + + metrics: tuple[MetricSpec, ...] = Field(min_length=1) + + @model_validator(mode="after") + def _validate_metrics(self) -> ScoringSpec: + names = [item.metric for item in self.metrics] + if len(set(names)) != len(names): + raise ValueError("scoring metrics must not contain duplicate names") + return self diff --git a/judgearena/tasks/schema/mt_bench.py b/judgearena/tasks/schema/mt_bench.py index e9b66e9a..ad959847 100644 --- a/judgearena/tasks/schema/mt_bench.py +++ b/judgearena/tasks/schema/mt_bench.py @@ -8,7 +8,8 @@ from judgearena.tasks.schema.base import StrictFrozenModel from judgearena.tasks.schema.baselines import BaselineSpec -from judgearena.tasks.schema.pairwise import PairwiseJudgeSpec, ScoringSpec +from judgearena.tasks.schema.metrics import ScoringSpec +from judgearena.tasks.schema.pairwise import PairwiseJudgeSpec class MultiTurnGeneration(StrictFrozenModel): diff --git a/judgearena/tasks/schema/pairwise.py b/judgearena/tasks/schema/pairwise.py index a534e1a7..9659a03b 100644 --- a/judgearena/tasks/schema/pairwise.py +++ b/judgearena/tasks/schema/pairwise.py @@ -8,6 +8,7 @@ from judgearena.tasks.schema.base import StrictFrozenModel from judgearena.tasks.schema.baselines import BaselineSpec +from judgearena.tasks.schema.metrics import ScoringSpec class SingleTurnGeneration(StrictFrozenModel): @@ -30,10 +31,6 @@ class PairwiseJudgeSpec(StrictFrozenModel): Arena-Hard v2.0 creative-writing judge prompt).""" -class ScoringSpec(StrictFrozenModel): - adapter: str = Field(min_length=1) - - class PairwiseProtocol(StrictFrozenModel): """Task-owned generation, baseline, judging, and scoring behavior.""" diff --git a/judgearena/utils/eval.py b/judgearena/utils/eval.py index 4663f59c..3eb12981 100644 --- a/judgearena/utils/eval.py +++ b/judgearena/utils/eval.py @@ -1,13 +1,11 @@ -"""Preference statistics and human-readable result reporting.""" +"""Preference statistics and compatibility report exports.""" from __future__ import annotations -import abc -import json -from pathlib import Path - import pandas as pd -from pydantic import BaseModel, ConfigDict, Field, computed_field, model_serializer +from pydantic import BaseModel + +from judgearena.reports import BattleReport, Report class PrefSummary(BaseModel): @@ -33,7 +31,7 @@ def compute_pref_summary(prefs: pd.Series) -> PrefSummary: num_ties = int((valid == 0.5).sum()) num_battles = int(len(prefs)) denom = num_wins + num_losses + num_ties - winrate = float((num_wins + 0.5 * num_ties) / denom) if denom > 0 else float("nan") + winrate = float((num_wins + 0.5 * num_ties) / denom) if denom else float("nan") return PrefSummary( num_battles=num_battles, winrate=winrate, @@ -44,141 +42,10 @@ def compute_pref_summary(prefs: pd.Series) -> PrefSummary: ) -class Report(BaseModel, abc.ABC): - """A reportable result that renders, serializes (versioned), and saves itself.""" - - # protected_namespaces=() allows model_* field names (model_a, model_name) - model_config = ConfigDict( - arbitrary_types_allowed=True, - populate_by_name=True, - protected_namespaces=(), - use_attribute_docstrings=True, - ) - - @computed_field - @property - def schema_version(self) -> str: - return "1" - - @computed_field - @property - def report_type(self) -> str: - return type(self).__name__ - - @model_serializer(mode="wrap") - def _flatten_summary(self, handler) -> dict: - data = handler(self) - summary = data.pop("summary", None) - if isinstance(summary, dict): - data = {**summary, **data} - return data - - @abc.abstractmethod - def render(self) -> None: ... - - def to_dict(self) -> dict: - return self.model_dump(by_alias=True, exclude_none=True) - - def save(self, path: str | Path) -> Path: - from judgearena.artifacts import to_jsonable # lazy: avoid an import cycle - - p = Path(path) - p.parent.mkdir(parents=True, exist_ok=True) - p.write_text(json.dumps(to_jsonable(self.to_dict()), indent=2) + "\n") - return p - - -class BattleReport(Report): - """Pairwise battle results for the arena and MT-Bench pipelines.""" - - task: str - """Evaluation task name.""" - model_a: str = Field(serialization_alias="model_A") - """Model in the A position.""" - model_b: str = Field(serialization_alias="model_B") - """Model in the B position.""" - judge_model: str - """LLM judge that scored the battles.""" - summary: PrefSummary - """Win/loss/tie statistics (flattened to the top level on serialization).""" - swap_mode: str | None = None - """Position-bias handling: "fixed" or "both".""" - result_folder: str | None = None - """Directory the run's artifacts were written to.""" - per_category: dict | None = None - """Per-category win/loss/tie breakdown (MT-Bench).""" - per_turn: dict | None = None - """Per-turn win/loss/tie breakdown (MT-Bench).""" - preferences: list = Field(default_factory=list) - """Raw per-battle preference values (0=A, 0.5=tie, 1=B).""" - metadata: dict = Field(default_factory=dict) - """Free-form run metadata (baseline assignment, prompt preset, ...).""" - - def render(self) -> None: - s = self.summary - print("\n" + "=" * 60) - print("🏆 MODEL BATTLE RESULTS 🏆".center(60)) - print(f"📊 Task: {self.task}") - print(f"🤖 Competitors: Model A: {self.model_a} vs Model B: {self.model_b}") - print(f"⚖️ Judge: {self.judge_model}") - print("📈 Results Summary:") - if s.num_missing > 0: - parsed = s.num_battles - s.num_missing - print( - f" Total Battles: {s.num_battles} ⚠️ {s.num_missing} unparseable " - f"(parsed: {parsed}/{s.num_battles})" - ) - elif self.swap_mode == "both": - print( - f" Total Battles: {s.num_battles} (2×{s.num_battles // 2} — each " - f"instruction judged in both orders to detect positional bias)" - ) - else: - print(f" Total Battles: {s.num_battles}") - print(f" Win Rate (A): {s.winrate:.1%}") - print(f" ✅ Wins: {s.num_wins}") - print(f" ❌ Losses: {s.num_losses}") - print(f" 🤝 Ties: {s.num_ties}") - - if self.per_category: - print("\nPer-Category Breakdown:") - print( - f" {'Category':<14} | {'Win Rate(A)':>11} | " - f"{'Wins':>4} | {'Losses':>6} | {'Ties':>4}" - ) - print(f" {'-' * 14}-+-{'-' * 11}-+-{'-' * 4}-+-{'-' * 6}-+-{'-' * 4}") - for cat, stats in sorted(self.per_category.items()): - print( - f" {cat:<14} | {stats['winrate']:>11.1%} | " - f"{stats['num_wins']:>4} | {stats['num_losses']:>6} | " - f"{stats['num_ties']:>4}" - ) - - if self.per_turn: - print("\nPer-Turn Breakdown:") - for turn, stats in sorted(self.per_turn.items()): - print( - f" Turn {turn} Win Rate(A): {stats['winrate']:.1%} " - f"(W:{stats['num_wins']} L:{stats['num_losses']} T:{stats['num_ties']})" - ) - - if self.result_folder: - print(f"📁 Results: {self.result_folder}") - print("=" * 60 + "\n") - - -def _compute_grouped_stats( - preferences: pd.Series, - metadata: list[dict[str, object]], - group_by: str, -) -> dict[object, dict[str, float | int]]: - grouped: dict[object, list[float]] = {} - for meta, pref in zip(metadata, preferences, strict=True): - key = meta.get(group_by) - if key is None: - continue - grouped.setdefault(key, []).append(pref) - return { - key: compute_pref_summary(pd.Series(vals)).to_dict() - for key, vals in grouped.items() - } +# Compatibility exports; report models now live together in judgearena.reports. +__all__ = [ + "BattleReport", + "PrefSummary", + "Report", + "compute_pref_summary", +] diff --git a/tests/test_elo_calibration.py b/tests/test_elo_calibration.py new file mode 100644 index 00000000..0c163cd0 --- /dev/null +++ b/tests/test_elo_calibration.py @@ -0,0 +1,58 @@ +"""Tests for PairScore temperature calibration.""" + +from copy import deepcopy +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from judgearena.benchmarks.elo import calibration as elo_calibration +from judgearena.benchmarks.elo.calibration import ( + calibrate_pairscore_temperature, + fit_temperature, +) + + +def test_fit_temperature_follows_human_preference_direction(): + score_differences = np.array([2.0, 1.0, -1.0, -2.0]) + outcomes = np.array([1.0, 1.0, 0.0, 0.0]) + + assert fit_temperature(score_differences, outcomes) > 0 + assert fit_temperature(score_differences, 1 - outcomes) < 0 + + +@pytest.mark.parametrize( + ("enabled", "prompt"), + [ + (False, None), + (True, SimpleNamespace(parser=object())), + ], +) +def test_skipped_calibration_does_not_consume_rng_or_build_judge( + monkeypatch, enabled, prompt +): + rng = np.random.default_rng(7) + state = deepcopy(rng.bit_generator.state) + + def fail_if_called(**_kwargs): + raise AssertionError("calibration judge was built") + + monkeypatch.setattr(elo_calibration, "make_model", fail_if_called) + result = calibrate_pairscore_temperature( + pd.DataFrame(), + pd.DataFrame(), + enabled=enabled, + soft_elo=True, + sample_size=None, + rng=rng, + judge_model="unused", + judge_model_kwargs={}, + swap_mode="fixed", + prompt=prompt, + truncate_input_chars=None, + default_temperature=0.3, + ) + + assert result is None + assert rng.bit_generator.state == state diff --git a/tests/test_elo_task_runtime.py b/tests/test_elo_task_runtime.py index aa4e4034..5f48af1d 100644 --- a/tests/test_elo_task_runtime.py +++ b/tests/test_elo_task_runtime.py @@ -3,8 +3,6 @@ import pandas as pd import judgearena.datasets.arena_battles as arena_battles -from judgearena.benchmarks.elo.rating import fit_bradley_terry -from judgearena.benchmarks.elo.scoring import ELO_SCORERS from judgearena.tasks.registry import get_packaged_task @@ -49,9 +47,3 @@ def test_elo_dataset_download_uses_pinned_task_source(monkeypatch, tmp_path): assert captured["repo_id"] == "lmarena-ai/arena-human-preference-100k" assert captured["revision"] == "72e85b3ddc9c81bf7b659d6b03d4126dfd8fb34a" assert captured["allow_patterns"] == ("data/*.parquet",) - - -def test_elo_scoring_adapter_resolves_task_selection(): - task = _elo_task() - - assert ELO_SCORERS[task.spec.protocol.scoring.adapter].fit is fit_bradley_terry diff --git a/tests/test_estimate_elo_ratings.py b/tests/test_estimate_elo_ratings.py index 7f2bec78..190cafa0 100644 --- a/tests/test_estimate_elo_ratings.py +++ b/tests/test_estimate_elo_ratings.py @@ -1,20 +1,26 @@ import math +from dataclasses import replace +from types import SimpleNamespace import numpy as np import pandas as pd import pytest +import judgearena.benchmarks.elo.calibration as elo_calibration import judgearena.benchmarks.elo.runner as estimate_elo_ratings from judgearena.benchmarks.elo.rating import ( arena_anchor_battles, fit_bradley_terry, + prefs_to_battle_results, winner_to_pref, ) from judgearena.benchmarks.elo.runner import run_elo +from judgearena.benchmarks.elo.scoring import BradleyTerryMetric from judgearena.config import RunConfig from judgearena.evaluate import JudgeAnnotation, judge_and_parse_prefs from judgearena.models import make_model from judgearena.tasks.registry import get_packaged_task +from judgearena.tasks.schema import MetricSpec N_BATTLES = 30 ARENA_MODELS = ["arena_model_alpha", "arena_model_beta", "arena_model_gamma"] @@ -90,7 +96,6 @@ def _run_without_cache(fun, **_kwargs): def _default_args(*, result_folder: str, **kwargs) -> RunConfig: task = kwargs.pop("task", "elo-comparia") - arena = kwargs.pop("arena", None) model = kwargs.pop("model", "Dummy/my model") judge_model = kwargs.pop("judge_model", "Dummy/score A: 0 score B: 10") n_instructions = kwargs.pop("n_instructions", 10) @@ -114,7 +119,6 @@ def _default_args(*, result_folder: str, **kwargs) -> RunConfig: judge=judge, generation={"n_instructions": n_instructions}, elo={ - "arena": arena, "n_bootstraps": n_bootstraps, "languages": languages, "calibrate_temperature": calibrate_temperature, @@ -123,6 +127,79 @@ def _default_args(*, result_folder: str, **kwargs) -> RunConfig: ) +def test_missing_preference_remains_missing_in_hard_battles(): + battles = prefs_to_battle_results([float("nan")], [True], ["opponent"], "candidate") + + assert pd.isna(battles.loc[0, "pref"]) + assert pd.isna(battles.loc[0, "pref_hard"]) + assert battles.loc[0, "winner"] is None + + +def test_bradley_terry_hard_mode_uses_hard_preferences(): + battles = pd.DataFrame( + { + "model_a": ["candidate", "candidate", "opponent", "opponent"], + "model_b": ["opponent", "opponent", "candidate", "candidate"], + "pref": [0.5, 0.5, 0.5, 0.5], + "pref_hard": [0.0, 0.0, 1.0, 1.0], + "source": ["llm-judge"] * 4, + "evaluation_model": ["candidate"] * 4, + } + ) + + soft = BradleyTerryMetric(soft=True).calculate(battles) + hard = BradleyTerryMetric(soft=False).calculate(battles) + + assert soft["ratings"]["candidate"] == pytest.approx(soft["ratings"]["opponent"]) + assert hard["ratings"]["candidate"] > hard["ratings"]["opponent"] + + +@pytest.mark.parametrize("baseline_model", [None, "anchor-b", "missing-baseline"]) +def test_bradley_terry_metric_owns_existing_bootstrap_outputs(baseline_model): + battles = pd.DataFrame( + { + "model_a": ["anchor-a", "anchor-b", "candidate", "anchor-a"], + "model_b": ["anchor-b", "anchor-a", "anchor-a", "candidate"], + "pref": [0.0, 1.0, 0.2, 0.8], + "pref_hard": [0.0, 1.0, 0.0, 1.0], + "source": ["human", "human", "llm-judge", "llm-judge"], + "evaluation_model": [None, None, "candidate", "candidate"], + } + ) + metric_rng = np.random.default_rng(17) + expected_rng = np.random.default_rng(17) + expected_bootstraps = [] + for _ in range(3): + sample = battles.sample( + n=len(battles), + replace=True, + random_state=int(expected_rng.integers(0, 2**31)), + ) + expected_bootstraps.append( + fit_bradley_terry(sample, baseline_model=baseline_model) + ) + + result = BradleyTerryMetric( + n_bootstraps=3, baseline_model=baseline_model + ).calculate(battles, rng=metric_rng) + + assert result["ratings"] == fit_bradley_terry( + battles, baseline_model=baseline_model + ) + assert result["human_ratings"] == fit_bradley_terry( + battles.iloc[:2], baseline_model=baseline_model + ) + assert result["bootstrap_ratings"] == expected_bootstraps + assert result["n_bootstraps"] == 3 + assert result["evaluation_model"] == "candidate" + assert result["battle_counts"] == { + "anchor-a": 4, + "anchor-b": 2, + "candidate": 2, + } + assert metric_rng.integers(0, 2**31) == expected_rng.integers(0, 2**31) + + # --- fit_bradley_terry unit tests --- @@ -183,21 +260,56 @@ def run_elo_with_task(cfg: RunConfig) -> dict: return run_elo(cfg, get_packaged_task(cfg.task)) -def test_run_elo_returns_summary(tmp_path): +def _pairwise_metric(run_result: dict) -> dict: + return run_result["metrics"]["pairwise_win_rate"] + + +def _rating_metric(run_result: dict) -> dict: + return run_result["metrics"]["bradley_terry"] + + +def _num_pairwise_rows(run_result: dict) -> int: + metric = _pairwise_metric(run_result) + return ( + metric["num_wins"] + + metric["num_losses"] + + metric["num_ties"] + + metric["num_missing"] + ) + + +def test_run_elo_returns_metrics(tmp_path): result = run_elo_with_task(_default_args(result_folder=str(tmp_path))) - assert set(result.keys()) >= { - "num_wins", - "num_losses", - "num_ties", - "winrate", - "bootstrap_ratings", + + assert set(result) >= { + "arena", + "judge_model", + "metrics", "model_name", + "num_battles", + "sampling_metadata", + "result_path", } + assert set(result["metrics"]) == {"pairwise_win_rate", "bradley_terry"} + assert "rating_entries" in _rating_metric(result) + assert "winrate" not in result + assert "bootstrap_ratings" not in result -def test_run_elo_winrate_in_valid_range(tmp_path): - result = run_elo_with_task(_default_args(result_folder=str(tmp_path))) - assert 0.0 <= result["winrate"] <= 1.0 +def test_run_elo_without_bradley_terry_skips_rating_artifacts(tmp_path): + cfg = _default_args(result_folder=str(tmp_path)) + task = get_packaged_task(cfg.task) + scoring = task.spec.protocol.scoring.model_copy( + update={"metrics": (MetricSpec(metric="pairwise_win_rate"),)} + ) + protocol = task.spec.protocol.model_copy(update={"scoring": scoring}) + task = replace(task, spec=task.spec.model_copy(update={"protocol": protocol})) + + result = run_elo(cfg, task) + + assert set(result["metrics"]) == {"pairwise_win_rate"} + assert not list(tmp_path.rglob("bootstrap_ratings.csv")) + assert not list(tmp_path.rglob("elo_ratings.json")) def test_run_elo_winrate_depends_on_judge(tmp_path): @@ -214,7 +326,10 @@ def test_run_elo_winrate_depends_on_judge(tmp_path): result_folder=str(tmp_path), judge_model="Dummy/score A: 10 score B: 0" ) ) - assert result_wins["winrate"] > result_loses["winrate"] + assert ( + _pairwise_metric(result_wins)["winrate"] + > _pairwise_metric(result_loses)["winrate"] + ) def test_run_elo_language_filter_reduces_battles(tmp_path): @@ -227,10 +342,8 @@ def test_run_elo_language_filter_reduces_battles(tmp_path): result_folder=str(tmp_path), n_instructions=None, languages=["en"] ) ) - total_all = ( - result_all["num_wins"] + result_all["num_losses"] + result_all["num_ties"] - ) - total_en = result_en["num_wins"] + result_en["num_losses"] + result_en["num_ties"] + total_all = _num_pairwise_rows(result_all) + total_en = _num_pairwise_rows(result_en) assert total_en < total_all @@ -238,7 +351,9 @@ def test_run_elo_model_in_bootstrap_ratings(tmp_path): """Our model should appear in the bootstrap ELO leaderboard.""" result = run_elo_with_task(_default_args(result_folder=str(tmp_path))) model_name = result["model_name"] - assert all(model_name in r for r in result["bootstrap_ratings"]) + assert all( + model_name in ratings for ratings in _rating_metric(result)["bootstrap_ratings"] + ) def test_run_elo_n_instructions_limits_battles(tmp_path): @@ -249,18 +364,8 @@ def test_run_elo_n_instructions_limits_battles(tmp_path): result_10 = run_elo_with_task( _default_args(result_folder=str(tmp_path), n_instructions=10) ) - total_5 = ( - result_5["num_wins"] - + result_5["num_losses"] - + result_5["num_ties"] - + result_5["num_missing"] - ) - total_10 = ( - result_10["num_wins"] - + result_10["num_losses"] - + result_10["num_ties"] - + result_10["num_missing"] - ) + total_5 = _num_pairwise_rows(result_5) + total_10 = _num_pairwise_rows(result_10) assert total_5 == 5 assert total_10 == 10 @@ -481,10 +586,8 @@ def test_elo_language_variant_resolves_and_filters(tmp_path): result_folder=str(tmp_path), task="elo-lmarena-140k", n_instructions=None ) ) - total_en = result_en["num_wins"] + result_en["num_losses"] + result_en["num_ties"] - total_all = ( - result_all["num_wins"] + result_all["num_losses"] + result_all["num_ties"] - ) + total_en = _num_pairwise_rows(result_en) + total_all = _num_pairwise_rows(result_all) assert 0 < total_en < total_all @@ -498,19 +601,18 @@ def fake_calibrate(delta_s, y): captured["n_pairs"] = len(delta_s) return 0.42 - monkeypatch.setattr(estimate_elo_ratings, "calibrate_temperature", fake_calibrate) + monkeypatch.setattr(elo_calibration, "fit_temperature", fake_calibrate) # Anchor battles require models with >= 500 appearances; the default # 30-battle fixture leaves the calibration pool empty. monkeypatch.setattr( estimate_elo_ratings, "load_battles", lambda _task: _arena_df(900) ) - result = run_elo_with_task( + run_elo_with_task( _default_args(result_folder=str(tmp_path), calibrate_temperature=True) ) assert captured["n_pairs"] >= 10 - assert 0.0 <= result["winrate"] <= 1.0 def test_extract_instruction_text_tolerates_moderated_turns(): @@ -541,3 +643,55 @@ def spy(*args, **kwargs): # The default preset's registered parser instance, not a fresh fallback. assert captured["parse"] is JUDGE_PARSERS["score"] + + +def test_run_elo_preserves_soft_preferences_from_non_pairscore_parser( + tmp_path, monkeypatch +): + soft_parser = object() + monkeypatch.setattr( + estimate_elo_ratings, + "resolve_run_judge_prompt", + lambda *_args, **_kwargs: SimpleNamespace( + parser=soft_parser, + preset_name="soft-parser", + system_prompt="system", + user_prompt_template="{instruction} {completion_A} {completion_B}", + ), + ) + + def fake_judge_and_parse_prefs(**kwargs): + annotations = [ + JudgeAnnotation( + instruction=instruction, + completion_A=completion_a, + completion_B=completion_b, + judge_completion="not a PairScore response", + judge_input="prompt", + ) + for instruction, completion_a, completion_b in zip( + kwargs["instructions"], + kwargs["completions_A"], + kwargs["completions_B"], + strict=True, + ) + ] + return annotations, None, pd.Series([0.75] * len(annotations)) + + captured_prefs = [] + convert = estimate_elo_ratings.prefs_to_battle_results + + def capture_prefs(prefs, *args, **kwargs): + captured_prefs.extend(prefs) + return convert(prefs, *args, **kwargs) + + monkeypatch.setattr( + estimate_elo_ratings, + "judge_and_parse_prefs", + fake_judge_and_parse_prefs, + ) + monkeypatch.setattr(estimate_elo_ratings, "prefs_to_battle_results", capture_prefs) + + run_elo_with_task(_default_args(result_folder=str(tmp_path))) + + assert captured_prefs and set(captured_prefs) == {0.75} diff --git a/tests/test_eval_reports.py b/tests/test_eval_reports.py index ca78bf52..da196ab4 100644 --- a/tests/test_eval_reports.py +++ b/tests/test_eval_reports.py @@ -1,194 +1,136 @@ -import io import json -from contextlib import redirect_stdout import pandas as pd -from judgearena.utils.eval import BattleReport, PrefSummary, compute_pref_summary +from judgearena.reports import BattleReport, EloReport +from judgearena.utils.eval import PrefSummary, compute_pref_summary -def test_compute_pref_summary_returns_prefsummary(): - # 0.0 = A wins, 1.0 = B wins, 0.5 = tie, None = missing - prefs = pd.Series([0.0, 0.0, 1.0, 0.5, None]) +def test_compute_pref_summary_returns_win_loss_tie_rate(): + prefs = pd.Series([0.0, 0.2, 1.0, 0.5, None]) summary = compute_pref_summary(prefs) + assert isinstance(summary, PrefSummary) assert summary.num_battles == 5 assert summary.num_wins == 2 assert summary.num_losses == 1 assert summary.num_ties == 1 assert summary.num_missing == 1 - assert summary.winrate == (2 + 0.5 * 1) / 4 - - -def test_prefsummary_to_dict_keys(): - prefs = pd.Series([0.0, 1.0]) - keys = set(compute_pref_summary(prefs).to_dict().keys()) - assert keys == { - "num_battles", - "winrate", - "num_wins", - "num_losses", - "num_ties", - "num_missing", - } + assert summary.winrate == (2 + 0.5) / 4 -def _summary( - num_battles=4, winrate=0.5, num_wins=2, num_losses=1, num_ties=1, num_missing=0 -): - return PrefSummary( - num_battles=num_battles, - winrate=winrate, - num_wins=num_wins, - num_losses=num_losses, - num_ties=num_ties, - num_missing=num_missing, - ) +def test_report_compatibility_exports(): + from judgearena.benchmarks.elo.runner import EloReport as RunnerEloReport + from judgearena.utils.eval import BattleReport as UtilsBattleReport + assert RunnerEloReport is EloReport + assert UtilsBattleReport is BattleReport -def test_battlereport_to_dict_arena_shape(): - report = BattleReport( - task="alpaca-eval", - model_a="my-model", - model_b="gpt4", - judge_model="judge", - summary=_summary(), - swap_mode="fixed", - result_folder="/tmp/run", - preferences=[0.0, 1.0, 0.5, None], - metadata={"baseline_assignment": "flat", "prompt_preset": "default"}, - ) - d = report.to_dict() - assert d["schema_version"] == "1" - assert d["report_type"] == "BattleReport" - assert d["task"] == "alpaca-eval" - assert d["model_A"] == "my-model" - assert d["model_B"] == "gpt4" - assert d["judge_model"] == "judge" - assert d["swap_mode"] == "fixed" - assert d["result_folder"] == "/tmp/run" - assert d["metadata"]["baseline_assignment"] == "flat" - assert d["metadata"]["prompt_preset"] == "default" - assert d["num_wins"] == 2 - assert d["preferences"] == [0.0, 1.0, 0.5, None] - assert "per_category" not in d - assert "per_turn" not in d - - -def test_battlereport_to_dict_mtbench_shape(): + +def test_battle_report_serializes_metrics_as_the_result(): + metrics = { + "pairwise_win_rate": { + "winrate": 0.5, + "num_battles": 4, + "groups": {"category": [{"group": "writing", "values": {"winrate": 0.6}}]}, + } + } report = BattleReport( task="mt-bench", model_a="my-model", model_b="baseline", judge_model="judge", - summary=_summary(), - per_category={ - "writing": {"winrate": 0.6, "num_wins": 3, "num_losses": 2, "num_ties": 0} - }, - per_turn={1: {"winrate": 0.5, "num_wins": 1, "num_losses": 1, "num_ties": 0}}, + metrics=metrics, preferences=[0.0, 1.0], metadata={"date": "2026-06-16", "user": "tester"}, ) - d = report.to_dict() - assert d["schema_version"] == "1" - assert d["report_type"] == "BattleReport" - assert d["per_category"]["writing"]["winrate"] == 0.6 - assert d["per_turn"][1]["winrate"] == 0.5 - assert d["metadata"]["date"] == "2026-06-16" - assert "swap_mode" not in d - assert "result_folder" not in d + result = report.to_dict() -def test_battlereport_render_arena_swap_both(): - report = BattleReport( - task="alpaca-eval", - model_a="A", - model_b="B", - judge_model="J", - summary=_summary(num_battles=4, winrate=0.5), - swap_mode="both", - result_folder="/tmp/x", - preferences=[], - metadata={}, - ) - buf = io.StringIO() - with redirect_stdout(buf): - report.render() - out = buf.getvalue() - assert "MODEL BATTLE RESULTS" in out - assert "Win Rate (A): 50.0%" in out - assert "both orders" in out - assert "/tmp/x" in out + assert result["schema_version"] == "1" + assert result["report_type"] == "BattleReport" + assert result["metrics"] == metrics + assert result["model_A"] == "my-model" + assert result["model_B"] == "baseline" + assert "winrate" not in result + assert "per_category" not in result + assert "per_turn" not in result -def test_battlereport_render_mtbench_breakdowns(): +def test_battle_report_renders_metrics_and_groups(capsys): report = BattleReport( - task="mt-bench", - model_a="A", - model_b="B", - judge_model="J", - summary=_summary(), - per_category={ - "writing": {"winrate": 0.6, "num_wins": 3, "num_losses": 2, "num_ties": 0} + task="demo", + model_a="candidate", + model_b="baseline", + judge_model="judge", + metrics={ + "length_controlled_winrate": { + "winrate": 0.52, + "num_scored": 10, + "num_pairs": 10, + "groups": { + "category": [ + { + "group": "writing", + "values": { + "winrate": 0.6, + "num_scored": 5, + "num_pairs": 5, + }, + } + ] + }, + } }, - per_turn={1: {"winrate": 0.5, "num_wins": 1, "num_losses": 1, "num_ties": 0}}, - preferences=[], - metadata={}, + result_folder="/tmp/run", ) - buf = io.StringIO() - with redirect_stdout(buf): - report.render() - out = buf.getvalue() - assert "Per-Category Breakdown:" in out - assert "writing" in out - assert "Per-Turn Breakdown:" in out + report.render() + output = capsys.readouterr().out -def test_battlereport_save_round_trip(tmp_path): + assert "length_controlled_winrate" in output + assert "category=writing" in output + assert "/tmp/run" in output + + +def test_battle_report_save_round_trip(tmp_path): report = BattleReport( task="alpaca-eval", model_a="my-model", model_b="gpt4", judge_model="judge", - summary=_summary(), + metrics={"pairwise_win_rate": {"winrate": 0.5}}, swap_mode="fixed", result_folder="/tmp/run", preferences=[0.0, 1.0, 0.5], metadata={"baseline_assignment": "flat"}, ) + path = report.save(tmp_path / "r.json") - assert path.exists() loaded = json.loads(path.read_text()) + assert loaded == report.to_dict() assert loaded["schema_version"] == "1" - assert loaded["report_type"] == "BattleReport" def test_eloreport_to_dict_envelope(): - from judgearena.benchmarks.elo.runner import EloReport - report = EloReport( arena="chatbot-arena", judge_model="judge", - summary=_summary(), + metrics={"bradley_terry": {"ratings": {"my-model": 1000.0}}}, num_battles=10, - llm_judged_battles=10, - human_anchor_battles=5, - elo_mean=1000.0, - elo_std=10.0, - elo_n_bootstraps=100, - mae_vs_human=5.0, - method="Soft-ELO", - n_bootstraps=100, model_name="my-model", - mean_ratings={"my-model": 1000.0}, - battle_counts={"my-model": 10}, - human_elo={"gpt4": 1100.0}, - bootstrap_ratings=[{"my-model": 1000.0}], sampling_metadata={"sampling_mode": "head"}, ) - d = report.to_dict() - assert d["schema_version"] == "1" - assert d["report_type"] == "EloReport" - assert d["arena"] == "chatbot-arena" - assert d["model_name"] == "my-model" + + result = report.to_dict() + assert result == { + "arena": "chatbot-arena", + "judge_model": "judge", + "metrics": {"bradley_terry": {"ratings": {"my-model": 1000.0}}}, + "num_battles": 10, + "model_name": "my-model", + "sampling_metadata": {"sampling_mode": "head"}, + "schema_version": "1", + "report_type": "EloReport", + } diff --git a/tests/test_generate_and_evaluate.py b/tests/test_generate_and_evaluate.py index 8f9b420f..6f4b9211 100644 --- a/tests/test_generate_and_evaluate.py +++ b/tests/test_generate_and_evaluate.py @@ -1,12 +1,17 @@ +import json +from dataclasses import replace from types import SimpleNamespace +import numpy as np import pandas as pd import pytest +from langchain_core.language_models.fake import FakeListLLM import judgearena.benchmarks.execution as benchmark_execution import judgearena.benchmarks.pairwise.runner as generate_and_evaluate import judgearena.benchmarks.registry as benchmark_registry import judgearena.benchmarks.runner as benchmark_runner +from judgearena.benchmarks.elo.rating import fit_bradley_terry from judgearena.benchmarks.pairwise.baselines import ( BaselinePlan, native_pairwise_baseline, @@ -17,6 +22,7 @@ from judgearena.config import RunConfig from judgearena.datasets.pairwise import PairwiseTaskData from judgearena.tasks.registry import get_packaged_task +from judgearena.tasks.schema import MetricSpec, ScoringSpec def _cfg( @@ -328,6 +334,99 @@ def batch(self, inputs, **_kwargs): assert captured["make_model"]["tensor_parallel_size"] == 4 +def test_pairwise_grouping_accepts_all_canonical_battle_columns(tmp_path): + task = get_packaged_task("alpaca-eval") + canonical_fields = ( + "model_a", + "model_b", + "completion_a", + "completion_b", + "evaluation_model", + "source", + "pref_hard", + ) + protocol = task.spec.protocol.model_copy( + update={ + "scoring": ScoringSpec( + metrics=( + MetricSpec( + metric="pairwise_win_rate", + breakdown_by=canonical_fields, + ), + ) + ) + } + ) + task = replace(task, spec=task.spec.model_copy(update={"protocol": protocol})) + + prefs = run_pairwise( + _cfg( + task="alpaca-eval", + model_A="Dummy/a", + model_B="Dummy/b", + judge_model="Dummy/score A: 10 score B: 0", + n_instructions=2, + result_folder=str(tmp_path), + ), + task, + ) + + assert len(prefs) == 2 + assert (prefs < 0.5).all() + + +@pytest.mark.parametrize("seed", [17, 29]) +def test_pairwise_bootstraps_use_run_seed(monkeypatch, tmp_path, seed): + monkeypatch.setattr( + benchmark_execution, + "make_model", + lambda **_kwargs: FakeListLLM( + responses=[ + "score A: 10 score B: 0", + "score A: 0 score B: 10", + "score A: 5 score B: 5", + ] + ), + ) + cfg = _cfg( + task="alpaca-eval", + model_A="Dummy/a", + model_B="Dummy/b", + judge_model="Dummy/judge", + n_instructions=3, + result_folder=str(tmp_path), + ) + cfg.run.seed = seed + task = get_packaged_task(cfg.task) + protocol = task.spec.protocol.model_copy( + update={ + "scoring": ScoringSpec( + metrics=( + MetricSpec(metric="bradley_terry", parameters={"n_bootstraps": 3}), + ) + ) + } + ) + task = replace(task, spec=task.spec.model_copy(update={"protocol": protocol})) + + prefs = run_pairwise(cfg, task) + + battles = pd.DataFrame( + {"model_a": cfg.model.name, "model_b": cfg.model.baseline, "pref": prefs} + ) + rng = np.random.default_rng(seed) + expected = [ + fit_bradley_terry( + battles.sample( + n=len(battles), replace=True, random_state=int(rng.integers(0, 2**31)) + ) + ) + for _ in range(3) + ] + saved = json.loads(next(tmp_path.glob("*/results-*.json")).read_text()) + assert saved["metrics"]["bradley_terry"]["bootstrap_ratings"] == expected + + def test_run_writes_roundtrippable_config(tmp_path): from judgearena.config import load_config @@ -343,6 +442,9 @@ def test_run_writes_roundtrippable_config(tmp_path): ) written = list(tmp_path.glob("*/config.yaml")) assert written, "config.yaml not written" + result = json.loads(next(tmp_path.glob("*/results-*.json")).read_text()) + assert result["metrics"]["pairwise_win_rate"]["num_battles"] == 2 + assert "winrate" not in result reloaded = load_config(written[0]) assert reloaded.task == "alpaca-eval" assert reloaded.model.name == "Dummy/no answer" diff --git a/tests/test_mt_bench_downloads.py b/tests/test_mt_bench_downloads.py index 93d26241..aae1a5ce 100644 --- a/tests/test_mt_bench_downloads.py +++ b/tests/test_mt_bench_downloads.py @@ -1,4 +1,6 @@ +import json from datetime import UTC, datetime +from types import SimpleNamespace import pandas as pd import pytest @@ -484,3 +486,67 @@ def fake_judge(**kwargs): "coding", "arena-hard-200", ) + + +def test_mt_bench_finalization_uses_shared_grouped_metric(monkeypatch, tmp_path): + task = get_packaged_task("mt-bench") + assert task is not None + monkeypatch.setattr( + mt_bench_runner, "write_run_metadata_safely", lambda **_kwargs: None + ) + cfg = RunConfig( + task="mt-bench", + model={"name": "candidate", "baseline": "reference"}, + judge={"model": "judge"}, + ) + prompt = SimpleNamespace( + metadata=lambda: {}, + system_prompt=None, + user_prompt_template="{instruction}", + ) + index = pd.Index([1, 2], name="question_id") + questions = pd.DataFrame( + {"turn_1": ["q1", "q2"], "turn_2": ["q1b", "q2b"]}, index=index + ) + completions_a = pd.DataFrame( + {"completion_turn_1": ["a1", "a2"], "completion_turn_2": ["a1b", "a2b"]}, + index=index, + ) + completions_b = pd.DataFrame( + {"completion_turn_1": ["b1", "b2"], "completion_turn_2": ["b1b", "b2b"]}, + index=index, + ) + preferences = pd.Series([0.0, 1.0, 0.0, 0.5]) + metadata = [ + {"question_id": 1, "category": "math", "turn": 1}, + {"question_id": 1, "category": "math", "turn": 2}, + {"question_id": 2, "category": "writing", "turn": 1}, + {"question_id": 2, "category": "writing", "turn": 2}, + ] + + returned = mt_bench_runner._finalize_mt_bench_run( + cfg=cfg, + protocol=task.spec.protocol, + res_folder=tmp_path, + result_name="result", + prefs=preferences, + annotations=[], + combined_metadata=metadata, + resolved_prompt=prompt, + questions_df=questions, + completions_a=completions_a, + completions_b=completions_b, + started_at_utc=datetime.now(UTC), + ) + + assert returned.equals(preferences) + saved = json.loads((tmp_path / "results-result.json").read_text()) + metric = saved["metrics"]["pairwise_win_rate"] + assert metric["winrate"] == pytest.approx(0.625) + assert [ + (item["group"], item["values"]["winrate"]) + for item in metric["groups"]["category"] + ] == [("math", 0.5), ("writing", 0.75)] + assert [ + (item["group"], item["values"]["winrate"]) for item in metric["groups"]["turn"] + ] == [(1, 1.0), (2, 0.25)] diff --git a/tests/test_mt_bench_preset_judging.py b/tests/test_mt_bench_preset_judging.py index 3a033cf2..e57e36e6 100644 --- a/tests/test_mt_bench_preset_judging.py +++ b/tests/test_mt_bench_preset_judging.py @@ -1,14 +1,31 @@ from __future__ import annotations +import json +from datetime import UTC, datetime +from types import SimpleNamespace + +import numpy as np import pandas as pd import pytest +import judgearena.benchmarks.mt_bench.runner as mt_bench_runner +from judgearena.benchmarks.elo.rating import fit_bradley_terry from judgearena.benchmarks.mt_bench.preset_judging import ( _build_mt_bench_preset_items, _select_preset_prompt, judge_mt_bench_with_preset, ) -from judgearena.prompts.registry import FASTCHAT_PAIRWISE_PROMPT_PRESET +from judgearena.benchmarks.mt_bench.runner import _build_mt_bench_battles +from judgearena.benchmarks.pairwise.scoring.metrics import ( + LengthControlledWinrateMetric, +) +from judgearena.config import RunConfig +from judgearena.prompts.registry import ( + FASTCHAT_PAIRWISE_PROMPT_PRESET, + resolve_judge_prompt, +) +from judgearena.tasks.registry import get_packaged_task +from judgearena.tasks.schema import MetricSpec, ScoringSpec REFERENCE_CATEGORIES = ("math", "reasoning", "coding", "arena-hard-200") @@ -155,9 +172,167 @@ def test_judge_mt_bench_with_preset_parses_and_inverts_swapped_scores(): assert annotations[1]["swapped"] is True assert "B1" in annotations[1]["user_prompt"] assert metadata == [ - {"question_id": 1, "category": "writing", "turn": 1}, - {"question_id": 1, "category": "writing", "turn": 1}, + { + "question_id": 1, + "category": "writing", + "turn": 1, + "orientation": "direct", + }, + { + "question_id": 1, + "category": "writing", + "turn": 1, + "orientation": "reversed", + }, + ] + battles = _build_mt_bench_battles( + cfg=SimpleNamespace( + model=SimpleNamespace(name="model-a", baseline="model-b"), + judge=SimpleNamespace(model="judge"), + ), + prefs=prefs, + combined_metadata=metadata, + completions_a=_completions_df("A"), + completions_b=_completions_df("B"), + ) + metric = LengthControlledWinrateMetric().calculate(battles) + assert metric == {"num_pairs": 1, "num_scored": 1, "winrate": None} + + +def test_fixed_preset_judgment_builds_one_single_orientation_battle(): + prefs, _, metadata = judge_mt_bench_with_preset( + judge_chat_model=SequenceJudge(["score_A: 10\nscore_B: 0"]), + judge_model="judge", + questions=_questions_df(category="writing"), + completions_a=_completions_df("A"), + completions_b=_completions_df("B"), + model_a="model-a", + model_b="model-b", + turns_mode="single", + swap_mode="fixed", + truncate_input_chars=None, + use_tqdm=False, + reference_categories=REFERENCE_CATEGORIES, + prompt_preset="default", + ) + + assert metadata[0]["orientation"] == "single" + battles = _build_mt_bench_battles( + cfg=SimpleNamespace( + model=SimpleNamespace(name="model-a", baseline="model-b"), + judge=SimpleNamespace(model="judge"), + ), + prefs=prefs, + combined_metadata=metadata, + completions_a=_completions_df("A"), + completions_b=_completions_df("B"), + ) + assert LengthControlledWinrateMetric().calculate(battles) == { + "num_pairs": 1, + "num_scored": 1, + "winrate": None, + } + + +def test_mt_bench_battles_preserve_preferences_and_turn_ids(): + prefs = pd.Series([0.2, 0.8, 0.5, np.nan]) + metadata = [ + {"question_id": question_id, "turn": turn} + for question_id, turn in [(1, 1), (1, 2), (2, 1), (2, 2)] + ] + battles = _build_mt_bench_battles( + cfg=SimpleNamespace( + model=SimpleNamespace(name="candidate", baseline="reference"), + judge=SimpleNamespace(model="judge"), + ), + prefs=prefs, + combined_metadata=metadata, + completions_a=pd.DataFrame( + {"completion_turn_1": ["a1", "a2"], "completion_turn_2": ["a1b", "a2b"]}, + index=[1, 2], + ), + completions_b=pd.DataFrame( + {"completion_turn_1": ["b1", "b2"], "completion_turn_2": ["b1b", "b2b"]}, + index=[1, 2], + ), + ) + + pd.testing.assert_series_equal(battles["pref"], prefs.rename("pref")) + pd.testing.assert_series_equal( + battles["pref_hard"], pd.Series([0.0, 1.0, 0.5, np.nan], name="pref_hard") + ) + assert { + "instruction_index", + "model", + "baseline", + "completion_model", + "completion_baseline", + "orientation", + "pref", + } <= set(battles) + assert battles["instruction_index"].tolist() == [ + "1:turn-1", + "1:turn-2", + "2:turn-1", + "2:turn-2", + ] + + +@pytest.mark.parametrize("seed", [17, 29]) +def test_mt_bench_hard_bootstraps_use_run_seed(monkeypatch, tmp_path, seed): + monkeypatch.setattr( + mt_bench_runner, "write_run_metadata_safely", lambda **_kwargs: None + ) + cfg = RunConfig( + task="mt-bench", + model={"name": "candidate", "baseline": "reference"}, + judge={"model": "judge", "prompt_preset": "default", "swap_mode": "fixed"}, + run={"seed": seed, "use_tqdm": False}, + ) + protocol = get_packaged_task(cfg.task).spec.protocol.model_copy( + update={ + "scoring": ScoringSpec( + metrics=( + MetricSpec( + metric="bradley_terry", + parameters={"soft": False, "n_bootstraps": 3}, + ), + ) + ) + } + ) + + mt_bench_runner._run_mt_bench_preset( + cfg=cfg, + protocol=protocol, + res_folder=tmp_path, + result_name="result", + questions_df=_questions_df(), + completions_a=_completions_df("A"), + completions_b=_completions_df("B"), + judge_chat_model=SequenceJudge( + ["score A: 10 score B: 0", "score A: 0 score B: 10"] + ), + resolved_prompt=resolve_judge_prompt(preset="default"), + started_at_utc=datetime.now(UTC), + ) + + battles = pd.DataFrame( + {"model_a": "candidate", "model_b": "reference", "pref": [0.0, 1.0]} + ) + rng = np.random.default_rng(seed) + expected = [ + fit_bradley_terry( + battles.sample( + n=len(battles), replace=True, random_state=int(rng.integers(0, 2**31)) + ) + ) + for _ in range(3) ] + saved = json.loads((tmp_path / "results-result.json").read_text()) + metric = saved["metrics"]["bradley_terry"] + assert metric["method"] == "ELO" + assert metric["bootstrap_ratings"] == expected def test_select_preset_prompt_forwards_named_parser(tmp_path, monkeypatch): diff --git a/tests/test_pairwise_scoring.py b/tests/test_pairwise_scoring.py index a685b0e8..f267d7cb 100644 --- a/tests/test_pairwise_scoring.py +++ b/tests/test_pairwise_scoring.py @@ -1,16 +1,315 @@ -"""Tests for runtime pairwise scoring adapters.""" +"""Behavior tests for composable pairwise metrics.""" +import math + +import numpy as np import pandas as pd import pytest -from judgearena.benchmarks.pairwise.scoring import PAIRWISE_SCORERS +from judgearena.benchmarks.pairwise.scoring import ( + LengthControlledWinrateMetric, + PairwiseWinRateMetric, + collapse_pairwise_battles, +) +from judgearena.benchmarks.scoring import ( + available_metrics, + build_metric, + build_metrics, + calculate_metrics, + render_metrics, +) +from judgearena.tasks.schema import MetricSpec + + +def _calculate_metrics( + battles: pd.DataFrame, requests: tuple[MetricSpec, ...] +) -> dict[str, dict[str, object]]: + return calculate_metrics(battles, build_metrics(requests)) + + +def _battle_rows( + length_differences: np.ndarray, + outcomes: np.ndarray, +) -> pd.DataFrame: + baseline_length = 200 + return pd.DataFrame( + { + "instruction_index": range(len(outcomes)), + "model": "candidate", + "baseline": "reference", + "completion_model": [ + "m" * (baseline_length + int(delta)) for delta in length_differences + ], + "completion_baseline": ["b" * baseline_length] * len(outcomes), + "pref": 1 - outcomes, + "orientation": "single", + } + ) + + +def test_pairwise_win_rate_reports_candidate_results(): + battles = pd.DataFrame({"pref": [0.0, 0.25, 1.0, None]}) + + result = PairwiseWinRateMetric().calculate(battles) + + assert result == { + "num_battles": 4, + "winrate": pytest.approx(2 / 3), + "num_wins": 2, + "num_losses": 1, + "num_ties": 0, + "num_missing": 1, + } + + +def test_length_controlled_winrate_predicts_at_equal_length(monkeypatch): + import judgearena.benchmarks.pairwise.scoring.metrics as metrics + + monkeypatch.setattr(metrics, "BOOTSTRAP_ROUNDS", 100) + differences = np.arange(-100, 101, 10, dtype=float) + scale = differences.std(ddof=1) + expected = 0.4 + intercept = math.log(expected / (1 - expected)) + outcomes = 1 / (1 + np.exp(-(intercept + 0.8 * differences / scale))) + + result = LengthControlledWinrateMetric().calculate( + _battle_rows(differences, outcomes) + ) + + assert result["winrate"] == pytest.approx(expected, abs=1e-5) + assert result["confidence_interval"] is not None + assert float(outcomes.mean()) != pytest.approx(expected) + assert "raw_winrate" not in result + + +def test_length_control_requires_complete_answer_order_pairs(): + direct = _battle_rows(np.array([-10, 0, 10]), np.array([0.2, 0.5, 0.8])) + direct["orientation"] = "direct" + reversed_rows = direct.copy() + reversed_rows["orientation"] = "reversed" + reversed_rows.loc[1, "pref"] = np.nan + battles = pd.concat([direct, reversed_rows], ignore_index=True) + + collapsed = collapse_pairwise_battles(battles) + + assert collapsed["n_parsed"].tolist() == [2, 1, 2] + result = LengthControlledWinrateMetric().calculate(battles) + assert result == {"num_pairs": 3, "num_scored": 2, "winrate": None} + + +def test_metrics_produce_separate_breakdowns_for_each_field(): + battles = pd.DataFrame( + { + "pref": [0.0, 1.0, 0.0, 0.0], + "category": ["a", "a", "b", "b"], + "turn": [1, 2, 1, 2], + } + ) + + results = _calculate_metrics( + battles, + (MetricSpec(metric="pairwise_win_rate", breakdown_by=("category", "turn")),), + ) + + metric = results["pairwise_win_rate"] + assert metric["winrate"] == 0.75 + assert set(metric["groups"]) == {"category", "turn"} + for field, expected in { + "category": [("a", 0.5), ("b", 1.0)], + "turn": [(1, 1.0), (2, 0.5)], + }.items(): + groups = metric["groups"][field] + assert [ + (item["group"], item["values"]["winrate"]) for item in groups + ] == expected + assert [item["values"]["num_battles"] for item in groups] == [2, 2] + + +def test_metric_spec_accepts_breakdown_by_and_rejects_old_group_by(): + spec = MetricSpec.model_validate( + {"metric": "pairwise_win_rate", "breakdown_by": ["category", "turn"]} + ) + assert spec.model_dump(mode="json")["breakdown_by"] == ["category", "turn"] + with pytest.raises(ValueError, match="group_by"): + MetricSpec.model_validate( + {"metric": "pairwise_win_rate", "group_by": ["category"]} + ) + + +def test_pairwise_metrics_reject_invalid_preferences(): + with pytest.raises(ValueError, match="finite values"): + PairwiseWinRateMetric().calculate(pd.DataFrame({"pref": [0.0, float("inf")]})) + + +def test_bootstrap_does_not_replace_undefined_draws(monkeypatch): + import judgearena.benchmarks.pairwise.scoring.metrics as metrics + + monkeypatch.setattr(metrics, "BOOTSTRAP_ROUNDS", 100) + differences = np.array([-10, 0, 10], dtype=float) + outcomes = np.array([0.0, 1.0, 0.0]) + + result = LengthControlledWinrateMetric().calculate( + _battle_rows(differences, outcomes) + ) + + assert result["winrate"] is not None + assert result["confidence_interval"] is None + + +def test_grouped_metric_rejects_missing_column(): + with pytest.raises(ValueError, match="missing column 'category'"): + _calculate_metrics( + pd.DataFrame({"pref": [0.0, 1.0]}), + (MetricSpec(metric="pairwise_win_rate", breakdown_by=("category",)),), + ) + + +def test_collapse_rejects_duplicate_or_incomplete_orientations(): + direct = _battle_rows(np.array([0]), np.array([0.5])) + direct["orientation"] = "direct" + duplicate = pd.concat([direct, direct], ignore_index=True) + with pytest.raises(ValueError, match="duplicate orientations"): + collapse_pairwise_battles(duplicate) + + reversed_row = direct.copy() + reversed_row["orientation"] = "reversed" + second_direct = _battle_rows(np.array([1]), np.array([0.5])) + second_direct["instruction_index"] = 1 + second_direct["orientation"] = "direct" + incomplete = pd.concat([direct, reversed_row, second_direct], ignore_index=True) + with pytest.raises(ValueError, match="expected orientations"): + collapse_pairwise_battles(incomplete) + + +def test_collapse_rejects_different_completions_across_orders(): + direct = _battle_rows(np.array([0]), np.array([0.5])) + direct["orientation"] = "direct" + reversed_row = direct.copy() + reversed_row["orientation"] = "reversed" + reversed_row["completion_model"] = "different" + + with pytest.raises(ValueError, match="different completions"): + collapse_pairwise_battles(pd.concat([direct, reversed_row])) + + +def test_length_controlled_winrate_rejects_mixed_baselines(): + battles = _battle_rows(np.array([-10, 0, 10]), np.array([0.2, 0.5, 0.8])) + battles.loc[2, "baseline"] = "other-reference" + + with pytest.raises(ValueError, match="exactly one baseline model"): + LengthControlledWinrateMetric().calculate(battles) + + +def test_grouped_metrics_preserve_distinct_group_values(): + battles = pd.DataFrame( + {"pref": [0.0, 0.0, 0.0, 0.0], "group": [1, "1", None, "missing"]} + ) + + result = _calculate_metrics( + battles, + (MetricSpec(metric="pairwise_win_rate", breakdown_by=("group",)),), + ) + + values = [item["group"] for item in result["pairwise_win_rate"]["groups"]["group"]] + assert {(type(value).__name__, value) for value in values} == { + ("int", 1), + ("str", "1"), + ("str", "missing"), + ("NoneType", None), + } + + +def test_pairwise_win_rate_selects_and_orients_evaluation_rows(): + battles = pd.DataFrame( + { + "model_a": ["candidate", "opponent", "anchor-a"], + "model_b": ["opponent", "candidate", "anchor-b"], + "evaluation_model": ["candidate", "candidate", None], + "pref": [0.0, 1.0, 1.0], + "source": ["llm-judge", "llm-judge", "human"], + } + ) + + result = PairwiseWinRateMetric().calculate(battles) + + assert result["num_battles"] == 2 + assert result["winrate"] == 1.0 + + +def test_shared_registry_calculates_and_renders_point_bradley_terry(): + battles = pd.DataFrame( + { + "model_a": ["a", "a", "b", "b"], + "model_b": ["b", "b", "a", "a"], + "pref": [0.0, 0.0, 1.0, 1.0], + } + ) + + results = _calculate_metrics(battles, (MetricSpec(metric="bradley_terry"),)) + + ratings = results["bradley_terry"]["ratings"] + assert set(ratings) == {"a", "b"} + assert ratings["a"] > ratings["b"] + rendered = render_metrics(results) + assert "bradley_terry" in rendered + assert "a:" in rendered + assert "b:" in rendered + + +def test_length_controlled_winrate_accepts_elo_shaped_evaluation_battles(): + battles = pd.DataFrame( + { + "instruction_index": [0, 1, 2], + "model_a": ["candidate", "opponent", "candidate"], + "model_b": ["opponent", "candidate", "opponent"], + "evaluation_model": ["candidate"] * 3, + "completion_a": ["a", "bbbb", "aaaaaa"], + "completion_b": ["bbb", "bb", "b"], + "pref": [0.8, 0.2, 0.1], + "orientation": ["single"] * 3, + } + ) + + result = LengthControlledWinrateMetric().calculate(battles) + + assert result["num_pairs"] == 3 + assert result["num_scored"] == 3 + assert 0.0 <= result["winrate"] <= 1.0 + + +def test_metric_builders_hide_registry_and_validate_parameters(): + assert available_metrics() == tuple(sorted(available_metrics())) + assert "bradley_terry" in available_metrics() + + metric = build_metric("pairwise_win_rate") + result = metric.calculate(pd.DataFrame({"pref": [0.1]})) + assert result["winrate"] == 1.0 + assert "100.00%" in metric.render(result) + + with pytest.raises(ValueError, match="Unknown metric"): + build_metric("missing") + with pytest.raises(ValueError, match="unexpected keyword argument"): + build_metric("pairwise_win_rate", {"soft": False}) + with pytest.raises(ValueError, match="soft must be a boolean"): + build_metric("bradley_terry", {"soft": "false"}) -def test_pairwise_win_rate_scorer_owns_metric_semantics(): - battles = pd.DataFrame({"pref": pd.Series([0.0, 0.0, 1.0, None], dtype=float)}) +def test_build_metrics_preserves_order_and_applies_overrides(): + requests = ( + MetricSpec(metric="pairwise_win_rate"), + MetricSpec(metric="bradley_terry", parameters={"n_bootstraps": 2}), + ) - summary = PAIRWISE_SCORERS["pairwise_win_rate"](battles) + configured = build_metrics( + requests, + parameter_overrides_by_metric={ + "bradley_terry": {"n_bootstraps": 3}, + }, + ) - assert summary.num_battles == 4 - assert summary.num_missing == 1 - assert summary.winrate == pytest.approx(2 / 3) + assert [request.metric for request, _ in configured] == [ + "pairwise_win_rate", + "bradley_terry", + ] + assert configured[1][1].n_bootstraps == 3 + assert configured[1][0].parameters == {"n_bootstraps": 2} diff --git a/tests/test_task_registry.py b/tests/test_task_registry.py index 969dd18b..ecfc4807 100644 --- a/tests/test_task_registry.py +++ b/tests/test_task_registry.py @@ -42,7 +42,7 @@ def _task_definition(task: str = "test-task") -> dict[str, object]: "default_prompt_preset": "default", "default_swap_mode": "fixed", }, - "scoring": {"adapter": "pairwise_win_rate"}, + "scoring": {"metrics": [{"metric": "pairwise_win_rate"}]}, }, } @@ -98,7 +98,10 @@ def test_packaged_registry_discovers_versioned_tasks(): ) assert elo_comparia.spec.protocol.runner == "elo" assert elo_comparia.spec.protocol.arena == "ComparIA" - assert elo_comparia.spec.protocol.scoring.adapter == "bradley_terry" + assert [metric.metric for metric in elo_comparia.spec.protocol.scoring.metrics] == [ + "pairwise_win_rate", + "bradley_terry", + ] assert elo_comparia.spec.dataset.sources["comparia"].revision == ( "7a40bce496c1f2aa3be4001da85a49cb4743042b" ) @@ -151,7 +154,7 @@ def test_packaged_registry_discovers_versioned_tasks(): assert mt_bench.spec.dataset.sources["benchmark"].revision == ( "a4b674ca573c24143824ac7f60d9173e7081e37d" ) - assert alpaca.spec.protocol.scoring.adapter == "pairwise_win_rate" + assert alpaca.spec.protocol.scoring.metrics[0].metric == "pairwise_win_rate" def test_find_returns_none_for_unregistered_task(): @@ -368,9 +371,9 @@ def test_registry_rejects_dataset_adapter_from_another_protocol(tmp_path): load_tasks(tmp_path) -def test_registry_rejects_unknown_scorer_id(tmp_path): +def test_registry_rejects_unknown_metric_id(tmp_path): definition = _task_definition() - definition["protocol"]["scoring"]["adapter"] = "missing_scorer" + definition["protocol"]["scoring"]["metrics"] = [{"metric": "missing_metric"}] _write_family( tmp_path, family="example", @@ -378,13 +381,15 @@ def test_registry_rejects_unknown_scorer_id(tmp_path): definition=definition, ) - with pytest.raises(TaskDefinitionError, match="unknown scorer"): + with pytest.raises(TaskDefinitionError, match="unknown metric"): load_tasks(tmp_path) -def test_registry_rejects_scorer_from_another_protocol(tmp_path): +def test_registry_validates_metric_parameters_with_source_path(tmp_path): definition = _task_definition() - definition["protocol"]["scoring"]["adapter"] = "bradley_terry" + definition["protocol"]["scoring"]["metrics"] = [ + {"metric": "pairwise_win_rate", "parameters": {"soft": False}} + ] _write_family( tmp_path, family="example", @@ -392,10 +397,33 @@ def test_registry_rejects_scorer_from_another_protocol(tmp_path): definition=definition, ) - with pytest.raises(TaskDefinitionError, match="unknown scorer"): + with pytest.raises( + TaskDefinitionError, + match=r"example/test-task.yaml: invalid metric 'pairwise_win_rate'.*unexpected", + ): load_tasks(tmp_path) +def test_metric_parameters_are_preserved_in_resolved_task(tmp_path): + definition = _task_definition() + definition["protocol"]["scoring"]["metrics"] = [ + {"metric": "bradley_terry", "parameters": {"n_bootstraps": 2}} + ] + _write_family( + tmp_path, + family="example", + filename="test-task.yaml", + definition=definition, + ) + + task = load_tasks(tmp_path)["test-task"] + + assert task.spec.protocol.scoring.metrics[0].parameters == {"n_bootstraps": 2} + assert task.spec.model_dump(mode="json")["protocol"]["scoring"]["metrics"][0][ + "parameters" + ] == {"n_bootstraps": 2} + + def test_official_outputs_must_reference_declared_source(tmp_path): definition = _task_definition() definition["protocol"]["baseline"] = { @@ -518,3 +546,46 @@ def unexpected_run_config(_argv): cli_module.cli(["tasks", "list"]) assert "alpaca-eval" in capsys.readouterr().out + + +def test_scoring_metrics_reject_duplicate_names(): + definition = _task_definition() + definition["protocol"]["scoring"]["metrics"] = [ + {"metric": "pairwise_win_rate"}, + {"metric": "pairwise_win_rate", "breakdown_by": ["category"]}, + ] + + with pytest.raises(ValueError, match="duplicate names"): + from judgearena.tasks.schema import TaskSpec + + TaskSpec.model_validate(definition) + + +def test_mt_bench_accepts_any_registered_metric(tmp_path): + definition = _task_definition("mt-test") + definition["protocol"] = { + "runner": "mt_bench", + "generation": {"mode": "multi_turn_chat"}, + "baseline": { + "strategy": "task_default", + "reference_id": "reference-output", + }, + "judge": { + "default_prompt_preset": "default", + "default_swap_mode": "fixed", + "turns_mode": "both", + "fastchat_prompt_preset": "default", + "fastchat_temperature": 0.0, + }, + "scoring": {"metrics": [{"metric": "length_controlled_winrate"}]}, + } + _write_family( + tmp_path, + family="mt", + filename="mt-test.yaml", + definition=definition, + ) + + task = load_tasks(tmp_path)["mt-test"] + + assert task.spec.protocol.scoring.metrics[0].metric == "length_controlled_winrate"