diff --git a/dokimi/changelog.d/20261026_130000_napetrov_report-eval-generate-agree.md b/dokimi/changelog.d/20261026_130000_napetrov_report-eval-generate-agree.md new file mode 100644 index 000000000..cb5ece550 --- /dev/null +++ b/dokimi/changelog.d/20261026_130000_napetrov_report-eval-generate-agree.md @@ -0,0 +1,28 @@ +### Changed + +- **`report generate` and the pipeline summary average each arm over the complete pairs, as `report eval` does** — the + Overall table, the topic and source breakdowns, and the `stats` block of `--format json` used + to average the context arm and the baseline arm over every question scored in each, so the two + means could cover different questions and their difference was not the delta printed beside + them. The `pipeline` run did the same for the summary it prints and stores in + `steps.evaluation.summary`. All of them now cover only the questions scored in both arms, and + the delta is the difference of those means. For an eval file with an unpaired question, the + context and baseline numbers change; for a file the judge writes, the delta does not. The report + says how many questions are unpaired (`paired_questions` and `unpaired_questions` in the JSON, + and a note plus the Count cell in the Markdown). To see the new numbers, run `report generate` + on an existing eval file again. + The top and bottom performer tables of `report generate` rank the complete pairs too. The + dashboard and the `report model-compare` run summary still average each arm over its own + scored questions, so for a file with an unpaired question their context and baseline means + differ from the ones `report generate` prints. + +### Fixed + +- **`report eval` computes the delta from unrounded means** — the Delta was the difference of the + two arm means after each had been rounded to one decimal, so it could be 0.1 away from the + difference of the unrounded means. It is now the unrounded difference rounded once for display, + so it can move by 0.1 in the Summary, Breakdown and Difficulty tables. The arm means themselves + print as before. Where questions are unpaired, the Count cell now shows them, and a note says the + means are over the complete pairs. + The win rate and the degradation rate in Key Observations divide by the complete pairs, not by + every question, so they can rise for a file with an unpaired question. diff --git a/dokimi/docs/quickstart.md b/dokimi/docs/quickstart.md index 329dc8245..c1015317d 100644 --- a/dokimi/docs/quickstart.md +++ b/dokimi/docs/quickstart.md @@ -161,7 +161,9 @@ python cli.py report generate \ `reports/oneTBB.md` contains summary stats (min/max/avg context-arm/baseline/delta), top-N best and worst answers, top improvements and degradations, topic and -persona clustering, and recommendations. The `reports/` directory is +persona clustering, and recommendations. The summary stats cover only the questions +scored in both arms; questions scored in one arm are counted and left out of them, and +`report eval` computes the same numbers from the same file. The `reports/` directory is git-ignored. ### Step 6 — Dashboard (optional) diff --git a/dokimi/dokimi/eval/stats.py b/dokimi/dokimi/eval/stats.py index 72101a0d9..fc0e22035 100644 --- a/dokimi/dokimi/eval/stats.py +++ b/dokimi/dokimi/eval/stats.py @@ -19,6 +19,10 @@ Holm-Bonferroni (default) or Benjamini-Hochberg adjusted p-values. paired_vectors(evaluations, arm_a, arm_b) Per-question paired score vectors for two arms, complete cases only. +paired_summary(evaluations) + Both arms and their delta over the complete pairs, with the unpaired count. + Called by the eval report, the report generator and the pipeline summary. The + dashboard and the model-compare summary still average each arm separately. mean_ci(values, ...) Student-t CI for the mean, coherent with ``p_ttest``; bootstrap fallback. point_biserial(flags, totals) @@ -45,9 +49,10 @@ import math import random +from dataclasses import dataclass from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple -from dokimi.records import RecordLike, records +from dokimi.records import BASELINE, CONTEXT, RecordLike, records try: from scipy import stats as _sp_stats @@ -333,6 +338,64 @@ def paired_vectors( return a_vals, b_vals +@dataclass(frozen=True) +class PairedSummary: + """Both arms of one question set over its complete pairs, and the delta between them. + + Every number is over the questions scored in both arms. The ``unpaired`` ones + (scored in one arm, or neither) are left out of all of them and counted here, + so no report can print a context mean over questions its baseline mean lacks. + The values are unrounded: round only when rendering. + """ + + total: int + paired: int + unpaired: int + with_mean: float + without_mean: float + delta_mean: float + with_min: float + with_max: float + without_min: float + without_max: float + delta_min: float + delta_max: float + improvements: int + degradations: int + neutral: int + + +def paired_summary(evaluations: Iterable[RecordLike]) -> PairedSummary: + """The arms and their delta over the complete pairs of ``evaluations``. + + The delta is the difference of the two paired means, which is also the mean of the + per-question deltas (``context - baseline``). Averaging each arm over *every* scored + question instead gives two means over different questions, whose difference is not + the delta the page prints beside them. An empty input gives zeros throughout. + """ + recs = records(evaluations, source="evaluations", kind="evaluations") + with_s, without_s = paired_vectors(recs, CONTEXT, BASELINE) + deltas = [w - wo for w, wo in zip(with_s, without_s, strict=True)] + paired = len(deltas) + return PairedSummary( + total=len(recs), + paired=paired, + unpaired=len(recs) - paired, + with_mean=_mean(with_s) if paired else 0.0, + without_mean=_mean(without_s) if paired else 0.0, + delta_mean=_mean(deltas) if paired else 0.0, + with_min=min(with_s, default=0.0), + with_max=max(with_s, default=0.0), + without_min=min(without_s, default=0.0), + without_max=max(without_s, default=0.0), + delta_min=min(deltas, default=0.0), + delta_max=max(deltas, default=0.0), + improvements=sum(1 for d in deltas if d > 0), + degradations=sum(1 for d in deltas if d < 0), + neutral=sum(1 for d in deltas if d == 0), + ) + + def point_biserial(flags: Sequence[bool], totals: Sequence[float]) -> Dict[str, Any]: """Discrimination of one binary item against respondents' total scores. diff --git a/dokimi/dokimi/orchestrator/pipeline.py b/dokimi/dokimi/orchestrator/pipeline.py index 5badfda32..99cd48029 100644 --- a/dokimi/dokimi/orchestrator/pipeline.py +++ b/dokimi/dokimi/orchestrator/pipeline.py @@ -10,8 +10,8 @@ from pathlib import Path from dokimi.defaults import DEFAULT_ANSWER_MODEL, DEFAULT_ANSWER_PROVIDER, DEFAULT_JUDGE_MODEL, DEFAULT_JUDGE_PROVIDER, DEFAULT_RETRIEVAL_TOP_K +from dokimi.eval.stats import paired_summary from dokimi.mcp.factory import create_doc_source_client -from dokimi.records import BASELINE, CONTEXT, records as read_records logger = logging.getLogger(__name__) @@ -380,17 +380,14 @@ def run(self, concurrency: int = 5) -> Dict[str, Any]: "path": str(self.eval_path) } - # Compute summary stats - scored = read_records(evaluations, source="evaluations", kind="evaluations") - with_scores = [s for s in (r.score(CONTEXT) for r in scored) if s is not None] - without_scores = [s for s in (r.score(BASELINE) for r in scored) if s is not None] - deltas = [r.delta for r in scored if r.delta is not None] - - if with_scores and without_scores and deltas: + # Summary stats over the complete pairs, the numbers the reports print for the same file. + summary = paired_summary(evaluations) + + if summary.paired: results["steps"]["evaluation"]["summary"] = { - "with_avg": round(sum(with_scores) / len(with_scores), 1), - "without_avg": round(sum(without_scores) / len(without_scores), 1), - "delta_avg": round(sum(deltas) / len(deltas), 1) + "with_avg": round(summary.with_mean, 1), + "without_avg": round(summary.without_mean, 1), + "delta_avg": round(summary.delta_mean, 1) } print(f"✓ Evaluated {len(evaluations)} answers: " f"context={results['steps']['evaluation']['summary']['with_avg']}, " diff --git a/dokimi/dokimi/report/eval_report.py b/dokimi/dokimi/report/eval_report.py index ebb8c657f..29d91ad3f 100755 --- a/dokimi/dokimi/report/eval_report.py +++ b/dokimi/dokimi/report/eval_report.py @@ -35,10 +35,6 @@ _TZ = None -def avg(lst): - return round(sum(lst) / len(lst), 1) if lst else 0.0 - - def significance_test(evals): """Paired t-test, Wilcoxon, and Cohen's d_z on (with_docs - without_docs) pairs. @@ -91,22 +87,41 @@ def _difficulty(e: Record, default): def stats(qs): - valid = [e for e in qs if has_scores(e)] - with_s = [e.score(CONTEXT) for e in valid] - without_s = [e.score(BASELINE) for e in valid] - deltas = [d for e in valid if (d := e.delta) is not None] + """The arms and delta over the complete pairs (:func:`eval.stats.paired_summary`). + + ``count`` is every question in ``qs``, ``unpaired`` the ones left out of the means. + Rounded here, once, for display: the delta is the difference of the unrounded means. + """ + s = eval_stats.paired_summary(qs) return { - "count": len(qs), - "valid": len(valid), - "with_avg": avg(with_s), - "without_avg": avg(without_s), - "delta_avg": round(avg(with_s) - avg(without_s), 1), - "improvements": sum(1 for d in deltas if d > 0), - "degradations": sum(1 for d in deltas if d < 0), - "neutral": sum(1 for d in deltas if d == 0), + "count": s.total, + "valid": s.paired, + "unpaired": s.unpaired, + "with_avg": round(s.with_mean, 1), + "without_avg": round(s.without_mean, 1), + "delta_avg": round(s.delta_mean, 1), + "improvements": s.improvements, + "degradations": s.degradations, + "neutral": s.neutral, } +def _count_cell(s: dict) -> str: + """The question count, with the unpaired questions the arm means leave out, if any.""" + return str(s["count"]) if not s["unpaired"] else f'{s["count"]} ({s["unpaired"]} unpaired)' + + +def _unpaired_note(s: dict) -> list[str]: + """A line saying what the Summary means leave out; nothing when every question is paired.""" + if not s["unpaired"]: + return [] + return [ + f'_Arm means and delta are over the {s["valid"]} complete pairs; ' + f'{s["unpaired"]} unpaired (scored in one arm only) left out._', + "", + ] + + def fmt_delta(d): if d is None: return "N/A" @@ -267,11 +282,12 @@ def generate_report(eval_path: str, out_path: str, qa_json_out: str | None = Non "## Summary", "| Set | Count | Context arm | Baseline | Delta |", "|---|---:|---:|---:|---:|", - f'| **All** | {all_stats["count"]} | {all_stats["with_avg"]} | {all_stats["without_avg"]} | **{all_stats["delta_avg"]:+}** |', - f'| Generated (dynamic) | {dyn_stats["count"]} | {dyn_stats["with_avg"]} | {dyn_stats["without_avg"]} | **{dyn_stats["delta_avg"]:+}** |', - f'| Golden (static) | {static_stats["count"]} | {static_stats["with_avg"]} | {static_stats["without_avg"]} | **{static_stats["delta_avg"]:+}** |', + f'| **All** | {_count_cell(all_stats)} | {all_stats["with_avg"]} | {all_stats["without_avg"]} | **{all_stats["delta_avg"]:+}** |', + f'| Generated (dynamic) | {_count_cell(dyn_stats)} | {dyn_stats["with_avg"]} | {dyn_stats["without_avg"]} | **{dyn_stats["delta_avg"]:+}** |', + f'| Golden (static) | {_count_cell(static_stats)} | {static_stats["with_avg"]} | {static_stats["without_avg"]} | **{static_stats["delta_avg"]:+}** |', "", ] + lines += _unpaired_note(all_stats) # Statistical significance sig = significance_test(evals) @@ -322,7 +338,7 @@ def _val(key): ("**Total**", all_stats), ]: lines.append( - f'| {label} | {s["count"]} | {s["with_avg"]} | {s["without_avg"]} ' + f'| {label} | {_count_cell(s)} | {s["with_avg"]} | {s["without_avg"]} ' f'| **{s["delta_avg"]:+}** | {s["improvements"]} | {s["degradations"]} | {s["neutral"]} |' ) lines.append("") @@ -337,7 +353,7 @@ def _val(key): if d in diff_groups: s = stats(diff_groups[d]) lines.append( - f'| {d} | {s["count"]} | {s["with_avg"]} | {s["without_avg"]} ' + f'| {d} | {_count_cell(s)} | {s["with_avg"]} | {s["without_avg"]} ' f'| **{s["delta_avg"]:+}** | {s["improvements"]} | {s["degradations"]} |' ) lines.append("") @@ -454,10 +470,10 @@ def _val(key): f'- **Dynamic vs Static agreement:** {dyn_stats["delta_avg"]:+} vs {static_stats["delta_avg"]:+} — ' + ("good consistency, benchmark is stable." if abs(dyn_stats["delta_avg"] - static_stats["delta_avg"]) < 3 else "notable gap — investigate question quality."), - f'- **Win rate:** {round(all_stats["improvements"]/all_stats["count"]*100) if all_stats["count"] else 0}% questions improved with context ' - f'({all_stats["improvements"]}/{all_stats["count"]}).', + f'- **Win rate:** {round(all_stats["improvements"]/all_stats["valid"]*100) if all_stats["valid"] else 0}% questions improved with context ' + f'({all_stats["improvements"]}/{all_stats["valid"]}).', f'- **Degradations:** {all_stats["degradations"]} questions ' - f'({round(all_stats["degradations"]/all_stats["count"]*100) if all_stats["count"] else 0}%).', + f'({round(all_stats["degradations"]/all_stats["valid"]*100) if all_stats["valid"] else 0}%).', "", ] diff --git a/dokimi/dokimi/report/generator.py b/dokimi/dokimi/report/generator.py index 1b022e4f7..bdd364ffc 100644 --- a/dokimi/dokimi/report/generator.py +++ b/dokimi/dokimi/report/generator.py @@ -14,6 +14,7 @@ DIAGNOSIS_LABELS, DIAGNOSIS_DESCRIPTIONS, ) +from dokimi.eval.stats import paired_summary from dokimi.records import BASELINE, CONTEXT, Record, records #: The persisted condition key each report column reads. @@ -151,33 +152,24 @@ def generate_report( ) def _compute_stats(self, evaluations: Sequence[Record]) -> Dict[str, Any]: - """Compute overall statistics.""" - with_scores, without_scores, deltas = _columns(evaluations) - + """Compute overall statistics over the complete pairs (see ``paired_summary``).""" + s = paired_summary(evaluations) return { - "total_questions": len(evaluations), - "with_docs": { - "avg": round(sum(with_scores) / len(with_scores), 1) if with_scores else 0, - "min": round(min(with_scores), 1) if with_scores else 0, - "max": round(max(with_scores), 1) if with_scores else 0, - }, + "total_questions": s.total, + "paired_questions": s.paired, + "unpaired_questions": s.unpaired, + "with_docs": {"avg": round(s.with_mean, 1), "min": round(s.with_min, 1), "max": round(s.with_max, 1)}, "without_docs": { - "avg": round(sum(without_scores) / len(without_scores), 1) if without_scores else 0, - "min": round(min(without_scores), 1) if without_scores else 0, - "max": round(max(without_scores), 1) if without_scores else 0, + "avg": round(s.without_mean, 1), "min": round(s.without_min, 1), "max": round(s.without_max, 1), }, - "delta": { - "avg": round(sum(deltas) / len(deltas), 1) if deltas else 0, - "min": round(min(deltas), 1) if deltas else 0, - "max": round(max(deltas), 1) if deltas else 0, - }, - "doc_improvements": sum(1 for d in deltas if d > 0), - "doc_degradations": sum(1 for d in deltas if d < 0), + "delta": {"avg": round(s.delta_mean, 1), "min": round(s.delta_min, 1), "max": round(s.delta_max, 1)}, + "doc_improvements": s.improvements, + "doc_degradations": s.degradations, } def _top_performers(self, evaluations: Sequence[Record], mode: str, limit: int) -> List[Dict]: - """Get top N by aggregate score.""" - scored = [(s, e) for e in evaluations if (s := e.score(_MODES[mode])) is not None] + """Get top N by aggregate score, over the complete pairs (the population the Overall Max covers).""" + scored = [(s, e) for e in _complete_pairs(evaluations) if (s := e.score(_MODES[mode])) is not None] scored.sort(reverse=True, key=lambda x: x[0]) return [ { @@ -189,8 +181,8 @@ def _top_performers(self, evaluations: Sequence[Record], mode: str, limit: int) ] def _bottom_performers(self, evaluations: Sequence[Record], mode: str, limit: int) -> List[Dict]: - """Get bottom N by aggregate score.""" - scored = [(s, e) for e in evaluations if (s := e.score(_MODES[mode])) is not None] + """Get bottom N by aggregate score, over the complete pairs (the population the Overall Max covers).""" + scored = [(s, e) for e in _complete_pairs(evaluations) if (s := e.score(_MODES[mode])) is not None] scored.sort(key=lambda x: x[0]) return [ { @@ -258,15 +250,7 @@ def _cluster_by_topic( # Compute stats per cluster clusters = [] for topic, evals in topic_map.items(): - with_scores, without_scores, deltas = _columns(evals) - - clusters.append({ - "topic": topic, - "count": len(evals), - "with_avg": round(sum(with_scores) / len(with_scores), 1) if with_scores else 0, - "without_avg": round(sum(without_scores) / len(without_scores), 1) if without_scores else 0, - "delta_avg": round(sum(deltas) / len(deltas), 1) if deltas else 0, - }) + clusters.append({"topic": topic, **_arm_means(evals)}) # Sort by delta (worst first) clusters.sort(key=lambda x: x["delta_avg"]) @@ -284,15 +268,7 @@ def _compute_source_summary(self, evaluations: Sequence[Record], q_lookup: Dict[ results = [] for stype, evals in source_map.items(): - with_scores, without_scores, deltas = _columns(evals) - - results.append({ - "source_type": stype, - "count": len(evals), - "with_avg": round(sum(with_scores) / len(with_scores), 1) if with_scores else 0, - "without_avg": round(sum(without_scores) / len(without_scores), 1) if without_scores else 0, - "delta_avg": round(sum(deltas) / len(deltas), 1) if deltas else 0, - }) + results.append({"source_type": stype, **_arm_means(evals)}) results.sort(key=lambda x: x["source_type"]) return results @@ -337,6 +313,9 @@ def _format_markdown( f"| Min | {stats['with_docs']['min']} | {stats['without_docs']['min']} | {stats['delta']['min']:+.1f} |", f"| Max | {stats['with_docs']['max']} | {stats['without_docs']['max']} | {stats['delta']['max']:+.1f} |", "", + ] + lines += _unpaired_note(stats) + lines += [ f"- **Improvements:** {stats['doc_improvements']} questions (context helped)", f"- **Degradations:** {stats['doc_degradations']} questions (context hurt)", "", @@ -429,7 +408,7 @@ def _format_markdown( for cluster in clusters: lines.append( - f"| {cluster['topic']} | {cluster['count']} | " + f"| {cluster['topic']} | {_count_cell(cluster)} | " f"{cluster['with_avg']:.1f} | {cluster['without_avg']:.1f} | " f"**{cluster['delta_avg']:+.1f}** |" ) @@ -447,7 +426,7 @@ def _format_markdown( ]) for src in sources_summary: lines.append( - f"| {src['source_type']} | {src['count']} | " + f"| {src['source_type']} | {_count_cell(src)} | " f"{src['with_avg']:.1f} | {src['without_avg']:.1f} | " f"**{src['delta_avg']:+.1f}** |" ) @@ -533,16 +512,34 @@ def _format_markdown( return "\n".join(lines) -def _columns(evaluations: Sequence[Record]) -> "tuple[List[float], List[float], List[float]]": - """Each arm's scores and the persisted deltas, each over the records that have one. +def _arm_means(evaluations: Sequence[Record]) -> Dict[str, Any]: + """The arms, delta and unpaired count of one group, rounded for display.""" + s = paired_summary(evaluations) + return { + "count": s.total, + "unpaired": s.unpaired, + "with_avg": round(s.with_mean, 1), + "without_avg": round(s.without_mean, 1), + "delta_avg": round(s.delta_mean, 1), + } - The arms are averaged over different question sets when a question is scored - in one arm only. The deltas are the persisted ``delta`` values, whatever the - arms' scores: a record with a delta and one unscored arm counts in the delta - mean but not in that arm's, so the delta mean need not equal the difference - of the arm means (and the top/bottom tables, which need both arms, leave it out). - """ - with_scores = [s for e in evaluations if (s := e.score(CONTEXT)) is not None] - without_scores = [s for e in evaluations if (s := e.score(BASELINE)) is not None] - deltas = [d for e in evaluations if (d := e.delta) is not None] - return with_scores, without_scores, deltas + +def _count_cell(entry: Dict[str, Any]) -> str: + """The question count, with the unpaired questions the arm means leave out, if any.""" + return str(entry["count"]) if not entry["unpaired"] else f"{entry['count']} ({entry['unpaired']} unpaired)" + + +def _complete_pairs(evaluations: Sequence[Record]) -> List[Record]: + """The records scored in both arms: the questions the Overall statistics are computed over.""" + return [e for e in evaluations if e.score(CONTEXT) is not None and e.score(BASELINE) is not None] + + +def _unpaired_note(stats: Dict[str, Any]) -> List[str]: + """A line saying what the overall means leave out; nothing when every question is paired.""" + if not stats["unpaired_questions"]: + return [] + return [ + f"_Averages, min/max, delta and the top/bottom tables are over the {stats['paired_questions']} complete pairs; " + f"{stats['unpaired_questions']} unpaired (scored in one arm only) left out._", + "", + ] diff --git a/dokimi/tests/test_pipeline_summary.py b/dokimi/tests/test_pipeline_summary.py index f07b2beb6..35e802037 100644 --- a/dokimi/tests/test_pipeline_summary.py +++ b/dokimi/tests/test_pipeline_summary.py @@ -39,18 +39,33 @@ def _run(tmp_path, monkeypatch: pytest.MonkeyPatch, evaluations): return pipeline.run(concurrency=1) -def test_summary_averages_the_judged_scores_and_skips_a_failed_question(tmp_path, monkeypatch): - """Each average is over the questions that have that score; a question whose judge - failed (null conditions, no delta) adds to none of them.""" +def test_summary_averages_the_complete_pairs_and_skips_a_failed_question(tmp_path, monkeypatch): + """Each average is over the complete pairs. A question whose judge failed (null + conditions, no delta) adds to none of them, and neither does a question scored in + one arm only: the context mean must not cover a question its baseline mean lacks.""" result = _run(tmp_path, monkeypatch, [ _evaluation("q1", 80.5, 50), _evaluation("q2", 59.5, 40), _evaluation("q3", None, None), + _evaluation("q4", 100, None), + _evaluation("q5", None, 30), ]) assert result["steps"]["evaluation"]["summary"] == {"with_avg": 70.0, "without_avg": 45.0, "delta_avg": 25.0} +def test_summary_rounds_the_delta_from_the_unrounded_means(tmp_path, monkeypatch): + """One pair, 60.04 against 50.06: the delta of the unrounded means is 9.98, which rounds to + 10.0, while the rounded means 60.0 and 50.1 would give 9.9. A baseline-only question must + not reach the baseline mean either (the per-arm formula would give 40.0).""" + result = _run(tmp_path, monkeypatch, [ + _evaluation("q1", 60.04, 50.06), + _evaluation("q2", None, 30), + ]) + + assert result["steps"]["evaluation"]["summary"] == {"with_avg": 60.0, "without_avg": 50.1, "delta_avg": 10.0} + + def test_summary_is_left_out_when_no_question_was_scored(tmp_path, monkeypatch): result = _run(tmp_path, monkeypatch, [_evaluation("q1", None, None)]) diff --git a/dokimi/tests/test_report_arm_agreement.py b/dokimi/tests/test_report_arm_agreement.py new file mode 100644 index 000000000..96381209f --- /dev/null +++ b/dokimi/tests/test_report_arm_agreement.py @@ -0,0 +1,240 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""`report eval` and `report generate` must agree on the arm means and the delta of one eval file. + +The fixture has two complete pairs and one question scored in the context arm only. +Both reports average the arms over the complete pairs, so both print the same numbers. +""" + +import json + +from dokimi.report.eval_report import generate_report +from dokimi.report.generator import ReportGenerator + +# q1 and q2 are complete pairs; q3 has a context score and no baseline score (unpaired). +EVALUATIONS = [ + {"question_id": "q1", "question_text": "How do I pin threads?", + "with_docs": {"aggregate": 80.0}, "without_docs": {"aggregate": 60.0}, "delta": 20.0}, + {"question_id": "q2", "question_text": "How do I size a pool?", + "with_docs": {"aggregate": 40.0}, "without_docs": {"aggregate": 40.0}, "delta": 0.0}, + {"question_id": "q3", "question_text": "How do I flush a queue?", + "with_docs": {"aggregate": 100.0}, "without_docs": None, "delta": None}, +] + + +def _eval_report_all_row(markdown: str) -> dict: + """The **All** row of the eval report's Summary table: context, baseline, delta.""" + row = next(line for line in markdown.splitlines() if line.startswith("| **All** |")) + cells = [c.strip() for c in row.strip("|").split("|")] + return {"with": float(cells[2]), "without": float(cells[3]), "delta": float(cells[4].strip("*+"))} + + +def test_report_eval_and_report_generate_print_the_same_arm_means_and_delta(tmp_path): + path = tmp_path / "eval" / "lib.json" + path.parent.mkdir() + path.write_text(json.dumps({"judge_model": "m", "evaluations": EVALUATIONS}), encoding="utf-8") + + md = tmp_path / "lib.md" + generate_report(str(path), str(md), str(tmp_path / "lib.qa.json")) + evaluated = _eval_report_all_row(md.read_text(encoding="utf-8")) + + data = json.loads(path.read_text(encoding="utf-8")) + generated = json.loads(ReportGenerator().generate_report(data, {"questions": []}, "json"))["stats"] + generated_row = { + "with": generated["with_docs"]["avg"], + "without": generated["without_docs"]["avg"], + "delta": generated["delta"]["avg"], + } + + # Over the complete pairs only: context (80, 40) -> 60, baseline (60, 40) -> 50, delta 10. + assert evaluated == {"with": 60.0, "without": 50.0, "delta": 10.0} + assert generated_row == evaluated + # The unpaired question is counted, not dropped silently. + assert generated["paired_questions"] == 2 + assert generated["unpaired_questions"] == 1 + + +def test_both_reports_say_which_questions_the_means_leave_out(tmp_path): + path = tmp_path / "eval" / "lib.json" + path.parent.mkdir() + path.write_text(json.dumps({"evaluations": EVALUATIONS}), encoding="utf-8") + md = tmp_path / "lib.md" + generate_report(str(path), str(md), str(tmp_path / "lib.qa.json")) + eval_md = md.read_text(encoding="utf-8") + assert "| **All** | 3 (1 unpaired) |" in eval_md + assert "over the 2 complete pairs; 1 unpaired (scored in one arm only) left out" in eval_md + + data = {"evaluations": EVALUATIONS} + gen_md = ReportGenerator().generate_report(data, {"questions": []}, "markdown") + assert "over the 2 complete pairs; 1 unpaired (scored in one arm only) left out" in gen_md + + +def test_paired_summary_counts_every_record_and_averages_only_the_pairs(): + from dokimi.eval.stats import paired_summary + + s = paired_summary(EVALUATIONS) + assert (s.total, s.paired, s.unpaired) == (3, 2, 1) + assert (s.with_mean, s.without_mean, s.delta_mean) == (60.0, 50.0, 10.0) + assert (s.with_min, s.with_max) == (40.0, 80.0) # the 100 of q3 is not in it + assert (s.without_min, s.without_max) == (40.0, 60.0) + assert (s.delta_min, s.delta_max) == (0.0, 20.0) + assert (s.improvements, s.degradations, s.neutral) == (1, 0, 1) + + +def test_paired_summary_delta_is_not_the_difference_of_the_rounded_means(): + from dokimi.eval.stats import paired_summary + + # 60.04 - 50.06 = 9.98 rounds to 10.0; the rounded means 60.0 and 50.1 would give 9.9. + s = paired_summary([{"question_id": "q1", "question_text": "Q?", + "with_docs": {"aggregate": 60.04}, "without_docs": {"aggregate": 50.06}}]) + assert round(s.delta_mean, 1) == 10.0 + assert round(round(s.with_mean, 1) - round(s.without_mean, 1), 1) == 9.9 + + +def test_paired_summary_of_nothing_is_all_zero(): + from dokimi.eval.stats import paired_summary + + s = paired_summary([]) + assert (s.total, s.paired, s.unpaired, s.with_mean, s.delta_mean) == (0, 0, 0, 0.0, 0.0) + + +# Pairs whose arm means are not exact at one decimal: (80.67, 70.33). Rounding each mean first +# gives 80.7 - 70.3 = +10.4; the delta of the unrounded means is +10.3, which both reports print. +INEXACT = [ + {"question_id": f"q{i}", "question_text": f"Question {i}?", + "with_docs": {"aggregate": w}, "without_docs": {"aggregate": wo}, "delta": w - wo} + for i, (w, wo) in enumerate([(80.0, 70.0), (81.0, 70.0), (81.0, 71.0)], 1) +] + + +def test_both_reports_take_the_delta_from_unrounded_means(tmp_path): + path = tmp_path / "eval" / "lib.json" + path.parent.mkdir() + path.write_text(json.dumps({"evaluations": INEXACT}), encoding="utf-8") + md = tmp_path / "lib.md" + generate_report(str(path), str(md), str(tmp_path / "lib.qa.json")) + eval_md = md.read_text(encoding="utf-8") + assert _eval_report_all_row(eval_md) == {"with": 80.7, "without": 70.3, "delta": 10.3} + assert "| **Total** | 3 | 80.7 | 70.3 | **+10.3** | 3 | 0 | 0 |" in eval_md + + generated = json.loads(ReportGenerator().generate_report({"evaluations": INEXACT}, {"questions": []}, "json"))["stats"] + assert generated["delta"] == {"avg": 10.3, "min": 10.0, "max": 11.0} + assert (generated["doc_improvements"], generated["doc_degradations"]) == (3, 0) + # One group holds every question here, so its delta is the same unrounded-mean delta. + assert json.loads(ReportGenerator().generate_report({"evaluations": INEXACT}, {"questions": []}, "json"))[ + "clusters"][0]["delta_avg"] == 10.3 + + +# The grouped tables: q1 and q3 share a persona and a source type, so their group holds the +# unpaired q3; q2 is a group of its own. +QUESTIONS = [ + {"id": "q1", "persona_id": "pins", "source_type": "generated"}, + {"id": "q2", "persona_id": "pools", "source_type": "manual"}, + {"id": "q3", "persona_id": "pins", "source_type": "generated"}, +] + + +def test_grouped_tables_average_each_group_over_its_pairs_and_count_the_unpaired(): + report = json.loads(ReportGenerator().generate_report( + {"evaluations": EVALUATIONS}, {"questions": QUESTIONS}, "json")) + clusters = {c["topic"]: c for c in report["clusters"]} + assert clusters["pins"] == {"topic": "pins", "count": 2, "unpaired": 1, + "with_avg": 80.0, "without_avg": 60.0, "delta_avg": 20.0} + assert clusters["pools"] == {"topic": "pools", "count": 1, "unpaired": 0, + "with_avg": 40.0, "without_avg": 40.0, "delta_avg": 0.0} + sources = {s["source_type"]: s for s in report["sources_summary"]} + assert sources["generated"] == {"source_type": "generated", "count": 2, "unpaired": 1, + "with_avg": 80.0, "without_avg": 60.0, "delta_avg": 20.0} + assert sources["manual"]["with_avg"] == 40.0 and sources["manual"]["delta_avg"] == 0.0 + + +def test_grouped_tables_print_the_count_and_unpaired_count_beside_their_means(): + gen_md = ReportGenerator().generate_report({"evaluations": EVALUATIONS}, {"questions": QUESTIONS}, "markdown") + assert "| pins | 2 (1 unpaired) | 80.0 | 60.0 | **+20.0** |" in gen_md + assert "| pools | 1 | 40.0 | 40.0 | **+0.0** |" in gen_md + assert "| generated | 2 (1 unpaired) | 80.0 | 60.0 | **+20.0** |" in gen_md + assert "| manual | 1 | 40.0 | 40.0 | **+0.0** |" in gen_md + + +# The eval report's groups: two golden questions (onetbb-Q prefix, one unpaired) and three +# dynamic ones (one unpaired), in two difficulties. Pairs: (80, 60), (40, 40), (60, 50). +GROUPED_EVALUATIONS = [ + {"question_id": "onetbb-Q001", "question_text": "How do I pin threads?", "difficulty": "easy", + "with_docs": {"aggregate": 80.0}, "without_docs": {"aggregate": 60.0}, "delta": 20.0}, + {"question_id": "onetbb-Q002", "question_text": "How do I size a pool?", "difficulty": "easy", + "with_docs": {"aggregate": 90.0}, "without_docs": None, "delta": None}, + {"question_id": "gen-1", "question_text": "How do I flush a queue?", "difficulty": "hard", + "with_docs": {"aggregate": 40.0}, "without_docs": {"aggregate": 40.0}, "delta": 0.0}, + {"question_id": "gen-2", "question_text": "How do I cancel a task?", "difficulty": "hard", + "with_docs": {"aggregate": 60.0}, "without_docs": {"aggregate": 50.0}, "delta": 10.0}, + {"question_id": "gen-3", "question_text": "How do I wait on a flag?", "difficulty": "hard", + "with_docs": {"aggregate": 100.0}, "without_docs": None, "delta": None}, +] + + +def test_eval_report_group_rows_count_their_unpaired_questions(tmp_path): + path = tmp_path / "eval" / "lib.json" + path.parent.mkdir() + path.write_text(json.dumps({"evaluations": GROUPED_EVALUATIONS}), encoding="utf-8") + md = tmp_path / "lib.md" + generate_report(str(path), str(md), str(tmp_path / "lib.qa.json")) + eval_md = md.read_text(encoding="utf-8") + + assert _eval_report_all_row(eval_md) == {"with": 60.0, "without": 50.0, "delta": 10.0} + assert "| **All** | 5 (2 unpaired) |" in eval_md + # Summary table: the dynamic and golden rows. Whole lines, because the breakdown rows below + # carry the same cells with more columns after them. + lines = eval_md.splitlines() + assert "| Generated (dynamic) | 3 (1 unpaired) | 50.0 | 45.0 | **+5.0** |" in lines + assert "| Golden (static) | 2 (1 unpaired) | 80.0 | 60.0 | **+20.0** |" in lines + # Breakdown by type. + assert "| Dynamic (generated) | 3 (1 unpaired) | 50.0 | 45.0 | **+5.0** |" in eval_md + assert "| Golden (static) | 2 (1 unpaired) | 80.0 | 60.0 | **+20.0** |" in eval_md + # Breakdown by difficulty. + assert "| easy | 2 (1 unpaired) | 80.0 | 60.0 | **+20.0** |" in eval_md + assert "| hard | 3 (1 unpaired) | 50.0 | 45.0 | **+5.0** |" in eval_md + + +def test_top_and_bottom_performers_rank_the_complete_pairs_like_the_overall_max(): + report = json.loads(ReportGenerator().generate_report({"evaluations": EVALUATIONS}, {"questions": []}, "json")) + # q3 scored 100 in the context arm alone; it is not in any table that the Overall Max bounds. + assert [(r["question_id"], r["score"]) for r in report["top_with_docs"]] == [("q1", 80.0), ("q2", 40.0)] + assert [(r["question_id"], r["score"]) for r in report["bottom_with_docs"]] == [("q2", 40.0), ("q1", 80.0)] + assert max(r["score"] for r in report["top_with_docs"]) == report["stats"]["with_docs"]["max"] + + gen_md = ReportGenerator().generate_report({"evaluations": EVALUATIONS}, {"questions": []}, "markdown") + assert "| q3 |" not in gen_md + assert "over the 2 complete pairs; 1 unpaired (scored in one arm only) left out" in gen_md + + +# Four complete pairs: two improved ((80, 60), (70, 50)), one degraded ((40, 50)), one neutral +# ((60, 60)). Two more questions are unpaired, one per arm. The win rate and the degradation rate +# are over the four pairs, the same set the Overall delta uses: 50% and 25%. Over all six +# questions they would be 33% and 17%. +WIN_RATE_EVALUATIONS = [ + {"question_id": "q1", "question_text": "How do I pin threads?", + "with_docs": {"aggregate": 80.0}, "without_docs": {"aggregate": 60.0}, "delta": 20.0}, + {"question_id": "q2", "question_text": "How do I size a pool?", + "with_docs": {"aggregate": 40.0}, "without_docs": {"aggregate": 50.0}, "delta": -10.0}, + {"question_id": "q3", "question_text": "How do I flush a queue?", + "with_docs": {"aggregate": 60.0}, "without_docs": {"aggregate": 60.0}, "delta": 0.0}, + {"question_id": "q4", "question_text": "How do I cancel a task?", + "with_docs": {"aggregate": 70.0}, "without_docs": {"aggregate": 50.0}, "delta": 20.0}, + {"question_id": "q5", "question_text": "How do I wait on a flag?", + "with_docs": {"aggregate": 90.0}, "without_docs": None, "delta": None}, + {"question_id": "q6", "question_text": "How do I join a thread?", + "with_docs": None, "without_docs": {"aggregate": 30.0}, "delta": None}, +] + + +def test_eval_report_win_and_degradation_rates_divide_by_the_complete_pairs(tmp_path): + path = tmp_path / "eval" / "lib.json" + path.parent.mkdir() + path.write_text(json.dumps({"evaluations": WIN_RATE_EVALUATIONS}), encoding="utf-8") + md = tmp_path / "lib.md" + generate_report(str(path), str(md), str(tmp_path / "lib.qa.json")) + eval_md = md.read_text(encoding="utf-8") + + assert "- **Win rate:** 50% questions improved with context (2/4)." in eval_md + assert "- **Degradations:** 1 questions (25%)." in eval_md