diff --git a/dokimi/changelog.d/20261026_130000_napetrov_cli-stale-hints.md b/dokimi/changelog.d/20261026_130000_napetrov_cli-stale-hints.md new file mode 100644 index 000000000..a5f6acfbb --- /dev/null +++ b/dokimi/changelog.d/20261026_130000_napetrov_cli-stale-hints.md @@ -0,0 +1,3 @@ +### Fixed + +- **The next-step hints name the `dokimi` command, and the `arms run` example in a baseline-only report runs as written** — `dokimi personas discover`, `dokimi personas approve`, `dokimi baseline list` and `dokimi baseline compare` used to print `python cli.py …` as the command to run; they now print `dokimi …`, the console script, with the same arguments. The example `arms run` command in a model-comparison report that has only baseline arms now carries the `--product` and `--questions` that `dokimi arms run` requires, in place of a `[other parameters]` placeholder. No command, flag or output changes otherwise. diff --git a/dokimi/dokimi/commands/baseline.py b/dokimi/dokimi/commands/baseline.py index 67630f6fb..20769c53d 100644 --- a/dokimi/dokimi/commands/baseline.py +++ b/dokimi/dokimi/commands/baseline.py @@ -185,7 +185,7 @@ def cmd_baseline_list(args: argparse.Namespace) -> None: if not baselines: print("No baselines saved yet.") - print("Run: python cli.py baseline save --from-eval eval/oneTBB.json --product oneTBB") + print("Run: dokimi baseline save --from-eval eval/oneTBB.json --product oneTBB") return print(f"{'Name':<35} {'Product':<12} {'Saved':<17} {'n':>4} {'with':>6} {'without':>8} {'delta':>7}") @@ -259,7 +259,7 @@ def cmd_baseline_compare(args: argparse.Namespace) -> None: if not baselines: print("No baselines found. Save one first:") - print(" python cli.py baseline save --from-eval eval/oneTBB.json --product oneTBB") + print(" dokimi baseline save --from-eval eval/oneTBB.json --product oneTBB") raise SystemExit(1) entry = _pick_entry(args, baselines, eval_data) diff --git a/dokimi/dokimi/commands/personas.py b/dokimi/dokimi/commands/personas.py index 0d25de884..73248afdd 100644 --- a/dokimi/dokimi/commands/personas.py +++ b/dokimi/dokimi/commands/personas.py @@ -70,7 +70,7 @@ def cmd_personas_discover(args: argparse.Namespace) -> None: print("\nNext steps:") print(f" 1. Review: cat {output_path}") print(" 2. Edit if needed") - print(f" 3. Approve: python cli.py personas approve --file {output_path}") + print(f" 3. Approve: dokimi personas approve --file {output_path}") def cmd_personas_approve(args: argparse.Namespace) -> None: @@ -117,7 +117,7 @@ def cmd_personas_approve(args: argparse.Namespace) -> None: print(f"✓ Validated {len(personas['personas'])} personas") print(f"✓ File approved: {persona_file}") print("\nReady for question generation:") - print(f" python cli.py questions generate --product {personas['product']} --personas {persona_file}") + print(f" dokimi questions generate --product {personas['product']} --personas {persona_file}") def register(sub, positive_int) -> None: diff --git a/dokimi/dokimi/commands/report.py b/dokimi/dokimi/commands/report.py index 58d47934a..8bf44ae8e 100644 --- a/dokimi/dokimi/commands/report.py +++ b/dokimi/dokimi/commands/report.py @@ -10,8 +10,6 @@ import json from pathlib import Path -from dokimi.report.markdown_report import write_compare_report, write_run_report - def _write_combined_report(args, regular_runs, golden_runs, run_ids, warnings) -> None: """Write the combined model-compare report, exiting cleanly if the output is unwritable.""" @@ -89,17 +87,6 @@ def cmd_report_model_compare(args: argparse.Namespace) -> None: _write_combined_report(args, regular_runs, golden_runs, run_ids, all_warnings) -def cmd_report(args: argparse.Namespace) -> None: - """Generate markdown report from JSON snapshot.""" - data = json.loads(Path(args.input).read_text(encoding="utf-8")) - out_md = Path(args.out_md).resolve() - - if "diff" in data: - write_compare_report(data, out_md) - else: - write_run_report(data, out_md) - - def cmd_report_eval(args: argparse.Namespace) -> None: """Generate quality report from eval JSON (dynamic vs static breakdown).""" from dokimi.records import RecordError diff --git a/dokimi/dokimi/report/model_compare.py b/dokimi/dokimi/report/model_compare.py index efa6e5267..f7078c496 100644 --- a/dokimi/dokimi/report/model_compare.py +++ b/dokimi/dokimi/report/model_compare.py @@ -882,11 +882,11 @@ def _models_compared_lines( "> ", "> **To generate full comparison with delta analysis, add a treatment arm:**", "> ```bash", - "> python cli.py arms run \\", + "> dokimi arms run \\", + "> --product oneTBB --questions questions/oneTBB.json \\", "> --arms \"baseline,docs\" \\", "> --judge \\", - "> --judge-model claude-sonnet-5-5 \\", - "> [other parameters]", + "> --judge-model claude-sonnet-5-5", "> ```", "> ", "> Treatment arm options: `docs` (Context7 documentation), `skill:` (local skill), `agent` (agentic doc retrieval).", diff --git a/dokimi/tests/fixtures/model_compare_sections.json b/dokimi/tests/fixtures/model_compare_sections.json index 115fe5f25..7bff66678 100644 --- a/dokimi/tests/fixtures/model_compare_sections.json +++ b/dokimi/tests/fixtures/model_compare_sections.json @@ -1,8 +1,8 @@ { - "baseline_only": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | alpha (p) | same (p) | — (baseline only) |\n| **b** | beta (p) | same (p) | — (baseline only) |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n> **⚠️ Report Limitation:** These runs contain only **baseline arm** (no treatment comparison).\n> \n> **Missing from this report:**\n> - Context-arm vs Baseline delta analysis\n> - Statistical significance tests\n> - Head-to-head winner comparisons\n> \n> **Available in this report:**\n> - Absolute baseline quality scores per model\n> - Resource usage metrics (cost, tokens, latency)\n> - Model ranking by absolute baseline quality\n> \n> **To generate full comparison with delta analysis, add a treatment arm:**\n> ```bash\n> python cli.py arms run \\\n> --arms \"baseline,docs\" \\\n> --judge \\\n> --judge-model claude-sonnet-5-5 \\\n> [other parameters]\n> ```\n> \n> Treatment arm options: `docs` (Context7 documentation), `skill:` (local skill), `agent` (agentic doc retrieval).\n> \n> See: `docs/benchmarking-and-comparison.md#treatment-arm-comparison`\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | Context arm | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 0 (9 unscored) | — | 56.2 | — | — | — |\n| **b** | 9 | 0 (9 unscored) | — | 56.2 | — | — | — |\n\n## Statistical Significance of Delta (α = 0.05)\n\n> **⚠️ Skipped:** Statistical tests require treatment arm (e.g., docs, skills).\n> Add `--arms \"baseline,docs\"` to enable delta analysis.\n\n\n## By Difficulty (common questions only)\n| Difficulty | N | a baseline | b baseline |\n|---|---:|| :---: | :---: |\n| easy | 3 | 60.0 | — |\n| beginner | 1 | 55.0 | — |\n| intermediate | 1 | 30.0 | — |\n| medium | 1 | 60.0 | — |\n| advanced | 1 | 65.0 | — |\n| hard | 1 | 40.0 | 56.2 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n\n> **⚠️ Skipped:** Head-to-head requires treatment arm scores.\n> Single-arm runs only have baseline scores.\n\n## Context-Arm Benefit — Delta Comparison\n\n> **⚠️ Skipped:** Delta comparison requires treatment arm.\n> Single-arm runs cannot compute context benefit.\n\n## Model Ranking\n\n### By absolute quality (baseline)\n| Rank | Run | Baseline avg |\n|---:|---|---:|\n| 1 | **a** | 56.2 |\n| 2 | **b** | 56.2 |\n\n> **Note:** Treatment utilization ranking requires treatment arm.\n\n", - "baseline_only_no_scipy": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | m (p) | same (p) | — (baseline only) |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n> **⚠️ Report Limitation:** These runs contain only **baseline arm** (no treatment comparison).\n> \n> **Missing from this report:**\n> - Context-arm vs Baseline delta analysis\n> - Statistical significance tests\n> - Head-to-head winner comparisons\n> \n> **Available in this report:**\n> - Absolute baseline quality scores per model\n> - Resource usage metrics (cost, tokens, latency)\n> - Model ranking by absolute baseline quality\n> \n> **To generate full comparison with delta analysis, add a treatment arm:**\n> ```bash\n> python cli.py arms run \\\n> --arms \"baseline,docs\" \\\n> --judge \\\n> --judge-model claude-sonnet-5-5 \\\n> [other parameters]\n> ```\n> \n> Treatment arm options: `docs` (Context7 documentation), `skill:` (local skill), `agent` (agentic doc retrieval).\n> \n> See: `docs/benchmarking-and-comparison.md#treatment-arm-comparison`\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | Context arm | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 0 (9 unscored) | — | 56.2 | — | — | — |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a baseline |\n|---|---:|| :---: |\n| easy | 3 | 60.0 |\n| beginner | 1 | 55.0 |\n| intermediate | 1 | 30.0 |\n| medium | 1 | 60.0 |\n| advanced | 1 | 65.0 |\n| hard | 1 | 40.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n\n> **⚠️ Skipped:** Head-to-head requires treatment arm scores.\n> Single-arm runs only have baseline scores.\n\n## Context-Arm Benefit — Delta Comparison\n\n> **⚠️ Skipped:** Delta comparison requires treatment arm.\n> Single-arm runs cannot compute context benefit.\n\n## Model Ranking\n\n### By absolute quality (baseline)\n| Rank | Run | Baseline avg |\n|---:|---|---:|\n| 1 | **a** | 56.2 |\n\n> **Note:** Treatment utilization ranking requires treatment arm.\n\n", - "baseline_zero": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | m (p) | same (p) | — (baseline only) |\n\nCommon questions evaluated: **3**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n> **⚠️ Report Limitation:** These runs contain only **baseline arm** (no treatment comparison).\n> \n> **Missing from this report:**\n> - Context-arm vs Baseline delta analysis\n> - Statistical significance tests\n> - Head-to-head winner comparisons\n> \n> **Available in this report:**\n> - Absolute baseline quality scores per model\n> - Resource usage metrics (cost, tokens, latency)\n> - Model ranking by absolute baseline quality\n> \n> **To generate full comparison with delta analysis, add a treatment arm:**\n> ```bash\n> python cli.py arms run \\\n> --arms \"baseline,docs\" \\\n> --judge \\\n> --judge-model claude-sonnet-5-5 \\\n> [other parameters]\n> ```\n> \n> Treatment arm options: `docs` (Context7 documentation), `skill:` (local skill), `agent` (agentic doc retrieval).\n> \n> See: `docs/benchmarking-and-comparison.md#treatment-arm-comparison`\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | Context arm | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 3 | 0 (3 unscored) | — | 30.0 | — | — | — |\n\n## Statistical Significance of Delta (α = 0.05)\n\n> **⚠️ Skipped:** Statistical tests require treatment arm (e.g., docs, skills).\n> Add `--arms \"baseline,docs\"` to enable delta analysis.\n\n\n## By Difficulty (common questions only)\n| Difficulty | N | a baseline |\n|---|---:|| :---: |\n| easy | 1 | 0.0 |\n| medium | 1 | 40.0 |\n| hard | 1 | 50.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n\n> **⚠️ Skipped:** Head-to-head requires treatment arm scores.\n> Single-arm runs only have baseline scores.\n\n## Context-Arm Benefit — Delta Comparison\n\n> **⚠️ Skipped:** Delta comparison requires treatment arm.\n> Single-arm runs cannot compute context benefit.\n\n## Model Ranking\n\n### By absolute quality (baseline)\n| Rank | Run | Baseline avg |\n|---:|---|---:|\n| 1 | **a** | 30.0 |\n\n> **Note:** Treatment utilization ranking requires treatment arm.\n\n", - "empty": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n\nCommon questions evaluated: **0**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n> **⚠️ Report Limitation:** These runs contain only **baseline arm** (no treatment comparison).\n> \n> **Missing from this report:**\n> - Context-arm vs Baseline delta analysis\n> - Statistical significance tests\n> - Head-to-head winner comparisons\n> \n> **Available in this report:**\n> - Absolute baseline quality scores per model\n> - Resource usage metrics (cost, tokens, latency)\n> - Model ranking by absolute baseline quality\n> \n> **To generate full comparison with delta analysis, add a treatment arm:**\n> ```bash\n> python cli.py arms run \\\n> --arms \"baseline,docs\" \\\n> --judge \\\n> --judge-model claude-sonnet-5-5 \\\n> [other parameters]\n> ```\n> \n> Treatment arm options: `docs` (Context7 documentation), `skill:` (local skill), `agent` (agentic doc retrieval).\n> \n> See: `docs/benchmarking-and-comparison.md#treatment-arm-comparison`\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | Context arm | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n\n## Statistical Significance of Delta (α = 0.05)\n\n> **⚠️ Skipped:** Statistical tests require treatment arm (e.g., docs, skills).\n> Add `--arms \"baseline,docs\"` to enable delta analysis.\n\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n\n> **⚠️ Skipped:** Head-to-head requires treatment arm scores.\n> Single-arm runs only have baseline scores.\n\n## Context-Arm Benefit — Delta Comparison\n\n> **⚠️ Skipped:** Delta comparison requires treatment arm.\n> Single-arm runs cannot compute context benefit.\n\n## Model Ranking\n\n### By absolute quality (baseline)\n| Rank | Run | Baseline avg |\n|---:|---|---:|\n\n> **Note:** Treatment utilization ranking requires treatment arm.\n\n", + "baseline_only": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | alpha (p) | same (p) | — (baseline only) |\n| **b** | beta (p) | same (p) | — (baseline only) |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n> **⚠️ Report Limitation:** These runs contain only **baseline arm** (no treatment comparison).\n> \n> **Missing from this report:**\n> - Context-arm vs Baseline delta analysis\n> - Statistical significance tests\n> - Head-to-head winner comparisons\n> \n> **Available in this report:**\n> - Absolute baseline quality scores per model\n> - Resource usage metrics (cost, tokens, latency)\n> - Model ranking by absolute baseline quality\n> \n> **To generate full comparison with delta analysis, add a treatment arm:**\n> ```bash\n> dokimi arms run \\\n> --product oneTBB --questions questions/oneTBB.json \\\n> --arms \"baseline,docs\" \\\n> --judge \\\n> --judge-model claude-sonnet-5-5\n> ```\n> \n> Treatment arm options: `docs` (Context7 documentation), `skill:` (local skill), `agent` (agentic doc retrieval).\n> \n> See: `docs/benchmarking-and-comparison.md#treatment-arm-comparison`\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | Context arm | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 0 (9 unscored) | — | 56.2 | — | — | — |\n| **b** | 9 | 0 (9 unscored) | — | 56.2 | — | — | — |\n\n## Statistical Significance of Delta (α = 0.05)\n\n> **⚠️ Skipped:** Statistical tests require treatment arm (e.g., docs, skills).\n> Add `--arms \"baseline,docs\"` to enable delta analysis.\n\n\n## By Difficulty (common questions only)\n| Difficulty | N | a baseline | b baseline |\n|---|---:|| :---: | :---: |\n| easy | 3 | 60.0 | — |\n| beginner | 1 | 55.0 | — |\n| intermediate | 1 | 30.0 | — |\n| medium | 1 | 60.0 | — |\n| advanced | 1 | 65.0 | — |\n| hard | 1 | 40.0 | 56.2 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n\n> **⚠️ Skipped:** Head-to-head requires treatment arm scores.\n> Single-arm runs only have baseline scores.\n\n## Context-Arm Benefit — Delta Comparison\n\n> **⚠️ Skipped:** Delta comparison requires treatment arm.\n> Single-arm runs cannot compute context benefit.\n\n## Model Ranking\n\n### By absolute quality (baseline)\n| Rank | Run | Baseline avg |\n|---:|---|---:|\n| 1 | **a** | 56.2 |\n| 2 | **b** | 56.2 |\n\n> **Note:** Treatment utilization ranking requires treatment arm.\n\n", + "baseline_only_no_scipy": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | m (p) | same (p) | — (baseline only) |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n> **⚠️ Report Limitation:** These runs contain only **baseline arm** (no treatment comparison).\n> \n> **Missing from this report:**\n> - Context-arm vs Baseline delta analysis\n> - Statistical significance tests\n> - Head-to-head winner comparisons\n> \n> **Available in this report:**\n> - Absolute baseline quality scores per model\n> - Resource usage metrics (cost, tokens, latency)\n> - Model ranking by absolute baseline quality\n> \n> **To generate full comparison with delta analysis, add a treatment arm:**\n> ```bash\n> dokimi arms run \\\n> --product oneTBB --questions questions/oneTBB.json \\\n> --arms \"baseline,docs\" \\\n> --judge \\\n> --judge-model claude-sonnet-5-5\n> ```\n> \n> Treatment arm options: `docs` (Context7 documentation), `skill:` (local skill), `agent` (agentic doc retrieval).\n> \n> See: `docs/benchmarking-and-comparison.md#treatment-arm-comparison`\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | Context arm | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 0 (9 unscored) | — | 56.2 | — | — | — |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a baseline |\n|---|---:|| :---: |\n| easy | 3 | 60.0 |\n| beginner | 1 | 55.0 |\n| intermediate | 1 | 30.0 |\n| medium | 1 | 60.0 |\n| advanced | 1 | 65.0 |\n| hard | 1 | 40.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n\n> **⚠️ Skipped:** Head-to-head requires treatment arm scores.\n> Single-arm runs only have baseline scores.\n\n## Context-Arm Benefit — Delta Comparison\n\n> **⚠️ Skipped:** Delta comparison requires treatment arm.\n> Single-arm runs cannot compute context benefit.\n\n## Model Ranking\n\n### By absolute quality (baseline)\n| Rank | Run | Baseline avg |\n|---:|---|---:|\n| 1 | **a** | 56.2 |\n\n> **Note:** Treatment utilization ranking requires treatment arm.\n\n", + "baseline_zero": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | m (p) | same (p) | — (baseline only) |\n\nCommon questions evaluated: **3**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n> **⚠️ Report Limitation:** These runs contain only **baseline arm** (no treatment comparison).\n> \n> **Missing from this report:**\n> - Context-arm vs Baseline delta analysis\n> - Statistical significance tests\n> - Head-to-head winner comparisons\n> \n> **Available in this report:**\n> - Absolute baseline quality scores per model\n> - Resource usage metrics (cost, tokens, latency)\n> - Model ranking by absolute baseline quality\n> \n> **To generate full comparison with delta analysis, add a treatment arm:**\n> ```bash\n> dokimi arms run \\\n> --product oneTBB --questions questions/oneTBB.json \\\n> --arms \"baseline,docs\" \\\n> --judge \\\n> --judge-model claude-sonnet-5-5\n> ```\n> \n> Treatment arm options: `docs` (Context7 documentation), `skill:` (local skill), `agent` (agentic doc retrieval).\n> \n> See: `docs/benchmarking-and-comparison.md#treatment-arm-comparison`\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | Context arm | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 3 | 0 (3 unscored) | — | 30.0 | — | — | — |\n\n## Statistical Significance of Delta (α = 0.05)\n\n> **⚠️ Skipped:** Statistical tests require treatment arm (e.g., docs, skills).\n> Add `--arms \"baseline,docs\"` to enable delta analysis.\n\n\n## By Difficulty (common questions only)\n| Difficulty | N | a baseline |\n|---|---:|| :---: |\n| easy | 1 | 0.0 |\n| medium | 1 | 40.0 |\n| hard | 1 | 50.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n\n> **⚠️ Skipped:** Head-to-head requires treatment arm scores.\n> Single-arm runs only have baseline scores.\n\n## Context-Arm Benefit — Delta Comparison\n\n> **⚠️ Skipped:** Delta comparison requires treatment arm.\n> Single-arm runs cannot compute context benefit.\n\n## Model Ranking\n\n### By absolute quality (baseline)\n| Rank | Run | Baseline avg |\n|---:|---|---:|\n| 1 | **a** | 30.0 |\n\n> **Note:** Treatment utilization ranking requires treatment arm.\n\n", + "empty": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n\nCommon questions evaluated: **0**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n> **⚠️ Report Limitation:** These runs contain only **baseline arm** (no treatment comparison).\n> \n> **Missing from this report:**\n> - Context-arm vs Baseline delta analysis\n> - Statistical significance tests\n> - Head-to-head winner comparisons\n> \n> **Available in this report:**\n> - Absolute baseline quality scores per model\n> - Resource usage metrics (cost, tokens, latency)\n> - Model ranking by absolute baseline quality\n> \n> **To generate full comparison with delta analysis, add a treatment arm:**\n> ```bash\n> dokimi arms run \\\n> --product oneTBB --questions questions/oneTBB.json \\\n> --arms \"baseline,docs\" \\\n> --judge \\\n> --judge-model claude-sonnet-5-5\n> ```\n> \n> Treatment arm options: `docs` (Context7 documentation), `skill:` (local skill), `agent` (agentic doc retrieval).\n> \n> See: `docs/benchmarking-and-comparison.md#treatment-arm-comparison`\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | Context arm | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n\n## Statistical Significance of Delta (α = 0.05)\n\n> **⚠️ Skipped:** Statistical tests require treatment arm (e.g., docs, skills).\n> Add `--arms \"baseline,docs\"` to enable delta analysis.\n\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n\n> **⚠️ Skipped:** Head-to-head requires treatment arm scores.\n> Single-arm runs only have baseline scores.\n\n## Context-Arm Benefit — Delta Comparison\n\n> **⚠️ Skipped:** Delta comparison requires treatment arm.\n> Single-arm runs cannot compute context benefit.\n\n## Model Ranking\n\n### By absolute quality (baseline)\n| Rank | Run | Baseline avg |\n|---:|---|---:|\n\n> **Note:** Treatment utilization ranking requires treatment arm.\n\n", "mixed": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | alpha (p) | same (p) | `with-context` |\n| **b** | beta (p) | same (p) | — (baseline only) |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 7 (2 unscored) | 68.8 | 56.2 | **+17.1** | 6 | 1 |\n| **b** | 9 | 0 (9 unscored) | — | 56.2 | — | — | — |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **a** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a context | a Δ | b context | b Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 63.3 | +5.0 | — | — |\n| beginner | 1 | 75.0 | +20.0 | — | — |\n| intermediate | 1 | 45.0 | +15.0 | — | — |\n| medium | 1 | 70.0 | +10.0 | — | — |\n| advanced | 1 | 90.0 | +25.0 | — | — |\n| hard | 1 | 80.0 | +40.0 | — | 56.2 (baseline) |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **a** | 8 | 89% |\n| **b** | 0 | 0% |\n| tie | 0 | 0% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **a** | **+17.1** | 6 (86%) | 1 (14%) |\n| **b** | — | — | — |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **a** | 68.8 |\n| 2 | **b** | — |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **a** | +17.1 |\n| 2 | **b** | N/A |\n", "no_scipy": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | alpha (p) | judge-j (jp) | `with-context` |\n| **b** | beta (p) | same (p) | `with-context` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Resource Usage (baseline arm)\n| Run | Total Cost | Prompt tok | Completion tok | Total tok | Avg latency/Q | Cache hit |\n|---|---:|---:|---:|---:|---:|---:|\n| **a** | $0.1235 (3/8) | 12,345 | 678 | 13,023 | 1.23s | 25.0% |\n| **b** | $1.5000 | — | — | — | — | — |\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | with-context | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 7 (2 unscored) | 68.8 | 56.2 | **+17.1** | 6 | 1 |\n| **b** | 9 | 9 | 62.2 | 52.2 | **+10.0** | 9 | 0 |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a context | a Δ | b context | b Δ |\n|---|---:| :---: | :---: | :---: | :---: |\n| easy | 3 | 63.3 | +5.0 | 56.7 | +10.0 |\n| beginner | 1 | 75.0 | +20.0 | 65.0 | +10.0 |\n| intermediate | 1 | 45.0 | +15.0 | 40.0 | +10.0 |\n| medium | 1 | 70.0 | +10.0 | 70.0 | +10.0 |\n| advanced | 1 | 90.0 | +25.0 | 75.0 | +10.0 |\n| hard | 1 | 80.0 | +40.0 | 50.0 | +10.0 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **a** | 6 | 67% |\n| **b** | 2 | 22% |\n| tie | 1 | 11% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `with-context`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **a** | **+17.1** | 6 (86%) | 1 (14%) |\n| **b** | **+10.0** | 9 (100%) | 0 (0%) |\n\n## Model Ranking\n\n### By absolute quality (with-context)\n| Rank | Run | with-context avg |\n|---:|---|---:|\n| 1 | **a** | 68.8 |\n| 2 | **b** | 62.2 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **a** | +17.1 |\n| 2 | **b** | +10.0 |\n", "pinned_arm": "## Models Compared\n| Run ID | Answer model | Judge model | Treatment arm |\n|---|---|---|---|\n| **a** | m (p) | same (p) | `docs` |\n\nCommon questions evaluated: **9**\n_All summary, significance, ranking, and head-to-head metrics use only this common question set._\n\n## Overall Summary\n_Delta is the mean of per-question paired differences. Scored pairs is the number of questions with a usable score in both arms; Delta, Improved and Degraded are computed over those only._\n| Run | N | Scored pairs | docs | Baseline | Delta | Improved | Degraded |\n|---|---:|---:|---:|---:|---:|---:|---:|\n| **a** | 9 | 7 (2 unscored) | 68.8 | 56.2 | **+17.1** | 6 | 1 |\n\n## Statistical Significance of Delta (α = 0.05)\n_p-values are adjusted across the 1 run(s) compared here (Holm-Bonferroni); the raw column is shown for reference. Significant = adjusted p < 0.05 and Wilcoxon < 0.05 (or unavailable)._\n\n| Run | Delta | p (t-test) | p (adjusted) | p (Wilcoxon) | Cohen's d_z | Effect | Significant? |\n|---|---:|---:|---:|---:|---:|---|---|\n| **a** | +17.1 | 0.0247 | 0.0247 | 0.0469 | +1.1 | large | OK Yes |\n\n## By Difficulty (common questions only)\n| Difficulty | N | a context | a Δ |\n|---|---:| :---: | :---: |\n| easy | 9 | 68.8 | +17.1 |\n\n## Head-to-Head (context arm, common questions)\n_On how many questions did each model achieve the highest absolute score (in the context arm)? This measures which model produces the best answers overall, not which benefits most from the treatment._\n\n| Winner | Count | % |\n|---|---:|---:|\n| **a** | 8 | 89% |\n| tie | 0 | 0% |\n\n## Context-Arm Benefit — Delta Comparison\n_Which model benefits more from the treatment arm `docs`?_\n\n| Run | Avg Delta | Questions context helped | Questions context hurt |\n|---|---:|---:|---:|\n| **a** | **+17.1** | 6 (86%) | 1 (14%) |\n\n## Model Ranking\n\n### By absolute quality (docs)\n| Rank | Run | docs avg |\n|---:|---|---:|\n| 1 | **a** | 68.8 |\n\n### By treatment utilisation (Delta)\n| Rank | Run | Delta |\n|---:|---|---:|\n| 1 | **a** | +17.1 |\n", diff --git a/dokimi/tests/test_printed_hints_parse.py b/dokimi/tests/test_printed_hints_parse.py new file mode 100644 index 000000000..48e2a8d0f --- /dev/null +++ b/dokimi/tests/test_printed_hints_parse.py @@ -0,0 +1,92 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Every command a printed hint tells the user to run names the `dokimi` console script and parses. + +The hints are read by people who copy them, so each one is captured from the code that prints it, +the argv is taken from the text, and the argv goes through the same parse_args the CLI uses. +""" + +import json +import re +import shlex + +import pytest + +from dokimi import cli +from dokimi.commands import personas as personas_cmd + +# A hint names its program, then a subcommand (lower case). `dokimi` is the console script from +# pyproject.toml [project.scripts]; `python cli.py` is the source-checkout shim, which hints must not name. +_HINT = re.compile(r"(?:^|\s)(?:python cli\.py|dokimi)\s+(?=[a-z])") + + +def _hinted_argvs(text: str) -> list[list[str]]: + """The argv (program first) of each command in ``text``, with backslash continuations joined.""" + argvs = [] + for line in text.replace("\\\n", " ").splitlines(): + match = _HINT.search(line) + if match: + argvs.append(shlex.split(line[match.start():].lstrip())) + return argvs + + +def _assert_hints_parse(text: str) -> None: + argvs = _hinted_argvs(text) + assert argvs, f"no hint found in:\n{text}" + for argv in argvs: + assert argv[0] == "dokimi", f"hint names {argv[0]!r}, not the console script: {' '.join(argv)}" + cli.parse_args(argv[1:]) # SystemExit if the hint does not parse + + +class _FakeGenerator: + def __init__(self, **_): + pass + + def generate_personas(self, **_): + return {"personas": [{"id": "p1", "name": "n", "skill_level": "novice"}]} + + def save_personas(self, personas, output_path): + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(personas), encoding="utf-8") + + +def test_personas_discover_next_step_parses(tmp_path, monkeypatch, capsys): + monkeypatch.setattr(personas_cmd, "PersonaGenerator", _FakeGenerator) + args = cli.parse_args(["personas", "discover", "--product", "P", "--description", "d", + "--output", str(tmp_path / "P.json")]) + args.func(args) + _assert_hints_parse(capsys.readouterr().out) + + +def test_personas_approve_next_step_parses(tmp_path, capsys): + persona_file = tmp_path / "P.json" + persona_file.write_text(json.dumps({"product": "P", "personas": [ + {"id": "p1", "name": "n", "skill_level": "novice"}]}), encoding="utf-8") + args = cli.parse_args(["personas", "approve", "--file", str(persona_file)]) + args.func(args) + _assert_hints_parse(capsys.readouterr().out) + + +def test_baseline_list_hint_parses(tmp_path, monkeypatch, capsys): + monkeypatch.chdir(tmp_path) # no baselines/eval/manifest.json here + args = cli.parse_args(["baseline", "list"]) + args.func(args) + _assert_hints_parse(capsys.readouterr().out) + + +def test_baseline_compare_hint_parses(tmp_path, monkeypatch, capsys): + monkeypatch.chdir(tmp_path) + eval_file = tmp_path / "eval.json" + eval_file.write_text("{}", encoding="utf-8") + args = cli.parse_args(["baseline", "compare", "--eval", str(eval_file)]) + with pytest.raises(SystemExit): # no baseline to compare against + args.func(args) + _assert_hints_parse(capsys.readouterr().out) + + +def test_baseline_only_report_hint_parses(): + from dokimi.report.model_compare import _BASELINE_ONLY_LIMITATION + + # The report quotes the command with a "> " prefix on every line. + _assert_hints_parse("\n".join(line.removeprefix("> ") for line in _BASELINE_ONLY_LIMITATION))