test: pin checks the reviews of #157, #187, #188, #213, #246 asked for

dokimi/tests/test_caching.py

@@ -271,6 +271,7 @@ def test_entry_without_text_is_a_miss(monkeypatch, tmp_path):
      '{"text": "answer", "usage": {"cost_usd": "free"}}',
      '{"text": "answer", "usage": {"latency_sec": true}}',
      '{"text": "answer", "usage": {"model": 5}}',
+     '{"text": "answer", "usage": {"model": null}}',
      '{"text": "answer", "usage": {"prompt_tokens": NaN}}'],
 )
 def test_entry_of_the_wrong_shape_is_a_miss_not_a_crash(monkeypatch, tmp_path, payload):

dokimi/tests/test_network_policy.py

@@ -93,6 +93,19 @@ def test_dokimi_egress_widens_the_agent_solve_into_an_allowlist(tmp_path):
     assert resolve(task).solve.mode == "none"  # the oracle is not the agent
 
 
+@pytest.mark.parametrize("mode, want", [("Public", "public"), ("No-Network", "none")])  # a mode, an alias
+def test_a_task_network_mode_is_read_whatever_its_case(tmp_path, mode, want):
+    task = load_task(_task(tmp_path, f'[environment]\nnetwork_mode = "{mode}"\n'))
+    assert resolve(task).verify.mode == want
+
+
+def test_a_single_host_given_as_a_string_is_one_host_not_its_letters(tmp_path):
+    allowed = '[environment]\nnetwork_mode = "allowlist"\nallowed_hosts = "pypi.org"\n'
+    assert resolve(load_task(_task(tmp_path / "a", allowed))).verify.hosts == ("pypi.org:443",)
+    egress = load_task(_task(tmp_path / "e", '[environment]\negress = "huggingface.co"\n'))
+    assert resolve(egress, agent=True).solve.hosts == ("huggingface.co:443",)
+
+
 def test_cli_overrides_win_over_the_task(tmp_path):
     task = load_task(_task(tmp_path, '[environment]\nnetwork_mode = "public"\n'))
     plan = resolve(task, ["verify=none", "solve=allowlist:httpbin.org:80"])
@@ -178,6 +191,7 @@ def test_an_allowlisted_verifier_goes_through_its_own_sidecar(tmp_path):
     assert side[side.index("--allow") + 1] == "files.pythonhosted.org:443,pypi.org:443"
     verify = args[_index(args, _exec_of("test.sh"))]
     assert "HTTPS_PROXY=http://egress:3128" in verify
+    assert "NO_PROXY=localhost,127.0.0.1,::1" in verify and "no_proxy=localhost,127.0.0.1,::1" in verify
     oracle = args[_index(args, _exec_of("solve.sh"))]
     assert not any("PROXY" in x for x in oracle)
     assert metrics["network"]["verify_egress"]["status"] == "ok"

dokimi/tests/test_repo_process_metrics.py

@@ -99,6 +99,13 @@ def test_guidance_read_is_the_workspace_guidance_files_the_agent_read():
     assert run_process_metrics(ctx, events)["guidance_read"] == ["AGENTS.md"]
 
 
+def test_a_read_that_only_the_second_guidance_glob_matches_is_guidance():
+    # MANIFEST's globs are "**/AGENTS.md" and ".github/instructions/*.md": each one counts.
+    ctx = SimpleNamespace(manifest=parse_manifest(MANIFEST))
+    events = [{"kind": "read", "path": ".github/instructions/c.md"}]
+    assert run_process_metrics(ctx, events)["guidance_read"] == [".github/instructions/c.md"]
+
+
 def test_a_command_the_permissions_denied_is_denied_not_failed():
     # A permission denial comes back as an is_error result (rc 1); a hook denial has rc null. Neither ran.
     events = [{"kind": "command", "text": "rm -rf build", "rc": 1, "denied": True},

dokimi/tests/test_repo_run.py

@@ -114,6 +114,15 @@ def refuse(*args, **kwargs):
     assert rec["grade"]["status"] == "grader_failure"
 
 
+def test_an_agent_that_left_no_trace_is_not_given_a_process_metrics_error(setup, monkeypatch):
+    # No events and no turns: the metrics are not run on the missing trace, so they record no failure.
+    monkeypatch.setenv("FAKE_ACTION", "silent")
+    _run(setup, "--arm", "guidance:raw", "--task", "fix_add")
+    rec = json.loads((setup["out"] / "b1/fix_add__guidance-raw__m-1__r1/result.json").read_text())
+    assert rec["process"] is None
+    assert not any("process metrics failed" in e for e in rec["agent"]["trace_errors"])
+
+
 def test_a_process_metrics_error_leaves_process_null_and_keeps_the_cost_and_the_grade(setup, monkeypatch):
     # The metric is best-effort: a counter that raises must not fail the task or drop the cost the harness reported.
     def refuse(*args, **kwargs):

dokimi/tests/test_stats_correction.py

@@ -13,6 +13,8 @@
 
 from __future__ import annotations
 
+import pytest
+
 from dokimi.eval import stats
 from dokimi.eval.cross_judge import apply_holm
 from dokimi.report import model_compare
@@ -93,12 +95,15 @@ def test_apply_correction_records_family_fields_and_none_entries():
         assert res["family_tested"] == 2
 
 
-def test_apply_correction_uses_the_requested_method():
-    results = [{"p_ttest": 0.04, "p_wilcoxon": None}, {"p_ttest": 0.01, "p_wilcoxon": None}]
-    stats._apply_correction(results, correction="bh", label="arms")
-    # BH on [0.04, 0.01]: step-up gives 0.04 and min(0.04, 0.01 * 2) = 0.02.
-    assert [r["p_adjusted"] for r in results] == [0.04, 0.02]
-    assert all(r["correction"] == "bh" for r in results)
+# On [0.01, 0.02, 0.03] the two methods differ. Holm steps down: 3 * 0.01 = 0.03,
+# max(0.03, 2 * 0.02) = 0.04, max(0.04, 1 * 0.03) = 0.04. BH steps up: 0.03,
+# min(0.03, 0.02 * 3/2) = 0.03, min(0.03, 0.01 * 3) = 0.03.
+@pytest.mark.parametrize("correction, adjusted", [("bh", [0.03, 0.03, 0.03]), ("holm", [0.03, 0.04, 0.04])])
+def test_apply_correction_uses_the_requested_method(correction, adjusted):
+    results = [{"p_ttest": p, "p_wilcoxon": None} for p in (0.01, 0.02, 0.03)]
+    stats._apply_correction(results, correction=correction, label="arms")
+    assert [r["p_adjusted"] for r in results] == adjusted
+    assert all(r["correction"] == correction for r in results)
 
 
 def test_apply_correction_on_an_empty_family_adds_nothing():

dokimi/tests/test_swebench_tasks.py

@@ -306,10 +306,11 @@ def test_an_empty_marked_section_falls_back_to_the_whole_log():
     assert grader.grade(log, _CONFIG)["resolved"] is True
 
 
-def test_a_bare_seaborn_status_word_is_skipped_not_an_index_error():
+@pytest.mark.parametrize("parser", ["seaborn", "pytest", "pytest_options", "pytest_v2"])
+def test_a_bare_status_word_is_skipped_not_an_index_error(parser):
     # A line that is only FAILED or PASSED has no test name; it must not raise.
     log = "FAILED\nPASSED\n PASSED \nPASSED t.py::f\n"
-    assert grader.parse_seaborn(log) == {"t.py::f": "PASSED"}
+    assert getattr(grader, f"parse_{parser}")(log) == {"t.py::f": "PASSED"}
 
 
 def test_pytest_v2_reads_coloured_and_old_style_lines():

dokimi/tests/test_tool_preference.py

@@ -817,11 +817,10 @@ def test_dry_run_names_the_groups_a_factor_is_skipped_for(capsys):
     args = build_parser().parse_args(["tools", "preference", "--groups", str(GROUPS), "--dry-run",
                                       "--factors", "assertive,exclusion,long_desc"])
     args.func(args)
-    out = capsys.readouterr().out
-    assert "exclusion: skipped for 2 of 3 groups (G02, N01)" in out
-    assert "long_desc: skipped for 2 of 3 groups (G02 twin B, N01)" in out
-    assert "assertive: skipped" not in out
-    assert "vendor_name: skipped" not in out
+    skipped = [line.strip() for line in capsys.readouterr().out.splitlines() if ": skipped for " in line]
+    # in the order --factors gave them (FACTORS has long_desc first), and only the factors asked for
+    assert skipped == ["exclusion: skipped for 2 of 3 groups (G02, N01)",
+                       "long_desc: skipped for 2 of 3 groups (G02 twin B, N01)"]
 
 
 def test_dry_run_names_twin_a_when_only_twin_a_lacks_the_field(tmp_path, capsys):