From 9e5dc10f4f3446ee4333fae68e7850d5c835eaa9 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Wed, 28 Oct 2026 15:41:20 +0000 Subject: [PATCH] test: pin checks the reviews of #157, #187, #188, #213, #246 asked for Each test below fails on a code change the reviews' mutation runs found no test catching. Tests only: no code changes. - #213 (network policy): the allowlist NO_PROXY value is asserted next to HTTPS_PROXY; a task network_mode of "Public" or "No-Network" is read case-insensitively; allowed_hosts or environment.egress given as one string is one host, not its letters. - #213 (SWE-bench grader): a bare status-word line is skipped by parse_pytest, parse_pytest_options and parse_pytest_v2, as it already was tested for parse_seaborn. - #246: _apply_correction is checked on [0.01, 0.02, 0.03], where BH (0.03, 0.03, 0.03) and Holm (0.03, 0.04, 0.04) differ. - #157: a read that only the second guidance glob matches counts as guidance, and an agent that left no trace gets no "process metrics failed" line in trace_errors. - #188: a cached usage with "model": null is a cache miss. - #187: the dry run prints the skipped-factor lines in --factors order. Signed-off-by: Anton Volkov --- dokimi/tests/test_caching.py | 1 + dokimi/tests/test_network_policy.py | 14 ++++++++++++++ dokimi/tests/test_repo_process_metrics.py | 7 +++++++ dokimi/tests/test_repo_run.py | 9 +++++++++ dokimi/tests/test_stats_correction.py | 17 +++++++++++------ dokimi/tests/test_swebench_tasks.py | 5 +++-- dokimi/tests/test_tool_preference.py | 9 ++++----- 7 files changed, 49 insertions(+), 13 deletions(-) diff --git a/dokimi/tests/test_caching.py b/dokimi/tests/test_caching.py index e2e5e3e90..2e538c525 100644 --- a/dokimi/tests/test_caching.py +++ b/dokimi/tests/test_caching.py @@ -271,6 +271,7 @@ def test_entry_without_text_is_a_miss(monkeypatch, tmp_path): '{"text": "answer", "usage": {"cost_usd": "free"}}', '{"text": "answer", "usage": {"latency_sec": true}}', '{"text": "answer", "usage": {"model": 5}}', + '{"text": "answer", "usage": {"model": null}}', '{"text": "answer", "usage": {"prompt_tokens": NaN}}'], ) def test_entry_of_the_wrong_shape_is_a_miss_not_a_crash(monkeypatch, tmp_path, payload): diff --git a/dokimi/tests/test_network_policy.py b/dokimi/tests/test_network_policy.py index b7db5657e..425b3425e 100644 --- a/dokimi/tests/test_network_policy.py +++ b/dokimi/tests/test_network_policy.py @@ -93,6 +93,19 @@ def test_dokimi_egress_widens_the_agent_solve_into_an_allowlist(tmp_path): assert resolve(task).solve.mode == "none" # the oracle is not the agent +@pytest.mark.parametrize("mode, want", [("Public", "public"), ("No-Network", "none")]) # a mode, an alias +def test_a_task_network_mode_is_read_whatever_its_case(tmp_path, mode, want): + task = load_task(_task(tmp_path, f'[environment]\nnetwork_mode = "{mode}"\n')) + assert resolve(task).verify.mode == want + + +def test_a_single_host_given_as_a_string_is_one_host_not_its_letters(tmp_path): + allowed = '[environment]\nnetwork_mode = "allowlist"\nallowed_hosts = "pypi.org"\n' + assert resolve(load_task(_task(tmp_path / "a", allowed))).verify.hosts == ("pypi.org:443",) + egress = load_task(_task(tmp_path / "e", '[environment]\negress = "huggingface.co"\n')) + assert resolve(egress, agent=True).solve.hosts == ("huggingface.co:443",) + + def test_cli_overrides_win_over_the_task(tmp_path): task = load_task(_task(tmp_path, '[environment]\nnetwork_mode = "public"\n')) plan = resolve(task, ["verify=none", "solve=allowlist:httpbin.org:80"]) @@ -178,6 +191,7 @@ def test_an_allowlisted_verifier_goes_through_its_own_sidecar(tmp_path): assert side[side.index("--allow") + 1] == "files.pythonhosted.org:443,pypi.org:443" verify = args[_index(args, _exec_of("test.sh"))] assert "HTTPS_PROXY=http://egress:3128" in verify + assert "NO_PROXY=localhost,127.0.0.1,::1" in verify and "no_proxy=localhost,127.0.0.1,::1" in verify oracle = args[_index(args, _exec_of("solve.sh"))] assert not any("PROXY" in x for x in oracle) assert metrics["network"]["verify_egress"]["status"] == "ok" diff --git a/dokimi/tests/test_repo_process_metrics.py b/dokimi/tests/test_repo_process_metrics.py index a38288178..4162025ba 100644 --- a/dokimi/tests/test_repo_process_metrics.py +++ b/dokimi/tests/test_repo_process_metrics.py @@ -99,6 +99,13 @@ def test_guidance_read_is_the_workspace_guidance_files_the_agent_read(): assert run_process_metrics(ctx, events)["guidance_read"] == ["AGENTS.md"] +def test_a_read_that_only_the_second_guidance_glob_matches_is_guidance(): + # MANIFEST's globs are "**/AGENTS.md" and ".github/instructions/*.md": each one counts. + ctx = SimpleNamespace(manifest=parse_manifest(MANIFEST)) + events = [{"kind": "read", "path": ".github/instructions/c.md"}] + assert run_process_metrics(ctx, events)["guidance_read"] == [".github/instructions/c.md"] + + def test_a_command_the_permissions_denied_is_denied_not_failed(): # A permission denial comes back as an is_error result (rc 1); a hook denial has rc null. Neither ran. events = [{"kind": "command", "text": "rm -rf build", "rc": 1, "denied": True}, diff --git a/dokimi/tests/test_repo_run.py b/dokimi/tests/test_repo_run.py index 64753444e..bc214de79 100644 --- a/dokimi/tests/test_repo_run.py +++ b/dokimi/tests/test_repo_run.py @@ -114,6 +114,15 @@ def refuse(*args, **kwargs): assert rec["grade"]["status"] == "grader_failure" +def test_an_agent_that_left_no_trace_is_not_given_a_process_metrics_error(setup, monkeypatch): + # No events and no turns: the metrics are not run on the missing trace, so they record no failure. + monkeypatch.setenv("FAKE_ACTION", "silent") + _run(setup, "--arm", "guidance:raw", "--task", "fix_add") + rec = json.loads((setup["out"] / "b1/fix_add__guidance-raw__m-1__r1/result.json").read_text()) + assert rec["process"] is None + assert not any("process metrics failed" in e for e in rec["agent"]["trace_errors"]) + + def test_a_process_metrics_error_leaves_process_null_and_keeps_the_cost_and_the_grade(setup, monkeypatch): # The metric is best-effort: a counter that raises must not fail the task or drop the cost the harness reported. def refuse(*args, **kwargs): diff --git a/dokimi/tests/test_stats_correction.py b/dokimi/tests/test_stats_correction.py index e61288c20..ff08d70ec 100644 --- a/dokimi/tests/test_stats_correction.py +++ b/dokimi/tests/test_stats_correction.py @@ -13,6 +13,8 @@ from __future__ import annotations +import pytest + from dokimi.eval import stats from dokimi.eval.cross_judge import apply_holm from dokimi.report import model_compare @@ -93,12 +95,15 @@ def test_apply_correction_records_family_fields_and_none_entries(): assert res["family_tested"] == 2 -def test_apply_correction_uses_the_requested_method(): - results = [{"p_ttest": 0.04, "p_wilcoxon": None}, {"p_ttest": 0.01, "p_wilcoxon": None}] - stats._apply_correction(results, correction="bh", label="arms") - # BH on [0.04, 0.01]: step-up gives 0.04 and min(0.04, 0.01 * 2) = 0.02. - assert [r["p_adjusted"] for r in results] == [0.04, 0.02] - assert all(r["correction"] == "bh" for r in results) +# On [0.01, 0.02, 0.03] the two methods differ. Holm steps down: 3 * 0.01 = 0.03, +# max(0.03, 2 * 0.02) = 0.04, max(0.04, 1 * 0.03) = 0.04. BH steps up: 0.03, +# min(0.03, 0.02 * 3/2) = 0.03, min(0.03, 0.01 * 3) = 0.03. +@pytest.mark.parametrize("correction, adjusted", [("bh", [0.03, 0.03, 0.03]), ("holm", [0.03, 0.04, 0.04])]) +def test_apply_correction_uses_the_requested_method(correction, adjusted): + results = [{"p_ttest": p, "p_wilcoxon": None} for p in (0.01, 0.02, 0.03)] + stats._apply_correction(results, correction=correction, label="arms") + assert [r["p_adjusted"] for r in results] == adjusted + assert all(r["correction"] == correction for r in results) def test_apply_correction_on_an_empty_family_adds_nothing(): diff --git a/dokimi/tests/test_swebench_tasks.py b/dokimi/tests/test_swebench_tasks.py index 35f40fe71..8196552ef 100644 --- a/dokimi/tests/test_swebench_tasks.py +++ b/dokimi/tests/test_swebench_tasks.py @@ -306,10 +306,11 @@ def test_an_empty_marked_section_falls_back_to_the_whole_log(): assert grader.grade(log, _CONFIG)["resolved"] is True -def test_a_bare_seaborn_status_word_is_skipped_not_an_index_error(): +@pytest.mark.parametrize("parser", ["seaborn", "pytest", "pytest_options", "pytest_v2"]) +def test_a_bare_status_word_is_skipped_not_an_index_error(parser): # A line that is only FAILED or PASSED has no test name; it must not raise. log = "FAILED\nPASSED\n PASSED \nPASSED t.py::f\n" - assert grader.parse_seaborn(log) == {"t.py::f": "PASSED"} + assert getattr(grader, f"parse_{parser}")(log) == {"t.py::f": "PASSED"} def test_pytest_v2_reads_coloured_and_old_style_lines(): diff --git a/dokimi/tests/test_tool_preference.py b/dokimi/tests/test_tool_preference.py index 034f99745..456453adf 100644 --- a/dokimi/tests/test_tool_preference.py +++ b/dokimi/tests/test_tool_preference.py @@ -817,11 +817,10 @@ def test_dry_run_names_the_groups_a_factor_is_skipped_for(capsys): args = build_parser().parse_args(["tools", "preference", "--groups", str(GROUPS), "--dry-run", "--factors", "assertive,exclusion,long_desc"]) args.func(args) - out = capsys.readouterr().out - assert "exclusion: skipped for 2 of 3 groups (G02, N01)" in out - assert "long_desc: skipped for 2 of 3 groups (G02 twin B, N01)" in out - assert "assertive: skipped" not in out - assert "vendor_name: skipped" not in out + skipped = [line.strip() for line in capsys.readouterr().out.splitlines() if ": skipped for " in line] + # in the order --factors gave them (FACTORS has long_desc first), and only the factors asked for + assert skipped == ["exclusion: skipped for 2 of 3 groups (G02, N01)", + "long_desc: skipped for 2 of 3 groups (G02 twin B, N01)"] def test_dry_run_names_twin_a_when_only_twin_a_lacks_the_field(tmp_path, capsys):