diff --git a/docs/CLI.md b/docs/CLI.md index 7f7a685..98ab8db 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -57,6 +57,16 @@ Load the project adapter + policy and start an evolutionary campaign. **Default execution mode is local** (Core + `.mutiny/adapter.py`). Hosted is opt-in only. +Verified findings are saved under `.mutiny/tests/`. The first uses +`cli_discovered_violation.json`; later saves use `_1`, `_2`, and so on, skipping +existing paths. Existing files are never overwritten, including during concurrent +runs. `mutiny test` discovers all saved JSON files. + +To deliberately replace an old regression, back up that file, remove it from +`.mutiny/tests/`, and run the campaign again. The next verified finding can reuse +the freed name; if no finding reproduces, restore the backup. `mutiny run` has no +`--force` flag (the `init --force` flag only controls scaffolding). + | Flag | Default | Meaning | |---|---|---| | `--path PATH` | cwd | Project root | diff --git a/packages/mutiny_cli/src/mutiny_cli/run_cmd.py b/packages/mutiny_cli/src/mutiny_cli/run_cmd.py index d00b601..116d013 100644 --- a/packages/mutiny_cli/src/mutiny_cli/run_cmd.py +++ b/packages/mutiny_cli/src/mutiny_cli/run_cmd.py @@ -389,9 +389,19 @@ def _maybe_minimize_and_save( return None out_dir = root / ".mutiny" / "tests" out_dir.mkdir(parents=True, exist_ok=True) - out_path = out_dir / f"{artifact.name}.json" - artifact_dict = artifact.model_dump(mode="json") - out_path.write_text(json.dumps(artifact_dict, indent=2), encoding="utf-8") + base_name = artifact.name + suffix = 0 + while True: + name = base_name if suffix == 0 else f"{base_name}_{suffix}" + out_path = out_dir / f"{name}.json" + artifact_dict = artifact.model_copy(update={"name": name}).model_dump(mode="json") + try: + # Exclusive creation also protects concurrent CLI runs from overwrites. + with out_path.open("x", encoding="utf-8") as stream: + json.dump(artifact_dict, stream, indent=2) + break + except FileExistsError: + suffix += 1 print(f" ✓ regression → {out_path.relative_to(root)}") print(" Next: fix the agent, then `mutiny test`") # Stable regression id for Hosted ingest (= local file stem). diff --git a/tests/unit/test_cli_hosted_sync.py b/tests/unit/test_cli_hosted_sync.py index 168e914..938f8a1 100644 --- a/tests/unit/test_cli_hosted_sync.py +++ b/tests/unit/test_cli_hosted_sync.py @@ -89,6 +89,43 @@ def _scaffold(tmp: Path, *, api_url: str = "http://127.0.0.1:8000") -> Path: return tmp +@pytest.mark.parametrize("existing", [False, True]) +def test_successive_findings_preserve_files_and_replay_both(tmp_path, monkeypatch, existing): + from mutiny_cli.test_cmd import discover_regressions, run_tests + from mutiny_core import load_project_policy + + _scaffold(tmp_path) + monkeypatch.setattr(run_cmd, "try_featherless_from_env", lambda: None) + policy, _ = load_project_policy(tmp_path) + config = yaml.safe_load((tmp_path / "mutiny.yaml").read_text()) + old_path = tmp_path / ".mutiny/tests/cli_discovered_violation.json" + if existing: + old_path.parent.mkdir() + old_path.write_bytes(b"existing user artifact\n") + first = run_cmd._run_local(tmp_path, config, policy) + first_path = tmp_path / first.regression_path + first_bytes = first_path.read_bytes() + second = run_cmd._run_local(tmp_path, config, policy) + assert first.regression_id != second.regression_id + assert first_path.read_bytes() == first_bytes + if existing: + assert old_path.read_bytes() == b"existing user artifact\n" + cases = discover_regressions(tmp_path) + valid = [case for case in cases if case["artifact"] is not None] + assert {case["id"] for case in valid} == {first.regression_id, second.regression_id} + for outcome in (first, second): + assert outcome.regression_artifact["name"] == outcome.regression_id + event = next(e for e in outcome.events if e.type == EventType.REGRESSION_CREATED) + assert event.payload["regression_id"] == outcome.regression_id + assert event.payload["path"] == outcome.regression_path + # Replay every saved case through the CLI's normal discovery path. + assert run_tests(project_root=tmp_path) == 1 # demo remains vulnerable + report = json.loads((tmp_path / ".mutiny/test-report.json").read_text()) + assert {r["id"] for r in report["results"] if r["status"] == "FAIL"} == { + first.regression_id, second.regression_id, + } + + def _scored( cid: str = "cand-1", *,