fix: preserve successive CLI regression findings

docs/CLI.md

@@ -57,6 +57,16 @@ Load the project adapter + policy and start an evolutionary campaign.
 **Default execution mode is local** (Core + `.mutiny/adapter.py`). Hosted is
 opt-in only.
 
+Verified findings are saved under `.mutiny/tests/`. The first uses
+`cli_discovered_violation.json`; later saves use `_1`, `_2`, and so on, skipping
+existing paths. Existing files are never overwritten, including during concurrent
+runs. `mutiny test` discovers all saved JSON files.
+
+To deliberately replace an old regression, back up that file, remove it from
+`.mutiny/tests/`, and run the campaign again. The next verified finding can reuse
+the freed name; if no finding reproduces, restore the backup. `mutiny run` has no
+`--force` flag (the `init --force` flag only controls scaffolding).
+
 | Flag | Default | Meaning |
 |---|---|---|
 | `--path PATH` | cwd | Project root |

packages/mutiny_cli/src/mutiny_cli/run_cmd.py

@@ -389,9 +389,19 @@ def _maybe_minimize_and_save(
         return None
     out_dir = root / ".mutiny" / "tests"
     out_dir.mkdir(parents=True, exist_ok=True)
-    out_path = out_dir / f"{artifact.name}.json"
-    artifact_dict = artifact.model_dump(mode="json")
-    out_path.write_text(json.dumps(artifact_dict, indent=2), encoding="utf-8")
+    base_name = artifact.name
+    suffix = 0
+    while True:
+        name = base_name if suffix == 0 else f"{base_name}_{suffix}"
+        out_path = out_dir / f"{name}.json"
+        artifact_dict = artifact.model_copy(update={"name": name}).model_dump(mode="json")
+        try:
+            # Exclusive creation also protects concurrent CLI runs from overwrites.
+            with out_path.open("x", encoding="utf-8") as stream:
+                json.dump(artifact_dict, stream, indent=2)
+            break
+        except FileExistsError:
+            suffix += 1
     print(f"  ✓ regression → {out_path.relative_to(root)}")
     print("  Next: fix the agent, then `mutiny test`")
     # Stable regression id for Hosted ingest (= local file stem).

tests/unit/test_cli_hosted_sync.py

@@ -89,6 +89,43 @@ def _scaffold(tmp: Path, *, api_url: str = "http://127.0.0.1:8000") -> Path:
     return tmp
 
 
+@pytest.mark.parametrize("existing", [False, True])
+def test_successive_findings_preserve_files_and_replay_both(tmp_path, monkeypatch, existing):
+    from mutiny_cli.test_cmd import discover_regressions, run_tests
+    from mutiny_core import load_project_policy
+
+    _scaffold(tmp_path)
+    monkeypatch.setattr(run_cmd, "try_featherless_from_env", lambda: None)
+    policy, _ = load_project_policy(tmp_path)
+    config = yaml.safe_load((tmp_path / "mutiny.yaml").read_text())
+    old_path = tmp_path / ".mutiny/tests/cli_discovered_violation.json"
+    if existing:
+        old_path.parent.mkdir()
+        old_path.write_bytes(b"existing user artifact\n")
+    first = run_cmd._run_local(tmp_path, config, policy)
+    first_path = tmp_path / first.regression_path
+    first_bytes = first_path.read_bytes()
+    second = run_cmd._run_local(tmp_path, config, policy)
+    assert first.regression_id != second.regression_id
+    assert first_path.read_bytes() == first_bytes
+    if existing:
+        assert old_path.read_bytes() == b"existing user artifact\n"
+    cases = discover_regressions(tmp_path)
+    valid = [case for case in cases if case["artifact"] is not None]
+    assert {case["id"] for case in valid} == {first.regression_id, second.regression_id}
+    for outcome in (first, second):
+        assert outcome.regression_artifact["name"] == outcome.regression_id
+        event = next(e for e in outcome.events if e.type == EventType.REGRESSION_CREATED)
+        assert event.payload["regression_id"] == outcome.regression_id
+        assert event.payload["path"] == outcome.regression_path
+    # Replay every saved case through the CLI's normal discovery path.
+    assert run_tests(project_root=tmp_path) == 1  # demo remains vulnerable
+    report = json.loads((tmp_path / ".mutiny/test-report.json").read_text())
+    assert {r["id"] for r in report["results"] if r["status"] == "FAIL"} == {
+        first.regression_id, second.regression_id,
+    }
+
+
 def _scored(
     cid: str = "cand-1",
     *,