fix(cli): exit non-zero when tests have skipped cases unless --allow-skip (#71)

packages/mutiny_cli/src/mutiny_cli/main.py

@@ -130,6 +130,11 @@ def main(argv: list[str] | None = None) -> int:
         action="store_true",
         help="Do not write .mutiny/test-report.json",
     )
+    test_p.add_argument(
+        "--allow-skip",
+        action="store_true",
+        help="Exit 0 when tests pass even if some regression files are skipped",
+    )
 
     args = parser.parse_args(argv)
     if args.command == "init":
@@ -155,6 +160,7 @@ def main(argv: list[str] | None = None) -> int:
             failed_only=args.failed,
             json_out=args.json_out,
             write_report=not args.no_report,
+            allow_skip=args.allow_skip,
         )
     if args.command == "db":
         from mutiny_cli.db_cmd import run_db_command

packages/mutiny_cli/src/mutiny_cli/test_cmd.py

@@ -65,8 +65,9 @@ def run_tests(
     failed_only: bool = False,
     json_out: bool = False,
     write_report: bool = True,
+    allow_skip: bool = False,
 ) -> int:
-    """Discover and replay project regressions. Exit 0 all pass, 1 failures, 2 error."""
+    """Discover and replay project regressions. Exit 0 all pass, 1 failures/unexpected skips, 2 error."""
     root = project_root.resolve()
     ensure_project_on_path(root)
 
@@ -199,8 +200,9 @@ def run_tests(
 
     if failed:
         return 1
-    if skipped and not passed:
-        return 1
+    if skipped:
+        if not allow_skip or not passed:
+            return 1
     return 0
 
 

tests/unit/test_mutiny_cli_test.py

@@ -131,3 +131,58 @@ def test_mutiny_test_summary_line_failed(tmp_path: Path, capsys: pytest.CaptureF
     out = capsys.readouterr().out
     assert "Summary: 0 passed, 1 failed, 0 skipped" in out
 
+
+def test_mutiny_test_pass_and_corrupt_json_fails_without_allow_skip(
+    tmp_path: Path, capsys: pytest.CaptureFixture[str]
+):
+    _scaffold(tmp_path, fixed=True)
+    # Add an invalid JSON regression fixture
+    (tmp_path / ".mutiny" / "tests" / "corrupt.json").write_text(
+        "{invalid json", encoding="utf-8"
+    )
+
+    code = run_tests(project_root=tmp_path, json_out=False)
+    assert code == 1
+    out = capsys.readouterr().out
+    assert "Summary: 1 passed, 0 failed, 1 skipped" in out
+
+    report = json.loads((tmp_path / ".mutiny" / "test-report.json").read_text())
+    assert report["passed"] == 1
+    assert report["failed"] == 0
+    assert report["skipped"] == 1
+    assert any(
+        r["status"] == "SKIPPED" and "invalid regression JSON: corrupt.json" in str(r.get("error"))
+        for r in report["results"]
+    )
+
+
+def test_mutiny_test_pass_and_corrupt_json_with_allow_skip(tmp_path: Path):
+    _scaffold(tmp_path, fixed=True)
+    (tmp_path / ".mutiny" / "tests" / "corrupt.json").write_text(
+        "{invalid json", encoding="utf-8"
+    )
+
+    # Function call with allow_skip=True
+    code = run_tests(project_root=tmp_path, allow_skip=True)
+    assert code == 0
+
+    # CLI call without --allow-skip
+    assert main(["test", "--path", str(tmp_path)]) == 1
+
+    # CLI call with --allow-skip
+    assert main(["test", "--path", str(tmp_path), "--allow-skip"]) == 0
+
+
+def test_mutiny_test_skip_only_fails_even_with_allow_skip(tmp_path: Path):
+    (tmp_path / ".mutiny" / "tests").mkdir(parents=True)
+    shutil.copy(DEMO_POLICY, tmp_path / "policy.json")
+    (tmp_path / ".mutiny" / "adapter.py").write_text(ADAPTER_SRC, encoding="utf-8")
+    (tmp_path / ".mutiny" / "tests" / "corrupt.json").write_text(
+        "{invalid json", encoding="utf-8"
+    )
+
+    # Skip-only must still return non-zero even if allow_skip=True
+    assert run_tests(project_root=tmp_path, allow_skip=False) == 1
+    assert run_tests(project_root=tmp_path, allow_skip=True) == 1
+
+