diff --git a/packages/mutiny_cli/src/mutiny_cli/main.py b/packages/mutiny_cli/src/mutiny_cli/main.py index cf68cba..e203a2d 100644 --- a/packages/mutiny_cli/src/mutiny_cli/main.py +++ b/packages/mutiny_cli/src/mutiny_cli/main.py @@ -130,6 +130,11 @@ def main(argv: list[str] | None = None) -> int: action="store_true", help="Do not write .mutiny/test-report.json", ) + test_p.add_argument( + "--allow-skip", + action="store_true", + help="Exit 0 when tests pass even if some regression files are skipped", + ) args = parser.parse_args(argv) if args.command == "init": @@ -155,6 +160,7 @@ def main(argv: list[str] | None = None) -> int: failed_only=args.failed, json_out=args.json_out, write_report=not args.no_report, + allow_skip=args.allow_skip, ) if args.command == "db": from mutiny_cli.db_cmd import run_db_command diff --git a/packages/mutiny_cli/src/mutiny_cli/test_cmd.py b/packages/mutiny_cli/src/mutiny_cli/test_cmd.py index 52ca0d1..66a9316 100644 --- a/packages/mutiny_cli/src/mutiny_cli/test_cmd.py +++ b/packages/mutiny_cli/src/mutiny_cli/test_cmd.py @@ -65,8 +65,9 @@ def run_tests( failed_only: bool = False, json_out: bool = False, write_report: bool = True, + allow_skip: bool = False, ) -> int: - """Discover and replay project regressions. Exit 0 all pass, 1 failures, 2 error.""" + """Discover and replay project regressions. Exit 0 all pass, 1 failures/unexpected skips, 2 error.""" root = project_root.resolve() ensure_project_on_path(root) @@ -199,8 +200,9 @@ def run_tests( if failed: return 1 - if skipped and not passed: - return 1 + if skipped: + if not allow_skip or not passed: + return 1 return 0 diff --git a/tests/unit/test_mutiny_cli_test.py b/tests/unit/test_mutiny_cli_test.py index 4ffba1d..e36ca9b 100644 --- a/tests/unit/test_mutiny_cli_test.py +++ b/tests/unit/test_mutiny_cli_test.py @@ -131,3 +131,58 @@ def test_mutiny_test_summary_line_failed(tmp_path: Path, capsys: pytest.CaptureF out = capsys.readouterr().out assert "Summary: 0 passed, 1 failed, 0 skipped" in out + +def test_mutiny_test_pass_and_corrupt_json_fails_without_allow_skip( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +): + _scaffold(tmp_path, fixed=True) + # Add an invalid JSON regression fixture + (tmp_path / ".mutiny" / "tests" / "corrupt.json").write_text( + "{invalid json", encoding="utf-8" + ) + + code = run_tests(project_root=tmp_path, json_out=False) + assert code == 1 + out = capsys.readouterr().out + assert "Summary: 1 passed, 0 failed, 1 skipped" in out + + report = json.loads((tmp_path / ".mutiny" / "test-report.json").read_text()) + assert report["passed"] == 1 + assert report["failed"] == 0 + assert report["skipped"] == 1 + assert any( + r["status"] == "SKIPPED" and "invalid regression JSON: corrupt.json" in str(r.get("error")) + for r in report["results"] + ) + + +def test_mutiny_test_pass_and_corrupt_json_with_allow_skip(tmp_path: Path): + _scaffold(tmp_path, fixed=True) + (tmp_path / ".mutiny" / "tests" / "corrupt.json").write_text( + "{invalid json", encoding="utf-8" + ) + + # Function call with allow_skip=True + code = run_tests(project_root=tmp_path, allow_skip=True) + assert code == 0 + + # CLI call without --allow-skip + assert main(["test", "--path", str(tmp_path)]) == 1 + + # CLI call with --allow-skip + assert main(["test", "--path", str(tmp_path), "--allow-skip"]) == 0 + + +def test_mutiny_test_skip_only_fails_even_with_allow_skip(tmp_path: Path): + (tmp_path / ".mutiny" / "tests").mkdir(parents=True) + shutil.copy(DEMO_POLICY, tmp_path / "policy.json") + (tmp_path / ".mutiny" / "adapter.py").write_text(ADAPTER_SRC, encoding="utf-8") + (tmp_path / ".mutiny" / "tests" / "corrupt.json").write_text( + "{invalid json", encoding="utf-8" + ) + + # Skip-only must still return non-zero even if allow_skip=True + assert run_tests(project_root=tmp_path, allow_skip=False) == 1 + assert run_tests(project_root=tmp_path, allow_skip=True) == 1 + +