diff --git a/docs/CLI.md b/docs/CLI.md index 7f7a685..d50d1f8 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -86,6 +86,10 @@ mutiny run --no-hosted # same as default local Replay regressions under `.mutiny/tests/` and print a PASS / FAIL / SKIPPED report. +Any failed or skipped case makes the command exit with code `1`, even if other +cases pass. This includes corrupt regression files and replay errors; an +incomplete run must not appear successful in CI. + | Arg / flag | Default | Meaning | |---|---|---| | `regression_id` | (all) | Optional id or name to run one case | diff --git a/packages/mutiny_cli/src/mutiny_cli/test_cmd.py b/packages/mutiny_cli/src/mutiny_cli/test_cmd.py index 52ca0d1..272f2bf 100644 --- a/packages/mutiny_cli/src/mutiny_cli/test_cmd.py +++ b/packages/mutiny_cli/src/mutiny_cli/test_cmd.py @@ -66,7 +66,7 @@ def run_tests( json_out: bool = False, write_report: bool = True, ) -> int: - """Discover and replay project regressions. Exit 0 all pass, 1 failures, 2 error.""" + """Replay regressions. Exit 0 all pass, 1 failed/skipped cases, 2 setup error.""" root = project_root.resolve() ensure_project_on_path(root) @@ -199,7 +199,7 @@ def run_tests( if failed: return 1 - if skipped and not passed: + if skipped: return 1 return 0 diff --git a/tests/unit/test_mutiny_cli_test.py b/tests/unit/test_mutiny_cli_test.py index 4ffba1d..700470f 100644 --- a/tests/unit/test_mutiny_cli_test.py +++ b/tests/unit/test_mutiny_cli_test.py @@ -70,6 +70,35 @@ def test_mutiny_test_fixed_passes(tmp_path: Path): assert report["results"][0]["status"] == "PASS" +@pytest.mark.parametrize("include_pass", [False, True]) +@pytest.mark.parametrize("contents", [b"{broken", b"{}", b"\xff"]) +def test_mutiny_test_corrupt_artifact_fails( + tmp_path: Path, capsys: pytest.CaptureFixture[str], include_pass: bool, contents: bytes +): + _scaffold(tmp_path, fixed=True) + cases = tmp_path / ".mutiny" / "tests" + if not include_pass: + (cases / "refund_limit.json").unlink() + (cases / "broken.json").write_bytes(contents) + + assert main(["test", "--path", str(tmp_path)]) == 1 + assert f"Summary: {int(include_pass)} passed, 0 failed, 1 skipped" in capsys.readouterr().out + report = json.loads((tmp_path / ".mutiny" / "test-report.json").read_text()) + assert (report["passed"], report["failed"], report["skipped"]) == (int(include_pass), 0, 1) + broken = next(r for r in report["results"] if r["id"] == "broken") + assert broken["status"] == "SKIPPED" + assert broken["error"] == "invalid regression JSON: broken.json" + + +def test_mutiny_test_mixed_skipped_json_output(tmp_path: Path, capsys: pytest.CaptureFixture[str]): + _scaffold(tmp_path, fixed=True) + (tmp_path / ".mutiny" / "tests" / "broken.json").write_text("{broken") + + assert run_tests(project_root=tmp_path, json_out=True) == 1 + report = json.loads(capsys.readouterr().out) + assert (report["passed"], report["failed"], report["skipped"]) == (1, 0, 1) + + def test_mutiny_test_by_id_and_json(tmp_path: Path, capsys: pytest.CaptureFixture[str]): _scaffold(tmp_path, fixed=True) code = run_tests(