From 4803de23f67f982f1c5d4e703b2277fcbd10a844 Mon Sep 17 00:00:00 2001 From: littfed Date: Thu, 24 Sep 2026 17:35:54 +0500 Subject: [PATCH 1/2] fix(cli,core): handle adapter/config errors and unify missing policy handling (#57) --- packages/mutiny_cli/src/mutiny_cli/run_cmd.py | 104 +++++++++--- .../mutiny_cli/src/mutiny_cli/test_cmd.py | 18 +- .../mutiny_core/src/mutiny_core/__init__.py | 2 + .../src/mutiny_core/campaign/engine.py | 23 ++- .../src/mutiny_core/policy/__init__.py | 2 + .../src/mutiny_core/policy/load.py | 6 +- tests/unit/test_campaign.py | 65 ++++++- tests/unit/test_mutiny_cli_run.py | 160 ++++++++++++++++++ tests/unit/test_policy_load.py | 9 + 9 files changed, 356 insertions(+), 33 deletions(-) create mode 100644 tests/unit/test_mutiny_cli_run.py diff --git a/packages/mutiny_cli/src/mutiny_cli/run_cmd.py b/packages/mutiny_cli/src/mutiny_cli/run_cmd.py index 116d013..8814920 100644 --- a/packages/mutiny_cli/src/mutiny_cli/run_cmd.py +++ b/packages/mutiny_cli/src/mutiny_cli/run_cmd.py @@ -12,12 +12,15 @@ import yaml +from pydantic import ValidationError + from mutiny_core import ( CampaignConfig, CampaignEngine, EventType, MutationEngine, MutinyEvent, + PolicyFileNotFoundError, PolicySet, PolicyValidationError, default_policy_seeds, @@ -84,13 +87,36 @@ def run_campaign( # Config api_url alone is never enough. want_hosted = bool(hosted or hosted_url) and not no_hosted - config = _load_mutiny_yaml(root / "mutiny.yaml") try: policy, policy_path = load_project_policy(root) + except PolicyFileNotFoundError as exc: + print( + f"error: {exc}\n" + " hint: run `mutiny init` first or add policy.yaml to your project", + file=sys.stderr, + ) + return 2 except PolicyValidationError as exc: print(f"error: invalid project policy — {exc}", file=sys.stderr) return 2 + config = _load_mutiny_yaml(root / "mutiny.yaml") + if config is None: + return 2 + + try: + core_cfg = CampaignConfig( + population_size=int(config.get("population_size", 8)), + max_generations=int(config.get("max_generations", 6)), + elite_count=int(config.get("elite_count", 2)), + max_turns=int(config.get("max_turns", 4)), + stop_on_first_violation=bool(config.get("stop_on_first_violation", True)), + wall_clock_seconds=config.get("wall_clock_seconds"), + ) + except (ValidationError, ValueError, TypeError) as exc: + print(f"error: invalid mutiny.yaml — {exc}", file=sys.stderr) + return 2 + if not attestation: print( "error: authorization attestation required " @@ -113,8 +139,8 @@ def run_campaign( f"{policy.target} · {len(policy.rules)} rule(s)" ) print( - f" search: N={config.get('population_size', 8)} " - f"Gmax={config.get('max_generations', 6)} " + f" search: N={core_cfg.population_size} " + f"Gmax={core_cfg.max_generations} " f"seed={config.get('rng_seed', 0)}" ) print(" safety: attestation ✓ · authorized testing only") @@ -142,9 +168,10 @@ def run_campaign( policy=policy, api_url=api_url, ui_url=ui_url, + core_cfg=core_cfg, ) - outcome = _run_local(root, config, policy) + outcome = _run_local(root, config, policy, core_cfg=core_cfg) return outcome.exit_code @@ -155,6 +182,8 @@ def _run_local_with_hosted_sync( policy: PolicySet, api_url: str, ui_url: str, + core_cfg: CampaignConfig | None = None, + adapter: Any | None = None, ) -> int: """Local Core campaign, then end-of-run Hosted ingest sync. @@ -167,7 +196,14 @@ def _run_local_with_hosted_sync( print(f" Hosted sync target: {api_url} (end-of-run ingest)") print() - outcome = _run_local(root, config, policy, campaign_id=campaign_id) + outcome = _run_local( + root, + config, + policy, + campaign_id=campaign_id, + core_cfg=core_cfg, + adapter=adapter, + ) if outcome.result is None: # Catastrophic local failure before a CampaignResult — still attempt @@ -225,6 +261,8 @@ def _run_local( policy: PolicySet, *, campaign_id: str | None = None, + core_cfg: CampaignConfig | None = None, + adapter: Any | None = None, ) -> LocalRunOutcome: cid = campaign_id or str(uuid.uuid4()) started_at = ( @@ -233,19 +271,36 @@ def _run_local( .isoformat() .replace("+00:00", "Z") ) - factory = load_adapter_factory(root) - adapter = factory() + if adapter is None: + try: + factory = load_adapter_factory(root) + adapter = factory() + if hasattr(adapter, "_get_agent"): + adapter._get_agent() + except Exception as exc: # noqa: BLE001 + print( + f"error: could not load adapter from .mutiny/adapter.py — {exc}\n" + " hint: run `mutiny init` or ensure create_adapter() imports cleanly", + file=sys.stderr, + ) + return LocalRunOutcome(exit_code=2, campaign_id=cid) + + if core_cfg is None: + try: + core_cfg = CampaignConfig( + population_size=int(config.get("population_size", 8)), + max_generations=int(config.get("max_generations", 6)), + elite_count=int(config.get("elite_count", 2)), + max_turns=int(config.get("max_turns", 4)), + stop_on_first_violation=bool(config.get("stop_on_first_violation", True)), + wall_clock_seconds=config.get("wall_clock_seconds"), + ) + except (ValidationError, ValueError, TypeError) as exc: + print(f"error: invalid mutiny.yaml — {exc}", file=sys.stderr) + return LocalRunOutcome(exit_code=2, campaign_id=cid) if campaign_id is None: print("→ Local campaign (Core + .mutiny/adapter.py)") - core_cfg = CampaignConfig( - population_size=int(config.get("population_size", 8)), - max_generations=int(config.get("max_generations", 6)), - elite_count=int(config.get("elite_count", 2)), - max_turns=int(config.get("max_turns", 4)), - stop_on_first_violation=bool(config.get("stop_on_first_violation", True)), - wall_clock_seconds=config.get("wall_clock_seconds"), - ) seeds = None if config.get("use_boundary_seeds", True): seeds = default_policy_seeds(policy) @@ -425,13 +480,22 @@ def _maybe_minimize_and_save( ) -def _load_mutiny_yaml(path: Path) -> dict[str, Any]: +def _load_mutiny_yaml(path: Path) -> dict[str, Any] | None: if not path.exists(): - print(f"error: missing {path}; run `mutiny init` first", file=sys.stderr) - raise SystemExit(2) - data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + print( + f"error: missing {path.name} in project\n" + " hint: run `mutiny init` first", + file=sys.stderr, + ) + return None + try: + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + except yaml.YAMLError as exc: + print(f"error: invalid {path.name} — {exc}", file=sys.stderr) + return None if not isinstance(data, dict): - raise SystemExit(f"error: {path} must be a mapping") + print(f"error: {path.name} must be a mapping", file=sys.stderr) + return None return data diff --git a/packages/mutiny_cli/src/mutiny_cli/test_cmd.py b/packages/mutiny_cli/src/mutiny_cli/test_cmd.py index 66a9316..29e1220 100644 --- a/packages/mutiny_cli/src/mutiny_cli/test_cmd.py +++ b/packages/mutiny_cli/src/mutiny_cli/test_cmd.py @@ -10,7 +10,7 @@ from pathlib import Path from typing import Any, Literal -from mutiny_core import PolicyValidationError, load_project_policy +from mutiny_core import PolicyFileNotFoundError, PolicyValidationError, load_project_policy from mutiny_core.redact import redact_secrets from mutiny_core.regress import RegressionTest, ReplayResult, replay_regression from mutiny_openai_agents.loader import ensure_project_on_path, load_adapter_factory @@ -73,16 +73,16 @@ def run_tests( try: policy, policy_path = load_project_policy(root) - except PolicyValidationError as exc: - print(f"error: invalid project policy — {exc}", file=sys.stderr) - return 2 - except FileNotFoundError as exc: + except PolicyFileNotFoundError as exc: print( f"error: {exc}\n" - " hint: run `mutiny init` or add policy.yaml / .mutiny/policy.yaml", + " hint: run `mutiny init` first or add policy.yaml to your project", file=sys.stderr, ) return 2 + except PolicyValidationError as exc: + print(f"error: invalid project policy — {exc}", file=sys.stderr) + return 2 cases = discover_regressions(root) if not cases: @@ -131,10 +131,12 @@ def run_tests( try: factory = load_adapter_factory(root) adapter = factory() + if hasattr(adapter, "_get_agent"): + adapter._get_agent() except Exception as exc: # noqa: BLE001 print( - f"error: could not load .mutiny/adapter.py — {exc}\n" - " hint: ensure create_adapter() imports cleanly", + f"error: could not load adapter from .mutiny/adapter.py — {exc}\n" + " hint: run `mutiny init` or ensure create_adapter() imports cleanly", file=sys.stderr, ) return 2 diff --git a/packages/mutiny_core/src/mutiny_core/__init__.py b/packages/mutiny_core/src/mutiny_core/__init__.py index 4564a58..fc502c7 100644 --- a/packages/mutiny_core/src/mutiny_core/__init__.py +++ b/packages/mutiny_core/src/mutiny_core/__init__.py @@ -42,6 +42,7 @@ ArgConstraint, PolicyEvaluator, PolicyEvidence, + PolicyFileNotFoundError, PolicyHit, PolicyRule, PolicySet, @@ -95,6 +96,7 @@ "MutinyEvent", "PolicyEvaluator", "PolicyEvidence", + "PolicyFileNotFoundError", "PolicyHit", "PolicyRule", "PolicySet", diff --git a/packages/mutiny_core/src/mutiny_core/campaign/engine.py b/packages/mutiny_core/src/mutiny_core/campaign/engine.py index 8d18ec7..9c16d98 100644 --- a/packages/mutiny_core/src/mutiny_core/campaign/engine.py +++ b/packages/mutiny_core/src/mutiny_core/campaign/engine.py @@ -175,13 +175,30 @@ def run(self) -> CampaignResult: except ToolsNotObservableError as exc: self._emit(EventType.CAMPAIGN_ERROR, {"error": str(exc)}) + best = max(all_scored, key=lambda c: c.fitness) if all_scored else None + violated = any(c.violated for c in all_scored) + gens = generations_done if generations_done > 0 else (1 if all_scored else 0) return CampaignResult( status="error", reason="tools_not_observable", - generations_completed=0, + generations_completed=gens, + candidates=all_scored, + best=best, + violated=violated, + events_emitted=self._events_emitted, + ) + except Exception as exc: # noqa: BLE001 + self._emit(EventType.CAMPAIGN_ERROR, {"error": str(exc)}) + best = max(all_scored, key=lambda c: c.fitness) if all_scored else None + violated = any(c.violated for c in all_scored) + gens = generations_done if generations_done > 0 else (1 if all_scored else 0) + return CampaignResult( + status="error", + reason=str(exc) or type(exc).__name__, + generations_completed=gens, candidates=all_scored, - best=None, - violated=False, + best=best, + violated=violated, events_emitted=self._events_emitted, ) diff --git a/packages/mutiny_core/src/mutiny_core/policy/__init__.py b/packages/mutiny_core/src/mutiny_core/policy/__init__.py index 721338e..e2dac11 100644 --- a/packages/mutiny_core/src/mutiny_core/policy/__init__.py +++ b/packages/mutiny_core/src/mutiny_core/policy/__init__.py @@ -7,6 +7,7 @@ ) from mutiny_core.policy.evaluator import PolicyEvaluator from mutiny_core.policy.load import ( + PolicyFileNotFoundError, PolicyValidationError, explain_rule, load_policy_file, @@ -29,6 +30,7 @@ "ArgConstraint", "PolicyEvaluator", "PolicyEvidence", + "PolicyFileNotFoundError", "PolicyHit", "PolicyRule", "PolicySet", diff --git a/packages/mutiny_core/src/mutiny_core/policy/load.py b/packages/mutiny_core/src/mutiny_core/policy/load.py index 8aa356f..1da4143 100644 --- a/packages/mutiny_core/src/mutiny_core/policy/load.py +++ b/packages/mutiny_core/src/mutiny_core/policy/load.py @@ -33,6 +33,10 @@ def __init__( super().__init__(" — ".join(parts)) +class PolicyFileNotFoundError(PolicyValidationError): + """Raised when no policy file exists in the project root.""" + + def resolve_policy_path(project_root: Path) -> Path: """Locate the project's policy file (``policy.yaml`` preferred — matches ``mutiny init``).""" root = project_root.resolve() @@ -41,7 +45,7 @@ def resolve_policy_path(project_root: Path) -> Path: if candidate.is_file(): return candidate expected = ", ".join(POLICY_FILENAMES) - raise PolicyValidationError( + raise PolicyFileNotFoundError( f"no policy file found in project (expected one of: {expected})", path=root, ) diff --git a/tests/unit/test_campaign.py b/tests/unit/test_campaign.py index cc32ad6..e029616 100644 --- a/tests/unit/test_campaign.py +++ b/tests/unit/test_campaign.py @@ -4,7 +4,7 @@ from pathlib import Path -from mutiny_core.adapter import TargetAdapter, execute_conversation +from mutiny_core.adapter import TargetAdapter, ToolsNotObservableError, execute_conversation from mutiny_core.campaign import CampaignConfig, CampaignEngine, CampaignResult from mutiny_core.events import EventType from mutiny_core.genome import AttackGenome, AttackMessage @@ -236,3 +236,66 @@ def test_demo_adapter_campaign_headless_smoke(): assert all(0.0 <= c.fitness <= 1.0 for c in result.candidates) # Report-friendly: may or may not find violation depending on search assert result.status in {"completed", "violation"} + + +class FlakyToolsAdapter(FakeRefundAdapter): + """Succeeds for a set number of steps, then raises ToolsNotObservableError.""" + + def __init__(self, succeed_turns: int = 1) -> None: + super().__init__() + self.succeed_turns = succeed_turns + + def step(self, session_id: str, user_message: str) -> AdapterTurnResult: + if self.calls >= self.succeed_turns: + raise ToolsNotObservableError("test tools dropped mid-campaign") + return super().step(session_id, user_message) + + +class CrashingAdapter(FakeRefundAdapter): + """Raises RuntimeError during execution.""" + + def step(self, session_id: str, user_message: str) -> AdapterTurnResult: + raise RuntimeError("adapter connection crashed") + + +def test_tools_not_observable_preserves_scored_candidate_metrics(): + config = CampaignConfig( + population_size=4, + max_generations=2, + elite_count=1, + stop_on_first_violation=False, + max_turns=1, + ) + # Allows 2 candidate turns to succeed, then fails on the 3rd + adapter = FlakyToolsAdapter(succeed_turns=2) + engine = CampaignEngine( + adapter=adapter, + policy_set=_policy(), + config=config, + rng_seed=42, + ) + result = engine.run() + assert result.status == "error" + assert result.reason == "tools_not_observable" + assert result.generations_completed > 0 + assert result.best is not None + assert len(result.candidates) >= 1 + assert result.best in result.candidates + + +def test_adapter_runtime_error_becomes_campaign_status_error(): + config = CampaignConfig( + population_size=2, + max_generations=1, + max_turns=1, + ) + adapter = CrashingAdapter() + engine = CampaignEngine( + adapter=adapter, + policy_set=_policy(), + config=config, + ) + result = engine.run() + assert result.status == "error" + assert "adapter connection crashed" in result.reason + diff --git a/tests/unit/test_mutiny_cli_run.py b/tests/unit/test_mutiny_cli_run.py new file mode 100644 index 0000000..5739c49 --- /dev/null +++ b/tests/unit/test_mutiny_cli_run.py @@ -0,0 +1,160 @@ +"""Unit tests for ``mutiny run`` CLI — crash prevention, validation, and error handling (Issue #57).""" + +from __future__ import annotations + +import shutil +from pathlib import Path + +import pytest +import yaml + +from mutiny_cli.init_cmd import run_init +from mutiny_cli.main import main + +ROOT = Path(__file__).resolve().parents[2] +DEMO_POLICY = ROOT / "examples" / "policies" / "demo_support.json" + +ADAPTER_MIN = '''\ +from demo_agent import DemoSupportAgent, InProcessDemoAdapter + +def create_adapter(): + return InProcessDemoAdapter(agent=DemoSupportAgent(enforce_refund_policy=False)) +''' + +ADAPTER_CRASHING = '''\ +from demo_agent import InProcessDemoAdapter + +class CrashingAdapter(InProcessDemoAdapter): + def step(self, session_id, user_message): + raise RuntimeError("simulated OpenAI SDK outage") + +def create_adapter(): + return CrashingAdapter() +''' + + +def _scaffold(tmp: Path) -> Path: + (tmp / ".mutiny").mkdir(parents=True, exist_ok=True) + shutil.copy(DEMO_POLICY, tmp / "policy.json") + (tmp / ".mutiny" / "adapter.py").write_text(ADAPTER_MIN, encoding="utf-8") + (tmp / "mutiny.yaml").write_text( + yaml.dump( + { + "population_size": 4, + "max_generations": 2, + "elite_count": 1, + } + ), + encoding="utf-8", + ) + return tmp + + +def test_empty_dir_mutiny_run_exits_2_with_clean_stderr( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + code = main(["run", "--path", str(tmp_path)]) + assert code == 2 + captured = capsys.readouterr() + assert "mutiny init" in captured.err + assert "Traceback" not in captured.err + assert "Traceback" not in captured.out + + +def test_empty_dir_mutiny_test_exits_2_with_clean_stderr( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + code = main(["test", "--path", str(tmp_path)]) + assert code == 2 + captured = capsys.readouterr() + assert "mutiny init" in captured.err + assert "invalid project policy" not in captured.err + assert "Traceback" not in captured.err + assert "Traceback" not in captured.out + + +def test_init_then_run_in_empty_project_exits_2_with_adapter_hint( + tmp_path: Path, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch +) -> None: + import sys + + monkeypatch.setattr( + sys, + "path", + [p for p in sys.path if "openai_support_agent" not in p], + ) + monkeypatch.delitem(sys.modules, "agent", raising=False) + + assert run_init(project_root=tmp_path) == 0 + capsys.readouterr() + + code = main(["run", "--path", str(tmp_path)]) + assert code == 2 + captured = capsys.readouterr() + assert "adapter" in captured.err.lower() + assert "init" in captured.err.lower() + assert "Traceback" not in captured.err + assert "ModuleNotFoundError" not in captured.err or "error: could not load adapter" in captured.err + + +def test_invalid_population_size_exits_2_before_search_banner( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + _scaffold(tmp_path) + (tmp_path / "mutiny.yaml").write_text( + yaml.dump({"population_size": 100}), encoding="utf-8" + ) + + code = main(["run", "--path", str(tmp_path)]) + assert code == 2 + captured = capsys.readouterr() + assert "search: N=" not in captured.out + assert "invalid mutiny.yaml" in captured.err + assert "Traceback" not in captured.err + + +def test_malformed_yaml_exits_2( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + _scaffold(tmp_path) + (tmp_path / "mutiny.yaml").write_text(": bad yaml ::", encoding="utf-8") + + code = main(["run", "--path", str(tmp_path)]) + assert code == 2 + captured = capsys.readouterr() + assert "invalid mutiny.yaml" in captured.err + assert "Traceback" not in captured.err + + +def test_missing_policy_with_existing_mutiny_yaml_exits_2_with_init_hint( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + (tmp_path / "mutiny.yaml").write_text("population_size: 4\n", encoding="utf-8") + + code_run = main(["run", "--path", str(tmp_path)]) + assert code_run == 2 + captured_run = capsys.readouterr() + assert "mutiny init" in captured_run.err + assert "invalid project policy" not in captured_run.err + assert "Traceback" not in captured_run.err + + code_test = main(["test", "--path", str(tmp_path)]) + assert code_test == 2 + captured_test = capsys.readouterr() + assert "mutiny init" in captured_test.err + assert "invalid project policy" not in captured_test.err + assert "Traceback" not in captured_test.err + + +def test_adapter_runtime_error_during_run_exits_1_without_traceback( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + _scaffold(tmp_path) + (tmp_path / ".mutiny" / "adapter.py").write_text(ADAPTER_CRASHING, encoding="utf-8") + + code = main(["run", "--path", str(tmp_path)]) + assert code == 1 + captured = capsys.readouterr() + assert "status=error" in captured.out + assert "simulated OpenAI SDK outage" in captured.out + assert "Traceback" not in captured.err diff --git a/tests/unit/test_policy_load.py b/tests/unit/test_policy_load.py index b8d7f18..3c1629b 100644 --- a/tests/unit/test_policy_load.py +++ b/tests/unit/test_policy_load.py @@ -8,6 +8,7 @@ import yaml from mutiny_core import ( + PolicyFileNotFoundError, PolicySet, PolicyValidationError, explain_rule, @@ -112,3 +113,11 @@ def test_regression_provenance_includes_policy_version(): policy_set=policy, ) assert art.provenance.policy_version == policy.version + + +def test_resolve_missing_policy_raises_policy_file_not_found(tmp_path: Path): + with pytest.raises(PolicyFileNotFoundError) as exc_info: + resolve_policy_path(tmp_path) + assert issubclass(PolicyFileNotFoundError, PolicyValidationError) + assert isinstance(exc_info.value, PolicyValidationError) + assert "no policy file found" in str(exc_info.value).lower() From c2db0309d3c1b90c1681186d05183918aa34d678 Mon Sep 17 00:00:00 2001 From: littfed Date: Fri, 25 Sep 2026 08:55:11 +0500 Subject: [PATCH 2/2] fix: bind generations_done on seed error and skip minimize on status=error --- packages/mutiny_cli/src/mutiny_cli/run_cmd.py | 17 ++++--- .../src/mutiny_core/campaign/engine.py | 29 +++++++----- tests/unit/test_campaign.py | 24 +++++++++- tests/unit/test_mutiny_cli_run.py | 44 +++++++++++++++++++ 4 files changed, 94 insertions(+), 20 deletions(-) diff --git a/packages/mutiny_cli/src/mutiny_cli/run_cmd.py b/packages/mutiny_cli/src/mutiny_cli/run_cmd.py index 8814920..78a86b0 100644 --- a/packages/mutiny_cli/src/mutiny_cli/run_cmd.py +++ b/packages/mutiny_cli/src/mutiny_cli/run_cmd.py @@ -356,13 +356,16 @@ def on_event(ev: MutinyEvent) -> None: regression_artifact: dict[str, Any] | None = None minimize_body: dict[str, Any] | None = None - if result.violated and result.best is not None: - saved = _maybe_minimize_and_save( - root, adapter, policy, result, campaign_id=cid, events=collected - ) - if saved is not None: - regression_id, regression_path, regression_artifact, minimize_body = saved - else: + if result.status != "error" and result.violated and result.best is not None: + try: + saved = _maybe_minimize_and_save( + root, adapter, policy, result, campaign_id=cid, events=collected + ) + if saved is not None: + regression_id, regression_path, regression_artifact, minimize_body = saved + except Exception as exc: # noqa: BLE001 + print(f"warning: minimization failed — {exc}", file=sys.stderr) + elif result.status != "error": print(" No violation this run — try different rng_seed or more generations.") print() diff --git a/packages/mutiny_core/src/mutiny_core/campaign/engine.py b/packages/mutiny_core/src/mutiny_core/campaign/engine.py index 9c16d98..9a4bd67 100644 --- a/packages/mutiny_core/src/mutiny_core/campaign/engine.py +++ b/packages/mutiny_core/src/mutiny_core/campaign/engine.py @@ -78,18 +78,17 @@ def run(self) -> CampaignResult: cfg = self.config started = time.monotonic() all_scored: list[ScoredCandidate] = [] - self._emit( - EventType.CAMPAIGN_STARTED, - { - "population_size": cfg.population_size, - "max_generations": cfg.max_generations, - "elite_count": cfg.elite_count, - }, - ) - + generations_done = 0 try: + self._emit( + EventType.CAMPAIGN_STARTED, + { + "population_size": cfg.population_size, + "max_generations": cfg.max_generations, + "elite_count": cfg.elite_count, + }, + ) population = self._initial_population() - generations_done = 0 for gen in range(cfg.max_generations): if self._budget_exceeded(started): @@ -174,7 +173,10 @@ def run(self) -> CampaignResult: ) except ToolsNotObservableError as exc: - self._emit(EventType.CAMPAIGN_ERROR, {"error": str(exc)}) + try: + self._emit(EventType.CAMPAIGN_ERROR, {"error": str(exc)}) + except Exception: # noqa: BLE001, S110 + pass best = max(all_scored, key=lambda c: c.fitness) if all_scored else None violated = any(c.violated for c in all_scored) gens = generations_done if generations_done > 0 else (1 if all_scored else 0) @@ -188,7 +190,10 @@ def run(self) -> CampaignResult: events_emitted=self._events_emitted, ) except Exception as exc: # noqa: BLE001 - self._emit(EventType.CAMPAIGN_ERROR, {"error": str(exc)}) + try: + self._emit(EventType.CAMPAIGN_ERROR, {"error": str(exc)}) + except Exception: # noqa: BLE001, S110 + pass best = max(all_scored, key=lambda c: c.fitness) if all_scored else None violated = any(c.violated for c in all_scored) gens = generations_done if generations_done > 0 else (1 if all_scored else 0) diff --git a/tests/unit/test_campaign.py b/tests/unit/test_campaign.py index e029616..73f9611 100644 --- a/tests/unit/test_campaign.py +++ b/tests/unit/test_campaign.py @@ -6,7 +6,7 @@ from mutiny_core.adapter import TargetAdapter, ToolsNotObservableError, execute_conversation from mutiny_core.campaign import CampaignConfig, CampaignEngine, CampaignResult -from mutiny_core.events import EventType +from mutiny_core.events import EventType, MutinyEvent from mutiny_core.genome import AttackGenome, AttackMessage from mutiny_core.policy import PolicySet from mutiny_core.trace import AdapterTurnResult, ToolCall @@ -299,3 +299,25 @@ def test_adapter_runtime_error_becomes_campaign_status_error(): assert result.status == "error" assert "adapter connection crashed" in result.reason + +def test_event_handler_raise_during_seed_emit_handled_as_status_error(): + config = CampaignConfig( + population_size=2, + max_generations=1, + max_turns=1, + ) + + def broken_handler(ev: MutinyEvent) -> None: + if ev.type == EventType.CANDIDATE_CREATED: + raise RuntimeError("event sink disconnected during seed emit") + + engine = CampaignEngine( + adapter=FakeRefundAdapter(), + policy_set=_policy(), + config=config, + on_event=broken_handler, + ) + result = engine.run() + assert result.status == "error" + assert "event sink disconnected during seed emit" in result.reason + assert result.generations_completed == 0 diff --git a/tests/unit/test_mutiny_cli_run.py b/tests/unit/test_mutiny_cli_run.py index 5739c49..054e721 100644 --- a/tests/unit/test_mutiny_cli_run.py +++ b/tests/unit/test_mutiny_cli_run.py @@ -32,6 +32,24 @@ def create_adapter(): return CrashingAdapter() ''' +ADAPTER_CRASH_AFTER_HIT = '''\ +from demo_agent import DemoSupportAgent, InProcessDemoAdapter + +class HitThenCrashAdapter(InProcessDemoAdapter): + def __init__(self): + super().__init__(agent=DemoSupportAgent(enforce_refund_policy=False)) + self._seen_sessions = set() + + def step(self, session_id, user_message): + self._seen_sessions.add(session_id) + if len(self._seen_sessions) > 1: + raise RuntimeError("sdk exploded after first candidate") + return super().step(session_id, user_message) + +def create_adapter(): + return HitThenCrashAdapter() +''' + def _scaffold(tmp: Path) -> Path: (tmp / ".mutiny").mkdir(parents=True, exist_ok=True) @@ -158,3 +176,29 @@ def test_adapter_runtime_error_during_run_exits_1_without_traceback( assert "status=error" in captured.out assert "simulated OpenAI SDK outage" in captured.out assert "Traceback" not in captured.err + + +def test_stop_on_first_violation_false_and_crash_after_hit_skips_minimize_and_exits_1( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + _scaffold(tmp_path) + (tmp_path / "mutiny.yaml").write_text( + yaml.dump( + { + "population_size": 4, + "max_generations": 2, + "elite_count": 1, + "stop_on_first_violation": False, + } + ), + encoding="utf-8", + ) + (tmp_path / ".mutiny" / "adapter.py").write_text(ADAPTER_CRASH_AFTER_HIT, encoding="utf-8") + + code = main(["run", "--path", str(tmp_path)]) + assert code == 1 + captured = capsys.readouterr() + assert "status=error" in captured.out + assert "sdk exploded after first candidate" in captured.out + assert "minimizing exploit" not in captured.out + assert "Traceback" not in captured.err