Skip to content

fix(cli,core): handle adapter/config errors and unify missing policy handling (#57) #91

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
littfed wants to merge 2 commits into
base: main
Choose a base branch
from
Open

fix(cli,core): handle adapter/config errors and unify missing policy handling (#57) #91

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4803de2
fix(cli,core): handle adapter/config errors and unify missing policy …
littfed Sep 24, 2026
c2db030
fix: bind generations_done on seed error and skip minimize on status=…
littfed Sep 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 94 additions & 27 deletions packages/mutiny_cli/src/mutiny_cli/run_cmd.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,15 @@

import yaml

from pydantic import ValidationError

from mutiny_core import (
CampaignConfig,
CampaignEngine,
EventType,
MutationEngine,
MutinyEvent,
PolicyFileNotFoundError,
PolicySet,
PolicyValidationError,
default_policy_seeds,
Expand Down Expand Up @@ -84,13 +87,36 @@ def run_campaign(
# Config api_url alone is never enough.
want_hosted = bool(hosted or hosted_url) and not no_hosted

config = _load_mutiny_yaml(root / "mutiny.yaml")
try:
policy, policy_path = load_project_policy(root)
except PolicyFileNotFoundError as exc:
print(
f"error: {exc}\n"
" hint: run `mutiny init` first or add policy.yaml to your project",
file=sys.stderr,
)
return 2
except PolicyValidationError as exc:
print(f"error: invalid project policy — {exc}", file=sys.stderr)
return 2

config = _load_mutiny_yaml(root / "mutiny.yaml")
if config is None:
return 2

try:
core_cfg = CampaignConfig(
population_size=int(config.get("population_size", 8)),
max_generations=int(config.get("max_generations", 6)),
elite_count=int(config.get("elite_count", 2)),
max_turns=int(config.get("max_turns", 4)),
stop_on_first_violation=bool(config.get("stop_on_first_violation", True)),
wall_clock_seconds=config.get("wall_clock_seconds"),
)
except (ValidationError, ValueError, TypeError) as exc:
print(f"error: invalid mutiny.yaml — {exc}", file=sys.stderr)
return 2

if not attestation:
print(
"error: authorization attestation required "
Expand All @@ -113,8 +139,8 @@ def run_campaign(
f"{policy.target} · {len(policy.rules)} rule(s)"
)
print(
f" search: N={config.get('population_size', 8)} "
f"Gmax={config.get('max_generations', 6)} "
f" search: N={core_cfg.population_size} "
f"Gmax={core_cfg.max_generations} "
f"seed={config.get('rng_seed', 0)}"
)
print(" safety: attestation ✓ · authorized testing only")
Expand Down Expand Up @@ -142,9 +168,10 @@ def run_campaign(
policy=policy,
api_url=api_url,
ui_url=ui_url,
core_cfg=core_cfg,
)

outcome = _run_local(root, config, policy)
outcome = _run_local(root, config, policy, core_cfg=core_cfg)
return outcome.exit_code


Expand All @@ -155,6 +182,8 @@ def _run_local_with_hosted_sync(
policy: PolicySet,
api_url: str,
ui_url: str,
core_cfg: CampaignConfig | None = None,
adapter: Any | None = None,
) -> int:
"""Local Core campaign, then end-of-run Hosted ingest sync.

Expand All @@ -167,7 +196,14 @@ def _run_local_with_hosted_sync(
print(f" Hosted sync target: {api_url} (end-of-run ingest)")
print()

outcome = _run_local(root, config, policy, campaign_id=campaign_id)
outcome = _run_local(
root,
config,
policy,
campaign_id=campaign_id,
core_cfg=core_cfg,
adapter=adapter,
)

if outcome.result is None:
# Catastrophic local failure before a CampaignResult — still attempt
Expand Down Expand Up @@ -225,6 +261,8 @@ def _run_local(
policy: PolicySet,
*,
campaign_id: str | None = None,
core_cfg: CampaignConfig | None = None,
adapter: Any | None = None,
) -> LocalRunOutcome:
cid = campaign_id or str(uuid.uuid4())
started_at = (
Expand All @@ -233,19 +271,36 @@ def _run_local(
.isoformat()
.replace("+00:00", "Z")
)
factory = load_adapter_factory(root)
adapter = factory()
if adapter is None:
try:
factory = load_adapter_factory(root)
adapter = factory()
if hasattr(adapter, "_get_agent"):
adapter._get_agent()

@CodewithJha CodewithJha Sep 25, 2026

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Non-blocking: this is what makes mutiny init && mutiny run exit 2 instead of falling into Core status=error on lazy agent_ref load. Fine for Adapter #1.

If you touch this file for the minimize guard, a one-line comment that this is an eager resolve for OpenAIAgentsAdapter (not part of TargetAdapter) would help the next reader. Do not expand into #27 AGENT_REF copy.

except Exception as exc: # noqa: BLE001
print(
f"error: could not load adapter from .mutiny/adapter.py — {exc}\n"
" hint: run `mutiny init` or ensure create_adapter() imports cleanly",
file=sys.stderr,
)
return LocalRunOutcome(exit_code=2, campaign_id=cid)

if core_cfg is None:
try:
core_cfg = CampaignConfig(
population_size=int(config.get("population_size", 8)),
max_generations=int(config.get("max_generations", 6)),
elite_count=int(config.get("elite_count", 2)),
max_turns=int(config.get("max_turns", 4)),
stop_on_first_violation=bool(config.get("stop_on_first_violation", True)),
wall_clock_seconds=config.get("wall_clock_seconds"),
)
except (ValidationError, ValueError, TypeError) as exc:
print(f"error: invalid mutiny.yaml — {exc}", file=sys.stderr)
return LocalRunOutcome(exit_code=2, campaign_id=cid)

if campaign_id is None:
print("→ Local campaign (Core + .mutiny/adapter.py)")
core_cfg = CampaignConfig(
population_size=int(config.get("population_size", 8)),
max_generations=int(config.get("max_generations", 6)),
elite_count=int(config.get("elite_count", 2)),
max_turns=int(config.get("max_turns", 4)),
stop_on_first_violation=bool(config.get("stop_on_first_violation", True)),
wall_clock_seconds=config.get("wall_clock_seconds"),
)
seeds = None
if config.get("use_boundary_seeds", True):
seeds = default_policy_seeds(policy)
Expand Down Expand Up @@ -301,13 +356,16 @@ def on_event(ev: MutinyEvent) -> None:
regression_artifact: dict[str, Any] | None = None
minimize_body: dict[str, Any] | None = None

if result.violated and result.best is not None:
saved = _maybe_minimize_and_save(
root, adapter, policy, result, campaign_id=cid, events=collected
)
if saved is not None:
regression_id, regression_path, regression_artifact, minimize_body = saved
else:
if result.status != "error" and result.violated and result.best is not None:
try:
saved = _maybe_minimize_and_save(
root, adapter, policy, result, campaign_id=cid, events=collected
)
if saved is not None:
regression_id, regression_path, regression_artifact, minimize_body = saved
except Exception as exc: # noqa: BLE001
print(f"warning: minimization failed — {exc}", file=sys.stderr)
elif result.status != "error":
print(" No violation this run — try different rng_seed or more generations.")

print()
Expand Down Expand Up @@ -425,13 +483,22 @@ def _maybe_minimize_and_save(
)


def _load_mutiny_yaml(path: Path) -> dict[str, Any]:
def _load_mutiny_yaml(path: Path) -> dict[str, Any] | None:
if not path.exists():
print(f"error: missing {path}; run `mutiny init` first", file=sys.stderr)
raise SystemExit(2)
data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
print(
f"error: missing {path.name} in project\n"
" hint: run `mutiny init` first",
file=sys.stderr,
)
return None
try:
data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
except yaml.YAMLError as exc:
print(f"error: invalid {path.name} — {exc}", file=sys.stderr)
return None
if not isinstance(data, dict):
raise SystemExit(f"error: {path} must be a mapping")
print(f"error: {path.name} must be a mapping", file=sys.stderr)
return None
return data


Expand Down
18 changes: 10 additions & 8 deletions packages/mutiny_cli/src/mutiny_cli/test_cmd.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from pathlib import Path
from typing import Any, Literal

from mutiny_core import PolicyValidationError, load_project_policy
from mutiny_core import PolicyFileNotFoundError, PolicyValidationError, load_project_policy
from mutiny_core.redact import redact_secrets
from mutiny_core.regress import RegressionTest, ReplayResult, replay_regression
from mutiny_openai_agents.loader import ensure_project_on_path, load_adapter_factory
Expand Down Expand Up @@ -73,16 +73,16 @@ def run_tests(

try:
policy, policy_path = load_project_policy(root)
except PolicyValidationError as exc:
print(f"error: invalid project policy — {exc}", file=sys.stderr)
return 2
except FileNotFoundError as exc:
except PolicyFileNotFoundError as exc:
print(
f"error: {exc}\n"
" hint: run `mutiny init` or add policy.yaml / .mutiny/policy.yaml",
" hint: run `mutiny init` first or add policy.yaml to your project",
file=sys.stderr,
)
return 2
except PolicyValidationError as exc:
print(f"error: invalid project policy — {exc}", file=sys.stderr)
return 2

cases = discover_regressions(root)
if not cases:
Expand Down Expand Up @@ -131,10 +131,12 @@ def run_tests(
try:
factory = load_adapter_factory(root)
adapter = factory()
if hasattr(adapter, "_get_agent"):
adapter._get_agent()
except Exception as exc: # noqa: BLE001
print(
f"error: could not load .mutiny/adapter.py — {exc}\n"
" hint: ensure create_adapter() imports cleanly",
f"error: could not load adapter from .mutiny/adapter.py — {exc}\n"
" hint: run `mutiny init` or ensure create_adapter() imports cleanly",
file=sys.stderr,
)
return 2
Expand Down
2 changes: 2 additions & 0 deletions packages/mutiny_core/src/mutiny_core/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
ArgConstraint,
PolicyEvaluator,
PolicyEvidence,
PolicyFileNotFoundError,
PolicyHit,
PolicyRule,
PolicySet,
Expand Down Expand Up @@ -95,6 +96,7 @@
"MutinyEvent",
"PolicyEvaluator",
"PolicyEvidence",
"PolicyFileNotFoundError",
"PolicyHit",
"PolicyRule",
"PolicySet",
Expand Down
50 changes: 36 additions & 14 deletions packages/mutiny_core/src/mutiny_core/campaign/engine.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,18 +78,17 @@ def run(self) -> CampaignResult:
cfg = self.config
started = time.monotonic()
all_scored: list[ScoredCandidate] = []
self._emit(
EventType.CAMPAIGN_STARTED,
{
"population_size": cfg.population_size,
"max_generations": cfg.max_generations,
"elite_count": cfg.elite_count,
},
)

generations_done = 0
try:
self._emit(
EventType.CAMPAIGN_STARTED,
{
"population_size": cfg.population_size,
"max_generations": cfg.max_generations,
"elite_count": cfg.elite_count,
},
)
population = self._initial_population()
generations_done = 0

for gen in range(cfg.max_generations):
if self._budget_exceeded(started):
Expand Down Expand Up @@ -174,14 +173,37 @@ def run(self) -> CampaignResult:
)

except ToolsNotObservableError as exc:
self._emit(EventType.CAMPAIGN_ERROR, {"error": str(exc)})
try:
self._emit(EventType.CAMPAIGN_ERROR, {"error": str(exc)})
except Exception: # noqa: BLE001, S110
pass
best = max(all_scored, key=lambda c: c.fitness) if all_scored else None
violated = any(c.violated for c in all_scored)
gens = generations_done if generations_done > 0 else (1 if all_scored else 0)
return CampaignResult(
status="error",
reason="tools_not_observable",
generations_completed=0,
generations_completed=gens,
candidates=all_scored,
best=None,
violated=False,
best=best,
violated=violated,
events_emitted=self._events_emitted,
)
except Exception as exc: # noqa: BLE001
try:
self._emit(EventType.CAMPAIGN_ERROR, {"error": str(exc)})
except Exception: # noqa: BLE001, S110
pass
best = max(all_scored, key=lambda c: c.fitness) if all_scored else None
violated = any(c.violated for c in all_scored)
gens = generations_done if generations_done > 0 else (1 if all_scored else 0)

@CodewithJha CodewithJha Sep 25, 2026

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Blocking: generations_done is assigned after _initial_population(). If seed construction or a CANDIDATE_CREATED on_event raises, this line (and the ToolsNotObservableError twin above) raises UnboundLocalError and swallows the original exception.

I reproduced it with an on_event that raises on EventType.CANDIDATE_CREATED — engine.run() escaped with UnboundLocalError: cannot access local variable 'generations_done'.

Fix: generations_done = 0 before the try (the 1 if all_scored else 0 fallback can stay). Please add a unit test for this path.

return CampaignResult(
status="error",
reason=str(exc) or type(exc).__name__,
generations_completed=gens,
candidates=all_scored,
best=best,

@CodewithJha CodewithJha Sep 25, 2026

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Preserving best / violated here is the right MUT-014 call for metrics. It does change CLI behavior: _run_local still does if result.violated and result.best is not None: _maybe_minimize_and_save(...) with no status != "error" guard.

Reproduced with stop_on_first_violation: false and an adapter that scores a refund violation then raises RuntimeError on the next candidate:

✓ Local finished: status=error ... violated=True candidates=1
  minimizing exploit …
RuntimeError: sdk exploded after first candidate

That reintroduces the traceback this PR is supposed to kill (default stop_on_first_violation: true hides it). In run_cmd.py, skip minimize when result.status == "error" — the adapter is not trustworthy. A CLI unit test for that path would lock it.

violated=violated,
events_emitted=self._events_emitted,
)

Expand Down
2 changes: 2 additions & 0 deletions packages/mutiny_core/src/mutiny_core/policy/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
)
from mutiny_core.policy.evaluator import PolicyEvaluator
from mutiny_core.policy.load import (
PolicyFileNotFoundError,
PolicyValidationError,
explain_rule,
load_policy_file,
Expand All @@ -29,6 +30,7 @@
"ArgConstraint",
"PolicyEvaluator",
"PolicyEvidence",
"PolicyFileNotFoundError",
"PolicyHit",
"PolicyRule",
"PolicySet",
Expand Down
6 changes: 5 additions & 1 deletion packages/mutiny_core/src/mutiny_core/policy/load.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,10 @@ def __init__(
super().__init__(" — ".join(parts))


class PolicyFileNotFoundError(PolicyValidationError):
"""Raised when no policy file exists in the project root."""


def resolve_policy_path(project_root: Path) -> Path:
"""Locate the project's policy file (``policy.yaml`` preferred — matches ``mutiny init``)."""
root = project_root.resolve()
Expand All @@ -41,7 +45,7 @@ def resolve_policy_path(project_root: Path) -> Path:
if candidate.is_file():
return candidate
expected = ", ".join(POLICY_FILENAMES)
raise PolicyValidationError(
raise PolicyFileNotFoundError(
f"no policy file found in project (expected one of: {expected})",
path=root,
)
Expand Down
Loading
Loading