diff --git a/.secrets.baseline b/.secrets.baseline index 26121825..358260d5 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -1537,21 +1537,21 @@ "filename": "tests/test_ai_setup.py", "hashed_secret": "06c3796a342a1f1172aa7a907d3c0a5da02a053a", "is_verified": false, - "line_number": 1092 + "line_number": 1098 }, { "type": "Secret Keyword", "filename": "tests/test_ai_setup.py", "hashed_secret": "6b1196369d5cd92be55dbdd72ef619957b6e6624", "is_verified": false, - "line_number": 1096 + "line_number": 1102 }, { "type": "Secret Keyword", "filename": "tests/test_ai_setup.py", "hashed_secret": "9ea9d5acda505555f09a5d2203f7078f1a3b6363", "is_verified": false, - "line_number": 1101 + "line_number": 1107 } ], "tests/test_ai_setup_import_keys.py": [ @@ -1860,70 +1860,70 @@ "filename": "tests/test_forge_copilot_runtime.py", "hashed_secret": "3fa437025c8f9e1c6f0f5547173acee9c2e22c3a", "is_verified": false, - "line_number": 190 + "line_number": 191 }, { "type": "Secret Keyword", "filename": "tests/test_forge_copilot_runtime.py", "hashed_secret": "c734e47630dda71619c696d88381f06f7511bd78", "is_verified": false, - "line_number": 191 + "line_number": 192 }, { "type": "Secret Keyword", "filename": "tests/test_forge_copilot_runtime.py", "hashed_secret": "a47110e348a3063541fb1f1f640d635d457181a0", "is_verified": false, - "line_number": 207 + "line_number": 208 }, { "type": "Secret Keyword", "filename": "tests/test_forge_copilot_runtime.py", "hashed_secret": "fd9a45bc0b07706d849cf85021ecf9123fa83d82", "is_verified": false, - "line_number": 266 + "line_number": 268 }, { "type": "Secret Keyword", "filename": "tests/test_forge_copilot_runtime.py", "hashed_secret": "e9a5f12a8ecbb3eb46eca5096b5c52aa5e7c9fdd", "is_verified": false, - "line_number": 299 + "line_number": 301 }, { "type": "Secret Keyword", "filename": "tests/test_forge_copilot_runtime.py", "hashed_secret": "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3", "is_verified": false, - "line_number": 338 + "line_number": 340 }, { "type": "Secret Keyword", "filename": "tests/test_forge_copilot_runtime.py", "hashed_secret": "8079944bb2ba4b2d6eb02908a6de57cb57c6f39a", "is_verified": false, - "line_number": 363 + "line_number": 365 }, { "type": "Secret Keyword", "filename": "tests/test_forge_copilot_runtime.py", "hashed_secret": "56e1d57b8db262b08bc73c60ed08d8c92e59503f", "is_verified": false, - "line_number": 434 + "line_number": 436 }, { "type": "Secret Keyword", "filename": "tests/test_forge_copilot_runtime.py", "hashed_secret": "a62f2225bf70bfaccbc7f1ef2a397836717377de", "is_verified": false, - "line_number": 1037 + "line_number": 1039 }, { "type": "Secret Keyword", "filename": "tests/test_forge_copilot_runtime.py", "hashed_secret": "63d185ab613a8f4ba8a3f2669f49e8df82bff53f", "is_verified": false, - "line_number": 1175 + "line_number": 1177 } ], "tests/test_forge_copilot_scenarios.py": [ @@ -2317,5 +2317,5 @@ } ] }, - "generated_at": "2026-11-05T14:05:52Z" + "generated_at": "2026-11-11T22:30:08Z" } diff --git a/CHANGELOG.md b/CHANGELOG.md index de6f36b3..c1272622 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,49 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 granted. The contract-tests local action grants its outputs the same way. The DuckDB acquisition runner's landing got this fix in 0.19.0 (#699). Found on the demo lab on 0.18.2, still in 0.28.0. +- **Local Ollama works through every automatic path, and `fluid ai test` probes the provider it + names.** With no endpoint set, `LlmConfig.endpoint` carried a telemetry label, + `litellm:///`. Ollama calls sent it to litellm as `api_base`, so no call + reached the daemon, and `fluid ai test` posted to it and failed with "AI test network error + ... (UnsupportedProtocol)" for every litellm provider. The endpoint now holds only a real + address: the one `--llm-endpoint` or `FLUID_LLM_ENDPOINT` states, else Ollama's `OLLAMA_HOST`, + else none (litellm's own default). A bare `OLLAMA_HOST=myhost` means `http://myhost:11434`, as + in Ollama's own client. The routing model's calls keep a stated endpoint. `fluid ai test` + sends its probe through the call `fluid forge` makes and exits 4 when the provider says the + model does not exist. `fluid doctor` shows "Provider default" instead of the label. +- **`fluid forge --agent` with `--llm-provider`, `--llm-model`, `--llm-endpoint` or `--context` + runs AI mode.** It dropped those flags, wrote a TODO placeholder contract and reported + `exit_code 0`, which an IDE agent reads as success. It now authors with forge's AI, + non-interactively, and a failed generation exits non-zero. AI-mode runs now emit + `forge.contract_written`, which the stream promised but only blank mode sent. `--agent` with + no AI flag still writes the blank contract. +- **A Claude Code login problem is reported as one.** `claude -p` reports a failure on stdout, + as a JSON result with `is_error: true`, and leaves stderr empty, so forge showed + "claude-code exited with code 1:" and no reason. Forge now reads that result: a login problem + says the agent "is installed but not authenticated" with the sign-in hint, any other failure + shows the agent's own message, and `is_error: true` is a failure even at exit code 0. +- **`fluid doctor` reports an installed coding agent as ready.** With `FLUID_LLM_PROVIDER` set to + `claude-code`, `codex`, `cursor` or `kiro`, readiness said "No API key found"; it now checks + that the agent's CLI is on PATH, and says so when it is not. Each CLI keeps its own sign-in. + +### Changed + +- **A non-interactive run with no LLM configured authors with an installed coding agent.** When + nothing names one (no `--llm-provider`, `--llm-endpoint` or `--llm-model`, no `FLUID_LLM_*`, + `OLLAMA_HOST` or provider key variable, no keyring key, no `fluid ai setup` provider) and stdin + is not a terminal or `--non-interactive` is set, forge uses a running local Ollama first, so + local data stays local, else the first of `claude`, `codex`, `cursor-agent` and `kiro-cli` on + PATH, and logs which one and how to override it. It works on the base install: the agents need + no litellm. Interactive runs still only offer the agent, and `--agent-loop` never picks one + (the agents have no tool use). +- **Forge runs `claude -p` apart from your own Claude Code setup.** Each call replaces Claude + Code's agent system prompt with forge's (`--system-prompt`) and turns off tools, MCP servers, + your settings (hooks, plugins, `CLAUDE.md`) and session persistence (`--tools ""`, + `--strict-mcp-config`, `--setting-sources=`, `--no-session-persistence`), and drops + `CLAUDECODE`, as Anthropic's Agent SDK does. Each flag is passed only when the installed + `claude --help` lists it. Claude Code authentication that lives in a setting (an + `apiKeyHelper`, or Bedrock or Vertex variables in `settings.json`) must come from the shell + environment instead. ## [0.28.0] — 2026-11-06 diff --git a/fluid_build/cli/_ai_setup_test.py b/fluid_build/cli/_ai_setup_test.py index 553635d9..7386e932 100644 --- a/fluid_build/cli/_ai_setup_test.py +++ b/fluid_build/cli/_ai_setup_test.py @@ -191,11 +191,6 @@ def _validate_ai_test_endpoint(provider_name: str, endpoint: str) -> Optional[st parsed = urlsplit(raw) if parsed.username or parsed.password: return "AI test endpoint URLs must not embed credentials." - # litellm-routed providers (the default) use a ``litellm://`` sentinel - # that's not a real URL — litellm owns auth + transport, so there's no - # plaintext-credential risk to validate against. Accept the sentinel. - if parsed.scheme == "litellm": - return None if parsed.scheme not in {"http", "https"} or not parsed.netloc: return "AI test endpoint must be an absolute HTTP(S) URL." host = (parsed.hostname or "").lower() @@ -253,10 +248,13 @@ def _build_for_provider( host = saved_for_provider.get("ollama_host") if host and not os.environ.get("OLLAMA_HOST"): os.environ["OLLAMA_HOST"] = _sanitize_ollama_host(str(host)) + # Only an operator-supplied endpoint is validated; with none, the + # call goes through litellm, which owns the address. + if endpoint_override: + endpoint_error = _validate_ai_test_endpoint(pname, endpoint_override) + if endpoint_error: + return None, endpoint_error endpoint = endpoint_override or provider.default_endpoint(model, dict(os.environ)) - endpoint_error = _validate_ai_test_endpoint(pname, endpoint) - if endpoint_error: - return None, endpoint_error return ( LlmConfig( provider=pname, @@ -272,9 +270,10 @@ def _build_for_provider( return None, f"No API key found for {PROVIDER_DISPLAY_NAMES.get(pname, pname)}." endpoint = endpoint_override or saved_for_provider.get("endpoint") config = _make_cloud_config(pname, api_key, model=model, endpoint=endpoint) - endpoint_error = _validate_ai_test_endpoint(pname, config.endpoint) - if endpoint_error: - return None, endpoint_error + if config.endpoint: + endpoint_error = _validate_ai_test_endpoint(pname, config.endpoint) + if endpoint_error: + return None, endpoint_error return ( config, None, @@ -391,8 +390,48 @@ def _check_ai_test_model_availability( return "available", available +def _is_litellm_backed(provider: Any) -> bool: + from fluid_build.llm.litellm_backend import LiteLLMProvider + + return isinstance(provider, LiteLLMProvider) + + +def _run_litellm_smoke_call(provider: Any, config: LlmConfig) -> tuple[str, dict, int]: + """The diagnostic call for a litellm-backed provider: the call forge itself makes. + + litellm owns the address, auth and transport, so there is no URL to POST to. + """ + from fluid_build.llm.providers import get_cumulative_token_usage + + cap = ( + _AI_TEST_GEMINI_OUTPUT_TOKENS + if config.provider == "gemini" + else _AI_TEST_DEFAULT_OUTPUT_TOKENS + ) + before = get_cumulative_token_usage() + started = time.perf_counter() + text = provider.invoke_blocking( + config, + _AI_TEST_SYSTEM_PROMPT, + _AI_TEST_USER_PROMPT, + extra_payload={"max_tokens": cap, "timeout": config.timeout_seconds}, + ) + latency_ms = int((time.perf_counter() - started) * 1000) + after = get_cumulative_token_usage() + usage = {key: after[key] - before[key] for key in after} + if "FLUID_OK" not in (text or ""): + raise CopilotGenerationError( + "ai_test_unexpected_response", + f"AI test response from {config.provider} was not the expected diagnostic token.", + suggestions=["Try again, or choose a different model with `fluid ai setup`."], + ) + return "FLUID_OK", usage, latency_ms + + def _run_ai_smoke_call(provider: Any, config: LlmConfig) -> tuple[str, dict, int]: """Issue the diagnostic call. Returns (text, usage, latency_ms).""" + if _is_litellm_backed(provider): + return _run_litellm_smoke_call(provider, config) headers, payload = _with_freeform_ai_test_payload(provider, config) started = time.perf_counter() try: @@ -454,6 +493,12 @@ def _classify_ai_test_error(error_code: str, status_code: Optional[int]) -> int: 0 OK / 2 config / 3 auth / 4 resource / 5 network. See AGENTS.md for the contract. """ + if error_code in {"copilot_llm_auth_failed", "copilot_llm_permission_denied"}: + return _AI_TEST_EXIT_AUTH + if error_code == "copilot_litellm_request_failed": + # A provider "not found" (a model it does not have) is a resource problem; + # timeouts, connection errors and the rest stay network. + return _AI_TEST_EXIT_RESOURCE if status_code == 404 else _AI_TEST_EXIT_NETWORK if error_code in {"ai_test_request_failed", "ai_test_model_preflight_failed"}: if status_code is None: return _AI_TEST_EXIT_NETWORK @@ -471,7 +516,10 @@ def _http_status_from_cause(exc: CopilotGenerationError) -> Optional[int]: cause = exc.__cause__ if isinstance(cause, httpx.HTTPStatusError): return cause.response.status_code - return None + # litellm's exceptions (the original error, chained by ``_translate_litellm_exception``) + # carry the provider's HTTP status. + status = getattr(cause, "status_code", None) + return status if isinstance(status, int) else None def _new_ai_test_report( @@ -653,7 +701,7 @@ def run_ai_test(console: Any, args: Any) -> tuple[int, dict]: table.add_column("Result", style="green") table.add_row("Provider", label) table.add_row("Model", _safe_ai_test_display(config.model)) - table.add_row("Endpoint", _safe_ai_test_display(endpoint)) + table.add_row("Endpoint", _safe_ai_test_display(endpoint) or "provider default") table.add_row( "Model availability", ( @@ -673,7 +721,7 @@ def run_ai_test(console: Any, args: Any) -> tuple[int, dict]: from fluid_build.cli.console import cprint cprint(f"AI Provider Test: ready ({label}, {_safe_ai_test_display(config.model)})") - cprint(f" Endpoint: {_safe_ai_test_display(endpoint)}") + cprint(f" Endpoint: {_safe_ai_test_display(endpoint) or 'provider default'}") cprint(f" Model availability: {availability}") cprint(f" Live call: {text}") cprint(f" Latency: {latency_text}") diff --git a/fluid_build/cli/doctor.py b/fluid_build/cli/doctor.py index 7a438978..f9871ee8 100644 --- a/fluid_build/cli/doctor.py +++ b/fluid_build/cli/doctor.py @@ -1411,7 +1411,10 @@ def _print_copilot_readiness(readiness: LlmReadinessCheck, verbose: bool = False """Render the copilot readiness summary without leaking secrets.""" status_text = "✅ Ready" if readiness.ready else "⚠️ Setup needed" auth_text = "Configured" if readiness.auth_available else "Missing" - endpoint_text = readiness.endpoint or "Not configured" + # No endpoint of ours is not "not configured": litellm picks the provider's own. + endpoint_text = readiness.endpoint or ( + "Provider default" if readiness.provider else "Not configured" + ) if RICH_AVAILABLE: console = Console() diff --git a/fluid_build/cli/forge.py b/fluid_build/cli/forge.py index 8acbc5f2..c2ab4624 100644 --- a/fluid_build/cli/forge.py +++ b/fluid_build/cli/forge.py @@ -568,10 +568,13 @@ def register(subparsers: argparse._SubParsersAction): "FLUID_FORGE_NO_{PICKER,PREVIEW,WELCOME,STREAMING_PREVIEW}=1 + " "emits JSON-Lines progress events (forge.start, forge.done, " "forge.contract_written, forge.cost) to stdout so the IDE's " - "shell tool can parse them. Defaults to --blank when no mode " - "flag is provided (avoids dropping into the interactive picker " - "the agent cannot navigate). Use this from inside any agentic " - "IDE (Kiro, Cursor, Claude Code, Cline)." + "shell tool can parse them. Any of --llm-provider, --llm-model, " + "--llm-endpoint or --context selects AI mode (strictly " + "non-interactive; a failed generation exits non-zero). With no " + "mode flag and no AI flag it defaults to --blank (avoids " + "dropping into the interactive picker the agent cannot " + "navigate). Use this from inside any agentic IDE (Kiro, " + "Cursor, Claude Code, Cline)." ), ) parser.add_argument( @@ -1282,6 +1285,12 @@ def _setup_agent_mode(args: Any) -> None: os.environ.setdefault(var, "1") args.yes = True + # An explicit AI flag is the agent choosing AI mode: no blank forcing, + # and the AI path must never block on a prompt (``non_interactive`` is + # what the rest of forge reads to skip inline setup and confirmations). + if _agent_chose_ai(args): + args.non_interactive = True + # If the agent didn't pick a mode, default to --blank: it's the # only mode that succeeds without an LLM round-trip, so it's the # safest fallback when invoked under ``--agent --data-product-type X`` @@ -1295,12 +1304,21 @@ def _setup_agent_mode(args: Any) -> None: bool(getattr(args, "refine", None)), bool(getattr(args, "from_product", None)), bool(getattr(args, "from_product_list", None)), + _agent_chose_ai(args), ] ) if no_mode: args.blank = True +def _agent_chose_ai(args: Any) -> bool: + """True when the agent passed any explicit AI flag (selects AI mode).""" + return any( + bool(getattr(args, flag, None)) + for flag in ("llm_provider", "llm_model", "llm_endpoint", "context") + ) + + def _agent_run_id() -> str: """Short, stable run-id for the JSONL event stream (12 hex chars).""" import uuid @@ -1431,9 +1449,10 @@ def _emit_forge_plan(args: Any, contract_path: Optional[Path]) -> None: "gate-check, then `fluid bundle && fluid plan` for the pipeline." ), note=( - "You (the IDE's agent) are the LLM. Fill in these fields using " - "your own Edit tool; do NOT shell-run `fluid forge --ai` (that " - "would need a second LLM API key)." + "You (the IDE's agent) are the LLM: fill in these fields using " + "your own Edit tool. Running `fluid forge --agent --llm-provider " + "` instead has forge's own AI author the contract, which " + "needs a second LLM." ), ) @@ -2026,6 +2045,10 @@ def load_context( ) +# ``diff_snapshots`` action -> the agent JSONL event's ``action``. +_AGENT_EVENT_ACTIONS = {"create": "created", "update": "modified"} + + def _write_forge_receipt( *, flow: str, @@ -2084,8 +2107,11 @@ def _write_forge_receipt( ) # Agent-mode JSONL: one event per changed file. The agent's # caller can highlight new contracts in its UI without - # parsing the receipt. - if is_agent and entry.action in ("created", "modified"): + # parsing the receipt. ``diff_snapshots`` says "create" / + # "update"; the events use the blank path's "created" / + # "modified" so every mode speaks one vocabulary. + event_action = _AGENT_EVENT_ACTIONS.get(entry.action) + if is_agent and event_action: _emit_agent_jsonl( ( "forge.contract_written" @@ -2093,7 +2119,7 @@ def _write_forge_receipt( else "forge.file_written" ), path=str(entry.path), - action=entry.action, + action=event_action, size=entry.size, ) diff --git a/fluid_build/cli/forge_modes.py b/fluid_build/cli/forge_modes.py index fd1a23dc..eab6baf7 100644 --- a/fluid_build/cli/forge_modes.py +++ b/fluid_build/cli/forge_modes.py @@ -306,6 +306,8 @@ def _forge_build_copilot_options( Extracted verbatim from ``run_ai_copilot_mode`` (behaviour-preserving) to shrink that god-function; the returned dict is mutated downstream. """ + from fluid_build.llm.providers import agent_loop_requested + return { "llm_provider": get_cli_arg_fn(args, "llm_provider"), "llm_model": get_cli_arg_fn(args, "llm_model"), @@ -322,6 +324,9 @@ def _forge_build_copilot_options( "fragment_first": bool(get_cli_arg_fn(args, "fragments", False)), "no_fragments": bool(get_cli_arg_fn(args, "no_fragments", False)), "no_generate": bool(get_cli_arg_fn(args, "no_generate", False)), + # The provider is resolved from these options, and the agent loop (by flag + # or FLUID_COPILOT_AGENT_LOOP) must never auto-pick a coding agent. + "agent_loop": agent_loop_requested(get_cli_arg_fn(args, "agent_loop", False)), # Pre-write preview UX (invariants I1/I4/I5): # auto_yes — skip the [Y/n] prompt, panel still renders # show_work — stream reasoning/tool calls live as the agent runs @@ -1034,12 +1039,9 @@ def _forge_project_creation_core( context = rc.context copilot_options = rc.copilot_options perf_stats = rc.perf_stats - # Parsed as a boolean, not by presence: ``FLUID_COPILOT_AGENT_LOOP=0`` is - # the documented way to turn the loop off, and a bare ``bool()`` of the - # string turned it on. - use_agent_loop = bool(get_cli_arg_fn(args, "agent_loop", False)) or os.environ.get( - "FLUID_COPILOT_AGENT_LOOP", "" - ).strip().lower() in {"1", "true", "yes", "on"} + from fluid_build.llm.providers import agent_loop_requested + + use_agent_loop = agent_loop_requested(get_cli_arg_fn(args, "agent_loop", False)) if scaffold_template: success_result = copilot.create_project( diff --git a/fluid_build/credentials/litellm_endpoints.py b/fluid_build/credentials/litellm_endpoints.py index 4514ae9f..2b50a1d6 100644 --- a/fluid_build/credentials/litellm_endpoints.py +++ b/fluid_build/credentials/litellm_endpoints.py @@ -13,9 +13,9 @@ project's own setting and follows the name-shape rule. Vendored because importing litellm takes seconds and dotenv hydration runs on -every command. Generated from the litellm 1.83.14, 1.97.0, 1.105.0, 1.106.0 -and 1.108.0 wheels (FLUID's floor, the last release Python 3.10 can import, and -the newest), as the union of: +every command. Generated from the litellm 1.83.14, 1.97.0, 1.105.0, 1.106.0, +1.108.0 and 1.109.0 wheels (FLUID's floor, the last release Python 3.10 can +import, and the newest), as the union of: * ``_API_BASE`` for every ``litellm.provider_list`` entry (``-`` read as ``_``); @@ -45,9 +45,11 @@ { "A2A_AGENT_API_BASE", "A2A_API_BASE", + "ABLITERATION_API_BASE", "AGENTCORE_GATEWAY_URL", "AI21_API_BASE", "AI21_CHAT_API_BASE", + "AIAND_API_BASE", "AIHUBMIX_API_BASE", "AIML_API_BASE", "AIOHTTP_OPENAI_API_BASE", @@ -75,6 +77,7 @@ "BEDROCK_MANTLE_API_BASE", "BESPOKE_API_BASE", "BFL_API_BASE", + "BIGMODEL_API_BASE", "BLACK_FOREST_LABS_API_BASE", "BRAVE_API_BASE", "BYTEZ_API_BASE", @@ -83,6 +86,7 @@ "CHATGPT_API_BASE", "CHUTES_API_BASE", "CLARIFAI_API_BASE", + "CLF_AI_GATEWAY_API_BASE", "CLOUDFLARE_API_BASE", "CODESTRAL_API_BASE", "COGNITION_API_BASE", @@ -120,6 +124,7 @@ "ELEVENLABS_API_BASE", "EMPIRIOLABS_API_BASE", "EMPOWER_API_BASE", + "EU_JEV_API_BASE", "EXA_API_BASE", "FAL_AI_API_BASE", "FAL_AI_QUEUE_API_BASE", @@ -148,6 +153,7 @@ "HEROKU_API_BASE", "HF_API_BASE", "HOSTED_VLLM_API_BASE", + "HOSTINGER_ROUTER_API_BASE", "HUGGINGFACE_API_BASE", "HUMANLOOP_API_BASE", "HYPERBOLIC_API_BASE", @@ -155,6 +161,7 @@ "INFERENCEAPIS_API_BASE", "INFINITY_API_BASE", "JINA_AI_API_BASE", + "KYMA_API_BASE", "LAMBDA_AI_API_BASE", "LAMBDA_API_BASE", "LANGFLOW_API_BASE", @@ -182,6 +189,7 @@ "MONGODB_API_BASE", "MOONSHOT_API_BASE", "MORPH_API_BASE", + "NACE_API_BASE", "NADIR_API_BASE", "NANO_GPT_API_BASE", "NEBIUS_API_BASE", @@ -199,7 +207,10 @@ "OPENAI_API_BASE", "OPENAI_CHATGPT_API_BASE", "OPENAI_LIKE_API_BASE", + "OPENCODE_GO_API_BASE", + "OPENINFER_API_BASE", "OPENROUTER_API_BASE", + "OPENSEARCH_API_BASE", "OPEN_SANDBOX_API_BASE", "OVHCLOUD_API_BASE", "PARALLEL_AI_API_BASE", @@ -223,6 +234,7 @@ "RAGFLOW_API_BASE", "RECRAFT_API_BASE", "REDUCTO_API_BASE", + "REFUSELESS_API_BASE", "REKA_API_BASE", "REPLICATE_API_BASE", "RUNWAYML_API_BASE", @@ -245,6 +257,7 @@ "SONIOX_API_BASE", "STABILITY_API_BASE", "STRANDS_DECIDER_API_BASE", + "STREAMLAKE_API_BASE", "SYNTHETIC_API_BASE", "TAVILY_API_BASE", "TENCENT_ANTHROPIC_API_BASE", @@ -283,6 +296,7 @@ "XINFERENCE_API_BASE", "YOUCOM_API_BASE", "ZAI_API_BASE", + "ZEROGPU_API_BASE", } ) diff --git a/fluid_build/llm/coding_agent.py b/fluid_build/llm/coding_agent.py index 7b57bbc4..254ad9eb 100644 --- a/fluid_build/llm/coding_agent.py +++ b/fluid_build/llm/coding_agent.py @@ -47,6 +47,7 @@ import tempfile import threading from dataclasses import dataclass +from functools import lru_cache from typing import Any, Dict, Iterator, List, Optional, Tuple from fluid_build.llm.providers import ( @@ -120,7 +121,7 @@ class AgentSpec: model_flag: str # "--model" | "-m" | "" (passthrough only when explicit) output_mode: str # "claude-json" | "result-json" | "raw" cost_field: Optional[str] # top-level JSON cost field, e.g. "total_cost_usd" - auth_markers: Tuple[str, ...] # stderr substrings -> not_authenticated + auth_markers: Tuple[str, ...] # stderr / reported-error substrings -> not_authenticated login_hint: str # actionable suggestion for missing-key / not-authenticated # Endpoint variables the agent reads, and the key variables it sends there # instead of its own stored login. A project dotenv file's endpoint reaches @@ -128,6 +129,16 @@ class AgentSpec: # ``_agent_environment``. endpoint_env: Tuple[str, ...] = () endpoint_key_env: Tuple[str, ...] = () + # Isolation from the user's own agent config (hooks, plugins, MCP servers, + # CLAUDE.md) and from the agent's default system prompt. Each entry is the + # argv tokens of one flag, passed only when the installed CLI's ``--help`` + # lists the flag (``_cli_help``), so older versions keep working. + isolation_args: Tuple[Tuple[str, ...], ...] = () + # Flag that REPLACES the default system prompt; ``system_prompt_flag`` (which + # only appends) is the fallback when the CLI does not list it. + system_prompt_replace_flag: str = "" + # Variables removed from the child environment. + drop_env: Tuple[str, ...] = () _AGENT_SPECS: Dict[str, AgentSpec] = { @@ -150,6 +161,9 @@ class AgentSpec: auth_markers=( "not logged in", "please run /login", + "/login", + "failed to authenticate", + "oauth session expired", "oauth", "unauthorized", "invalid api key", @@ -158,6 +172,21 @@ class AgentSpec: login_hint="Run `claude` once and sign in (subscription), or set ANTHROPIC_API_KEY.", endpoint_env=("ANTHROPIC_BASE_URL",), endpoint_key_env=("ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_API_KEY"), + # Borrowed from claude-agent-sdk-python (MIT) @ 3cf3446b0f, + # src/claude_agent_sdk/_internal/transport/subprocess_cli.py + # ``_build_command``: replace the system prompt, no tools, strict MCP, + # no filesystem settings. An EMPTY setting-sources list must be the + # single token ``--setting-sources=``: a separate "" argument misparses + # the flags after it (SDK #778). The SDK also drops CLAUDECODE from the + # child env (SDK #732). + isolation_args=( + ("--tools", ""), + ("--strict-mcp-config",), + ("--setting-sources=",), + ("--no-session-persistence",), + ), + system_prompt_replace_flag="--system-prompt", + drop_env=("CLAUDECODE",), ), "codex": AgentSpec( name="codex", @@ -295,6 +324,44 @@ def _run_agent( return proc.returncode, proc.stdout, proc.stderr +_HELP_TIMEOUT_SECONDS = 10 + + +def _run_help(binary: str) -> str: + """`` --help`` text, or ``""`` when it cannot be read. + + The second subprocess seam (tests monkeypatch it, like ``_run_agent``). + """ + try: + proc = subprocess.run( # noqa: S603 — list argv, no shell, fixed binary set + [binary, "--help"], + capture_output=True, + text=True, + timeout=_HELP_TIMEOUT_SECONDS, + cwd=tempfile.gettempdir(), + check=False, + ) + except (OSError, subprocess.SubprocessError): + return "" + return f"{proc.stdout or ''}\n{proc.stderr or ''}" + + +@lru_cache(maxsize=None) +def _cli_help(binary: str) -> str: + """Help text of ``binary``, probed once per process.""" + return _run_help(binary) + + +def _lists_flag(help_text: str, flag: str) -> bool: + """True when ``flag`` appears in ``help_text`` as a whole flag name. + + ``--tools`` must not match ``--allowedTools``, nor ``--system-prompt`` match + ``--append-system-prompt``. + """ + name = flag.split("=", 1)[0] + return re.search(rf"(? Dict[str, str]: CODEX_API_KEY in place of its login. """ env = dict(os.environ) + for key in spec.drop_env: + env.pop(key, None) if not spec.endpoint_env: return env from fluid_build.credentials.dotenv_store import project_supplied @@ -417,6 +486,27 @@ def _extract_envelope(spec: AgentSpec, stdout: str) -> str: return text # "raw" — stdout is the final message verbatim +def _reported_error(spec: AgentSpec, stdout: str) -> Optional[str]: + """The agent's own failure report from its JSON stdout, else ``None``. + + ``claude -p --output-format json`` reports a failure as a result object with + ``is_error: true`` on STDOUT (its ``subtype`` can still say ``success`` and + stderr stays empty). Returns the ``result`` text (``""`` when absent) so the + caller treats it as a failure even at exit code 0, before the envelope parser + sees it. + """ + if spec.output_mode not in ("claude-json", "result-json"): + return None + try: + data = json.loads(_ANSI_RE.sub("", stdout or "")) + except (ValueError, TypeError): + return None + if not isinstance(data, dict) or data.get("is_error") is not True: + return None + result = data.get("result") + return result.strip() if isinstance(result, str) else "" + + def _record_agent_usage_and_cost(spec: AgentSpec, stdout: str) -> None: """Stash Claude's per-call USD + bump the shared usage counters. @@ -528,9 +618,19 @@ def invoke_blocking( if spec.model_flag and config.model and config.model != spec.name: argv += [spec.model_flag, str(config.model)] + help_text = _cli_help(binary) if spec.isolation_args else "" + for tokens in spec.isolation_args: + if _lists_flag(help_text, tokens[0]): + argv += tokens + combined_prompt = user_prompt if spec.system_prompt_mode == "flag": - argv += [spec.system_prompt_flag, system_prompt] + system_flag = spec.system_prompt_flag + if spec.system_prompt_replace_flag and _lists_flag( + help_text, spec.system_prompt_replace_flag + ): + system_flag = spec.system_prompt_replace_flag + argv += [system_flag, system_prompt] else: combined_prompt = f"{system_prompt}{_PROMPT_DELIM}{user_prompt}" @@ -581,17 +681,20 @@ def invoke_blocking( except OSError: # pragma: no cover — best-effort cleanup pass - if rc != 0: - lowered = (stderr or "").lower() + reported = _reported_error(spec, stdout) + if rc != 0 or reported is not None: + reason = "\n".join(part for part in (reported, (stderr or "").strip()) if part) + lowered = reason.lower() if any(marker in lowered for marker in spec.auth_markers): raise CopilotGenerationError( "coding_agent_not_authenticated", f"{spec.name} is installed but not authenticated.", suggestions=[spec.login_hint], ) + outcome = f"exited with code {rc}" if rc != 0 else "reported an error" raise CopilotGenerationError( "coding_agent_failed", - f"{spec.name} exited with code {rc}: {_redact(stderr)[:500]}", + f"{spec.name} {outcome}: {_redact(reason)[:500]}", suggestions=[ f"Run `{spec.binary}` manually to confirm it works.", "Or fall back to an API key / Ollama.", diff --git a/fluid_build/llm/litellm_backend.py b/fluid_build/llm/litellm_backend.py index f4ffee96..d4a8da48 100644 --- a/fluid_build/llm/litellm_backend.py +++ b/fluid_build/llm/litellm_backend.py @@ -18,12 +18,14 @@ from __future__ import annotations +import ipaddress import json import logging import os import sys import threading import time +import urllib.parse from typing import Any, Dict, Iterator, List, Mapping, Optional, Tuple from fluid_build.llm.providers import ( @@ -255,6 +257,43 @@ def _is_anthropic_model(model_id: str) -> bool: return "claude" in lower +def _ollama_base_url(env: Mapping[str, str]) -> str: + """``OLLAMA_HOST`` as a URL, or ``""`` when unset. + + Ollama's own variable accepts forms litellm and ``urllib`` cannot dial as + they are (a bare ``host``, ``host:port``, ``:port``), so it is normalised + with the convention of ollama-python's ``_parse_host`` (MIT, + github.com/ollama/ollama-python, ``ollama/_client.py``): no scheme means + ``http`` on the default port 11434; an explicit ``http://`` or ``https://`` + without a port means port 80 or 443; a path is kept. Both reads of + ``OLLAMA_HOST`` in this module go through here, so they agree. + """ + host = (env.get("OLLAMA_HOST") or "").strip() + if not host: + return "" + port = 11434 + scheme, _, hostport = host.partition("://") + if not hostport: + scheme, hostport = "http", host + elif scheme == "http": + port = 80 + elif scheme == "https": + port = 443 + try: + split = urllib.parse.urlsplit(f"{scheme}://{hostport}") + name = split.hostname or "127.0.0.1" + port = split.port or port + except ValueError: # e.g. a non-numeric port: leave it to fail visibly at the dial + return f"{scheme}://{hostport}".rstrip("/") + try: + if isinstance(ipaddress.ip_address(name), ipaddress.IPv6Address): + name = f"[{name}]" # urlsplit strips the brackets + except ValueError: + pass # a hostname, not an IP literal + path = split.path.strip("/") + return f"{scheme}://{name}:{port}" + (f"/{path}" if path else "") + + # --------------------------------------------------------------------------- # Adapter # --------------------------------------------------------------------------- @@ -283,8 +322,27 @@ def __init__(self, provider_name: str, *, default_model: str = ""): # ----- LlmProvider abstract API ------------------------------------ def default_endpoint(self, model: str, env: Mapping[str, str]) -> str: - """litellm owns auth/endpoints; return a sentinel for telemetry.""" - return f"litellm://{self.name}/{model}" + """The endpoint the operator pointed us at; empty when litellm picks it. + + litellm owns auth and transport, so a hosted provider has no endpoint of + ours to report and ``LlmConfig.endpoint`` stays empty. Ollama is the one + exception: its own ``OLLAMA_HOST`` is a real daemon address, so it is + reported. Never a placeholder URL: every consumer of ``config.endpoint`` + treats a non-empty value as an address to dial. + """ + return _ollama_base_url(env) if self.name == "ollama" else "" + + def _apply_ollama_api_base(self, payload: Dict[str, Any], config: LlmConfig) -> None: + """Point an Ollama call at an explicit endpoint, else ``OLLAMA_HOST``. + + With neither set, ``api_base`` is left out and litellm uses its own + default (``http://localhost:11434``, ``litellm/llms/ollama/common_utils.py``). + """ + if self.name != "ollama": + return + api_base = config.endpoint or _ollama_base_url(os.environ) + if api_base: + payload["api_base"] = api_base def build_request( self, config: LlmConfig, system_prompt: str, user_prompt: str @@ -312,11 +370,7 @@ def build_request( payload["max_tokens"] = int(max_tokens) if config.api_key: payload["api_key"] = config.api_key - # Ollama: litellm needs the daemon endpoint as api_base. - if self.name == "ollama": - payload["api_base"] = config.endpoint or os.environ.get( - "OLLAMA_HOST", "http://localhost:11434" - ) + self._apply_ollama_api_base(payload, config) _maybe_inject_cache_control(payload, model_id) return ({}, payload) @@ -502,10 +556,7 @@ def build_tool_request( } if config.api_key: payload["api_key"] = config.api_key - if self.name == "ollama": - payload["api_base"] = config.endpoint or os.environ.get( - "OLLAMA_HOST", "http://localhost:11434" - ) + self._apply_ollama_api_base(payload, config) _maybe_inject_cache_control(payload, model_id) return ("litellm://internal", {}, payload) @@ -734,7 +785,7 @@ def _list_ollama_models(env: Mapping[str, str]) -> Optional[List[str]]: reachable so preflight degrades to "unavailable" rather than falsely rejecting models. """ - base = env.get("OLLAMA_HOST", "http://localhost:11434").rstrip("/") + base = _ollama_base_url(env) or "http://localhost:11434" try: import urllib.error import urllib.request diff --git a/fluid_build/llm/provider_resolve.py b/fluid_build/llm/provider_resolve.py index 2b13110a..04691d7d 100644 --- a/fluid_build/llm/provider_resolve.py +++ b/fluid_build/llm/provider_resolve.py @@ -28,6 +28,8 @@ from __future__ import annotations import logging +import shutil +import sys from typing import Mapping, Optional LOG = logging.getLogger("fluid.cli.llm_providers.resolve") @@ -91,8 +93,121 @@ def _infer_provider_from_explicit_keys(env: Mapping[str, str]) -> Optional[str]: return None -def _infer_provider_from_ambient(env: Mapping[str, str]) -> Optional[str]: - """Last-resort detection: Ollama already running on localhost. +def _is_non_interactive(non_interactive: bool = False) -> bool: + """True when the run cannot answer a prompt: ``--non-interactive`` or no TTY on stdin.""" + if non_interactive: + return True + try: + return not sys.stdin.isatty() + except (AttributeError, ValueError): # no stdin, or a closed one + return True + + +# Variables that mean the operator configured an LLM: an API key, or a stated +# endpoint, model or Ollama host. A coding agent is never auto-picked over any of them, so a +# run never sends its prompt somewhere other than the destination it was given. +_CONFIG_ENV_VARS = ( + "FLUID_LLM_API_KEY", + "FLUID_LLM_ENDPOINT", + "FLUID_LLM_MODEL", + "FLUID_LLM_ROUTING_ENDPOINT", + "FLUID_LLM_ROUTING_MODEL", + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + "GEMINI_API_KEY", + "GOOGLE_API_KEY", + "OLLAMA_HOST", +) + + +def _configured_by_env(env: Mapping[str, str]) -> bool: + """Whether the environment configures an LLM (see :data:`_CONFIG_ENV_VARS`). + + Every provider's key in the host's ``PROVIDER_ENV_VARS`` counts too. + """ + provider_keys = tuple(getattr(_host(), "PROVIDER_ENV_VARS", {}).values()) + return any(env.get(var) for var in _CONFIG_ENV_VARS + provider_keys) + + +def _operator_configured_a_provider(env: Mapping[str, str]) -> bool: + """Whether anything but ambient discovery already names a provider. + + An API-key, endpoint or model env var (even several keys, which + :func:`_infer_provider_from_explicit_keys` refuses to choose between), a + keyring key, or a saved ``~/.fluid`` provider. A coding agent is never + auto-picked over any of these. + """ + if _configured_by_env(env): + return True + keyring_fn = getattr(_host(), "_get_api_key_from_keyring", _get_api_key_from_keyring) + if any(keyring_fn(name) for name in ("openai", "anthropic", "gemini")): + return True + try: + from fluid_build._ai_config_shared import load_ai_config + + saved = load_ai_config() + except Exception: # noqa: BLE001 — detection must never hard-fail + return False + return bool(saved and saved.get("provider")) + + +def _first_coding_agent_on_path(env: Mapping[str, str]) -> Optional[str]: + """The first installed coding-agent CLI, in ``CODING_AGENT_NAMES`` order.""" + from fluid_build.llm.coding_agent import _AGENT_SPECS, CODING_AGENT_NAMES + + # Search the PATH of the environment we were given, never the process's: + # an environment without PATH finds nothing, as it would for a child process. + path = env.get("PATH", "") + for name in CODING_AGENT_NAMES: + if shutil.which(_AGENT_SPECS[name].binary, path=path): + return name + return None + + +def _lean_keyless_autopick( + env: Mapping[str, str], *, non_interactive: bool = False, allow_agents: bool = True +) -> Optional[str]: + """The coding agent a run without litellm auto-picks, else ``None``. + + The ambient rung of the ladder without its keyring read, so the litellm + preflight stays cheap: a non-interactive run, no configuring variable, no + saved provider, an agent CLI on PATH, and no local Ollama (it needs litellm, + and a running one is never silently swapped for a cloud agent). + ``allow_agents=False`` (the agent loop, or a stated endpoint or model) + disables it. + """ + if not allow_agents or not _is_non_interactive(non_interactive): + return None + if _configured_by_env(env): + return None + agent = _first_coding_agent_on_path(env) + if agent is None: + return None + try: + from fluid_build._ai_config_shared import load_ai_config + + saved = load_ai_config() + except Exception: # noqa: BLE001 — detection must never hard-fail + saved = None + if saved and saved.get("provider"): + return None + detect_fn = getattr(_host(), "detect_ollama_available", None) + if detect_fn is not None and detect_fn(env): + return None + return agent + + +def _infer_provider_from_ambient( + env: Mapping[str, str], *, non_interactive: bool = False, allow_agents: bool = True +) -> Optional[str]: + """Last-resort detection, shared by the readiness ladder and the real selection. + + 1. Ollama already running on localhost: data stays local. + 2. Non-interactive runs only, and only when nothing is configured: the first + coding-agent CLI on PATH. Interactive runs keep "offer, don't force" + (the ``fluid ai setup`` inline offer). ``allow_agents=False`` skips this + rung: for the agent loop, which needs tool use, and when the operator + stated an endpoint or model. Only call this AFTER the saved config and the keyring have been consulted — ambient services must not override an explicit @@ -107,6 +222,12 @@ def _infer_provider_from_ambient(env: Mapping[str, str]) -> Optional[str]: detect_fn = getattr(_host(), "detect_ollama_available", None) if detect_fn is not None and detect_fn(env): return "ollama" + if ( + allow_agents + and _is_non_interactive(non_interactive) + and not _operator_configured_a_provider(env) + ): + return _first_coding_agent_on_path(env) return None @@ -114,14 +235,18 @@ def _infer_provider_from_ambient(env: Mapping[str, str]) -> Optional[str]: # imported ``_infer_provider_from_env``. The new code paths inside # :func:`check_llm_readiness` use the split functions above so the # saved config gets a chance to win over ambient Ollama. -def _infer_provider_from_env(env: Mapping[str, str]) -> Optional[str]: +def _infer_provider_from_env( + env: Mapping[str, str], *, non_interactive: bool = False, allow_agents: bool = True +) -> Optional[str]: explicit = _infer_provider_from_explicit_keys(env) if explicit: return explicit keyring_match = _infer_provider_from_keyring() if keyring_match: return keyring_match - return _infer_provider_from_ambient(env) + return _infer_provider_from_ambient( + env, non_interactive=non_interactive, allow_agents=allow_agents + ) def _infer_provider_from_keyring() -> Optional[str]: diff --git a/fluid_build/llm/providers.py b/fluid_build/llm/providers.py index 921804f1..7cbe98d6 100644 --- a/fluid_build/llm/providers.py +++ b/fluid_build/llm/providers.py @@ -30,6 +30,7 @@ "normalize_llm_provider_name", "query_ollama_models", "reset_llm_caches", + "agent_loop_requested", "preflight_llm_provider", "resolve_llm_config", "resolve_model_name", @@ -45,6 +46,7 @@ import logging import os import re +import shutil import threading from abc import ABC, abstractmethod from dataclasses import dataclass, field @@ -239,6 +241,10 @@ class LlmConfig: # contract.fluid.yaml into the workspace with its own tools). Ignored by # every other provider. agent_mode: str = "envelope" + # True when the operator stated ``endpoint`` (``--llm-endpoint`` or + # ``FLUID_LLM_ENDPOINT``) rather than the provider deriving it. A stated + # endpoint is where every call goes, so routing calls keep it. + endpoint_explicit: bool = False def for_routing(self) -> "LlmConfig": """Return a shallow copy configured for the routing model. @@ -253,7 +259,7 @@ def for_routing(self) -> "LlmConfig": overrides: Dict[str, Any] = {"model": self.routing_model} if self.routing_endpoint: overrides["endpoint"] = self.routing_endpoint - else: + elif not self.endpoint_explicit: # Re-derive endpoint for the routing model using the same # provider's default-endpoint logic. provider = BUILTIN_LLM_PROVIDERS.get(self.provider) @@ -771,15 +777,61 @@ def has_llm_api_key( {"ollama", "mcp-sampling", "mcp_sampling", "claude-code", "codex", "cursor", "kiro"} ) +# The keyless providers that do not go through litellm: the call is made by the +# MCP client or by an installed agent CLI, so a missing litellm extra is no obstacle. +_NON_LITELLM_PROVIDERS: frozenset = _KEYLESS_PROVIDERS - {"ollama"} + def _provider_name_from(args: Any, env: Mapping[str, str]) -> str: """The provider name the flags and environment select (flag, env, key sniff, default).""" + return ( + _explicit_provider_name(args, env) + or _infer_provider_from_env( + env, + non_interactive=bool(getattr(args, "non_interactive", False)), + allow_agents=_agents_allowed(args, env), + ) + or "gemini" + ) + + +def agent_loop_requested(flag: Any, environ: Optional[Mapping[str, str]] = None) -> bool: + """Whether this run asks for the multi-turn agent loop. + + ``flag`` is the ``--agent-loop`` value (callers read it their own way, e.g. + the CLI's ``get_cli_arg``), or ``FLUID_COPILOT_AGENT_LOOP`` set to a truthy + value. It is parsed as a boolean, not by presence: + ``FLUID_COPILOT_AGENT_LOOP=0`` is the documented way to turn the loop off. + """ + env = os.environ if environ is None else environ + return bool(flag) or ( + env.get("FLUID_COPILOT_AGENT_LOOP", "").strip().lower() in {"1", "true", "yes", "on"} + ) + + +def _agents_allowed(args: Any, env: Mapping[str, str]) -> bool: + """Whether the keyless auto-pick may choose a coding agent for this run. + + Not for the agent loop (it needs tool use, which the agents do not offer), + and not when ``--llm-endpoint`` / ``--llm-model`` (or their routing twins) + state where or what to call: that destination is never swapped for an agent. + """ + return not ( + agent_loop_requested(getattr(args, "agent_loop", False), env) + or any( + getattr(args, name, None) + for name in ("llm_endpoint", "llm_model", "llm_routing_endpoint", "llm_routing_model") + ) + ) + + +def _explicit_provider_name(args: Any, env: Mapping[str, str]) -> Optional[str]: + """The provider the operator named: ``--llm-provider``, else the env selectors.""" return ( getattr(args, "llm_provider", None) or env.get("FLUID_LLM_PROVIDER") or env.get("FLUID_FORGE_AGENT") - or _infer_provider_from_env(env) - or "gemini" + or None ) @@ -793,8 +845,9 @@ def _provider_name_from(args: Any, env: Mapping[str, str]) -> str: def preflight_llm_provider(args: Any, environ: Optional[Mapping[str, str]] = None) -> None: """Fail fast when the selected provider cannot be used on this install. - Cheap and side-effect free (no key lookup, no network, no litellm import): - it only resolves the provider adapter, which raises + Cheap and side-effect free (no key lookup, no litellm import, no network + beyond the localhost Ollama probe of the keyless auto-pick): it only + resolves the provider adapter, which raises :class:`~fluid_build._optional.MissingOptionalDependency` when a litellm-backed provider is chosen and litellm is absent. Call it before anything that reads workspace context or memory. @@ -805,14 +858,20 @@ def preflight_llm_provider(args: Any, environ: Optional[Mapping[str, str]] = Non from fluid_build._optional import MissingOptionalDependency, ensure_installed env = dict(environ or os.environ) - agent_loop = bool(getattr(args, "agent_loop", False)) - if not ( - getattr(args, "llm_provider", None) - or env.get("FLUID_LLM_PROVIDER") - or env.get("FLUID_FORGE_AGENT") - ): + agent_loop = agent_loop_requested(getattr(args, "agent_loop", False), env) + if not _explicit_provider_name(args, env): # Nothing chosen: inference reads the keyring and probes Ollama, and every - # provider it can pick (or the gemini default) needs litellm. Fail first. + # provider it can pick (or the gemini default) needs litellm, except the + # coding agent a non-interactive run auto-picks. Without litellm, that + # agent is the only provider that can run; otherwise fail first. + from fluid_build._optional import is_available + + if not is_available("litellm") and _lean_keyless_autopick( + env, + non_interactive=bool(getattr(args, "non_interactive", False)), + allow_agents=_agents_allowed(args, env), + ): + return ensure_installed( "litellm", "litellm", @@ -887,6 +946,15 @@ def resolve_llm_config(args: Any, environ: Optional[Mapping[str, str]] = None) - ) model_notes: List[str] = [] + if not _explicit_provider_name(args, env) and provider.name in _NON_LITELLM_PROVIDERS: + # Nothing named a provider and it is not a key-based one, so the keyless + # auto-pick chose it: say which, and how to override. + note = ( + f"No LLM provider is configured; using {provider.name}, the first coding agent " + f"found on PATH. Override with --llm-provider." + ) + model_notes.append(note) + LOG.warning(note) preflight_requested = _model_preflight_requested(args, env) available_models: Optional[List[str]] = None @@ -956,6 +1024,7 @@ def _available_models() -> Optional[List[str]]: model_source = "live_preflight" endpoint = getattr(args, "llm_endpoint", None) or env.get("FLUID_LLM_ENDPOINT") + endpoint_explicit = bool(endpoint) if not endpoint: endpoint = provider.default_endpoint(model, env) @@ -1025,6 +1094,7 @@ def _available_models() -> Optional[List[str]]: model_resolution_notes=model_notes, tier_models=tier_models, agent_mode=agent_mode, + endpoint_explicit=endpoint_explicit, ) @@ -1405,6 +1475,7 @@ def call_llm_streaming( _infer_provider_from_explicit_keys, _infer_provider_from_keyring, _keyring_key, + _lean_keyless_autopick, _resolve_api_key, clear_api_key_from_keyring, save_api_key_to_keyring, @@ -1573,7 +1644,26 @@ def check_llm_readiness(environ: Optional[Mapping[str, str]] = None) -> LlmReadi # Resolve API key: env vars → config file api_key = _resolve_api_key(provider.name, env) or saved_key - auth_ok = bool(api_key) or provider.name == "ollama" + from fluid_build.llm.coding_agent import _AGENT_SPECS + + agent = _AGENT_SPECS.get(provider.name) + if agent is not None: + # A coding agent is ready when its CLI is installed; its login or key is the + # CLI's own business and is not pre-checked. Search the PATH of the + # environment we were given, as the real call's child process would. + if not shutil.which(agent.binary, path=env.get("PATH", "")): + return LlmReadinessCheck( + ready=False, + provider=provider.name, + model=provider.default_model, + auth_available=False, + error=f"The '{agent.binary}' CLI for {provider.name} is not on PATH.", + ) + auth_ok = True + else: + auth_ok = ( + bool(api_key) or provider.name == "ollama" or bool(getattr(provider, "keyless", False)) + ) if not auth_ok: LOG.debug("LLM readiness: provider=%s found but no API key", provider.name) @@ -1588,11 +1678,11 @@ def check_llm_readiness(environ: Optional[Mapping[str, str]] = None) -> LlmReadi model = env.get("FLUID_LLM_MODEL") or saved_model or provider.default_model endpoint = env.get("FLUID_LLM_ENDPOINT") or provider.default_endpoint(model, env) - # Every built-in provider runs through litellm, which is an extra since 0.28.0. - # A configured key is not "ready" without it. + # Every built-in provider except the keyless agent/MCP ones runs through + # litellm, which is an extra since 0.28.0. A configured key is not "ready" without it. from fluid_build._optional import is_available - if not is_available("litellm"): + if provider.name not in _NON_LITELLM_PROVIDERS and not is_available("litellm"): return LlmReadinessCheck( ready=False, provider=provider.name, diff --git a/tests/cli/test_forge_agent_mode.py b/tests/cli/test_forge_agent_mode.py index f06f5e30..026b1d92 100644 --- a/tests/cli/test_forge_agent_mode.py +++ b/tests/cli/test_forge_agent_mode.py @@ -18,7 +18,9 @@ from __future__ import annotations +import io import json +import logging import os import subprocess import sys @@ -243,9 +245,11 @@ def test_emit_plan_emits_forge_plan_event(tmp_path: Path): consumes_step = next(s for s in plan["next_steps"] if "consumes" in s["step"].lower()) assert "list_source_lineage" in consumes_step["mcp_tools"] - # The architectural note must remind the agent NOT to shell-run fluid forge --ai. - assert "do NOT" in plan["note"] or "do not" in plan["note"].lower() - assert "API key" in plan["note"] or "api key" in plan["note"].lower() + # The note must describe the real split: the agent fills the fields itself, + # while --agent plus an AI flag runs forge's own AI with a second LLM. + assert "--ai" not in plan["note"] + assert "--llm-provider" in plan["note"] + assert "second LLM" in plan["note"] # Order: plan event must come BEFORE forge.done so consumers parsing # the stream see "what to do next" before the terminal signal. @@ -330,3 +334,123 @@ def test_agent_flag_registered_in_argparse(): args = parser.parse_args(["forge", "--agent", "--blank"]) assert getattr(args, "agent", False) is True assert getattr(args, "blank", False) is True + + +# --------------------------------------------------------------------------- +# 6. Explicit AI flags under --agent select AI mode (never a silent blank) +# --------------------------------------------------------------------------- + + +def _agent_args(argv: list[str], tmp_path: Path): + import argparse + + from fluid_build.cli import forge as forge_mod + + parser = argparse.ArgumentParser() + sp = parser.add_subparsers() + forge_mod.register(sp) + return parser.parse_args(["forge", "--agent", *argv, "-d", str(tmp_path / "product")]) + + +def _run_agent(args, capsys, monkeypatch, *, ai_result=0): + """Run forge.run under --agent with the AI generation seam mocked. + + Returns (events, ai_mock). stdin is a non-TTY and ``input`` must never be + called: an agent's shell cannot answer a prompt. + """ + from unittest.mock import MagicMock + + from fluid_build.cli import forge as forge_mod + + ai = MagicMock(side_effect=ai_result if isinstance(ai_result, Exception) else None) + if not isinstance(ai_result, Exception): + ai.return_value = ai_result + monkeypatch.setattr(forge_mod, "run_ai_copilot_mode", ai) + monkeypatch.setattr( + "builtins.input", lambda *a, **k: pytest.fail("input() called under --agent") + ) + monkeypatch.setattr(forge_mod, "_write_forge_receipt", lambda **k: None) + forge_mod.run(args, logging.getLogger("test")) + return _parse_jsonl_events(capsys.readouterr().out), ai + + +def test_agent_alone_is_blank_in_process(tmp_path, capsys, monkeypatch): + args = _agent_args(["--data-product-type", "SDP"], tmp_path) + events, ai = _run_agent(args, capsys, monkeypatch) + assert next(e for e in events if e["event"] == "forge.start")["mode"] == "blank" + assert args.blank is True + ai.assert_not_called() + + +def test_agent_with_llm_provider_selects_ai_mode(tmp_path, capsys, monkeypatch): + args = _agent_args(["--llm-provider", "claude-code"], tmp_path) + events, ai = _run_agent(args, capsys, monkeypatch) + assert next(e for e in events if e["event"] == "forge.start")["mode"] == "ai" + assert not args.blank + ai.assert_called_once() + assert next(e for e in events if e["event"] == "forge.done")["exit_code"] == 0 + + +@pytest.mark.parametrize( + "flags", + [ + ["--context", '{"project_goal": "daily revenue"}'], + ["--llm-model", "some-model"], + ["--llm-endpoint", "http://localhost:1234"], + ], +) +def test_agent_with_other_ai_flags_selects_ai_mode(flags, tmp_path, capsys, monkeypatch): + args = _agent_args(flags, tmp_path) + events, ai = _run_agent(args, capsys, monkeypatch) + assert next(e for e in events if e["event"] == "forge.start")["mode"] == "ai" + assert not args.blank + ai.assert_called_once() + + +def test_agent_ai_mode_is_non_interactive(tmp_path, capsys, monkeypatch): + """AI mode under --agent must never reach the inline-setup prompt flow.""" + from fluid_build.cli import ai_setup + + monkeypatch.setattr( + ai_setup, + "run_ai_setup_inline", + lambda *a, **k: pytest.fail("interactive AI setup reached under --agent"), + ) + monkeypatch.setattr("sys.stdin", io.StringIO("")) + args = _agent_args(["--llm-provider", "claude-code"], tmp_path) + events, ai = _run_agent(args, capsys, monkeypatch) + assert args.non_interactive is True + ai.assert_called_once() + + +@pytest.mark.parametrize("failure", [1, RuntimeError("provider down")]) +def test_agent_ai_failure_is_not_success(failure, tmp_path, capsys, monkeypatch): + args = _agent_args(["--llm-provider", "claude-code"], tmp_path) + events, _ = _run_agent(args, capsys, monkeypatch, ai_result=failure) + done = next(e for e in events if e["event"] == "forge.done") + assert done["exit_code"] != 0 + + +def test_agent_ai_mode_emits_contract_written(tmp_path, capsys, monkeypatch): + """The receipt path's "create" rows reach the stream as forge.contract_written.""" + from fluid_build.cli import forge as forge_mod + + product = tmp_path / "product" + + def _fake_ai(args, logger): + product.mkdir(parents=True, exist_ok=True) + (product / "contract.fluid.yaml").write_text("fluidVersion: 0.7.6\n") + return 0 + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(forge_mod, "run_ai_copilot_mode", _fake_ai) + monkeypatch.setattr( + "builtins.input", lambda *a, **k: pytest.fail("input() called under --agent") + ) + args = _agent_args(["--llm-provider", "claude-code"], tmp_path) + forge_mod.run(args, logging.getLogger("test")) + events = _parse_jsonl_events(capsys.readouterr().out) + written = [e for e in events if e["event"] == "forge.contract_written"] + assert written, [e["event"] for e in events] + assert written[0]["action"] == "created" + assert written[0]["path"].endswith("contract.fluid.yaml") diff --git a/tests/test_ai_setup.py b/tests/test_ai_setup.py index f7d5545c..0f775361 100644 --- a/tests/test_ai_setup.py +++ b/tests/test_ai_setup.py @@ -1055,6 +1055,12 @@ def test_resolve_llm_config_falls_back_to_gemini_when_unconfigured(self, monkeyp "fluid_build.cli.forge_copilot_llm_providers.detect_ollama_available", return_value=False, ), + # A coding agent on this machine's PATH would be auto-picked + # (non-interactive); "unconfigured" means none is installed. + patch( + "fluid_build.llm.provider_resolve._first_coding_agent_on_path", + return_value=None, + ), patch( "fluid_build.cli.forge_copilot_llm_providers._resolve_api_key", return_value=None, diff --git a/tests/test_coding_agent_isolation.py b/tests/test_coding_agent_isolation.py new file mode 100644 index 00000000..adb23b04 --- /dev/null +++ b/tests/test_coding_agent_isolation.py @@ -0,0 +1,280 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""Claude Code driver: failure reporting on stdout, and isolation from user config. + +No real CLI is spawned: ``_run_agent`` (the agent call) and ``_run_help`` (the +one-shot ``--help`` probe) are the two seams, both monkeypatched. +""" + +from __future__ import annotations + +import json + +import pytest + +from fluid_build.llm import coding_agent as cap +from fluid_build.llm.providers import CopilotGenerationError, LlmConfig + +pytestmark = pytest.mark.unit + +_CLAUDE_AUTH_FAILURE = ( + '{"type":"result","subtype":"success","is_error":true,' + '"result":"Failed to authenticate: OAuth session expired and could not be refreshed",' + '"num_turns":1,"duration_ms":105}' +) + +_ISOLATION_FLAGS = ( + "--tools", + "--strict-mcp-config", + "--setting-sources", + "--no-session-persistence", +) + +_FULL_HELP = """\ +Usage: claude [options] [prompt] + -p, --print Print response and exit + --system-prompt System prompt to use for the session + --append-system-prompt Append a system prompt + --allowedTools Tools to allow + --tools Specify the list of available tools + --strict-mcp-config Only use MCP servers from --mcp-config + --setting-sources Comma-separated list of setting sources + --no-session-persistence Disable session persistence +""" + + +class _Recorder: + def __init__(self, *, rc=0, stdout="", stderr=""): + self.rc, self.stdout, self.stderr = rc, stdout, stderr + self.calls = [] + + def __call__(self, argv, *, stdin, timeout, cwd, env): + self.calls.append({"argv": list(argv), "stdin": stdin, "env": env}) + return (self.rc, self.stdout, self.stderr) + + +def _cfg(): + return LlmConfig( + provider="claude-code", + model="claude-code", + endpoint="coding-agent://claude-code", + api_key=None, + ) + + +@pytest.fixture +def run(monkeypatch): + """Install a recorder; returns ``(recorder, set_help, help_probe_calls)``.""" + cap._cli_help.cache_clear() + probes = [] + state = {"help": ""} + + def _help(binary): + probes.append(binary) + return state["help"] + + monkeypatch.setattr(cap, "_run_help", _help) + monkeypatch.setattr(cap.shutil, "which", lambda b: f"/usr/local/bin/{b}") + + def _install(recorder, help_text=""): + state["help"] = help_text + monkeypatch.setattr(cap, "_run_agent", recorder) + return recorder + + yield _install, probes + cap._cli_help.cache_clear() + + +def _invoke(system="SYS", user="USER"): + return cap.get_coding_agent_provider("claude-code").invoke_blocking(_cfg(), system, user) + + +# --------------------------------------------------------------------------- +# Failures reported on stdout +# --------------------------------------------------------------------------- + + +def test_stdout_auth_failure_is_not_authenticated_with_hint(run): + install, _ = run + install(_Recorder(rc=1, stdout=_CLAUDE_AUTH_FAILURE, stderr="")) + with pytest.raises(CopilotGenerationError) as exc: + _invoke() + assert exc.value.event == "coding_agent_not_authenticated" + assert cap._AGENT_SPECS["claude-code"].login_hint in exc.value.suggestions + + +def test_stdout_error_with_other_text_is_failed_and_carries_the_text(run): + install, _ = run + out = json.dumps({"type": "result", "is_error": True, "result": "Prompt is too long"}) + install(_Recorder(rc=1, stdout=out)) + with pytest.raises(CopilotGenerationError) as exc: + _invoke() + assert exc.value.event == "coding_agent_failed" + assert "Prompt is too long" in exc.value.message + + +def test_stdout_error_text_is_redacted_and_capped(run): + install, _ = run + out = json.dumps({"is_error": True, "result": "word " * 400}) + install(_Recorder(rc=1, stdout=out)) + with pytest.raises(CopilotGenerationError) as exc: + _invoke() + assert exc.value.event == "coding_agent_failed" + # "claude-code exited with code 1: " + at most 500 characters of reason + assert len(exc.value.message) <= len("claude-code exited with code 1: ") + 500 + + +def test_exit_zero_with_is_error_is_a_clean_failure(run): + install, _ = run + out = json.dumps({"is_error": True, "result": "Something broke"}) + install(_Recorder(rc=0, stdout=out)) + with pytest.raises(CopilotGenerationError) as exc: + _invoke() + assert exc.value.event == "coding_agent_failed" + assert "Something broke" in exc.value.message + + +def test_exit_zero_auth_is_error_is_not_authenticated(run): + install, _ = run + install(_Recorder(rc=0, stdout=_CLAUDE_AUTH_FAILURE)) + with pytest.raises(CopilotGenerationError) as exc: + _invoke() + assert exc.value.event == "coding_agent_not_authenticated" + + +def test_stderr_is_combined_with_the_reported_reason(run): + install, _ = run + out = json.dumps({"is_error": True, "result": "boom"}) + install(_Recorder(rc=1, stdout=out, stderr="Please run /login")) + with pytest.raises(CopilotGenerationError) as exc: + _invoke() + assert exc.value.event == "coding_agent_not_authenticated" + + +def test_stderr_only_failure_still_works(run): + install, _ = run + install(_Recorder(rc=1, stderr="Error: invalid API key")) + with pytest.raises(CopilotGenerationError) as exc: + _invoke() + assert exc.value.event == "coding_agent_not_authenticated" + install(_Recorder(rc=2, stderr="some unexpected explosion")) + with pytest.raises(CopilotGenerationError) as exc: + _invoke() + assert exc.value.event == "coding_agent_failed" + assert "some unexpected explosion" in exc.value.message + + +@pytest.mark.parametrize( + "text", + [ + "Failed to authenticate", + "OAuth session expired", + "Not logged in", + "Invalid API key", + "run /login to continue", + ], +) +def test_claude_auth_markers_cover_the_known_wordings(text): + markers = cap._AGENT_SPECS["claude-code"].auth_markers + assert any(m in text.lower() for m in markers) + + +def test_is_error_false_is_not_a_failure(run): + install, _ = run + install(_Recorder(stdout=json.dumps({"is_error": False, "result": "{}"}))) + assert _invoke() == "{}" + + +# --------------------------------------------------------------------------- +# Isolation flags +# --------------------------------------------------------------------------- + + +def test_all_isolation_flags_present_when_help_lists_them(run): + install, _ = run + rec = install(_Recorder(stdout=json.dumps({"result": "{}"})), _FULL_HELP) + _invoke("THE-SYSTEM-PROMPT", "THE-USER-PROMPT") + argv = rec.calls[0]["argv"] + assert argv[argv.index("--tools") + 1] == "" + assert "--strict-mcp-config" in argv + assert "--no-session-persistence" in argv + # the system prompt replaces the default one instead of being appended + assert argv[argv.index("--system-prompt") + 1] == "THE-SYSTEM-PROMPT" + assert "--append-system-prompt" not in argv + # unchanged: stdin prompt, json output, schema + assert rec.calls[0]["stdin"] == "THE-USER-PROMPT" + assert "--json-schema" in argv and "--output-format" in argv + + +def test_setting_sources_is_one_token(run): + install, _ = run + rec = install(_Recorder(stdout=json.dumps({"result": "{}"})), _FULL_HELP) + _invoke() + argv = rec.calls[0]["argv"] + assert "--setting-sources=" in argv + assert "--setting-sources" not in argv + + +@pytest.mark.parametrize("missing", _ISOLATION_FLAGS) +def test_a_flag_help_lacks_is_omitted(run, missing): + install, _ = run + help_text = "\n".join(line for line in _FULL_HELP.splitlines() if missing not in line) + rec = install(_Recorder(stdout=json.dumps({"result": "{}"})), help_text) + _invoke() + argv = rec.calls[0]["argv"] + assert not any(tok.split("=")[0] == missing for tok in argv) + for flag in _ISOLATION_FLAGS: + if flag != missing: + assert any(tok.split("=")[0] == flag for tok in argv) + + +def test_tools_is_not_matched_by_allowed_tools(run): + install, _ = run + rec = install( + _Recorder(stdout=json.dumps({"result": "{}"})), " --allowedTools Tools\n" + ) + _invoke() + assert "--tools" not in rec.calls[0]["argv"] + + +def test_old_claude_without_system_prompt_flag_keeps_append_fallback(run): + install, _ = run + rec = install( + _Recorder(stdout=json.dumps({"result": "{}"})), + " --append-system-prompt Append\n", + ) + _invoke("SYS") + argv = rec.calls[0]["argv"] + assert argv[argv.index("--append-system-prompt") + 1] == "SYS" + assert "--system-prompt" not in argv + + +def test_child_env_lacks_claudecode(run, monkeypatch): + install, _ = run + monkeypatch.setenv("CLAUDECODE", "1") + rec = install(_Recorder(stdout=json.dumps({"result": "{}"})), _FULL_HELP) + _invoke() + assert "CLAUDECODE" not in rec.calls[0]["env"] + + +def test_other_agents_keep_their_env_and_argv(run, monkeypatch): + install, probes = run + monkeypatch.setenv("CLAUDECODE", "1") + rec = install(_Recorder(stdout="TEXT"), _FULL_HELP) + cap.get_coding_agent_provider("codex").invoke_blocking( + LlmConfig(provider="codex", model="codex", endpoint="x", api_key=None), "SYS", "USER" + ) + argv = rec.calls[0]["argv"] + assert rec.calls[0]["env"].get("CLAUDECODE") == "1" + assert not any(tok.startswith("--setting-sources") for tok in argv) + assert "--strict-mcp-config" not in argv + assert probes == [] # no help probe for an agent without isolation flags + + +def test_help_probe_runs_once_across_two_invocations(run): + install, probes = run + install(_Recorder(stdout=json.dumps({"result": "{}"})), _FULL_HELP) + _invoke() + _invoke() + assert probes == ["/usr/local/bin/claude"] diff --git a/tests/test_coding_agent_provider.py b/tests/test_coding_agent_provider.py index ed60427f..5d9ccbe8 100644 --- a/tests/test_coding_agent_provider.py +++ b/tests/test_coding_agent_provider.py @@ -74,6 +74,15 @@ def _argv(rec): return rec.calls[0]["argv"] +@pytest.fixture(autouse=True) +def _no_real_help_probe(monkeypatch): + """Never run the real ``claude --help``; an empty help lists no flag.""" + cap._cli_help.cache_clear() + monkeypatch.setattr(cap, "_run_help", lambda binary: "") + yield + cap._cli_help.cache_clear() + + # --------------------------------------------------------------------------- # Per-agent argv + output parsing # --------------------------------------------------------------------------- @@ -103,7 +112,8 @@ def test_claude_code_argv_and_structured_output(monkeypatch): assert "--json-schema" in argv schema_str = argv[argv.index("--json-schema") + 1] assert "contract" in schema_str and "recommended_template" in schema_str - # system via flag, user via stdin + # system via flag (help lists no --system-prompt here, so the append fallback), + # user via stdin assert "--append-system-prompt" in argv assert argv[argv.index("--append-system-prompt") + 1] == "SYS-PROMPT" assert rec.calls[0]["stdin"] == "USER-PROMPT" @@ -224,6 +234,15 @@ def test_not_authenticated_marker(monkeypatch): assert exc.value.event == "coding_agent_not_authenticated" +def test_not_authenticated_marker_wording_from_stderr_only(monkeypatch): + rec = _Recorder(rc=1, stderr="Failed to authenticate: OAuth session expired") + _install(monkeypatch, rec) + provider = get_llm_provider("claude-code") + with pytest.raises(CopilotGenerationError) as exc: + provider.invoke_blocking(_cfg("claude-code"), "SYS", "USER") + assert exc.value.event == "coding_agent_not_authenticated" + + def test_generic_failure(monkeypatch): rec = _Recorder(rc=2, stderr="some unexpected explosion") _install(monkeypatch, rec) diff --git a/tests/test_forge_copilot_runtime.py b/tests/test_forge_copilot_runtime.py index 19873b92..8bd92bc9 100644 --- a/tests/test_forge_copilot_runtime.py +++ b/tests/test_forge_copilot_runtime.py @@ -31,6 +31,7 @@ clear_api_key_from_keyring, detect_provider_from_api_key, get_catalog_default, + get_llm_provider, has_llm_api_key, live_provider_models, resolve_llm_config, @@ -221,10 +222,11 @@ def test_ollama_uses_default_endpoint_without_api_key(self): assert config.provider == "ollama" assert config.api_key is None - # litellm owns auth/endpoint resolution; we emit a sentinel - # ``litellm://ollama/`` so telemetry can identify the - # routed call without leaking the daemon URL into the config. - assert config.endpoint.startswith("litellm://ollama/") + # OLLAMA_HOST is a real daemon address, so it is the endpoint; litellm + # owns everything else and no placeholder URL is ever recorded. + assert config.endpoint == "http://localhost:11434" + _, payload = get_llm_provider("ollama").build_request(config, "s", "u") + assert payload["api_base"] == "http://localhost:11434" def test_llm_timeout_seconds_can_be_configured_from_env(self): args = SimpleNamespace(llm_provider="ollama", llm_model=None, llm_endpoint=None) @@ -381,7 +383,7 @@ def test_required_llm_preflight_accepts_local_ollama_latest_alias(self): assert config.provider == "ollama" assert config.model == "gemma4:latest" - assert config.endpoint.startswith("litellm://ollama/") + assert config.endpoint == "http://localhost:11434" def test_required_llm_preflight_rejects_unavailable_ollama_model(self): args = SimpleNamespace( @@ -422,8 +424,8 @@ def test_marks_ollama_ready_without_api_key(self): assert readiness.ready is True assert readiness.provider == "ollama" assert readiness.auth_available is True - # litellm-routed sentinel — see test_ollama_uses_default_endpoint_without_api_key. - assert readiness.endpoint.startswith("litellm://ollama/") + # See test_ollama_uses_default_endpoint_without_api_key. + assert readiness.endpoint == "http://localhost:11434" def test_redacts_endpoint_in_readiness_output(self): readiness = check_llm_readiness( diff --git a/tests/test_litellm_absent.py b/tests/test_litellm_absent.py index 5de67fda..4f75f9e8 100644 --- a/tests/test_litellm_absent.py +++ b/tests/test_litellm_absent.py @@ -419,6 +419,119 @@ def test_init_handoff_prints_the_hint_and_touches_no_credential_store( forge_run.assert_not_called() +class TestLeanKeylessAutoPick: + """Without litellm, a non-interactive run may still author with an installed agent.""" + + @staticmethod + def _lean(block, monkeypatch, tmp_path, *, tty=False, ollama=False, saved=None): # noqa: F811 + import stat + import sys + from types import SimpleNamespace + + from fluid_build.llm import providers + + block("litellm") + bindir = tmp_path / "bin" + bindir.mkdir() + exe = bindir / "claude" + exe.write_text("#!/bin/sh\nexit 0\n") + exe.chmod(exe.stat().st_mode | stat.S_IXUSR) + monkeypatch.setattr(sys, "stdin", SimpleNamespace(isatty=lambda: tty)) + monkeypatch.setattr(providers, "detect_ollama_available", lambda env: ollama) + monkeypatch.setattr(providers, "_get_api_key_from_keyring", lambda name: None) + monkeypatch.setattr("fluid_build._ai_config_shared.load_ai_config", lambda *a, **k: saved) + return {"PATH": str(bindir)} + + @staticmethod + def _args(**kw): + from types import SimpleNamespace + + return SimpleNamespace(**{"llm_provider": None, "agent_loop": False, **kw}) + + def test_agent_on_path_passes_the_preflight(self, block, monkeypatch, tmp_path): # noqa: F811 + from fluid_build.llm.providers import _provider_name_from, preflight_llm_provider + + env = self._lean(block, monkeypatch, tmp_path) + preflight_llm_provider(self._args(), env) + assert _provider_name_from(self._args(), env) == "claude-code" + + def test_running_ollama_still_fails_fast(self, block, monkeypatch, tmp_path): # noqa: F811 + from fluid_build.llm.providers import preflight_llm_provider + + env = self._lean(block, monkeypatch, tmp_path, ollama=True) + with pytest.raises(MissingOptionalDependency): + preflight_llm_provider(self._args(), env) + + def test_an_api_key_still_fails_fast(self, block, monkeypatch, tmp_path): # noqa: F811 + from fluid_build.llm.providers import preflight_llm_provider + + env = self._lean(block, monkeypatch, tmp_path) + env["GEMINI_API_KEY"] = "AIza-test" # pragma: allowlist secret + with pytest.raises(MissingOptionalDependency): + preflight_llm_provider(self._args(), env) + + def test_a_stated_ollama_host_still_fails_fast( + self, block, monkeypatch, tmp_path + ): # noqa: F811 + from fluid_build.llm.providers import preflight_llm_provider + + env = self._lean(block, monkeypatch, tmp_path) + env["OLLAMA_HOST"] = "http://127.0.0.1:11500" + with pytest.raises(MissingOptionalDependency): + preflight_llm_provider(self._args(), env) + + def test_a_saved_provider_still_fails_fast(self, block, monkeypatch, tmp_path): # noqa: F811 + from fluid_build.llm.providers import preflight_llm_provider + + env = self._lean(block, monkeypatch, tmp_path, saved={"provider": "openai"}) + with pytest.raises(MissingOptionalDependency): + preflight_llm_provider(self._args(), env) + + def test_a_tty_still_fails_fast(self, block, monkeypatch, tmp_path): # noqa: F811 + from fluid_build.llm.providers import preflight_llm_provider + + env = self._lean(block, monkeypatch, tmp_path, tty=True) + with pytest.raises(MissingOptionalDependency): + preflight_llm_provider(self._args(), env) + + def test_the_agent_loop_never_auto_picks_an_agent( + self, block, monkeypatch, tmp_path + ): # noqa: F811 + from fluid_build.llm.providers import _provider_name_from, preflight_llm_provider + + env = self._lean(block, monkeypatch, tmp_path) + with pytest.raises(MissingOptionalDependency) as info: + preflight_llm_provider(self._args(agent_loop=True), env) + assert "--template starter" in str(info.value) + assert _provider_name_from(self._args(agent_loop=True), env) != "claude-code" + + def test_the_agent_loop_env_route_never_auto_picks_an_agent( + self, block, monkeypatch, tmp_path + ): # noqa: F811 + from fluid_build.llm.providers import preflight_llm_provider + + env = {**self._lean(block, monkeypatch, tmp_path), "FLUID_COPILOT_AGENT_LOOP": "1"} + with pytest.raises(MissingOptionalDependency) as info: + preflight_llm_provider(self._args(), env) + assert "--template starter" in str(info.value) + + +def test_agent_search_uses_only_the_given_environment(monkeypatch, tmp_path): + """An environment without PATH finds no agent, even if the process PATH has one.""" + import stat + + from fluid_build.llm.provider_resolve import _first_coding_agent_on_path + + bindir = tmp_path / "bin" + bindir.mkdir() + exe = bindir / "claude" + exe.write_text("#!/bin/sh\nexit 0\n") + exe.chmod(exe.stat().st_mode | stat.S_IXUSR) + monkeypatch.setenv("PATH", str(bindir)) + assert _first_coding_agent_on_path({}) is None + assert _first_coding_agent_on_path({"PATH": str(bindir)}) == "claude-code" + + class TestDoctorStatus: def _row(self, monkeypatch): from fluid_build.cli.doctor import _check_fluid_features diff --git a/tests/test_llm_endpoint_sentinel.py b/tests/test_llm_endpoint_sentinel.py new file mode 100644 index 00000000..fb745115 --- /dev/null +++ b/tests/test_llm_endpoint_sentinel.py @@ -0,0 +1,416 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""``LlmConfig.endpoint`` holds only a real endpoint, never a telemetry sentinel. + +litellm-backed providers used to resolve to ``litellm:///``. +Consumers then dialled it: Ollama's ``api_base`` carried the sentinel, ``fluid +ai test`` POSTed to it, and forge copied it into ``args.llm_endpoint`` where it +read as an explicit endpoint on the next resolve. These tests drive the real +``resolve_llm_config`` -> ``build_request`` path so the class cannot return. +""" + +from __future__ import annotations + +import sys +from types import SimpleNamespace +from unittest import mock + +import pytest + +from fluid_build.llm.providers import ( + BUILTIN_LLM_PROVIDERS, + LlmConfig, + resolve_llm_config, +) + +pytestmark = pytest.mark.unit + +_OLLAMA_URL = "http://ollama.internal:11434" + + +@pytest.fixture(autouse=True) +def _clean_env(monkeypatch): + for var in ("OLLAMA_HOST", "FLUID_LLM_ENDPOINT", "FLUID_LLM_PROVIDER", "FLUID_FORGE_AGENT"): + monkeypatch.delenv(var, raising=False) + + +def _args(**kw): + base = {"llm_provider": None, "llm_model": None, "llm_endpoint": None} + base.update(kw) + return SimpleNamespace(**base) + + +def _payload(config: LlmConfig) -> dict: + _, payload = BUILTIN_LLM_PROVIDERS[config.provider].build_request(config, "sys", "usr") + return payload + + +_KEYS = { + "openai": {"OPENAI_API_KEY": "k"}, + "anthropic": {"ANTHROPIC_API_KEY": "k"}, + "gemini": {"GEMINI_API_KEY": "k"}, + "ollama": {}, +} + + +@pytest.mark.parametrize("provider", sorted(_KEYS)) +def test_endpoint_is_never_the_litellm_sentinel(provider): + config = resolve_llm_config(_args(llm_provider=provider), _KEYS[provider]) + assert "litellm://" not in config.endpoint + assert config.endpoint == "" + assert "litellm://" not in config.redacted_endpoint + + +@pytest.mark.parametrize("provider", sorted(_KEYS)) +def test_routing_config_never_carries_the_sentinel(provider): + config = resolve_llm_config(_args(llm_provider=provider), _KEYS[provider]) + assert "litellm://" not in config.for_routing().endpoint + + +class TestRoutingKeepsAStatedEndpoint: + """Routing calls go to the daemon the operator named, not a re-derived default.""" + + _STATED = "http://127.0.0.1:9999" + + def test_flag_endpoint_reaches_routing_calls(self): + config = resolve_llm_config( + _args(llm_provider="ollama", llm_model="llama3", llm_endpoint=self._STATED), {} + ) + routed = config.for_routing() + assert routed.model == config.routing_model != config.model + assert routed.endpoint == self._STATED + assert _payload(routed)["api_base"] == self._STATED + + def test_env_endpoint_reaches_routing_calls(self): + config = resolve_llm_config( + _args(llm_provider="ollama", llm_model="llama3"), + {"FLUID_LLM_ENDPOINT": self._STATED}, + ) + assert config.for_routing().endpoint == self._STATED + + def test_a_routing_endpoint_still_wins(self): + config = resolve_llm_config( + _args( + llm_provider="ollama", + llm_model="llama3", + llm_endpoint=self._STATED, + llm_routing_endpoint="http://127.0.0.1:9998", + ), + {}, + ) + assert config.for_routing().endpoint == "http://127.0.0.1:9998" + + def test_a_derived_endpoint_is_still_rederived(self): + # Nothing stated: the provider's own default is asked again for the routing model. + config = resolve_llm_config(_args(llm_provider="ollama", llm_model="llama3"), {}) + assert config.endpoint_explicit is False + seen = [] + provider = BUILTIN_LLM_PROVIDERS["ollama"] + with mock.patch.object( + type(provider), "default_endpoint", lambda self, m, e: seen.append(m) or "derived" + ): + assert config.for_routing().endpoint == "derived" + assert seen == [config.routing_model] + + def test_hosted_provider_without_an_endpoint_stays_empty(self): + config = resolve_llm_config(_args(llm_provider="openai"), _KEYS["openai"]) + assert config.for_routing().endpoint == "" + + +def test_readiness_endpoint_is_not_a_fake_url(): + from fluid_build.llm.providers import check_llm_readiness + + readiness = check_llm_readiness({"FLUID_LLM_PROVIDER": "ollama"}) + assert "litellm://" not in (readiness.endpoint or "") + + +@pytest.mark.parametrize( + "provider,expected", [("openai", "Provider default"), (None, "Not configured")] +) +def test_doctor_prints_no_fake_endpoint(capsys, provider, expected): + from fluid_build.cli import doctor + from fluid_build.llm.providers import LlmReadinessCheck + + readiness = LlmReadinessCheck(ready=bool(provider), provider=provider, endpoint="") + doctor._print_copilot_readiness(readiness) + + out = capsys.readouterr().out + assert expected in out + assert "litellm://" not in out + + +class TestOllamaApiBase: + def test_no_host_leaves_api_base_to_litellm(self): + config = resolve_llm_config(_args(llm_provider="ollama"), {}) + assert "api_base" not in _payload(config) + + def test_ollama_host_becomes_api_base(self): + config = resolve_llm_config(_args(llm_provider="ollama"), {"OLLAMA_HOST": _OLLAMA_URL}) + assert config.endpoint == _OLLAMA_URL + assert _payload(config)["api_base"] == _OLLAMA_URL + + def test_schemeless_ollama_host_gets_http(self): + # Ollama's own variable accepts ``host:port``; litellm needs a URL. + config = resolve_llm_config( + _args(llm_provider="ollama"), {"OLLAMA_HOST": "127.0.0.1:11500"} + ) + assert _payload(config)["api_base"] == "http://127.0.0.1:11500" + + def test_flag_beats_ollama_host(self): + config = resolve_llm_config( + _args(llm_provider="ollama", llm_endpoint="http://localhost:9999"), + {"OLLAMA_HOST": _OLLAMA_URL}, + ) + assert _payload(config)["api_base"] == "http://localhost:9999" + + def test_env_endpoint_beats_ollama_host(self): + config = resolve_llm_config( + _args(llm_provider="ollama"), + {"FLUID_LLM_ENDPOINT": "http://localhost:9998", "OLLAMA_HOST": _OLLAMA_URL}, + ) + assert _payload(config)["api_base"] == "http://localhost:9998" + + def test_tool_request_uses_the_same_api_base(self): + provider = BUILTIN_LLM_PROVIDERS["ollama"] + bare = resolve_llm_config(_args(llm_provider="ollama"), {}) + hosted = resolve_llm_config(_args(llm_provider="ollama"), {"OLLAMA_HOST": _OLLAMA_URL}) + assert "api_base" not in provider.build_tool_request(bare, "s", [], [])[2] + assert provider.build_tool_request(hosted, "s", [], [])[2]["api_base"] == _OLLAMA_URL + + def test_streaming_request_uses_the_same_api_base(self): + provider = BUILTIN_LLM_PROVIDERS["ollama"] + hosted = resolve_llm_config(_args(llm_provider="ollama"), {"OLLAMA_HOST": _OLLAMA_URL}) + assert provider.build_streaming_request(hosted, "s", "u")[2]["api_base"] == _OLLAMA_URL + + @pytest.mark.parametrize("host", [None, _OLLAMA_URL]) + def test_copying_the_endpoint_into_args_changes_nothing(self, host): + # forge.py / init.py do ``args.llm_endpoint = llm_config.endpoint`` and + # resolve again; that round trip must be the identity. + env = {"OLLAMA_HOST": host} if host else {} + args = _args(llm_provider="ollama") + first = resolve_llm_config(args, env) + args.llm_endpoint = first.endpoint + second = resolve_llm_config(args, env) + assert second.endpoint == first.endpoint + assert _payload(second).get("api_base") == _payload(first).get("api_base") + + +# Forms of OLLAMA_HOST -> the URL both reads of it use. Same rules as +# ollama-python's ``_parse_host``: no scheme means http on 11434; an explicit +# scheme without a port means that scheme's default port; a path is kept. +_OLLAMA_HOST_FORMS = [ + ("myhost", "http://myhost:11434"), + ("myhost:56789", "http://myhost:56789"), + ("1.2.3.4", "http://1.2.3.4:11434"), + (":56789", "http://127.0.0.1:56789"), + ("http://myhost", "http://myhost:80"), + ("https://myhost", "https://myhost:443"), + ("http://myhost:56789", "http://myhost:56789"), + ("https://myhost:56789", "https://myhost:56789"), + ("myhost/", "http://myhost:11434"), + ("myhost:56789/path/", "http://myhost:56789/path"), + ("https://myhost:56789/path", "https://myhost:56789/path"), + (" myhost:56789 ", "http://myhost:56789"), + ("[::1]", "http://[::1]:11434"), + ("[::1]:56789", "http://[::1]:56789"), + ("https://[::1]", "https://[::1]:443"), +] + + +class TestOllamaHostForms: + @pytest.mark.parametrize("host, expected", _OLLAMA_HOST_FORMS) + def test_base_url(self, host, expected): + from fluid_build.llm.litellm_backend import _ollama_base_url + + assert _ollama_base_url({"OLLAMA_HOST": host}) == expected + + @pytest.mark.parametrize("env", [{}, {"OLLAMA_HOST": ""}, {"OLLAMA_HOST": " "}]) + def test_unset_stays_empty_for_litellm_to_choose(self, env): + from fluid_build.llm.litellm_backend import _ollama_base_url + + assert _ollama_base_url(env) == "" + + def test_an_unparseable_port_is_left_to_fail_at_the_dial(self): + from fluid_build.llm.litellm_backend import _ollama_base_url + + assert _ollama_base_url({"OLLAMA_HOST": "myhost:abc"}) == "http://myhost:abc" + + @pytest.mark.parametrize("host, expected", _OLLAMA_HOST_FORMS) + def test_the_model_list_dials_the_same_url(self, host, expected): + from fluid_build.llm.litellm_backend import _list_ollama_models + + with mock.patch("urllib.request.urlopen", side_effect=OSError("down")) as urlopen: + assert _list_ollama_models({"OLLAMA_HOST": host}) is None + assert urlopen.call_args.args[0].full_url == f"{expected}/api/tags" + + def test_the_model_list_defaults_to_localhost(self): + from fluid_build.llm.litellm_backend import _list_ollama_models + + with mock.patch("urllib.request.urlopen", side_effect=OSError("down")) as urlopen: + _list_ollama_models({}) + assert urlopen.call_args.args[0].full_url == "http://localhost:11434/api/tags" + + def test_resolved_endpoint_and_api_base_use_the_bare_host_port(self): + config = resolve_llm_config(_args(llm_provider="ollama"), {"OLLAMA_HOST": "myhost"}) + assert config.endpoint == "http://myhost:11434" + assert _payload(config)["api_base"] == "http://myhost:11434" + + +class TestAiTestNeverDialsASentinel: + """``fluid ai test`` probes litellm-backed providers through their own call path.""" + + def _fake_litellm(self): + fake = mock.MagicMock(spec=["completion", "completion_cost"]) + fake.completion.return_value = { + "choices": [{"message": {"content": "FLUID_OK"}}], + "usage": {"prompt_tokens": 7, "completion_tokens": 2, "total_tokens": 9}, + } + fake.completion_cost.return_value = 0.0 + return fake + + @pytest.mark.parametrize("provider", sorted(_KEYS)) + def test_smoke_call_goes_through_litellm_not_http(self, monkeypatch, provider): + from fluid_build.cli.ai_setup import _run_ai_smoke_call + + fake = self._fake_litellm() + monkeypatch.setitem(sys.modules, "litellm", fake) + config = resolve_llm_config(_args(llm_provider=provider), _KEYS[provider]) + + with mock.patch("httpx.Client", side_effect=AssertionError("raw HTTP POST")): + text, usage, latency_ms = _run_ai_smoke_call(BUILTIN_LLM_PROVIDERS[provider], config) + + assert text == "FLUID_OK" + assert usage["total_tokens"] == 9 + assert latency_ms >= 0 + kwargs = fake.completion.call_args.kwargs + assert kwargs["max_tokens"] >= 1 + assert kwargs["timeout"] == config.timeout_seconds + + def test_smoke_call_unexpected_text_is_reported(self, monkeypatch): + from fluid_build.cli.ai_setup import _run_ai_smoke_call + from fluid_build.llm.providers import CopilotGenerationError + + fake = self._fake_litellm() + fake.completion.return_value = {"choices": [{"message": {"content": "hi"}}]} + monkeypatch.setitem(sys.modules, "litellm", fake) + config = resolve_llm_config(_args(llm_provider="openai"), _KEYS["openai"]) + + with pytest.raises(CopilotGenerationError) as exc_info: + _run_ai_smoke_call(BUILTIN_LLM_PROVIDERS["openai"], config) + assert exc_info.value.event == "ai_test_unexpected_response" + + @pytest.mark.parametrize( + "event,exit_code", + [ + ("copilot_llm_auth_failed", 3), + ("copilot_llm_permission_denied", 3), + ("copilot_litellm_request_failed", 5), + ], + ) + def test_litellm_errors_classify_to_the_documented_exit_codes(self, event, exit_code): + from fluid_build.cli.ai_setup import _classify_ai_test_error + + assert _classify_ai_test_error(event, None) == exit_code + + @staticmethod + def _exit_code_for(monkeypatch, error): + """The exit code ``fluid ai test`` picks when the litellm call raises *error*.""" + from fluid_build.cli.ai_setup import ( + _classify_ai_test_error, + _http_status_from_cause, + _run_ai_smoke_call, + ) + from fluid_build.llm.providers import CopilotGenerationError + + fake = mock.MagicMock(spec=["completion", "completion_cost"]) + fake.completion.side_effect = error + monkeypatch.setitem(sys.modules, "litellm", fake) + config = resolve_llm_config(_args(llm_provider="openai"), _KEYS["openai"]) + with pytest.raises(CopilotGenerationError) as exc_info: + _run_ai_smoke_call(BUILTIN_LLM_PROVIDERS["openai"], config) + exc = exc_info.value + return _classify_ai_test_error(exc.event, _http_status_from_cause(exc)) + + @pytest.mark.parametrize( + "name, exit_code", + [ + ("NotFoundError", 4), # a model the provider does not have + ("Timeout", 5), + ("APIConnectionError", 5), + ("RateLimitError", 5), + ("ServiceUnavailableError", 5), + ], + ) + def test_litellm_not_found_is_a_resource_error_the_rest_stay_network( + self, monkeypatch, name, exit_code + ): + litellm = pytest.importorskip("litellm") + error = getattr(litellm, name)(message="boom", model="m", llm_provider="openai") + assert self._exit_code_for(monkeypatch, error) == exit_code + + def test_a_404_from_an_unlisted_exception_class_is_also_a_resource_error(self, monkeypatch): + class _Gone(Exception): + status_code = 404 + + assert self._exit_code_for(monkeypatch, _Gone("no such model")) == 4 + + def test_an_error_without_a_status_stays_network(self, monkeypatch): + assert self._exit_code_for(monkeypatch, RuntimeError("socket closed")) == 5 + + @pytest.mark.parametrize("provider", sorted(_KEYS)) + def test_resolved_test_config_has_no_sentinel_and_validates(self, provider): + from fluid_build.cli.ai_setup import _resolve_ai_test_config + + args = _args(llm_provider=provider, llm_timeout_seconds=None) + with ( + mock.patch("fluid_build.cli.ai_setup._load_ai_config", return_value=None), + mock.patch("fluid_build.cli.ai_setup._load_key_from_keyring", return_value="k"), + mock.patch.dict("os.environ", _KEYS[provider], clear=True), + ): + config, error = _resolve_ai_test_config(args) + + assert error is None + assert config is not None + assert "litellm://" not in config.endpoint + + def test_a_litellm_url_is_not_a_valid_test_endpoint(self): + from fluid_build.cli.ai_setup import _validate_ai_test_endpoint + + assert _validate_ai_test_endpoint("openai", "litellm://openai/gpt-4o") is not None + + def test_run_ai_test_never_posts_and_reports_no_fake_endpoint(self, monkeypatch): + from fluid_build.cli.ai_setup import run_ai_test + + monkeypatch.setitem(sys.modules, "litellm", self._fake_litellm()) + args = _args(llm_provider="openai", llm_timeout_seconds=None, json=True) + with ( + mock.patch("fluid_build.cli.ai_setup._load_ai_config", return_value=None), + mock.patch("fluid_build.cli.ai_setup._load_key_from_keyring", return_value="k"), + mock.patch.dict("os.environ", {"OPENAI_API_KEY": "k"}, clear=True), + mock.patch("httpx.Client", side_effect=AssertionError("raw HTTP POST")), + mock.patch("fluid_build.cli.ai_setup._check_ai_test_model_availability") as avail, + ): + avail.return_value = ("unavailable", None) + code, report = run_ai_test(None, args) + + assert code == 0 + assert report["ok"] is True + assert report["endpoint"] is None + + +@pytest.mark.parametrize( + "provider,key_env,expected", + [ + ("openai", "OPENAI_API_KEY", ("openai", "gpt-4.1-mini")), + ("anthropic", "ANTHROPIC_API_KEY", ("anthropic", "claude-sonnet-4-6")), + ("claude", "ANTHROPIC_API_KEY", ("anthropic", "claude-sonnet-4-6")), + ("gemini", "GEMINI_API_KEY", ("gemini", "gemini-2.5-pro")), + ], +) +def test_command_center_provider_and_model_are_unchanged(provider, key_env, expected): + """Command Center runs ``fluid forge data-model from-intent --llm-provider X`` + with the key in env and no ``--llm-model``. Only the endpoint may change.""" + config = resolve_llm_config(_args(llm_provider=provider), {key_env: "k"}) + assert (config.provider, config.model) == expected + assert config.endpoint == "" diff --git a/tests/test_llm_keyless_autopick.py b/tests/test_llm_keyless_autopick.py new file mode 100644 index 00000000..4e7252c5 --- /dev/null +++ b/tests/test_llm_keyless_autopick.py @@ -0,0 +1,426 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""Keyless auto-pick: a non-interactive run with no provider configured. + +Order: a running local Ollama (data stays local), then the first coding-agent CLI +on PATH in ``CODING_AGENT_NAMES`` order, then the hardcoded ``gemini`` last +resort. Interactive runs keep "offer, don't force" (the ``fluid ai setup`` +inline offer). ``check_llm_readiness`` and the real selection share one helper, +so ``fluid doctor`` never disagrees with what forge then does. + +Hermetic: stub executables on a tmp PATH, stdin's ``isatty`` patched, and no +read of the real ``~/.fluid`` config or keyring. +""" + +from __future__ import annotations + +import logging +import stat +import sys +from types import SimpleNamespace +from unittest import mock + +import pytest + +from fluid_build.llm import providers +from fluid_build.llm.coding_agent import _AGENT_SPECS, CODING_AGENT_NAMES +from fluid_build.llm.providers import ( + _provider_name_from, + check_llm_readiness, + resolve_llm_config, +) + +pytestmark = pytest.mark.unit + +_SELECTOR_ENV = ( + "FLUID_LLM_PROVIDER", + "FLUID_FORGE_AGENT", + "FLUID_LLM_API_KEY", + "FLUID_LLM_ENDPOINT", + "FLUID_LLM_MODEL", + "FLUID_LLM_ROUTING_ENDPOINT", + "FLUID_LLM_ROUTING_MODEL", + "OLLAMA_HOST", +) +_KEY_ENV = ( + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + "GEMINI_API_KEY", + "GOOGLE_API_KEY", + "GITHUB_API_KEY", +) + + +def _args(**kw): + base = {"llm_provider": None, "llm_model": None, "llm_endpoint": None} + base.update(kw) + return SimpleNamespace(**base) + + +@pytest.fixture +def world(monkeypatch, tmp_path): + """An empty world: no keys, no saved config, no keyring, no Ollama, empty PATH.""" + for var in _SELECTOR_ENV + _KEY_ENV: + monkeypatch.delenv(var, raising=False) + bindir = tmp_path / "bin" + bindir.mkdir() + monkeypatch.setenv("PATH", str(bindir)) + monkeypatch.setattr(sys, "stdin", SimpleNamespace(isatty=lambda: False)) + monkeypatch.setattr(providers, "_get_api_key_from_keyring", lambda name: None) + monkeypatch.setattr(providers, "detect_ollama_available", lambda env: False) + monkeypatch.setattr("fluid_build._ai_config_shared.load_ai_config", lambda *a, **k: None) + return SimpleNamespace( + bindir=bindir, + env={"PATH": str(bindir)}, + monkeypatch=monkeypatch, + install=lambda *names: [_install(bindir, n) for n in names], + ) + + +def _install(bindir, agent): + exe = bindir / _AGENT_SPECS[agent].binary + exe.write_text("#!/bin/sh\nexit 0\n") + exe.chmod(exe.stat().st_mode | stat.S_IXUSR) + + +def _tty(world, value): + world.monkeypatch.setattr(sys, "stdin", SimpleNamespace(isatty=lambda: value)) + + +class TestPick: + def test_claude_only(self, world): + world.install("claude-code") + assert _provider_name_from(_args(), dict(world.env)) == "claude-code" + + def test_codex_only(self, world): + world.install("codex") + assert _provider_name_from(_args(), dict(world.env)) == "codex" + + @pytest.mark.parametrize("agent", CODING_AGENT_NAMES) + def test_each_agent_is_found_by_its_own_binary(self, world, agent): + world.install(agent) + assert _provider_name_from(_args(), dict(world.env)) == agent + + def test_first_in_coding_agent_order_wins(self, world): + world.install(*reversed(CODING_AGENT_NAMES)) + assert _provider_name_from(_args(), dict(world.env)) == CODING_AGENT_NAMES[0] + + def test_codex_beats_later_agents(self, world): + world.install("kiro", "cursor", "codex") + assert _provider_name_from(_args(), dict(world.env)) == "codex" + + def test_running_ollama_beats_an_agent(self, world): + world.install("claude-code") + world.monkeypatch.setattr(providers, "detect_ollama_available", lambda env: True) + assert _provider_name_from(_args(), dict(world.env)) == "ollama" + + def test_nothing_detected_keeps_the_gemini_last_resort(self, world): + assert _provider_name_from(_args(), dict(world.env)) == "gemini" + + def test_non_interactive_flag_overrides_a_tty(self, world): + world.install("claude-code") + _tty(world, True) + assert _provider_name_from(_args(non_interactive=True), dict(world.env)) == "claude-code" + + def test_path_comes_from_the_env_mapping_when_given(self, world, tmp_path): + other = tmp_path / "other" + other.mkdir() + _install(other, "codex") + assert _provider_name_from(_args(), {**world.env, "PATH": str(other)}) == "codex" + + +class TestInteractiveKeepsOfferDontForce: + def test_tty_does_not_auto_pick_an_agent(self, world): + world.install("claude-code") + _tty(world, True) + assert _provider_name_from(_args(), dict(world.env)) == "gemini" + + def test_tty_still_picks_a_running_ollama(self, world): + world.install("claude-code") + _tty(world, True) + world.monkeypatch.setattr(providers, "detect_ollama_available", lambda env: True) + assert _provider_name_from(_args(), dict(world.env)) == "ollama" + + def test_closed_stdin_counts_as_non_interactive(self, world): + world.install("claude-code") + world.monkeypatch.setattr(sys, "stdin", None) + assert _provider_name_from(_args(), dict(world.env)) == "claude-code" + + +class TestConfiguredMeansNoAutoPick: + def test_explicit_flag_gemini_beats_claude_on_path(self, world): + world.install("claude-code") + assert _provider_name_from(_args(llm_provider="gemini"), dict(world.env)) == "gemini" + + def test_explicit_gemini_key_beats_claude_on_path(self, world): + world.install("claude-code") + assert _provider_name_from(_args(), {**world.env, "GEMINI_API_KEY": "k"}) == "gemini" + + def test_resolve_with_gemini_key_stays_gemini(self, world): + world.install("claude-code") + config = resolve_llm_config(_args(), {"GEMINI_API_KEY": "k"}) + assert config.provider == "gemini" + + @pytest.mark.parametrize("var", ["FLUID_LLM_PROVIDER", "FLUID_FORGE_AGENT"]) + def test_env_selector_wins(self, world, var): + world.install("claude-code") + assert _provider_name_from(_args(), {**world.env, var: "codex"}) == "codex" + + def test_two_explicit_keys_are_still_configured(self, world): + # Ambiguous keys fall through to the default, never to an agent. + world.install("claude-code") + env = {**world.env, "OPENAI_API_KEY": "k", "ANTHROPIC_API_KEY": "k"} + assert _provider_name_from(_args(), env) == "gemini" + + def test_stated_ollama_host_is_never_swapped_for_an_agent(self, world): + # Selection treats OLLAMA_HOST as an explicit Ollama choice; the agent + # rung must agree, in both the real pick and the lean (no litellm) pick. + world.install("claude-code") + env = {**world.env, "OLLAMA_HOST": "http://127.0.0.1:11500"} + assert _provider_name_from(_args(), env) == "ollama" + assert providers._lean_keyless_autopick(env, non_interactive=True) is None + + def test_generic_api_key_is_configured(self, world): + world.install("claude-code") + assert _provider_name_from(_args(), {**world.env, "FLUID_LLM_API_KEY": "k"}) == "gemini" + + def test_keyring_key_wins(self, world): + world.install("claude-code") + world.monkeypatch.setattr( + providers, "_get_api_key_from_keyring", lambda n: "k" if n == "anthropic" else None + ) + assert _provider_name_from(_args(), dict(world.env)) == "anthropic" + + def test_ambiguous_keyring_is_still_configured(self, world): + world.install("claude-code") + world.monkeypatch.setattr(providers, "_get_api_key_from_keyring", lambda n: "k") + assert _provider_name_from(_args(), dict(world.env)) == "gemini" + + def test_saved_config_is_still_configured(self, world): + world.install("claude-code") + world.monkeypatch.setattr( + "fluid_build._ai_config_shared.load_ai_config", + lambda *a, **k: {"provider": "openai", "model": "gpt-4.1-mini"}, + ) + assert _provider_name_from(_args(), dict(world.env)) == "gemini" + + +class TestSaysWhichWasPicked: + def test_resolve_keyless_pick_logs_one_line_naming_the_override(self, world, caplog): + world.install("claude-code") + with caplog.at_level(logging.WARNING, logger=providers.LOG.name): + config = resolve_llm_config(_args(), {}) + assert config.provider == "claude-code" + lines = [r.getMessage() for r in caplog.records if "claude-code" in r.getMessage()] + assert len(lines) == 1 + assert "--llm-provider" in lines[0] + assert any("--llm-provider" in note for note in config.model_resolution_notes) + + def test_explicit_agent_choice_is_not_announced(self, world, caplog): + world.install("claude-code") + with caplog.at_level(logging.WARNING, logger=providers.LOG.name): + resolve_llm_config(_args(llm_provider="claude-code"), {}) + assert not [r for r in caplog.records if "--llm-provider" in r.getMessage()] + + +class TestReadinessAgreesWithSelection: + def test_readiness_reports_the_same_provider_non_interactive(self, world): + world.install("codex") + ready = check_llm_readiness({}) + assert ready.provider == _provider_name_from(_args(), dict(world.env)) == "codex" + assert ready.ready is True + assert ready.auth_available is True + + def test_readiness_ready_without_litellm_for_an_agent(self, world): + world.install("claude-code") + with mock.patch("fluid_build._optional.is_available", return_value=False): + ready = check_llm_readiness({}) + assert ready.provider == "claude-code" + assert ready.ready is True + + def test_readiness_on_a_tty_reports_not_configured(self, world): + world.install("claude-code") + _tty(world, True) + ready = check_llm_readiness({}) + assert ready.ready is False + assert ready.provider is None + + def test_readiness_prefers_running_ollama(self, world): + world.install("claude-code") + world.monkeypatch.setattr(providers, "detect_ollama_available", lambda env: True) + assert check_llm_readiness({}).provider == "ollama" + + def test_readiness_nothing_detected(self, world): + assert check_llm_readiness({}).ready is False + + +class TestStatedDestinationBlocksAutoPick: + """A stated endpoint, model or provider key is never swapped for a cloud agent.""" + + _ONPREM = "http://onprem-llm.internal:8000/v1" + + def test_llm_endpoint_flag(self, world): + world.install("claude-code") + assert _provider_name_from(_args(llm_endpoint=self._ONPREM), dict(world.env)) != ( + "claude-code" + ) + + @pytest.mark.parametrize("flag", ["llm_routing_endpoint", "llm_routing_model"]) + def test_routing_flags(self, world, flag): + world.install("claude-code") + assert _provider_name_from(_args(**{flag: self._ONPREM}), dict(world.env)) != ( + "claude-code" + ) + + def test_llm_model_flag(self, world): + world.install("claude-code") + assert _provider_name_from(_args(llm_model="gpt-4o"), dict(world.env)) != "claude-code" + + def test_resolve_keeps_the_stated_endpoint_off_the_agent(self, world): + # The security repro: the agent is not picked, so with no key for the + # default provider the run stops before any call leaves the machine. + world.install("claude-code") + with pytest.raises(providers.CopilotGenerationError) as info: + resolve_llm_config(_args(llm_endpoint=self._ONPREM, llm_model="my-model"), {}) + assert "copilot_missing_llm_api_key" in repr(vars(info.value)) + str(info.value) + + @pytest.mark.parametrize( + "var, value", + [ + ("FLUID_LLM_ENDPOINT", _ONPREM), + ("FLUID_LLM_MODEL", "my-model"), + ("FLUID_LLM_ROUTING_ENDPOINT", _ONPREM), + ("FLUID_LLM_ROUTING_MODEL", "my-model"), + ("GITHUB_API_KEY", "ghp-test"), + ], + ) + def test_env_destination_or_provider_key(self, world, var, value): + world.install("claude-code") + env = {**world.env, var: value} + assert _provider_name_from(_args(), env) != "claude-code" + world.monkeypatch.setenv(var, value) + assert check_llm_readiness({}).provider != "claude-code" + + +class TestAgentLoopNeverAutoPicksAnAgent: + """The agent loop needs tool use, which no coding agent offers. + + It is requested by ``--agent-loop`` or ``FLUID_COPILOT_AGENT_LOOP``, and the + provider is resolved from the options ``_forge_build_copilot_options`` builds, + so both routes must reach the auto-pick. + """ + + @staticmethod + def _options(**kw): + from fluid_build.cli.forge_modes import _forge_build_copilot_options + + options = _forge_build_copilot_options( + SimpleNamespace(**kw), + get_cli_arg_fn=lambda a, k, d=None: getattr(a, k, d), + is_non_interactive=True, + ) + return type("Args", (), options)() + + def test_the_options_carry_the_flag(self): + assert self._options(agent_loop=True).agent_loop is True + assert self._options().agent_loop is False + + @pytest.mark.parametrize("value", ["1", "true", "YES", " on "]) + def test_the_options_carry_the_env_route(self, world, value): + world.monkeypatch.setenv("FLUID_COPILOT_AGENT_LOOP", value) + assert self._options().agent_loop is True + + def test_flag_route_ends_at_the_missing_key_error(self, world): + world.install("claude-code") + with pytest.raises(providers.CopilotGenerationError) as info: + resolve_llm_config(self._options(agent_loop=True), {}) + assert info.value.event == "copilot_missing_llm_api_key" + + def test_env_route_ends_at_the_missing_key_error(self, world): + world.install("claude-code") + world.monkeypatch.setenv("FLUID_COPILOT_AGENT_LOOP", "1") + with pytest.raises(providers.CopilotGenerationError) as info: + resolve_llm_config(self._options(), {}) + assert info.value.event == "copilot_missing_llm_api_key" + + def test_env_route_blocks_the_pick_in_selection(self, world): + world.install("claude-code") + env = {**world.env, "FLUID_COPILOT_AGENT_LOOP": "1"} + assert _provider_name_from(_args(), env) != "claude-code" + + @pytest.mark.parametrize("value", ["0", "false", "no", "off", ""]) + def test_a_falsey_env_value_keeps_the_pick(self, world, value): + world.install("claude-code") + env = {**world.env, "FLUID_COPILOT_AGENT_LOOP": value} + assert _provider_name_from(_args(), env) == "claude-code" + + def test_the_loop_check_is_one_rule(self): + from fluid_build.llm.providers import agent_loop_requested + + assert agent_loop_requested(True, {}) is True + assert agent_loop_requested(False, {"FLUID_COPILOT_AGENT_LOOP": "on"}) is True + assert agent_loop_requested(False, {"FLUID_COPILOT_AGENT_LOOP": "0"}) is False + + +class TestAgentReadinessNeedsTheBinary: + """``fluid doctor`` says "Ready" for a coding agent only when its CLI is on PATH. + + Auth stays each CLI's own job, so no key is looked for. + """ + + @pytest.mark.parametrize("agent", CODING_AGENT_NAMES) + def test_uninstalled_agent_is_not_ready(self, world, agent): + ready = check_llm_readiness({"FLUID_LLM_PROVIDER": agent, "PATH": "/usr/bin:/bin"}) + assert ready.ready is False + assert ready.provider == agent + assert ready.auth_available is False + assert _AGENT_SPECS[agent].binary in ready.error + + @pytest.mark.parametrize("agent", CODING_AGENT_NAMES) + def test_installed_agent_is_ready_without_a_key(self, world, agent): + world.install(agent) + ready = check_llm_readiness({**world.env, "FLUID_LLM_PROVIDER": agent}) + assert ready.ready is True + assert ready.auth_available is True + + def test_installed_agent_is_ready_without_litellm(self, world): + world.install("codex") + with mock.patch("fluid_build._optional.is_available", return_value=False): + ready = check_llm_readiness({**world.env, "FLUID_LLM_PROVIDER": "codex"}) + assert ready.ready is True + + def test_the_binary_is_searched_on_the_given_environments_path(self, world): + # Installed on the process PATH, absent from the environment handed in. + world.install("codex") + ready = check_llm_readiness({"FLUID_LLM_PROVIDER": "codex", "PATH": "/usr/bin:/bin"}) + assert ready.ready is False + + def test_ollama_is_unchanged(self, world): + with mock.patch("fluid_build._optional.is_available", return_value=True): + ready = check_llm_readiness({"FLUID_LLM_PROVIDER": "ollama", "PATH": "/usr/bin:/bin"}) + assert ready.ready is True + assert ready.provider == "ollama" + + def test_mcp_sampling_is_not_ready_through_this_check(self, world): + # It only works inside an MCP host that selects it explicitly. + ready = check_llm_readiness( + {"FLUID_LLM_PROVIDER": "mcp-sampling", "PATH": str(world.bindir)} + ) + assert ready.ready is False + + def test_a_plugin_declaring_keyless_is_still_ready(self, world): + class _Plugin: + name = "myplugin" + default_model = "m" + keyless = True + + def default_endpoint(self, model, env): + return "" + + world.monkeypatch.setitem(providers.BUILTIN_LLM_PROVIDERS, "myplugin", _Plugin()) + with mock.patch("fluid_build._optional.is_available", return_value=True): + ready = check_llm_readiness( + {"FLUID_LLM_PROVIDER": "myplugin", "PATH": str(world.bindir)} + ) + assert ready.ready is True diff --git a/tests/test_slice_ux_j_ai_caching.py b/tests/test_slice_ux_j_ai_caching.py index 01eb8694..7ceda2dc 100644 --- a/tests/test_slice_ux_j_ai_caching.py +++ b/tests/test_slice_ux_j_ai_caching.py @@ -72,13 +72,9 @@ def test_for_routing_re_derives_endpoint_when_no_routing_endpoint(self): routing_model="claude-3-5-haiku-latest", ) routed = cfg.for_routing() - # litellm path: routed endpoint is the ``litellm://anthropic/`` - # sentinel (litellm owns auth + transport, no real URL needed). - # Legacy path: ``https://api.anthropic.com/...`` — accept either. - assert "anthropic" in routed.endpoint - assert ( - routed.endpoint.startswith("litellm://anthropic/") or "anthropic.com" in routed.endpoint - ) + # litellm owns auth + transport, so the re-derived endpoint is empty: + # never a placeholder URL for something to dial. + assert routed.endpoint == "" def test_for_routing_uses_explicit_routing_endpoint(self): cfg = LlmConfig(