From 8fa6a051040c77d50ddfaffcf19dcbbca9d93125 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Wed, 11 Nov 2026 23:02:56 +0100 Subject: [PATCH 1/6] fix(coding-agent): say why a coding agent failed, and run claude -p apart from the user's setup claude -p reports a failure on stdout as a JSON result with is_error: true and leaves stderr empty, so the not-authenticated check (stderr only) never fired and the error read "claude-code exited with code 1:" with no reason. The provider now reads that result: the auth markers are matched against it plus stderr, any other failure carries the agent's own redacted message, and is_error: true fails even at exit code 0. Each claude-code call now replaces Claude Code's agent system prompt (--system-prompt), passes --tools "", --strict-mcp-config, --setting-sources= and --no-session-persistence, and drops CLAUDECODE from the child env, so the user's hooks, plugins, MCP servers and CLAUDE.md no longer reach contract generation. Each flag is used only when the installed claude --help lists it (probed once per process), so older CLIs keep working. Pattern from claude-agent-sdk-python (MIT) @ 3cf3446b0f, subprocess_cli.py _build_command (SDK #778, #732). --- fluid_build/llm/coding_agent.py | 113 ++++++++++- tests/test_coding_agent_isolation.py | 280 +++++++++++++++++++++++++++ tests/test_coding_agent_provider.py | 21 +- 3 files changed, 408 insertions(+), 6 deletions(-) create mode 100644 tests/test_coding_agent_isolation.py diff --git a/fluid_build/llm/coding_agent.py b/fluid_build/llm/coding_agent.py index 7b57bbc4..254ad9eb 100644 --- a/fluid_build/llm/coding_agent.py +++ b/fluid_build/llm/coding_agent.py @@ -47,6 +47,7 @@ import tempfile import threading from dataclasses import dataclass +from functools import lru_cache from typing import Any, Dict, Iterator, List, Optional, Tuple from fluid_build.llm.providers import ( @@ -120,7 +121,7 @@ class AgentSpec: model_flag: str # "--model" | "-m" | "" (passthrough only when explicit) output_mode: str # "claude-json" | "result-json" | "raw" cost_field: Optional[str] # top-level JSON cost field, e.g. "total_cost_usd" - auth_markers: Tuple[str, ...] # stderr substrings -> not_authenticated + auth_markers: Tuple[str, ...] # stderr / reported-error substrings -> not_authenticated login_hint: str # actionable suggestion for missing-key / not-authenticated # Endpoint variables the agent reads, and the key variables it sends there # instead of its own stored login. A project dotenv file's endpoint reaches @@ -128,6 +129,16 @@ class AgentSpec: # ``_agent_environment``. endpoint_env: Tuple[str, ...] = () endpoint_key_env: Tuple[str, ...] = () + # Isolation from the user's own agent config (hooks, plugins, MCP servers, + # CLAUDE.md) and from the agent's default system prompt. Each entry is the + # argv tokens of one flag, passed only when the installed CLI's ``--help`` + # lists the flag (``_cli_help``), so older versions keep working. + isolation_args: Tuple[Tuple[str, ...], ...] = () + # Flag that REPLACES the default system prompt; ``system_prompt_flag`` (which + # only appends) is the fallback when the CLI does not list it. + system_prompt_replace_flag: str = "" + # Variables removed from the child environment. + drop_env: Tuple[str, ...] = () _AGENT_SPECS: Dict[str, AgentSpec] = { @@ -150,6 +161,9 @@ class AgentSpec: auth_markers=( "not logged in", "please run /login", + "/login", + "failed to authenticate", + "oauth session expired", "oauth", "unauthorized", "invalid api key", @@ -158,6 +172,21 @@ class AgentSpec: login_hint="Run `claude` once and sign in (subscription), or set ANTHROPIC_API_KEY.", endpoint_env=("ANTHROPIC_BASE_URL",), endpoint_key_env=("ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_API_KEY"), + # Borrowed from claude-agent-sdk-python (MIT) @ 3cf3446b0f, + # src/claude_agent_sdk/_internal/transport/subprocess_cli.py + # ``_build_command``: replace the system prompt, no tools, strict MCP, + # no filesystem settings. An EMPTY setting-sources list must be the + # single token ``--setting-sources=``: a separate "" argument misparses + # the flags after it (SDK #778). The SDK also drops CLAUDECODE from the + # child env (SDK #732). + isolation_args=( + ("--tools", ""), + ("--strict-mcp-config",), + ("--setting-sources=",), + ("--no-session-persistence",), + ), + system_prompt_replace_flag="--system-prompt", + drop_env=("CLAUDECODE",), ), "codex": AgentSpec( name="codex", @@ -295,6 +324,44 @@ def _run_agent( return proc.returncode, proc.stdout, proc.stderr +_HELP_TIMEOUT_SECONDS = 10 + + +def _run_help(binary: str) -> str: + """`` --help`` text, or ``""`` when it cannot be read. + + The second subprocess seam (tests monkeypatch it, like ``_run_agent``). + """ + try: + proc = subprocess.run( # noqa: S603 — list argv, no shell, fixed binary set + [binary, "--help"], + capture_output=True, + text=True, + timeout=_HELP_TIMEOUT_SECONDS, + cwd=tempfile.gettempdir(), + check=False, + ) + except (OSError, subprocess.SubprocessError): + return "" + return f"{proc.stdout or ''}\n{proc.stderr or ''}" + + +@lru_cache(maxsize=None) +def _cli_help(binary: str) -> str: + """Help text of ``binary``, probed once per process.""" + return _run_help(binary) + + +def _lists_flag(help_text: str, flag: str) -> bool: + """True when ``flag`` appears in ``help_text`` as a whole flag name. + + ``--tools`` must not match ``--allowedTools``, nor ``--system-prompt`` match + ``--append-system-prompt``. + """ + name = flag.split("=", 1)[0] + return re.search(rf"(? Dict[str, str]: CODEX_API_KEY in place of its login. """ env = dict(os.environ) + for key in spec.drop_env: + env.pop(key, None) if not spec.endpoint_env: return env from fluid_build.credentials.dotenv_store import project_supplied @@ -417,6 +486,27 @@ def _extract_envelope(spec: AgentSpec, stdout: str) -> str: return text # "raw" — stdout is the final message verbatim +def _reported_error(spec: AgentSpec, stdout: str) -> Optional[str]: + """The agent's own failure report from its JSON stdout, else ``None``. + + ``claude -p --output-format json`` reports a failure as a result object with + ``is_error: true`` on STDOUT (its ``subtype`` can still say ``success`` and + stderr stays empty). Returns the ``result`` text (``""`` when absent) so the + caller treats it as a failure even at exit code 0, before the envelope parser + sees it. + """ + if spec.output_mode not in ("claude-json", "result-json"): + return None + try: + data = json.loads(_ANSI_RE.sub("", stdout or "")) + except (ValueError, TypeError): + return None + if not isinstance(data, dict) or data.get("is_error") is not True: + return None + result = data.get("result") + return result.strip() if isinstance(result, str) else "" + + def _record_agent_usage_and_cost(spec: AgentSpec, stdout: str) -> None: """Stash Claude's per-call USD + bump the shared usage counters. @@ -528,9 +618,19 @@ def invoke_blocking( if spec.model_flag and config.model and config.model != spec.name: argv += [spec.model_flag, str(config.model)] + help_text = _cli_help(binary) if spec.isolation_args else "" + for tokens in spec.isolation_args: + if _lists_flag(help_text, tokens[0]): + argv += tokens + combined_prompt = user_prompt if spec.system_prompt_mode == "flag": - argv += [spec.system_prompt_flag, system_prompt] + system_flag = spec.system_prompt_flag + if spec.system_prompt_replace_flag and _lists_flag( + help_text, spec.system_prompt_replace_flag + ): + system_flag = spec.system_prompt_replace_flag + argv += [system_flag, system_prompt] else: combined_prompt = f"{system_prompt}{_PROMPT_DELIM}{user_prompt}" @@ -581,17 +681,20 @@ def invoke_blocking( except OSError: # pragma: no cover — best-effort cleanup pass - if rc != 0: - lowered = (stderr or "").lower() + reported = _reported_error(spec, stdout) + if rc != 0 or reported is not None: + reason = "\n".join(part for part in (reported, (stderr or "").strip()) if part) + lowered = reason.lower() if any(marker in lowered for marker in spec.auth_markers): raise CopilotGenerationError( "coding_agent_not_authenticated", f"{spec.name} is installed but not authenticated.", suggestions=[spec.login_hint], ) + outcome = f"exited with code {rc}" if rc != 0 else "reported an error" raise CopilotGenerationError( "coding_agent_failed", - f"{spec.name} exited with code {rc}: {_redact(stderr)[:500]}", + f"{spec.name} {outcome}: {_redact(reason)[:500]}", suggestions=[ f"Run `{spec.binary}` manually to confirm it works.", "Or fall back to an API key / Ollama.", diff --git a/tests/test_coding_agent_isolation.py b/tests/test_coding_agent_isolation.py new file mode 100644 index 00000000..adb23b04 --- /dev/null +++ b/tests/test_coding_agent_isolation.py @@ -0,0 +1,280 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""Claude Code driver: failure reporting on stdout, and isolation from user config. + +No real CLI is spawned: ``_run_agent`` (the agent call) and ``_run_help`` (the +one-shot ``--help`` probe) are the two seams, both monkeypatched. +""" + +from __future__ import annotations + +import json + +import pytest + +from fluid_build.llm import coding_agent as cap +from fluid_build.llm.providers import CopilotGenerationError, LlmConfig + +pytestmark = pytest.mark.unit + +_CLAUDE_AUTH_FAILURE = ( + '{"type":"result","subtype":"success","is_error":true,' + '"result":"Failed to authenticate: OAuth session expired and could not be refreshed",' + '"num_turns":1,"duration_ms":105}' +) + +_ISOLATION_FLAGS = ( + "--tools", + "--strict-mcp-config", + "--setting-sources", + "--no-session-persistence", +) + +_FULL_HELP = """\ +Usage: claude [options] [prompt] + -p, --print Print response and exit + --system-prompt System prompt to use for the session + --append-system-prompt Append a system prompt + --allowedTools Tools to allow + --tools Specify the list of available tools + --strict-mcp-config Only use MCP servers from --mcp-config + --setting-sources Comma-separated list of setting sources + --no-session-persistence Disable session persistence +""" + + +class _Recorder: + def __init__(self, *, rc=0, stdout="", stderr=""): + self.rc, self.stdout, self.stderr = rc, stdout, stderr + self.calls = [] + + def __call__(self, argv, *, stdin, timeout, cwd, env): + self.calls.append({"argv": list(argv), "stdin": stdin, "env": env}) + return (self.rc, self.stdout, self.stderr) + + +def _cfg(): + return LlmConfig( + provider="claude-code", + model="claude-code", + endpoint="coding-agent://claude-code", + api_key=None, + ) + + +@pytest.fixture +def run(monkeypatch): + """Install a recorder; returns ``(recorder, set_help, help_probe_calls)``.""" + cap._cli_help.cache_clear() + probes = [] + state = {"help": ""} + + def _help(binary): + probes.append(binary) + return state["help"] + + monkeypatch.setattr(cap, "_run_help", _help) + monkeypatch.setattr(cap.shutil, "which", lambda b: f"/usr/local/bin/{b}") + + def _install(recorder, help_text=""): + state["help"] = help_text + monkeypatch.setattr(cap, "_run_agent", recorder) + return recorder + + yield _install, probes + cap._cli_help.cache_clear() + + +def _invoke(system="SYS", user="USER"): + return cap.get_coding_agent_provider("claude-code").invoke_blocking(_cfg(), system, user) + + +# --------------------------------------------------------------------------- +# Failures reported on stdout +# --------------------------------------------------------------------------- + + +def test_stdout_auth_failure_is_not_authenticated_with_hint(run): + install, _ = run + install(_Recorder(rc=1, stdout=_CLAUDE_AUTH_FAILURE, stderr="")) + with pytest.raises(CopilotGenerationError) as exc: + _invoke() + assert exc.value.event == "coding_agent_not_authenticated" + assert cap._AGENT_SPECS["claude-code"].login_hint in exc.value.suggestions + + +def test_stdout_error_with_other_text_is_failed_and_carries_the_text(run): + install, _ = run + out = json.dumps({"type": "result", "is_error": True, "result": "Prompt is too long"}) + install(_Recorder(rc=1, stdout=out)) + with pytest.raises(CopilotGenerationError) as exc: + _invoke() + assert exc.value.event == "coding_agent_failed" + assert "Prompt is too long" in exc.value.message + + +def test_stdout_error_text_is_redacted_and_capped(run): + install, _ = run + out = json.dumps({"is_error": True, "result": "word " * 400}) + install(_Recorder(rc=1, stdout=out)) + with pytest.raises(CopilotGenerationError) as exc: + _invoke() + assert exc.value.event == "coding_agent_failed" + # "claude-code exited with code 1: " + at most 500 characters of reason + assert len(exc.value.message) <= len("claude-code exited with code 1: ") + 500 + + +def test_exit_zero_with_is_error_is_a_clean_failure(run): + install, _ = run + out = json.dumps({"is_error": True, "result": "Something broke"}) + install(_Recorder(rc=0, stdout=out)) + with pytest.raises(CopilotGenerationError) as exc: + _invoke() + assert exc.value.event == "coding_agent_failed" + assert "Something broke" in exc.value.message + + +def test_exit_zero_auth_is_error_is_not_authenticated(run): + install, _ = run + install(_Recorder(rc=0, stdout=_CLAUDE_AUTH_FAILURE)) + with pytest.raises(CopilotGenerationError) as exc: + _invoke() + assert exc.value.event == "coding_agent_not_authenticated" + + +def test_stderr_is_combined_with_the_reported_reason(run): + install, _ = run + out = json.dumps({"is_error": True, "result": "boom"}) + install(_Recorder(rc=1, stdout=out, stderr="Please run /login")) + with pytest.raises(CopilotGenerationError) as exc: + _invoke() + assert exc.value.event == "coding_agent_not_authenticated" + + +def test_stderr_only_failure_still_works(run): + install, _ = run + install(_Recorder(rc=1, stderr="Error: invalid API key")) + with pytest.raises(CopilotGenerationError) as exc: + _invoke() + assert exc.value.event == "coding_agent_not_authenticated" + install(_Recorder(rc=2, stderr="some unexpected explosion")) + with pytest.raises(CopilotGenerationError) as exc: + _invoke() + assert exc.value.event == "coding_agent_failed" + assert "some unexpected explosion" in exc.value.message + + +@pytest.mark.parametrize( + "text", + [ + "Failed to authenticate", + "OAuth session expired", + "Not logged in", + "Invalid API key", + "run /login to continue", + ], +) +def test_claude_auth_markers_cover_the_known_wordings(text): + markers = cap._AGENT_SPECS["claude-code"].auth_markers + assert any(m in text.lower() for m in markers) + + +def test_is_error_false_is_not_a_failure(run): + install, _ = run + install(_Recorder(stdout=json.dumps({"is_error": False, "result": "{}"}))) + assert _invoke() == "{}" + + +# --------------------------------------------------------------------------- +# Isolation flags +# --------------------------------------------------------------------------- + + +def test_all_isolation_flags_present_when_help_lists_them(run): + install, _ = run + rec = install(_Recorder(stdout=json.dumps({"result": "{}"})), _FULL_HELP) + _invoke("THE-SYSTEM-PROMPT", "THE-USER-PROMPT") + argv = rec.calls[0]["argv"] + assert argv[argv.index("--tools") + 1] == "" + assert "--strict-mcp-config" in argv + assert "--no-session-persistence" in argv + # the system prompt replaces the default one instead of being appended + assert argv[argv.index("--system-prompt") + 1] == "THE-SYSTEM-PROMPT" + assert "--append-system-prompt" not in argv + # unchanged: stdin prompt, json output, schema + assert rec.calls[0]["stdin"] == "THE-USER-PROMPT" + assert "--json-schema" in argv and "--output-format" in argv + + +def test_setting_sources_is_one_token(run): + install, _ = run + rec = install(_Recorder(stdout=json.dumps({"result": "{}"})), _FULL_HELP) + _invoke() + argv = rec.calls[0]["argv"] + assert "--setting-sources=" in argv + assert "--setting-sources" not in argv + + +@pytest.mark.parametrize("missing", _ISOLATION_FLAGS) +def test_a_flag_help_lacks_is_omitted(run, missing): + install, _ = run + help_text = "\n".join(line for line in _FULL_HELP.splitlines() if missing not in line) + rec = install(_Recorder(stdout=json.dumps({"result": "{}"})), help_text) + _invoke() + argv = rec.calls[0]["argv"] + assert not any(tok.split("=")[0] == missing for tok in argv) + for flag in _ISOLATION_FLAGS: + if flag != missing: + assert any(tok.split("=")[0] == flag for tok in argv) + + +def test_tools_is_not_matched_by_allowed_tools(run): + install, _ = run + rec = install( + _Recorder(stdout=json.dumps({"result": "{}"})), " --allowedTools Tools\n" + ) + _invoke() + assert "--tools" not in rec.calls[0]["argv"] + + +def test_old_claude_without_system_prompt_flag_keeps_append_fallback(run): + install, _ = run + rec = install( + _Recorder(stdout=json.dumps({"result": "{}"})), + " --append-system-prompt Append\n", + ) + _invoke("SYS") + argv = rec.calls[0]["argv"] + assert argv[argv.index("--append-system-prompt") + 1] == "SYS" + assert "--system-prompt" not in argv + + +def test_child_env_lacks_claudecode(run, monkeypatch): + install, _ = run + monkeypatch.setenv("CLAUDECODE", "1") + rec = install(_Recorder(stdout=json.dumps({"result": "{}"})), _FULL_HELP) + _invoke() + assert "CLAUDECODE" not in rec.calls[0]["env"] + + +def test_other_agents_keep_their_env_and_argv(run, monkeypatch): + install, probes = run + monkeypatch.setenv("CLAUDECODE", "1") + rec = install(_Recorder(stdout="TEXT"), _FULL_HELP) + cap.get_coding_agent_provider("codex").invoke_blocking( + LlmConfig(provider="codex", model="codex", endpoint="x", api_key=None), "SYS", "USER" + ) + argv = rec.calls[0]["argv"] + assert rec.calls[0]["env"].get("CLAUDECODE") == "1" + assert not any(tok.startswith("--setting-sources") for tok in argv) + assert "--strict-mcp-config" not in argv + assert probes == [] # no help probe for an agent without isolation flags + + +def test_help_probe_runs_once_across_two_invocations(run): + install, probes = run + install(_Recorder(stdout=json.dumps({"result": "{}"})), _FULL_HELP) + _invoke() + _invoke() + assert probes == ["/usr/local/bin/claude"] diff --git a/tests/test_coding_agent_provider.py b/tests/test_coding_agent_provider.py index ed60427f..5d9ccbe8 100644 --- a/tests/test_coding_agent_provider.py +++ b/tests/test_coding_agent_provider.py @@ -74,6 +74,15 @@ def _argv(rec): return rec.calls[0]["argv"] +@pytest.fixture(autouse=True) +def _no_real_help_probe(monkeypatch): + """Never run the real ``claude --help``; an empty help lists no flag.""" + cap._cli_help.cache_clear() + monkeypatch.setattr(cap, "_run_help", lambda binary: "") + yield + cap._cli_help.cache_clear() + + # --------------------------------------------------------------------------- # Per-agent argv + output parsing # --------------------------------------------------------------------------- @@ -103,7 +112,8 @@ def test_claude_code_argv_and_structured_output(monkeypatch): assert "--json-schema" in argv schema_str = argv[argv.index("--json-schema") + 1] assert "contract" in schema_str and "recommended_template" in schema_str - # system via flag, user via stdin + # system via flag (help lists no --system-prompt here, so the append fallback), + # user via stdin assert "--append-system-prompt" in argv assert argv[argv.index("--append-system-prompt") + 1] == "SYS-PROMPT" assert rec.calls[0]["stdin"] == "USER-PROMPT" @@ -224,6 +234,15 @@ def test_not_authenticated_marker(monkeypatch): assert exc.value.event == "coding_agent_not_authenticated" +def test_not_authenticated_marker_wording_from_stderr_only(monkeypatch): + rec = _Recorder(rc=1, stderr="Failed to authenticate: OAuth session expired") + _install(monkeypatch, rec) + provider = get_llm_provider("claude-code") + with pytest.raises(CopilotGenerationError) as exc: + provider.invoke_blocking(_cfg("claude-code"), "SYS", "USER") + assert exc.value.event == "coding_agent_not_authenticated" + + def test_generic_failure(monkeypatch): rec = _Recorder(rc=2, stderr="some unexpected explosion") _install(monkeypatch, rec) From f2aac8eda8601a9eb6e96e6af3cd24401c3f38b6 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Wed, 11 Nov 2026 23:02:56 +0100 Subject: [PATCH 2/6] fix(llm): use only real endpoints, and auto-pick an installed coding agent when nothing is configured LlmConfig.endpoint carried the telemetry label litellm:/// whenever no endpoint was set. Ollama sent it to litellm as api_base, so no call reached the daemon, and fluid ai test posted to it and failed with UnsupportedProtocol. The endpoint now holds only a real address (the stated one, else OLLAMA_HOST, else litellm's own default). OLLAMA_HOST is read as ollama-python's _parse_host reads it (MIT), routing calls keep a stated endpoint, fluid ai test probes through the provider call and exits 4 on a 404, and doctor shows "Provider default". A non-interactive run with nothing configured now uses a running local Ollama first, else the first coding-agent CLI on PATH (claude-code, codex, cursor, kiro), and logs which. A key, a keyring entry, a saved provider, OLLAMA_HOST, or a stated endpoint or model (flags or env, the routing ones too) blocks it, as does the agent loop. The litellm preflight lets it through on the base install, where the agents are the only provider that can run. Readiness reports an agent ready only when its CLI is on PATH. --- fluid_build/cli/_ai_setup_test.py | 76 ++++- fluid_build/cli/doctor.py | 5 +- fluid_build/cli/forge_modes.py | 14 +- fluid_build/llm/litellm_backend.py | 75 ++++- fluid_build/llm/provider_resolve.py | 133 ++++++++- fluid_build/llm/providers.py | 122 ++++++-- tests/test_ai_setup.py | 6 + tests/test_forge_copilot_runtime.py | 16 +- tests/test_litellm_absent.py | 113 ++++++++ tests/test_llm_endpoint_sentinel.py | 416 +++++++++++++++++++++++++++ tests/test_llm_keyless_autopick.py | 426 ++++++++++++++++++++++++++++ tests/test_slice_ux_j_ai_caching.py | 10 +- 12 files changed, 1345 insertions(+), 67 deletions(-) create mode 100644 tests/test_llm_endpoint_sentinel.py create mode 100644 tests/test_llm_keyless_autopick.py diff --git a/fluid_build/cli/_ai_setup_test.py b/fluid_build/cli/_ai_setup_test.py index 553635d9..7386e932 100644 --- a/fluid_build/cli/_ai_setup_test.py +++ b/fluid_build/cli/_ai_setup_test.py @@ -191,11 +191,6 @@ def _validate_ai_test_endpoint(provider_name: str, endpoint: str) -> Optional[st parsed = urlsplit(raw) if parsed.username or parsed.password: return "AI test endpoint URLs must not embed credentials." - # litellm-routed providers (the default) use a ``litellm://`` sentinel - # that's not a real URL — litellm owns auth + transport, so there's no - # plaintext-credential risk to validate against. Accept the sentinel. - if parsed.scheme == "litellm": - return None if parsed.scheme not in {"http", "https"} or not parsed.netloc: return "AI test endpoint must be an absolute HTTP(S) URL." host = (parsed.hostname or "").lower() @@ -253,10 +248,13 @@ def _build_for_provider( host = saved_for_provider.get("ollama_host") if host and not os.environ.get("OLLAMA_HOST"): os.environ["OLLAMA_HOST"] = _sanitize_ollama_host(str(host)) + # Only an operator-supplied endpoint is validated; with none, the + # call goes through litellm, which owns the address. + if endpoint_override: + endpoint_error = _validate_ai_test_endpoint(pname, endpoint_override) + if endpoint_error: + return None, endpoint_error endpoint = endpoint_override or provider.default_endpoint(model, dict(os.environ)) - endpoint_error = _validate_ai_test_endpoint(pname, endpoint) - if endpoint_error: - return None, endpoint_error return ( LlmConfig( provider=pname, @@ -272,9 +270,10 @@ def _build_for_provider( return None, f"No API key found for {PROVIDER_DISPLAY_NAMES.get(pname, pname)}." endpoint = endpoint_override or saved_for_provider.get("endpoint") config = _make_cloud_config(pname, api_key, model=model, endpoint=endpoint) - endpoint_error = _validate_ai_test_endpoint(pname, config.endpoint) - if endpoint_error: - return None, endpoint_error + if config.endpoint: + endpoint_error = _validate_ai_test_endpoint(pname, config.endpoint) + if endpoint_error: + return None, endpoint_error return ( config, None, @@ -391,8 +390,48 @@ def _check_ai_test_model_availability( return "available", available +def _is_litellm_backed(provider: Any) -> bool: + from fluid_build.llm.litellm_backend import LiteLLMProvider + + return isinstance(provider, LiteLLMProvider) + + +def _run_litellm_smoke_call(provider: Any, config: LlmConfig) -> tuple[str, dict, int]: + """The diagnostic call for a litellm-backed provider: the call forge itself makes. + + litellm owns the address, auth and transport, so there is no URL to POST to. + """ + from fluid_build.llm.providers import get_cumulative_token_usage + + cap = ( + _AI_TEST_GEMINI_OUTPUT_TOKENS + if config.provider == "gemini" + else _AI_TEST_DEFAULT_OUTPUT_TOKENS + ) + before = get_cumulative_token_usage() + started = time.perf_counter() + text = provider.invoke_blocking( + config, + _AI_TEST_SYSTEM_PROMPT, + _AI_TEST_USER_PROMPT, + extra_payload={"max_tokens": cap, "timeout": config.timeout_seconds}, + ) + latency_ms = int((time.perf_counter() - started) * 1000) + after = get_cumulative_token_usage() + usage = {key: after[key] - before[key] for key in after} + if "FLUID_OK" not in (text or ""): + raise CopilotGenerationError( + "ai_test_unexpected_response", + f"AI test response from {config.provider} was not the expected diagnostic token.", + suggestions=["Try again, or choose a different model with `fluid ai setup`."], + ) + return "FLUID_OK", usage, latency_ms + + def _run_ai_smoke_call(provider: Any, config: LlmConfig) -> tuple[str, dict, int]: """Issue the diagnostic call. Returns (text, usage, latency_ms).""" + if _is_litellm_backed(provider): + return _run_litellm_smoke_call(provider, config) headers, payload = _with_freeform_ai_test_payload(provider, config) started = time.perf_counter() try: @@ -454,6 +493,12 @@ def _classify_ai_test_error(error_code: str, status_code: Optional[int]) -> int: 0 OK / 2 config / 3 auth / 4 resource / 5 network. See AGENTS.md for the contract. """ + if error_code in {"copilot_llm_auth_failed", "copilot_llm_permission_denied"}: + return _AI_TEST_EXIT_AUTH + if error_code == "copilot_litellm_request_failed": + # A provider "not found" (a model it does not have) is a resource problem; + # timeouts, connection errors and the rest stay network. + return _AI_TEST_EXIT_RESOURCE if status_code == 404 else _AI_TEST_EXIT_NETWORK if error_code in {"ai_test_request_failed", "ai_test_model_preflight_failed"}: if status_code is None: return _AI_TEST_EXIT_NETWORK @@ -471,7 +516,10 @@ def _http_status_from_cause(exc: CopilotGenerationError) -> Optional[int]: cause = exc.__cause__ if isinstance(cause, httpx.HTTPStatusError): return cause.response.status_code - return None + # litellm's exceptions (the original error, chained by ``_translate_litellm_exception``) + # carry the provider's HTTP status. + status = getattr(cause, "status_code", None) + return status if isinstance(status, int) else None def _new_ai_test_report( @@ -653,7 +701,7 @@ def run_ai_test(console: Any, args: Any) -> tuple[int, dict]: table.add_column("Result", style="green") table.add_row("Provider", label) table.add_row("Model", _safe_ai_test_display(config.model)) - table.add_row("Endpoint", _safe_ai_test_display(endpoint)) + table.add_row("Endpoint", _safe_ai_test_display(endpoint) or "provider default") table.add_row( "Model availability", ( @@ -673,7 +721,7 @@ def run_ai_test(console: Any, args: Any) -> tuple[int, dict]: from fluid_build.cli.console import cprint cprint(f"AI Provider Test: ready ({label}, {_safe_ai_test_display(config.model)})") - cprint(f" Endpoint: {_safe_ai_test_display(endpoint)}") + cprint(f" Endpoint: {_safe_ai_test_display(endpoint) or 'provider default'}") cprint(f" Model availability: {availability}") cprint(f" Live call: {text}") cprint(f" Latency: {latency_text}") diff --git a/fluid_build/cli/doctor.py b/fluid_build/cli/doctor.py index 7a438978..f9871ee8 100644 --- a/fluid_build/cli/doctor.py +++ b/fluid_build/cli/doctor.py @@ -1411,7 +1411,10 @@ def _print_copilot_readiness(readiness: LlmReadinessCheck, verbose: bool = False """Render the copilot readiness summary without leaking secrets.""" status_text = "✅ Ready" if readiness.ready else "⚠️ Setup needed" auth_text = "Configured" if readiness.auth_available else "Missing" - endpoint_text = readiness.endpoint or "Not configured" + # No endpoint of ours is not "not configured": litellm picks the provider's own. + endpoint_text = readiness.endpoint or ( + "Provider default" if readiness.provider else "Not configured" + ) if RICH_AVAILABLE: console = Console() diff --git a/fluid_build/cli/forge_modes.py b/fluid_build/cli/forge_modes.py index fd1a23dc..eab6baf7 100644 --- a/fluid_build/cli/forge_modes.py +++ b/fluid_build/cli/forge_modes.py @@ -306,6 +306,8 @@ def _forge_build_copilot_options( Extracted verbatim from ``run_ai_copilot_mode`` (behaviour-preserving) to shrink that god-function; the returned dict is mutated downstream. """ + from fluid_build.llm.providers import agent_loop_requested + return { "llm_provider": get_cli_arg_fn(args, "llm_provider"), "llm_model": get_cli_arg_fn(args, "llm_model"), @@ -322,6 +324,9 @@ def _forge_build_copilot_options( "fragment_first": bool(get_cli_arg_fn(args, "fragments", False)), "no_fragments": bool(get_cli_arg_fn(args, "no_fragments", False)), "no_generate": bool(get_cli_arg_fn(args, "no_generate", False)), + # The provider is resolved from these options, and the agent loop (by flag + # or FLUID_COPILOT_AGENT_LOOP) must never auto-pick a coding agent. + "agent_loop": agent_loop_requested(get_cli_arg_fn(args, "agent_loop", False)), # Pre-write preview UX (invariants I1/I4/I5): # auto_yes — skip the [Y/n] prompt, panel still renders # show_work — stream reasoning/tool calls live as the agent runs @@ -1034,12 +1039,9 @@ def _forge_project_creation_core( context = rc.context copilot_options = rc.copilot_options perf_stats = rc.perf_stats - # Parsed as a boolean, not by presence: ``FLUID_COPILOT_AGENT_LOOP=0`` is - # the documented way to turn the loop off, and a bare ``bool()`` of the - # string turned it on. - use_agent_loop = bool(get_cli_arg_fn(args, "agent_loop", False)) or os.environ.get( - "FLUID_COPILOT_AGENT_LOOP", "" - ).strip().lower() in {"1", "true", "yes", "on"} + from fluid_build.llm.providers import agent_loop_requested + + use_agent_loop = agent_loop_requested(get_cli_arg_fn(args, "agent_loop", False)) if scaffold_template: success_result = copilot.create_project( diff --git a/fluid_build/llm/litellm_backend.py b/fluid_build/llm/litellm_backend.py index f4ffee96..d4a8da48 100644 --- a/fluid_build/llm/litellm_backend.py +++ b/fluid_build/llm/litellm_backend.py @@ -18,12 +18,14 @@ from __future__ import annotations +import ipaddress import json import logging import os import sys import threading import time +import urllib.parse from typing import Any, Dict, Iterator, List, Mapping, Optional, Tuple from fluid_build.llm.providers import ( @@ -255,6 +257,43 @@ def _is_anthropic_model(model_id: str) -> bool: return "claude" in lower +def _ollama_base_url(env: Mapping[str, str]) -> str: + """``OLLAMA_HOST`` as a URL, or ``""`` when unset. + + Ollama's own variable accepts forms litellm and ``urllib`` cannot dial as + they are (a bare ``host``, ``host:port``, ``:port``), so it is normalised + with the convention of ollama-python's ``_parse_host`` (MIT, + github.com/ollama/ollama-python, ``ollama/_client.py``): no scheme means + ``http`` on the default port 11434; an explicit ``http://`` or ``https://`` + without a port means port 80 or 443; a path is kept. Both reads of + ``OLLAMA_HOST`` in this module go through here, so they agree. + """ + host = (env.get("OLLAMA_HOST") or "").strip() + if not host: + return "" + port = 11434 + scheme, _, hostport = host.partition("://") + if not hostport: + scheme, hostport = "http", host + elif scheme == "http": + port = 80 + elif scheme == "https": + port = 443 + try: + split = urllib.parse.urlsplit(f"{scheme}://{hostport}") + name = split.hostname or "127.0.0.1" + port = split.port or port + except ValueError: # e.g. a non-numeric port: leave it to fail visibly at the dial + return f"{scheme}://{hostport}".rstrip("/") + try: + if isinstance(ipaddress.ip_address(name), ipaddress.IPv6Address): + name = f"[{name}]" # urlsplit strips the brackets + except ValueError: + pass # a hostname, not an IP literal + path = split.path.strip("/") + return f"{scheme}://{name}:{port}" + (f"/{path}" if path else "") + + # --------------------------------------------------------------------------- # Adapter # --------------------------------------------------------------------------- @@ -283,8 +322,27 @@ def __init__(self, provider_name: str, *, default_model: str = ""): # ----- LlmProvider abstract API ------------------------------------ def default_endpoint(self, model: str, env: Mapping[str, str]) -> str: - """litellm owns auth/endpoints; return a sentinel for telemetry.""" - return f"litellm://{self.name}/{model}" + """The endpoint the operator pointed us at; empty when litellm picks it. + + litellm owns auth and transport, so a hosted provider has no endpoint of + ours to report and ``LlmConfig.endpoint`` stays empty. Ollama is the one + exception: its own ``OLLAMA_HOST`` is a real daemon address, so it is + reported. Never a placeholder URL: every consumer of ``config.endpoint`` + treats a non-empty value as an address to dial. + """ + return _ollama_base_url(env) if self.name == "ollama" else "" + + def _apply_ollama_api_base(self, payload: Dict[str, Any], config: LlmConfig) -> None: + """Point an Ollama call at an explicit endpoint, else ``OLLAMA_HOST``. + + With neither set, ``api_base`` is left out and litellm uses its own + default (``http://localhost:11434``, ``litellm/llms/ollama/common_utils.py``). + """ + if self.name != "ollama": + return + api_base = config.endpoint or _ollama_base_url(os.environ) + if api_base: + payload["api_base"] = api_base def build_request( self, config: LlmConfig, system_prompt: str, user_prompt: str @@ -312,11 +370,7 @@ def build_request( payload["max_tokens"] = int(max_tokens) if config.api_key: payload["api_key"] = config.api_key - # Ollama: litellm needs the daemon endpoint as api_base. - if self.name == "ollama": - payload["api_base"] = config.endpoint or os.environ.get( - "OLLAMA_HOST", "http://localhost:11434" - ) + self._apply_ollama_api_base(payload, config) _maybe_inject_cache_control(payload, model_id) return ({}, payload) @@ -502,10 +556,7 @@ def build_tool_request( } if config.api_key: payload["api_key"] = config.api_key - if self.name == "ollama": - payload["api_base"] = config.endpoint or os.environ.get( - "OLLAMA_HOST", "http://localhost:11434" - ) + self._apply_ollama_api_base(payload, config) _maybe_inject_cache_control(payload, model_id) return ("litellm://internal", {}, payload) @@ -734,7 +785,7 @@ def _list_ollama_models(env: Mapping[str, str]) -> Optional[List[str]]: reachable so preflight degrades to "unavailable" rather than falsely rejecting models. """ - base = env.get("OLLAMA_HOST", "http://localhost:11434").rstrip("/") + base = _ollama_base_url(env) or "http://localhost:11434" try: import urllib.error import urllib.request diff --git a/fluid_build/llm/provider_resolve.py b/fluid_build/llm/provider_resolve.py index 2b13110a..04691d7d 100644 --- a/fluid_build/llm/provider_resolve.py +++ b/fluid_build/llm/provider_resolve.py @@ -28,6 +28,8 @@ from __future__ import annotations import logging +import shutil +import sys from typing import Mapping, Optional LOG = logging.getLogger("fluid.cli.llm_providers.resolve") @@ -91,8 +93,121 @@ def _infer_provider_from_explicit_keys(env: Mapping[str, str]) -> Optional[str]: return None -def _infer_provider_from_ambient(env: Mapping[str, str]) -> Optional[str]: - """Last-resort detection: Ollama already running on localhost. +def _is_non_interactive(non_interactive: bool = False) -> bool: + """True when the run cannot answer a prompt: ``--non-interactive`` or no TTY on stdin.""" + if non_interactive: + return True + try: + return not sys.stdin.isatty() + except (AttributeError, ValueError): # no stdin, or a closed one + return True + + +# Variables that mean the operator configured an LLM: an API key, or a stated +# endpoint, model or Ollama host. A coding agent is never auto-picked over any of them, so a +# run never sends its prompt somewhere other than the destination it was given. +_CONFIG_ENV_VARS = ( + "FLUID_LLM_API_KEY", + "FLUID_LLM_ENDPOINT", + "FLUID_LLM_MODEL", + "FLUID_LLM_ROUTING_ENDPOINT", + "FLUID_LLM_ROUTING_MODEL", + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + "GEMINI_API_KEY", + "GOOGLE_API_KEY", + "OLLAMA_HOST", +) + + +def _configured_by_env(env: Mapping[str, str]) -> bool: + """Whether the environment configures an LLM (see :data:`_CONFIG_ENV_VARS`). + + Every provider's key in the host's ``PROVIDER_ENV_VARS`` counts too. + """ + provider_keys = tuple(getattr(_host(), "PROVIDER_ENV_VARS", {}).values()) + return any(env.get(var) for var in _CONFIG_ENV_VARS + provider_keys) + + +def _operator_configured_a_provider(env: Mapping[str, str]) -> bool: + """Whether anything but ambient discovery already names a provider. + + An API-key, endpoint or model env var (even several keys, which + :func:`_infer_provider_from_explicit_keys` refuses to choose between), a + keyring key, or a saved ``~/.fluid`` provider. A coding agent is never + auto-picked over any of these. + """ + if _configured_by_env(env): + return True + keyring_fn = getattr(_host(), "_get_api_key_from_keyring", _get_api_key_from_keyring) + if any(keyring_fn(name) for name in ("openai", "anthropic", "gemini")): + return True + try: + from fluid_build._ai_config_shared import load_ai_config + + saved = load_ai_config() + except Exception: # noqa: BLE001 — detection must never hard-fail + return False + return bool(saved and saved.get("provider")) + + +def _first_coding_agent_on_path(env: Mapping[str, str]) -> Optional[str]: + """The first installed coding-agent CLI, in ``CODING_AGENT_NAMES`` order.""" + from fluid_build.llm.coding_agent import _AGENT_SPECS, CODING_AGENT_NAMES + + # Search the PATH of the environment we were given, never the process's: + # an environment without PATH finds nothing, as it would for a child process. + path = env.get("PATH", "") + for name in CODING_AGENT_NAMES: + if shutil.which(_AGENT_SPECS[name].binary, path=path): + return name + return None + + +def _lean_keyless_autopick( + env: Mapping[str, str], *, non_interactive: bool = False, allow_agents: bool = True +) -> Optional[str]: + """The coding agent a run without litellm auto-picks, else ``None``. + + The ambient rung of the ladder without its keyring read, so the litellm + preflight stays cheap: a non-interactive run, no configuring variable, no + saved provider, an agent CLI on PATH, and no local Ollama (it needs litellm, + and a running one is never silently swapped for a cloud agent). + ``allow_agents=False`` (the agent loop, or a stated endpoint or model) + disables it. + """ + if not allow_agents or not _is_non_interactive(non_interactive): + return None + if _configured_by_env(env): + return None + agent = _first_coding_agent_on_path(env) + if agent is None: + return None + try: + from fluid_build._ai_config_shared import load_ai_config + + saved = load_ai_config() + except Exception: # noqa: BLE001 — detection must never hard-fail + saved = None + if saved and saved.get("provider"): + return None + detect_fn = getattr(_host(), "detect_ollama_available", None) + if detect_fn is not None and detect_fn(env): + return None + return agent + + +def _infer_provider_from_ambient( + env: Mapping[str, str], *, non_interactive: bool = False, allow_agents: bool = True +) -> Optional[str]: + """Last-resort detection, shared by the readiness ladder and the real selection. + + 1. Ollama already running on localhost: data stays local. + 2. Non-interactive runs only, and only when nothing is configured: the first + coding-agent CLI on PATH. Interactive runs keep "offer, don't force" + (the ``fluid ai setup`` inline offer). ``allow_agents=False`` skips this + rung: for the agent loop, which needs tool use, and when the operator + stated an endpoint or model. Only call this AFTER the saved config and the keyring have been consulted — ambient services must not override an explicit @@ -107,6 +222,12 @@ def _infer_provider_from_ambient(env: Mapping[str, str]) -> Optional[str]: detect_fn = getattr(_host(), "detect_ollama_available", None) if detect_fn is not None and detect_fn(env): return "ollama" + if ( + allow_agents + and _is_non_interactive(non_interactive) + and not _operator_configured_a_provider(env) + ): + return _first_coding_agent_on_path(env) return None @@ -114,14 +235,18 @@ def _infer_provider_from_ambient(env: Mapping[str, str]) -> Optional[str]: # imported ``_infer_provider_from_env``. The new code paths inside # :func:`check_llm_readiness` use the split functions above so the # saved config gets a chance to win over ambient Ollama. -def _infer_provider_from_env(env: Mapping[str, str]) -> Optional[str]: +def _infer_provider_from_env( + env: Mapping[str, str], *, non_interactive: bool = False, allow_agents: bool = True +) -> Optional[str]: explicit = _infer_provider_from_explicit_keys(env) if explicit: return explicit keyring_match = _infer_provider_from_keyring() if keyring_match: return keyring_match - return _infer_provider_from_ambient(env) + return _infer_provider_from_ambient( + env, non_interactive=non_interactive, allow_agents=allow_agents + ) def _infer_provider_from_keyring() -> Optional[str]: diff --git a/fluid_build/llm/providers.py b/fluid_build/llm/providers.py index 921804f1..7cbe98d6 100644 --- a/fluid_build/llm/providers.py +++ b/fluid_build/llm/providers.py @@ -30,6 +30,7 @@ "normalize_llm_provider_name", "query_ollama_models", "reset_llm_caches", + "agent_loop_requested", "preflight_llm_provider", "resolve_llm_config", "resolve_model_name", @@ -45,6 +46,7 @@ import logging import os import re +import shutil import threading from abc import ABC, abstractmethod from dataclasses import dataclass, field @@ -239,6 +241,10 @@ class LlmConfig: # contract.fluid.yaml into the workspace with its own tools). Ignored by # every other provider. agent_mode: str = "envelope" + # True when the operator stated ``endpoint`` (``--llm-endpoint`` or + # ``FLUID_LLM_ENDPOINT``) rather than the provider deriving it. A stated + # endpoint is where every call goes, so routing calls keep it. + endpoint_explicit: bool = False def for_routing(self) -> "LlmConfig": """Return a shallow copy configured for the routing model. @@ -253,7 +259,7 @@ def for_routing(self) -> "LlmConfig": overrides: Dict[str, Any] = {"model": self.routing_model} if self.routing_endpoint: overrides["endpoint"] = self.routing_endpoint - else: + elif not self.endpoint_explicit: # Re-derive endpoint for the routing model using the same # provider's default-endpoint logic. provider = BUILTIN_LLM_PROVIDERS.get(self.provider) @@ -771,15 +777,61 @@ def has_llm_api_key( {"ollama", "mcp-sampling", "mcp_sampling", "claude-code", "codex", "cursor", "kiro"} ) +# The keyless providers that do not go through litellm: the call is made by the +# MCP client or by an installed agent CLI, so a missing litellm extra is no obstacle. +_NON_LITELLM_PROVIDERS: frozenset = _KEYLESS_PROVIDERS - {"ollama"} + def _provider_name_from(args: Any, env: Mapping[str, str]) -> str: """The provider name the flags and environment select (flag, env, key sniff, default).""" + return ( + _explicit_provider_name(args, env) + or _infer_provider_from_env( + env, + non_interactive=bool(getattr(args, "non_interactive", False)), + allow_agents=_agents_allowed(args, env), + ) + or "gemini" + ) + + +def agent_loop_requested(flag: Any, environ: Optional[Mapping[str, str]] = None) -> bool: + """Whether this run asks for the multi-turn agent loop. + + ``flag`` is the ``--agent-loop`` value (callers read it their own way, e.g. + the CLI's ``get_cli_arg``), or ``FLUID_COPILOT_AGENT_LOOP`` set to a truthy + value. It is parsed as a boolean, not by presence: + ``FLUID_COPILOT_AGENT_LOOP=0`` is the documented way to turn the loop off. + """ + env = os.environ if environ is None else environ + return bool(flag) or ( + env.get("FLUID_COPILOT_AGENT_LOOP", "").strip().lower() in {"1", "true", "yes", "on"} + ) + + +def _agents_allowed(args: Any, env: Mapping[str, str]) -> bool: + """Whether the keyless auto-pick may choose a coding agent for this run. + + Not for the agent loop (it needs tool use, which the agents do not offer), + and not when ``--llm-endpoint`` / ``--llm-model`` (or their routing twins) + state where or what to call: that destination is never swapped for an agent. + """ + return not ( + agent_loop_requested(getattr(args, "agent_loop", False), env) + or any( + getattr(args, name, None) + for name in ("llm_endpoint", "llm_model", "llm_routing_endpoint", "llm_routing_model") + ) + ) + + +def _explicit_provider_name(args: Any, env: Mapping[str, str]) -> Optional[str]: + """The provider the operator named: ``--llm-provider``, else the env selectors.""" return ( getattr(args, "llm_provider", None) or env.get("FLUID_LLM_PROVIDER") or env.get("FLUID_FORGE_AGENT") - or _infer_provider_from_env(env) - or "gemini" + or None ) @@ -793,8 +845,9 @@ def _provider_name_from(args: Any, env: Mapping[str, str]) -> str: def preflight_llm_provider(args: Any, environ: Optional[Mapping[str, str]] = None) -> None: """Fail fast when the selected provider cannot be used on this install. - Cheap and side-effect free (no key lookup, no network, no litellm import): - it only resolves the provider adapter, which raises + Cheap and side-effect free (no key lookup, no litellm import, no network + beyond the localhost Ollama probe of the keyless auto-pick): it only + resolves the provider adapter, which raises :class:`~fluid_build._optional.MissingOptionalDependency` when a litellm-backed provider is chosen and litellm is absent. Call it before anything that reads workspace context or memory. @@ -805,14 +858,20 @@ def preflight_llm_provider(args: Any, environ: Optional[Mapping[str, str]] = Non from fluid_build._optional import MissingOptionalDependency, ensure_installed env = dict(environ or os.environ) - agent_loop = bool(getattr(args, "agent_loop", False)) - if not ( - getattr(args, "llm_provider", None) - or env.get("FLUID_LLM_PROVIDER") - or env.get("FLUID_FORGE_AGENT") - ): + agent_loop = agent_loop_requested(getattr(args, "agent_loop", False), env) + if not _explicit_provider_name(args, env): # Nothing chosen: inference reads the keyring and probes Ollama, and every - # provider it can pick (or the gemini default) needs litellm. Fail first. + # provider it can pick (or the gemini default) needs litellm, except the + # coding agent a non-interactive run auto-picks. Without litellm, that + # agent is the only provider that can run; otherwise fail first. + from fluid_build._optional import is_available + + if not is_available("litellm") and _lean_keyless_autopick( + env, + non_interactive=bool(getattr(args, "non_interactive", False)), + allow_agents=_agents_allowed(args, env), + ): + return ensure_installed( "litellm", "litellm", @@ -887,6 +946,15 @@ def resolve_llm_config(args: Any, environ: Optional[Mapping[str, str]] = None) - ) model_notes: List[str] = [] + if not _explicit_provider_name(args, env) and provider.name in _NON_LITELLM_PROVIDERS: + # Nothing named a provider and it is not a key-based one, so the keyless + # auto-pick chose it: say which, and how to override. + note = ( + f"No LLM provider is configured; using {provider.name}, the first coding agent " + f"found on PATH. Override with --llm-provider." + ) + model_notes.append(note) + LOG.warning(note) preflight_requested = _model_preflight_requested(args, env) available_models: Optional[List[str]] = None @@ -956,6 +1024,7 @@ def _available_models() -> Optional[List[str]]: model_source = "live_preflight" endpoint = getattr(args, "llm_endpoint", None) or env.get("FLUID_LLM_ENDPOINT") + endpoint_explicit = bool(endpoint) if not endpoint: endpoint = provider.default_endpoint(model, env) @@ -1025,6 +1094,7 @@ def _available_models() -> Optional[List[str]]: model_resolution_notes=model_notes, tier_models=tier_models, agent_mode=agent_mode, + endpoint_explicit=endpoint_explicit, ) @@ -1405,6 +1475,7 @@ def call_llm_streaming( _infer_provider_from_explicit_keys, _infer_provider_from_keyring, _keyring_key, + _lean_keyless_autopick, _resolve_api_key, clear_api_key_from_keyring, save_api_key_to_keyring, @@ -1573,7 +1644,26 @@ def check_llm_readiness(environ: Optional[Mapping[str, str]] = None) -> LlmReadi # Resolve API key: env vars → config file api_key = _resolve_api_key(provider.name, env) or saved_key - auth_ok = bool(api_key) or provider.name == "ollama" + from fluid_build.llm.coding_agent import _AGENT_SPECS + + agent = _AGENT_SPECS.get(provider.name) + if agent is not None: + # A coding agent is ready when its CLI is installed; its login or key is the + # CLI's own business and is not pre-checked. Search the PATH of the + # environment we were given, as the real call's child process would. + if not shutil.which(agent.binary, path=env.get("PATH", "")): + return LlmReadinessCheck( + ready=False, + provider=provider.name, + model=provider.default_model, + auth_available=False, + error=f"The '{agent.binary}' CLI for {provider.name} is not on PATH.", + ) + auth_ok = True + else: + auth_ok = ( + bool(api_key) or provider.name == "ollama" or bool(getattr(provider, "keyless", False)) + ) if not auth_ok: LOG.debug("LLM readiness: provider=%s found but no API key", provider.name) @@ -1588,11 +1678,11 @@ def check_llm_readiness(environ: Optional[Mapping[str, str]] = None) -> LlmReadi model = env.get("FLUID_LLM_MODEL") or saved_model or provider.default_model endpoint = env.get("FLUID_LLM_ENDPOINT") or provider.default_endpoint(model, env) - # Every built-in provider runs through litellm, which is an extra since 0.28.0. - # A configured key is not "ready" without it. + # Every built-in provider except the keyless agent/MCP ones runs through + # litellm, which is an extra since 0.28.0. A configured key is not "ready" without it. from fluid_build._optional import is_available - if not is_available("litellm"): + if provider.name not in _NON_LITELLM_PROVIDERS and not is_available("litellm"): return LlmReadinessCheck( ready=False, provider=provider.name, diff --git a/tests/test_ai_setup.py b/tests/test_ai_setup.py index f7d5545c..0f775361 100644 --- a/tests/test_ai_setup.py +++ b/tests/test_ai_setup.py @@ -1055,6 +1055,12 @@ def test_resolve_llm_config_falls_back_to_gemini_when_unconfigured(self, monkeyp "fluid_build.cli.forge_copilot_llm_providers.detect_ollama_available", return_value=False, ), + # A coding agent on this machine's PATH would be auto-picked + # (non-interactive); "unconfigured" means none is installed. + patch( + "fluid_build.llm.provider_resolve._first_coding_agent_on_path", + return_value=None, + ), patch( "fluid_build.cli.forge_copilot_llm_providers._resolve_api_key", return_value=None, diff --git a/tests/test_forge_copilot_runtime.py b/tests/test_forge_copilot_runtime.py index 19873b92..8bd92bc9 100644 --- a/tests/test_forge_copilot_runtime.py +++ b/tests/test_forge_copilot_runtime.py @@ -31,6 +31,7 @@ clear_api_key_from_keyring, detect_provider_from_api_key, get_catalog_default, + get_llm_provider, has_llm_api_key, live_provider_models, resolve_llm_config, @@ -221,10 +222,11 @@ def test_ollama_uses_default_endpoint_without_api_key(self): assert config.provider == "ollama" assert config.api_key is None - # litellm owns auth/endpoint resolution; we emit a sentinel - # ``litellm://ollama/`` so telemetry can identify the - # routed call without leaking the daemon URL into the config. - assert config.endpoint.startswith("litellm://ollama/") + # OLLAMA_HOST is a real daemon address, so it is the endpoint; litellm + # owns everything else and no placeholder URL is ever recorded. + assert config.endpoint == "http://localhost:11434" + _, payload = get_llm_provider("ollama").build_request(config, "s", "u") + assert payload["api_base"] == "http://localhost:11434" def test_llm_timeout_seconds_can_be_configured_from_env(self): args = SimpleNamespace(llm_provider="ollama", llm_model=None, llm_endpoint=None) @@ -381,7 +383,7 @@ def test_required_llm_preflight_accepts_local_ollama_latest_alias(self): assert config.provider == "ollama" assert config.model == "gemma4:latest" - assert config.endpoint.startswith("litellm://ollama/") + assert config.endpoint == "http://localhost:11434" def test_required_llm_preflight_rejects_unavailable_ollama_model(self): args = SimpleNamespace( @@ -422,8 +424,8 @@ def test_marks_ollama_ready_without_api_key(self): assert readiness.ready is True assert readiness.provider == "ollama" assert readiness.auth_available is True - # litellm-routed sentinel — see test_ollama_uses_default_endpoint_without_api_key. - assert readiness.endpoint.startswith("litellm://ollama/") + # See test_ollama_uses_default_endpoint_without_api_key. + assert readiness.endpoint == "http://localhost:11434" def test_redacts_endpoint_in_readiness_output(self): readiness = check_llm_readiness( diff --git a/tests/test_litellm_absent.py b/tests/test_litellm_absent.py index 5de67fda..3f6c0a70 100644 --- a/tests/test_litellm_absent.py +++ b/tests/test_litellm_absent.py @@ -419,6 +419,119 @@ def test_init_handoff_prints_the_hint_and_touches_no_credential_store( forge_run.assert_not_called() +class TestLeanKeylessAutoPick: + """Without litellm, a non-interactive run may still author with an installed agent.""" + + @staticmethod + def _lean(block, monkeypatch, tmp_path, *, tty=False, ollama=False, saved=None): # noqa: F811 + import stat + import sys + from types import SimpleNamespace + + from fluid_build.llm import providers + + block("litellm") + bindir = tmp_path / "bin" + bindir.mkdir() + exe = bindir / "claude" + exe.write_text("#!/bin/sh\nexit 0\n") + exe.chmod(exe.stat().st_mode | stat.S_IXUSR) + monkeypatch.setattr(sys, "stdin", SimpleNamespace(isatty=lambda: tty)) + monkeypatch.setattr(providers, "detect_ollama_available", lambda env: ollama) + monkeypatch.setattr(providers, "_get_api_key_from_keyring", lambda name: None) + monkeypatch.setattr("fluid_build._ai_config_shared.load_ai_config", lambda *a, **k: saved) + return {"PATH": str(bindir)} + + @staticmethod + def _args(**kw): + from types import SimpleNamespace + + return SimpleNamespace(**{"llm_provider": None, "agent_loop": False, **kw}) + + def test_agent_on_path_passes_the_preflight(self, block, monkeypatch, tmp_path): # noqa: F811 + from fluid_build.llm.providers import _provider_name_from, preflight_llm_provider + + env = self._lean(block, monkeypatch, tmp_path) + preflight_llm_provider(self._args(), env) + assert _provider_name_from(self._args(), env) == "claude-code" + + def test_running_ollama_still_fails_fast(self, block, monkeypatch, tmp_path): # noqa: F811 + from fluid_build.llm.providers import preflight_llm_provider + + env = self._lean(block, monkeypatch, tmp_path, ollama=True) + with pytest.raises(MissingOptionalDependency): + preflight_llm_provider(self._args(), env) + + def test_an_api_key_still_fails_fast(self, block, monkeypatch, tmp_path): # noqa: F811 + from fluid_build.llm.providers import preflight_llm_provider + + env = self._lean(block, monkeypatch, tmp_path) + env["GEMINI_API_KEY"] = "AIza-test" + with pytest.raises(MissingOptionalDependency): + preflight_llm_provider(self._args(), env) + + def test_a_stated_ollama_host_still_fails_fast( + self, block, monkeypatch, tmp_path + ): # noqa: F811 + from fluid_build.llm.providers import preflight_llm_provider + + env = self._lean(block, monkeypatch, tmp_path) + env["OLLAMA_HOST"] = "http://127.0.0.1:11500" + with pytest.raises(MissingOptionalDependency): + preflight_llm_provider(self._args(), env) + + def test_a_saved_provider_still_fails_fast(self, block, monkeypatch, tmp_path): # noqa: F811 + from fluid_build.llm.providers import preflight_llm_provider + + env = self._lean(block, monkeypatch, tmp_path, saved={"provider": "openai"}) + with pytest.raises(MissingOptionalDependency): + preflight_llm_provider(self._args(), env) + + def test_a_tty_still_fails_fast(self, block, monkeypatch, tmp_path): # noqa: F811 + from fluid_build.llm.providers import preflight_llm_provider + + env = self._lean(block, monkeypatch, tmp_path, tty=True) + with pytest.raises(MissingOptionalDependency): + preflight_llm_provider(self._args(), env) + + def test_the_agent_loop_never_auto_picks_an_agent( + self, block, monkeypatch, tmp_path + ): # noqa: F811 + from fluid_build.llm.providers import _provider_name_from, preflight_llm_provider + + env = self._lean(block, monkeypatch, tmp_path) + with pytest.raises(MissingOptionalDependency) as info: + preflight_llm_provider(self._args(agent_loop=True), env) + assert "--template starter" in str(info.value) + assert _provider_name_from(self._args(agent_loop=True), env) != "claude-code" + + def test_the_agent_loop_env_route_never_auto_picks_an_agent( + self, block, monkeypatch, tmp_path + ): # noqa: F811 + from fluid_build.llm.providers import preflight_llm_provider + + env = {**self._lean(block, monkeypatch, tmp_path), "FLUID_COPILOT_AGENT_LOOP": "1"} + with pytest.raises(MissingOptionalDependency) as info: + preflight_llm_provider(self._args(), env) + assert "--template starter" in str(info.value) + + +def test_agent_search_uses_only_the_given_environment(monkeypatch, tmp_path): + """An environment without PATH finds no agent, even if the process PATH has one.""" + import stat + + from fluid_build.llm.provider_resolve import _first_coding_agent_on_path + + bindir = tmp_path / "bin" + bindir.mkdir() + exe = bindir / "claude" + exe.write_text("#!/bin/sh\nexit 0\n") + exe.chmod(exe.stat().st_mode | stat.S_IXUSR) + monkeypatch.setenv("PATH", str(bindir)) + assert _first_coding_agent_on_path({}) is None + assert _first_coding_agent_on_path({"PATH": str(bindir)}) == "claude-code" + + class TestDoctorStatus: def _row(self, monkeypatch): from fluid_build.cli.doctor import _check_fluid_features diff --git a/tests/test_llm_endpoint_sentinel.py b/tests/test_llm_endpoint_sentinel.py new file mode 100644 index 00000000..fb745115 --- /dev/null +++ b/tests/test_llm_endpoint_sentinel.py @@ -0,0 +1,416 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""``LlmConfig.endpoint`` holds only a real endpoint, never a telemetry sentinel. + +litellm-backed providers used to resolve to ``litellm:///``. +Consumers then dialled it: Ollama's ``api_base`` carried the sentinel, ``fluid +ai test`` POSTed to it, and forge copied it into ``args.llm_endpoint`` where it +read as an explicit endpoint on the next resolve. These tests drive the real +``resolve_llm_config`` -> ``build_request`` path so the class cannot return. +""" + +from __future__ import annotations + +import sys +from types import SimpleNamespace +from unittest import mock + +import pytest + +from fluid_build.llm.providers import ( + BUILTIN_LLM_PROVIDERS, + LlmConfig, + resolve_llm_config, +) + +pytestmark = pytest.mark.unit + +_OLLAMA_URL = "http://ollama.internal:11434" + + +@pytest.fixture(autouse=True) +def _clean_env(monkeypatch): + for var in ("OLLAMA_HOST", "FLUID_LLM_ENDPOINT", "FLUID_LLM_PROVIDER", "FLUID_FORGE_AGENT"): + monkeypatch.delenv(var, raising=False) + + +def _args(**kw): + base = {"llm_provider": None, "llm_model": None, "llm_endpoint": None} + base.update(kw) + return SimpleNamespace(**base) + + +def _payload(config: LlmConfig) -> dict: + _, payload = BUILTIN_LLM_PROVIDERS[config.provider].build_request(config, "sys", "usr") + return payload + + +_KEYS = { + "openai": {"OPENAI_API_KEY": "k"}, + "anthropic": {"ANTHROPIC_API_KEY": "k"}, + "gemini": {"GEMINI_API_KEY": "k"}, + "ollama": {}, +} + + +@pytest.mark.parametrize("provider", sorted(_KEYS)) +def test_endpoint_is_never_the_litellm_sentinel(provider): + config = resolve_llm_config(_args(llm_provider=provider), _KEYS[provider]) + assert "litellm://" not in config.endpoint + assert config.endpoint == "" + assert "litellm://" not in config.redacted_endpoint + + +@pytest.mark.parametrize("provider", sorted(_KEYS)) +def test_routing_config_never_carries_the_sentinel(provider): + config = resolve_llm_config(_args(llm_provider=provider), _KEYS[provider]) + assert "litellm://" not in config.for_routing().endpoint + + +class TestRoutingKeepsAStatedEndpoint: + """Routing calls go to the daemon the operator named, not a re-derived default.""" + + _STATED = "http://127.0.0.1:9999" + + def test_flag_endpoint_reaches_routing_calls(self): + config = resolve_llm_config( + _args(llm_provider="ollama", llm_model="llama3", llm_endpoint=self._STATED), {} + ) + routed = config.for_routing() + assert routed.model == config.routing_model != config.model + assert routed.endpoint == self._STATED + assert _payload(routed)["api_base"] == self._STATED + + def test_env_endpoint_reaches_routing_calls(self): + config = resolve_llm_config( + _args(llm_provider="ollama", llm_model="llama3"), + {"FLUID_LLM_ENDPOINT": self._STATED}, + ) + assert config.for_routing().endpoint == self._STATED + + def test_a_routing_endpoint_still_wins(self): + config = resolve_llm_config( + _args( + llm_provider="ollama", + llm_model="llama3", + llm_endpoint=self._STATED, + llm_routing_endpoint="http://127.0.0.1:9998", + ), + {}, + ) + assert config.for_routing().endpoint == "http://127.0.0.1:9998" + + def test_a_derived_endpoint_is_still_rederived(self): + # Nothing stated: the provider's own default is asked again for the routing model. + config = resolve_llm_config(_args(llm_provider="ollama", llm_model="llama3"), {}) + assert config.endpoint_explicit is False + seen = [] + provider = BUILTIN_LLM_PROVIDERS["ollama"] + with mock.patch.object( + type(provider), "default_endpoint", lambda self, m, e: seen.append(m) or "derived" + ): + assert config.for_routing().endpoint == "derived" + assert seen == [config.routing_model] + + def test_hosted_provider_without_an_endpoint_stays_empty(self): + config = resolve_llm_config(_args(llm_provider="openai"), _KEYS["openai"]) + assert config.for_routing().endpoint == "" + + +def test_readiness_endpoint_is_not_a_fake_url(): + from fluid_build.llm.providers import check_llm_readiness + + readiness = check_llm_readiness({"FLUID_LLM_PROVIDER": "ollama"}) + assert "litellm://" not in (readiness.endpoint or "") + + +@pytest.mark.parametrize( + "provider,expected", [("openai", "Provider default"), (None, "Not configured")] +) +def test_doctor_prints_no_fake_endpoint(capsys, provider, expected): + from fluid_build.cli import doctor + from fluid_build.llm.providers import LlmReadinessCheck + + readiness = LlmReadinessCheck(ready=bool(provider), provider=provider, endpoint="") + doctor._print_copilot_readiness(readiness) + + out = capsys.readouterr().out + assert expected in out + assert "litellm://" not in out + + +class TestOllamaApiBase: + def test_no_host_leaves_api_base_to_litellm(self): + config = resolve_llm_config(_args(llm_provider="ollama"), {}) + assert "api_base" not in _payload(config) + + def test_ollama_host_becomes_api_base(self): + config = resolve_llm_config(_args(llm_provider="ollama"), {"OLLAMA_HOST": _OLLAMA_URL}) + assert config.endpoint == _OLLAMA_URL + assert _payload(config)["api_base"] == _OLLAMA_URL + + def test_schemeless_ollama_host_gets_http(self): + # Ollama's own variable accepts ``host:port``; litellm needs a URL. + config = resolve_llm_config( + _args(llm_provider="ollama"), {"OLLAMA_HOST": "127.0.0.1:11500"} + ) + assert _payload(config)["api_base"] == "http://127.0.0.1:11500" + + def test_flag_beats_ollama_host(self): + config = resolve_llm_config( + _args(llm_provider="ollama", llm_endpoint="http://localhost:9999"), + {"OLLAMA_HOST": _OLLAMA_URL}, + ) + assert _payload(config)["api_base"] == "http://localhost:9999" + + def test_env_endpoint_beats_ollama_host(self): + config = resolve_llm_config( + _args(llm_provider="ollama"), + {"FLUID_LLM_ENDPOINT": "http://localhost:9998", "OLLAMA_HOST": _OLLAMA_URL}, + ) + assert _payload(config)["api_base"] == "http://localhost:9998" + + def test_tool_request_uses_the_same_api_base(self): + provider = BUILTIN_LLM_PROVIDERS["ollama"] + bare = resolve_llm_config(_args(llm_provider="ollama"), {}) + hosted = resolve_llm_config(_args(llm_provider="ollama"), {"OLLAMA_HOST": _OLLAMA_URL}) + assert "api_base" not in provider.build_tool_request(bare, "s", [], [])[2] + assert provider.build_tool_request(hosted, "s", [], [])[2]["api_base"] == _OLLAMA_URL + + def test_streaming_request_uses_the_same_api_base(self): + provider = BUILTIN_LLM_PROVIDERS["ollama"] + hosted = resolve_llm_config(_args(llm_provider="ollama"), {"OLLAMA_HOST": _OLLAMA_URL}) + assert provider.build_streaming_request(hosted, "s", "u")[2]["api_base"] == _OLLAMA_URL + + @pytest.mark.parametrize("host", [None, _OLLAMA_URL]) + def test_copying_the_endpoint_into_args_changes_nothing(self, host): + # forge.py / init.py do ``args.llm_endpoint = llm_config.endpoint`` and + # resolve again; that round trip must be the identity. + env = {"OLLAMA_HOST": host} if host else {} + args = _args(llm_provider="ollama") + first = resolve_llm_config(args, env) + args.llm_endpoint = first.endpoint + second = resolve_llm_config(args, env) + assert second.endpoint == first.endpoint + assert _payload(second).get("api_base") == _payload(first).get("api_base") + + +# Forms of OLLAMA_HOST -> the URL both reads of it use. Same rules as +# ollama-python's ``_parse_host``: no scheme means http on 11434; an explicit +# scheme without a port means that scheme's default port; a path is kept. +_OLLAMA_HOST_FORMS = [ + ("myhost", "http://myhost:11434"), + ("myhost:56789", "http://myhost:56789"), + ("1.2.3.4", "http://1.2.3.4:11434"), + (":56789", "http://127.0.0.1:56789"), + ("http://myhost", "http://myhost:80"), + ("https://myhost", "https://myhost:443"), + ("http://myhost:56789", "http://myhost:56789"), + ("https://myhost:56789", "https://myhost:56789"), + ("myhost/", "http://myhost:11434"), + ("myhost:56789/path/", "http://myhost:56789/path"), + ("https://myhost:56789/path", "https://myhost:56789/path"), + (" myhost:56789 ", "http://myhost:56789"), + ("[::1]", "http://[::1]:11434"), + ("[::1]:56789", "http://[::1]:56789"), + ("https://[::1]", "https://[::1]:443"), +] + + +class TestOllamaHostForms: + @pytest.mark.parametrize("host, expected", _OLLAMA_HOST_FORMS) + def test_base_url(self, host, expected): + from fluid_build.llm.litellm_backend import _ollama_base_url + + assert _ollama_base_url({"OLLAMA_HOST": host}) == expected + + @pytest.mark.parametrize("env", [{}, {"OLLAMA_HOST": ""}, {"OLLAMA_HOST": " "}]) + def test_unset_stays_empty_for_litellm_to_choose(self, env): + from fluid_build.llm.litellm_backend import _ollama_base_url + + assert _ollama_base_url(env) == "" + + def test_an_unparseable_port_is_left_to_fail_at_the_dial(self): + from fluid_build.llm.litellm_backend import _ollama_base_url + + assert _ollama_base_url({"OLLAMA_HOST": "myhost:abc"}) == "http://myhost:abc" + + @pytest.mark.parametrize("host, expected", _OLLAMA_HOST_FORMS) + def test_the_model_list_dials_the_same_url(self, host, expected): + from fluid_build.llm.litellm_backend import _list_ollama_models + + with mock.patch("urllib.request.urlopen", side_effect=OSError("down")) as urlopen: + assert _list_ollama_models({"OLLAMA_HOST": host}) is None + assert urlopen.call_args.args[0].full_url == f"{expected}/api/tags" + + def test_the_model_list_defaults_to_localhost(self): + from fluid_build.llm.litellm_backend import _list_ollama_models + + with mock.patch("urllib.request.urlopen", side_effect=OSError("down")) as urlopen: + _list_ollama_models({}) + assert urlopen.call_args.args[0].full_url == "http://localhost:11434/api/tags" + + def test_resolved_endpoint_and_api_base_use_the_bare_host_port(self): + config = resolve_llm_config(_args(llm_provider="ollama"), {"OLLAMA_HOST": "myhost"}) + assert config.endpoint == "http://myhost:11434" + assert _payload(config)["api_base"] == "http://myhost:11434" + + +class TestAiTestNeverDialsASentinel: + """``fluid ai test`` probes litellm-backed providers through their own call path.""" + + def _fake_litellm(self): + fake = mock.MagicMock(spec=["completion", "completion_cost"]) + fake.completion.return_value = { + "choices": [{"message": {"content": "FLUID_OK"}}], + "usage": {"prompt_tokens": 7, "completion_tokens": 2, "total_tokens": 9}, + } + fake.completion_cost.return_value = 0.0 + return fake + + @pytest.mark.parametrize("provider", sorted(_KEYS)) + def test_smoke_call_goes_through_litellm_not_http(self, monkeypatch, provider): + from fluid_build.cli.ai_setup import _run_ai_smoke_call + + fake = self._fake_litellm() + monkeypatch.setitem(sys.modules, "litellm", fake) + config = resolve_llm_config(_args(llm_provider=provider), _KEYS[provider]) + + with mock.patch("httpx.Client", side_effect=AssertionError("raw HTTP POST")): + text, usage, latency_ms = _run_ai_smoke_call(BUILTIN_LLM_PROVIDERS[provider], config) + + assert text == "FLUID_OK" + assert usage["total_tokens"] == 9 + assert latency_ms >= 0 + kwargs = fake.completion.call_args.kwargs + assert kwargs["max_tokens"] >= 1 + assert kwargs["timeout"] == config.timeout_seconds + + def test_smoke_call_unexpected_text_is_reported(self, monkeypatch): + from fluid_build.cli.ai_setup import _run_ai_smoke_call + from fluid_build.llm.providers import CopilotGenerationError + + fake = self._fake_litellm() + fake.completion.return_value = {"choices": [{"message": {"content": "hi"}}]} + monkeypatch.setitem(sys.modules, "litellm", fake) + config = resolve_llm_config(_args(llm_provider="openai"), _KEYS["openai"]) + + with pytest.raises(CopilotGenerationError) as exc_info: + _run_ai_smoke_call(BUILTIN_LLM_PROVIDERS["openai"], config) + assert exc_info.value.event == "ai_test_unexpected_response" + + @pytest.mark.parametrize( + "event,exit_code", + [ + ("copilot_llm_auth_failed", 3), + ("copilot_llm_permission_denied", 3), + ("copilot_litellm_request_failed", 5), + ], + ) + def test_litellm_errors_classify_to_the_documented_exit_codes(self, event, exit_code): + from fluid_build.cli.ai_setup import _classify_ai_test_error + + assert _classify_ai_test_error(event, None) == exit_code + + @staticmethod + def _exit_code_for(monkeypatch, error): + """The exit code ``fluid ai test`` picks when the litellm call raises *error*.""" + from fluid_build.cli.ai_setup import ( + _classify_ai_test_error, + _http_status_from_cause, + _run_ai_smoke_call, + ) + from fluid_build.llm.providers import CopilotGenerationError + + fake = mock.MagicMock(spec=["completion", "completion_cost"]) + fake.completion.side_effect = error + monkeypatch.setitem(sys.modules, "litellm", fake) + config = resolve_llm_config(_args(llm_provider="openai"), _KEYS["openai"]) + with pytest.raises(CopilotGenerationError) as exc_info: + _run_ai_smoke_call(BUILTIN_LLM_PROVIDERS["openai"], config) + exc = exc_info.value + return _classify_ai_test_error(exc.event, _http_status_from_cause(exc)) + + @pytest.mark.parametrize( + "name, exit_code", + [ + ("NotFoundError", 4), # a model the provider does not have + ("Timeout", 5), + ("APIConnectionError", 5), + ("RateLimitError", 5), + ("ServiceUnavailableError", 5), + ], + ) + def test_litellm_not_found_is_a_resource_error_the_rest_stay_network( + self, monkeypatch, name, exit_code + ): + litellm = pytest.importorskip("litellm") + error = getattr(litellm, name)(message="boom", model="m", llm_provider="openai") + assert self._exit_code_for(monkeypatch, error) == exit_code + + def test_a_404_from_an_unlisted_exception_class_is_also_a_resource_error(self, monkeypatch): + class _Gone(Exception): + status_code = 404 + + assert self._exit_code_for(monkeypatch, _Gone("no such model")) == 4 + + def test_an_error_without_a_status_stays_network(self, monkeypatch): + assert self._exit_code_for(monkeypatch, RuntimeError("socket closed")) == 5 + + @pytest.mark.parametrize("provider", sorted(_KEYS)) + def test_resolved_test_config_has_no_sentinel_and_validates(self, provider): + from fluid_build.cli.ai_setup import _resolve_ai_test_config + + args = _args(llm_provider=provider, llm_timeout_seconds=None) + with ( + mock.patch("fluid_build.cli.ai_setup._load_ai_config", return_value=None), + mock.patch("fluid_build.cli.ai_setup._load_key_from_keyring", return_value="k"), + mock.patch.dict("os.environ", _KEYS[provider], clear=True), + ): + config, error = _resolve_ai_test_config(args) + + assert error is None + assert config is not None + assert "litellm://" not in config.endpoint + + def test_a_litellm_url_is_not_a_valid_test_endpoint(self): + from fluid_build.cli.ai_setup import _validate_ai_test_endpoint + + assert _validate_ai_test_endpoint("openai", "litellm://openai/gpt-4o") is not None + + def test_run_ai_test_never_posts_and_reports_no_fake_endpoint(self, monkeypatch): + from fluid_build.cli.ai_setup import run_ai_test + + monkeypatch.setitem(sys.modules, "litellm", self._fake_litellm()) + args = _args(llm_provider="openai", llm_timeout_seconds=None, json=True) + with ( + mock.patch("fluid_build.cli.ai_setup._load_ai_config", return_value=None), + mock.patch("fluid_build.cli.ai_setup._load_key_from_keyring", return_value="k"), + mock.patch.dict("os.environ", {"OPENAI_API_KEY": "k"}, clear=True), + mock.patch("httpx.Client", side_effect=AssertionError("raw HTTP POST")), + mock.patch("fluid_build.cli.ai_setup._check_ai_test_model_availability") as avail, + ): + avail.return_value = ("unavailable", None) + code, report = run_ai_test(None, args) + + assert code == 0 + assert report["ok"] is True + assert report["endpoint"] is None + + +@pytest.mark.parametrize( + "provider,key_env,expected", + [ + ("openai", "OPENAI_API_KEY", ("openai", "gpt-4.1-mini")), + ("anthropic", "ANTHROPIC_API_KEY", ("anthropic", "claude-sonnet-4-6")), + ("claude", "ANTHROPIC_API_KEY", ("anthropic", "claude-sonnet-4-6")), + ("gemini", "GEMINI_API_KEY", ("gemini", "gemini-2.5-pro")), + ], +) +def test_command_center_provider_and_model_are_unchanged(provider, key_env, expected): + """Command Center runs ``fluid forge data-model from-intent --llm-provider X`` + with the key in env and no ``--llm-model``. Only the endpoint may change.""" + config = resolve_llm_config(_args(llm_provider=provider), {key_env: "k"}) + assert (config.provider, config.model) == expected + assert config.endpoint == "" diff --git a/tests/test_llm_keyless_autopick.py b/tests/test_llm_keyless_autopick.py new file mode 100644 index 00000000..4e7252c5 --- /dev/null +++ b/tests/test_llm_keyless_autopick.py @@ -0,0 +1,426 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""Keyless auto-pick: a non-interactive run with no provider configured. + +Order: a running local Ollama (data stays local), then the first coding-agent CLI +on PATH in ``CODING_AGENT_NAMES`` order, then the hardcoded ``gemini`` last +resort. Interactive runs keep "offer, don't force" (the ``fluid ai setup`` +inline offer). ``check_llm_readiness`` and the real selection share one helper, +so ``fluid doctor`` never disagrees with what forge then does. + +Hermetic: stub executables on a tmp PATH, stdin's ``isatty`` patched, and no +read of the real ``~/.fluid`` config or keyring. +""" + +from __future__ import annotations + +import logging +import stat +import sys +from types import SimpleNamespace +from unittest import mock + +import pytest + +from fluid_build.llm import providers +from fluid_build.llm.coding_agent import _AGENT_SPECS, CODING_AGENT_NAMES +from fluid_build.llm.providers import ( + _provider_name_from, + check_llm_readiness, + resolve_llm_config, +) + +pytestmark = pytest.mark.unit + +_SELECTOR_ENV = ( + "FLUID_LLM_PROVIDER", + "FLUID_FORGE_AGENT", + "FLUID_LLM_API_KEY", + "FLUID_LLM_ENDPOINT", + "FLUID_LLM_MODEL", + "FLUID_LLM_ROUTING_ENDPOINT", + "FLUID_LLM_ROUTING_MODEL", + "OLLAMA_HOST", +) +_KEY_ENV = ( + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + "GEMINI_API_KEY", + "GOOGLE_API_KEY", + "GITHUB_API_KEY", +) + + +def _args(**kw): + base = {"llm_provider": None, "llm_model": None, "llm_endpoint": None} + base.update(kw) + return SimpleNamespace(**base) + + +@pytest.fixture +def world(monkeypatch, tmp_path): + """An empty world: no keys, no saved config, no keyring, no Ollama, empty PATH.""" + for var in _SELECTOR_ENV + _KEY_ENV: + monkeypatch.delenv(var, raising=False) + bindir = tmp_path / "bin" + bindir.mkdir() + monkeypatch.setenv("PATH", str(bindir)) + monkeypatch.setattr(sys, "stdin", SimpleNamespace(isatty=lambda: False)) + monkeypatch.setattr(providers, "_get_api_key_from_keyring", lambda name: None) + monkeypatch.setattr(providers, "detect_ollama_available", lambda env: False) + monkeypatch.setattr("fluid_build._ai_config_shared.load_ai_config", lambda *a, **k: None) + return SimpleNamespace( + bindir=bindir, + env={"PATH": str(bindir)}, + monkeypatch=monkeypatch, + install=lambda *names: [_install(bindir, n) for n in names], + ) + + +def _install(bindir, agent): + exe = bindir / _AGENT_SPECS[agent].binary + exe.write_text("#!/bin/sh\nexit 0\n") + exe.chmod(exe.stat().st_mode | stat.S_IXUSR) + + +def _tty(world, value): + world.monkeypatch.setattr(sys, "stdin", SimpleNamespace(isatty=lambda: value)) + + +class TestPick: + def test_claude_only(self, world): + world.install("claude-code") + assert _provider_name_from(_args(), dict(world.env)) == "claude-code" + + def test_codex_only(self, world): + world.install("codex") + assert _provider_name_from(_args(), dict(world.env)) == "codex" + + @pytest.mark.parametrize("agent", CODING_AGENT_NAMES) + def test_each_agent_is_found_by_its_own_binary(self, world, agent): + world.install(agent) + assert _provider_name_from(_args(), dict(world.env)) == agent + + def test_first_in_coding_agent_order_wins(self, world): + world.install(*reversed(CODING_AGENT_NAMES)) + assert _provider_name_from(_args(), dict(world.env)) == CODING_AGENT_NAMES[0] + + def test_codex_beats_later_agents(self, world): + world.install("kiro", "cursor", "codex") + assert _provider_name_from(_args(), dict(world.env)) == "codex" + + def test_running_ollama_beats_an_agent(self, world): + world.install("claude-code") + world.monkeypatch.setattr(providers, "detect_ollama_available", lambda env: True) + assert _provider_name_from(_args(), dict(world.env)) == "ollama" + + def test_nothing_detected_keeps_the_gemini_last_resort(self, world): + assert _provider_name_from(_args(), dict(world.env)) == "gemini" + + def test_non_interactive_flag_overrides_a_tty(self, world): + world.install("claude-code") + _tty(world, True) + assert _provider_name_from(_args(non_interactive=True), dict(world.env)) == "claude-code" + + def test_path_comes_from_the_env_mapping_when_given(self, world, tmp_path): + other = tmp_path / "other" + other.mkdir() + _install(other, "codex") + assert _provider_name_from(_args(), {**world.env, "PATH": str(other)}) == "codex" + + +class TestInteractiveKeepsOfferDontForce: + def test_tty_does_not_auto_pick_an_agent(self, world): + world.install("claude-code") + _tty(world, True) + assert _provider_name_from(_args(), dict(world.env)) == "gemini" + + def test_tty_still_picks_a_running_ollama(self, world): + world.install("claude-code") + _tty(world, True) + world.monkeypatch.setattr(providers, "detect_ollama_available", lambda env: True) + assert _provider_name_from(_args(), dict(world.env)) == "ollama" + + def test_closed_stdin_counts_as_non_interactive(self, world): + world.install("claude-code") + world.monkeypatch.setattr(sys, "stdin", None) + assert _provider_name_from(_args(), dict(world.env)) == "claude-code" + + +class TestConfiguredMeansNoAutoPick: + def test_explicit_flag_gemini_beats_claude_on_path(self, world): + world.install("claude-code") + assert _provider_name_from(_args(llm_provider="gemini"), dict(world.env)) == "gemini" + + def test_explicit_gemini_key_beats_claude_on_path(self, world): + world.install("claude-code") + assert _provider_name_from(_args(), {**world.env, "GEMINI_API_KEY": "k"}) == "gemini" + + def test_resolve_with_gemini_key_stays_gemini(self, world): + world.install("claude-code") + config = resolve_llm_config(_args(), {"GEMINI_API_KEY": "k"}) + assert config.provider == "gemini" + + @pytest.mark.parametrize("var", ["FLUID_LLM_PROVIDER", "FLUID_FORGE_AGENT"]) + def test_env_selector_wins(self, world, var): + world.install("claude-code") + assert _provider_name_from(_args(), {**world.env, var: "codex"}) == "codex" + + def test_two_explicit_keys_are_still_configured(self, world): + # Ambiguous keys fall through to the default, never to an agent. + world.install("claude-code") + env = {**world.env, "OPENAI_API_KEY": "k", "ANTHROPIC_API_KEY": "k"} + assert _provider_name_from(_args(), env) == "gemini" + + def test_stated_ollama_host_is_never_swapped_for_an_agent(self, world): + # Selection treats OLLAMA_HOST as an explicit Ollama choice; the agent + # rung must agree, in both the real pick and the lean (no litellm) pick. + world.install("claude-code") + env = {**world.env, "OLLAMA_HOST": "http://127.0.0.1:11500"} + assert _provider_name_from(_args(), env) == "ollama" + assert providers._lean_keyless_autopick(env, non_interactive=True) is None + + def test_generic_api_key_is_configured(self, world): + world.install("claude-code") + assert _provider_name_from(_args(), {**world.env, "FLUID_LLM_API_KEY": "k"}) == "gemini" + + def test_keyring_key_wins(self, world): + world.install("claude-code") + world.monkeypatch.setattr( + providers, "_get_api_key_from_keyring", lambda n: "k" if n == "anthropic" else None + ) + assert _provider_name_from(_args(), dict(world.env)) == "anthropic" + + def test_ambiguous_keyring_is_still_configured(self, world): + world.install("claude-code") + world.monkeypatch.setattr(providers, "_get_api_key_from_keyring", lambda n: "k") + assert _provider_name_from(_args(), dict(world.env)) == "gemini" + + def test_saved_config_is_still_configured(self, world): + world.install("claude-code") + world.monkeypatch.setattr( + "fluid_build._ai_config_shared.load_ai_config", + lambda *a, **k: {"provider": "openai", "model": "gpt-4.1-mini"}, + ) + assert _provider_name_from(_args(), dict(world.env)) == "gemini" + + +class TestSaysWhichWasPicked: + def test_resolve_keyless_pick_logs_one_line_naming_the_override(self, world, caplog): + world.install("claude-code") + with caplog.at_level(logging.WARNING, logger=providers.LOG.name): + config = resolve_llm_config(_args(), {}) + assert config.provider == "claude-code" + lines = [r.getMessage() for r in caplog.records if "claude-code" in r.getMessage()] + assert len(lines) == 1 + assert "--llm-provider" in lines[0] + assert any("--llm-provider" in note for note in config.model_resolution_notes) + + def test_explicit_agent_choice_is_not_announced(self, world, caplog): + world.install("claude-code") + with caplog.at_level(logging.WARNING, logger=providers.LOG.name): + resolve_llm_config(_args(llm_provider="claude-code"), {}) + assert not [r for r in caplog.records if "--llm-provider" in r.getMessage()] + + +class TestReadinessAgreesWithSelection: + def test_readiness_reports_the_same_provider_non_interactive(self, world): + world.install("codex") + ready = check_llm_readiness({}) + assert ready.provider == _provider_name_from(_args(), dict(world.env)) == "codex" + assert ready.ready is True + assert ready.auth_available is True + + def test_readiness_ready_without_litellm_for_an_agent(self, world): + world.install("claude-code") + with mock.patch("fluid_build._optional.is_available", return_value=False): + ready = check_llm_readiness({}) + assert ready.provider == "claude-code" + assert ready.ready is True + + def test_readiness_on_a_tty_reports_not_configured(self, world): + world.install("claude-code") + _tty(world, True) + ready = check_llm_readiness({}) + assert ready.ready is False + assert ready.provider is None + + def test_readiness_prefers_running_ollama(self, world): + world.install("claude-code") + world.monkeypatch.setattr(providers, "detect_ollama_available", lambda env: True) + assert check_llm_readiness({}).provider == "ollama" + + def test_readiness_nothing_detected(self, world): + assert check_llm_readiness({}).ready is False + + +class TestStatedDestinationBlocksAutoPick: + """A stated endpoint, model or provider key is never swapped for a cloud agent.""" + + _ONPREM = "http://onprem-llm.internal:8000/v1" + + def test_llm_endpoint_flag(self, world): + world.install("claude-code") + assert _provider_name_from(_args(llm_endpoint=self._ONPREM), dict(world.env)) != ( + "claude-code" + ) + + @pytest.mark.parametrize("flag", ["llm_routing_endpoint", "llm_routing_model"]) + def test_routing_flags(self, world, flag): + world.install("claude-code") + assert _provider_name_from(_args(**{flag: self._ONPREM}), dict(world.env)) != ( + "claude-code" + ) + + def test_llm_model_flag(self, world): + world.install("claude-code") + assert _provider_name_from(_args(llm_model="gpt-4o"), dict(world.env)) != "claude-code" + + def test_resolve_keeps_the_stated_endpoint_off_the_agent(self, world): + # The security repro: the agent is not picked, so with no key for the + # default provider the run stops before any call leaves the machine. + world.install("claude-code") + with pytest.raises(providers.CopilotGenerationError) as info: + resolve_llm_config(_args(llm_endpoint=self._ONPREM, llm_model="my-model"), {}) + assert "copilot_missing_llm_api_key" in repr(vars(info.value)) + str(info.value) + + @pytest.mark.parametrize( + "var, value", + [ + ("FLUID_LLM_ENDPOINT", _ONPREM), + ("FLUID_LLM_MODEL", "my-model"), + ("FLUID_LLM_ROUTING_ENDPOINT", _ONPREM), + ("FLUID_LLM_ROUTING_MODEL", "my-model"), + ("GITHUB_API_KEY", "ghp-test"), + ], + ) + def test_env_destination_or_provider_key(self, world, var, value): + world.install("claude-code") + env = {**world.env, var: value} + assert _provider_name_from(_args(), env) != "claude-code" + world.monkeypatch.setenv(var, value) + assert check_llm_readiness({}).provider != "claude-code" + + +class TestAgentLoopNeverAutoPicksAnAgent: + """The agent loop needs tool use, which no coding agent offers. + + It is requested by ``--agent-loop`` or ``FLUID_COPILOT_AGENT_LOOP``, and the + provider is resolved from the options ``_forge_build_copilot_options`` builds, + so both routes must reach the auto-pick. + """ + + @staticmethod + def _options(**kw): + from fluid_build.cli.forge_modes import _forge_build_copilot_options + + options = _forge_build_copilot_options( + SimpleNamespace(**kw), + get_cli_arg_fn=lambda a, k, d=None: getattr(a, k, d), + is_non_interactive=True, + ) + return type("Args", (), options)() + + def test_the_options_carry_the_flag(self): + assert self._options(agent_loop=True).agent_loop is True + assert self._options().agent_loop is False + + @pytest.mark.parametrize("value", ["1", "true", "YES", " on "]) + def test_the_options_carry_the_env_route(self, world, value): + world.monkeypatch.setenv("FLUID_COPILOT_AGENT_LOOP", value) + assert self._options().agent_loop is True + + def test_flag_route_ends_at_the_missing_key_error(self, world): + world.install("claude-code") + with pytest.raises(providers.CopilotGenerationError) as info: + resolve_llm_config(self._options(agent_loop=True), {}) + assert info.value.event == "copilot_missing_llm_api_key" + + def test_env_route_ends_at_the_missing_key_error(self, world): + world.install("claude-code") + world.monkeypatch.setenv("FLUID_COPILOT_AGENT_LOOP", "1") + with pytest.raises(providers.CopilotGenerationError) as info: + resolve_llm_config(self._options(), {}) + assert info.value.event == "copilot_missing_llm_api_key" + + def test_env_route_blocks_the_pick_in_selection(self, world): + world.install("claude-code") + env = {**world.env, "FLUID_COPILOT_AGENT_LOOP": "1"} + assert _provider_name_from(_args(), env) != "claude-code" + + @pytest.mark.parametrize("value", ["0", "false", "no", "off", ""]) + def test_a_falsey_env_value_keeps_the_pick(self, world, value): + world.install("claude-code") + env = {**world.env, "FLUID_COPILOT_AGENT_LOOP": value} + assert _provider_name_from(_args(), env) == "claude-code" + + def test_the_loop_check_is_one_rule(self): + from fluid_build.llm.providers import agent_loop_requested + + assert agent_loop_requested(True, {}) is True + assert agent_loop_requested(False, {"FLUID_COPILOT_AGENT_LOOP": "on"}) is True + assert agent_loop_requested(False, {"FLUID_COPILOT_AGENT_LOOP": "0"}) is False + + +class TestAgentReadinessNeedsTheBinary: + """``fluid doctor`` says "Ready" for a coding agent only when its CLI is on PATH. + + Auth stays each CLI's own job, so no key is looked for. + """ + + @pytest.mark.parametrize("agent", CODING_AGENT_NAMES) + def test_uninstalled_agent_is_not_ready(self, world, agent): + ready = check_llm_readiness({"FLUID_LLM_PROVIDER": agent, "PATH": "/usr/bin:/bin"}) + assert ready.ready is False + assert ready.provider == agent + assert ready.auth_available is False + assert _AGENT_SPECS[agent].binary in ready.error + + @pytest.mark.parametrize("agent", CODING_AGENT_NAMES) + def test_installed_agent_is_ready_without_a_key(self, world, agent): + world.install(agent) + ready = check_llm_readiness({**world.env, "FLUID_LLM_PROVIDER": agent}) + assert ready.ready is True + assert ready.auth_available is True + + def test_installed_agent_is_ready_without_litellm(self, world): + world.install("codex") + with mock.patch("fluid_build._optional.is_available", return_value=False): + ready = check_llm_readiness({**world.env, "FLUID_LLM_PROVIDER": "codex"}) + assert ready.ready is True + + def test_the_binary_is_searched_on_the_given_environments_path(self, world): + # Installed on the process PATH, absent from the environment handed in. + world.install("codex") + ready = check_llm_readiness({"FLUID_LLM_PROVIDER": "codex", "PATH": "/usr/bin:/bin"}) + assert ready.ready is False + + def test_ollama_is_unchanged(self, world): + with mock.patch("fluid_build._optional.is_available", return_value=True): + ready = check_llm_readiness({"FLUID_LLM_PROVIDER": "ollama", "PATH": "/usr/bin:/bin"}) + assert ready.ready is True + assert ready.provider == "ollama" + + def test_mcp_sampling_is_not_ready_through_this_check(self, world): + # It only works inside an MCP host that selects it explicitly. + ready = check_llm_readiness( + {"FLUID_LLM_PROVIDER": "mcp-sampling", "PATH": str(world.bindir)} + ) + assert ready.ready is False + + def test_a_plugin_declaring_keyless_is_still_ready(self, world): + class _Plugin: + name = "myplugin" + default_model = "m" + keyless = True + + def default_endpoint(self, model, env): + return "" + + world.monkeypatch.setitem(providers.BUILTIN_LLM_PROVIDERS, "myplugin", _Plugin()) + with mock.patch("fluid_build._optional.is_available", return_value=True): + ready = check_llm_readiness( + {"FLUID_LLM_PROVIDER": "myplugin", "PATH": str(world.bindir)} + ) + assert ready.ready is True diff --git a/tests/test_slice_ux_j_ai_caching.py b/tests/test_slice_ux_j_ai_caching.py index 01eb8694..7ceda2dc 100644 --- a/tests/test_slice_ux_j_ai_caching.py +++ b/tests/test_slice_ux_j_ai_caching.py @@ -72,13 +72,9 @@ def test_for_routing_re_derives_endpoint_when_no_routing_endpoint(self): routing_model="claude-3-5-haiku-latest", ) routed = cfg.for_routing() - # litellm path: routed endpoint is the ``litellm://anthropic/`` - # sentinel (litellm owns auth + transport, no real URL needed). - # Legacy path: ``https://api.anthropic.com/...`` — accept either. - assert "anthropic" in routed.endpoint - assert ( - routed.endpoint.startswith("litellm://anthropic/") or "anthropic.com" in routed.endpoint - ) + # litellm owns auth + transport, so the re-derived endpoint is empty: + # never a placeholder URL for something to dial. + assert routed.endpoint == "" def test_for_routing_uses_explicit_routing_endpoint(self): cfg = LlmConfig( From 331f680380b692c83e4702d20f68b42da9b317f5 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Wed, 11 Nov 2026 23:02:57 +0100 Subject: [PATCH 3/6] fix(forge): --agent with an AI flag runs AI mode, and reports the contract it wrote fluid forge --agent forced --blank unless a mode flag was given, so --llm-provider, --llm-model, --llm-endpoint and --context were dropped and the run wrote a TODO contract with exit_code 0, which an IDE agent reads as success. Those flags now select AI mode, run non-interactively, and a failed generation exits non-zero. The receipt path compared diff_snapshots actions against "created"/"modified" while it emits "create"/"update", so AI-mode runs never sent forge.contract_written. The actions are now mapped to the blank path's vocabulary. --- fluid_build/cli/forge.py | 46 +++++++--- tests/cli/test_forge_agent_mode.py | 130 ++++++++++++++++++++++++++++- 2 files changed, 163 insertions(+), 13 deletions(-) diff --git a/fluid_build/cli/forge.py b/fluid_build/cli/forge.py index 8acbc5f2..c2ab4624 100644 --- a/fluid_build/cli/forge.py +++ b/fluid_build/cli/forge.py @@ -568,10 +568,13 @@ def register(subparsers: argparse._SubParsersAction): "FLUID_FORGE_NO_{PICKER,PREVIEW,WELCOME,STREAMING_PREVIEW}=1 + " "emits JSON-Lines progress events (forge.start, forge.done, " "forge.contract_written, forge.cost) to stdout so the IDE's " - "shell tool can parse them. Defaults to --blank when no mode " - "flag is provided (avoids dropping into the interactive picker " - "the agent cannot navigate). Use this from inside any agentic " - "IDE (Kiro, Cursor, Claude Code, Cline)." + "shell tool can parse them. Any of --llm-provider, --llm-model, " + "--llm-endpoint or --context selects AI mode (strictly " + "non-interactive; a failed generation exits non-zero). With no " + "mode flag and no AI flag it defaults to --blank (avoids " + "dropping into the interactive picker the agent cannot " + "navigate). Use this from inside any agentic IDE (Kiro, " + "Cursor, Claude Code, Cline)." ), ) parser.add_argument( @@ -1282,6 +1285,12 @@ def _setup_agent_mode(args: Any) -> None: os.environ.setdefault(var, "1") args.yes = True + # An explicit AI flag is the agent choosing AI mode: no blank forcing, + # and the AI path must never block on a prompt (``non_interactive`` is + # what the rest of forge reads to skip inline setup and confirmations). + if _agent_chose_ai(args): + args.non_interactive = True + # If the agent didn't pick a mode, default to --blank: it's the # only mode that succeeds without an LLM round-trip, so it's the # safest fallback when invoked under ``--agent --data-product-type X`` @@ -1295,12 +1304,21 @@ def _setup_agent_mode(args: Any) -> None: bool(getattr(args, "refine", None)), bool(getattr(args, "from_product", None)), bool(getattr(args, "from_product_list", None)), + _agent_chose_ai(args), ] ) if no_mode: args.blank = True +def _agent_chose_ai(args: Any) -> bool: + """True when the agent passed any explicit AI flag (selects AI mode).""" + return any( + bool(getattr(args, flag, None)) + for flag in ("llm_provider", "llm_model", "llm_endpoint", "context") + ) + + def _agent_run_id() -> str: """Short, stable run-id for the JSONL event stream (12 hex chars).""" import uuid @@ -1431,9 +1449,10 @@ def _emit_forge_plan(args: Any, contract_path: Optional[Path]) -> None: "gate-check, then `fluid bundle && fluid plan` for the pipeline." ), note=( - "You (the IDE's agent) are the LLM. Fill in these fields using " - "your own Edit tool; do NOT shell-run `fluid forge --ai` (that " - "would need a second LLM API key)." + "You (the IDE's agent) are the LLM: fill in these fields using " + "your own Edit tool. Running `fluid forge --agent --llm-provider " + "` instead has forge's own AI author the contract, which " + "needs a second LLM." ), ) @@ -2026,6 +2045,10 @@ def load_context( ) +# ``diff_snapshots`` action -> the agent JSONL event's ``action``. +_AGENT_EVENT_ACTIONS = {"create": "created", "update": "modified"} + + def _write_forge_receipt( *, flow: str, @@ -2084,8 +2107,11 @@ def _write_forge_receipt( ) # Agent-mode JSONL: one event per changed file. The agent's # caller can highlight new contracts in its UI without - # parsing the receipt. - if is_agent and entry.action in ("created", "modified"): + # parsing the receipt. ``diff_snapshots`` says "create" / + # "update"; the events use the blank path's "created" / + # "modified" so every mode speaks one vocabulary. + event_action = _AGENT_EVENT_ACTIONS.get(entry.action) + if is_agent and event_action: _emit_agent_jsonl( ( "forge.contract_written" @@ -2093,7 +2119,7 @@ def _write_forge_receipt( else "forge.file_written" ), path=str(entry.path), - action=entry.action, + action=event_action, size=entry.size, ) diff --git a/tests/cli/test_forge_agent_mode.py b/tests/cli/test_forge_agent_mode.py index f06f5e30..026b1d92 100644 --- a/tests/cli/test_forge_agent_mode.py +++ b/tests/cli/test_forge_agent_mode.py @@ -18,7 +18,9 @@ from __future__ import annotations +import io import json +import logging import os import subprocess import sys @@ -243,9 +245,11 @@ def test_emit_plan_emits_forge_plan_event(tmp_path: Path): consumes_step = next(s for s in plan["next_steps"] if "consumes" in s["step"].lower()) assert "list_source_lineage" in consumes_step["mcp_tools"] - # The architectural note must remind the agent NOT to shell-run fluid forge --ai. - assert "do NOT" in plan["note"] or "do not" in plan["note"].lower() - assert "API key" in plan["note"] or "api key" in plan["note"].lower() + # The note must describe the real split: the agent fills the fields itself, + # while --agent plus an AI flag runs forge's own AI with a second LLM. + assert "--ai" not in plan["note"] + assert "--llm-provider" in plan["note"] + assert "second LLM" in plan["note"] # Order: plan event must come BEFORE forge.done so consumers parsing # the stream see "what to do next" before the terminal signal. @@ -330,3 +334,123 @@ def test_agent_flag_registered_in_argparse(): args = parser.parse_args(["forge", "--agent", "--blank"]) assert getattr(args, "agent", False) is True assert getattr(args, "blank", False) is True + + +# --------------------------------------------------------------------------- +# 6. Explicit AI flags under --agent select AI mode (never a silent blank) +# --------------------------------------------------------------------------- + + +def _agent_args(argv: list[str], tmp_path: Path): + import argparse + + from fluid_build.cli import forge as forge_mod + + parser = argparse.ArgumentParser() + sp = parser.add_subparsers() + forge_mod.register(sp) + return parser.parse_args(["forge", "--agent", *argv, "-d", str(tmp_path / "product")]) + + +def _run_agent(args, capsys, monkeypatch, *, ai_result=0): + """Run forge.run under --agent with the AI generation seam mocked. + + Returns (events, ai_mock). stdin is a non-TTY and ``input`` must never be + called: an agent's shell cannot answer a prompt. + """ + from unittest.mock import MagicMock + + from fluid_build.cli import forge as forge_mod + + ai = MagicMock(side_effect=ai_result if isinstance(ai_result, Exception) else None) + if not isinstance(ai_result, Exception): + ai.return_value = ai_result + monkeypatch.setattr(forge_mod, "run_ai_copilot_mode", ai) + monkeypatch.setattr( + "builtins.input", lambda *a, **k: pytest.fail("input() called under --agent") + ) + monkeypatch.setattr(forge_mod, "_write_forge_receipt", lambda **k: None) + forge_mod.run(args, logging.getLogger("test")) + return _parse_jsonl_events(capsys.readouterr().out), ai + + +def test_agent_alone_is_blank_in_process(tmp_path, capsys, monkeypatch): + args = _agent_args(["--data-product-type", "SDP"], tmp_path) + events, ai = _run_agent(args, capsys, monkeypatch) + assert next(e for e in events if e["event"] == "forge.start")["mode"] == "blank" + assert args.blank is True + ai.assert_not_called() + + +def test_agent_with_llm_provider_selects_ai_mode(tmp_path, capsys, monkeypatch): + args = _agent_args(["--llm-provider", "claude-code"], tmp_path) + events, ai = _run_agent(args, capsys, monkeypatch) + assert next(e for e in events if e["event"] == "forge.start")["mode"] == "ai" + assert not args.blank + ai.assert_called_once() + assert next(e for e in events if e["event"] == "forge.done")["exit_code"] == 0 + + +@pytest.mark.parametrize( + "flags", + [ + ["--context", '{"project_goal": "daily revenue"}'], + ["--llm-model", "some-model"], + ["--llm-endpoint", "http://localhost:1234"], + ], +) +def test_agent_with_other_ai_flags_selects_ai_mode(flags, tmp_path, capsys, monkeypatch): + args = _agent_args(flags, tmp_path) + events, ai = _run_agent(args, capsys, monkeypatch) + assert next(e for e in events if e["event"] == "forge.start")["mode"] == "ai" + assert not args.blank + ai.assert_called_once() + + +def test_agent_ai_mode_is_non_interactive(tmp_path, capsys, monkeypatch): + """AI mode under --agent must never reach the inline-setup prompt flow.""" + from fluid_build.cli import ai_setup + + monkeypatch.setattr( + ai_setup, + "run_ai_setup_inline", + lambda *a, **k: pytest.fail("interactive AI setup reached under --agent"), + ) + monkeypatch.setattr("sys.stdin", io.StringIO("")) + args = _agent_args(["--llm-provider", "claude-code"], tmp_path) + events, ai = _run_agent(args, capsys, monkeypatch) + assert args.non_interactive is True + ai.assert_called_once() + + +@pytest.mark.parametrize("failure", [1, RuntimeError("provider down")]) +def test_agent_ai_failure_is_not_success(failure, tmp_path, capsys, monkeypatch): + args = _agent_args(["--llm-provider", "claude-code"], tmp_path) + events, _ = _run_agent(args, capsys, monkeypatch, ai_result=failure) + done = next(e for e in events if e["event"] == "forge.done") + assert done["exit_code"] != 0 + + +def test_agent_ai_mode_emits_contract_written(tmp_path, capsys, monkeypatch): + """The receipt path's "create" rows reach the stream as forge.contract_written.""" + from fluid_build.cli import forge as forge_mod + + product = tmp_path / "product" + + def _fake_ai(args, logger): + product.mkdir(parents=True, exist_ok=True) + (product / "contract.fluid.yaml").write_text("fluidVersion: 0.7.6\n") + return 0 + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(forge_mod, "run_ai_copilot_mode", _fake_ai) + monkeypatch.setattr( + "builtins.input", lambda *a, **k: pytest.fail("input() called under --agent") + ) + args = _agent_args(["--llm-provider", "claude-code"], tmp_path) + forge_mod.run(args, logging.getLogger("test")) + events = _parse_jsonl_events(capsys.readouterr().out) + written = [e for e in events if e["event"] == "forge.contract_written"] + assert written, [e["event"] for e in events] + assert written[0]["action"] == "created" + assert written[0]["path"].endswith("contract.fluid.yaml") From fadbdbf581d1ae22221669315d30c7960c057ed1 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Wed, 11 Nov 2026 23:02:57 +0100 Subject: [PATCH 4/6] docs(changelog): the copilot keyless fixes and the claude -p isolation --- CHANGELOG.md | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index de6f36b3..c1272622 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,49 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 granted. The contract-tests local action grants its outputs the same way. The DuckDB acquisition runner's landing got this fix in 0.19.0 (#699). Found on the demo lab on 0.18.2, still in 0.28.0. +- **Local Ollama works through every automatic path, and `fluid ai test` probes the provider it + names.** With no endpoint set, `LlmConfig.endpoint` carried a telemetry label, + `litellm:///`. Ollama calls sent it to litellm as `api_base`, so no call + reached the daemon, and `fluid ai test` posted to it and failed with "AI test network error + ... (UnsupportedProtocol)" for every litellm provider. The endpoint now holds only a real + address: the one `--llm-endpoint` or `FLUID_LLM_ENDPOINT` states, else Ollama's `OLLAMA_HOST`, + else none (litellm's own default). A bare `OLLAMA_HOST=myhost` means `http://myhost:11434`, as + in Ollama's own client. The routing model's calls keep a stated endpoint. `fluid ai test` + sends its probe through the call `fluid forge` makes and exits 4 when the provider says the + model does not exist. `fluid doctor` shows "Provider default" instead of the label. +- **`fluid forge --agent` with `--llm-provider`, `--llm-model`, `--llm-endpoint` or `--context` + runs AI mode.** It dropped those flags, wrote a TODO placeholder contract and reported + `exit_code 0`, which an IDE agent reads as success. It now authors with forge's AI, + non-interactively, and a failed generation exits non-zero. AI-mode runs now emit + `forge.contract_written`, which the stream promised but only blank mode sent. `--agent` with + no AI flag still writes the blank contract. +- **A Claude Code login problem is reported as one.** `claude -p` reports a failure on stdout, + as a JSON result with `is_error: true`, and leaves stderr empty, so forge showed + "claude-code exited with code 1:" and no reason. Forge now reads that result: a login problem + says the agent "is installed but not authenticated" with the sign-in hint, any other failure + shows the agent's own message, and `is_error: true` is a failure even at exit code 0. +- **`fluid doctor` reports an installed coding agent as ready.** With `FLUID_LLM_PROVIDER` set to + `claude-code`, `codex`, `cursor` or `kiro`, readiness said "No API key found"; it now checks + that the agent's CLI is on PATH, and says so when it is not. Each CLI keeps its own sign-in. + +### Changed + +- **A non-interactive run with no LLM configured authors with an installed coding agent.** When + nothing names one (no `--llm-provider`, `--llm-endpoint` or `--llm-model`, no `FLUID_LLM_*`, + `OLLAMA_HOST` or provider key variable, no keyring key, no `fluid ai setup` provider) and stdin + is not a terminal or `--non-interactive` is set, forge uses a running local Ollama first, so + local data stays local, else the first of `claude`, `codex`, `cursor-agent` and `kiro-cli` on + PATH, and logs which one and how to override it. It works on the base install: the agents need + no litellm. Interactive runs still only offer the agent, and `--agent-loop` never picks one + (the agents have no tool use). +- **Forge runs `claude -p` apart from your own Claude Code setup.** Each call replaces Claude + Code's agent system prompt with forge's (`--system-prompt`) and turns off tools, MCP servers, + your settings (hooks, plugins, `CLAUDE.md`) and session persistence (`--tools ""`, + `--strict-mcp-config`, `--setting-sources=`, `--no-session-persistence`), and drops + `CLAUDECODE`, as Anthropic's Agent SDK does. Each flag is passed only when the installed + `claude --help` lists it. Claude Code authentication that lives in a setting (an + `apiKeyHelper`, or Bedrock or Vertex variables in `settings.json`) must come from the shell + environment instead. ## [0.28.0] — 2026-11-06 From f27b6a40a536edae066ca3eb7bf3530d14eaf351 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Wed, 11 Nov 2026 23:30:08 +0100 Subject: [PATCH 5/6] chore(secrets): mark a test's fake key, and move the baseline's line numbers The lean-install preflight test sets GEMINI_API_KEY to a dummy value, which detect-secrets flags as a Secret Keyword. It now carries the inline allowlist pragma the other tests use. The baseline's line numbers for already-audited fixtures in the touched test files moved with this PR's edits; detect-secrets-hook rewrote them. --- .secrets.baseline | 28 ++++++++++++++-------------- tests/test_litellm_absent.py | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index 26121825..358260d5 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -1537,21 +1537,21 @@ "filename": "tests/test_ai_setup.py", "hashed_secret": "06c3796a342a1f1172aa7a907d3c0a5da02a053a", "is_verified": false, - "line_number": 1092 + "line_number": 1098 }, { "type": "Secret Keyword", "filename": "tests/test_ai_setup.py", "hashed_secret": "6b1196369d5cd92be55dbdd72ef619957b6e6624", "is_verified": false, - "line_number": 1096 + "line_number": 1102 }, { "type": "Secret Keyword", "filename": "tests/test_ai_setup.py", "hashed_secret": "9ea9d5acda505555f09a5d2203f7078f1a3b6363", "is_verified": false, - "line_number": 1101 + "line_number": 1107 } ], "tests/test_ai_setup_import_keys.py": [ @@ -1860,70 +1860,70 @@ "filename": "tests/test_forge_copilot_runtime.py", "hashed_secret": "3fa437025c8f9e1c6f0f5547173acee9c2e22c3a", "is_verified": false, - "line_number": 190 + "line_number": 191 }, { "type": "Secret Keyword", "filename": "tests/test_forge_copilot_runtime.py", "hashed_secret": "c734e47630dda71619c696d88381f06f7511bd78", "is_verified": false, - "line_number": 191 + "line_number": 192 }, { "type": "Secret Keyword", "filename": "tests/test_forge_copilot_runtime.py", "hashed_secret": "a47110e348a3063541fb1f1f640d635d457181a0", "is_verified": false, - "line_number": 207 + "line_number": 208 }, { "type": "Secret Keyword", "filename": "tests/test_forge_copilot_runtime.py", "hashed_secret": "fd9a45bc0b07706d849cf85021ecf9123fa83d82", "is_verified": false, - "line_number": 266 + "line_number": 268 }, { "type": "Secret Keyword", "filename": "tests/test_forge_copilot_runtime.py", "hashed_secret": "e9a5f12a8ecbb3eb46eca5096b5c52aa5e7c9fdd", "is_verified": false, - "line_number": 299 + "line_number": 301 }, { "type": "Secret Keyword", "filename": "tests/test_forge_copilot_runtime.py", "hashed_secret": "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3", "is_verified": false, - "line_number": 338 + "line_number": 340 }, { "type": "Secret Keyword", "filename": "tests/test_forge_copilot_runtime.py", "hashed_secret": "8079944bb2ba4b2d6eb02908a6de57cb57c6f39a", "is_verified": false, - "line_number": 363 + "line_number": 365 }, { "type": "Secret Keyword", "filename": "tests/test_forge_copilot_runtime.py", "hashed_secret": "56e1d57b8db262b08bc73c60ed08d8c92e59503f", "is_verified": false, - "line_number": 434 + "line_number": 436 }, { "type": "Secret Keyword", "filename": "tests/test_forge_copilot_runtime.py", "hashed_secret": "a62f2225bf70bfaccbc7f1ef2a397836717377de", "is_verified": false, - "line_number": 1037 + "line_number": 1039 }, { "type": "Secret Keyword", "filename": "tests/test_forge_copilot_runtime.py", "hashed_secret": "63d185ab613a8f4ba8a3f2669f49e8df82bff53f", "is_verified": false, - "line_number": 1175 + "line_number": 1177 } ], "tests/test_forge_copilot_scenarios.py": [ @@ -2317,5 +2317,5 @@ } ] }, - "generated_at": "2026-11-05T14:05:52Z" + "generated_at": "2026-11-11T22:30:08Z" } diff --git a/tests/test_litellm_absent.py b/tests/test_litellm_absent.py index 3f6c0a70..4f75f9e8 100644 --- a/tests/test_litellm_absent.py +++ b/tests/test_litellm_absent.py @@ -466,7 +466,7 @@ def test_an_api_key_still_fails_fast(self, block, monkeypatch, tmp_path): # noq from fluid_build.llm.providers import preflight_llm_provider env = self._lean(block, monkeypatch, tmp_path) - env["GEMINI_API_KEY"] = "AIza-test" + env["GEMINI_API_KEY"] = "AIza-test" # pragma: allowlist secret with pytest.raises(MissingOptionalDependency): preflight_llm_provider(self._args(), env) From ad913502eac38c334a706875ca8368878bae887a Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Wed, 11 Nov 2026 23:52:28 +0100 Subject: [PATCH 6/6] chore(credentials): cover the 14 endpoint names litellm 1.109.0 reads litellm 1.109.0 (released 2026-11-11) reads fourteen more _API_BASE names: ABLITERATION, AIAND, BIGMODEL, CLF_AI_GATEWAY, EU_JEV, HOSTINGER_ROUTER, KYMA, NACE, OPENCODE_GO, OPENINFER, OPENSEARCH, REFUSELESS, STREAMLAKE and ZEROGPU. Each comes from litellm.provider_list, llms/openai_like/providers.json or the provider's own module, so each picks a provider host and joins LITELLM_ENDPOINT_NAMES. tests/test_litellm_endpoint_names.py failed on every CI leg that installs it (Python 3.11 and up; 3.10 stays below 1.98). It passes on litellm 1.109.0 and 1.104.0. --- fluid_build/credentials/litellm_endpoints.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/fluid_build/credentials/litellm_endpoints.py b/fluid_build/credentials/litellm_endpoints.py index 4514ae9f..2b50a1d6 100644 --- a/fluid_build/credentials/litellm_endpoints.py +++ b/fluid_build/credentials/litellm_endpoints.py @@ -13,9 +13,9 @@ project's own setting and follows the name-shape rule. Vendored because importing litellm takes seconds and dotenv hydration runs on -every command. Generated from the litellm 1.83.14, 1.97.0, 1.105.0, 1.106.0 -and 1.108.0 wheels (FLUID's floor, the last release Python 3.10 can import, and -the newest), as the union of: +every command. Generated from the litellm 1.83.14, 1.97.0, 1.105.0, 1.106.0, +1.108.0 and 1.109.0 wheels (FLUID's floor, the last release Python 3.10 can +import, and the newest), as the union of: * ``_API_BASE`` for every ``litellm.provider_list`` entry (``-`` read as ``_``); @@ -45,9 +45,11 @@ { "A2A_AGENT_API_BASE", "A2A_API_BASE", + "ABLITERATION_API_BASE", "AGENTCORE_GATEWAY_URL", "AI21_API_BASE", "AI21_CHAT_API_BASE", + "AIAND_API_BASE", "AIHUBMIX_API_BASE", "AIML_API_BASE", "AIOHTTP_OPENAI_API_BASE", @@ -75,6 +77,7 @@ "BEDROCK_MANTLE_API_BASE", "BESPOKE_API_BASE", "BFL_API_BASE", + "BIGMODEL_API_BASE", "BLACK_FOREST_LABS_API_BASE", "BRAVE_API_BASE", "BYTEZ_API_BASE", @@ -83,6 +86,7 @@ "CHATGPT_API_BASE", "CHUTES_API_BASE", "CLARIFAI_API_BASE", + "CLF_AI_GATEWAY_API_BASE", "CLOUDFLARE_API_BASE", "CODESTRAL_API_BASE", "COGNITION_API_BASE", @@ -120,6 +124,7 @@ "ELEVENLABS_API_BASE", "EMPIRIOLABS_API_BASE", "EMPOWER_API_BASE", + "EU_JEV_API_BASE", "EXA_API_BASE", "FAL_AI_API_BASE", "FAL_AI_QUEUE_API_BASE", @@ -148,6 +153,7 @@ "HEROKU_API_BASE", "HF_API_BASE", "HOSTED_VLLM_API_BASE", + "HOSTINGER_ROUTER_API_BASE", "HUGGINGFACE_API_BASE", "HUMANLOOP_API_BASE", "HYPERBOLIC_API_BASE", @@ -155,6 +161,7 @@ "INFERENCEAPIS_API_BASE", "INFINITY_API_BASE", "JINA_AI_API_BASE", + "KYMA_API_BASE", "LAMBDA_AI_API_BASE", "LAMBDA_API_BASE", "LANGFLOW_API_BASE", @@ -182,6 +189,7 @@ "MONGODB_API_BASE", "MOONSHOT_API_BASE", "MORPH_API_BASE", + "NACE_API_BASE", "NADIR_API_BASE", "NANO_GPT_API_BASE", "NEBIUS_API_BASE", @@ -199,7 +207,10 @@ "OPENAI_API_BASE", "OPENAI_CHATGPT_API_BASE", "OPENAI_LIKE_API_BASE", + "OPENCODE_GO_API_BASE", + "OPENINFER_API_BASE", "OPENROUTER_API_BASE", + "OPENSEARCH_API_BASE", "OPEN_SANDBOX_API_BASE", "OVHCLOUD_API_BASE", "PARALLEL_AI_API_BASE", @@ -223,6 +234,7 @@ "RAGFLOW_API_BASE", "RECRAFT_API_BASE", "REDUCTO_API_BASE", + "REFUSELESS_API_BASE", "REKA_API_BASE", "REPLICATE_API_BASE", "RUNWAYML_API_BASE", @@ -245,6 +257,7 @@ "SONIOX_API_BASE", "STABILITY_API_BASE", "STRANDS_DECIDER_API_BASE", + "STREAMLAKE_API_BASE", "SYNTHETIC_API_BASE", "TAVILY_API_BASE", "TENCENT_ANTHROPIC_API_BASE", @@ -283,6 +296,7 @@ "XINFERENCE_API_BASE", "YOUCOM_API_BASE", "ZAI_API_BASE", + "ZEROGPU_API_BASE", } )