diff --git a/CHANGELOG.md b/CHANGELOG.md index 8dd209b1..6c1dbc29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -146,6 +146,40 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 calls a stated endpoint too, as the routing model already did; it called the provider's default address instead. +### Security + +- **`FLUID_LLM_API_KEY` goes only to a provider you name.** The key does not say which vendor it + is for, yet forge sent it to whatever provider it picked, including one it only inferred: a + running Ollama or the one `OLLAMA_HOST` names (as `Authorization: Bearer`, in cleartext when + the host is http), the provider of the one key saved in the keyring, or the provider of the one + key variable set. So an OpenAI key could reach a local or remote Ollama, or Anthropic. Forge + now uses it only when `--llm-provider` or `FLUID_LLM_PROVIDER` names the provider the run uses. + A provider forge picks itself gets its own key variable, then its keyring key, and a warning + says the generic key was not used. In interactive `fluid forge` and `fluid init`, the provider + that setup finds counts as picked, not named. `fluid ai test` reads the key only with + `--llm-provider`, and `fluid doctor` counts it only with `FLUID_LLM_PROVIDER`. A named Ollama + still gets the key (ollama.com, a token proxy). If you used the key with a provider forge + inferred, name that provider: `--llm-provider ` or `FLUID_LLM_PROVIDER=`. +- **A project `.env` can no longer pair an OpenAI, Anthropic, Gemini or GitHub endpoint with + `FLUID_LLM_API_KEY`.** Forge can pick these providers without a name, and then it sends the + operator's keyring key, not the project's `FLUID_LLM_API_KEY`, so `OPENAI_BASE_URL`, + `ANTHROPIC_BASE_URL`, `GEMINI_API_BASE`, `GITHUB_API_BASE` and the like in a project file now + need the provider's own key variable beside them (`OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, + `GEMINI_API_KEY` or `GOOGLE_API_KEY`, `GITHUB_API_KEY`). Endpoints of providers forge reaches + only by name (`GROQ_API_BASE`, `DEEPSEEK_API_BASE` ...) still pair with it. +- **Interactive setup no longer sends your saved key to a host a project's `.env` names.** A + project file may point a provider at its own host (`OPENAI_BASE_URL`, `ANTHROPIC_BASE_URL`, + `GEMINI_API_BASE`, `GITHUB_API_BASE`) next to a key of its own. Interactive `fluid forge` and + `fluid init` then loaded the provider `fluid ai setup` saved and copied its keyring key into the + provider's key variable, over the project's key (for Gemini into `GOOGLE_API_KEY`, which forge + reads first), so your key went to the project's host. While a project endpoint of that provider + is in effect, setup now leaves the variable alone, the project's own key goes there, and a + warning says your key was not used. `fluid ai setup` no longer checks a typed or imported key + against such a host either: the check is one litellm call, which reads the endpoint from the + environment. It now asks for no key there, skips a key it would import, and says to run + `fluid ai setup` outside the project. For a provider other than OpenAI, Anthropic, Gemini and + GitHub, any LLM endpoint a project file set counts. + ## [0.28.0] — 2026-11-06 ### Added diff --git a/fluid_build/cli/_ai_setup_test.py b/fluid_build/cli/_ai_setup_test.py index 6226587c..9d6dc80d 100644 --- a/fluid_build/cli/_ai_setup_test.py +++ b/fluid_build/cli/_ai_setup_test.py @@ -202,8 +202,16 @@ def _validate_ai_test_endpoint(provider_name: str, endpoint: str) -> Optional[st return None -def _resolve_cloud_api_key(provider: str, saved: Optional[dict] = None) -> Optional[str]: - if os.environ.get("FLUID_LLM_API_KEY"): +def _resolve_cloud_api_key( + provider: str, saved: Optional[dict] = None, *, named: bool = False +) -> Optional[str]: + """The key ``fluid ai test`` sends to *provider*. + + ``FLUID_LLM_API_KEY`` counts only when *named* (``--llm-provider``): it does + not say which vendor it is for, so a saved or env-inferred provider never + gets it (``llm/provider_resolve._resolve_api_key``). + """ + if named and os.environ.get("FLUID_LLM_API_KEY"): return os.environ["FLUID_LLM_API_KEY"] if provider == "gemini": for env_var in ("GEMINI_API_KEY", "GOOGLE_API_KEY"): @@ -236,7 +244,7 @@ def _resolve_ai_test_config(args: Any) -> tuple[Optional[LlmConfig], Optional[st saved = _load_ai_config() or {} def _build_for_provider( - pname: str, *, allow_saved: bool = True + pname: str, *, allow_saved: bool = True, named: bool = False ) -> tuple[Optional[LlmConfig], Optional[str]]: provider = BUILTIN_LLM_PROVIDERS.get(pname) if not provider: @@ -264,7 +272,7 @@ def _build_for_provider( ), None, ) - api_key = _resolve_cloud_api_key(pname, saved_for_provider) + api_key = _resolve_cloud_api_key(pname, saved_for_provider, named=named) if not api_key: return None, f"No API key found for {PROVIDER_DISPLAY_NAMES.get(pname, pname)}." endpoint = endpoint_override or saved_for_provider.get("endpoint") @@ -279,7 +287,7 @@ def _build_for_provider( ) if provider_override: - config, error = _build_for_provider(provider_override) + config, error = _build_for_provider(provider_override, named=True) if config: config.timeout_seconds = timeout return config, error diff --git a/fluid_build/cli/ai_setup.py b/fluid_build/cli/ai_setup.py index 4fb09e42..0e57f80d 100644 --- a/fluid_build/cli/ai_setup.py +++ b/fluid_build/cli/ai_setup.py @@ -232,7 +232,15 @@ def _validate_api_key(provider: Any, api_key: str) -> Optional[str]: renders as a one-line install hint. Returns ``None`` on success or a short human-readable error message. + + Nothing is sent while a project dotenv file points the provider at its own + host: litellm reads that endpoint from the environment, so the check would + hand the operator's key (typed, or imported from another tool) to it. """ + notice = _project_host_notice(provider.name.lower()) + if notice: + return f"not checked: {notice}" + from fluid_build._optional import require litellm = require("litellm", "litellm", "Validating an AI provider key") @@ -272,21 +280,108 @@ def _validate_api_key(provider: Any, api_key: str) -> Optional[str]: return f"validation failed: {exc}" +# The ``credentials.protected_env`` endpoint group of each provider whose +# PROVIDER_ENV_VARS name ``set_session_env`` writes. +_PROVIDER_ENDPOINT_SERVICES = { + "openai": "OpenAI", + "anthropic": "Anthropic", + "gemini": "Google Gemini", + "github": "GitHub Models", +} + + +def _endpoint_service(provider: str) -> Optional[str]: + """The ``credentials.protected_env`` group of *provider*, or ``None``. + + Picked by comparison, so the result is the table's own constant. A provider + name can come from classifying a typed key (``_classify_key_shape``), and + what setup logs must not be derived from a key (CodeQL + py/clear-text-logging-sensitive-data). + """ + return next( + (service for name, service in _PROVIDER_ENDPOINT_SERVICES.items() if name == provider), + None, + ) + + +def _project_llm_endpoints(provider: str) -> List[str]: + """The endpoint variables in effect that project dotenv files point *provider* at. + + ``credentials.protected_env`` loaded such an endpoint only because the + project files supply a key for it, the one its readers send there. For a + provider whose key ``set_session_env`` writes, the endpoints of its own + group. For any other provider, every LLM endpoint the project set (the + groups that list FLUID_LLM_API_KEY, which every LLM group does, and the + litellm names no group lists): litellm's routes are not mapped one by one. + """ + from fluid_build.credentials import dotenv_store + + service = _endpoint_service(provider) + if service is not None: + return dotenv_store.project_endpoints_in_effect(service) + from fluid_build.credentials.protected_env import ( + ENDPOINT_CREDENTIAL_GROUPS, + UNLISTED_LLM_ENDPOINTS, + is_endpoint_of, + ) + + groups = [g for g in ENDPOINT_CREDENTIAL_GROUPS if "FLUID_LLM_API_KEY" in g.credentials] + groups.append(UNLISTED_LLM_ENDPOINTS) + return sorted( + name + for name in dotenv_store.project_values_in_effect() + if any(is_endpoint_of(group, name) for group in groups) + ) + + +def _project_host_notice(provider: str) -> Optional[str]: + """Why setup sends no key of the operator's for *provider*, or ``None``. + + Built from the endpoint group's name and the endpoint variables' names, never + from *provider* itself or a key, because it is logged (CodeQL + py/clear-text-logging-sensitive-data). + """ + names = _project_llm_endpoints(provider) + if not names: + return None + who = _endpoint_service(provider) or "this LLM provider" + return ( + f"a project dotenv file points {who} at its own host ({', '.join(names)}), " + "so your key is not sent there; run 'fluid ai setup' outside this project" + ) + + +def _warn_key_not_used(provider: str) -> None: + """Log that the operator's key for *provider* is not used for this run.""" + notice = _project_host_notice(provider) + if notice: + LOG.warning("Your key is not used: %s.", notice) + + def set_session_env(provider: str, api_key: str) -> None: """Set the provider-specific env var for the current process only. This is necessary so that ``resolve_llm_config()`` can find the key during this session. The key is **not** written to disk or exported to child processes beyond the current process tree. + + Not while a project dotenv file points the provider at its own host: the + key is the operator's (keyring, ``ai_config.json``, a prompt, another + tool's login), and the readers would send it there in place of the key + the project paired that host with. """ from fluid_build.cli.ai_setup import PROVIDER_ENV_VARS env_var = PROVIDER_ENV_VARS.get(provider) - if env_var: - os.environ[env_var] = api_key - # Don't interpolate env_var/provider — ``api_key`` is in scope - # at this LOG site (CodeQL py/clear-text-logging-sensitive-data). - LOG.debug("Set session env var (provider configured)") + if not env_var: + return + if _project_llm_endpoints(provider): + _warn_key_not_used(provider) + return + os.environ[env_var] = api_key + # Don't interpolate env_var/provider — ``api_key`` is in scope + # at this LOG site (CodeQL py/clear-text-logging-sensitive-data). + LOG.debug("Set session env var (provider configured)") # --------------------------------------------------------------------------- @@ -579,6 +674,12 @@ def _collect_and_validate_api_key( label = PROVIDER_DISPLAY_NAMES.get(provider_choice, provider_choice) signup_url = _SIGNUP_URLS.get(provider_choice, "") last_error: Optional[str] = None + # A key typed here could be neither checked nor used (_validate_api_key, + # set_session_env), so ask for none. + notice = _project_host_notice(provider_choice) + if notice: + console.print(literal_error_text(f"Not asking for a {label} key: {notice}.", "yellow")) + return None for attempt in range(1, max_attempts + 1): url_hint = ( @@ -1250,6 +1351,10 @@ def _import_and_persist(console: Any, candidate: _ImportedKey) -> Optional[LlmCo if provider_obj is None: console.print(f"[yellow]Could not initialise {label}; skipping.[/yellow]") return None + notice = _project_host_notice(provider_obj.name.lower()) + if notice: # the key could be neither checked nor used (_validate_api_key) + console.print(literal_error_text(f"Skipping the {label} key: {notice}.", "yellow")) + return None console.print("[dim]Verifying the imported key...[/dim]") # Don't interpolate the key at any LOG site in this frame # (CodeQL py/clear-text-logging-sensitive-data — ``candidate.key`` in scope). diff --git a/fluid_build/cli/forge.py b/fluid_build/cli/forge.py index 92d6cbad..90804c98 100644 --- a/fluid_build/cli/forge.py +++ b/fluid_build/cli/forge.py @@ -1879,6 +1879,11 @@ def _run_main(args, logger: logging.Logger) -> int: # 3. Nothing found → prompts user interactively llm_config = run_ai_setup_inline(console) if llm_config: + from fluid_build.llm.providers import remember_named_llm_provider + + # First keep what the operator named: FLUID_LLM_API_KEY goes only to + # that provider, never to the one setup found. + remember_named_llm_provider(args) # Inject into args so copilot's resolve_llm_config() finds them args.llm_provider = llm_config.provider args.llm_model = llm_config.model diff --git a/fluid_build/cli/forge_modes.py b/fluid_build/cli/forge_modes.py index eab6baf7..ee867798 100644 --- a/fluid_build/cli/forge_modes.py +++ b/fluid_build/cli/forge_modes.py @@ -308,7 +308,7 @@ def _forge_build_copilot_options( """ from fluid_build.llm.providers import agent_loop_requested - return { + options = { "llm_provider": get_cli_arg_fn(args, "llm_provider"), "llm_model": get_cli_arg_fn(args, "llm_model"), "llm_endpoint": get_cli_arg_fn(args, "llm_endpoint"), @@ -357,6 +357,14 @@ def _forge_build_copilot_options( "seed_allow_remote": bool(get_cli_arg_fn(args, "seed_allow_remote", False)), "seed_no_remote": bool(get_cli_arg_fn(args, "seed_no_remote", False)), } + # Interactive setup may have put the provider it found into llm_provider. + # Carry what the operator named, so FLUID_LLM_API_KEY still goes only to that + # provider (providers.remember_named_llm_provider). + not_recorded = object() + named = get_cli_arg_fn(args, "llm_provider_named", not_recorded) + if named is not not_recorded: + options["llm_provider_named"] = named + return options def _forge_install_pause_handler( diff --git a/fluid_build/cli/init.py b/fluid_build/cli/init.py index cd85cf02..2b1eda66 100644 --- a/fluid_build/cli/init.py +++ b/fluid_build/cli/init.py @@ -915,6 +915,10 @@ def _ai_mode(args, logger: logging.Logger) -> int: _console = console if RICH_AVAILABLE else None llm_config = run_ai_setup_inline(_console) if llm_config: + from fluid_build.llm.providers import remember_named_llm_provider + + # FLUID_LLM_API_KEY goes only to a provider the operator named. + remember_named_llm_provider(args) args.llm_provider = llm_config.provider args.llm_model = llm_config.model args.llm_endpoint = llm_config.endpoint diff --git a/fluid_build/credentials/litellm_endpoints.py b/fluid_build/credentials/litellm_endpoints.py index 2b50a1d6..c853cab9 100644 --- a/fluid_build/credentials/litellm_endpoints.py +++ b/fluid_build/credentials/litellm_endpoints.py @@ -4,13 +4,14 @@ """Where litellm reads an LLM provider's endpoint, vendored. ``credentials.protected_env`` pairs these with ``FLUID_LLM_API_KEY`` only: -FLUID routes any provider litellm knows and passes it the key it resolved -(``FLUID_LLM_API_KEY``, else the keyring), which beats a provider key in the -project's files. A group in ``protected_env`` that lists one of the names -(``IBM watsonx``, ``OpenAI`` ...) pairs it by its own rule instead, and a name -litellm sends something else to (``LITELLM_PROXY_API_BASE``, the GitHub -Copilot ones) is protected outright. Any other ``_API_BASE`` is the -project's own setting and follows the name-shape rule. +FLUID routes any provider litellm knows, only when the operator names it, and +passes it the key it resolved (``FLUID_LLM_API_KEY``, else the keyring), which +beats a provider key in the project's files. A group in ``protected_env`` that +lists one of the names (``IBM watsonx``, ``OpenAI`` ...) pairs it by its own +rule instead, and a name litellm sends something else to +(``LITELLM_PROXY_API_BASE``, the GitHub Copilot ones) is protected outright. +Any other ``_API_BASE`` is the project's own setting and follows the +name-shape rule. Vendored because importing litellm takes seconds and dotenv hydration runs on every command. Generated from the litellm 1.83.14, 1.97.0, 1.105.0, 1.106.0, diff --git a/fluid_build/credentials/protected_env.py b/fluid_build/credentials/protected_env.py index 3ceb6ceb..6150ec73 100644 --- a/fluid_build/credentials/protected_env.py +++ b/fluid_build/credentials/protected_env.py @@ -83,8 +83,8 @@ (keyring, a stored CLI login, an SDK config file, cloud default credentials), whenever the project files do not supply one that every reader sends in its place. A project credential a reader consults after the keyring (FLUID passes -the key it resolved, FLUID_LLM_API_KEY else the keyring, as litellm's -``api_key``, which beats ``GROQ_API_KEY``), or never (``CLAUDE_API_KEY`` for +the key it resolved, FLUID_LLM_API_KEY for a named provider else the keyring, +as litellm's ``api_key``, which beats ``GROQ_API_KEY``), or never (``CLAUDE_API_KEY`` for litellm), is a decoy and pairs nothing; :attr:`EndpointGroup.paired_by` lists, per code path, the credentials that do. :data:`ENDPOINT_CREDENTIAL_GROUPS` lists the services and where each variable is read; the names litellm reads @@ -356,7 +356,8 @@ "AWS_IGNORE_CONFIGURED_ENDPOINT_URLS", "OLLAMA_HOST", # litellm's ollama/ route reads this one, and sends it the key FLUID - # resolved (FLUID_LLM_API_KEY or the keyring) as a bearer token + # resolved (FLUID_LLM_API_KEY for a named ollama, or the keyring) as a + # bearer token "OLLAMA_API_BASE", "VAULT_ADDR", # litellm routes every model call to this proxy when the switch is @@ -883,17 +884,22 @@ class EndpointGroup: connection_string_passwords: Tuple[str, ...] = () -# ``llm/provider_resolve._resolve_api_key`` prefers it for every provider and -# passes the result to litellm as ``api_key``, which beats the provider's own -# key variable. +# ``llm/provider_resolve._resolve_api_key`` prefers it for a provider the +# operator named (``--llm-provider``, ``FLUID_LLM_PROVIDER``) and passes the +# result to litellm as ``api_key``, which beats the provider's own key +# variable. A provider forge picks itself never gets it, and forge can pick +# openai, anthropic and gemini (the one keyring key or key variable), github +# (interactive setup reads PROVIDER_ENV_VARS) and ollama. A project +# FLUID_LLM_API_KEY is a decoy for those (the operator's keyring key goes out), +# so it pairs only the endpoints of providers forge reaches by being named. _LLM_KEY = "FLUID_LLM_API_KEY" # ``_resolve_api_key`` then reads the provider's PROVIDER_ENV_VARS name (only # openai, anthropic, gemini and github have one) and falls back to the key # ``fluid ai setup`` saved in the keyring, which also beats a provider key # variable that litellm would read. _LLM_KEYRING = ( - "the key comes from FLUID_LLM_API_KEY, the provider's PROVIDER_ENV_VARS name, then the " - "keyring (llm/provider_resolve.py)" + "the key comes from FLUID_LLM_API_KEY when the provider is named, the provider's " + "PROVIDER_ENV_VARS name, then the keyring (llm/provider_resolve.py)" ) _GOOGLE_CREDENTIALS: Tuple[str, ...] = ( "GOOGLE_APPLICATION_CREDENTIALS", @@ -929,7 +935,8 @@ class EndpointGroup: "project OPENAI_BASE_URL from the codex environment unless the project " "supplies CODEX_API_KEY too" ), - paired_by=(("OPENAI_API_KEY", _LLM_KEY),), + # not _LLM_KEY: forge can pick openai without it being named + paired_by=(("OPENAI_API_KEY",),), ), EndpointGroup( service="Anthropic", @@ -949,7 +956,8 @@ class EndpointGroup: "stored subscription login to ANTHROPIC_BASE_URL; only ANTHROPIC_API_KEY " "comes before the ambient credential on both paths" ), - paired_by=(("ANTHROPIC_API_KEY", _LLM_KEY), ("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN")), + # not _LLM_KEY: forge can pick anthropic without it being named + paired_by=(("ANTHROPIC_API_KEY",), ("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN")), ), EndpointGroup( service="Azure OpenAI", @@ -980,7 +988,8 @@ class EndpointGroup: "GEMINI_API_KEY); google-genai (GOOGLE_GEMINI_BASE_URL); " f"{_LLM_KEYRING}, and for gemini also GEMINI_API_KEY and GOOGLE_API_KEY" ), - paired_by=((_LLM_KEY, "GOOGLE_API_KEY", "GEMINI_API_KEY"),), + # not _LLM_KEY: forge can pick gemini without it being named + paired_by=(("GOOGLE_API_KEY", "GEMINI_API_KEY"),), ), EndpointGroup( service="Groq", @@ -1018,7 +1027,8 @@ class EndpointGroup: "litellm's github/ route (GITHUB_API_BASE; the key is GITHUB_API_KEY, " f"llm/providers.py PROVIDER_ENV_VARS); {_LLM_KEYRING}" ), - paired_by=((_LLM_KEY, "GITHUB_API_KEY"),), + # not _LLM_KEY: interactive setup can pick github without it being named + paired_by=(("GITHUB_API_KEY",),), ), EndpointGroup( service="Amazon Bedrock", @@ -1460,7 +1470,9 @@ class EndpointGroup: #: account ids its provider code chooses a host from (DATAROBOT_ENDPOINT, #: SNOWFLAKE_ACCOUNT_ID, GIGACHAT_AUTH_URL ...); FLUID routes any provider name #: litellm knows (``llm/providers.py`` falls back to litellm's catch-all), -#: handing it the key ``_resolve_api_key`` found. Only the names litellm reads +#: handing it the key ``_resolve_api_key`` found. Forge never picks one of these +#: itself: it reaches them only by being named, so FLUID_LLM_API_KEY comes first +#: and pairs them. Only the names litellm reads #: (``litellm_endpoints.py``): any other ``_API_BASE`` is the project's own #: and follows the name-shape rule. UNLISTED_LLM_ENDPOINTS = EndpointGroup( diff --git a/fluid_build/llm/provider_resolve.py b/fluid_build/llm/provider_resolve.py index a7ce42de..7cddf340 100644 --- a/fluid_build/llm/provider_resolve.py +++ b/fluid_build/llm/provider_resolve.py @@ -11,7 +11,8 @@ :func:`_infer_provider_from_env` / :func:`_infer_provider_from_keyring` — pick a provider hint from whatever signal we can find. -* :func:`_resolve_api_key` — env var first, keyring fallback. +* :func:`_resolve_api_key` — env var first, keyring fallback; + ``FLUID_LLM_API_KEY`` only for a provider the operator named. * :func:`save_api_key_to_keyring` / :func:`clear_api_key_from_keyring` / :func:`_get_api_key_from_keyring` / :func:`_keyring_key` — keyring round-trips for cloud-provider secrets. @@ -371,8 +372,19 @@ def _infer_provider_from_keyring() -> Optional[str]: return None -def _resolve_api_key(provider: str, env: Mapping[str, str]) -> Optional[str]: - if env.get("FLUID_LLM_API_KEY"): +def _resolve_api_key( + provider: str, env: Mapping[str, str], *, named: bool = False +) -> Optional[str]: + """The API key for *provider*: ``FLUID_LLM_API_KEY``, its own variable, the keyring. + + ``FLUID_LLM_API_KEY`` does not say which provider the key is for, so it is + read only when *named* is true: the operator named *provider* for this run + (``--llm-provider``, ``FLUID_LLM_PROVIDER``). A provider that forge found + itself (a running Ollama, ``OLLAMA_HOST``, the one keyring key, the one + provider key variable, interactive setup's pick) gets its own key or none, + never a key that may belong to another vendor. + """ + if named and env.get("FLUID_LLM_API_KEY"): return env["FLUID_LLM_API_KEY"] env_var = PROVIDER_ENV_VARS.get(provider) if env_var: diff --git a/fluid_build/llm/providers.py b/fluid_build/llm/providers.py index 13f982bb..7c5f0f20 100644 --- a/fluid_build/llm/providers.py +++ b/fluid_build/llm/providers.py @@ -32,6 +32,7 @@ "reset_llm_caches", "agent_loop_requested", "preflight_llm_provider", + "remember_named_llm_provider", "resolve_llm_config", "resolve_model_name", "resolve_ollama_model", @@ -758,23 +759,35 @@ def live_provider_models( *, api_key: Optional[str] = None, environ: Optional[Mapping[str, str]] = None, + named: bool = False, ) -> Optional[List[str]]: - """Fetch live provider model ids when supported by the adapter.""" + """Fetch live provider model ids when supported by the adapter. + + ``named``: the operator named this provider, so ``FLUID_LLM_API_KEY`` may + be its key (see ``provider_resolve._resolve_api_key``). + """ env = dict(environ or os.environ) provider = get_llm_provider(provider_name) - return provider.list_available_models(api_key or _resolve_api_key(provider.name, env), env) + return provider.list_available_models( + api_key or _resolve_api_key(provider.name, env, named=named), env + ) def has_llm_api_key( provider_name: str, environ: Optional[Mapping[str, str]] = None, + *, + named: bool = False, ) -> bool: - """Return whether the provider can resolve credentials without exposing them.""" + """Return whether the provider can resolve credentials without exposing them. + + ``named`` as for :func:`live_provider_models`. + """ env = dict(environ or os.environ) provider = get_llm_provider(provider_name) if provider.name == "ollama": return detect_ollama_available(env) - return bool(_resolve_api_key(provider.name, env)) + return bool(_resolve_api_key(provider.name, env, named=named)) # Providers that never require an LLM API key — the call is paid for @@ -865,6 +878,84 @@ def _explicit_provider_name(args: Any, env: Mapping[str, str]) -> Optional[str]: ) +# Interactive forge puts the provider its inline setup found into +# ``args.llm_provider``, where it would read as named. Before that, it records +# here what the operator named (``None`` for nothing), so ``FLUID_LLM_API_KEY`` +# still goes only to a provider the operator named. +_NAMED_PROVIDER_ATTR = "llm_provider_named" +_NOT_RECORDED = object() + + +def _recorded_named_provider(args: Any) -> Any: + """What :func:`remember_named_llm_provider` recorded on *args*, else ``_NOT_RECORDED``. + + Only a string or ``None`` is a record; anything else (no attribute, a + test double's auto-attribute) is not. + """ + value = getattr(args, _NAMED_PROVIDER_ATTR, _NOT_RECORDED) + return value if value is None or isinstance(value, str) else _NOT_RECORDED + + +def remember_named_llm_provider(args: Any, environ: Optional[Mapping[str, str]] = None) -> None: + """Record which provider the operator named, before ``args.llm_provider`` is replaced. + + Call it before putting a provider that setup found into ``args.llm_provider``. + A record already made is kept, so a second setup cannot turn its own pick + into a named provider. + """ + if _recorded_named_provider(args) is not _NOT_RECORDED: + return + env = os.environ if environ is None else environ + setattr(args, _NAMED_PROVIDER_ATTR, _explicit_provider_name(args, env)) + + +def _unnamed_provider_note( + args: Any, env: Mapping[str, str], provider_name: str, named: bool +) -> Optional[str]: + """Why ``FLUID_LLM_API_KEY`` is not used for *provider_name*, else ``None``. + + ``FLUID_LLM_API_KEY`` does not say which vendor it is for, so a provider + forge picked itself never gets it (``provider_resolve._resolve_api_key``). + The note is logged, so it is built from provider names alone and never from + anything that holds or resolves a key (CodeQL + py/clear-text-logging-sensitive-data). + """ + if named or not env.get("FLUID_LLM_API_KEY"): + return None + recorded = _recorded_named_provider(args) + if isinstance(recorded, str) and recorded: # interactive setup replaced the choice + return ( + f"FLUID_LLM_API_KEY is set but not used: it is for {recorded}, the provider " + f"you named, and this run uses {provider_name}." + ) + return ( + "FLUID_LLM_API_KEY is set but not used: it goes only to a provider named with " + f"--llm-provider or FLUID_LLM_PROVIDER, and forge picked {provider_name} itself. " + f"Pass --llm-provider {provider_name} to use the key with it." + ) + + +def _same_llm_provider(first: str, second: str) -> bool: + """Whether two provider names select the same provider (``claude`` is ``anthropic``).""" + return normalize_llm_provider_name(first) == normalize_llm_provider_name(second) + + +def _provider_is_named(args: Any, env: Mapping[str, str], provider_name: str) -> bool: + """Whether *provider_name*, the provider this run uses, is one the operator named. + + Without a record, a named provider always wins inference, so the run uses + it whenever one is named. With one, ``args.llm_provider`` may be what setup + found, which counts only when it is the provider the operator named. + """ + recorded = _recorded_named_provider(args) + explicit = _explicit_provider_name(args, env) + if recorded is _NOT_RECORDED: + return bool(explicit) + if not recorded: + return False + return recorded == explicit or _same_llm_provider(recorded, provider_name) + + _AGENT_LOOP_ALTERNATIVES = ( "Or author without it: --template starter. The agent loop needs a provider that " "supports tool use, which the keyless providers (claude-code, codex, cursor, kiro, " @@ -968,7 +1059,9 @@ def resolve_llm_config(args: Any, environ: Optional[Mapping[str, str]] = None) - ], ) - api_key = _resolve_api_key(provider.name, env) + named = _provider_is_named(args, env, provider.name) + api_key = _resolve_api_key(provider.name, env, named=named) + unnamed_note = _unnamed_provider_note(args, env, provider.name, named) # A third-party provider plugin may declare ``keyless = True`` (class attr) # when it authenticates by some other means (e.g. an ambient cloud identity # or its own env var validated at call time) — same escape the built-in @@ -981,8 +1074,9 @@ def resolve_llm_config(args: Any, environ: Optional[Mapping[str, str]] = None) - "copilot_missing_llm_api_key", f"No API key was configured for the {provider.name} copilot adapter.", suggestions=[ + *([unnamed_note] if unnamed_note else []), "Run 'fluid ai setup' to configure a provider interactively", - "Set FLUID_LLM_API_KEY or the provider-specific API key environment variable", + "Set the provider's own key variable, or FLUID_LLM_API_KEY with --llm-provider", "Examples: OPENAI_API_KEY, ANTHROPIC_API_KEY, GEMINI_API_KEY, GOOGLE_API_KEY", "For local models, use --llm-provider ollama and optionally --llm-endpoint", "For keyless authoring: run forge from your IDE (mcp-sampling), or use a " @@ -991,6 +1085,9 @@ def resolve_llm_config(args: Any, environ: Optional[Mapping[str, str]] = None) - ) model_notes: List[str] = [] + if unnamed_note: + model_notes.append(unnamed_note) + LOG.warning(unnamed_note) if ( not _explicit_provider_name(args, env) and not saved @@ -1643,7 +1740,10 @@ def check_llm_readiness(environ: Optional[Mapping[str, str]] = None) -> LlmReadi saved = _saved_choice_for(None, env, provider.name) # Resolve API key: env vars → config file - api_key = _resolve_api_key(provider.name, env) or saved.get("api_key") + # FLUID_LLM_API_KEY counts only for a provider named with FLUID_LLM_PROVIDER or + # FLUID_FORGE_AGENT, never for one the ladder picked, the saved choice included. + named = _provider_is_named(None, env, provider.name) + api_key = _resolve_api_key(provider.name, env, named=named) or saved.get("api_key") from fluid_build.llm.coding_agent import _AGENT_SPECS agent = _AGENT_SPECS.get(provider.name) diff --git a/tests/copilot/agents/test_judge_live.py b/tests/copilot/agents/test_judge_live.py index 93779ff6..298fb2b8 100644 --- a/tests/copilot/agents/test_judge_live.py +++ b/tests/copilot/agents/test_judge_live.py @@ -43,10 +43,7 @@ def _resolve_live_config_or_skip(): """Resolve real LLM config; skip cleanly if no provider configured.""" - from fluid_build.cli.forge_copilot_llm_providers import ( - has_llm_api_key, - resolve_llm_config, - ) + from fluid_build.cli.forge_copilot_llm_providers import resolve_llm_config class _Args: pass @@ -62,7 +59,7 @@ class _Args: if not detect_ollama_available(None): pytest.skip("provider resolved as ollama but no local server detected") - elif not has_llm_api_key(config.provider): + elif not config.api_key: # the key the judge sends (FLUID_LLM_API_KEY only if named) pytest.skip( f"no API key resolved for provider {config.provider!r}; " f"set FLUID_LLM_PROVIDER/FLUID_LLM_MODEL + the relevant API key env var to run" diff --git a/tests/test_dotenv_pairing_matrix.py b/tests/test_dotenv_pairing_matrix.py index af47f9c0..80272caf 100644 --- a/tests/test_dotenv_pairing_matrix.py +++ b/tests/test_dotenv_pairing_matrix.py @@ -141,10 +141,22 @@ def _skip_where_litellm_cannot_reply() -> None: ) +# The providers forge picks without a name: the one keyring key or key variable +# (openai, anthropic, gemini), interactive setup's scan of PROVIDER_ENV_VARS +# (github too), a running Ollama. Such a run never gets FLUID_LLM_API_KEY. Forge +# reaches every other provider only by being named. +_PICKED_WITHOUT_A_NAME = frozenset({"openai", "anthropic", "gemini", "github", "ollama"}) + + def _llm_call(provider: str, endpoint_var: str) -> None: """What ``fluid forge`` does with FLUID_LLM_BACKEND=litellm: resolve the key, hand it to litellm, and let litellm read the endpoint from the environment. + The key is resolved as for a named provider and, for one forge can pick + itself (:data:`_PICKED_WITHOUT_A_NAME`), as for a picked one too, since a + pairing must hold however the run chose its provider. Each distinct key is + sent once, so where both ways find the same key the listener sees one call. + litellm is called only for a loopback endpoint; a test that gets that far is skipped where litellm cannot build its reply (:func:`_skip_where_litellm_cannot_reply`). ``pytest.skip`` raises a @@ -154,19 +166,25 @@ def _llm_call(provider: str, endpoint_var: str) -> None: from fluid_build.llm.litellm_backend import _litellm_model_for from fluid_build.llm.provider_resolve import _resolve_api_key - key = _resolve_api_key(provider, os.environ) + keys: List[Optional[str]] = [] + for named in (True, False) if provider in _PICKED_WITHOUT_A_NAME else (True,): + key = _resolve_api_key(provider, os.environ, named=named) + if key not in keys: + keys.append(key) - def complete() -> Any: - _skip_where_litellm_cannot_reply() - return litellm.completion( - model=_litellm_model_for(provider, "test-model"), - messages=[{"role": "user", "content": "ping"}], - api_key=key, - num_retries=0, - timeout=5, - ) + for key in keys: - send_if_loopback(os.environ.get(endpoint_var), complete) + def complete(key: Optional[str] = key) -> Any: + _skip_where_litellm_cannot_reply() + return litellm.completion( + model=_litellm_model_for(provider, "test-model"), + messages=[{"role": "user", "content": "ping"}], + api_key=key, + num_retries=0, + timeout=5, + ) + + send_if_loopback(os.environ.get(endpoint_var), complete) def _openai_sdk_call() -> None: @@ -576,7 +594,7 @@ def test_a_decoy_credential_does_not_pair_an_llm_endpoint_with_the_keyring_key( "provider,endpoint,credential", [ ("openai", "OPENAI_BASE_URL", "OPENAI_API_KEY"), - ("openai", "OPENAI_API_BASE", "FLUID_LLM_API_KEY"), + ("openai", "OPENAI_API_BASE", "OPENAI_API_KEY"), ("anthropic", "ANTHROPIC_BASE_URL", "ANTHROPIC_API_KEY"), ("gemini", "GEMINI_API_BASE", "GEMINI_API_KEY"), ("gemini", "GEMINI_API_BASE", "GOOGLE_API_KEY"), @@ -606,6 +624,36 @@ def test_the_credential_the_reader_sends_pairs_its_endpoint( assert listener.saw(PROJECT_KEY) != [] +@pytest.mark.parametrize( + "provider,endpoint", + [ + ("openai", "OPENAI_API_BASE"), + ("openai", "OPENAI_BASE_URL"), + ("anthropic", "ANTHROPIC_BASE_URL"), + ("gemini", "GEMINI_API_BASE"), + ("github", "GITHUB_API_BASE"), + ], +) +def test_a_project_fluid_llm_api_key_does_not_pair_a_provider_forge_can_pick( + project: Path, + monkeypatch: pytest.MonkeyPatch, + listener: Listener, + provider: str, + endpoint: str, +) -> None: + # The keyring holds one key, so forge picks its provider without a name, and + # a provider forge picked never gets FLUID_LLM_API_KEY: the keyring key is + # what goes out. The project's FLUID_LLM_API_KEY is a decoy and pairs nothing. + _keyring_holds(monkeypatch, provider) + (project / ".env").write_text(f"{endpoint}={listener.url}\nFLUID_LLM_API_KEY={PROJECT_KEY}\n") + + hydrate_dotenv(project) + _llm_call(provider, endpoint) + + assert endpoint not in os.environ + assert listener.saw(OPERATOR_KEY) == [] + + @pytest.mark.parametrize( "provider,endpoint", [("groq", "GROQ_API_BASE"), ("openai", "OPENAI_BASE_URL")] ) @@ -630,7 +678,8 @@ def test_the_harness_sees_a_key_sent_to_an_operator_endpoint( def test_an_exported_fluid_llm_api_key_refuses_a_project_openai_endpoint( project: Path, monkeypatch: pytest.MonkeyPatch, listener: Listener ) -> None: - # _resolve_api_key prefers FLUID_LLM_API_KEY to the project's OPENAI_API_KEY. + # For a named openai, _resolve_api_key prefers FLUID_LLM_API_KEY to the + # project's OPENAI_API_KEY. monkeypatch.setenv("FLUID_LLM_API_KEY", OPERATOR_KEY) (project / ".env").write_text(f"OPENAI_BASE_URL={listener.url}\nOPENAI_API_KEY={DECOY_KEY}\n") @@ -641,6 +690,124 @@ def test_an_exported_fluid_llm_api_key_refuses_a_project_openai_endpoint( assert "OPENAI_BASE_URL" not in os.environ +# ── interactive setup puts no stored key where a project endpoint reads ───── + + +@pytest.mark.parametrize( + "provider,endpoint,credential", + [ + ("openai", "OPENAI_BASE_URL", "OPENAI_API_KEY"), + ("anthropic", "ANTHROPIC_BASE_URL", "ANTHROPIC_API_KEY"), + # setup writes GOOGLE_API_KEY, which forge reads before GEMINI_API_KEY + ("gemini", "GEMINI_API_BASE", "GEMINI_API_KEY"), + ], +) +def test_interactive_setup_never_sends_the_saved_key_to_a_project_endpoint( + project: Path, + monkeypatch: pytest.MonkeyPatch, + listener: Listener, + caplog: pytest.LogCaptureFixture, + provider: str, + endpoint: str, + credential: str, +) -> None: + # The project pairs its own endpoint with its own key; ``fluid ai setup`` + # saved this provider with a keyring key. Interactive forge's setup copied + # the saved key into the provider's variable (set_session_env), over the + # project's, so the saved key went to the project's host. + import fluid_build.cli.ai_setup as ai_setup + import fluid_build.llm.providers as llm_providers + + (project / ".env").write_text(f"{endpoint}={listener.url}\n{credential}={PROJECT_KEY}\n") + hydrate_dotenv(project) + assert os.environ[endpoint] == listener.url # the pairing holds + _keyring_holds(monkeypatch, provider) + monkeypatch.setattr(ai_setup, "_ai_setup_skipped", False) + monkeypatch.setattr(ai_setup, "_load_ai_config", lambda: {"provider": provider, "model": "m"}) + monkeypatch.setattr( + ai_setup, "_load_key_from_keyring", lambda name: OPERATOR_KEY if name == provider else None + ) + + # What interactive ``fluid forge`` does (cli/forge.py): setup, then its key. + with caplog.at_level(logging.WARNING): + config = ai_setup.run_ai_setup_inline(None) + args = types.SimpleNamespace(llm_provider=None, llm_model=None, llm_endpoint=None) + llm_providers.remember_named_llm_provider(args, os.environ) + args.llm_provider = config.provider + if config.api_key: + ai_setup.set_session_env(config.provider, config.api_key) + _llm_call(provider, endpoint) + + assert listener.saw(OPERATOR_KEY) == [] + assert listener.saw(PROJECT_KEY) != [] + # The warning names the endpoint, never the provider string setup worked + # with: that can come from classifying a typed key (CodeQL + # py/clear-text-logging-sensitive-data). + warned = [r.getMessage() for r in caplog.records if "Your key is not used" in r.getMessage()] + assert warned and all(endpoint in message for message in warned) + + +@pytest.mark.parametrize( + "provider,endpoint,credential", + [ + ("openai", "OPENAI_BASE_URL", "OPENAI_API_KEY"), + # no group is mapped to groq, so every LLM endpoint the project set counts + ("groq", "GROQ_API_BASE", "FLUID_LLM_API_KEY"), + ], +) +def test_setup_checks_no_key_against_a_project_endpoint( + project: Path, listener: Listener, provider: str, endpoint: str, credential: str +) -> None: + # ``fluid ai setup`` checks a typed or imported key with one litellm call, + # and litellm reads the endpoint from the environment. + import fluid_build.cli.ai_setup as ai_setup + from fluid_build.cli.forge_copilot_llm_litellm import get_litellm_provider + + _skip_where_litellm_cannot_reply() + (project / ".env").write_text(f"{endpoint}={listener.url}\n{credential}={PROJECT_KEY}\n") + hydrate_dotenv(project) + assert os.environ[endpoint] == listener.url # the pairing holds + + error = ai_setup._validate_api_key(get_litellm_provider(provider), OPERATOR_KEY) + + assert listener.saw(OPERATOR_KEY) == [] + assert error is not None and error.startswith("not checked:") and endpoint in error + + +def test_setup_asks_for_no_key_it_could_not_check( + project: Path, monkeypatch: pytest.MonkeyPatch, listener: Listener +) -> None: + # Asking three times for a key that can be neither checked nor used, with + # "2 attempt(s) left", helps nobody: setup says why once and stops. + import fluid_build.cli.ai_setup as ai_setup + + (project / ".env").write_text(f"OPENAI_BASE_URL={listener.url}\nOPENAI_API_KEY={PROJECT_KEY}\n") + hydrate_dotenv(project) + asked: List[Any] = [] + monkeypatch.setattr(ai_setup.Prompt, "ask", lambda *a, **k: asked.append(a) or OPERATOR_KEY) + printed: List[str] = [] + console = types.SimpleNamespace(print=lambda *a, **k: printed.append(str(a[0]) if a else "")) + + result = ai_setup._collect_and_validate_api_key( + console, provider_choice="openai", tier="balanced" + ) + + assert result is None + assert asked == [] + assert listener.requests == [] + assert any("Not asking for a" in line and "OPENAI_BASE_URL" in line for line in printed) + + +def test_setup_still_sets_the_key_where_no_project_endpoint_reads(project: Path) -> None: + import fluid_build.cli.ai_setup as ai_setup + + (project / ".env").write_text(f"OPENAI_API_KEY={PROJECT_KEY}\n") # a key, no endpoint + hydrate_dotenv(project) + ai_setup.set_session_env("openai", OPERATOR_KEY) + + assert os.environ["OPENAI_API_KEY"] == OPERATOR_KEY + + _FAKE_CODEX = """#!{python} # codex exec, as far as credentials go: OPENAI_BASE_URL is where it sends, # CODEX_API_KEY is the key (blank counts as unset), else the login stored in @@ -678,10 +845,10 @@ def _install_fake_codex(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None def _run_codex() -> None: + import fluid_build.llm.providers as llm_providers from fluid_build.llm.coding_agent import get_coding_agent_provider - from fluid_build.llm.providers import LlmConfig - config = LlmConfig(provider="codex", model="codex", endpoint="", api_key=None) + config = llm_providers.LlmConfig(provider="codex", model="codex", endpoint="", api_key=None) try: get_coding_agent_provider("codex").invoke_blocking(config, "system", "user") except Exception: # noqa: BLE001 — the listener answers 401 diff --git a/tests/test_dotenv_snowflake_project_account.py b/tests/test_dotenv_snowflake_project_account.py index b20ec474..d4a3ee16 100644 --- a/tests/test_dotenv_snowflake_project_account.py +++ b/tests/test_dotenv_snowflake_project_account.py @@ -337,11 +337,14 @@ def test_a_blank_operator_credential_pairs_only_an_account_behind_a_blank_variab def test_a_blank_operator_llm_key_still_unpairs_an_llm_endpoint() -> None: # FLUID reads a blank FLUID_LLM_API_KEY as unset and takes the keyring key. - project = {"OPENAI_API_BASE": "https://llm.project.test", "FLUID_LLM_API_KEY": PROJECT_KEY} + # (DEEPSEEK_API_BASE: forge reaches deepseek only by name, so FLUID_LLM_API_KEY + # pairs it at all.) + project = {"DEEPSEEK_API_BASE": "https://llm.project.test", "FLUID_LLM_API_KEY": PROJECT_KEY} - refused = unpaired_endpoints(project, {"OPENAI_API_BASE": "", "FLUID_LLM_API_KEY": ""}) + refused = unpaired_endpoints(project, {"DEEPSEEK_API_BASE": "", "FLUID_LLM_API_KEY": ""}) - assert "OPENAI_API_BASE" in refused + assert unpaired_endpoints(project, {}) == {} + assert "DEEPSEEK_API_BASE" in refused def test_a_dropped_account_is_reported_while_the_shell_s_variable_is_blank( diff --git a/tests/test_forge_copilot_runtime.py b/tests/test_forge_copilot_runtime.py index 8bd92bc9..7021de54 100644 --- a/tests/test_forge_copilot_runtime.py +++ b/tests/test_forge_copilot_runtime.py @@ -1177,8 +1177,14 @@ def test_provider_specific_env_keys(self): assert has_llm_api_key("gemini", {"GEMINI_API_KEY": "AIza-test"}) is True assert has_llm_api_key("gemini", {"GOOGLE_API_KEY": "AIza-test"}) is True - def test_generic_fluid_key_counts_for_hosted_provider(self): - assert has_llm_api_key("openai", {"FLUID_LLM_API_KEY": "sk-test"}) is True + def test_generic_fluid_key_counts_only_for_a_named_provider(self, monkeypatch): + monkeypatch.setattr( + "fluid_build.cli.forge_copilot_llm_providers._get_api_key_from_keyring", + lambda provider: None, + ) + assert has_llm_api_key("openai", {"FLUID_LLM_API_KEY": "sk-test"}, named=True) is True + # FLUID_LLM_API_KEY names no vendor, so it is no key for a provider forge picked. + assert has_llm_api_key("openai", {"FLUID_LLM_API_KEY": "sk-test"}) is False def test_missing_hosted_key_is_false(self, monkeypatch): monkeypatch.setattr( @@ -1207,13 +1213,16 @@ def test_env_var_takes_priority_over_keyring(self): result = _resolve_api_key("openai", {"OPENAI_API_KEY": "env-secret"}) assert result == "env-secret" - def test_fluid_llm_api_key_takes_priority_over_keyring(self): + def test_fluid_llm_api_key_takes_priority_over_keyring_for_a_named_provider(self): with patch( "fluid_build.cli.forge_copilot_llm_providers._get_api_key_from_keyring", return_value="keyring-secret", ): - result = _resolve_api_key("openai", {"FLUID_LLM_API_KEY": "generic-key"}) - assert result == "generic-key" + named = _resolve_api_key("openai", {"FLUID_LLM_API_KEY": "generic-key"}, named=True) + picked = _resolve_api_key("openai", {"FLUID_LLM_API_KEY": "generic-key"}) + assert named == "generic-key" + # A provider forge picked itself gets its own key, never the generic one. + assert picked == "keyring-secret" def test_keyring_returns_none_when_not_available(self): with patch( diff --git a/tests/test_llm_api_key_scope.py b/tests/test_llm_api_key_scope.py new file mode 100644 index 00000000..30dd6a77 --- /dev/null +++ b/tests/test_llm_api_key_scope.py @@ -0,0 +1,433 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""``FLUID_LLM_API_KEY`` goes only to a provider the operator named. + +The key does not say which vendor it is for. Forge used to send it to any +provider it picked, so an OpenAI key went to a running Ollama, to the host +``OLLAMA_HOST`` names, or to Anthropic when the keyring held one Anthropic key. +Named means ``--llm-provider`` or ``FLUID_LLM_PROVIDER`` names the provider the +run uses; every other way forge picks one leaves the key unused. + +Hermetic: the keyring, the Ollama probe, the saved ``fluid ai setup`` config and +``HOME`` are patched, so nothing reads the real ``~/.fluid`` or dials a daemon +(the one listener is a loopback stand-in). +""" + +from __future__ import annotations + +import logging +import os +import sys +from argparse import Namespace +from pathlib import Path +from types import SimpleNamespace +from typing import Dict, Iterator, Optional +from unittest import mock + +import pytest + +import fluid_build.llm.providers as providers +from tests._dotenv_pairing_support import Listener, refuse_outbound_dns, send_if_loopback + +GENERIC_KEY = "generic-llm-key-test-0001" # pragma: allowlist secret +ANTHROPIC_KEY = "anthropic-keyring-key-test-0002" # pragma: allowlist secret +OPENAI_KEY = "openai-env-key-test-0003" # pragma: allowlist secret + + +class _World: + """What the machine holds: keyring keys, a running Ollama, a saved provider.""" + + def __init__(self, bin_dir: Path) -> None: + self.keyring: Dict[str, str] = {} + self.ollama_up = False + self.saved: Optional[dict] = None + self.bin_dir = bin_dir + + def env(self, **values: str) -> Dict[str, str]: + """An environment with only *values*; PATH finds no coding-agent CLI.""" + return {"PATH": str(self.bin_dir), **values} + + +@pytest.fixture +def world(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Iterator[_World]: + import fluid_build._ai_config_shared as ai_config + + home = tmp_path / "home" + home.mkdir() + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("PYTHON_KEYRING_BACKEND", "keyring.backends.null.Keyring") + w = _World(bin_dir) + monkeypatch.setattr(providers, "_get_api_key_from_keyring", lambda name: w.keyring.get(name)) + monkeypatch.setattr(providers, "detect_ollama_available", lambda env: w.ollama_up) + monkeypatch.setattr(providers, "resolve_ollama_model", lambda env, endpoint=None: "llama3.2") + monkeypatch.setattr(ai_config, "load_ai_config", lambda config_file=None: w.saved) + monkeypatch.setattr( + ai_config, + "load_ai_config_for", + lambda name, config_file=None: w.saved if (w.saved or {}).get("provider") == name else None, + ) + providers.reset_llm_caches() + yield w + providers.reset_llm_caches() + + +def _args(**values: object) -> SimpleNamespace: + base = {"llm_provider": None, "llm_model": None, "llm_endpoint": None} + return SimpleNamespace(**{**base, **values}) + + +# ── a provider forge picks itself never gets the key ─────────────────────── + + +def test_a_running_ollama_never_gets_the_generic_key(world: _World) -> None: + world.ollama_up = True + + config = providers.resolve_llm_config(_args(), world.env(FLUID_LLM_API_KEY=GENERIC_KEY)) + + assert config.provider == "ollama" + assert config.api_key is None + assert any("FLUID_LLM_API_KEY is set but not used" in n for n in config.model_resolution_notes) + + +def test_the_host_ollama_host_names_never_gets_the_generic_key(world: _World) -> None: + # OLLAMA_HOST alone picks ollama; it does not name it. The key would go out as + # "Authorization: Bearer", here in cleartext to another machine. + remote = "http://gpu-box.example.test:11434" + env = world.env(FLUID_LLM_API_KEY=GENERIC_KEY, OLLAMA_HOST=remote) + + config = providers.resolve_llm_config(_args(), env) + with mock.patch.dict(os.environ, {"OLLAMA_HOST": remote}): + _headers, payload = providers.get_llm_provider("ollama").build_request(config, "s", "u") + + assert config.provider == "ollama" + assert config.api_key is None + assert "api_key" not in payload + assert payload["api_base"] == remote + + +def test_the_one_keyring_key_s_provider_gets_that_key(world: _World) -> None: + world.keyring["anthropic"] = ANTHROPIC_KEY + + config = providers.resolve_llm_config(_args(), world.env(FLUID_LLM_API_KEY=GENERIC_KEY)) + + assert config.provider == "anthropic" + assert config.api_key == ANTHROPIC_KEY + + +def test_the_one_key_variable_s_provider_gets_that_key(world: _World) -> None: + env = world.env(FLUID_LLM_API_KEY=GENERIC_KEY, OPENAI_API_KEY=OPENAI_KEY) + + config = providers.resolve_llm_config(_args(), env) + + assert config.provider == "openai" + assert config.api_key == OPENAI_KEY + + +def test_the_saved_choice_s_provider_gets_its_own_key(world: _World) -> None: + # A run without a TTY uses the provider ``fluid ai setup`` saved. That + # provider is picked, not named, so it gets its keyring key, never the + # generic one. Two keyring keys, so the keyring rung picks nothing itself. + world.saved = {"provider": "anthropic", "model": "claude-sonnet-4-5"} + world.keyring.update(anthropic=ANTHROPIC_KEY, openai=OPENAI_KEY) + env = world.env(FLUID_LLM_API_KEY=GENERIC_KEY) + + config = providers.resolve_llm_config(_args(non_interactive=True), env) + readiness = providers.check_llm_readiness(env) + + assert config.provider == "anthropic" + assert config.api_key == ANTHROPIC_KEY + assert any("FLUID_LLM_API_KEY is set but not used" in n for n in config.model_resolution_notes) + assert readiness.provider == "anthropic" + + +def test_the_saved_choice_without_a_key_of_its_own_is_not_ready( + world: _World, monkeypatch: pytest.MonkeyPatch +) -> None: + # The generic key does not make the saved provider ready either. + monkeypatch.setattr("fluid_build._optional.is_available", lambda name: True) + world.saved = {"provider": "anthropic", "model": "claude-sonnet-4-5"} + env = world.env(FLUID_LLM_API_KEY=GENERIC_KEY) + + with pytest.raises(providers.CopilotGenerationError) as info: + providers.resolve_llm_config(_args(non_interactive=True), env) + readiness = providers.check_llm_readiness(env) + + assert info.value.event == "copilot_missing_llm_api_key" + assert readiness.provider == "anthropic" and readiness.ready is False + + +def test_the_key_never_reaches_an_ollama_on_the_wire( + world: _World, monkeypatch: pytest.MonkeyPatch +) -> None: + # The reviewer's capture: litellm's ollama/ route sends the api_key it is + # handed as "Authorization: Bearer". A loopback listener stands in for Ollama. + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") # no price-map fetch + refuse_outbound_dns(monkeypatch) + litellm = pytest.importorskip("litellm") + listener = Listener() + try: + monkeypatch.setenv("OLLAMA_HOST", listener.url) + env = world.env(FLUID_LLM_API_KEY=GENERIC_KEY, OLLAMA_HOST=listener.url) + config = providers.resolve_llm_config(_args(), env) + _headers, payload = providers.get_llm_provider("ollama").build_request(config, "s", "u") + send_if_loopback( + payload.get("api_base"), + lambda: litellm.completion(**{**payload, "num_retries": 0, "timeout": 5}), + ) + + if not listener.requests: + pytest.skip("litellm sent no request to the listener, so it proves nothing") + assert listener.saw(GENERIC_KEY) == [] + finally: + listener.close() + + +# ── a named provider still gets it ───────────────────────────────────────── + + +@pytest.mark.parametrize("how", ["flag", "env"]) +@pytest.mark.parametrize("provider", ["ollama", "anthropic"]) +def test_a_named_provider_gets_the_generic_key_first( + world: _World, how: str, provider: str +) -> None: + # A named Ollama may be ollama.com or a token proxy; a named vendor's own + # variable still comes second. + env = world.env(FLUID_LLM_API_KEY=GENERIC_KEY, ANTHROPIC_API_KEY=ANTHROPIC_KEY) + if how == "env": + env["FLUID_LLM_PROVIDER"] = provider + args = _args() + else: + args = _args(llm_provider=provider) + + config = providers.resolve_llm_config(args, env) + + assert config.provider == provider + assert config.api_key == GENERIC_KEY + assert not any("FLUID_LLM_API_KEY" in n for n in config.model_resolution_notes) + + +def test_a_provider_with_no_key_of_its_own_says_why_the_generic_one_was_not_used( + world: _World, +) -> None: + # Forge seldom picks a provider that then has no key of its own; patch the + # pick to reach that error, whose first suggestion is the way out. + with mock.patch.object(providers, "_provider_name_from", return_value="openai"): + with pytest.raises(providers.CopilotGenerationError) as info: + providers.resolve_llm_config(_args(), world.env(FLUID_LLM_API_KEY=GENERIC_KEY)) + + assert info.value.event == "copilot_missing_llm_api_key" + assert "Pass --llm-provider openai" in info.value.suggestions[0] + + +# ── interactive setup's pick is not a name ───────────────────────────────── + + +def test_interactive_setup_s_pick_does_not_count_as_named(world: _World) -> None: + + args = _args() + providers.remember_named_llm_provider(args, {}) + args.llm_provider = "openai" # what forge does with run_ai_setup_inline's config + env = world.env(FLUID_LLM_API_KEY=GENERIC_KEY, OPENAI_API_KEY=OPENAI_KEY) + + config = providers.resolve_llm_config(args, env) + + assert config.api_key == OPENAI_KEY + + +def test_setup_replacing_a_named_provider_does_not_carry_the_key_over(world: _World) -> None: + + args = _args(llm_provider="anthropic") + providers.remember_named_llm_provider(args, {}) + args.llm_provider = "openai" # the saved provider, loaded over the flag + env = world.env(FLUID_LLM_API_KEY=ANTHROPIC_KEY, OPENAI_API_KEY=OPENAI_KEY) + + config = providers.resolve_llm_config(args, env) + + assert config.api_key == OPENAI_KEY + notes = " ".join(config.model_resolution_notes) + assert "it is for anthropic, the provider you named" in notes + assert "Pass --llm-provider" not in notes + + +@pytest.mark.parametrize("named,picked", [("claude", "anthropic"), ("openai", "openai")]) +def test_setup_keeping_the_named_provider_keeps_the_key( + world: _World, named: str, picked: str +) -> None: + + args = _args(llm_provider=named) + providers.remember_named_llm_provider(args, {}) + args.llm_provider = picked + + config = providers.resolve_llm_config(args, world.env(FLUID_LLM_API_KEY=GENERIC_KEY)) + + assert config.api_key == GENERIC_KEY + + +def test_the_first_record_is_kept(world: _World) -> None: + + args = _args() + providers.remember_named_llm_provider(args, {}) + args.llm_provider = "openai" + providers.remember_named_llm_provider(args, {}) # a second setup must not name its own pick + + assert args.llm_provider_named is None + + +def test_the_copilot_options_carry_the_record(world: _World) -> None: + from fluid_build.cli.forge import get_cli_arg + from fluid_build.cli.forge_modes import _forge_build_copilot_options + + args = Namespace(llm_provider="openai", llm_provider_named=None) + options = _forge_build_copilot_options( + args, get_cli_arg_fn=get_cli_arg, is_non_interactive=False + ) + env = world.env(FLUID_LLM_API_KEY=GENERIC_KEY, OPENAI_API_KEY=OPENAI_KEY) + + config = providers.resolve_llm_config(SimpleNamespace(**options), env) + + assert options["llm_provider_named"] is None + assert config.api_key == OPENAI_KEY + # without a record nothing is added, so a flag still names the provider + plain = _forge_build_copilot_options( + Namespace(llm_provider="openai"), get_cli_arg_fn=get_cli_arg, is_non_interactive=False + ) + assert "llm_provider_named" not in plain + + +@pytest.fixture +def fake_tty(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(sys.stdin, "isatty", lambda: True) + + +@pytest.mark.parametrize("flag", [None, "anthropic"]) +def test_interactive_forge_records_what_was_named_before_setup_s_pick( + world: _World, + fake_tty: None, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + flag: Optional[str], +) -> None: + from fluid_build.cli.forge import run as forge_run + + monkeypatch.setattr("fluid_build.cli._welcome_scan.Path.home", lambda: tmp_path / "home") + monkeypatch.setenv("FLUID_FORGE_NO_PREVIEW", "1") + for name in ("FLUID_LLM_PROVIDER", "FLUID_FORGE_AGENT"): + monkeypatch.delenv(name, raising=False) + seen: Dict[str, Namespace] = {} + + def _copilot(args: Namespace, _logger: logging.Logger) -> int: + seen["args"] = args + return 0 + + def _setup(_console: object) -> providers.LlmConfig: + return providers.LlmConfig( + provider="openai", model="gpt-4o", endpoint="", api_key=OPENAI_KEY + ) + + args = Namespace( + command="forge", + forge_subcommand=None, + blank=False, + target_dir=str(tmp_path), + non_interactive=False, + dry_run=False, + llm_provider=flag, + llm_model=None, + llm_endpoint=None, + agent_loop=False, + refine=None, + from_product=[], + from_product_list=None, + template=None, + scaffold=None, + ) + with ( + mock.patch("fluid_build.cli._forge_mode_picker.pick_mode", lambda a, **k: "ai"), + mock.patch("fluid_build.cli._forge_mode_picker.should_show_picker", lambda a: True), + mock.patch("fluid_build.cli.forge.run_ai_copilot_mode", _copilot), + mock.patch("fluid_build.cli.ai_setup.run_ai_setup_inline", _setup), + mock.patch("fluid_build.cli.ai_setup.set_session_env", lambda provider, key: None), + ): + forge_run(args, logging.getLogger("test")) + + ran_with = seen["args"] + assert ran_with.llm_provider == "openai" + assert ran_with.llm_provider_named == flag + env = world.env(FLUID_LLM_API_KEY=GENERIC_KEY, OPENAI_API_KEY=OPENAI_KEY) + assert providers.resolve_llm_config(ran_with, env).api_key == OPENAI_KEY + + +# ── fluid doctor and fluid ai test ───────────────────────────────────────── + + +def test_readiness_counts_the_key_only_for_a_named_provider( + world: _World, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr("fluid_build._optional.is_available", lambda name: True) + world.saved = {"provider": "openai", "model": "gpt-4o"} + + picked = providers.check_llm_readiness(world.env(FLUID_LLM_API_KEY=GENERIC_KEY)) + named = providers.check_llm_readiness( + world.env(FLUID_LLM_API_KEY=GENERIC_KEY, FLUID_LLM_PROVIDER="openai") + ) + + assert picked.provider == "openai" + assert picked.ready is False + assert named.ready is True + + +def _ai_test_config(args: Namespace, env: Dict[str, str], saved: Optional[dict], keyring: str): + from fluid_build.cli.ai_setup import _resolve_ai_test_config + + with ( + mock.patch("fluid_build.cli.ai_setup._load_ai_config", return_value=saved), + mock.patch("fluid_build.cli.ai_setup._load_key_from_keyring", return_value=keyring), + mock.patch("fluid_build.cli.ai_setup.detect_ollama_available", return_value=False), + # keep the fixture's HOME: without one, ``~`` is the real home again + mock.patch.dict(os.environ, {**env, "HOME": os.environ["HOME"]}, clear=True), + ): + return _resolve_ai_test_config(args) + + +def _ai_test_args(provider: Optional[str]) -> Namespace: + return Namespace( + llm_provider=provider, llm_model=None, llm_endpoint=None, llm_timeout_seconds=None + ) + + +def test_ai_test_uses_the_saved_provider_s_own_key(world: _World) -> None: + config, error = _ai_test_config( + _ai_test_args(None), + {"FLUID_LLM_API_KEY": GENERIC_KEY}, + {"provider": "anthropic", "model": "claude-sonnet-4-5"}, + ANTHROPIC_KEY, + ) + + assert error is None + assert config.provider == "anthropic" + assert config.api_key == ANTHROPIC_KEY + + +def test_ai_test_uses_the_key_variable_s_own_key(world: _World) -> None: + config, error = _ai_test_config( + _ai_test_args(None), + {"FLUID_LLM_API_KEY": GENERIC_KEY, "OPENAI_API_KEY": OPENAI_KEY}, + None, + "", + ) + + assert error is None + assert config.provider == "openai" + assert config.api_key == OPENAI_KEY + + +def test_ai_test_with_llm_provider_uses_the_generic_key(world: _World) -> None: + config, error = _ai_test_config( + _ai_test_args("openai"), {"FLUID_LLM_API_KEY": GENERIC_KEY}, None, OPENAI_KEY + ) + + assert error is None + assert config.api_key == GENERIC_KEY