From a35e8e9c3abd17aac0f15bf8a8ba258cd6330185 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 12 Nov 2026 11:51:27 +0100 Subject: [PATCH 1/6] security(ai): FLUID_LLM_API_KEY goes only to a provider you name FLUID_LLM_API_KEY does not say which vendor it is for, yet forge sent it to whatever provider it picked, including one it only inferred: a running Ollama or the one OLLAMA_HOST names (as a bearer token, in cleartext over http), the provider of the one keyring key, or the provider of the one key variable set. So an OpenAI key could reach a local or remote Ollama, or Anthropic. The key is now used only when --llm-provider or FLUID_LLM_PROVIDER names the provider the run uses (founder decision, 2026-11-12). A provider forge picks itself gets its own key variable, then its keyring key, and a warning says the generic key was not used. Interactive fluid forge and fluid init record what the operator named before their setup replaces args.llm_provider, so setup's pick counts as picked. fluid ai test reads the key only with --llm-provider, fluid doctor only with FLUID_LLM_PROVIDER. A project .env can no longer pair an OpenAI, Anthropic, Gemini or GitHub endpoint with FLUID_LLM_API_KEY: forge can pick those providers without a name, and then the operator's keyring key, not the project's key, would go out. --- CHANGELOG.md | 22 + fluid_build/cli/_ai_setup_test.py | 18 +- fluid_build/cli/forge.py | 5 + fluid_build/cli/forge_modes.py | 10 +- fluid_build/cli/init.py | 4 + fluid_build/credentials/litellm_endpoints.py | 15 +- fluid_build/credentials/protected_env.py | 38 +- fluid_build/llm/provider_resolve.py | 18 +- fluid_build/llm/providers.py | 121 +++++- tests/copilot/agents/test_judge_live.py | 7 +- tests/test_dotenv_pairing_matrix.py | 75 +++- .../test_dotenv_snowflake_project_account.py | 9 +- tests/test_forge_copilot_runtime.py | 19 +- tests/test_llm_api_key_scope.py | 404 ++++++++++++++++++ 14 files changed, 701 insertions(+), 64 deletions(-) create mode 100644 tests/test_llm_api_key_scope.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 066646dd..f5f471ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -96,6 +96,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 go to, passed only when the operator states one. A provider plugin that overrides the method with the two-argument signature keeps working until an endpoint is stated. +### Security + +- **`FLUID_LLM_API_KEY` goes only to a provider you name.** The key does not say which vendor it + is for, yet forge sent it to whatever provider it picked, including one it only inferred: a + running Ollama or the one `OLLAMA_HOST` names (as `Authorization: Bearer`, in cleartext when + the host is http), the provider of the one key saved in the keyring, or the provider of the one + key variable set. So an OpenAI key could reach a local or remote Ollama, or Anthropic. Forge + now uses it only when `--llm-provider` or `FLUID_LLM_PROVIDER` names the provider the run uses. + A provider forge picks itself gets its own key variable, then its keyring key, and a warning + says the generic key was not used. In interactive `fluid forge` and `fluid init`, the provider + that setup finds counts as picked, not named. `fluid ai test` reads the key only with + `--llm-provider`, and `fluid doctor` counts it only with `FLUID_LLM_PROVIDER`. A named Ollama + still gets the key (ollama.com, a token proxy). If you used the key with a provider forge + inferred, name that provider: `--llm-provider ` or `FLUID_LLM_PROVIDER=`. +- **A project `.env` can no longer pair an OpenAI, Anthropic, Gemini or GitHub endpoint with + `FLUID_LLM_API_KEY`.** Forge can pick these providers without a name, and then it sends the + operator's keyring key, not the project's `FLUID_LLM_API_KEY`, so `OPENAI_BASE_URL`, + `ANTHROPIC_BASE_URL`, `GEMINI_API_BASE`, `GITHUB_API_BASE` and the like in a project file now + need the provider's own key variable beside them (`OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, + `GEMINI_API_KEY` or `GOOGLE_API_KEY`, `GITHUB_API_KEY`). Endpoints of providers forge reaches + only by name (`GROQ_API_BASE`, `DEEPSEEK_API_BASE` ...) still pair with it. + ## [0.28.0] — 2026-11-06 ### Added diff --git a/fluid_build/cli/_ai_setup_test.py b/fluid_build/cli/_ai_setup_test.py index 6226587c..9d6dc80d 100644 --- a/fluid_build/cli/_ai_setup_test.py +++ b/fluid_build/cli/_ai_setup_test.py @@ -202,8 +202,16 @@ def _validate_ai_test_endpoint(provider_name: str, endpoint: str) -> Optional[st return None -def _resolve_cloud_api_key(provider: str, saved: Optional[dict] = None) -> Optional[str]: - if os.environ.get("FLUID_LLM_API_KEY"): +def _resolve_cloud_api_key( + provider: str, saved: Optional[dict] = None, *, named: bool = False +) -> Optional[str]: + """The key ``fluid ai test`` sends to *provider*. + + ``FLUID_LLM_API_KEY`` counts only when *named* (``--llm-provider``): it does + not say which vendor it is for, so a saved or env-inferred provider never + gets it (``llm/provider_resolve._resolve_api_key``). + """ + if named and os.environ.get("FLUID_LLM_API_KEY"): return os.environ["FLUID_LLM_API_KEY"] if provider == "gemini": for env_var in ("GEMINI_API_KEY", "GOOGLE_API_KEY"): @@ -236,7 +244,7 @@ def _resolve_ai_test_config(args: Any) -> tuple[Optional[LlmConfig], Optional[st saved = _load_ai_config() or {} def _build_for_provider( - pname: str, *, allow_saved: bool = True + pname: str, *, allow_saved: bool = True, named: bool = False ) -> tuple[Optional[LlmConfig], Optional[str]]: provider = BUILTIN_LLM_PROVIDERS.get(pname) if not provider: @@ -264,7 +272,7 @@ def _build_for_provider( ), None, ) - api_key = _resolve_cloud_api_key(pname, saved_for_provider) + api_key = _resolve_cloud_api_key(pname, saved_for_provider, named=named) if not api_key: return None, f"No API key found for {PROVIDER_DISPLAY_NAMES.get(pname, pname)}." endpoint = endpoint_override or saved_for_provider.get("endpoint") @@ -279,7 +287,7 @@ def _build_for_provider( ) if provider_override: - config, error = _build_for_provider(provider_override) + config, error = _build_for_provider(provider_override, named=True) if config: config.timeout_seconds = timeout return config, error diff --git a/fluid_build/cli/forge.py b/fluid_build/cli/forge.py index fd30309b..d6e03641 100644 --- a/fluid_build/cli/forge.py +++ b/fluid_build/cli/forge.py @@ -1861,6 +1861,11 @@ def _run_main(args, logger: logging.Logger) -> int: # 3. Nothing found → prompts user interactively llm_config = run_ai_setup_inline(console) if llm_config: + from fluid_build.llm.providers import remember_named_llm_provider + + # First keep what the operator named: FLUID_LLM_API_KEY goes only to + # that provider, never to the one setup found. + remember_named_llm_provider(args) # Inject into args so copilot's resolve_llm_config() finds them args.llm_provider = llm_config.provider args.llm_model = llm_config.model diff --git a/fluid_build/cli/forge_modes.py b/fluid_build/cli/forge_modes.py index eab6baf7..ee867798 100644 --- a/fluid_build/cli/forge_modes.py +++ b/fluid_build/cli/forge_modes.py @@ -308,7 +308,7 @@ def _forge_build_copilot_options( """ from fluid_build.llm.providers import agent_loop_requested - return { + options = { "llm_provider": get_cli_arg_fn(args, "llm_provider"), "llm_model": get_cli_arg_fn(args, "llm_model"), "llm_endpoint": get_cli_arg_fn(args, "llm_endpoint"), @@ -357,6 +357,14 @@ def _forge_build_copilot_options( "seed_allow_remote": bool(get_cli_arg_fn(args, "seed_allow_remote", False)), "seed_no_remote": bool(get_cli_arg_fn(args, "seed_no_remote", False)), } + # Interactive setup may have put the provider it found into llm_provider. + # Carry what the operator named, so FLUID_LLM_API_KEY still goes only to that + # provider (providers.remember_named_llm_provider). + not_recorded = object() + named = get_cli_arg_fn(args, "llm_provider_named", not_recorded) + if named is not not_recorded: + options["llm_provider_named"] = named + return options def _forge_install_pause_handler( diff --git a/fluid_build/cli/init.py b/fluid_build/cli/init.py index cd85cf02..2b1eda66 100644 --- a/fluid_build/cli/init.py +++ b/fluid_build/cli/init.py @@ -915,6 +915,10 @@ def _ai_mode(args, logger: logging.Logger) -> int: _console = console if RICH_AVAILABLE else None llm_config = run_ai_setup_inline(_console) if llm_config: + from fluid_build.llm.providers import remember_named_llm_provider + + # FLUID_LLM_API_KEY goes only to a provider the operator named. + remember_named_llm_provider(args) args.llm_provider = llm_config.provider args.llm_model = llm_config.model args.llm_endpoint = llm_config.endpoint diff --git a/fluid_build/credentials/litellm_endpoints.py b/fluid_build/credentials/litellm_endpoints.py index 2b50a1d6..c853cab9 100644 --- a/fluid_build/credentials/litellm_endpoints.py +++ b/fluid_build/credentials/litellm_endpoints.py @@ -4,13 +4,14 @@ """Where litellm reads an LLM provider's endpoint, vendored. ``credentials.protected_env`` pairs these with ``FLUID_LLM_API_KEY`` only: -FLUID routes any provider litellm knows and passes it the key it resolved -(``FLUID_LLM_API_KEY``, else the keyring), which beats a provider key in the -project's files. A group in ``protected_env`` that lists one of the names -(``IBM watsonx``, ``OpenAI`` ...) pairs it by its own rule instead, and a name -litellm sends something else to (``LITELLM_PROXY_API_BASE``, the GitHub -Copilot ones) is protected outright. Any other ``_API_BASE`` is the -project's own setting and follows the name-shape rule. +FLUID routes any provider litellm knows, only when the operator names it, and +passes it the key it resolved (``FLUID_LLM_API_KEY``, else the keyring), which +beats a provider key in the project's files. A group in ``protected_env`` that +lists one of the names (``IBM watsonx``, ``OpenAI`` ...) pairs it by its own +rule instead, and a name litellm sends something else to +(``LITELLM_PROXY_API_BASE``, the GitHub Copilot ones) is protected outright. +Any other ``_API_BASE`` is the project's own setting and follows the +name-shape rule. Vendored because importing litellm takes seconds and dotenv hydration runs on every command. Generated from the litellm 1.83.14, 1.97.0, 1.105.0, 1.106.0, diff --git a/fluid_build/credentials/protected_env.py b/fluid_build/credentials/protected_env.py index 3ceb6ceb..6150ec73 100644 --- a/fluid_build/credentials/protected_env.py +++ b/fluid_build/credentials/protected_env.py @@ -83,8 +83,8 @@ (keyring, a stored CLI login, an SDK config file, cloud default credentials), whenever the project files do not supply one that every reader sends in its place. A project credential a reader consults after the keyring (FLUID passes -the key it resolved, FLUID_LLM_API_KEY else the keyring, as litellm's -``api_key``, which beats ``GROQ_API_KEY``), or never (``CLAUDE_API_KEY`` for +the key it resolved, FLUID_LLM_API_KEY for a named provider else the keyring, +as litellm's ``api_key``, which beats ``GROQ_API_KEY``), or never (``CLAUDE_API_KEY`` for litellm), is a decoy and pairs nothing; :attr:`EndpointGroup.paired_by` lists, per code path, the credentials that do. :data:`ENDPOINT_CREDENTIAL_GROUPS` lists the services and where each variable is read; the names litellm reads @@ -356,7 +356,8 @@ "AWS_IGNORE_CONFIGURED_ENDPOINT_URLS", "OLLAMA_HOST", # litellm's ollama/ route reads this one, and sends it the key FLUID - # resolved (FLUID_LLM_API_KEY or the keyring) as a bearer token + # resolved (FLUID_LLM_API_KEY for a named ollama, or the keyring) as a + # bearer token "OLLAMA_API_BASE", "VAULT_ADDR", # litellm routes every model call to this proxy when the switch is @@ -883,17 +884,22 @@ class EndpointGroup: connection_string_passwords: Tuple[str, ...] = () -# ``llm/provider_resolve._resolve_api_key`` prefers it for every provider and -# passes the result to litellm as ``api_key``, which beats the provider's own -# key variable. +# ``llm/provider_resolve._resolve_api_key`` prefers it for a provider the +# operator named (``--llm-provider``, ``FLUID_LLM_PROVIDER``) and passes the +# result to litellm as ``api_key``, which beats the provider's own key +# variable. A provider forge picks itself never gets it, and forge can pick +# openai, anthropic and gemini (the one keyring key or key variable), github +# (interactive setup reads PROVIDER_ENV_VARS) and ollama. A project +# FLUID_LLM_API_KEY is a decoy for those (the operator's keyring key goes out), +# so it pairs only the endpoints of providers forge reaches by being named. _LLM_KEY = "FLUID_LLM_API_KEY" # ``_resolve_api_key`` then reads the provider's PROVIDER_ENV_VARS name (only # openai, anthropic, gemini and github have one) and falls back to the key # ``fluid ai setup`` saved in the keyring, which also beats a provider key # variable that litellm would read. _LLM_KEYRING = ( - "the key comes from FLUID_LLM_API_KEY, the provider's PROVIDER_ENV_VARS name, then the " - "keyring (llm/provider_resolve.py)" + "the key comes from FLUID_LLM_API_KEY when the provider is named, the provider's " + "PROVIDER_ENV_VARS name, then the keyring (llm/provider_resolve.py)" ) _GOOGLE_CREDENTIALS: Tuple[str, ...] = ( "GOOGLE_APPLICATION_CREDENTIALS", @@ -929,7 +935,8 @@ class EndpointGroup: "project OPENAI_BASE_URL from the codex environment unless the project " "supplies CODEX_API_KEY too" ), - paired_by=(("OPENAI_API_KEY", _LLM_KEY),), + # not _LLM_KEY: forge can pick openai without it being named + paired_by=(("OPENAI_API_KEY",),), ), EndpointGroup( service="Anthropic", @@ -949,7 +956,8 @@ class EndpointGroup: "stored subscription login to ANTHROPIC_BASE_URL; only ANTHROPIC_API_KEY " "comes before the ambient credential on both paths" ), - paired_by=(("ANTHROPIC_API_KEY", _LLM_KEY), ("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN")), + # not _LLM_KEY: forge can pick anthropic without it being named + paired_by=(("ANTHROPIC_API_KEY",), ("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN")), ), EndpointGroup( service="Azure OpenAI", @@ -980,7 +988,8 @@ class EndpointGroup: "GEMINI_API_KEY); google-genai (GOOGLE_GEMINI_BASE_URL); " f"{_LLM_KEYRING}, and for gemini also GEMINI_API_KEY and GOOGLE_API_KEY" ), - paired_by=((_LLM_KEY, "GOOGLE_API_KEY", "GEMINI_API_KEY"),), + # not _LLM_KEY: forge can pick gemini without it being named + paired_by=(("GOOGLE_API_KEY", "GEMINI_API_KEY"),), ), EndpointGroup( service="Groq", @@ -1018,7 +1027,8 @@ class EndpointGroup: "litellm's github/ route (GITHUB_API_BASE; the key is GITHUB_API_KEY, " f"llm/providers.py PROVIDER_ENV_VARS); {_LLM_KEYRING}" ), - paired_by=((_LLM_KEY, "GITHUB_API_KEY"),), + # not _LLM_KEY: interactive setup can pick github without it being named + paired_by=(("GITHUB_API_KEY",),), ), EndpointGroup( service="Amazon Bedrock", @@ -1460,7 +1470,9 @@ class EndpointGroup: #: account ids its provider code chooses a host from (DATAROBOT_ENDPOINT, #: SNOWFLAKE_ACCOUNT_ID, GIGACHAT_AUTH_URL ...); FLUID routes any provider name #: litellm knows (``llm/providers.py`` falls back to litellm's catch-all), -#: handing it the key ``_resolve_api_key`` found. Only the names litellm reads +#: handing it the key ``_resolve_api_key`` found. Forge never picks one of these +#: itself: it reaches them only by being named, so FLUID_LLM_API_KEY comes first +#: and pairs them. Only the names litellm reads #: (``litellm_endpoints.py``): any other ``_API_BASE`` is the project's own #: and follows the name-shape rule. UNLISTED_LLM_ENDPOINTS = EndpointGroup( diff --git a/fluid_build/llm/provider_resolve.py b/fluid_build/llm/provider_resolve.py index d7c43e65..fb553887 100644 --- a/fluid_build/llm/provider_resolve.py +++ b/fluid_build/llm/provider_resolve.py @@ -11,7 +11,8 @@ :func:`_infer_provider_from_env` / :func:`_infer_provider_from_keyring` — pick a provider hint from whatever signal we can find. -* :func:`_resolve_api_key` — env var first, keyring fallback. +* :func:`_resolve_api_key` — env var first, keyring fallback; + ``FLUID_LLM_API_KEY`` only for a provider the operator named. * :func:`save_api_key_to_keyring` / :func:`clear_api_key_from_keyring` / :func:`_get_api_key_from_keyring` / :func:`_keyring_key` — keyring round-trips for cloud-provider secrets. @@ -310,8 +311,19 @@ def _infer_provider_from_keyring() -> Optional[str]: return None -def _resolve_api_key(provider: str, env: Mapping[str, str]) -> Optional[str]: - if env.get("FLUID_LLM_API_KEY"): +def _resolve_api_key( + provider: str, env: Mapping[str, str], *, named: bool = False +) -> Optional[str]: + """The API key for *provider*: ``FLUID_LLM_API_KEY``, its own variable, the keyring. + + ``FLUID_LLM_API_KEY`` does not say which provider the key is for, so it is + read only when *named* is true: the operator named *provider* for this run + (``--llm-provider``, ``FLUID_LLM_PROVIDER``). A provider that forge found + itself (a running Ollama, ``OLLAMA_HOST``, the one keyring key, the one + provider key variable, interactive setup's pick) gets its own key or none, + never a key that may belong to another vendor. + """ + if named and env.get("FLUID_LLM_API_KEY"): return env["FLUID_LLM_API_KEY"] env_var = PROVIDER_ENV_VARS.get(provider) if env_var: diff --git a/fluid_build/llm/providers.py b/fluid_build/llm/providers.py index fccf1617..2c809a9f 100644 --- a/fluid_build/llm/providers.py +++ b/fluid_build/llm/providers.py @@ -32,6 +32,7 @@ "reset_llm_caches", "agent_loop_requested", "preflight_llm_provider", + "remember_named_llm_provider", "resolve_llm_config", "resolve_model_name", "resolve_ollama_model", @@ -51,7 +52,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass, field from pathlib import Path -from typing import Any, Dict, Iterator, List, Mapping, Optional +from typing import Any, Dict, Iterator, List, Mapping, Optional, Tuple import httpx @@ -758,23 +759,35 @@ def live_provider_models( *, api_key: Optional[str] = None, environ: Optional[Mapping[str, str]] = None, + named: bool = False, ) -> Optional[List[str]]: - """Fetch live provider model ids when supported by the adapter.""" + """Fetch live provider model ids when supported by the adapter. + + ``named``: the operator named this provider, so ``FLUID_LLM_API_KEY`` may + be its key (see ``provider_resolve._resolve_api_key``). + """ env = dict(environ or os.environ) provider = get_llm_provider(provider_name) - return provider.list_available_models(api_key or _resolve_api_key(provider.name, env), env) + return provider.list_available_models( + api_key or _resolve_api_key(provider.name, env, named=named), env + ) def has_llm_api_key( provider_name: str, environ: Optional[Mapping[str, str]] = None, + *, + named: bool = False, ) -> bool: - """Return whether the provider can resolve credentials without exposing them.""" + """Return whether the provider can resolve credentials without exposing them. + + ``named`` as for :func:`live_provider_models`. + """ env = dict(environ or os.environ) provider = get_llm_provider(provider_name) if provider.name == "ollama": return detect_ollama_available(env) - return bool(_resolve_api_key(provider.name, env)) + return bool(_resolve_api_key(provider.name, env, named=named)) # Providers that never require an LLM API key — the call is paid for @@ -850,6 +863,89 @@ def _explicit_provider_name(args: Any, env: Mapping[str, str]) -> Optional[str]: ) +# Interactive forge puts the provider its inline setup found into +# ``args.llm_provider``, where it would read as named. Before that, it records +# here what the operator named (``None`` for nothing), so ``FLUID_LLM_API_KEY`` +# still goes only to a provider the operator named. +_NAMED_PROVIDER_ATTR = "llm_provider_named" +_NOT_RECORDED = object() + + +def _recorded_named_provider(args: Any) -> Any: + """What :func:`remember_named_llm_provider` recorded on *args*, else ``_NOT_RECORDED``. + + Only a string or ``None`` is a record; anything else (no attribute, a + test double's auto-attribute) is not. + """ + value = getattr(args, _NAMED_PROVIDER_ATTR, _NOT_RECORDED) + return value if value is None or isinstance(value, str) else _NOT_RECORDED + + +def remember_named_llm_provider(args: Any, environ: Optional[Mapping[str, str]] = None) -> None: + """Record which provider the operator named, before ``args.llm_provider`` is replaced. + + Call it before putting a provider that setup found into ``args.llm_provider``. + A record already made is kept, so a second setup cannot turn its own pick + into a named provider. + """ + if _recorded_named_provider(args) is not _NOT_RECORDED: + return + env = os.environ if environ is None else environ + setattr(args, _NAMED_PROVIDER_ATTR, _explicit_provider_name(args, env)) + + +def _unused_generic_key_note(args: Any, provider_name: str) -> str: + """Why ``FLUID_LLM_API_KEY`` is not the key of *provider_name*, which nothing named.""" + recorded = _recorded_named_provider(args) + if isinstance(recorded, str) and recorded: # interactive setup replaced the choice + return ( + f"FLUID_LLM_API_KEY is set but not used: it is for {recorded}, the provider " + f"you named, and this run uses {provider_name}." + ) + return ( + "FLUID_LLM_API_KEY is set but not used: it goes only to a provider named with " + f"--llm-provider or FLUID_LLM_PROVIDER, and forge picked {provider_name} itself. " + f"Pass --llm-provider {provider_name} to use the key with it." + ) + + +def _api_key_for_run( + args: Any, env: Mapping[str, str], provider_name: str +) -> Tuple[Optional[str], Optional[str]]: + """The key this run sends to *provider_name*, and a note when FLUID_LLM_API_KEY is unused. + + ``FLUID_LLM_API_KEY`` does not say which vendor it is for, so a provider + forge picked itself never gets it (``provider_resolve._resolve_api_key``), + and the note says so. + """ + named = _provider_is_named(args, env, provider_name) + note = None + if env.get("FLUID_LLM_API_KEY") and not named: + note = _unused_generic_key_note(args, provider_name) + return _resolve_api_key(provider_name, env, named=named), note + + +def _same_llm_provider(first: str, second: str) -> bool: + """Whether two provider names select the same provider (``claude`` is ``anthropic``).""" + return normalize_llm_provider_name(first) == normalize_llm_provider_name(second) + + +def _provider_is_named(args: Any, env: Mapping[str, str], provider_name: str) -> bool: + """Whether *provider_name*, the provider this run uses, is one the operator named. + + Without a record, a named provider always wins inference, so the run uses + it whenever one is named. With one, ``args.llm_provider`` may be what setup + found, which counts only when it is the provider the operator named. + """ + recorded = _recorded_named_provider(args) + explicit = _explicit_provider_name(args, env) + if recorded is _NOT_RECORDED: + return bool(explicit) + if not recorded: + return False + return recorded == explicit or _same_llm_provider(recorded, provider_name) + + _AGENT_LOOP_ALTERNATIVES = ( "Or author without it: --template starter. The agent loop needs a provider that " "supports tool use, which the keyless providers (claude-code, codex, cursor, kiro, " @@ -946,7 +1042,7 @@ def resolve_llm_config(args: Any, environ: Optional[Mapping[str, str]] = None) - ], ) - api_key = _resolve_api_key(provider.name, env) + api_key, unused_generic_key = _api_key_for_run(args, env, provider.name) # A third-party provider plugin may declare ``keyless = True`` (class attr) # when it authenticates by some other means (e.g. an ambient cloud identity # or its own env var validated at call time) — same escape the built-in @@ -959,8 +1055,10 @@ def resolve_llm_config(args: Any, environ: Optional[Mapping[str, str]] = None) - "copilot_missing_llm_api_key", f"No API key was configured for the {provider.name} copilot adapter.", suggestions=[ + *([unused_generic_key] if unused_generic_key else []), "Run 'fluid ai setup' to configure a provider interactively", - "Set FLUID_LLM_API_KEY or the provider-specific API key environment variable", + "Set the provider-specific API key environment variable, or FLUID_LLM_API_KEY " + "with --llm-provider", "Examples: OPENAI_API_KEY, ANTHROPIC_API_KEY, GEMINI_API_KEY, GOOGLE_API_KEY", "For local models, use --llm-provider ollama and optionally --llm-endpoint", "For keyless authoring: run forge from your IDE (mcp-sampling), or use a " @@ -969,6 +1067,9 @@ def resolve_llm_config(args: Any, environ: Optional[Mapping[str, str]] = None) - ) model_notes: List[str] = [] + if unused_generic_key: + model_notes.append(unused_generic_key) + LOG.warning(unused_generic_key) if not _explicit_provider_name(args, env) and provider.name in _NON_LITELLM_PROVIDERS: # Nothing named a provider and it is not a key-based one, so the keyless # auto-pick chose it: say which, and how to override. @@ -1598,8 +1699,10 @@ def check_llm_readiness(environ: Optional[Mapping[str, str]] = None) -> LlmReadi saved_model = None saved_key = None - # Step 1 — explicit env-var selector. + # Step 1 — explicit env-var selector. Only this step names the provider, so + # only here may FLUID_LLM_API_KEY count as its key (_resolve_api_key). provider_name = env.get("FLUID_LLM_PROVIDER") + named = bool(provider_name) # Step 2 — explicit API-key env vars (deliberate per-run override). if not provider_name: @@ -1670,7 +1773,7 @@ def check_llm_readiness(environ: Optional[Mapping[str, str]] = None) -> LlmReadi env["OLLAMA_HOST"] = saved["ollama_host"] # Resolve API key: env vars → config file - api_key = _resolve_api_key(provider.name, env) or saved_key + api_key = _resolve_api_key(provider.name, env, named=named) or saved_key from fluid_build.llm.coding_agent import _AGENT_SPECS agent = _AGENT_SPECS.get(provider.name) diff --git a/tests/copilot/agents/test_judge_live.py b/tests/copilot/agents/test_judge_live.py index 93779ff6..298fb2b8 100644 --- a/tests/copilot/agents/test_judge_live.py +++ b/tests/copilot/agents/test_judge_live.py @@ -43,10 +43,7 @@ def _resolve_live_config_or_skip(): """Resolve real LLM config; skip cleanly if no provider configured.""" - from fluid_build.cli.forge_copilot_llm_providers import ( - has_llm_api_key, - resolve_llm_config, - ) + from fluid_build.cli.forge_copilot_llm_providers import resolve_llm_config class _Args: pass @@ -62,7 +59,7 @@ class _Args: if not detect_ollama_available(None): pytest.skip("provider resolved as ollama but no local server detected") - elif not has_llm_api_key(config.provider): + elif not config.api_key: # the key the judge sends (FLUID_LLM_API_KEY only if named) pytest.skip( f"no API key resolved for provider {config.provider!r}; " f"set FLUID_LLM_PROVIDER/FLUID_LLM_MODEL + the relevant API key env var to run" diff --git a/tests/test_dotenv_pairing_matrix.py b/tests/test_dotenv_pairing_matrix.py index af47f9c0..c1daf329 100644 --- a/tests/test_dotenv_pairing_matrix.py +++ b/tests/test_dotenv_pairing_matrix.py @@ -141,10 +141,22 @@ def _skip_where_litellm_cannot_reply() -> None: ) +# The providers forge picks without a name: the one keyring key or key variable +# (openai, anthropic, gemini), interactive setup's scan of PROVIDER_ENV_VARS +# (github too), a running Ollama. Such a run never gets FLUID_LLM_API_KEY. Forge +# reaches every other provider only by being named. +_PICKED_WITHOUT_A_NAME = frozenset({"openai", "anthropic", "gemini", "github", "ollama"}) + + def _llm_call(provider: str, endpoint_var: str) -> None: """What ``fluid forge`` does with FLUID_LLM_BACKEND=litellm: resolve the key, hand it to litellm, and let litellm read the endpoint from the environment. + The key is resolved as for a named provider and, for one forge can pick + itself (:data:`_PICKED_WITHOUT_A_NAME`), as for a picked one too, since a + pairing must hold however the run chose its provider. Each distinct key is + sent once, so where both ways find the same key the listener sees one call. + litellm is called only for a loopback endpoint; a test that gets that far is skipped where litellm cannot build its reply (:func:`_skip_where_litellm_cannot_reply`). ``pytest.skip`` raises a @@ -154,19 +166,25 @@ def _llm_call(provider: str, endpoint_var: str) -> None: from fluid_build.llm.litellm_backend import _litellm_model_for from fluid_build.llm.provider_resolve import _resolve_api_key - key = _resolve_api_key(provider, os.environ) + keys: List[Optional[str]] = [] + for named in (True, False) if provider in _PICKED_WITHOUT_A_NAME else (True,): + key = _resolve_api_key(provider, os.environ, named=named) + if key not in keys: + keys.append(key) - def complete() -> Any: - _skip_where_litellm_cannot_reply() - return litellm.completion( - model=_litellm_model_for(provider, "test-model"), - messages=[{"role": "user", "content": "ping"}], - api_key=key, - num_retries=0, - timeout=5, - ) + for key in keys: - send_if_loopback(os.environ.get(endpoint_var), complete) + def complete(key: Optional[str] = key) -> Any: + _skip_where_litellm_cannot_reply() + return litellm.completion( + model=_litellm_model_for(provider, "test-model"), + messages=[{"role": "user", "content": "ping"}], + api_key=key, + num_retries=0, + timeout=5, + ) + + send_if_loopback(os.environ.get(endpoint_var), complete) def _openai_sdk_call() -> None: @@ -576,7 +594,7 @@ def test_a_decoy_credential_does_not_pair_an_llm_endpoint_with_the_keyring_key( "provider,endpoint,credential", [ ("openai", "OPENAI_BASE_URL", "OPENAI_API_KEY"), - ("openai", "OPENAI_API_BASE", "FLUID_LLM_API_KEY"), + ("openai", "OPENAI_API_BASE", "OPENAI_API_KEY"), ("anthropic", "ANTHROPIC_BASE_URL", "ANTHROPIC_API_KEY"), ("gemini", "GEMINI_API_BASE", "GEMINI_API_KEY"), ("gemini", "GEMINI_API_BASE", "GOOGLE_API_KEY"), @@ -606,6 +624,36 @@ def test_the_credential_the_reader_sends_pairs_its_endpoint( assert listener.saw(PROJECT_KEY) != [] +@pytest.mark.parametrize( + "provider,endpoint", + [ + ("openai", "OPENAI_API_BASE"), + ("openai", "OPENAI_BASE_URL"), + ("anthropic", "ANTHROPIC_BASE_URL"), + ("gemini", "GEMINI_API_BASE"), + ("github", "GITHUB_API_BASE"), + ], +) +def test_a_project_fluid_llm_api_key_does_not_pair_a_provider_forge_can_pick( + project: Path, + monkeypatch: pytest.MonkeyPatch, + listener: Listener, + provider: str, + endpoint: str, +) -> None: + # The keyring holds one key, so forge picks its provider without a name, and + # a provider forge picked never gets FLUID_LLM_API_KEY: the keyring key is + # what goes out. The project's FLUID_LLM_API_KEY is a decoy and pairs nothing. + _keyring_holds(monkeypatch, provider) + (project / ".env").write_text(f"{endpoint}={listener.url}\nFLUID_LLM_API_KEY={PROJECT_KEY}\n") + + hydrate_dotenv(project) + _llm_call(provider, endpoint) + + assert endpoint not in os.environ + assert listener.saw(OPERATOR_KEY) == [] + + @pytest.mark.parametrize( "provider,endpoint", [("groq", "GROQ_API_BASE"), ("openai", "OPENAI_BASE_URL")] ) @@ -630,7 +678,8 @@ def test_the_harness_sees_a_key_sent_to_an_operator_endpoint( def test_an_exported_fluid_llm_api_key_refuses_a_project_openai_endpoint( project: Path, monkeypatch: pytest.MonkeyPatch, listener: Listener ) -> None: - # _resolve_api_key prefers FLUID_LLM_API_KEY to the project's OPENAI_API_KEY. + # For a named openai, _resolve_api_key prefers FLUID_LLM_API_KEY to the + # project's OPENAI_API_KEY. monkeypatch.setenv("FLUID_LLM_API_KEY", OPERATOR_KEY) (project / ".env").write_text(f"OPENAI_BASE_URL={listener.url}\nOPENAI_API_KEY={DECOY_KEY}\n") diff --git a/tests/test_dotenv_snowflake_project_account.py b/tests/test_dotenv_snowflake_project_account.py index b20ec474..d4a3ee16 100644 --- a/tests/test_dotenv_snowflake_project_account.py +++ b/tests/test_dotenv_snowflake_project_account.py @@ -337,11 +337,14 @@ def test_a_blank_operator_credential_pairs_only_an_account_behind_a_blank_variab def test_a_blank_operator_llm_key_still_unpairs_an_llm_endpoint() -> None: # FLUID reads a blank FLUID_LLM_API_KEY as unset and takes the keyring key. - project = {"OPENAI_API_BASE": "https://llm.project.test", "FLUID_LLM_API_KEY": PROJECT_KEY} + # (DEEPSEEK_API_BASE: forge reaches deepseek only by name, so FLUID_LLM_API_KEY + # pairs it at all.) + project = {"DEEPSEEK_API_BASE": "https://llm.project.test", "FLUID_LLM_API_KEY": PROJECT_KEY} - refused = unpaired_endpoints(project, {"OPENAI_API_BASE": "", "FLUID_LLM_API_KEY": ""}) + refused = unpaired_endpoints(project, {"DEEPSEEK_API_BASE": "", "FLUID_LLM_API_KEY": ""}) - assert "OPENAI_API_BASE" in refused + assert unpaired_endpoints(project, {}) == {} + assert "DEEPSEEK_API_BASE" in refused def test_a_dropped_account_is_reported_while_the_shell_s_variable_is_blank( diff --git a/tests/test_forge_copilot_runtime.py b/tests/test_forge_copilot_runtime.py index 8bd92bc9..7021de54 100644 --- a/tests/test_forge_copilot_runtime.py +++ b/tests/test_forge_copilot_runtime.py @@ -1177,8 +1177,14 @@ def test_provider_specific_env_keys(self): assert has_llm_api_key("gemini", {"GEMINI_API_KEY": "AIza-test"}) is True assert has_llm_api_key("gemini", {"GOOGLE_API_KEY": "AIza-test"}) is True - def test_generic_fluid_key_counts_for_hosted_provider(self): - assert has_llm_api_key("openai", {"FLUID_LLM_API_KEY": "sk-test"}) is True + def test_generic_fluid_key_counts_only_for_a_named_provider(self, monkeypatch): + monkeypatch.setattr( + "fluid_build.cli.forge_copilot_llm_providers._get_api_key_from_keyring", + lambda provider: None, + ) + assert has_llm_api_key("openai", {"FLUID_LLM_API_KEY": "sk-test"}, named=True) is True + # FLUID_LLM_API_KEY names no vendor, so it is no key for a provider forge picked. + assert has_llm_api_key("openai", {"FLUID_LLM_API_KEY": "sk-test"}) is False def test_missing_hosted_key_is_false(self, monkeypatch): monkeypatch.setattr( @@ -1207,13 +1213,16 @@ def test_env_var_takes_priority_over_keyring(self): result = _resolve_api_key("openai", {"OPENAI_API_KEY": "env-secret"}) assert result == "env-secret" - def test_fluid_llm_api_key_takes_priority_over_keyring(self): + def test_fluid_llm_api_key_takes_priority_over_keyring_for_a_named_provider(self): with patch( "fluid_build.cli.forge_copilot_llm_providers._get_api_key_from_keyring", return_value="keyring-secret", ): - result = _resolve_api_key("openai", {"FLUID_LLM_API_KEY": "generic-key"}) - assert result == "generic-key" + named = _resolve_api_key("openai", {"FLUID_LLM_API_KEY": "generic-key"}, named=True) + picked = _resolve_api_key("openai", {"FLUID_LLM_API_KEY": "generic-key"}) + assert named == "generic-key" + # A provider forge picked itself gets its own key, never the generic one. + assert picked == "keyring-secret" def test_keyring_returns_none_when_not_available(self): with patch( diff --git a/tests/test_llm_api_key_scope.py b/tests/test_llm_api_key_scope.py new file mode 100644 index 00000000..68e67190 --- /dev/null +++ b/tests/test_llm_api_key_scope.py @@ -0,0 +1,404 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""``FLUID_LLM_API_KEY`` goes only to a provider the operator named. + +The key does not say which vendor it is for. Forge used to send it to any +provider it picked, so an OpenAI key went to a running Ollama, to the host +``OLLAMA_HOST`` names, or to Anthropic when the keyring held one Anthropic key. +Named means ``--llm-provider`` or ``FLUID_LLM_PROVIDER`` names the provider the +run uses; every other way forge picks one leaves the key unused. + +Hermetic: the keyring, the Ollama probe, the saved ``fluid ai setup`` config and +``HOME`` are patched, so nothing reads the real ``~/.fluid`` or dials a daemon +(the one listener is a loopback stand-in). +""" + +from __future__ import annotations + +import logging +import os +import sys +from argparse import Namespace +from pathlib import Path +from types import SimpleNamespace +from typing import Dict, Iterator, Optional +from unittest import mock + +import pytest + +import fluid_build.llm.providers as providers +from fluid_build.llm.providers import check_llm_readiness, resolve_llm_config +from tests._dotenv_pairing_support import Listener, refuse_outbound_dns, send_if_loopback + +GENERIC_KEY = "generic-llm-key-test-0001" # pragma: allowlist secret +ANTHROPIC_KEY = "anthropic-keyring-key-test-0002" # pragma: allowlist secret +OPENAI_KEY = "openai-env-key-test-0003" # pragma: allowlist secret + + +class _World: + """What the machine holds: keyring keys, a running Ollama, a saved provider.""" + + def __init__(self, bin_dir: Path) -> None: + self.keyring: Dict[str, str] = {} + self.ollama_up = False + self.saved: Optional[dict] = None + self.bin_dir = bin_dir + + def env(self, **values: str) -> Dict[str, str]: + """An environment with only *values*; PATH finds no coding-agent CLI.""" + return {"PATH": str(self.bin_dir), **values} + + +@pytest.fixture +def world(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Iterator[_World]: + import fluid_build._ai_config_shared as ai_config + + home = tmp_path / "home" + home.mkdir() + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("PYTHON_KEYRING_BACKEND", "keyring.backends.null.Keyring") + w = _World(bin_dir) + monkeypatch.setattr(providers, "_get_api_key_from_keyring", lambda name: w.keyring.get(name)) + monkeypatch.setattr(providers, "detect_ollama_available", lambda env: w.ollama_up) + monkeypatch.setattr(providers, "resolve_ollama_model", lambda env, endpoint=None: "llama3.2") + monkeypatch.setattr(ai_config, "load_ai_config", lambda config_file=None: w.saved) + monkeypatch.setattr( + ai_config, + "load_ai_config_for", + lambda name, config_file=None: w.saved if (w.saved or {}).get("provider") == name else None, + ) + providers.reset_llm_caches() + yield w + providers.reset_llm_caches() + + +def _args(**values: object) -> SimpleNamespace: + base = {"llm_provider": None, "llm_model": None, "llm_endpoint": None} + return SimpleNamespace(**{**base, **values}) + + +# ── a provider forge picks itself never gets the key ─────────────────────── + + +def test_a_running_ollama_never_gets_the_generic_key(world: _World) -> None: + world.ollama_up = True + + config = resolve_llm_config(_args(), world.env(FLUID_LLM_API_KEY=GENERIC_KEY)) + + assert config.provider == "ollama" + assert config.api_key is None + assert any("FLUID_LLM_API_KEY is set but not used" in n for n in config.model_resolution_notes) + + +def test_the_host_ollama_host_names_never_gets_the_generic_key(world: _World) -> None: + # OLLAMA_HOST alone picks ollama; it does not name it. The key would go out as + # "Authorization: Bearer", here in cleartext to another machine. + remote = "http://gpu-box.example.test:11434" + env = world.env(FLUID_LLM_API_KEY=GENERIC_KEY, OLLAMA_HOST=remote) + + config = resolve_llm_config(_args(), env) + with mock.patch.dict(os.environ, {"OLLAMA_HOST": remote}): + _headers, payload = providers.get_llm_provider("ollama").build_request(config, "s", "u") + + assert config.provider == "ollama" + assert config.api_key is None + assert "api_key" not in payload + assert payload["api_base"] == remote + + +def test_the_one_keyring_key_s_provider_gets_that_key(world: _World) -> None: + world.keyring["anthropic"] = ANTHROPIC_KEY + + config = resolve_llm_config(_args(), world.env(FLUID_LLM_API_KEY=GENERIC_KEY)) + + assert config.provider == "anthropic" + assert config.api_key == ANTHROPIC_KEY + + +def test_the_one_key_variable_s_provider_gets_that_key(world: _World) -> None: + env = world.env(FLUID_LLM_API_KEY=GENERIC_KEY, OPENAI_API_KEY=OPENAI_KEY) + + config = resolve_llm_config(_args(), env) + + assert config.provider == "openai" + assert config.api_key == OPENAI_KEY + + +def test_the_key_never_reaches_an_ollama_on_the_wire( + world: _World, monkeypatch: pytest.MonkeyPatch +) -> None: + # The reviewer's capture: litellm's ollama/ route sends the api_key it is + # handed as "Authorization: Bearer". A loopback listener stands in for Ollama. + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") # no price-map fetch + refuse_outbound_dns(monkeypatch) + litellm = pytest.importorskip("litellm") + listener = Listener() + try: + monkeypatch.setenv("OLLAMA_HOST", listener.url) + env = world.env(FLUID_LLM_API_KEY=GENERIC_KEY, OLLAMA_HOST=listener.url) + config = resolve_llm_config(_args(), env) + _headers, payload = providers.get_llm_provider("ollama").build_request(config, "s", "u") + send_if_loopback( + payload.get("api_base"), + lambda: litellm.completion(**{**payload, "num_retries": 0, "timeout": 5}), + ) + + if not listener.requests: + pytest.skip("litellm sent no request to the listener, so it proves nothing") + assert listener.saw(GENERIC_KEY) == [] + finally: + listener.close() + + +# ── a named provider still gets it ───────────────────────────────────────── + + +@pytest.mark.parametrize("how", ["flag", "env"]) +@pytest.mark.parametrize("provider", ["ollama", "anthropic"]) +def test_a_named_provider_gets_the_generic_key_first( + world: _World, how: str, provider: str +) -> None: + # A named Ollama may be ollama.com or a token proxy; a named vendor's own + # variable still comes second. + env = world.env(FLUID_LLM_API_KEY=GENERIC_KEY, ANTHROPIC_API_KEY=ANTHROPIC_KEY) + if how == "env": + env["FLUID_LLM_PROVIDER"] = provider + args = _args() + else: + args = _args(llm_provider=provider) + + config = resolve_llm_config(args, env) + + assert config.provider == provider + assert config.api_key == GENERIC_KEY + assert not any("FLUID_LLM_API_KEY" in n for n in config.model_resolution_notes) + + +def test_a_provider_with_no_key_of_its_own_says_why_the_generic_one_was_not_used( + world: _World, +) -> None: + # Forge seldom picks a provider that then has no key of its own; patch the + # pick to reach that error, whose first suggestion is the way out. + with mock.patch.object(providers, "_provider_name_from", return_value="openai"): + with pytest.raises(providers.CopilotGenerationError) as info: + resolve_llm_config(_args(), world.env(FLUID_LLM_API_KEY=GENERIC_KEY)) + + assert info.value.event == "copilot_missing_llm_api_key" + assert "Pass --llm-provider openai" in info.value.suggestions[0] + + +# ── interactive setup's pick is not a name ───────────────────────────────── + + +def test_interactive_setup_s_pick_does_not_count_as_named(world: _World) -> None: + from fluid_build.llm.providers import remember_named_llm_provider + + args = _args() + remember_named_llm_provider(args, {}) + args.llm_provider = "openai" # what forge does with run_ai_setup_inline's config + env = world.env(FLUID_LLM_API_KEY=GENERIC_KEY, OPENAI_API_KEY=OPENAI_KEY) + + config = resolve_llm_config(args, env) + + assert config.api_key == OPENAI_KEY + + +def test_setup_replacing_a_named_provider_does_not_carry_the_key_over(world: _World) -> None: + from fluid_build.llm.providers import remember_named_llm_provider + + args = _args(llm_provider="anthropic") + remember_named_llm_provider(args, {}) + args.llm_provider = "openai" # the saved provider, loaded over the flag + env = world.env(FLUID_LLM_API_KEY=ANTHROPIC_KEY, OPENAI_API_KEY=OPENAI_KEY) + + config = resolve_llm_config(args, env) + + assert config.api_key == OPENAI_KEY + notes = " ".join(config.model_resolution_notes) + assert "it is for anthropic, the provider you named" in notes + assert "Pass --llm-provider" not in notes + + +@pytest.mark.parametrize("named,picked", [("claude", "anthropic"), ("openai", "openai")]) +def test_setup_keeping_the_named_provider_keeps_the_key( + world: _World, named: str, picked: str +) -> None: + from fluid_build.llm.providers import remember_named_llm_provider + + args = _args(llm_provider=named) + remember_named_llm_provider(args, {}) + args.llm_provider = picked + + config = resolve_llm_config(args, world.env(FLUID_LLM_API_KEY=GENERIC_KEY)) + + assert config.api_key == GENERIC_KEY + + +def test_the_first_record_is_kept(world: _World) -> None: + from fluid_build.llm.providers import remember_named_llm_provider + + args = _args() + remember_named_llm_provider(args, {}) + args.llm_provider = "openai" + remember_named_llm_provider(args, {}) # a second setup must not name its own pick + + assert args.llm_provider_named is None + + +def test_the_copilot_options_carry_the_record(world: _World) -> None: + from fluid_build.cli.forge import get_cli_arg + from fluid_build.cli.forge_modes import _forge_build_copilot_options + + args = Namespace(llm_provider="openai", llm_provider_named=None) + options = _forge_build_copilot_options( + args, get_cli_arg_fn=get_cli_arg, is_non_interactive=False + ) + env = world.env(FLUID_LLM_API_KEY=GENERIC_KEY, OPENAI_API_KEY=OPENAI_KEY) + + config = resolve_llm_config(SimpleNamespace(**options), env) + + assert options["llm_provider_named"] is None + assert config.api_key == OPENAI_KEY + # without a record nothing is added, so a flag still names the provider + plain = _forge_build_copilot_options( + Namespace(llm_provider="openai"), get_cli_arg_fn=get_cli_arg, is_non_interactive=False + ) + assert "llm_provider_named" not in plain + + +@pytest.fixture +def fake_tty(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(sys.stdin, "isatty", lambda: True) + + +@pytest.mark.parametrize("flag", [None, "anthropic"]) +def test_interactive_forge_records_what_was_named_before_setup_s_pick( + world: _World, + fake_tty: None, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + flag: Optional[str], +) -> None: + from fluid_build.cli.forge import run as forge_run + from fluid_build.llm.providers import LlmConfig + + monkeypatch.setattr("fluid_build.cli._welcome_scan.Path.home", lambda: tmp_path / "home") + monkeypatch.setenv("FLUID_FORGE_NO_PREVIEW", "1") + for name in ("FLUID_LLM_PROVIDER", "FLUID_FORGE_AGENT"): + monkeypatch.delenv(name, raising=False) + seen: Dict[str, Namespace] = {} + + def _copilot(args: Namespace, _logger: logging.Logger) -> int: + seen["args"] = args + return 0 + + def _setup(_console: object) -> LlmConfig: + return LlmConfig(provider="openai", model="gpt-4o", endpoint="", api_key=OPENAI_KEY) + + args = Namespace( + command="forge", + forge_subcommand=None, + blank=False, + target_dir=str(tmp_path), + non_interactive=False, + dry_run=False, + llm_provider=flag, + llm_model=None, + llm_endpoint=None, + agent_loop=False, + refine=None, + from_product=[], + from_product_list=None, + template=None, + scaffold=None, + ) + with ( + mock.patch("fluid_build.cli._forge_mode_picker.pick_mode", lambda a, **k: "ai"), + mock.patch("fluid_build.cli._forge_mode_picker.should_show_picker", lambda a: True), + mock.patch("fluid_build.cli.forge.run_ai_copilot_mode", _copilot), + mock.patch("fluid_build.cli.ai_setup.run_ai_setup_inline", _setup), + mock.patch("fluid_build.cli.ai_setup.set_session_env", lambda provider, key: None), + ): + forge_run(args, logging.getLogger("test")) + + ran_with = seen["args"] + assert ran_with.llm_provider == "openai" + assert ran_with.llm_provider_named == flag + env = world.env(FLUID_LLM_API_KEY=GENERIC_KEY, OPENAI_API_KEY=OPENAI_KEY) + assert resolve_llm_config(ran_with, env).api_key == OPENAI_KEY + + +# ── fluid doctor and fluid ai test ───────────────────────────────────────── + + +def test_readiness_counts_the_key_only_for_a_named_provider( + world: _World, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr("fluid_build._optional.is_available", lambda name: True) + world.saved = {"provider": "openai", "model": "gpt-4o"} + + picked = check_llm_readiness(world.env(FLUID_LLM_API_KEY=GENERIC_KEY)) + named = check_llm_readiness( + world.env(FLUID_LLM_API_KEY=GENERIC_KEY, FLUID_LLM_PROVIDER="openai") + ) + + assert picked.provider == "openai" + assert picked.ready is False + assert named.ready is True + + +def _ai_test_config(args: Namespace, env: Dict[str, str], saved: Optional[dict], keyring: str): + from fluid_build.cli.ai_setup import _resolve_ai_test_config + + with ( + mock.patch("fluid_build.cli.ai_setup._load_ai_config", return_value=saved), + mock.patch("fluid_build.cli.ai_setup._load_key_from_keyring", return_value=keyring), + mock.patch("fluid_build.cli.ai_setup.detect_ollama_available", return_value=False), + # keep the fixture's HOME: without one, ``~`` is the real home again + mock.patch.dict(os.environ, {**env, "HOME": os.environ["HOME"]}, clear=True), + ): + return _resolve_ai_test_config(args) + + +def _ai_test_args(provider: Optional[str]) -> Namespace: + return Namespace( + llm_provider=provider, llm_model=None, llm_endpoint=None, llm_timeout_seconds=None + ) + + +def test_ai_test_uses_the_saved_provider_s_own_key(world: _World) -> None: + config, error = _ai_test_config( + _ai_test_args(None), + {"FLUID_LLM_API_KEY": GENERIC_KEY}, + {"provider": "anthropic", "model": "claude-sonnet-4-5"}, + ANTHROPIC_KEY, + ) + + assert error is None + assert config.provider == "anthropic" + assert config.api_key == ANTHROPIC_KEY + + +def test_ai_test_uses_the_key_variable_s_own_key(world: _World) -> None: + config, error = _ai_test_config( + _ai_test_args(None), + {"FLUID_LLM_API_KEY": GENERIC_KEY, "OPENAI_API_KEY": OPENAI_KEY}, + None, + "", + ) + + assert error is None + assert config.provider == "openai" + assert config.api_key == OPENAI_KEY + + +def test_ai_test_with_llm_provider_uses_the_generic_key(world: _World) -> None: + config, error = _ai_test_config( + _ai_test_args("openai"), {"FLUID_LLM_API_KEY": GENERIC_KEY}, None, OPENAI_KEY + ) + + assert error is None + assert config.api_key == GENERIC_KEY From 266ca78e507c3263abf2712b0ec624b1926b7362 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 12 Nov 2026 12:00:15 +0100 Subject: [PATCH 2/6] security(ai): interactive setup keeps your saved key off a project's host A project .env may point a provider at its own host (OPENAI_BASE_URL, ANTHROPIC_BASE_URL, GEMINI_API_BASE, GITHUB_API_BASE) next to a key of its own; credentials/protected_env pairs it because that key is what the readers send. Interactive fluid forge and fluid init then loaded the provider that fluid ai setup saved and copied its keyring key into the provider's key variable with set_session_env, over the project's key (for Gemini into GOOGLE_API_KEY, which forge reads before GEMINI_API_KEY). The operator's key went to the project's host. set_session_env now leaves the variable alone while a project endpoint of that provider is in effect (dotenv_store.project_endpoints_in_effect, as the Snowflake readers already do), so the project's own key goes there, and logs that the operator's key was not used. fluid ai setup checked a typed or imported key with one litellm call, which reads the endpoint from the environment, so the check sent that key to the project's host too. While such an endpoint is in effect, setup now asks for no key, skips a key it would import, and _validate_api_key sends nothing. For a provider without a mapped group, any LLM endpoint a project file set counts. --- CHANGELOG.md | 12 +++ fluid_build/cli/ai_setup.py | 99 ++++++++++++++++++++++-- tests/test_dotenv_pairing_matrix.py | 114 ++++++++++++++++++++++++++++ 3 files changed, 220 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5f471ca..7eab5419 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -117,6 +117,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 need the provider's own key variable beside them (`OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `GEMINI_API_KEY` or `GOOGLE_API_KEY`, `GITHUB_API_KEY`). Endpoints of providers forge reaches only by name (`GROQ_API_BASE`, `DEEPSEEK_API_BASE` ...) still pair with it. +- **Interactive setup no longer sends your saved key to a host a project's `.env` names.** A + project file may point a provider at its own host (`OPENAI_BASE_URL`, `ANTHROPIC_BASE_URL`, + `GEMINI_API_BASE`, `GITHUB_API_BASE`) next to a key of its own. Interactive `fluid forge` and + `fluid init` then loaded the provider `fluid ai setup` saved and copied its keyring key into the + provider's key variable, over the project's key (for Gemini into `GOOGLE_API_KEY`, which forge + reads first), so your key went to the project's host. While a project endpoint of that provider + is in effect, setup now leaves the variable alone, the project's own key goes there, and a + warning says your key was not used. `fluid ai setup` no longer checks a typed or imported key + against such a host either: the check is one litellm call, which reads the endpoint from the + environment. It now asks for no key there, skips a key it would import, and says to run + `fluid ai setup` outside the project. For a provider other than OpenAI, Anthropic, Gemini and + GitHub, any LLM endpoint a project file set counts. ## [0.28.0] — 2026-11-06 diff --git a/fluid_build/cli/ai_setup.py b/fluid_build/cli/ai_setup.py index 4fb09e42..52d568a3 100644 --- a/fluid_build/cli/ai_setup.py +++ b/fluid_build/cli/ai_setup.py @@ -232,7 +232,15 @@ def _validate_api_key(provider: Any, api_key: str) -> Optional[str]: renders as a one-line install hint. Returns ``None`` on success or a short human-readable error message. + + Nothing is sent while a project dotenv file points the provider at its own + host: litellm reads that endpoint from the environment, so the check would + hand the operator's key (typed, or imported from another tool) to it. """ + notice = _project_host_notice(provider.name.lower()) + if notice: + return f"not checked: {notice}" + from fluid_build._optional import require litellm = require("litellm", "litellm", "Validating an AI provider key") @@ -272,21 +280,92 @@ def _validate_api_key(provider: Any, api_key: str) -> Optional[str]: return f"validation failed: {exc}" +# The ``credentials.protected_env`` endpoint group of each provider whose +# PROVIDER_ENV_VARS name ``set_session_env`` writes. +_PROVIDER_ENDPOINT_SERVICES = { + "openai": "OpenAI", + "anthropic": "Anthropic", + "gemini": "Google Gemini", + "github": "GitHub Models", +} + + +def _project_llm_endpoints(provider: str) -> List[str]: + """The endpoint variables in effect that project dotenv files point *provider* at. + + ``credentials.protected_env`` loaded such an endpoint only because the + project files supply a key for it, the one its readers send there. For a + provider whose key ``set_session_env`` writes, the endpoints of its own + group. For any other provider, every LLM endpoint the project set (the + groups that list FLUID_LLM_API_KEY, which every LLM group does, and the + litellm names no group lists): litellm's routes are not mapped one by one. + """ + from fluid_build.credentials import dotenv_store + + service = _PROVIDER_ENDPOINT_SERVICES.get(provider) + if service is not None: + return dotenv_store.project_endpoints_in_effect(service) + from fluid_build.credentials.protected_env import ( + ENDPOINT_CREDENTIAL_GROUPS, + UNLISTED_LLM_ENDPOINTS, + is_endpoint_of, + ) + + groups = [g for g in ENDPOINT_CREDENTIAL_GROUPS if "FLUID_LLM_API_KEY" in g.credentials] + groups.append(UNLISTED_LLM_ENDPOINTS) + return sorted( + name + for name in dotenv_store.project_values_in_effect() + if any(is_endpoint_of(group, name) for group in groups) + ) + + +def _project_host_notice(provider: str) -> Optional[str]: + """Why setup sends no key of the operator's for *provider*, or ``None``. + + Kept apart from every frame that holds a key, so no key can reach the text + (CodeQL py/clear-text-logging-sensitive-data). + """ + names = _project_llm_endpoints(provider) + if not names: + return None + return ( + f"a project dotenv file points {provider} at its own host ({', '.join(names)}), " + "so your key is not sent there; run 'fluid ai setup' outside this project" + ) + + +def _warn_key_not_used(provider: str) -> None: + """Log that the operator's key for *provider* is not used for this run.""" + notice = _project_host_notice(provider) + if notice: + LOG.warning("Your %s key is not used: %s.", provider, notice) + + def set_session_env(provider: str, api_key: str) -> None: """Set the provider-specific env var for the current process only. This is necessary so that ``resolve_llm_config()`` can find the key during this session. The key is **not** written to disk or exported to child processes beyond the current process tree. + + Not while a project dotenv file points the provider at its own host: the + key is the operator's (keyring, ``ai_config.json``, a prompt, another + tool's login), and the readers would send it there in place of the key + the project paired that host with. """ from fluid_build.cli.ai_setup import PROVIDER_ENV_VARS env_var = PROVIDER_ENV_VARS.get(provider) - if env_var: - os.environ[env_var] = api_key - # Don't interpolate env_var/provider — ``api_key`` is in scope - # at this LOG site (CodeQL py/clear-text-logging-sensitive-data). - LOG.debug("Set session env var (provider configured)") + if not env_var: + return + if _project_llm_endpoints(provider): + _warn_key_not_used(provider) + return + os.environ[env_var] = api_key + # Don't interpolate env_var/provider — ``api_key`` is in scope + # at this LOG site (CodeQL py/clear-text-logging-sensitive-data). + LOG.debug("Set session env var (provider configured)") # --------------------------------------------------------------------------- @@ -579,6 +658,12 @@ def _collect_and_validate_api_key( label = PROVIDER_DISPLAY_NAMES.get(provider_choice, provider_choice) signup_url = _SIGNUP_URLS.get(provider_choice, "") last_error: Optional[str] = None + # A key typed here could be neither checked nor used (_validate_api_key, + # set_session_env), so ask for none. + notice = _project_host_notice(provider_choice) + if notice: + console.print(literal_error_text(f"Not asking for a {label} key: {notice}.", "yellow")) + return None for attempt in range(1, max_attempts + 1): url_hint = ( @@ -1250,6 +1335,10 @@ def _import_and_persist(console: Any, candidate: _ImportedKey) -> Optional[LlmCo if provider_obj is None: console.print(f"[yellow]Could not initialise {label}; skipping.[/yellow]") return None + notice = _project_host_notice(provider_obj.name.lower()) + if notice: # the key could be neither checked nor used (_validate_api_key) + console.print(literal_error_text(f"Skipping the {label} key: {notice}.", "yellow")) + return None console.print("[dim]Verifying the imported key...[/dim]") # Don't interpolate the key at any LOG site in this frame # (CodeQL py/clear-text-logging-sensitive-data — ``candidate.key`` in scope). diff --git a/tests/test_dotenv_pairing_matrix.py b/tests/test_dotenv_pairing_matrix.py index c1daf329..657cf312 100644 --- a/tests/test_dotenv_pairing_matrix.py +++ b/tests/test_dotenv_pairing_matrix.py @@ -690,6 +690,120 @@ def test_an_exported_fluid_llm_api_key_refuses_a_project_openai_endpoint( assert "OPENAI_BASE_URL" not in os.environ +# ── interactive setup puts no stored key where a project endpoint reads ───── + + +@pytest.mark.parametrize( + "provider,endpoint,credential", + [ + ("openai", "OPENAI_BASE_URL", "OPENAI_API_KEY"), + ("anthropic", "ANTHROPIC_BASE_URL", "ANTHROPIC_API_KEY"), + # setup writes GOOGLE_API_KEY, which forge reads before GEMINI_API_KEY + ("gemini", "GEMINI_API_BASE", "GEMINI_API_KEY"), + ], +) +def test_interactive_setup_never_sends_the_saved_key_to_a_project_endpoint( + project: Path, + monkeypatch: pytest.MonkeyPatch, + listener: Listener, + caplog: pytest.LogCaptureFixture, + provider: str, + endpoint: str, + credential: str, +) -> None: + # The project pairs its own endpoint with its own key; ``fluid ai setup`` + # saved this provider with a keyring key. Interactive forge's setup copied + # the saved key into the provider's variable (set_session_env), over the + # project's, so the saved key went to the project's host. + import fluid_build.cli.ai_setup as ai_setup + from fluid_build.llm.providers import remember_named_llm_provider + + (project / ".env").write_text(f"{endpoint}={listener.url}\n{credential}={PROJECT_KEY}\n") + hydrate_dotenv(project) + assert os.environ[endpoint] == listener.url # the pairing holds + _keyring_holds(monkeypatch, provider) + monkeypatch.setattr(ai_setup, "_ai_setup_skipped", False) + monkeypatch.setattr(ai_setup, "_load_ai_config", lambda: {"provider": provider, "model": "m"}) + monkeypatch.setattr( + ai_setup, "_load_key_from_keyring", lambda name: OPERATOR_KEY if name == provider else None + ) + + # What interactive ``fluid forge`` does (cli/forge.py): setup, then its key. + with caplog.at_level(logging.WARNING): + config = ai_setup.run_ai_setup_inline(None) + args = types.SimpleNamespace(llm_provider=None, llm_model=None, llm_endpoint=None) + remember_named_llm_provider(args, os.environ) + args.llm_provider = config.provider + if config.api_key: + ai_setup.set_session_env(config.provider, config.api_key) + _llm_call(provider, endpoint) + + assert listener.saw(OPERATOR_KEY) == [] + assert listener.saw(PROJECT_KEY) != [] + assert any(f"Your {provider} key is not used" in r.getMessage() for r in caplog.records) + + +@pytest.mark.parametrize( + "provider,endpoint,credential", + [ + ("openai", "OPENAI_BASE_URL", "OPENAI_API_KEY"), + # no group is mapped to groq, so every LLM endpoint the project set counts + ("groq", "GROQ_API_BASE", "FLUID_LLM_API_KEY"), + ], +) +def test_setup_checks_no_key_against_a_project_endpoint( + project: Path, listener: Listener, provider: str, endpoint: str, credential: str +) -> None: + # ``fluid ai setup`` checks a typed or imported key with one litellm call, + # and litellm reads the endpoint from the environment. + import fluid_build.cli.ai_setup as ai_setup + from fluid_build.cli.forge_copilot_llm_litellm import get_litellm_provider + + _skip_where_litellm_cannot_reply() + (project / ".env").write_text(f"{endpoint}={listener.url}\n{credential}={PROJECT_KEY}\n") + hydrate_dotenv(project) + assert os.environ[endpoint] == listener.url # the pairing holds + + error = ai_setup._validate_api_key(get_litellm_provider(provider), OPERATOR_KEY) + + assert listener.saw(OPERATOR_KEY) == [] + assert error is not None and error.startswith("not checked:") and endpoint in error + + +def test_setup_asks_for_no_key_it_could_not_check( + project: Path, monkeypatch: pytest.MonkeyPatch, listener: Listener +) -> None: + # Asking three times for a key that can be neither checked nor used, with + # "2 attempt(s) left", helps nobody: setup says why once and stops. + import fluid_build.cli.ai_setup as ai_setup + + (project / ".env").write_text(f"OPENAI_BASE_URL={listener.url}\nOPENAI_API_KEY={PROJECT_KEY}\n") + hydrate_dotenv(project) + asked: List[Any] = [] + monkeypatch.setattr(ai_setup.Prompt, "ask", lambda *a, **k: asked.append(a) or OPERATOR_KEY) + printed: List[str] = [] + console = types.SimpleNamespace(print=lambda *a, **k: printed.append(str(a[0]) if a else "")) + + result = ai_setup._collect_and_validate_api_key( + console, provider_choice="openai", tier="balanced" + ) + + assert result is None + assert asked == [] + assert listener.requests == [] + assert any("Not asking for a" in line and "OPENAI_BASE_URL" in line for line in printed) + + +def test_setup_still_sets_the_key_where_no_project_endpoint_reads(project: Path) -> None: + import fluid_build.cli.ai_setup as ai_setup + + (project / ".env").write_text(f"OPENAI_API_KEY={PROJECT_KEY}\n") # a key, no endpoint + hydrate_dotenv(project) + ai_setup.set_session_env("openai", OPERATOR_KEY) + + assert os.environ["OPENAI_API_KEY"] == OPERATOR_KEY + + _FAKE_CODEX = """#!{python} # codex exec, as far as credentials go: OPENAI_BASE_URL is where it sends, # CODEX_API_KEY is the key (blank counts as unset), else the login stored in From bcc5eb58d58e08585ff63e542e88256813708395 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 12 Nov 2026 13:30:39 +0100 Subject: [PATCH 3/6] fix(ai): build the unused-key warnings from names, never from a key CodeQL (py/clear-text-logging-sensitive-data) flagged the two warnings this branch adds: - resolve_llm_config logged a note unpacked from the same tuple as the API key, returned by _api_key_for_run, a call whose name reads as a password source. The note now comes from _unnamed_provider_note, from the provider names and the named flag alone; the key comes from _resolve_api_key on its own. - set_session_env's warning logged the provider string, which setup can get by classifying a typed key (_classify_key_shape -> detect_provider_from_ api_key). The notice now names the endpoint group from _PROVIDER_ENDPOINT_SERVICES, picked by comparison so it is the table's own constant, and the endpoint variables; the provider string is never logged. --- fluid_build/cli/ai_setup.py | 26 +++++++++++++---- fluid_build/llm/providers.py | 45 ++++++++++++++--------------- tests/test_dotenv_pairing_matrix.py | 6 +++- 3 files changed, 47 insertions(+), 30 deletions(-) diff --git a/fluid_build/cli/ai_setup.py b/fluid_build/cli/ai_setup.py index 52d568a3..0e57f80d 100644 --- a/fluid_build/cli/ai_setup.py +++ b/fluid_build/cli/ai_setup.py @@ -290,6 +290,20 @@ def _validate_api_key(provider: Any, api_key: str) -> Optional[str]: } +def _endpoint_service(provider: str) -> Optional[str]: + """The ``credentials.protected_env`` group of *provider*, or ``None``. + + Picked by comparison, so the result is the table's own constant. A provider + name can come from classifying a typed key (``_classify_key_shape``), and + what setup logs must not be derived from a key (CodeQL + py/clear-text-logging-sensitive-data). + """ + return next( + (service for name, service in _PROVIDER_ENDPOINT_SERVICES.items() if name == provider), + None, + ) + + def _project_llm_endpoints(provider: str) -> List[str]: """The endpoint variables in effect that project dotenv files point *provider* at. @@ -302,7 +316,7 @@ def _project_llm_endpoints(provider: str) -> List[str]: """ from fluid_build.credentials import dotenv_store - service = _PROVIDER_ENDPOINT_SERVICES.get(provider) + service = _endpoint_service(provider) if service is not None: return dotenv_store.project_endpoints_in_effect(service) from fluid_build.credentials.protected_env import ( @@ -323,14 +337,16 @@ def _project_llm_endpoints(provider: str) -> List[str]: def _project_host_notice(provider: str) -> Optional[str]: """Why setup sends no key of the operator's for *provider*, or ``None``. - Kept apart from every frame that holds a key, so no key can reach the text - (CodeQL py/clear-text-logging-sensitive-data). + Built from the endpoint group's name and the endpoint variables' names, never + from *provider* itself or a key, because it is logged (CodeQL + py/clear-text-logging-sensitive-data). """ names = _project_llm_endpoints(provider) if not names: return None + who = _endpoint_service(provider) or "this LLM provider" return ( - f"a project dotenv file points {provider} at its own host ({', '.join(names)}), " + f"a project dotenv file points {who} at its own host ({', '.join(names)}), " "so your key is not sent there; run 'fluid ai setup' outside this project" ) @@ -339,7 +355,7 @@ def _warn_key_not_used(provider: str) -> None: """Log that the operator's key for *provider* is not used for this run.""" notice = _project_host_notice(provider) if notice: - LOG.warning("Your %s key is not used: %s.", provider, notice) + LOG.warning("Your key is not used: %s.", notice) def set_session_env(provider: str, api_key: str) -> None: diff --git a/fluid_build/llm/providers.py b/fluid_build/llm/providers.py index 2c809a9f..305a9813 100644 --- a/fluid_build/llm/providers.py +++ b/fluid_build/llm/providers.py @@ -52,7 +52,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass, field from pathlib import Path -from typing import Any, Dict, Iterator, List, Mapping, Optional, Tuple +from typing import Any, Dict, Iterator, List, Mapping, Optional import httpx @@ -894,8 +894,19 @@ def remember_named_llm_provider(args: Any, environ: Optional[Mapping[str, str]] setattr(args, _NAMED_PROVIDER_ATTR, _explicit_provider_name(args, env)) -def _unused_generic_key_note(args: Any, provider_name: str) -> str: - """Why ``FLUID_LLM_API_KEY`` is not the key of *provider_name*, which nothing named.""" +def _unnamed_provider_note( + args: Any, env: Mapping[str, str], provider_name: str, named: bool +) -> Optional[str]: + """Why ``FLUID_LLM_API_KEY`` is not used for *provider_name*, else ``None``. + + ``FLUID_LLM_API_KEY`` does not say which vendor it is for, so a provider + forge picked itself never gets it (``provider_resolve._resolve_api_key``). + The note is logged, so it is built from provider names alone and never from + anything that holds or resolves a key (CodeQL + py/clear-text-logging-sensitive-data). + """ + if named or not env.get("FLUID_LLM_API_KEY"): + return None recorded = _recorded_named_provider(args) if isinstance(recorded, str) and recorded: # interactive setup replaced the choice return ( @@ -909,22 +920,6 @@ def _unused_generic_key_note(args: Any, provider_name: str) -> str: ) -def _api_key_for_run( - args: Any, env: Mapping[str, str], provider_name: str -) -> Tuple[Optional[str], Optional[str]]: - """The key this run sends to *provider_name*, and a note when FLUID_LLM_API_KEY is unused. - - ``FLUID_LLM_API_KEY`` does not say which vendor it is for, so a provider - forge picked itself never gets it (``provider_resolve._resolve_api_key``), - and the note says so. - """ - named = _provider_is_named(args, env, provider_name) - note = None - if env.get("FLUID_LLM_API_KEY") and not named: - note = _unused_generic_key_note(args, provider_name) - return _resolve_api_key(provider_name, env, named=named), note - - def _same_llm_provider(first: str, second: str) -> bool: """Whether two provider names select the same provider (``claude`` is ``anthropic``).""" return normalize_llm_provider_name(first) == normalize_llm_provider_name(second) @@ -1042,7 +1037,9 @@ def resolve_llm_config(args: Any, environ: Optional[Mapping[str, str]] = None) - ], ) - api_key, unused_generic_key = _api_key_for_run(args, env, provider.name) + named = _provider_is_named(args, env, provider.name) + api_key = _resolve_api_key(provider.name, env, named=named) + unnamed_note = _unnamed_provider_note(args, env, provider.name, named) # A third-party provider plugin may declare ``keyless = True`` (class attr) # when it authenticates by some other means (e.g. an ambient cloud identity # or its own env var validated at call time) — same escape the built-in @@ -1055,7 +1052,7 @@ def resolve_llm_config(args: Any, environ: Optional[Mapping[str, str]] = None) - "copilot_missing_llm_api_key", f"No API key was configured for the {provider.name} copilot adapter.", suggestions=[ - *([unused_generic_key] if unused_generic_key else []), + *([unnamed_note] if unnamed_note else []), "Run 'fluid ai setup' to configure a provider interactively", "Set the provider-specific API key environment variable, or FLUID_LLM_API_KEY " "with --llm-provider", @@ -1067,9 +1064,9 @@ def resolve_llm_config(args: Any, environ: Optional[Mapping[str, str]] = None) - ) model_notes: List[str] = [] - if unused_generic_key: - model_notes.append(unused_generic_key) - LOG.warning(unused_generic_key) + if unnamed_note: + model_notes.append(unnamed_note) + LOG.warning(unnamed_note) if not _explicit_provider_name(args, env) and provider.name in _NON_LITELLM_PROVIDERS: # Nothing named a provider and it is not a key-based one, so the keyless # auto-pick chose it: say which, and how to override. diff --git a/tests/test_dotenv_pairing_matrix.py b/tests/test_dotenv_pairing_matrix.py index 657cf312..5baf41f8 100644 --- a/tests/test_dotenv_pairing_matrix.py +++ b/tests/test_dotenv_pairing_matrix.py @@ -740,7 +740,11 @@ def test_interactive_setup_never_sends_the_saved_key_to_a_project_endpoint( assert listener.saw(OPERATOR_KEY) == [] assert listener.saw(PROJECT_KEY) != [] - assert any(f"Your {provider} key is not used" in r.getMessage() for r in caplog.records) + # The warning names the endpoint, never the provider string setup worked + # with: that can come from classifying a typed key (CodeQL + # py/clear-text-logging-sensitive-data). + warned = [r.getMessage() for r in caplog.records if "Your key is not used" in r.getMessage()] + assert warned and all(endpoint in message for message in warned) @pytest.mark.parametrize( From 12ec8ba66bb25facd1595a4f871d1f2cb2706361 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 12 Nov 2026 15:35:44 +0100 Subject: [PATCH 4/6] chore(ai): one literal per suggestion, one import style in the key-scope tests Addresses the two code-quality review threads on #853: - the copilot_missing_llm_api_key suggestion this branch added was two adjacent string literals in a list (implicit concatenation, read as a possibly missing comma); it is one literal now; - tests/test_llm_api_key_scope.py imported fluid_build.llm.providers both as a module and with from-imports; it uses the module import only, and the new interactive-setup matrix test does the same. --- fluid_build/llm/providers.py | 3 +- tests/test_dotenv_pairing_matrix.py | 4 +-- tests/test_llm_api_key_scope.py | 50 +++++++++++++---------------- 3 files changed, 26 insertions(+), 31 deletions(-) diff --git a/fluid_build/llm/providers.py b/fluid_build/llm/providers.py index 305a9813..17c26841 100644 --- a/fluid_build/llm/providers.py +++ b/fluid_build/llm/providers.py @@ -1054,8 +1054,7 @@ def resolve_llm_config(args: Any, environ: Optional[Mapping[str, str]] = None) - suggestions=[ *([unnamed_note] if unnamed_note else []), "Run 'fluid ai setup' to configure a provider interactively", - "Set the provider-specific API key environment variable, or FLUID_LLM_API_KEY " - "with --llm-provider", + "Set the provider's own key variable, or FLUID_LLM_API_KEY with --llm-provider", "Examples: OPENAI_API_KEY, ANTHROPIC_API_KEY, GEMINI_API_KEY, GOOGLE_API_KEY", "For local models, use --llm-provider ollama and optionally --llm-endpoint", "For keyless authoring: run forge from your IDE (mcp-sampling), or use a " diff --git a/tests/test_dotenv_pairing_matrix.py b/tests/test_dotenv_pairing_matrix.py index 5baf41f8..09237daa 100644 --- a/tests/test_dotenv_pairing_matrix.py +++ b/tests/test_dotenv_pairing_matrix.py @@ -716,7 +716,7 @@ def test_interactive_setup_never_sends_the_saved_key_to_a_project_endpoint( # the saved key into the provider's variable (set_session_env), over the # project's, so the saved key went to the project's host. import fluid_build.cli.ai_setup as ai_setup - from fluid_build.llm.providers import remember_named_llm_provider + import fluid_build.llm.providers as llm_providers (project / ".env").write_text(f"{endpoint}={listener.url}\n{credential}={PROJECT_KEY}\n") hydrate_dotenv(project) @@ -732,7 +732,7 @@ def test_interactive_setup_never_sends_the_saved_key_to_a_project_endpoint( with caplog.at_level(logging.WARNING): config = ai_setup.run_ai_setup_inline(None) args = types.SimpleNamespace(llm_provider=None, llm_model=None, llm_endpoint=None) - remember_named_llm_provider(args, os.environ) + llm_providers.remember_named_llm_provider(args, os.environ) args.llm_provider = config.provider if config.api_key: ai_setup.set_session_env(config.provider, config.api_key) diff --git a/tests/test_llm_api_key_scope.py b/tests/test_llm_api_key_scope.py index 68e67190..fee14dd7 100644 --- a/tests/test_llm_api_key_scope.py +++ b/tests/test_llm_api_key_scope.py @@ -28,7 +28,6 @@ import pytest import fluid_build.llm.providers as providers -from fluid_build.llm.providers import check_llm_readiness, resolve_llm_config from tests._dotenv_pairing_support import Listener, refuse_outbound_dns, send_if_loopback GENERIC_KEY = "generic-llm-key-test-0001" # pragma: allowlist secret @@ -86,7 +85,7 @@ def _args(**values: object) -> SimpleNamespace: def test_a_running_ollama_never_gets_the_generic_key(world: _World) -> None: world.ollama_up = True - config = resolve_llm_config(_args(), world.env(FLUID_LLM_API_KEY=GENERIC_KEY)) + config = providers.resolve_llm_config(_args(), world.env(FLUID_LLM_API_KEY=GENERIC_KEY)) assert config.provider == "ollama" assert config.api_key is None @@ -99,7 +98,7 @@ def test_the_host_ollama_host_names_never_gets_the_generic_key(world: _World) -> remote = "http://gpu-box.example.test:11434" env = world.env(FLUID_LLM_API_KEY=GENERIC_KEY, OLLAMA_HOST=remote) - config = resolve_llm_config(_args(), env) + config = providers.resolve_llm_config(_args(), env) with mock.patch.dict(os.environ, {"OLLAMA_HOST": remote}): _headers, payload = providers.get_llm_provider("ollama").build_request(config, "s", "u") @@ -112,7 +111,7 @@ def test_the_host_ollama_host_names_never_gets_the_generic_key(world: _World) -> def test_the_one_keyring_key_s_provider_gets_that_key(world: _World) -> None: world.keyring["anthropic"] = ANTHROPIC_KEY - config = resolve_llm_config(_args(), world.env(FLUID_LLM_API_KEY=GENERIC_KEY)) + config = providers.resolve_llm_config(_args(), world.env(FLUID_LLM_API_KEY=GENERIC_KEY)) assert config.provider == "anthropic" assert config.api_key == ANTHROPIC_KEY @@ -121,7 +120,7 @@ def test_the_one_keyring_key_s_provider_gets_that_key(world: _World) -> None: def test_the_one_key_variable_s_provider_gets_that_key(world: _World) -> None: env = world.env(FLUID_LLM_API_KEY=GENERIC_KEY, OPENAI_API_KEY=OPENAI_KEY) - config = resolve_llm_config(_args(), env) + config = providers.resolve_llm_config(_args(), env) assert config.provider == "openai" assert config.api_key == OPENAI_KEY @@ -139,7 +138,7 @@ def test_the_key_never_reaches_an_ollama_on_the_wire( try: monkeypatch.setenv("OLLAMA_HOST", listener.url) env = world.env(FLUID_LLM_API_KEY=GENERIC_KEY, OLLAMA_HOST=listener.url) - config = resolve_llm_config(_args(), env) + config = providers.resolve_llm_config(_args(), env) _headers, payload = providers.get_llm_provider("ollama").build_request(config, "s", "u") send_if_loopback( payload.get("api_base"), @@ -170,7 +169,7 @@ def test_a_named_provider_gets_the_generic_key_first( else: args = _args(llm_provider=provider) - config = resolve_llm_config(args, env) + config = providers.resolve_llm_config(args, env) assert config.provider == provider assert config.api_key == GENERIC_KEY @@ -184,7 +183,7 @@ def test_a_provider_with_no_key_of_its_own_says_why_the_generic_one_was_not_used # pick to reach that error, whose first suggestion is the way out. with mock.patch.object(providers, "_provider_name_from", return_value="openai"): with pytest.raises(providers.CopilotGenerationError) as info: - resolve_llm_config(_args(), world.env(FLUID_LLM_API_KEY=GENERIC_KEY)) + providers.resolve_llm_config(_args(), world.env(FLUID_LLM_API_KEY=GENERIC_KEY)) assert info.value.event == "copilot_missing_llm_api_key" assert "Pass --llm-provider openai" in info.value.suggestions[0] @@ -194,27 +193,25 @@ def test_a_provider_with_no_key_of_its_own_says_why_the_generic_one_was_not_used def test_interactive_setup_s_pick_does_not_count_as_named(world: _World) -> None: - from fluid_build.llm.providers import remember_named_llm_provider args = _args() - remember_named_llm_provider(args, {}) + providers.remember_named_llm_provider(args, {}) args.llm_provider = "openai" # what forge does with run_ai_setup_inline's config env = world.env(FLUID_LLM_API_KEY=GENERIC_KEY, OPENAI_API_KEY=OPENAI_KEY) - config = resolve_llm_config(args, env) + config = providers.resolve_llm_config(args, env) assert config.api_key == OPENAI_KEY def test_setup_replacing_a_named_provider_does_not_carry_the_key_over(world: _World) -> None: - from fluid_build.llm.providers import remember_named_llm_provider args = _args(llm_provider="anthropic") - remember_named_llm_provider(args, {}) + providers.remember_named_llm_provider(args, {}) args.llm_provider = "openai" # the saved provider, loaded over the flag env = world.env(FLUID_LLM_API_KEY=ANTHROPIC_KEY, OPENAI_API_KEY=OPENAI_KEY) - config = resolve_llm_config(args, env) + config = providers.resolve_llm_config(args, env) assert config.api_key == OPENAI_KEY notes = " ".join(config.model_resolution_notes) @@ -226,24 +223,22 @@ def test_setup_replacing_a_named_provider_does_not_carry_the_key_over(world: _Wo def test_setup_keeping_the_named_provider_keeps_the_key( world: _World, named: str, picked: str ) -> None: - from fluid_build.llm.providers import remember_named_llm_provider args = _args(llm_provider=named) - remember_named_llm_provider(args, {}) + providers.remember_named_llm_provider(args, {}) args.llm_provider = picked - config = resolve_llm_config(args, world.env(FLUID_LLM_API_KEY=GENERIC_KEY)) + config = providers.resolve_llm_config(args, world.env(FLUID_LLM_API_KEY=GENERIC_KEY)) assert config.api_key == GENERIC_KEY def test_the_first_record_is_kept(world: _World) -> None: - from fluid_build.llm.providers import remember_named_llm_provider args = _args() - remember_named_llm_provider(args, {}) + providers.remember_named_llm_provider(args, {}) args.llm_provider = "openai" - remember_named_llm_provider(args, {}) # a second setup must not name its own pick + providers.remember_named_llm_provider(args, {}) # a second setup must not name its own pick assert args.llm_provider_named is None @@ -258,7 +253,7 @@ def test_the_copilot_options_carry_the_record(world: _World) -> None: ) env = world.env(FLUID_LLM_API_KEY=GENERIC_KEY, OPENAI_API_KEY=OPENAI_KEY) - config = resolve_llm_config(SimpleNamespace(**options), env) + config = providers.resolve_llm_config(SimpleNamespace(**options), env) assert options["llm_provider_named"] is None assert config.api_key == OPENAI_KEY @@ -283,7 +278,6 @@ def test_interactive_forge_records_what_was_named_before_setup_s_pick( flag: Optional[str], ) -> None: from fluid_build.cli.forge import run as forge_run - from fluid_build.llm.providers import LlmConfig monkeypatch.setattr("fluid_build.cli._welcome_scan.Path.home", lambda: tmp_path / "home") monkeypatch.setenv("FLUID_FORGE_NO_PREVIEW", "1") @@ -295,8 +289,10 @@ def _copilot(args: Namespace, _logger: logging.Logger) -> int: seen["args"] = args return 0 - def _setup(_console: object) -> LlmConfig: - return LlmConfig(provider="openai", model="gpt-4o", endpoint="", api_key=OPENAI_KEY) + def _setup(_console: object) -> providers.LlmConfig: + return providers.LlmConfig( + provider="openai", model="gpt-4o", endpoint="", api_key=OPENAI_KEY + ) args = Namespace( command="forge", @@ -328,7 +324,7 @@ def _setup(_console: object) -> LlmConfig: assert ran_with.llm_provider == "openai" assert ran_with.llm_provider_named == flag env = world.env(FLUID_LLM_API_KEY=GENERIC_KEY, OPENAI_API_KEY=OPENAI_KEY) - assert resolve_llm_config(ran_with, env).api_key == OPENAI_KEY + assert providers.resolve_llm_config(ran_with, env).api_key == OPENAI_KEY # ── fluid doctor and fluid ai test ───────────────────────────────────────── @@ -340,8 +336,8 @@ def test_readiness_counts_the_key_only_for_a_named_provider( monkeypatch.setattr("fluid_build._optional.is_available", lambda name: True) world.saved = {"provider": "openai", "model": "gpt-4o"} - picked = check_llm_readiness(world.env(FLUID_LLM_API_KEY=GENERIC_KEY)) - named = check_llm_readiness( + picked = providers.check_llm_readiness(world.env(FLUID_LLM_API_KEY=GENERIC_KEY)) + named = providers.check_llm_readiness( world.env(FLUID_LLM_API_KEY=GENERIC_KEY, FLUID_LLM_PROVIDER="openai") ) From b2298038463318a1cfe7e6d90100ff8746d1dc22 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 12 Nov 2026 15:37:39 +0100 Subject: [PATCH 5/6] test(ai): the saved fluid ai setup choice never gets FLUID_LLM_API_KEY Runs without a TTY use the provider fluid ai setup saved (#854). That provider is picked, not named, so it gets its own keyring key and the generic key is unused; without a key of its own it is not ready and resolve_llm_config raises copilot_missing_llm_api_key. Both tests fail on main 3b7e8c10. --- tests/test_llm_api_key_scope.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/test_llm_api_key_scope.py b/tests/test_llm_api_key_scope.py index fee14dd7..30dd6a77 100644 --- a/tests/test_llm_api_key_scope.py +++ b/tests/test_llm_api_key_scope.py @@ -126,6 +126,39 @@ def test_the_one_key_variable_s_provider_gets_that_key(world: _World) -> None: assert config.api_key == OPENAI_KEY +def test_the_saved_choice_s_provider_gets_its_own_key(world: _World) -> None: + # A run without a TTY uses the provider ``fluid ai setup`` saved. That + # provider is picked, not named, so it gets its keyring key, never the + # generic one. Two keyring keys, so the keyring rung picks nothing itself. + world.saved = {"provider": "anthropic", "model": "claude-sonnet-4-5"} + world.keyring.update(anthropic=ANTHROPIC_KEY, openai=OPENAI_KEY) + env = world.env(FLUID_LLM_API_KEY=GENERIC_KEY) + + config = providers.resolve_llm_config(_args(non_interactive=True), env) + readiness = providers.check_llm_readiness(env) + + assert config.provider == "anthropic" + assert config.api_key == ANTHROPIC_KEY + assert any("FLUID_LLM_API_KEY is set but not used" in n for n in config.model_resolution_notes) + assert readiness.provider == "anthropic" + + +def test_the_saved_choice_without_a_key_of_its_own_is_not_ready( + world: _World, monkeypatch: pytest.MonkeyPatch +) -> None: + # The generic key does not make the saved provider ready either. + monkeypatch.setattr("fluid_build._optional.is_available", lambda name: True) + world.saved = {"provider": "anthropic", "model": "claude-sonnet-4-5"} + env = world.env(FLUID_LLM_API_KEY=GENERIC_KEY) + + with pytest.raises(providers.CopilotGenerationError) as info: + providers.resolve_llm_config(_args(non_interactive=True), env) + readiness = providers.check_llm_readiness(env) + + assert info.value.event == "copilot_missing_llm_api_key" + assert readiness.provider == "anthropic" and readiness.ready is False + + def test_the_key_never_reaches_an_ollama_on_the_wire( world: _World, monkeypatch: pytest.MonkeyPatch ) -> None: From 90145615a80630eadfafc4e919b85530b26e6370 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 12 Nov 2026 15:54:49 +0100 Subject: [PATCH 6/6] chore(tests): one import style for fluid_build.llm.providers in the pairing matrix The file imported the module as llm_providers in two places and with 'from ... import LlmConfig' in _run_codex (github-code-quality on #853). _run_codex now uses llm_providers.LlmConfig as well. --- tests/test_dotenv_pairing_matrix.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_dotenv_pairing_matrix.py b/tests/test_dotenv_pairing_matrix.py index 09237daa..80272caf 100644 --- a/tests/test_dotenv_pairing_matrix.py +++ b/tests/test_dotenv_pairing_matrix.py @@ -845,10 +845,10 @@ def _install_fake_codex(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None def _run_codex() -> None: + import fluid_build.llm.providers as llm_providers from fluid_build.llm.coding_agent import get_coding_agent_provider - from fluid_build.llm.providers import LlmConfig - config = LlmConfig(provider="codex", model="codex", endpoint="", api_key=None) + config = llm_providers.LlmConfig(provider="codex", model="codex", endpoint="", api_key=None) try: get_coding_agent_provider("codex").invoke_blocking(config, "system", "user") except Exception: # noqa: BLE001 — the listener answers 401