diff --git a/CHANGELOG.md b/CHANGELOG.md index ab0be845..1c6ac723 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 script needs a SQL warehouse that runs SQL scripting. A dbt build that `--mode amend-and-build` runs locally is followed by the same re-declaration. +### Changed + +- **Keys for two or more providers stop the run, even with Ollama running.** With + `OPENAI_API_KEY` and `ANTHROPIC_API_KEY` both set (or one provider's key and `OLLAMA_HOST`) and + nothing naming a provider, a run without a TTY skipped both keys and used a running local + Ollama, and interactive setup used the first key it found. Forge now stops with + `copilot_no_llm_provider` and names the keys, without asking Ollama; interactive setup, + `fluid ai test` and `fluid doctor` say the same and use none of them, and `fluid init`'s AI + path exits 1 where it used to write a template instead. A `fluid ai setup` + choice and a single keyring key still come first, and `GEMINI_API_KEY` with `GOOGLE_API_KEY` is + one provider. Pick one with `--llm-provider` or `FLUID_LLM_PROVIDER`. + ### Fixed - **`fluid ai models --provider` works, and takes the names `fluid forge --llm-provider` @@ -74,6 +86,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `forge.contract_written` and the receipt sent another, so the stream named one file twice, first by its absolute path and then by its path from the current directory. Only the receipt's event is left, as in every other mode. +- **Interactive `fluid forge` keeps the provider, model and endpoint you state.** Its inline + setup put the saved or first-found provider over `--llm-provider`, `--llm-model` and + `--llm-endpoint`, and over `FLUID_LLM_PROVIDER`, `FLUID_FORGE_AGENT`, `FLUID_LLM_MODEL` and + `FLUID_LLM_ENDPOINT`, so `fluid forge --llm-provider ollama` could call Anthropic. A run that + names its provider now skips inline setup and resolves as a run without a TTY does. Otherwise + setup fills only what the run leaves unstated, and an agent it finds is not used when the run + states an endpoint or a model, or runs the agent loop. The welcome panel and the interview's + "What I see" panel name the provider the run uses, not the saved one. `fluid init`'s AI path + does the same, and its handoff no longer offers AI setup for a provider named by a variable. +- **Interactive setup no longer picks GitHub Models from `GITHUB_API_KEY`.** It took any variable + in `PROVIDER_ENV_VARS`, so `GITHUB_API_KEY` alone selected github, which setup then failed to + build (`KeyError: 'github'`). It now reads the key variables a run without a TTY reads: + `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, and `GOOGLE_API_KEY` or `GEMINI_API_KEY` (it missed the + second). GitHub Models stays a provider you name with `--llm-provider github`. `fluid ai test` + and `fluid init`'s check for a configured AI follow the same rule. +- **`fluid ai test` tests the provider `FLUID_LLM_PROVIDER` or `FLUID_FORGE_AGENT` names.** It + read only `--llm-provider`, so with `FLUID_LLM_PROVIDER=anthropic` it tested the saved + provider, which forge would not use. A provider named this way also gets `FLUID_LLM_API_KEY`, + as with the flag. A coding agent or `mcp-sampling` named either way is not tested, and the + command now says so, where `claude-code` failed as "Unsupported AI provider 'claude_code'". - **A contract that types its text columns `STRING` passes the DuckDB acquisition's schema gate.** DuckDB reports every text column as `VARCHAR`, and the gate read `STRING` and `VARCHAR` as two types, so a contract written for Spark or Databricks stopped every ingest with "source diff --git a/fluid_build/cli/_ai_setup_test.py b/fluid_build/cli/_ai_setup_test.py index 9d6dc80d..83cc2e24 100644 --- a/fluid_build/cli/_ai_setup_test.py +++ b/fluid_build/cli/_ai_setup_test.py @@ -207,9 +207,10 @@ def _resolve_cloud_api_key( ) -> Optional[str]: """The key ``fluid ai test`` sends to *provider*. - ``FLUID_LLM_API_KEY`` counts only when *named* (``--llm-provider``): it does - not say which vendor it is for, so a saved or env-inferred provider never - gets it (``llm/provider_resolve._resolve_api_key``). + ``FLUID_LLM_API_KEY`` counts only when *named* (``--llm-provider``, + ``FLUID_LLM_PROVIDER``, ``FLUID_FORGE_AGENT``): it does not say which vendor + it is for, so a saved or env-inferred provider never gets it + (``llm/provider_resolve._resolve_api_key``). """ if named and os.environ.get("FLUID_LLM_API_KEY"): return os.environ["FLUID_LLM_API_KEY"] @@ -234,8 +235,63 @@ def _resolve_cloud_api_key( return None +def _untested_agent_message(named: Optional[str]) -> Optional[str]: + """Why ``fluid ai test`` does not test *named*, when it is an agent; else ``None``. + + Forge hands an agent's calls to its CLI, or to the IDE for ``mcp-sampling``, + so there is no API call to test. Any spelling forge accepts counts + (``MCP-Sampling``, ``claude_code``): the registry does not list + ``mcp-sampling``, so the name is normalized here. + """ + # From ``providers``, never ``provider_resolve`` first (a circular import). + from fluid_build.llm.providers import _NON_LITELLM_PROVIDERS, _canonical_provider_name + + agent = (_canonical_provider_name(named) or "").lower().replace("_", "-") + if agent not in _NON_LITELLM_PROVIDERS: + return None + if agent == "mcp-sampling": + return ( + "fluid ai test does not test mcp-sampling: the IDE that runs forge's MCP " + "server answers those calls with its own model." + ) + return ( + f"fluid ai test does not test {agent}: forge hands its calls to that agent's " + "CLI, which signs in by itself. 'fluid doctor' checks that the CLI is installed." + ) + + +def _key_variable_choice() -> tuple[Optional[str], Optional[str]]: + """The provider the key variables select, as in forge, else why none. + + One provider's key variables select it (never ``GITHUB_API_KEY``; + ``OLLAMA_HOST`` is checked after this). Two or more providers' select none: + exactly one keyring key then decides. Returns ``(provider, None)``, + ``(None, message)`` to stop, or ``(None, None)`` when no key variable applies. + """ + # From ``providers``, never ``provider_resolve`` first (a circular import). + from fluid_build.llm.providers import ( + _explicit_key_signals, + _infer_provider_from_keyring, + ambiguous_keys_message, + ) + + ambiguous = ambiguous_keys_message(os.environ) + if ambiguous: + kept = _infer_provider_from_keyring() + return (kept, None) if kept else (None, ambiguous) + return next((p for p in _explicit_key_signals(os.environ) if p != "ollama"), None), None + + def _resolve_ai_test_config(args: Any) -> tuple[Optional[LlmConfig], Optional[str]]: - provider_override = _normalize_ai_test_provider(getattr(args, "llm_provider", None)) + from fluid_build.llm.providers import named_llm_provider + + # The provider the run names, read as forge reads it: --llm-provider, then + # FLUID_LLM_PROVIDER, then FLUID_FORGE_AGENT. + named = named_llm_provider(args, os.environ) + untested = _untested_agent_message(named) + if untested: + return None, untested + provider_override = _normalize_ai_test_provider(named) model_override = getattr(args, "llm_model", None) or os.environ.get("FLUID_LLM_MODEL") endpoint_override = getattr(args, "llm_endpoint", None) or os.environ.get("FLUID_LLM_ENDPOINT") timeout = _coerce_ai_test_timeout( @@ -277,40 +333,30 @@ def _build_for_provider( return None, f"No API key found for {PROVIDER_DISPLAY_NAMES.get(pname, pname)}." endpoint = endpoint_override or saved_for_provider.get("endpoint") config = _make_cloud_config(pname, api_key, model=model, endpoint=endpoint) + config.timeout_seconds = timeout if config.endpoint: endpoint_error = _validate_ai_test_endpoint(pname, config.endpoint) if endpoint_error: return None, endpoint_error - return ( - config, - None, - ) + return config, None if provider_override: - config, error = _build_for_provider(provider_override, named=True) - if config: - config.timeout_seconds = timeout - return config, error + return _build_for_provider(provider_override, named=True) if saved.get("provider"): config, error = _build_for_provider(str(saved["provider"])) if config: - config.timeout_seconds = timeout return config, None LOG.debug("Saved AI config was not test-ready: %s", error) - for pname, env_var in PROVIDER_ENV_VARS.items(): - if os.environ.get(env_var) or (pname == "gemini" and os.environ.get("GEMINI_API_KEY")): - config, error = _build_for_provider(pname, allow_saved=False) - if config: - config.timeout_seconds = timeout - return config, error + pname, stop = _key_variable_choice() + if stop: + return None, stop + if pname: + return _build_for_provider(pname, allow_saved=False) if os.environ.get("OLLAMA_HOST") or detect_ollama_available(os.environ): - config, error = _build_for_provider("ollama", allow_saved=False) - if config: - config.timeout_seconds = timeout - return config, error + return _build_for_provider("ollama", allow_saved=False) return None, "No AI provider configured. Run 'fluid ai setup' or set a provider API key." diff --git a/fluid_build/cli/_world_class_interview.py b/fluid_build/cli/_world_class_interview.py index d1db350c..0e96c983 100644 --- a/fluid_build/cli/_world_class_interview.py +++ b/fluid_build/cli/_world_class_interview.py @@ -250,6 +250,7 @@ def collect_signals( target_dir: Optional[Path] = None, project_memory: Any = None, active_agent: str = "", + ai_provider: Optional[str] = None, ) -> InterviewSignals: """Pull every signal the system already has into a single record. @@ -261,6 +262,9 @@ def collect_signals( data-product-type picker to the codes the agent's spec declares in ``supported_data_product_types``. Empty / unknown agent names leave the picker unfiltered (all three types). + + ``ai_provider``: the provider the run resolved. It replaces the scan's + guess, which reads the saved choice and cannot see ``--llm-provider``. """ cwd = (target_dir or Path.cwd()).resolve() sig = InterviewSignals(cwd_name=cwd.name, active_agent=active_agent) @@ -289,6 +293,9 @@ def collect_signals( sig.ai_provider = findings.ai_provider_hint except Exception as exc: # noqa: BLE001 LOG.debug("welcome_scan_unavailable: %s", exc) + if isinstance(ai_provider, str) and ai_provider: + sig.ai_configured = True + sig.ai_provider = ai_provider try: from fluid_build.cli.workspace_config import ( @@ -439,6 +446,7 @@ def run_world_class_bootstrap( project_memory: Any = None, auto_mode: bool = False, active_agent: str = "", + ai_provider: Optional[str] = None, ) -> None: """Drive the world-class interview against ``state``. @@ -451,6 +459,9 @@ def run_world_class_bootstrap( ``supported_data_product_types`` allowlist. The interview's inference path falls back to the first allowed type when the workspace lock or scan-suggested type is forbidden by the agent. + + ``ai_provider``: the provider the run uses, shown in the "What I see" + panel (see :func:`collect_signals`). """ from fluid_build.cli.forge_dialogs import ask_friendly_text @@ -462,6 +473,7 @@ def run_world_class_bootstrap( target_dir=target_dir, project_memory=project_memory, active_agent=active_agent, + ai_provider=ai_provider, ) # Pre-fill anything we can infer without asking, BEFORE rendering diff --git a/fluid_build/cli/ai_setup.py b/fluid_build/cli/ai_setup.py index eacd2118..0846fc35 100644 --- a/fluid_build/cli/ai_setup.py +++ b/fluid_build/cli/ai_setup.py @@ -1440,24 +1440,87 @@ def _offer_import_from_other_clis(console: Any) -> Optional[LlmConfig]: return None +# What :func:`_setup_from_key_variables` returns when setup must stop. +_STOP_SETUP = object() + + +def _setup_from_key_variables(console: Any) -> Any: + """Step 2 of :func:`run_ai_setup_inline`: the key variables a run without a TTY reads. + + ``OPENAI_API_KEY``, ``ANTHROPIC_API_KEY`` and ``GEMINI_API_KEY`` / + ``GOOGLE_API_KEY`` select their provider. ``GITHUB_API_KEY`` never does: + github is used only when named (``PROVIDER_ENV_VARS``). ``OLLAMA_HOST`` is + step 3. Variables for two or more providers select none: as without a TTY, + exactly one keyring key then decides, else this names them. + + Returns the config, :data:`_STOP_SETUP` when setup must stop, or ``None`` to + go on to step 3. + """ + # Resolve deferred symbols via the module ``__getattr__`` lazy-import seam. + from fluid_build.cli.ai_setup import PROVIDER_DISPLAY_NAMES, PROVIDER_ENV_VARS + + # From ``providers``, never ``provider_resolve`` first (a circular import). + from fluid_build.llm.providers import ( + _explicit_key_signals, + _infer_provider_from_keyring, + _resolve_api_key, + ambiguous_keys_message, + ) + + ambiguous = ambiguous_keys_message(os.environ) + if ambiguous: + kept = _infer_provider_from_keyring() + kept_key = _resolve_api_key(kept, os.environ) if kept else None + if kept and kept_key: + if console and RICH_AVAILABLE: + label = PROVIDER_DISPLAY_NAMES.get(kept, kept) + console.print(f"[dim]Using {label}, the one provider with a keyring key.[/dim]") + LOG.info("Inline AI setup: several key variables, one keyring key") + return _make_cloud_config(kept, kept_key) + if console and RICH_AVAILABLE: + console.print(literal_error_text(ambiguous, "yellow")) + LOG.info("Inline AI setup: key variables for several providers, none chosen") + return _STOP_SETUP + signals = _explicit_key_signals(os.environ) # one provider at most here + pname = next(iter(signals), None) + if not pname or pname == "ollama": # nothing, or OLLAMA_HOST: step 3 + return None + # GOOGLE_API_KEY before GEMINI_API_KEY, as the key resolver reads them. + env_var = PROVIDER_ENV_VARS.get(pname) + if not env_var or not os.environ.get(env_var): + env_var = signals[pname][0] + env_key = os.environ[env_var] + if console and RICH_AVAILABLE: + label = PROVIDER_DISPLAY_NAMES.get(pname, pname) + console.print(f"[dim]Using {label} from environment.[/dim]") + LOG.info("Inline AI setup: loaded %s from env var", pname) + # 2a. Opt-in import: the key is in this shell's environment + # (e.g. exported for another tool). Offer to persist it to + # fluid's own keyring so it survives the next shell — strictly + # opt-in, never in CI, and only the env value we already read. + _offer_import_env_api_key(console, pname, env_key, env_var) + return _make_cloud_config(pname, env_key) + + def run_ai_setup_inline(console: Any) -> Optional[LlmConfig]: """Compact inline setup triggered when forge starts without a configured provider. Resolves an LLM config in this priority order: 1. saved config file + OS keyring (``~/.fluid/ai_config.json``) - 2. cloud provider env vars (``OPENAI_API_KEY`` etc.) + 2. cloud provider env vars (``OPENAI_API_KEY`` etc.), never ``GITHUB_API_KEY`` 3. explicit ``OLLAMA_HOST`` env var 4. auto-detected local Ollama (with user confirmation on TTY) 5. interactive provider picker - Returns ``None`` if the user skips setup or stdin is not a TTY. + Returns ``None`` if the user skips setup, stdin is not a TTY, or key + variables name two or more providers (setup names them instead of choosing). + Callers do not run it for a provider the operator named. """ # Resolve deferred symbols via the module ``__getattr__`` lazy-import seam. from fluid_build.cli.ai_setup import ( BUILTIN_LLM_PROVIDERS, PROVIDER_DISPLAY_NAMES, - PROVIDER_ENV_VARS, ask_confirmation, detect_ollama_available, ) @@ -1518,19 +1581,12 @@ def run_ai_setup_inline(console: Any) -> Optional[LlmConfig]: # 2. Check cloud-provider env vars (e.g. CI environments that # inject ANTHROPIC_API_KEY / OPENAI_API_KEY directly). - for pname, env_var in PROVIDER_ENV_VARS.items() if has_litellm else (): - env_key = os.environ.get(env_var) - if env_key: - if console and RICH_AVAILABLE: - label = PROVIDER_DISPLAY_NAMES.get(pname, pname) - console.print(f"[dim]Using {label} from environment.[/dim]") - LOG.info("Inline AI setup: loaded %s from env var", pname) - # 2a. Opt-in import: the key is in this shell's environment - # (e.g. exported for another tool). Offer to persist it to - # fluid's own keyring so it survives the next shell — strictly - # opt-in, never in CI, and only the env value we already read. - _offer_import_env_api_key(console, pname, env_key, env_var) - return _make_cloud_config(pname, env_key) + if has_litellm: + chosen = _setup_from_key_variables(console) + if chosen is _STOP_SETUP: + return None + if chosen is not None: + return chosen # 3. Explicit OLLAMA_HOST env var. if has_litellm and os.environ.get("OLLAMA_HOST"): diff --git a/fluid_build/cli/forge.py b/fluid_build/cli/forge.py index 14de12c6..9ae637c8 100644 --- a/fluid_build/cli/forge.py +++ b/fluid_build/cli/forge.py @@ -1892,34 +1892,29 @@ def _run_main(args, logger: logging.Logger) -> int: if console and not get_cli_arg(args, "non_interactive", False) and not _picker_ran: print_welcome_panel(console) - # Check LLM readiness; load saved config or offer inline setup - if not get_cli_arg(args, "non_interactive", False): + # Check LLM readiness; load saved config or offer inline setup. Setup + # cannot see the run's flags, so a run that names its provider + # (--llm-provider, FLUID_LLM_PROVIDER, FLUID_FORGE_AGENT) skips it and + # resolves as a run without a TTY does. The copilot's preflight still + # fails first on a provider this install cannot run (no litellm), before + # anything reads the keyring. + from fluid_build.llm.providers import named_llm_provider + + if not get_cli_arg(args, "non_interactive", False) and not named_llm_provider(args): from fluid_build.cli.ai_setup import run_ai_setup_inline - # An explicit litellm-backed --llm-provider on an install without - # litellm fails here, before the setup flow reads the OS keyring or - # offers to import or prompt for a key it could not use. - if get_cli_arg(args, "llm_provider", None): - from fluid_build.llm.providers import preflight_llm_provider - - preflight_llm_provider(args) - - # Always go through inline setup — it handles all cases: + # Inline setup handles every run that names no provider: # 1. Config file exists with key → loads it, sets env vars, returns config # 2. Keyring has key → loads it, sets env vars, returns config # 3. Nothing found → prompts user interactively llm_config = run_ai_setup_inline(console) if llm_config: - from fluid_build.llm.providers import remember_named_llm_provider - - # First keep what the operator named: FLUID_LLM_API_KEY goes only to - # that provider, never to the one setup found. - remember_named_llm_provider(args) - # Inject into args so copilot's resolve_llm_config() finds them - args.llm_provider = llm_config.provider - args.llm_model = llm_config.model - args.llm_endpoint = llm_config.endpoint - if llm_config.api_key: + from fluid_build.llm.providers import fill_llm_args_from_setup + + # Setup fills only what the operator left unstated + # (--llm-model, --llm-endpoint and their FLUID_LLM_* twins win), + # and its key goes into the session only with its own provider. + if fill_llm_args_from_setup(args, llm_config) and llm_config.api_key: from fluid_build.cli.ai_setup import set_session_env set_session_env(llm_config.provider, llm_config.api_key) diff --git a/fluid_build/cli/forge_copilot_interview.py b/fluid_build/cli/forge_copilot_interview.py index 99e76237..ed7b19ad 100644 --- a/fluid_build/cli/forge_copilot_interview.py +++ b/fluid_build/cli/forge_copilot_interview.py @@ -594,6 +594,7 @@ def run_adaptive_copilot_interview( console=console, target_dir=target_dir, project_memory=project_memory, + ai_provider=getattr(llm_config, "provider", None), ) # Schema-coverage gate: if anything required is missing, # fall through to the gap-filler bootstrap so the LLM still diff --git a/fluid_build/cli/forge_modes.py b/fluid_build/cli/forge_modes.py index a5b4fc25..859226ed 100644 --- a/fluid_build/cli/forge_modes.py +++ b/fluid_build/cli/forge_modes.py @@ -1111,6 +1111,22 @@ def _render_ai_copilot_error(console: Any, exc: BaseException) -> None: ) +def _with_run_provider(findings: Any, args: Any) -> Any: + """The welcome scan's *findings*, showing the provider this run uses. + + The scan reads the saved choice and the environment, never the flags, so + ``--llm-provider`` (or the provider inline setup put into *args*) replaces + its guess. + """ + from fluid_build.llm.providers import named_llm_provider + + provider = named_llm_provider(args) + if provider: + findings.ai_configured = True + findings.ai_provider_hint = provider + return findings + + def run_ai_copilot_mode( args: Any, logger: logging.Logger, @@ -1163,7 +1179,7 @@ def run_ai_copilot_mode( run_welcome_scan, ) - _findings = run_welcome_scan() + _findings = _with_run_provider(run_welcome_scan(), args) render_welcome(_findings, console=console) except Exception: # noqa: BLE001 — welcome must never block forge pass diff --git a/fluid_build/cli/init.py b/fluid_build/cli/init.py index 2b1eda66..b0d93e3f 100644 --- a/fluid_build/cli/init.py +++ b/fluid_build/cli/init.py @@ -528,33 +528,41 @@ def run(args, logger: logging.Logger) -> int: def _is_ai_configured() -> bool: """Return True if a saved AI config (or environment fallback) is usable. - Mirrors the resolution order in :func:`ai_setup.run_ai_setup_inline`: - saved config → cloud provider env vars (including the - ``GEMINI_API_KEY`` alias for ``GOOGLE_API_KEY``) → ``OLLAMA_HOST``. + Mirrors what forge does before :func:`ai_setup.run_ai_setup_inline`: a + provider named by ``FLUID_LLM_PROVIDER`` / ``FLUID_FORGE_AGENT`` (forge + then runs no setup) → saved config → the key variables of exactly one + provider (``OPENAI_API_KEY``, ``ANTHROPIC_API_KEY``, ``GEMINI_API_KEY`` / + ``GOOGLE_API_KEY``, or ``OLLAMA_HOST``), or of several when exactly one + provider has a keyring key, which setup then uses. ``GITHUB_API_KEY`` does + not count. Used to decide whether the post-init handoff should offer to set AI up before launching forge. """ try: - from .ai_setup import PROVIDER_ENV_VARS, _load_ai_config + from .ai_setup import _load_ai_config except Exception: # pragma: no cover — import guard for partial installs return False + # From ``providers``: importing ``provider_resolve`` first is a circular import, + # and a plain ``fluid init`` has loaded neither by now. + from fluid_build.llm.providers import ( + _infer_provider_from_explicit_keys, + _infer_provider_from_keyring, + ambiguous_keys_message, + named_llm_provider, + ) + + if named_llm_provider(None): + return True + saved = _load_ai_config() if saved and saved.get("provider"): return True - for env_var in PROVIDER_ENV_VARS.values(): - if os.environ.get(env_var): - return True - - # ``GEMINI_API_KEY`` is the user-facing alias for ``GOOGLE_API_KEY`` - # accepted by the copilot provider resolver — keep the post-init - # handoff in sync so a user who pastes that env var doesn't get - # asked to "set up AI" again. - if os.environ.get("GEMINI_API_KEY"): + if _infer_provider_from_explicit_keys(os.environ): return True - - return bool(os.environ.get("OLLAMA_HOST")) + # Keys for several providers: setup goes on with the one keyring key, if any. + return bool(ambiguous_keys_message(os.environ) and _infer_provider_from_keyring()) def _offer_first_forge(args, logger: logging.Logger) -> None: @@ -894,6 +902,36 @@ def _ensure_gitignore_template(workspace_root: Path) -> None: pass # Best-effort — gitignore is a nice-to-have, not load-bearing. +def _setup_ai_mode_llm(args) -> str: + """Pick the LLM for ``fluid init``'s AI path, as ``fluid forge``'s ``run()`` does. + + A provider the operator named (``FLUID_LLM_PROVIDER``, ``FLUID_FORGE_AGENT``) + skips inline setup. Otherwise setup fills only what the run leaves unstated + (``FLUID_LLM_MODEL`` and ``FLUID_LLM_ENDPOINT`` win). Returns ``"ready"``, + ``"stop"`` when setup named the keys of two or more providers and chose + none, or ``"template"`` when no provider is available. + """ + from fluid_build._optional import is_available + from fluid_build.llm.providers import ( + ambiguous_keys_message, + fill_llm_args_from_setup, + named_llm_provider, + ) + + from .ai_setup import run_ai_setup_inline, set_session_env + + if named_llm_provider(args): + return "ready" + llm_config = run_ai_setup_inline(console if RICH_AVAILABLE else None) + if llm_config: + if fill_llm_args_from_setup(args, llm_config) and llm_config.api_key: + set_session_env(llm_config.provider, llm_config.api_key) + return "ready" + if is_available("litellm") and ambiguous_keys_message(os.environ): + return "stop" + return "template" + + def _ai_mode(args, logger: logging.Logger) -> int: """Let AI design the data product — delegates to forge copilot inline.""" if RICH_AVAILABLE: @@ -908,23 +946,14 @@ def _ai_mode(args, logger: logging.Logger) -> int: ) try: - from .ai_setup import run_ai_setup_inline, set_session_env from .forge import run_ai_copilot_mode - # Run inline LLM setup — same as forge.py:run() does. - _console = console if RICH_AVAILABLE else None - llm_config = run_ai_setup_inline(_console) - if llm_config: - from fluid_build.llm.providers import remember_named_llm_provider - - # FLUID_LLM_API_KEY goes only to a provider the operator named. - remember_named_llm_provider(args) - args.llm_provider = llm_config.provider - args.llm_model = llm_config.model - args.llm_endpoint = llm_config.endpoint - if llm_config.api_key: - set_session_env(llm_config.provider, llm_config.api_key) - else: + choice = _setup_ai_mode_llm(args) + if choice == "stop": + # Setup named the keys of two or more providers and chose none: + # stop, as forge does without a TTY, rather than write a template. + return 1 + if choice == "template": if RICH_AVAILABLE: from .forge_dialogs import print_dialog_status diff --git a/fluid_build/credentials/protected_env.py b/fluid_build/credentials/protected_env.py index 6150ec73..9e8ba252 100644 --- a/fluid_build/credentials/protected_env.py +++ b/fluid_build/credentials/protected_env.py @@ -889,7 +889,7 @@ class EndpointGroup: # result to litellm as ``api_key``, which beats the provider's own key # variable. A provider forge picks itself never gets it, and forge can pick # openai, anthropic and gemini (the one keyring key or key variable), github -# (interactive setup reads PROVIDER_ENV_VARS) and ollama. A project +# (interactive setup's key import offers a gh login) and ollama. A project # FLUID_LLM_API_KEY is a decoy for those (the operator's keyring key goes out), # so it pairs only the endpoints of providers forge reaches by being named. _LLM_KEY = "FLUID_LLM_API_KEY" @@ -1028,6 +1028,7 @@ class EndpointGroup: f"llm/providers.py PROVIDER_ENV_VARS); {_LLM_KEYRING}" ), # not _LLM_KEY: interactive setup can pick github without it being named + # (it imports a gh login; GITHUB_API_KEY alone selects nothing) paired_by=(("GITHUB_API_KEY",),), ), EndpointGroup( diff --git a/fluid_build/llm/provider_resolve.py b/fluid_build/llm/provider_resolve.py index 7cddf340..a5f55e06 100644 --- a/fluid_build/llm/provider_resolve.py +++ b/fluid_build/llm/provider_resolve.py @@ -87,6 +87,24 @@ def _explicit_key_signals(env: Mapping[str, str]) -> Dict[str, List[str]]: return signals +def ambiguous_keys_message(env: Mapping[str, str]) -> Optional[str]: + """Why the key variables select no provider, when they name two or more. + + ``None`` when they name one provider or none. Forge, ``fluid doctor``, + interactive setup and ``fluid ai test`` all stop with this message. None of + them picks one of the keys, or a running Ollama, in the operator's place + (founder decision, 2026-11-12: stop and name the keys). + """ + signals = _explicit_key_signals(env) + if len(signals) < 2: + return None + names = ", ".join(name for present in signals.values() for name in present) + return ( + f"No LLM provider is selected: {names} are all set, so none of them is " + "chosen. Pick one with --llm-provider or FLUID_LLM_PROVIDER." + ) + + def _canonical_provider_name(name: Any) -> Optional[str]: """*name* as the provider registry spells it, ``None`` when it is not a name. @@ -159,13 +177,9 @@ def no_provider_message(env: Mapping[str, str]) -> str: "is not used when the run states an endpoint or a model, or runs the agent " "loop. Name a provider with --llm-provider." ) - signals = _explicit_key_signals(env) - if len(signals) > 1: - names = ", ".join(name for present in signals.values() for name in present) - return ( - f"No LLM provider is selected: {names} are all set, so none of them is " - "chosen. Pick one with --llm-provider or FLUID_LLM_PROVIDER." - ) + ambiguous = ambiguous_keys_message(env) + if ambiguous: + return ambiguous if env.get("FLUID_LLM_API_KEY"): return ( "No LLM provider is selected: FLUID_LLM_API_KEY does not say which provider " @@ -311,12 +325,19 @@ def _infer_provider_from_ambient( rung: for the agent loop, which needs tool use, and when the operator stated an endpoint or model. + Neither rung answers when key variables name two or more providers + (:func:`ambiguous_keys_message`): the operator set those keys, and a + running Ollama is not a choice between them. The run stops and names them. + Only call this AFTER the saved config and the keyring have been consulted — ambient services must not override an explicit operator choice. See :func:`_infer_provider_from_env` for the full resolution ladder. """ + if len(_explicit_key_signals(env)) > 1: + # Checked before the Ollama probe, so nothing is dialled either. + return None # Resolve via the host module: ``detect_ollama_available`` is # defined later in the host's module body, so an import-time # binding would be ``None``. Late-resolution at call time picks @@ -341,7 +362,8 @@ def _infer_provider_from_env( 1. Exactly one explicit key variable: a deliberate per-run choice. 2. The provider ``fluid ai setup`` saved, unless :func:`_passed_over`. 3. Exactly one keyring key. - 4. Ambient discovery (:func:`_infer_provider_from_ambient`). + 4. Ambient discovery (:func:`_infer_provider_from_ambient`), except when + key variables name two or more providers: then nothing is selected. The rest of the ladder after a named provider, shared by the real selection and :func:`check_llm_readiness`, so ``fluid doctor`` names what a run uses. diff --git a/fluid_build/llm/providers.py b/fluid_build/llm/providers.py index 7c5f0f20..809233b9 100644 --- a/fluid_build/llm/providers.py +++ b/fluid_build/llm/providers.py @@ -32,6 +32,8 @@ "reset_llm_caches", "agent_loop_requested", "preflight_llm_provider", + "fill_llm_args_from_setup", + "named_llm_provider", "remember_named_llm_provider", "resolve_llm_config", "resolve_model_name", @@ -52,6 +54,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass, field from pathlib import Path +from types import SimpleNamespace from typing import Any, Dict, Iterator, List, Mapping, Optional import httpx @@ -909,6 +912,78 @@ def remember_named_llm_provider(args: Any, environ: Optional[Mapping[str, str]] setattr(args, _NAMED_PROVIDER_ATTR, _explicit_provider_name(args, env)) +def named_llm_provider(args: Any, environ: Optional[Mapping[str, str]] = None) -> Optional[str]: + """The provider the operator named for this run, else ``None``. + + ``--llm-provider``, else ``FLUID_LLM_PROVIDER``, else ``FLUID_FORGE_AGENT``: + the first thing every selection reads (:func:`_provider_name_from`). A run + that names one does not run interactive setup, which cannot know the name. + """ + env = os.environ if environ is None else environ + flag = _stated_arg(args, "llm_provider") + return flag or _explicit_provider_name(None, env) + + +def _stated_arg(args: Any, name: str) -> Optional[str]: + """``args.`` when the run states it, else ``None``. + + Only a non-empty string counts, so a test double's invented attribute does not. + """ + value = getattr(args, name, None) + return value if isinstance(value, str) and value else None + + +def fill_llm_args_from_setup( + args: Any, config: LlmConfig, environ: Optional[Mapping[str, str]] = None +) -> bool: + """Put what interactive setup found into *args*, under what the operator stated. + + Inline setup (``cli.ai_setup.run_ai_setup_inline``) finds a provider without + reading the run's flags, so what the operator stated wins: + + * A named provider (:func:`named_llm_provider`) takes nothing from setup. + * An agent that setup found (a coding-agent CLI or MCP sampling) is not used + when the run states an endpoint or a model, or runs the agent loop, as in + a run without a TTY (``provider_resolve._passed_over``). + * Otherwise setup's provider goes into ``args.llm_provider``, recorded first + as not named (:func:`remember_named_llm_provider`). Its model and endpoint + fill ``args.llm_model`` and ``args.llm_endpoint`` only where the run states + neither the flag nor ``FLUID_LLM_MODEL`` / ``FLUID_LLM_ENDPOINT``. + + Returns whether setup's provider is used. Only then may the caller put its + key into the session (``cli.ai_setup.set_session_env``). + """ + env = os.environ if environ is None else environ + remember_named_llm_provider(args, env) + recorded = _recorded_named_provider(args) + if isinstance(recorded, str) and recorded: # the operator named a provider + return False + # The run's own statements only, so a test double's invented attribute + # does not read as a stated endpoint, model or agent loop. + stated = SimpleNamespace( + agent_loop=getattr(args, "agent_loop", False) is True, + **{ + name: _stated_arg(args, name) + for name in ("llm_endpoint", "llm_model", "llm_routing_endpoint", "llm_routing_model") + }, + ) + if config.provider in _NON_LITELLM_PROVIDERS and not _agents_allowed(stated, env): + # A constant line: *config* holds a key (CodeQL + # py/clear-text-logging-sensitive-data). + LOG.warning( + "Not using the agent setup found: an agent is not used when the run states " + "an endpoint or a model, or runs the agent loop. Name a provider with " + "--llm-provider." + ) + return False + args.llm_provider = config.provider + if not (_stated_arg(args, "llm_model") or env.get("FLUID_LLM_MODEL")): + args.llm_model = config.model + if not (_stated_arg(args, "llm_endpoint") or env.get("FLUID_LLM_ENDPOINT")): + args.llm_endpoint = config.endpoint + return True + + def _unnamed_provider_note( args: Any, env: Mapping[str, str], provider_name: str, named: bool ) -> Optional[str]: @@ -1616,14 +1691,21 @@ def call_llm_streaming( # extracted to ``cli/_llm_provider_resolve.py``. Re-exported here # under the same names so existing call sites and test patches keep # resolving. +# Import these from here, not from ``provider_resolve``: that module binds this +# one's names when it loads, so importing it first is a circular import. from fluid_build.llm.provider_resolve import ( # noqa: E402,F401 _LLM_KEYRING_PREFIX, + _canonical_provider_name, + _explicit_key_signals, _get_api_key_from_keyring, _infer_provider_from_env, + _infer_provider_from_explicit_keys, + _infer_provider_from_keyring, _keyring_key, _lean_keyless_autopick, _resolve_api_key, _saved_choice_for, + ambiguous_keys_message, clear_api_key_from_keyring, no_provider_message, save_api_key_to_keyring, diff --git a/tests/conftest.py b/tests/conftest.py index afa2a5dc..21658796 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -344,6 +344,61 @@ def _hermetic_saved_ai_config(monkeypatch, _empty_fluid_home): monkeypatch.setenv("FLUID_CONFIG", str(home / "config.yaml")) +# What interactive setup writes straight into ``os.environ``: ``set_session_env`` +# exports the provider key it found (``PROVIDER_ENV_VARS``), and a saved Ollama +# choice sets ``OLLAMA_HOST``. Monkeypatch cannot undo a direct assignment. +_SESSION_ENV_VARS = ( + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + "GOOGLE_API_KEY", + "GEMINI_API_KEY", + "GITHUB_API_KEY", + "OLLAMA_HOST", +) + + +@pytest.fixture(autouse=True) +def _session_env_is_restored(monkeypatch): + """No test leaves a provider key or ``OLLAMA_HOST`` behind for the next one. + + A key one test exports (the import tests' canary, for one) made a later + test in the same worker select that provider: interactive setup reads the + key variables from ``os.environ``. The values are put back as the test found + them. Requesting ``monkeypatch`` tears this down first, so what monkeypatch + set is then undone by monkeypatch itself. + """ + import os + + found = {name: os.environ.get(name) for name in _SESSION_ENV_VARS} + yield + for name, value in found.items(): + if value is None: + os.environ.pop(name, None) + else: + os.environ[name] = value + + +@pytest.fixture +def no_llm_key_variables(monkeypatch): + """No provider key variable or LLM selector is set. + + Provider selection reads the key variables (``OPENAI_API_KEY``, + ``ANTHROPIC_API_KEY``, ``GEMINI_API_KEY`` / ``GOOGLE_API_KEY``, + ``OLLAMA_HOST``) and the named selectors from the environment. A test that + needs none of them set asks for this fixture. + """ + for name in ( + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + "GEMINI_API_KEY", + "GOOGLE_API_KEY", + "OLLAMA_HOST", + "FLUID_LLM_PROVIDER", + "FLUID_FORGE_AGENT", + ): + monkeypatch.delenv(name, raising=False) + + def _noop_assert_safe_url(url, *, allow_private=False): # noqa: ARG001 """Replacement for safe_http.assert_safe_url used in unit tests. diff --git a/tests/test_ai_setup_import_keys.py b/tests/test_ai_setup_import_keys.py index d7d78291..f45ee514 100644 --- a/tests/test_ai_setup_import_keys.py +++ b/tests/test_ai_setup_import_keys.py @@ -336,11 +336,11 @@ def test_preserves_other_saved_providers(self, monkeypatch, tmp_path): class TestWiring: + @pytest.mark.usefixtures("no_llm_key_variables") def test_inline_offers_import_before_interactive_picker(self, monkeypatch): monkeypatch.setattr(ai_setup, "_ai_setup_skipped", False, raising=False) monkeypatch.setattr(ai_setup, "_load_ai_config", lambda: None) monkeypatch.setattr(ai_setup, "PROVIDER_ENV_VARS", {}, raising=False) - monkeypatch.delenv("OLLAMA_HOST", raising=False) monkeypatch.setattr(ai_setup, "detect_ollama_available", lambda env: False) monkeypatch.setattr(ai_setup.shutil, "which", lambda b: None) monkeypatch.setattr(ai_setup.sys, "stdin", types.SimpleNamespace(isatty=lambda: True)) diff --git a/tests/test_coding_agent_ai_setup.py b/tests/test_coding_agent_ai_setup.py index 2ff9bae4..f27c6aab 100644 --- a/tests/test_coding_agent_ai_setup.py +++ b/tests/test_coding_agent_ai_setup.py @@ -40,13 +40,13 @@ def test_inline_setup_loads_saved_coding_agent_keyless(monkeypatch): assert cfg.api_key is None +@pytest.mark.usefixtures("no_llm_key_variables") def test_inline_setup_auto_selects_claude_code_when_only_agent(monkeypatch): # No saved config, no cloud key, no Ollama, claude on PATH, non-interactive # -> step 4.5 auto-selects keyless Claude Code (mirrors the Ollama branch). monkeypatch.setattr(ai_setup, "_ai_setup_skipped", False, raising=False) monkeypatch.setattr(ai_setup, "_load_ai_config", lambda: None) monkeypatch.setattr(ai_setup, "PROVIDER_ENV_VARS", {}, raising=False) - monkeypatch.delenv("OLLAMA_HOST", raising=False) monkeypatch.setattr(ai_setup, "detect_ollama_available", lambda env: False) monkeypatch.setattr( ai_setup.shutil, "which", lambda b: "/usr/local/bin/claude" if b == "claude" else None @@ -142,12 +142,12 @@ def test_offer_import_env_key_skips_on_shape_mismatch(monkeypatch): assert called["asked"] is False +@pytest.mark.usefixtures("no_llm_key_variables") def test_inline_setup_no_agent_no_key_returns_none(monkeypatch): # Nothing available + non-interactive -> None (existing behavior preserved). monkeypatch.setattr(ai_setup, "_ai_setup_skipped", False, raising=False) monkeypatch.setattr(ai_setup, "_load_ai_config", lambda: None) monkeypatch.setattr(ai_setup, "PROVIDER_ENV_VARS", {}, raising=False) - monkeypatch.delenv("OLLAMA_HOST", raising=False) monkeypatch.setattr(ai_setup, "detect_ollama_available", lambda env: False) monkeypatch.setattr(ai_setup.shutil, "which", lambda b: None) monkeypatch.setattr(ai_setup.sys, "stdin", types.SimpleNamespace(isatty=lambda: False)) diff --git a/tests/test_dotenv_pairing_matrix.py b/tests/test_dotenv_pairing_matrix.py index 80272caf..cddf14e0 100644 --- a/tests/test_dotenv_pairing_matrix.py +++ b/tests/test_dotenv_pairing_matrix.py @@ -142,8 +142,8 @@ def _skip_where_litellm_cannot_reply() -> None: # The providers forge picks without a name: the one keyring key or key variable -# (openai, anthropic, gemini), interactive setup's scan of PROVIDER_ENV_VARS -# (github too), a running Ollama. Such a run never gets FLUID_LLM_API_KEY. Forge +# (openai, anthropic, gemini), interactive setup's key import (a gh login: +# github), a running Ollama. Such a run never gets FLUID_LLM_API_KEY. Forge # reaches every other provider only by being named. _PICKED_WITHOUT_A_NAME = frozenset({"openai", "anthropic", "gemini", "github", "ollama"}) diff --git a/tests/test_init_forge_integration.py b/tests/test_init_forge_integration.py index 2afb4aab..1fe0e011 100644 --- a/tests/test_init_forge_integration.py +++ b/tests/test_init_forge_integration.py @@ -653,10 +653,12 @@ def test_returns_true_when_saved_config_has_provider(self): ): assert _is_ai_configured() is True + @pytest.mark.usefixtures("no_llm_key_variables") def test_returns_true_when_provider_env_var_set(self, monkeypatch): from fluid_build.cli.init import _is_ai_configured - # Make sure no saved config interferes. + # Make sure no saved config interferes, nor a second provider's key: + # two providers' keys select neither. monkeypatch.setenv("OPENAI_API_KEY", "sk-test-fake") with patch("fluid_build.cli.ai_setup._load_ai_config", return_value=None): assert _is_ai_configured() is True @@ -699,6 +701,8 @@ def test_returns_false_when_no_config_or_env(self, monkeypatch): "GEMINI_API_KEY", "GOOGLE_API_KEY", "OLLAMA_HOST", + "FLUID_LLM_PROVIDER", + "FLUID_FORGE_AGENT", ): monkeypatch.delenv(key, raising=False) with patch("fluid_build.cli.ai_setup._load_ai_config", return_value=None): diff --git a/tests/test_llm_api_key_scope.py b/tests/test_llm_api_key_scope.py index 30dd6a77..bf7d228d 100644 --- a/tests/test_llm_api_key_scope.py +++ b/tests/test_llm_api_key_scope.py @@ -354,10 +354,17 @@ def _setup(_console: object) -> providers.LlmConfig: forge_run(args, logging.getLogger("test")) ran_with = seen["args"] - assert ran_with.llm_provider == "openai" - assert ran_with.llm_provider_named == flag env = world.env(FLUID_LLM_API_KEY=GENERIC_KEY, OPENAI_API_KEY=OPENAI_KEY) - assert providers.resolve_llm_config(ran_with, env).api_key == OPENAI_KEY + config = providers.resolve_llm_config(ran_with, env) + if flag is None: + assert ran_with.llm_provider == "openai" + assert ran_with.llm_provider_named is None + assert config.api_key == OPENAI_KEY + else: + # A named provider skips setup, so the key goes to the provider named. + assert ran_with.llm_provider == flag + assert config.provider == flag + assert config.api_key == GENERIC_KEY # ── fluid doctor and fluid ai test ───────────────────────────────────────── diff --git a/tests/test_llm_provider_selection_flags.py b/tests/test_llm_provider_selection_flags.py new file mode 100644 index 00000000..35f20043 --- /dev/null +++ b/tests/test_llm_provider_selection_flags.py @@ -0,0 +1,765 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""Provider selection: what the operator states wins, and ambiguous keys stop. + +1. Key variables for two or more providers fell through to a running Ollama, + which then answered in their place. Founder decision (2026-11-12): stop and + name the keys. The saved choice and one keyring key still come first. +2. Inline setup inferred ``github`` from ``GITHUB_API_KEY``, and then failed to + build it, though ``PROVIDER_ENV_VARS`` says github is used only when named. + Setup and ``fluid ai test`` now read the key variables a run without a TTY + reads, and treat keys for two or more providers as that run does. +3. Interactive ``fluid forge`` and ``fluid init`` put inline setup's pick over + ``--llm-provider``, ``--llm-model``, ``--llm-endpoint`` and their + ``FLUID_LLM_*`` twins. A named provider now skips setup, and setup fills + only what the run leaves unstated. +4. ``fluid ai test`` ignored ``FLUID_LLM_PROVIDER`` and ``FLUID_FORGE_AGENT``. + +Hermetic: the keyring, the Ollama probe, the saved ``fluid ai setup`` choice, +HOME and PATH are patched, so nothing reads the real ``~/.fluid``, finds an +agent CLI, or dials a daemon. +""" + +from __future__ import annotations + +import io +import logging +import os +import sys +from argparse import Namespace +from pathlib import Path +from types import SimpleNamespace +from typing import Dict, Iterator, List, Optional, Tuple +from unittest import mock + +import pytest + +import fluid_build.cli.ai_setup as ai_setup +import fluid_build.llm.providers as providers + +pytestmark = pytest.mark.unit + +OPENAI_KEY = "openai-env-key-test-0001" # pragma: allowlist secret +ANTHROPIC_KEY = "anthropic-env-key-test-0002" # pragma: allowlist secret +GEMINI_KEY = "gemini-env-key-test-0003" # pragma: allowlist secret +GITHUB_KEY = "github-env-key-test-0004" # pragma: allowlist secret +ONPREM = "https://llm.internal.test/v1" +GENERIC_KEY = "generic-llm-key-test-0005" # pragma: allowlist secret + + +_VARS = ( + "FLUID_LLM_PROVIDER", + "FLUID_FORGE_AGENT", + "FLUID_LLM_API_KEY", + "FLUID_LLM_MODEL", + "FLUID_LLM_ENDPOINT", + "FLUID_LLM_ROUTING_MODEL", + "FLUID_LLM_ROUTING_ENDPOINT", + "FLUID_COPILOT_AGENT_LOOP", + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + "GEMINI_API_KEY", + "GOOGLE_API_KEY", + "GITHUB_API_KEY", + "GITHUB_TOKEN", + "OLLAMA_HOST", +) + + +class _World: + """What the machine holds: keyring keys, a running Ollama, a saved provider.""" + + def __init__(self, monkeypatch: pytest.MonkeyPatch, bin_dir: Path) -> None: + self.keyring: Dict[str, str] = {} + self.ollama_up = False + self.ollama_probes = 0 + self.saved: Optional[dict] = None + self.monkeypatch = monkeypatch + self.bin_dir = bin_dir + + def env(self, **values: str) -> Dict[str, str]: + """An environment with only *values*; PATH finds no coding-agent CLI.""" + return {"PATH": str(self.bin_dir), **values} + + def tty(self, value: bool) -> None: + self.monkeypatch.setattr(sys, "stdin", SimpleNamespace(isatty=lambda: value)) + + +@pytest.fixture +def world(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Iterator[_World]: + import fluid_build._ai_config_shared as ai_config + + for var in _VARS: + monkeypatch.delenv(var, raising=False) + home = tmp_path / "home" + home.mkdir() + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("PATH", str(bin_dir)) + monkeypatch.setenv("PYTHON_KEYRING_BACKEND", "keyring.backends.null.Keyring") + w = _World(monkeypatch, bin_dir) + + def _ollama(env: object) -> bool: + w.ollama_probes += 1 + return w.ollama_up + + monkeypatch.setattr(providers, "_get_api_key_from_keyring", lambda name: w.keyring.get(name)) + monkeypatch.setattr(providers, "detect_ollama_available", _ollama) + monkeypatch.setattr(providers, "resolve_ollama_model", lambda env, endpoint=None: "llama3.2") + monkeypatch.setattr(ai_config, "load_ai_config", lambda config_file=None: w.saved) + monkeypatch.setattr( + ai_config, + "load_ai_config_for", + lambda name, config_file=None: w.saved if (w.saved or {}).get("provider") == name else None, + ) + monkeypatch.setattr(ai_setup, "_load_ai_config", lambda *a, **k: w.saved) + monkeypatch.setattr(ai_setup, "_load_key_from_keyring", lambda name: w.keyring.get(name)) + monkeypatch.setattr(ai_setup, "detect_ollama_available", _ollama) + monkeypatch.setattr(ai_setup, "_ai_setup_skipped", False) + w.tty(False) + providers.reset_llm_caches() + yield w + providers.reset_llm_caches() + + +def _args(**values: object) -> SimpleNamespace: + base = {"llm_provider": None, "llm_model": None, "llm_endpoint": None} + return SimpleNamespace(**{**base, **values}) + + +# ── 1. key variables for two or more providers stop the run ───────────────── + + +_TWO_KEYS = {"OPENAI_API_KEY": OPENAI_KEY, "ANTHROPIC_API_KEY": ANTHROPIC_KEY} + + +def test_two_keys_never_fall_through_to_a_running_ollama(world: _World) -> None: + world.ollama_up = True + env = world.env(**_TWO_KEYS) + + assert providers._provider_name_from(_args(), env) is None + with pytest.raises(providers.CopilotGenerationError) as info: + providers.resolve_llm_config(_args(), env) + assert info.value.event == "copilot_no_llm_provider" + assert "OPENAI_API_KEY, ANTHROPIC_API_KEY" in info.value.message + assert world.ollama_probes == 0 # not even asked + + +def test_doctor_reports_the_same(world: _World) -> None: + world.ollama_up = True + env = world.env(**_TWO_KEYS) + + readiness = providers.check_llm_readiness(env) + + assert readiness.ready is False + assert readiness.provider is None + assert readiness.error == providers.no_provider_message(env) + assert "OPENAI_API_KEY, ANTHROPIC_API_KEY" in readiness.error + + +def test_ollama_host_and_a_cloud_key_are_two_providers(world: _World) -> None: + world.ollama_up = True + env = world.env(OPENAI_API_KEY=OPENAI_KEY, OLLAMA_HOST="http://127.0.0.1:11500") + + assert providers._provider_name_from(_args(), env) is None + assert "OPENAI_API_KEY, OLLAMA_HOST" in providers.check_llm_readiness(env).error + + +def test_gemini_s_two_variables_are_one_provider(world: _World) -> None: + world.ollama_up = True + env = world.env(GEMINI_API_KEY=GEMINI_KEY, GOOGLE_API_KEY=GEMINI_KEY) + + assert providers._provider_name_from(_args(), env) == "gemini" + + +def test_the_saved_choice_still_breaks_the_tie(world: _World) -> None: + world.ollama_up = True + world.saved = {"provider": "anthropic", "model": "claude-saved-test"} + env = world.env(**_TWO_KEYS) + + config = providers.resolve_llm_config(_args(), env) + + assert (config.provider, config.model, config.api_key) == ( + "anthropic", + "claude-saved-test", + ANTHROPIC_KEY, + ) + assert providers.check_llm_readiness(env).provider == "anthropic" + + +def test_one_keyring_key_still_comes_before_the_stop(world: _World) -> None: + world.ollama_up = True + world.keyring["gemini"] = GEMINI_KEY + env = world.env(**_TWO_KEYS) + + assert providers._provider_name_from(_args(), env) == "gemini" + assert providers.check_llm_readiness(env).provider == "gemini" + + +def test_two_keyring_keys_still_reach_a_running_ollama(world: _World) -> None: + # Only key variables stop the ambient rung; keyring keys are not per-run signals. + world.ollama_up = True + world.keyring.update(openai=OPENAI_KEY, anthropic=ANTHROPIC_KEY) + + assert providers._provider_name_from(_args(), world.env()) == "ollama" + + +def test_a_named_provider_is_unaffected(world: _World) -> None: + world.ollama_up = True + + config = providers.resolve_llm_config(_args(llm_provider="anthropic"), world.env(**_TWO_KEYS)) + + assert (config.provider, config.api_key) == ("anthropic", ANTHROPIC_KEY) + + +# ── 2. inline setup and fluid ai test read the ladder's key variables ─────── + + +def test_setup_does_not_pick_github_from_its_key(world: _World) -> None: + # It used to pick github here, then raise KeyError: 'github' building it. + world.monkeypatch.setenv("GITHUB_API_KEY", GITHUB_KEY) + + assert ai_setup.run_ai_setup_inline(None) is None + + +def test_setup_still_reads_one_provider_s_key(world: _World) -> None: + world.monkeypatch.setenv("GITHUB_API_KEY", GITHUB_KEY) + world.monkeypatch.setenv("ANTHROPIC_API_KEY", ANTHROPIC_KEY) + + config = ai_setup.run_ai_setup_inline(None) + + assert (config.provider, config.api_key) == ("anthropic", ANTHROPIC_KEY) + + +def test_setup_reads_the_gemini_alias_like_the_resolver(world: _World) -> None: + world.monkeypatch.setenv("GEMINI_API_KEY", GEMINI_KEY) + + config = ai_setup.run_ai_setup_inline(None) + + assert (config.provider, config.api_key) == ("gemini", GEMINI_KEY) + + +def test_init_does_not_count_github_s_key_as_ai_configured(world: _World) -> None: + from fluid_build.cli.init import _is_ai_configured + + world.monkeypatch.setenv("GITHUB_API_KEY", GITHUB_KEY) + + assert _is_ai_configured() is False + + +def test_interactive_setup_names_two_keys_and_picks_neither(world: _World) -> None: + from rich.console import Console + + for name, value in _TWO_KEYS.items(): + world.monkeypatch.setenv(name, value) + out = io.StringIO() + + assert ai_setup.run_ai_setup_inline(Console(file=out, width=250)) is None + assert "OPENAI_API_KEY, ANTHROPIC_API_KEY are all set" in out.getvalue() + + +def test_setup_lets_one_keyring_key_break_the_tie_as_the_ladder_does(world: _World) -> None: + # Without a TTY the one keyring key selects gemini; setup must agree. + for name, value in _TWO_KEYS.items(): + world.monkeypatch.setenv(name, value) + world.keyring["gemini"] = GEMINI_KEY + + config = ai_setup.run_ai_setup_inline(None) + + assert (config.provider, config.api_key) == ("gemini", GEMINI_KEY) + assert providers.check_llm_readiness(dict(os.environ)).provider == "gemini" + + +def test_init_s_check_agrees_with_setup_on_two_keys(world: _World) -> None: + from fluid_build.cli.init import _is_ai_configured + + for name, value in _TWO_KEYS.items(): + world.monkeypatch.setenv(name, value) + + assert _is_ai_configured() is False # setup would stop + world.keyring["gemini"] = GEMINI_KEY + assert _is_ai_configured() is True # setup would use gemini + + +def _ai_test(**flags: object) -> Tuple[Optional[providers.LlmConfig], Optional[str]]: + values = { + "llm_provider": None, + "llm_model": None, + "llm_endpoint": None, + "llm_timeout_seconds": None, + } + values.update(flags) + return ai_setup._resolve_ai_test_config(Namespace(**values)) + + +def test_ai_test_does_not_pick_github_from_its_key(world: _World) -> None: + world.monkeypatch.setenv("GITHUB_API_KEY", GITHUB_KEY) + + config, error = _ai_test() + + assert config is None + assert "github" not in error.lower() + + +def test_ai_test_names_two_keys_and_tests_neither(world: _World) -> None: + for name, value in _TWO_KEYS.items(): + world.monkeypatch.setenv(name, value) + + config, error = _ai_test() + + assert config is None + assert "OPENAI_API_KEY, ANTHROPIC_API_KEY are all set" in error + + +def test_ai_test_still_tests_the_saved_choice_over_two_keys(world: _World) -> None: + world.saved = {"provider": "anthropic", "model": "claude-saved-test"} + for name, value in _TWO_KEYS.items(): + world.monkeypatch.setenv(name, value) + + config, error = _ai_test() + + assert error is None + assert config.provider == "anthropic" + + +def test_ai_test_lets_one_keyring_key_break_the_tie(world: _World) -> None: + for name, value in _TWO_KEYS.items(): + world.monkeypatch.setenv(name, value) + world.keyring["gemini"] = GEMINI_KEY + + config, error = _ai_test() + + assert error is None + assert (config.provider, config.api_key) == ("gemini", GEMINI_KEY) + + +# ── 3. interactive forge and init: what the operator states wins ──────────── + + +def _openai_setup_pick() -> providers.LlmConfig: + """What inline setup returns for a saved OpenAI choice with its keyring key.""" + return providers.LlmConfig( + provider="openai", model="gpt-saved-test", endpoint="", api_key=OPENAI_KEY + ) + + +def _claude_code_setup_pick() -> providers.LlmConfig: + return providers.LlmConfig( + provider="claude-code", + model="claude-code", + endpoint="coding-agent://claude-code", + api_key=None, + ) + + +def _interactive_forge( + world: _World, tmp_path: Path, pick: providers.LlmConfig, **flags: object +) -> Tuple[Optional[Namespace], List[str], List[str]]: + """Run interactive ``fluid forge`` in AI mode with setup returning *pick*. + + Returns the args the copilot ran with, setup's calls and the providers + whose key setup's pick put into the session. + """ + from fluid_build.cli.forge import run as forge_run + + world.tty(True) + world.monkeypatch.setattr("fluid_build.cli._welcome_scan.Path.home", lambda: tmp_path / "home") + world.monkeypatch.setenv("FLUID_FORGE_NO_PREVIEW", "1") + seen: Dict[str, Namespace] = {} + setup_calls: List[str] = [] + session: List[str] = [] + + def _copilot(args: Namespace, _logger: logging.Logger) -> int: + seen["args"] = args + return 0 + + def _setup(_console: object) -> providers.LlmConfig: + setup_calls.append("setup") + return pick + + values = { + "command": "forge", + "forge_subcommand": None, + "blank": False, + "target_dir": str(tmp_path / "product"), + "non_interactive": False, + "dry_run": False, + "llm_provider": None, + "llm_model": None, + "llm_endpoint": None, + "agent_loop": False, + "refine": None, + "from_product": [], + "from_product_list": None, + "template": None, + "scaffold": None, + } + values.update(flags) + with ( + mock.patch("fluid_build.cli._forge_mode_picker.pick_mode", lambda a, **k: "ai"), + mock.patch("fluid_build.cli._forge_mode_picker.should_show_picker", lambda a: True), + mock.patch("fluid_build.cli.forge.run_ai_copilot_mode", _copilot), + mock.patch("fluid_build.cli.ai_setup.run_ai_setup_inline", _setup), + mock.patch( + "fluid_build.cli.ai_setup.set_session_env", + lambda provider, key: session.append(provider), + ), + ): + forge_run(Namespace(**values), logging.getLogger("test")) + return seen.get("args"), setup_calls, session + + +def test_a_named_provider_flag_is_not_replaced_by_setup_s_pick( + world: _World, tmp_path: Path +) -> None: + ran, setup_calls, session = _interactive_forge( + world, tmp_path, _openai_setup_pick(), llm_provider="anthropic" + ) + + assert setup_calls == [] # setup cannot know the name, so it is not asked + assert ran.llm_provider == "anthropic" + assert ran.llm_model is None # the named provider's catalog model, as without a TTY + assert session == [] + + +@pytest.mark.parametrize( + "var,value", [("FLUID_LLM_PROVIDER", "anthropic"), ("FLUID_FORGE_AGENT", "codex")] +) +def test_a_provider_named_by_a_variable_skips_setup( + world: _World, tmp_path: Path, var: str, value: str +) -> None: + world.monkeypatch.setenv(var, value) + + ran, setup_calls, _ = _interactive_forge(world, tmp_path, _openai_setup_pick()) + + assert setup_calls == [] + assert ran.llm_provider is None + assert providers._provider_name_from(ran, os.environ) == value + + +def test_setup_keeps_a_stated_model(world: _World, tmp_path: Path) -> None: + ran, setup_calls, session = _interactive_forge( + world, tmp_path, _openai_setup_pick(), llm_model="gpt-stated-test" + ) + + assert setup_calls == ["setup"] + assert ran.llm_provider == "openai" # nothing named one, so setup's pick is used + assert ran.llm_model == "gpt-stated-test" + assert session == ["openai"] + + +def test_setup_keeps_a_stated_endpoint(world: _World, tmp_path: Path) -> None: + ran, _, _ = _interactive_forge(world, tmp_path, _openai_setup_pick(), llm_endpoint=ONPREM) + + assert ran.llm_provider == "openai" + assert ran.llm_endpoint == ONPREM + + +def test_setup_keeps_the_model_and_endpoint_variables(world: _World, tmp_path: Path) -> None: + world.monkeypatch.setenv("FLUID_LLM_MODEL", "gpt-env-test") + world.monkeypatch.setenv("FLUID_LLM_ENDPOINT", ONPREM) + + ran, _, _ = _interactive_forge(world, tmp_path, _openai_setup_pick()) + config = providers.resolve_llm_config( + ran, + world.env( + OPENAI_API_KEY=OPENAI_KEY, FLUID_LLM_MODEL="gpt-env-test", FLUID_LLM_ENDPOINT=ONPREM + ), + ) + + assert (ran.llm_model, ran.llm_endpoint) == (None, None) + assert config.provider == "openai" + assert config.model == "gpt-env-test" + assert config.endpoint == ONPREM + + +def test_setup_fills_what_the_run_leaves_unstated(world: _World, tmp_path: Path) -> None: + pick = providers.LlmConfig( + provider="openai", + model="gpt-saved-test", + endpoint="https://saved.example.test/v1", + api_key=OPENAI_KEY, + ) + + ran, _, session = _interactive_forge(world, tmp_path, pick) + + assert (ran.llm_provider, ran.llm_model, ran.llm_endpoint) == ( + "openai", + "gpt-saved-test", + "https://saved.example.test/v1", + ) + assert ran.llm_provider_named is None # setup's pick never counts as named + assert session == ["openai"] + + +def test_setup_s_agent_never_takes_a_stated_endpoint(world: _World, tmp_path: Path) -> None: + # An agent cannot call the endpoint, so it is not used, as without a TTY. + world.saved = {"provider": "claude-code"} + + ran, setup_calls, _ = _interactive_forge( + world, tmp_path, _claude_code_setup_pick(), llm_endpoint=ONPREM + ) + + assert setup_calls == ["setup"] + assert ran.llm_provider is None + assert ran.llm_endpoint == ONPREM + with pytest.raises(providers.CopilotGenerationError) as info: + providers.resolve_llm_config(ran, world.env()) + assert info.value.event == "copilot_no_llm_provider" + assert "saved claude-code" in info.value.message + + +def test_setup_s_agent_is_used_for_test_double_args(world: _World) -> None: + # A MagicMock invents llm_endpoint, llm_model and agent_loop; none is stated. + args = mock.MagicMock() + + assert providers.fill_llm_args_from_setup(args, _claude_code_setup_pick(), world.env()) + assert args.llm_provider == "claude-code" + + +def test_named_llm_provider_reads_the_flag_then_the_variables(world: _World) -> None: + env = world.env(FLUID_LLM_PROVIDER="anthropic", FLUID_FORGE_AGENT="codex") + + assert providers.named_llm_provider(_args(llm_provider="openai"), env) == "openai" + assert providers.named_llm_provider(_args(), env) == "anthropic" + assert providers.named_llm_provider(_args(), world.env(FLUID_FORGE_AGENT="codex")) == "codex" + assert providers.named_llm_provider(mock.MagicMock(), world.env()) is None + + +def _init_ai_mode( + world: _World, tmp_path: Path, pick: providers.LlmConfig +) -> Tuple[Optional[Namespace], List[str]]: + """``fluid init``'s AI path with setup returning *pick*; the copilot is stubbed.""" + from fluid_build.cli import init as init_mod + + world.monkeypatch.chdir(tmp_path) + seen: Dict[str, Namespace] = {} + setup_calls: List[str] = [] + + def _copilot(args: Namespace, _logger: logging.Logger) -> int: + seen["args"] = args + return 1 # no success panel + + def _setup(_console: object) -> providers.LlmConfig: + setup_calls.append("setup") + return pick + + with ( + mock.patch("fluid_build.cli.ai_setup.run_ai_setup_inline", _setup), + mock.patch("fluid_build.cli.ai_setup.set_session_env", lambda provider, key: None), + mock.patch("fluid_build.cli.forge.run_ai_copilot_mode", _copilot), + ): + init_mod._ai_mode(Namespace(name="demo-product", yes=True), logging.getLogger("test")) + return seen.get("args"), setup_calls + + +def test_init_keeps_a_provider_named_by_a_variable(world: _World, tmp_path: Path) -> None: + world.monkeypatch.setenv("FLUID_LLM_PROVIDER", "anthropic") + + ran, setup_calls = _init_ai_mode(world, tmp_path, _openai_setup_pick()) + + assert setup_calls == [] + assert getattr(ran, "llm_provider", None) is None + assert providers._provider_name_from(ran, os.environ) == "anthropic" + + +def test_init_keeps_the_model_variable(world: _World, tmp_path: Path) -> None: + world.monkeypatch.setenv("FLUID_LLM_MODEL", "gpt-env-test") + + ran, setup_calls = _init_ai_mode(world, tmp_path, _openai_setup_pick()) + + assert setup_calls == ["setup"] + assert ran.llm_provider == "openai" + assert getattr(ran, "llm_model", None) is None # FLUID_LLM_MODEL then applies + + +def test_init_counts_a_named_provider_as_ai_configured(world: _World) -> None: + # Forge runs no setup for it, so init offers none either. + from fluid_build.cli.init import _is_ai_configured + + world.monkeypatch.setenv("FLUID_LLM_PROVIDER", "anthropic") + + assert _is_ai_configured() is True + + +def test_the_interview_panel_names_the_run_s_provider( + world: _World, tmp_path: Path, deterministic_welcome_scan: None +) -> None: + # The scan reads the saved choice; the run was named anthropic. + from fluid_build.cli._world_class_interview import collect_signals + + world.monkeypatch.setattr("fluid_build.cli._welcome_scan.Path.home", lambda: tmp_path / "home") + world.saved = {"provider": "openai", "model": "gpt-saved-test"} + + assert collect_signals(target_dir=tmp_path).ai_provider == "openai" + sig = collect_signals(target_dir=tmp_path, ai_provider="anthropic") + + assert (sig.ai_configured, sig.ai_provider) == (True, "anthropic") + + +def test_the_welcome_panel_names_the_run_s_provider(world: _World) -> None: + from fluid_build.cli._welcome_scan import WelcomeFindings + from fluid_build.cli.forge_modes import _with_run_provider + + def _saved_openai() -> WelcomeFindings: + return WelcomeFindings(ai_configured=True, ai_provider_hint="openai") + + named = _with_run_provider(_saved_openai(), Namespace(llm_provider="anthropic")) + unnamed = _with_run_provider(_saved_openai(), Namespace(llm_provider=None)) + + assert named.ai_provider_hint == "anthropic" + assert unnamed.ai_provider_hint == "openai" + + +# ── 4. fluid ai test honours the provider the run names ───────────────────── + + +def test_ai_test_honours_fluid_llm_provider(world: _World) -> None: + world.saved = {"provider": "openai", "model": "gpt-saved-test"} + world.keyring["openai"] = OPENAI_KEY + world.monkeypatch.setenv("FLUID_LLM_PROVIDER", "anthropic") + world.monkeypatch.setenv("ANTHROPIC_API_KEY", ANTHROPIC_KEY) + + config, error = _ai_test() + + assert error is None + assert (config.provider, config.api_key) == ("anthropic", ANTHROPIC_KEY) + + +def test_ai_test_gives_a_provider_named_by_a_variable_the_generic_key(world: _World) -> None: + world.monkeypatch.setenv("FLUID_LLM_PROVIDER", "openai") + world.monkeypatch.setenv("FLUID_LLM_API_KEY", GENERIC_KEY) + + config, error = _ai_test() + + assert error is None + assert (config.provider, config.api_key) == ("openai", GENERIC_KEY) + + +def test_ai_test_honours_fluid_forge_agent(world: _World) -> None: + # It tested the saved provider, which this run would not use. + world.saved = {"provider": "openai", "model": "gpt-saved-test"} + world.keyring["openai"] = OPENAI_KEY + world.monkeypatch.setenv("FLUID_FORGE_AGENT", "codex") + + config, error = _ai_test() + + assert config is None + assert "codex" in error + + +def test_ai_test_flag_beats_the_variable(world: _World) -> None: + world.monkeypatch.setenv("FLUID_LLM_PROVIDER", "anthropic") + world.monkeypatch.setenv("OPENAI_API_KEY", OPENAI_KEY) + + config, error = _ai_test(llm_provider="openai") + + assert error is None + assert config.provider == "openai" + + +@pytest.mark.parametrize("value", ["claude-code", "codex"]) +def test_ai_test_says_it_does_not_test_a_named_agent(world: _World, value: str) -> None: + world.monkeypatch.setenv("FLUID_FORGE_AGENT", value) + + config, error = _ai_test() + + assert config is None + assert f"does not test {value}" in error + + +# ── 5. review follow-up: the helpers load in a fresh interpreter ─────────── + + +_FRESH_CALLS = ( + "from fluid_build.cli.init import _is_ai_configured; print(_is_ai_configured())", + "from argparse import Namespace; from fluid_build.cli.ai_setup import _resolve_ai_test_config; " + "print(_resolve_ai_test_config(Namespace(llm_provider=None, llm_model=None, " + "llm_endpoint=None, llm_timeout_seconds=None))[1])", + "from fluid_build.cli.ai_setup import run_ai_setup_inline; print(run_ai_setup_inline(None))", +) + + +@pytest.mark.parametrize("call", _FRESH_CALLS, ids=["init", "ai-test", "setup"]) +def test_the_helpers_work_in_a_fresh_interpreter(tmp_path: Path, call: str) -> None: + # ``provider_resolve`` and ``providers`` import each other, so importing + # ``provider_resolve`` first fails. ``fluid init`` loads neither before its + # post-init check, which then raised and lost the "first data product" offer. + import subprocess + + env = { + "PATH": "/usr/bin:/bin", + "HOME": str(tmp_path), + "PYTHONPATH": str(Path(__file__).resolve().parents[1]), + "PYTHON_KEYRING_BACKEND": "keyring.backends.null.Keyring", + # Two providers' keys: every call stops before it could probe an Ollama. + "OPENAI_API_KEY": OPENAI_KEY, + "ANTHROPIC_API_KEY": ANTHROPIC_KEY, + "HTTP_PROXY": "http://127.0.0.1:9", + "HTTPS_PROXY": "http://127.0.0.1:9", + } + result = subprocess.run( + [sys.executable, "-c", call], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + timeout=120, + ) + + assert result.returncode == 0, result.stderr[-2000:] + + +@pytest.mark.parametrize("spelling", ["MCP-Sampling", "Mcp_Sampling", "Claude-Code", "claude_code"]) +def test_ai_test_knows_an_agent_in_any_spelling(world: _World, spelling: str) -> None: + world.monkeypatch.setenv("FLUID_LLM_PROVIDER", spelling) + + config, error = _ai_test() + + assert config is None + assert "does not test" in error + assert spelling.lower().replace("_", "-") in error + + +def _init_ai_mode_without_setup(world: _World, tmp_path: Path) -> Tuple[int, List[str]]: + """``fluid init``'s AI path where setup found nothing; returns (rc, templates written).""" + from fluid_build.cli import init as init_mod + + world.monkeypatch.chdir(tmp_path) + templates: List[str] = [] + + def _template(args: Namespace, _logger: logging.Logger) -> int: + templates.append(args.template) + return 0 + + with ( + mock.patch("fluid_build.cli.ai_setup.run_ai_setup_inline", lambda console: None), + mock.patch.object(init_mod, "template_mode", _template), + ): + rc = init_mod._ai_mode(Namespace(name="demo-product", yes=True), logging.getLogger("t")) + return rc, templates + + +def test_init_stops_when_setup_named_two_keys(world: _World, tmp_path: Path) -> None: + # Setup printed the keys and chose none; init must not write a template instead. + for name, value in _TWO_KEYS.items(): + world.monkeypatch.setenv(name, value) + + assert _init_ai_mode_without_setup(world, tmp_path) == (1, []) + + +def test_init_still_uses_a_template_when_nothing_is_configured( + world: _World, tmp_path: Path +) -> None: + assert _init_ai_mode_without_setup(world, tmp_path) == (0, ["customer-360"]) + + +def test_ai_test_says_the_ide_answers_mcp_sampling(world: _World) -> None: + world.monkeypatch.setenv("FLUID_LLM_PROVIDER", "MCP-Sampling") + + config, error = _ai_test() + + assert config is None + assert "does not test mcp-sampling" in error and "IDE" in error and "doctor" not in error