Stability fixes: provider routing, license grace, exports, scanner and regression tests

.github/workflows/python-tests.yml

@@ -0,0 +1,27 @@
+name: Python tests
+
+on:
+  push:
+    branches: ["main", "fix/**"]
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: dataforge-cli
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: pip
+          cache-dependency-path: dataforge-cli/requirements.txt
+      - name: Install dependencies
+        run: python -m pip install -r requirements.txt
+      - name: Compile Python files
+        run: python -m compileall -q .
+      - name: Run regression tests
+        run: python -m unittest discover -s tests -v

dataforge-cli/dataforge/api_handler.py

@@ -1,56 +1,116 @@
-import os
-import json
-import urllib.request
-from groq import Groq
-from dataforge.config_manager import get_config
-
-def ask_enterprise_ai(prompt):
-    """Enruta el prompt al proveedor de IA configurado (Groq, OpenAI, Gemini, Claude u Ollama local)"""
-    config = get_config()
-    if not config:
-        config = {}
-
-    # Soporta tanto 'ai_provider' como 'provider' según la versión del diccionario de configuración
-    provider = config.get("ai_provider", config.get("provider", "groq")).lower()
-
-    # 1. Proveedor Ollama (100% Offline / Local)
-    if provider == "ollama":
-        url = "http://localhost:11434/api/generate"
-        payload = {
-            "model": "llama3",  # O el modelo local que prefiera el usuario
-            "prompt": prompt,
-            "stream": False
-        }
-        data = json.dumps(payload).encode("utf-8")
-        req = urllib.request.Request(
-            url, 
-            data=data, 
-            headers={"Content-Type": "application/json"}
-        )
-        try:
-            with urllib.request.urlopen(req, timeout=60) as response:
-                result = json.loads(response.read().decode("utf-8"))
-                return result.get("response", "Error: Respuesta vacía de Ollama.")
-        except Exception as e:
-            return f"⚠️ Error de conexión con Ollama local (Verifica que esté abierto y el modelo instalado): {e}"
-
-    # 2. Proveedor Groq (Nube Ultra Rápida)
-    elif provider == "groq":
-        # Revisa tanto 'groq_api_key' como 'api_key'
-        api_key = config.get("groq_api_key", config.get("api_key"))
-        if not api_key:
-            raise ValueError("Falta la API Key de Groq. Ejecuta 'python main.py setup'")
-        client = Groq(api_key=api_key)
-        chat_completion = client.chat.completions.create(
-            messages=[{"role": "user", "content": prompt}],
-            model="llama-3.3-70b-versatile",
-        )
-        return chat_completion.choices[0].message.content
-
-    # 3. Otros proveedores en nube (OpenAI, Gemini, Claude)
-    else:
-        raise ValueError(f"Proveedor de IA '{provider}' no soportado o pendiente de credenciales.")
-
-def ask_groq(prompt, api_key=None):
-    """Puente de compatibilidad para llamadas antiguas a ask_groq"""
-    return ask_enterprise_ai(prompt)
+import json
+import urllib.error
+import urllib.request
+import urllib.parse
+
+from groq import Groq
+from dataforge.config_manager import get_config
+
+
+def _post_json(url, payload, headers=None, timeout=60):
+    request_headers = {"Content-Type": "application/json"}
+    request_headers.update(headers or {})
+    request = urllib.request.Request(
+        url,
+        data=json.dumps(payload).encode("utf-8"),
+        headers=request_headers,
+        method="POST",
+    )
+    with urllib.request.urlopen(request, timeout=timeout) as response:
+        return json.loads(response.read().decode("utf-8"))
+
+
+def ask_enterprise_ai(prompt):
+    """Route a prompt to the provider selected in ~/.dataforge/config.json."""
+    config = get_config() or {}
+    provider = str(config.get("ai_provider", config.get("provider", "groq"))).strip().lower()
+    api_key = config.get(f"{provider}_api_key") or config.get("api_key")
+
+    if provider == "ollama":
+        model = config.get("ollama_model", "llama3")
+        try:
+            result = _post_json(
+                config.get("ollama_url", "http://localhost:11434/api/generate"),
+                {"model": model, "prompt": prompt, "stream": False},
+                timeout=60,
+            )
+        except Exception as exc:
+            raise RuntimeError(
+                "Could not connect to Ollama. Start Ollama and verify the configured model."
+            ) from exc
+        answer = result.get("response")
+        if not answer:
+            raise RuntimeError("Ollama returned an empty response.")
+        return answer
+
+    if not api_key:
+        raise ValueError(
+            f"Missing API key for {provider}. Run 'python main.py setup' and configure credentials."
+        )
+
+    if provider == "groq":
+        client = Groq(api_key=api_key)
+        completion = client.chat.completions.create(
+            messages=[{"role": "user", "content": prompt}],
+            model=config.get("groq_model", "llama-3.3-70b-versatile"),
+        )
+        return completion.choices[0].message.content
+
+    if provider == "openai":
+        result = _post_json(
+            "https://api.openai.com/v1/chat/completions",
+            {
+                "model": config.get("openai_model", "gpt-4o-mini"),
+                "messages": [{"role": "user", "content": prompt}],
+            },
+            {"Authorization": f"Bearer {api_key}"},
+        )
+        try:
+            return result["choices"][0]["message"]["content"]
+        except (KeyError, IndexError, TypeError) as exc:
+            raise RuntimeError("OpenAI returned an unexpected response format.") from exc
+
+    if provider == "gemini":
+        url = (
+            "https://generativelanguage.googleapis.com/v1beta/models/"
+            + config.get("gemini_model", "gemini-2.5-flash")
+            + ":generateContent?key="
+            + urllib.parse.quote(api_key, safe="")
+        )
+        result = _post_json(
+            url,
+            {"contents": [{"parts": [{"text": prompt}]}]},
+        )
+        try:
+            return "".join(part["text"] for part in result["candidates"][0]["content"]["parts"])
+        except (KeyError, IndexError, TypeError) as exc:
+            raise RuntimeError("Gemini returned an unexpected response format.") from exc
+
+    if provider == "claude":
+        result = _post_json(
+            "https://api.anthropic.com/v1/messages",
+            {
+                "model": config.get("claude_model", "claude-3-5-haiku-latest"),
+                "max_tokens": 4096,
+                "messages": [{"role": "user", "content": prompt}],
+            },
+            {
+                "x-api-key": api_key,
+                "anthropic-version": "2023-06-01",
+            },
+        )
+        try:
+            return "".join(
+                block["text"] for block in result["content"] if block.get("type") == "text"
+            )
+        except (KeyError, TypeError) as exc:
+            raise RuntimeError("Claude returned an unexpected response format.") from exc
+
+    raise ValueError(
+        f"Unsupported AI provider '{provider}'. Supported providers: groq, openai, gemini, claude, ollama."
+    )
+
+
+def ask_groq(prompt, api_key=None):
+    """Backward-compatible alias; routing follows the configured provider."""
+    return ask_enterprise_ai(prompt)

dataforge-cli/dataforge/core.py

@@ -1,69 +1,110 @@
-import os
-from datetime import datetime
-
-# Extensiones que nos interesan
-VALID_EXTENSIONS = {'.py', '.js', '.ts', '.txt', '.md', '.html', '.css', '.json', '.go', '.rs', '.java', '.cpp'}
-
-# Directorios excluidos por seguridad y rendimiento
-EXCLUDED_DIRS = {'.git', '__pycache__', 'node_modules', '.venv', 'venv', 'dataforge-reports', 'dist', 'build', '.idea', '.vscode', 'target', 'bin', 'obj', '.dataforge'}
-
-def get_all_files(root_path):
-    files_to_process = []
-    for root, dirs, files in os.walk(root_path):
-        # Ignorar carpetas conflictivas de forma segura in-place
-        dirs[:] = [d for d in dirs if d not in EXCLUDED_DIRS]
-        for file in files:
-            ext = os.path.splitext(file)[1].lower()
-            if ext in VALID_EXTENSIONS:
-                full_path = os.path.join(root, file)
-                files_to_process.append(full_path)
-    return files_to_process
-
-def read_file_content(file_path):
-    try:
-        # Usamos errors='ignore' para evitar bloqueos por codificación de caracteres raros
-        with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
-            return f.read()
-    except Exception as e:
-        return f"Error al leer {file_path}: {e}"
-
-def get_single_file_content(file_path):
-    if not os.path.exists(file_path):
-        return None
-    return read_file_content(file_path)
-
-def prepare_code_payload(files_list, max_chars=35000):
-    """Combina el contenido de los archivos y aplica un recorte estricto para evitar el Error 413 de tokens."""
-    project_data = []
-    total_chars = 0
-
-    for file_path in files_list:
-        content = read_file_content(file_path)
-        file_header = f"--- FILE: {file_path} ---\n"
-        file_block = f"{file_header}{content}\n\n"
-
-        # Si agregar este archivo rebasa el límite seguro de caracteres, recortamos o detenemos
-        if total_chars + len(file_block) > max_chars:
-            remaining_chars = max_chars - total_chars
-            if remaining_chars > 100:
-                project_data.append(file_block[:remaining_chars] + "\n[... Archivo recortado por límite de presupuesto de tokens ...]")
-            project_data.append("\n⚠️ [Aviso del Sistema]: El proyecto excede el presupuesto seguro de tokens. Se omitieron algunos archivos restantes.")
-            break
-
-        project_data.append(file_block)
-        total_chars += len(file_block)
-
-    return "".join(project_data)
-
-def save_report(content, target_path):
-    report_dir = os.path.join(target_path, "dataforge-reports")
-    if not os.path.exists(report_dir):
-        os.makedirs(report_dir)
-
-    timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
-    filename = f"reporte_{timestamp}.txt"
-    file_path = os.path.join(report_dir, filename)
-
-    with open(file_path, 'w', encoding='utf-8') as f:
-        f.write(content)
+import os
+import fnmatch
+from datetime import datetime
+
+# Extensiones que nos interesan
+VALID_EXTENSIONS = {'.py', '.js', '.ts', '.txt', '.md', '.html', '.css', '.json', '.go', '.rs', '.java', '.cpp'}
+
+# Directorios excluidos por seguridad y rendimiento
+EXCLUDED_DIRS = {'.git', '__pycache__', 'node_modules', '.venv', 'venv', 'dataforge-reports', 'dist', 'build', '.idea', '.vscode', 'target', 'bin', 'obj', '.dataforge'}
+EXCLUDED_FILES = {'.env', '.env.local', '.env.production', 'id_rsa', 'id_ed25519', 'credentials.json', 'secrets.json'}
+MAX_FILE_BYTES = 1_000_000
+
+
+def _load_gitignore_patterns(root_path):
+    """Read simple .gitignore patterns; unsupported negation patterns are ignored safely."""
+    patterns = []
+    ignore_path = os.path.join(root_path, ".gitignore")
+    try:
+        with open(ignore_path, "r", encoding="utf-8", errors="ignore") as handle:
+            for raw in handle:
+                pattern = raw.strip()
+                if pattern and not pattern.startswith("#") and not pattern.startswith("!"):
+                    patterns.append(pattern.rstrip("/"))
+    except OSError:
+        pass
+    return patterns
+
+
+def _is_ignored(relative_path, patterns):
+    normalized = relative_path.replace(os.sep, "/")
+    parts = normalized.split("/")
+    for pattern in patterns:
+        if "/" in pattern:
+            if fnmatch.fnmatch(normalized, pattern) or normalized == pattern:
+                return True
+        elif any(fnmatch.fnmatch(part, pattern) for part in parts):
+            return True
+    return False
+
+def get_all_files(root_path):
+    root_path = os.path.abspath(root_path)
+    patterns = _load_gitignore_patterns(root_path)
+    files_to_process = []
+    for root, dirs, files in os.walk(root_path):
+        dirs[:] = [
+            d for d in dirs
+            if d not in EXCLUDED_DIRS
+            and not _is_ignored(os.path.relpath(os.path.join(root, d), root_path), patterns)
+        ]
+        for file in files:
+            if file.lower() in EXCLUDED_FILES or file.lower().startswith(".env."):
+                continue
+            relative_path = os.path.relpath(os.path.join(root, file), root_path)
+            if _is_ignored(relative_path, patterns):
+                continue
+            ext = os.path.splitext(file)[1].lower()
+            if ext in VALID_EXTENSIONS:
+                files_to_process.append(os.path.join(root, file))
+    return files_to_process
+
+def read_file_content(file_path, max_bytes=MAX_FILE_BYTES):
+    try:
+        with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
+            content = f.read(max_bytes + 1)
+        if len(content) > max_bytes:
+            return content[:max_bytes] + "\\n[DataForge: file truncated at configured safety limit.]"
+        return content
+    except Exception as e:
+        return f"Error al leer {file_path}: {e}"
+
+def get_single_file_content(file_path):
+    if not os.path.exists(file_path):
+        return None
+    return read_file_content(file_path)
+
+def prepare_code_payload(files_list, max_chars=35000):
+    """Combina el contenido de los archivos y aplica un recorte estricto para evitar el Error 413 de tokens."""
+    project_data = []
+    total_chars = 0
+
+    for file_path in files_list:
+        content = read_file_content(file_path)
+        file_header = f"--- FILE: {file_path} ---\n"
+        file_block = f"{file_header}{content}\n\n"
+
+        # Si agregar este archivo rebasa el límite seguro de caracteres, recortamos o detenemos
+        if total_chars + len(file_block) > max_chars:
+            remaining_chars = max_chars - total_chars
+            if remaining_chars > 100:
+                project_data.append(file_block[:remaining_chars] + "\n[... Archivo recortado por límite de presupuesto de tokens ...]")
+            project_data.append("\n⚠️ [Aviso del Sistema]: El proyecto excede el presupuesto seguro de tokens. Se omitieron algunos archivos restantes.")
+            break
+
+        project_data.append(file_block)
+        total_chars += len(file_block)
+
+    return "".join(project_data)
+
+def save_report(content, target_path):
+    report_dir = os.path.join(target_path, "dataforge-reports")
+    if not os.path.exists(report_dir):
+        os.makedirs(report_dir)
+
+    timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+    filename = f"reporte_{timestamp}.txt"
+    file_path = os.path.join(report_dir, filename)
+
+    with open(file_path, 'w', encoding='utf-8') as f:
+        f.write(content)
     return file_path