diff --git a/README.md b/README.md index 8683bcce3..03d9ffd04 100644 --- a/README.md +++ b/README.md @@ -301,7 +301,7 @@ ARTEMIS supports two execution profiles tailored for different automation requir - [ ] **Android Studio Integration**: Native IDE plugin and workflow integration to enable in-editor debugging, test recording, and automated device control directly within Android Studio. - [ ] **iOS Platform Expansion**: Extending multimodal perception and mobile automation to iOS devices and simulators. -- [ ] **On-Device Lightweight VLMs**: Local execution with lightweight edge vision models for low-latency, privacy-first automation. +- [x] **On-Device Lightweight VLMs**: Local execution with lightweight edge vision models for low-latency, privacy-first automation via OpenAI-compatible local providers — see the [local models guide](./docs/local_models.md) for install/pull/run steps (`config/artemis.gemma4.jsonc` is the tested Gemma 4 E4B setup). - [ ] **Real-time Duplex Voice Interaction**: Voice-driven task dispatch with real-time conversational control and interruption handling. ## Community & Contributing diff --git a/apps/admin_console/routers/system.py b/apps/admin_console/routers/system.py index c8ce18343..be27dbeef 100644 --- a/apps/admin_console/routers/system.py +++ b/apps/admin_console/routers/system.py @@ -16,6 +16,7 @@ import ipaddress import os +import re import secrets from urllib.parse import urlsplit @@ -624,3 +625,188 @@ async def _shutdown_after_response() -> None: "message": "Artemis server is shutting down.", "pid": os.getpid(), } + + +# --- Local model lifecycle (artemis model pull/serve/stop) ------------------ +# +# The console surfaces the local_model_endpoint probe's guided actions; these +# endpoints let the user run them without leaving the browser. Each action +# spawns `python -m artemis model ...` and streams its output into a capped +# in-memory log the frontend polls. + + +class LocalModelActionRequest(BaseModel): + """Payload for a local-model lifecycle action.""" + + alias: str = Field(..., description="Catalog alias or org/repo id (e.g. gemma4-e4b)") + port: int | None = Field( + default=None, + ge=1, + le=65535, + description="Serve/stop port; the CLI default (8080) applies when omitted", + ) + + +_MODEL_TASKS: dict[str, dict] = {} +_MODEL_TASK_LOG_LINES = 80 + + +def _model_cli_available() -> bool: + """Whether the ``artemis model`` command group exists in this build.""" + import importlib.util + + try: + return importlib.util.find_spec("artemis.interfaces.cli.commands.model") is not None + except Exception: + return False + + +def _task_view(key: str, task: dict) -> dict: + return { + "key": key, + "action": task["action"], + "alias": task["alias"], + "cmd": task["cmd"], + "status": task["status"], + "returncode": task["returncode"], + "started_at": task["started_at"], + "log": task["log"][-12:], + } + + +def _task_snapshot() -> dict: + return {k: _task_view(k, t) for k, t in _MODEL_TASKS.items()} + + +async def _drain_model_task(task: dict) -> None: + proc = task["process"] + try: + if proc.stdout is not None: + async for raw in proc.stdout: + line = raw.decode(errors="replace").rstrip() + if line: + task["log"].append(line) + del task["log"][:-_MODEL_TASK_LOG_LINES] + rc = await proc.wait() + except Exception as e: # pylint: disable=broad-exception-caught + rc = -1 + task["log"].append(f"task monitor error: {e}") + task["returncode"] = rc + task["status"] = "done" if rc == 0 else "failed" + + +async def _start_model_task(action: str, alias: str, extra_args: list[str]) -> dict: + """Spawn `artemis model ` as a subprocess with a capped log.""" + import asyncio + import sys + import time + + key = f"{action}:{alias}" + existing = _MODEL_TASKS.get(key) + if existing and existing["status"] == "running": + return _task_view(key, existing) + + # ``stop`` is port-scoped on the CLI (no positional model arg). + cmd = [sys.executable, "-m", "artemis", "model", action] + if action != "stop": + cmd.append(alias) + cmd += extra_args + try: + proc = await asyncio.create_subprocess_exec( + *cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT, + ) + except Exception as e: + raise HTTPException( + status_code=500, detail=f"Failed to start `artemis model {action}`: {e}" + ) from e + + task = { + "action": action, + "alias": alias, + "cmd": " ".join(cmd), + "status": "running", + "returncode": None, + "started_at": time.time(), + "log": [], + "process": proc, + } + _MODEL_TASKS[key] = task + task["watcher"] = asyncio.create_task(_drain_model_task(task)) + return _task_view(key, task) + + +def _require_model_cli() -> None: + if not _model_cli_available(): + raise HTTPException( + status_code=501, + detail=( + "`artemis model` commands are not available in this build. " + "Run the shown command in a terminal, or manage the model with " + "your own OpenAI-compatible server." + ), + ) + + +_MODEL_REF_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*(/[A-Za-z0-9][A-Za-z0-9_.-]*)?$") + + +def _require_model_ref(alias: str) -> None: + """Reject aliases the managed CLI could never resolve. + + ``resolve_model_ref`` accepts catalog aliases and ``org/repo`` ids — the + same inputs ``artemis model pull`` takes. Anything else (bare junk, flag + injection like ``--port``, or argv that doesn't look like a model ref) is + refused before a subprocess is spawned. + """ + from artemis.config.local_models import resolve_model_ref + + if not _MODEL_REF_RE.match(alias) or resolve_model_ref(alias) is None: + raise HTTPException( + status_code=400, + detail=( + f"Unknown model '{alias}'. `artemis model` accepts catalog " + "aliases (see `artemis model list`) or a Hugging Face " + "'org/repo' id." + ), + ) + + +@router.get("/local-models/status") +async def local_model_status(): + """Command availability plus in-flight/finished model task states.""" + return { + "commands_available": _model_cli_available(), + "tasks": _task_snapshot(), + } + + +@router.post("/local-models/pull") +async def pull_local_model(payload: LocalModelActionRequest, request: Request): + """Download model weights via `artemis model pull` (long-running).""" + _require_loopback_request(request, "Local model lifecycle is local-only.") + _require_model_cli() + alias = payload.alias.strip() + _require_model_ref(alias) + return await _start_model_task("pull", alias, []) + + +@router.post("/local-models/serve") +async def serve_local_model(payload: LocalModelActionRequest, request: Request): + """Start the managed model server via `artemis model serve`.""" + _require_loopback_request(request, "Local model lifecycle is local-only.") + _require_model_cli() + alias = payload.alias.strip() + _require_model_ref(alias) + extra = ["--port", str(payload.port)] if payload.port else [] + return await _start_model_task("serve", alias, extra) + + +@router.post("/local-models/stop") +async def stop_local_model(payload: LocalModelActionRequest, request: Request): + """Stop a managed model server via `artemis model stop`.""" + _require_loopback_request(request, "Local model lifecycle is local-only.") + _require_model_cli() + extra = ["--port", str(payload.port)] if payload.port else [] + return await _start_model_task("stop", payload.alias.strip(), extra) diff --git a/apps/showcase_ui/src/app/pages/home/home.component.html b/apps/showcase_ui/src/app/pages/home/home.component.html index 060a01d25..ef11bb5a4 100644 --- a/apps/showcase_ui/src/app/pages/home/home.component.html +++ b/apps/showcase_ui/src/app/pages/home/home.component.html @@ -349,7 +349,7 @@

AI Model Setup

Custom Configuration

Define models in config/artemis.jsonc & keys in .env - (OpenAI, Claude, Ollama).

+ (OpenAI, Claude, Ollama — or a local model via artemis model serve).

@@ -604,6 +604,87 @@

AI Model Setup

} + + @if (localModelProbe(); as lm) { + @if (lm.status !== 'pass' || modelConfigEnv()?.default_model?.provider === 'custom') { +
+
+
+ dns +
+ Local Model Endpoint + {{ lm.summary }} +
+
+
+ @if (lm.status === 'pass') { + + check_circle + Ready + + } @else { + + error + Action Needed + + } +
+
+ +

{{ lm.description }}

+ + + @for (act of lm.actions; track act.label) { + @if (act.action_type === 'command') { +
+ {{ act.payload }} + @if (modelActionArgs(act.payload); as args) { + @if (localModelCommandsAvailable()) { + @if (localModelTaskFor(args.action, args.alias)?.status === 'running') { + progress_activity + Running… + } @else { + + } + } + } + +
+ @if (modelActionArgs(act.payload); as args) { + @if (localModelTaskFor(args.action, args.alias); as task) { + @if (task.status === 'failed' || task.log.length) { +
+ @if (task.status === 'done') { +
✓ {{ act.label }} finished
+ } + @if (task.status === 'failed') { +
✗ {{ act.label }} failed (exit {{ task.returncode }}) — check the log below or run the command in a terminal
+ } + @for (line of task.log.slice(-4); track $index) { +
{{ line }}
+ } +
+ } + } + } + } @else { +

{{ act.label }}: {{ act.payload }}

+ } + } +
+ } + } +
diff --git a/apps/showcase_ui/src/app/pages/home/home.component.scss b/apps/showcase_ui/src/app/pages/home/home.component.scss index c1a5e68d6..8d426c814 100644 --- a/apps/showcase_ui/src/app/pages/home/home.component.scss +++ b/apps/showcase_ui/src/app/pages/home/home.component.scss @@ -1421,6 +1421,18 @@ $text-light: #94a3b8; border-radius: 10px; overflow: hidden; + .inspector-desc { + font-size: 12px; + line-height: 1.5; + color: $text-secondary; + margin: 0; + padding: 10px 14px 0; + } + + > .code-box { + margin: 10px 14px; + } + .inspector-card-header { display: flex; align-items: center; @@ -3778,6 +3790,89 @@ $text-light: #94a3b8; } } } + + .btn-run-cmd { + display: inline-flex; + align-items: center; + justify-content: center; + gap: 3px; + padding: 0 8px; + height: 22px; + background: rgba(56, 189, 248, 0.25); + color: #7dd3fc !important; + border: 1px solid rgba(56, 189, 248, 0.4); + border-radius: 5px; + font-family: $font-sans; + font-size: 10.5px; + font-weight: 600; + cursor: pointer; + transition: all 0.18s ease; + flex-shrink: 0; + + .material-symbols-outlined, span { + color: #7dd3fc !important; + } + + .material-symbols-outlined { + font-size: 13px; + } + + &:hover { + background: rgba(56, 189, 248, 0.4); + border-color: rgba(56, 189, 248, 0.6); + } + + &:active { + transform: scale(0.96); + } + } + + .lm-spin { + font-size: 15px; + color: #38bdf8; + flex-shrink: 0; + animation: spin 1s linear infinite; + } + + .lm-running-label { + font-size: 10.5px; + font-weight: 600; + color: #7dd3fc; + flex-shrink: 0; + white-space: nowrap; + } + } + + .lm-task-log { + margin-top: 6px; + padding: 8px 10px; + background: #0f172a; + border-radius: 6px; + font-family: $font-mono; + font-size: 10.5px; + line-height: 1.5; + max-height: 110px; + overflow-y: auto; + + .lm-log-line { + color: #94a3b8; + white-space: pre-wrap; + word-break: break-all; + } + + .lm-log-done { + color: #34d399; + font-weight: 600; + } + + .lm-log-failed { + color: #f87171; + font-weight: 600; + } + + &.lm-task-failed { + border: 1px solid rgba(248, 113, 113, 0.35); + } } /* ======================================================================= */ diff --git a/apps/showcase_ui/src/app/pages/home/home.component.ts b/apps/showcase_ui/src/app/pages/home/home.component.ts index e189b9ce5..b7446bc73 100644 --- a/apps/showcase_ui/src/app/pages/home/home.component.ts +++ b/apps/showcase_ui/src/app/pages/home/home.component.ts @@ -358,6 +358,9 @@ export class HomeComponent implements OnInit, OnDestroy { public configProbe = computed(() => this.systemService.configProbe()); public adbProbe = computed(() => this.systemService.adbProbe()); public llmProbe = computed(() => this.systemService.llmProbe()); + public localModelProbe = computed(() => this.systemService.localModelProbe()); + public localModelTasks = computed(() => this.systemService.localModelTasks()); + public localModelCommandsAvailable = computed(() => this.systemService.localModelCommandsAvailable()); public geminiProbe = computed(() => this.systemService.geminiProbe()); public ocrProbe = computed(() => this.systemService.ocrProbe()); public toolchainProbe = computed(() => this.systemService.toolchainProbe()); @@ -541,6 +544,7 @@ export class HomeComponent implements OnInit, OnDestroy { // Initial fetch of system readiness & model configuration this.systemService.fetchReadiness().subscribe(); this.systemService.fetchModelConfigEnv().subscribe(); + this.systemService.fetchLocalModelStatus().subscribe(); this.systemService.fetchAdbServerStatus().subscribe({ next: status => { if (status.endpoint.mode === 'remote') { @@ -564,6 +568,7 @@ export class HomeComponent implements OnInit, OnDestroy { if (tab === 'diagnostics') { this.systemService.fetchReadiness().subscribe(); this.systemService.fetchModelConfigEnv().subscribe(); + this.systemService.fetchLocalModelStatus().subscribe(); } } @@ -1035,6 +1040,35 @@ export class HomeComponent implements OnInit, OnDestroy { this.selectedProfile.set(profile); } + // --- Local model lifecycle (artemis model pull/serve/stop) --------------- + + /** Parse an `artemis model [--port N]` action payload. */ + public modelActionArgs(payload: string): { action: 'pull' | 'serve' | 'stop'; alias: string; port?: number } | null { + const m = payload.match(/artemis\s+model\s+(pull|serve|stop)\s+([^\s]+)/); + if (!m) return null; + const portMatch = payload.match(/--port\s+(\d+)/); + return { + action: m[1] as 'pull' | 'serve' | 'stop', + alias: m[2], + port: portMatch ? Number(portMatch[1]) : undefined, + }; + } + + /** Latest task state for an action+alias pair, if any. */ + public localModelTaskFor(action: string, alias: string) { + return this.localModelTasks()[`${action}:${alias}`] || null; + } + + public runLocalModelAction(args: { action: 'pull' | 'serve' | 'stop'; alias: string; port?: number }): void { + if (args.action === 'pull') { + this.systemService.pullLocalModel(args.alias).subscribe(); + } else if (args.action === 'serve') { + this.systemService.serveLocalModel(args.alias, args.port).subscribe(); + } else { + this.systemService.stopLocalModel(args.alias, args.port).subscribe(); + } + } + public setCategory(cat: SuggestionCategory): void { this.selectedCategory.set(cat); } diff --git a/apps/showcase_ui/src/app/services/system.service.ts b/apps/showcase_ui/src/app/services/system.service.ts index b69697d9f..1cb5b49e9 100644 --- a/apps/showcase_ui/src/app/services/system.service.ts +++ b/apps/showcase_ui/src/app/services/system.service.ts @@ -58,6 +58,7 @@ export class SystemService { public adbProbe = computed(() => this.probes().find(p => p.id === 'android_adb') || null); public llmProbe = computed(() => this.probes().find(p => p.id === 'gemini_api_key' || p.id === 'llm_api_key') || null); public geminiProbe = computed(() => this.llmProbe()); + public localModelProbe = computed(() => this.probes().find(p => p.id === 'local_model_endpoint') || null); public ocrProbe = computed(() => this.probes().find(p => p.id === 'vision_ocr_key' || p.id === 'ocr_api_key') || null); // Grouped readiness helpers for the 3-step onboarding flow @@ -80,7 +81,12 @@ export class SystemService { return true; } const llm = this.llmProbe(); - return llm?.status === 'pass'; + // The local-endpoint probe is a blocker server-side: when artemis.jsonc + // binds a node to a custom api_base (Artemis-served or BYO), a dead + // endpoint or missing model must gate the run the same way a missing + // API key does. It self-passes ("Not used") for cloud-only configs. + const local = this.localModelProbe(); + return llm?.status === 'pass' && (local == null || local.status === 'pass'); }); public isDeviceReady = computed(() => { @@ -314,6 +320,79 @@ export class SystemService { } } + // --- Local model lifecycle (artemis model pull/serve/stop) --------------- + + public localModelCommandsAvailable = signal(null); + public localModelTasks = signal>({}); + private localModelPollTimer: any = null; + + public fetchLocalModelStatus(): Observable { + return this.http.get('/api/system/local-models/status').pipe( + tap({ + next: (resp) => { + this.localModelCommandsAvailable.set(resp.commands_available); + this.localModelTasks.set(resp.tasks || {}); + const anyRunning = Object.values(resp.tasks || {}).some(t => t.status === 'running'); + if (!anyRunning) { + this.stopLocalModelPolling(); + } + } + }) + ); + } + + public startLocalModelPolling(): void { + if (this.localModelPollTimer) { + clearInterval(this.localModelPollTimer); + } + this.localModelPollTimer = setInterval(() => { + this.fetchLocalModelStatus().subscribe({ error: () => {} }); + }, 2000); + } + + public stopLocalModelPolling(): void { + if (this.localModelPollTimer) { + clearInterval(this.localModelPollTimer); + this.localModelPollTimer = null; + // A finished action may have changed model readiness — re-check. + this.fetchReadiness().subscribe({ error: () => {} }); + } + } + + private runLocalModelAction( + endpoint: 'pull' | 'serve' | 'stop', + alias: string, + port?: number + ): Observable { + return this.http + .post(`/api/system/local-models/${endpoint}`, { alias, port: port ?? null }) + .pipe( + tap({ + next: (task) => { + this.localModelTasks.update(tasks => ({ ...tasks, [task.key]: task })); + if (task.status === 'running') { + this.startLocalModelPolling(); + } else { + this.fetchReadiness().subscribe({ error: () => {} }); + } + }, + error: () => {} + }) + ); + } + + public pullLocalModel(alias: string): Observable { + return this.runLocalModelAction('pull', alias); + } + + public serveLocalModel(alias: string, port?: number): Observable { + return this.runLocalModelAction('serve', alias, port); + } + + public stopLocalModel(alias: string, port?: number): Observable { + return this.runLocalModelAction('stop', alias, port); + } + /** * Launch a local AVD emulator in the background and continuously track boot progress */ @@ -608,3 +687,19 @@ export interface ModelConfigEnvResponse { }>; } + +export interface LocalModelTask { + key: string; + action: 'pull' | 'serve' | 'stop' | string; + alias: string; + cmd: string; + status: 'running' | 'done' | 'failed' | string; + returncode: number | null; + started_at: number; + log: string[]; +} + +export interface LocalModelStatusResponse { + commands_available: boolean; + tasks: Record; +} diff --git a/artemis/agents/checker/checker.py b/artemis/agents/checker/checker.py index 52dff2ca2..ad474395b 100644 --- a/artemis/agents/checker/checker.py +++ b/artemis/agents/checker/checker.py @@ -67,6 +67,7 @@ ) from artemis.utils.visualization import format_minimal_list_with_elements from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_codec import image_data_uri logger = get_logger(__name__) @@ -836,15 +837,18 @@ async def run_final_check( f"# Final Screen Elements\n{minimal_list}" ) - content: list = [{"type": "text", "text": human_text}] + content: list = [] if screenshot_b64: - content.append({"type": "text", "text": "--- Final Screenshot ---"}) + # Image-first ordering: vision models ground better when the + # screenshot precedes the instruction/context text. content.append( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{screenshot_b64}"}, + "image_url": {"url": image_data_uri(screenshot_b64)}, } ) + content.append({"type": "text", "text": "--- Final Screenshot ---"}) + content.append({"type": "text", "text": human_text}) messages = [ SystemMessage(content=system_content), diff --git a/artemis/agents/diagnoser/diagnoser.py b/artemis/agents/diagnoser/diagnoser.py index 13d80317c..d94376a5d 100644 --- a/artemis/agents/diagnoser/diagnoser.py +++ b/artemis/agents/diagnoser/diagnoser.py @@ -40,6 +40,7 @@ from artemis.tools.log_tool import get_analyze_logs_tool from artemis.tools.mobile.read_hierarchy import get_ui_hierarchy_tool from artemis.tools.scratchpad import get_list_notes_tool, get_read_note_tool +from artemis.utils.image_codec import image_data_uri from artemis.tools.tool_wrapper import ( get_tool_result_content, invoke_tool_with_injection, @@ -227,7 +228,7 @@ def _build_initial_messages( content.append( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{latest_screenshot_b64}"}, + "image_url": {"url": image_data_uri(latest_screenshot_b64)}, } ) diff --git a/artemis/agents/explorer/universal_runner.py b/artemis/agents/explorer/universal_runner.py index c5b5bc89a..219a15a54 100644 --- a/artemis/agents/explorer/universal_runner.py +++ b/artemis/agents/explorer/universal_runner.py @@ -36,6 +36,7 @@ from artemis.graph.state import State from artemis.services.llm import get_llm from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_codec import image_data_uri logger = get_logger(__name__) @@ -161,6 +162,10 @@ def _build_universal_messages( img_b64 = base64.b64encode(f.read()).decode("utf-8") user_content: list[dict[str, Any]] = [ + { + "type": "image_url", + "image_url": {"url": image_data_uri(img_b64)}, + }, { "type": "text", "text": ( @@ -172,7 +177,7 @@ def _build_universal_messages( }, { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{img_b64}"}, + "image_url": {"url": image_data_uri(img_b64)}, }, ] @@ -266,16 +271,14 @@ def _build_tool_images_message(self, images: list[tuple[str, str]]) -> HumanMess continue if tool_name not in tool_names: tool_names.append(tool_name) - blocks.append( - {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{img_b64}"}} - ) + blocks.append({"type": "image_url", "image_url": {"url": image_data_uri(img_b64)}}) if not blocks: return None header = { "type": "text", "text": f"[Annotated image(s) returned by: {', '.join(tool_names)}]", } - return HumanMessage(content=[header, *blocks]) + return HumanMessage(content=[*blocks, header]) async def _universal_dispatch_tools( self, tool_calls: list, messages: list[BaseMessage], turn: int diff --git a/artemis/agents/flash/runner.py b/artemis/agents/flash/runner.py index 3ea3f6865..6f0927564 100644 --- a/artemis/agents/flash/runner.py +++ b/artemis/agents/flash/runner.py @@ -70,17 +70,22 @@ MemoryTranscriptConfig, StepSummarizerConfig, load_agent_config, + settings, ) from artemis.context import ArtemisContext from artemis.controllers.unified_controller import UnifiedMobileController from artemis.data_engine.trace import trace from artemis.graph.perception import _check_injected_instruction_file from artemis.graph.state import State +from artemis.llm.router import ModelProvider from artemis.llm.structured import ParseFailure, parse_structured +from artemis.core.decision_client import DecisionClient from artemis.mcp.action_executor import McpActionExecutor +from artemis.utils.image_codec import image_data_uri from artemis.mcp.observation import observe from artemis.memory.transcript import PRO_UI_LIST_MARKER, TranscriptLedger, mark_ephemeral from artemis.services.llm import ( + LLMCallError, RobustChatModelWrapper, acomplete, get_google_llm, @@ -103,8 +108,112 @@ " action tool or 'report_task_status'." ) +# Hard ceiling on consecutive narration-only turns. Some local VLMs (e.g. +# Gemma 4 E4B under mlx_vlm) can fall into an apology loop — seeing the +# goal, narrating a plan, never emitting a tool call — and with an +# unlimited turn budget that loop never exits on its own. +_MAX_CONSECUTIVE_NO_TOOL_CALLS = 8 + +# Gemma-family tool-call envelopes observed in the wild: the model emits +# `call:{}` and some servers (mlx_vlm) wrap it into a function +# literally named "call" whose arguments are the unparseable remainder, or +# the envelope leaks into text content. +_GEMMA_CALL_RE = re.compile(r"(?:<\|?tool_call\|?>?\s*)?call:([a-zA-Z_]\w*)\s*\{") +_GEMMA_ENVELOPED_RE = re.compile(r'\s*\{?\s*"?([a-zA-Z_]\w*)"?\s*\{') +_GEMMA_QUOTE_RE = re.compile(r'<\|\\?\"\|>') + + +def _loads_relaxed_args(raw: str): + """Best-effort JSON parse of a mangled Gemma `name{key:value}` args object.""" + s = raw.strip() + s = _GEMMA_QUOTE_RE.sub('"', s) + # Quote bare keys: {key: -> {"key": and ,key: -> ,"key": + s = re.sub(r'([{,])\s*"?([a-zA-Z_]\w*)"?\s*:', r'\1"\2":', s) + try: + return json.loads(s) + except ValueError: + return None + + +def _parse_call_syntax(raw: str, match=None): + """Recovers `name{args}` at ``match`` — `call:name{...}` or `{name{...}`.""" + if match is None: + match = _GEMMA_ENVELOPED_RE.match(raw) + if not match: + return None + name = match.group(1) + start = match.end() - 1 # opening '{' of the args object + args = None + depth = 0 + for i in range(start, len(raw)): + ch = raw[i] + if ch == '{': + depth += 1 + elif ch == '}': + depth -= 1 + if depth == 0: + args = _loads_relaxed_args(raw[start:i + 1]) + break + if depth: + args = _loads_relaxed_args(raw[start:] + '}') + if not isinstance(args, dict): + return None + return {"name": name, "args": args, "id": str(uuid.uuid4())} + + _FINAL_TURN_WARNING = "[WARNING] This is your final turn; only 'report_task_status' is available." +# Hybrid Flash loop (decision.flash_loop): the VLM perceives, the typed +# decision model acts. The describe call carries NO tool declarations, so +# the tool-call envelope syntax that local VLMs mangle can never be +# emitted; action selection is a fixed-space `choose` over the live +# element list plus meta-actions instead. +_HYBRID_PLAN_PROMPT = """You are the planning stage of a UI testing agent. +GOAL: $goal + +Decompose the goal into 3-8 short ordered milestones (one per line, starting +with '- '). Each milestone names a screen state or interaction step, not a +tool call. Output only the milestone list.""" + +_HYBRID_DESCRIBE_PROMPT = """You are the perception stage of a UI testing agent. +GOAL: $goal +PLAN: +$plan + +Describe the CURRENT screenshot factually in 3-5 short sentences: which app +and screen is shown, what has changed since the previous observation, and +where the elements relevant to the goal sit. For every element relevant to +the goal give one line: [index] label ~[x,y] — the index from the attached +element list when the element is listed there, and a normalized 0-1000 +[x,y] coordinate only when it is visible on screen but NOT in the list. +Never claim an action was taken. Do not decide what to do next — a separate +decision stage reads your description.""" + +# Menu entries the decision model may always pick, alongside the element +# taps generated from the live index list. Free-text input is excluded by +# design: a fixed answer space cannot produce the text to type, so a goal +# that needs text input escalates to the standard tool-calling turn. +_HYBRID_META_ACTIONS = { + "swipe_up": "scroll the screen up to reveal content below", + "swipe_down": "scroll the screen down to reveal content above", + "swipe_left": "swipe left (e.g. carousel, page forward)", + "swipe_right": "swipe right (e.g. carousel, page back)", + "press_back": "press the Android back button", + "press_home": "press the Android home button", + "wait": "wait briefly for the screen to settle or load", + "done": "the goal is satisfied — finish the task as completed", + "fail": "the goal cannot be achieved from here — finish as failed", +} + +# Cap on element-tap entries in the decision menu: the decision model's +# context is short (~512-1024 tokens) and taps beyond the first screenful +# are rarely the right action anyway. +_HYBRID_MAX_MENU_ELEMENTS = 15 + +# Distilled state sent to the decision model is capped for the same reason: +# goal + plan + screen description, truncated tail-first. +_HYBRID_MAX_STATE_CHARS = 2000 + @dataclass class _TurnRecord: @@ -286,6 +395,169 @@ def _init_llm(self): return RobustChatModelWrapper(get_google_llm(model_name="gemini-2.5-flash"), self.ctx) + # ------------------------------------------------------------------ + # Hybrid loop (decision.flash_loop): VLM describes, decision model acts + # ------------------------------------------------------------------ + + async def _hybrid_plan(self, llm) -> str | None: + """The one upfront planning call — Pro's planner leg grafted into Flash. + + A plain completion (no tools): the VLM emits a milestone list that the + per-turn describe prompt carries as context. Failure is non-fatal — the + loop still runs, just without plan guidance. + """ + prompt = Template(_HYBRID_PLAN_PROMPT).render(goal=self.goal) + try: + response = await invoke_llm_with_timeout_message( + acomplete(llm, [HumanMessage(content=prompt)]), + timeout_seconds=10, + hard_timeout=settings.LLM_FLASH_HARD_TIMEOUT_SECONDS, + ) + except LLMCallError as e: + logger.warning(f"Hybrid plan call failed, running unplanned: {e}") + return None + plan = self._extract_response_text(response).strip() + logger.info(f"Hybrid plan ({len(plan)} chars): {plan[:200]}") + return plan or None + + @staticmethod + def _hybrid_action_menu(state: State) -> dict[str, str]: + """The fixed action menu for one turn: element taps + meta-actions. + + Tap keys embed the element index of the CURRENT observation's list — + index drift across turns is impossible because the menu is rebuilt + every turn and the executor resolves the index against the same list + snapshot the decision was made on. + """ + elements = getattr(state, "indexed_elements", None) or [] + menu: dict[str, str] = {} + for el in elements[:_HYBRID_MAX_MENU_ELEMENTS]: + idx = el.get("index") + if idx is None: + continue + label = ( + el.get("text") + or el.get("content_desc") + or el.get("resource_id") + or el.get("class_name") + or "element" + ) + menu[f"tap_{idx}"] = f"tap element {idx}: {str(label)[:60]}" + menu.update(_HYBRID_META_ACTIONS) + return menu + + @staticmethod + def _menu_choice_to_call(choice: str, description: str) -> dict | None: + """Maps a menu key to a tool-call dict for the normal execution path.""" + tid = f"call_{uuid.uuid4().hex[:8]}" + if choice.startswith("tap_"): + try: + idx = int(choice[len("tap_"):]) + except ValueError: + return None + return {"id": tid, "name": "click", "args": {"target": idx}} + if choice in ("swipe_up", "swipe_down", "swipe_left", "swipe_right"): + return { + "id": tid, + "name": "swipe", + "args": {"direction": choice.split("_", 1)[1]}, + } + if choice in ("press_back", "press_home"): + return { + "id": tid, + "name": "press_key", + "args": {"key": "back" if choice == "press_back" else "home"}, + } + if choice == "wait": + return {"id": tid, "name": "wait_for_delay", "args": {"timeout_ms": 1500}} + if choice in ("done", "fail"): + return { + "id": tid, + "name": "report_task_status", + "args": { + "status": "completed" if choice == "done" else "failed", + "explanation": description[:500] or "Decision layer verdict.", + }, + } + return None + + async def _hybrid_decide( + self, + llm, + state: State, + tail: HumanMessage, + plan_text: str | None, + cfg, + ) -> tuple | None: + """One hybrid turn: describe → choose → tool call. + + Returns ``(describe_response, [tool_call])`` on a confident decision, + ``None`` to escalate to the standard tool-calling model turn. The + describe call binds no tools — the VLM answers with a screen + narrative (plus approximate coords for unlisted targets) that becomes + the turn's thought text, so envelope-mangling and narration-only + stalls cannot occur on this path. + """ + describe_prompt = Template(_HYBRID_DESCRIBE_PROMPT).render( + goal=self.goal, plan=plan_text or "(no plan — describe and list elements)" + ) + try: + response = await invoke_llm_with_timeout_message( + acomplete(llm, [SystemMessage(content=describe_prompt), tail]), + timeout_seconds=10, + hard_timeout=settings.LLM_FLASH_HARD_TIMEOUT_SECONDS, + ) + except LLMCallError as e: + logger.warning(f"Hybrid describe call failed, escalating: {e}") + return None + description = self._extract_response_text(response) + logger.info(f"Hybrid describe: {description[:200]}") + + menu = self._hybrid_action_menu(state) + client = DecisionClient( + cfg.url, model=cfg.model, timeout_seconds=cfg.timeout_seconds + ) + state_text = ( + f"Goal: {self.goal}\n" + f"Plan: {plan_text or 'none'}\n" + f"Current screen: {description}" + ) + verdict = await client.achoose( + state_text[-_HYBRID_MAX_STATE_CHARS:], + "Which single action moves the task forward?", + menu, + ) + if ( + not verdict.answered + or verdict.confidence < cfg.confidence_threshold + or verdict.verdict not in menu + ): + logger.info( + f"Hybrid decision inconclusive (verdict={verdict.verdict!r} " + f"confidence={verdict.confidence:.2f}), escalating to tool-call turn" + ) + return None + tool_call = self._menu_choice_to_call(str(verdict.verdict), description) + if tool_call is None: + return None + logger.info( + f"Hybrid decision: {tool_call['name']} " + f"(confidence {verdict.confidence:.2f})" + ) + # Stamp the chosen call onto the describe message: the ToolMessages the + # executor appends then have a legitimate parent tool_call, which strict + # providers require when the ledger replays them into later prompts. + response.tool_calls = [ + { + "name": tool_call["name"], + "args": tool_call["args"], + "id": tool_call["id"], + "type": "tool_call", + } + ] + response.invalid_tool_calls = [] + return response, [tool_call] + def _render_system_prompt(self, tools_declaration: list) -> str: """Renders the system prompt from the flash_runner.md template. @@ -363,15 +635,16 @@ def _build_tail( """ # ``turns`` is kept for call-site symmetry: the tail has the same shape # on every turn. - blocks: list[dict] = [ - {"type": "text", "text": f"# CURRENT OBSERVATION [{ledger.elapsed_label()}]"} - ] + blocks: list[dict] = [] if img_bytes: img_b64 = base64.b64encode(img_bytes).decode("utf-8") + # Image-first: vision models ground better with the screenshot + # ahead of the observation text. + blocks.append({"type": "image_url", "image_url": {"url": image_data_uri(img_b64)}}) + blocks.append({"type": "text", "text": f"# CURRENT OBSERVATION [{ledger.elapsed_label()}]"}) + if img_bytes: blocks.append({"type": "text", "text": "--- Current Screenshot ---"}) - blocks.append( - {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{img_b64}"}} - ) + blocks.append({"type": "image_url", "image_url": {"url": image_data_uri(img_b64)}}) if xml_list: blocks.append({"type": "text", "text": f"{PRO_UI_LIST_MARKER}\n{xml_list}"}) ephemeral: list[int] = [] @@ -534,8 +807,40 @@ def _resolve_tool_calls(self, response, raw_text: str) -> list: "id": str(uuid.uuid4()), } ] + + # Local-server envelope repair: mlx_vlm can wrap the model's native + # `call:name{args}` output into a function literally named "call" whose + # arguments are the unparseable remainder — langchain then drops it to + # invalid_tool_calls and the call is silently lost. + if not tool_calls: + tool_calls = self._repair_enveloped_tool_calls( + getattr(response, "invalid_tool_calls", None) + ) + + # Same envelope can leak into text content outright. + if not tool_calls and raw_text: + call_match = _GEMMA_CALL_RE.search(raw_text) + if call_match: + repaired = _parse_call_syntax(raw_text, match=call_match) + if repaired: + tool_calls = [repaired] return tool_calls + @staticmethod + def _repair_enveloped_tool_calls(invalid_tool_calls) -> list: + """Recovers calls mangled into `name="call"` + raw `name{args}` args.""" + repaired = [] + for tc in invalid_tool_calls or []: + name = tc.get("name") + raw_args = tc.get("args") + if name != "call" or not isinstance(raw_args, str): + continue + parsed = _parse_call_syntax(raw_args) + if parsed: + parsed["id"] = tc.get("id") or parsed["id"] + repaired.append(parsed) + return repaired + # ------------------------------------------------------------------ # Per-tool-call helpers (act / record / report) # ------------------------------------------------------------------ @@ -883,14 +1188,27 @@ async def _execute_and_record_action( async def _invoke_model(self, llm, current_tools: list, messages: list[BaseMessage]): """Binds the active tools and invokes the model through the LLM gateway.""" - # Bind active tools - bound_llm = llm.bind_tools(current_tools) + # Keep only the latest screenshot: strict gateways cap images per + # request, and earlier frames are stale context anyway. + self._prune_intermediate_screenshots(messages) + # Bind active tools. Local OpenAI-compatible servers (mlx_vlm and + # friends) may let the model answer with narration only, which the + # loop reads as a no-progress turn; tool_choice="required" holds the + # provider to the loop's contract that every turn is a tool call. + endpoint = getattr(llm, "endpoint", None) + provider = getattr(endpoint, "provider", None) + if provider == ModelProvider.CUSTOM: + bound_llm = llm.bind_tools(current_tools, tool_choice="required") + else: + bound_llm = llm.bind_tools(current_tools) # Invoke Model. Streaming, live-token UI deltas, classified # retries, and pause/resume are all owned by the LLM gateway # (acomplete); a typed LLMCallError propagates if it gives up. return await invoke_llm_with_timeout_message( - acomplete(bound_llm, messages), timeout_seconds=10, hard_timeout=180 + acomplete(bound_llm, messages), + timeout_seconds=10, + hard_timeout=settings.LLM_FLASH_HARD_TIMEOUT_SECONDS, ) async def _process_tool_calls( @@ -1044,6 +1362,20 @@ async def run(self, state: State) -> dict: llm = self._init_llm() self._llm = llm + # Hybrid loop (decision.flash_loop): VLM describes, a typed decision + # model picks the action. Both flags must be on; a missing/unreachable + # decision endpoint escalates per-turn to the standard tool-call path. + decision_cfg = getattr(getattr(self.ctx, "agent_config", None), "decision", None) + hybrid = bool( + decision_cfg + and getattr(decision_cfg, "enabled", False) + and getattr(decision_cfg, "flash_loop", False) + ) + plan_text = None + if hybrid: + logger.info("Hybrid Flash loop enabled: VLM describes, decision model acts.") + plan_text = await self._hybrid_plan(llm) + tools_declaration = self._get_tools() report_only_tools = [t for t in tools_declaration if t.name == "report_task_status"] @@ -1056,6 +1388,7 @@ async def run(self, state: State) -> dict: current_xml_list = xml_list previous_turn: _TurnRecord | None = None pending_notices: list[str] = [] + no_tool_call_streak = 0 while limit is None or turns < limit: turns += 1 @@ -1095,7 +1428,19 @@ async def run(self, state: State) -> dict: turn_base = len(messages) - 1 current_tools = report_only_tools if is_final else tools_declaration - response = await self._invoke_model(llm, current_tools, messages) + # Hybrid turn: describe → decision-model action pick. The final + # turn stays on the standard path (report_task_status only). + response = None + tool_calls = None + if hybrid and not is_final: + decided = await self._hybrid_decide( + llm, state, tail, plan_text, decision_cfg + ) + if decided is not None: + response, tool_calls = decided + + if response is None: + response = await self._invoke_model(llm, current_tools, messages) if response is None: break @@ -1121,7 +1466,8 @@ async def run(self, state: State) -> dict: self._record_llm_trace(step_token_usage, raw_text) - tool_calls = self._resolve_tool_calls(response, raw_text) + if tool_calls is None: + tool_calls = self._resolve_tool_calls(response, raw_text) if not tool_calls: logger.info( @@ -1131,11 +1477,24 @@ async def run(self, state: State) -> dict: if is_final: final_report = {"status": "failed", "explanation": raw_text} break + no_tool_call_streak += 1 + if no_tool_call_streak >= _MAX_CONSECUTIVE_NO_TOOL_CALLS: + final_report = { + "status": "failed", + "explanation": ( + f"Stopped after {no_tool_call_streak} consecutive responses" + " without a tool call; the model kept narrating instead" + " of acting." + ), + } + break pending_notices.append(_NO_TOOL_CALL_NOTICE) ledger.stage_turn(messages[turn_base:]) previous_turn = _TurnRecord() continue + no_tool_call_streak = 0 + # Process tool calls turn = _TurnRecord() ( diff --git a/artemis/agents/flash/summarizer.py b/artemis/agents/flash/summarizer.py index 5b8a3cf98..0147b652c 100644 --- a/artemis/agents/flash/summarizer.py +++ b/artemis/agents/flash/summarizer.py @@ -41,6 +41,7 @@ from artemis.utils.task_tree import format_actions_clean from artemis.utils.visualization import draw_action_overlay_on_image from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_codec import image_data_uri logger = get_logger(__name__) @@ -430,7 +431,7 @@ async def _attempt(self, key: JobKey) -> bool: } ) content_blocks.append( - {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{b64_pre}"}} + {"type": "image_url", "image_url": {"url": image_data_uri(b64_pre)}} ) if post_bytes: @@ -449,7 +450,7 @@ async def _attempt(self, key: JobKey) -> bool: content_blocks.append( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{b64_post}"}, + "image_url": {"url": image_data_uri(b64_post)}, } ) diff --git a/artemis/agents/image_processor/image_processor.py b/artemis/agents/image_processor/image_processor.py index 4ce526984..54e60a8e4 100644 --- a/artemis/agents/image_processor/image_processor.py +++ b/artemis/agents/image_processor/image_processor.py @@ -35,6 +35,7 @@ from artemis.services.llm import get_llm from artemis.utils.python_executor import PythonExecutor from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_codec import image_data_uri logger = get_logger(__name__) @@ -115,11 +116,11 @@ async def run(self, instruction: str, target_image_path: str) -> dict: SystemMessage(content=prompt_template), HumanMessage( content=[ - {"type": "text", "text": "Here is the target image. Begin writing your code."}, { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{img_b64}"}, + "image_url": {"url": image_data_uri(img_b64)}, }, + {"type": "text", "text": "Here is the target image. Begin writing your code."}, ] ), ] @@ -186,9 +187,7 @@ async def run(self, instruction: str, target_image_path: str) -> dict: image_blocks.append( { "type": "image_url", - "image_url": { - "url": f"data:image/jpeg;base64,{b64_data}" - }, + "image_url": {"url": image_data_uri(b64_data)}, } ) image_blocks.append({"type": "text", "text": f"\n{label}\n"}) diff --git a/artemis/agents/object_detector/object_detector.py b/artemis/agents/object_detector/object_detector.py index b2acb4f5f..acfb3e14f 100644 --- a/artemis/agents/object_detector/object_detector.py +++ b/artemis/agents/object_detector/object_detector.py @@ -16,7 +16,9 @@ import asyncio import base64 +import io import json +import math import os from pathlib import Path import re @@ -25,12 +27,176 @@ from artemis.llm.structured import ParseFailure, parse_structured from artemis.services.llm import get_llm +from artemis.utils.image_codec import image_data_uri from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) _DETECTOR_SEMAPHORE = asyncio.Semaphore(6) +# The detector's output contract: points are [x, y] on a normalized 0-1000 +# grid over the image that was sent (consumers divide by 1000 to get pixels). +# Items may also carry ``box_2d`` corner pairs in the same declared convention +# (Gemma 4's trained detection output is [ymin, xmin, ymax, xmax] / 1000, i.e. +# the default ``yx_1000``); a box is normalized in place to [x1, y1, x2, y2] +# and synthesizes ``point`` from its center when the item lacks one. +# Models differ in what they natively emit; ``coordinate_format`` on the +# resolved endpoint declares the convention so results can be normalized: +# axis order — ``yx`` (Gemini ER style) or ``xy`` (most vision models) +# scale — ``1000`` grid, ``px`` absolute pixels, ``norm`` 0-1 floats +_DEFAULT_COORDINATE_FORMAT = "yx_1000" +_PIXEL_GRID = 1000.0 + +# Qwen-family processors smart-resize inputs to a multiple of the patch-merge +# factor within min/max pixel bounds; pixel-space coordinates land on that +# resized grid, not the raw image grid. +_QWEN_RESIZE_FACTOR = 32 +_QWEN_MIN_PIXELS = 65536 +_QWEN_MAX_PIXELS = 16777216 # size.longest_edge in the shipped preprocessor config + + +def _image_dimensions(image_data: bytes) -> tuple[int, int] | None: + """Return (width, height) of encoded image bytes, or None when undecodable.""" + try: + from PIL import Image + + with Image.open(io.BytesIO(image_data)) as img: + return img.width, img.height + except Exception: + return None + + +def _qwen_resized_dimensions(width: int, height: int) -> tuple[int, int]: + """Grid a Qwen-family processor sees after its smart resize.""" + factor = _QWEN_RESIZE_FACTOR + w_bar = round(width / factor) * factor + h_bar = round(height / factor) * factor + if w_bar * h_bar > _QWEN_MAX_PIXELS: + beta = math.sqrt((width * height) / _QWEN_MAX_PIXELS) + w_bar = math.floor(width / beta / factor) * factor + h_bar = math.floor(height / beta / factor) * factor + elif w_bar * h_bar < _QWEN_MIN_PIXELS: + beta = math.sqrt(_QWEN_MIN_PIXELS / (width * height)) + w_bar = math.ceil(width * beta / factor) * factor + h_bar = math.ceil(height * beta / factor) * factor + return w_bar, h_bar + + +def _endpoint_coordinate_format(llm) -> str: + fmt = getattr(getattr(llm, "endpoint", None), "coordinate_format", None) + if isinstance(fmt, str) and fmt in _COORDINATE_FORMATS: + return fmt + return _DEFAULT_COORDINATE_FORMAT + + +_COORDINATE_FORMATS = frozenset({"yx_1000", "xy_1000", "yx_px", "xy_px", "yx_norm", "xy_norm"}) + + +def _detected_batch_scale(items: list) -> str | None: + """Infer the actual scale a model used across a whole response. + + The contract is 0-1000, but when a model ignores it the whole batch + shares one convention: every coordinate fitting in 0-1 means ``norm``; + any coordinate above 1000 means ``px``. Mixed magnitudes stay on the + declared grid so sub-1000 corner points are not rescaled twice. + """ + coords = [ + abs(float(c)) + for item in items + if isinstance(item, dict) + for key in ("point", "box_2d") + if isinstance(item.get(key), (list, tuple)) + and len(item[key]) == (2 if key == "point" else 4) + for c in item[key] + if isinstance(c, (int, float)) and not isinstance(c, bool) + ] + if not coords: + return None + max_coord = max(coords) + if max_coord <= 1.0 and any(c != 0.0 for c in coords): + return "norm" + if max_coord > _PIXEL_GRID: + return "px" + return None + + +def _normalize_detected_point( + item: dict, + coordinate_format: str, + image_size: tuple[int, int] | None, + batch_scale: str | None, +) -> None: + """Rewrite ``item['point']`` to the [x, y] 0-1000 contract in place. + + Axis order and scale come from the endpoint's declared coordinate + format; ``batch_scale`` (``norm``/``px``) overrides the declared scale + only when the whole response clearly violated the contract. + """ + point = item.get("point") + if not (isinstance(point, (list, tuple)) and len(point) == 2): + return + try: + first, second = float(point[0]), float(point[1]) + except (TypeError, ValueError): + return + + x, y = (second, first) if coordinate_format.startswith("yx") else (first, second) + + scale = batch_scale or coordinate_format.rsplit("_", 1)[-1] + if scale == "px": + if image_size: + grid_w, grid_h = _qwen_resized_dimensions(*image_size) + x, y = x * _PIXEL_GRID / grid_w, y * _PIXEL_GRID / grid_h + elif scale == "norm": + x, y = x * _PIXEL_GRID, y * _PIXEL_GRID + + item["point"] = [x, y] + + +def _normalize_detected_box( + item: dict, + coordinate_format: str, + image_size: tuple[int, int] | None, + batch_scale: str | None, +) -> bool: + """Rewrite ``item['box_2d']`` to [x1, y1, x2, y2] on the 0-1000 contract. + + When the item has no usable ``point``, the box center becomes it. Returns + True only in that case: a synthesized point is already on the contract + grid and must not go through ``_normalize_detected_point`` again. + """ + box = item.get("box_2d") + if not (isinstance(box, (list, tuple)) and len(box) == 4): + return False + try: + a, b, c, d = (float(v) for v in box) + except (TypeError, ValueError): + return False + + x1, y1, x2, y2 = (b, a, d, c) if coordinate_format.startswith("yx") else (a, b, c, d) + + scale = batch_scale or coordinate_format.rsplit("_", 1)[-1] + if scale == "px": + if image_size: + grid_w, grid_h = _qwen_resized_dimensions(*image_size) + x1, x2 = x1 * _PIXEL_GRID / grid_w, x2 * _PIXEL_GRID / grid_w + y1, y2 = y1 * _PIXEL_GRID / grid_h, y2 * _PIXEL_GRID / grid_h + elif scale == "norm": + x1, y1, x2, y2 = x1 * _PIXEL_GRID, y1 * _PIXEL_GRID, x2 * _PIXEL_GRID, y2 * _PIXEL_GRID + + item["box_2d"] = [x1, y1, x2, y2] + + point = item.get("point") + if isinstance(point, (list, tuple)) and len(point) == 2: + try: + float(point[0]) + float(point[1]) + return False + except (TypeError, ValueError): + pass + item["point"] = [(x1 + x2) / 2, (y1 + y2) / 2] + return True + async def _detect_single_label( llm, @@ -53,7 +219,7 @@ async def _detect_single_label( content=[ { "type": "image_url", - "image_url": {"url": f"data:{mime_type};base64,{img_b64}"}, + "image_url": {"url": image_data_uri(img_b64)}, }, {"type": "text", "text": prompt}, ] @@ -124,7 +290,7 @@ async def _run_object_detection( queries = queries or [] templates = templates or ["Point to the following objects: {labels_str}"] try: - llm = get_llm(ctx, name="object_detector") + llm = get_llm(ctx, name="object_detector", is_utils=True) except Exception: llm = get_llm(ctx, name="operator") @@ -165,16 +331,18 @@ async def _run_object_detection( except Exception as e: logger.error(f"Task raised exception: {e}") - # Swap coordinates from [y, x] normalized to [x, y] normalized if needed + # Normalize model points to the [x, y] 0-1000 contract. Axis order and + # scale follow the endpoint's declared coordinate_format (Gemini ER's + # [y, x] stays the default); a batch that clearly arrived in 0-1 or raw + # pixels is rescaled instead. + coordinate_format = _endpoint_coordinate_format(llm) + image_size = _image_dimensions(image_data) + batch_scale = _detected_batch_scale(fused_results) for item in fused_results: - if ( - isinstance(item, dict) - and "point" in item - and isinstance(item["point"], list) - and len(item["point"]) == 2 - ): - y_norm, x_norm = item["point"] - item["point"] = [x_norm, y_norm] + if isinstance(item, dict): + box_derived = _normalize_detected_box(item, coordinate_format, image_size, batch_scale) + if not box_derived: + _normalize_detected_point(item, coordinate_format, image_size, batch_scale) detected_labels = set( item["label"] for item in fused_results if isinstance(item, dict) and "label" in item diff --git a/artemis/agents/operator/operator.py b/artemis/agents/operator/operator.py index d24536270..94496f272 100644 --- a/artemis/agents/operator/operator.py +++ b/artemis/agents/operator/operator.py @@ -88,6 +88,7 @@ BackgroundTasksPromptComponent, render_plan_ledger_bounce, ReasoningReminderPromptComponent, + ActiveFocusPromptComponent, unwritten_action_streak, ToolLimitWarningPromptComponent, InjectedInstructionPromptComponent, @@ -255,11 +256,13 @@ async def _build_prompt( (FeedbackPromptComponent(), {}), (ReasoningReminderPromptComponent(), {}), (ToolLimitWarningPromptComponent(), {}), + (ActiveFocusPromptComponent(), {}), ] else: components = [(c, {}) for c in components] + [ (ReasoningReminderPromptComponent(), {}), (ToolLimitWarningPromptComponent(), {}), + (ActiveFocusPromptComponent(), {}), ] # Active and finished tasks are now passed as arguments @@ -426,6 +429,9 @@ async def _build_prompt_transcript( (FeedbackPromptComponent(), {}), (ReasoningReminderPromptComponent(), {}), (ToolLimitWarningPromptComponent(), {}), + # Last block: keeps the live sub-goal + bounce rules inside the + # local attention window of sliding-window models. + (ActiveFocusPromptComponent(), {}), ] for component, extra_kwargs in components: kwargs_to_pass = { diff --git a/artemis/agents/operator/prompts.py b/artemis/agents/operator/prompts.py index e800de234..d572ec91c 100644 --- a/artemis/agents/operator/prompts.py +++ b/artemis/agents/operator/prompts.py @@ -31,6 +31,7 @@ _format_long_output_response, _is_output_long, ) +from artemis.utils.image_codec import image_data_uri from artemis.utils.plan_grammar import parse_plan, render_plan_grammar_spec from artemis.utils.task_tree import SELF_DESCRIBED_MARKER, action_intent_phrase from third_party.mobile_use.utils.logger import get_logger @@ -384,7 +385,7 @@ async def __call__(self, builder: PromptBuilder, state: State, ctx: ArtemisConte builder.add_human_content( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{latest_screenshot_b64}"}, + "image_url": {"url": image_data_uri(latest_screenshot_b64)}, } ) builder.add_human_content(f"--- Visible UI Elements ---\n{minimal_list}") @@ -751,6 +752,63 @@ async def __call__(self, builder: PromptBuilder, state: State, ctx: ArtemisConte ) +#: Header of the standing focus block rendered last in every observation tail. +FOCUS_REMINDER_MARKER = "--- Now ---" + + +class ActiveFocusPromptComponent(PromptComponent): + """Standing tail-end recitation of the active milestone/sub-goal plus the + two rules the bounce checks enforce. + + Rendered last in the observation so these lines sit inside the local + attention window of sliding-window models (e.g. Gemma's window covers only + the final tokens of the whole sequence — the deep system-prompt sections + restating these rules are outside it once the prompt grows). The block is + ephemeral: it leaves the transcript when the turn commits. + """ + + async def __call__(self, builder: PromptBuilder, state: State, ctx: ArtemisContext, **kwargs): + snapshot = parse_plan(kwargs.get("task_plan") or "") + if not snapshot.has_top_level or snapshot.all_top_level_done: + return + milestone = snapshot.active_milestone() + leaf = snapshot.active_leaf(milestone) if milestone else None + lines = [FOCUS_REMINDER_MARKER] + if milestone is not None and leaf is not None: + lines.append( + f'In progress: milestone "{_clip(milestone.text)}" → sub-goal "{_clip(leaf.text)}".' + ) + lines.append( + "Settle it (`[x]` / `[!]`) or open the next `[/]` sub-goal via" + " `update_note` (key `task_plan`) in the same tool-call list as" + " the action." + ) + elif milestone is not None: + lines.append( + f'In progress: milestone "{_clip(milestone.text)}" has no `[/]`' + " sub-goal — open one for what you are doing now via" + " `update_note` (key `task_plan`)." + ) + else: + lines.append("No milestone is marked in progress — mark one `[/]`.") + lines.append( + "If the goal is already achieved on screen, settle the plan and call" + " `mark_done` — do not take further actions. If the goal cannot be" + " achieved (e.g., a required app is not installed), mark the" + " milestone `[!]` blocked instead of searching indefinitely." + ) + lines.append("End the turn only with a Turn-Ending Action.") + text = "\n".join(lines) + if builder.human_footer is not None: + # Legacy path: the template footer renders after every component, + # so the focus block must ride inside it to stay last. + builder.set_human_footer(f"{builder.human_footer.rstrip()}\n\n{text}") + else: + # Transcript path: the tail is the last message; the ephemeral flag + # keeps the block out of the committed transcript. + builder.add_human_content(text, ephemeral=True) + + #: Header of the user-guidance block (shared by the Flash runner and the Pro #: operator; the operator's static prompt explains the channel once). USER_GUIDANCE_MARKER = "--- User Guidance ---" diff --git a/artemis/agents/outputter/outputter.py b/artemis/agents/outputter/outputter.py index 9b75b427f..9a63fcb0c 100644 --- a/artemis/agents/outputter/outputter.py +++ b/artemis/agents/outputter/outputter.py @@ -46,6 +46,7 @@ from artemis.utils.task_tree import get_active_subgoal_hashes from third_party.mobile_use.utils.logger import get_logger from pydantic import BaseModel +from artemis.utils.image_codec import image_data_uri logger = get_logger(__name__) @@ -257,27 +258,31 @@ def _initial_messages( target_schema=_target_schema_json(output_config), plan_and_history=plan_and_history, ) - content: list[dict[str, Any]] = [{"type": "text", "text": human_message}] - - if ctx.data_engine: - verification = _verification_block(ctx, graph_output) - if verification: - content.append({"type": "text", "text": verification}) - - environment = _environment_block(ctx) - if environment: - content.append({"type": "text", "text": environment}) + content: list[dict[str, Any]] = [] raw_data = graph_output.operator_raw_data screenshot_b64 = raw_data.get("screenshot_b64") if raw_data else None if screenshot_b64: + # Image-first: vision models ground better when the screenshot + # precedes the instruction/context text. content.append( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{screenshot_b64}"}, + "image_url": {"url": image_data_uri(screenshot_b64)}, } ) + content.append({"type": "text", "text": human_message}) + + if ctx.data_engine: + verification = _verification_block(ctx, graph_output) + if verification: + content.append({"type": "text", "text": verification}) + + environment = _environment_block(ctx) + if environment: + content.append({"type": "text", "text": environment}) + return [SystemMessage(content=_SYSTEM_MESSAGE), HumanMessage(content=content)] diff --git a/artemis/agents/planner/planner.py b/artemis/agents/planner/planner.py index 02be5c7c4..588bf8e81 100644 --- a/artemis/agents/planner/planner.py +++ b/artemis/agents/planner/planner.py @@ -54,6 +54,7 @@ from artemis.utils.cython_compat import CyFunctionDetector from third_party.mobile_use.utils.decorators import agent_lifecycle_logging from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_codec import image_data_uri logger = get_logger(__name__) @@ -83,6 +84,56 @@ def validate_plan_format(content: str) -> tuple[bool, str]: return True, "" +async def _try_decision_layer( + ctx: ArtemisContext, + initial_goal: str, + content_before: str, + content_after: str, +) -> dict | None: + """Asks the decision layer for the milestone review verdict. + + Returns the ``{"status", "feedback"}`` dict when the endpoint answers at + or above the configured confidence threshold; ``None`` to escalate to + the LLM path. State is text-distilled (goal + plan diff) — decision + models take no pixels. + """ + agent_config = getattr(ctx, "agent_config", None) + cfg = getattr(agent_config, "decision", None) + if not cfg or not cfg.enabled or not getattr(cfg, "planner_validation", True): + return None + from artemis.core.decision_client import DecisionClient + + state = ( + f"Goal: {initial_goal[:500]}\n" + f"Plan before: {content_before[:800]}\n" + f"Plan after: {content_after[:800]}" + ) + client = DecisionClient( + url=cfg.url, model=cfg.model, timeout_seconds=cfg.timeout_seconds + ) + result = await client.anoul( + state, + "Is the revised plan coherent with the goal and strictly better than " + "the previous plan (no lost milestones, no contradictory steps)?", + ) + if not result.answered or result.confidence < cfg.confidence_threshold: + logger.info( + "Decision layer escalated planner validation " + f"(answered={result.answered}, confidence={result.confidence:.2f})" + ) + return None + verdict = bool(result.verdict) + logger.info(f"Decision layer answered planner validation: {verdict} (confidence {result.confidence:.2f})") + return { + "status": "success" if verdict else "failed", + "feedback": ( + "Decision-layer review " + f"({'approved' if verdict else 'flagged'}, " + f"confidence {result.confidence:.2f})." + ), + } + + def _check_gates(ctx: ArtemisContext) -> tuple[bool, bool]: """(midway_checks, final_check): the two check gates, read separately so the planner is taught the judgment moment that actually applies. With both @@ -183,6 +234,14 @@ async def run_async_planner_validation( HumanMessage(content=human_message), ] + # Decision layer: a local decision model (Laya/Jev) can answer the + # advisory yes/no in ~100ms instead of a full LLM turn. Below the + # confidence threshold — or any endpoint failure — fall through to + # the existing model path. + decision_answer = await _try_decision_layer(ctx, initial_goal, content_before, content_after) + if decision_answer is not None: + return decision_answer + # Advisory review on the lightweight judge node (shared default with # the pixel safety net), never the heavyweight Planner model. llm = get_llm(ctx=ctx, name="planner_validation").with_structured_output(ValidationResult) @@ -274,14 +333,31 @@ async def __call__(self, state: State): human_message = Template(human_content).render(**render_kwargs) - human_message_content = [{"type": "text", "text": human_message}] + human_message_content = [] if screenshot_b64: human_message_content.append( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{screenshot_b64}"}, + "image_url": {"url": image_data_uri(screenshot_b64)}, } ) + # Multimodal models ground more reliably when the image precedes the + # instruction text (image-first ordering). + human_message_content.append({"type": "text", "text": human_message}) + # Sliding-window models attend reliably to the tail of the sequence; + # the submission protocol reminder must sit after the screenshot, or + # the image tokens push it out of the local window. + human_message_content.append( + { + "type": "text", + "text": ( + "Reminder: submit the plan by calling `save_note` (key" + " `task_plan`), or `update_note` when revising an existing" + " plan. A tool call is required — a plan written only in" + " your text response is not recognized." + ), + } + ) messages = [ SystemMessage(content=system_message), HumanMessage(content=human_message_content), diff --git a/artemis/agents/validator/precondition_pixel.py b/artemis/agents/validator/precondition_pixel.py index 80a542fb8..784bfe22c 100644 --- a/artemis/agents/validator/precondition_pixel.py +++ b/artemis/agents/validator/precondition_pixel.py @@ -36,6 +36,7 @@ from artemis.services.llm import acomplete_structured from artemis.utils import visualization from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_codec import image_data_uri logger = get_logger(__name__) @@ -116,17 +117,16 @@ def _build_messages( action_name = action_item.get("action") user_content = [ - {"type": "text", "text": _describe_target(action_item)}, + # Images first, then the target description: vision models ground + # better when pixel content precedes the instruction text. + {"type": "image_url", "image_url": {"url": image_data_uri(orig_b64)}}, {"type": "text", "text": "[Image 1 (Reference)]"}, { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{orig_b64}"}, + "image_url": {"url": image_data_uri(live_b64)}, }, {"type": "text", "text": "[Image 2 (Current State)]"}, - { - "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{live_b64}"}, - }, + {"type": "text", "text": _describe_target(action_item)}, ] if state: thoughts = [] diff --git a/artemis/agents/validator/tool_declarations.py b/artemis/agents/validator/tool_declarations.py index fba8dd751..889b41cb2 100644 --- a/artemis/agents/validator/tool_declarations.py +++ b/artemis/agents/validator/tool_declarations.py @@ -45,6 +45,7 @@ READ_NOTE_DOCSTRING, ) from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_codec import image_data_uri logger = get_logger(__name__) @@ -79,7 +80,7 @@ def to_langchain_tool_message(self) -> ToolMessage: content_blocks.append( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{b64_img}"}, + "image_url": {"url": image_data_uri(b64_img)}, } ) return ToolMessage( diff --git a/artemis/agents/video_analyzer/universal_engine.py b/artemis/agents/video_analyzer/universal_engine.py index 96ade3fe7..767a9ec98 100644 --- a/artemis/agents/video_analyzer/universal_engine.py +++ b/artemis/agents/video_analyzer/universal_engine.py @@ -42,6 +42,7 @@ ) from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_codec import image_data_uri logger = get_logger(__name__) @@ -159,7 +160,7 @@ async def _build_universal_user_blocks( user_blocks.append( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{b64_str}"}, + "image_url": {"url": image_data_uri(b64_str)}, } ) diff --git a/artemis/config/agent.py b/artemis/config/agent.py index 6fb00d398..25d66990a 100644 --- a/artemis/config/agent.py +++ b/artemis/config/agent.py @@ -152,6 +152,53 @@ class PlannerValidationConfig(BaseModel): model_config = {"extra": "allow"} +class DecisionLayerConfig(BaseModel): + """System-One decision layer: cheap typed junctions (noul/choice/score) + answered by a local decision model (e.g. Laya) instead of a full LLM call. + + Every junction is advisory — a decision-layer miss or low-confidence + answer always escalates to the existing model path, never rolls back. + """ + + enabled: bool = Field( + default=False, + description="Master switch for the decision layer. Off = identical to upstream behavior.", + ) + url: str = Field( + default="http://127.0.0.1:11434/v1/decide", + description="Jev/Laya-compatible decision endpoint.", + ) + model: str | None = Field( + default="laya", + description="Decision model name sent in the request payload, if the endpoint uses it.", + ) + confidence_threshold: float = Field( + default=0.85, + ge=0.0, + le=1.0, + description="Answers below this confidence escalate to the LLM path.", + ) + timeout_seconds: float = Field( + default=5.0, + gt=0, + description="Per-request decision timeout; expiry escalates to the LLM path.", + ) + planner_validation: bool = Field( + default=True, + description="Offload the advisory planner milestone review to the decision layer.", + ) + flash_loop: bool = Field( + default=False, + description=( + "Hybrid Flash loop: the VLM describes the screen (text only, no tool " + "calls), the decision model picks the next action from a fixed menu " + "built on the live element list. Off → the VLM emits tool calls " + "directly (upstream behavior)." + ), + ) + model_config = {"extra": "allow"} + + class CommitteeConfig(BaseModel): """Configuration specific to Multi-Agent Committee / Council.""" @@ -880,6 +927,10 @@ class AgentGlobalConfig(BaseModel): " ``explorer.flash_mode`` / ``explorer.pro_mode`` decide." ), ) + decision: DecisionLayerConfig = Field( + default_factory=DecisionLayerConfig, + description="System-One decision layer (Laya/Jev) offloading cheap typed junctions.", + ) planner_validation: PlannerValidationConfig = Field( default_factory=PlannerValidationConfig, description="Planner milestone validation runtime options.", diff --git a/artemis/config/local_models.py b/artemis/config/local_models.py new file mode 100644 index 000000000..90b24efa4 --- /dev/null +++ b/artemis/config/local_models.py @@ -0,0 +1,62 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Catalog of local models ARTEMIS can pull and serve itself. + +Each alias maps to a Hugging Face repository servable by ``mlx_vlm.server`` +(``pip install artemis[local]`` on Apple Silicon). Aliases are what the +``"model"`` field in ``artemis.jsonc`` references when ``provider`` is +``"custom"`` and ``api_base`` points at the managed server. +""" + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class LocalModelSpec: + """A local model ARTEMIS can pull and serve.""" + + repo: str + description: str = "" + # Recommended ``max_soft_tokens`` for the model's image processor — + # the vision-token budget per image. Gemma 4 ships with 280, which + # downsamples a phone screenshot to ~528x1152 effective; 1120 keeps + # near-original detail (~1056x2352) and measurably improves detection + # and not-found discipline. ``None`` leaves the shipped default. + vision_soft_tokens: int | None = None + + +LOCAL_MODELS: dict[str, LocalModelSpec] = { + "gemma4-e4b": LocalModelSpec( + repo="mlx-community/gemma-4-e4b-it-4bit", + description="Gemma 4 E4B 4-bit multimodal (tested; ~5 GB pull, ~6 GB resident)", + vision_soft_tokens=1120, + ), +} + +DEFAULT_LOCAL_MODEL = "gemma4-e4b" + + +def resolve_model_ref(alias_or_repo: str) -> tuple[str, str] | None: + """Map a catalog alias or ``org/repo`` id to ``(alias, repo)``. + + Returns ``None`` for bare names that match neither the catalog nor the + ``org/repo`` shape — those cannot be served by ARTEMIS itself. + """ + spec = LOCAL_MODELS.get(alias_or_repo) + if spec is not None: + return alias_or_repo, spec.repo + if "/" in alias_or_repo: + return alias_or_repo, alias_or_repo + return None diff --git a/artemis/config/settings.py b/artemis/config/settings.py index 182d7ec77..6558f22c4 100644 --- a/artemis/config/settings.py +++ b/artemis/config/settings.py @@ -144,6 +144,14 @@ class Settings(BaseSettings): " forever (legacy interactive behavior)." ), ) + LLM_FLASH_HARD_TIMEOUT_SECONDS: float | None = Field( + default=180.0, + description=( + "Hard per-call timeout for Flash operator turns, in seconds." + " Busy time only: paused waits (PAUSE_FILE) do not accrue." + " Raise for slow local providers; None waits indefinitely." + ), + ) # Paths & Storage TRACES_PATH: Path = Field(default_factory=get_default_traces_path) diff --git a/artemis/context.py b/artemis/context.py index 57360c8c8..d07f1f9e6 100644 --- a/artemis/context.py +++ b/artemis/context.py @@ -147,6 +147,9 @@ class ArtemisContext(DeviceClientAccessors, BaseModel): """checkpoint_id -> number of verify-fail repairs already applied.""" assert_halt: bool = False """Latched by an assert failure under ``assert_failure_policy='halt'``.""" + operator_done_claimed: bool = False + """Latched by the operator's ``mark_done`` tool; consumed once by the + convergence gate, which routes the claim to terminal verification.""" final_check_attempts: int = 0 """Number of final-check passes already executed at exit settlement.""" diff --git a/artemis/core/decision_client.py b/artemis/core/decision_client.py new file mode 100644 index 000000000..8d6e120b2 --- /dev/null +++ b/artemis/core/decision_client.py @@ -0,0 +1,177 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Typed decision-model client — the "System One" layer. + +Decision models (Jev from TypeSafe AI, or the open-weight Laya from Convai +Innovations — served locally via Ollama MLX builds) answer *typed* questions +about a distilled text/JSON state in a single non-autoregressive pass: +``choice`` (pick one of labelled options), ``score`` (ordered rubric), or +``noul`` (calibrated yes/no probability). They never emit free text, so +there is nothing to parse and no format to hallucinate. + +Wire format (TypeSafe-compatible, which Laya's ``typed-decisions`` checkpoint +also speaks):: + + POST {url} + {"model": "...", "state": "...", + "questions": {"gate": {"type": "noul", "instructions": "..."}}} + -> + {"answers": {"gate": {"noul": 0.83, "confidence": 0.91}}, ...} + +Used to offload cheap boolean/categorical junctions (validator gates, +planner review, routing triage) from the heavyweight multimodal model. +Fails closed: any transport/parse problem yields ``DecisionResult`` with +``verdict=None`` and ``confidence=0`` so callers escalate to the LLM path. +""" + +import asyncio +from dataclasses import dataclass, field +import json +import urllib.error +import urllib.request + +from third_party.mobile_use.utils.logger import get_logger + +logger = get_logger(__name__) + +DEFAULT_TIMEOUT_SECONDS = 5.0 + + +@dataclass +class DecisionResult: + """One answered question. + + ``verdict`` is the typed answer — bool for ``noul`` (probability >= 0.5), + the winning key for ``choice``, a float for ``score`` — or ``None`` when + the decision failed and the caller should escalate to the model path. + """ + + verdict: bool | str | float | None + confidence: float = 0.0 + probabilities: dict = field(default_factory=dict) + raw: dict = field(default_factory=dict) + + @property + def answered(self) -> bool: + return self.verdict is not None + + +class DecisionClient: + """Minimal urllib client for Jev/Laya-compatible decision endpoints.""" + + def __init__( + self, + url: str, + model: str | None = None, + timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS, + ): + self._url = url + self._model = model + self._timeout = timeout_seconds + + def decide(self, state: str, questions: dict) -> dict[str, DecisionResult]: + """One request, all questions answered in a single forward pass.""" + payload: dict = {"state": state, "questions": questions} + if self._model: + payload["model"] = self._model + request = urllib.request.Request( + self._url, + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=self._timeout) as resp: + body = json.loads(resp.read().decode("utf-8")) + except (urllib.error.URLError, TimeoutError, OSError, ValueError) as e: + logger.warning(f"Decision endpoint {self._url} failed: {e}") + return {name: DecisionResult(verdict=None) for name in questions} + answers = body.get("answers") or {} + return { + name: self._parse_answer(spec, answers.get(name) or {}) + for name, spec in questions.items() + } + + async def adecide(self, state: str, questions: dict) -> dict[str, DecisionResult]: + return await asyncio.to_thread(self.decide, state, questions) + + def noul(self, state: str, question: str) -> DecisionResult: + """Calibrated yes/no — the gate/guardrail primitive.""" + results = self.decide( + state, {"q": {"type": "noul", "instructions": question}} + ) + return results["q"] + + async def anoul(self, state: str, question: str) -> DecisionResult: + return await asyncio.to_thread(self.noul, state, question) + + def choose( + self, state: str, question: str, criteria: dict[str, str] + ) -> DecisionResult: + """Categorical pick over labelled options.""" + results = self.decide( + state, + {"q": {"type": "choice", "instructions": question, "criteria": criteria}}, + ) + return results["q"] + + async def achoose( + self, state: str, question: str, criteria: dict[str, str] + ) -> DecisionResult: + return await asyncio.to_thread(self.choose, state, question, criteria) + + def _parse_answer(self, spec: dict, answer: dict) -> DecisionResult: + qtype = spec.get("type", "noul") + confidence = answer.get("confidence") + try: + confidence = float(confidence) if confidence is not None else 0.0 + except (TypeError, ValueError): + confidence = 0.0 + probabilities = answer.get("probabilities") or {} + if qtype == "noul": + prob = answer.get("noul") + if prob is None: + return DecisionResult(verdict=None, confidence=confidence, raw=answer) + try: + prob = float(prob) + except (TypeError, ValueError): + return DecisionResult(verdict=None, confidence=confidence, raw=answer) + return DecisionResult( + verdict=prob >= 0.5, + confidence=confidence, + probabilities={"yes": prob, "no": 1.0 - prob}, + raw=answer, + ) + if qtype == "choice": + verdict = answer.get("choice") + return DecisionResult( + verdict=verdict, + confidence=confidence, + probabilities=probabilities, + raw=answer, + ) + if qtype == "score": + score = answer.get("score") + try: + verdict = float(score) if score is not None else None + except (TypeError, ValueError): + verdict = None + return DecisionResult( + verdict=verdict, + confidence=confidence, + probabilities=probabilities, + raw=answer, + ) + return DecisionResult(verdict=None, confidence=confidence, raw=answer) diff --git a/artemis/core/diagnostics/engine.py b/artemis/core/diagnostics/engine.py index 97f9d20ca..5b3c91e26 100644 --- a/artemis/core/diagnostics/engine.py +++ b/artemis/core/diagnostics/engine.py @@ -27,6 +27,9 @@ LLMCredentialsProbe, VisionOCRProbe, ) +from artemis.core.diagnostics.probes.local_model_probe import ( + LocalModelEndpointProbe, +) from artemis.core.diagnostics.probes.runtime_probe import ( PythonRuntimeProbe, SystemConfigProbe, @@ -67,6 +70,7 @@ def __init__(self): self._credentials_probe = LLMCredentialsProbe() self._ocr_probe = VisionOCRProbe() self._adb_probe = AdbDeviceProbe() + self._local_model_probe = LocalModelEndpointProbe() self._report_cache: SystemReadinessReport | None = None self._report_cache_time = 0.0 self._report_cache_generation = -1 @@ -80,6 +84,7 @@ def __init__(self): self.register_probe(self._credentials_probe) self.register_probe(self._ocr_probe) self.register_probe(self._adb_probe) + self.register_probe(self._local_model_probe) def register_probe(self, probe: BaseProbe) -> None: """Register a new diagnostic probe.""" diff --git a/artemis/core/diagnostics/probes/credentials_probe.py b/artemis/core/diagnostics/probes/credentials_probe.py index 683e216c3..b07705383 100644 --- a/artemis/core/diagnostics/probes/credentials_probe.py +++ b/artemis/core/diagnostics/probes/credentials_probe.py @@ -144,6 +144,30 @@ async def probe(self) -> ProbeResult: ) api_keys_map[prov_id] = val.strip() + # Config-bound local endpoints (artemis-served models, Ollama, vLLM, + # any OpenAI-compatible server) need no API key: the custom provider + # authenticates to a loopback or self-hosted server. + try: + from artemis.core.diagnostics.probes.local_model_probe import ( + _local_endpoints, + ) + + for base, models in _local_endpoints().items(): + configured_providers.append( + { + "provider": "custom", + "label": "Local Model Endpoint", + "masked": base, + "raw_key": base, + "key": base, + } + ) + api_keys_map["custom"] = base + for alias in models: + api_keys_map.setdefault(alias, base) + except Exception: + pass + if ocr_key and not is_placeholder_key(ocr_key): api_keys_map["ocr"] = ocr_key.get_secret_value() diff --git a/artemis/core/diagnostics/probes/local_model_probe.py b/artemis/core/diagnostics/probes/local_model_probe.py new file mode 100644 index 000000000..f43dbc051 --- /dev/null +++ b/artemis/core/diagnostics/probes/local_model_probe.py @@ -0,0 +1,273 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Local Model Endpoint Readiness Probe. + +When artemis.jsonc routes nodes to a ``custom`` provider (an OpenAI-compatible +endpoint at ``api_base``), every configured model must actually be served +there. This probe verifies the endpoint is reachable and that each required +model appears in ``GET {api_base}/models``. + +Guidance is model-aware: aliases in the Artemis local-model catalog get +``artemis model pull`` / ``artemis model serve`` commands; other models are +BYO — the probe suggests serving them with the endpoint's own tooling +(Ollama, vLLM, LM Studio, ...) instead of a command Artemis cannot run. +""" + +from typing import Any +from urllib.parse import urlparse + +import httpx + +from artemis.config.local_models import resolve_model_ref +from artemis.core.diagnostics.probes.base import BaseProbe +from artemis.core.diagnostics.schema import ( + ProbeAction, + ProbeCategory, + ProbeResult, + ProbeStatus, +) +from third_party.mobile_use.config.llm import LLM +from third_party.mobile_use.utils.logger import get_logger + +logger = get_logger(__name__) + +_PROBE_HTTP_TIMEOUT_S = 5.0 + + +def _artemis_managed(model: str) -> bool: + """Whether the model is an Artemis catalog alias (``artemis model`` + lifecycle commands apply) or a BYO name the endpoint serves itself.""" + try: + from artemis.config.local_models import LOCAL_MODELS + + return model in LOCAL_MODELS + except Exception: + return False + + +def _serve_port(api_base: str) -> str | None: + """Port from an api_base URL, for ``--port`` in serve guidance.""" + try: + from urllib.parse import urlparse + + port = urlparse(api_base).port + return str(port) if port else None + except Exception: + return None + + +def _iter_configured_llms() -> list[LLM]: + """Every LLM/LLMWithFallback configured in the active artemis.jsonc, + including per-node fallbacks.""" + from artemis.config.llm import parse_llm_config + + llms: list[LLM] = [] + cfg = parse_llm_config() + for name in type(cfg).model_fields: + node = getattr(cfg, name, None) + if isinstance(node, LLM): + llms.append(node) + fallback = getattr(node, "fallback", None) + if isinstance(fallback, LLM): + llms.append(fallback) + elif node is not None and hasattr(node, "model_fields_set"): + # Container (e.g. LLMConfigUtils) - walk its LLM fields too. + for sub_name in type(node).model_fields: + sub = getattr(node, sub_name, None) + if isinstance(sub, LLM): + llms.append(sub) + fallback = getattr(sub, "fallback", None) + if isinstance(fallback, LLM): + llms.append(fallback) + return llms + + +def _local_endpoints() -> dict[str, set[str]]: + """Map api_base -> set of model aliases required on it (custom provider).""" + endpoints: dict[str, set[str]] = {} + for llm in _iter_configured_llms(): + if llm.provider == "custom" and llm.api_base: + base = llm.api_base.rstrip("/") + endpoints.setdefault(base, set()).add(llm.model) + return endpoints + + +class LocalModelEndpointProbe(BaseProbe): + """Verify local model endpoints serve every configured custom model.""" + + @property + def probe_id(self) -> str: + return "local_model_endpoint" + + @property + def category(self) -> ProbeCategory: + return ProbeCategory.RUNTIME + + @property + def is_blocker(self) -> bool: + return True + + async def probe(self) -> ProbeResult: + try: + endpoints = _local_endpoints() + except Exception as e: + # A broken config is the SystemConfigProbe's job - don't double-report. + logger.debug(f"local model probe skipped: cannot parse LLM config: {e}") + endpoints = {} + + if not endpoints: + return ProbeResult( + id=self.probe_id, + category=self.category, + title="Local Model Endpoint", + status=ProbeStatus.PASS, + is_blocker=self.is_blocker, + summary="Not used", + description="No node is bound to a local custom endpoint; cloud providers serve all models.", + metadata={"endpoints": {}}, + actions=[ + ProbeAction( + action_type="hint", + label="Cloud Providers", + payload="Configured providers are remote; no local model install is required.", + ) + ], + ) + + reports: list[dict[str, Any]] = [] + missing_actions: list[ProbeAction] = [] + worst_summary = "" + managed_all: set[str] = set() + async with httpx.AsyncClient(timeout=_PROBE_HTTP_TIMEOUT_S) as client: + for base, required in sorted(endpoints.items()): + managed_all.update(a for a in required if _artemis_managed(a)) + try: + resp = await client.get(f"{base}/models") + resp.raise_for_status() + served = { + m["id"] + for m in resp.json().get("data", []) + if isinstance(m, dict) and isinstance(m.get("id"), str) + } + except Exception as e: + reports.append({"api_base": base, "reachable": False, "error": str(e)}) + worst_summary = f"{base} unreachable" + managed = sorted(a for a in required if _artemis_managed(a)) + if managed: + port = _serve_port(base) + port_arg = f" --port {port}" if port else "" + missing_actions.append( + ProbeAction( + action_type="command", + label="Start Model Server", + payload=f"artemis model serve {managed[0]}{port_arg}", + ) + ) + else: + missing_actions.append( + ProbeAction( + action_type="hint", + label="Start Model Server", + payload=( + f"Start the OpenAI-compatible model server for {base}" + " and re-run diagnostics." + ), + ) + ) + continue + missing = sorted(required - served) + reports.append( + { + "api_base": base, + "reachable": True, + "required": sorted(required), + "served": sorted(served), + "missing": missing, + } + ) + for alias in missing: + worst_summary = f"model '{alias}' not installed" + if _artemis_managed(alias): + missing_actions.append( + ProbeAction( + action_type="command", + label=f"Pull {alias}", + payload=f"artemis model pull {alias}", + ) + ) + else: + missing_actions.append( + ProbeAction( + action_type="hint", + label=f"Serve {alias}", + payload=( + f"'{alias}' is not served at {base}. Pull and serve" + " it with that server's own tooling" + " (e.g. `ollama pull`, a vLLM/LM Studio config)" + " or choose a model the endpoint already serves." + ), + ) + ) + + unreachable = [r for r in reports if not r.get("reachable")] + with_missing = [r for r in reports if r.get("missing")] + + if not unreachable and not with_missing: + served_total = sorted({m for r in reports for m in r.get("served", [])}) + return ProbeResult( + id=self.probe_id, + category=self.category, + title="Local Model Endpoint", + status=ProbeStatus.PASS, + is_blocker=self.is_blocker, + summary=f"Serving {len(served_total)} model(s)", + description=( + f"{len(reports)} local endpoint(s) reachable and all configured " + f"model aliases are pulled and serving." + ), + metadata={"endpoints": reports, "managed": sorted(managed_all)}, + actions=[ + ProbeAction( + action_type="hint", + label="Models Ready", + payload=f"Serving: {', '.join(served_total)}", + ) + ], + ) + + if unreachable: + description = ( + "Local model endpoint(s) unreachable: " + + ", ".join(r["api_base"] for r in unreachable) + + ". The agent cannot run until the server is up." + ) + else: + missing_all = sorted({m for r in with_missing for m in r["missing"]}) + description = ( + "Configured model(s) not installed on the local endpoint: " + + ", ".join(missing_all) + + ". Pull them before running a task." + ) + return ProbeResult( + id=self.probe_id, + category=self.category, + title="Local Model Endpoint", + status=ProbeStatus.FAIL, + is_blocker=self.is_blocker, + summary=worst_summary, + description=description, + metadata={"endpoints": reports, "managed": sorted(managed_all)}, + actions=missing_actions, + ) diff --git a/artemis/graph/graph.py b/artemis/graph/graph.py index 499739110..3abbedbc3 100644 --- a/artemis/graph/graph.py +++ b/artemis/graph/graph.py @@ -68,6 +68,7 @@ run_adb_command_wrapper, ) from artemis.tools.diagnostic_tool import ask_diagnoser_wrapper +from artemis.tools.done_tool import mark_done_wrapper from artemis.tools.explorer_tool import ask_explorer_wrapper from artemis.tools.index import get_tools_from_wrappers from artemis.tools.scratchpad import ( @@ -913,6 +914,15 @@ def _terminal_route() -> Literal["exit_settlement", "end"]: logger.info("Plan validation failed, returning to operator for retry.") return "continue" + # An explicit completion claim via the mark_done tool routes to terminal + # verification regardless of plan-ledger state. The claim is consumed once: + # a failed settlement returns the run to the operator, which may claim + # again after acting. + if getattr(ctx, "operator_done_claimed", False): + ctx.operator_done_claimed = False + logger.info("Operator claimed completion via mark_done; routing to settlement.") + return _terminal_route() + file_path = get_note_file_path(ctx.data_engine.base_dir, "task_plan") if not file_path.exists(): logger.warning(f"Task plan file not found at {file_path}") @@ -942,6 +952,14 @@ def _terminal_route() -> Literal["exit_settlement", "end"]: logger.info("All subgoals are completed, ending the goal") return _terminal_route() + # Every top-level milestone settled with at least one [!] blocked: the + # operator has declared the goal infeasible. Route to settlement so the + # final check arbitrates — it either repairs (reverts a subgoal to + # in-progress) or ends with a blocked outcome once its budget is spent. + if snapshot.all_top_level_resolved: + logger.info("All subgoals are resolved with blocked items; routing to settlement.") + return _terminal_route() + return "continue" @@ -982,6 +1000,7 @@ async def get_graph(ctx: ArtemisContext) -> CompiledStateGraph: run_adb_command_wrapper, manage_task_wrapper, ask_explorer_wrapper, + mark_done_wrapper, *HISTORY_TOOL_WRAPPERS, ] # ask_committee does not follow the Operator's pre-decision / turn-ending contract. diff --git a/artemis/interfaces/cli/commands/model.py b/artemis/interfaces/cli/commands/model.py new file mode 100644 index 000000000..e0fab5d49 --- /dev/null +++ b/artemis/interfaces/cli/commands/model.py @@ -0,0 +1,323 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Local on-device model management (artemis model). + +ARTEMIS can pull and serve a local multimodal model itself — no external +serving stack required. ``pull`` downloads weights through huggingface_hub; +``serve`` runs ``mlx_vlm.server`` (installed via the ``local`` extra) as a +managed background process exposing an OpenAI-compatible endpoint that the +``custom`` LLM provider and the local-model readiness probe consume. +""" + +import json +from pathlib import Path +import signal +import subprocess +import sys +import time +from typing import Annotated + +from artemis.config.local_models import DEFAULT_LOCAL_MODEL, LOCAL_MODELS, resolve_model_ref +from third_party.mobile_use.utils.logger import get_logger +from rich.console import Console +from rich.table import Table +import httpx +import typer + +logger = get_logger(__name__) +model_app = typer.Typer(help="Pull and serve local on-device models.") + +_STATE_DIR = Path.home() / ".artemis" / "local_models" +_SERVE_TIMEOUT_S = 300.0 + + +def _state_file(port: int) -> Path: + return _STATE_DIR / f"server-{port}.json" + + +def _read_state(port: int) -> dict | None: + path = _state_file(port) + if not path.exists(): + return None + try: + return json.loads(path.read_text()) + except (OSError, json.JSONDecodeError): + return None + + +def _pid_alive(pid: int) -> bool: + if pid <= 0: + return False + try: + import os + + os.kill(pid, 0) + except OSError: + return False + return True + + +def _require_local_extra() -> None: + try: + import mlx_vlm # noqa: F401 + except ImportError as e: + typer.secho( + "mlx-vlm is not installed. Install the local-model extra first:\n" + " pip install 'artemis[local]' # Apple Silicon only\n" + "On other platforms, run an OpenAI-compatible server (e.g. Ollama) " + "and point api_base at it instead.", + fg=typer.colors.RED, + ) + raise typer.Exit(code=1) from e + + +@model_app.command("list") +def list_models() -> None: + """List local model aliases ARTEMIS can pull and serve.""" + console = Console() + table = Table(title="Local Models") + table.add_column("Alias", style="cyan") + table.add_column("Repository", style="green") + table.add_column("Notes", style="white") + for alias, spec in LOCAL_MODELS.items(): + table.add_row(alias, spec.repo, spec.description) + console.print(table) + console.print( + "\nAliases resolve through [bold]artemis model pull/serve [/bold]; " + "an org/repo id may be used instead of an alias." + ) + + +@model_app.command("pull") +def pull_model( + model: Annotated[ + str, + typer.Argument(help="Catalog alias or org/repo id to download."), + ] = DEFAULT_LOCAL_MODEL, +) -> None: + """Download model weights so they are ready to serve.""" + ref = resolve_model_ref(model) + if ref is None: + typer.secho( + f"Unknown model '{model}'. Run `artemis model list` or pass an org/repo id.", + fg=typer.colors.RED, + ) + raise typer.Exit(code=1) + _require_local_extra() + alias, repo = ref + typer.secho(f"Pulling {alias} ({repo})...", fg=typer.colors.CYAN) + try: + from huggingface_hub import snapshot_download + + path = snapshot_download(repo) + except ImportError as e: + typer.secho( + "huggingface_hub is not installed. Install 'artemis[local]' first.", + fg=typer.colors.RED, + ) + raise typer.Exit(code=1) from e + except Exception as e: + typer.secho(f"Pull failed for {repo}: {e}", fg=typer.colors.RED) + raise typer.Exit(code=1) from e + typer.secho(f"Ready: {alias} cached at {path}", fg=typer.colors.GREEN) + + +_VISION_SOFT_TOKENS_SUPPORTED = (70, 140, 280, 560, 1120) + + +def _snapshot_path(repo: str) -> Path | None: + """Locate the local HF snapshot for ``repo`` without hitting the network.""" + try: + from huggingface_hub import snapshot_download + + return Path(snapshot_download(repo, local_files_only=True)) + except Exception: + return None + + +def _apply_vision_soft_tokens(repo: str, tokens: int) -> bool: + """Write ``image_processor.max_soft_tokens`` into the pulled snapshot. + + ``mlx_vlm.server`` exposes no flag for the vision-token budget; the + image processor reads it from ``processor_config.json`` at load time, + so we adjust the pulled snapshot before spawning. Idempotent. + """ + snapshot = _snapshot_path(repo) + if snapshot is None: + return False + cfg_path = snapshot / "processor_config.json" + try: + cfg = json.loads(cfg_path.read_text()) + except (OSError, json.JSONDecodeError): + return False + image_proc = cfg.get("image_processor") + if not isinstance(image_proc, dict) or "max_soft_tokens" not in image_proc: + return False + image_proc["max_soft_tokens"] = tokens + cfg_path.write_text(json.dumps(cfg, indent=2)) + return True + + +@model_app.command("serve") +def serve_model( + model: Annotated[ + str, + typer.Argument(help="Catalog alias or org/repo id to serve."), + ] = DEFAULT_LOCAL_MODEL, + host: Annotated[str, typer.Option("--host", help="Bind address.")] = "127.0.0.1", + port: Annotated[int, typer.Option("--port", "-p", help="Bind port.")] = 8080, + vision_tokens: Annotated[ + int | None, + typer.Option( + "--vision-tokens", + help="Image soft-token budget (Gemma 4 supports " + f"{_VISION_SOFT_TOKENS_SUPPORTED}; catalog default used when omitted).", + ), + ] = None, +) -> None: + """Start a managed OpenAI-compatible model server in the background.""" + ref = resolve_model_ref(model) + if ref is None: + typer.secho( + f"Unknown model '{model}'. Run `artemis model list` or pass an org/repo id.", + fg=typer.colors.RED, + ) + raise typer.Exit(code=1) + if vision_tokens is not None and vision_tokens not in _VISION_SOFT_TOKENS_SUPPORTED: + typer.secho( + f"--vision-tokens must be one of {_VISION_SOFT_TOKENS_SUPPORTED}.", + fg=typer.colors.RED, + ) + raise typer.Exit(code=1) + _require_local_extra() + alias, repo = ref + tokens = vision_tokens + if tokens is None: + tokens = LOCAL_MODELS[alias].vision_soft_tokens if alias in LOCAL_MODELS else None + if tokens is not None: + if _apply_vision_soft_tokens(repo, tokens): + typer.secho(f"Vision soft tokens per image: {tokens}", fg=typer.colors.CYAN) + else: + typer.secho( + "Could not set the vision-token budget (model not pulled, or its " + "processor has no max_soft_tokens); serving the model default. " + "Run `artemis model pull` first.", + fg=typer.colors.YELLOW, + ) + existing = _read_state(port) + if existing and _pid_alive(int(existing.get("pid", -1))): + typer.secho( + f"A server (pid {existing['pid']}) is already running on {host}:{port}.", + fg=typer.colors.YELLOW, + ) + return + _STATE_DIR.mkdir(parents=True, exist_ok=True) + log_path = _STATE_DIR / f"server-{port}.log" + argv = [ + sys.executable, + "-m", + "mlx_vlm.server", + "--model", + repo, + "--host", + host, + "--port", + str(port), + ] + typer.secho( + f"Starting {alias} ({repo}) on http://{host}:{port}/v1 — log: {log_path}", + fg=typer.colors.CYAN, + ) + with open(log_path, "ab") as log: + proc = subprocess.Popen( + argv, + stdout=log, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + _state_file(port).write_text( + json.dumps({"pid": proc.pid, "alias": alias, "repo": repo, "host": host, "port": port}) + ) + base = f"http://{host}:{port}/v1" + deadline = time.monotonic() + _SERVE_TIMEOUT_S + while time.monotonic() < deadline: + if proc.poll() is not None: + break + try: + resp = httpx.get(f"{base}/models", timeout=2.0) + if resp.status_code == 200: + typer.secho( + f"Serving {alias} at {base} (pid {proc.pid}). Stop with " + f"`artemis model stop --port {port}`.", + fg=typer.colors.GREEN, + ) + return + except httpx.HTTPError: + pass + time.sleep(2.0) + typer.secho( + f"Server did not become ready within {_SERVE_TIMEOUT_S:.0f}s; " + f"check the log at {log_path}.", + fg=typer.colors.RED, + ) + raise typer.Exit(code=1) + + +@model_app.command("stop") +def stop_model( + port: Annotated[int, typer.Option("--port", "-p", help="Port the server was started on.")] = 8080, +) -> None: + """Stop a managed model server started by `artemis model serve`.""" + state = _read_state(port) + if not state: + typer.secho(f"No managed server recorded on port {port}.", fg=typer.colors.YELLOW) + return + pid = int(state.get("pid", -1)) + if not _pid_alive(pid): + _state_file(port).unlink(missing_ok=True) + typer.secho("Server was not running; cleaned up state.", fg=typer.colors.YELLOW) + return + import os + + os.kill(pid, signal.SIGTERM) + _state_file(port).unlink(missing_ok=True) + typer.secho(f"Stopped {state.get('alias', 'server')} (pid {pid}).", fg=typer.colors.GREEN) + + +@model_app.command("status") +def status_model( + port: Annotated[int, typer.Option("--port", "-p", help="Port to check.")] = 8080, + host: Annotated[str, typer.Option("--host", help="Server host.")] = "127.0.0.1", +) -> None: + """Report whether a managed local model server is up.""" + state = _read_state(port) + if state and _pid_alive(int(state.get("pid", -1))): + typer.secho( + f"Running: {state.get('alias')} (pid {state['pid']}) on {state.get('host')}:{port}", + fg=typer.colors.GREEN, + ) + return + try: + resp = httpx.get(f"http://{host}:{port}/v1/models", timeout=3.0) + if resp.status_code == 200: + ids = [m.get("id") for m in resp.json().get("data", [])] + typer.secho( + f"An unmanaged server is serving {ids} on {host}:{port}.", + fg=typer.colors.YELLOW, + ) + return + except httpx.HTTPError: + pass + typer.secho(f"No model server on {host}:{port}.", fg=typer.colors.RED) diff --git a/artemis/interfaces/cli/main.py b/artemis/interfaces/cli/main.py index 629d6b2cb..301904a4f 100644 --- a/artemis/interfaces/cli/main.py +++ b/artemis/interfaces/cli/main.py @@ -22,6 +22,7 @@ from artemis.interfaces.cli.commands.helper import helper_app from artemis.interfaces.cli.commands.init import init_command from artemis.interfaces.cli.commands.mcp import mcp_command +from artemis.interfaces.cli.commands.model import model_app from artemis.interfaces.cli.commands.run import run_command from artemis.interfaces.cli.commands.server import server_app from artemis.interfaces.cli.commands.server_lifecycle import ( @@ -64,6 +65,9 @@ app.command(name="mcp", help="Start the Artemis Model Context Protocol (MCP) server.")(mcp_command) app.add_typer(server_app, name="server", help="Cloud Run proxy and web dashboard server.") app.add_typer(trace_app, name="trace", help="Inspect and query execution traces.") +app.add_typer( + model_app, name="model", help="Pull and serve local on-device models." +) app.add_typer( helper_app, name="helper", help="Manage the Accessibility Helper APK on attached devices." ) diff --git a/artemis/llm/router.py b/artemis/llm/router.py index f2e6b42b6..4e94a7151 100644 --- a/artemis/llm/router.py +++ b/artemis/llm/router.py @@ -111,6 +111,13 @@ class ModelEndpoint(BaseModel): default=False, description="Whether to enable Google Search grounding for Gemini endpoints", ) + coordinate_format: str | None = Field( + default=None, + description=( + "Vision detector point convention, e.g. 'yx_1000' (Gemini default)," + " 'xy_1000', 'yx_px', 'xy_px', 'yx_norm', 'xy_norm'" + ), + ) def cache_key(self) -> tuple: """Key the client cache by endpoint settings, hashing the API key.""" @@ -286,7 +293,8 @@ def create_model(cls, endpoint: ModelEndpoint) -> BaseChatModel: api_key = ( endpoint.api_key or (settings.OPENAI_API_KEY.get_secret_value() if settings.OPENAI_API_KEY else None) - or os.environ.get("OPENAI_API_KEY", "EMPTY") + or os.environ.get("OPENAI_API_KEY") + or "EMPTY" ) base_url = endpoint.api_base or ( str(settings.OPENAI_BASE_URL) if settings.OPENAI_BASE_URL else None @@ -369,7 +377,7 @@ def create_model(cls, endpoint: ModelEndpoint) -> BaseChatModel: elif provider in (ModelProvider.OLLAMA, ModelProvider.VLLM, ModelProvider.CUSTOM): from langchain_openai import ChatOpenAI - api_key = endpoint.api_key or os.environ.get("OPENAI_API_KEY", "EMPTY") + api_key = endpoint.api_key or os.environ.get("OPENAI_API_KEY") or "EMPTY" base_url = endpoint.api_base or os.environ.get( "OPENAI_BASE_URL", "http://localhost:8000/v1" ) diff --git a/artemis/resources/config/artemis.jsonc b/artemis/resources/config/artemis.jsonc index 517d73840..71e59add7 100644 --- a/artemis/resources/config/artemis.jsonc +++ b/artemis/resources/config/artemis.jsonc @@ -45,6 +45,20 @@ "provider": "openai", "model": "llama3.2-vision", "fallback": { "provider": "openai", "model": "llama3.2-vision" } + }, + // Fully local route via any OpenAI-compatible server — Artemis-managed + // models (`artemis model serve `) or BYO (Ollama, vLLM, LM Studio). + // Copy the provider/model/api_base keys into "default" or a node below; + // "custom" skips cloud credential checks and needs no API key. + "local-qwen": { + "provider": "custom", + "model": "qwen3.8-9b", + "api_base": "http://127.0.0.1:8080/v1", + "fallback": { + "provider": "custom", + "model": "qwen3.8-9b", + "api_base": "http://127.0.0.1:8080/v1" + } } }, @@ -74,6 +88,11 @@ // CRITICAL REQUIREMENT: Standard LLMs (including standard Gemini Flash, GPT-4o, Claude 3.7) lack sub-pixel spatial coordinate fine-tuning. // To achieve accurate [x, y] point and bounding box localization, this node MUST use specialized Gemini ER (Embodied Reasoning / Robotics) models: // e.g., "gemini-robotics-er-2-preview". Non-ER models will fail spatial coordinate detection. + // For a local vision model (e.g. qwen-vl via a local server) also set + // "coordinate_format" — detector points are normalized to [x, y] 0-1000: + // "yx_1000" (default, Gemini ER) | "xy_1000" (Qwen-VL verified) | + // "xy_px"/"yx_px" (absolute pixels of the processed image grid) | + // "xy_norm"/"yx_norm" (0-1 floats) "object_detector": { "model": "gemini-robotics-er-2-preview", "fallback": { diff --git a/artemis/services/llm.py b/artemis/services/llm.py index ec91e6fa0..d9e936c90 100644 --- a/artemis/services/llm.py +++ b/artemis/services/llm.py @@ -595,6 +595,11 @@ def with_config(self, *args, **kwargs): def with_structured_output(self, *args, **kwargs): if hasattr(self.base_model, "with_structured_output"): + if self._provider_value() == ModelProvider.CUSTOM.value: + # The local OpenAI-compatible endpoint rejects strict JSON-schema + # response_format requests; function calling yields the same + # structured result through tool calls it does support. + kwargs.setdefault("method", "function_calling") return RobustChatModelWrapper( self.base_model.with_structured_output(*args, **kwargs), self.ctx, @@ -929,6 +934,9 @@ def _get_val(obj, attr, expected_type): return ModelEndpoint( provider=ModelProvider.from_string(provider_val), model_name=str(model_val), + api_base=_get_val(cfg, "api_base", str), + max_tokens=_get_val(cfg, "max_tokens", int), + coordinate_format=_get_val(cfg, "coordinate_format", str), temperature=_get_val(cfg, "temperature", (int, float)) or 0.0, timeout_seconds=_get_val(cfg, "timeout", (int, float)) or 60.0, thinking_budget=_get_val(cfg, "thinking_budget", int), diff --git a/artemis/tools/committee_tool.py b/artemis/tools/committee_tool.py index 28735c903..fbb2da5ed 100644 --- a/artemis/tools/committee_tool.py +++ b/artemis/tools/committee_tool.py @@ -55,6 +55,7 @@ get_recent_subgoal_hashes, ) from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_codec import image_data_uri logger = get_logger(__name__) @@ -273,19 +274,20 @@ async def _execute_committee( async def run_agent_turn(llm, system_prompt, tools, agent_name): current_blackboard = blackboard_path.read_text(encoding="utf-8") - human_content = [ - { - "type": "text", - "text": f"Current Blackboard:\n{current_blackboard}", - } - ] + human_content = [] if screenshot_b64: human_content.append( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{screenshot_b64}"}, + "image_url": {"url": image_data_uri(screenshot_b64)}, } ) + human_content.append( + { + "type": "text", + "text": f"Current Blackboard:\n{current_blackboard}", + } + ) messages = [ SystemMessage(content=system_prompt), @@ -367,19 +369,20 @@ async def run_tool(tc): # Final Synthesis by Planner Avatar current_blackboard = blackboard_path.read_text(encoding="utf-8") - final_human_content = [ - { - "type": "text", - "text": f"Current Blackboard:\n{current_blackboard}", - } - ] + final_human_content = [] if screenshot_b64: final_human_content.append( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{screenshot_b64}"}, + "image_url": {"url": image_data_uri(screenshot_b64)}, } ) + final_human_content.append( + { + "type": "text", + "text": f"Current Blackboard:\n{current_blackboard}", + } + ) msg_final = [ SystemMessage( diff --git a/artemis/tools/done_tool.py b/artemis/tools/done_tool.py new file mode 100644 index 000000000..2d78e19d6 --- /dev/null +++ b/artemis/tools/done_tool.py @@ -0,0 +1,94 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any + +from langchain_core.tools import BaseTool +from pydantic import BaseModel, Field + +from artemis.context import ArtemisContext +from artemis.data_engine.trace import trace_langchain_tool +from artemis.drivers.base import BaseDeviceDriver +from artemis.tools.base import ArtemisTool, ToolCategory +from artemis.tools.tool_wrapper import ToolWrapper +from artemis.utils.cython_compat import CyFunctionDetector +from third_party.mobile_use.utils.logger import get_logger + +logger = get_logger(__name__) + + +class MarkDoneArgs(BaseModel): + """Arguments schema for mark_done tool.""" + + model_config = {"ignored_types": (CyFunctionDetector,)} + reason: str | None = Field( + None, + description="One short sentence stating which goal evidence is on screen.", + ) + + +MARK_DONE_DOCSTRING = ( + "[SHELL] Claims the task goal is complete and requests final verification.\n\n" + "Call this when the user's goal is already satisfied on screen and no further " + "device actions are needed — including goals that were already true when the " + "run started. After this call the checker verifies the screen; if verification " + "fails the run continues with feedback. Do NOT call this while actions remain " + "to be performed.\n\n" + "Prefer this over taking more actions once the goal is visibly achieved." +) + + +class MarkDoneTool(ArtemisTool): + """Structural completion claim: routes the run to final verification.""" + + def __init__(self, category: ToolCategory = "system"): + super().__init__( + name="mark_done", + description=MARK_DONE_DOCSTRING, + args_schema=MarkDoneArgs, + category=category, + ) + + async def execute( + self, + driver: BaseDeviceDriver | None = None, # pylint: disable=unused-argument + ctx: ArtemisContext | None = None, + reason: str | None = None, + **kwargs: Any, + ) -> str: + r = reason if reason is not None else (kwargs.get("reason") or "") + logger.info(f"mark_done called — completion claimed. reason={r!r}") + if ctx is not None: + ctx.operator_done_claimed = True + return ( + "Completion claim recorded. The run now enters final verification: " + "the checker will confirm the goal on screen. If it cannot verify, " + "the run continues — do not take further actions in the meantime." + ) + + +mark_done = MarkDoneTool() +MarkDone = MarkDoneTool + + +def get_mark_done_tool(ctx: ArtemisContext) -> BaseTool: + """Exports mark_done as a LangChain BaseTool.""" + return trace_langchain_tool(mark_done.to_langchain_tool(ctx), ctx) + + +mark_done_wrapper = ToolWrapper( + tool_fn_getter=get_mark_done_tool, + on_success_fn=lambda output: output, + on_failure_fn=lambda err: f"mark_done failed: {err}", +) diff --git a/artemis/tools/explorer_tool.py b/artemis/tools/explorer_tool.py index 14a95b94f..83e86c1f0 100644 --- a/artemis/tools/explorer_tool.py +++ b/artemis/tools/explorer_tool.py @@ -62,6 +62,7 @@ from artemis.utils.element_hit_test import find_element_at_point from artemis.utils.visualization import draw_dots from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_codec import image_data_uri logger = get_logger(__name__) @@ -479,8 +480,8 @@ def render_operator_blocks( logger.error(f"Failed to read annotated explorer image: {e}") return text return [ + {"type": "image_url", "image_url": {"url": image_data_uri(img_b64)}}, {"type": "text", "text": text}, - {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{img_b64}"}}, ] diff --git a/artemis/tools/history/screenshot.py b/artemis/tools/history/screenshot.py index 94f55d343..50b886a51 100644 --- a/artemis/tools/history/screenshot.py +++ b/artemis/tools/history/screenshot.py @@ -32,6 +32,7 @@ from artemis.core.tool_failure import ToolFailure from artemis.utils.visualization import overlay_action_on_screenshot from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_codec import image_data_uri logger = get_logger(__name__) @@ -64,8 +65,8 @@ def to_content_blocks(self) -> list[dict[str, Any]] | str: return self.description encoded = base64.b64encode(self.image_bytes).decode("utf-8") return [ + {"type": "image_url", "image_url": {"url": image_data_uri(encoded)}}, {"type": "text", "text": self.description}, - {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{encoded}"}}, ] diff --git a/artemis/utils/apple_vision_ocr.py b/artemis/utils/apple_vision_ocr.py new file mode 100644 index 000000000..e03df770c --- /dev/null +++ b/artemis/utils/apple_vision_ocr.py @@ -0,0 +1,106 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Standalone Apple Vision OCR worker — run as ``python apple_vision_ocr.py``. + +Reads image bytes on stdin and writes a JSON list of +``{"text": str, "position": [{"x": int, "y": int}, ...]}`` on stdout, +matching the Google Vision ``boundingPoly.vertices`` contract. + +This module is deliberately free of artemis imports and runs in a clean +interpreter: once cv2 (and its GPU/OpenCL contexts) is loaded in the host +process, ``VNRecognizeTextRequest`` fails intermittently with e5rt +compute-stream errors. A subprocess keeps the ML stack isolated. +""" + +# pyright: reportAttributeAccessIssue=false +# pyobjc exposes AppKit/Vision classes dynamically; static stubs have no symbols. + +import io +import json +import sys + + +def _vision_box_to_vertices(bounding_box, width: int, height: int) -> list[dict[str, int]]: + """Convert a Vision normalized bounding box (origin bottom-left) to + pixel-space vertices in Google's boundingPoly order: TL, TR, BR, BL.""" + x0 = bounding_box.origin.x * width + x1 = (bounding_box.origin.x + bounding_box.size.width) * width + y_top = (1.0 - bounding_box.origin.y - bounding_box.size.height) * height + y_bottom = (1.0 - bounding_box.origin.y) * height + return [ + {"x": round(x0), "y": round(y_top)}, + {"x": round(x1), "y": round(y_top)}, + {"x": round(x1), "y": round(y_bottom)}, + {"x": round(x0), "y": round(y_bottom)}, + ] + + +def ocr_image(image_bytes: bytes) -> list[dict]: + """On-device text recognition via Apple's Vision framework.""" + import Vision + from Foundation import NSData + from PIL import Image + + with Image.open(io.BytesIO(image_bytes)) as img: + width, height = img.width, img.height + + request = Vision.VNRecognizeTextRequest.alloc().init() + request.setRecognitionLevel_(Vision.VNRequestTextRecognitionLevelAccurate) + request.setUsesLanguageCorrection_(True) + # The first entry in recognitionLanguages dominates per-request — en-US + # first garbles CJK text. Per-observation language detection (revision 3) + # avoids ordering games: every supported language is enabled and Vision + # picks the right one for each text region. + request.setAutomaticallyDetectsLanguage_(True) + langs, _ = ( + Vision.VNRecognizeTextRequest.supportedRecognitionLanguagesForTextRecognitionLevel_revision_error_( + Vision.VNRequestTextRecognitionLevelAccurate, + request.revision(), + None, + ) + ) + if langs: + request.setRecognitionLanguages_(list(langs)) + + ns_data = NSData.dataWithBytes_length_(image_bytes, len(image_bytes)) + handler = Vision.VNImageRequestHandler.alloc().initWithData_options_(ns_data, None) + ok, error = handler.performRequests_error_([request], None) + if not ok: + raise RuntimeError(f"VNRecognizeTextRequest failed: {error}") + + results = [] + for observation in request.results() or []: + candidates = observation.topCandidates_(1) + if not candidates: + continue + text = str(candidates[0].string()).strip() + if not text: + continue + results.append( + { + "text": text, + "position": _vision_box_to_vertices(observation.boundingBox(), width, height), + } + ) + return results + + +def main() -> None: + image_bytes = sys.stdin.buffer.read() + json.dump(ocr_image(image_bytes), sys.stdout) + + +if __name__ == "__main__": + main() diff --git a/artemis/utils/image_codec.py b/artemis/utils/image_codec.py new file mode 100644 index 000000000..f791019fb --- /dev/null +++ b/artemis/utils/image_codec.py @@ -0,0 +1,52 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Base64 image data-URI helpers. + +OpenAI-compatible servers differ in how strictly they treat the declared +MIME type: some (e.g. strict local gateways) validate it against the actual +payload and reject a PNG byte stream labeled ``image/jpeg``. Always sniff the +magic bytes instead of assuming JPEG. +""" + +import base64 + +_JPEG_SOI = b"\xff\xd8\xff" +_PNG_SIG = b"\x89PNG" +_WEBP_RIFF = b"RIFF" +_WEBP_TAG = b"WEBP" + + +def sniff_image_mime(image_bytes: bytes) -> str: + """Return the ``image/*`` MIME type for encoded image bytes.""" + if image_bytes[:4] == _PNG_SIG: + return "image/png" + if image_bytes[:3] == _JPEG_SOI: + return "image/jpeg" + if image_bytes[:4] == _WEBP_RIFF and image_bytes[8:12] == _WEBP_TAG: + return "image/webp" + return "image/jpeg" + + +def image_data_uri(b64_data: str) -> str: + """Wrap a base64 payload in a data URI with the MIME type sniffed from its + decoded head. Unknown or corrupt prefixes keep the historic ``image/jpeg`` + label. + """ + try: + head = base64.b64decode(b64_data[:24]) + mime = sniff_image_mime(head) + except Exception: + mime = "image/jpeg" + return f"data:{mime};base64,{b64_data}" diff --git a/artemis/utils/ocr_api.py b/artemis/utils/ocr_api.py index 56b7c2356..8ad937ea8 100644 --- a/artemis/utils/ocr_api.py +++ b/artemis/utils/ocr_api.py @@ -12,7 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import asyncio +import base64 +import json import os +from pathlib import Path +import sys from typing import Any import httpx @@ -22,6 +27,13 @@ logger = get_logger(__name__) _HTTP_CLIENT: httpx.AsyncClient | None = None +# Provider selection via ARTEMIS_OCR_PROVIDER: "auto" (default) prefers +# on-device Apple Vision on macOS and falls back to Google Cloud Vision when +# an API key exists; "apple" / "google" pin a single provider. +_OCR_PROVIDER_ENV = "ARTEMIS_OCR_PROVIDER" +_APPLE_VISION_SUPPORTED: bool | None = None +_APPLE_OCR_WORKER = Path(__file__).with_name("apple_vision_ocr.py") + def get_http_client() -> httpx.AsyncClient: global _HTTP_CLIENT @@ -30,29 +42,131 @@ def get_http_client() -> httpx.AsyncClient: return _HTTP_CLIENT -def is_ocr_configured() -> bool: - """Checks if a valid Google Vision OCR API key is configured.""" +def _ocr_provider() -> str: + return os.environ.get(_OCR_PROVIDER_ENV, "auto").strip().lower() + + +def _apple_vision_supported() -> bool: + """True on macOS when the pyobjc Vision bindings are importable and the + standalone worker script exists. The real OCR runs in a subprocess: once + cv2 is loaded in this process, VNRecognizeTextRequest intermittently + fails compiling its ANE compute stream (e5rt errors).""" + global _APPLE_VISION_SUPPORTED + if _APPLE_VISION_SUPPORTED is None: + if sys.platform != "darwin" or getattr(sys, "frozen", False): + _APPLE_VISION_SUPPORTED = False + else: + try: + import Vision # noqa: F401 + + _APPLE_VISION_SUPPORTED = _APPLE_OCR_WORKER.is_file() + except ImportError: + _APPLE_VISION_SUPPORTED = False + return _APPLE_VISION_SUPPORTED + + +def _google_vision_key() -> str | None: + """Raw Google Vision API key, or None (placeholders are not filtered).""" ocr_secret = settings.get_api_key("ocr") - if ocr_secret and ocr_secret.get_secret_value().strip(): - val = ocr_secret.get_secret_value().strip() - if val not in ("API_KEY", "your_google_cloud_vision_api_key_here"): - return True - for env_var in ("OCR_API_KEY", "VISION_API_KEY"): - val = os.environ.get(env_var) + return ( + (ocr_secret.get_secret_value() if ocr_secret else None) + or os.environ.get("OCR_API_KEY") + or os.environ.get("VISION_API_KEY") + ) + + +def _google_vision_key_present() -> bool: + """True when a non-placeholder Google Vision key is configured.""" + for val in (_google_vision_key(),): if ( val and val.strip() - and val.strip() not in ("API_KEY", "your_google_cloud_vision_api_key_here") + and val.strip() + not in ( + "API_KEY", + "your_google_cloud_vision_api_key_here", + ) ): return True return False +def is_ocr_configured() -> bool: + """True when some OCR provider is usable: Apple Vision on macOS, or a + configured Google Cloud Vision API key.""" + provider = _ocr_provider() + if provider == "apple": + return _apple_vision_available() + if provider == "google": + return _google_vision_key_present() + return _apple_vision_available() or _google_vision_key_present() + + +# A failing Vision attempt can stall for minutes inside the ObjC call (e5rt +# compute-stream compile), so every subprocess invocation is time-bounded and +# retried once in a fresh interpreter before degrading. When the OS-level +# Vision ML service is wedged, every call would burn 2x timeout forever — +# after consecutive failures the provider is disabled until a success resets +# the counter. +_APPLE_OCR_TIMEOUT_S = float(os.environ.get("APPLE_OCR_TIMEOUT", "20")) +_APPLE_OCR_MAX_CONSECUTIVE_FAILURES = 2 +_apple_vision_consecutive_failures = 0 + + +def _apple_vision_available() -> bool: + return ( + _apple_vision_supported() + and _apple_vision_consecutive_failures < _APPLE_OCR_MAX_CONSECUTIVE_FAILURES + ) + + +async def _apple_vision_subprocess(image_bytes: bytes) -> list[dict[str, Any]]: + proc = await asyncio.create_subprocess_exec( + sys.executable, + str(_APPLE_OCR_WORKER), + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + out, err = await asyncio.wait_for( + proc.communicate(input=image_bytes), timeout=_APPLE_OCR_TIMEOUT_S + ) + except BaseException: + proc.kill() + await proc.wait() + raise + if proc.returncode != 0: + tail = err.decode(errors="replace")[-300:] if err else "" + raise RuntimeError(f"Apple Vision worker exited {proc.returncode}: {tail}") + return json.loads(out or b"[]") + + +async def _run_apple_vision(image_bytes: bytes) -> list[dict[str, Any]]: + global _apple_vision_consecutive_failures + try: + results = await _apple_vision_subprocess(image_bytes) + except Exception as e: + logger.warning(f"Apple Vision OCR attempt failed ({e}); retrying once") + try: + results = await _apple_vision_subprocess(image_bytes) + except Exception: + _apple_vision_consecutive_failures += 1 + raise + _apple_vision_consecutive_failures = 0 + return results + + async def perform_ocr( screenshot_b64: str, client: httpx.AsyncClient | None = None, ) -> list[dict[str, Any]]: - """Calls Google Vision API for text recognition on an image. + """Runs text recognition on an image with the selected provider. + + Provider order under "auto": Apple Vision on-device (macOS) first, then + Google Cloud Vision when an API key is configured. An empty Apple result + is authoritative — Google is only consulted when Apple Vision is + unavailable or raises. Args: screenshot_b64: Base64 encoded screenshot image. @@ -61,12 +175,20 @@ async def perform_ocr( Returns: A list of dictionaries containing detected text and position vertices. """ - ocr_secret = settings.get_api_key("ocr") - api_key = ( - (ocr_secret.get_secret_value() if ocr_secret else None) - or os.environ.get("OCR_API_KEY") - or os.environ.get("VISION_API_KEY") - ) + provider = _ocr_provider() + if provider in ("auto", "apple"): + if _apple_vision_available(): + try: + return await _run_apple_vision(base64.b64decode(screenshot_b64)) + except Exception as e: + logger.warning(f"Apple Vision OCR failed: {e}") + elif provider == "apple": + logger.warning("ARTEMIS_OCR_PROVIDER=apple but Apple Vision is unavailable") + return [] + if provider == "apple": + return [] + + api_key = _google_vision_key() if not api_key: return [] diff --git a/artemis/utils/plan_grammar.py b/artemis/utils/plan_grammar.py index 374eef74c..4f786039c 100644 --- a/artemis/utils/plan_grammar.py +++ b/artemis/utils/plan_grammar.py @@ -102,6 +102,10 @@ def is_active(self) -> bool: def is_done(self) -> bool: return self.status == STATUS_DONE + @property + def is_blocked(self) -> bool: + return self.status == STATUS_BLOCKED + @property def is_loop(self) -> bool: return bool(_LOOP_TAG_RE.search(self.text)) @@ -155,6 +159,17 @@ def all_top_level_done(self) -> bool: top = self.top_level return bool(top) and all(i.is_done for i in top) + @property + def all_top_level_resolved(self) -> bool: + """Every top-level milestone is settled — completed or declared blocked. + + Unlike ``all_top_level_done`` this includes ``[!]`` items: an + all-blocked plan is a terminal state the settlement/checker arbitrates, + not a reason to loop the operator forever. + """ + top = self.top_level + return bool(top) and all(i.is_done or i.is_blocked for i in top) + @property def continuous_top_level(self) -> tuple[PlanItem, ...]: return tuple(i for i in self.top_level if i.is_continuous) diff --git a/config/artemis.gemma4.jsonc b/config/artemis.gemma4.jsonc new file mode 100644 index 000000000..8fd317ee7 --- /dev/null +++ b/config/artemis.gemma4.jsonc @@ -0,0 +1,331 @@ +{ + // ============================================================================== + // ☕ ARTEMIS Autonomous Agent & Framework Configuration — GEMMA 4 SPIKE + // ============================================================================== + // Single-model deployment: every node (reasoning + vision + grounding) resolves + // to one gemma4-e4b endpoint on the local OpenAI-compatible model server. + // + // ACTIVATION (pick one): + // export ARTEMIS_ARTEMIS_JSONC=config/artemis.gemma4.jsonc + // # or rename this file over artemis.jsonc + // + // PREREQUISITE: the :8080 platform must actually serve the "gemma4-e4b" alias + // (curl http://127.0.0.1:8080/v1/models). Until it does, keep artemis.jsonc. + // + // Grounding notes: + // - Gemma 4 emits detection natively as {"box_2d": [ymin,xmin,ymax,xmax]} + // on a 0-1000 grid = this repo's default "yx_1000" coordinate_format, so + // no override is set. The detector now derives "point" from the box center. + // - Operator targets already use [x,y] on the same 0-1000 grid. + + // ------------------------------------------------------------------------------ + // 1. Global Default Model (Applied to all sub-agents unless overridden below) + // ------------------------------------------------------------------------------ + "default": { + "provider": "custom", + // One unified multimodal model serves every role; the server alias must + // match the platform's registry entry (Gemma 4 E4B via the mlx-vlm provider, + // e.g. mlx-community/gemma-4-E4B-it-4bit). + "model": "gemma4-e4b", + "api_base": "http://127.0.0.1:8080/v1", + // Local reasoning + vision encoder; keep the same generous timeout the + // qwen deployment needed for thinking preambles. + "timeout": 300, + "fallback": { + "provider": "custom", + "model": "gemma4-e4b", + "api_base": "http://127.0.0.1:8080/v1", + "timeout": 300 + } + }, + + // ------------------------------------------------------------------------------ + // 2. Ready-to-Use Presets (Switch easily in code or CLI) + // ------------------------------------------------------------------------------ + "presets": { + "gemini-flagship": { + "provider": "google", + "model": "gemini-3.8-flash", + "fallback": { "provider": "google", "model": "gemini-3.7-flash" } + }, + "gemini-flash": { + "provider": "google", + "model": "gemini-2.5-flash", + "fallback": { "provider": "google", "model": "gemini-2.0-flash" } + }, + "openai-gpt4o": { + "provider": "openai", + "model": "gpt-4o", + "fallback": { "provider": "openai", "model": "gpt-4o-mini" } + }, + "cost-saving": { + "provider": "google", + "model": "gemini-3.5-flash-lite", + "fallback": { "provider": "google", "model": "gemini-3.1-flash-lite" } + }, + "local-ollama": { + "provider": "openai", + "model": "llama3.2-vision", + "fallback": { "provider": "openai", "model": "llama3.2-vision" } + }, + // Unified single-model route via a local OpenAI-compatible server. + "local-gemma4": { + "provider": "custom", + "model": "gemma4-e4b", + "api_base": "http://127.0.0.1:8080/v1", + "fallback": { + "provider": "custom", + "model": "gemma4-e4b", + "api_base": "http://127.0.0.1:8080/v1" + } + } + }, + + // ------------------------------------------------------------------------------ + // 3. Agent & Sub-Agent Node Customizations (Optional - Inherits from 'default') + // ------------------------------------------------------------------------------ + "nodes": { + // 🧠 Planner: High-level goal decomposition and strategic step planning + "planner": { + "thinking_level": "high" + }, + + // 🎯 Operator: vision-capable by inheritance now — one model serves all roles. + "operator": { + "thinking_level": "medium", + "include_thoughts": true + }, + + // 🎬 Video Analyzer: inherits the unified endpoint; universal multi-model path + // feeds extracted keyframes as images. + "video_analyzer": {}, + + // 👁️ Object Detector: Gemma 4 native box_2d output is yx_1000 = the detector's + // declared default, so no coordinate_format override. box_2d collapses to a + // center point (and stays available as normalized bounds) downstream. + "object_detector": { + "timeout": 300 + }, + + // 🔍 Explorer: inherits the unified endpoint. + "explorer": { + "thinking_level": "medium" + }, + + // 📋 Summarizer: Aggregates step history and generates structured outcome + "summarizer": {}, + + // 🔽 Hopper: Fast lightweight package name locator + "hopper": {}, + + // 📝 Outputter: Synthesizes execution trace into final output report + "outputter": {}, + + // 🪵 Log Analyzer: grounding is a Gemini-only feature — ineffective here. + "log_analyzer": {}, + + // 🩺 Diagnoser Expert + "diagnoser_expert": {}, + // 🏛️ Committee Council Members + "planner_avatar": {}, + "history_analyzer_expert": {}, + // 🔎 Checker: Plan-declared checkpoint verification and exit final review. + "checker": { + "thinking_level": "medium", + "include_thoughts": true + } + }, + + // ------------------------------------------------------------------------------ + // 4. Runtime Mode & Agent Capabilities Configuration (Dual-Engine: Flash vs. Pro) + // ------------------------------------------------------------------------------ + "agent": { + // ========================================================================== + // ⚡ 4.1 FLASH Profile Options (Reactive Single-Agent Lightweight Loop) + // ========================================================================== + // Best for: Simple, fast, deterministic tasks; runs unbounded by default. + // Execution: Observe-Think-Act single LLM reactive loop without graph overhead. + "flash": { + "max_turns": 0, // Maximum reactive turns before stopping (0 = unlimited; history is bounded by transcript compression, not by a turn cap) + "explorer_mode": "flash", // Explorer tier for the Flash runner: 'flash' | 'pro' | 'ultra' (see top-level "explorer") + // 🧠 Step Summarizer: Asynchronous visual context compressor + // Summarizes intermediate steps in background without blocking the runner, + // replacing images only after strictly objective, high-density summaries are ready. + // LOCAL-MODEL NOTE: the visual-transition lens bypasses the LLM router + // (get_google_llm) — with no GOOGLE_API_KEY it crashes runner init. + // Disabled for the local-provider deployment. + "step_summarizer": { + "enabled": false, // Whether background step summarization is active + "model": "gemini-3.5-flash-lite", // Lightweight model for fast background inference + "prune_history_xml": true // Prune outdated heavy XML trees from historical steps + } + }, + // ========================================================================== + // 🚀 4.2 PRO Profile Options (Deep Multi-Agent Graph Closed-Loop) + // ========================================================================== + // Best for: Complex, exploratory, dynamic multi-branch tasks. + // Execution: LangGraph multi-node graph with a living task plan, a pre-execution safety net (single actions) / fast-action bursts (multi-action turns), + // execution-incident recovery handled by the Operator itself (no separate repair agent), and a read-only Checker for checkpoint verification + exit final review. + "pro": { + // 🧭 Explorer: Perception sub-agent under Pro profile ('flash', 'pro', or 'ultra') + "explorer": { + "mode": "flash" // Explorer tier for Pro nodes (Operator/Validator): 'flash' | 'pro' | 'ultra' + }, + // 🎯 Planner Validation: Async verification when task plan milestones change + "planner_validation": { + "enabled": true // On by default: every milestone text change is reviewed (advisory: hint + reason only, never rolled back) + }, + // 🏛️ Committee: Multi-Agent Council debate tool for Operator + "committee": { + "enabled": false, + "debate_rounds": 2 + }, + // 🔎 Checker: Plan-declared checkpoint verification (optimistic async) + exit final review + "checker": { + "enabled": true, // Master switch (compat alias): false disables both gates below + "midway_checks": false, // Plan-declared midway checkpoints (off in the factory layering) + "final_check": true, // Exit final review against the user's original goal (on by default) + "max_iterations": 20, + "final_check_max_attempts": 3, + "checkpoint_max_repairs": 2, + "max_concurrent_checkpoints": 3, + "checkpoint_timeout": 180.0, + "settlement_timeout": 120.0, + "assert_failure_policy": "continue", // 'continue' or 'halt' on assert failures + "device_probes": true // Checker's enumerated read-only device probes + }, + // 🎬 Video Analyzer & Screen Recording: Continuous screen recording via scrcpy and temporal analysis + "video_analyzer": { + "enabled": true, // Whether continuous video recording and video tools are active (default: auto-detected) + "enable_ledger": true + } + }, + // ========================================================================== + // 🌐 4.3 Shared Capabilities & Global Overrides + // ========================================================================== + // Shared step memory and history compression + // Transcript ledger + layered compression for the Pro operator. + "memory": { + // Background summarization runtime (visual/capsule lenses) + "runtime": { + "max_concurrency": 2, // 1→2 per 2026-09-01 baseline: serial queue pushed summary-ready P90 past the scrub window + "retry_limit": 3, + "flush_timeout_s": 30 + }, + "transcript": { + // ROLLBACK SWITCH: set false to restore the legacy per-turn 2-message + // prompt path byte-for-byte (L2/L3 chunk compression rides this same + // flag). See services/token_meter.py for measured session token usage + // and cache-hit ratios. + // LOCAL-MODEL NOTE: transcript + capsule lens are Google-bound + // (get_google_llm) — disabled under the local-provider deployment. + "enabled": false, + "context_budget_tokens": 80000, + // Below start_ratio of the operator's own measured prompt size, closed + // segments only prepare their summary; the original turns are kept in + // context and replaced once occupancy reaches this ratio. Measurements + // come from the operator; other agents' calls do not affect this gate. + "start_ratio": 0.35, + "soft_ratio": 0.7, + "hard_ratio": 0.9, + "min_active_steps": 5, + // Screenshot edge: 6 while the operator's measured context is below + // start_ratio (nothing is being replaced yet), 3 from there up. A + // screenshot reaching the edge is replaced in place by its visual + // summary (the 'Current Screenshot' label leaves with it) and never + // restored; the depth only decides how long new ones stay. + "image_scrub_depth": 3, + "image_scrub_depth_relaxed": 6, + "pending_grace_steps": 3, + // Text edge: once a newer observation exists (depth 1) the UI element + // list, the plan recitation and the per-turn ephemeral blocks + // (reminders, hints, guidance wrappers) leave the older observation. + // User instruction bodies remain in history. The text edge stays + // shallow on purpose: only the live observation carries an indexed + // element list, so a stale [n] index can never be picked up as a + // target. + "xml_scrub_depth": 1, + // Local historical-state hint (no model call); distance calibrated + // 2026-09-01 from 460 dHash-stamped on-device steps (same screen <=4, + // different screens >=7). + "similarity_hint": true, + "similarity_max_distance": 5 + }, + // Segment (L2) chunk compression: three-band chunks, era folding + "chunking": { + "max_steps": 12, + // Size/pressure closes wait until at least this many turns are + // eligible, so heavy turns do not become one-step chunks (one capsule + // call each); milestone closes are exempt, the hard line waives it. + "min_steps": 3, + "target_source_tokens": 2000, + "model": "gemini-3.8-flash", + "max_chunks": 8 + // "max_eras": 8 // independent era cap; omitted = follows max_chunks + }, + // search_history: deterministic keyword / step-range lookup over stored history + "recall": { + "enabled": true, + "max_results": 5, + "max_text_tokens": 2000, + "screen_scan_steps": 150 + }, + // replay_steps: full step replays (whole trailing steps are dropped over budget) + "replay": { + "max_steps": 5, + "max_tokens": 12000 + } + }, + // 🎬 Screen Recording & Video Analyzer Configuration + "video_analyzer": { + "enabled": true, // Whether continuous screen recording via scrcpy is active (default: auto-detected) + "enable_ledger": true + // "processing": "auto", // How Gemini sub-agents watch a clip: auto (agentic on Gemini 3.6+ Flash, static elsewhere) | agentic | static + // "agentic_chunk_size_seconds": 600, // Longest clip one agentic sub-agent takes; static mode keeps chunk_size_seconds (60) + // "agentic_call_timeout_seconds": 300 // Hard timeout for one agentic sub-agent response + }, + // 📝 Outputter: Post-execution report synthesis and structured schema extraction + // Reviews action history, notes, and verifies goal achievement to synthesize final output. + "outputter": { + "enabled": true, // Whether Outputter is mounted to synthesize final outputs + "force_synthesis": false // Force Outputter synthesis even without explicit output schema + }, + // 🧠 Decision Layer (System One): a local decision model (Laya via Ollama + // or a Jev-compatible endpoint) answers cheap typed junctions — yes/no + // gates, categorical picks, rubric scores — in ~100ms instead of a full + // Gemma turn. Advisory only: below-threshold or unreachable answers + // escalate to the existing model path. Text/JSON state only — no pixels. + "decision": { + "enabled": false, // opt-in; off = upstream-identical behavior + "url": "http://127.0.0.1:11434/v1/decide", + "model": "laya", + "confidence_threshold": 0.85, + "timeout_seconds": 5.0, + "planner_validation": true, // offload the advisory milestone review first + // Hybrid Flash loop: Gemma plans once, then per turn describes the + // screen (text only — no tool-call envelopes to mangle), and the + // decision model picks the next action from a fixed menu of element + // taps + meta-actions. Inconclusive/low-confidence verdicts escalate + // to the standard tool-call turn. + "flash_loop": false + }, + // 🧭 Explorer tier: the perception sub-agent behind `ask_explorer`. + // Tiers are a user setting; calling agents never see which tier is active. + // flash - one-shot visual detection on the screenshot (fastest) + // pro - 3-turn reasoning loop: UI-tree search + coordinate audit + detection + // ultra - 8-turn loop adding zoom, OCR and pixel-level image processing (slowest) + // `flash.explorer_mode` and `pro.explorer.mode` above are the per-profile knobs; + // `caching` (Gemini explicit context caching) defaults per tier: off for pro, on for ultra. + "explorer": { + "default_version": "flash" // Fallback tier when no profile knob applies + // "caching": true, // Uncomment to force context caching for pro/ultra + }, + // Advanced: per-agent tier override, e.g. {"validator": "ultra"}. Takes + // precedence over the profile knobs; leave empty to let them decide. + "explorer_versions": {}, +// 🚫 Per-agent Tool Denylists + "denylisted_tools": { + "explorer": [] // Perception tools to withhold from the Explorer, e.g. "ask_image_processor" + } + } +} diff --git a/config/artemis.jsonc b/config/artemis.jsonc index 517d73840..c357ee9d2 100644 --- a/config/artemis.jsonc +++ b/config/artemis.jsonc @@ -7,13 +7,20 @@ // 1. Global Default Model (Applied to all sub-agents unless overridden below) // ------------------------------------------------------------------------------ "default": { - "provider": "google", - "model": "gemini-3.8-flash", - "thinking_level": "medium", + "provider": "custom", + // gemma4-e4b via a local OpenAI-compatible model server (mlx-vlm gemma4 route); + // vision nodes use gemma4-e4b (semantic/grounding); vision-hybrid + // is the OCR-only route (Apple Vision first, qwen-vl escalation). + "model": "gemma4-e4b", + "api_base": "http://127.0.0.1:8080/v1", + // Local 9B reasoning runs ~15-30 tok/s with a thinking preamble; + // the 60s endpoint default times out mid-response on dense prompts. + "timeout": 300, "fallback": { - "provider": "google", - "model": "gemini-3.7-flash", - "thinking_level": "medium" + "provider": "custom", + "model": "qwen3.8-9b", + "api_base": "http://127.0.0.1:8080/v1", + "timeout": 300 } }, @@ -58,27 +65,52 @@ }, // 🎯 Operator: Screen perception, element interaction, and direct execution + // MUST stay on a vision-capable model — it consumes the screenshot every + // turn; the platform hard-rejects image parts on text-only models (400). "operator": { + "model": "gemma4-e4b", "thinking_level": "medium", - "include_thoughts": true + "include_thoughts": true, + "fallback": { + "provider": "custom", + "model": "qwen-vl", + "api_base": "http://127.0.0.1:8080/v1" + } }, // 🎬 Video Analyzer: Multimodal screen recording verification and temporal analysis - // NOTE: Dual-path engine automatically routes: - // - Native Gemini Files API for Gemini models with Google API keys (long video file stream); - // Gemini 3.6+ Flash sub-agents use agentic video understanding by default (see video_analyzer.processing below) - // - Universal Multi-Model Engine for Claude 3.5/3.7, GPT-4o, OpenRouter, Vertex AI (dynamic keyframe extraction + multimodal tool calling) - "video_analyzer": {}, + // Local route: universal multi-model path feeds extracted keyframes to + // qwen-vl (native Gemini Files API path only exists for google provider). + "video_analyzer": { + "provider": "custom", + "model": "gemma4-e4b", + "api_base": "http://127.0.0.1:8080/v1", + "fallback": { + "provider": "custom", + "model": "qwen-vl", + "api_base": "http://127.0.0.1:8080/v1" + } + }, // 👁️ Object Detector & Explorer (Flash Mode): Visual coordinates grounding and bounding boxes - // CRITICAL REQUIREMENT: Standard LLMs (including standard Gemini Flash, GPT-4o, Claude 3.7) lack sub-pixel spatial coordinate fine-tuning. - // To achieve accurate [x, y] point and bounding box localization, this node MUST use specialized Gemini ER (Embodied Reasoning / Robotics) models: - // e.g., "gemini-robotics-er-2-preview". Non-ER models will fail spatial coordinate detection. + // Local gemma4-e4b via a local OpenAI-compatible model server. + // grounding verified on synthetic UI: gemma4-e4b emits box_2d as + // [ymin,xmin,ymax,xmax]/1000 = the detector's native yx_1000 contract + // (pixel-accurate on a settings-screen mock; real-UI eval still pending). + // qwen-vl fallback keeps "xy_1000" — it emits [x, y] on the 0-1000 grid. + // Off-contract batches still auto-rescale (0-1 or raw pixels). "object_detector": { - "model": "gemini-robotics-er-2-preview", + "provider": "custom", + "model": "gemma4-e4b", + "api_base": "http://127.0.0.1:8080/v1", + "coordinate_format": "yx_1000", + "timeout": 300, "fallback": { - "provider": "google", - "model": "gemini-robotics-er-2-preview" + "provider": "custom", + "model": "qwen-vl", + "api_base": "http://127.0.0.1:8080/v1", + "coordinate_format": "xy_1000", + "timeout": 300 } }, @@ -86,21 +118,25 @@ // NOTE: // - "flash" version delegates directly to object_detector (requires Gemini ER model above). // - "pro" / "ultra" versions run multi-turn ReAct loops (supports both Native Gemini and Universal Multi-Model LangChain engines). + // pro/ultra tiers consume screenshots directly — must be vision-capable. "explorer": { - "thinking_level": "medium" + "model": "gemma4-e4b", + "thinking_level": "medium", + "fallback": { + "provider": "custom", + "model": "qwen-vl", + "api_base": "http://127.0.0.1:8080/v1" + } }, // 📋 Summarizer: Aggregates step history and generates structured outcome "summarizer": {}, // 🔽 Hopper: Fast lightweight package name locator - "hopper": { - "model": "gemini-3.5-flash-lite", - "fallback": { - "provider": "google", - "model": "gemini-3.1-flash-lite" - } - }, + // Inherits the local default; a Gemini model name here would be sent to + // the local server and fail. qwen-small is registered if a lighter hopper + // is wanted: {"model": "qwen-small"}. + "hopper": {}, // 📝 Outputter: Synthesizes execution trace into final output report "outputter": {}, @@ -143,8 +179,11 @@ // 🧠 Step Summarizer: Asynchronous visual context compressor // Summarizes intermediate steps in background without blocking the runner, // replacing images only after strictly objective, high-density summaries are ready. + // LOCAL-MODEL NOTE: the visual-transition lens bypasses the LLM router + // (get_google_llm) — with no GOOGLE_API_KEY it crashes runner init. + // Disabled for the local-provider deployment. "step_summarizer": { - "enabled": true, // Whether background step summarization is active + "enabled": false, // Whether background step summarization is active "model": "gemini-3.5-flash-lite", // Lightweight model for fast background inference "prune_history_xml": true // Prune outdated heavy XML trees from historical steps } @@ -206,7 +245,9 @@ // prompt path byte-for-byte (L2/L3 chunk compression rides this same // flag). See services/token_meter.py for measured session token usage // and cache-hit ratios. - "enabled": true, + // LOCAL-MODEL NOTE: transcript + capsule lens are Google-bound + // (get_google_llm) — disabled under the local-provider deployment. + "enabled": false, "context_budget_tokens": 80000, // Below start_ratio of the operator's own measured prompt size, closed // segments only prepare their summary; the original turns are kept in diff --git a/docs/assets/gemma4-e4b-flash-safari.png b/docs/assets/gemma4-e4b-flash-safari.png new file mode 100644 index 000000000..839cf178c Binary files /dev/null and b/docs/assets/gemma4-e4b-flash-safari.png differ diff --git a/docs/assets/gemma4-e4b-pro-reminders.png b/docs/assets/gemma4-e4b-pro-reminders.png new file mode 100644 index 000000000..73259751d Binary files /dev/null and b/docs/assets/gemma4-e4b-pro-reminders.png differ diff --git a/docs/local_models.md b/docs/local_models.md new file mode 100644 index 000000000..4ec728fbb --- /dev/null +++ b/docs/local_models.md @@ -0,0 +1,117 @@ +# Running ARTEMIS on a Local Model + +ARTEMIS can drive every agent node (planner, operator, checker, flash, +grounding, OCR helpers) from a single local multimodal model — and it can +pull and serve that model itself, no external serving stack or cloud key +required. + +Tested reference setup: **Gemma 4 E4B (4-bit, MLX)** on Apple Silicon — +~6 GB resident, ~40 tok/s decode, both `flash` and `pro` profiles verified +end-to-end on a real device. + +## 1. Install the local-model extra (Apple Silicon) + +```bash +pip install 'artemis[local]' # or: uv pip install -e '.[local]' +``` + +This installs `mlx-vlm` + `huggingface-hub`, the serving stack ARTEMIS +manages for you. (On other platforms, skip to +[BYO server](#byo-any-openai-compatible-server).) + +## 2. Pull and serve the model + +```bash +artemis model list # known aliases +artemis model pull gemma4-e4b # download weights (~5 GB) +artemis model serve gemma4-e4b # OpenAI-compatible server on :8080 +``` + +`serve` runs `mlx_vlm.server` as a managed background process, waits for +readiness, and prints the endpoint. Lifecycle: + +```bash +artemis model status # is the managed server up? +artemis model stop # shut it down +``` + +The served aliases appear under `GET /v1/models` — the same names the +ARTEMIS config's `"model"` field must match. + +## 3. Point ARTEMIS at it + +A ready-made config ships with this repo: + +```bash +export ARTEMIS_ARTEMIS_JSONC=config/artemis.gemma4.jsonc +# or copy it over your artemis.jsonc +``` + +The `default` block (every node inherits it unless overridden): + +```jsonc +"default": { + "provider": "custom", // skips cloud credential checks + "model": "gemma4-e4b", // must match /v1/models exactly + "api_base": "http://127.0.0.1:8080/v1", // the managed server + "timeout": 300, // local decode is slower; keep generous +} +``` + +## 4. Verify + +```bash +uv run artemis doctor +``` + +The **Local Model Endpoint** probe checks the endpoint is reachable and that +every configured alias is actually served: + +- ✅ `Serving N model(s)` — ready +- ❌ `model 'X' not installed` → action: `artemis model pull X` +- ❌ `endpoint unreachable` → action: `artemis model serve X --port 8080` + +The same probe backs the web console's setup wizard and `mobile_diagnose`. +For catalog aliases the console shows **Run** buttons that execute +`artemis model pull` / `serve` in the background with live logs; other +servers get copyable commands for their own tooling. + +## 5. Run + +```bash +uv run artemis run "Open Settings" --profile flash +# or the heavier planner/operator/checker loop: +uv run artemis run "Open Reminders" --profile pro +``` + +## BYO: any OpenAI-compatible server + +ARTEMIS only needs `/v1/chat/completions` with text + `image_url` + tool +calls. Ollama, LM Studio, llama.cpp, vLLM all work — pull the model there, +set `api_base` + `"model"` to match its `/v1/models`, and `artemis doctor` +verifies the same way (remediation hints fall back to generic pull/serve +guidance for non-catalog aliases). + +## Tuning notes for small local models + +- **Timeouts**: `LLM_FLASH_HARD_TIMEOUT_SECONDS` (default 180 s) caps Flash + turns; keep `timeout` ≥ 300 in the config for thinking-style models. +- **Images**: strict servers cap images-per-request — Flash prunes + intermediate screenshots automatically; no action needed. +- **Vision detail**: Gemma 4's image processor defaults to 280 soft tokens + per image (≈528×1152 effective for a phone screenshot). `artemis model + serve` applies the catalog's recommended 1120 (≈near-original detail, + +840 prompt tokens/image) — measured to fix grounding misses and + hallucinated detections on dense screens. Override with + `--vision-tokens {70,140,280,560,1120}`; BYO servers keep their own + defaults. +- **Structured output**: strict JSON-schema `response_format` is not + required — ARTEMIS uses function-calling for structured nodes. +- **Termination**: 4B-class models can recognize the goal yet keep + deliberating instead of ending the turn. The Pro/operator loop exposes a + `mark_done` tool for this: the model calls it when the goal is met, and the + convergence gate routes the claim to final verification — a false claim is + rejected and the run continues. Goals that cannot be achieved (e.g. a + required app is not installed) should be marked `[!]` blocked in the plan; + an all-resolved plan also routes to settlement instead of looping. +- **Memory**: budget ~6 GB for a 4-bit 4B vision model + KV cache. diff --git a/pyproject.toml b/pyproject.toml index e5cd095fe..98772717a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,9 @@ dependencies = [ "scipy>=1.13.0", # System, Utilities & Telemetry "psutil>=5.9.0", + # On-device OCR via Apple Vision (macOS only); guarded import — the OCR + # seam degrades to Google Cloud Vision or disabled without it. + "pyobjc-framework-Vision>=10.0,<13; sys_platform == 'darwin'", "retry>=0.9.2", "posthog>=3.5.0", "uuid-utils>=0.9.0", @@ -86,6 +89,13 @@ dev = [ "pyright>=1.1.389", "pre-commit>=3.5.0", ] +# `artemis model pull/serve`: managed local model serving via mlx-vlm. +# Apple Silicon only; other platforms should run an OpenAI-compatible +# server (e.g. Ollama) and point api_base at it. +local = [ + "mlx-vlm>=0.7.6,<1.0; sys_platform == 'darwin' and platform_machine == 'arm64'", + "huggingface-hub>=0.30,<2.0", +] [project.scripts] artemis = "artemis.interfaces.cli.main:cli" diff --git a/tests/conftest.py b/tests/conftest.py index b99e8d4dd..0b148fed0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -32,6 +32,20 @@ def mock_driver(): return MockDeviceDriver(device_id="fixture-mock-device", width=1080, height=2400) +@pytest.fixture(autouse=True) +def _reset_apple_vision_ocr_breaker(): + """Isolate the Apple Vision OCR circuit-breaker counter between tests. + + Tests exercising the real ``perform_ocr`` path with undecodable images + trip the process-global failure counter and would silently disable the + provider for every later test in the session.""" + import artemis.utils.ocr_api as ocr_api + + ocr_api._apple_vision_consecutive_failures = 0 + yield + ocr_api._apple_vision_consecutive_failures = 0 + + def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: """Attach test-layer markers according to the owning test directory.""" for item in items: diff --git a/tests/unit/admin_console/test_local_model_endpoints.py b/tests/unit/admin_console/test_local_model_endpoints.py new file mode 100644 index 000000000..90675ac0d --- /dev/null +++ b/tests/unit/admin_console/test_local_model_endpoints.py @@ -0,0 +1,233 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for the local-model lifecycle endpoints. + +These guard the click-to-run console path: only resolvable model refs may +spawn ``artemis model`` subprocesses, ``stop`` stays port-scoped, and task +state/logs surface for polling. +""" + +import asyncio + +import pytest +from httpx import ASGITransport, AsyncClient + +from apps.admin_console.routers import system +from apps.admin_console.server import app + + +@pytest.fixture(autouse=True) +def _clear_model_tasks(): + system._MODEL_TASKS.clear() + yield + system._MODEL_TASKS.clear() + + +class _FakeStdout: + def __init__(self, lines): + self._lines = lines + + def __aiter__(self): + return self._iter() + + async def _iter(self): + for line in self._lines: + yield line + + +class _FakeProc: + def __init__(self, rc=0, lines=(b"done\n",)): + self.stdout = _FakeStdout(lines) + self._rc = rc + + async def wait(self): + return self._rc + + +def _patch_subprocess(monkeypatch, captured: list, rc=0, lines=(b"done\n",)): + async def _fake_exec(*cmd, **kwargs): + captured.append(list(cmd)) + return _FakeProc(rc=rc, lines=lines) + + monkeypatch.setattr(asyncio, "create_subprocess_exec", _fake_exec) + + +def _client(**kwargs): + return AsyncClient(transport=ASGITransport(app=app, **kwargs), base_url="http://localhost") + + +@pytest.mark.asyncio +async def test_status_reports_cli_unavailable_without_model_module(): + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://localhost") as ac: + res = await ac.get("/api/system/local-models/status") + assert res.status_code == 200 + data = res.json() + # The `artemis model` CLI ships with the local-models feature; on builds + # without it the console must fall back to copy-to-terminal guidance. + assert data["commands_available"] == system._model_cli_available() + assert data["tasks"] == {} + + +@pytest.mark.asyncio +async def test_pull_returns_501_when_cli_missing(): + async with _client() as ac: + res = await ac.post("/api/system/local-models/pull", json={"alias": "gemma4-e4b"}) + if system._model_cli_available(): + pytest.skip("model CLI present in this build") + assert res.status_code == 501 + assert "terminal" in res.json()["detail"] + + +@pytest.mark.asyncio +async def test_pull_rejects_unresolvable_alias(monkeypatch): + monkeypatch.setattr(system, "_model_cli_available", lambda: True) + async with _client() as ac: + for bad in ("not-a-model", "--port", "gemma4-e4b; rm -rf /"): + res = await ac.post("/api/system/local-models/pull", json={"alias": bad}) + assert res.status_code == 400, bad + assert system._MODEL_TASKS == {} + + +@pytest.mark.asyncio +async def test_pull_rejects_remote_client(monkeypatch): + monkeypatch.setattr(system, "_model_cli_available", lambda: True) + async with _client(client=("192.168.1.20", 42000)) as ac: + res = await ac.post("/api/system/local-models/pull", json={"alias": "gemma4-e4b"}) + assert res.status_code == 403 + assert "local-only" in res.json()["detail"] + + +@pytest.mark.asyncio +async def test_pull_starts_task_and_completes(monkeypatch): + monkeypatch.setattr(system, "_model_cli_available", lambda: True) + captured = [] + _patch_subprocess(monkeypatch, captured, rc=0, lines=(b"downloading\n", b"done\n")) + + async with _client() as ac: + res = await ac.post("/api/system/local-models/pull", json={"alias": "gemma4-e4b"}) + assert res.status_code == 200 + body = res.json() + assert body["status"] == "running" + assert body["key"] == "pull:gemma4-e4b" + + assert captured[0][-3:] == ["model", "pull", "gemma4-e4b"] + + await system._MODEL_TASKS["pull:gemma4-e4b"]["watcher"] + task = system._MODEL_TASKS["pull:gemma4-e4b"] + assert task["status"] == "done" + assert task["returncode"] == 0 + assert "downloading" in "\n".join(task["log"]) + + +@pytest.mark.asyncio +async def test_pull_is_idempotent_while_running(monkeypatch): + monkeypatch.setattr(system, "_model_cli_available", lambda: True) + started = asyncio.Event() + release = asyncio.Event() + + class _HangingProc(_FakeProc): + async def wait(self): + release.set() + return 0 + + async def _fake_exec(*cmd, **kwargs): + proc = _HangingProc() + started.set() + return proc + + monkeypatch.setattr(asyncio, "create_subprocess_exec", _fake_exec) + + async with _client() as ac: + res1 = await ac.post("/api/system/local-models/pull", json={"alias": "gemma4-e4b"}) + res2 = await ac.post("/api/system/local-models/pull", json={"alias": "gemma4-e4b"}) + assert res1.json()["status"] == res2.json()["status"] == "running" + + +@pytest.mark.asyncio +async def test_serve_passes_port_and_serve_cmd(monkeypatch): + monkeypatch.setattr(system, "_model_cli_available", lambda: True) + captured = [] + _patch_subprocess(monkeypatch, captured) + + async with _client() as ac: + res = await ac.post( + "/api/system/local-models/serve", json={"alias": "gemma4-e4b", "port": 8097} + ) + assert res.status_code == 200 + assert captured[0][-5:] == ["model", "serve", "gemma4-e4b", "--port", "8097"] + + +@pytest.mark.asyncio +async def test_stop_is_port_scoped_without_positional_alias(monkeypatch): + monkeypatch.setattr(system, "_model_cli_available", lambda: True) + captured = [] + _patch_subprocess(monkeypatch, captured) + + async with _client() as ac: + res = await ac.post( + "/api/system/local-models/stop", json={"alias": "gemma4-e4b", "port": 8080} + ) + assert res.status_code == 200 + cmd = captured[0] + # CLI: `artemis model stop --port N` — the alias must not appear as argv. + assert cmd[-3:] == ["model", "stop", "--port"] or cmd[-4:] == [ + "model", + "stop", + "--port", + "8080", + ] + assert "gemma4-e4b" not in cmd + + +@pytest.mark.asyncio +async def test_failed_process_surfaces_log_and_returncode(monkeypatch): + monkeypatch.setattr(system, "_model_cli_available", lambda: True) + _patch_subprocess(monkeypatch, [], rc=2, lines=(b"error: no space left\n",)) + + async with _client() as ac: + res = await ac.post("/api/system/local-models/pull", json={"alias": "gemma4-e4b"}) + assert res.status_code == 200 + + await system._MODEL_TASKS["pull:gemma4-e4b"]["watcher"] + async with _client() as ac: + status = (await ac.get("/api/system/local-models/status")).json() + task = status["tasks"]["pull:gemma4-e4b"] + assert task["status"] == "failed" + assert task["returncode"] == 2 + assert any("no space left" in line for line in task["log"]) + + +@pytest.mark.asyncio +async def test_org_repo_alias_is_accepted(monkeypatch): + monkeypatch.setattr(system, "_model_cli_available", lambda: True) + captured = [] + _patch_subprocess(monkeypatch, captured) + + async with _client() as ac: + res = await ac.post( + "/api/system/local-models/pull", json={"alias": "mlx-community/gemma-4-e4b-it-4bit"} + ) + assert res.status_code == 200 + assert captured[0][-1] == "mlx-community/gemma-4-e4b-it-4bit" + + +@pytest.mark.asyncio +async def test_invalid_port_rejected(): + async with _client() as ac: + res = await ac.post( + "/api/system/local-models/serve", json={"alias": "gemma4-e4b", "port": 70000} + ) + assert res.status_code == 422 diff --git a/tests/unit/agents/flash/test_hybrid_loop.py b/tests/unit/agents/flash/test_hybrid_loop.py new file mode 100644 index 000000000..f3b61d5f3 --- /dev/null +++ b/tests/unit/agents/flash/test_hybrid_loop.py @@ -0,0 +1,241 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Hybrid Flash loop (decision.flash_loop): VLM describes, decision model acts. + +The describe call binds no tools, so malformed tool-call envelopes cannot be +emitted; the decision model picks from a fixed menu (element taps + +meta-actions). Any inconclusive verdict — unreachable endpoint, low +confidence, or a key outside the menu — escalates to the standard +tool-calling model turn. +""" + +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch + +import pytest +from langchain_core.messages import AIMessage, HumanMessage + +from artemis.agents.flash.runner import FlashRunner, _HYBRID_META_ACTIONS +from artemis.core.decision_client import DecisionResult +from artemis.llm.reliability import Failure, FailureCategory, LLMCallError + + +_FAILURE = Failure(FailureCategory.UNKNOWN, retryable=True, should_fallback=True) + + +def _runner() -> FlashRunner: + r = FlashRunner.__new__(FlashRunner) + r.goal = "enable dark theme" + return r + + +def _cfg(threshold: float = 0.85): + return SimpleNamespace( + url="http://127.0.0.1:11434/v1/decide", + model="laya", + timeout_seconds=5.0, + confidence_threshold=threshold, + ) + + +def _state(elements): + return SimpleNamespace(indexed_elements=elements) + + +def _verdict(verdict, confidence=0.9): + return DecisionResult(verdict=verdict, confidence=confidence) + + +# --- action menu -------------------------------------------------------------- + + +def test_menu_lists_element_taps_and_meta_actions(): + state = _state( + [ + {"index": 1, "text": "Settings"}, + {"index": 2, "content_desc": "Search"}, + {"index": 3, "resource_id": "id/more"}, + ] + ) + menu = FlashRunner._hybrid_action_menu(state) + assert menu["tap_1"] == "tap element 1: Settings" + assert menu["tap_2"] == "tap element 2: Search" + assert menu["tap_3"] == "tap element 3: id/more" + for key in _HYBRID_META_ACTIONS: + assert key in menu + + +def test_menu_handles_missing_elements(): + assert "done" in FlashRunner._hybrid_action_menu(_state(None)) + assert "tap_" not in " ".join(FlashRunner._hybrid_action_menu(_state(None))) + + +# --- menu -> tool call mapping ------------------------------------------------- + + +def test_choice_maps_element_tap_to_click(): + call = FlashRunner._menu_choice_to_call("tap_4", "desc") + assert call["name"] == "click" + assert call["args"] == {"target": 4} + assert call["id"] + + +def test_choice_maps_swipe_and_keys(): + assert FlashRunner._menu_choice_to_call("swipe_up", "")["args"] == { + "direction": "up" + } + back = FlashRunner._menu_choice_to_call("press_back", "") + assert back["name"] == "press_key" and back["args"] == {"key": "back"} + + +def test_choice_maps_done_and_fail_to_report(): + done = FlashRunner._menu_choice_to_call("done", "all set") + assert done["name"] == "report_task_status" + assert done["args"]["status"] == "completed" + assert done["args"]["explanation"] == "all set" + fail = FlashRunner._menu_choice_to_call("fail", "blocked") + assert fail["args"]["status"] == "failed" + + +def test_choice_rejects_unknown_key(): + assert FlashRunner._menu_choice_to_call("launch_rocket", "") is None + + +# --- _hybrid_decide ------------------------------------------------------------ + + +@pytest.mark.asyncio +async def test_hybrid_decide_returns_call_on_confident_verdict(): + runner = _runner() + state = _state([{"index": 1, "text": "Display"}]) + describe_resp = AIMessage(content="Settings list; [1] Display ~[500,300]") + + with ( + patch( + "artemis.agents.flash.runner.acomplete", + new=AsyncMock(return_value=describe_resp), + ), + patch("artemis.agents.flash.runner.DecisionClient") as MockClient, + ): + MockClient.return_value.achoose = AsyncMock( + return_value=_verdict("tap_1") + ) + out = await runner._hybrid_decide( + AsyncMock(), state, HumanMessage(content="obs"), "plan", _cfg() + ) + + assert out is not None + response, tool_calls = out + assert tool_calls[0]["name"] == "click" + assert tool_calls[0]["args"] == {"target": 1} + # The choice is stamped onto the describe message so ToolMessages keep a + # legitimate parent when the ledger replays them. + assert response.tool_calls[0]["name"] == "click" + # The describe response is pure text — no envelope parsing needed. + assert "Settings list" in response.content + + +@pytest.mark.asyncio +async def test_hybrid_decide_escalates_on_low_confidence(): + runner = _runner() + with ( + patch( + "artemis.agents.flash.runner.acomplete", + new=AsyncMock(return_value=AIMessage(content="desc")), + ), + patch("artemis.agents.flash.runner.DecisionClient") as MockClient, + ): + MockClient.return_value.achoose = AsyncMock( + return_value=_verdict("tap_1", confidence=0.40) + ) + out = await runner._hybrid_decide( + AsyncMock(), _state([{"index": 1}]), HumanMessage(content="x"), None, _cfg() + ) + assert out is None + + +@pytest.mark.asyncio +async def test_hybrid_decide_escalates_when_unanswered(): + runner = _runner() + with ( + patch( + "artemis.agents.flash.runner.acomplete", + new=AsyncMock(return_value=AIMessage(content="desc")), + ), + patch("artemis.agents.flash.runner.DecisionClient") as MockClient, + ): + MockClient.return_value.achoose = AsyncMock( + return_value=DecisionResult(verdict=None) + ) + out = await runner._hybrid_decide( + AsyncMock(), _state([]), HumanMessage(content="x"), None, _cfg() + ) + assert out is None + + +@pytest.mark.asyncio +async def test_hybrid_decide_escalates_on_out_of_menu_verdict(): + runner = _runner() + with ( + patch( + "artemis.agents.flash.runner.acomplete", + new=AsyncMock(return_value=AIMessage(content="desc")), + ), + patch("artemis.agents.flash.runner.DecisionClient") as MockClient, + ): + MockClient.return_value.achoose = AsyncMock( + return_value=_verdict("tap_99", confidence=0.95) + ) + out = await runner._hybrid_decide( + AsyncMock(), _state([]), HumanMessage(content="x"), None, _cfg() + ) + assert out is None + + +@pytest.mark.asyncio +async def test_hybrid_decide_escalates_on_describe_failure(): + runner = _runner() + with patch( + "artemis.agents.flash.runner.acomplete", + new=AsyncMock(side_effect=LLMCallError("server down", failure=_FAILURE)), + ): + out = await runner._hybrid_decide( + AsyncMock(), _state([]), HumanMessage(content="x"), None, _cfg() + ) + assert out is None + + +# --- _hybrid_plan -------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_hybrid_plan_returns_milestone_text(): + runner = _runner() + with patch( + "artemis.agents.flash.runner.acomplete", + new=AsyncMock(return_value=AIMessage(content="- open Settings\n- tap Display")), + ): + plan = await runner._hybrid_plan(AsyncMock()) + assert "open Settings" in plan + + +@pytest.mark.asyncio +async def test_hybrid_plan_failure_is_non_fatal(): + runner = _runner() + with patch( + "artemis.agents.flash.runner.acomplete", + new=AsyncMock(side_effect=LLMCallError("boom", failure=_FAILURE)), + ): + assert await runner._hybrid_plan(AsyncMock()) is None diff --git a/tests/unit/agents/test_explorer_tiers.py b/tests/unit/agents/test_explorer_tiers.py index e8c400d3d..ef747e172 100644 --- a/tests/unit/agents/test_explorer_tiers.py +++ b/tests/unit/agents/test_explorer_tiers.py @@ -244,11 +244,11 @@ async def test_universal_dispatch_appends_image_message(tmp_path): assert [type(m) for m in messages] == [ToolMessage, ToolMessage, HumanMessage] assert messages[0].content == "perception text" human = messages[2] - assert human.content[0] == { + assert human.content[-1] == { "type": "text", "text": "[Annotated image(s) returned by: ask_perception_tool, inspect_region]", } - image_blocks = human.content[1:] + image_blocks = human.content[:-1] assert len(image_blocks) == 2 # the missing file is skipped assert all(b["type"] == "image_url" for b in image_blocks) assert image_blocks[0]["image_url"]["url"].startswith("data:image/jpeg;base64,") diff --git a/tests/unit/agents/test_flash_runner.py b/tests/unit/agents/test_flash_runner.py index f188b5e1b..2633a50de 100644 --- a/tests/unit/agents/test_flash_runner.py +++ b/tests/unit/agents/test_flash_runner.py @@ -556,3 +556,64 @@ async def test_final_report_persists_native_thinking(mock_context): assert kwargs["operator_raw_thinking"] == "final text" assert kwargs["operator_native_thinking"] == "native summary" assert isinstance(messages[-1], ToolMessage) + + +# --- Gemma local-server tool-call envelope repair -------------------------------- + + +def _make_runner(mock_context): + with patch("artemis.controllers.unified_controller.get_driver"): + return FlashRunner(mock_context, goal="test goal") + + +def test_resolve_tool_calls_keeps_native_calls(mock_context): + runner = _make_runner(mock_context) + native = [{"name": "click", "args": {"target": 1}, "id": "n1"}] + resp = SimpleNamespace(tool_calls=native, invalid_tool_calls=[]) + assert runner._resolve_tool_calls(resp, "") == native + + +def test_resolve_tool_calls_repairs_enveloped_invalid_call(mock_context): + """mlx_vlm wraps `call:name{args}` as name="call" + unparseable arguments.""" + runner = _make_runner(mock_context) + resp = SimpleNamespace( + tool_calls=[], + invalid_tool_calls=[ + { + "name": "call", + "args": '{"report_task_status{explanation": "Settings is open."}', + "id": "broken-1", + "error": "json error", + } + ], + ) + calls = runner._resolve_tool_calls(resp, "") + assert calls == [ + { + "name": "report_task_status", + "args": {"explanation": "Settings is open."}, + "id": "broken-1", + } + ] + + +def test_resolve_tool_calls_repairs_call_envelope_in_text(mock_context): + """The `call:` envelope can leak into content instead of tool_calls.""" + runner = _make_runner(mock_context) + resp = SimpleNamespace(tool_calls=[], invalid_tool_calls=[]) + text = 'Opening settings. call:click{target: 34, target_description: "Display & touch"}' + calls = runner._resolve_tool_calls(resp, text) + assert calls[0]["name"] == "click" + assert calls[0]["args"] == {"target": 34, "target_description": "Display & touch"} + + +def test_resolve_tool_calls_unrecoverable_envelope_returns_empty(mock_context): + """Genuinely garbled args still surface as no-call (notice path), not a crash.""" + runner = _make_runner(mock_context) + resp = SimpleNamespace( + tool_calls=[], + invalid_tool_calls=[ + {"name": "call", "args": '{"click{target": "6,broken', "id": "b2", "error": "x"} + ], + ) + assert runner._resolve_tool_calls(resp, "") == [] diff --git a/tests/unit/agents/test_object_detector_coordinates.py b/tests/unit/agents/test_object_detector_coordinates.py new file mode 100644 index 000000000..6033db2f6 --- /dev/null +++ b/tests/unit/agents/test_object_detector_coordinates.py @@ -0,0 +1,298 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Coordinate normalization for multi-provider object detection. + +The detector contract is ``[x, y]`` on a 0-1000 grid over the sent image. +Each endpoint declares the convention its model actually emits via +``coordinate_format``; clearly off-contract batches are rescaled. +""" + +from types import SimpleNamespace + +import pytest + +from artemis.agents.object_detector.object_detector import ( + _detected_batch_scale, + _endpoint_coordinate_format, + _image_dimensions, + _normalize_detected_box, + _normalize_detected_point, + _qwen_resized_dimensions, +) + + +def _item(a, b): + return {"point": [a, b], "label": "x"} + + +# --------------------------------------------------------------------------- +# Declared formats +# --------------------------------------------------------------------------- + + +def test_yx_1000_swaps_axes_and_keeps_scale(): + item = _item(800, 200) + _normalize_detected_point(item, "yx_1000", (1000, 2000), None) + assert item["point"] == [200.0, 800.0] + + +def test_xy_1000_keeps_axes_and_scale(): + item = _item(200, 800) + _normalize_detected_point(item, "xy_1000", (1000, 2000), None) + assert item["point"] == [200.0, 800.0] + + +def test_xy_norm_scales_zero_to_one(): + item = _item(0.2, 0.8) + _normalize_detected_point(item, "xy_norm", (1000, 2000), None) + assert item["point"] == [200.0, 800.0] + + +def test_yx_norm_swaps_and_scales(): + item = _item(0.8, 0.2) + _normalize_detected_point(item, "yx_norm", (1000, 2000), None) + assert item["point"] == [200.0, 800.0] + + +def test_xy_px_scales_by_qwen_resize_grid(): + # 1000x2000 snaps to 992x1984 under the factor-32 smart resize. + item = _item(496, 992) # center of the resized grid + _normalize_detected_point(item, "xy_px", (1000, 2000), None) + assert item["point"] == [500.0, 500.0] + + +def test_yx_px_swaps_axes_and_scales(): + item = _item(992, 496) + _normalize_detected_point(item, "yx_px", (1000, 2000), None) + assert item["point"] == [500.0, 500.0] + + +def test_px_without_image_size_leaves_point_unchanged(): + item = _item(1500, 700) + _normalize_detected_point(item, "xy_px", None, None) + assert item["point"] == [1500.0, 700.0] + + +# --------------------------------------------------------------------------- +# Batch-level scale detection +# --------------------------------------------------------------------------- + + +def test_batch_scale_detects_norm_when_all_below_one(): + items = [_item(0.5, 0.2), _item(0.9, 0.1)] + assert _detected_batch_scale(items) == "norm" + + +def test_batch_scale_detects_px_when_any_above_1000(): + items = [_item(500, 200), _item(1500, 300)] + assert _detected_batch_scale(items) == "px" + + +def test_batch_scale_mixed_grid_stays_undeclared(): + # A legit 0-1000 corner point coexisting with mid-grid values must not + # trip the norm heuristic — the batch still reads as the declared grid. + items = [_item(0.5, 0.3), _item(500, 400)] + assert _detected_batch_scale(items) is None + + +def test_batch_scale_all_zero_is_undeclared(): + assert _detected_batch_scale([_item(0, 0)]) is None + + +def test_batch_scale_scans_box_2d_coords(): + boxes = [ + {"box_2d": [0.1, 0.2, 0.3, 0.4], "label": "a"}, + {"box_2d": [0.5, 0.6, 0.7, 0.8], "label": "b"}, + ] + assert _detected_batch_scale(boxes) == "norm" + assert _detected_batch_scale([{"box_2d": [100, 200, 300, 1400]}]) == "px" + + +# --------------------------------------------------------------------------- +# box_2d detection output (Gemma 4 native [ymin, xmin, ymax, xmax] / 1000) +# --------------------------------------------------------------------------- + + +def test_box_2d_yx_1000_derives_center_point(): + # Gemma-style [ymin, xmin, ymax, xmax] -> contract [x1, y1, x2, y2]. + item = {"box_2d": [200, 100, 400, 300], "label": "icon"} + assert _normalize_detected_box(item, "yx_1000", (1000, 2000), None) is True + assert item["box_2d"] == [100.0, 200.0, 300.0, 400.0] + assert item["point"] == [200.0, 300.0] + + +def test_box_2d_xy_1000_keeps_axes(): + item = {"box_2d": [100, 200, 300, 400]} + assert _normalize_detected_box(item, "xy_1000", (1000, 2000), None) is True + assert item["box_2d"] == [100.0, 200.0, 300.0, 400.0] + assert item["point"] == [200.0, 300.0] + + +def test_box_2d_norm_scale_scales_and_swaps(): + item = {"box_2d": [0.2, 0.1, 0.4, 0.3]} + assert _normalize_detected_box(item, "yx_norm", (1000, 2000), None) is True + assert item["box_2d"] == [100.0, 200.0, 300.0, 400.0] + assert item["point"] == [200.0, 300.0] + + +def test_box_2d_keeps_valid_point(): + item = {"point": [800, 200], "box_2d": [200, 100, 400, 300]} + assert _normalize_detected_box(item, "yx_1000", (1000, 2000), None) is False + _normalize_detected_point(item, "yx_1000", (1000, 2000), None) + assert item["point"] == [200.0, 800.0] + assert item["box_2d"] == [100.0, 200.0, 300.0, 400.0] + + +def test_box_2d_replaces_unusable_point(): + item = {"point": ["a", "b"], "box_2d": [200, 100, 400, 300]} + assert _normalize_detected_box(item, "yx_1000", (1000, 2000), None) is True + assert item["point"] == [200.0, 300.0] + + +def test_invalid_box_2d_is_untouched(): + for bad in ({"box_2d": [1, 2]}, {"box_2d": "x"}, {"box_2d": [1, 2, "c", 4]}): + item = dict(bad) + assert _normalize_detected_box(item, "yx_1000", (100, 100), None) is False + assert item == bad + + +def test_batch_scale_override_fixes_norm_batch_under_1000_contract(): + item = _item(0.5, 0.2) + _normalize_detected_point(item, "yx_1000", (1000, 2000), "norm") + # yx order + norm scale: x=0.2*1000, y=0.5*1000 + assert item["point"] == [200.0, 500.0] + + +def test_batch_scale_override_fixes_px_batch_under_1000_contract(): + item = _item(992, 496) # [y_px, x_px] on a 992x1984 grid + _normalize_detected_point(item, "yx_1000", (1000, 2000), "px") + assert item["point"] == [500.0, 500.0] + + +# --------------------------------------------------------------------------- +# Endpoint format resolution + image dims +# --------------------------------------------------------------------------- + + +def test_endpoint_coordinate_format_defaults_to_gemini(): + assert _endpoint_coordinate_format(SimpleNamespace(endpoint=None)) == "yx_1000" + wrapper = SimpleNamespace(endpoint=SimpleNamespace(coordinate_format="xy_px")) + assert _endpoint_coordinate_format(wrapper) == "xy_px" + bogus = SimpleNamespace(endpoint=SimpleNamespace(coordinate_format="bogus")) + assert _endpoint_coordinate_format(bogus) == "yx_1000" + + +def test_qwen_resize_snaps_to_factor_32(): + assert _qwen_resized_dimensions(1000, 2000) == (992, 1984) + assert _qwen_resized_dimensions(2940, 1912) == (2944, 1920) + + +def test_image_dimensions_reads_png_header(): + import io + + from PIL import Image + + buf = io.BytesIO() + Image.new("RGB", (640, 480)).save(buf, format="PNG") + assert _image_dimensions(buf.getvalue()) == (640, 480) + assert _image_dimensions(b"not an image") is None + + +def test_invalid_points_are_untouched(): + for bad in ({}, {"point": "x"}, {"point": [1]}, {"point": ["a", "b"]}): + item = dict(bad) + _normalize_detected_point(item, "yx_1000", (100, 100), None) + assert item == bad + + +@pytest.mark.asyncio +async def test_run_object_detection_uses_endpoint_coordinate_format(): + """End-to-end: an xy_px endpoint turns pixel output into the 0-1000 contract.""" + import io + from unittest.mock import AsyncMock, patch + + from PIL import Image + + from artemis.agents.object_detector.object_detector import _run_object_detection + + buf = io.BytesIO() + Image.new("RGB", (1000, 2000)).save(buf, format="PNG") + image_bytes = buf.getvalue() + + fake_llm = SimpleNamespace(endpoint=SimpleNamespace(coordinate_format="xy_px")) + # Model reports grid-center in pixels of the 992x1984 resized image. + vlm_output = [{"point": [496, 992], "label": "home button"}] + + with ( + patch( + "artemis.agents.object_detector.object_detector.get_llm", + return_value=fake_llm, + ), + patch( + "artemis.agents.object_detector.object_detector._detect_single_label", + new_callable=AsyncMock, + return_value=vlm_output, + ), + ): + ctx = SimpleNamespace(llm_config=SimpleNamespace()) + result = await _run_object_detection( + ctx=ctx, + image_bytes=image_bytes, + queries=["home button"], + ) + + point = result["detected"][0]["point"] + # 496px/992w -> 500x, 992px/1984h -> 500y on the 0-1000 grid. + assert point == [pytest.approx(500.0), pytest.approx(500.0)] + + +@pytest.mark.asyncio +async def test_run_object_detection_accepts_box_2d_output(): + """End-to-end: a box-only response (Gemma-style) still yields a contract point.""" + import io + from unittest.mock import AsyncMock, patch + + from PIL import Image + + from artemis.agents.object_detector.object_detector import _run_object_detection + + buf = io.BytesIO() + Image.new("RGB", (1000, 2000)).save(buf, format="PNG") + image_bytes = buf.getvalue() + + fake_llm = SimpleNamespace(endpoint=SimpleNamespace(coordinate_format=None)) + vlm_output = [{"box_2d": [200, 100, 400, 300], "label": "home button"}] + + with ( + patch( + "artemis.agents.object_detector.object_detector.get_llm", + return_value=fake_llm, + ), + patch( + "artemis.agents.object_detector.object_detector._detect_single_label", + new_callable=AsyncMock, + return_value=vlm_output, + ), + ): + ctx = SimpleNamespace(llm_config=SimpleNamespace()) + result = await _run_object_detection( + ctx=ctx, + image_bytes=image_bytes, + queries=["home button"], + ) + + detected = result["detected"][0] + assert detected["point"] == [pytest.approx(200.0), pytest.approx(300.0)] + assert detected["box_2d"] == [100.0, 200.0, 300.0, 400.0] diff --git a/tests/unit/agents/test_operator_focus_reminder.py b/tests/unit/agents/test_operator_focus_reminder.py new file mode 100644 index 000000000..7582e5dad --- /dev/null +++ b/tests/unit/agents/test_operator_focus_reminder.py @@ -0,0 +1,163 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the Operator standing focus reminder. + +Sliding-window models (e.g. Gemma's 512-token local window) reliably attend +only to the tail end of the whole prompt; the static system-prompt sections +restating the ledger protocol are outside it once the prompt grows. The +ActiveFocusPromptComponent re-states the active sub-goal plus the two rules +the bounce checks enforce as the last block of every observation, marked +ephemeral so it never reaches the committed transcript. +""" + +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from artemis.agents.operator.operator import OperatorNode +from artemis.agents.operator.prompts import FOCUS_REMINDER_MARKER +from artemis.config.agent import MemoryTranscriptConfig +from artemis.context import ArtemisContext + +LEGACY_TRANSCRIPT = MemoryTranscriptConfig(enabled=False) + +PLAN_WITH_LEAF = """- [x] Open Google Maps and search for SFO +- [/] Read the commute duration and record it into note `commute_eta_info` + - [/] Read the driving duration + - [/] Tap the "Driving" tab so the fastest route is highlighted +- [ ] Draft the ETA message +""" + +PLAN_NO_LEAF = """- [x] Open Google Maps and search for SFO +- [/] Read the commute duration and record it into note `commute_eta_info` + - [ ] Read the driving duration +- [ ] Draft the ETA message +""" + +PLAN_ALL_DONE = """- [x] Open Google Maps and search for SFO +- [x] Read the commute duration and record it into note `commute_eta_info` +""" + + +def _make_node(tmp_path, plan: str): + notes = tmp_path / "notes" + notes.mkdir(parents=True) + (notes / "task_plan.md").write_text(plan, encoding="utf-8") + + ctx = MagicMock(spec=ArtemisContext) + ctx.execution_setup = None + ctx.data_engine = MagicMock() + ctx.data_engine.base_dir = str(tmp_path) + ctx.data_engine.current_session_id = "s" + ctx.data_engine.get_agent_friendly_steps.return_value = [] + + state = MagicMock() + state.subagent_calls = [] + state.initial_goal = "goal" + state.open_incident = None + return OperatorNode(ctx, transcript_config=LEGACY_TRANSCRIPT), state + + +def _click_response(content="ok"): + response = MagicMock() + response.content = content + response.tool_calls = [ + { + "name": "click", + "args": {"target": [50, 50], "target_description": "button"}, + "id": "call_0", + } + ] + return response + + +def _texts(messages): + out = [] + for m in messages: + content = getattr(m, "content", None) + if isinstance(content, str): + out.append(content) + elif isinstance(content, list): + out.extend(b.get("text", "") for b in content if isinstance(b, dict)) + return "\n".join(out) + + +def _last_human_texts(messages): + """The text blocks of the final human message, in order.""" + human = [m for m in messages if getattr(m, "type", None) == "human"][-1] + return [b.get("text", "") for b in human.content if isinstance(b, dict) and b.get("text")] + + +@pytest.mark.asyncio +async def test_focus_block_is_the_last_tail_block(tmp_path): + node, state = _make_node(tmp_path, PLAN_WITH_LEAF) + seen = [] + + async def ainvoke(messages, *args, **kwargs): + seen.append(list(messages)) + return _click_response() + + llm = MagicMock() + llm.ainvoke = AsyncMock(side_effect=ainvoke) + llm.bind_tools.return_value = llm + with patch("artemis.agents.operator.operator.get_llm", return_value=llm): + await node(state) + + blocks = _last_human_texts(seen[0]) + focus = [b for b in blocks if FOCUS_REMINDER_MARKER in b] + assert len(focus) == 1, "exactly one focus block" + assert blocks[-1] == focus[0], "focus block must be the tail's last block" + assert "Read the commute duration" in focus[0] + assert 'Tap the "Driving" tab' in focus[0] + assert "update_note" in focus[0] + assert "Turn-Ending Action" in focus[0] + + +@pytest.mark.asyncio +async def test_focus_block_nudges_a_leafless_milestone(tmp_path): + node, state = _make_node(tmp_path, PLAN_NO_LEAF) + seen = [] + + async def ainvoke(messages, *args, **kwargs): + seen.append(list(messages)) + return _click_response() + + llm = MagicMock() + llm.ainvoke = AsyncMock(side_effect=ainvoke) + llm.bind_tools.return_value = llm + with patch("artemis.agents.operator.operator.get_llm", return_value=llm): + await node(state) + + focus = [b for b in _last_human_texts(seen[0]) if FOCUS_REMINDER_MARKER in b] + assert len(focus) == 1 + assert "no `[/]` sub-goal" in focus[0] + + +@pytest.mark.asyncio +async def test_focus_block_skipped_when_plan_is_done(tmp_path): + node, state = _make_node(tmp_path, PLAN_ALL_DONE) + seen = [] + + async def ainvoke(messages, *args, **kwargs): + seen.append(list(messages)) + return _click_response() + + llm = MagicMock() + llm.ainvoke = AsyncMock(side_effect=ainvoke) + llm.bind_tools.return_value = llm + with patch("artemis.agents.operator.operator.get_llm", return_value=llm): + await node(state) + + assert FOCUS_REMINDER_MARKER not in _texts(seen[0]) diff --git a/tests/unit/agents/test_outputter.py b/tests/unit/agents/test_outputter.py index 8def19fa9..259cf0b41 100644 --- a/tests/unit/agents/test_outputter.py +++ b/tests/unit/agents/test_outputter.py @@ -147,11 +147,11 @@ def __init__(self, messages, initial_goal, operator_raw_data): human_content = messages[1].content assert isinstance(human_content, list) - assert human_content[0]["type"] == "text" - assert "Step 1: Opened browser" in human_content[0]["text"] - assert "Plan: Find capital" in human_content[0]["text"] - assert human_content[1]["type"] == "image_url" - assert human_content[1]["image_url"]["url"] == "data:image/jpeg;base64,mock_b64_data" + assert human_content[0]["type"] == "image_url" + assert human_content[0]["image_url"]["url"] == "data:image/jpeg;base64,mock_b64_data" + assert human_content[1]["type"] == "text" + assert "Step 1: Opened browser" in human_content[1]["text"] + assert "Plan: Find capital" in human_content[1]["text"] @patch("artemis.agents.outputter.outputter.get_llm") diff --git a/tests/unit/agents/test_planner_node.py b/tests/unit/agents/test_planner_node.py index 9fabb1143..9aef9a384 100644 --- a/tests/unit/agents/test_planner_node.py +++ b/tests/unit/agents/test_planner_node.py @@ -82,8 +82,8 @@ async def mock_astream(*args, **kwargs): human_msg_content = messages[1].content assert isinstance(human_msg_content, list) - assert human_msg_content[0]["type"] == "text" - assert "Goal: Test Goal" in human_msg_content[0]["text"] + assert human_msg_content[0]["type"] == "image_url" + assert "Goal: Test Goal" in human_msg_content[1]["text"] def test_validate_plan_format_single_sourced_status_alphabet(): diff --git a/tests/unit/agents/test_precondition_pixel_target.py b/tests/unit/agents/test_precondition_pixel_target.py index 99ddfa20a..a3a5851ae 100644 --- a/tests/unit/agents/test_precondition_pixel_target.py +++ b/tests/unit/agents/test_precondition_pixel_target.py @@ -28,7 +28,7 @@ def _texts(messages: list) -> list[str]: return [part["text"] for part in human.content if part["type"] == "text"] -def test_named_control_target_block_precedes_images(): +def test_named_control_target_block_follows_images(): item = { "action": "tap", "coordinates": [1001, 718], @@ -45,14 +45,18 @@ def test_named_control_target_block_precedes_images(): assert isinstance(messages[0], SystemMessage) assert isinstance(messages[1], HumanMessage) - first_text = messages[1].content[0]["text"] - assert first_text.startswith("[Target]") - assert "Kind: specific UI control" in first_text - assert "Label: 进入全屏模式" in first_text - assert "Resource ID: com.google.android.youtube:id/fullscreen_button" in first_text - assert "Class: android.widget.ImageView" in first_text - assert "[922, 655, 1080, 781]" in first_text - assert messages[1].content[1]["text"] == "[Image 1 (Reference)]" + blocks = messages[1].content + assert blocks[0]["type"] == "image_url" + assert blocks[1]["text"] == "[Image 1 (Reference)]" + assert blocks[2]["type"] == "image_url" + assert blocks[3]["text"] == "[Image 2 (Current State)]" + target_text = blocks[4]["text"] + assert target_text.startswith("[Target]") + assert "Kind: specific UI control" in target_text + assert "Label: 进入全屏模式" in target_text + assert "Resource ID: com.google.android.youtube:id/fullscreen_button" in target_text + assert "Class: android.widget.ImageView" in target_text + assert "[922, 655, 1080, 781]" in target_text assert any("Original Thinking" in t for t in _texts(messages)) assert any("Action: tap" in t for t in _texts(messages)) @@ -62,7 +66,8 @@ def test_coordinates_only_target_is_labelled_as_surface(): messages = pp._build_messages("SYSTEM", _PNG, _PNG, item, state=None) - first_text = messages[1].content[0]["text"] + blocks = messages[1].content + first_text = next(b["text"] for b in blocks if b.get("text", "").startswith("[Target]")) assert "Kind: coordinates only" in first_text # No hit test runs any more: the block states that nothing was recorded, it # does not claim the point was looked up in the UI hierarchy. diff --git a/tests/unit/config/test_local_models.py b/tests/unit/config/test_local_models.py new file mode 100644 index 000000000..740d2cba7 --- /dev/null +++ b/tests/unit/config/test_local_models.py @@ -0,0 +1,109 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the local model catalog and the `artemis model` CLI surface.""" + +from typer.testing import CliRunner + +from artemis.config.local_models import ( + DEFAULT_LOCAL_MODEL, + LOCAL_MODELS, + resolve_model_ref, +) +from artemis.interfaces.cli.commands.model import model_app + +runner = CliRunner() + + +class TestResolveModelRef: + def test_catalog_alias(self): + assert resolve_model_ref("gemma4-e4b") == ( + "gemma4-e4b", + LOCAL_MODELS["gemma4-e4b"].repo, + ) + + def test_default_alias_is_cataloged(self): + assert DEFAULT_LOCAL_MODEL in LOCAL_MODELS + + def test_repo_id_passthrough(self): + assert resolve_model_ref("org/some-model") == ( + "org/some-model", + "org/some-model", + ) + + def test_unknown_bare_name_rejected(self): + assert resolve_model_ref("not-a-model") is None + + +class TestModelCli: + def test_list_shows_catalog(self): + result = runner.invoke(model_app, ["list"]) + assert result.exit_code == 0 + assert "gemma4-e4b" in result.output + + def test_pull_rejects_unknown_alias(self): + result = runner.invoke(model_app, ["pull", "bogus-name"]) + assert result.exit_code == 1 + assert "Unknown model" in result.output + + def test_serve_rejects_unknown_alias(self): + result = runner.invoke(model_app, ["serve", "bogus-name"]) + assert result.exit_code == 1 + assert "Unknown model" in result.output + + def test_stop_without_state_is_clean(self, tmp_path, monkeypatch): + import artemis.interfaces.cli.commands.model as model_mod + + monkeypatch.setattr(model_mod, "_STATE_DIR", tmp_path) + result = runner.invoke(model_app, ["stop", "--port", "19191"]) + assert result.exit_code == 0 + assert "No managed server" in result.output + + +class TestVisionSoftTokens: + def test_catalog_default(self): + from artemis.config.local_models import LOCAL_MODELS + + assert LOCAL_MODELS["gemma4-e4b"].vision_soft_tokens == 1120 + + def test_apply_writes_processor_config(self, tmp_path, monkeypatch): + import json as _json + + import artemis.interfaces.cli.commands.model as model_mod + + snap = tmp_path / "snap" + snap.mkdir() + (snap / "processor_config.json").write_text( + _json.dumps({"image_processor": {"max_soft_tokens": 280}}) + ) + monkeypatch.setattr(model_mod, "_snapshot_path", lambda repo: snap) + assert model_mod._apply_vision_soft_tokens("any/repo", 1120) + cfg = _json.loads((snap / "processor_config.json").read_text()) + assert cfg["image_processor"]["max_soft_tokens"] == 1120 + + def test_apply_missing_key_is_noop(self, tmp_path, monkeypatch): + import json as _json + + import artemis.interfaces.cli.commands.model as model_mod + + snap = tmp_path / "snap" + snap.mkdir() + (snap / "processor_config.json").write_text(_json.dumps({"other": {}})) + monkeypatch.setattr(model_mod, "_snapshot_path", lambda repo: snap) + assert not model_mod._apply_vision_soft_tokens("any/repo", 1120) + + def test_serve_rejects_bad_token_value(self): + result = runner.invoke(model_app, ["serve", "--vision-tokens", "999"]) + assert result.exit_code == 1 + assert "vision-tokens" in result.output diff --git a/tests/unit/core/test_decision_client.py b/tests/unit/core/test_decision_client.py new file mode 100644 index 000000000..aa5d85112 --- /dev/null +++ b/tests/unit/core/test_decision_client.py @@ -0,0 +1,184 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for the System-One decision client.""" + +import io +import json +from types import SimpleNamespace +from unittest.mock import patch +import urllib.error + +import pytest + +from artemis.core.decision_client import DecisionClient + + +def _mock_response(body: dict): + return io.BytesIO(json.dumps(body).encode("utf-8")) + + +class _Closing(io.BytesIO): + """BytesIO that survives use as a urlopen context manager.""" + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + +def _mock_ctx(body: dict): + return _Closing(json.dumps(body).encode("utf-8")) + + +def test_noul_yes_verdict(): + client = DecisionClient(url="http://x/v1/decide", model="laya") + body = {"answers": {"q": {"noul": 0.92, "confidence": 0.9}}} + with patch("urllib.request.urlopen", return_value=_mock_ctx(body)) as mock: + result = client.noul("state text", "is it ok?") + assert result.answered + assert result.verdict is True + assert result.confidence == 0.9 + assert result.probabilities["yes"] == pytest.approx(0.92) + payload = json.loads(mock.call_args.args[0].data.decode()) + assert payload["questions"]["q"]["type"] == "noul" + assert payload["model"] == "laya" + + +def test_noul_no_verdict(): + client = DecisionClient(url="http://x/v1/decide") + body = {"answers": {"q": {"noul": 0.1, "confidence": 0.88}}} + with patch("urllib.request.urlopen", return_value=_mock_ctx(body)): + result = client.noul("s", "q?") + assert result.verdict is False + assert result.probabilities["no"] == pytest.approx(0.9) + + +def test_choice_verdict(): + client = DecisionClient(url="http://x/v1/decide") + body = { + "answers": { + "q": { + "choice": "settings", + "probabilities": {"settings": 0.8, "chrome": 0.2}, + "confidence": 0.8, + } + } + } + with patch("urllib.request.urlopen", return_value=_mock_ctx(body)): + result = client.choose("s", "pick", {"settings": "a", "chrome": "b"}) + assert result.verdict == "settings" + assert result.probabilities["settings"] == 0.8 + + +def test_network_error_fails_closed(): + client = DecisionClient(url="http://x/v1/decide") + with patch( + "urllib.request.urlopen", side_effect=urllib.error.URLError("down") + ): + result = client.noul("s", "q?") + assert not result.answered + assert result.verdict is None + assert result.confidence == 0.0 + + +def test_malformed_body_fails_closed(): + client = DecisionClient(url="http://x/v1/decide") + with patch("urllib.request.urlopen", return_value=_Closing(b"not json")): + result = client.noul("s", "q?") + assert not result.answered + + +def test_missing_answer_fields_fail_closed(): + client = DecisionClient(url="http://x/v1/decide") + body = {"answers": {"q": {}}} + with patch("urllib.request.urlopen", return_value=_mock_ctx(body)): + result = client.noul("s", "q?") + assert result.verdict is None + + +@pytest.mark.asyncio +async def test_anoul_async_wrapper(): + client = DecisionClient(url="http://x/v1/decide") + body = {"answers": {"q": {"noul": 0.7, "confidence": 0.95}}} + with patch("urllib.request.urlopen", return_value=_mock_ctx(body)): + result = await client.anoul("s", "q?") + assert result.verdict is True + assert result.confidence == 0.95 + + +# --- Planner wiring (_try_decision_layer) ------------------------------------- + + +@pytest.mark.asyncio +async def test_planner_decision_layer_disabled_returns_none(): + from artemis.agents.planner.planner import _try_decision_layer + + ctx = SimpleNamespace(agent_config=SimpleNamespace( + decision=SimpleNamespace(enabled=False))) + assert await _try_decision_layer(ctx, "goal", "before", "after") is None + + +@pytest.mark.asyncio +async def test_planner_decision_layer_answers_above_threshold(): + from artemis.agents.planner.planner import _try_decision_layer + from artemis.core.decision_client import DecisionResult + + ctx = SimpleNamespace(agent_config=SimpleNamespace( + decision=SimpleNamespace( + enabled=True, planner_validation=True, + url="http://x", model="laya", + confidence_threshold=0.85, timeout_seconds=5.0))) + with patch( + "artemis.core.decision_client.DecisionClient.anoul", + return_value=DecisionResult(verdict=True, confidence=0.9), + ): + out = await _try_decision_layer(ctx, "goal", "before", "after") + assert out["status"] == "success" + assert "approved" in out["feedback"] + + +@pytest.mark.asyncio +async def test_planner_decision_layer_escalates_below_threshold(): + from artemis.agents.planner.planner import _try_decision_layer + from artemis.core.decision_client import DecisionResult + + ctx = SimpleNamespace(agent_config=SimpleNamespace( + decision=SimpleNamespace( + enabled=True, planner_validation=True, + url="http://x", model="laya", + confidence_threshold=0.85, timeout_seconds=5.0))) + with patch( + "artemis.core.decision_client.DecisionClient.anoul", + return_value=DecisionResult(verdict=True, confidence=0.4), + ): + assert await _try_decision_layer(ctx, "goal", "before", "after") is None + + +@pytest.mark.asyncio +async def test_planner_decision_layer_escalates_on_failure(): + from artemis.agents.planner.planner import _try_decision_layer + from artemis.core.decision_client import DecisionResult + + ctx = SimpleNamespace(agent_config=SimpleNamespace( + decision=SimpleNamespace( + enabled=True, planner_validation=True, + url="http://x", model="laya", + confidence_threshold=0.85, timeout_seconds=5.0))) + with patch( + "artemis.core.decision_client.DecisionClient.anoul", + return_value=DecisionResult(verdict=None, confidence=0.0), + ): + assert await _try_decision_layer(ctx, "goal", "before", "after") is None diff --git a/tests/unit/core/test_local_model_probe.py b/tests/unit/core/test_local_model_probe.py new file mode 100644 index 000000000..5120ea7fe --- /dev/null +++ b/tests/unit/core/test_local_model_probe.py @@ -0,0 +1,131 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Local model endpoint probe: guidance must be model-aware. + +Artemis catalog aliases get ``artemis model pull`` / ``artemis model serve`` +commands; BYO models on third-party OpenAI-compatible servers get neutral +hints — the probe must never emit a command Artemis cannot run on that +endpoint (or reference a foreign binary). +""" + +import sys +import types +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from artemis.core.diagnostics.probes.local_model_probe import ( + LocalModelEndpointProbe, + _artemis_managed, + _serve_port, +) + + +class _FakeLLM: + def __init__(self, model, api_base="http://127.0.0.1:8097/v1", provider="custom"): + self.provider = provider + self.model = model + self.api_base = api_base + + +@pytest.fixture +def catalog(): + """Inject a stand-in catalog module; restore the import miss afterwards.""" + fake = types.ModuleType("artemis.config.local_models") + fake.LOCAL_MODELS = {"gemma4-e4b": object(), "qwen-vl": object()} + sys.modules["artemis.config.local_models"] = fake + yield fake + del sys.modules["artemis.config.local_models"] + + +def _llms(*models, base="http://127.0.0.1:8097/v1"): + return [_FakeLLM(m, api_base=base) for m in models] + + +def _run(llms): + with patch( + "artemis.core.diagnostics.probes.local_model_probe._iter_configured_llms", + side_effect=lambda: llms, + ): + return __import__("asyncio").run(LocalModelEndpointProbe().probe()) + + +def _commands(result): + return [a.payload for a in result.actions if a.action_type == "command"] + + +def _hints(result): + return [a.payload for a in result.actions if a.action_type == "hint"] + + +def test_catalog_detection_and_port_parsing(catalog): + assert _artemis_managed("gemma4-e4b") is True + assert _artemis_managed("llama3.2-vision") is False + assert _serve_port("http://127.0.0.1:8097/v1") == "8097" + assert _serve_port("https://api.example.com/v1") is None + + +def test_unreachable_catalog_model_suggests_artemis_serve(catalog): + result = _run(_llms("gemma4-e4b", base="http://127.0.0.1:9/v1")) + assert _commands(result) == ["artemis model serve gemma4-e4b --port 9"] + assert result.status.name == "FAIL" + + +def test_unreachable_byo_model_suggests_neutral_hint(catalog): + result = _run(_llms("llama3.2-vision", base="http://127.0.0.1:9/v1")) + assert _commands(result) == [] + assert any("OpenAI-compatible" in h for h in _hints(result)) + assert "ondevice-agent-platform" not in " ".join(_hints(result)) + + +def _serve_models(*ids): + """Patch httpx so ``GET /models`` returns the given ids.""" + resp = MagicMock() + resp.raise_for_status = MagicMock() + resp.json = MagicMock(return_value={"data": [{"id": i} for i in ids]}) + client = MagicMock() + client.get = AsyncMock(return_value=resp) + cm = MagicMock() + cm.__aenter__ = AsyncMock(return_value=client) + cm.__aexit__ = AsyncMock(return_value=False) + return patch( + "artemis.core.diagnostics.probes.local_model_probe.httpx.AsyncClient", + return_value=cm, + ) + + +def test_missing_catalog_model_suggests_artemis_pull(catalog): + """A reachable endpoint lacking a catalog alias gets `artemis model pull`.""" + with _serve_models("something-else"): + result = _run(_llms("qwen-vl")) + assert _commands(result) == ["artemis model pull qwen-vl"] + + +def test_missing_byo_model_suggests_own_tooling(catalog): + with _serve_models("gemma4-e4b"): + result = _run(_llms("llama3.2-vision")) + assert _commands(result) == [] + assert any("own tooling" in h for h in _hints(result)) + + +def test_cloud_only_config_is_not_used_pass(catalog): + class _Cloud: + provider = "google" + model = "gemini-3.8-flash" + api_base = None + + result = _run([_Cloud()]) + assert result.status.name == "PASS" + assert result.summary == "Not used" diff --git a/tests/unit/graph/test_exit_settlement.py b/tests/unit/graph/test_exit_settlement.py index 80a897e15..bed9c2aa2 100644 --- a/tests/unit/graph/test_exit_settlement.py +++ b/tests/unit/graph/test_exit_settlement.py @@ -62,6 +62,7 @@ def _make_ctx(tmp_path, **setup_kwargs): ctx.checkpoint_attempt_seq = {} ctx.checkpoint_repairs = {} ctx.assert_halt = False + ctx.operator_done_claimed = False ctx.final_check_attempts = 0 return ctx @@ -164,6 +165,47 @@ def test_gate_explicit_validation_failure_still_continues(tmp_path): assert "Plan validation failed" in logged +def test_gate_mark_done_claim_ends_when_nothing_to_settle(tmp_path): + """A mark_done claim with an incomplete plan and no check items routes to + the terminal path — with nothing to verify, that path is ``end``.""" + _write_plan(tmp_path, "- [/] Create alarm\n") + ctx = _make_ctx(tmp_path, disable_final_check=True) + ctx.operator_done_claimed = True + assert convergence_gate(_make_state(), ctx) == "end" + # The claim is consumed once: a bounced settlement must not re-route. + assert ctx.operator_done_claimed is False + + +def test_gate_mark_done_claim_routes_to_settlement_with_checks(tmp_path): + _write_plan(tmp_path, DONE_PLAN_WITH_CHECKS) + ctx = _make_ctx(tmp_path) + ctx.operator_done_claimed = True + assert convergence_gate(_make_state(), ctx) == "exit_settlement" + + +def test_gate_mark_done_claim_yields_to_validation_failure(tmp_path): + _write_plan(tmp_path, "- [/] Create alarm\n") + ctx = _make_ctx(tmp_path) + ctx.operator_done_claimed = True + assert convergence_gate(_make_state(checker_success=False), ctx) == "continue" + # Rejected validation leaves the claim latched for the next pass. + assert ctx.operator_done_claimed is True + + +def test_gate_all_blocked_plan_routes_to_settlement(tmp_path): + """A plan whose milestones are all [!] (declared infeasible) is terminal — + settlement/final-check arbitrates instead of looping forever.""" + _write_plan(tmp_path, "- [!] Open App Store\n") + ctx = _make_ctx(tmp_path) + assert convergence_gate(_make_state(), ctx) == "exit_settlement" + + +def test_gate_mixed_done_blocked_plan_routes_terminal(tmp_path): + _write_plan(tmp_path, "- [x] Open Photos\n- [!] Open App Store\n") + ctx = _make_ctx(tmp_path, disable_final_check=True) + assert convergence_gate(_make_state(), ctx) == "end" + + def test_gate_assert_halt_terminates_mid_plan(tmp_path): _write_plan(tmp_path, "- [/] Create alarm\n - assert: toast appeared\n") ctx = _make_ctx(tmp_path) diff --git a/tests/unit/test_llm_local_endpoint.py b/tests/unit/test_llm_local_endpoint.py new file mode 100644 index 000000000..0355cb54c --- /dev/null +++ b/tests/unit/test_llm_local_endpoint.py @@ -0,0 +1,188 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Endpoint plumbing for local OpenAI-compatible providers (e.g. a local OpenAI-compatible server).""" + +from types import SimpleNamespace + +from artemis.config.llm import LLMConfig +from artemis.llm.router import ModelEndpoint, ModelFactory, ModelProvider +from artemis.services.llm import _resolve_endpoint +from third_party.mobile_use.utils.file import strip_json_comments + + +def _ctx(**node_overrides) -> SimpleNamespace: + utils = { + "outputter": { + "provider": "google", + "model": "g", + "fallback": {"provider": "google", "model": "g"}, + }, + "hopper": { + "provider": "google", + "model": "g", + "fallback": {"provider": "google", "model": "g"}, + }, + "video_analyzer": None, + "object_detector": None, + } + utils.update(node_overrides.pop("utils", {})) + config = LLMConfig.model_validate( + { + "planner": { + "provider": "google", + "model": "g", + "fallback": {"provider": "google", "model": "g"}, + }, + "summarizer": { + "provider": "google", + "model": "g", + "fallback": {"provider": "google", "model": "g"}, + }, + "operator": { + "provider": "google", + "model": "g", + "fallback": {"provider": "google", "model": "g"}, + }, + "operator_summarizer": { + "provider": "google", + "model": "g", + "fallback": {"provider": "google", "model": "g"}, + }, + "log_reader_sub_agent": { + "provider": "google", + "model": "g", + "fallback": {"provider": "google", "model": "g"}, + }, + "log_analyzer": { + "provider": "google", + "model": "g", + "fallback": {"provider": "google", "model": "g"}, + }, + "diagnoser": { + "provider": "google", + "model": "g", + "fallback": {"provider": "google", "model": "g"}, + }, + "checker": { + "provider": "google", + "model": "g", + "fallback": {"provider": "google", "model": "g"}, + }, + "planner_avatar": { + "provider": "google", + "model": "g", + "fallback": {"provider": "google", "model": "g"}, + }, + "history_analyzer_expert": { + "provider": "google", + "model": "g", + "fallback": {"provider": "google", "model": "g"}, + }, + "diagnoser_expert": { + "provider": "google", + "model": "g", + "fallback": {"provider": "google", "model": "g"}, + }, + "explorer": { + "provider": "google", + "model": "g", + "fallback": {"provider": "google", "model": "g"}, + }, + "utils": utils, + **node_overrides, + } + ) + return SimpleNamespace(llm_config=config) + + +_LOCAL_NODE = { + "provider": "custom", + "model": "qwen-vl", + "api_base": "http://127.0.0.1:8080/v1", + "coordinate_format": "xy_px", + "max_tokens": 1024, + "fallback": { + "provider": "custom", + "model": "qwen-vl", + "api_base": "http://127.0.0.1:8080/v1", + "coordinate_format": "xy_px", + }, +} + + +def test_resolve_endpoint_forwards_api_base_and_coordinate_format(): + ctx = _ctx(utils={"object_detector": _LOCAL_NODE}) + ep = _resolve_endpoint(ctx, "object_detector", is_utils=True) + assert ep.provider == ModelProvider.CUSTOM + assert ep.model_name == "qwen-vl" + assert ep.api_base == "http://127.0.0.1:8080/v1" + assert ep.coordinate_format == "xy_px" + assert ep.max_tokens == 1024 + + +def test_resolve_endpoint_fallback_carries_local_fields(): + ctx = _ctx(utils={"object_detector": _LOCAL_NODE}) + ep = _resolve_endpoint(ctx, "object_detector", is_utils=True, use_fallback=True) + assert ep.api_base == "http://127.0.0.1:8080/v1" + assert ep.coordinate_format == "xy_px" + + +def test_resolve_endpoint_defaults_keep_google_shape(): + ctx = _ctx() + ep = _resolve_endpoint(ctx, "operator") + assert ep.provider == ModelProvider.GOOGLE + assert ep.api_base is None + assert ep.coordinate_format is None + + +def test_custom_endpoint_builds_chat_openai_with_local_base(monkeypatch): + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + ep = ModelEndpoint( + provider=ModelProvider.CUSTOM, + model_name="qwen-vl", + api_base="http://127.0.0.1:8080/v1", + ) + model = ModelFactory.create_model(ep) + assert type(model).__name__ == "ChatOpenAI" + assert str(model.openai_api_base) == "http://127.0.0.1:8080/v1" + + +def test_empty_openai_env_key_falls_back_to_placeholder(monkeypatch): + """An exported-but-empty OPENAI_API_KEY must not reach ChatOpenAI.""" + monkeypatch.setenv("OPENAI_API_KEY", "") + ep = ModelEndpoint( + provider=ModelProvider.CUSTOM, + model_name="qwen3.8-9b", + api_base="http://127.0.0.1:8080/v1", + ) + model = ModelFactory.create_model(ep) + assert type(model).__name__ == "ChatOpenAI" + assert model.openai_api_key.get_secret_value() == "EMPTY" + + +def test_strip_json_comments_preserves_urls_and_strings(): + text = ( + "{ // header\n" + ' "api_base": "http://127.0.0.1:8080/v1", // trailing\n' + ' "note": "x /* not a comment */ y",\n' + ' "escaped": "a \\"//\\" b"\n' + "}" + ) + import json + + parsed = json.loads(strip_json_comments(text)) + assert parsed["api_base"] == "http://127.0.0.1:8080/v1" + assert parsed["note"] == "x /* not a comment */ y" + assert parsed["escaped"] == 'a "//" b' diff --git a/tests/unit/utils/test_optional_ocr.py b/tests/unit/utils/test_optional_ocr.py index d8aa0dcd7..e7733e5b3 100644 --- a/tests/unit/utils/test_optional_ocr.py +++ b/tests/unit/utils/test_optional_ocr.py @@ -39,15 +39,19 @@ async def test_is_ocr_configured_false_when_unset(monkeypatch): monkeypatch.delenv("API_KEY", raising=False) monkeypatch.delenv("GOOGLE_API_KEY", raising=False) - with patch.object(Settings, "get_api_key", return_value=None): + with ( + patch.object(Settings, "get_api_key", return_value=None), + patch("artemis.utils.ocr_api._apple_vision_supported", return_value=False), + ): assert not is_ocr_configured() @pytest.mark.asyncio async def test_perform_ocr_returns_empty_list_when_unconfigured(monkeypatch): - """Verify perform_ocr returns [] when no OCR key is present.""" + """Verify perform_ocr returns [] when no OCR provider is available.""" with ( patch.object(Settings, "get_api_key", return_value=None), + patch("artemis.utils.ocr_api._apple_vision_supported", return_value=False), patch.dict("os.environ", {}, clear=True), ): result = await perform_ocr(screenshot_b64="dummy_b64") @@ -62,6 +66,7 @@ async def test_perform_ocr_executes_api_call_when_configured(): with ( patch.object(Settings, "get_api_key", return_value=mock_key), + patch("artemis.utils.ocr_api._apple_vision_supported", return_value=False), patch("artemis.utils.ocr_api.get_http_client") as mock_client, ): mock_resp = MagicMock() @@ -90,6 +95,158 @@ async def test_perform_ocr_executes_api_call_when_configured(): assert result[0]["text"] == "Login" +def test_is_ocr_configured_true_with_apple_vision(monkeypatch): + """Apple Vision alone (no Google key) makes OCR available on macOS.""" + monkeypatch.delenv("OCR_API_KEY", raising=False) + monkeypatch.delenv("VISION_API_KEY", raising=False) + with ( + patch.object(Settings, "get_api_key", return_value=None), + patch("artemis.utils.ocr_api._apple_vision_supported", return_value=True), + ): + assert is_ocr_configured() + + +@pytest.mark.asyncio +async def test_perform_ocr_uses_apple_vision_first(monkeypatch): + """Under auto provider, on-device Apple Vision answers without any HTTP call.""" + monkeypatch.delenv("OCR_API_KEY", raising=False) + monkeypatch.delenv("VISION_API_KEY", raising=False) + apple_result = [{"text": "Login", "position": [{"x": 1, "y": 2}]}] + with ( + patch.object(Settings, "get_api_key", return_value=None), + patch("artemis.utils.ocr_api._apple_vision_supported", return_value=True), + patch( + "artemis.utils.ocr_api._run_apple_vision", + new_callable=AsyncMock, + return_value=apple_result, + ) as mock_apple, + ): + result = await perform_ocr(screenshot_b64="aGVsbG8=") + assert result == apple_result + mock_apple.assert_awaited_once_with(b"hello") + + +@pytest.mark.asyncio +async def test_perform_ocr_falls_back_to_google_on_apple_failure(): + """An Apple Vision error under auto falls back to a keyed Google Vision.""" + mock_key = MagicMock() + mock_key.get_secret_value.return_value = "test_key" + + with ( + patch.object(Settings, "get_api_key", return_value=mock_key), + patch("artemis.utils.ocr_api._apple_vision_supported", return_value=True), + patch( + "artemis.utils.ocr_api._run_apple_vision", + new_callable=AsyncMock, + side_effect=RuntimeError("boom"), + ), + patch("artemis.utils.ocr_api.get_http_client") as mock_client, + ): + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.json.return_value = { + "responses": [ + { + "textAnnotations": [ + {"description": "Full", "boundingPoly": {}}, + { + "description": "Login", + "boundingPoly": {"vertices": [{"x": 1, "y": 2}]}, + }, + ] + } + ] + } + mock_http = AsyncMock() + mock_http.post.return_value = mock_resp + mock_client.return_value = mock_http + + result = await perform_ocr(screenshot_b64="aGVsbG8=") + assert result[0]["text"] == "Login" + + +@pytest.mark.asyncio +async def test_perform_ocr_apple_failure_without_key_returns_empty(monkeypatch): + """Apple failure with no Google key degrades to [] instead of raising.""" + monkeypatch.delenv("OCR_API_KEY", raising=False) + monkeypatch.delenv("VISION_API_KEY", raising=False) + with ( + patch.object(Settings, "get_api_key", return_value=None), + patch("artemis.utils.ocr_api._apple_vision_supported", return_value=True), + patch( + "artemis.utils.ocr_api._run_apple_vision", + new_callable=AsyncMock, + side_effect=RuntimeError("boom"), + ), + ): + assert await perform_ocr(screenshot_b64="aGVsbG8=") == [] + + +@pytest.mark.asyncio +async def test_perform_ocr_forced_apple_never_calls_google(monkeypatch): + """ARTEMIS_OCR_PROVIDER=apple must not fall through to the cloud provider.""" + monkeypatch.setenv("ARTEMIS_OCR_PROVIDER", "apple") + mock_key = MagicMock() + mock_key.get_secret_value.return_value = "test_key" + with ( + patch.object(Settings, "get_api_key", return_value=mock_key), + patch("artemis.utils.ocr_api._apple_vision_supported", return_value=True), + patch( + "artemis.utils.ocr_api._run_apple_vision", + new_callable=AsyncMock, + side_effect=RuntimeError("boom"), + ), + patch("artemis.utils.ocr_api.get_http_client") as mock_client, + ): + assert await perform_ocr(screenshot_b64="aGVsbG8=") == [] + mock_client.assert_not_called() + + +@pytest.mark.asyncio +async def test_perform_ocr_apple_breaker_stops_dispatch_after_failures(monkeypatch): + """Consecutive Apple subprocess failures disable the provider; a third + call must not spawn another worker.""" + import artemis.utils.ocr_api as ocr_api + + monkeypatch.delenv("OCR_API_KEY", raising=False) + monkeypatch.delenv("VISION_API_KEY", raising=False) + monkeypatch.setattr(ocr_api, "_apple_vision_consecutive_failures", 0) + with ( + patch.object(Settings, "get_api_key", return_value=None), + patch("artemis.utils.ocr_api._apple_vision_supported", return_value=True), + patch( + "artemis.utils.ocr_api._apple_vision_subprocess", + new_callable=AsyncMock, + side_effect=RuntimeError("wedged"), + ) as mock_worker, + ): + assert await perform_ocr(screenshot_b64="aGVsbG8=") == [] + assert await perform_ocr(screenshot_b64="aGVsbG8=") == [] + assert mock_worker.await_count == 4 + assert await perform_ocr(screenshot_b64="aGVsbG8=") == [] + assert not ocr_api.is_ocr_configured() + assert mock_worker.await_count == 4 + + +def test_vision_box_to_vertices_converts_bottom_left_origin(): + """Vision's normalized bottom-left box maps to TL/TR/BR/BL pixel vertices.""" + from types import SimpleNamespace + + from artemis.utils.apple_vision_ocr import _vision_box_to_vertices + + bb = SimpleNamespace( + origin=SimpleNamespace(x=0.1, y=0.8), + size=SimpleNamespace(width=0.2, height=0.05), + ) + vertices = _vision_box_to_vertices(bb, width=1000, height=2000) + assert vertices == [ + {"x": 100, "y": 300}, + {"x": 300, "y": 300}, + {"x": 300, "y": 400}, + {"x": 100, "y": 400}, + ] + + def test_fuse_ocr_with_xml_empty_ocr_retains_xml_pruning(): """Verify fuse_ocr_with_xml preserves full XML structure and allows downstream pruning.""" sample_xml = [ diff --git a/third_party/mobile_use/config/llm.py b/third_party/mobile_use/config/llm.py index be9f509a9..3f764c1b4 100644 --- a/third_party/mobile_use/config/llm.py +++ b/third_party/mobile_use/config/llm.py @@ -58,11 +58,17 @@ class LLM(BaseModel): provider: LLMProvider model: str temperature: float | None = None + api_base: str | None = None + max_tokens: int | None = None + coordinate_format: ( + Literal["yx_1000", "xy_1000", "yx_px", "xy_px", "yx_norm", "xy_norm"] | None + ) = None thinking_budget: int | None = None thinking_level: Literal["minimal", "low", "medium", "high"] | None = None reasoning_effort: Literal["none", "low", "medium", "high"] | None = None include_thoughts: bool | None = None enable_grounding: bool | None = None + timeout: float | None = None def validate_provider(self, name: str) -> None: """Ensure the required API key or credentials exist in settings for this provider.""" diff --git a/third_party/mobile_use/utils/file.py b/third_party/mobile_use/utils/file.py index e7930014e..055adc105 100644 --- a/third_party/mobile_use/utils/file.py +++ b/third_party/mobile_use/utils/file.py @@ -17,14 +17,51 @@ # See third_party/mobile_use/METADATA for the upstream source and local modifications. import json -import re from typing import IO def strip_json_comments(text: str) -> str: - text = re.sub(r"//.*?$", "", text, flags=re.MULTILINE) - text = re.sub(r"/\*.*?\*/", "", text, flags=re.DOTALL) - return text + """Remove ``//`` and ``/* */`` comments while preserving string literals. + + The naive regex version stripped ``//`` sequences inside quoted strings, + which made URL values (e.g. ``"api_base": "http://127.0.0.1:8080/v1"``) + unparseable. This scan tracks string/escape state and only treats the + comment markers that appear outside strings as comments. + """ + out: list[str] = [] + i, n = 0, len(text) + in_string = False + while i < n: + char = text[i] + if in_string: + out.append(char) + if char == "\\" and i + 1 < n: + out.append(text[i + 1]) + i += 2 + continue + if char == '"': + in_string = False + i += 1 + continue + if char == '"': + in_string = True + out.append(char) + i += 1 + continue + if char == "/" and i + 1 < n: + if text[i + 1] == "/": + while i < n and text[i] != "\n": + i += 1 + continue + if text[i + 1] == "*": + i += 2 + while i + 1 < n and not (text[i] == "*" and text[i + 1] == "/"): + i += 1 + i += 2 + continue + out.append(char) + i += 1 + return "".join(out) def load_jsonc(file: IO) -> dict: