Repository navigation
Expand file tree
/
Copy pathscreen_ui.py
More file actions
450 lines (360 loc) · 16.7 KB
/
Copy pathscreen_ui.py
File metadata and controls
450 lines (360 loc) · 16.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
"""
screen_ui.py — Gemma's tool layer for ScreenKitTY navigation.
Wires the four teammate modules (capture, navigator, voiceover_announcer,
click_at) into a single `ScreenUI` object that Gemma can drive via its
JSON tool-call output.
Three tools exposed to Gemma:
announce(text)
One-shot spoken message via VoiceOver. Non-blocking.
Use when there's something to say but no user choice needed.
navigate(payload)
Render an interactive menu overlay and block until the user picks
something or dismisses with Escape. Returns a structured result.
find_text_on_screen(text) [internal, auto-invoked during navigate]
Runs Apple Vision OCR on the most recent screenshot, finds the best
bounding-box match for `text`, and returns its center in LOGICAL
screen points (the coordinate space the Clicker expects).
Gemma only needs to emit the `announce` or `navigate` tool. For `navigate`,
it attaches a `target` field to any clickable item (the exact visible text
of the element). The orchestrator resolves each target → (x, y) via OCR
before handing the payload to Navigator. Items without a `target` become
high-level / conceptual options and come back to Gemma as a label.
"""
from __future__ import annotations
import json
import subprocess
import time
import traceback
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Callable, Optional
from capture import ScreenKit
from click_at import _detect_scale
from navigator import NavigationResult, Navigator
from voiceover_announcer import VoiceOver
from screenkitty.ocr import OcrError, OcrResult, TextMatch, ocr as run_ocr
from screenkitty.pipeline import describe_screen
# ---------------------------------------------------------------------------
# System prompt — tells Gemma the tool schema it must emit
# ---------------------------------------------------------------------------
SYSTEM_PROMPT = """\
You are an accessibility agent for a BLIND user.
You receive a screenshot and you describe what you see.
You have ONE tool:
{"tool": "announce", "text": "..."}
Use it to speak everything on screen to the user:
• Page title, main heading, what the page is about.
• Read article text, paragraphs, captions verbatim — do not summarise.
• Describe every image in plain words: people, objects, colours,
setting, any text visible in the image. The user cannot see it.
• Describe the layout: navigation links, buttons, search box, etc.
"text" can be as long as needed. Read the actual words on the screen.
Rules:
• Never click anything. Never mention clicking.
• Never present menus or options.
• Never ask the user what they want.
• Never invent content not visible on screen.
• No markdown fences. One JSON object. Nothing else.
"""
USER_PROMPT_FIRST_LOOK = """\
Here is the current screenshot. Describe what you see to the blind user
(announce), or click the most relevant interactive element (click).
Do not offer options. Decide and act.
"""
# ---------------------------------------------------------------------------
# Dispatch result — what one Gemma turn produced
# ---------------------------------------------------------------------------
@dataclass
class TurnResult:
"""Outcome of one Gemma → tool-dispatch cycle."""
kind: str # "clicked" | "conceptual" | "announced" | "dismissed" | "error"
label: Optional[str] = None # for conceptual/clicked: what the user picked
detail: Optional[str] = None # for error: message to log
# ---------------------------------------------------------------------------
# ScreenUI — tool layer
# ---------------------------------------------------------------------------
class ScreenUI:
"""High-level tool layer Gemma's JSON drives into."""
def __init__(self, announce_on_startup: bool = True) -> None:
self.sk = ScreenKit()
self.navigator = Navigator()
self.voice = VoiceOver()
# Scale factor for OCR pixel → logical point conversion.
# ScreenKit uses `screencapture` which writes physical-resolution PNGs,
# so OCR px_cx/px_cy are physical; divide by scale for logical points.
self._scale = _detect_scale()
self._last_screenshot: Optional[Path] = None
self._last_ocr: Optional[OcrResult] = None
# Pop the Navigator panel up immediately so the user sees it from
# the moment ScreenUI boots — not only on the first Gemma session.
self.navigator.ensure_visible()
if announce_on_startup:
self.announce("Screen Kitty ready. Analyzing your screen.")
# -- Screenshot + OCR cache --------------------------------------------
def screenshot(self) -> Path:
"""Capture a fresh full-screen screenshot and invalidate OCR cache."""
path = self.sk.screenshot_fullscreen()
self._last_screenshot = path
self._last_ocr = None
return path
@property
def last_screenshot(self) -> Optional[Path]:
return self._last_screenshot
def _ensure_ocr(self) -> Optional[OcrResult]:
"""Lazy-run OCR on the most recent screenshot."""
if self._last_screenshot is None:
return None
if self._last_ocr is None:
try:
self._last_ocr = run_ocr(self._last_screenshot)
except OcrError as exc:
print(f"[screen_ui] OCR failed: {exc}")
return None
return self._last_ocr
# -- Tool 1: announce ---------------------------------------------------
def announce(self, text: str, priority: str = "high") -> None:
"""Show `text` in the Navigator overlay AND speak it via `say`.
Routing announcements through the Navigator panel means the user
sees every announcement on screen (instead of depending on the
flaky voiceover_announcer subprocess), and hears it spoken via
macOS `say` (which works without VoiceOver being enabled).
"""
text = (text or "").strip()
if not text:
return
print(f"[screen_ui] announce: {text!r}")
# show_message speaks internally via navigator._speak, so this
# call gives us both the visual + audio in one shot.
self.navigator.show_message(text, speak=True)
def show_status(self, text: str) -> None:
"""Show a status message in the overlay WITHOUT speaking it.
Used for transient UI feedback like "Thinking..." where the user
doesn't need audio (or would find it annoying to hear repeatedly).
"""
text = (text or "").strip()
if not text:
return
self.navigator.show_message(text, speak=False)
# -- Tool 2: find_text_on_screen ---------------------------------------
def find_text_on_screen(self, text: str) -> Optional[dict]:
"""
Resolve a visible text string to its center point on screen.
Returns:
{"x": float, "y": float, "matched_text": str} in LOGICAL points,
or None if OCR found no match.
"""
ocr_result = self._ensure_ocr()
if ocr_result is None:
return None
matches = ocr_result.find(text)
if not matches:
return None
# Prefer an exact (case-insensitive) match; fall back to shortest
# containing line (more specific than long paragraphs).
needle = text.strip().lower()
exact = [m for m in matches if m.text.strip().lower() == needle]
chosen: TextMatch
if exact:
chosen = max(exact, key=lambda m: m.confidence)
else:
chosen = min(matches, key=lambda m: len(m.text))
# Convert physical pixel center → logical points.
lx = chosen.px_cx / self._scale
ly = chosen.px_cy / self._scale
return {"x": lx, "y": ly, "matched_text": chosen.text}
# -- Tool 3: navigate ---------------------------------------------------
def navigate(self, payload: dict) -> Optional[NavigationResult]:
"""
Resolve any `target` fields via OCR, load into Navigator, show overlay.
Must be called on the main thread — Navigator uses AppKit.
"""
resolved = self._resolve_targets(payload)
self.navigator.load(resolved)
return self.navigator.show(auto_click=True)
def _resolve_targets(self, payload: dict) -> dict:
"""
Walk menus → items, convert each item's `target` text to (x, y) via OCR.
Items without `target`, or with `target` not found on screen, are left
without coordinates — Navigator treats them as conceptual.
"""
resolved = {
"context": payload.get("context", ""),
"menus": [],
}
unresolved: list[str] = []
for menu in payload.get("menus", []) or []:
out_items = []
for item in menu.get("items", []) or []:
new_item = {"label": str(item.get("label", ""))}
target = item.get("target")
# Gemma may have already attached x/y itself — respect that.
if item.get("x") is not None and item.get("y") is not None:
new_item["x"] = float(item["x"])
new_item["y"] = float(item["y"])
new_item["coords"] = str(item.get("coords", "logical"))
elif target:
hit = self.find_text_on_screen(str(target))
if hit is not None:
new_item["x"] = hit["x"]
new_item["y"] = hit["y"]
new_item["coords"] = "logical"
print(f"[screen_ui] resolved '{target}' → "
f"({hit['x']:.0f}, {hit['y']:.0f})")
else:
unresolved.append(str(target))
print(f"[screen_ui] '{target}' not found on screen — "
f"item will be conceptual")
out_items.append(new_item)
resolved["menus"].append({
"label": str(menu.get("label", "")),
"items": out_items,
})
if unresolved:
print(f"[screen_ui] {len(unresolved)} target(s) unresolved: {unresolved}")
return resolved
# ---------------------------------------------------------------------------
# Gemma response parser
# ---------------------------------------------------------------------------
class GemmaResponseError(ValueError):
"""Raised when Gemma's output can't be parsed into a valid tool call."""
def parse_gemma_response(raw: str) -> dict:
"""
Parse Gemma's text output into a tool-call dict.
Tolerates leading/trailing prose and markdown code fences — extracts
the first balanced JSON object it finds and loads it.
Raises GemmaResponseError if nothing parseable is in the output.
"""
if not raw or not raw.strip():
raise GemmaResponseError("empty response")
text = raw.strip()
# Strip ```json … ``` or plain ``` … ``` fences
if text.startswith("```"):
lines = text.splitlines()
lines = lines[1:] # drop opening fence
if lines and lines[-1].strip().startswith("```"):
lines = lines[:-1]
text = "\n".join(lines).strip()
# Snap to the outermost JSON object span
first = text.find("{")
last = text.rfind("}")
if first < 0 or last < 0 or last < first:
raise GemmaResponseError(f"no JSON object in response: {raw[:200]!r}")
candidate = text[first : last + 1]
try:
parsed = json.loads(candidate)
except json.JSONDecodeError as exc:
raise GemmaResponseError(f"JSON decode failed: {exc} raw={candidate[:200]!r}") from exc
if not isinstance(parsed, dict):
raise GemmaResponseError(f"response not a JSON object: {type(parsed).__name__}")
tool = parsed.get("tool")
if tool != "announce":
raise GemmaResponseError(f"unknown tool: {tool!r}")
return parsed
# ---------------------------------------------------------------------------
# Session loop — shared by screen_reader.py and the orchestrator hotkey
# ---------------------------------------------------------------------------
# Upper bound on high-level conceptual hops inside a single session before we
# bail (prevents Gemma loops that never land on a clickable item).
MAX_CONCEPTUAL_HOPS = 8
# Seconds to wait after a real click before taking the next screenshot.
POST_CLICK_SETTLE = 0.8
def _format_prev_menu(response: dict) -> str:
"""Serialize a prior `navigate` response as a compact text summary.
Used in the concept-followup prompt so Gemma can see exactly which
options it already offered and is forbidden from re-emitting them.
"""
if not isinstance(response, dict):
return "(unavailable)"
menus = response.get("menus") or []
if not menus:
return "(no previous menu)"
lines = []
for menu in menus:
name = str(menu.get("label", "")).strip() or "(unnamed)"
lines.append(f"- Category: {name}")
for it in (menu.get("items") or []):
lbl = str(it.get("label", "")).strip()
tgt = it.get("target")
if tgt:
lines.append(f" * {lbl} [target: {tgt}]")
else:
lines.append(f" * {lbl}")
return "\n".join(lines)
def _ask_gemma(image_path: Path, user_prompt: str, page_context: Optional[str]) -> dict:
"""One Gemma turn → parsed tool-call dict. Raises on parse failure."""
result = describe_screen(
str(image_path),
prompt=user_prompt,
system_prompt=SYSTEM_PROMPT,
page_context=page_context,
# 1024 tokens so an `announce` can read a full article paragraph
# aloud without getting truncated mid-sentence.
max_tokens=1024,
temperature=0.1,
)
print(f"[gemma] {result.generation_tokens} tok in "
f"{result.elapsed_seconds:.1f}s ({result.generation_tps:.0f} tps)")
return parse_gemma_response(result.text)
def run_session(
ui: ScreenUI,
*,
screenshot_fn: Optional[Callable[[], Path]] = None,
page_context_fn: Optional[Callable[[], Optional[str]]] = None,
auto_continue_after_click: bool = True,
max_hops: int = MAX_CONCEPTUAL_HOPS,
) -> bool:
"""
Run one screen-reader session on the *current* thread.
Must be called from the main thread — Navigator uses AppKit and will
crash from any other thread.
Args:
ui: Shared ScreenUI instance (owns Navigator, VoiceOver, ScreenKit).
screenshot_fn: Callable that captures + returns a screenshot path.
Defaults to ui.screenshot() (full-screen via ScreenKit).
Override to capture an app-specific window.
page_context_fn: Callable that returns optional page-context text to
attach to Gemma's prompt (e.g., VisionGuide DOM JSON).
Called fresh for every Gemma turn.
auto_continue_after_click: If True, the session loops after every
real click (screen has changed, start over).
If False, returns after one click.
max_hops: Max conceptual hops before bailing on a single-screen chain.
Returns:
True if the caller should continue (another session may be started),
False if the user dismissed with Escape and the app should exit.
"""
shot_fn = screenshot_fn or ui.screenshot
ctx_fn = page_context_fn or (lambda: None)
while True:
try:
ui.show_status("Taking screenshot…")
path = shot_fn()
except Exception as exc:
print(f"[session] screenshot failed: {exc}")
ui.announce("Sorry, I could not take a screenshot.")
return True
ui._last_screenshot = path
ui._last_ocr = None
ui.show_status("Thinking…")
page_context = ctx_fn()
try:
response = _ask_gemma(path, USER_PROMPT_FIRST_LOOK, page_context)
except GemmaResponseError as exc:
print(f"[session] Gemma parse error: {exc}")
ui.announce("Sorry, I could not understand the screen.")
return True
except Exception as exc:
print(f"[session] Gemma inference failed: {exc}")
traceback.print_exc()
ui.announce("Sorry, something went wrong.")
return True
tool = response.get("tool")
if tool == "announce":
text = str(response.get("text", "")).strip()
if text:
ui.announce(text)
time.sleep(0.2)
return True
print(f"[session] unexpected tool: {tool!r}")
ui.announce("Something went wrong. Please try again.")
return True