From 082512e58177597584a2bf0a62dbb3b64537eeb8 Mon Sep 17 00:00:00 2001 From: Ranjithkumar Ragavan <43761047+RanjithRagavan@users.noreply.github.com> Date: Sun, 11 Oct 2026 23:35:19 -0700 Subject: [PATCH] fix(perception): keep system dialogs clickable, only filter true status/nav bars MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit com.android.systemui was treated as a high-confidence fixed-bar identity with no height bound, so any full-width SystemUI window anchored to the top or bottom screen edge — including full-screen alert dialog windows (USB debugging authorization, crash/ANR alerts) and tall bottom-anchored dialog panels — was classified as a fixed bar. The bogus bar then made _clamp_by_fixed_bars clamp or silently discard neighbouring elements and shift tap targets. Similarly, _detect_status_bar_height accepted the first systemui node with top == 0 without a height check; a full-screen SystemUI dialog window yielded a 2400px 'status bar', pushing every OCR coordinate off screen in _map_coordinates_back and silently dropping all OCR results for the frame. - Cap fixed-bar height in all cases (<=15% of screen height at the bottom, <=12% at the top), even for high-confidence system packages. - Never classify com.android.permissioncontroller nodes as bars; the package only owns dialogs. - Only accept thin top-anchored strips as the status bar in _detect_status_bar_height, and keep scanning for the real one. Fixes #180 --- artemis/utils/ocr_xml_fusion.py | 12 +- artemis/utils/ui_filter.py | 39 +++- tests/unit/utils/test_ocr_xml_fusion.py | 18 ++ tests/unit/utils/test_ui_filter.py | 267 ++++++++++++++++++++++++ 4 files changed, 328 insertions(+), 8 deletions(-) diff --git a/artemis/utils/ocr_xml_fusion.py b/artemis/utils/ocr_xml_fusion.py index ab8ee5d5c..707746552 100644 --- a/artemis/utils/ocr_xml_fusion.py +++ b/artemis/utils/ocr_xml_fusion.py @@ -89,6 +89,10 @@ def _calculate_overlap_ratio(ocr_bounds: dict[str, int], xml_bounds: dict[str, i def _detect_status_bar_height(xml_hierarchy: list[dict[str, Any]], screen_height: int) -> int: """Try to detect status bar height from XML, fallback to 4% of screen height.""" + # A status bar is a thin top-anchored strip; never taller than 12% of the + # screen (mirrors the top-bar bound in ui_filter._detect_fixed_system_bars). + max_status_bar_height = int(screen_height * 0.12) + for node in xml_hierarchy: package = node.get("package") or node.get("packageName") bounds_str = node.get("bounds") @@ -96,7 +100,13 @@ def _detect_status_bar_height(xml_hierarchy: list[dict[str, Any]], screen_height # Look for systemui at the top if package == "com.android.systemui": bounds = _parse_bounds(bounds_str) - if bounds and bounds["top"] == 0: + # Full-screen SystemUI dialog/alert windows (USB debugging + # authorization, crash/ANR alerts, ...) also report top == 0. + # Accepting their bottom as the "status bar height" pushes every + # OCR coordinate off screen in _map_coordinates_back and silently + # drops all OCR results for the frame, so require a plausible + # thin-bar height and keep scanning for the real status bar. + if bounds and bounds["top"] == 0 and bounds["bottom"] <= max_status_bar_height: return bounds["bottom"] return int(screen_height * 0.04) diff --git a/artemis/utils/ui_filter.py b/artemis/utils/ui_filter.py index 64aebfe72..92f47d70b 100644 --- a/artemis/utils/ui_filter.py +++ b/artemis/utils/ui_filter.py @@ -434,12 +434,19 @@ def _filter_and_prune( return [filtered_node] +#: Packages that only ever own dialogs or overlays and never own the status +#: or navigation bar. Nodes from these packages are never classified as fixed +#: bars, no matter how bar-like their bounds look (e.g. a bottom-anchored, +#: full-width runtime permission dialog). +_NEVER_FIXED_BAR_PACKAGES = frozenset({"com.android.permissioncontroller"}) + + def _detect_fixed_system_bars( ui_hierarchy: list[dict[str, Any]], screen_width: int, screen_height: int ) -> list[dict[str, int]]: """Detect fixed, non-scrollable system or navigation bars anchored to the extreme top/bottom edges. - Enforces strict triple guardrails to guarantee near-zero false positives: + Enforces strict quadruple guardrails to guarantee near-zero false positives: 1. Edge & Width Guarantee: Must anchor to top (top==0) or bottom (bottom==screen_height) and span >= 95% of screen width. @@ -447,9 +454,19 @@ def _detect_fixed_system_bars( descendants/ancestors must NOT be scrollable. 3. Identity & Height Bounds: High-confidence system package/IDs OR strict navigation bar height bounds. + 4. Thin Bar Guarantee: Even a high-confidence system package must be a + thin strip (<= 15% of screen height at the bottom, <= 12% at the top). + SystemUI also owns full-screen dialog/alert windows (USB debugging + authorization, crash/ANR alerts, ...); without a height cap those were + classified as bars, which made ``_clamp_by_fixed_bars`` mangle or + silently discard the dialog's neighbours and shifted tap targets. + Dialog-only packages (``_NEVER_FIXED_BAR_PACKAGES``) are never bars. """ fixed_bars = [] + max_bottom_bar_height = int(screen_height * 0.15) + max_top_bar_height = int(screen_height * 0.12) + def check_scrollable(node: dict[str, Any]) -> bool: if node.get("scrollable") is True or str(node.get("scrollable", "")).lower() == "true": return True @@ -487,16 +504,24 @@ def scan_for_bars(node: dict[str, Any], has_scrollable_ancestor: bool): ) # Check bottom fixed bar (e.g., bottom navigation bar / system navigation bar) - if bounds["bottom"] >= screen_height and is_full_width and height > 0: - if is_high_conf_id or (25 <= height <= int(screen_height * 0.15)): + if ( + package_name not in _NEVER_FIXED_BAR_PACKAGES + and bounds["bottom"] >= screen_height + and is_full_width + and 0 < height <= max_bottom_bar_height + ): + if is_high_conf_id or height >= 25: fixed_bars.append(bounds.copy()) return # Don't recurse into children of confirmed fixed bars # Check top fixed bar (e.g., status bar / top action bar) - if bounds["top"] <= 0 and is_full_width and height > 0: - if package_name == "com.android.systemui" or ( - 15 <= height <= int(screen_height * 0.12) - ): + if ( + package_name not in _NEVER_FIXED_BAR_PACKAGES + and bounds["top"] <= 0 + and is_full_width + and 0 < height <= max_top_bar_height + ): + if package_name == "com.android.systemui" or height >= 15: fixed_bars.append(bounds.copy()) return diff --git a/tests/unit/utils/test_ocr_xml_fusion.py b/tests/unit/utils/test_ocr_xml_fusion.py index e99e60f43..9a0818b84 100644 --- a/tests/unit/utils/test_ocr_xml_fusion.py +++ b/tests/unit/utils/test_ocr_xml_fusion.py @@ -75,6 +75,24 @@ def test_detect_status_bar_height(self): xml = [{"package": "com.example.app", "bounds": "[0,0][1080,2400]"}] self.assertEqual(_detect_status_bar_height(xml, screen_height=2400), 96) + def test_detect_status_bar_height_ignores_full_screen_systemui_dialog(self): + # Issue #180: a full-screen SystemUI dialog/alert window (USB debugging + # authorization, crash/ANR alerts) also reports top == 0. Treating its + # bottom as the status bar height (2400) would shift every OCR + # coordinate off screen in _map_coordinates_back and silently drop all + # OCR results for the frame. The scan must skip it and find the real + # status bar even when the dialog window comes first in the dump. + xml = [ + {"package": "com.android.systemui", "bounds": "[0,0][1080,2400]"}, + {"package": "com.android.systemui", "bounds": "[0,0][1080,120]"}, + {"package": "com.example.app", "bounds": "[0,0][1080,2400]"}, + ] + self.assertEqual(_detect_status_bar_height(xml, screen_height=2400), 120) + + # Only the dialog window visible: fall back to 4% of screen height. + xml = [{"package": "com.android.systemui", "bounds": "[0,0][1080,2400]"}] + self.assertEqual(_detect_status_bar_height(xml, screen_height=2400), 96) + def test_crop_image_remove_status_bar(self): # Create a dummy image img = Image.new("RGB", (100, 100), color="red") diff --git a/tests/unit/utils/test_ui_filter.py b/tests/unit/utils/test_ui_filter.py index f7628389b..292d1ee75 100644 --- a/tests/unit/utils/test_ui_filter.py +++ b/tests/unit/utils/test_ui_filter.py @@ -14,6 +14,7 @@ from artemis.utils.ui_filter import ( _clip_bounds, + _detect_fixed_system_bars, _is_semantic_empty, filter_node, filter_ui_hierarchy, @@ -347,3 +348,269 @@ def test_mutual_occlusion_warning_injection(): ) assert "WARNING: may overlap with [2], possible occlusion" in minimal_list assert "WARNING: may overlap with [1], possible occlusion" in minimal_list + + +# --------------------------------------------------------------------------- +# Issue #180: system-owned dialogs (runtime permission dialogs, SystemUI +# alerts) must stay clickable; only true thin status/nav bars are filtered. +# Fixtures use the flat element shape produced by +# parse_hierarchy_xml_to_elements (document order, no nesting). +# --------------------------------------------------------------------------- + +_STATUS_BAR = { + "package": "com.android.systemui", + "class": "android.widget.FrameLayout", + "bounds": "[0,0][1080,80]", + "clickable": "false", + "scrollable": "false", +} +_NAV_BAR = { + "package": "com.android.systemui", + "class": "android.widget.FrameLayout", + "bounds": "[0,2268][1080,2400]", + "clickable": "false", + "scrollable": "false", +} + + +def _texts_of(elements): + return {el.get("text") for el in elements} + + +def test_permission_controller_dialog_stays_clickable(): + # Runtime permission dialog (com.android.permissioncontroller) over an app + # window, with real status/nav bars present. The Allow/Deny buttons must + # survive filtering with their bounds untouched. + hierarchy = [ + dict(_STATUS_BAR), + dict(_NAV_BAR), + # Full-screen dialog window root, as uiautomator reports it. + { + "package": "com.android.permissioncontroller", + "class": "android.widget.FrameLayout", + "bounds": "[0,0][1080,2400]", + "clickable": "false", + "scrollable": "false", + }, + { + "package": "com.android.permissioncontroller", + "class": "android.widget.TextView", + "text": "Allow Camera to take pictures and record video?", + "bounds": "[90,1000][990,1080]", + "clickable": "false", + }, + { + "package": "com.android.permissioncontroller", + "class": "android.widget.Button", + "resource-id": "com.android.permissioncontroller:id/permission_allow_button", + "text": "While using the app", + "bounds": "[90,1900][990,2000]", + "clickable": "true", + }, + { + "package": "com.android.permissioncontroller", + "class": "android.widget.Button", + "resource-id": "com.android.permissioncontroller:id/permission_deny_button", + "text": "Deny", + "bounds": "[90,2100][990,2200]", + "clickable": "true", + }, + { + "package": "com.example.app", + "class": "android.widget.Button", + "text": "AppButton", + "bounds": "[100,300][980,420]", + "clickable": "true", + }, + ] + + bars = _detect_fixed_system_bars(hierarchy, 1080, 2400) + # Only the real status bar and nav bar are fixed bars. + assert bars == [ + {"left": 0, "top": 0, "right": 1080, "bottom": 80}, + {"left": 0, "top": 2268, "right": 1080, "bottom": 2400}, + ] + + filtered = filter_ui_hierarchy(hierarchy, screen_width=1080, screen_height=2400) + texts = _texts_of(filtered) + assert "While using the app" in texts + assert "Deny" in texts + allow = next(el for el in filtered if el.get("text") == "While using the app") + deny = next(el for el in filtered if el.get("text") == "Deny") + assert allow["bounds"] == "[90,1900][990,2000]" + assert deny["bounds"] == "[90,2100][990,2200]" + + +def test_permission_controller_never_classified_as_bar(): + # A bottom-anchored, full-width permission dialog variant (height 300px + # falls inside the generic 25..15%-of-screen bottom-bar range) must NOT be + # classified as a fixed bar: permissioncontroller only ever owns dialogs. + hierarchy = [ + { + "package": "com.android.permissioncontroller", + "class": "android.widget.FrameLayout", + "bounds": "[0,2100][1080,2400]", + "clickable": "false", + "scrollable": "false", + }, + { + "package": "com.android.permissioncontroller", + "class": "android.widget.Button", + "text": "Allow", + "bounds": "[90,2200][990,2300]", + "clickable": "true", + }, + ] + + assert _detect_fixed_system_bars(hierarchy, 1080, 2400) == [] + + filtered = filter_ui_hierarchy(hierarchy, screen_width=1080, screen_height=2400) + allow = next(el for el in filtered if el.get("text") == "Allow") + assert allow["bounds"] == "[90,2200][990,2300]" + + +def test_systemui_full_screen_dialog_window_not_a_bar(): + # SystemUI alert (USB debugging authorization): the dialog lives in a + # full-screen SystemUI window. That window is not a fixed bar and the + # dialog buttons must stay clickable. + hierarchy = [ + dict(_STATUS_BAR), + dict(_NAV_BAR), + { + "package": "com.android.systemui", + "class": "android.widget.FrameLayout", + "bounds": "[0,0][1080,2400]", + "clickable": "false", + "scrollable": "false", + }, + { + "package": "com.android.systemui", + "class": "android.widget.TextView", + "text": "Allow USB debugging?", + "bounds": "[90,1000][990,1060]", + "clickable": "false", + }, + { + "package": "com.android.systemui", + "class": "android.widget.Button", + "text": "Cancel", + "bounds": "[90,1340][520,1420]", + "clickable": "true", + }, + { + "package": "com.android.systemui", + "class": "android.widget.Button", + "text": "OK", + "bounds": "[560,1340][990,1420]", + "clickable": "true", + }, + ] + + bars = _detect_fixed_system_bars(hierarchy, 1080, 2400) + assert bars == [ + {"left": 0, "top": 0, "right": 1080, "bottom": 80}, + {"left": 0, "top": 2268, "right": 1080, "bottom": 2400}, + ] + + filtered = filter_ui_hierarchy(hierarchy, screen_width=1080, screen_height=2400) + texts = _texts_of(filtered) + assert "Cancel" in texts + assert "OK" in texts + ok = next(el for el in filtered if el.get("text") == "OK") + assert ok["bounds"] == "[560,1340][990,1420]" + + +def test_systemui_tall_bottom_dialog_panel_not_a_bar(): + # A full-width, bottom-anchored SystemUI alert panel taller than 15% of + # the screen is a dialog, not a navigation bar. Previously it was + # classified as a bar (systemui bypassed the height bound), and the bogus + # bar clamped or silently discarded elements crossing the dialog's top + # edge. + hierarchy = [ + { + "package": "com.android.systemui", + "class": "android.widget.FrameLayout", + "bounds": "[0,1700][1080,2400]", + "clickable": "false", + "scrollable": "false", + }, + { + "package": "com.android.systemui", + "class": "android.widget.Button", + "text": "OK", + "bounds": "[560,2200][990,2300]", + "clickable": "true", + }, + # App element crossing the dialog's top edge. + { + "package": "com.example.app", + "class": "android.widget.TextView", + "text": "CrossingItem", + "bounds": "[100,1600][980,1750]", + "clickable": "true", + }, + # App element mostly covered by the dialog panel. + { + "package": "com.example.app", + "class": "android.widget.TextView", + "text": "CoveredItem", + "bounds": "[100,1690][980,1800]", + "clickable": "true", + }, + ] + + assert _detect_fixed_system_bars(hierarchy, 1080, 2400) == [] + + filtered = filter_ui_hierarchy(hierarchy, screen_width=1080, screen_height=2400) + texts = _texts_of(filtered) + assert "OK" in texts + assert "CrossingItem" in texts + assert "CoveredItem" in texts + crossing = next(el for el in filtered if el.get("text") == "CrossingItem") + assert crossing["bounds"] == "[100,1600][980,1750]" + + +def test_real_status_and_nav_bars_still_filtered(): + # Regression guard: genuine thin bars are still detected and still clamp + # occluded elements, including a thin gesture nav bar (height < 25px) + # owned by SystemUI. + hierarchy = [ + dict(_STATUS_BAR), + dict(_NAV_BAR), + # Thin gesture-navigation bar (20px tall, below the generic 25px bound, + # matched via the high-confidence systemui identity). + { + "package": "com.android.systemui", + "class": "android.widget.FrameLayout", + "bounds": "[0,2380][1080,2400]", + "clickable": "false", + "scrollable": "false", + }, + { + "package": "com.example.app", + "class": "android.widget.TextView", + "text": "Partially occluded item", + "bounds": "[0,2100][1080,2300]", + "clickable": "true", + }, + { + "package": "com.example.app", + "class": "android.widget.TextView", + "text": "Almost entirely covered sliver", + "bounds": "[0,2260][1080,2300]", + "clickable": "true", + }, + ] + + bars = _detect_fixed_system_bars(hierarchy, 1080, 2400) + assert {"left": 0, "top": 0, "right": 1080, "bottom": 80} in bars + assert {"left": 0, "top": 2268, "right": 1080, "bottom": 2400} in bars + assert {"left": 0, "top": 2380, "right": 1080, "bottom": 2400} in bars + + filtered = filter_ui_hierarchy(hierarchy, screen_width=1080, screen_height=2400) + texts = _texts_of(filtered) + assert "Partially occluded item" in texts + assert "Almost entirely covered sliver" not in texts + clamped = next(el for el in filtered if el.get("text") == "Partially occluded item") + # Clamped against the nav bar top at 2268. + assert clamped["bounds"] == "[0,2100][1080,2268]"