From 8be55a1eeaa678a9f572b66597b83c214dda88c0 Mon Sep 17 00:00:00 2001 From: Lana Begunova <70295997+lana-20@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:07:38 -0800 Subject: [PATCH 1/8] Drive Android Automotive: every window, the driver's user (CHALLENGES 393-399) Measured on the Android Automotive 15 emulator, 1024x768, 2026-11-11. On main, map found nothing on the home screen; automotive.sh fails there at its first step and passes on this branch. Morpho Market's 26 phone tests: 12 passed before, 15 after. The other 11 wait for elements below a short landscape screen's fold, which wait_for does not scroll to; checked for a cart total and the gift card. No regression on a Pixel 7 AVD, Android 15: dialogs, clear-data, keyboard, gestures, crashes and otp checks pass. Quest not re-run; its fixture tests pass. --- README.md | 16 +- docs/CHALLENGES.md | 106 ++++++- docs/RELEASE-CHECKLIST.md | 2 +- docs/ROADMAP.md | 12 + docs/checks/README.md | 1 + docs/checks/automotive.sh | 164 +++++++++++ internal/agent/dialogs.go | 34 +-- internal/device/adb.go | 35 ++- internal/device/cleardata.go | 7 +- internal/device/logcat.go | 6 +- internal/device/permreset.go | 8 +- internal/device/users.go | 87 ++++++ internal/device/users_test.go | 51 ++++ internal/mobiumdriver/alerts.go | 22 +- internal/mobiumdriver/uia2.go | 22 +- internal/uitree/car_test.go | 136 +++++++++ internal/uitree/cover_test.go | 2 + internal/uitree/parse.go | 12 + internal/uitree/snapshot.go | 26 ++ internal/uitree/testdata/car-app-grid.xml | 178 ++++++++++++ .../testdata/car-mobiumapp-late-alert.xml | 128 +++++++++ .../testdata/car-mobiumapp-letterboxed.xml | 126 +++++++++ .../uitree/testdata/car-permission-prompt.xml | 133 +++++++++ internal/uitree/testdata/car-shop-dialog.xml | 122 ++++++++ .../uitree/testdata/car-shop-keyboard.xml | 263 ++++++++++++++++++ internal/uitree/windows.go | 106 ++++++- 26 files changed, 1749 insertions(+), 56 deletions(-) create mode 100755 docs/checks/automotive.sh create mode 100644 internal/device/users.go create mode 100644 internal/device/users_test.go create mode 100644 internal/uitree/car_test.go create mode 100644 internal/uitree/testdata/car-app-grid.xml create mode 100644 internal/uitree/testdata/car-mobiumapp-late-alert.xml create mode 100644 internal/uitree/testdata/car-mobiumapp-letterboxed.xml create mode 100644 internal/uitree/testdata/car-permission-prompt.xml create mode 100644 internal/uitree/testdata/car-shop-dialog.xml create mode 100644 internal/uitree/testdata/car-shop-keyboard.xml diff --git a/README.md b/README.md index 6d219106..e6ad704f 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,8 @@ *Mutatis mutandis.* **Mobile app automation for AI agents and people.** Android emulators, Android -phones, iOS simulators and iPhones — and TVs, Wear OS and Meta Quest's 2D app -panels — driven through one tool layer from a single Go binary with no runtime +phones, iOS simulators and iPhones — and TVs, Wear OS, Android Automotive and +Meta Quest's 2D app panels — driven through one tool layer from a single Go binary with no runtime dependencies. ```sh @@ -168,8 +168,8 @@ app may, and reporting "empty" would be a different and usually false claim. Verified on devices, not only compiled: a Pixel 7 emulator on Android 15 and Android 17, a Pixel 8 Pro on Android 17, an iPhone 17 Pro simulator on iOS 26.5 and an iPhone 18 Pro simulator on iOS 27.0, and an iPhone 15 Plus on iOS -26.6.2 and, since its update, iOS 27.0.1. TVs, Wear OS and Meta Quest are -below. The end-to-end flows in +26.6.2 and, since its update, iOS 27.0.1. TVs, Wear OS, cars and Meta Quest +are below. The end-to-end flows in [docs/checks/](docs/checks/) each drive a real device and assert on what happened rather than on an exit code. @@ -211,6 +211,14 @@ answers — with swiping right as a watch's back ([docs/checks/wear-os.sh](docs/checks/wear-os.sh)). Not yet a real watch, and not the crown: `press` has no rotary input. +**A car's screen** is Android too: since 2026-11-11 the UiAutomator2 backend +drives the Android Automotive 15 emulator — the system bars and the app +between them, the app's dialogs and the car's permission prompt, with the +keyboard left out of `map` as a phone leaves it — and acts for user 10, the +driver, as the car's apps do ([docs/checks/automotive.sh](docs/checks/automotive.sh)). +Not yet a real car, nor the instrument cluster's displays, and the +`uiautomator` driver reads only the app's window there. + **Meta Quest's 2D apps** are Android too: since 2026-10-25 the Android backends drive the panels on the Meta Spatial Simulator, Meta's Horizon OS emulator, where every app is a panel and every panel stays open. Not yet a diff --git a/docs/CHALLENGES.md b/docs/CHALLENGES.md index b7588086..b4e4ec0b 100644 --- a/docs/CHALLENGES.md +++ b/docs/CHALLENGES.md @@ -7,7 +7,7 @@ the source. The pattern across them is the reason the document exists: **almost nothing here was found by reading code or by a test written from imagination.** Of -392 defects, 314 were found only by running against a real device. Seventeen +399 defects, 321 were found only by running against a real device. Seventeen more — 255, 280, 282, 327, 329, 353, 355, 356, 359, 371, 375, 380, 382, 384, 386, 387 and 388 — were met while running against one, though the fault lay in a check, the host or Mobium's own design rather than in the platform. The other sixty-one — 4, 5, 6, 14, 23, @@ -8674,6 +8674,110 @@ from the other side: a wrapper around a smaller button still has no ref. A sweep test IDs in 14 screens of real apps on both platforms gave list refs only to such containers. iOS still labels it `ScrollView` where Android says `table`. +### 393. A car's home screen mapped as nothing + +**Found by:** asked whether Mobium drives a car, on the Android Automotive 15 +emulator (AVD on `automotive_1024p_landscape`, 1024x768). `map` on the home +screen answered "No actionable elements found" over a screen of climate +controls, an app grid and a dock. The car's screen is several windows: the top +and bottom bars (`com.android.systemui`, user 0), the launcher and a maps panel +embedded in it (user 10), and two more displays for the instrument cluster. +UiAutomator2 read the active window alone, the empty maps placeholder. + +**Fixed:** `ro.build.characteristics` holding `automotive` makes the device +windowed, as Horizon OS is (CHALLENGES 364): UiAutomator2 reads every window. +The `uiautomator` driver still reads one window, so on a car it maps only the +app. + +### 394. Read as windows, a car's system bars hid the app + +**Found by:** the same emulator, once 393 read every window. The app grid +mapped as the bars alone, and `current` answered `com.android.systemui`. The +windowed rule takes the topmost window as the app in front and keeps every other +app's out of `map`, which is right on a Quest, where a touch on a panel behind +only raises it. A car's bars are always on top, and both they and the app take +touches. + +**Fixed:** the bars' package is the tree's `SystemBars`: never the app in +front, and always reachable. The input method's window is `KeyboardPackage`: +never the app in front either, it covers what lies under it, and it is left out +of `map`, as a phone's hierarchy never holds it. Its keys were in `map` with the +bars treated as chrome alone. + +### 395. An app's dialog on a car was no dialog + +**Found by:** Morpho Market's "Remove Wait For It Mug?" dialog on the car +emulator. `map` listed Cancel and Remove; `app_alert` said no dialog was on +screen. While the dialog is up the hierarchy holds the bars and the dialog's +window, 443x206 at 290,271, and not the app's window behind it, so the windowed +rule found no panel the dialog lay in. + +**Fixed:** with system bars, a window in front is a dialog when it spans +neither the full height nor the full width of the screen less the bars. +Filling that area was the first rule, and it read MobiumApp's main screen as a +dialog: the car letterboxes a portrait app to 576x768 at 224,0, which keeps the +height whole. The first form also read the keyboard, a full-width window, as +"an untitled dialog"; it keeps the width whole, and is not the app in front +(394). Neither a bottom sheet nor a car's portrait screen has been seen. + +### 396. An untargeted swipe had no screen to cross + +**Found by:** `mobium swipe up` on the car emulator: "could not determine the +screen size to swipe across". The hierarchy's root adopts its window's bounds +only when there is one window. With several it had none, and about ten places +read the root's bounds as the screen: the swipe, `back`'s edge, `zoom`'s center, +the tap's bounds check and the scroll limit among them. A phone with a popup +window open has two windows too. + +**Fixed:** a root with no bounds takes the screen's: the display UiAutomator2 +names, widened to every window on it (CHALLENGES 326). + +### 397. grant, clear-data and an app's logs acted for user 0 on a car + +**Found by:** `clear-data` on the car emulator reported "cleared" and +"/sdcard/Android/data/dev.mobium.shop (read back empty)", and Morpho Market +relaunched with its cart still holding a mug. A car's apps run as user 10, the +driver; `pm` acts for user 0 unless told. `pm clear --user 10` emptied the cart. +Then, with CAMERA revoked for both users, `mobium grant dev.mobium.mobiumapp +camera` reported it granted, and `dumpsys package` showed it granted to user 0 +and still denied to user 10: the read-back took the first per-user block, user +0's. And `logs --app` filtered by uid 10252, user 0's, while the app ran as +1010252: none of its own lines against 4,273 under that uid. + +**Fixed:** `am get-current-user` names the user in front. When it is not 0, +`pm clear`, `pm grant`/`revoke` and `pm clear-permission-flags` pass `--user`, +the permission read starts at that user's section of `dumpsys package` and +refuses a package with none, and the app's uid is that user's. A phone runs the +same commands as before. User 10's shared storage cannot be listed from the +shell, "Permission denied", so `clear-data` no longer claims it read it. +Per-app locale was right already: `cmd locale` acts for the current user. + +### 398. app_alert quoted the instrument cluster + +**Found by:** MobiumApp's late alert on the car emulator: `app_alert` found it, +with SIGN OUT and STAY, titled "100\n31\n0\n0.0\nP\nR\nN\nD", which is +nowhere in the hierarchy. It is the cluster's gear selector and gauges, on +another display. UiAutomator2's alert endpoint finds the dialog and quotes the +text it gathers across displays. Answering through it pressed the right buttons. + +**Fixed:** on a car, a dialog the hierarchy holds is quoted from its own nodes. +The endpoint still finds and answers it. + +### 399. A car's permission prompt had buttons with no names + +**Found by:** MobiumApp's notification prompt on the car emulator. Its buttons +mapped as `car_ui_list_item_touch_interceptor` twice, and its text read +"Allow MobiumApp to send you notifications?\nAllow\nDon’t allow". A car's +list item is a nameless clickable laid over a sibling that holds its title. The +alert endpoint cannot press its buttons: "The expected button cannot be +detected on the alert". + +**Fixed:** a clickable with no name of its own, from itself or what it holds, +is named by the text of the siblings wholly under it, and a dialog's text +leaves that text out. `alert --button "Don’t allow"` presses it, and MobiumApp +received "notifications: denied". `accept` and `dismiss` stay refused, as +they are for any dialog whose buttons the platform does not mark. + ## Findings that were not defects Worth recording because each one closed off an approach that looked obvious. diff --git a/docs/RELEASE-CHECKLIST.md b/docs/RELEASE-CHECKLIST.md index 555d8473..e48974ce 100644 --- a/docs/RELEASE-CHECKLIST.md +++ b/docs/RELEASE-CHECKLIST.md @@ -4,7 +4,7 @@ [CI workflow](../.github/workflows/ci.yml) runs on every push — formatting, vet, both modules' tests, the four non-Go clients' own tests, and six cross-compile targets. This file is the rest, and the rest is where most -serious defects in this project have come from: 314 of 392 were found only by +serious defects in this project have come from: 321 of 399 were found only by running against a real device. Work through it before tagging a release, on three substrates: an Android diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index ebc52a2e..703b04e8 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -27,6 +27,18 @@ this is what is not. which `press` cannot send — Wear's lists also scroll by swipe, so nothing needs it yet. HarmonyOS, KaiOS and BlackBerry: no plans to implement at the moment; anyone can write a driver. + **Android Automotive measured 2026-11-11** (CHALLENGES 393–399): the + UiAutomator2 backend drives the car emulator after seven fixes + (`automotive.sh`). Left for a car, each seen once and not fixed: + `boot` gives up after 120 seconds while a cold car boot takes longer, + and the emulator boots anyway; the launcher's app grid maps each tile + twice, as its label and as `app_icon`; `text` reads a car-ui container's + id, `ROTARY_CONTAINER`, as screen text; `devices` says a car has + three-button navigation; the car's back button on the screen's + left edge is not mapped. Not tried: the `uiautomator` driver on a car, + which reads one window; the instrument cluster's displays; a car's + portrait screen; Android Auto, whose head unit has no hierarchy; CarPlay + in the iOS simulator. 3. **`press-tap` and `press-drag` on Android 16 and later**, refused since CHALLENGES 86: the refusals and comments said a restart would recover a device left with fingers down, which 86 measured it does diff --git a/docs/checks/README.md b/docs/checks/README.md index 596b178e..6dd001bc 100644 --- a/docs/checks/README.md +++ b/docs/checks/README.md @@ -65,6 +65,7 @@ checks do not: `clean-stop.sh`, which stops the caller's daemon, and | [flutter.sh](flutter.sh) | **A Flutter app** — MobiumApp's `flutter/` demo, installed by hand — on any device, through the semantics tree and no driver: `map` names every control; typing reaches the app, judged by what the app says it got, the password not echoed; a Semantics identifier taps its button; check, uncheck, a counted tap; `scroll-to` the fortieth row; back from a pushed screen. Passed 2026-10-01/02 on the Pixel 7 AVD, the iPhone 17 Pro simulator, the Pixel 8 Pro and the iPhone 15 Plus ([CHALLENGES 206](../CHALLENGES.md)) | | [tv-app.sh](tv-app.sh) | **A TV app driven with a remote** — mobium-app's `tv/` app, MobiumTV — on any Android device: a D-pad press says where focus went and the app agrees; select and a tap arrive as select and as touch; a tap on a tile that opens on focus reaches it as focus, not a click; nine presses right scroll a row to Card 10; `app_alert` sees a dialog, accept is its positive button and back cancels it; play-pause, fast-forward and stop arrive as the app counts them. Passed 2026-10-08 on the Android TV emulator (API 34) and on a Fire TV (Fire OS 7.7.1.7) over Wi-Fi, in 2 minutes 9 seconds; the TV found that a read can land before the screen it expects, so every screen and line is waited for | | [wear-os.sh](wear-os.sh) | **A watch** — Wear OS's own Settings, by locator: `text="Gestures"` is one match though a chip repeats its title's text; map lists a toggle chip's switch once, not three times; `app_alert` reads Wear's ✓/✕ dialog, dismiss leaves Tilt-to-wake off and accept presses OK; swiping right goes back. Refuses anything that is not a watch. Passed 2026-10-13 three times in three on a Wear OS 5.1 emulator (API 35, round 454×454), about 20 seconds; on `main` it failed at its first tap (CHALLENGES 315–317) | +| [automotive.sh](automotive.sh) | **A car** — Android Automotive, its screen of system bars, app and keyboard windows, its apps run as user 10: `map` lists the bars and the launcher; a tap in the bottom bar raises the temperature, read back and put back; an untargeted swipe; letterboxed MobiumApp is the app in front and no dialog; `app_alert` quotes the app's dialog from its own words, not the instrument cluster, and the car's permission prompt by its named buttons, `dismiss` refused and `--button` pressing; with the keyboard up no key is mapped and nothing reads as a dialog; `grant`, `logs --app` and `clear-data` act for user 10, each read back from the device or the app. Refuses anything that is not a car, or a car in front for user 0. Passed 2026-11-11 on the Android Automotive 15 emulator (1024×768); on `main` it failed at its first step (CHALLENGES 393–399) | | [twa.sh](twa.sh) | **A PWA from the Play Store** — a Trusted Web Activity, OYO Lite by default, installed by hand — on an Android phone: checked to be a TWA by its task, `launch` saying so, its page attached by the site's host and standalone, a tap counted by the page, and back through a tapped link's history and then out, each naming the app. Passed on the Pixel 8 Pro on 2026-10-01 ([CHALLENGES 205](../CHALLENGES.md)) | | [chrome.sh](chrome.sh) | **A page in Chrome on Android**, on an emulator or a phone, with the network: listed by its title and URL, `text`, `eval` and `map` answered by the page, a tap on a ref counted by a button put on the page, and the page's own link followed to iana.org. On an emulator it starts Chrome fresh first ([CHALLENGES 200](../CHALLENGES.md)); on a phone it leaves Chrome running, prints no listing, and closes its tab | | [pwa.sh](pwa.sh) | **A progressive web app**, Squoosh, on an emulator, an Android phone or a simulator, with the network: installed through Chrome's menu or Safari's share sheet when its icon is absent, launched from the icon, running standalone with an active service worker, its context found by its start URL. iOS: a tap in the web context, placed from text the page and the native tree both report, and one from `NATIVE_APP`, each counted by the page ([CHALLENGES 248, 290](../CHALLENGES.md)). Android phone: installed as a WebAPK, launched by its package with `app_launch` saying so ([CHALLENGES 202](../CHALLENGES.md)), the tap counted by the page, and the WebAPK and the tabs removed afterwards. Emulator: the tap counted, or refused because Chrome stopped reporting its WebView and the page counted nothing ([CHALLENGES 200](../CHALLENGES.md)). An iPhone refused | diff --git a/docs/checks/automotive.sh b/docs/checks/automotive.sh new file mode 100755 index 00000000..676f0509 --- /dev/null +++ b/docs/checks/automotive.sh @@ -0,0 +1,164 @@ +#!/bin/sh +# Android Automotive: the Android backend on a car's screen. +# +# A car's screen is several windows: the system bars, the app between them, +# and the keyboard in a window of its own. Its apps run as user 10, the +# driver, while user 0 runs only the system. On the Android Automotive 15 +# emulator (2026-11-11), before this check: +# - map found nothing on the home screen (CHALLENGES 393); +# - read as windows, the bars hid the app (394); +# - an app's dialog was no dialog to app_alert (395); +# - an untargeted swipe had no screen to cross (396); +# - grant, clear-data and an app's logs acted for user 0 (397); +# - app_alert quoted the instrument cluster (398); +# - the permission prompt's buttons had no names (399). +# +# Needs MobiumApp installed (mobiumdev/mobium-app). It is letterboxed on a +# landscape car screen, which is a case the dialog rule must tell from a +# dialog. Every change is put back: the temperature, and MobiumApp's +# permissions, reset to asking. +# +# docs/checks/automotive.sh an Android Automotive emulator +set -e +DEV="$1" +if [ -z "$DEV" ]; then echo "usage: $0 " >&2; exit 2; fi +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +. "$ROOT/docs/checks/lib.sh" +check_platform android "$DEV" +case "$(adb -s "$DEV" shell getprop ro.build.characteristics 2>&1)" in + *automotive*) ;; + *) echo "$(basename "$0") is for an Android Automotive device, and $DEV is not one" >&2; exit 2 ;; +esac +check_lock "$DEV" +M="$ROOT/bin/mobium --device $DEV" +APP=dev.mobium.mobiumapp +USER_ID=$(adb -s "$DEV" shell am get-current-user | tr -d '\r') +[ "$USER_ID" != 0 ] || { echo "$DEV is in front for user 0; this check is about a driver who is not" >&2; exit 2; } + +fail() { echo "FAIL: $*" >&2; exit 1; } +row() { printf " %-12s ok — %s\n" "$1" "$2"; } +no_dialog() { + out=$($M alert 2>&1) || fail "alert: $out" + [ "$out" = "no dialog is on screen" ] || fail "$1: $out" +} +# camera : whether that user's MobiumApp has the camera, from the +# device rather than from Mobium. +camera() { + adb -s "$DEV" shell dumpsys package $APP | awk -v u="User $1:" ' + index($0, u) { on = 1; next } + on && /User [0-9]+:/ { exit } + on && /android.permission.CAMERA:/ { sub(/.*granted=/, ""); sub(/,.*/, ""); print; exit }' +} + +echo "--- $DEV (user $USER_ID)" +$M press home >/dev/null +out=$($M map 2>&1) || fail "map: $out" +echo "$out" | grep -q "Raise temperature (button)" || fail "map lists no climate control: $out" +echo "$out" | grep -q "Applications (button" || fail "map lists no app grid button: $out" +[ "$($M current)" = com.android.car.carlauncher ] || fail "current is $($M current), not the launcher" +no_dialog "the home screen read as a dialog" +row "home" "map lists the bars and the launcher; current is the launcher" + +# The driver's side, the first of two, reads back after a tap, and is put +# back. Both sides share their ids, so they are taken by ref. +temp() { $M find testid=hvac_temperature_text | head -1 | awk '{print $2}'; } +ref() { $M find "label=$1" | head -1 | awk '{print $1}'; } +t=$(temp) +case "$t" in *[!0-9]*|"") fail "the temperature reads \"$t\"" ;; esac +at_exit "$M tap \$($M find 'label=Lower temperature' | head -1 | awk '{print \$1}') >/dev/null 2>&1 || true" +$M tap "$(ref 'Raise temperature')" >/dev/null +sleep 1 +t2=$(temp) +[ "$t2" = $((t + 1)) ] || fail "raising $t gave $t2" +row "bar" "a tap in the bottom bar raised $t to $t2" + +out=$($M swipe up 2>&1) || fail "swipe: $out" +row "swipe" "$out" + +$M terminate $APP >/dev/null 2>&1 || true +$M reset-permissions $APP >/dev/null +at_exit "$M reset-permissions $APP >/dev/null 2>&1 || true" +$M launch $APP >/dev/null +$M wait 'label=WebViews' --timeout 15s >/dev/null || fail "MobiumApp did not show its menu" +[ "$($M current)" = $APP ] || fail "current is $($M current), not MobiumApp" +no_dialog "letterboxed MobiumApp read as a dialog" +row "letterbox" "MobiumApp, letterboxed, is the app in front and no dialog" + +$M tap 'label=Dialog Demo' >/dev/null +$M tap testid=lateAlertBtn >/dev/null +i=0; until $M alert 2>/dev/null | grep -q 'a dialog is on screen'; do + i=$((i + 1)); [ $i -lt 8 ] || fail "the late alert never came"; sleep 1 +done +out=$($M alert) +case "$out" in + 'a dialog is on screen: "Session expiring\nStay signed in?"'*) ;; + *) fail "app_alert does not quote the dialog itself: $out" ;; +esac +$M alert --button Stay >/dev/null || fail "--button Stay was refused" +sleep 1 +$M scroll-to testid=dialogOutcome --direction up >/dev/null 2>&1 || true +[ "$($M text testid=dialogOutcome)" = "late: Stay" ] || fail "the app did not receive Stay" +row "dialog" "app_alert reads the app's dialog by its own words; Stay pressed" + +$M tap testid=notificationsBtn >/dev/null +i=0; until $M alert 2>/dev/null | grep -q 'a dialog is on screen'; do + i=$((i + 1)); [ $i -lt 8 ] || fail "the notification prompt never came"; sleep 1 +done +out=$($M alert) +case "$out" in + *'"Allow MobiumApp to send you notifications?"'*'buttons: "Allow", "Don’t allow"'*) ;; + *) fail "the permission prompt reads: $out" ;; +esac +st=0; out=$($M alert dismiss 2>&1) || st=$? +[ $st -eq 5 ] || fail "dismiss on the car's prompt exited $st, want 5 (refused): $out" +$M alert --button "Don’t allow" >/dev/null || fail "--button \"Don’t allow\" was refused" +sleep 1 +$M scroll-to testid=dialogOutcome --direction up >/dev/null 2>&1 || true +[ "$($M text testid=dialogOutcome)" = "notifications: denied" ] || fail "the app did not receive the denial" +row "prompt" "the car's prompt names Allow and Don’t allow; dismiss refused; --button denies" + +$M terminate $APP >/dev/null +$M launch $APP >/dev/null +$M tap 'label=Login Demo' >/dev/null +$M tap testid=username >/dev/null +$M keyboard | grep -q "the keyboard is up" || fail "tapping the username field raised no keyboard" +n=$($M map | grep -c -E '@e[0-9]+ (q|w|Space|Shift) \(button') || true +[ "$n" = 0 ] || fail "map lists $n keyboard keys" +no_dialog "the keyboard read as a dialog" +[ "$($M current)" = $APP ] || fail "with the keyboard up, current is $($M current)" +$M keyboard --hide >/dev/null +row "keyboard" "up, it is no dialog, not mapped, and MobiumApp stays in front" + +# User 10's own state, read from the device. Revoked for both users first, +# so a grant to user 0 alone leaves user $USER_ID without it. +adb -s "$DEV" shell pm revoke --user 0 $APP android.permission.CAMERA +adb -s "$DEV" shell pm revoke --user "$USER_ID" $APP android.permission.CAMERA +[ "$(camera "$USER_ID")" = false ] || fail "the revoke did not take for user $USER_ID" +$M grant $APP camera >/dev/null || fail "grant camera was refused" +[ "$(camera "$USER_ID")" = true ] || fail "grant camera left user $USER_ID without it (user 0: $(camera 0))" +row "grant" "camera granted to user $USER_ID, read back from the device" + +$M logs --app $APP >/dev/null 2>&1 +$M terminate $APP >/dev/null +$M launch $APP >/dev/null +sleep 3 +own=$($M logs --app $APP 2>/dev/null | grep -c -v -E 'ActivityManager|ActivityTaskManager|WindowManager|CoreBackPreview|ImeTracker|AppsFilter|Car[A-Za-z]*:' ) || true +[ "${own:-0}" -gt 0 ] || fail "logs --app gave none of MobiumApp's own lines" +row "logs" "$own of MobiumApp's own lines, from user $USER_ID's process" + +# clear-data: a count MobiumApp keeps, raised, then cleared. +$M terminate $APP >/dev/null +$M launch $APP >/dev/null +$M tap 'label=Storage Demo' >/dev/null +$M tap testid=saveOneMoreBtn >/dev/null +before=$($M text testid=storedCount) +[ "$before" != "stored: 0" ] || fail "saving left the count at 0" +$M clear-data $APP >/dev/null || fail "clear-data was refused" +$M launch $APP >/dev/null +$M tap 'label=Storage Demo' >/dev/null +after=$($M text testid=storedCount) +[ "$after" = "stored: 0" ] || fail "after clear-data the count reads \"$after\", was \"$before\"" +row "clear-data" "MobiumApp's count went from \"$before\" to \"$after\"" + +$M press home >/dev/null +echo "PASS" diff --git a/internal/agent/dialogs.go b/internal/agent/dialogs.go index 73d2a49a..bdcf87a3 100644 --- a/internal/agent/dialogs.go +++ b/internal/agent/dialogs.go @@ -203,43 +203,13 @@ func appDialogText(t *uitree.Tree) string { if d == nil { return "" } - if words := dialogWords(d); words != "" { + if words := uitree.DialogWords(d); words != "" { return words } // Only buttons: still a dialog in the way, with nothing to quote. return "an untitled dialog" } -// dialogWords is what a dialog or popover says — its texts in order, title -// first, one to a line — leaving out its buttons' captions, or "" when it -// has only buttons. -func dialogWords(d *uitree.Node) string { - var parts []string - var walk func(n *uitree.Node) - walk = func(n *uitree.Node) { - // A button's words are its caption, not what the dialog says: a - // platform alert's text is its title and message alone. - if n.Clickable { - return - } - text := strings.TrimSpace(n.Text) - if masked, ok := uitree.Redact(n); ok { - text = masked - } - if text == "" { - text = strings.TrimSpace(n.Label) - } - if text != "" { - parts = append(parts, text) - } - for _, c := range n.Children { - walk(c) - } - } - walk(d) - return strings.Join(parts, "\n") -} - // popoverNote says that a popover is in front, what it says, and how it // closes, or "" when none is. A popover is no dialog: the platform's alert // endpoint does not know it, and Pocket Casts' first-run tip, "Add @@ -263,7 +233,7 @@ func popoverNote(t *uitree.Tree) string { return "" } note := "a popover is in front" - if words := strings.ReplaceAll(dialogWords(p), "\n", " — "); words != "" { + if words := strings.ReplaceAll(uitree.DialogWords(p), "\n", " — "); words != "" { note += fmt.Sprintf(": %q", words) } return note + fmt.Sprintf(" — iOS reports what is behind it hidden until it closes, and it closes when "+ diff --git a/internal/device/adb.go b/internal/device/adb.go index 0870c2d6..1960fe22 100644 --- a/internal/device/adb.go +++ b/internal/device/adb.go @@ -732,11 +732,10 @@ var runtimePermsRe = regexp.MustCompile(`^\s+(android\.permission\.[A-Z_0-9]+|[a // com.android.settings and BODY_SENSORS. Anything that trusts the command // instead of reading the state back is reporting its own hope. func (a *ADB) RuntimePermissions(ctx context.Context, pkg string) (map[string]bool, error) { - out, err := a.Shell(ctx, "dumpsys", "package", pkg) + text, err := a.packageDump(ctx, pkg) if err != nil { return nil, err } - text := string(out) if strings.Contains(text, "Unable to find package") { return nil, mobiumerr.New(mobiumerr.InvalidArgument, "%s is not installed", pkg). WithRemedy("app_list_apps lists the installed apps by package id; app_launch takes that id, not the name on the screen") @@ -755,8 +754,8 @@ func (a *ADB) RuntimePermissions(ctx context.Context, pkg string) (map[string]bo m := runtimePermsRe.FindStringSubmatch(line) if m == nil { // The block ends at the first line that is not a permission. It - // appears once per user on a multi-user device; taking only the - // first is right, since that is the user adb is acting as. + // appears once per user; packageDump starts at the foreground + // user's. if strings.TrimSpace(line) != "" { break } @@ -785,7 +784,7 @@ func (a *ADB) SetPermission(ctx context.Context, pkg, perm string, grant bool) e if grant { verb = "grant" } - out, diag, err := a.ShellDiagnostics(ctx, "pm", verb, pkg, perm) + out, diag, err := a.ShellDiagnostics(ctx, a.pmArgs(ctx, verb, pkg, perm)...) if err != nil { return err } @@ -1192,6 +1191,32 @@ func (a *ADB) HorizonOS(ctx context.Context) (bool, error) { return strings.TrimSpace(string(out)) != "", nil } +// Automotive reports whether the device runs Android Automotive, whose +// screen is the app's window between system bars that are windows of their +// own (CHALLENGES 393). Known by `ro.build.characteristics`, which holds +// "automotive" on the Android 15 car emulator. +func (a *ADB) Automotive(ctx context.Context) (bool, error) { + out, err := a.Shell(ctx, "getprop", "ro.build.characteristics") + if err != nil { + return false, err + } + return strings.Contains(string(out), "automotive"), nil +} + +// InputMethodPackage is the package of the device's default keyboard, or "" +// when it names none. +func (a *ADB) InputMethodPackage(ctx context.Context) string { + out, err := a.Shell(ctx, "settings", "get", "secure", "default_input_method") + if err != nil { + return "" + } + pkg, _, ok := strings.Cut(strings.TrimSpace(string(out)), "/") + if !ok { + return "" + } + return pkg +} + // DeviceLocale reports the device's own locale, which an app follows unless it // has been pinned. func (a *ADB) DeviceLocale(ctx context.Context) (string, error) { diff --git a/internal/device/cleardata.go b/internal/device/cleardata.go index dd262694..77322c74 100644 --- a/internal/device/cleardata.go +++ b/internal/device/cleardata.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "sort" + "strconv" "strings" "time" @@ -43,7 +44,8 @@ func (a *ADB) ClearAppData(ctx context.Context, pkg string) (ClearedData, error) if _, err := a.Shell(ctx, "pm", "path", shellQuote(pkg)); err != nil { return ClearedData{}, mobiumerr.New(mobiumerr.InvalidArgument, "%s is not installed on this device", pkg) } - out, err := a.Shell(ctx, "pm", "clear", shellQuote(pkg)) + userFlag, user := a.userArgs(ctx) + out, err := a.Shell(ctx, append(append([]string{"pm", "clear"}, userFlag...), shellQuote(pkg))...) if err != nil { return ClearedData{}, mobiumerr.New(mobiumerr.DeviceServer, "pm clear %s: %w", pkg, err) } @@ -57,6 +59,9 @@ func (a *ADB) ClearAppData(ctx context.Context, pkg string) (ClearedData, error) Kept: []string{"the app itself"}, } ext := "/sdcard/Android/data/" + pkg + if user != 0 { + ext = "/storage/emulated/" + strconv.Itoa(user) + "/Android/data/" + pkg + } left, diag, err := a.ShellDiagnostics(ctx, "ls", "-A", shellQuote(ext)) switch { case err == nil && strings.TrimSpace(string(left)) != "": diff --git a/internal/device/logcat.go b/internal/device/logcat.go index 4a451565..1b7117b3 100644 --- a/internal/device/logcat.go +++ b/internal/device/logcat.go @@ -228,13 +228,15 @@ func filterLog(entries []LogEntry, q LogQuery) (kept []LogEntry, skipped int) { return kept, skipped } -// AppUID is the Linux UID an installed app runs as. +// AppUID is the Linux UID an installed app runs as, for the foreground user: +// each user runs it under its own (CHALLENGES 397). // // `pm list packages -U ` matches by substring, so asking for // com.android.settings also returns com.android.settings.auto_generated_rro_… // under a different UID; only the exact name is taken. func (a *ADB) AppUID(ctx context.Context, pkg string) (int, error) { - out, err := a.Shell(ctx, "pm", "list", "packages", "-U", shellQuote(pkg)) + user, _ := a.userArgs(ctx) + out, err := a.Shell(ctx, append(append([]string{"pm", "list", "packages", "-U"}, user...), shellQuote(pkg))...) if err != nil { return 0, err } diff --git a/internal/device/permreset.go b/internal/device/permreset.go index 774ee382..d0969a8e 100644 --- a/internal/device/permreset.go +++ b/internal/device/permreset.go @@ -94,15 +94,15 @@ func parseRuntimePermissions(dump string) []permissionRecord { } func (a *ADB) runtimePermissionRecords(ctx context.Context, pkg string) ([]permissionRecord, error) { - out, err := a.Shell(ctx, "dumpsys", "package", pkg) + out, err := a.packageDump(ctx, pkg) if err != nil { return nil, err } - if strings.Contains(string(out), "Unable to find package") { + if strings.Contains(out, "Unable to find package") { return nil, mobiumerr.New(mobiumerr.InvalidArgument, "%s is not installed", pkg). WithRemedy("app_list_apps lists the installed apps by package id; app_launch takes that id, not the name on the screen") } - return parseRuntimePermissions(string(out)), nil + return parseRuntimePermissions(out), nil } // ResetAppPermissions puts one app's runtime permissions back to asking, as a @@ -142,7 +142,7 @@ func (a *ADB) ResetAppPermissions(ctx context.Context, pkg string) (PermissionRe } } if p.decided() { - out, diag, err := a.ShellDiagnostics(ctx, "pm", "clear-permission-flags", pkg, p.name, "user-set", "user-fixed") + out, diag, err := a.ShellDiagnostics(ctx, a.pmArgs(ctx, "clear-permission-flags", pkg, p.name, "user-set", "user-fixed")...) if err != nil { return res, err } diff --git a/internal/device/users.go b/internal/device/users.go new file mode 100644 index 00000000..a99006ab --- /dev/null +++ b/internal/device/users.go @@ -0,0 +1,87 @@ +package device + +import ( + "context" + "regexp" + "strconv" + "strings" + + "github.com/mobiumdev/mobium/internal/mobiumerr" +) + +// CurrentUser is the Android user in the foreground. It is 0 on a phone, and +// 10 on Android Automotive, where user 0 runs only the system and the driver +// is user 10. `pm` and `dumpsys package` act for or list user 0 first unless +// told otherwise (CHALLENGES 397). A device that cannot say is taken to be +// user 0, as every command here assumed before. +func (a *ADB) CurrentUser(ctx context.Context) int { + out, err := a.Shell(ctx, "am", "get-current-user") + if err != nil { + return 0 + } + return parseCurrentUser(string(out)) +} + +func parseCurrentUser(out string) int { + n, err := strconv.Atoi(strings.TrimSpace(out)) + if err != nil || n < 0 { + return 0 + } + return n +} + +// userArgs is `--user N` for a `pm` subcommand when the foreground user is not +// user 0, and nothing otherwise, so a phone runs the same command as before. +func (a *ADB) userArgs(ctx context.Context) (args []string, user int) { + user = a.CurrentUser(ctx) + if user == 0 { + return nil, 0 + } + return []string{"--user", strconv.Itoa(user)}, user +} + +// pmArgs is a `pm` command for the foreground user: the subcommand, then +// userArgs, then the rest. +func (a *ADB) pmArgs(ctx context.Context, sub string, rest ...string) []string { + user, _ := a.userArgs(ctx) + return append(append([]string{"pm", sub}, user...), rest...) +} + +// packageDump is `dumpsys package ` cut to start at the foreground +// user's section, so a parser that takes the first runtime-permissions block +// reads that user's. User 0's section comes first, so the dump is unchanged +// for it. A dump with no section for the user is refused rather than read as +// user 0's. +func (a *ADB) packageDump(ctx context.Context, pkg string) (string, error) { + out, err := a.Shell(ctx, "dumpsys", "package", shellQuote(pkg)) + if err != nil { + return "", err + } + dump := string(out) + if strings.Contains(dump, "Unable to find package") { + return dump, nil + } + user := a.CurrentUser(ctx) + section, ok := userSection(dump, user) + if !ok { + return "", mobiumerr.New(mobiumerr.InvalidArgument, "%s is not installed for user %d, the one in front", pkg, user). + WithRemedy("app_install installs it for every user") + } + return section, nil +} + +var userHeaderRe = regexp.MustCompile(`(?m)^\s*User (\d+):`) + +// userSection is dump from the first line that opens user's section, and +// whether there is one. User 0 is the whole dump. +func userSection(dump string, user int) (string, bool) { + if user == 0 { + return dump, true + } + for _, m := range userHeaderRe.FindAllStringSubmatchIndex(dump, -1) { + if dump[m[2]:m[3]] == strconv.Itoa(user) { + return dump[m[0]:], true + } + } + return "", false +} diff --git a/internal/device/users_test.go b/internal/device/users_test.go new file mode 100644 index 00000000..8a5673c4 --- /dev/null +++ b/internal/device/users_test.go @@ -0,0 +1,51 @@ +package device + +import ( + "strings" + "testing" +) + +func TestParseCurrentUser(t *testing.T) { + for in, want := range map[string]int{"10\n": 10, "0": 0, "": 0, "Unknown command: get-current-user": 0, "-1": 0} { + if got := parseCurrentUser(in); got != want { + t.Errorf("parseCurrentUser(%q) = %d, want %d", in, got, want) + } + } +} + +// The shape of `dumpsys package` on the Android Automotive 15 emulator, with +// CAMERA granted to user 0 and not to user 10, the driver. +const twoUserDump = `Packages: + Package [dev.mobium.mobiumapp] (5c1e0a2): + User 0: ceDataInode=557714 installed=true hidden=false stopped=true + runtime permissions: + android.permission.CAMERA: granted=true, flags=[ USER_SENSITIVE_WHEN_GRANTED|USER_SENSITIVE_WHEN_DENIED] + User 10: ceDataInode=492273 installed=true hidden=false stopped=true + runtime permissions: + android.permission.CAMERA: granted=false, flags=[ USER_SENSITIVE_WHEN_GRANTED|USER_SENSITIVE_WHEN_DENIED] +Dexopt state: + User 0: + User 10: +` + +func TestUserSection(t *testing.T) { + all, ok := userSection(twoUserDump, 0) + if !ok || all != twoUserDump { + t.Error("user 0 is not the whole dump") + } + ten, ok := userSection(twoUserDump, 10) + if !ok || !strings.HasPrefix(strings.TrimSpace(ten), "User 10: ceDataInode") { + t.Fatalf("user 10's section starts %q", firstLine(ten)) + } + perms := parseRuntimePermissions(ten) + if len(perms) != 1 || perms[0].granted { + t.Errorf("user 10's permissions = %+v, want CAMERA not granted", perms) + } + if _, ok := userSection(twoUserDump, 11); ok { + t.Error("a user with no section was found") + } + // User 1 must not match User 10. + if _, ok := userSection(twoUserDump, 1); ok { + t.Error("user 1 matched user 10's header") + } +} diff --git a/internal/mobiumdriver/alerts.go b/internal/mobiumdriver/alerts.go index 7e45bdc2..f92afdbc 100644 --- a/internal/mobiumdriver/alerts.go +++ b/internal/mobiumdriver/alerts.go @@ -4,6 +4,7 @@ import ( "context" "github.com/mobiumdev/mobium/internal/mobiumerr" + "github.com/mobiumdev/mobium/internal/uitree" ) // Both device-side servers implement the W3C alert endpoints, so both backends @@ -12,8 +13,25 @@ import ( // label: the platforms differ in what the dialog says and agree on how to // answer it. -// AlertText reports what a system dialog says. -func (u *UIA2) AlertText(ctx context.Context) (string, error) { return u.w3c.alertText(ctx) } +// AlertText reports what a system dialog says. On a car the endpoint +// quotes text from the instrument cluster's display too, so a dialog in the +// hierarchy is quoted from its own nodes (CHALLENGES 398). +func (u *UIA2) AlertText(ctx context.Context) (string, error) { + text, err := u.w3c.alertText(ctx) + if err != nil || u.bars == nil { + return text, err + } + tree, err := u.Snapshot(ctx) + if err != nil { + return "", err + } + if d := tree.Dialog(); d != nil { + if words := uitree.DialogWords(d); words != "" { + return words, nil + } + } + return text, nil +} // AnswerAlert accepts or dismisses a system dialog. func (u *UIA2) AnswerAlert(ctx context.Context, accept bool) error { diff --git a/internal/mobiumdriver/uia2.go b/internal/mobiumdriver/uia2.go index 65236150..22ad117e 100644 --- a/internal/mobiumdriver/uia2.go +++ b/internal/mobiumdriver/uia2.go @@ -52,8 +52,16 @@ type UIA2 struct { // windowed is a device whose apps are windows over one another, read // every window at once (uitree.MarkWindows). windowed bool + // bars and keyboard are what a car's tree takes as uitree.SystemBars and + // uitree.KeyboardPackage. + bars []string + keyboard string } +// carSystemBars is the package that draws a car's top and bottom bars, on +// the Android Automotive 15 emulator. +var carSystemBars = []string{"com.android.systemui"} + // NewUIA2 prepares a driver. Nothing touches the device until Start. func NewUIA2(adb *device.ADB) *UIA2 { return &UIA2{adb: adb, w3c: newW3CClient(requestTimeout)} @@ -123,12 +131,21 @@ func (u *UIA2) Start(ctx context.Context, progress func(string)) error { u.teardownLocked(ctx) return err } - windowed, err := u.adb.HorizonOS(ctx) + horizon, err := u.adb.HorizonOS(ctx) + if err != nil { + u.teardownLocked(ctx) + return err + } + car, err := u.adb.Automotive(ctx) if err != nil { u.teardownLocked(ctx) return err } - u.windowed = windowed + u.windowed = horizon || car + u.bars, u.keyboard = nil, "" + if car { + u.bars, u.keyboard = carSystemBars, u.adb.InputMethodPackage(ctx) + } if err := u.configure(ctx); err != nil { u.teardownLocked(ctx) return err @@ -315,6 +332,7 @@ func (u *UIA2) Snapshot(ctx context.Context) (*uitree.Tree, error) { } if u.windowed { tree.MarkWindows() + tree.SystemBars, tree.KeyboardPackage = u.bars, u.keyboard } return tree, nil } diff --git a/internal/uitree/car_test.go b/internal/uitree/car_test.go new file mode 100644 index 00000000..e5bdbe5d --- /dev/null +++ b/internal/uitree/car_test.go @@ -0,0 +1,136 @@ +package uitree + +import ( + "strings" + "testing" +) + +// The car-*.xml fixtures are UiAutomator2's /source with enableMultiWindows +// on, from the Android Automotive 15 emulator (1024x768), 2026-11-11: every +// window, top first. + +const ( + carBars = "com.android.systemui" + carKeyboard = "com.google.android.apps.automotive.inputmethod" +) + +func carFixture(t *testing.T, file string) *Tree { + t.Helper() + tree := windowedFixture(t, file) + tree.SystemBars, tree.KeyboardPackage = []string{carBars}, carKeyboard + return tree +} + +// The app grid, under the top bar and over nothing but the bottom bar. Read +// as Quest's windows, the bars were the app in front (CHALLENGES 394). +func TestCarAppGridIsTheAppInFront(t *testing.T) { + plain := windowedFixture(t, "car-app-grid.xml") + if pkg := plain.Package(); pkg != carBars { + t.Fatalf("control: without bars the app in front should be the bars, as on the device; got %q", pkg) + } + tree := carFixture(t, "car-app-grid.xml") + if pkg := tree.Package(); pkg != "com.android.car.carlauncher" { + t.Errorf("the app in front is %q, want the launcher", pkg) + } + got := mapLabels(tree) + for _, want := range []string{"Settings", "Calendar", "Raise temperature", "Applications"} { + if _, ok := got[want]; !ok { + t.Errorf("map has no %q", want) + } + } + if d := tree.Dialog(); d != nil { + t.Errorf("the app grid read as a dialog: %s", d.Path) + } + if tree.Root.Bounds != tree.Screen || tree.Screen.Empty() { + t.Errorf("the root's bounds are %v, want the screen %v (CHALLENGES 396)", tree.Root.Bounds, tree.Screen) + } +} + +// Morpho Market's dialog: its window, the bars, and not the app's window +// behind it (CHALLENGES 395). +func TestCarAppDialog(t *testing.T) { + plain := windowedFixture(t, "car-shop-dialog.xml") + if plain.Dialog() != nil { + t.Fatal("control: without bars the dialog should go unseen, as it did on the device") + } + tree := carFixture(t, "car-shop-dialog.xml") + d := tree.Dialog() + if d == nil { + t.Fatal("the dialog is not a dialog") + } + if words := DialogWords(d); !strings.Contains(words, "Remove Wait For It Mug?") { + t.Errorf("the dialog says %q", words) + } +} + +// MobiumApp is portrait-locked, and the car letterboxes it to 576x768. It +// keeps the area's height whole, which a dialog does not. +func TestCarLetterboxedAppIsNoDialog(t *testing.T) { + tree := carFixture(t, "car-mobiumapp-letterboxed.xml") + if d := tree.Dialog(); d != nil { + t.Errorf("letterboxed MobiumApp read as a dialog: %s", d.Path) + } + if pkg := tree.Package(); pkg != "dev.mobium.mobiumapp" { + t.Errorf("the app in front is %q", pkg) + } +} + +// The keyboard is a window of its own over the app. It is no dialog, the app +// is still in front, and its keys are not mapped. +func TestCarKeyboard(t *testing.T) { + unmarked := carFixture(t, "car-shop-keyboard.xml") + unmarked.KeyboardPackage = "" + if pkg := unmarked.Package(); pkg != carKeyboard { + t.Fatalf("control: with the keyboard taken for an app, it should be the app in front; got %q", pkg) + } + if _, ok := mapLabels(unmarked)["q"]; !ok { + t.Fatal("control: with the keyboard taken for an app, its keys should be mapped") + } + tree := carFixture(t, "car-shop-keyboard.xml") + if d := tree.Dialog(); d != nil { + t.Errorf("the keyboard read as a dialog: %s", d.Path) + } + if pkg := tree.Package(); pkg != "dev.mobium.shop" { + t.Errorf("the app in front is %q", pkg) + } + got := mapLabels(tree) + for _, key := range []string{"q", "w", "Space", "Shift"} { + if _, ok := got[key]; ok { + t.Errorf("map lists the key %q", key) + } + } + if _, ok := got["Search"]; !ok { + t.Errorf("map has no search field: %v", got) + } +} + +// MobiumApp's late alert, quoted from its own nodes (CHALLENGES 398). +func TestCarDialogWords(t *testing.T) { + tree := carFixture(t, "car-mobiumapp-late-alert.xml") + d := tree.Dialog() + if d == nil { + t.Fatal("the late alert is not a dialog") + } + if words := DialogWords(d); words != "Session expiring\nStay signed in?" { + t.Errorf("the dialog says %q", words) + } +} + +// The car's permission prompt: each button a nameless interceptor over a +// sibling holding its caption (CHALLENGES 399). +func TestCarPermissionPrompt(t *testing.T) { + tree := carFixture(t, "car-permission-prompt.xml") + got := mapLabels(tree) + for _, want := range []string{"Allow", "Don’t allow"} { + if _, ok := got[want]; !ok { + t.Errorf("map has no %q: %v", want, got) + } + } + d := tree.Dialog() + if d == nil { + t.Fatal("the prompt is not a dialog") + } + if words := DialogWords(d); words != "Allow MobiumApp to send you notifications?" { + t.Errorf("the prompt says %q", words) + } +} diff --git a/internal/uitree/cover_test.go b/internal/uitree/cover_test.go index 90a991c4..d943e4fd 100644 --- a/internal/uitree/cover_test.go +++ b/internal/uitree/cover_test.go @@ -86,6 +86,8 @@ var coveredCaptures = map[string]string{ "panel, over MobiumApp's", "quest-shop-over-browser.xml": "Horizon OS, every window: Morpho Market's panel over MobiumApp's, the " + "browser's and Gallery's", + "car-permission-prompt.xml": "a car's permission prompt draws each button's caption over its nameless " + + "touch interceptor, so the tap moves off the caption to the rest of the button (CHALLENGES 399)", } func TestTheCoverRuleLeavesOrdinaryScreensAlone(t *testing.T) { diff --git a/internal/uitree/parse.go b/internal/uitree/parse.go index 68e0961e..32211eea 100644 --- a/internal/uitree/parse.go +++ b/internal/uitree/parse.go @@ -157,6 +157,13 @@ type Tree struct { // Windowed is set when the hierarchy holds every window on the display, // bottom first, each root marked Window (MarkWindows). Windowed bool + // SystemBars names the packages whose windows lie over every app on a + // windowed display and take touches with it: a car's top and bottom bars. + // They are never the app in front (CHALLENGES 394). + SystemBars []string + // KeyboardPackage is the package of the input method, whose window covers the + // app beneath it but is not mapped, as a phone's hierarchy never holds it. + KeyboardPackage string } var boundsRe = regexp.MustCompile(`\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]`) @@ -267,6 +274,11 @@ func ParseAndroid(data []byte) (*Tree, error) { for _, w := range root.Children { tree.Screen = tree.Screen.union(w.Bounds) } + // Several windows leave the root with no bounds of its own, and an + // untargeted swipe had no screen to cross (CHALLENGES 396). + if root.Bounds.Empty() { + root.Bounds = tree.Screen + } } foldCheckableRows(root) nameWebSlidersByLabel(root) diff --git a/internal/uitree/snapshot.go b/internal/uitree/snapshot.go index 034f6fec..e11e714a 100644 --- a/internal/uitree/snapshot.go +++ b/internal/uitree/snapshot.go @@ -231,6 +231,9 @@ func describe(n *Node) string { if s := descendantText(n, 3); s != "" { return truncate(s) } + if s := coveredSiblingText(n); s != "" { + return truncate(s) + } } if s := n.ShortTestID(); s != "" { return s @@ -238,6 +241,29 @@ func describe(n *Node) string { return n.ShortClass() } +// coveredSiblingText is the text a clickable with none of its own lies over: +// that of the nodes beside it, under the same parent, wholly inside its +// bounds. A car's list item is a nameless touch interceptor over a sibling +// holding its title, and "Allow" mapped as car_ui_list_item_touch_interceptor +// (CHALLENGES 399). +func coveredSiblingText(n *Node) string { + if !n.Clickable || n.Parent == nil || n.Bounds.Empty() { + return "" + } + var parts []string + for _, sib := range n.Parent.Children { + if sib == n || Actionable(sib) || sib.Bounds.Empty() || !enclosesRect(n.Bounds, sib.Bounds) { + continue + } + if s := clean(sib.Text); s != "" { + parts = append(parts, s) + } else if s := descendantText(sib, 2); s != "" { + parts = append(parts, s) + } + } + return strings.Join(parts, " ") +} + // onceIfTwice is a label that is one phrase said twice, said once. WebKit // names a link from everything in it, and Kiwix's article tiles hold a // picture whose alt text is the article's title, then the title: "America diff --git a/internal/uitree/testdata/car-app-grid.xml b/internal/uitree/testdata/car-app-grid.xml new file mode 100644 index 00000000..514b4969 --- /dev/null +++ b/internal/uitree/testdata/car-app-grid.xml @@ -0,0 +1,178 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/internal/uitree/testdata/car-mobiumapp-late-alert.xml b/internal/uitree/testdata/car-mobiumapp-late-alert.xml new file mode 100644 index 00000000..800e3de0 --- /dev/null +++ b/internal/uitree/testdata/car-mobiumapp-late-alert.xml @@ -0,0 +1,128 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/internal/uitree/testdata/car-mobiumapp-letterboxed.xml b/internal/uitree/testdata/car-mobiumapp-letterboxed.xml new file mode 100644 index 00000000..c3ffacc2 --- /dev/null +++ b/internal/uitree/testdata/car-mobiumapp-letterboxed.xml @@ -0,0 +1,126 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/internal/uitree/testdata/car-permission-prompt.xml b/internal/uitree/testdata/car-permission-prompt.xml new file mode 100644 index 00000000..ca3fac53 --- /dev/null +++ b/internal/uitree/testdata/car-permission-prompt.xml @@ -0,0 +1,133 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/internal/uitree/testdata/car-shop-dialog.xml b/internal/uitree/testdata/car-shop-dialog.xml new file mode 100644 index 00000000..f82bc8fc --- /dev/null +++ b/internal/uitree/testdata/car-shop-dialog.xml @@ -0,0 +1,122 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/internal/uitree/testdata/car-shop-keyboard.xml b/internal/uitree/testdata/car-shop-keyboard.xml new file mode 100644 index 00000000..b60f523b --- /dev/null +++ b/internal/uitree/testdata/car-shop-keyboard.xml @@ -0,0 +1,263 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/internal/uitree/windows.go b/internal/uitree/windows.go index 2d1e6b50..72755a25 100644 --- a/internal/uitree/windows.go +++ b/internal/uitree/windows.go @@ -1,5 +1,7 @@ package uitree +import "strings" + // A device where apps are windows over one another. Horizon OS, on the Meta // Spatial Simulator, keeps every app open as a panel smaller than the display, // over its shell's own windows: a full-screen one with the home screen, a @@ -40,13 +42,51 @@ func (t *Tree) frontWindow() *Node { return nil } for i := len(t.Root.Children) - 1; i >= 0; i-- { - if w := t.Root.Children[i]; !w.Bounds.Empty() { + if w := t.Root.Children[i]; !w.Bounds.Empty() && !t.isBar(w) && !t.isKeyboard(w) { return w } } return nil } +func (t *Tree) isBar(w *Node) bool { + for _, p := range t.SystemBars { + if w.Package == p { + return true + } + } + return false +} + +func (t *Tree) isKeyboard(w *Node) bool { + return t.KeyboardPackage != "" && w.Package == t.KeyboardPackage +} + +// appArea is the screen less the system bars along its edges: what an app +// has to draw in. False when no bar lies along an edge. +func (t *Tree) appArea() (Rect, bool) { + area, found := t.Screen, false + for _, w := range t.Root.Children { + b := w.Bounds + if !t.isBar(w) || b.Empty() { + continue + } + wide := b.X1 <= area.X1 && b.X2 >= area.X2 + tall := b.Y1 <= area.Y1 && b.Y2 >= area.Y2 + switch { + case wide && b.Y1 <= area.Y1 && b.Y2 > area.Y1: + area.Y1, found = b.Y2, true + case wide && b.Y2 >= area.Y2 && b.Y1 < area.Y2: + area.Y2, found = b.Y1, true + case tall && b.X1 <= area.X1 && b.X2 > area.X1: + area.X1, found = b.X2, true + case tall && b.X2 >= area.X2 && b.X1 < area.X2: + area.X2, found = b.X1, true + } + } + return area, found && !area.Empty() +} + // windowedDialog is the window in front when it is a dialog: inside a lower // window that is itself no bigger than a panel. A dialog an app opens lies // over its own panel — the permission prompt's window had MobiumApp's @@ -58,6 +98,18 @@ func (t *Tree) windowedDialog() *Node { if front == nil { return nil } + // Under system bars the app's own window is not in the hierarchy while + // its dialog is up. A letterboxed app keeps one side of the app's area + // whole, and a dialog neither (CHALLENGES 395). + if area, ok := t.appArea(); ok { + b := front.Bounds + tall := b.Y1 <= area.Y1 && b.Y2 >= area.Y2 + wide := b.X1 <= area.X1 && b.X2 >= area.X2 + if !tall && !wide { + return front + } + return nil + } for _, w := range t.Root.Children { if w == front { break @@ -93,7 +145,11 @@ func (t *Tree) behindFront(target *Node) *Node { if front == nil { return nil } - if w := t.windowOf(target); w != nil && w.Package != front.Package { + w := t.windowOf(target) + if w == nil || t.isBar(w) { + return nil + } + if w.Package != front.Package || t.isKeyboard(w) { return front } return nil @@ -160,3 +216,49 @@ func (t *Tree) WalkFront(fn func(*Node) bool) { } } } + +// DialogWords is what a dialog or popover says — its texts in order, title +// first, one to a line — leaving out its buttons' captions, or "" when it +// has only buttons. +func DialogWords(d *Node) string { + var parts []string + var walk func(n *Node) + walk = func(n *Node) { + // A button's words are its caption, not what the dialog says: a + // platform alert's text is its title and message alone. So is the + // text a nameless button lies over (coveredSiblingText). + if n.Clickable || underClickableSibling(n) { + return + } + text := strings.TrimSpace(n.Text) + if masked, ok := Redact(n); ok { + text = masked + } + if text == "" { + text = strings.TrimSpace(n.Label) + } + if text != "" { + parts = append(parts, text) + } + for _, c := range n.Children { + walk(c) + } + } + walk(d) + return strings.Join(parts, "\n") +} + +// underClickableSibling reports whether a clickable beside n, with no text of +// its own, lies wholly over it. +func underClickableSibling(n *Node) bool { + if n.Parent == nil || n.Bounds.Empty() { + return false + } + for _, sib := range n.Parent.Children { + if sib != n && sib.Clickable && clean(sib.Text) == "" && clean(sib.Label) == "" && + enclosesRect(sib.Bounds, n.Bounds) { + return true + } + } + return false +} From 279389f3a6e80ec9075661d2206af0068eabc46e Mon Sep 17 00:00:00 2001 From: Lana Begunova <70295997+lana-20@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:33:07 -0800 Subject: [PATCH 2/8] Record that CarPlay is not reachable in Xcode 27.1's simulator Looked for on 2026-11-11 with an iPhone 18 Pro simulator on iOS 27.0, through DeviceHub's menus and panels and simctl io enumerate. --- docs/ROADMAP.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 703b04e8..0c80d310 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -37,8 +37,15 @@ this is what is not. three-button navigation; the car's back button on the screen's left edge is not mapped. Not tried: the `uiautomator` driver on a car, which reads one window; the instrument cluster's displays; a car's - portrait screen; Android Auto, whose head unit has no hierarchy; CarPlay - in the iOS simulator. + portrait screen; Android Auto, whose head unit has no hierarchy. + **CarPlay is not reachable in Xcode 27.1's simulator** (looked for + 2026-11-11, iPhone 18 Pro on iOS 27.0): DeviceHub, which replaces + Simulator.app, has no CarPlay or external-display item in any menu or + panel, and `simctl io enumerate` lists LCD, Resizable, TVOut and + Wireless screens, none of them CarPlay; a screenshot of the external + one hangs. Left: Apple's standalone CarPlay Simulator, which drives a + real iPhone, or an older Xcode's Simulator.app. Seen on the way: + `boot --window` is ignored for a simulator, which boots with no window. 3. **`press-tap` and `press-drag` on Android 16 and later**, refused since CHALLENGES 86: the refusals and comments said a restart would recover a device left with fingers down, which 86 measured it does From bf949303c75eb200cabbaa85eaeda213bf0d8a75 Mon Sep 17 00:00:00 2001 From: Lana Begunova <70295997+lana-20@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:26:18 -0800 Subject: [PATCH 3/8] Fix what the car and DeviceHub turned up (CHALLENGES 400-406) Measured on the Android Automotive 15 emulator, the Pixel 7 AVD (Android 15) and an iPhone 18 Pro simulator (iOS 27.0, Xcode 27.1), 2026-11-11. boot read the cluster display's focus and never answered on a car; the same cold boot now answers in 11s. The 212s first quoted for it was the uptime when looked at, not a boot time; the longer limits it led to are not in this change. The car's climate control drops about one press in ten a second apart, from adb as from Mobium; automotive.sh presses again when it does. automotive.sh passes on the car, headed; boot.sh on the car and the Pixel 7 AVD with the simulator; back, dialogs, gestures, keyboard, clear-data, locators, mobium-app, login and audit on the Pixel 7 AVD; audit on the simulator. --- cmd/mobium/boot.go | 2 +- docs/CHALLENGES.md | 101 +++++++++++- docs/RELEASE-CHECKLIST.md | 2 +- docs/ROADMAP.md | 19 +-- docs/SETUP.md | 14 +- docs/checks/README.md | 2 +- docs/checks/audit.sh | 4 +- docs/checks/automotive.sh | 62 +++++-- docs/checks/lib-selftest.sh | 6 + docs/checks/lib.sh | 6 + docs/checks/login.sh | 6 +- docs/quickstart/README.md | 3 +- internal/agent/back.go | 7 + internal/agent/boot.go | 38 +++++ internal/agent/boot_test.go | 32 ++++ internal/agent/outputschema_gen.go | 2 +- internal/agent/schema.go | 2 +- internal/device/boot.go | 32 +++- internal/device/navmode.go | 9 ++ internal/device/simwindow.go | 65 ++++++++ internal/device/users_test.go | 31 ++++ internal/uitree/car_test.go | 42 +++++ internal/uitree/parse.go | 15 +- internal/uitree/snapshot.go | 41 ++++- internal/uitree/testdata/car-settings.xml | 187 ++++++++++++++++++++++ 25 files changed, 682 insertions(+), 48 deletions(-) create mode 100644 internal/device/simwindow.go create mode 100644 internal/uitree/testdata/car-settings.xml diff --git a/cmd/mobium/boot.go b/cmd/mobium/boot.go index 80d0f764..2601667f 100644 --- a/cmd/mobium/boot.go +++ b/cmd/mobium/boot.go @@ -22,7 +22,7 @@ func newBootCmd() *cobra.Command { return runTool("app_boot", call) }, } - cmd.Flags().BoolVar(&window, "window", false, "Show the emulator's window (default: headless)") + cmd.Flags().BoolVar(&window, "window", false, "Show the emulator's window, or the simulator in DeviceHub or Simulator.app (default: headless)") return cmd } diff --git a/docs/CHALLENGES.md b/docs/CHALLENGES.md index b4e4ec0b..44a0433c 100644 --- a/docs/CHALLENGES.md +++ b/docs/CHALLENGES.md @@ -7,8 +7,8 @@ the source. The pattern across them is the reason the document exists: **almost nothing here was found by reading code or by a test written from imagination.** Of -399 defects, 321 were found only by running against a real device. Seventeen -more — 255, 280, 282, 327, 329, 353, 355, 356, 359, 371, 375, 380, 382, 384, 386, 387 and 388 — were met while +406 defects, 327 were found only by running against a real device. Eighteen +more — 255, 280, 282, 327, 329, 353, 355, 356, 359, 371, 375, 380, 382, 384, 386, 387, 388 and 406 — were met while running against one, though the fault lay in a check, the host or Mobium's own design rather than in the platform. The other sixty-one — 4, 5, 6, 14, 23, 24, 32, 35, 36, 39, 44, 50, 53, 54, 57, 66, 89, 99, 100, 121, 123, 129, 130, @@ -8778,10 +8778,102 @@ leaves that text out. `alert --button "Don’t allow"` presses it, and MobiumApp received "notifications: denied". `accept` and `dismiss` stay refused, as they are for any dialog whose buttons the platform does not mark. +### 400. boot never answered on a car + +**Found by:** `mobium boot mobium-auto` on the Android Automotive 15 +emulator: "read response: … i/o timeout" after 120 seconds, and with the +client's deadline moved on, "did not finish booting within 3m30s", while the +emulator had booted. A boot waits for boot completed and then for a focused +window, read from the first `mCurrentFocus` line of `dumpsys window`. A car +lists a section for each display, and the first is an instrument cluster's, +display 3, whose focus is null for good. The device's uptime when it was +looked at, 212 seconds, was first taken for the time a boot takes and the +limits raised to fit; that was wrong, and is undone. + +**Fixed:** the main display's section, display 0's, decides. The same cold +boot of the car then answered in 11 seconds. A boot also reports "booting +mobium-auto: 30s so far" every 30 seconds: the client waits 120 seconds for an +answer, and a boot waits up to 210. + +### 401. A car launcher's tile mapped twice + +**Found by:** the car's app grid: Calendar, then `app_icon`, for every tile. A +tile is a clickable `app_item` holding a clickable, nameless `app_icon` image +over the tile's name. + +**Fixed:** a clickable with no words of its own or inside it, held by a +clickable parent that has words, is the parent's picture, as the iOS parser +already holds for an inaccessible button in a labeled one (CHALLENGES 244). Of +every captured hierarchy, it changes the car's app grid alone: 14 tiles. + +### 402. text read car-ui's marker as the screen's first line + +**Found by:** `mobium text` in car Settings opened with +`com.android.car.ui.utils.ROTARY_CONTAINER`, and `map` named the list by it. +car-ui writes that id into a container's content-desc for the rotary +controller. It names nothing. + +**Fixed:** a content-desc in car-ui's `com.android.car.ui.utils.` namespace is +not read as a label. + +### 403. A car said it had three-button navigation + +**Found by:** `mobium devices` on the car emulator: "navigation: three-button". +`navigation_mode` reads 0 there. A car has its own bars and a back button laid +over the app. A swipe in from the left edge, of the screen or of a letterboxed +app, changed nothing; the back key went back. + +**Fixed:** a car's navigation is "car bars", and `press back --gesture` is +refused there, naming the back key, as with buttons. + +### 404. A car's back button was not in map + +**Found by:** the car's back button, at the left edge over every app, was +never mapped. It is an ImageButton named "Back Button" that reports +`clickable=false`, `focusable=true`. A tap on it went back in MobiumApp. + +**Fixed:** a focusable, enabled button with a name is a control even when it +reports itself not clickable. Of every captured hierarchy, it adds the car's +back button alone. + +### 405. boot --window opened no window for a simulator + +**Found by:** `mobium boot "iPhone 18 Pro" --window` booted the simulator and +showed nothing: the flag was read only for an emulator. Xcode 27.1 has no +Simulator.app. DeviceHub, inside Xcode.app, shows simulators, and ignores +Simulator.app's `-CurrentDeviceUDID`. It opens on the device it last showed, +kept as `lastSelectedDeviceIdentifier` in `com.apple.dt.Devices`. + +**Fixed:** `--window` opens the simulator: in Simulator.app with +`-CurrentDeviceUDID` where Xcode has one, else in DeviceHub, with that +preference written before it starts. DeviceHub then opened on the iPhone 18 +Pro. A DeviceHub already running is not quit, since quitting it shut down the +simulator it showed; it is brought forward, and the answer names the device to +pick in its sidebar. + +### 406. Checks branched on find's status, which no match leaves 0 + +**Found by:** `automotive.sh` failing "the car's back button did not go back" +after it had gone back. `mobium find` answers "No element matches" with status +0, so `while $M find …` never ended, and `$M find … || fail` could not fail. +`audit.sh` guarded a finding's locator the same way, twice, and `login.sh` +took the save-password branch whether or not the sheet was up. + +**Fixed:** `shows`, in `lib.sh`, reads `find`'s answer. `lib-selftest.sh` +holds it to both answers. `login.sh` and `audit.sh` passed on the Pixel 7 AVD, +and `audit.sh` on the iPhone 18 Pro simulator. + ## Findings that were not defects Worth recording because each one closed off an approach that looked obvious. +- **A car's climate control drops a press.** On the Android Automotive 15 + emulator, ten presses of Lower temperature a second apart moved it nine + steps, from `adb shell input tap` and from Mobium alike. A first count said + `adb` lost two of ten raising; it had reached 82, the top, where a press + does nothing. `automotive.sh` presses once more when a press is dropped, + and says so. + - **Predictive back on Android 15 closes a React Native app.** MobiumApp built with predictive back turned on went back to closing on every back from a demo on the Android 15 AVD, `BackHandler` never called, while on @@ -9163,6 +9255,11 @@ Worth recording because each one closed off an approach that looked obvious. None of these is a Mobium defect; each cost real time and no error named its cause. +- **Xcode 27.1 has no Simulator.app.** DeviceHub, in + `Xcode.app/Contents/Applications`, replaces it, with no CarPlay or external + display in any menu or panel; `simctl io enumerate` lists no CarPlay screen. + Quitting DeviceHub shut down the simulator it showed. + - **`platform-tools` must be installed into the SDK root** with `sdkmanager`, not only as the Homebrew cask. The emulator validates the SDK root by looking for that directory and dies with `Broken AVD system path` — while `adb` is on diff --git a/docs/RELEASE-CHECKLIST.md b/docs/RELEASE-CHECKLIST.md index e48974ce..ec6234eb 100644 --- a/docs/RELEASE-CHECKLIST.md +++ b/docs/RELEASE-CHECKLIST.md @@ -4,7 +4,7 @@ [CI workflow](../.github/workflows/ci.yml) runs on every push — formatting, vet, both modules' tests, the four non-Go clients' own tests, and six cross-compile targets. This file is the rest, and the rest is where most -serious defects in this project have come from: 321 of 399 were found only by +serious defects in this project have come from: 327 of 406 were found only by running against a real device. Work through it before tagging a release, on three substrates: an Android diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 0c80d310..19e22bcf 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -27,25 +27,18 @@ this is what is not. which `press` cannot send — Wear's lists also scroll by swipe, so nothing needs it yet. HarmonyOS, KaiOS and BlackBerry: no plans to implement at the moment; anyone can write a driver. - **Android Automotive measured 2026-11-11** (CHALLENGES 393–399): the - UiAutomator2 backend drives the car emulator after seven fixes - (`automotive.sh`). Left for a car, each seen once and not fixed: - `boot` gives up after 120 seconds while a cold car boot takes longer, - and the emulator boots anyway; the launcher's app grid maps each tile - twice, as its label and as `app_icon`; `text` reads a car-ui container's - id, `ROTARY_CONTAINER`, as screen text; `devices` says a car has - three-button navigation; the car's back button on the screen's - left edge is not mapped. Not tried: the `uiautomator` driver on a car, - which reads one window; the instrument cluster's displays; a car's - portrait screen; Android Auto, whose head unit has no hierarchy. + **Android Automotive measured 2026-11-11** (CHALLENGES 393–404): the + UiAutomator2 backend drives the car emulator (`automotive.sh`). Not + tried: the `uiautomator` driver on a car, which reads one window; the + instrument cluster's displays; a car's portrait screen; Android Auto, + whose head unit has no hierarchy. **CarPlay is not reachable in Xcode 27.1's simulator** (looked for 2026-11-11, iPhone 18 Pro on iOS 27.0): DeviceHub, which replaces Simulator.app, has no CarPlay or external-display item in any menu or panel, and `simctl io enumerate` lists LCD, Resizable, TVOut and Wireless screens, none of them CarPlay; a screenshot of the external one hangs. Left: Apple's standalone CarPlay Simulator, which drives a - real iPhone, or an older Xcode's Simulator.app. Seen on the way: - `boot --window` is ignored for a simulator, which boots with no window. + real iPhone, or an older Xcode's Simulator.app. 3. **`press-tap` and `press-drag` on Android 16 and later**, refused since CHALLENGES 86: the refusals and comments said a restart would recover a device left with fingers down, which 86 measured it does diff --git a/docs/SETUP.md b/docs/SETUP.md index 88b9272b..d29c2643 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -147,7 +147,7 @@ emulator -avd mobium-test -no-window -no-snapshot-save -no-boot-anim -no-audio & ``` **Unlike an iOS simulator, the window cannot be attached afterwards.** `simctl -boot` is always headless and `open -a Simulator` shows it later; the Android +boot` is always headless and `mobium boot --window` shows it later; the Android emulator decides at launch, and the console (`adb emu help`) has no command to raise a window. To see the screen on a `-no-window` emulator, take a screenshot — or restart it without the flag. @@ -455,7 +455,7 @@ xcodebuild -downloadPlatform iOS # the runtime, if it is missing xcrun simctl list devices available xcrun simctl boot # boots headless — no window appears -open -a Simulator # only this puts it on screen +mobium boot --window # only this puts it on screen mobium --device map ``` @@ -464,9 +464,11 @@ has, and `mobium shutdown "iPhone 17 Pro"` ends the session and shuts it down. **`simctl boot` shows you nothing.** It starts the runtime with no window, so a simulator can be fully booted and driveable while the screen stays empty. -`open -a Simulator` attaches the GUI to whatever is already booted. Mobium -neither needs nor opens it — `xcrun simctl io screenshot` works -headless — but it is the first thing anyone asks about. +`mobium boot --window` opens it on screen: in Simulator.app, or from +Xcode 27.1, which has none, in DeviceHub. A DeviceHub already open is brought +forward and left on the device it shows, and the answer names the one to pick. +Mobium needs no window — `xcrun simctl io screenshot` works headless — +but it is the first thing anyone asks about. **Install Xcode from the Mac App Store.** `xcodes install` has been broken since 2026-09-11: Apple removed the `/olympus/v1/app/config` endpoint it uses @@ -544,7 +546,7 @@ xcodebuild -downloadPlatform tvOS # or Xcode > Settings > Components xcrun simctl create "Apple TV" \ com.apple.CoreSimulator.SimDeviceType.Apple-TV-4K-3rd-generation-1080p \ com.apple.CoreSimulator.SimRuntime.tvOS-26-5 -xcrun simctl boot && open -a Simulator +mobium boot --window ``` `mobium devices` lists it as an `apple tv simulator`, and the first command diff --git a/docs/checks/README.md b/docs/checks/README.md index 6dd001bc..8ef86ba7 100644 --- a/docs/checks/README.md +++ b/docs/checks/README.md @@ -65,7 +65,7 @@ checks do not: `clean-stop.sh`, which stops the caller's daemon, and | [flutter.sh](flutter.sh) | **A Flutter app** — MobiumApp's `flutter/` demo, installed by hand — on any device, through the semantics tree and no driver: `map` names every control; typing reaches the app, judged by what the app says it got, the password not echoed; a Semantics identifier taps its button; check, uncheck, a counted tap; `scroll-to` the fortieth row; back from a pushed screen. Passed 2026-10-01/02 on the Pixel 7 AVD, the iPhone 17 Pro simulator, the Pixel 8 Pro and the iPhone 15 Plus ([CHALLENGES 206](../CHALLENGES.md)) | | [tv-app.sh](tv-app.sh) | **A TV app driven with a remote** — mobium-app's `tv/` app, MobiumTV — on any Android device: a D-pad press says where focus went and the app agrees; select and a tap arrive as select and as touch; a tap on a tile that opens on focus reaches it as focus, not a click; nine presses right scroll a row to Card 10; `app_alert` sees a dialog, accept is its positive button and back cancels it; play-pause, fast-forward and stop arrive as the app counts them. Passed 2026-10-08 on the Android TV emulator (API 34) and on a Fire TV (Fire OS 7.7.1.7) over Wi-Fi, in 2 minutes 9 seconds; the TV found that a read can land before the screen it expects, so every screen and line is waited for | | [wear-os.sh](wear-os.sh) | **A watch** — Wear OS's own Settings, by locator: `text="Gestures"` is one match though a chip repeats its title's text; map lists a toggle chip's switch once, not three times; `app_alert` reads Wear's ✓/✕ dialog, dismiss leaves Tilt-to-wake off and accept presses OK; swiping right goes back. Refuses anything that is not a watch. Passed 2026-10-13 three times in three on a Wear OS 5.1 emulator (API 35, round 454×454), about 20 seconds; on `main` it failed at its first tap (CHALLENGES 315–317) | -| [automotive.sh](automotive.sh) | **A car** — Android Automotive, its screen of system bars, app and keyboard windows, its apps run as user 10: `map` lists the bars and the launcher; a tap in the bottom bar raises the temperature, read back and put back; an untargeted swipe; letterboxed MobiumApp is the app in front and no dialog; `app_alert` quotes the app's dialog from its own words, not the instrument cluster, and the car's permission prompt by its named buttons, `dismiss` refused and `--button` pressing; with the keyboard up no key is mapped and nothing reads as a dialog; `grant`, `logs --app` and `clear-data` act for user 10, each read back from the device or the app. Refuses anything that is not a car, or a car in front for user 0. Passed 2026-11-11 on the Android Automotive 15 emulator (1024×768); on `main` it failed at its first step (CHALLENGES 393–399) | +| [automotive.sh](automotive.sh) | **A car** — Android Automotive, its screen of system bars, app and keyboard windows, its apps run as user 10: `map` lists the bars and the launcher; a tap in the bottom bar moves the temperature, read back and put back, pressed again if the car drops the press; an untargeted swipe; each app tile mapped once and car-ui's marker not read as text; `devices` says car bars and a back gesture is refused; the car's back button mapped, and a tap on it goes back; letterboxed MobiumApp is the app in front and no dialog; `app_alert` quotes the app's dialog from its own words, not the instrument cluster, and the car's permission prompt by its named buttons, `dismiss` refused and `--button` pressing; with the keyboard up no key is mapped and nothing reads as a dialog; `grant`, `logs --app` and `clear-data` act for user 10, each read back from the device or the app. Refuses anything that is not a car, or a car in front for user 0. Passed 2026-11-11 on the Android Automotive 15 emulator (1024×768); on `main` it failed at its first step (CHALLENGES 393–404) | | [twa.sh](twa.sh) | **A PWA from the Play Store** — a Trusted Web Activity, OYO Lite by default, installed by hand — on an Android phone: checked to be a TWA by its task, `launch` saying so, its page attached by the site's host and standalone, a tap counted by the page, and back through a tapped link's history and then out, each naming the app. Passed on the Pixel 8 Pro on 2026-10-01 ([CHALLENGES 205](../CHALLENGES.md)) | | [chrome.sh](chrome.sh) | **A page in Chrome on Android**, on an emulator or a phone, with the network: listed by its title and URL, `text`, `eval` and `map` answered by the page, a tap on a ref counted by a button put on the page, and the page's own link followed to iana.org. On an emulator it starts Chrome fresh first ([CHALLENGES 200](../CHALLENGES.md)); on a phone it leaves Chrome running, prints no listing, and closes its tab | | [pwa.sh](pwa.sh) | **A progressive web app**, Squoosh, on an emulator, an Android phone or a simulator, with the network: installed through Chrome's menu or Safari's share sheet when its icon is absent, launched from the icon, running standalone with an active service worker, its context found by its start URL. iOS: a tap in the web context, placed from text the page and the native tree both report, and one from `NATIVE_APP`, each counted by the page ([CHALLENGES 248, 290](../CHALLENGES.md)). Android phone: installed as a WebAPK, launched by its package with `app_launch` saying so ([CHALLENGES 202](../CHALLENGES.md)), the tap counted by the page, and the WebAPK and the tabs removed afterwards. Emulator: the tap counted, or refused because Chrome stopped reporting its WebView and the page counted nothing ([CHALLENGES 200](../CHALLENGES.md)). An iPhone refused | diff --git a/docs/checks/audit.sh b/docs/checks/audit.sh index 1ea97ad2..10c6ad95 100755 --- a/docs/checks/audit.sh +++ b/docs/checks/audit.sh @@ -60,7 +60,7 @@ if [ "$KIND" = android ]; then [ "$got" = "LabelInName:testid=sendMismatch,RedundantDescription:testid=saveRedundant,SpeakableTextPresent:testid=iconUnnamed" ] || fail "the Screen Reader Demo's findings were $got, want its nameless icon, its Save button and its Submit form alone" echo "$out" | json '",".join(d.get("unchecked", []))' | grep -q TextContrast || fail "contrast is not named as unchecked" - $M find testid=iconUnnamed >/dev/null && $M find testid=saveRedundant >/dev/null || fail "a finding's locator does not resolve" + shows testid=iconUnnamed && shows testid=saveRedundant || fail "a finding's locator does not resolve" row "spoken" "the nameless icon, Save button and Submit form, not twins" $M terminate $APP >/dev/null 2>&1 || true echo PASS; exit 0 @@ -84,7 +84,7 @@ types=$(echo "$out" | json '",".join(sorted(set(f["type"] for f in d["findings"] nameless=$(echo "$out" | json 'sum(1 for f in d["findings"] if not f.get("bounds"))') loc=$(echo "$out" | json 'next((f["locator"] for f in d["findings"] if f.get("locator","").startswith("testid=")), "")') if [ -n "$loc" ]; then - $M find "$loc" >/dev/null 2>&1 || fail "a finding's locator does not resolve" + shows "$loc" || fail "a finding's locator does not resolve" fi row "settings" "$n finding(s): $types; $nameless with no element; a locator resolves" $M terminate com.apple.Preferences >/dev/null 2>&1 || true diff --git a/docs/checks/automotive.sh b/docs/checks/automotive.sh index 676f0509..439282ee 100755 --- a/docs/checks/automotive.sh +++ b/docs/checks/automotive.sh @@ -11,7 +11,10 @@ # - an untargeted swipe had no screen to cross (396); # - grant, clear-data and an app's logs acted for user 0 (397); # - app_alert quoted the instrument cluster (398); -# - the permission prompt's buttons had no names (399). +# - the permission prompt's buttons had no names (399); +# - boot gave up on a cold car boot (400), a tile mapped twice (401), +# car-ui's marker read as text (402), a car said three-button +# navigation (403), and its back button was not mapped (404). # # Needs MobiumApp installed (mobiumdev/mobium-app). It is letterboxed on a # landscape car screen, which is a case the dialog rule must tell from a @@ -59,22 +62,50 @@ echo "$out" | grep -q "Applications (button" || fail "map lists no app grid butt no_dialog "the home screen read as a dialog" row "home" "map lists the bars and the launcher; current is the launcher" -# The driver's side, the first of two, reads back after a tap, and is put -# back. Both sides share their ids, so they are taken by ref. +# The driver's side, the first of two, moves by one after a tap, and is put +# back. Both sides share their ids, so they are taken by ref. The emulator's +# climate control drops a press now and then, adb's as much as Mobium's (9 of +# 10 each, a second apart): one dropped press is pressed again, and said. At +# its highest, 82, raising does nothing, so it is lowered instead. temp() { $M find testid=hvac_temperature_text | head -1 | awk '{print $2}'; } ref() { $M find "label=$1" | head -1 | awk '{print $1}'; } t=$(temp) case "$t" in *[!0-9]*|"") fail "the temperature reads \"$t\"" ;; esac -at_exit "$M tap \$($M find 'label=Lower temperature' | head -1 | awk '{print \$1}') >/dev/null 2>&1 || true" -$M tap "$(ref 'Raise temperature')" >/dev/null -sleep 1 -t2=$(temp) -[ "$t2" = $((t + 1)) ] || fail "raising $t gave $t2" -row "bar" "a tap in the bottom bar raised $t to $t2" +if [ "$t" -ge 82 ]; then button="Lower temperature"; back="Raise temperature"; want=$((t - 1)) +else button="Raise temperature"; back="Lower temperature"; want=$((t + 1)); fi +moved() { + $M tap "$(ref "$button")" >/dev/null + i=0; while [ $i -lt 10 ]; do [ "$(temp)" = "$want" ] && return 0; sleep 0.5; i=$((i + 1)); done + return 1 +} +again="" +moved || { again=" (pressed twice: the first was dropped)"; moved; } || fail "$button from $t gave $(temp), twice" +at_exit "$M tap \$($M find 'label=$back' | head -1 | awk '{print \$1}') >/dev/null 2>&1 || true" +row "bar" "a tap in the bottom bar, $button, took $t to $want$again" out=$($M swipe up 2>&1) || fail "swipe: $out" row "swipe" "$out" +$M tap 'label=Applications' >/dev/null +$M wait 'label=Settings' --timeout 10s >/dev/null || fail "the app grid did not open" +out=$($M map) +n=$(echo "$out" | grep -c "app_icon") || true +[ "$n" = 0 ] || fail "map lists $n tile icons as controls of their own" +[ "$(echo "$out" | grep -c '^@e[0-9]* Settings (button')" -ge 1 ] || fail "map has no Settings tile: $out" +$M tap 'label=Settings,role=button' >/dev/null 2>&1 || $M tap "$(echo "$out" | grep -m1 ' Settings (button' | awk '{print $1}')" >/dev/null +$M wait 'label=Display' --timeout 10s >/dev/null 2>&1 || $M wait 'text=Display' --timeout 10s >/dev/null || fail "Settings did not open" +$M text | grep -q "ROTARY" && fail "text reads car-ui's ROTARY marker as screen text" +$M map | grep -q "ROTARY" && fail "map names a list by car-ui's ROTARY marker" +row "launcher" "each tile maps once; Settings opens; car-ui's marker is not read as text" + +case "$($ROOT/bin/mobium devices 2>&1 | grep "^$DEV ")" in + *"navigation: car bars"*) ;; + *) fail "devices does not say car bars: $($ROOT/bin/mobium devices 2>&1 | grep "^$DEV ")" ;; +esac +st=0; out=$($M press back --gesture 2>&1) || st=$? +[ $st -eq 5 ] || fail "a back gesture on a car exited $st, want 5 (refused): $out" +row "navigation" "devices says car bars; a back gesture is refused for the back key" + $M terminate $APP >/dev/null 2>&1 || true $M reset-permissions $APP >/dev/null at_exit "$M reset-permissions $APP >/dev/null 2>&1 || true" @@ -84,6 +115,19 @@ $M wait 'label=WebViews' --timeout 15s >/dev/null || fail "MobiumApp did not sho no_dialog "letterboxed MobiumApp read as a dialog" row "letterbox" "MobiumApp, letterboxed, is the app in front and no dialog" +$M tap 'label=Dialog Demo' >/dev/null +# The car's back button lies over the app's left edge: mapped, and a tap on +# it goes back. It shows while an app is in front. +i=0; until $M map | grep -q "Back Button (button"; do + i=$((i + 1)); [ $i -lt 6 ] || fail "map lists no car back button"; sleep 1 +done +shows 'label=One-button alert' || fail "the Dialog Demo is not open" +$M tap 'label=Back Button' >/dev/null +i=0; while shows 'label=One-button alert'; do + i=$((i + 1)); [ $i -lt 10 ] || fail "the car's back button did not go back"; sleep 0.5 +done +[ "$($M current)" = $APP ] || fail "the car's back button left MobiumApp" +row "back" "the car's back button is mapped, and a tap goes back in the app" $M tap 'label=Dialog Demo' >/dev/null $M tap testid=lateAlertBtn >/dev/null i=0; until $M alert 2>/dev/null | grep -q 'a dialog is on screen'; do diff --git a/docs/checks/lib-selftest.sh b/docs/checks/lib-selftest.sh index 18d5a472..40944014 100755 --- a/docs/checks/lib-selftest.sh +++ b/docs/checks/lib-selftest.sh @@ -48,6 +48,12 @@ out=$(sh -c '. "$1"; check_platform android 457C7DC2-C706-45D9-8D68-1D26953E28B1 out=$(sh -c '. "$1"; check_platform ios emulator-5554' x "$LIB" 2>&1) && bad "an iOS check took an emulator" || st=$? [ "$st" = 2 ] && pass "an iOS check refuses an emulator" || bad "platform: exit $st" sh -c '. "$1"; check_platform ios 00008120-000A75EA1ED8A01E' x "$LIB" && pass "an iPhone's UDID is iOS" || bad "an iPhone's UDID was refused" +# 7. shows reads find's answer, not its status: find answers "No element +# matches" with status 0, and a check guarded on the status never failed. +out=$(sh -c '. "$1"; M=fake; fake() { echo "No element matches label=x"; }; shows label=x && echo shown || echo absent' x "$LIB") +[ "$out" = absent ] && pass "shows: no match is absent" || bad "shows: no match read as $out" +out=$(sh -c '. "$1"; M=fake; fake() { echo "@e3 x (button)"; }; shows label=x && echo shown || echo absent' x "$LIB") +[ "$out" = shown ] && pass "shows: a match is shown" || bad "shows: a match read as $out" # 8. every check, but for these, and none sets its own EXIT # trap, which would replace the prelude's. clean-stop.sh stops the # caller's daemon, so it must not have one of its own; the test-runner, diff --git a/docs/checks/lib.sh b/docs/checks/lib.sh index 43264270..a8e858a4 100644 --- a/docs/checks/lib.sh +++ b/docs/checks/lib.sh @@ -67,6 +67,12 @@ esac CHECK_LOCK="" _check_handlers="" +# shows : whether `$M find` matched it. find answers "No element +# matches" with status 0, so a check that branches on its status cannot fail. +shows() { + ! $M find "$1" 2>&1 | grep -q '^No element matches' +} + at_exit() { # Newest first. _check_handlers="$1 diff --git a/docs/checks/login.sh b/docs/checks/login.sh index d9ac31b2..249ccc24 100755 --- a/docs/checks/login.sh +++ b/docs/checks/login.sh @@ -72,15 +72,15 @@ save_password() { # choice. Nothing of either is printed: they name the account. save_password_android() { sleep 1 # either one slides in; it can be found before a tap reaches it - if $M find "text=Not now" >/dev/null 2>&1; then + if shows "text=Not now"; then $M tap "text=Not now" >/dev/null || fail "a save-password sheet is up and Not now could not be pressed" - elif $M find "label=Never,role=button" >/dev/null 2>&1 || $M find "text=Never" >/dev/null 2>&1; then + elif shows "label=Never,role=button" || shows "text=Never"; then $M tap "label=Close,role=button" >/dev/null || fail "a save-password dialog is up and Close could not be pressed" else return 1 fi sleep 1 - { $M find "text=Not now" || $M find "label=Never,role=button" || $M find "text=Never"; } >/dev/null 2>&1 && + { shows "text=Not now" || shows "label=Never,role=button" || shows "text=Never"; } && fail "the save-password offer is still up after it was answered" row "save password" "Android offered to save it; answered without saving, the offer gone" } diff --git a/docs/quickstart/README.md b/docs/quickstart/README.md index 2175a031..971c156b 100644 --- a/docs/quickstart/README.md +++ b/docs/quickstart/README.md @@ -113,8 +113,7 @@ runtime (Settings → Components). Then boot a simulator by name: ```sh xcrun simctl list devices available | grep iPhone # pick one -mobium boot "iPhone 17 Pro" -open -a Simulator # to watch it; optional +mobium boot "iPhone 17 Pro" --window # --window to watch it; optional ``` ### iPhone (macOS) diff --git a/internal/agent/back.go b/internal/agent/back.go index 8b2d2486..85adf2dd 100644 --- a/internal/agent/back.go +++ b/internal/agent/back.go @@ -48,6 +48,13 @@ func (h *Handlers) gestureBack(ctx context.Context, s *session, tree *uitree.Tre if err != nil { return "", err } + if mode == device.NavCar { + return "", mobiumerr.New(mobiumerr.Unsupported, "a car has no back gesture — measured: a swipe in "+ + "from the left edge changed nothing. Its back button is the driver's back: `mobium press back`, "+ + "app_press back without gesture"). + WithRemedy("app_press back without gesture: the back button is the driver's back here"). + WithDetail("navigation", mode) + } if mode != device.NavGestures { if mode == "" { mode = "an unknown" diff --git a/internal/agent/boot.go b/internal/agent/boot.go index 259986b2..9d1c9402 100644 --- a/internal/agent/boot.go +++ b/internal/agent/boot.go @@ -6,6 +6,7 @@ import ( "path/filepath" "runtime" "strings" + "time" "github.com/mobiumdev/mobium/internal/device" "github.com/mobiumdev/mobium/internal/mobiumerr" @@ -18,12 +19,42 @@ import ( // could not start a device at all. A real phone is refused by both: it is // somebody's. +// bootHeartbeat is how often a boot says it is still going. Each notice +// moves the client's read deadline on: the client waits 120 seconds for an +// answer, and a boot waits up to 210 (CHALLENGES 400). +var bootHeartbeat = 30 * time.Second + +// heartbeat reports what is going on every interval until the returned +// function is called. It does nothing where no one hears progress. +func (h *Handlers) heartbeat(what string, interval time.Duration) func() { + notify := h.progress + if notify == nil { + return func() {} + } + done := make(chan struct{}) + go func() { + start := time.Now() + t := time.NewTicker(interval) + defer t.Stop() + for { + select { + case <-done: + return + case <-t.C: + notify(fmt.Sprintf("%s: %s so far", what, time.Since(start).Round(time.Second))) + } + } + }() + return func() { close(done) } +} + // BootView is app_boot's answer. type BootView = device.Booted // bootAVDs and bootSimulators are what a name could be, replaceable in tests. var ( bootAVDs = device.AVDs + showSimulator = device.ShowSimulator bootSimulator = func(ctx context.Context, name string) bool { if runtime.GOOS != "darwin" { return false @@ -49,6 +80,7 @@ func (h *Handlers) boot(ctx context.Context, args map[string]interface{}) (*Tool isSim := bootSimulator(ctx, name) var b device.Booted var err error + defer h.heartbeat("booting "+name, bootHeartbeat)() switch { case isAVD && isSim: return nil, mobiumerr.New(mobiumerr.InvalidArgument, "%q names both an AVD and a simulator — give the "+ @@ -58,6 +90,9 @@ func (h *Handlers) boot(ctx context.Context, args map[string]interface{}) (*Tool b, err = device.BootEmulator(ctx, name, boolArg(args, "window"), log) case isSim: b, err = device.BootSimulator(ctx, name) + if err == nil && boolArg(args, "window") { + b.Note, err = showSimulator(ctx, b.Serial, b.Name) + } case avdErr != nil: // The emulator could not be found or asked, and the name is not a // simulator: say that, not "no AVDs" — a daemon started without @@ -78,6 +113,9 @@ func (h *Handlers) boot(ctx context.Context, args map[string]interface{}) (*Tool if b.Already { msg = fmt.Sprintf("%s was already running, as %s", b.Name, b.Serial) } + if b.Note != "" { + msg += "\n" + b.Note + } return Result(msg, BootView(b)), nil } diff --git a/internal/agent/boot_test.go b/internal/agent/boot_test.go index 8f7f53df..cf0d4c81 100644 --- a/internal/agent/boot_test.go +++ b/internal/agent/boot_test.go @@ -6,6 +6,8 @@ import ( "testing" "github.com/mobiumdev/mobium/internal/mobiumerr" + "sync" + "time" ) // What app_boot and app_shutdown refuse before touching anything: no name, a @@ -64,3 +66,33 @@ func TestBootSaysWhyItCannotListAVDs(t *testing.T) { t.Errorf("got %v", err) } } + +// A boot says it is still going while it waits, and stops saying so once it +// returns: each notice moves the client's read deadline on. +func TestBootHeartbeat(t *testing.T) { + h := NewHandlers() + var mu sync.Mutex + var got []string + h.SetProgress(func(m string) { mu.Lock(); got = append(got, m); mu.Unlock() }) + stop := h.heartbeat("booting mobium-auto", 20*time.Millisecond) + time.Sleep(90 * time.Millisecond) + stop() + mu.Lock() + n := len(got) + mu.Unlock() + if n < 2 { + t.Fatalf("%d notices in 90ms at one per 20ms", n) + } + if !strings.HasPrefix(got[0], "booting mobium-auto: ") { + t.Errorf("notice %q", got[0]) + } + time.Sleep(60 * time.Millisecond) + mu.Lock() + defer mu.Unlock() + if len(got) != n { + t.Errorf("%d more notices after the boot returned", len(got)-n) + } + + quiet := NewHandlers() + quiet.heartbeat("booting", time.Millisecond)() +} diff --git a/internal/agent/outputschema_gen.go b/internal/agent/outputschema_gen.go index 2d470cdb..41a58086 100644 --- a/internal/agent/outputschema_gen.go +++ b/internal/agent/outputschema_gen.go @@ -17,7 +17,7 @@ var outputSchemas = map[string]string{ "app_background": "{\"properties\":{\"app\":{\"type\":\"string\"},\"away_ms\":{\"type\":\"integer\"},\"device\":{\"type\":\"string\"},\"seconds\":{\"type\":\"number\"}},\"required\":[\"app\",\"away_ms\",\"device\",\"seconds\"],\"type\":\"object\"}", "app_battery": "{\"properties\":{\"device\":{\"type\":\"string\"},\"level\":{\"type\":[\"integer\",\"null\"]},\"plugged\":{\"type\":\"string\"},\"present\":{\"type\":\"boolean\"},\"state\":{\"type\":\"string\"}},\"required\":[\"device\",\"present\",\"state\"],\"type\":\"object\"}", "app_biometric": "{\"properties\":{\"device\":{\"type\":\"string\"},\"enrolled\":{\"type\":\"boolean\"},\"kind\":{\"type\":\"string\"},\"locked_out\":{\"type\":\"boolean\"},\"note\":{\"type\":\"string\"},\"outcome\":{\"type\":\"string\"}},\"required\":[\"device\",\"enrolled\",\"kind\"],\"type\":\"object\"}", - "app_boot": "{\"properties\":{\"already\":{\"type\":\"boolean\"},\"device\":{\"type\":\"string\"},\"name\":{\"type\":\"string\"},\"platform\":{\"type\":\"string\"},\"took\":{\"type\":\"string\"}},\"required\":[\"already\",\"device\",\"name\",\"platform\"],\"type\":\"object\"}", + "app_boot": "{\"properties\":{\"already\":{\"type\":\"boolean\"},\"device\":{\"type\":\"string\"},\"name\":{\"type\":\"string\"},\"note\":{\"type\":\"string\"},\"platform\":{\"type\":\"string\"},\"took\":{\"type\":\"string\"}},\"required\":[\"already\",\"device\",\"name\",\"platform\"],\"type\":\"object\"}", "app_call": "{\"properties\":{\"action\":{\"type\":\"string\"},\"device\":{\"type\":\"string\"},\"number\":{\"type\":\"string\"}},\"required\":[\"action\",\"device\",\"number\"],\"type\":\"object\"}", "app_check": "{\"properties\":{\"claim\":{\"type\":\"string\"},\"evidence\":{\"items\":{\"properties\":{\"summary\":{\"type\":\"string\"},\"type\":{\"type\":\"string\"}},\"required\":[\"summary\",\"type\"],\"type\":\"object\"},\"type\":[\"array\",\"null\"]},\"model\":{\"type\":\"string\"},\"record\":{\"type\":\"string\"},\"status\":{\"type\":\"string\"},\"steps\":{\"type\":\"integer\"},\"summary\":{\"type\":\"string\"}},\"required\":[\"claim\",\"evidence\",\"model\",\"record\",\"status\",\"steps\",\"summary\"],\"type\":\"object\"}", "app_clear_data": "{\"properties\":{\"app\":{\"type\":\"string\"},\"device\":{\"type\":\"string\"},\"emptied\":{\"items\":{\"type\":\"string\"},\"type\":[\"array\",\"null\"]},\"kept\":{\"items\":{\"type\":\"string\"},\"type\":[\"array\",\"null\"]},\"not_read_back\":{\"items\":{\"type\":\"string\"},\"type\":[\"array\",\"null\"]},\"still_granted\":{\"items\":{\"type\":\"string\"},\"type\":[\"array\",\"null\"]}},\"required\":[\"app\",\"device\",\"emptied\",\"still_granted\"],\"type\":\"object\"}", diff --git a/internal/agent/schema.go b/internal/agent/schema.go index b8135b6e..2e47189e 100644 --- a/internal/agent/schema.go +++ b/internal/agent/schema.go @@ -64,7 +64,7 @@ func GetToolSchemas() []Tool { }, "window": map[string]interface{}{ "type": "boolean", - "description": "Show the emulator's window. Default false: headless.", + "description": "Show the emulator's window, or open the simulator in DeviceHub or Simulator.app. Default false: headless.", }, }, "required": []string{"name"}, diff --git a/internal/device/boot.go b/internal/device/boot.go index 38fefdb6..8df44f2a 100644 --- a/internal/device/boot.go +++ b/internal/device/boot.go @@ -111,6 +111,8 @@ type Booted struct { // Already says it was running before it was asked for. Already bool `json:"already"` Took string `json:"took,omitempty"` + // Note is what a person must still do to see it, when anything. + Note string `json:"note,omitempty"` } // BootEmulator starts the AVD named avd and waits until Android has booted: @@ -203,12 +205,36 @@ func windowFocused(ctx context.Context, a *ADB) bool { if err != nil { return false } - for _, l := range strings.Split(string(out), "\n") { - if l = strings.TrimSpace(l); strings.HasPrefix(l, "mCurrentFocus=") { + return focusedIn(string(out)) +} + +// focusedIn reads the main display's focus from `dumpsys window`. A car lists +// a section for each display, and its first, an instrument cluster's display +// 3, has no focus ever, so a boot waiting on the first line never ended +// (CHALLENGES 400). Display 0's section decides, where there are sections. +func focusedIn(dump string) bool { + first, inMain, sections := "", false, false + for _, l := range strings.Split(dump, "\n") { + l = strings.TrimSpace(l) + if strings.HasPrefix(l, "Display: mDisplayId=") { + sections = true + inMain = strings.HasPrefix(l, "Display: mDisplayId=0 ") || l == "Display: mDisplayId=0" + continue + } + if !strings.HasPrefix(l, "mCurrentFocus=") { + continue + } + if inMain { return !strings.Contains(l, "=null") } + if first == "" { + first = l + } } - return false + if sections { + return false + } + return first != "" && !strings.Contains(first, "=null") } // ShutdownEmulator stops a running emulator and waits until adb no longer diff --git a/internal/device/navmode.go b/internal/device/navmode.go index b3eb0d61..3f74eca7 100644 --- a/internal/device/navmode.go +++ b/internal/device/navmode.go @@ -13,12 +13,21 @@ const ( NavGestures = "gestures" NavThreeButton = "three-button" NavTwoButton = "two-button" + // NavCar is a car's: its system bars and a back button laid over the + // app. navigation_mode reads 0 there, which is no three buttons, and a + // swipe in from the left edge changed nothing (CHALLENGES 403). + NavCar = "car bars" ) // NavigationMode reads how the device is navigated, or "" when the setting // says nothing this knows: unset reads "null", and a value added later reads // as itself, neither of which is a mode to act on. func (a *ADB) NavigationMode(ctx context.Context) (string, error) { + if car, err := a.Automotive(ctx); err != nil { + return "", err + } else if car { + return NavCar, nil + } out, err := a.Shell(ctx, "settings", "get", "secure", "navigation_mode") if err != nil { return "", err diff --git a/internal/device/simwindow.go b/internal/device/simwindow.go new file mode 100644 index 00000000..898c0409 --- /dev/null +++ b/internal/device/simwindow.go @@ -0,0 +1,65 @@ +package device + +import ( + "context" + "encoding/hex" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/mobiumdev/mobium/internal/mobiumerr" +) + +// ShowSimulator opens the app that shows a simulator's screen, on the +// simulator udid, and says what a person must still do, or "". +// +// Xcode 27.1 has no Simulator.app: DeviceHub, inside Xcode.app, shows +// simulators, and ignores Simulator.app's -CurrentDeviceUDID. It opens on +// the device it last showed, which it keeps as lastSelectedDeviceIdentifier, +// the UDID as a JSON string; written before it starts, it opened on that +// simulator (CHALLENGES 405). A DeviceHub already running reads it no more, +// and is not quit: it is brought forward, and the note names the device to +// pick in its sidebar. +func ShowSimulator(ctx context.Context, udid, name string) (string, error) { + dev, err := exec.CommandContext(ctx, "xcode-select", "-p").Output() + if err != nil { + return "", mobiumerr.New(mobiumerr.ToolchainMissing, "xcode-select -p: %v", err). + WithRemedy("install Xcode and run `sudo xcode-select -s /Applications/Xcode.app`") + } + developer := strings.TrimSpace(string(dev)) + if sim := filepath.Join(developer, "Applications", "Simulator.app"); isDir(sim) { + return "", openApp(ctx, sim, "--args", "-CurrentDeviceUDID", udid) + } + hub := filepath.Join(filepath.Dir(developer), "Applications", "DeviceHub.app") + if !isDir(hub) { + return "", mobiumerr.New(mobiumerr.ToolchainMissing, "neither Simulator.app nor DeviceHub.app is in %s's Xcode", + developer) + } + if exec.CommandContext(ctx, "pgrep", "-x", "DeviceHub").Run() == nil { + if err := openApp(ctx, hub); err != nil { + return "", err + } + return "DeviceHub was already open, so it shows the device it showed: select " + name + + " in its sidebar", nil + } + id := hex.EncodeToString([]byte(`"` + udid + `"`)) + if out, err := exec.CommandContext(ctx, "defaults", "write", "com.apple.dt.Devices", + "lastSelectedDeviceIdentifier", "-data", id).CombinedOutput(); err != nil { + return "", mobiumerr.New(mobiumerr.DeviceServer, "set DeviceHub's device: %v: %s", err, strings.TrimSpace(string(out))) + } + return "", openApp(ctx, hub) +} + +func openApp(ctx context.Context, app string, args ...string) error { + out, err := exec.CommandContext(ctx, "open", append([]string{"-a", app}, args...)...).CombinedOutput() + if err != nil { + return mobiumerr.New(mobiumerr.DeviceServer, "open %s: %v: %s", filepath.Base(app), err, strings.TrimSpace(string(out))) + } + return nil +} + +func isDir(p string) bool { + st, err := os.Stat(p) + return err == nil && st.IsDir() +} diff --git a/internal/device/users_test.go b/internal/device/users_test.go index 8a5673c4..1882fc1c 100644 --- a/internal/device/users_test.go +++ b/internal/device/users_test.go @@ -1,6 +1,7 @@ package device import ( + "fmt" "strings" "testing" ) @@ -49,3 +50,33 @@ func TestUserSection(t *testing.T) { t.Error("user 1 matched user 10's header") } } + +// The main display's focus decides. On the Android Automotive 15 emulator the +// first section is the cluster's display 3, never focused; display 0 comes +// last. A Pixel 7 AVD has the one section. +func TestFocusedIn(t *testing.T) { + car := `WINDOW MANAGER DISPLAY CONTENTS (dumpsys window displays) + Display: mDisplayId=3 + mCurrentFocus=null + Display: mDisplayId=2 (organized) + mCurrentFocus=Window{dffec1b u0 com.android.car.cluster.osdouble/com.android.car.cluster.osdouble.ClusterOsDoubleActivity} + Display: mDisplayId=0 (organized) + mCurrentFocus=%s +` + if !focusedIn(fmt.Sprintf(car, "Window{12f7970 u10 com.android.car.carlauncher/.CarLauncher}")) { + t.Error("a car whose main display has focus read as not focused") + } + if focusedIn(fmt.Sprintf(car, "null")) { + t.Error("a car whose main display has no focus read as focused, from the cluster's") + } + phone := " Display: mDisplayId=0 (organized)\n mCurrentFocus=Window{904043c u0 dev.mobium.mobiumapp/.MainActivity}\n" + if !focusedIn(phone) || focusedIn(strings.Replace(phone, "Window{904043c u0 dev.mobium.mobiumapp/.MainActivity}", "null", 1)) { + t.Error("a phone's one display misread") + } + if !focusedIn(" mCurrentFocus=Window{1 u0 a/b}\n") || focusedIn(" mCurrentFocus=null\n") { + t.Error("a dump with no display sections misread") + } + if focusedIn(" Display: mDisplayId=10\n mCurrentFocus=Window{1 u0 a/b}\n") { + t.Error("display 10 read as display 0") + } +} diff --git a/internal/uitree/car_test.go b/internal/uitree/car_test.go index e5bdbe5d..469ac3ff 100644 --- a/internal/uitree/car_test.go +++ b/internal/uitree/car_test.go @@ -1,6 +1,7 @@ package uitree import ( + "os" "strings" "testing" ) @@ -38,6 +39,10 @@ func TestCarAppGridIsTheAppInFront(t *testing.T) { t.Errorf("map has no %q", want) } } + // Each tile once: its icon is part of it (CHALLENGES 401). + if _, ok := got["app_icon"]; ok { + t.Error("map lists a tile's icon as a control of its own") + } if d := tree.Dialog(); d != nil { t.Errorf("the app grid read as a dialog: %s", d.Path) } @@ -134,3 +139,40 @@ func TestCarPermissionPrompt(t *testing.T) { t.Errorf("the prompt says %q", words) } } + +// car-ui marks a list for the rotary controller with an id in its +// content-desc, which is no name (CHALLENGES 402). +func TestCarUIMarkerIsNoLabel(t *testing.T) { + raw, err := os.ReadFile("testdata/car-settings.xml") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(raw), "com.android.car.ui.utils.ROTARY_CONTAINER") { + t.Fatal("control: the capture should hold the marker") + } + tree := carFixture(t, "car-settings.xml") + tree.Walk(func(n *Node) bool { + if strings.Contains(n.Label, "ROTARY") { + t.Errorf("%s keeps the marker as its label", n.Path) + } + return true + }) + for _, e := range tree.Map() { + if strings.Contains(e.Label, "ROTARY") { + t.Errorf("map lists %q", e.Label) + } + } +} + +// The car's back button says it is not clickable, and a tap on it goes back +// (CHALLENGES 404). +func TestCarBackButtonIsMapped(t *testing.T) { + tree := carFixture(t, "car-mobiumapp-letterboxed.xml") + e, ok := mapLabels(tree)["Back Button"] + if !ok { + t.Fatal("map has no Back Button") + } + if e.Node.Clickable { + t.Fatal("control: the capture should hold a back button that says it is not clickable") + } +} diff --git a/internal/uitree/parse.go b/internal/uitree/parse.go index 32211eea..f5091d00 100644 --- a/internal/uitree/parse.go +++ b/internal/uitree/parse.go @@ -410,7 +410,7 @@ func nodeFrom(e xml.StartElement, parent *Node, sibling int) *Node { n := &Node{ Text: attr(e, "text"), - Label: attr(e, "content-desc"), + Label: androidLabel(attr(e, "content-desc")), TestID: attr(e, "resource-id"), Class: class, Package: attr(e, "package"), @@ -454,6 +454,19 @@ func nodeFrom(e xml.StartElement, parent *Node, sibling int) *Node { return n } +// carUIMarker is the prefix of the ids car-ui writes into a container's +// content-desc for the rotary controller, such as ROTARY_CONTAINER on car +// Settings' list: a marker, not a name, and `text` read it out as the +// screen's first line (CHALLENGES 402). +const carUIMarker = "com.android.car.ui.utils." + +func androidLabel(desc string) string { + if strings.HasPrefix(desc, carUIMarker) { + return "" + } + return desc +} + // insideWebView reports a node a WebView draws for its page. func insideWebView(n *Node) bool { for p := n.Parent; p != nil; p = p.Parent { diff --git a/internal/uitree/snapshot.go b/internal/uitree/snapshot.go index e11e714a..e2b4a22f 100644 --- a/internal/uitree/snapshot.go +++ b/internal/uitree/snapshot.go @@ -128,7 +128,7 @@ func Actionable(n *Node) bool { if !n.Displayed { return false } - if partOfToggle(n) { + if partOfToggle(n) || insideNamedTile(n) { return false } if !n.Clickable && !n.LongClickable && !n.Checkable && !n.Scrollable { @@ -136,13 +136,20 @@ func Actionable(n *Node) bool { // slider: Android marks a seek bar neither clickable nor scrollable, // so MobiumApp's two sliders were missing from map on the Pixel 8 // Pro (CHALLENGES 227). - if !(n.Focusable && HasRole(n, "input")) && !HasClassRole(n, "slider") { + // Nor a named button that says it is not clickable: a car's back + // button, an ImageButton, takes a tap that goes back (CHALLENGES 404). + if !(n.Focusable && HasRole(n, "input")) && !HasClassRole(n, "slider") && !focusableNamedButton(n) { return false } } return true } +func focusableNamedButton(n *Node) bool { + return n.Focusable && n.Enabled && !IsIOS(n) && HasClassRole(n, "button") && + (clean(n.Label) != "" || clean(n.Text) != "") +} + // partOfToggle reports a checkable node that takes no touch of its own, // inside a control that does and that has the checked state itself: the // switch drawn on a toggle, not a second one. Wear OS's toggle chip is a @@ -162,6 +169,36 @@ func partOfToggle(n *Node) bool { return false } +// insideNamedTile reports a clickable with no words of its own or inside +// it, held by a clickable parent that has words: the parent's picture, not a +// second control. A car launcher's tile is a clickable app_item holding a +// clickable, nameless app_icon over its name, and map listed every tile twice, +// once as "app_icon" (CHALLENGES 401). +func insideNamedTile(n *Node) bool { + p := n.Parent + if !n.Clickable || n.Checkable || n.Scrollable || IsIOS(n) || HasRole(n, "input") || p == nil || !p.Clickable { + return false + } + if hasWords(n) || !hasWords(p) || !enclosesRect(p.Bounds, n.Bounds) { + return false + } + return true +} + +// hasWords reports whether n, or a node inside it that takes no touch of its +// own, has text or a label. +func hasWords(n *Node) bool { + if clean(n.Text) != "" || clean(n.Label) != "" { + return true + } + for _, c := range n.Children { + if !c.Clickable && hasWords(c) { + return true + } + } + return false +} + // maxLabel is how many runes of label `map` will print for one element. // // A label exists so an agent can choose between elements; past a sentence or diff --git a/internal/uitree/testdata/car-settings.xml b/internal/uitree/testdata/car-settings.xml new file mode 100644 index 00000000..c21174c1 --- /dev/null +++ b/internal/uitree/testdata/car-settings.xml @@ -0,0 +1,187 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From efea39c56b933feb1d3aafcb51aa3839bb61f178 Mon Sep 17 00:00:00 2001 From: Lana Begunova <70295997+lana-20@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:26:18 -0800 Subject: [PATCH 4/8] Replace a real phone's serial and UDID in tests with made-up ones The Pixel 8 Pro's serial and the iPhone 15 Plus's UDID stood in for any phone in four tests; a made-up id of the same shape does the same. --- docs/checks/lib-selftest.sh | 2 +- internal/agent/session_test.go | 2 +- internal/device/emuaudio_test.go | 2 +- internal/mobiumdriver/audio_test.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/checks/lib-selftest.sh b/docs/checks/lib-selftest.sh index 40944014..bae278af 100755 --- a/docs/checks/lib-selftest.sh +++ b/docs/checks/lib-selftest.sh @@ -47,7 +47,7 @@ out=$(sh -c '. "$1"; check_platform android 457C7DC2-C706-45D9-8D68-1D26953E28B1 [ "$st" = 2 ] && echo "$out" | grep -q "is for an Android device" && pass "an Android check refuses a simulator" || bad "platform: exit $st, $out" out=$(sh -c '. "$1"; check_platform ios emulator-5554' x "$LIB" 2>&1) && bad "an iOS check took an emulator" || st=$? [ "$st" = 2 ] && pass "an iOS check refuses an emulator" || bad "platform: exit $st" -sh -c '. "$1"; check_platform ios 00008120-000A75EA1ED8A01E' x "$LIB" && pass "an iPhone's UDID is iOS" || bad "an iPhone's UDID was refused" +sh -c '. "$1"; check_platform ios 00008120-0001234567890ABC' x "$LIB" && pass "an iPhone's UDID is iOS" || bad "an iPhone's UDID was refused" # 7. shows reads find's answer, not its status: find answers "No element # matches" with status 0, and a check guarded on the status never failed. out=$(sh -c '. "$1"; M=fake; fake() { echo "No element matches label=x"; }; shows label=x && echo shown || echo absent' x "$LIB") diff --git a/internal/agent/session_test.go b/internal/agent/session_test.go index c02abf15..84c566e7 100644 --- a/internal/agent/session_test.go +++ b/internal/agent/session_test.go @@ -366,7 +366,7 @@ func TestNotAnsweringADBNamesTheDevice(t *testing.T) { cause := mobiumerr.New(mobiumerr.Timeout, "adb shell true timed out after 30s").WithDetail(device.ADBTimeoutKey, "shell true") for serial, want := range map[string]string{ "emulator-5554": "`mobium shutdown emulator-5554`, which ends a frozen emulator too", - "3C191FDJG001QX": "`adb -s 3C191FDJG001QX shell true`", + "1A2B3C4D5E6F7G": "`adb -s 1A2B3C4D5E6F7G shell true`", } { e, _ := mobiumerr.As(notAnsweringADB(serial, cause)) if e == nil || e.Code != mobiumerr.DeviceNotReady || strings.Contains(e.Error(), "--driver") || !strings.Contains(e.Remedy, want) { diff --git a/internal/device/emuaudio_test.go b/internal/device/emuaudio_test.go index 51e620d6..8e5fe59e 100644 --- a/internal/device/emuaudio_test.go +++ b/internal/device/emuaudio_test.go @@ -57,7 +57,7 @@ func TestFindEmulatorEndpointMissing(t *testing.T) { if e, _ := mobiumerr.As(err); !strings.Contains(e.Remedy, "`mobium shutdown emulator-5554`, then `mobium boot") { t.Fatalf("remedy %q", e.Remedy) } - if _, err := findEmulatorEndpoint("3C191FDJG001QX", []string{dir}); mobiumerr.CodeOf(err) != mobiumerr.Unsupported { + if _, err := findEmulatorEndpoint("1A2B3C4D5E6F7G", []string{dir}); mobiumerr.CodeOf(err) != mobiumerr.Unsupported { t.Fatalf("a phone: got %v", err) } } diff --git a/internal/mobiumdriver/audio_test.go b/internal/mobiumdriver/audio_test.go index cf85dcf8..d10869bb 100644 --- a/internal/mobiumdriver/audio_test.go +++ b/internal/mobiumdriver/audio_test.go @@ -38,7 +38,7 @@ func TestAndroidAudioOnAPhoneAndAnEmulator(t *testing.T) { return device.StartAudioEvents(context.Background(), nil, nil), nil } defer func() { startEmulatorAudio = device.StartEmulatorAudio }() - r, err := androidAudioOn(context.Background(), phoneOrEmulator(false), "3C191FDJG001QX", "") + r, err := androidAudioOn(context.Background(), phoneOrEmulator(false), "1A2B3C4D5E6F7G", "") if err != nil || r.Captures() || asked != "" { t.Fatalf("a phone: %v, captures %v, asked %q", err, r != nil && r.Captures(), asked) } From 1d5e351b1409296815634553c4d474382949b38e Mon Sep 17 00:00:00 2001 From: Lana Begunova <70295997+lana-20@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:40:25 -0800 Subject: [PATCH 5/8] Replace a phone's name, LAN address and serial prefix in docs and tests with made-up ones CHALLENGES quoted the iPhone's device name, SETUP a real adb mdns capture, and two tests a Fire TV's home address; the address is now from 192.0.2.0/24, kept for documentation. --- docs/CHALLENGES.md | 8 ++++---- docs/SETUP.md | 6 +++--- internal/device/unreachable_test.go | 20 ++++++++++---------- internal/testrun/link_test.go | 4 ++-- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/CHALLENGES.md b/docs/CHALLENGES.md index 44a0433c..0ca3bf61 100644 --- a/docs/CHALLENGES.md +++ b/docs/CHALLENGES.md @@ -3729,7 +3729,7 @@ trip each, and now takes 16, one visit reading every switch on its page. **Found by:** writing the grid guide, with the iPhone 15 Plus attached to the Mac standing in for a node. -`mobium grid status` listed the phone with "Lana Begunova's iPhone" in its +`mobium grid status` listed the phone with "'s iPhone" in its MODEL column. The structured answer of `app_devices` — which the grid's status and page, `MOBIUM_GRID_MODEL`'s matching and every client read — put a physical iPhone's name where its model belongs, while the CLI's own line @@ -3974,7 +3974,7 @@ not connected was reported as MobiumApp not being installed. ### 175. Every phone message named its owner **Found by:** the iPhone runs above, whose output began "starting -WebDriverAgent on Lana Begunova's iPhone". +WebDriverAgent on 's iPhone". 161 took the owner's name for the phone out of `app_devices`' structured answer and left it in two places: the `mobium devices` line, read by the @@ -4727,8 +4727,8 @@ that adb does not know at all now names `adb connect` too, where it named `mobium devices`, which cannot list a device adb was never connected to. The unit test uses adb's exact lines from the TV; against the old code it failed with the TV's own message. On the TV, with its link dropped on purpose, the -lookup answered "no device with serial "10.0.0.77:5555" — `adb connect -10.0.0.77:5555`, then retry". +lookup answered "no device with serial "192.0.2.10:5555" — `adb connect +192.0.2.10:5555`, then retry". ### 211. A Fire TV's screenshot was refused on one backend and slow on the other diff --git a/docs/SETUP.md b/docs/SETUP.md index d29c2643..95bd5070 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -385,15 +385,15 @@ Android 11+. The phone and the Mac must be on the same network. ```sh adb mdns services - # adb-3C19...-FIOvFv _adb-tls-pairing._tcp 10.0.0.96:33169 - # adb-3C19...-FIOvFv _adb-tls-connect._tcp 10.0.0.96:43943 + # adb-XXXXXXXX-abcdef _adb-tls-pairing._tcp 192.0.2.20:33169 + # adb-XXXXXXXX-abcdef _adb-tls-connect._tcp 192.0.2.20:43943 ``` 4. Pair against the **pairing** port, which is not the one on the main Wireless debugging screen: ```sh - adb pair 10.0.0.96:33169 123456 + adb pair 192.0.2.20:33169 123456 ``` `adb pair` may answer `protocol fault (couldn't read status message)` **and diff --git a/internal/device/unreachable_test.go b/internal/device/unreachable_test.go index 5e3a717c..bfa0af55 100644 --- a/internal/device/unreachable_test.go +++ b/internal/device/unreachable_test.go @@ -19,8 +19,8 @@ func TestUnreachable(t *testing.T) { want bool remedy string }{ - {"10.0.0.77:5555", "adb: device offline\n", true, "adb connect 10.0.0.77:5555"}, - {"10.0.0.77:5555", "error: device offline\n", true, "adb connect 10.0.0.77:5555"}, + {"192.0.2.10:5555", "adb: device offline\n", true, "adb connect 192.0.2.10:5555"}, + {"192.0.2.10:5555", "error: device offline\n", true, "adb connect 192.0.2.10:5555"}, {"emulator-5554", "error: device 'emulator-5554' not found\n", true, "adb devices"}, {"emulator-5554", "adb: no devices/emulators found\n", true, "adb devices"}, // A command on the device printing similar words is not adb saying @@ -55,7 +55,7 @@ func TestEnsureUIA2InstalledOnAnOfflineDevice(t *testing.T) { *install*) echo x >> `+fakecmd.Path(installs)+`; echo "adb: device offline" >&2; exit 1 ;; *) echo "adb: device offline" >&2; exit 1 ;; esac`) - a := &ADB{Path: bin, Serial: "10.0.0.77:5555"} + a := &ADB{Path: bin, Serial: "192.0.2.10:5555"} err := a.EnsureUIA2Installed(context.Background(), nil) if mobiumerr.CodeOf(err) != mobiumerr.DeviceNotReady { t.Fatalf("got %v, want device_not_ready", err) @@ -72,13 +72,13 @@ func netADB(t *testing.T, stayDown bool) (string, string) { t.Helper() dir := t.TempDir() state := filepath.Join(dir, "state") - connect := `echo "connected to 10.0.0.77:5555"; [ -f ` + fakecmd.Path(state) + `.cut ] && echo up > ` + fakecmd.Path(state) + connect := `echo "connected to 192.0.2.10:5555"; [ -f ` + fakecmd.Path(state) + `.cut ] && echo up > ` + fakecmd.Path(state) if stayDown { - connect = `echo "failed to connect to 10.0.0.77:5555"` + connect = `echo "failed to connect to 192.0.2.10:5555"` } bin := fakecmd.Script(t, dir, "adb", `case "$*" in - *"devices -l"*) echo "List of devices attached"; if [ -f `+fakecmd.Path(state)+` ]; then echo "10.0.0.77:5555 device product:x model:AFTHA001"; else echo "10.0.0.77:5555 offline"; fi ;; - *disconnect*) touch `+fakecmd.Path(state)+`.cut; echo "disconnected 10.0.0.77:5555" ;; + *"devices -l"*) echo "List of devices attached"; if [ -f `+fakecmd.Path(state)+` ]; then echo "192.0.2.10:5555 device product:x model:AFTHA001"; else echo "192.0.2.10:5555 offline"; fi ;; + *disconnect*) touch `+fakecmd.Path(state)+`.cut; echo "disconnected 192.0.2.10:5555" ;; *connect*) `+connect+` ;; esac`) t.Setenv("MOBIUM_ADB_PATH", bin) @@ -87,7 +87,7 @@ esac`) func TestSelectConnectsANetworkDeviceAgain(t *testing.T) { netADB(t, false) - _, d, err := Select(context.Background(), "10.0.0.77:5555") + _, d, err := Select(context.Background(), "192.0.2.10:5555") if err != nil || d == nil || !d.Ready() { t.Fatalf("got %v, %+v; want the device back after one reconnect", err, d) } @@ -97,7 +97,7 @@ func TestSelectConnectsANetworkDeviceAgain(t *testing.T) { // call that never reached it — the mark the test runner waits on. func TestSelectMarksAnUnreachedDevice(t *testing.T) { netADB(t, true) - _, _, err := Select(context.Background(), "10.0.0.77:5555") + _, _, err := Select(context.Background(), "192.0.2.10:5555") if mobiumerr.CodeOf(err) != mobiumerr.DeviceNotReady || !Unreached(err) { t.Fatalf("got %v; want device_not_ready marked as never reaching the device", err) } @@ -106,7 +106,7 @@ func TestSelectMarksAnUnreachedDevice(t *testing.T) { } // A failure from inside a call carries no mark: it may have reached // the device. - if Unreached(unreachable("10.0.0.77:5555", "adb: device offline\n")) { + if Unreached(unreachable("192.0.2.10:5555", "adb: device offline\n")) { t.Error("an error from inside a call was marked as never reaching the device") } } diff --git a/internal/testrun/link_test.go b/internal/testrun/link_test.go index 6f2545f0..b8aae27d 100644 --- a/internal/testrun/link_test.go +++ b/internal/testrun/link_test.go @@ -12,7 +12,7 @@ import ( // unreached is the error a call gets while a device's link is down: the // lookup failed, so nothing was sent. func unreached() error { - return mobiumerr.New(mobiumerr.DeviceNotReady, "device 10.0.0.77:5555 is offline, not ready"). + return mobiumerr.New(mobiumerr.DeviceNotReady, "device 192.0.2.10:5555 is offline, not ready"). WithDetail(device.ReachedKey, false) } @@ -55,7 +55,7 @@ func TestAFailureThatMayHaveReachedTheDeviceIsNotRepeated(t *testing.T) { calls := 0 f := stepsRun(t, []Step{press("select")}, true, func(string, map[string]interface{}, ...map[string]interface{}) (*agent.ToolsCallResult, error) { calls++ - return nil, mobiumerr.New(mobiumerr.DeviceNotReady, "10.0.0.77:5555 cannot be reached: device offline") + return nil, mobiumerr.New(mobiumerr.DeviceNotReady, "192.0.2.10:5555 cannot be reached: device offline") }) if f == nil || calls != 1 { t.Errorf("failure %+v after %d calls; want it to fail on the first", f, calls) From a919c517102a7ada2ce1155e907f2dac2ab58b46 Mon Sep 17 00:00:00 2001 From: Lana Begunova <70295997+lana-20@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:48:14 -0800 Subject: [PATCH 6/8] Refuse real device and home identifiers in tracked files (CHALLENGES 407) Scanned every public mobiumdev and lana-20 repository's main and history on 2026-11-11. The values this replaces stay in main's history. private-data.py knows no real value, only shapes and the made-up values allowed for them. Its selftest flags each rule's case; a 10.0.0.x address planted in a staged file failed make docs-check, and the report held no value. --- Makefile | 4 +- docs/CHALLENGES.md | 34 +++++++-- docs/RELEASE-CHECKLIST.md | 2 +- docs/checks/private-data.py | 147 ++++++++++++++++++++++++++++++++++++ 4 files changed, 179 insertions(+), 8 deletions(-) create mode 100644 docs/checks/private-data.py diff --git a/Makefile b/Makefile index 21a4b368..b304c7b0 100644 --- a/Makefile +++ b/Makefile @@ -237,7 +237,9 @@ docs-check: @python3 docs/quickstart/build.py --check >/dev/null \ || { python3 docs/quickstart/build.py --check; exit 1; } @sh docs/checks/lib-selftest.sh >/dev/null || { sh docs/checks/lib-selftest.sh; exit 1; } - @echo "docs: spelling, anchors, quick-start pages and the checks' prelude clean" + @python3 docs/checks/private-data.py selftest >/dev/null || { python3 docs/checks/private-data.py selftest; exit 1; } + @python3 docs/checks/private-data.py scan $$(git ls-files) || exit 1 + @echo "docs: spelling, anchors, quick-start pages, the checks' prelude and private data clean" ci: fmt-check vet lint test clients crosscompile java api-check flags-check license-check docs-check dotnet diff --git a/docs/CHALLENGES.md b/docs/CHALLENGES.md index 0ca3bf61..72af4b3b 100644 --- a/docs/CHALLENGES.md +++ b/docs/CHALLENGES.md @@ -7,21 +7,21 @@ the source. The pattern across them is the reason the document exists: **almost nothing here was found by reading code or by a test written from imagination.** Of -406 defects, 327 were found only by running against a real device. Eighteen +407 defects, 327 were found only by running against a real device. Eighteen more — 255, 280, 282, 327, 329, 353, 355, 356, 359, 371, 375, 380, 382, 384, 386, 387, 388 and 406 — were met while running against one, though the fault lay in a check, the host or Mobium's own -design rather than in the platform. The other sixty-one — 4, 5, 6, 14, 23, +design rather than in the platform. The other sixty-two — 4, 5, 6, 14, 23, 24, 32, 35, 36, 39, 44, 50, 53, 54, 57, 66, 89, 99, 100, 121, 123, 129, 130, 133, 134, 139, 140, 141, 142, 144, 150, 158, 162, 164, 165, 172, 222, 231, 232, 243, 274, 278, 279, 286, 287, 305, 306, 308, 309, 310, 313, 314, 321, 361, 362, -363, 372, 373, 374, 378 and 381 — came from reading code, the compiler, a test, a linter, +363, 372, 373, 374, 378, 381 and 407 — came from reading code, the compiler, a test, a linter, cross-checking a computed number against a screenshot, using the tooling on itself, typing a negative number at a command line, and driving the clients against a stand-in daemon, CI on Windows, following the quick start from a fresh clone, type-checking — or writing — the documentation site's examples, reviewing the questions of the documentation's evaluation set, synthesized -sounds, the host's own shell, a clean Windows machine, and a person at a -Windows laptop's keyboard. +sounds, the host's own shell, a clean Windows machine, a person at a +Windows laptop's keyboard, and scanning the public repositories. Read it before writing a test that asserts platform behavior. @@ -8142,7 +8142,7 @@ both new lines. Tests hold each platform to its own install. **Found by:** the checks at a Windows laptop's keyboard, as a standard user with v0.1.0-rc1. `doctor` reported one problem: "android sdk root — -C:\Users\begun\AppData\Local\Android\Sdk has no platform-tools directory", +C:\Users\you\AppData\Local\Android\Sdk has no platform-tools directory", with the fix `sdkmanager platform-tools`, which that account did not have. The folder had one: a machine-wide ANDROID_HOME, left by Android Studio for the administrator, pointed into the administrator's profile, which a @@ -8863,6 +8863,28 @@ took the save-password branch whether or not the sheet was up. holds it to both answers. `login.sh` and `audit.sh` passed on the Pixel 7 AVD, and `audit.sh` on the iPhone 18 Pro simulator. +### 407. A real phone's identifiers were in the public repositories + +**Found by:** scanning every public repository's main and history for the +identifiers of the phones Mobium was built on, after a test was seen to name +the Pixel 8 Pro by its serial. In the files of main: the Pixel's serial in +three tests, the iPhone's UDID in `lib-selftest.sh`, the iPhone's device name +in two quotes here, a Fire TV's home-network address in two tests and an +entry here, a wireless-debugging capture in SETUP with the Pixel's address +and the start of its serial, and a Windows home directory's name here. The +same capture was in the RAG project's review page; the documentation site +tracked Python bytecode, which records the path it was built in. Each came +from a real capture or a test written beside a real device. The history keeps +them: main refuses a rewrite, and copies of a public repository outlive one. + +**Fixed:** each replaced with a made-up value of the same shape; addresses +come from 192.0.2.0/24, kept for documentation. `docs/checks/private-data.py`, +run by `make ci`, refuses a UDID, an adb mdns name, a home network's address +or a home path that is not on its list of made-up values, and tracked +bytecode. It prints where and what kind, never the value, since a public +repository's CI log is public. Its selftest holds each rule to a case it must +flag, and a leak planted in a staged file failed `make docs-check`. + ## Findings that were not defects Worth recording because each one closed off an approach that looked obvious. diff --git a/docs/RELEASE-CHECKLIST.md b/docs/RELEASE-CHECKLIST.md index ec6234eb..750cf222 100644 --- a/docs/RELEASE-CHECKLIST.md +++ b/docs/RELEASE-CHECKLIST.md @@ -4,7 +4,7 @@ [CI workflow](../.github/workflows/ci.yml) runs on every push — formatting, vet, both modules' tests, the four non-Go clients' own tests, and six cross-compile targets. This file is the rest, and the rest is where most -serious defects in this project have come from: 327 of 406 were found only by +serious defects in this project have come from: 327 of 407 were found only by running against a real device. Work through it before tagging a release, on three substrates: an Android diff --git a/docs/checks/private-data.py b/docs/checks/private-data.py new file mode 100644 index 00000000..072878ea --- /dev/null +++ b/docs/checks/private-data.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Refuse a real device's or person's identifiers in tracked files. + +This repository is public. A real iPhone's UDID, a phone's serial in an adb +mdns name, a home network's address and a home directory's path all reached +it from captures and test fixtures (CHALLENGES 407). This check knows no real +value: it knows the shapes, and the made-up values that may stand in for one. +A value of a flagged shape that is not on the list fails `make ci`. + +What it prints names the file, the line and the kind, never the value: a CI +log of a public repository is public too. + + docs/checks/private-data.py scan ... + docs/checks/private-data.py selftest +""" +import re +import sys + +# Made-up values a test or a page may use. Add one here, never a real one. +ALLOWED = { + "udid": { + "00008120-0000000000000001", + "00008120-0000000000000002", + "00008120-0001234567890ABC", + }, + # 10.0.2.2 is the emulator's own name for the host. The rest are fixtures + # that stand for any address; new ones belong in 192.0.2.0/24, + # 198.51.100.0/24 or 203.0.113.0/24, kept for documentation and never + # flagged. + "lan-ip": {"10.0.2.2", "10.0.2.15", "10.0.0.1", "10.0.0.5", "192.168.1.4", "192.168.1.5"}, + "home-path": {"/home/runner", "/Users/x", "/Users/you", "/home/you", "C:\\Users\\you"}, + "mdns-name": set(), +} + +RULES = [ + ("udid", re.compile(r"(?, /home/runner/work, $HOME", []), + ("a.go", "pipeName(`C:\\Users\\a\\.mobium`) and /Users/b/x", []), + ("a.md", "C:\\Users\\someone\\AppData", ["home-path"]), + ("scripts/__pycache__/x.cpython-314.pyc", "", ["bytecode"]), + ] + failed = 0 + for path, text, want in cases: + got = [k for _, k in findings(path, text)] + if got != want: + failed += 1 + print("FAIL %r: got %s, want %s" % (text or path, got, want)) + # What it prints never holds the value it flags. + import io + import contextlib + import os + import tempfile + secret = "10.0.0.42" + with tempfile.TemporaryDirectory() as d: + p = os.path.join(d, "leak.md") + with open(p, "w") as f: + f.write("adb connect %s:5555\n" % secret) + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + n = scan([p]) + if n != 1 or secret in buf.getvalue(): + failed += 1 + print("FAIL the report should flag one and print no value: %r" % buf.getvalue()) + if failed: + return 1 + print("private-data: all ok") + return 0 + + +if __name__ == "__main__": + if len(sys.argv) >= 2 and sys.argv[1] == "selftest": + sys.exit(selftest()) + if len(sys.argv) >= 2 and sys.argv[1] == "scan": + sys.exit(1 if scan(sys.argv[2:]) else 0) + print(__doc__, file=sys.stderr) + sys.exit(2) From d5ae4d6b00ff6aa1239acf933c81565d5d9a46af Mon Sep 17 00:00:00 2001 From: Lana Begunova <70295997+lana-20@users.noreply.github.com> Date: Wed, 11 Nov 2026 16:00:43 -0800 Subject: [PATCH 7/8] Name the self-test's planted address for what it is CodeQL read a variable named secret, written to a temporary file, as clear-text storage of sensitive data. The value is a made-up 10.0.0.x address. --- docs/checks/private-data.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/checks/private-data.py b/docs/checks/private-data.py index 072878ea..e61a6e9e 100644 --- a/docs/checks/private-data.py +++ b/docs/checks/private-data.py @@ -121,15 +121,15 @@ def selftest(): import contextlib import os import tempfile - secret = "10.0.0.42" + planted = "10.0.0.42" # made up, in a range the scan flags with tempfile.TemporaryDirectory() as d: p = os.path.join(d, "leak.md") with open(p, "w") as f: - f.write("adb connect %s:5555\n" % secret) + f.write("adb connect %s:5555\n" % planted) buf = io.StringIO() with contextlib.redirect_stdout(buf): n = scan([p]) - if n != 1 or secret in buf.getvalue(): + if n != 1 or planted in buf.getvalue(): failed += 1 print("FAIL the report should flag one and print no value: %r" % buf.getvalue()) if failed: From 78273cd5d5b9e8549b942f59e49153a49a5d3c63 Mon Sep 17 00:00:00 2001 From: Lana Begunova <70295997+lana-20@users.noreply.github.com> Date: Wed, 11 Nov 2026 16:11:25 -0800 Subject: [PATCH 8/8] Send no boot notice after the boot has answered CI on Windows: TestBootHeartbeat saw a notice after stop returned. A tick and the stop were ready together and the tick won. On this Mac 200 runs of the old code passed; at one tick a millisecond, the new test fails the old code within two runs. --- internal/agent/boot.go | 15 ++++++++++++--- internal/agent/boot_test.go | 31 +++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/internal/agent/boot.go b/internal/agent/boot.go index 9d1c9402..5b268751 100644 --- a/internal/agent/boot.go +++ b/internal/agent/boot.go @@ -25,14 +25,16 @@ import ( var bootHeartbeat = 30 * time.Second // heartbeat reports what is going on every interval until the returned -// function is called. It does nothing where no one hears progress. +// function is called, and that function returns only once no notice can +// follow. It does nothing where no one hears progress. func (h *Handlers) heartbeat(what string, interval time.Duration) func() { notify := h.progress if notify == nil { return func() {} } - done := make(chan struct{}) + done, finished := make(chan struct{}), make(chan struct{}) go func() { + defer close(finished) start := time.Now() t := time.NewTicker(interval) defer t.Stop() @@ -41,11 +43,18 @@ func (h *Handlers) heartbeat(what string, interval time.Duration) func() { case <-done: return case <-t.C: + // A tick and the stop can be ready together; the stop wins, + // so no notice follows the answer. + select { + case <-done: + return + default: + } notify(fmt.Sprintf("%s: %s so far", what, time.Since(start).Round(time.Second))) } } }() - return func() { close(done) } + return func() { close(done); <-finished } } // BootView is app_boot's answer. diff --git a/internal/agent/boot_test.go b/internal/agent/boot_test.go index cf0d4c81..a362937f 100644 --- a/internal/agent/boot_test.go +++ b/internal/agent/boot_test.go @@ -96,3 +96,34 @@ func TestBootHeartbeat(t *testing.T) { quiet := NewHandlers() quiet.heartbeat("booting", time.Millisecond)() } + +// At one tick a millisecond a tick is always waiting when the boot returns, +// and the stop must still win: a notice after the answer reached the CLI +// after its result on Windows, where timers are coarse. +func TestBootHeartbeatStopsBeforeAWaitingTick(t *testing.T) { + h := NewHandlers() + for i := 0; i < 50; i++ { + var mu sync.Mutex + stopped, late := false, 0 + h.SetProgress(func(string) { + mu.Lock() + if stopped { + late++ + } + mu.Unlock() + }) + stop := h.heartbeat("booting", time.Millisecond) + time.Sleep(5 * time.Millisecond) + stop() + mu.Lock() + stopped = true + mu.Unlock() + time.Sleep(3 * time.Millisecond) + mu.Lock() + n := late + mu.Unlock() + if n != 0 { + t.Fatalf("run %d: %d notices after the boot returned", i, n) + } + } +}