From c1bc786a20fc1cfb81b3b966b67115984614dfe6 Mon Sep 17 00:00:00 2001 From: Lana Begunova <70295997+lana-20@users.noreply.github.com> Date: Wed, 4 Nov 2026 16:22:12 -0800 Subject: [PATCH] Name the lock, however an action meets it (CHALLENGES 389) Fixes #327. lock lock now waits until a read sees the lock. Any tool that fails reading on a locked device answers device_not_ready. A touch at a point on an Android screen that is off is refused, and scroll-to leaves a lock screen unswiped. Measured, tap after lock: Android 15 and 17, 8 of 8 exit 3, none landed; Android 15 before, 8 of 8 false successes. iOS 26.5: scroll-to 8 of 8 exit 3, still locked. lock takes about 0.6 s on Android and 1.5 s on iOS. lock.sh fails on main and passes here. device-state, gestures, locators, mobium-app and dialogs pass on Android; biometric, gestures, locators and mobium-app on iOS. The emulator froze once during a run and passed gestures after a restart. A sleep from outside Mobium, tapped by element within the half second, is still reported: 4 of 4 with input keyevent KEYCODE_SLEEP. --- docs/CHALLENGES.md | 40 ++++++- docs/RELEASE-CHECKLIST.md | 2 +- docs/checks/README.md | 1 + docs/checks/lock.sh | 79 +++++++++++++ internal/agent/batch.go | 2 +- internal/agent/drag.go | 3 + internal/agent/handlers.go | 84 +++++++++++++- internal/agent/lock_test.go | 173 ++++++++++++++++++++++++++++ internal/agent/scroll.go | 4 + internal/device/adb.go | 45 ++++++++ internal/device/adb_test.go | 14 +++ internal/mobiumdriver/buttons.go | 39 +++++++ internal/mobiumdriver/capability.go | 6 + 13 files changed, 485 insertions(+), 7 deletions(-) create mode 100755 docs/checks/lock.sh create mode 100644 internal/agent/lock_test.go diff --git a/docs/CHALLENGES.md b/docs/CHALLENGES.md index 575bd06b..97335a50 100644 --- a/docs/CHALLENGES.md +++ b/docs/CHALLENGES.md @@ -7,7 +7,7 @@ the source. The pattern across them is the reason the document exists: **almost nothing here was found by reading code or by a test written from imagination.** Of -388 defects, 310 were found only by running against a real device. Seventeen +389 defects, 311 were found only by running against a real device. Seventeen more — 255, 280, 282, 327, 329, 353, 355, 356, 359, 371, 375, 380, 382, 384, 386, 387 and 388 — were met while running against one, though the fault lay in a check, the host or Mobium's own design rather than in the platform. The other sixty-one — 4, 5, 6, 14, 23, @@ -8551,6 +8551,44 @@ TalkBack here does not log at VERBOSE. ends it, so `mobium record` cannot film a client's run. It now warns, and the CLI's stop names why. `mobium test` over a running recording kept it, 1 run. +### 389. A tap straight after a lock was reported, and reached nothing + +**Found by:** a person, 2026-11-04 (mobiumdev/mobium#327), on MobiumApp on the +Android 15 emulator and the iOS 26.5 simulator. `dumpsys power` says asleep at +once, so `lock lock` answered. Reads went on showing the app for about half a +second, then hung for 10 seconds and failed. A tap in that half second answered +`tapped`, exit 0. Going through every action found the rest: on Android, a +swipe, long press or drag at a point touched a screen that was off and answered +done, since no read stands in front of them; and `find`, `wait`, `scroll-to`, +`swipe up`, `keyboard --text` and `current` blamed a changing screen, exit 1. On +iOS, reads showed the app for a moment too: `scroll-to` swiped the lock screen, +which can unlock it, and a miss said only "no element matches". + +**Fixed:** `lock lock` waits until the lock is what a read will see: on Android +for `dumpsys window`'s `interactiveState` to leave +`INTERACTIVE_STATE_GOING_TO_SLEEP`, about 0.55 s; on iOS for SpringBoard in front. +Each is bounded at 3 s. Any tool that fails reading on a locked device answers +`device_not_ready`, in one place, `lockedFailure`. A touch at a point asks whether +the screen is off, one `dumpsys power`, 57 ms for the refusal. A miss or a +`scroll-to` with the system shell in front asks whether the device is locked. + +"Asleep" is not "locked": a lock screen that is up takes touches, a swipe up or a +PIN pad, so nothing refuses those. iOS's lock screen stays on and is never asked. + +**Measured:** tap after lock, 8 of 8 exit 3 on Android 15 and 17, none landing, +where Android 15 gave 8 of 8 false successes. On iOS, tap 10 of 10 exit 3, and +`scroll-to` 8 of 8 exit 3 still locked. `lock.sh` fails on `main` and passes here. +Unlocked point gestures still land, and a tap straight after `lock unlock` landed +6 of 6. A lock screen that stays awake on Android was not measured: the Android +15 emulator showed none. + +**Still so:** a device put to sleep by anything else, then tapped by element +within that half second, reports the tap: 4 of 4 with `input keyevent +KEYCODE_SLEEP`. Asking before every touch would close it for 45 ms of a 250 ms +tap on the emulator. `press back` and `press home` on a sleeping Android answer +after 20 s, saying what they could not read. A read on a sleeping Android takes +10 s to fail, so these refusals do too. + ## Findings that were not defects Worth recording because each one closed off an approach that looked obvious. diff --git a/docs/RELEASE-CHECKLIST.md b/docs/RELEASE-CHECKLIST.md index c608f9c8..38b0269d 100644 --- a/docs/RELEASE-CHECKLIST.md +++ b/docs/RELEASE-CHECKLIST.md @@ -4,7 +4,7 @@ [CI workflow](../.github/workflows/ci.yml) runs on every push — formatting, vet, both modules' tests, the four non-Go clients' own tests, and six cross-compile targets. This file is the rest, and the rest is where most -serious defects in this project have come from: 310 of 388 were found only by +serious defects in this project have come from: 311 of 389 were found only by running against a real device. Work through it before tagging a release, on three substrates: an Android diff --git a/docs/checks/README.md b/docs/checks/README.md index acfa9f56..abfd92ff 100644 --- a/docs/checks/README.md +++ b/docs/checks/README.md @@ -76,6 +76,7 @@ checks do not: `clean-stop.sh`, which stops the caller's daemon, and | [audio.sh](audio.sh) | **Audio capture on an Android emulator**, on MobiumApp's Audio Demo: 440 Hz for 2 s heard as such, 440 then a second of silence then 880 in order, 660 Hz played as an alarm, a tone ended at Stop, and the silence control heard as silence while `dumpsys audio` reports a player started for it — the contrast the capture exists for. Each assertion was shown to fail on a wrong expectation. Refuses a phone and an iOS device by name | | [audio-sim.sh](audio-sim.sh) | **Audio on an iOS simulator**, on MobiumApp's Audio Demo, through a process tap on the app alone: a tone heard as 440 Hz for about two seconds, the sequence as 440 then 880 Hz, the Silent button's player writing zeros heard as silence, an expect of 660 Hz for a 440 tone failing with what was heard and the WAV kept, and Settings, which opens no audio, said never to have. Needs Xcode and System Audio Recording allowed for what runs mobium. Passed three of three on the iPhone 17 Pro simulator, 2026-10-22 | | [record.sh](record.sh) | **Screen recording**: something moving recorded, saved where the command ran, and judged by the video's own header — frames and duration — then a still screen, fewer frames and still a valid file (on a real iPhone, whose stream sends frames at a steady rate, a steady rate of them), the refusals, and nothing left recording or on the device | +| [lock.sh](lock.sh) | **A locked device**, straight after `lock lock`: a tap is refused as device_not_ready and reaches nothing, on Android a read names the lock and a touch at a point is refused because the screen is off, and on iOS `scroll-to` leaves the lock screen unswiped. Unlocked, the same tap lands. Emulators and simulators only (CHALLENGES 389) | | [locale-ios.sh](locale-ios.sh) | **Per-app language on iOS**, read back from the app: Settings pinned to ja-JP reads 一般 for General — relaunched by the pin, and again on a later launch — and General once cleared; a malformed tag is refused. iOS has no stored per-app language, so this holds Mobium's launch arguments to what the app shows. Android's half is `device-state.sh` | | [notifications-ios.sh](notifications-ios.sh) | **Notifications on an iOS simulator**, read back from Notification Center: a post as MobiumApp confirmed by its banner with MobiumApp still in front; the read finding it by app and body and closing Notification Center again; the shade opened so `map` reads Notification Center and finds it, then closed back to MobiumApp; and a post with the home screen in front refused. On a real iPhone, only the refusal is checked | | [timezone-ios.sh](timezone-ios.sh) | **The time zone on iOS**, read back from Calendar, which marks the hour in progress: the device's hour, then Asia/Tokyo's once set — the app in front relaunched, and a later launch too — and the device's again once set back; an unknown zone refused. Only hour rows are read, so a phone's events are never printed | diff --git a/docs/checks/lock.sh b/docs/checks/lock.sh new file mode 100755 index 00000000..7d532e58 --- /dev/null +++ b/docs/checks/lock.sh @@ -0,0 +1,79 @@ +#!/bin/sh +# A locked device, end to end: straight after `lock lock`, an action is refused +# as device_not_ready naming the lock, and nothing is reported as done. Then +# `lock unlock`, and the same tap lands, which is what shows the refusal was the +# lock's. +# +# docs/checks/lock.sh +# +# CHALLENGES 389. On Android a touch at a point is refused too, since the screen +# is off. On iOS the lock screen stays on and takes touches, so `scroll-to` is +# what is asked: it must not swipe it, which can unlock it. Not on a phone: one +# with a PIN cannot be unlocked from outside. +# +# Needs MobiumApp installed (mobiumdev/mobium-app). +set -e +DEV="$1" +if [ -z "$DEV" ]; then echo "usage: $0 " >&2; exit 2; fi +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +. "$ROOT/docs/checks/lib.sh" +check_lock "$DEV" +fail() { echo "FAIL: $*" >&2; exit 1; } +row() { printf ' %-16s %-52s ok\n' "$1" "$2"; } +APP=dev.mobium.mobiumapp + +case "$DEV" in + *-*-*-*-*) PLATFORM=ios; M="$ROOT/bin/mobium --driver wda --device $DEV" ;; + emulator-*) PLATFORM=android; M="$ROOT/bin/mobium --device $DEV" ;; + *) echo "lock.sh is for an emulator or a simulator: a phone with a PIN cannot be unlocked from outside" >&2; exit 2 ;; +esac +echo "--- $DEV ($PLATFORM)" +at_exit "$M lock unlock >/dev/null 2>&1 || true" + +# refused : exit 3, device_not_ready, naming the lock. +refused() { + what="$1"; shift + set +e; r=$($M "$@" 2>&1); st=$?; set -e + [ $st -eq 3 ] || fail "$what: expected device_not_ready (3), got $st: $r" + echo "$r" | grep -q 'lock unlock' || fail "$what: the refusal does not name the unlock: $r" +} + +fresh() { + $M lock unlock >/dev/null + $M terminate $APP >/dev/null 2>&1 || true + $M launch $APP >/dev/null + $M wait "label=Login Demo" >/dev/null + $M lock lock >/dev/null +} + +fresh +refused "a tap" tap "label=Login Demo" +$M lock unlock >/dev/null +sleep 1 +if $M find testid=username 2>/dev/null | grep -q 'username (input)'; then + fail "a refused tap opened the Login Demo" +fi +row "tap" "refused, and it reached nothing" + +fresh +if [ "$PLATFORM" = android ]; then + refused "a read" find testid=username + row "read" "names the lock, not a changing screen" + fresh + refused "a swipe at a point" swipe 540 1500 540 1000 + refused "a long press at a point" long-press 540 1500 + row "at a point" "refused: the screen is off" +else + refused "scroll-to" scroll-to testid=nothing + $M lock | grep -q '^locked' || fail "scroll-to unlocked the device" + row "scroll-to" "refused, and the device is still locked" +fi + +$M lock unlock >/dev/null +$M wait "label=Login Demo" >/dev/null +$M tap "label=Login Demo" >/dev/null +$M wait testid=username >/dev/null || fail "the tap did not land once unlocked" +row "unlocked" "the same tap lands" + +$M terminate $APP >/dev/null 2>&1 || true +echo PASS diff --git a/internal/agent/batch.go b/internal/agent/batch.go index bfda7a69..079b473a 100644 --- a/internal/agent/batch.go +++ b/internal/agent/batch.go @@ -118,7 +118,7 @@ func (h *Handlers) batchStep(ctx context.Context, st batchStep) (*ToolsCallResul ctx, cancel := context.WithTimeout(ctx, callTimeout) defer cancel() h.handled = nil - res, err := h.reportHandled(h.dispatch(ctx, st.name, st.args)) + res, err := h.reportHandled(h.callTool(ctx, st.name, st.args)) h.handled = nil return res, err } diff --git a/internal/agent/drag.go b/internal/agent/drag.go index c6c27fff..7501a099 100644 --- a/internal/agent/drag.go +++ b/internal/agent/drag.go @@ -83,6 +83,9 @@ func (h *Handlers) dragOn(ctx context.Context, s *session, args map[string]inter } if coords { + if err := h.refuseAsleep(ctx, s); err != nil { + return nil, err + } if err := dragger.Drag(ctx, x1, y1, x2, y2, hold, move); err != nil { return nil, err } diff --git a/internal/agent/handlers.go b/internal/agent/handlers.go index 6c66eae0..2f631497 100644 --- a/internal/agent/handlers.go +++ b/internal/agent/handlers.go @@ -332,7 +332,7 @@ func (h *Handlers) CallMeta(name string, args, meta map[string]interface{}) (*To if s := h.tracedSession(args); s != nil { t := s.trace id := traceBefore(t, name, args, meta[MetaExpect]) - res, err := h.dispatch(ctx, name, args) + res, err := h.callTool(ctx, name, args) res, err = h.reportIdle(h.reportHandled(res, err)) // A call that ended the session ended the trace with it. if s.trace == t { @@ -341,12 +341,47 @@ func (h *Handlers) CallMeta(name string, args, meta map[string]interface{}) (*To return res, err } } - res, err := h.dispatch(ctx, name, args) + res, err := h.callTool(ctx, name, args) return h.reportIdle(h.reportHandled(res, err)) } // dispatch runs one tool by name, if the policy allows it. A batch is held // to the policy step by step, since each step comes back through here. +func (h *Handlers) callTool(ctx context.Context, name string, args map[string]interface{}) (*ToolsCallResult, error) { + res, err := h.dispatch(ctx, name, args) + return res, h.lockedFailure(ctx, name, args, err) +} + +// lockedFailure names the lock when a tool failed for want of a screen to read: +// the read blames a changing screen. CHALLENGES 389. +func (h *Handlers) lockedFailure(ctx context.Context, name string, args map[string]interface{}, err error) error { + if code := mobiumerr.CodeOf(err); code != mobiumerr.Unclassified && code != mobiumerr.DeviceServer { + return err + } + switch name { + case "app_lock", "app_session", "app_shutdown", "app_boot", "app_batch": + return err + } + serial := stringArg(args, "device") + if serial == "" { + serial = h.defaultDevice + } + var open *session + for _, s := range h.sessions { + if serial != "" && s.dev.Serial != serial { + continue + } + if open != nil { + return err + } + open = s + } + if open == nil { + return err + } + return h.lockedRead(ctx, open, err) +} + func (h *Handlers) dispatch(ctx context.Context, name string, args map[string]interface{}) (*ToolsCallResult, error) { if err := h.permitCall(ctx, name, args); err != nil { return nil, err @@ -769,6 +804,28 @@ func (h *Handlers) lockedRead(ctx context.Context, s *session, err error) error "or password, or unlock it by hand").WithDetail("read_error", err.Error()) } +// refuseAsleep refuses a touch on a screen that is off, which reaches nothing. +// For a touch at a point, which no read stands in front of. CHALLENGES 389. +func (h *Handlers) refuseAsleep(ctx context.Context, s *session) error { + sl, ok := mobiumdriver.AsScreenSleeper(s.driver) + if !ok { + return nil + } + if asleep, err := sl.ScreenAsleep(ctx); err != nil || !asleep { + return nil + } + return mobiumerr.New(mobiumerr.DeviceNotReady, "the screen is off, so a touch reaches nothing — run `mobium "+ + "lock unlock` (app_lock with state \"unlock\"), which wakes it and unlocks a device with no PIN, pattern "+ + "or password") +} + +// lockedOver refuses a target that a lock screen stands in front of. +func lockedOver(loc uitree.Locator) error { + return mobiumerr.New(mobiumerr.DeviceNotReady, "%s is not on the screen because the device is locked — run "+ + "`mobium lock unlock` (app_lock with state \"unlock\"), which unlocks a device with no PIN, pattern or "+ + "password, or unlock it by hand", loc) +} + // inferredNote says when which elements are shown was worked out from where // they are, on a screen too large for iOS to be asked (CHALLENGES 258): an // element something covers may be listed, and a tap on it is refused by @@ -881,13 +938,22 @@ func (h *Handlers) tapOn(ctx context.Context, s *session, args map[string]interf } // A point off the screen touches nothing, and an iPhone simulator // answered a tap at (50000, 50000) as done (CHALLENGES 187). - if tree, err := driver.Snapshot(ctx); err == nil && tree.Root != nil { + tree, err := driver.Snapshot(ctx) + if err != nil { + if lerr := h.lockedRead(ctx, s, err); mobiumerr.CodeOf(lerr) == mobiumerr.DeviceNotReady { + return nil, lerr + } + } + if err == nil && tree.Root != nil { if b := tree.Root.Bounds; !b.Empty() && (x < b.X1 || y < b.Y1 || x >= b.X2 || y >= b.Y2) { return nil, mobiumerr.New(mobiumerr.InvalidArgument, "(%d, %d) is off the screen, which is %s in device "+ "pixels — a touch there reaches nothing", x, y, b). WithRemedy("give a point on the screen, or tap an element by its ref or locator") } } + if err := h.refuseAsleep(ctx, s); err != nil { + return nil, err + } if err := touch(ctx, x, y); err != nil { return nil, err } @@ -2004,6 +2070,10 @@ func (h *Handlers) resolveNodeOnce(ctx context.Context, s *session, target strin if text := appDialogText(tree); text != "" { return nil, nil, appDialogOver(text, loc, tree.Root != nil && !uitree.IsIOS(tree.Root)) } + // The lock screen, which holds nothing of the app. CHALLENGES 389. + if tree != nil && h.lockedNote(ctx, s, tree) != "" { + return nil, nil, lockedOver(loc) + } default: // It resolved, which is not the same as being reachable. A row @@ -2188,7 +2258,7 @@ func (h *Handlers) settle(ctx context.Context, s *session, loc uitree.Locator, // it: this is the snapshot taken *because* something might be // moving, which is exactly when the hierarchy is unreadable. if time.Now().After(deadline) { - return nil, nil, err + return nil, nil, h.lockedRead(ctx, s, err) } continue } @@ -2564,6 +2634,9 @@ func (h *Handlers) swipe(ctx context.Context, args map[string]interface{}) (*Too if err := h.awaitAppIdle(ctx, s, fmt.Sprintf("(%d,%d) to (%d,%d)", x1, y1, x2, y2)); err != nil { return nil, err } + if err := h.refuseAsleep(ctx, s); err != nil { + return nil, err + } if err := gest.Swipe(ctx, x1, y1, x2, y2, duration); err != nil { return nil, err } @@ -2667,6 +2740,9 @@ func (h *Handlers) longPress(ctx context.Context, args map[string]interface{}) ( if err := h.awaitAppIdle(ctx, s, fmt.Sprintf("(%d, %d)", x, y)); err != nil { return nil, err } + if err := h.refuseAsleep(ctx, s); err != nil { + return nil, err + } if err := gest.LongPress(ctx, x, y, duration); err != nil { return nil, err } diff --git a/internal/agent/lock_test.go b/internal/agent/lock_test.go new file mode 100644 index 00000000..269acff6 --- /dev/null +++ b/internal/agent/lock_test.go @@ -0,0 +1,173 @@ +package agent + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/mobiumdev/mobium/internal/mobiumerr" + "github.com/mobiumdev/mobium/internal/uitree" +) + +// lockedDriver is a device whose screen reads come from a script, for the +// ways a lock meets an action. CHALLENGES 389. +type lockedDriver struct { + *buttonDriver + reads []func() (*uitree.Tree, error) + tapped int + asleep bool +} + +func (d *lockedDriver) ScreenAsleep(ctx context.Context) (bool, error) { return d.asleep, nil } + +func (d *lockedDriver) Snapshot(ctx context.Context) (*uitree.Tree, error) { + read := d.reads[0] + if len(d.reads) > 1 { + d.reads = d.reads[1:] + } + return read() +} + +func (d *lockedDriver) Tap(ctx context.Context, x, y int) error { + d.tapped++ + return nil +} + +func withLock(t *testing.T, locked bool, reads ...func() (*uitree.Tree, error)) (*Handlers, *session, *lockedDriver) { + t.Helper() + d := &lockedDriver{buttonDriver: &buttonDriver{locked: locked}, reads: reads} + h := NewHandlers() + h.implicitWait = 0 + h.settleWindow = 0 + s := &session{dev: fakeDevice(), driver: d, backend: BackendUIA2} + h.sessions["fake"] = s + return h, s, d +} + +func unreadable() (*uitree.Tree, error) { return nil, errors.New("could not read the UI hierarchy") } + +func keyguard(t *testing.T) func() (*uitree.Tree, error) { + t.Helper() + tree, err := uitree.ParseAndroid([]byte(`` + + `` + + ``)) + if err != nil { + t.Fatalf("parse fixture: %v", err) + } + return func() (*uitree.Tree, error) { return tree, nil } +} + +func TestAMissOnALockScreenSaysTheDeviceIsLocked(t *testing.T) { + h, s, d := withLock(t, true, keyguard(t)) + _, err := tap(h, s, map[string]interface{}{"target": "text=Login"}) + if mobiumerr.CodeOf(err) != mobiumerr.DeviceNotReady || !strings.Contains(err.Error(), "mobium lock unlock") { + t.Errorf("err = %v, want device_not_ready naming the unlock", err) + } + if d.tapped != 0 { + t.Errorf("touched a locked device %d times", d.tapped) + } +} + +func TestAMissOnTheShellOfAnUnlockedDeviceStaysAMiss(t *testing.T) { + h, s, _ := withLock(t, false, keyguard(t)) + _, err := tap(h, s, map[string]interface{}{"target": "text=Login"}) + if mobiumerr.CodeOf(err) != mobiumerr.NoSuchElement { + t.Errorf("err = %v, want no_such_element", err) + } +} + +func TestAPointIsNotTouchedOnADeviceThatCannotBeRead(t *testing.T) { + h, s, d := withLock(t, true, unreadable) + _, err := tap(h, s, map[string]interface{}{"x": 10, "y": 10}) + if mobiumerr.CodeOf(err) != mobiumerr.DeviceNotReady { + t.Errorf("err = %v, want device_not_ready", err) + } + if d.tapped != 0 { + t.Errorf("touched a locked device %d times", d.tapped) + } +} + +func TestAPointIsTouchedWhenAnUnlockedDeviceCannotBeRead(t *testing.T) { + h, s, d := withLock(t, false, unreadable) + if _, err := tap(h, s, map[string]interface{}{"x": 10, "y": 10}); err != nil { + t.Errorf("err = %v", err) + } + if d.tapped != 1 { + t.Errorf("tapped %d times, want 1", d.tapped) + } +} + +func TestAReadLostWhileSettlingSaysTheDeviceIsLocked(t *testing.T) { + first := screen(t, "Login") + h, s, d := withLock(t, true, func() (*uitree.Tree, error) { return first, nil }, unreadable) + h.settleWindow = time.Millisecond + h.settleTimeout = 10 * time.Millisecond + _, err := tap(h, s, map[string]interface{}{"target": "text=Login"}) + if mobiumerr.CodeOf(err) != mobiumerr.DeviceNotReady { + t.Errorf("err = %v, want device_not_ready", err) + } + if d.tapped != 0 { + t.Errorf("touched a locked device %d times", d.tapped) + } +} + +func TestAPointIsNotTouchedOnAScreenThatIsOff(t *testing.T) { + h, s, d := withLock(t, true, keyguard(t)) + d.asleep = true + _, err := tap(h, s, map[string]interface{}{"x": 10, "y": 10}) + if mobiumerr.CodeOf(err) != mobiumerr.DeviceNotReady || !strings.Contains(err.Error(), "the screen is off") { + t.Errorf("err = %v, want device_not_ready naming the screen", err) + } + if d.tapped != 0 { + t.Errorf("touched a screen that is off %d times", d.tapped) + } +} + +// A lock screen that is up takes touches: a swipe up, a PIN pad. +func TestAPointIsTouchedOnALockScreenThatIsUp(t *testing.T) { + h, s, d := withLock(t, true, keyguard(t)) + if _, err := tap(h, s, map[string]interface{}{"x": 10, "y": 10}); err != nil { + t.Errorf("err = %v", err) + } + if d.tapped != 1 { + t.Errorf("tapped %d times, want 1", d.tapped) + } +} + +func TestARefusalOfAnotherCodeIsNotBlamedOnTheLock(t *testing.T) { + h, _, _ := withLock(t, true, keyguard(t)) + refused := mobiumerr.New(mobiumerr.NoSuchElement, "no element") + if got := h.lockedFailure(context.Background(), "app_find", nil, refused); got != refused { + t.Errorf("got %v", got) + } +} + +func TestAFailedReadOnALockedDeviceNamesTheLock(t *testing.T) { + h, _, _ := withLock(t, true, keyguard(t)) + read := errors.New("could not read the UI hierarchy") + got := h.lockedFailure(context.Background(), "app_find", nil, read) + if mobiumerr.CodeOf(got) != mobiumerr.DeviceNotReady { + t.Errorf("got %v", got) + } + if got := h.lockedFailure(context.Background(), "app_lock", nil, read); got != read { + t.Errorf("app_lock's own failure: %v", got) + } +} + +func TestAFailedReadOnAnUnlockedDeviceStaysAsItIs(t *testing.T) { + h, _, _ := withLock(t, false, keyguard(t)) + read := errors.New("could not read the UI hierarchy") + if got := h.lockedFailure(context.Background(), "app_find", nil, read); got != read { + t.Errorf("got %v", got) + } +} + +func TestScrollToDoesNotSwipeALockScreen(t *testing.T) { + h, s, _ := withLock(t, true, keyguard(t)) + _, err := h.scrollToOn(context.Background(), s, map[string]interface{}{"target": "text=Login"}) + if mobiumerr.CodeOf(err) != mobiumerr.DeviceNotReady { + t.Errorf("err = %v, want device_not_ready", err) + } +} diff --git a/internal/agent/scroll.go b/internal/agent/scroll.go index 5da3ef37..8b00e6a1 100644 --- a/internal/agent/scroll.go +++ b/internal/agent/scroll.go @@ -221,6 +221,10 @@ func (h *Handlers) scrollIntoViewFrom(ctx context.Context, s *session, tree *uit if lostAmongLookAlikes(resolveErr) || !matchedNothing(resolveErr) && !errors.Is(resolveErr, errOffScreen) { return nil, nil, 0, resolveErr } + // A swipe up on a lock screen unlocks it. CHALLENGES 389. + if h.lockedNote(ctx, s, tree) != "" { + return nil, nil, 0, lockedOver(loc) + } // A screen in the middle of changing can read, for a moment, as having // nothing that scrolls: measured on Settings, a scroll-to 60ms after // pressing back failed 3 times in 15, and the list was there a moment diff --git a/internal/device/adb.go b/internal/device/adb.go index fdd857dd..0870c2d6 100644 --- a/internal/device/adb.go +++ b/internal/device/adb.go @@ -1282,6 +1282,48 @@ func (a *ADB) ScreenLocked(ctx context.Context) (bool, error) { return false, nil } +// screenSleepBudget bounds awaitSleep: a device still on its way to sleep after +// it is locked as asked. Measured at about 0.55 s on the Android 15 emulator. +const screenSleepBudget = 3 * time.Second + +// ScreenAsleep reports the screen off. A lock screen that is up is awake, and +// takes touches. It costs one dumpsys, where ScreenLocked may cost two. +func (a *ADB) ScreenAsleep(ctx context.Context) (bool, error) { + power, err := a.Shell(ctx, "dumpsys", "power") + if err != nil { + return false, err + } + m := wakefulnessRe.FindSubmatch(power) + if m == nil { + return false, mobiumerr.New(mobiumerr.DeviceServer, "could not read whether the device is awake") + } + return !strings.EqualFold(string(m[1]), "Awake"), nil +} + +// awaitSleep waits for the device to finish going to sleep. Wakefulness changes +// at once, while reads go on showing the app for about half a second, and a tap +// then is reported and reaches nothing. CHALLENGES 389. +func (a *ADB) awaitSleep(ctx context.Context) { + deadline := time.Now().Add(screenSleepBudget) + for time.Now().Before(deadline) { + win, err := a.Shell(ctx, "dumpsys", "window") + if err != nil || !goingToSleep(win) { + return + } + select { + case <-ctx.Done(): + return + case <-time.After(50 * time.Millisecond): + } + } +} + +// goingToSleep reports a device on its way to sleep. An Android without the +// field answers false, and nothing is waited for. +func goingToSleep(dumpsysWindow []byte) bool { + return bytes.Contains(dumpsysWindow, []byte("interactiveState=INTERACTIVE_STATE_GOING_TO_SLEEP")) +} + // SetScreenLocked puts the screen to sleep or wakes it, and confirms it. // // KEYCODE_SLEEP and KEYCODE_WAKEUP are used rather than KEYCODE_POWER because @@ -1311,6 +1353,9 @@ func (a *ADB) SetScreenLocked(ctx context.Context, locked bool) error { return err } if got == locked { + if locked { + a.awaitSleep(ctx) + } return nil } if time.Now().After(deadline) { diff --git a/internal/device/adb_test.go b/internal/device/adb_test.go index 81891654..464da8f4 100644 --- a/internal/device/adb_test.go +++ b/internal/device/adb_test.go @@ -455,3 +455,17 @@ func TestForwardsToPicksOneDevicesForwardsToOnePort(t *testing.T) { t.Errorf("no forwards read as %v", got) } } + +// CHALLENGES 389. +func TestGoingToSleep(t *testing.T) { + for dumpsys, want := range map[string]bool{ + " interactiveState=INTERACTIVE_STATE_GOING_TO_SLEEP\n": true, + " interactiveState=INTERACTIVE_STATE_SLEEP\n": false, + " interactiveState=INTERACTIVE_STATE_AWAKE\n": false, + "an Android that does not print the field\n": false, + } { + if got := goingToSleep([]byte(dumpsys)); got != want { + t.Errorf("goingToSleep(%q) = %v", dumpsys, got) + } + } +} diff --git a/internal/mobiumdriver/buttons.go b/internal/mobiumdriver/buttons.go index 6d6fd998..dfe398eb 100644 --- a/internal/mobiumdriver/buttons.go +++ b/internal/mobiumdriver/buttons.go @@ -6,6 +6,7 @@ import ( "github.com/mobiumdev/mobium/internal/mobiumerr" "sort" "strings" + "time" ) // Mobium's hardware button vocabulary. @@ -74,6 +75,12 @@ type ScreenLock interface { SetScreenLocked(ctx context.Context, locked bool) error } +// ScreenSleeper reports a screen that is off, which a touch cannot reach. Only +// Android: an iOS lock screen stays on, and takes touches. +type ScreenSleeper interface { + ScreenAsleep(ctx context.Context) (bool, error) +} + // androidKeycodes maps the vocabulary to what `input keyevent` understands. var androidKeycodes = map[string]string{ ButtonBack: "KEYCODE_BACK", @@ -146,6 +153,11 @@ func (a *Android) ScreenLocked(ctx context.Context) (bool, error) { return a.adb.ScreenLocked(ctx) } +// ScreenAsleep reports the screen off. +func (a *Android) ScreenAsleep(ctx context.Context) (bool, error) { + return a.adb.ScreenAsleep(ctx) +} + // SetScreenLocked locks or unlocks the screen and confirms it. func (a *Android) SetScreenLocked(ctx context.Context, locked bool) error { return a.adb.SetScreenLocked(ctx, locked) @@ -164,6 +176,11 @@ func (u *UIA2) ScreenLocked(ctx context.Context) (bool, error) { return u.adb.ScreenLocked(ctx) } +// ScreenAsleep reports the screen off. +func (u *UIA2) ScreenAsleep(ctx context.Context) (bool, error) { + return u.adb.ScreenAsleep(ctx) +} + // SetScreenLocked locks or unlocks the screen and confirms it. func (u *UIA2) SetScreenLocked(ctx context.Context, locked bool) error { return u.adb.SetScreenLocked(ctx, locked) @@ -296,5 +313,27 @@ func (w *WDA) SetScreenLocked(ctx context.Context, locked bool) error { map[bool]string{true: "lock", false: "unlock"}[locked], map[bool]string{true: "locked", false: "unlocked"}[got]) } + if locked { + w.awaitLockScreen(ctx) + } return nil } + +// awaitLockScreen waits for SpringBoard to come in front. /wda/locked says yes +// at once, while reads go on showing the app, and a scroll-to swiped there. +// CHALLENGES 389. +func (w *WDA) awaitLockScreen(ctx context.Context) { + deadline := time.Now().Add(lockScreenBudget) + for time.Now().Before(deadline) && ctx.Err() == nil { + if w.foregroundBundle(ctx) == springboardBundleID { + return + } + select { + case <-ctx.Done(): + case <-time.After(50 * time.Millisecond): + } + } +} + +// lockScreenBudget bounds awaitLockScreen. +const lockScreenBudget = 3 * time.Second diff --git a/internal/mobiumdriver/capability.go b/internal/mobiumdriver/capability.go index 01a61483..7138e5f8 100644 --- a/internal/mobiumdriver/capability.go +++ b/internal/mobiumdriver/capability.go @@ -259,6 +259,12 @@ func AsScreenLock(d Driver) (ScreenLock, bool) { return s, ok && has(d, CapScreenLock) } +// AsScreenSleeper returns the driver's asleep question, if it has one. +func AsScreenSleeper(d Driver) (ScreenSleeper, bool) { + s, ok := d.(ScreenSleeper) + return s, ok && has(d, CapScreenLock) +} + // AsInterruptions returns the driver's call and message support, if it has any. func AsInterruptions(d Driver) (Interruptions, bool) { i, ok := d.(Interruptions)