Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion clients/dotnet/Mobium/Device.cs
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,8 @@ public byte[] Screenshot(string path)
/// <summary>
/// Blocks until the screen agrees, instead of sleeping. On success the
/// screen is remapped, so the element returned already has a ref that
/// can be tapped. Waiting for something to go away returns
/// can be tapped. Its ref is empty when the map filters it out, as a
/// plain text in a list. Waiting for something to go away returns
/// <c>null</c>: there is nothing left to point at.
/// </summary>
public Element? WaitFor(string target, Until condition) =>
Expand Down
9 changes: 6 additions & 3 deletions clients/dotnet/Mobium/Element.cs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ namespace Mobium
/// <summary>One actionable thing on screen.</summary>
public sealed class Element
{
/// <summary>The <c>@e1</c> handle, valid only for the screen it came from.</summary>
/// <summary>The <c>@e1</c> handle, valid only for the screen it came from. Empty for a match the map filters out.</summary>
public string Ref { get; }

/// <summary>What a person would call it.</summary>
Expand Down Expand Up @@ -66,7 +66,10 @@ internal static Element From(IDictionary<string, object?> m)
}

/// <summary>The ref and label, with the role in brackets when there is one.</summary>
public override string ToString() =>
string.IsNullOrEmpty(Role) ? Ref + " " + Label : Ref + " " + Label + " (" + Role + ")";
public override string ToString()
{
var name = (Ref + " " + Label).Trim();
return string.IsNullOrEmpty(Role) ? name : name + " (" + Role + ")";
}
}
}
4 changes: 3 additions & 1 deletion clients/go/mobium.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ func (l Locator) String() string {

// Element is one actionable thing on screen.
type Element struct {
// Ref is the @e1 handle, empty for a match the map filters out.
Ref string `json:"ref"`
Label string `json:"label"`
Role string `json:"role,omitempty"`
Expand Down Expand Up @@ -473,7 +474,8 @@ type WaitOptions struct {
// WaitFor blocks until the screen agrees, instead of sleeping.
//
// On success the screen is remapped, so the returned element already has a ref
// that can be tapped without calling Map first. Waiting for something to go
// that can be tapped without calling Map first. Its ref is empty when the map
// filters it out, as a plain text in a list. Waiting for something to go
// away returns a nil element and a nil error, since there is nothing left to
// point at. If the condition never holds the error says what was on screen
// instead, which is usually the answer.
Expand Down
6 changes: 4 additions & 2 deletions clients/java/src/main/java/dev/mobium/Element.java
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@
/**
* One actionable thing on screen.
*
* @param ref the {@code @e1} handle, valid only for the screen it came from
* @param ref the {@code @e1} handle, valid only for the screen it came from;
* empty for a match the map filters out
* @param label what a person would call it
* @param role button, input, list and so on; empty if Mobium could not say
* @param locator how the ref resolves on a later screen, as {@code kind=value}
Expand Down Expand Up @@ -45,6 +46,7 @@ static Element from(Map<String, Object> m) {
}

@Override public String toString() {
return role.isEmpty() ? ref + " " + label : ref + " " + label + " (" + role + ")";
String name = (ref + " " + label).strip();
return role.isEmpty() ? name : name + " (" + role + ")";
}
}
3 changes: 2 additions & 1 deletion clients/java/src/main/java/dev/mobium/Mobium.java
Original file line number Diff line number Diff line change
Expand Up @@ -402,7 +402,8 @@ public byte[] screenshot(Path path) {
* Blocks until the screen agrees, instead of sleeping.
*
* <p>On success the screen is remapped, so the element returned already
* has a ref that can be tapped without calling {@link #map()} first.
* has a ref that can be tapped without calling {@link #map()} first. Its
* ref is empty when the map filters it out, as a plain text in a list.
* Waiting for something to go away returns {@code null}: there is nothing
* left to point at. If the condition never holds, the exception says what
* was on screen instead, which is usually the answer.
Expand Down
2 changes: 1 addition & 1 deletion clients/javascript/index.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ export interface Bounds {

/** One actionable thing on screen, from `map()` or `find()`. */
export interface Element {
/** The `@e1` handle, valid only for the screen it came from. */
/** The `@e1` handle, valid only for the screen it came from. Empty for a match the map filters out. */
ref: string
/** What a person would call it. */
label: string
Expand Down
3 changes: 2 additions & 1 deletion clients/javascript/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -627,7 +627,8 @@ export class Device {
* it never happens, saying what was on screen instead.
*
* On success the screen is remapped, so the element returned already has a
* ref that can be tapped without calling map() first. Nothing is returned
* ref that can be tapped without calling map() first. Its ref is empty when
* the map filters it out, as a plain text in a list. Nothing is returned
* when waiting for something to go away.
*/
async waitFor(target, { condition = 'visible', text, timeoutMs = 10000, not, exact, count } = {}) {
Expand Down
7 changes: 5 additions & 2 deletions clients/python/mobium/_device.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ class Element:
"""One actionable element on screen."""

ref: str
"""The @e1 handle, valid only for the screen it came from; empty for a match the map filters out."""
label: str
role: str = ""
bounds: Bounds = Bounds(0, 0, 0, 0)
Expand All @@ -53,7 +54,8 @@ class Element:
"""What a slider reads, as the app states it ("80%", "1.2"); empty for anything else."""

def __str__(self) -> str:
return f"{self.ref} {self.label} ({self.role})" if self.role else f"{self.ref} {self.label}"
name = f"{self.ref} {self.label}".strip()
return f"{name} ({self.role})" if self.role else name


@dataclass(frozen=True)
Expand Down Expand Up @@ -341,7 +343,8 @@ def wait_for(
if it never happens, saying what was on screen instead.

On success the screen is remapped, so the element returned already has
a ref that can be tapped without calling map() first. Nothing is
a ref that can be tapped without calling map() first. Its ref is empty
when the map filters it out, as a plain text in a list. Nothing is
returned when waiting for something to go away.
"""
args: dict[str, Any] = {
Expand Down
26 changes: 24 additions & 2 deletions docs/CHALLENGES.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ the source.

The pattern across them is the reason the document exists: **almost nothing
here was found by reading code or by a test written from imagination.** Of
386 defects, 310 were found only by running against a real device. Fifteen
more — 255, 280, 282, 327, 329, 353, 355, 356, 359, 371, 375, 380, 382, 384 and 386 — were met while
387 defects, 310 were found only by running against a real device. Sixteen
more — 255, 280, 282, 327, 329, 353, 355, 356, 359, 371, 375, 380, 382, 384, 386 and 387 — were met while
running against one, though the fault lay in a check, the host or Mobium's own
design rather than in the platform. The other sixty-one — 4, 5, 6, 14, 23,
24, 32, 35, 36, 39, 44, 50, 53, 54, 57, 66, 89, 99, 100, 121, 123, 129, 130,
Expand Down Expand Up @@ -8502,6 +8502,28 @@ scrolls back. A list that fits takes one swipe that moves nothing; that miss
stays as it was. Measured here on both devices: the issue's repro names the
4 scrolls. A test fails on the old call site with the issue's message.

### 387. A wait for a text in a list answered with the list

**Found by:** a person writing a client test, 2026-11-04
(mobiumdev/mobium#329), on MobiumApp's Dialog Demo on the Android 15 emulator
and the iOS 26.5 simulator. `wait testid=dialogOutcome` answered `@e2
ScrollView (list)`, and the ref tapped the list's center. The walk to the
nearest mapped ancestor is right for a text in a clickable row. A plain text
has no such row, so the walk ended at the list. `scroll-to` shares the walk.

**Fixed:** the walk stops at a list, unless the list is what matched. The
match is reported with no ref, as `find` reports it: `(no ref) nothing yet
[42,515][1038,572]`, and an element with `ref` empty in every client. It is
labeled as `map` labels: cleaned, bounded, and never a password. `app_scroll_to`
is marked as device text.

**Measured:** a sweep waited on the first 14 visible texts of each screen, across
9 Android apps and 8 iOS apps, Compose, SwiftUI and Flutter among them. On `main`,
22 answers on Android and 18 on iOS were the list, from MobiumApp and iOS Settings
alone. With the fix, 0 of 203 and 0 of 193. All five clients and MCP took the
empty ref on the Android 15 emulator. `wait-states.sh` fails on `main` on both
devices and passes with the fix.

## Findings that were not defects

Worth recording because each one closed off an approach that looked obvious.
Expand Down
2 changes: 1 addition & 1 deletion docs/RELEASE-CHECKLIST.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
[CI workflow](../.github/workflows/ci.yml) runs on every push — formatting,
vet, both modules' tests, the four non-Go clients' own tests, and six
cross-compile targets. This file is the rest, and the rest is where most
serious defects in this project have come from: 310 of 386 were found only by
serious defects in this project have come from: 310 of 387 were found only by
running against a real device.

Work through it before tagging a release, on three substrates: an Android
Expand Down
2 changes: 1 addition & 1 deletion docs/checks/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ checks do not: `clean-stop.sh`, which stops the caller's daemon, and
| [speech.sh](speech.sh) | **What the screen reader says**, `mobium speech`, on MobiumApp's Login Demo and Screen Reader Demo: refused with the screen reader off, naming the setting; a sentence about a typed password field withheld; a walk hearing the icon with no name as a bare "Button" and Send as "Submit form" — on Android "Save button" with its kind said twice and every sentence beside its element, on an iPhone both Save buttons said alike and neither named — and ending back round on Android, where the cursor stops on an iPhone; listening around a wrong sign-in hearing the app's announcement on both, and on Android around the carts hearing "Live cart: 1" as a change and never the quiet count — on an iPhone neither, since React Native's live region is Android's; the screen reader off after the session. On Android it sets TalkBack's log output level to VERBOSE through TalkBack's own settings and back as found, and closes the Accessibility Suite's notification prompt with back, answering nothing. A simulator is refused; a phone needs `ALLOW_PHONE=1` |
| [login.sh](login.sh) | **The login demo** — mobile automation's hello world, on MobiumApp's Login screen, and the script to show when showing Mobium. Negative paths first, each judged by the message the app shows: an empty form, a username too short, one with characters it does not allow, a password too short, a wrong password and an unknown user, told apart by nothing. Then positive: one field valid while the other is not, a username sanitized on submit, the welcome screen waited for rather than slept for, and log-out. A typed password is never printed. If iOS raises "Save Password?" over the app, log-out is skipped and the dialog quoted, rather than a swallowed tap reported as a timeout |
| [otp.sh](otp.sh) | **A one-time code**, on MobiumApp's OTP Demo: the code read where it arrives — the notification shade on Android, the banner on iOS, read through rather than landing on SpringBoard (CHALLENGES 155) — six one-digit boxes that move focus on, filled digit by digit and then as one string into the first box, which either arrives whole or names the digit that never did (156); no code, verified, already used, wrong twice, locked, and expired after `background 62`; and Resend refused while the screen disables it |
| [wait-states.sh](wait-states.sh) | **Waiting for a state**, on MobiumApp's Form and Login Demos: `wait --for` checked and unchecked on a custom checkbox, a radio and a real Switch; focused, arriving on a tap and moving to the next field — on iOS by asking WebDriverAgent which element is active, since the tree says every field is unfocused; and value, the whole of it where `text` is a part, with a field showing its placeholder holding "". Each timed out, saying what it saw, before it succeeded. A checked state on a text field and a password's value are refused at once, and a refusal's `details.check` is read from a real one (CHALLENGES 158) |
| [wait-states.sh](wait-states.sh) | **Waiting for a state**, on MobiumApp's Form and Login Demos: `wait --for` checked and unchecked on a custom checkbox, a radio and a real Switch; focused, arriving on a tap and moving to the next field — on iOS by asking WebDriverAgent which element is active, since the tree says every field is unfocused; and value, the whole of it where `text` is a part, with a field showing its placeholder holding "". Each timed out, saying what it saw, before it succeeded. A checked state on a text field and a password's value are refused at once, and a refusal's `details.check` is read from a real one (CHALLENGES 158). A plain text in a list answers with no ref, while a row and the list itself keep theirs (CHALLENGES 387) |
| [network.sh](network.sh) | **Network conditions**, judged by the traffic: on an emulator, latency by ping and download and upload by timing 2MB and 1MB against servers the script runs on the Mac, each before, shaped and after; offline by a connection that must fail; the download limit airplane mode removes, put back on coming online; and a session ended offline and shaped leaving the device online and unshaped. On a phone, offline and back, and shaping refused. The emulator console's own `network speed`/`delay` read back as set and change nothing — see CHALLENGES, findings |
| [proxy.sh](proxy.sh) | **A proxy**, judged by what the app received: MobiumApp's Network Demo fetches a port nothing on the device listens on, through a proxy the script runs on the Mac that answers every request itself — failed with no proxy, the proxy's body with `--proxy 127.0.0.1:<port>` (reversed with `adb reverse`, and logged by the proxy), failed again with `--no-proxy`, every proxy key unset and the reverse gone; set again and the session ended, the device as found; a proxy with a scheme refused. An emulator; a phone with `ALLOW_PHONE=1`. Passed 2026-10-22 on the Android 15 emulator and the Pixel 8 Pro |
| [test-runner.sh](test-runner.sh) | **`mobium test`**, held to its controls: a project whose device is unset refused before any test runs; the suite in `tests/mobiumapp` passing on every project, the projects at once and faster than one after another; every test in `tests/controls/must-fail.test.json` failing with its step, code and a screenshot, exit 1, and the JUnit file read by Python's XML parser agreeing; `--last-failed` running exactly those; and the flaky control failing with no retry and reported flaky with one. Iteration 2: the soft control reporting both soft failures and still reaching its last step, a retain-on-failure trace kept for a failed test and not a passing one, with its recording in Vibium's record format, and `--debug` stopping before each step and quitting. The suite's login test uses `each`; `--ui` is `test-ui.sh`'s. Emulators and simulators (the flaky control clears app data). [the test runner guide](../guides/test-runner.md) |
Expand Down
12 changes: 12 additions & 0 deletions docs/checks/wait-states.sh
Original file line number Diff line number Diff line change
Expand Up @@ -114,5 +114,17 @@ row "value" "whole, not part; empty past the placeholder"
refused "a password's value" "never read" wait testid=password --for value --text x --timeout 5s
row "password" "refused, and nothing compared"

# --- what a wait answers with (CHALLENGES 387) --------------------------------
open "Dialog Demo"
r=$($M wait testid=dialogOutcome) || fail "a text in a list: $r"
echo "$r" | grep -q '(no ref)' || fail "a text in a list did not answer with no ref: $r"
r=$($M scroll-to testid=dialogOutcome --direction up) || fail "scroll-to a text in a list: $r"
echo "$r" | grep -q '(no ref)' || fail "scroll-to a text in a list did not answer with no ref: $r"
r=$($M wait testid=twoButtonBtn) || fail "a row: $r"
echo "$r" | grep -q '@e[0-9]' || fail "a row lost its ref: $r"
r=$($M wait role=list) || fail "the list itself: $r"
echo "$r" | grep -q '@e[0-9].*(list' || fail "the list itself lost its ref: $r"
row "answer" "a text has no ref; a row and a list keep theirs"

$M terminate $APP >/dev/null 2>&1 || true
echo PASS
3 changes: 2 additions & 1 deletion docs/guides/autowait.md
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,8 @@ Every condition:

The default timeout is 10 seconds, `--timeout` changes it, up to two minutes.
A successful wait remaps the screen, so the element it found already has a
ref: `mobium tap @e3` straight after, with no `map` in between.
ref: `mobium tap @e3` straight after, with no `map` in between. A match the
map filters out, such as a plain text in a list, answers `(no ref)`.

## 6. Reading a refusal from code

Expand Down
2 changes: 1 addition & 1 deletion docs/guides/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -289,7 +289,7 @@ $ mobium batch steps.json
2. app_fill: typed "mobium" into testid=username
3. app_fill: typed 7 characters into testid=password, a password field — not echoed
4. app_tap: tapped testid=loginBtn at (540, 1457)
5. app_wait_for: testid=welcomeText is visible after 1.622s
5. app_wait_for: testid=welcomeText is visible after 1.637s — (no ref) Welcome, mobium! [42,334][1038,412]
```

![The Login Demo after the batch: "Welcome, mobium! You are logged in."](images/cli-3-welcome.jpg)
Expand Down
10 changes: 3 additions & 7 deletions internal/agent/handlers.go
Original file line number Diff line number Diff line change
Expand Up @@ -1064,13 +1064,9 @@ func (h *Handlers) find(ctx context.Context, args map[string]interface{}) (*Tool
view.Elements = append(view.Elements, elementView(e))
continue
}
// Matched something the map filters out — a label, say. Still worth
// reporting, but it has no ref to act on.
lines = append(lines, fmt.Sprintf("(no ref) %s %s", uitree.OneLine(n.Text+" "+n.Label), n.Bounds))
view.Elements = append(view.Elements, ElementView{
Label: strings.TrimSpace(n.Text + " " + n.Label),
Bounds: boundsView(n.Bounds),
})
line, ev := noRefElement(n)
lines = append(lines, line)
view.Elements = append(view.Elements, ev)
}
if len(lines) == 0 {
return Result(fmt.Sprintf("No element matches %s", loc), view), nil
Expand Down
1 change: 1 addition & 0 deletions internal/agent/provenance.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ var deviceTextFields = map[string][]string{
"app_text": {"text"},
"app_source": {"source"},
"app_wait_for": {"element.label", "element.value", "element.locator.value"},
"app_scroll_to": {"element.label", "element.value", "element.locator.value"},
"app_alert": {"text", "buttons[]", "pressed"},
"app_notifications": {"notifications[].title", "notifications[].text"},
"app_clipboard": {"text"},
Expand Down
4 changes: 4 additions & 0 deletions internal/agent/scroll.go
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,10 @@ func (h *Handlers) scrollToOn(ctx context.Context, s *session, args map[string]i
ev := elementView(e)
view.Element = &ev
msg += " — " + e.Line()
} else {
line, ev := noRefElement(node)
view.Element = &ev
msg += " — " + line
}
return Result(msg, view), nil
}
Expand Down
Loading
Loading