diff --git a/data/runs.json b/data/runs.json index 47f0eb9..7f146dc 100644 --- a/data/runs.json +++ b/data/runs.json @@ -777,5 +777,30 @@ "exit": 0, "output": "asked Google Assistant (typed, not spoken): \"Open Clock\"\nit showed no text a screen read can see\nin front: com.google.android.deskclock\ncom.google.android.deskclock came forward", "after": "/img/shots/app_assistant-0-after.jpg" + }, + "app_speech[2]": { + "tool": "app_speech", + "index": 2, + "command": "mobium speech label=Pro", + "verbatim": true, + "exit": 0, + "output": "TalkBack says: \"not checked. Pro. Radio button\" — Pro (radio)", + "after": "/img/shots/app_speech-2-after.jpg" + }, + "app_speech[0]": { + "tool": "app_speech", + "index": 0, + "command": "mobium speech", + "verbatim": true, + "exit": 0, + "output": "TalkBack says: \"not checked. Pro. Radio button\" — Pro (radio)" + }, + "app_speech[1]": { + "tool": "app_speech", + "index": 1, + "command": "mobium speech --walk --limit 20", + "verbatim": true, + "exit": 0, + "output": "TalkBack says:\n 1. \"not checked. Team. Radio button\" — Team (radio)\n 2. \"OFF. Dark mode. Switch\" — Dark mode (switch)\n 3. \"Dark mode, for the whole app\" — Dark mode, for the whole app, not in map\n 4. \"Read-only\" — Read-only, not in map\n 5. \"This field cannot be edited. Edit box. disabled\"\n 6. \"notify=false terms=false plan=free theme=light\" — notify=false terms=false plan=free theme=light, not in map\n 7. (nothing)\n 8. \"Back. Button\" — Back (button)\n 9. \"Form\" — Form, not in map\n 10. \"not checked. Email me. Check box\" — Email me (checkbox)\n 11. \"not checked. Accept terms. Check box\" — Accept terms (checkbox)\n 12. \"checked. Free. Radio button\" — Free (radio)\n 13. \"not checked. Pro. Radio button\" — Pro (radio)\nended: the cursor came back round to the first element" } } diff --git a/examples/canonical.json b/examples/canonical.json index 37364c6..72ff483 100644 --- a/examples/canonical.json +++ b/examples/canonical.json @@ -917,6 +917,28 @@ } } ], + "app_speech": [ + { + "title": "Say what the screen reader says about the element its cursor is on", + "cli": "mobium speech", + "args": {} + }, + { + "title": "Walk the screen in the screen reader's order, each sentence beside its element", + "cli": "mobium speech --walk --limit 20", + "args": { + "walk": true, + "limit": 20 + } + }, + { + "title": "Put the screen reader's cursor on an element, and hear what it says", + "cli": "mobium speech label=Pro", + "args": { + "target": "label=Pro" + } + } + ], "app_hit_test": [ { "title": "Ask where a tap on an element would really land", diff --git a/examples/dotnet.json b/examples/dotnet.json index 1981263..1a7bc3a 100644 --- a/examples/dotnet.json +++ b/examples/dotnet.json @@ -459,6 +459,17 @@ "code": "var got = device.Assistant(\"What is 15 times 3\", timeoutSeconds: 20);\nConsole.WriteLine(got[\"answer\"]);" } ], + "app_speech": [ + { + "code": "var said = device.Speech();\nConsole.WriteLine(said[\"steps\"]);" + }, + { + "code": "var walk = device.Speech(walk: true, limit: 20);\nConsole.WriteLine($\"{walk[\"steps\"]} ended {walk[\"ended\"]}\");" + }, + { + "code": "var said = device.Speech(\"label=Pro\");\nConsole.WriteLine(said[\"steps\"]);" + } + ], "app_hit_test": [ { "code": "var hit = device.HitTest(\"testid=submit\");\nConsole.WriteLine(hit[\"reaches\"]);" diff --git a/examples/go.json b/examples/go.json index b67ddb0..be6527d 100644 --- a/examples/go.json +++ b/examples/go.json @@ -459,6 +459,17 @@ "code": "got, err := dev.Assistant(ctx, \"What is 15 times 3\", mobium.AssistantOptions{Timeout: 20 * time.Second})\nif err != nil {\n\treturn err\n}\nfmt.Println(got.Answer)" } ], + "app_speech": [ + { + "code": "said, err := dev.Speech(ctx, mobium.SpeechOptions{})\nif err != nil {\n\treturn err\n}\nfmt.Println(said.Steps[0].Said)" + }, + { + "code": "walk, err := dev.Speech(ctx, mobium.SpeechOptions{Walk: true, Limit: 20})\nif err != nil {\n\treturn err\n}\nfor _, step := range walk.Steps {\n\tfmt.Println(step.Said)\n}\nfmt.Println(\"ended:\", walk.Ended)" + }, + { + "code": "said, err := dev.Speech(ctx, mobium.SpeechOptions{Target: \"label=Pro\"})\nif err != nil {\n\treturn err\n}\nfmt.Println(said.Steps[0].Said)" + } + ], "app_hit_test": [ { "code": "hit, err := dev.HitTest(ctx, \"testid=submit\")\nif err != nil {\n\treturn err\n}\nfmt.Println(hit.Reaches)" diff --git a/examples/java.json b/examples/java.json index 7a5474a..0952461 100644 --- a/examples/java.json +++ b/examples/java.json @@ -459,6 +459,17 @@ "code": "Map got = device.assistant(\"What is 15 times 3\", null, 20);\nSystem.out.println(got.get(\"answer\"));" } ], + "app_speech": [ + { + "code": "Map said = device.speech(null, false, 0);\nSystem.out.println(said.get(\"steps\"));" + }, + { + "code": "Map walk = device.speech(null, true, 20);\nSystem.out.println(walk.get(\"steps\") + \" ended \" + walk.get(\"ended\"));" + }, + { + "code": "Map said = device.speech(\"label=Pro\", false, 0);\nSystem.out.println(said.get(\"steps\"));" + } + ], "app_hit_test": [ { "code": "Map hit = device.hitTest(\"testid=submit\");\nSystem.out.println(hit.get(\"reaches\"));" diff --git a/examples/javascript.json b/examples/javascript.json index 5b55590..5ac165f 100644 --- a/examples/javascript.json +++ b/examples/javascript.json @@ -459,6 +459,17 @@ "code": "const got = await device.assistant('What is 15 times 3', { timeout: 20 })\nconsole.log(got.answer)" } ], + "app_speech": [ + { + "code": "const said = await device.speech()\nconsole.log(said.steps)" + }, + { + "code": "const walk = await device.speech({ walk: true, limit: 20 })\nconsole.log(walk.steps, walk.ended)" + }, + { + "code": "const said = await device.speech({ target: 'label=Pro' })\nconsole.log(said.steps)" + } + ], "app_hit_test": [ { "code": "const hit = await device.hitTest('testid=submit')\nconsole.log(hit.reaches)" diff --git a/examples/python.json b/examples/python.json index 891a8f7..64c42df 100644 --- a/examples/python.json +++ b/examples/python.json @@ -459,6 +459,17 @@ "code": "got = device.assistant(\"What is 15 times 3\", timeout=20)\nprint(got[\"answer\"])" } ], + "app_speech": [ + { + "code": "said = device.speech()\nprint(said[\"steps\"][0][\"said\"])" + }, + { + "code": "walk = device.speech(walk=True, limit=20)\nfor step in walk[\"steps\"]:\n print(step[\"said\"], step.get(\"element\", {}).get(\"label\"))" + }, + { + "code": "said = device.speech(\"label=Pro\")\nprint(said[\"steps\"])" + } + ], "app_hit_test": [ { "code": "hit = device.hit_test(\"testid=submit\")\nprint(hit[\"reaches\"])" diff --git a/scripts/capture.py b/scripts/capture.py index abc3d23..129f437 100644 --- a/scripts/capture.py +++ b/scripts/capture.py @@ -157,6 +157,15 @@ def app_screen(name): # terminated first, so the picture shows the assistant bringing it forward. dict(tool="app_assistant", i=0, setup=["terminate com.google.android.deskclock", "press home"], after=True, pause=2.0, restore=["terminate com.google.android.deskclock", "press home"]), + # The screen reader on the Form Demo, TalkBack turned on after the screen + # is up. The cursor is put on Pro first, so the picture shows TalkBack's + # green rectangle there and the plain read after it says Pro; the walk + # goes last, and TalkBack goes off. Needs TalkBack's log output level at + # VERBOSE, which is TalkBack's own setting: set it there before, and back. + dict(tool="app_speech", i=2, setup=app_screen("Form Demo") + ["wait label=Pro", "accessibility screen_reader on", + "$sleep 3"], after=True, pause=1.0), + dict(tool="app_speech", i=0), + dict(tool="app_speech", i=1, restore=["accessibility screen_reader off"]), dict(tool="app_list_apps", i=0), dict(tool="app_contexts", i=0, setup=app_screen("WebViews") + ["tap 'text=Plain page'", "wait text=Back"], after=True), diff --git a/static/img/shots/app_speech-2-after.jpg b/static/img/shots/app_speech-2-after.jpg new file mode 100644 index 0000000..d8639f3 Binary files /dev/null and b/static/img/shots/app_speech-2-after.jpg differ