diff --git a/data/runs.json b/data/runs.json index 9c76a00..47f0eb9 100644 --- a/data/runs.json +++ b/data/runs.json @@ -768,5 +768,14 @@ "exit": 0, "output": "pressed \"DON'T SAVE\" on the dialog \"Save this draft?\\nYou can come back to it later.\"", "after": "/img/shots/app_alert-2-after.jpg" + }, + "app_assistant[0]": { + "tool": "app_assistant", + "index": 0, + "command": "mobium assistant \"Open Clock\" --app com.google.android.deskclock", + "verbatim": true, + "exit": 0, + "output": "asked Google Assistant (typed, not spoken): \"Open Clock\"\nit showed no text a screen read can see\nin front: com.google.android.deskclock\ncom.google.android.deskclock came forward", + "after": "/img/shots/app_assistant-0-after.jpg" } } diff --git a/examples/canonical.json b/examples/canonical.json index cbc4a66..37364c6 100644 --- a/examples/canonical.json +++ b/examples/canonical.json @@ -899,6 +899,24 @@ "args": {} } ], + "app_assistant": [ + { + "title": "Ask the assistant to open an app, and check that it came forward", + "cli": "mobium assistant \"Open Clock\" --app com.google.android.deskclock", + "args": { + "request": "Open Clock", + "app": "com.google.android.deskclock" + } + }, + { + "title": "Ask a question, and read the answer it showed", + "cli": "mobium assistant \"What is 15 times 3\" --timeout 20", + "args": { + "request": "What is 15 times 3", + "timeout": 20 + } + } + ], "app_hit_test": [ { "title": "Ask where a tap on an element would really land", diff --git a/examples/dotnet.json b/examples/dotnet.json index 479a8af..1981263 100644 --- a/examples/dotnet.json +++ b/examples/dotnet.json @@ -451,6 +451,14 @@ "code": "var found = device.Audit();\nConsole.WriteLine(found[\"findings\"]);" } ], + "app_assistant": [ + { + "code": "var got = device.Assistant(\"Open Clock\", app: \"com.google.android.deskclock\");\nif (!(got[\"reached\"] is bool reached && reached)) throw new Exception($\"in front: {got[\"front\"]}\");" + }, + { + "code": "var got = device.Assistant(\"What is 15 times 3\", timeoutSeconds: 20);\nConsole.WriteLine(got[\"answer\"]);" + } + ], "app_hit_test": [ { "code": "var hit = device.HitTest(\"testid=submit\");\nConsole.WriteLine(hit[\"reaches\"]);" diff --git a/examples/go.json b/examples/go.json index 346ee9f..b67ddb0 100644 --- a/examples/go.json +++ b/examples/go.json @@ -451,6 +451,14 @@ "code": "found, err := dev.Audit(ctx)\nif err != nil {\n\treturn err\n}\nfor _, f := range found.Findings {\n\tfmt.Println(f.Type, f.Summary)\n}" } ], + "app_assistant": [ + { + "code": "got, err := dev.Assistant(ctx, \"Open Clock\", mobium.AssistantOptions{App: \"com.google.android.deskclock\"})\nif err != nil {\n\treturn err\n}\nif got.Reached == nil || !*got.Reached {\n\treturn fmt.Errorf(\"Clock did not come forward: %s is in front\", got.Front)\n}" + }, + { + "code": "got, err := dev.Assistant(ctx, \"What is 15 times 3\", mobium.AssistantOptions{Timeout: 20 * time.Second})\nif err != nil {\n\treturn err\n}\nfmt.Println(got.Answer)" + } + ], "app_hit_test": [ { "code": "hit, err := dev.HitTest(ctx, \"testid=submit\")\nif err != nil {\n\treturn err\n}\nfmt.Println(hit.Reaches)" diff --git a/examples/java.json b/examples/java.json index b13ac9e..7a5474a 100644 --- a/examples/java.json +++ b/examples/java.json @@ -451,6 +451,14 @@ "code": "Map found = device.audit();\nSystem.out.println(found.get(\"findings\"));" } ], + "app_assistant": [ + { + "code": "Map got = device.assistant(\"Open Clock\", \"com.google.android.deskclock\", 0);\nif (!Boolean.TRUE.equals(got.get(\"reached\"))) throw new AssertionError(\"in front: \" + got.get(\"front\"));" + }, + { + "code": "Map got = device.assistant(\"What is 15 times 3\", null, 20);\nSystem.out.println(got.get(\"answer\"));" + } + ], "app_hit_test": [ { "code": "Map hit = device.hitTest(\"testid=submit\");\nSystem.out.println(hit.get(\"reaches\"));" diff --git a/examples/javascript.json b/examples/javascript.json index 8276f36..5b55590 100644 --- a/examples/javascript.json +++ b/examples/javascript.json @@ -451,6 +451,14 @@ "code": "const found = await device.audit()\nconsole.log(found.findings)" } ], + "app_assistant": [ + { + "code": "const got = await device.assistant('Open Clock', { app: 'com.google.android.deskclock' })\nif (!got.reached) throw new Error(`in front: ${got.front}`)" + }, + { + "code": "const got = await device.assistant('What is 15 times 3', { timeout: 20 })\nconsole.log(got.answer)" + } + ], "app_hit_test": [ { "code": "const hit = await device.hitTest('testid=submit')\nconsole.log(hit.reaches)" diff --git a/examples/python.json b/examples/python.json index 65586ad..891a8f7 100644 --- a/examples/python.json +++ b/examples/python.json @@ -451,6 +451,14 @@ "code": "found = device.audit()\nfor f in found[\"findings\"]:\n print(f[\"type\"], f[\"summary\"])" } ], + "app_assistant": [ + { + "code": "got = device.assistant(\"Open Clock\", app=\"com.google.android.deskclock\")\nassert got[\"reached\"], got[\"front\"]" + }, + { + "code": "got = device.assistant(\"What is 15 times 3\", timeout=20)\nprint(got[\"answer\"])" + } + ], "app_hit_test": [ { "code": "hit = device.hit_test(\"testid=submit\")\nprint(hit[\"reaches\"])" diff --git a/scripts/capture.py b/scripts/capture.py index da0d041..abc3d23 100644 --- a/scripts/capture.py +++ b/scripts/capture.py @@ -153,6 +153,10 @@ def app_screen(name): # played for two seconds, so the stop's expect has what it asks for. dict(tool="app_audio", i=0, setup=app_screen("Audio Demo") + ["wait testid=audioState"]), dict(tool="app_audio", i=1, setup=["tap testid=audioTone", "$sleep 3"]), + # The assistant, asked in typed text from the home screen: Clock is + # terminated first, so the picture shows the assistant bringing it forward. + dict(tool="app_assistant", i=0, setup=["terminate com.google.android.deskclock", "press home"], after=True, + pause=2.0, restore=["terminate com.google.android.deskclock", "press home"]), dict(tool="app_list_apps", i=0), dict(tool="app_contexts", i=0, setup=app_screen("WebViews") + ["tap 'text=Plain page'", "wait text=Back"], after=True), diff --git a/static/img/shots/app_assistant-0-after.jpg b/static/img/shots/app_assistant-0-after.jpg new file mode 100644 index 0000000..cc22327 Binary files /dev/null and b/static/img/shots/app_assistant-0-after.jpg differ