From 6ea3a021eb1a89cac9915ad551f5ad655c8f920a Mon Sep 17 00:00:00 2001 From: Lana Begunova <70295997+lana-20@users.noreply.github.com> Date: Mon, 26 Oct 2026 12:25:23 -0700 Subject: [PATCH] The verifier: check, recording and verdict, in every client (mobiumdev/mobium#281) An example for each of the three tools on every surface, the guide copied from Mobium's docs, and why none has a picture: each reads a recording, not a device. --- examples/canonical.json | 48 ++++++++++++++++++++++++++++++++++++++++ examples/dotnet.json | 18 +++++++++++++++ examples/go.json | 18 +++++++++++++++ examples/java.json | 18 +++++++++++++++ examples/javascript.json | 18 +++++++++++++++ examples/no-picture.json | 5 ++++- examples/python.json | 18 +++++++++++++++ scripts/generate.py | 2 +- 8 files changed, 143 insertions(+), 2 deletions(-) diff --git a/examples/canonical.json b/examples/canonical.json index f1dff02..0f71413 100644 --- a/examples/canonical.json +++ b/examples/canonical.json @@ -139,6 +139,17 @@ } } ], + "app_check": [ + { + "title": "Ask a model on this machine whether a claim holds in a recording", + "cli": "mobium check \"After card 3 was tapped, it was shown face up as the Blue Morpho.\" --input card.zip --model openai:qwen2.5:14b", + "args": { + "claim": "After card 3 was tapped, it was shown face up as the Blue Morpho.", + "record": "card.zip", + "model": "openai:qwen2.5:14b" + } + } + ], "app_clear_data": [ { "title": "Delete an app's data, and keep it installed", @@ -577,6 +588,25 @@ } } ], + "app_recording": [ + { + "title": "See what a recording holds", + "cli": "mobium recording summary --input card.zip", + "args": { + "read": "summary", + "record": "card.zip" + } + }, + { + "title": "Find the calls that mention something", + "cli": "mobium recording search \"Blue Morpho\" --input card.zip", + "args": { + "read": "search", + "query": "Blue Morpho", + "record": "card.zip" + } + } + ], "app_reset_permissions": [ { "title": "Put one app's permissions back to their defaults", @@ -849,6 +879,24 @@ } } ], + "app_verdict": [ + { + "title": "Judge a claim yourself, citing the call it rests on", + "cli": "mobium verdict \"After card 3 was tapped, it was shown face up as the Blue Morpho.\" --input card.zip --status passed --summary \"Card 3 reads Blue Morpho after the tap\" --evidence \"call@7: the screen after the tap shows Card 3, Blue Morpho\"", + "args": { + "claim": "After card 3 was tapped, it was shown face up as the Blue Morpho.", + "record": "card.zip", + "status": "passed", + "summary": "Card 3 reads Blue Morpho after the tap", + "evidence": [ + { + "type": "observation", + "summary": "call@7: the screen after the tap shows Card 3, Blue Morpho" + } + ] + } + } + ], "app_wait_for": [ { "title": "Wait for text to appear", diff --git a/examples/dotnet.json b/examples/dotnet.json index 865a99d..bcd5464 100644 --- a/examples/dotnet.json +++ b/examples/dotnet.json @@ -63,6 +63,11 @@ "code": "device.Unset(\"@e4\");" } ], + "app_check": [ + { + "code": "var v = device.Check(\"After card 3 was tapped, it was shown face up as the Blue Morpho.\", \"card.zip\", \"openai:qwen2.5:14b\");\nConsole.WriteLine($\"{v[\"status\"]} {v[\"summary\"]}\");" + } + ], "app_clear_data": [ { "code": "device.ClearData(\"com.example.shop\");" @@ -287,6 +292,14 @@ "code": "var video = device.Record(\"stop\", \"flow.mp4\");" } ], + "app_recording": [ + { + "code": "var sum = device.Recording(\"card.zip\", \"summary\");\nConsole.WriteLine(sum[\"title\"]);" + }, + { + "code": "var hits = device.Recording(\"card.zip\", \"search\", \"Blue Morpho\");\nConsole.WriteLine(hits[\"calls\"]);" + } + ], "app_reset_permissions": [ { "code": "device.ResetPermissions(\"com.example.shop\");" @@ -427,6 +440,11 @@ "code": "var sent = device.PushPath(\"seed\", \"/sdcard/seed\");\nConsole.WriteLine($\"{sent[\"files\"]} {sent[\"checked\"]}\");" } ], + "app_verdict": [ + { + "code": "device.Verdict(\"After card 3 was tapped, it was shown face up as the Blue Morpho.\", \"card.zip\", \"passed\",\n \"Card 3 reads Blue Morpho after the tap\",\n \"call@7: the screen after the tap shows Card 3, Blue Morpho\");" + } + ], "app_wait_for": [ { "code": "var welcome = device.WaitFor(\"text=Welcome back\");" diff --git a/examples/go.json b/examples/go.json index fd3c24f..16b7753 100644 --- a/examples/go.json +++ b/examples/go.json @@ -63,6 +63,11 @@ "code": "if err := dev.Unset(ctx, \"@e4\"); err != nil {\n\treturn err\n}" } ], + "app_check": [ + { + "code": "v, err := dev.Check(ctx, \"After card 3 was tapped, it was shown face up as the Blue Morpho.\", \"card.zip\",\n\t&mobium.CheckOptions{Model: \"openai:qwen2.5:14b\"})\nif err != nil {\n\treturn err\n}\nfmt.Println(v.Status, v.Summary)" + } + ], "app_clear_data": [ { "code": "cleared, err := dev.ClearData(ctx, \"com.example.shop\")\nif err != nil {\n\treturn err\n}\nfmt.Println(\"emptied:\", cleared.Emptied, \"still granted:\", cleared.StillGranted)" @@ -287,6 +292,14 @@ "code": "rec, err := dev.Record(ctx, \"stop\", \"flow.mp4\")\nif err != nil {\n\treturn err\n}\nfmt.Println(rec.Path, rec.Frames, \"frames\")" } ], + "app_recording": [ + { + "code": "sum, err := dev.Recording(ctx, \"card.zip\", \"summary\", nil)\nif err != nil {\n\treturn err\n}\nfmt.Println(sum[\"calls\"], sum[\"title\"])" + }, + { + "code": "hits, err := dev.Recording(ctx, \"card.zip\", \"search\", map[string]any{\"query\": \"Blue Morpho\"})\nif err != nil {\n\treturn err\n}\nfmt.Println(hits[\"calls\"])" + } + ], "app_reset_permissions": [ { "code": "if err := dev.ResetPermissions(ctx, \"com.example.shop\"); err != nil {\n\treturn err\n}" @@ -427,6 +440,11 @@ "code": "sent, err := dev.PushPath(ctx, \"seed\", \"/sdcard/seed\", \"\")\nif err != nil {\n\treturn err\n}\nfmt.Println(sent.Files, sent.Checked)" } ], + "app_verdict": [ + { + "code": "if _, err := dev.Verdict(ctx, \"After card 3 was tapped, it was shown face up as the Blue Morpho.\", \"card.zip\", \"passed\",\n\t\"Card 3 reads Blue Morpho after the tap\",\n\t\"call@7: the screen after the tap shows Card 3, Blue Morpho\"); err != nil {\n\treturn err\n}" + } + ], "app_wait_for": [ { "code": "el, err := dev.WaitFor(ctx, \"text=Welcome back\", nil)\nif err != nil {\n\treturn err\n}\nfmt.Println(el.Ref)" diff --git a/examples/java.json b/examples/java.json index 1a84f83..cffd876 100644 --- a/examples/java.json +++ b/examples/java.json @@ -63,6 +63,11 @@ "code": "device.unset(\"@e4\");" } ], + "app_check": [ + { + "code": "Map v = device.check(\"After card 3 was tapped, it was shown face up as the Blue Morpho.\", \"card.zip\", \"openai:qwen2.5:14b\");\nSystem.out.println(v.get(\"status\") + \" \" + v.get(\"summary\"));" + } + ], "app_clear_data": [ { "code": "Map result = device.clearData(\"com.example.shop\");\nSystem.out.println(result.get(\"emptied\"));" @@ -287,6 +292,14 @@ "code": "Map video = device.record(\"stop\", \"flow.mp4\");\nSystem.out.println(video.get(\"frames\") + \" frames\");" } ], + "app_recording": [ + { + "code": "System.out.println(device.recording(\"card.zip\", \"summary\", null));" + }, + { + "code": "Map hits = device.recording(\"card.zip\", \"search\", \"Blue Morpho\");\nSystem.out.println(hits.get(\"calls\"));" + } + ], "app_reset_permissions": [ { "code": "device.resetPermissions(\"com.example.shop\");" @@ -427,6 +440,11 @@ "code": "Map sent = device.pushPath(\"seed\", \"/sdcard/seed\");\nSystem.out.println(sent.get(\"files\") + \" \" + sent.get(\"checked\"));" } ], + "app_verdict": [ + { + "code": "device.verdict(\"After card 3 was tapped, it was shown face up as the Blue Morpho.\", \"card.zip\", \"passed\",\n \"Card 3 reads Blue Morpho after the tap\",\n \"call@7: the screen after the tap shows Card 3, Blue Morpho\");" + } + ], "app_wait_for": [ { "code": "Element welcome = device.waitFor(\"text=Welcome back\");\ndevice.tap(welcome.ref());" diff --git a/examples/javascript.json b/examples/javascript.json index 550bea0..a1a6e30 100644 --- a/examples/javascript.json +++ b/examples/javascript.json @@ -63,6 +63,11 @@ "code": "await device.unset('@e4')" } ], + "app_check": [ + { + "code": "const v = await device.check(\"After card 3 was tapped, it was shown face up as the Blue Morpho.\", 'card.zip', { model: 'openai:qwen2.5:14b' })\nconsole.log(v.status, v.summary)" + } + ], "app_clear_data": [ { "code": "const cleared = await device.clearData('com.example.shop')\nconsole.log(cleared.emptied, cleared.kept)" @@ -287,6 +292,14 @@ "code": "const video = await device.record({ action: 'stop', path: 'flow.mp4' })\nconsole.log(video.frames, video.duration)" } ], + "app_recording": [ + { + "code": "console.log(await device.recording('card.zip', 'summary'))" + }, + { + "code": "const hits = await device.recording('card.zip', 'search', { query: 'Blue Morpho' })\nconsole.log(hits.calls)" + } + ], "app_reset_permissions": [ { "code": "await device.resetPermissions('com.example.shop')" @@ -427,6 +440,11 @@ "code": "const sent = await device.pushPath('seed', '/sdcard/seed')\nconsole.log(sent.files, sent.checked)" } ], + "app_verdict": [ + { + "code": "await device.verdict(\"After card 3 was tapped, it was shown face up as the Blue Morpho.\", 'card.zip', 'passed',\n 'Card 3 reads Blue Morpho after the tap',\n [\"call@7: the screen after the tap shows Card 3, Blue Morpho\"])" + } + ], "app_wait_for": [ { "code": "const welcome = await device.waitFor('text=Welcome back')\nconsole.log(welcome?.ref)" diff --git a/examples/no-picture.json b/examples/no-picture.json index 827d9f2..dde3e61 100644 --- a/examples/no-picture.json +++ b/examples/no-picture.json @@ -5,6 +5,7 @@ "background": "the app is off the screen for the seconds it is away, and back in front when the command returns.", "biometric": "what it presents goes to a prompt the app put up, and the app's own screen says what it made of it.", "boot": "it starts a device, which has no screen of its own until it is up.", + "check": "it reads a recording, not a device, and needs no screen at all.", "clear-data": "it deletes an app's data, which shows only the next time the app starts.", "crashes": "it reads the device's crash reports, and nothing on the screen changes.", "current": "it names the app in front, and nothing on the screen changes.", @@ -21,6 +22,7 @@ "lock": "a locked emulator's screen is off, so its screenshot is black.", "logs": "it reads a log, and nothing on the screen changes.", "record": "what it keeps is a video of the screen.", + "recording": "it reads a recording, not a device, and needs no screen at all.", "reset-permissions": "it changes what an app may do, not what is on the screen.", "revoke": "it changes what an app may do, not what is on the screen.", "session": "a session is the connection to the device, and the screen shows nothing of it.", @@ -30,5 +32,6 @@ "state": "it reads whether an app is running, and nothing on the screen changes.", "time": "it reads the device's clock, and nothing on the screen changes.", "trace": "what it keeps is a zip in Vibium's record format, which opens in Vibium's player at player.vibium.dev: every step, and the screen after each with the map drawn over it.", - "uninstall": "it removes an app, which shows nowhere on the screen in front." + "uninstall": "it removes an app, which shows nowhere on the screen in front.", + "verdict": "it judges a recording, not a device, and needs no screen at all." } diff --git a/examples/python.json b/examples/python.json index 9afb279..7599b73 100644 --- a/examples/python.json +++ b/examples/python.json @@ -63,6 +63,11 @@ "code": "device.unset(\"@e4\")" } ], + "app_check": [ + { + "code": "v = device.check(\"After card 3 was tapped, it was shown face up as the Blue Morpho.\", \"card.zip\", model=\"openai:qwen2.5:14b\")\nprint(v[\"status\"], v[\"summary\"])" + } + ], "app_clear_data": [ { "code": "result = device.clear_data(\"com.example.shop\")\nprint(result[\"emptied\"])" @@ -287,6 +292,14 @@ "code": "video = device.record(\"stop\", path=\"flow.mp4\")\nprint(video[\"frames\"], video[\"duration\"])" } ], + "app_recording": [ + { + "code": "print(device.recording(\"card.zip\", \"summary\"))" + }, + { + "code": "hits = device.recording(\"card.zip\", \"search\", query=\"Blue Morpho\")\nfor c in hits[\"calls\"]:\n print(c[\"id\"], c[\"title\"])" + } + ], "app_reset_permissions": [ { "code": "device.reset_permissions(\"com.example.shop\")" @@ -427,6 +440,11 @@ "code": "sent = device.push_path(\"seed\", \"/sdcard/seed\")\nprint(sent[\"files\"], sent[\"checked\"])" } ], + "app_verdict": [ + { + "code": "device.verdict(\"After card 3 was tapped, it was shown face up as the Blue Morpho.\", \"card.zip\", \"passed\",\n \"Card 3 reads Blue Morpho after the tap\",\n \"call@7: the screen after the tap shows Card 3, Blue Morpho\")" + } + ], "app_wait_for": [ { "code": "el = device.wait_for(\"text=Welcome back\")\ndevice.tap(el.ref)" diff --git a/scripts/generate.py b/scripts/generate.py index 870a23e..59f17f6 100644 --- a/scripts/generate.py +++ b/scripts/generate.py @@ -536,7 +536,7 @@ def doc_text(doc, lang): SYNC = { "quickstart": ["README.md", "cli.md", "mcp.md", "python.md", "javascript.md", "go.md", "java.md", "dotnet.md"], "guides": ["README.md", "autowait.md", "graybox.md", "graybox-tutorial.md", "cli.md", "test-runner.md", "grid.md", - "network.md", "inspector.md", "mcp.md"], + "network.md", "inspector.md", "mcp.md", "verifier.md"], }