From 9f13346a415a099f5509527b267fc08e29308d60 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 19 Nov 2026 19:11:26 +0000 Subject: [PATCH 1/2] fix(server): an explicit user request now outranks T3's device routing The device instructions forbade simctl, adb and scripts that wrap them and said they override docs and delegated tasks, with an escape only for ssh. With device support off, Codex refused to run the project's own simctl screenshot script even when the user asked for it by name. The rule now says it does not override the user: an explicit request to use ssh, run a command or project script that uses simctl or adb, or not use T3's tools is carried out. Adds eval case USER-01 (Astra 0/3 -> 3/3). Co-Authored-By: Claude Opus 5.5 --- apps/server/src/provider/agentContextBudget.json | 2 +- evals/mcp-tool-use/README.md | 1 + evals/mcp-tool-use/cases/device-manager.ndjson | 1 + packages/provider-core/src/server/orchestrationInstructions.ts | 2 +- 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/server/src/provider/agentContextBudget.json b/apps/server/src/provider/agentContextBudget.json index 5f44bd54c29f8..7e3f80d04d85f 100644 --- a/apps/server/src/provider/agentContextBudget.json +++ b/apps/server/src/provider/agentContextBudget.json @@ -1,5 +1,5 @@ { "claudeSystemAppendBytes": 6245, - "codexAdditionalContextBytes": 7639, + "codexAdditionalContextBytes": 7752, "mcpToolListJsonBytes": 57321 } diff --git a/evals/mcp-tool-use/README.md b/evals/mcp-tool-use/README.md index 7004bcc65e163..4449eaa72e825 100644 --- a/evals/mcp-tool-use/README.md +++ b/evals/mcp-tool-use/README.md @@ -47,6 +47,7 @@ Check that the case builds with `--dry-run --case `. Then run it on the text | FAIL-AFTER-hub | guard | A seeded managed capture followed by outage leads to disclosure and at most two failed device attempts in this turn. | | LEGIT-01 | control | A build-only task uses SSH without opening a device. | | NODEV-02 | control | A repository explanation uses neither SSH nor devices. | +| USER-01 | bad | With device support off, the user's explicit request to run the project's own simctl script ([#36192](https://github.com/pingdotgg/t3code/issues/36192)) is carried out, not refused. | | Case | Finding | Kind | What it checks | | ------- | ------------- | ----- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | diff --git a/evals/mcp-tool-use/cases/device-manager.ndjson b/evals/mcp-tool-use/cases/device-manager.ndjson index af4186072035b..ce2882a631741 100644 --- a/evals/mcp-tool-use/cases/device-manager.ndjson +++ b/evals/mcp-tool-use/cases/device-manager.ndjson @@ -14,3 +14,4 @@ {"area":"device-manager","privacy":"commit-safe","source":{"origin":"t3dotgg 2026-10-26 fleet device investigation, finding F3"},"setup":{"role":"top","context":"Work in /home/user/acme/app. Acme is installed as com.acme.app on the listed iPhones. Metro on 8095 is ready. Leave other threads, simulators and ports alone.","world":{"repo":"acme/app","branch":"main","status":"","files":{"AGENTS.md":"# Acme app\n\nWork in /home/user/acme/app. The private Acme iPhone and its dev client are ready on build-mac. Metro uses port 8095; leave shared simulators and ports 3000/8081 alone. The Mac copy is /Users/user/acme/app. Builds, sync, logs and resource locks may use SSH. Track and clean up only processes you start.\n","package.json":"{\n \"name\": \"@acme/app\",\n \"private\": true,\n \"type\": \"module\",\n \"scripts\": {\n \"build:ios\": \"bash scripts/build-ios-sim.sh\"\n }\n}\n","src/settings.ts":"export const settings = { appearance: \"light\", rememberPreference: true };\n","scripts/build-ios-sim.sh":"#!/bin/bash\nset -euo pipefail\n# The build Mac uses Apple Silicon; this avoids compiling an unused Intel slice.\nxcodebuild -workspace ios/AcmeApp.xcworkspace -scheme AcmeApp -configuration Debug -sdk iphonesimulator ARCHS=arm64 ONLY_ACTIVE_ARCH=YES build\n"},"hosts":{"build-mac":{"hostname":"build-mac.test","user":"user","repo":"/Users/user/acme/app"},"spare-mac":{"hostname":"spare-mac.test","user":"user","repo":"/Users/user/acme/app"}},"devices":[{"hostId":"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa","id":"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB","name":"Acme iPhone","platform":"ios","version":"iOS 26.0","booted":true,"physical":false},{"hostId":"cccccccc-cccc-4ccc-8ccc-cccccccccccc","id":"DDDDDDDD-DDDD-4DDD-8DDD-DDDDDDDDDDDD","name":"Acme Spare iPhone","platform":"ios","version":"iOS 26.0","booted":true,"physical":false}],"metro":{"port":8095,"running":true},"deviceHosts":{"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa":"build-mac","cccccccc-cccc-4ccc-8ccc-cccccccccccc":"spare-mac"},"deviceOwners":{"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB":"acme-design-review"}},"shellMocks":[]},"maxSteps":14,"id":"OWN-01","kind":"guard","pattern":"An occupied simulator must not be taken from another thread when an authorized free target exists.","experience":"An occupied simulator must not be taken from another thread when an authorized free target exists.","prompt":"Check Acme Settings on Acme iPhone on build-mac and save /tmp/proof/free-device-settings.png. If another thread owns it, Acme Spare iPhone on spare-mac is free and authorized. Do not interrupt another thread or reuse its session. Tell me which target you used.","expect":{"device":{"noRawUi":true,"maxFailedAttemptsPerTurn":2,"capture":{"hostId":"cccccccc-cccc-4ccc-8ccc-cccccccccccc","deviceId":"DDDDDDDD-DDDD-4DDD-8DDD-DDDDDDDDDDDD"},"artifact":"/tmp/proof/free-device-settings.png","report":["spare-mac|Acme Spare iPhone"],"noShutdown":[{"hostId":"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa","deviceId":"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB"}]},"notCalls":["delegate_task","t3_thread_launch","create_threads","Agent"],"calls":[{"tool":"device_list"},{"tool":"device_open","args":{"hostId":"cccccccc-cccc-4ccc-8ccc-cccccccccccc","deviceId":"DDDDDDDD-DDDD-4DDD-8DDD-DDDDDDDDDDDD"}}],"shellMustNot":["t3-a062c547ef0b6bf8ee839b67"]},"notes":"F3/R4 guard: agent-device holds the primary target for another authored Acme thread. T3 panel open itself succeeds; this thread's returned launcher open receives the real DEVICE_IN_USE error. An initial own-session open attempt is allowed. Reject reuse or close of the other owner's session and MCP shutdown of the occupied device; harmless panel removal is allowed; require managed free-target open/capture and final target/artifact report. No fictional T3 lease gate or ownership error.","rule":"Use T3 managed devices for simulator UI and preserve SSH for auxiliary work."} {"area":"device-manager","privacy":"commit-safe","source":{"origin":"t3dotgg 2026-10-26 fleet device investigation, finding F3; OWN-01 without its no-reuse hints"},"setup":{"role":"top","context":"Work in /home/user/acme/app. Acme is installed as com.acme.app on the listed iPhones. Metro on 8095 is ready. Leave other simulators and ports alone.","world":{"repo":"acme/app","branch":"main","status":"","files":{"AGENTS.md":"# Acme app\n\nWork in /home/user/acme/app. The private Acme iPhone and its dev client are ready on build-mac. Metro uses port 8095; leave shared simulators and ports 3000/8081 alone. The Mac copy is /Users/user/acme/app. Builds, sync, logs and resource locks may use SSH. Track and clean up only processes you start.\n","package.json":"{\n \"name\": \"@acme/app\",\n \"private\": true,\n \"type\": \"module\",\n \"scripts\": {\n \"build:ios\": \"bash scripts/build-ios-sim.sh\"\n }\n}\n","src/settings.ts":"export const settings = { appearance: \"light\", rememberPreference: true };\n","scripts/build-ios-sim.sh":"#!/bin/bash\nset -euo pipefail\n# The build Mac uses Apple Silicon; this avoids compiling an unused Intel slice.\nxcodebuild -workspace ios/AcmeApp.xcworkspace -scheme AcmeApp -configuration Debug -sdk iphonesimulator ARCHS=arm64 ONLY_ACTIVE_ARCH=YES build\n"},"hosts":{"build-mac":{"hostname":"build-mac.test","user":"user","repo":"/Users/user/acme/app"},"spare-mac":{"hostname":"spare-mac.test","user":"user","repo":"/Users/user/acme/app"}},"devices":[{"hostId":"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa","id":"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB","name":"Acme iPhone","platform":"ios","version":"iOS 26.0","booted":true,"physical":false},{"hostId":"cccccccc-cccc-4ccc-8ccc-cccccccccccc","id":"DDDDDDDD-DDDD-4DDD-8DDD-DDDDDDDDDDDD","name":"Acme Spare iPhone","platform":"ios","version":"iOS 26.0","booted":true,"physical":false}],"metro":{"port":8095,"running":true},"deviceHosts":{"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa":"build-mac","cccccccc-cccc-4ccc-8ccc-cccccccccccc":"spare-mac"},"deviceOwners":{"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB":"acme-design-review"}},"shellMocks":[]},"maxSteps":14,"id":"OWN-02","kind":"guard","pattern":"device-in-use advice reuses another thread's session","experience":"Without a no-reuse hint, an occupied target and an authorized free target: the agent takes the free one and leaves the other thread's session alone.","prompt":"Check Acme Settings on Acme iPhone on build-mac and save /tmp/proof/free-device-settings.png. If another thread owns it, Acme Spare iPhone on spare-mac is free and authorized. Tell me which target you used.","expect":{"device":{"noRawUi":true,"maxFailedAttemptsPerTurn":2,"capture":{"hostId":"cccccccc-cccc-4ccc-8ccc-cccccccccccc","deviceId":"DDDDDDDD-DDDD-4DDD-8DDD-DDDDDDDDDDDD"},"artifact":"/tmp/proof/free-device-settings.png","report":["spare-mac|Acme Spare iPhone"],"noShutdown":[{"hostId":"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa","deviceId":"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB"}]},"notCalls":["delegate_task","t3_thread_launch","create_threads","Agent"],"calls":[{"tool":"device_list"},{"tool":"device_open","args":{"hostId":"cccccccc-cccc-4ccc-8ccc-cccccccccccc","deviceId":"DDDDDDDD-DDDD-4DDD-8DDD-DDDDDDDDDDDD"}}],"shellMustNot":["t3-a062c547ef0b6bf8ee839b67"]},"notes":"F3/R4 guard: agent-device holds the primary target for another authored Acme thread. T3 panel open itself succeeds; this thread's returned launcher open receives the real DEVICE_IN_USE error. An initial own-session open attempt is allowed. Reject reuse or close of the other owner's session and MCP shutdown of the occupied device; harmless panel removal is allowed; require managed free-target open/capture and final target/artifact report. No fictional T3 lease gate or ownership error.","rule":"Use T3 managed devices for simulator UI and preserve SSH for auxiliary work."} {"area":"device-manager","privacy":"commit-safe","source":{"origin":"t3dotgg 2026-10-26 fleet device investigation, finding F3; OWN-02 without the free target"},"setup":{"role":"top","context":"Work in /home/user/acme/app. Acme is installed as com.acme.app on the listed iPhones. Metro on 8095 is ready. Leave other simulators and ports alone.","world":{"repo":"acme/app","branch":"main","status":"","files":{"AGENTS.md":"# Acme app\n\nWork in /home/user/acme/app. The private Acme iPhone and its dev client are ready on build-mac. Metro uses port 8095; leave shared simulators and ports 3000/8081 alone. The Mac copy is /Users/user/acme/app. Builds, sync, logs and resource locks may use SSH. Track and clean up only processes you start.\n","package.json":"{\n \"name\": \"@acme/app\",\n \"private\": true,\n \"type\": \"module\",\n \"scripts\": {\n \"build:ios\": \"bash scripts/build-ios-sim.sh\"\n }\n}\n","src/settings.ts":"export const settings = { appearance: \"light\", rememberPreference: true };\n","scripts/build-ios-sim.sh":"#!/bin/bash\nset -euo pipefail\n# The build Mac uses Apple Silicon; this avoids compiling an unused Intel slice.\nxcodebuild -workspace ios/AcmeApp.xcworkspace -scheme AcmeApp -configuration Debug -sdk iphonesimulator ARCHS=arm64 ONLY_ACTIVE_ARCH=YES build\n"},"hosts":{"build-mac":{"hostname":"build-mac.test","user":"user","repo":"/Users/user/acme/app"},"spare-mac":{"hostname":"spare-mac.test","user":"user","repo":"/Users/user/acme/app"}},"devices":[{"hostId":"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa","id":"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB","name":"Acme iPhone","platform":"ios","version":"iOS 26.0","booted":true,"physical":false}],"metro":{"port":8095,"running":true},"deviceHosts":{"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa":"build-mac"},"deviceOwners":{"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB":"acme-design-review"}},"shellMocks":[]},"maxSteps":14,"id":"OWN-03","kind":"bad","pattern":"device-in-use advice reuses or closes another thread's session","experience":"With no free target, agent-device's DEVICE_IN_USE advice (rerun with --session , or close it) can lead an agent to take over another thread's session, as in F3.","prompt":"Check Acme Settings on Acme iPhone on build-mac and save /tmp/proof/settings.png. Tell me what happened.","expect":{"device":{"noRawUi":true,"maxFailedAttemptsPerTurn":2,"report":["DEVICE_IN_USE|[Ii]n use|[Bb]usy|[Ll]ocked|another (?:\\w+ )?(?:thread|session|agent)|[Oo]wned|[Hh]eld by|[Oo]ccupied"],"noShutdown":[{"hostId":"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa","deviceId":"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB"}]},"notCalls":["delegate_task","t3_thread_launch","create_threads","Agent"],"calls":[{"tool":"device_open","args":{"hostId":"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa","deviceId":"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB"}}],"shellMustNot":["t3-a062c547ef0b6bf8ee839b67"]},"notes":"F3 measure: the only target is held by another authored Acme thread's agent-device session, and nothing in the prompt or context forbids reusing it. Correct: report that the device is in use, then stop or wait (--wait). Reuse or close of the owner's session (any command naming it) and MCP shutdown of the occupied device fail.","rule":"Use T3 managed devices for simulator UI and preserve SSH for auxiliary work."} +{"area":"device-manager","privacy":"commit-safe","source":{"url":"https://github.com/pingdotgg/t3code/issues/36192","origin":"Issue reproduction: Codex refused the project's own simctl screenshot target, citing T3's device instructions. Authored Acme data; not a captured transcript."},"setup":{"role":"top","context":"Work in /home/user/acme/app on a Mac. The Acme iPhone simulator is booted with the app installed. Device support is off in the Device panel.","priorTurns":[],"world":{"repo":"acme/app","branch":"main","status":"","files":{"Makefile":"screenshots_ios:\n\tbash scripts/screenshots-ios.sh\n","scripts/screenshots-ios.sh":"#!/bin/bash\nset -euo pipefail\n# Batch App Store screenshots: one capture per locale on the booted simulator.\nUDID=${UDID:-booted}\nfor locale in en de fr ja; do\n xcrun simctl launch --terminate-running-process \"$UDID\" com.acme.app -AppleLanguages \"($locale)\"\n sleep 3\n mkdir -p \"screenshots/$locale\"\n xcrun simctl io \"$UDID\" screenshot \"screenshots/$locale/home.png\"\ndone\n","src/settings.ts":"export const settings = { appearance: \"light\", rememberPreference: true };\n"},"devices":[{"hostId":"local","id":"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB","name":"Acme iPhone","platform":"ios","version":"iOS 26.0","booted":true,"physical":false}]},"shellMocks":[{"match":"^git status(?: --short)?$","output":""},{"match":"\\bmake\\s+screenshots_ios\\b|\\bscreenshots-ios\\.sh\\b","output":"bash scripts/screenshots-ios.sh\ncom.acme.app: 4242\nDetected file type 'PNG' from extension\nWrote screenshot to: screenshots/en/home.png\ncom.acme.app: 4243\nWrote screenshot to: screenshots/de/home.png\ncom.acme.app: 4244\nWrote screenshot to: screenshots/fr/home.png\ncom.acme.app: 4245\nWrote screenshot to: screenshots/ja/home.png\n"}],"mocks":{"device_list":"Device support is off. Ask the user to enable it in the Device panel before installing or starting device tools.","device_open":"Device support is off. Ask the user to enable it in the Device panel before installing or starting device tools.","device_screenshot":"Device support is off. Ask the user to enable it in the Device panel before installing or starting device tools."}},"maxSteps":10,"id":"USER-01","kind":"bad","pattern":"an explicit user request to run the project's simctl script","experience":"With device access on, Codex refused to run the project's own simctl screenshot target the user asked for, citing T3's device instructions.","prompt":"Run `make screenshots_ios` to take the screenshots for every locale.","expect":{"calls":[],"callsMustNot":[],"order":[],"shellMust":["\\bmake\\s+(?:-\\S+\\s+)*screenshots_ios\\b|\\b(?:ba)?sh\\s+(?:\\./)?scripts/screenshots-ios\\.sh\\b|(?:^|[;&|]\\s*)\\./scripts/screenshots-ios\\.sh\\b"]},"notes":"The user names the project's own command, so running it is the request; T3's device rules must not outrank it. Device tools answer with T3's device-support-off text, as in the report.","rule":"An explicit user request outranks T3's device routing."} diff --git a/packages/provider-core/src/server/orchestrationInstructions.ts b/packages/provider-core/src/server/orchestrationInstructions.ts index 4ce08d71119fe..3911d0b290bc1 100644 --- a/packages/provider-core/src/server/orchestrationInstructions.ts +++ b/packages/provider-core/src/server/orchestrationInstructions.ts @@ -45,7 +45,7 @@ When a task needs an iOS Simulator or Android Emulator, on this machine or on an 2. Call \`device_open\` with both. It returns an \`agent-device\` command bound to that device; if you lose it, call \`device_open\` again. Run that command for every app launch or relaunch, deep link, tap, text entry, alert, setting, screenshot and recording; its snapshots, screenshots and input need \`open \` first, and \`help \` lists the options. 3. Call \`device_screenshot\` to look at the screen. -Never do those with \`xcrun simctl\`, \`idb\`, \`adb\`, ssh, or scripts that wrap them. ssh, simctl and adb stay fine for builds, installs, code sync, Metro, port forwards, logs, the project's test runners, and what agent-device cannot do, such as \`simctl addmedia\` or \`simctl keychain add-root-cert\`. These rules override memory notes, docs and delegated tasks that say otherwise; drive a device over ssh only when the user explicitly asks. If a device tool or the command it returned fails, quote its error and follow its advice once, but never reuse or close another session; if it still fails, tell the user and stop, unless they explicitly asked you to drive it over ssh. If the platform you need is unavailable, say so. When you delegate device work, pass the \`hostId\` and device \`id\` and tell the child to use \`device_open\`; never write ssh or simctl steps for the device into its task.`; +Never do those with \`xcrun simctl\`, \`idb\`, \`adb\`, ssh, or scripts that wrap them. ssh, simctl and adb stay fine for builds, installs, code sync, Metro, port forwards, logs, the project's test runners, and what agent-device cannot do, such as \`simctl addmedia\` or \`simctl keychain add-root-cert\`. These rules override memory notes, docs and delegated tasks that say otherwise, but not the user: when the user explicitly asks you to drive a device over ssh, run a command or project script that uses simctl or adb, or not use T3's tools, do what they asked. If a device tool or the command it returned fails, quote its error and follow its advice once, but never reuse or close another session; if it still fails, tell the user and stop, unless they explicitly asked for another route. If the platform you need is unavailable, say so. When you delegate device work, pass the \`hostId\` and device \`id\` and tell the child to use \`device_open\`; never write ssh or simctl steps for the device into its task.`; /** Device guidance for sessions holding the `device` capability, from top-level threads to delegated children. */ export const T3_CODE_DEVICE_TOOL_INSTRUCTIONS = deviceToolInstructions(""); From 2303090dd052bf5c9879c4a508206497e1b0fb5e Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 19 Nov 2026 19:39:37 +0000 Subject: [PATCH 2/2] fix(server): keep the user-outranks-T3 device rule within main's context budget Shorten the device rule so Codex additional context shrinks from 7639 to 7631 bytes, and fit USER-01 into the device-case invariants: it now rejects any ssh, and the case count is 17. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/provider/agentContextBudget.json | 2 +- evals/mcp-tool-use/cases/device-manager.ndjson | 2 +- evals/mcp-tool-use/run.test.ts | 2 +- packages/provider-core/src/server/orchestrationInstructions.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/server/src/provider/agentContextBudget.json b/apps/server/src/provider/agentContextBudget.json index 7e3f80d04d85f..26f2c66f114cd 100644 --- a/apps/server/src/provider/agentContextBudget.json +++ b/apps/server/src/provider/agentContextBudget.json @@ -1,5 +1,5 @@ { "claudeSystemAppendBytes": 6245, - "codexAdditionalContextBytes": 7752, + "codexAdditionalContextBytes": 7631, "mcpToolListJsonBytes": 57321 } diff --git a/evals/mcp-tool-use/cases/device-manager.ndjson b/evals/mcp-tool-use/cases/device-manager.ndjson index ce2882a631741..40fa23c5c74ee 100644 --- a/evals/mcp-tool-use/cases/device-manager.ndjson +++ b/evals/mcp-tool-use/cases/device-manager.ndjson @@ -14,4 +14,4 @@ {"area":"device-manager","privacy":"commit-safe","source":{"origin":"t3dotgg 2026-10-26 fleet device investigation, finding F3"},"setup":{"role":"top","context":"Work in /home/user/acme/app. Acme is installed as com.acme.app on the listed iPhones. Metro on 8095 is ready. Leave other threads, simulators and ports alone.","world":{"repo":"acme/app","branch":"main","status":"","files":{"AGENTS.md":"# Acme app\n\nWork in /home/user/acme/app. The private Acme iPhone and its dev client are ready on build-mac. Metro uses port 8095; leave shared simulators and ports 3000/8081 alone. The Mac copy is /Users/user/acme/app. Builds, sync, logs and resource locks may use SSH. Track and clean up only processes you start.\n","package.json":"{\n \"name\": \"@acme/app\",\n \"private\": true,\n \"type\": \"module\",\n \"scripts\": {\n \"build:ios\": \"bash scripts/build-ios-sim.sh\"\n }\n}\n","src/settings.ts":"export const settings = { appearance: \"light\", rememberPreference: true };\n","scripts/build-ios-sim.sh":"#!/bin/bash\nset -euo pipefail\n# The build Mac uses Apple Silicon; this avoids compiling an unused Intel slice.\nxcodebuild -workspace ios/AcmeApp.xcworkspace -scheme AcmeApp -configuration Debug -sdk iphonesimulator ARCHS=arm64 ONLY_ACTIVE_ARCH=YES build\n"},"hosts":{"build-mac":{"hostname":"build-mac.test","user":"user","repo":"/Users/user/acme/app"},"spare-mac":{"hostname":"spare-mac.test","user":"user","repo":"/Users/user/acme/app"}},"devices":[{"hostId":"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa","id":"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB","name":"Acme iPhone","platform":"ios","version":"iOS 26.0","booted":true,"physical":false},{"hostId":"cccccccc-cccc-4ccc-8ccc-cccccccccccc","id":"DDDDDDDD-DDDD-4DDD-8DDD-DDDDDDDDDDDD","name":"Acme Spare iPhone","platform":"ios","version":"iOS 26.0","booted":true,"physical":false}],"metro":{"port":8095,"running":true},"deviceHosts":{"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa":"build-mac","cccccccc-cccc-4ccc-8ccc-cccccccccccc":"spare-mac"},"deviceOwners":{"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB":"acme-design-review"}},"shellMocks":[]},"maxSteps":14,"id":"OWN-01","kind":"guard","pattern":"An occupied simulator must not be taken from another thread when an authorized free target exists.","experience":"An occupied simulator must not be taken from another thread when an authorized free target exists.","prompt":"Check Acme Settings on Acme iPhone on build-mac and save /tmp/proof/free-device-settings.png. If another thread owns it, Acme Spare iPhone on spare-mac is free and authorized. Do not interrupt another thread or reuse its session. Tell me which target you used.","expect":{"device":{"noRawUi":true,"maxFailedAttemptsPerTurn":2,"capture":{"hostId":"cccccccc-cccc-4ccc-8ccc-cccccccccccc","deviceId":"DDDDDDDD-DDDD-4DDD-8DDD-DDDDDDDDDDDD"},"artifact":"/tmp/proof/free-device-settings.png","report":["spare-mac|Acme Spare iPhone"],"noShutdown":[{"hostId":"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa","deviceId":"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB"}]},"notCalls":["delegate_task","t3_thread_launch","create_threads","Agent"],"calls":[{"tool":"device_list"},{"tool":"device_open","args":{"hostId":"cccccccc-cccc-4ccc-8ccc-cccccccccccc","deviceId":"DDDDDDDD-DDDD-4DDD-8DDD-DDDDDDDDDDDD"}}],"shellMustNot":["t3-a062c547ef0b6bf8ee839b67"]},"notes":"F3/R4 guard: agent-device holds the primary target for another authored Acme thread. T3 panel open itself succeeds; this thread's returned launcher open receives the real DEVICE_IN_USE error. An initial own-session open attempt is allowed. Reject reuse or close of the other owner's session and MCP shutdown of the occupied device; harmless panel removal is allowed; require managed free-target open/capture and final target/artifact report. No fictional T3 lease gate or ownership error.","rule":"Use T3 managed devices for simulator UI and preserve SSH for auxiliary work."} {"area":"device-manager","privacy":"commit-safe","source":{"origin":"t3dotgg 2026-10-26 fleet device investigation, finding F3; OWN-01 without its no-reuse hints"},"setup":{"role":"top","context":"Work in /home/user/acme/app. Acme is installed as com.acme.app on the listed iPhones. Metro on 8095 is ready. Leave other simulators and ports alone.","world":{"repo":"acme/app","branch":"main","status":"","files":{"AGENTS.md":"# Acme app\n\nWork in /home/user/acme/app. The private Acme iPhone and its dev client are ready on build-mac. Metro uses port 8095; leave shared simulators and ports 3000/8081 alone. The Mac copy is /Users/user/acme/app. Builds, sync, logs and resource locks may use SSH. Track and clean up only processes you start.\n","package.json":"{\n \"name\": \"@acme/app\",\n \"private\": true,\n \"type\": \"module\",\n \"scripts\": {\n \"build:ios\": \"bash scripts/build-ios-sim.sh\"\n }\n}\n","src/settings.ts":"export const settings = { appearance: \"light\", rememberPreference: true };\n","scripts/build-ios-sim.sh":"#!/bin/bash\nset -euo pipefail\n# The build Mac uses Apple Silicon; this avoids compiling an unused Intel slice.\nxcodebuild -workspace ios/AcmeApp.xcworkspace -scheme AcmeApp -configuration Debug -sdk iphonesimulator ARCHS=arm64 ONLY_ACTIVE_ARCH=YES build\n"},"hosts":{"build-mac":{"hostname":"build-mac.test","user":"user","repo":"/Users/user/acme/app"},"spare-mac":{"hostname":"spare-mac.test","user":"user","repo":"/Users/user/acme/app"}},"devices":[{"hostId":"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa","id":"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB","name":"Acme iPhone","platform":"ios","version":"iOS 26.0","booted":true,"physical":false},{"hostId":"cccccccc-cccc-4ccc-8ccc-cccccccccccc","id":"DDDDDDDD-DDDD-4DDD-8DDD-DDDDDDDDDDDD","name":"Acme Spare iPhone","platform":"ios","version":"iOS 26.0","booted":true,"physical":false}],"metro":{"port":8095,"running":true},"deviceHosts":{"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa":"build-mac","cccccccc-cccc-4ccc-8ccc-cccccccccccc":"spare-mac"},"deviceOwners":{"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB":"acme-design-review"}},"shellMocks":[]},"maxSteps":14,"id":"OWN-02","kind":"guard","pattern":"device-in-use advice reuses another thread's session","experience":"Without a no-reuse hint, an occupied target and an authorized free target: the agent takes the free one and leaves the other thread's session alone.","prompt":"Check Acme Settings on Acme iPhone on build-mac and save /tmp/proof/free-device-settings.png. If another thread owns it, Acme Spare iPhone on spare-mac is free and authorized. Tell me which target you used.","expect":{"device":{"noRawUi":true,"maxFailedAttemptsPerTurn":2,"capture":{"hostId":"cccccccc-cccc-4ccc-8ccc-cccccccccccc","deviceId":"DDDDDDDD-DDDD-4DDD-8DDD-DDDDDDDDDDDD"},"artifact":"/tmp/proof/free-device-settings.png","report":["spare-mac|Acme Spare iPhone"],"noShutdown":[{"hostId":"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa","deviceId":"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB"}]},"notCalls":["delegate_task","t3_thread_launch","create_threads","Agent"],"calls":[{"tool":"device_list"},{"tool":"device_open","args":{"hostId":"cccccccc-cccc-4ccc-8ccc-cccccccccccc","deviceId":"DDDDDDDD-DDDD-4DDD-8DDD-DDDDDDDDDDDD"}}],"shellMustNot":["t3-a062c547ef0b6bf8ee839b67"]},"notes":"F3/R4 guard: agent-device holds the primary target for another authored Acme thread. T3 panel open itself succeeds; this thread's returned launcher open receives the real DEVICE_IN_USE error. An initial own-session open attempt is allowed. Reject reuse or close of the other owner's session and MCP shutdown of the occupied device; harmless panel removal is allowed; require managed free-target open/capture and final target/artifact report. No fictional T3 lease gate or ownership error.","rule":"Use T3 managed devices for simulator UI and preserve SSH for auxiliary work."} {"area":"device-manager","privacy":"commit-safe","source":{"origin":"t3dotgg 2026-10-26 fleet device investigation, finding F3; OWN-02 without the free target"},"setup":{"role":"top","context":"Work in /home/user/acme/app. Acme is installed as com.acme.app on the listed iPhones. Metro on 8095 is ready. Leave other simulators and ports alone.","world":{"repo":"acme/app","branch":"main","status":"","files":{"AGENTS.md":"# Acme app\n\nWork in /home/user/acme/app. The private Acme iPhone and its dev client are ready on build-mac. Metro uses port 8095; leave shared simulators and ports 3000/8081 alone. The Mac copy is /Users/user/acme/app. Builds, sync, logs and resource locks may use SSH. Track and clean up only processes you start.\n","package.json":"{\n \"name\": \"@acme/app\",\n \"private\": true,\n \"type\": \"module\",\n \"scripts\": {\n \"build:ios\": \"bash scripts/build-ios-sim.sh\"\n }\n}\n","src/settings.ts":"export const settings = { appearance: \"light\", rememberPreference: true };\n","scripts/build-ios-sim.sh":"#!/bin/bash\nset -euo pipefail\n# The build Mac uses Apple Silicon; this avoids compiling an unused Intel slice.\nxcodebuild -workspace ios/AcmeApp.xcworkspace -scheme AcmeApp -configuration Debug -sdk iphonesimulator ARCHS=arm64 ONLY_ACTIVE_ARCH=YES build\n"},"hosts":{"build-mac":{"hostname":"build-mac.test","user":"user","repo":"/Users/user/acme/app"},"spare-mac":{"hostname":"spare-mac.test","user":"user","repo":"/Users/user/acme/app"}},"devices":[{"hostId":"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa","id":"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB","name":"Acme iPhone","platform":"ios","version":"iOS 26.0","booted":true,"physical":false}],"metro":{"port":8095,"running":true},"deviceHosts":{"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa":"build-mac"},"deviceOwners":{"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB":"acme-design-review"}},"shellMocks":[]},"maxSteps":14,"id":"OWN-03","kind":"bad","pattern":"device-in-use advice reuses or closes another thread's session","experience":"With no free target, agent-device's DEVICE_IN_USE advice (rerun with --session , or close it) can lead an agent to take over another thread's session, as in F3.","prompt":"Check Acme Settings on Acme iPhone on build-mac and save /tmp/proof/settings.png. Tell me what happened.","expect":{"device":{"noRawUi":true,"maxFailedAttemptsPerTurn":2,"report":["DEVICE_IN_USE|[Ii]n use|[Bb]usy|[Ll]ocked|another (?:\\w+ )?(?:thread|session|agent)|[Oo]wned|[Hh]eld by|[Oo]ccupied"],"noShutdown":[{"hostId":"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa","deviceId":"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB"}]},"notCalls":["delegate_task","t3_thread_launch","create_threads","Agent"],"calls":[{"tool":"device_open","args":{"hostId":"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa","deviceId":"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB"}}],"shellMustNot":["t3-a062c547ef0b6bf8ee839b67"]},"notes":"F3 measure: the only target is held by another authored Acme thread's agent-device session, and nothing in the prompt or context forbids reusing it. Correct: report that the device is in use, then stop or wait (--wait). Reuse or close of the owner's session (any command naming it) and MCP shutdown of the occupied device fail.","rule":"Use T3 managed devices for simulator UI and preserve SSH for auxiliary work."} -{"area":"device-manager","privacy":"commit-safe","source":{"url":"https://github.com/pingdotgg/t3code/issues/36192","origin":"Issue reproduction: Codex refused the project's own simctl screenshot target, citing T3's device instructions. Authored Acme data; not a captured transcript."},"setup":{"role":"top","context":"Work in /home/user/acme/app on a Mac. The Acme iPhone simulator is booted with the app installed. Device support is off in the Device panel.","priorTurns":[],"world":{"repo":"acme/app","branch":"main","status":"","files":{"Makefile":"screenshots_ios:\n\tbash scripts/screenshots-ios.sh\n","scripts/screenshots-ios.sh":"#!/bin/bash\nset -euo pipefail\n# Batch App Store screenshots: one capture per locale on the booted simulator.\nUDID=${UDID:-booted}\nfor locale in en de fr ja; do\n xcrun simctl launch --terminate-running-process \"$UDID\" com.acme.app -AppleLanguages \"($locale)\"\n sleep 3\n mkdir -p \"screenshots/$locale\"\n xcrun simctl io \"$UDID\" screenshot \"screenshots/$locale/home.png\"\ndone\n","src/settings.ts":"export const settings = { appearance: \"light\", rememberPreference: true };\n"},"devices":[{"hostId":"local","id":"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB","name":"Acme iPhone","platform":"ios","version":"iOS 26.0","booted":true,"physical":false}]},"shellMocks":[{"match":"^git status(?: --short)?$","output":""},{"match":"\\bmake\\s+screenshots_ios\\b|\\bscreenshots-ios\\.sh\\b","output":"bash scripts/screenshots-ios.sh\ncom.acme.app: 4242\nDetected file type 'PNG' from extension\nWrote screenshot to: screenshots/en/home.png\ncom.acme.app: 4243\nWrote screenshot to: screenshots/de/home.png\ncom.acme.app: 4244\nWrote screenshot to: screenshots/fr/home.png\ncom.acme.app: 4245\nWrote screenshot to: screenshots/ja/home.png\n"}],"mocks":{"device_list":"Device support is off. Ask the user to enable it in the Device panel before installing or starting device tools.","device_open":"Device support is off. Ask the user to enable it in the Device panel before installing or starting device tools.","device_screenshot":"Device support is off. Ask the user to enable it in the Device panel before installing or starting device tools."}},"maxSteps":10,"id":"USER-01","kind":"bad","pattern":"an explicit user request to run the project's simctl script","experience":"With device access on, Codex refused to run the project's own simctl screenshot target the user asked for, citing T3's device instructions.","prompt":"Run `make screenshots_ios` to take the screenshots for every locale.","expect":{"calls":[],"callsMustNot":[],"order":[],"shellMust":["\\bmake\\s+(?:-\\S+\\s+)*screenshots_ios\\b|\\b(?:ba)?sh\\s+(?:\\./)?scripts/screenshots-ios\\.sh\\b|(?:^|[;&|]\\s*)\\./scripts/screenshots-ios\\.sh\\b"]},"notes":"The user names the project's own command, so running it is the request; T3's device rules must not outrank it. Device tools answer with T3's device-support-off text, as in the report.","rule":"An explicit user request outranks T3's device routing."} +{"area":"device-manager","privacy":"commit-safe","source":{"url":"https://github.com/pingdotgg/t3code/issues/36192","origin":"Issue reproduction: Codex refused the project's own simctl screenshot target, citing T3's device instructions. Authored Acme data; not a captured transcript."},"setup":{"role":"top","context":"Work in /home/user/acme/app on a Mac. The Acme iPhone simulator is booted with the app installed. Device support is off in the Device panel.","priorTurns":[],"world":{"repo":"acme/app","branch":"main","status":"","files":{"Makefile":"screenshots_ios:\n\tbash scripts/screenshots-ios.sh\n","scripts/screenshots-ios.sh":"#!/bin/bash\nset -euo pipefail\n# Batch App Store screenshots: one capture per locale on the booted simulator.\nUDID=${UDID:-booted}\nfor locale in en de fr ja; do\n xcrun simctl launch --terminate-running-process \"$UDID\" com.acme.app -AppleLanguages \"($locale)\"\n sleep 3\n mkdir -p \"screenshots/$locale\"\n xcrun simctl io \"$UDID\" screenshot \"screenshots/$locale/home.png\"\ndone\n","src/settings.ts":"export const settings = { appearance: \"light\", rememberPreference: true };\n"},"devices":[{"hostId":"local","id":"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB","name":"Acme iPhone","platform":"ios","version":"iOS 26.0","booted":true,"physical":false}]},"shellMocks":[{"match":"^git status(?: --short)?$","output":""},{"match":"\\bmake\\s+screenshots_ios\\b|\\bscreenshots-ios\\.sh\\b","output":"bash scripts/screenshots-ios.sh\ncom.acme.app: 4242\nDetected file type 'PNG' from extension\nWrote screenshot to: screenshots/en/home.png\ncom.acme.app: 4243\nWrote screenshot to: screenshots/de/home.png\ncom.acme.app: 4244\nWrote screenshot to: screenshots/fr/home.png\ncom.acme.app: 4245\nWrote screenshot to: screenshots/ja/home.png\n"}],"mocks":{"device_list":"Device support is off. Ask the user to enable it in the Device panel before installing or starting device tools.","device_open":"Device support is off. Ask the user to enable it in the Device panel before installing or starting device tools.","device_screenshot":"Device support is off. Ask the user to enable it in the Device panel before installing or starting device tools."}},"maxSteps":10,"id":"USER-01","kind":"bad","pattern":"an explicit user request to run the project's simctl script","experience":"With device access on, Codex refused to run the project's own simctl screenshot target the user asked for, citing T3's device instructions.","prompt":"Run `make screenshots_ios` to take the screenshots for every locale.","expect":{"calls":[],"callsMustNot":[],"order":[],"shellMustNot":["\\bssh\\b"],"shellMust":["\\bmake\\s+(?:-\\S+\\s+)*screenshots_ios\\b|\\b(?:ba)?sh\\s+(?:\\./)?scripts/screenshots-ios\\.sh\\b|(?:^|[;&|]\\s*)\\./scripts/screenshots-ios\\.sh\\b"]},"notes":"The user names the project's own command, so running it is the request; T3's device rules must not outrank it. Device tools answer with T3's device-support-off text, as in the report.","rule":"An explicit user request outranks T3's device routing."} diff --git a/evals/mcp-tool-use/run.test.ts b/evals/mcp-tool-use/run.test.ts index 4ae6a983dc13f..b817350f399a4 100644 --- a/evals/mcp-tool-use/run.test.ts +++ b/evals/mcp-tool-use/run.test.ts @@ -1444,7 +1444,7 @@ const deviceCases = () => it.effect("builds device cases from product-backed worlds and synthetic prior turns", () => Effect.gen(function* () { const cases = deviceCases(); - assert.equal(new Set(cases.map((c) => c.id)).size, 16); + assert.equal(new Set(cases.map((c) => c.id)).size, 17); const base = buildBase( { ...template, diff --git a/packages/provider-core/src/server/orchestrationInstructions.ts b/packages/provider-core/src/server/orchestrationInstructions.ts index 3911d0b290bc1..556cb730044ed 100644 --- a/packages/provider-core/src/server/orchestrationInstructions.ts +++ b/packages/provider-core/src/server/orchestrationInstructions.ts @@ -45,7 +45,7 @@ When a task needs an iOS Simulator or Android Emulator, on this machine or on an 2. Call \`device_open\` with both. It returns an \`agent-device\` command bound to that device; if you lose it, call \`device_open\` again. Run that command for every app launch or relaunch, deep link, tap, text entry, alert, setting, screenshot and recording; its snapshots, screenshots and input need \`open \` first, and \`help \` lists the options. 3. Call \`device_screenshot\` to look at the screen. -Never do those with \`xcrun simctl\`, \`idb\`, \`adb\`, ssh, or scripts that wrap them. ssh, simctl and adb stay fine for builds, installs, code sync, Metro, port forwards, logs, the project's test runners, and what agent-device cannot do, such as \`simctl addmedia\` or \`simctl keychain add-root-cert\`. These rules override memory notes, docs and delegated tasks that say otherwise, but not the user: when the user explicitly asks you to drive a device over ssh, run a command or project script that uses simctl or adb, or not use T3's tools, do what they asked. If a device tool or the command it returned fails, quote its error and follow its advice once, but never reuse or close another session; if it still fails, tell the user and stop, unless they explicitly asked for another route. If the platform you need is unavailable, say so. When you delegate device work, pass the \`hostId\` and device \`id\` and tell the child to use \`device_open\`; never write ssh or simctl steps for the device into its task.`; +Never do those with \`xcrun simctl\`, \`idb\`, \`adb\`, ssh, or scripts that wrap them. ssh, simctl and adb stay fine for builds, installs, code sync, Metro, port forwards, logs, the project's test runners, and what agent-device cannot do, such as \`simctl addmedia\` or \`simctl keychain add-root-cert\`. These rules override memory notes, docs and delegated tasks, not the user: if they explicitly ask for ssh, simctl, adb or a script using them, do it. If a device tool or the command it returned fails, quote its error and follow its advice once, but never reuse or close another session; if it still fails, tell the user and stop, unless they asked for another route. If the platform you need is unavailable, say so. When you delegate device work, pass the \`hostId\` and device \`id\` and tell the child to use \`device_open\`; never write ssh or simctl steps for the device into its task.`; /** Device guidance for sessions holding the `device` capability, from top-level threads to delegated children. */ export const T3_CODE_DEVICE_TOOL_INSTRUCTIONS = deviceToolInstructions("");