From 8e45088f3f906740e90ef76cc37d4e233db636ff Mon Sep 17 00:00:00 2001 From: Jake Leventhal Date: Sat, 24 Oct 2026 14:09:12 -0400 Subject: [PATCH 01/19] feat: Codex voice conversations on web, desktop, and mobile Replayed onto d53873b6a4 as one commit because upstream split ChatView and ChatComposer into hooks; the voice send path now lives in useComposerSend, the /voice menu item in useComposerMenuItems, and the footer toggle in ComposerVoiceButton. Squashes: - 084aa6ec30 feat: Codex voice conversations on web, desktop, and mobile - 6ee113a7c7 fix: harden voice handoff resubmits and mobile voice controls - 1fce1a3b3f refactor: use Toggle for the voice button and a schema voice error - 3ad2f92fb8 fix(web): dim the disabled voice glyph instead of restyling Toggle - 823d3eb679 fix: keep failed voice handoffs and let /voice always stop a live conversation - 02a72c63a2 fix(server): don't let a conditional default shadow a user's unconditional shortcut - ae50f45aa0 fix: follow the base's namespace imports in voice code - 4da0e63fdb fix(server): preserve late defaults with voice shortcut migration - 4dc0a55264 fix(mobile): combine unused Android permission exclusions - 6225c3babe fix(voice): adapt reactivity imports and lockfile to current upstream - 0515f1eb6d fix(voice): reconcile current upstream dependency lockfile - 6bcc540de3 fix(voice): use renamed upstream test layers - 19978df1cb fix(server): use shared RPC instrumentation for voice streams - ee9edee4d8 test(voice): preserve custom shortcuts and provide MCP session service - e8e06bc974 fix(voice): use the provider protocol error from provider-core - e6fb3339f2 test(desktop): isolate the primary Windows native probe assertion - a5dcca51ee fix: address open PR review feedback after rebase - 16361d912a fix: preserve source behavior against current upstream contracts - 310c408577 fix(voice): retain sessions and respect navigation boundaries - a4bd27ac27 fix(voice): preserve typed session validation errors Co-Authored-By: Claude Opus 5.5 --- apps/desktop/src/window/DesktopWindow.test.ts | 4 + apps/desktop/src/window/DesktopWindow.ts | 5 + .../src/window/MainWindowPermissions.test.ts | 60 +++ .../src/window/MainWindowPermissions.ts | 72 +++ apps/mobile/app.config.ts | 27 +- apps/mobile/package.json | 2 + apps/mobile/src/components/ui/AppSymbol.tsx | 4 + .../src/features/threads/ThreadComposer.tsx | 39 +- .../features/threads/ThreadDetailScreen.tsx | 11 + .../voice-input/useVoiceInputController.ts | 1 + .../features/voice-mode/ComposerVoiceMode.tsx | 172 +++++++ .../features/voice-mode/useVoiceMode.test.tsx | 72 +++ .../src/features/voice-mode/useVoiceMode.ts | 71 +++ .../src/features/voice-mode/voiceMode.ts | 120 +++++ .../voice-mode/voiceModePresentation.test.ts | 70 +++ .../voice-mode/voiceModePresentation.ts | 68 +++ apps/mobile/vite.config.ts | 1 + apps/server/src/auth/RpcAuthorization.ts | 1 + apps/server/src/keybindings.test.ts | 26 + apps/server/src/keybindings.ts | 31 +- .../src/observability/RpcInstrumentation.ts | 1 + .../Adapters/CodexAdapterV2.test.ts | 126 +++++ .../Adapters/CodexAdapterV2.ts | 118 +++++ .../src/orchestration-v2/EffectWorker.test.ts | 2 + .../ProviderContinuationService.test.ts | 32 ++ .../ProviderContinuationService.ts | 48 +- .../ProviderSessionManager.test.ts | 73 +++ .../ProviderSessionManager.ts | 55 +++ .../ProviderTurnControlService.test.ts | 1 + apps/server/src/provider/CodexProvider.ts | 1 + apps/server/src/provider/CodexVoice.test.ts | 330 +++++++++++++ apps/server/src/provider/CodexVoice.ts | 295 +++++++++++ .../provider/ProviderInstanceRegistry.test.ts | 1 + apps/server/src/ws.ts | 2 + apps/web/src/components/CommandPalette.tsx | 68 ++- apps/web/src/components/chat/ChatComposer.tsx | 87 +++- .../src/components/chat/VoiceModeStrip.tsx | 209 ++++++++ .../components/chat/useComposerMenuItems.ts | 17 + .../src/components/chat/useComposerSend.ts | 43 +- apps/web/src/composer-logic.test.ts | 11 + apps/web/src/composer-logic.ts | 5 +- apps/web/src/keybindings.test.ts | 17 + apps/web/src/voice/voiceAvailability.test.ts | 40 ++ apps/web/src/voice/voiceAvailability.ts | 46 ++ apps/web/src/voice/voiceMode.ts | 153 ++++++ docs/internals/voice-conversations.md | 34 ++ docs/user/providers-codex.md | 23 + packages/client-runtime/package.json | 4 + packages/client-runtime/src/rpc/client.ts | 1 + .../src/voice-mode/controller.test.ts | 194 ++++++++ .../src/voice-mode/controller.ts | 462 ++++++++++++++++++ .../client-runtime/src/voice-mode/index.ts | 16 + .../client-runtime/src/voice-mode/session.ts | 102 ++++ packages/contracts/src/keybindings.ts | 2 + packages/contracts/src/provider.ts | 53 ++ packages/contracts/src/rpc.ts | 12 + packages/contracts/src/server.ts | 2 + packages/contracts/src/settings.ts | 29 +- .../src/server/ProviderAdapter.ts | 13 +- .../server/ProviderContinuationRequests.ts | 5 + .../provider-core/src/server/snapshotProbe.ts | 2 + packages/shared/src/composerTrigger.ts | 2 +- packages/shared/src/keybindings.ts | 1 + pnpm-lock.yaml | 48 ++ scripts/build-desktop-artifact.ts | 2 + third-party-licenses.config.json | 15 + 66 files changed, 3604 insertions(+), 56 deletions(-) create mode 100644 apps/desktop/src/window/MainWindowPermissions.test.ts create mode 100644 apps/desktop/src/window/MainWindowPermissions.ts create mode 100644 apps/mobile/src/features/voice-mode/ComposerVoiceMode.tsx create mode 100644 apps/mobile/src/features/voice-mode/useVoiceMode.test.tsx create mode 100644 apps/mobile/src/features/voice-mode/useVoiceMode.ts create mode 100644 apps/mobile/src/features/voice-mode/voiceMode.ts create mode 100644 apps/mobile/src/features/voice-mode/voiceModePresentation.test.ts create mode 100644 apps/mobile/src/features/voice-mode/voiceModePresentation.ts create mode 100644 apps/server/src/provider/CodexVoice.test.ts create mode 100644 apps/server/src/provider/CodexVoice.ts create mode 100644 apps/web/src/components/chat/VoiceModeStrip.tsx create mode 100644 apps/web/src/voice/voiceAvailability.test.ts create mode 100644 apps/web/src/voice/voiceAvailability.ts create mode 100644 apps/web/src/voice/voiceMode.ts create mode 100644 docs/internals/voice-conversations.md create mode 100644 packages/client-runtime/src/voice-mode/controller.test.ts create mode 100644 packages/client-runtime/src/voice-mode/controller.ts create mode 100644 packages/client-runtime/src/voice-mode/index.ts create mode 100644 packages/client-runtime/src/voice-mode/session.ts diff --git a/apps/desktop/src/window/DesktopWindow.test.ts b/apps/desktop/src/window/DesktopWindow.test.ts index 4f8949823e354..79c0568480b0e 100644 --- a/apps/desktop/src/window/DesktopWindow.test.ts +++ b/apps/desktop/src/window/DesktopWindow.test.ts @@ -93,6 +93,10 @@ function makeFakeBrowserWindow() { reloadIgnoringCache: vi.fn(), replaceMisspelling: vi.fn(), send: vi.fn(), + session: { + setPermissionCheckHandler: vi.fn(), + setPermissionRequestHandler: vi.fn(), + }, setBackgroundThrottling: vi.fn(), setWindowOpenHandler: vi.fn(), toggleDevTools: vi.fn(), diff --git a/apps/desktop/src/window/DesktopWindow.ts b/apps/desktop/src/window/DesktopWindow.ts index 5e02f4cfd8bbe..d995a5cf42650 100644 --- a/apps/desktop/src/window/DesktopWindow.ts +++ b/apps/desktop/src/window/DesktopWindow.ts @@ -35,6 +35,7 @@ import * as DesktopAppSettings from "../settings/DesktopAppSettings.ts"; import * as DesktopClientSettings from "../settings/DesktopClientSettings.ts"; import * as ElectronApp from "../electron/ElectronApp.ts"; import * as DesktopRendererHistory from "../telemetry/DesktopRendererHistory.ts"; +import { installMainWindowPermissionHandlers } from "./MainWindowPermissions.ts"; import { makeQuitShortcutHandler } from "./QuitHold.ts"; import { copyContextMenuImage } from "./ContextMenuImage.ts"; @@ -420,6 +421,10 @@ export const make = Effect.gen(function* () { if (environment.platform === "darwin") { window.setAutoHideCursor(false); } + installMainWindowPermissionHandlers(window.webContents.session, { + appUrl: applicationUrl, + platform: environment.platform, + }); let boundsPersistFiber: Fiber.Fiber | undefined; let pendingBoundsPersistFiber: Fiber.Fiber | undefined; let boundsPersistenceEnabled = persistedBounds === null || restoredPersistedBounds; diff --git a/apps/desktop/src/window/MainWindowPermissions.test.ts b/apps/desktop/src/window/MainWindowPermissions.test.ts new file mode 100644 index 0000000000000..e8c13053f26cf --- /dev/null +++ b/apps/desktop/src/window/MainWindowPermissions.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { + decideMainWindowMediaRequest, + isAppUrl, + isMainWindowPermissionAllowed, +} from "./MainWindowPermissions.ts"; + +const APP_URL = "t3code://app/"; + +function decide(overrides: Partial[0]> = {}) { + return decideMainWindowMediaRequest({ + requestingUrl: "t3code://app/threads/abc", + appUrl: APP_URL, + mediaTypes: ["audio"], + platform: "darwin", + microphoneStatus: () => "granted", + ...overrides, + }); +} + +describe("isAppUrl", () => { + it("matches the app shell and nothing that merely shares its prefix", () => { + expect(isAppUrl("t3code://app", APP_URL)).toBe(true); + expect(isAppUrl("t3code://app/settings", APP_URL)).toBe(true); + expect(isAppUrl("t3code://application/", APP_URL)).toBe(false); + expect(isAppUrl("https://example.com/", APP_URL)).toBe(false); + expect(isAppUrl(undefined, APP_URL)).toBe(false); + }); +}); + +describe("decideMainWindowMediaRequest", () => { + it("denies media to frames outside the app shell", () => { + expect(decide({ requestingUrl: "https://example.com/embed" })).toBe("deny"); + }); + + it("asks macOS for the microphone until the user has granted it", () => { + expect(decide({ microphoneStatus: () => "not-determined" })).toBe("ask-microphone"); + expect(decide({ microphoneStatus: () => "denied" })).toBe("ask-microphone"); + expect(decide()).toBe("grant"); + }); + + it("grants the microphone directly off macOS", () => { + expect(decide({ platform: "win32", microphoneStatus: () => "unknown" })).toBe("grant"); + }); + + it("keeps granting video-only capture such as preview tab recording", () => { + expect(decide({ mediaTypes: ["video"], microphoneStatus: () => "denied" })).toBe("grant"); + }); +}); + +it("denies synchronous clipboard reads for embedded origins and the app shell", () => { + for (const origin of ["https://example.com/frame", APP_URL]) { + expect(isMainWindowPermissionAllowed("deprecated-sync-clipboard-read", origin, APP_URL)).toBe( + false, + ); + } + expect(isMainWindowPermissionAllowed("media", "https://example.com/frame", APP_URL)).toBe(false); + expect(isMainWindowPermissionAllowed("media", APP_URL, APP_URL)).toBe(true); +}); diff --git a/apps/desktop/src/window/MainWindowPermissions.ts b/apps/desktop/src/window/MainWindowPermissions.ts new file mode 100644 index 0000000000000..0df6b97cef304 --- /dev/null +++ b/apps/desktop/src/window/MainWindowPermissions.ts @@ -0,0 +1,72 @@ +import * as Electron from "electron"; + +export type MediaPermissionDecision = "grant" | "deny" | "ask-microphone"; + +/** Whether `url` belongs to the app shell served from `appUrl` (e.g. `t3code://app/`). */ +export function isAppUrl(url: string | undefined, appUrl: string): boolean { + if (!url) return false; + // Custom schemes have an opaque WHATWG origin, so compare the serialized prefix. + const appOrigin = appUrl.endsWith("/") ? appUrl.slice(0, -1) : appUrl; + return url === appOrigin || url.startsWith(`${appOrigin}/`); +} + +/** + * Media requests come from the app shell (voice conversations use the + * microphone; preview tab recording routes `getDisplayMedia` through the same + * permission). Frames from any other origin never get media. On macOS the + * microphone also needs the system's consent, asked for on first use. + */ +export function decideMainWindowMediaRequest(input: { + readonly requestingUrl: string; + readonly appUrl: string; + readonly mediaTypes: ReadonlyArray<"audio" | "video">; + readonly platform: NodeJS.Platform; + readonly microphoneStatus: () => string; +}): MediaPermissionDecision { + if (!isAppUrl(input.requestingUrl, input.appUrl)) return "deny"; + if (!input.mediaTypes.includes("audio") || input.platform !== "darwin") return "grant"; + return input.microphoneStatus() === "granted" ? "grant" : "ask-microphone"; +} + +export function isMainWindowPermissionAllowed( + permission: string, + origin: string | undefined, + appUrl: string, +): boolean { + return permission === "media" + ? isAppUrl(origin, appUrl) + : permission !== "deprecated-sync-clipboard-read"; +} + +/** + * Installs the main window session's permission handlers. Only `media` is + * decided here. Synchronous clipboard reads keep Electron's default denial. + */ +export function installMainWindowPermissionHandlers( + session: Electron.Session, + options: { readonly appUrl: string; readonly platform: NodeJS.Platform }, +): void { + session.setPermissionRequestHandler((_webContents, permission, callback, details) => { + if (permission !== "media") { + callback(isMainWindowPermissionAllowed(permission, details.requestingUrl, options.appUrl)); + return; + } + const decision = decideMainWindowMediaRequest({ + requestingUrl: details.requestingUrl, + appUrl: options.appUrl, + mediaTypes: "mediaTypes" in details ? (details.mediaTypes ?? []) : [], + platform: options.platform, + microphoneStatus: () => Electron.systemPreferences.getMediaAccessStatus("microphone"), + }); + if (decision !== "ask-microphone") { + callback(decision === "grant"); + return; + } + Electron.systemPreferences.askForMediaAccess("microphone").then(callback, () => { + callback(false); + }); + }); + session.setPermissionCheckHandler((_webContents, permission, requestingOrigin) => + isMainWindowPermissionAllowed(permission, requestingOrigin, options.appUrl), + ); +} diff --git a/apps/mobile/app.config.ts b/apps/mobile/app.config.ts index e6d388b9a9856..e0ae761718715 100644 --- a/apps/mobile/app.config.ts +++ b/apps/mobile/app.config.ts @@ -221,6 +221,10 @@ const sharingPlugin: NonNullable[number] = [ }, ]; +const MICROPHONE_PERMISSION = + "Allow T3 Code to use your microphone for voice input and voice conversations."; +const CAMERA_PERMISSION = "Allow T3 Code to access your camera so you can scan pairing QR codes."; + // Match iOS PostScript names on Android so RN and the native composer share fonts before runtime loading. const config: ExpoConfig = { @@ -300,8 +304,11 @@ const config: ExpoConfig = { }, // JS back dispatch needs RN's Android 16 shim and withAndroidPredictiveBackCompat on Android 13-15. predictiveBackGestureEnabled: true, - // expo-sensors declares this for its pedometer, which the app does not use. - blockedPermissions: ["android.permission.ACTIVITY_RECOGNITION"], + // The app uses neither expo-sensors' pedometer nor WebRTC screen sharing. + blockedPermissions: [ + "android.permission.ACTIVITY_RECOGNITION", + "android.permission.SYSTEM_ALERT_WINDOW", + ], }, web: { favicon: variant.assets.appIcon, @@ -366,22 +373,30 @@ const config: ExpoConfig = { [ "expo-audio", { - microphonePermission: "Allow T3 Code to use your microphone for voice input.", - recordAudioAndroid: false, + microphonePermission: MICROPHONE_PERMISSION, + recordAudioAndroid: true, enableBackgroundPlayback: false, enableBackgroundRecording: false, }, ], + // Voice conversations run over WebRTC. Pass both usage strings so its + // plugin cannot fall back to generic ones. + [ + "@config-plugins/react-native-webrtc", + { cameraPermission: CAMERA_PERMISSION, microphonePermission: MICROPHONE_PERMISSION }, + ], [ "expo-camera", { - cameraPermission: "Allow T3 Code to access your camera so you can scan pairing QR codes.", + cameraPermission: CAMERA_PERMISSION, microphonePermission: false, barcodeScannerEnabled: true, recordAudioAndroid: false, }, ], - ["expo-image-picker", { photosPermission: false, microphonePermission: false }], + // `microphonePermission: false` would block RECORD_AUDIO on Android for every + // package, including voice conversations, so share the app's usage string. + ["expo-image-picker", { photosPermission: false, microphonePermission: MICROPHONE_PERMISSION }], [ "expo-splash-screen", { diff --git a/apps/mobile/package.json b/apps/mobile/package.json index a49c3856864f1..cca018d013373 100644 --- a/apps/mobile/package.json +++ b/apps/mobile/package.json @@ -131,6 +131,7 @@ "react-native-safe-area-context": "~5.10.1", "react-native-screens": "file:deps/react-native-screens-5.0.0-t3.48.tgz", "react-native-svg": "15.16.0", + "react-native-webrtc": "124.0.8", "react-native-webview": "14.0.1", "react-native-worklets": "0.13.0", "remark-gfm": "catalog:", @@ -141,6 +142,7 @@ }, "devDependencies": { "@babel/core": "^7.29.7", + "@config-plugins/react-native-webrtc": "15.0.2", "@effect/platform-node": "catalog:", "@effect/vitest": "catalog:", "@pierre/trees": "1.0.0-beta.6", diff --git a/apps/mobile/src/components/ui/AppSymbol.tsx b/apps/mobile/src/components/ui/AppSymbol.tsx index 4b17bddd3c324..63ecde90c5bc0 100644 --- a/apps/mobile/src/components/ui/AppSymbol.tsx +++ b/apps/mobile/src/components/ui/AppSymbol.tsx @@ -84,6 +84,7 @@ import IconLayoutSidebar from "@tabler/icons-react-native/IconLayoutSidebar"; import IconLayoutSidebarRight from "@tabler/icons-react-native/IconLayoutSidebarRight"; import IconLetterSpacing from "@tabler/icons-react-native/IconLetterSpacing"; import IconMicrophone from "@tabler/icons-react-native/IconMicrophone"; +import IconMicrophoneOff from "@tabler/icons-react-native/IconMicrophoneOff"; import IconLink from "@tabler/icons-react-native/IconLink"; import IconListNumbers from "@tabler/icons-react-native/IconListNumbers"; import IconMenu2 from "@tabler/icons-react-native/IconMenu2"; @@ -131,6 +132,7 @@ import IconUserQuestion from "@tabler/icons-react-native/IconUserQuestion"; import IconUserPlus from "@tabler/icons-react-native/IconUserPlus"; import IconUsers from "@tabler/icons-react-native/IconUsers"; import IconWifi from "@tabler/icons-react-native/IconWifi"; +import IconWaveSine from "@tabler/icons-react-native/IconWaveSine"; import IconWifiOff from "@tabler/icons-react-native/IconWifiOff"; import IconWorld from "@tabler/icons-react-native/IconWorld"; import IconX from "@tabler/icons-react-native/IconX"; @@ -229,6 +231,7 @@ const ANDROID_ICON_BY_SF_SYMBOL = { macstudio: IconDeviceDesktop, magnifyingglass: IconSearch, mic: IconMicrophone, + "mic.slash": IconMicrophoneOff, paintbrush: IconPalette, pencil: IconPencil, "person.crop.circle": IconUserCircle, @@ -280,6 +283,7 @@ const ANDROID_ICON_BY_SF_SYMBOL = { "tray.and.arrow.up": IconUpload, trash: IconTrash, wifi: IconWifi, + waveform: IconWaveSine, "wifi.slash": IconWifiOff, xmark: IconX, "xmark.circle.fill": IconCircleXFilled, diff --git a/apps/mobile/src/features/threads/ThreadComposer.tsx b/apps/mobile/src/features/threads/ThreadComposer.tsx index ea2928f531374..8ed80966f4798 100644 --- a/apps/mobile/src/features/threads/ThreadComposer.tsx +++ b/apps/mobile/src/features/threads/ThreadComposer.tsx @@ -115,6 +115,9 @@ import { import { useVoiceInputController } from "../voice-input/useVoiceInputController"; import { resolveVoiceComposerPresentation } from "../voice-input/voiceInputPresentation"; import type { ExistingThreadSettingsRouteParams } from "./ThreadSettingsSheet"; +import { ComposerVoiceModeButton, ComposerVoiceStrip } from "../voice-mode/ComposerVoiceMode"; +import { useVoiceModePhase, useVoiceModeThreadLifecycle } from "../voice-mode/useVoiceMode"; +import { voiceMode, voiceModeSupported } from "../voice-mode/voiceMode"; import { useThreadSettingsSheetPresentation, type NavigationWithFinishTransitioning, @@ -561,17 +564,37 @@ export const ThreadComposer = function ThreadComposer(props: ThreadComposerProps onUsageLimits: usageLimitsOffered && draftAttachments.length === 0 ? openUsageLimits : undefined, }); + const voiceTarget = { environmentId: props.environmentId, threadId: props.selectedThread.id }; + const voicePhase = useVoiceModePhase(voiceTarget); + useVoiceModeThreadLifecycle(voiceTarget); + // Dictation and a voice conversation both need the microphone. + const voiceConversationLive = voicePhase !== "idle"; const voiceInput = useVoiceInputController({ ownerKey: composerDraftKey, label: props.selectedThread.title || "Untitled thread", readDraftMessage: () => getComposerDraftSnapshot(composerDraftKey).text, subscribeToDraftChanges: (onChange) => appAtomRegistry.subscribe(composerDraftsAtom, onChange), selection: composerMenu.selection, + disabled: voiceConversationLive, onChangeDraftMessage: (text) => setComposerDraftText(composerDraftKey, text), onChangeSelection: composerMenu.onSelectionChange, }); const isSubmissionLocked = props.interactionLocked === true || queuedEdit?.saving === true || voiceInput.blocksSubmission; + const offersVoiceConversation = + voiceModeSupported && + props.serverConfig?.providers.find( + (provider) => provider.instanceId === props.selectedThread.providerInstanceId, + )?.supportsVoice === true; + // The server attaches voice to the thread's provider thread, which the first + // run creates. An optimistic pending thread already has a message timestamp. + const canToggleVoiceConversation = + voiceConversationLive || + (props.connectionState === "connected" && + props.canOperateThread && + props.selectedThread.activeProviderThreadId !== null && + !voiceInput.isGloballyBusy); + const toggleVoiceConversation = () => voiceMode.toggle(voiceTarget); const voicePresentation = resolveVoiceComposerPresentation( voiceInput.state, voiceInput.elapsedSeconds, @@ -583,6 +606,14 @@ export const ThreadComposer = function ThreadComposer(props: ThreadComposerProps const usesLargeControls = useSidebarWorkspaceTabs() === "picker" || (Platform.OS === "ios" && Platform.isPad); const surfacePaddingVertical = usesLargeControls ? 9 : floatingSidebarComposer ? 6 : 2; + const voiceModeButton = offersVoiceConversation ? ( + + ) : null; const [compactRowWidth, setCompactRowWidth] = useState(0); const compactAttachmentLimit = compactRowWidth >= 500 ? 3 : 1; const compactEditor = useCompactComposerEditor(bodyText, draftMessage); @@ -864,6 +895,9 @@ export const ThreadComposer = function ThreadComposer(props: ThreadComposerProps {selectedProviderStatus.compatibilityAdvisory.message} ) : null} + + {voiceConversationLive ? : null} + {modelUnavailable ? ( + + + + {captions.length > 0 ? ( + + {captions.map((caption, index) => ( +

+ {caption.text} +

+ ))} +
+ ) : null} + + ); +}); + +/** Starts or stops the thread's voice conversation from the composer footer. */ +export function ComposerVoiceButton(props: { + readonly engaged: boolean; + /** Why voice cannot start yet; the button stays focusable so the tooltip explains it. */ + readonly disabledReason: string | null; + readonly onToggle: () => void; +}) { + const label = props.engaged ? "Stop voice conversation" : "Start voice conversation"; + return ( + + event.preventDefault()} + onPressedChange={() => { + if (props.disabledReason === null) props.onToggle(); + }} + aria-disabled={props.disabledReason !== null || undefined} + aria-label={label} + /> + } + > + {/* Dim the glyph, not the Toggle, which owns its own disabled styling. */} + + + {props.disabledReason ?? label} + + ); +} diff --git a/apps/web/src/components/chat/useComposerMenuItems.ts b/apps/web/src/components/chat/useComposerMenuItems.ts index 5f778ac6cac1e..45033b73e74b3 100644 --- a/apps/web/src/components/chat/useComposerMenuItems.ts +++ b/apps/web/src/components/chat/useComposerMenuItems.ts @@ -67,6 +67,7 @@ export function useComposerMenuItems({ sideChatAvailable, sideChatDisabledReason, compactSlashCommandAvailable, + voiceSlashCommand, selectedProvider, selectedProviderSkills, selectedProviderSlashCommands, @@ -86,6 +87,8 @@ export function useComposerMenuItems({ readonly sideChatAvailable: boolean; readonly sideChatDisabledReason: string | null; readonly compactSlashCommandAvailable: boolean; + /** Whether /voice starts or stops a conversation; null hides it. */ + readonly voiceSlashCommand: "start" | "stop" | null; readonly selectedProvider: ProviderDriverKind; readonly selectedProviderSkills: ServerProvider["skills"]; readonly selectedProviderSlashCommands: ServerProvider["slashCommands"]; @@ -260,6 +263,20 @@ export function useComposerMenuItems({ }, ] as const) : []), + ...(voiceSlashCommand !== null + ? ([ + { + id: "slash:voice", + type: "slash-command", + command: "voice", + label: "/voice", + description: + voiceSlashCommand === "stop" + ? "Stop the voice conversation" + : "Start a voice conversation", + }, + ] as const) + : []), ] satisfies ReadonlyArray>; const slashMenuSkills = getProviderSkillsForSlashMenu( selectedProviderSkills, diff --git a/apps/web/src/components/chat/useComposerSend.ts b/apps/web/src/components/chat/useComposerSend.ts index 98084e9795443..7b2bb3519d40d 100644 --- a/apps/web/src/components/chat/useComposerSend.ts +++ b/apps/web/src/components/chat/useComposerSend.ts @@ -58,6 +58,8 @@ import { parseStandaloneComposerSlashCommand, } from "../../composer-logic"; import { resolvePlanFollowUpSubmission } from "../../proposedPlan"; +import { resolveVoiceAvailability } from "../../voice/voiceAvailability"; +import { toggleThreadVoice, voiceMode } from "../../voice/voiceMode"; import { type ChatMessage } from "../../types"; import { recordPromptSent } from "../GitHubStarPrompt"; import { stackedThreadToast, toastManager } from "../ui/toast"; @@ -716,6 +718,35 @@ export function useComposerSend(input: ComposerSendInput) { return; } + const parsedSlashCommand = + composerImages.length === 0 && + composerFiles.length === 0 && + sendableComposerTerminalContexts.length === 0 && + composerPreviewAnnotations.length === 0 && + composerReviewComments.length === 0 && + composerThreadContexts.length === 0 && + multipleModelSelections === null + ? parseStandaloneComposerSlashCommand(trimmed) + : null; + if (parsedSlashCommand === "voice") { + const voiceTarget = scopeThreadRef(environmentId, activeThread.id); + // Providers without voice receive "/voice" as a normal message. + const voiceAvailability = resolveVoiceAvailability({ + provider: providerInstanceEntries.find( + (entry) => entry.instanceId === activeThread.providerInstanceId, + )?.snapshot, + hasMessages: activeThread.latestUserMessageAt !== null, + hasSession: activeThread.activeProviderThreadId !== null, + }); + // An active conversation always stops, even if the provider stopped offering voice. + if (voiceAvailability.kind !== "unsupported" || voiceMode.isActiveFor(voiceTarget)) { + toggleThreadVoice(voiceTarget, voiceAvailability); + promptRef.current = ""; + clearComposerDraftContent(composerDraftTarget); + composerRef.current?.resetCursorState(); + return; + } + } if ( !directAnnotation && sendInteractionModeEnabled && @@ -797,17 +828,9 @@ export function useComposerSend(input: ComposerSendInput) { } return; } + // Providers without voice receive their native commands unchanged. const standaloneSlashCommand = - sendInteractionModeEnabled && - composerImages.length === 0 && - composerFiles.length === 0 && - sendableComposerTerminalContexts.length === 0 && - composerPreviewAnnotations.length === 0 && - composerReviewComments.length === 0 && - composerThreadContexts.length === 0 && - composerEnvironmentContexts.length === 0 - ? parseStandaloneComposerSlashCommand(trimmed) - : null; + sendInteractionModeEnabled && parsedSlashCommand !== "voice" ? parsedSlashCommand : null; if (standaloneSlashCommand && multipleModelSelections === null) { handleInteractionModeChange(standaloneSlashCommand); promptRef.current = ""; diff --git a/apps/web/src/composer-logic.test.ts b/apps/web/src/composer-logic.test.ts index 51635a651d476..f9d699958574f 100644 --- a/apps/web/src/composer-logic.test.ts +++ b/apps/web/src/composer-logic.test.ts @@ -23,6 +23,7 @@ import { formatAssistantCitationForComposer, isPathSuggestionCurrent, parseSideChatCommand, + parseStandaloneComposerSlashCommand, replaceTextRange, } from "./composer-logic"; import { slashCommandItemsForPromptPosition } from "./components/chat/composerSlashCommandSearch"; @@ -512,6 +513,16 @@ describe("assistant citation cursor offsets", () => { }); }); +describe("parseStandaloneComposerSlashCommand", () => { + it("parses standalone /voice command", () => { + expect(parseStandaloneComposerSlashCommand("/Voice ")).toBe("voice"); + }); + + it("ignores /voice with extra message text", () => { + expect(parseStandaloneComposerSlashCommand("/voice please")).toBeNull(); + }); +}); + describe("literal editor answer cursor state", () => { const prefill = "- [X] done $my-skill "; diff --git a/apps/web/src/composer-logic.ts b/apps/web/src/composer-logic.ts index 9b7223797190e..66910f8afd510 100644 --- a/apps/web/src/composer-logic.ts +++ b/apps/web/src/composer-logic.ts @@ -18,7 +18,7 @@ import { import { resolveShortcutCommand, type ShortcutEventLike } from "./keybindings"; export type ComposerTriggerKind = "path" | "pull-request" | "slash-command" | "skill"; -export type ComposerSlashCommand = "model" | "plan" | "default" | "side" | "btw"; +export type ComposerSlashCommand = "model" | "plan" | "default" | "side" | "btw" | "voice"; export type ComposerSubmissionIntent = "foreground" | "background" | "alternate"; export interface ComposerTrigger { @@ -327,12 +327,13 @@ export function parseSideChatCommand(text: string): { prompt: string } | null { export function parseStandaloneComposerSlashCommand( text: string, ): Exclude | null { - const match = /^\/(plan|default)\s*$/i.exec(text.trim()); + const match = /^\/(plan|default|voice)\s*$/i.exec(text.trim()); if (!match) { return null; } const command = match[1]?.toLowerCase(); if (command === "plan") return "plan"; + if (command === "voice") return "voice"; return "default"; } diff --git a/apps/web/src/keybindings.test.ts b/apps/web/src/keybindings.test.ts index 1f0a096fccaec..8b26e86a4f22e 100644 --- a/apps/web/src/keybindings.test.ts +++ b/apps/web/src/keybindings.test.ts @@ -397,6 +397,23 @@ describe("thread navigation helpers", () => { ), ); }); + + it("toggles voice with a bare F8 unless the terminal has focus", () => { + const input = event({ key: "F8", code: "F8" }); + assert.strictEqual( + resolveShortcutCommand(input, DEFAULT_RESOLVED_KEYBINDINGS, { + platform: "MacIntel", + context: { terminalFocus: false }, + }), + "voice.toggle", + ); + assert.isNull( + resolveShortcutCommand(input, DEFAULT_RESOLVED_KEYBINDINGS, { + platform: "MacIntel", + context: { terminalFocus: true }, + }), + ); + }); }); describe("model picker navigation helpers", () => { diff --git a/apps/web/src/voice/voiceAvailability.test.ts b/apps/web/src/voice/voiceAvailability.test.ts new file mode 100644 index 0000000000000..0d5b0ff85e1f5 --- /dev/null +++ b/apps/web/src/voice/voiceAvailability.test.ts @@ -0,0 +1,40 @@ +import { ProviderInstanceId, type ServerProvider } from "@t3tools/contracts"; +import { describe, expect, it } from "vite-plus/test"; + +import { resolveThreadVoiceAvailability } from "./voiceAvailability"; + +const codex = ProviderInstanceId.make("codex"); +const claude = ProviderInstanceId.make("claudeAgent"); +const providers = [ + { instanceId: codex, supportsVoice: true }, + { instanceId: claude }, +] as unknown as ReadonlyArray; + +function thread(input: { readonly provider: ProviderInstanceId; readonly messages?: boolean }) { + return { + providerInstanceId: input.provider, + latestUserMessageAt: input.messages ? "2026-09-22T00:00:00.000Z" : null, + activeProviderThreadId: input.messages ? "provider-thread-1" : null, + }; +} + +describe("resolveThreadVoiceAvailability", () => { + it("hides voice for providers that do not support it", () => { + expect( + resolveThreadVoiceAvailability(thread({ provider: claude, messages: true }), providers), + ).toEqual({ kind: "unsupported" }); + }); + + it("explains that a new thread needs a first message", () => { + expect(resolveThreadVoiceAvailability(thread({ provider: codex }), providers)).toEqual({ + kind: "unavailable", + reason: "Send a message before starting voice", + }); + }); + + it("is ready once a Codex thread has run", () => { + expect( + resolveThreadVoiceAvailability(thread({ provider: codex, messages: true }), providers), + ).toEqual({ kind: "ready" }); + }); +}); diff --git a/apps/web/src/voice/voiceAvailability.ts b/apps/web/src/voice/voiceAvailability.ts new file mode 100644 index 0000000000000..1e919f3b8d203 --- /dev/null +++ b/apps/web/src/voice/voiceAvailability.ts @@ -0,0 +1,46 @@ +import type { ProviderInstanceId, ServerProvider } from "@t3tools/contracts"; + +/** + * Whether a thread can hold a voice conversation. `unsupported` hides every + * voice entry point; `unavailable` shows them disabled with `reason`. + */ +export type VoiceAvailability = + | { readonly kind: "unsupported" } + | { readonly kind: "unavailable"; readonly reason: string } + | { readonly kind: "ready" }; + +const VOICE_UNSUPPORTED: VoiceAvailability = { kind: "unsupported" }; + +export function resolveVoiceAvailability(input: { + /** The provider instance that runs (or will run) the thread's turns. */ + readonly provider: Pick | null | undefined; + readonly hasMessages: boolean; + readonly hasSession: boolean; +}): VoiceAvailability { + if (input.provider?.supportsVoice !== true) return VOICE_UNSUPPORTED; + // The server hands voice requests to the thread's bound provider session, + // which only exists once a turn has been sent. + if (!input.hasMessages || !input.hasSession) { + return { kind: "unavailable", reason: "Send a message before starting voice" }; + } + return { kind: "ready" }; +} + +/** The thread fields voice availability reads (orchestrator V2 thread shells). */ +export interface VoiceThread { + readonly providerInstanceId: ProviderInstanceId; + readonly latestUserMessageAt: string | null; + readonly activeProviderThreadId: string | null; +} + +/** Voice follows the provider instance that owns the thread. */ +export function resolveThreadVoiceAvailability( + thread: VoiceThread, + providers: ReadonlyArray, +): VoiceAvailability { + return resolveVoiceAvailability({ + provider: providers.find((provider) => provider.instanceId === thread.providerInstanceId), + hasMessages: thread.latestUserMessageAt !== null, + hasSession: thread.activeProviderThreadId !== null, + }); +} diff --git a/apps/web/src/voice/voiceMode.ts b/apps/web/src/voice/voiceMode.ts new file mode 100644 index 0000000000000..3a87277ef465b --- /dev/null +++ b/apps/web/src/voice/voiceMode.ts @@ -0,0 +1,153 @@ +import { EnvironmentId, ThreadId } from "@t3tools/contracts"; +import { scopeThreadRef } from "@t3tools/client-runtime/environment"; +import { + VoiceModeController, + describeVoiceEnd, + makeVoiceSessionOpener, + makeVoiceSessionValidator, + type VoiceMediaStream, + type VoiceModePhase, + type VoiceModePlatform, + type VoiceModeState, + type VoicePeerConnection, + type VoiceSessionTarget, +} from "@t3tools/client-runtime/voice-mode"; +import * as Effect from "effect/Effect"; +import { AtomRegistry } from "effect/reactivity"; +import { useSyncExternalStore } from "react"; + +import { stackedThreadToast, toastManager } from "../components/ui/toast"; +import { connectionAtomRuntime } from "../connection/runtime"; +import { appAtomRegistry } from "../rpc/atomRegistry"; +import type { VoiceAvailability } from "./voiceAvailability"; + +function getMicrophone(): Promise { + const mediaDevices = typeof navigator === "undefined" ? undefined : navigator.mediaDevices; + // Browsers only expose the microphone API on secure origins, so a remote + // environment opened over plain HTTP has no `mediaDevices` at all. + if (!mediaDevices?.getUserMedia) { + return Promise.reject( + new Error("Voice needs microphone access, which requires HTTPS when connecting remotely."), + ); + } + return mediaDevices.getUserMedia({ + audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true }, + }); +} + +function playRemoteAudio(stream: VoiceMediaStream): () => void { + const audio = new Audio(); + audio.autoplay = true; + audio.srcObject = stream as MediaStream; + void audio.play().catch(() => undefined); + return () => { + audio.pause(); + audio.srcObject = null; + }; +} + +// The voice stream runs on the connection runtime, which owns the environment +// registry. Mounting keeps that runtime alive for the conversation. +const runOnConnectionRuntime = ( + effect: Effect.Effect< + void, + never, + import("@t3tools/client-runtime/connection").EnvironmentRegistry.EnvironmentRegistry + >, +) => + Effect.runFork( + Effect.scoped( + AtomRegistry.mount(appAtomRegistry, connectionAtomRuntime).pipe( + Effect.andThen(AtomRegistry.getResult(appAtomRegistry, connectionAtomRuntime)), + Effect.flatMap((context) => Effect.provideContext(effect, context)), + ), + ).pipe(Effect.orDie), + ); + +const openSession = makeVoiceSessionOpener(runOnConnectionRuntime); + +const platform: VoiceModePlatform = { + // The DOM's overloaded addEventListener does not fit the structural `never` + // listener in VoicePeerConnection, though the runtime shape matches. + createPeerConnection: () => new RTCPeerConnection() as unknown as VoicePeerConnection, + validateTarget: makeVoiceSessionValidator(runOnConnectionRuntime), + getMicrophone, + playRemoteAudio, + openSession, +}; + +/** The app's single voice conversation. The microphone is exclusive, so there is one per client. */ +export const voiceMode = new VoiceModeController(platform); + +export function isVoiceTargetFor( + state: VoiceModeState, + target: VoiceSessionTarget | null, +): boolean { + return ( + target !== null && + state.target !== null && + state.target.environmentId === target.environmentId && + state.target.threadId === target.threadId + ); +} + +export function useVoiceModeState(): VoiceModeState { + return useSyncExternalStore(voiceMode.subscribe, voiceMode.getState, voiceMode.getState); +} + +/** The conversation phase on `target`; "idle" while voice is off or on another thread. */ +export function useVoicePhaseFor(target: VoiceSessionTarget | null): VoiceModePhase { + const getPhase = () => { + const state = voiceMode.getState(); + return isVoiceTargetFor(state, target) ? state.phase : "idle"; + }; + return useSyncExternalStore(voiceMode.subscribe, getPhase, getPhase); +} + +/** + * Starts or stops voice on `target`. An unavailable thread explains why + * instead of failing on the server. + */ +export function toggleThreadVoice(target: VoiceSessionTarget, availability: VoiceAvailability) { + if (voiceMode.isActiveFor(target)) { + voiceMode.stop(); + return; + } + if (availability.kind === "unsupported") return; + if (availability.kind === "unavailable") { + toastManager.add({ type: "info", title: availability.reason }); + return; + } + void voiceMode.start(target); +} + +// Errors and provider-side ends surface once as a toast on the conversation's +// thread. A user stop leaves neither set, so it stays silent. +let lastPhase: VoiceModePhase = "idle"; +voiceMode.subscribe((state) => { + const previousPhase = lastPhase; + lastPhase = state.phase; + if (state.phase !== "idle" || state.target === null) return; + if (state.error === null && state.endedReason === null) return; + const threadRef = scopeThreadRef( + EnvironmentId.make(state.target.environmentId), + ThreadId.make(state.target.threadId), + ); + if (state.error !== null) { + toastManager.add( + stackedThreadToast({ + type: "error", + title: previousPhase === "active" ? "Voice conversation ended" : "Couldn't start voice", + description: state.error, + data: { threadRef }, + }), + ); + } else if (state.endedReason !== null) { + toastManager.add({ + type: "info", + title: describeVoiceEnd(state.endedReason), + data: { threadRef }, + }); + } + voiceMode.dismiss(); +}); diff --git a/docs/internals/voice-conversations.md b/docs/internals/voice-conversations.md new file mode 100644 index 0000000000000..1b9dc06e08ea1 --- /dev/null +++ b/docs/internals/voice-conversations.md @@ -0,0 +1,34 @@ +# Voice conversations + +A voice conversation is a Codex realtime session bound to one thread. The user +talks to a voice model, and when it wants work done it starts an ordinary Codex +turn on that thread. This is separate from [voice input](voice-input.md), which +only transcribes into the composer. + +Audio never touches the T3 socket. The client owns the microphone, speaker, and +WebRTC peer, and audio flows directly between the client and OpenAI. The +`provider.voice.session` stream carries only the SDP handshake, captions, and +lifecycle. That keeps remote and tunneled clients as cheap as local ones, and it +is why the [shared controller](../../packages/client-runtime/src/voice-mode/controller.ts) +takes a platform object: browsers and react-native-webrtc supply the same peer API. + +The stream's lifetime is the conversation's lifetime. When the client +unsubscribes or disconnects, the server stops the realtime session, so a dropped +client cannot leave a billed session running. + +Handoffs are client-managed, as in the Codex TUI. The +[Codex voice module](../../apps/server/src/provider/Layers/CodexVoice.ts) hands +each voice turn's final answer back with `thread/realtime/appendSpeech`, which +keeps commentary and reasoning unspoken. + +Codex starts or steers the delegated turn itself, with a user message wrapped in +``, and it cannot be told not to. Steering an active run +needs nothing special. An idle thread is different: V2 runs are +orchestrator-owned and there is no provider-initiated run yet, so +[CodexAdapterV2](../../apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts) +interrupts that turn and, once it settles, offers a `voice` continuation. The +request then runs as an ordinary user message, and Codex's history keeps a stub +of the interrupted turn. Captions are ephemeral and are not persisted. + +A thread needs an existing provider session binding before voice can start, +because Codex attaches realtime to an existing provider thread. diff --git a/docs/user/providers-codex.md b/docs/user/providers-codex.md index bdc87902a63b2..b044b74cc8c6e 100644 --- a/docs/user/providers-codex.md +++ b/docs/user/providers-codex.md @@ -92,6 +92,29 @@ in the thread on web, desktop, or mobile. Some tools offer access for one reques the current session, or permanently. See [Permission modes](./permission-modes.md) for command and file approvals. +## Voice conversations + +A Codex thread can hold a spoken conversation. You talk with +a voice model that hands coding requests to Codex and reads its answers back to +you. Send at least one message in the thread first. + +Start or stop it with the waveform button in the composer. On web and desktop you +can also send `/voice` or press `F8`. While it runs, the strip above the composer shows whether it is +listening or speaking, with buttons to mute your microphone and stop. The command +palette offers the same actions, and **Voice: Toggle Mute** can be bound in +**Settings > Keybindings**. + +Choose the voice in **Settings > Providers > Codex > Voice**. The change applies +to your next conversation. + +On mobile, the conversation stops when you leave the thread or the app. The mobile +app needs an update that includes voice support. + +Voice needs a microphone and permission to use it. When you open T3 Code from +another device, the page must be served over HTTPS for the browser to allow the +microphone. Live captions are not saved: the thread keeps only the requests +handed to Codex and Codex's replies. + ## Codex says I hit a usage limit When Codex stops on a usage limit, the thread names the window that ran out and diff --git a/packages/client-runtime/package.json b/packages/client-runtime/package.json index d1b48e1d76f4f..1d69802203530 100644 --- a/packages/client-runtime/package.json +++ b/packages/client-runtime/package.json @@ -135,6 +135,10 @@ "types": "./src/voice-input/index.ts", "default": "./src/voice-input/index.ts" }, + "./voice-mode": { + "types": "./src/voice-mode/index.ts", + "default": "./src/voice-mode/index.ts" + }, "./t3ToolSummary": { "types": "./src/t3ToolSummary.ts", "default": "./src/t3ToolSummary.ts" diff --git a/packages/client-runtime/src/rpc/client.ts b/packages/client-runtime/src/rpc/client.ts index 2832a4e231556..81476d470ebcd 100644 --- a/packages/client-runtime/src/rpc/client.ts +++ b/packages/client-runtime/src/rpc/client.ts @@ -77,6 +77,7 @@ export type EnvironmentStreamCommandRpcTag = | typeof WS_METHODS.chatGptHandoffSubscribe | typeof WS_METHODS.cloudInstallRelayClient | typeof WS_METHODS.serverUpdateServerWithProgress + | typeof WS_METHODS.providerVoiceSession | typeof WS_METHODS.gitRunStackedAction; export type EnvironmentStreamRpcTag = diff --git a/packages/client-runtime/src/voice-mode/controller.test.ts b/packages/client-runtime/src/voice-mode/controller.test.ts new file mode 100644 index 0000000000000..710904adc497c --- /dev/null +++ b/packages/client-runtime/src/voice-mode/controller.test.ts @@ -0,0 +1,194 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { + appendVoiceCaption, + finishVoiceCaption, + VoiceModeController, + type VoiceMediaStream, + type VoiceModePlatform, + type VoicePeerConnection, + type VoiceSessionHandlers, +} from "./controller.ts"; + +const target = { environmentId: "env-1", threadId: "thread-1" }; + +function makeFakePlatform(options: { readonly denyMicrophone?: boolean } = {}) { + const track = { enabled: true, stopped: false, stop: () => void (track.stopped = true) }; + const microphone: VoiceMediaStream = { getTracks: () => [track], getAudioTracks: () => [track] }; + const listeners = new Map void>(); + const peer = { + connectionState: "new", + iceGatheringState: "complete", + localDescription: { sdp: "v=offer\r\n" }, + closed: false, + remoteSdp: null as string | null, + dataChannels: [] as string[], + addTrack: () => undefined, + createDataChannel: (label: string) => void peer.dataChannels.push(label), + createOffer: async () => ({ sdp: "v=offer\r\n" }), + setLocalDescription: async () => undefined, + setRemoteDescription: async (description: { sdp: string }) => { + peer.remoteSdp = description.sdp; + }, + getStats: async () => ({ forEach: () => undefined }), + addEventListener: (type: string, listener: (event: never) => void) => + void listeners.set(type, listener), + close: () => void (peer.closed = true), + connect: () => { + peer.connectionState = "connected"; + listeners.get("connectionstatechange")?.({} as never); + }, + }; + const sessions: Array<{ + readonly offerSdp: string; + readonly handlers: VoiceSessionHandlers; + cancelled: boolean; + }> = []; + const platform: VoiceModePlatform = { + createPeerConnection: () => peer as unknown as VoicePeerConnection, + getMicrophone: async () => { + if (options.denyMicrophone) { + throw Object.assign(new Error("denied"), { name: "NotAllowedError" }); + } + return microphone; + }, + playRemoteAudio: () => () => undefined, + openSession: (input, handlers) => { + const session = { offerSdp: input.offerSdp, handlers, cancelled: false }; + sessions.push(session); + return () => void (session.cancelled = true); + }, + }; + return { platform, peer, track, sessions }; +} + +async function startConversation() { + const fake = makeFakePlatform(); + const controller = new VoiceModeController(fake.platform); + await controller.start(target); + const session = fake.sessions[0]!; + return { ...fake, controller, session }; +} + +describe("VoiceModeController", () => { + it("sends the locally gathered ICE candidates with the offer", async () => { + const fake = makeFakePlatform(); + const ready = Promise.withResolvers(); + fake.peer.iceGatheringState = "gathering"; + fake.peer.setLocalDescription = async () => { + ready.resolve(); + }; + const controller = new VoiceModeController(fake.platform); + const starting = controller.start(target); + await ready.promise; + expect(fake.sessions).toHaveLength(0); + fake.peer.localDescription = { sdp: "v=offer\r\na=candidate:gathered\r\n" }; + fake.peer.iceGatheringState = "complete"; + await starting; + expect(fake.sessions[0]?.offerSdp).toContain("a=candidate:gathered"); + controller.stop(); + }); + + it("checks the provider session before requesting microphone access", async () => { + const fake = makeFakePlatform(); + let microphoneRequested = false; + const controller = new VoiceModeController({ + ...fake.platform, + validateTarget: () => Promise.reject(new Error("Send a message first.")), + getMicrophone: async () => { + microphoneRequested = true; + return fake.platform.getMicrophone(); + }, + }); + await controller.start(target); + expect(microphoneRequested).toBe(false); + expect(fake.sessions).toHaveLength(0); + expect(controller.getState().error).toContain("Send a message first."); + }); + + it("negotiates WebRTC through the voice session and activates once both sides are up", async () => { + const { controller, peer, session } = await startConversation(); + expect(peer.dataChannels).toEqual(["oai-events"]); + expect(session.offerSdp).toBe("v=offer\r\n"); + expect(controller.getState().phase).toBe("connecting"); + + session.handlers.onEvent({ type: "answer", sdp: "v=answer\r\n" }); + await Promise.resolve(); + expect(peer.remoteSdp).toBe("v=answer\r\n"); + + session.handlers.onEvent({ type: "started" }); + expect(controller.getState().phase).toBe("connecting"); + peer.connect(); + expect(controller.getState().phase).toBe("active"); + controller.stop(); + }); + + it("keeps ephemeral captions and reports why Codex ended the conversation", async () => { + const { controller, peer, session, track } = await startConversation(); + session.handlers.onEvent({ type: "transcript.delta", role: "user", delta: "run the" }); + session.handlers.onEvent({ type: "transcript.delta", role: "user", delta: " tests" }); + expect(controller.getState().captions).toEqual([ + { role: "user", text: "run the tests", final: false }, + ]); + + session.handlers.onEvent({ type: "closed", reason: "transport_closed" }); + const state = controller.getState(); + expect(state.phase).toBe("idle"); + expect(state.captions).toEqual([]); + expect(state.endedReason).toBe("transport_closed"); + expect(state.target).toEqual(target); + expect(peer.closed).toBe(true); + expect(track.stopped).toBe(true); + }); + + it("ends the server session and releases the microphone when the user stops", async () => { + const { controller, peer, session, track } = await startConversation(); + controller.toggle(target); + expect(session.cancelled).toBe(true); + expect(peer.closed).toBe(true); + expect(track.stopped).toBe(true); + expect(controller.getState()).toMatchObject({ phase: "idle", endedReason: null, error: null }); + }); + + it("mutes by disabling the microphone track", async () => { + const { controller, track } = await startConversation(); + controller.toggleMuted(); + expect(track.enabled).toBe(false); + expect(controller.getState().muted).toBe(true); + controller.toggleMuted(); + expect(track.enabled).toBe(true); + controller.stop(); + }); + + it("explains a denied microphone without opening a session", async () => { + const fake = makeFakePlatform({ denyMicrophone: true }); + const controller = new VoiceModeController(fake.platform); + await controller.start(target); + expect(fake.sessions).toHaveLength(0); + expect(controller.getState()).toMatchObject({ + phase: "idle", + error: "Microphone access was denied.", + }); + }); +}); + +describe("voice captions", () => { + it("interleaves speakers and settles each with its final text", () => { + let captions = appendVoiceCaption([], "user", "hello"); + captions = appendVoiceCaption(captions, "assistant", "Hi"); + captions = appendVoiceCaption(captions, "user", " there"); + captions = finishVoiceCaption(captions, "user", "Hello there."); + expect(captions).toEqual([ + { role: "user", text: "Hello there.", final: true }, + { role: "assistant", text: "Hi", final: false }, + ]); + }); + + it("keeps only the most recent captions", () => { + let captions = finishVoiceCaption([], "user", "one"); + for (const text of ["two", "three", "four", "five"]) { + captions = finishVoiceCaption(captions, "assistant", text); + } + expect(captions.map((caption) => caption.text)).toEqual(["two", "three", "four", "five"]); + }); +}); diff --git a/packages/client-runtime/src/voice-mode/controller.ts b/packages/client-runtime/src/voice-mode/controller.ts new file mode 100644 index 0000000000000..2f69757de78e7 --- /dev/null +++ b/packages/client-runtime/src/voice-mode/controller.ts @@ -0,0 +1,462 @@ +// @effect-diagnostics globalTimers:off - WebRTC lifecycle runs on platform callbacks, outside an Effect runtime. +import type { ProviderVoiceSessionEvent, ProviderVoiceTranscriptRole } from "@t3tools/contracts"; + +/** + * Voice conversations (Codex realtime voice). The client owns the microphone, + * speaker, and WebRTC peer; the environment only relays the SDP handshake and + * captions over `provider.voice.session`. Audio never crosses the T3 socket, + * so voice works the same locally, over a relay, or through a tunnel. + * + * The WebRTC surface is structural so browsers and react-native-webrtc both fit. + */ + +export interface VoiceMediaTrack { + enabled: boolean; + stop(): void; +} + +export interface VoiceMediaStream { + getTracks(): ReadonlyArray; + getAudioTracks(): ReadonlyArray; + /** react-native-webrtc frees the native stream here; browsers have no equivalent. */ + release?(): void; +} + +interface VoiceSessionDescription { + readonly sdp?: string | null | undefined; +} + +interface VoiceStatsReport { + forEach(callback: (stat: { readonly type?: string; readonly kind?: string }) => void): void; +} + +export interface VoicePeerConnection { + readonly connectionState: string; + readonly iceGatheringState: string; + readonly localDescription: VoiceSessionDescription | null; + addTrack(track: never, stream: never): unknown; + createDataChannel(label: string, options?: { readonly ordered?: boolean }): unknown; + createOffer(): Promise; + setLocalDescription(description: never): Promise; + setRemoteDescription(description: never): Promise; + getStats(): Promise; + addEventListener( + type: "track" | "connectionstatechange" | "icegatheringstatechange", + listener: (event: never) => void, + ): void; + removeEventListener?(type: "icegatheringstatechange", listener: () => void): void; + close(): void; +} + +export interface VoiceSessionTarget { + readonly environmentId: string; + readonly threadId: string; +} + +export interface VoiceSessionHandlers { + readonly onEvent: (event: ProviderVoiceSessionEvent) => void; + readonly onError: (message: string) => void; + readonly onEnd: () => void; +} + +export interface VoiceModePlatform { + readonly validateTarget?: (target: VoiceSessionTarget) => Promise; + readonly createPeerConnection: () => VoicePeerConnection; + /** Resolves an echo-cancelled microphone stream, prompting for permission if needed. */ + readonly getMicrophone: () => Promise; + /** Plays the assistant's audio. Returns a disposer. */ + readonly playRemoteAudio: (stream: VoiceMediaStream) => () => void; + /** Opens `provider.voice.session`. Returns a cancel function that ends the conversation. */ + readonly openSession: ( + input: VoiceSessionTarget & { readonly offerSdp: string }, + handlers: VoiceSessionHandlers, + ) => () => void; +} + +export type VoiceModePhase = "idle" | "connecting" | "active"; + +export interface VoiceCaption { + readonly role: ProviderVoiceTranscriptRole; + readonly text: string; + readonly final: boolean; +} + +export interface VoiceModeState { + readonly phase: VoiceModePhase; + /** The conversation's thread; kept after it ends so its error or end notice can show there. */ + readonly target: VoiceSessionTarget | null; + readonly muted: boolean; + /** Recent captions, oldest first. Ephemeral: they vanish when the conversation ends. */ + readonly captions: ReadonlyArray; + readonly error: string | null; + /** Why the last conversation ended when the user did not end it. */ + readonly endedReason: string | null; +} + +/** Mic and speaker loudness in 0..1, sampled while a conversation is active. */ +export interface VoiceLevels { + readonly microphone: number; + readonly speaker: number; +} + +const VOICE_CONNECT_TIMEOUT_MS = 20_000; +const LEVEL_SAMPLE_INTERVAL_MS = 100; +const MAX_CAPTIONS = 4; +const MAX_CAPTION_CHARS = 1024; + +export const INITIAL_VOICE_MODE_STATE: VoiceModeState = { + phase: "idle", + target: null, + muted: false, + captions: [], + error: null, + endedReason: null, +}; + +/** Appends a streaming caption delta, continuing the speaker's open caption. */ +export function appendVoiceCaption( + captions: ReadonlyArray, + role: ProviderVoiceTranscriptRole, + delta: string, +): ReadonlyArray { + const openIndex = captions.findLastIndex((caption) => caption.role === role && !caption.final); + if (openIndex === -1) { + return [...captions, { role, text: delta.trimStart(), final: false }].slice(-MAX_CAPTIONS); + } + const open = captions[openIndex]!; + const text = (open.text + delta).slice(-MAX_CAPTION_CHARS); + return captions.map((caption, index) => (index === openIndex ? { ...caption, text } : caption)); +} + +/** Settles the speaker's open caption with the final transcript text. */ +export function finishVoiceCaption( + captions: ReadonlyArray, + role: ProviderVoiceTranscriptRole, + text: string, +): ReadonlyArray { + const finalText = text.trim().slice(-MAX_CAPTION_CHARS); + const openIndex = captions.findLastIndex((caption) => caption.role === role && !caption.final); + if (openIndex === -1) { + return finalText + ? [...captions, { role, text: finalText, final: true }].slice(-MAX_CAPTIONS) + : captions; + } + if (!finalText) return captions.filter((_, index) => index !== openIndex); + return captions.map((caption, index) => + index === openIndex ? { role, text: finalText, final: true } : caption, + ); +} + +/** User-facing text for why a conversation ended without the user stopping it. */ +export function describeVoiceEnd(reason: string): string { + switch (reason) { + case "replaced": + return "Voice conversation moved to another window or device."; + case "ended": + return "Voice conversation ended."; + default: + return `Voice conversation ended: ${reason.replaceAll("_", " ")}.`; + } +} + +function describeError(cause: unknown): string { + if (cause && typeof cause === "object" && "name" in cause) { + const name = String((cause as { name: unknown }).name); + if (name === "NotAllowedError" || name === "SecurityError") { + return "Microphone access was denied."; + } + if (name === "NotFoundError") return "No microphone was found."; + } + if (cause instanceof Error && cause.message) return cause.message; + return "Voice conversation failed."; +} + +interface ActiveSession { + readonly target: VoiceSessionTarget; + readonly id: number; + microphone: VoiceMediaStream | null; + peer: VoicePeerConnection | null; + cancelSession: (() => void) | null; + cancelIceGathering: (() => void) | null; + stopRemoteAudio: (() => void) | null; + backendStarted: boolean; + peerConnected: boolean; + connectTimer: ReturnType | null; + levelTimer: ReturnType | null; + sampling: boolean; +} + +/** + * Owns at most one voice conversation. Starting on another thread ends the + * current one first, since the microphone is exclusive. + */ +export class VoiceModeController { + private state: VoiceModeState = INITIAL_VOICE_MODE_STATE; + private session: ActiveSession | null = null; + private nextSessionId = 1; + private readonly listeners = new Set<(state: VoiceModeState) => void>(); + private readonly levelListeners = new Set<(levels: VoiceLevels) => void>(); + + private readonly platform: VoiceModePlatform; + + constructor(platform: VoiceModePlatform) { + this.platform = platform; + } + + getState = (): VoiceModeState => this.state; + + subscribe = (listener: (state: VoiceModeState) => void): (() => void) => { + this.listeners.add(listener); + return () => this.listeners.delete(listener); + }; + + subscribeLevels = (listener: (levels: VoiceLevels) => void): (() => void) => { + this.levelListeners.add(listener); + return () => this.levelListeners.delete(listener); + }; + + isActiveFor(target: VoiceSessionTarget): boolean { + const current = this.state.target; + return ( + this.state.phase !== "idle" && + current !== null && + current.environmentId === target.environmentId && + current.threadId === target.threadId + ); + } + + toggle(target: VoiceSessionTarget): void { + if (this.isActiveFor(target)) { + this.stop(); + } else { + void this.start(target); + } + } + + async start(target: VoiceSessionTarget): Promise { + this.stop(); + const session: ActiveSession = { + target, + id: this.nextSessionId++, + microphone: null, + peer: null, + cancelSession: null, + cancelIceGathering: null, + stopRemoteAudio: null, + backendStarted: false, + peerConnected: false, + connectTimer: null, + levelTimer: null, + sampling: false, + }; + this.session = session; + this.setState({ + ...INITIAL_VOICE_MODE_STATE, + phase: "connecting", + target, + muted: this.state.muted, + }); + + try { + await this.platform.validateTarget?.(target); + if (this.session !== session) return; + const microphone = await this.platform.getMicrophone(); + if (this.session !== session) { + for (const track of microphone.getTracks()) track.stop(); + microphone.release?.(); + return; + } + session.microphone = microphone; + // Timed from here so a first-run permission prompt doesn't count against it. + session.connectTimer = setTimeout( + () => this.fail(session, "Voice connection timed out."), + VOICE_CONNECT_TIMEOUT_MS, + ); + for (const track of microphone.getAudioTracks()) track.enabled = !this.state.muted; + + const peer = this.platform.createPeerConnection(); + session.peer = peer; + peer.addEventListener( + "track", + (event: { readonly streams?: ReadonlyArray }) => { + const [remote] = event.streams ?? []; + if (!remote || this.session !== session) return; + session.stopRemoteAudio?.(); + session.stopRemoteAudio = this.platform.playRemoteAudio(remote); + }, + ); + peer.addEventListener("connectionstatechange", () => { + if (this.session !== session) return; + if (peer.connectionState === "connected") { + session.peerConnected = true; + this.maybeActivate(session); + } else if (peer.connectionState === "failed") { + this.fail(session, "Voice connection was lost."); + } + }); + for (const track of microphone.getAudioTracks()) { + peer.addTrack(track as never, microphone as never); + } + // Codex's realtime transport requires the ordered events channel in the offer. + peer.createDataChannel("oai-events", { ordered: true }); + const offer = await peer.createOffer(); + await peer.setLocalDescription(offer as never); + if (this.session !== session) return; + if (peer.iceGatheringState !== "complete") { + await new Promise((resolve) => { + const finish = () => { + peer.removeEventListener?.("icegatheringstatechange", onChange); + session.cancelIceGathering = null; + resolve(); + }; + const onChange = () => { + if (peer.iceGatheringState === "complete") finish(); + }; + session.cancelIceGathering = finish; + peer.addEventListener("icegatheringstatechange", onChange); + onChange(); + }); + } + if (this.session !== session) return; + const offerSdp = peer.localDescription?.sdp; + if (!offerSdp) throw new Error("Could not create a voice connection offer."); + + session.cancelSession = this.platform.openSession( + { ...target, offerSdp }, + { + onEvent: (event) => this.handleEvent(session, event), + onError: (message) => this.fail(session, message), + onEnd: () => this.end(session, null), + }, + ); + } catch (cause) { + this.fail(session, describeError(cause)); + } + } + + stop(): void { + const session = this.session; + if (!session) return; + this.teardown(session); + this.setState({ ...INITIAL_VOICE_MODE_STATE, muted: this.state.muted }); + } + + setMuted(muted: boolean): void { + for (const track of this.session?.microphone?.getAudioTracks() ?? []) track.enabled = !muted; + this.setState({ ...this.state, muted }); + } + + toggleMuted(): void { + this.setMuted(!this.state.muted); + } + + /** Clears a finished conversation's error or end notice. */ + dismiss(): void { + if (this.state.phase !== "idle") return; + this.setState({ ...this.state, error: null, endedReason: null }); + } + + private handleEvent(session: ActiveSession, event: ProviderVoiceSessionEvent): void { + if (this.session !== session) return; + switch (event.type) { + case "answer": + session.peer + ?.setRemoteDescription({ type: "answer", sdp: event.sdp } as never) + .catch((cause: unknown) => this.fail(session, describeError(cause))); + return; + case "started": + session.backendStarted = true; + this.maybeActivate(session); + return; + case "transcript.delta": + this.setState({ + ...this.state, + captions: appendVoiceCaption(this.state.captions, event.role, event.delta), + }); + return; + case "transcript.done": + this.setState({ + ...this.state, + captions: finishVoiceCaption(this.state.captions, event.role, event.text), + }); + return; + case "closed": + this.end(session, event.reason ?? null); + return; + } + } + + private maybeActivate(session: ActiveSession): void { + if (!session.backendStarted || !session.peerConnected || this.state.phase === "active") return; + if (session.connectTimer) clearTimeout(session.connectTimer); + session.connectTimer = null; + session.levelTimer = setInterval(() => this.sampleLevels(session), LEVEL_SAMPLE_INTERVAL_MS); + this.setState({ ...this.state, phase: "active" }); + } + + private sampleLevels(session: ActiveSession): void { + // Native getStats serializes the whole report, so never stack calls. + if (this.levelListeners.size === 0 || !session.peer || session.sampling) return; + session.sampling = true; + session.peer + .getStats() + .finally(() => { + session.sampling = false; + }) + .then((report) => { + if (this.session !== session) return; + let microphone = 0; + let speaker = 0; + report.forEach((stat) => { + const level = (stat as { readonly audioLevel?: unknown }).audioLevel; + if (typeof level !== "number" || stat.kind !== "audio") return; + if (stat.type === "media-source") microphone = Math.max(microphone, level); + else if (stat.type === "inbound-rtp") speaker = Math.max(speaker, level); + }); + const levels = { microphone: this.state.muted ? 0 : microphone, speaker }; + for (const listener of this.levelListeners) listener(levels); + }) + .catch(() => undefined); + } + + private end(session: ActiveSession, reason: string | null): void { + if (this.session !== session) return; + this.teardown(session); + this.setState({ + ...INITIAL_VOICE_MODE_STATE, + target: session.target, + muted: this.state.muted, + endedReason: reason ?? "ended", + }); + } + + private fail(session: ActiveSession, message: string): void { + if (this.session !== session) return; + this.teardown(session); + this.setState({ + ...INITIAL_VOICE_MODE_STATE, + target: session.target, + muted: this.state.muted, + error: message, + }); + } + + private teardown(session: ActiveSession): void { + if (this.session === session) this.session = null; + if (session.connectTimer) clearTimeout(session.connectTimer); + if (session.levelTimer) clearInterval(session.levelTimer); + session.cancelIceGathering?.(); + session.cancelSession?.(); + session.stopRemoteAudio?.(); + session.peer?.close(); + for (const track of session.microphone?.getTracks() ?? []) track.stop(); + session.microphone?.release?.(); + for (const listener of this.levelListeners) listener({ microphone: 0, speaker: 0 }); + } + + private setState(next: VoiceModeState): void { + this.state = next; + // Read the latest state per listener: an earlier listener may have changed it. + for (const listener of [...this.listeners]) listener(this.state); + } +} diff --git a/packages/client-runtime/src/voice-mode/index.ts b/packages/client-runtime/src/voice-mode/index.ts new file mode 100644 index 0000000000000..f05fef5af545b --- /dev/null +++ b/packages/client-runtime/src/voice-mode/index.ts @@ -0,0 +1,16 @@ +export { + INITIAL_VOICE_MODE_STATE, + VoiceModeController, + describeVoiceEnd, + type VoiceCaption, + type VoiceLevels, + type VoiceMediaStream, + type VoiceMediaTrack, + type VoiceModePhase, + type VoiceModePlatform, + type VoiceModeState, + type VoicePeerConnection, + type VoiceSessionHandlers, + type VoiceSessionTarget, +} from "./controller.ts"; +export { makeVoiceSessionOpener, makeVoiceSessionValidator } from "./session.ts"; diff --git a/packages/client-runtime/src/voice-mode/session.ts b/packages/client-runtime/src/voice-mode/session.ts new file mode 100644 index 0000000000000..10364b98da7d2 --- /dev/null +++ b/packages/client-runtime/src/voice-mode/session.ts @@ -0,0 +1,102 @@ +import { + EnvironmentId, + ThreadId, + WS_METHODS, + ORCHESTRATION_V2_WS_METHODS, +} from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; +import * as Data from "effect/Data"; +import * as Effect from "effect/Effect"; +import type * as Fiber from "effect/Fiber"; +import * as Stream from "effect/Stream"; + +import { EnvironmentRegistry } from "../connection/registry.ts"; +import { request, runStream } from "../rpc/client.ts"; +import type { VoiceModePlatform } from "./controller.ts"; + +class VoiceSessionUnavailable extends Data.TaggedError("VoiceSessionUnavailable")<{ + readonly message: string; +}> {} + +function failureMessage(cause: Cause.Cause): string { + const error = Cause.squash(cause); + if (error && typeof error === "object") { + if ("detail" in error && typeof error.detail === "string") return error.detail; + if ("message" in error && typeof error.message === "string" && error.message) { + return error.message; + } + } + return "Voice conversation failed."; +} + +/** + * Builds `VoiceModePlatform.openSession` on an app runtime that provides the + * environment registry. Interrupting the returned fiber ends the conversation + * server-side. + */ +export function makeVoiceSessionOpener( + runFork: (effect: Effect.Effect) => Fiber.Fiber, +): VoiceModePlatform["openSession"] { + return ({ environmentId, threadId, offerSdp }, handlers) => { + const fiber = runFork( + EnvironmentRegistry.pipe( + Effect.flatMap((registry) => + registry + .runStream( + EnvironmentId.make(environmentId), + runStream(WS_METHODS.providerVoiceSession, { + threadId: ThreadId.make(threadId), + offerSdp, + }), + ) + .pipe(Stream.runForEach((event) => Effect.sync(() => handlers.onEvent(event)))), + ), + Effect.matchCause({ + onFailure: (cause) => { + if (!Cause.hasInterruptsOnly(cause)) handlers.onError(failureMessage(cause)); + }, + onSuccess: () => handlers.onEnd(), + }), + ), + ); + return () => fiber.interruptUnsafe(); + }; +} + +/** Refreshes the bound provider session before requesting microphone access. */ +export function makeVoiceSessionValidator( + runFork: (effect: Effect.Effect) => Fiber.Fiber, +): NonNullable { + return ({ environmentId, threadId }) => + new Promise((resolve, reject) => { + runFork( + Effect.gen(function* () { + const registry = yield* EnvironmentRegistry; + const projection = yield* registry.run( + EnvironmentId.make(environmentId), + request(ORCHESTRATION_V2_WS_METHODS.getThreadProjection, { + threadId: ThreadId.make(threadId), + }), + ); + const providerThread = projection.providerThreads.find( + (thread) => thread.id === projection.thread.activeProviderThreadId, + ); + const session = projection.providerSessions.find( + (session) => session.id === providerThread?.providerSessionId, + ); + if (!session || !["ready", "running", "waiting"].includes(session.status)) { + return yield* Effect.fail( + new VoiceSessionUnavailable({ + message: "The provider session is no longer running. Send a message first.", + }), + ); + } + }).pipe( + Effect.matchCause({ + onSuccess: () => resolve(), + onFailure: (cause) => reject(new Error(failureMessage(cause))), + }), + ), + ); + }); +} diff --git a/packages/contracts/src/keybindings.ts b/packages/contracts/src/keybindings.ts index 714e0a1d486a1..b2c412f028185 100644 --- a/packages/contracts/src/keybindings.ts +++ b/packages/contracts/src/keybindings.ts @@ -99,6 +99,8 @@ export const STATIC_KEYBINDING_COMMANDS = [ "composer.workspace", "composer.previousWorktree", "composer.branch", + "voice.toggle", + "voice.toggleMute", "chat.new", "chat.newLocal", "chat.newWithoutProject", diff --git a/packages/contracts/src/provider.ts b/packages/contracts/src/provider.ts index e7942e08fddc8..6504791bb2759 100644 --- a/packages/contracts/src/provider.ts +++ b/packages/contracts/src/provider.ts @@ -48,3 +48,56 @@ export class ProviderUploadFeedbackError extends Schema.TaggedError()( + "ProviderVoiceSessionError", + { + threadId: ThreadId, + detail: Schema.String, + cause: Schema.optional(Schema.Defect()), + }, +) { + override get message(): string { + return this.detail; + } +} + diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index 01989ff8a474f..4920d89f9f96f 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -163,6 +163,9 @@ import { ProviderUploadFeedbackError, ProviderUploadFeedbackInput, ProviderUploadFeedbackResult, + ProviderVoiceSessionError, + ProviderVoiceSessionEvent, + ProviderVoiceSessionInput, } from "./provider.ts"; import { ProviderInstanceId, ProviderInstanceMutation } from "./providerInstance.ts"; import { @@ -434,6 +437,7 @@ export const WS_METHODS = { mcpAppsUpdateModelContext: "mcpApps.updateModelContext", providerUploadFeedback: "provider.uploadFeedback", + providerVoiceSession: "provider.voice.session", providerAuthStart: "provider.auth.start", providerConsumeResetCredit: "provider.consumeResetCredit", providerAuthComplete: "provider.auth.complete", @@ -1434,6 +1438,13 @@ const WsProviderUploadFeedbackRpc = Rpc.make(WS_METHODS.providerUploadFeedback, error: Schema.Union([ProviderUploadFeedbackError, EnvironmentAuthorizationError]), }); +const WsProviderVoiceSessionRpc = Rpc.make(WS_METHODS.providerVoiceSession, { + payload: ProviderVoiceSessionInput, + success: ProviderVoiceSessionEvent, + error: Schema.Union([ProviderVoiceSessionError, EnvironmentAuthorizationError]), + stream: true, +}); + const WsSubscribeVcsStatusRpc = Rpc.make(WS_METHODS.subscribeVcsStatus, { payload: VcsStatusSubscriptionInput, success: VcsStatusStreamEvent, @@ -2118,6 +2129,7 @@ export const WsRpcGroup = RpcGroup.make( WsMcpAppsReadResourceRpc, WsMcpAppsUpdateModelContextRpc, WsProviderUploadFeedbackRpc, + WsProviderVoiceSessionRpc, WsSubscribeVcsStatusRpc, WsSubscribeWorktreeSetupRpc, WsWorktreeSetupCancelRpc, diff --git a/packages/contracts/src/server.ts b/packages/contracts/src/server.ts index e027930c116ba..67d0f8e57e5e1 100644 --- a/packages/contracts/src/server.ts +++ b/packages/contracts/src/server.ts @@ -219,6 +219,8 @@ export const ServerProvider = Schema.Struct({ supportedRuntimeModes: Schema.optional(ForwardCompatibleArray(RuntimeMode)), supportsConversationRollback: Schema.optional(Schema.Boolean), supportsTextGeneration: Schema.optional(Schema.Boolean), + // The driver can hold realtime voice conversations (`provider.voice.session`). + supportsVoice: Schema.optional(Schema.Boolean), setup: Schema.optional( Schema.Struct({ canAuthenticate: Schema.Boolean, diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index f291cec115f73..e81c96839a61e 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -627,6 +627,21 @@ export function makeProviderSettingsSchema; + export const CodexSettings = makeProviderSettingsSchema( { setupMode: Schema.optionalKey(Schema.Literals(["managed", "existing"])).pipe( @@ -677,9 +692,21 @@ export const CodexSettings = makeProviderSettingsSchema( Schema.withDecodingDefault(Effect.succeed([])), Schema.annotateKey({ providerSettingsForm: { hidden: true } }), ), + voice: TrimmedString.pipe( + Schema.withDecodingDefault(Effect.succeed("")), + Schema.annotateKey({ + title: "Voice", + description: "Voice used by voice conversations. Applies to the next conversation.", + providerSettingsForm: { + control: "select", + options: CODEX_VOICE_OPTIONS, + clearWhenEmpty: "omit", + }, + }), + ), }, { - order: ["binaryPath", "homePath", "shadowHomePath", "launchArgs"], + order: ["binaryPath", "homePath", "shadowHomePath", "launchArgs", "voice"], }, ); export type CodexSettings = typeof CodexSettings.Type; diff --git a/packages/provider-core/src/server/ProviderAdapter.ts b/packages/provider-core/src/server/ProviderAdapter.ts index a92256d7dd791..09967b2e349a7 100644 --- a/packages/provider-core/src/server/ProviderAdapter.ts +++ b/packages/provider-core/src/server/ProviderAdapter.ts @@ -1,4 +1,7 @@ -import type { OrchestrationV2HistoricalMessage } from "@t3tools/contracts"; +import type { + OrchestrationV2HistoricalMessage, + ProviderVoiceSessionEvent, +} from "@t3tools/contracts"; import { ChatAttachment, CheckpointId, @@ -653,6 +656,14 @@ export interface ProviderAdapterV2SessionRuntime { }) => Effect.Effect<{ readonly feedbackId: string }, ProviderAdapterV2Error>; /** Absent means non-interactive apps with plain tool rows. */ readonly mcpApps?: ProviderAdapterV2McpApps; + /** + * Providers with realtime voice (Codex) hold a voice conversation on the + * thread. Interrupting the stream ends the conversation. + */ + readonly startVoiceSession?: (input: { + readonly providerThread: OrchestrationV2ProviderThread; + readonly offerSdp: string; + }) => Stream.Stream; readonly rollbackThread: ( input: ProviderAdapterV2RollbackThreadInput, ) => Effect.Effect; diff --git a/packages/provider-core/src/server/ProviderContinuationRequests.ts b/packages/provider-core/src/server/ProviderContinuationRequests.ts index 936769c46c3e4..80adc7b6c1e96 100644 --- a/packages/provider-core/src/server/ProviderContinuationRequests.ts +++ b/packages/provider-core/src/server/ProviderContinuationRequests.ts @@ -26,6 +26,11 @@ export interface ProviderContinuationRequest { /** adapter_buffered (default) ingests existing CLI output; message_text delivers app-owned wake text. */ readonly delivery?: "adapter_buffered" | "message_text"; readonly messageId?: MessageId; + /** + * `voice` resubmits a request the user spoke in a voice conversation, so the + * run reads as theirs: user-authored, with no background notification. + */ + readonly origin?: "voice"; readonly dispatchIfCurrent?: ( effect: Effect.Effect, ) => Effect.Effect, E, R>; diff --git a/packages/provider-core/src/server/snapshotProbe.ts b/packages/provider-core/src/server/snapshotProbe.ts index 3303b9d6ff51a..b7f112ea055cc 100644 --- a/packages/provider-core/src/server/snapshotProbe.ts +++ b/packages/provider-core/src/server/snapshotProbe.ts @@ -88,6 +88,7 @@ export interface ServerProviderPresentation { readonly reportsContextWindow?: boolean; readonly supportedRuntimeModes?: ReadonlyArray; readonly supportsConversationRollback?: boolean; + readonly supportsVoice?: boolean; } export type ServerProviderDraft = Omit; @@ -254,6 +255,7 @@ export function buildServerProvider(input: { ...(input.presentation.supportedRuntimeModes === undefined ? {} : { supportedRuntimeModes: [...input.presentation.supportedRuntimeModes] }), + ...(input.presentation.supportsVoice ? { supportsVoice: true } : {}), enabled: input.enabled, installed: input.probe.installed, version: input.probe.version, diff --git a/packages/shared/src/composerTrigger.ts b/packages/shared/src/composerTrigger.ts index 77566a2b09608..d77b906e9b952 100644 --- a/packages/shared/src/composerTrigger.ts +++ b/packages/shared/src/composerTrigger.ts @@ -4,7 +4,7 @@ export type ComposerTriggerKind = | "slash-command" | "slash-model" | "skill"; -export type ComposerSlashCommand = "model" | "plan" | "default"; +export type ComposerSlashCommand = "model" | "plan" | "default" | "voice"; export interface ComposerTrigger { kind: ComposerTriggerKind; diff --git a/packages/shared/src/keybindings.ts b/packages/shared/src/keybindings.ts index 32292c0f9ef21..2c0a86d8b1d58 100644 --- a/packages/shared/src/keybindings.ts +++ b/packages/shared/src/keybindings.ts @@ -210,6 +210,7 @@ export const DEFAULT_KEYBINDINGS: ReadonlyArray = [ { key: "mod+shift+g", command: "composer.branch", when: "!terminalFocus" }, { key: "mod+shift+l", command: "composer.previousWorktree", when: "!terminalFocus" }, { key: "mod+shift+k", command: "pullRequest.copyNumber", when: "!terminalFocus" }, + { key: "f8", command: "voice.toggle", when: "!terminalFocus" }, { key: "mod+shift+arrowup", command: "modelPicker.previousProvider", when: "modelPickerOpen" }, { key: "mod+shift+arrowdown", command: "modelPicker.nextProvider", when: "modelPickerOpen" }, { key: "mod+o", command: "editor.openFavorite" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fa3d823395445..5b3ea4608f803 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -768,6 +768,9 @@ importers: react-native-svg: specifier: 15.16.0 version: 15.16.0(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0) + react-native-webrtc: + specifier: 124.0.8 + version: 124.0.8(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(supports-color@8.1.1) react-native-webview: specifier: 14.0.1 version: 14.0.1(patch_hash=81caa279ec3dc1f81209062133b66e8f824e13f106cca57f1fa732a72c2ab378)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0) @@ -793,6 +796,9 @@ importers: '@babel/core': specifier: ^7.29.7 version: 7.29.7(supports-color@8.1.1) + '@config-plugins/react-native-webrtc': + specifier: 15.0.2 + version: 15.0.2(expo@58.0.14) '@effect/platform-node': specifier: 4.0.5 version: 4.0.5(bufferutil@4.1.0)(effect@4.0.5)(redis@6.2.1)(utf-8-validate@6.0.6) @@ -3422,6 +3428,11 @@ packages: '@coderabbitai/config@1.1.0': resolution: {integrity: sha512-NvQKaoOD7fWb7lMKhid57ABVlIBSczw/S+ePk5YvoPuWdrDkyY+PWdqVtq3TRq7VY8f3SNRlMFjmAtXVab66pw==} + '@config-plugins/react-native-webrtc@15.0.2': + resolution: {integrity: sha512-sH4T7Z4P2RowV91k9CEwEr0unn+396NBexxCNZwRXuJXwguDU1qZWpE6fcwN0730B8uiS83F7+CLTyhpC7qRHQ==} + peerDependencies: + expo: '>=56' + '@connectrpc/connect-web@1.7.0': resolution: {integrity: sha512-qyP0YOnUPRWwCc/VfsoydMJvkb7EyUPr2q9sHgBuJzbADjiqck1gKH5V5ZPzPhTLBvmz5UvG+wiZ5sMRQHU1MQ==} peerDependencies: @@ -8219,6 +8230,15 @@ packages: supports-color: optional: true + debug@4.3.4: + resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + debug@4.4.3: resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} engines: {node: '>=6.0'} @@ -10266,6 +10286,9 @@ packages: ms@2.0.0: resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==} + ms@2.1.2: + resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==} + ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} @@ -11035,6 +11058,11 @@ packages: peerDependencies: react-native: '*' + react-native-webrtc@124.0.8: + resolution: {integrity: sha512-uuQxvmk+mvnk5U0tr+1N42sKZqgm41fJrBA+fmCvML9J9P4roSh2So82t5RHAlu/vE9vxu5AKgivAiH61clCBg==} + peerDependencies: + react-native: '>=0.60.0' + react-native-webview@14.0.1: resolution: {integrity: sha512-lBUuvFk459Mo+5vUNqwzGvn+I8rDJE6zvOulG2hWWudeEOJfWTsQwOfuYnRu+27djUIXrESFfC+EQE3PGu+FWA==} peerDependencies: @@ -13441,6 +13469,10 @@ snapshots: dependencies: zod: 4.5.4 + '@config-plugins/react-native-webrtc@15.0.2(expo@58.0.14)': + dependencies: + expo: 58.0.14(277a83826bb26ba15256633fefeb0bc2) + '@connectrpc/connect-web@1.7.0(@bufbuild/protobuf@1.10.0)(@connectrpc/connect@1.7.0(@bufbuild/protobuf@1.10.0))': dependencies: '@bufbuild/protobuf': 1.10.0 @@ -17952,6 +17984,12 @@ snapshots: optionalDependencies: supports-color: 8.1.1 + debug@4.3.4(supports-color@8.1.1): + dependencies: + ms: 2.1.2 + optionalDependencies: + supports-color: 8.1.1 + debug@4.4.3(supports-color@8.1.1): dependencies: ms: 2.1.3 @@ -20400,6 +20438,8 @@ snapshots: ms@2.0.0: {} + ms@2.1.2: {} + ms@2.1.3: {} muggle-string@0.4.1: {} @@ -21309,6 +21349,14 @@ snapshots: dependencies: react-native: 0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6) + react-native-webrtc@124.0.8(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(supports-color@8.1.1): + dependencies: + base64-js: 1.5.1 + debug: 4.3.4(supports-color@8.1.1) + react-native: 0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6) + transitivePeerDependencies: + - supports-color + react-native-webview@14.0.1(patch_hash=81caa279ec3dc1f81209062133b66e8f824e13f106cca57f1fa732a72c2ab378)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0): dependencies: escape-string-regexp: 4.0.0 diff --git a/scripts/build-desktop-artifact.ts b/scripts/build-desktop-artifact.ts index 5de3c068fc6c6..8d6ed598d9d5d 100644 --- a/scripts/build-desktop-artifact.ts +++ b/scripts/build-desktop-artifact.ts @@ -2844,6 +2844,8 @@ export const createBuildConfig = Effect.fn("createBuildConfig")(function* ( LSItemContentTypes: ["public.html", "public.xhtml"], }, ], + NSMicrophoneUsageDescription: + "T3 Code uses the microphone when you start a voice conversation with an agent.", }, protocols: [ { diff --git a/third-party-licenses.config.json b/third-party-licenses.config.json index e6d38a17e2d87..70dfd6309ff58 100644 --- a/third-party-licenses.config.json +++ b/third-party-licenses.config.json @@ -140,6 +140,21 @@ } ] }, + { + "bundles": ["android", "ios", "mobile"], + "license": "BSD-3-Clause", + "name": "WebRTC", + "sourceUrl": "https://webrtc.googlesource.com/src", + "generatedNotices": [ + { + "licenseId": "BSD-3-Clause", + "copyrights": ["Copyright (c) 2011, The WebRTC project authors. All rights reserved."], + "preamble": [ + "WebRTC\n\nVoice conversations use the libwebrtc build that react-native-webrtc links into the iOS and Android apps." + ] + } + ] + }, { "bundles": ["android", "assets", "mobile"], "license": "Apache-2.0", From e397aec4c755c4a124f26124d82e5ec6a04bcabf Mon Sep 17 00:00:00 2001 From: Jake Leventhal Date: Sat, 24 Oct 2026 14:11:03 -0400 Subject: [PATCH 02/19] fix(voice): only interrupt voice handoffs that can be resubmitted Codex voice handoffs are interrupted so they can be resubmitted as ordinary runs. Without a continuation sink the request would be lost, so leave such turns alone. Also pin the replacement-session ordering: the predecessor is stopped once, before the successor starts. Co-Authored-By: Claude Opus 5.5 --- .../src/orchestration-v2/Adapters/CodexAdapterV2.ts | 2 ++ apps/server/src/provider/CodexVoice.test.ts | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index 6d9fc799b30ae..bc00d9dbf303a 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -4809,8 +4809,10 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( if (payload.item.type === "userMessage") { const handoff = readRealtimeDelegationFromItem(payload.item); const voiceThread = yield* Ref.get(voiceProviderThread); + // Without a continuation sink an interrupted handoff could not be resubmitted. if ( handoff !== undefined && + continuationRequests !== undefined && voiceThread !== undefined && voiceThread.nativeThreadRef?.nativeId === payload.threadId && !(yield* Ref.get(activeTurns)).has(payload.turnId) && diff --git a/apps/server/src/provider/CodexVoice.test.ts b/apps/server/src/provider/CodexVoice.test.ts index cc4ebcb4df910..5d1b5e928d895 100644 --- a/apps/server/src/provider/CodexVoice.test.ts +++ b/apps/server/src/provider/CodexVoice.test.ts @@ -214,6 +214,14 @@ describe("makeCodexVoice", () => { { type: "answer", sdp: "replacement-answer" }, { type: "closed", reason: "replacement-done" }, ]); + // The predecessor is stopped once, before the successor starts, and + // its finalizer must not stop the successor's session. + NodeAssert.deepEqual( + fake.requests + .map((request) => request.method) + .filter((method) => method.startsWith("thread/realtime/")), + ["thread/realtime/start", "thread/realtime/stop", "thread/realtime/start"], + ); }), ), ); From e25906cbf3cb589b6294b11f21c5181dc88adb23 Mon Sep 17 00:00:00 2001 From: Jake Leventhal Date: Mon, 26 Oct 2026 09:26:46 -0400 Subject: [PATCH 03/19] refactor(voice): trace session teardown --- apps/server/src/provider/CodexVoice.ts | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/apps/server/src/provider/CodexVoice.ts b/apps/server/src/provider/CodexVoice.ts index 2287707834f74..2f4ae6ba8dbc0 100644 --- a/apps/server/src/provider/CodexVoice.ts +++ b/apps/server/src/provider/CodexVoice.ts @@ -226,15 +226,14 @@ export const makeCodexVoice = Effect.fn("makeCodexVoice")(function* ( ), ); - const stop = (active: ActiveVoice) => - Effect.gen(function* () { - yield* Ref.update(activeRef, (current) => (current === active ? undefined : current)); - if (active.closed) return; - active.closed = true; - yield* client.raw - .request("thread/realtime/stop", { threadId: active.providerThreadId }) - .pipe(Effect.ignore); - }); + const stop = Effect.fn("CodexVoice.stop")(function* (active: ActiveVoice) { + yield* Ref.update(activeRef, (current) => (current === active ? undefined : current)); + if (active.closed) return; + active.closed = true; + yield* client.raw + .request("thread/realtime/stop", { threadId: active.providerThreadId }) + .pipe(Effect.ignore); + }); /** * Starts a voice conversation. The stream ends when Codex closes the From 1fa4024c49d5daea2bde5d70b122943f0fb20eb7 Mon Sep 17 00:00:00 2001 From: Jake Leventhal Date: Mon, 26 Oct 2026 09:55:17 -0400 Subject: [PATCH 04/19] fix(voice): preserve upstream microphone permissions and validation --- .../src/features/threads/ThreadDetailScreen.tsx | 1 + .../src/features/voice-mode/ComposerVoiceMode.tsx | 1 + apps/mobile/src/features/voice-mode/useVoiceMode.ts | 3 +++ apps/mobile/src/features/voice-mode/voiceMode.ts | 9 ++------- apps/server/src/provider/CodexVoice.test.ts | 4 +++- apps/server/src/provider/CodexVoice.ts | 12 ++---------- apps/server/src/ws.ts | 1 - apps/web/src/components/chat/VoiceModeStrip.tsx | 1 + packages/client-runtime/src/voice-mode/controller.ts | 11 +++-------- scripts/build-desktop-artifact.ts | 4 +--- 10 files changed, 17 insertions(+), 30 deletions(-) diff --git a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx index 9734c68ece0c3..b603d71adc6d0 100644 --- a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx +++ b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx @@ -917,6 +917,7 @@ function ThreadDetailBody( null; const voiceTarget = { environmentId: props.environmentId, threadId: props.selectedThread.id }; const voiceLiveHere = useVoiceModePhase(voiceTarget) !== "idle"; + // Keep live quota data visible, but close the panel when a turn consumes quota. const [usageLimitsPanel, setUsageLimitsPanel] = useState<{ readonly key: string; readonly threadKey: string; diff --git a/apps/mobile/src/features/voice-mode/ComposerVoiceMode.tsx b/apps/mobile/src/features/voice-mode/ComposerVoiceMode.tsx index 2808a38945e46..4d590a8e0ca44 100644 --- a/apps/mobile/src/features/voice-mode/ComposerVoiceMode.tsx +++ b/apps/mobile/src/features/voice-mode/ComposerVoiceMode.tsx @@ -108,6 +108,7 @@ export function ComposerVoiceStrip() { // Levels arrive at 10Hz; they drive shared values directly, and the // speaking flag only re-renders when it flips. + // oxlint-disable-next-line t3code/react-effect-budget -- Subscribe native meter animations to the audio level stream. useEffect(() => { const isSpeaking = createSpeakingDetector(); const unsubscribe = voiceMode.subscribeLevels((levels) => { diff --git a/apps/mobile/src/features/voice-mode/useVoiceMode.ts b/apps/mobile/src/features/voice-mode/useVoiceMode.ts index 62ae21e52d708..1c9a3bb64512e 100644 --- a/apps/mobile/src/features/voice-mode/useVoiceMode.ts +++ b/apps/mobile/src/features/voice-mode/useVoiceMode.ts @@ -39,11 +39,13 @@ export function useVoiceModeThreadLifecycle(target: VoiceSessionTarget): void { const params = route?.params; return params && "threadId" in params ? String(params.threadId) : null; }); + // oxlint-disable-next-line t3code/react-effect-budget -- Synchronize the WebRTC lifetime with the visible native thread. useEffect(() => { if (visibleThreadId !== threadId && voiceMode.isActiveFor({ environmentId, threadId })) voiceMode.stop(); }, [environmentId, threadId, visibleThreadId]); + // oxlint-disable-next-line t3code/react-effect-budget -- Release the native call when its owning thread screen unmounts. useEffect( () => () => { if (routeName === "Thread" && voiceMode.isActiveFor({ environmentId, threadId })) @@ -52,6 +54,7 @@ export function useVoiceModeThreadLifecycle(target: VoiceSessionTarget): void { [environmentId, threadId, routeName], ); + // oxlint-disable-next-line t3code/react-effect-budget -- Subscribe to call lifecycle failures and remote endings. useEffect(() => { // Reads the live state: dismissing notifies listeners re-entrantly, so a // listener's argument can already be stale. diff --git a/apps/mobile/src/features/voice-mode/voiceMode.ts b/apps/mobile/src/features/voice-mode/voiceMode.ts index 8730f654efffb..2ab09284cb6c1 100644 --- a/apps/mobile/src/features/voice-mode/voiceMode.ts +++ b/apps/mobile/src/features/voice-mode/voiceMode.ts @@ -60,13 +60,8 @@ export const voiceMode = new VoiceModeController({ openSession: makeVoiceSessionOpener(runOnConnectionRuntime), }); -/* - * WebRTC's iOS audio session uses voice-chat mode, which plays through the - * earpiece. Once the call is live, re-apply play-and-record with the speaker - * as the default route (headphones and Bluetooth still win). WebRTC only sets - * its category when its audio unit starts, which has happened by then. On - * Android the same call turns the speakerphone on. - */ +/* WebRTC resets iOS routing when its audio unit starts; apply speaker routing afterward. + * Headphones and Bluetooth still take precedence. */ async function routeVoiceToSpeaker(): Promise { await setAudioModeAsync({ allowsRecording: true, diff --git a/apps/server/src/provider/CodexVoice.test.ts b/apps/server/src/provider/CodexVoice.test.ts index 5d1b5e928d895..9612e836f2453 100644 --- a/apps/server/src/provider/CodexVoice.test.ts +++ b/apps/server/src/provider/CodexVoice.test.ts @@ -18,6 +18,8 @@ import { readSpeakableAnswer, } from "./CodexVoice.ts"; +const isCodexVoiceSessionError = Schema.is(CodexVoiceSessionError); + const PROVIDER_THREAD = "provider-thread-1"; function delegation(input: string) { @@ -248,7 +250,7 @@ describe("makeCodexVoice", () => { const exit = yield* Fiber.await(voice.fiber); NodeAssert.ok(Exit.isFailure(exit)); const error = exit.cause.reasons[0]; - NodeAssert.ok(error?._tag === "Fail" && Schema.is(CodexVoiceSessionError)(error.error)); + NodeAssert.ok(error?._tag === "Fail" && isCodexVoiceSessionError(error.error)); NodeAssert.equal(error.error.detail, "realtime unavailable"); }), ); diff --git a/apps/server/src/provider/CodexVoice.ts b/apps/server/src/provider/CodexVoice.ts index 2f4ae6ba8dbc0..0332ac8d8508f 100644 --- a/apps/server/src/provider/CodexVoice.ts +++ b/apps/server/src/provider/CodexVoice.ts @@ -1,13 +1,5 @@ -/** - * Codex voice conversations: a thread-scoped realtime session whose audio runs - * over WebRTC between the client and OpenAI. The app-server only relays the - * SDP handshake, captions, and lifecycle; this module owns that signaling. - * - * Handoffs are client-managed, matching the Codex TUI: the realtime model - * delegates work by starting a Codex turn whose user message is wrapped in - * ``, and when that turn finishes we hand its final - * answer back with `thread/realtime/appendSpeech` so the voice reads it out. - */ +/** Thread-scoped WebRTC signaling; audio stays between the client and OpenAI. + * Client handoffs start Codex turns, then append their final answer to live speech. */ import type { ProviderVoiceSessionEvent } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; import * as Effect from "effect/Effect"; diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 60ba12f6faa9e..d4a3fb578fb71 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -75,7 +75,6 @@ import { ProjectWriteFileError, ProjectMutationError, ProviderUploadFeedbackError, - ProviderVoiceSessionError, ProviderSetupError, RelayClientInstallFailedError, type RelayClientInstallProgressEvent, diff --git a/apps/web/src/components/chat/VoiceModeStrip.tsx b/apps/web/src/components/chat/VoiceModeStrip.tsx index 903d559f18097..8cdcf52deb1ef 100644 --- a/apps/web/src/components/chat/VoiceModeStrip.tsx +++ b/apps/web/src/components/chat/VoiceModeStrip.tsx @@ -84,6 +84,7 @@ export const VoiceModeStrip = memo(function VoiceModeStrip({ const speakerMeterRef = useRef(null); const [speaking, setSpeaking] = useState(false); + // oxlint-disable-next-line t3code/react-effect-budget -- Subscribe visible meters to the audio level stream. useEffect(() => { if (!isActive) return; // Teardown publishes a silent sample, which clears `speaking` for the next conversation. diff --git a/packages/client-runtime/src/voice-mode/controller.ts b/packages/client-runtime/src/voice-mode/controller.ts index 2f69757de78e7..b14d7a0107829 100644 --- a/packages/client-runtime/src/voice-mode/controller.ts +++ b/packages/client-runtime/src/voice-mode/controller.ts @@ -1,14 +1,8 @@ // @effect-diagnostics globalTimers:off - WebRTC lifecycle runs on platform callbacks, outside an Effect runtime. import type { ProviderVoiceSessionEvent, ProviderVoiceTranscriptRole } from "@t3tools/contracts"; -/** - * Voice conversations (Codex realtime voice). The client owns the microphone, - * speaker, and WebRTC peer; the environment only relays the SDP handshake and - * captions over `provider.voice.session`. Audio never crosses the T3 socket, - * so voice works the same locally, over a relay, or through a tunnel. - * - * The WebRTC surface is structural so browsers and react-native-webrtc both fit. - */ +/** Clients own WebRTC audio; environments relay only signaling and captions. + * Structural interfaces support both browser and React Native WebRTC. */ export interface VoiceMediaTrack { enabled: boolean; @@ -457,6 +451,7 @@ export class VoiceModeController { private setState(next: VoiceModeState): void { this.state = next; // Read the latest state per listener: an earlier listener may have changed it. + // oxlint-disable-next-line unicorn/no-useless-spread -- A notification can mutate subscriptions re-entrantly; iterate a snapshot. for (const listener of [...this.listeners]) listener(this.state); } } diff --git a/scripts/build-desktop-artifact.ts b/scripts/build-desktop-artifact.ts index 8d6ed598d9d5d..f87decd03934e 100644 --- a/scripts/build-desktop-artifact.ts +++ b/scripts/build-desktop-artifact.ts @@ -2833,7 +2833,7 @@ export const createBuildConfig = Effect.fn("createBuildConfig")(function* ( NSCameraUsageDescription: "T3 Code lets commands run by terminals and coding agents use the camera.", NSMicrophoneUsageDescription: - "T3 Code lets commands run by terminals and coding agents use the microphone.", + "T3 Code uses the microphone for voice conversations and commands run by terminals and coding agents.", // macOS lists an app under Default web browser only when it opens web // pages as documents as well as http and https links (see protocols). CFBundleDocumentTypes: [ @@ -2844,8 +2844,6 @@ export const createBuildConfig = Effect.fn("createBuildConfig")(function* ( LSItemContentTypes: ["public.html", "public.xhtml"], }, ], - NSMicrophoneUsageDescription: - "T3 Code uses the microphone when you start a voice conversation with an agent.", }, protocols: [ { From 987f30c7371492283061dae4aea44216a08abaff Mon Sep 17 00:00:00 2001 From: Jake Leventhal Date: Tue, 27 Oct 2026 15:11:19 -0400 Subject: [PATCH 05/19] fix(voice): stop inspector sessions and bound initial captions --- .../src/features/voice-mode/useVoiceMode.test.tsx | 10 ++++++++-- apps/mobile/src/features/voice-mode/useVoiceMode.ts | 5 ++++- .../client-runtime/src/voice-mode/controller.test.ts | 6 ++++++ packages/client-runtime/src/voice-mode/controller.ts | 5 ++++- packages/client-runtime/src/voice-mode/session.ts | 8 +++----- 5 files changed, 25 insertions(+), 9 deletions(-) diff --git a/apps/mobile/src/features/voice-mode/useVoiceMode.test.tsx b/apps/mobile/src/features/voice-mode/useVoiceMode.test.tsx index fa762657f6be1..e18371cc63a90 100644 --- a/apps/mobile/src/features/voice-mode/useVoiceMode.test.tsx +++ b/apps/mobile/src/features/voice-mode/useVoiceMode.test.tsx @@ -42,7 +42,7 @@ beforeEach(() => { root = createRoot(document.createElement("div")); }); afterEach(() => act(() => root.unmount())); -const render = (overlay: boolean) => +const render = (overlay: boolean, overlayRoute = "ThreadFiles") => act(() => root.render( <> @@ -50,7 +50,7 @@ const render = (overlay: boolean) => {overlay ? ( - + ) : null} @@ -70,3 +70,9 @@ it("stops the previous thread when navigation selects a different thread", () => render(false); expect(fixture.stop).toHaveBeenCalledTimes(1); }); + +it("stops voice when its fullscreen thread closes over the still-mounted thread", () => { + render(true, "ThreadInspectorFullscreen"); + render(false, "ThreadInspectorFullscreen"); + expect(fixture.stop).toHaveBeenCalledTimes(1); +}); diff --git a/apps/mobile/src/features/voice-mode/useVoiceMode.ts b/apps/mobile/src/features/voice-mode/useVoiceMode.ts index 1c9a3bb64512e..4d9edd10ceef6 100644 --- a/apps/mobile/src/features/voice-mode/useVoiceMode.ts +++ b/apps/mobile/src/features/voice-mode/useVoiceMode.ts @@ -48,7 +48,10 @@ export function useVoiceModeThreadLifecycle(target: VoiceSessionTarget): void { // oxlint-disable-next-line t3code/react-effect-budget -- Release the native call when its owning thread screen unmounts. useEffect( () => () => { - if (routeName === "Thread" && voiceMode.isActiveFor({ environmentId, threadId })) + if ( + (routeName === "Thread" || routeName === "ThreadInspectorFullscreen") && + voiceMode.isActiveFor({ environmentId, threadId }) + ) voiceMode.stop(); }, [environmentId, threadId, routeName], diff --git a/packages/client-runtime/src/voice-mode/controller.test.ts b/packages/client-runtime/src/voice-mode/controller.test.ts index 710904adc497c..70fa8579c7375 100644 --- a/packages/client-runtime/src/voice-mode/controller.test.ts +++ b/packages/client-runtime/src/voice-mode/controller.test.ts @@ -173,6 +173,12 @@ describe("VoiceModeController", () => { }); describe("voice captions", () => { + it("bounds the first streaming caption as well as later deltas", () => { + const text = "a".repeat(1500) + "end"; + expect(appendVoiceCaption([], "user", text)).toEqual([ + { role: "user", text: text.slice(-1024), final: false }, + ]); + }); it("interleaves speakers and settles each with its final text", () => { let captions = appendVoiceCaption([], "user", "hello"); captions = appendVoiceCaption(captions, "assistant", "Hi"); diff --git a/packages/client-runtime/src/voice-mode/controller.ts b/packages/client-runtime/src/voice-mode/controller.ts index b14d7a0107829..ac714f767bf60 100644 --- a/packages/client-runtime/src/voice-mode/controller.ts +++ b/packages/client-runtime/src/voice-mode/controller.ts @@ -115,7 +115,10 @@ export function appendVoiceCaption( ): ReadonlyArray { const openIndex = captions.findLastIndex((caption) => caption.role === role && !caption.final); if (openIndex === -1) { - return [...captions, { role, text: delta.trimStart(), final: false }].slice(-MAX_CAPTIONS); + return [ + ...captions, + { role, text: delta.trimStart().slice(-MAX_CAPTION_CHARS), final: false }, + ].slice(-MAX_CAPTIONS); } const open = captions[openIndex]!; const text = (open.text + delta).slice(-MAX_CAPTION_CHARS); diff --git a/packages/client-runtime/src/voice-mode/session.ts b/packages/client-runtime/src/voice-mode/session.ts index 10364b98da7d2..ae12202941794 100644 --- a/packages/client-runtime/src/voice-mode/session.ts +++ b/packages/client-runtime/src/voice-mode/session.ts @@ -85,11 +85,9 @@ export function makeVoiceSessionValidator( (session) => session.id === providerThread?.providerSessionId, ); if (!session || !["ready", "running", "waiting"].includes(session.status)) { - return yield* Effect.fail( - new VoiceSessionUnavailable({ - message: "The provider session is no longer running. Send a message first.", - }), - ); + return yield* new VoiceSessionUnavailable({ + message: "The provider session is no longer running. Send a message first.", + }); } }).pipe( Effect.matchCause({ From ad51f203328782d6923c623b8d2938a4bf34ad31 Mon Sep 17 00:00:00 2001 From: Jake Leventhal Date: Wed, 28 Oct 2026 09:00:15 -0400 Subject: [PATCH 06/19] fix(voice): preserve microphone consent and loaded provider sessions --- .../src/window/MainWindowPermissions.test.ts | 12 ++++++++++++ apps/desktop/src/window/MainWindowPermissions.ts | 9 +++++++-- .../src/features/voice-mode/ComposerVoiceMode.tsx | 2 +- .../mobile/src/features/voice-mode/useVoiceMode.ts | 6 +++--- apps/server/src/keybindings.test.ts | 7 +++++-- apps/server/src/keybindings.ts | 11 ++++------- .../ProviderSessionManager.test.ts | 1 + .../src/orchestration-v2/ProviderSessionManager.ts | 14 ++++++++++++-- apps/web/src/components/chat/ChatComposer.tsx | 1 + apps/web/src/components/chat/VoiceModeStrip.tsx | 2 +- packages/contracts/src/provider.ts | 1 - 11 files changed, 47 insertions(+), 19 deletions(-) diff --git a/apps/desktop/src/window/MainWindowPermissions.test.ts b/apps/desktop/src/window/MainWindowPermissions.test.ts index e8c13053f26cf..f091d052eb1b8 100644 --- a/apps/desktop/src/window/MainWindowPermissions.test.ts +++ b/apps/desktop/src/window/MainWindowPermissions.test.ts @@ -38,6 +38,18 @@ describe("decideMainWindowMediaRequest", () => { expect(decide({ microphoneStatus: () => "not-determined" })).toBe("ask-microphone"); expect(decide({ microphoneStatus: () => "denied" })).toBe("ask-microphone"); expect(decide()).toBe("grant"); + expect( + isMainWindowPermissionAllowed("media", APP_URL, APP_URL, { + platform: "darwin", + microphoneStatus: () => "not-determined", + }), + ).toBe(false); + expect( + isMainWindowPermissionAllowed("media", APP_URL, APP_URL, { + platform: "darwin", + microphoneStatus: () => "granted", + }), + ).toBe(true); }); it("grants the microphone directly off macOS", () => { diff --git a/apps/desktop/src/window/MainWindowPermissions.ts b/apps/desktop/src/window/MainWindowPermissions.ts index 0df6b97cef304..ec926731257b3 100644 --- a/apps/desktop/src/window/MainWindowPermissions.ts +++ b/apps/desktop/src/window/MainWindowPermissions.ts @@ -32,9 +32,11 @@ export function isMainWindowPermissionAllowed( permission: string, origin: string | undefined, appUrl: string, + media?: { readonly platform: NodeJS.Platform; readonly microphoneStatus: () => string }, ): boolean { return permission === "media" - ? isAppUrl(origin, appUrl) + ? isAppUrl(origin, appUrl) && + (media?.platform !== "darwin" || media.microphoneStatus() !== "not-determined") : permission !== "deprecated-sync-clipboard-read"; } @@ -67,6 +69,9 @@ export function installMainWindowPermissionHandlers( }); }); session.setPermissionCheckHandler((_webContents, permission, requestingOrigin) => - isMainWindowPermissionAllowed(permission, requestingOrigin, options.appUrl), + isMainWindowPermissionAllowed(permission, requestingOrigin, options.appUrl, { + platform: options.platform, + microphoneStatus: () => Electron.systemPreferences.getMediaAccessStatus("microphone"), + }), ); } diff --git a/apps/mobile/src/features/voice-mode/ComposerVoiceMode.tsx b/apps/mobile/src/features/voice-mode/ComposerVoiceMode.tsx index 4d590a8e0ca44..a7dffe22d3f57 100644 --- a/apps/mobile/src/features/voice-mode/ComposerVoiceMode.tsx +++ b/apps/mobile/src/features/voice-mode/ComposerVoiceMode.tsx @@ -108,7 +108,7 @@ export function ComposerVoiceStrip() { // Levels arrive at 10Hz; they drive shared values directly, and the // speaking flag only re-renders when it flips. - // oxlint-disable-next-line t3code/react-effect-budget -- Subscribe native meter animations to the audio level stream. + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- Subscribe native meter animations to the audio level stream. useEffect(() => { const isSpeaking = createSpeakingDetector(); const unsubscribe = voiceMode.subscribeLevels((levels) => { diff --git a/apps/mobile/src/features/voice-mode/useVoiceMode.ts b/apps/mobile/src/features/voice-mode/useVoiceMode.ts index 4d9edd10ceef6..f9347f04ef51e 100644 --- a/apps/mobile/src/features/voice-mode/useVoiceMode.ts +++ b/apps/mobile/src/features/voice-mode/useVoiceMode.ts @@ -39,13 +39,13 @@ export function useVoiceModeThreadLifecycle(target: VoiceSessionTarget): void { const params = route?.params; return params && "threadId" in params ? String(params.threadId) : null; }); - // oxlint-disable-next-line t3code/react-effect-budget -- Synchronize the WebRTC lifetime with the visible native thread. + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- Synchronize the WebRTC lifetime with the visible native thread. useEffect(() => { if (visibleThreadId !== threadId && voiceMode.isActiveFor({ environmentId, threadId })) voiceMode.stop(); }, [environmentId, threadId, visibleThreadId]); - // oxlint-disable-next-line t3code/react-effect-budget -- Release the native call when its owning thread screen unmounts. + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- Release the native call when its owning thread screen unmounts. useEffect( () => () => { if ( @@ -57,7 +57,7 @@ export function useVoiceModeThreadLifecycle(target: VoiceSessionTarget): void { [environmentId, threadId, routeName], ); - // oxlint-disable-next-line t3code/react-effect-budget -- Subscribe to call lifecycle failures and remote endings. + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- Subscribe to call lifecycle failures and remote endings. useEffect(() => { // Reads the live state: dismissing notifies listeners re-entrantly, so a // listener's argument can already be stale. diff --git a/apps/server/src/keybindings.test.ts b/apps/server/src/keybindings.test.ts index 14fc8a4f6b7a3..c48432b7888f0 100644 --- a/apps/server/src/keybindings.test.ts +++ b/apps/server/src/keybindings.test.ts @@ -355,7 +355,7 @@ it.layer(NodeServices.layer)("keybindings", (it) => { ); }); - it.effect.each([undefined, "(!terminalFocus)"])( + it.effect.each([undefined, "(!terminalFocus)", "composerFocus"])( "preserves a user voice shortcut with condition %s", (when) => Effect.gen(function* () { @@ -374,7 +374,10 @@ it.layer(NodeServices.layer)("keybindings", (it) => { }); const persisted = yield* readKeybindingsConfig(keybindingsConfigPath); - assert.isFalse(persisted.some((entry) => entry.command === "voice.toggle")); + assert.equal( + persisted.some((entry) => entry.command === "voice.toggle"), + when === "composerFocus", + ); assert.isTrue(persisted.some((entry) => entry.command === "navigation.back")); }).pipe(Effect.provide(layerKeybindings())), ); diff --git a/apps/server/src/keybindings.ts b/apps/server/src/keybindings.ts index 3cfb4724b24f1..8ffbf5a5752d2 100644 --- a/apps/server/src/keybindings.ts +++ b/apps/server/src/keybindings.ts @@ -119,18 +119,15 @@ function encodedRuleShortcut(rule: KeybindingRule): string | null { return parsed ? encodeShortcut(parsed) : null; } -/** - * Whether appending `rule` after `existing` would shadow it: same shortcut, and - * either the same `when` or one of them applies in every context. - */ +// Persisted custom rules follow defaults, so a conditional override leaves the default available elsewhere. function shadowsShortcut(existing: KeybindingRule, rule: KeybindingRule): boolean { const existingShortcut = encodedRuleShortcut(existing); if (!existingShortcut || existingShortcut !== encodedRuleShortcut(rule)) return false; + if (existing.when === undefined) return true; + if (rule.when === undefined) return false; return ( - existing.when === undefined || - rule.when === undefined || JSON.stringify(parseKeybindingWhenExpression(existing.when)) === - JSON.stringify(parseKeybindingWhenExpression(rule.when)) + JSON.stringify(parseKeybindingWhenExpression(rule.when)) ); } diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index 6de85422c02e2..d747e0c3ab232 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -3874,6 +3874,7 @@ it.effect( .startVoiceSession({ threadId, offerSdp: "offer" }) .pipe(Stream.runDrain, Effect.forkChild); yield* Deferred.await(started); + assert.equal((yield* Ref.get(state)).resumeCount, 1); yield* TestClock.adjust("2 seconds"); assert.isTrue(Option.isSome(yield* manager.get(providerSessionId))); assert.equal((yield* Ref.get(state)).closeCount, 0); diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index 0feb52bc254a9..9ffc57c638502 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -2158,7 +2158,14 @@ export const layerWithOptions = ( () => markIdle(providerSessionId, providerThread.id, voiceOrdinal).pipe(Effect.ignore), ); - return runtime.startVoiceSession({ providerThread, offerSdp: input.offerSdp }); + const loadedThread = yield* runtime.resumeThread({ + providerThread, + threadId: input.threadId, + }); + return runtime.startVoiceSession({ + providerThread: loadedThread, + offerSdp: input.offerSdp, + }); }), ).pipe( Stream.mapError((cause) => @@ -2166,7 +2173,10 @@ export const layerWithOptions = ( ? cause : new ProviderVoiceSessionError({ threadId: input.threadId, - detail: cause.message, + detail: + cause._tag === "ProviderAdapterProtocolError" + ? cause.detail + : "The voice conversation could not continue.", cause, }), ), diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index ec7df9e72ec91..aacbfd063def5 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -1140,6 +1140,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) }; }, [backgroundThread]); + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- binds voice shortcuts to window keyboard events for the mounted composer useEffect(() => { const handler = (event: globalThis.KeyboardEvent) => { const command = resolveShortcutCommand(event, keybindings, { diff --git a/apps/web/src/components/chat/VoiceModeStrip.tsx b/apps/web/src/components/chat/VoiceModeStrip.tsx index 8cdcf52deb1ef..d901fa01aef39 100644 --- a/apps/web/src/components/chat/VoiceModeStrip.tsx +++ b/apps/web/src/components/chat/VoiceModeStrip.tsx @@ -84,7 +84,7 @@ export const VoiceModeStrip = memo(function VoiceModeStrip({ const speakerMeterRef = useRef(null); const [speaking, setSpeaking] = useState(false); - // oxlint-disable-next-line t3code/react-effect-budget -- Subscribe visible meters to the audio level stream. + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- Subscribe visible meters to the audio level stream. useEffect(() => { if (!isActive) return; // Teardown publishes a silent sample, which clears `speaking` for the next conversation. diff --git a/packages/contracts/src/provider.ts b/packages/contracts/src/provider.ts index 6504791bb2759..d1ceee445aded 100644 --- a/packages/contracts/src/provider.ts +++ b/packages/contracts/src/provider.ts @@ -100,4 +100,3 @@ export class ProviderVoiceSessionError extends Schema.TaggedError Date: Wed, 28 Oct 2026 10:13:20 -0400 Subject: [PATCH 07/19] fix(voice): isolate sessions and respect client interaction boundaries --- .../features/voice-mode/useVoiceMode.test.tsx | 38 +++++++--- .../src/features/voice-mode/useVoiceMode.ts | 15 ++-- .../Adapters/CodexAdapterV2.test.ts | 75 +++++++++++++++++++ .../Adapters/CodexAdapterV2.ts | 4 +- apps/web/src/components/chat/ChatComposer.tsx | 5 +- .../src/components/chat/VoiceModeStrip.tsx | 6 +- 6 files changed, 123 insertions(+), 20 deletions(-) diff --git a/apps/mobile/src/features/voice-mode/useVoiceMode.test.tsx b/apps/mobile/src/features/voice-mode/useVoiceMode.test.tsx index e18371cc63a90..e920552399476 100644 --- a/apps/mobile/src/features/voice-mode/useVoiceMode.test.tsx +++ b/apps/mobile/src/features/voice-mode/useVoiceMode.test.tsx @@ -3,7 +3,12 @@ import { createRoot, type Root } from "react-dom/client"; import { afterEach, beforeEach, expect, it, vi } from "vite-plus/test"; import { EnvironmentId, ThreadId } from "@t3tools/contracts"; -const fixture = vi.hoisted(() => ({ active: true, currentThread: "thread-a", stop: vi.fn() })); +const fixture = vi.hoisted(() => ({ + active: true, + currentThread: "thread-a", + currentEnvironment: "environment", + stop: vi.fn(), +})); const RouteName = createContext("Thread"); const target = { environmentId: EnvironmentId.make("environment"), @@ -12,8 +17,18 @@ const target = { vi.mock("@react-navigation/native", () => ({ useRoute: () => ({ name: useContext(RouteName) }), useNavigationState: ( - select: (state: { routes: Array<{ name: string; params: { threadId: string } }> }) => unknown, - ) => select({ routes: [{ name: "Thread", params: { threadId: fixture.currentThread } }] }), + select: (state: { + routes: Array<{ name: string; params: { threadId: string; environmentId: string } }>; + }) => unknown, + ) => + select({ + routes: [ + { + name: "Thread", + params: { threadId: fixture.currentThread, environmentId: fixture.currentEnvironment }, + }, + ], + }), })); vi.mock("react-native", () => ({ Alert: { alert: vi.fn() } })); vi.mock("./voiceModePresentation", () => ({ resolveVoiceModeNotice: () => null })); @@ -37,6 +52,7 @@ let root: Root; beforeEach(() => { fixture.active = true; fixture.currentThread = "thread-a"; + fixture.currentEnvironment = "environment"; fixture.stop.mockClear(); (globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true; root = createRoot(document.createElement("div")); @@ -64,12 +80,16 @@ it("keeps voice running when Files closes and stops when the owning Thread leave act(() => root.unmount()); expect(fixture.stop).toHaveBeenCalledTimes(1); }); -it("stops the previous thread when navigation selects a different thread", () => { - render(false); - fixture.currentThread = "thread-b"; - render(false); - expect(fixture.stop).toHaveBeenCalledTimes(1); -}); +it.each(["thread", "environment"])( + "stops the previous thread when navigation selects a different %s", + (changed) => { + render(false); + if (changed === "thread") fixture.currentThread = "thread-b"; + else fixture.currentEnvironment = "another-environment"; + render(false); + expect(fixture.stop).toHaveBeenCalledTimes(1); + }, +); it("stops voice when its fullscreen thread closes over the still-mounted thread", () => { render(true, "ThreadInspectorFullscreen"); diff --git a/apps/mobile/src/features/voice-mode/useVoiceMode.ts b/apps/mobile/src/features/voice-mode/useVoiceMode.ts index f9347f04ef51e..4f480b5baaa7b 100644 --- a/apps/mobile/src/features/voice-mode/useVoiceMode.ts +++ b/apps/mobile/src/features/voice-mode/useVoiceMode.ts @@ -34,16 +34,21 @@ export function useVoiceModePhase(target: VoiceSessionTarget): VoiceModePhase { export function useVoiceModeThreadLifecycle(target: VoiceSessionTarget): void { const { environmentId, threadId } = target; const routeName = useRoute().name; - const visibleThreadId = useNavigationState((state) => { + const isVisibleThread = useNavigationState((state) => { const route = state.routes.findLast((route) => route.name === "Thread"); const params = route?.params; - return params && "threadId" in params ? String(params.threadId) : null; + return ( + params != null && + "threadId" in params && + "environmentId" in params && + String(params.threadId) === threadId && + String(params.environmentId) === environmentId + ); }); // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- Synchronize the WebRTC lifetime with the visible native thread. useEffect(() => { - if (visibleThreadId !== threadId && voiceMode.isActiveFor({ environmentId, threadId })) - voiceMode.stop(); - }, [environmentId, threadId, visibleThreadId]); + if (!isVisibleThread && voiceMode.isActiveFor({ environmentId, threadId })) voiceMode.stop(); + }, [environmentId, threadId, isVisibleThread]); // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- Release the native call when its owning thread screen unmounts. useEffect( diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts index 29cf3c7b1e6d9..cdb15ef375ced 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts @@ -1380,6 +1380,81 @@ describe("CodexAdapterV2 post-settle continuation", () => { ), ); + it.effect.each(["request", "realtime"] as const)( + "only displays intentional voice errors from a %s failure", + (failure) => + Effect.scoped( + Effect.gen(function* () { + const nativeThreadId = "voice-error-thread"; + const detail = + failure === "request" ? "private protocol payload" : "Voice service is unavailable"; + const harness = yield* makeCodexReplayHarness( + makeCodexReplayTranscript({ + scenario: `voice-error-${failure}`, + entries: [ + ...codexReplayPreamble({ + nativeThreadId, + nativeTurnId: "unused", + prompt: "", + }).slice(0, -3), + { + type: "expect_outbound", + label: "thread/realtime/start", + frame: { + id: 3, + method: "thread/realtime/start", + params: { + threadId: nativeThreadId, + outputModality: "audio", + transport: { type: "webrtc", sdp: "v=offer\r\n" }, + version: "v3", + clientManagedHandoffs: true, + includeStartupContext: false, + }, + }, + }, + { + type: "emit_inbound", + label: "thread/realtime/start", + frame: + failure === "request" + ? { id: 3, error: { code: -32603, message: detail } } + : { id: 3, result: {} }, + }, + ...(failure === "realtime" + ? [ + { + type: "emit_inbound" as const, + label: "thread/realtime/error", + frame: { + method: "thread/realtime/error", + params: { threadId: nativeThreadId, message: detail }, + }, + }, + ] + : []), + ], + }), + ); + const error = yield* harness.runtime.startVoiceSession!({ + providerThread: harness.providerThread, + offerSdp: "v=offer\r\n", + }).pipe(Stream.runDrain, Effect.flip); + assert.equal(error._tag, "ProviderAdapterProtocolError"); + if (error._tag === "ProviderAdapterProtocolError") { + assert.equal( + error.detail, + failure === "request" ? "Voice conversation failed." : detail, + ); + } + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + it.effect( "resubmits a voice handoff Codex started only once its turn is interrupted or failed", () => diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index bc00d9dbf303a..f73853e5ac1d6 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -7856,7 +7856,9 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( : new ProviderAdapter.ProviderAdapterProtocolError({ driver: CODEX_PROVIDER, detail: - cause._tag === "CodexVoiceSessionError" ? cause.detail : cause.message, + cause._tag === "CodexVoiceSessionError" + ? cause.detail + : "Voice conversation failed.", payload: cause, }), ), diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index aacbfd063def5..4c25351b3238d 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -967,7 +967,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) const voiceEngaged = voicePhase !== "idle"; // A conversation stays stoppable here even if its provider stopped offering voice. const voiceSupported = voiceAvailability.kind !== "unsupported" || voiceEngaged; - const toggleVoice = () => toggleThreadVoice(routeThreadRef, voiceAvailability); + const toggleVoice = useStableCallback(() => toggleThreadVoice(routeThreadRef, voiceAvailability)); const selectedProviderSkills = selectedProviderStatus ? resolveProviderSkillsForCwd(selectedProviderStatus, gitCwd) : NO_PROVIDER_SKILLS; @@ -1143,6 +1143,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- binds voice shortcuts to window keyboard events for the mounted composer useEffect(() => { const handler = (event: globalThis.KeyboardEvent) => { + if (isCommandPaletteOpen()) return; const command = resolveShortcutCommand(event, keybindings, { context: { terminalFocus: getTerminalFocusOwner() !== null, @@ -1162,7 +1163,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) if (!event.repeat) voiceMode.toggleMuted(); return; } - if (command !== "voice.toggle" || !voiceSupported || isCommandPaletteOpen()) return; + if (command !== "voice.toggle" || !voiceSupported) return; event.preventDefault(); event.stopPropagation(); if (!event.repeat) toggleVoice(); diff --git a/apps/web/src/components/chat/VoiceModeStrip.tsx b/apps/web/src/components/chat/VoiceModeStrip.tsx index d901fa01aef39..e812041b6ecb6 100644 --- a/apps/web/src/components/chat/VoiceModeStrip.tsx +++ b/apps/web/src/components/chat/VoiceModeStrip.tsx @@ -115,9 +115,10 @@ export const VoiceModeStrip = memo(function VoiceModeStrip({ {isActive ? : } - - {status} + + {isActive ? "Voice connected" : "Connecting voice…"} + {status} {isActive ? ( @@ -190,7 +191,6 @@ export function ComposerVoiceButton(props: { variant="ghost" size="sm" // aria-disabled rather than disabled keeps the tooltip explaining why. - className="aria-disabled:cursor-not-allowed" pressed={props.engaged} onPointerDown={(event) => event.preventDefault()} onPressedChange={() => { From 870f3c5186bef0ac2d4f0c19189848a181382546 Mon Sep 17 00:00:00 2001 From: Jake Leventhal Date: Wed, 28 Oct 2026 10:43:39 -0400 Subject: [PATCH 08/19] fix(mobile): end parent voice when a different fullscreen thread opens --- .../src/features/voice-mode/useVoiceMode.test.tsx | 14 ++++++++++++-- .../mobile/src/features/voice-mode/useVoiceMode.ts | 4 +++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/apps/mobile/src/features/voice-mode/useVoiceMode.test.tsx b/apps/mobile/src/features/voice-mode/useVoiceMode.test.tsx index e920552399476..9f8852e9b4903 100644 --- a/apps/mobile/src/features/voice-mode/useVoiceMode.test.tsx +++ b/apps/mobile/src/features/voice-mode/useVoiceMode.test.tsx @@ -7,6 +7,7 @@ const fixture = vi.hoisted(() => ({ active: true, currentThread: "thread-a", currentEnvironment: "environment", + fullscreenTarget: null as { threadId: string; environmentId: string } | null, stop: vi.fn(), })); const RouteName = createContext("Thread"); @@ -27,6 +28,9 @@ vi.mock("@react-navigation/native", () => ({ name: "Thread", params: { threadId: fixture.currentThread, environmentId: fixture.currentEnvironment }, }, + ...(fixture.fullscreenTarget === null + ? [] + : [{ name: "ThreadInspectorFullscreen", params: fixture.fullscreenTarget }]), ], }), })); @@ -53,6 +57,7 @@ beforeEach(() => { fixture.active = true; fixture.currentThread = "thread-a"; fixture.currentEnvironment = "environment"; + fixture.fullscreenTarget = null; fixture.stop.mockClear(); (globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true; root = createRoot(document.createElement("div")); @@ -80,12 +85,17 @@ it("keeps voice running when Files closes and stops when the owning Thread leave act(() => root.unmount()); expect(fixture.stop).toHaveBeenCalledTimes(1); }); -it.each(["thread", "environment"])( +it.each(["thread", "environment", "fullscreen thread", "fullscreen environment"])( "stops the previous thread when navigation selects a different %s", (changed) => { render(false); if (changed === "thread") fixture.currentThread = "thread-b"; - else fixture.currentEnvironment = "another-environment"; + else if (changed === "environment") fixture.currentEnvironment = "another-environment"; + else + fixture.fullscreenTarget = { + threadId: changed === "fullscreen thread" ? "thread-b" : "thread-a", + environmentId: changed === "fullscreen environment" ? "another-environment" : "environment", + }; render(false); expect(fixture.stop).toHaveBeenCalledTimes(1); }, diff --git a/apps/mobile/src/features/voice-mode/useVoiceMode.ts b/apps/mobile/src/features/voice-mode/useVoiceMode.ts index 4f480b5baaa7b..b9933a7932654 100644 --- a/apps/mobile/src/features/voice-mode/useVoiceMode.ts +++ b/apps/mobile/src/features/voice-mode/useVoiceMode.ts @@ -35,7 +35,9 @@ export function useVoiceModeThreadLifecycle(target: VoiceSessionTarget): void { const { environmentId, threadId } = target; const routeName = useRoute().name; const isVisibleThread = useNavigationState((state) => { - const route = state.routes.findLast((route) => route.name === "Thread"); + const route = state.routes.findLast( + (route) => route.name === "Thread" || route.name === "ThreadInspectorFullscreen", + ); const params = route?.params; return ( params != null && From ac97fd5d12aac64c2f2b402f718bf212aa77633d Mon Sep 17 00:00:00 2001 From: Jake Leventhal Date: Wed, 28 Oct 2026 11:06:15 -0400 Subject: [PATCH 09/19] fix(server): serialize Codex voice lifecycle ownership --- .../Adapters/CodexAdapterV2.ts | 22 +- apps/server/src/provider/CodexVoice.test.ts | 204 ++++++++++++++++++ apps/server/src/provider/CodexVoice.ts | 83 ++++--- 3 files changed, 273 insertions(+), 36 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index f73853e5ac1d6..6d3cccdf38a23 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -7825,15 +7825,19 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( Effect.gen(function* () { const nativeThreadId = yield* getNativeThreadId(voiceInput.providerThread); yield* ensureInitialized; - yield* Ref.set(voiceSession, owner); - yield* Ref.set(voiceProviderThread, voiceInput.providerThread); - return voice.start({ - providerThreadId: nativeThreadId, - offerSdp: voiceInput.offerSdp, - ...(adapterOptions.settings.voice - ? { voice: adapterOptions.settings.voice } - : {}), - }); + return voice.start( + { + providerThreadId: nativeThreadId, + offerSdp: voiceInput.offerSdp, + ...(adapterOptions.settings.voice + ? { voice: adapterOptions.settings.voice } + : {}), + }, + Effect.gen(function* () { + yield* Ref.set(voiceSession, owner); + yield* Ref.set(voiceProviderThread, voiceInput.providerThread); + }), + ); }), ).pipe( // A replaced conversation must not clear its successor's thread. diff --git a/apps/server/src/provider/CodexVoice.test.ts b/apps/server/src/provider/CodexVoice.test.ts index 9612e836f2453..74fcae20c9d37 100644 --- a/apps/server/src/provider/CodexVoice.test.ts +++ b/apps/server/src/provider/CodexVoice.test.ts @@ -6,10 +6,12 @@ import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; +import * as Logger from "effect/Logger"; import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import type * as CodexClient from "effect-codex-app-server/client"; +import * as CodexErrors from "effect-codex-app-server/errors"; import { CodexVoiceSessionError, @@ -30,8 +32,11 @@ const makeFakeClient = Effect.gen(function* () { const handlers = new Map Effect.Effect>>(); const requests: Array<{ readonly method: string; readonly params: unknown }> = []; let closeOnStop = false; + let onStart: Effect.Effect = Effect.void; + let onStop: Effect.Effect = Effect.void; const startRequested = yield* Deferred.make(); const secondStartRequested = yield* Deferred.make(); + const thirdStartRequested = yield* Deferred.make(); const speechRequested = yield* Deferred.make(); const stopRequested = yield* Deferred.make(); const client = { @@ -50,12 +55,16 @@ const makeFakeClient = Effect.gen(function* () { yield* Deferred.succeed(startRequested, undefined); if (requests.filter((request) => request.method === method).length === 2) yield* Deferred.succeed(secondStartRequested, undefined); + if (requests.filter((request) => request.method === method).length === 3) + yield* Deferred.succeed(thirdStartRequested, undefined); + yield* onStart; } if (method === "thread/realtime/appendSpeech") { yield* Deferred.succeed(speechRequested, undefined); } if (method === "thread/realtime/stop") { yield* Deferred.succeed(stopRequested, undefined); + yield* onStop; if (closeOnStop) yield* emit("thread/realtime/closed", { threadId: PROVIDER_THREAD, @@ -76,7 +85,14 @@ const makeFakeClient = Effect.gen(function* () { emit, startRequested, secondStartRequested, + thirdStartRequested, speechRequested, + setStart: (effect: typeof onStart) => { + onStart = effect; + }, + setStop: (effect: typeof onStop) => { + onStop = effect; + }, stopRequested, closePreviousOnStop: () => { closeOnStop = true; @@ -228,6 +244,194 @@ describe("makeCodexVoice", () => { ), ); + it.effect("serializes concurrent replacements before routing the successor's SDP", () => + Effect.scoped( + Effect.gen(function* () { + const fake = yield* makeFakeClient; + const voice = yield* makeCodexVoice(fake.client, yield* Effect.scope); + const owners: Array = []; + const start = (owner: string) => + voice + .start( + { providerThreadId: PROVIDER_THREAD, offerSdp: "offer" }, + Effect.sync(() => { + owners.push(owner); + }), + ) + .pipe(Stream.runCollect, Effect.forkChild({ startImmediately: true })); + const first = yield* start("first"); + yield* Deferred.await(fake.startRequested); + const releaseStop = yield* Deferred.make(); + fake.setStop(Deferred.await(releaseStop)); + const second = yield* start("second"); + yield* Deferred.await(fake.stopRequested); + const third = yield* start("third"); + const cancelled = yield* start("cancelled"); + const cancellation = yield* Fiber.interrupt(cancelled).pipe(Effect.forkChild); + const whileStopping = fake.requests.map(({ method }) => method); + yield* Deferred.succeed(releaseStop, undefined); + NodeAssert.deepEqual(whileStopping, ["thread/realtime/start", "thread/realtime/stop"]); + yield* Deferred.await(fake.thirdStartRequested); + yield* Fiber.join(cancellation); + NodeAssert.deepEqual(owners, ["first", "second", "third"]); + yield* fake.emit("thread/realtime/sdp", { threadId: PROVIDER_THREAD, sdp: "third-answer" }); + yield* fake.emit("thread/realtime/closed", { threadId: PROVIDER_THREAD }); + NodeAssert.deepEqual(yield* Fiber.join(first), [{ type: "closed", reason: "replaced" }]); + NodeAssert.deepEqual(yield* Fiber.join(second), [{ type: "closed", reason: "replaced" }]); + NodeAssert.deepEqual(yield* Fiber.join(third), [ + { type: "answer", sdp: "third-answer" }, + { type: "closed" }, + ]); + NodeAssert.deepEqual( + fake.requests.map(({ method }) => method), + [ + "thread/realtime/start", + "thread/realtime/stop", + "thread/realtime/start", + "thread/realtime/stop", + "thread/realtime/start", + ], + ); + }), + ), + ); + + it.effect("waits for unsubscribe cleanup before starting another native thread", () => + Effect.scoped( + Effect.gen(function* () { + const fake = yield* makeFakeClient; + const voice = yield* makeCodexVoice(fake.client, yield* Effect.scope); + const first = yield* voice + .start({ providerThreadId: PROVIDER_THREAD, offerSdp: "offer" }) + .pipe(Stream.runCollect, Effect.forkChild); + yield* Deferred.await(fake.startRequested); + const releaseStop = yield* Deferred.make(); + fake.setStop(Deferred.await(releaseStop)); + const cleanup = yield* Fiber.interrupt(first).pipe(Effect.forkChild); + yield* Deferred.await(fake.stopRequested); + const replacement = yield* voice + .start({ providerThreadId: "another-thread", offerSdp: "offer" }) + .pipe(Stream.runCollect, Effect.forkChild({ startImmediately: true })); + const whileStopping = fake.requests.map(({ method }) => method); + yield* Deferred.succeed(releaseStop, undefined); + NodeAssert.deepEqual(whileStopping, ["thread/realtime/start", "thread/realtime/stop"]); + yield* Fiber.join(cleanup); + yield* Deferred.await(fake.secondStartRequested); + yield* fake.emit("thread/realtime/sdp", { threadId: PROVIDER_THREAD, sdp: "ignored" }); + yield* fake.emit("thread/realtime/sdp", { threadId: "another-thread", sdp: "answer" }); + yield* fake.emit("thread/realtime/closed", { threadId: "another-thread" }); + NodeAssert.deepEqual(yield* Fiber.join(replacement), [ + { type: "answer", sdp: "answer" }, + { type: "closed" }, + ]); + }), + ), + ); + + it.effect.each(["stop response", "closed notification"] as const)( + "retains failed cleanup until a %s confirms termination", + (confirmation) => { + const messages: Array = []; + const logger = Logger.make(({ message }) => { + messages.push(message); + }); + return Effect.scoped( + Effect.gen(function* () { + const fake = yield* makeFakeClient; + const voice = yield* makeCodexVoice(fake.client, yield* Effect.scope); + const start = () => + voice + .start({ providerThreadId: PROVIDER_THREAD, offerSdp: "offer" }) + .pipe(Stream.runCollect, Effect.forkChild); + const first = yield* start(); + yield* Deferred.await(fake.startRequested); + const stopFailure = new CodexErrors.CodexAppServerRequestError({ + code: -32603, + errorMessage: "stop failed", + method: "thread/realtime/stop", + }); + fake.setStop(Effect.fail(stopFailure)); + fake.setStart( + Effect.fail( + new CodexErrors.CodexAppServerRequestError({ + code: -32603, + errorMessage: "unexpected start", + }), + ), + ); + yield* Fiber.interrupt(first); + const replacementExit = yield* voice + .start({ providerThreadId: PROVIDER_THREAD, offerSdp: "offer" }) + .pipe(Stream.runCollect, Effect.exit); + NodeAssert.ok(Exit.isFailure(replacementExit)); + NodeAssert.deepEqual( + fake.requests.map(({ method }) => method), + ["thread/realtime/start", "thread/realtime/stop", "thread/realtime/stop"], + ); + NodeAssert.ok( + messages.flat().includes("Codex voice could not stop the realtime session"), + ); + fake.setStop(Effect.void); + fake.setStart(Effect.void); + if (confirmation === "closed notification") + yield* fake.emit("thread/realtime/closed", { threadId: PROVIDER_THREAD }); + const recovered = yield* start(); + yield* Deferred.await(fake.secondStartRequested); + yield* fake.emit("thread/realtime/closed", { threadId: PROVIDER_THREAD }); + yield* Fiber.join(recovered); + NodeAssert.deepEqual( + fake.requests.map(({ method }) => method), + [ + "thread/realtime/start", + "thread/realtime/stop", + "thread/realtime/stop", + ...(confirmation === "stop response" ? ["thread/realtime/stop"] : []), + "thread/realtime/start", + ], + ); + }), + ).pipe(Effect.provide(Logger.layer([logger], { mergeWithExisting: false }))); + }, + ); + + it.effect("retries predecessor cleanup after an interrupted replacement", () => + Effect.scoped( + Effect.gen(function* () { + const fake = yield* makeFakeClient; + const voice = yield* makeCodexVoice(fake.client, yield* Effect.scope); + const start = () => + voice + .start({ providerThreadId: PROVIDER_THREAD, offerSdp: "offer" }) + .pipe(Stream.runCollect, Effect.forkChild({ startImmediately: true })); + const first = yield* start(); + yield* Deferred.await(fake.startRequested); + const stopInterrupted = yield* Deferred.make(); + const releaseStop = yield* Deferred.make(); + fake.setStop( + Deferred.await(releaseStop).pipe( + Effect.onInterrupt(() => Deferred.succeed(stopInterrupted, undefined)), + ), + ); + const replacement = yield* start(); + yield* Deferred.await(fake.stopRequested); + const interrupted = yield* Fiber.interrupt(replacement).pipe(Effect.forkChild); + yield* Deferred.await(stopInterrupted); + fake.setStop(Effect.void); + yield* Deferred.succeed(releaseStop, undefined); + yield* Fiber.join(interrupted); + yield* Fiber.join(first); + NodeAssert.deepEqual( + fake.requests.map(({ method }) => method), + ["thread/realtime/start", "thread/realtime/stop", "thread/realtime/stop"], + ); + const recovered = yield* start(); + yield* Deferred.await(fake.secondStartRequested); + yield* fake.emit("thread/realtime/closed", { threadId: PROVIDER_THREAD }); + yield* Fiber.join(recovered); + }), + ), + ); + it.effect("stops the realtime session when the client unsubscribes", () => Effect.gen(function* () { const voice = yield* startVoice; diff --git a/apps/server/src/provider/CodexVoice.ts b/apps/server/src/provider/CodexVoice.ts index 0332ac8d8508f..8e2494a2656a4 100644 --- a/apps/server/src/provider/CodexVoice.ts +++ b/apps/server/src/provider/CodexVoice.ts @@ -6,6 +6,7 @@ import * as Effect from "effect/Effect"; import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; import type * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import type * as CodexClient from "effect-codex-app-server/client"; @@ -98,6 +99,7 @@ interface ActiveVoice { /** Handoffs resubmitted as ordinary runs, awaiting their native turn. */ readonly expectedHandoffs: Array; closed: boolean; + stopped: boolean; } export interface CodexVoiceStartInput { @@ -112,6 +114,7 @@ export const makeCodexVoice = Effect.fn("makeCodexVoice")(function* ( runtimeScope: Scope.Scope, ) { const activeRef = yield* Ref.make(undefined); + const transition = yield* Semaphore.make(1); const withActive = ( threadId: string, @@ -163,9 +166,16 @@ export const makeCodexVoice = Effect.fn("makeCodexVoice")(function* ( ), ); yield* client.handleServerNotification("thread/realtime/closed", (payload) => - withActive(payload.threadId, (active) => - finish(active, { type: "closed", ...(payload.reason ? { reason: payload.reason } : {}) }), - ), + Effect.flatMap(Ref.get(activeRef), (active) => { + if (!active || active.providerThreadId !== payload.threadId) return Effect.void; + active.stopped = true; + return active.closed + ? Effect.void + : finish(active, { + type: "closed", + ...(payload.reason ? { reason: payload.reason } : {}), + }); + }), ); yield* client.handleServerNotification("item/started", (payload) => @@ -219,12 +229,13 @@ export const makeCodexVoice = Effect.fn("makeCodexVoice")(function* ( ); const stop = Effect.fn("CodexVoice.stop")(function* (active: ActiveVoice) { - yield* Ref.update(activeRef, (current) => (current === active ? undefined : current)); - if (active.closed) return; + if ((yield* Ref.get(activeRef)) !== active) return; active.closed = true; - yield* client.raw - .request("thread/realtime/stop", { threadId: active.providerThreadId }) - .pipe(Effect.ignore); + if (!active.stopped) { + yield* client.raw.request("thread/realtime/stop", { threadId: active.providerThreadId }); + active.stopped = true; + } + yield* Ref.set(activeRef, undefined); }); /** @@ -233,6 +244,7 @@ export const makeCodexVoice = Effect.fn("makeCodexVoice")(function* ( */ const start = ( input: CodexVoiceStartInput, + onAcquired: Effect.Effect = Effect.void, ): Stream.Stream< ProviderVoiceSessionEvent, CodexVoiceSessionError | CodexErrors.CodexAppServerError @@ -249,26 +261,43 @@ export const makeCodexVoice = Effect.fn("makeCodexVoice")(function* ( delegatedTurns: new Map(), expectedHandoffs: [], closed: false, + stopped: false, }; - const previous = yield* Ref.get(activeRef); - if (previous && !previous.closed) { - // One conversation per thread: a newer client takes over. - yield* finish(previous, { type: "closed", reason: "replaced" }); - yield* client.raw - .request("thread/realtime/stop", { threadId: previous.providerThreadId }) - .pipe(Effect.ignore); - } - yield* Ref.set(activeRef, active); - yield* Effect.addFinalizer(() => stop(active)); - yield* client.raw.request("thread/realtime/start", { - threadId: input.providerThreadId, - outputModality: "audio", - transport: { type: "webrtc", sdp: input.offerSdp }, - version: "v3", - clientManagedHandoffs: true, - includeStartupContext: false, - ...(input.voice ? { voice: input.voice } : {}), - }); + yield* Effect.addFinalizer(() => + transition + .withPermit(stop(active)) + .pipe( + Effect.catch((cause) => + Effect.logWarning("Codex voice could not stop the realtime session", { cause }), + ), + ), + ); + yield* transition.withPermit( + Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + const previous = yield* Ref.get(activeRef); + if (previous) { + // The adapter supports one conversation per provider session, across native threads. + if (!previous.closed) + yield* finish(previous, { type: "closed", reason: "replaced" }); + yield* restore(stop(previous)); + } + yield* Ref.set(activeRef, active); + yield* onAcquired; + yield* restore( + client.raw.request("thread/realtime/start", { + threadId: input.providerThreadId, + outputModality: "audio", + transport: { type: "webrtc", sdp: input.offerSdp }, + version: "v3", + clientManagedHandoffs: true, + includeStartupContext: false, + ...(input.voice ? { voice: input.voice } : {}), + }), + ); + }), + ), + ); return Stream.fromQueue(queue); }), ); From ee958a5b4f59e2f55abd427eb5a0c2225a3e965e Mon Sep 17 00:00:00 2001 From: Jake Leventhal Date: Mon, 2 Nov 2026 10:13:29 -0500 Subject: [PATCH 10/19] fix(voice): identify handoffs and serialize mobile audio transitions --- .../features/threads/ThreadDetailScreen.tsx | 5 --- .../features/voice-mode/voiceAudio.test.ts | 31 ++++++++++++++ .../src/features/voice-mode/voiceAudio.ts | 22 ++++++++++ .../src/features/voice-mode/voiceMode.ts | 24 ++--------- .../Adapters/CodexAdapterV2.ts | 6 ++- .../ProviderSessionManager.ts | 24 +++++++++-- apps/server/src/provider/CodexVoice.test.ts | 40 +++++++++++++++--- apps/server/src/provider/CodexVoice.ts | 42 ++++++------------- apps/web/src/components/chat/ChatComposer.tsx | 2 + .../src/components/chat/useComposerSend.ts | 1 + docs/internals/voice-conversations.md | 6 +-- 11 files changed, 136 insertions(+), 67 deletions(-) create mode 100644 apps/mobile/src/features/voice-mode/voiceAudio.test.ts create mode 100644 apps/mobile/src/features/voice-mode/voiceAudio.ts diff --git a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx index b603d71adc6d0..f90858bcdd437 100644 --- a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx +++ b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx @@ -917,7 +917,6 @@ function ThreadDetailBody( null; const voiceTarget = { environmentId: props.environmentId, threadId: props.selectedThread.id }; const voiceLiveHere = useVoiceModePhase(voiceTarget) !== "idle"; - // Keep live quota data visible, but close the panel when a turn consumes quota. const [usageLimitsPanel, setUsageLimitsPanel] = useState<{ readonly key: string; readonly threadKey: string; @@ -1708,10 +1707,6 @@ function ThreadDetailBody( {/* The composer below hides for a question card; keep a live voice conversation's mute and stop controls reachable. */} {activeUserInputRequestId !== null && voiceLiveHere ? : null} - {/* Hidden (not unmounted) while a user-input request owns the - composer slot, so composer drafts and editor state survive. - A rejected creation has no thread to send to; the failure card - owns the slot instead. */} {isProviderSubagent ? ( ({ + setMode: vi.fn<(mode: { allowsRecording?: boolean }) => Promise>(), +})); +vi.mock("expo-audio", () => ({ setAudioModeAsync: audio.setMode })); + +import { setVoiceAudioActive } from "./voiceAudio"; + +it("finishes an old release before activating its successor", async () => { + let recording = true; + const releaseStarted = Promise.withResolvers(); + const finishRelease = Promise.withResolvers(); + audio.setMode.mockImplementation(async (mode) => { + if (mode.allowsRecording === false) { + releaseStarted.resolve(); + await finishRelease.promise; + } + recording = mode.allowsRecording === true; + }); + const release = setVoiceAudioActive(false); + await releaseStarted.promise; + const successor = setVoiceAudioActive(true); + finishRelease.resolve(); + await Promise.all([release, successor]); + expect(recording).toBe(true); + audio.setMode.mockRejectedValueOnce(new Error("release failed")); + await expect(setVoiceAudioActive(false)).rejects.toThrow("release failed"); + await setVoiceAudioActive(true); + expect(recording).toBe(true); +}); diff --git a/apps/mobile/src/features/voice-mode/voiceAudio.ts b/apps/mobile/src/features/voice-mode/voiceAudio.ts new file mode 100644 index 0000000000000..27273ed38d7b7 --- /dev/null +++ b/apps/mobile/src/features/voice-mode/voiceAudio.ts @@ -0,0 +1,22 @@ +import { setAudioModeAsync } from "expo-audio"; + +// Expo audio-mode changes must finish before a successor configures the shared native session. +let pending = Promise.resolve(); + +export function setVoiceAudioActive(active: boolean): Promise { + const result = pending.then(() => + setAudioModeAsync( + active + ? { + allowsRecording: true, + playsInSilentMode: true, + interruptionMode: "doNotMix", + shouldPlayInBackground: false, + shouldRouteThroughEarpiece: false, + } + : { allowsRecording: false }, + ), + ); + pending = result.catch(() => {}); + return result; +} diff --git a/apps/mobile/src/features/voice-mode/voiceMode.ts b/apps/mobile/src/features/voice-mode/voiceMode.ts index 2ab09284cb6c1..8d0795864d883 100644 --- a/apps/mobile/src/features/voice-mode/voiceMode.ts +++ b/apps/mobile/src/features/voice-mode/voiceMode.ts @@ -8,13 +8,13 @@ import { } from "@t3tools/client-runtime/voice-mode"; import * as Effect from "effect/Effect"; import { AtomRegistry } from "effect/reactivity"; -import { setAudioModeAsync } from "expo-audio"; import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake"; import { AppState, NativeModules } from "react-native"; import type * as WebRTC from "react-native-webrtc"; import { connectionAtomRuntime } from "../../connection/runtime"; import { appAtomRegistry } from "../../state/atom-registry"; +import { setVoiceAudioActive } from "./voiceAudio"; /** * False on binaries built before react-native-webrtc was added. Importing the @@ -60,23 +60,6 @@ export const voiceMode = new VoiceModeController({ openSession: makeVoiceSessionOpener(runOnConnectionRuntime), }); -/* WebRTC resets iOS routing when its audio unit starts; apply speaker routing afterward. - * Headphones and Bluetooth still take precedence. */ -async function routeVoiceToSpeaker(): Promise { - await setAudioModeAsync({ - allowsRecording: true, - playsInSilentMode: true, - interruptionMode: "doNotMix", - shouldPlayInBackground: false, - shouldRouteThroughEarpiece: false, - }); -} - -/** Mirrors dictation's release so interrupted audio in other apps resumes. */ -async function releaseVoiceAudio(): Promise { - await setAudioModeAsync({ allowsRecording: false }); -} - const KEEP_AWAKE_TAG = "voice-mode"; let previousPhase: VoiceModePhase = "idle"; let routedAudio = false; @@ -87,7 +70,8 @@ voiceMode.subscribe(({ phase }) => { previousPhase = phase; if (phase === "active" && !routedAudio) { routedAudio = true; - routeVoiceToSpeaker().catch(() => undefined); + // WebRTC resets iOS routing when its audio unit starts, so route after activation. + setVoiceAudioActive(true).catch(() => undefined); } if (phase !== "idle" && !keepAwake) { // A hands-free conversation must not auto-lock: backgrounding ends it. @@ -96,7 +80,7 @@ voiceMode.subscribe(({ phase }) => { if (phase === "idle") { if (routedAudio) { routedAudio = false; - releaseVoiceAudio().catch(() => undefined); + setVoiceAudioActive(false).catch(() => undefined); } const activation = keepAwake; keepAwake = null; diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index 6d3cccdf38a23..cd17c23e9d2f1 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -30,6 +30,7 @@ import { } from "../../provider/codexUsageLimits.ts"; import { CodexSettings, + MessageId, defaultInstanceIdForDriver, isOrchestrationV2WorkActive, type OrchestrationV2GoalCommand, @@ -2063,6 +2064,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( if (existing !== undefined) { return existing; } + yield* voice.registerTurn(input.turnInput.message.messageId, input.nativeTurnId); const providerTurnId = idAllocator.derive.providerTurn({ driver: CODEX_PROVIDER, nativeTurnId: input.nativeTurnId, @@ -6567,8 +6569,10 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( voiceResubmit.session !== undefined && voiceResubmit.session === (yield* Ref.get(voiceSession)) ) { - yield* voice.expectSpokenTurn(voiceResubmit.text); + const messageId = MessageId.make(`voice:${payload.threadId}:${payload.turn.id}`); + yield* voice.expectSpokenTurn(messageId); yield* continuationRequests.offer({ + messageId, threadId: voiceResubmit.providerThread.appThreadId ?? input.threadId, providerThreadId: voiceResubmit.providerThread.id, driver: CODEX_PROVIDER, diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index 9ffc57c638502..de7798a121859 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -2132,11 +2132,11 @@ export const layerWithOptions = ( const providerThread = projection.providerThreads.find( (thread) => thread.id === projection.thread.activeProviderThreadId, ); - const runtime = providerThread?.providerSessionId + const entry = providerThread?.providerSessionId ? (yield* Ref.get(sessions)).get(sessionKey(providerThread.providerSessionId)) - ?.exposedRuntime : undefined; - if (!providerThread || !runtime) + const runtime = entry?.exposedRuntime; + if (!providerThread || !entry || !runtime) return yield* new ProviderVoiceSessionError({ threadId: input.threadId, detail: "The provider session is no longer running. Send a message first.", @@ -2154,7 +2154,23 @@ export const layerWithOptions = ( providerThread, }); yield* Effect.acquireRelease( - markBusy(providerSessionId, busyTurnKey(providerThread.id, voiceOrdinal)), + markBusy( + providerSessionId, + busyTurnKey(providerThread.id, voiceOrdinal), + entry.runtime, + ).pipe( + Effect.flatMap((marked) => + marked + ? Effect.void + : Effect.fail( + new ProviderVoiceSessionError({ + threadId: input.threadId, + detail: + "The provider session was released while voice started. Send a message first.", + }), + ), + ), + ), () => markIdle(providerSessionId, providerThread.id, voiceOrdinal).pipe(Effect.ignore), ); diff --git a/apps/server/src/provider/CodexVoice.test.ts b/apps/server/src/provider/CodexVoice.test.ts index 74fcae20c9d37..c80da28d972f2 100644 --- a/apps/server/src/provider/CodexVoice.test.ts +++ b/apps/server/src/provider/CodexVoice.test.ts @@ -469,7 +469,32 @@ describe("makeCodexVoice", () => { .pipe(Stream.runCollect, Effect.forkChild); yield* Deferred.await(fake.startRequested); - yield* voice.expectSpokenTurn("run the tests"); + yield* voice.expectSpokenTurn("voice-message"); + yield* voice.registerTurn("typed-message", "typed"); + yield* fake.emit("item/started", { + threadId: PROVIDER_THREAD, + turnId: "typed", + item: { + type: "userMessage", + id: "user-typed", + content: [{ type: "text", text: "run the tests please" }], + }, + }); + yield* fake.emit("item/completed", { + threadId: PROVIDER_THREAD, + turnId: "typed", + item: { + type: "agentMessage", + id: "typed-answer", + text: "Typed answer.", + phase: "final_answer", + }, + }); + yield* fake.emit("turn/completed", { + threadId: PROVIDER_THREAD, + turn: { id: "typed", status: "completed" }, + }); + yield* voice.registerTurn("voice-message", "resubmitted"); yield* fake.emit("item/started", { threadId: PROVIDER_THREAD, turnId: "resubmitted", @@ -490,10 +515,15 @@ describe("makeCodexVoice", () => { }); yield* Deferred.await(fake.speechRequested); - NodeAssert.deepEqual(fake.requests.at(-1), { - method: "thread/realtime/appendSpeech", - params: { threadId: PROVIDER_THREAD, text: "All green." }, - }); + NodeAssert.deepEqual( + fake.requests.filter(({ method }) => method === "thread/realtime/appendSpeech"), + [ + { + method: "thread/realtime/appendSpeech", + params: { threadId: PROVIDER_THREAD, text: "All green." }, + }, + ], + ); yield* Fiber.interrupt(fiber); }), ); diff --git a/apps/server/src/provider/CodexVoice.ts b/apps/server/src/provider/CodexVoice.ts index 8e2494a2656a4..19d658183fe65 100644 --- a/apps/server/src/provider/CodexVoice.ts +++ b/apps/server/src/provider/CodexVoice.ts @@ -32,17 +32,6 @@ export function readRealtimeDelegationInput(text: string): string | undefined { return input.length > 0 ? input : undefined; } -function readUserMessageText(item: unknown): string | undefined { - if (!item || typeof item !== "object") return undefined; - const record = item as { type?: unknown; content?: unknown }; - if (record.type !== "userMessage" || !Array.isArray(record.content)) return undefined; - return (record.content as ReadonlyArray<{ type?: unknown; text?: unknown }>) - .flatMap((entry) => - entry.type === "text" && typeof entry.text === "string" ? [entry.text] : [], - ) - .join("\n"); -} - /** Reads the delegation input from a Codex `userMessage` item. */ export function readRealtimeDelegationFromItem(item: unknown): string | undefined { if (!item || typeof item !== "object") return undefined; @@ -97,7 +86,7 @@ interface ActiveVoice { /** Delegated turn id -> latest speakable answer in that turn. */ readonly delegatedTurns: Map; /** Handoffs resubmitted as ordinary runs, awaiting their native turn. */ - readonly expectedHandoffs: Array; + readonly expectedHandoffs: Set; closed: boolean; stopped: boolean; } @@ -185,15 +174,6 @@ export const makeCodexVoice = Effect.fn("makeCodexVoice")(function* ( active.delegatedTurns.set(payload.turnId, undefined); return; } - const text = readUserMessageText(payload.item); - const expectedIndex = - text === undefined - ? -1 - : active.expectedHandoffs.findIndex((expected) => text.includes(expected)); - if (expectedIndex !== -1) { - active.expectedHandoffs.splice(expectedIndex, 1); - active.delegatedTurns.set(payload.turnId, undefined); - } }), ), ); @@ -259,7 +239,7 @@ export const makeCodexVoice = Effect.fn("makeCodexVoice")(function* ( providerThreadId: input.providerThreadId, queue, delegatedTurns: new Map(), - expectedHandoffs: [], + expectedHandoffs: new Set(), closed: false, stopped: false, }; @@ -302,14 +282,18 @@ export const makeCodexVoice = Effect.fn("makeCodexVoice")(function* ( }), ); - /** - * Marks the next turn whose user message carries `text` as a voice handoff, - * for hosts that resubmit Codex's delegation as their own run. - */ - const expectSpokenTurn = (text: string) => + /** Registers the app message identity before its continuation is queued. */ + const expectSpokenTurn = (messageId: string) => + Effect.map(Ref.get(activeRef), (active) => { + if (active && !active.closed) active.expectedHandoffs.add(messageId); + }); + + const registerTurn = (messageId: string, turnId: string) => Effect.map(Ref.get(activeRef), (active) => { - if (active && !active.closed) active.expectedHandoffs.push(text); + if (active && !active.closed && active.expectedHandoffs.delete(messageId)) { + active.delegatedTurns.set(turnId, undefined); + } }); - return { start, expectSpokenTurn }; + return { start, expectSpokenTurn, registerTurn }; }); diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index 4c25351b3238d..a4ed708973361 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -1142,6 +1142,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- binds voice shortcuts to window keyboard events for the mounted composer useEffect(() => { + if (backgroundThread) return; const handler = (event: globalThis.KeyboardEvent) => { if (isCommandPaletteOpen()) return; const command = resolveShortcutCommand(event, keybindings, { @@ -1171,6 +1172,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) window.addEventListener("keydown", handler, true); return () => window.removeEventListener("keydown", handler, true); }, [ + backgroundThread, isComposerModelPickerOpen, keybindings, phase, diff --git a/apps/web/src/components/chat/useComposerSend.ts b/apps/web/src/components/chat/useComposerSend.ts index 7b2bb3519d40d..a3dce1ea4d403 100644 --- a/apps/web/src/components/chat/useComposerSend.ts +++ b/apps/web/src/components/chat/useComposerSend.ts @@ -725,6 +725,7 @@ export function useComposerSend(input: ComposerSendInput) { composerPreviewAnnotations.length === 0 && composerReviewComments.length === 0 && composerThreadContexts.length === 0 && + composerEnvironmentContexts.length === 0 && multipleModelSelections === null ? parseStandaloneComposerSlashCommand(trimmed) : null; diff --git a/docs/internals/voice-conversations.md b/docs/internals/voice-conversations.md index 1b9dc06e08ea1..2603e448228b1 100644 --- a/docs/internals/voice-conversations.md +++ b/docs/internals/voice-conversations.md @@ -13,11 +13,11 @@ is why the [shared controller](../../packages/client-runtime/src/voice-mode/cont takes a platform object: browsers and react-native-webrtc supply the same peer API. The stream's lifetime is the conversation's lifetime. When the client -unsubscribes or disconnects, the server stops the realtime session, so a dropped -client cannot leave a billed session running. +unsubscribes or disconnects, the server requests realtime stop. A failed stop +retains provider ownership and blocks replacement until termination is confirmed. Handoffs are client-managed, as in the Codex TUI. The -[Codex voice module](../../apps/server/src/provider/Layers/CodexVoice.ts) hands +[Codex voice module](../../apps/server/src/provider/CodexVoice.ts) hands each voice turn's final answer back with `thread/realtime/appendSpeech`, which keeps commentary and reasoning unspoken. From dacdc77c79a4dcadc39d81efb8cb2ef6d8250561 Mon Sep 17 00:00:00 2001 From: Jake Leventhal Date: Mon, 2 Nov 2026 10:54:55 -0500 Subject: [PATCH 11/19] fix(voice): retain retry handoffs and reset released audio mode --- .../features/voice-mode/voiceAudio.test.ts | 13 ++++++- .../src/features/voice-mode/voiceAudio.ts | 8 ++++- apps/server/src/provider/CodexVoice.test.ts | 34 +++++++++++++++++-- apps/server/src/provider/CodexVoice.ts | 27 +++++++++------ 4 files changed, 67 insertions(+), 15 deletions(-) diff --git a/apps/mobile/src/features/voice-mode/voiceAudio.test.ts b/apps/mobile/src/features/voice-mode/voiceAudio.test.ts index cba699d412b54..1af4c349f31d0 100644 --- a/apps/mobile/src/features/voice-mode/voiceAudio.test.ts +++ b/apps/mobile/src/features/voice-mode/voiceAudio.test.ts @@ -1,7 +1,8 @@ import { expect, it, vi } from "vite-plus/test"; +import type { AudioMode } from "expo-audio"; const audio = vi.hoisted(() => ({ - setMode: vi.fn<(mode: { allowsRecording?: boolean }) => Promise>(), + setMode: vi.fn<(mode: Partial) => Promise>(), })); vi.mock("expo-audio", () => ({ setAudioModeAsync: audio.setMode })); @@ -9,6 +10,7 @@ import { setVoiceAudioActive } from "./voiceAudio"; it("finishes an old release before activating its successor", async () => { let recording = true; + let currentMode: Partial = {}; const releaseStarted = Promise.withResolvers(); const finishRelease = Promise.withResolvers(); audio.setMode.mockImplementation(async (mode) => { @@ -17,6 +19,7 @@ it("finishes an old release before activating its successor", async () => { await finishRelease.promise; } recording = mode.allowsRecording === true; + currentMode = { ...currentMode, ...mode }; }); const release = setVoiceAudioActive(false); await releaseStarted.promise; @@ -28,4 +31,12 @@ it("finishes an old release before activating its successor", async () => { await expect(setVoiceAudioActive(false)).rejects.toThrow("release failed"); await setVoiceAudioActive(true); expect(recording).toBe(true); + await setVoiceAudioActive(false); + expect(currentMode).toEqual({ + allowsRecording: false, + playsInSilentMode: true, + interruptionMode: "mixWithOthers", + shouldPlayInBackground: false, + shouldRouteThroughEarpiece: false, + }); }); diff --git a/apps/mobile/src/features/voice-mode/voiceAudio.ts b/apps/mobile/src/features/voice-mode/voiceAudio.ts index 27273ed38d7b7..cb0235293175a 100644 --- a/apps/mobile/src/features/voice-mode/voiceAudio.ts +++ b/apps/mobile/src/features/voice-mode/voiceAudio.ts @@ -14,7 +14,13 @@ export function setVoiceAudioActive(active: boolean): Promise { shouldPlayInBackground: false, shouldRouteThroughEarpiece: false, } - : { allowsRecording: false }, + : { + allowsRecording: false, + playsInSilentMode: true, + interruptionMode: "mixWithOthers", + shouldPlayInBackground: false, + shouldRouteThroughEarpiece: false, + }, ), ); pending = result.catch(() => {}); diff --git a/apps/server/src/provider/CodexVoice.test.ts b/apps/server/src/provider/CodexVoice.test.ts index c80da28d972f2..8cde0a3749d12 100644 --- a/apps/server/src/provider/CodexVoice.test.ts +++ b/apps/server/src/provider/CodexVoice.test.ts @@ -31,6 +31,7 @@ function delegation(input: string) { const makeFakeClient = Effect.gen(function* () { const handlers = new Map Effect.Effect>>(); const requests: Array<{ readonly method: string; readonly params: unknown }> = []; + let scheduledSpeechCount = 0; let closeOnStop = false; let onStart: Effect.Effect = Effect.void; let onStop: Effect.Effect = Effect.void; @@ -48,8 +49,9 @@ const makeFakeClient = Effect.gen(function* () { handlers.set(method, [...(handlers.get(method) ?? []), handler]); }), raw: { - request: (method: string, params: unknown) => - Effect.gen(function* () { + request: (method: string, params: unknown) => { + if (method === "thread/realtime/appendSpeech") scheduledSpeechCount += 1; + return Effect.gen(function* () { requests.push({ method, params }); if (method === "thread/realtime/start") { yield* Deferred.succeed(startRequested, undefined); @@ -72,7 +74,8 @@ const makeFakeClient = Effect.gen(function* () { }); } return {}; - }), + }); + }, }, } as unknown as CodexClient.CodexAppServerClient["Service"]; const emit = (method: string, payload: unknown) => @@ -87,6 +90,9 @@ const makeFakeClient = Effect.gen(function* () { secondStartRequested, thirdStartRequested, speechRequested, + get scheduledSpeechCount() { + return scheduledSpeechCount; + }, setStart: (effect: typeof onStart) => { onStart = effect; }, @@ -494,6 +500,11 @@ describe("makeCodexVoice", () => { threadId: PROVIDER_THREAD, turn: { id: "typed", status: "completed" }, }); + yield* voice.registerTurn("voice-message", "capacity-failed"); + yield* fake.emit("turn/completed", { + threadId: PROVIDER_THREAD, + turn: { id: "capacity-failed", status: "failed" }, + }); yield* voice.registerTurn("voice-message", "resubmitted"); yield* fake.emit("item/started", { threadId: PROVIDER_THREAD, @@ -514,7 +525,24 @@ describe("makeCodexVoice", () => { turn: { id: "resubmitted", status: "completed" }, }); + NodeAssert.equal(fake.scheduledSpeechCount, 1); yield* Deferred.await(fake.speechRequested); + yield* voice.registerTurn("voice-message", "after-success"); + yield* fake.emit("item/completed", { + threadId: PROVIDER_THREAD, + turnId: "after-success", + item: { + type: "agentMessage", + id: "duplicate", + text: "Do not speak.", + phase: "final_answer", + }, + }); + yield* fake.emit("turn/completed", { + threadId: PROVIDER_THREAD, + turn: { id: "after-success", status: "completed" }, + }); + NodeAssert.equal(fake.scheduledSpeechCount, 1); NodeAssert.deepEqual( fake.requests.filter(({ method }) => method === "thread/realtime/appendSpeech"), [ diff --git a/apps/server/src/provider/CodexVoice.ts b/apps/server/src/provider/CodexVoice.ts index 19d658183fe65..c59a39a91060a 100644 --- a/apps/server/src/provider/CodexVoice.ts +++ b/apps/server/src/provider/CodexVoice.ts @@ -83,9 +83,8 @@ type VoiceQueue = Queue.Queue latest speakable answer in that turn. */ - readonly delegatedTurns: Map; - /** Handoffs resubmitted as ordinary runs, awaiting their native turn. */ + readonly delegatedTurns: Map; + /** Handoff message identities survive native retries until successful completion. */ readonly expectedHandoffs: Set; closed: boolean; stopped: boolean; @@ -171,7 +170,9 @@ export const makeCodexVoice = Effect.fn("makeCodexVoice")(function* ( withActive(payload.threadId, (active) => Effect.sync(() => { if (readRealtimeDelegationFromItem(payload.item) !== undefined) { - active.delegatedTurns.set(payload.turnId, undefined); + if (!active.delegatedTurns.has(payload.turnId)) { + active.delegatedTurns.set(payload.turnId, { answer: undefined }); + } return; } }), @@ -180,18 +181,24 @@ export const makeCodexVoice = Effect.fn("makeCodexVoice")(function* ( yield* client.handleServerNotification("item/completed", (payload) => withActive(payload.threadId, (active) => Effect.sync(() => { - if (!active.delegatedTurns.has(payload.turnId)) return; + const delegatedTurn = active.delegatedTurns.get(payload.turnId); + if (delegatedTurn === undefined) return; const answer = readSpeakableAnswer(payload.item); - if (answer !== undefined) active.delegatedTurns.set(payload.turnId, answer); + if (answer !== undefined) delegatedTurn.answer = answer; }), ), ); yield* client.handleServerNotification("turn/completed", (payload) => withActive(payload.threadId, (active) => Effect.suspend(() => { - const answer = active.delegatedTurns.get(payload.turn.id); + const delegatedTurn = active.delegatedTurns.get(payload.turn.id); active.delegatedTurns.delete(payload.turn.id); - if (answer === undefined || payload.turn.status !== "completed") return Effect.void; + if (payload.turn.status !== "completed") return Effect.void; + if (delegatedTurn?.messageId !== undefined) { + active.expectedHandoffs.delete(delegatedTurn.messageId); + } + const answer = delegatedTurn?.answer; + if (answer === undefined) return Effect.void; return client.raw .request("thread/realtime/appendSpeech", { threadId: active.providerThreadId, @@ -290,8 +297,8 @@ export const makeCodexVoice = Effect.fn("makeCodexVoice")(function* ( const registerTurn = (messageId: string, turnId: string) => Effect.map(Ref.get(activeRef), (active) => { - if (active && !active.closed && active.expectedHandoffs.delete(messageId)) { - active.delegatedTurns.set(turnId, undefined); + if (active && !active.closed && active.expectedHandoffs.has(messageId)) { + active.delegatedTurns.set(turnId, { messageId, answer: undefined }); } }); From 92d9898cb659257bbcbc29008ff69faf64b01991 Mon Sep 17 00:00:00 2001 From: Jake Leventhal Date: Mon, 2 Nov 2026 12:50:48 -0500 Subject: [PATCH 12/19] fix(codex): keep final speech within its owning voice session --- apps/server/src/provider/CodexVoice.test.ts | 71 ++++++++++++++++++++- apps/server/src/provider/CodexVoice.ts | 24 +++++-- 2 files changed, 87 insertions(+), 8 deletions(-) diff --git a/apps/server/src/provider/CodexVoice.test.ts b/apps/server/src/provider/CodexVoice.test.ts index 8cde0a3749d12..2e641b99471fe 100644 --- a/apps/server/src/provider/CodexVoice.test.ts +++ b/apps/server/src/provider/CodexVoice.test.ts @@ -35,6 +35,7 @@ const makeFakeClient = Effect.gen(function* () { let closeOnStop = false; let onStart: Effect.Effect = Effect.void; let onStop: Effect.Effect = Effect.void; + let onSpeech: Effect.Effect = Effect.void; const startRequested = yield* Deferred.make(); const secondStartRequested = yield* Deferred.make(); const thirdStartRequested = yield* Deferred.make(); @@ -52,6 +53,7 @@ const makeFakeClient = Effect.gen(function* () { request: (method: string, params: unknown) => { if (method === "thread/realtime/appendSpeech") scheduledSpeechCount += 1; return Effect.gen(function* () { + if (method === "thread/realtime/appendSpeech") yield* onSpeech; requests.push({ method, params }); if (method === "thread/realtime/start") { yield* Deferred.succeed(startRequested, undefined); @@ -99,6 +101,9 @@ const makeFakeClient = Effect.gen(function* () { setStop: (effect: typeof onStop) => { onStop = effect; }, + setSpeech: (effect: typeof onSpeech) => { + onSpeech = effect; + }, stopRequested, closePreviousOnStop: () => { closeOnStop = true; @@ -136,9 +141,21 @@ describe("readSpeakableAnswer", () => { readSpeakableAnswer({ type: "agentMessage", text: "[FINAL] All green.", phase: null }), "All green.", ); + NodeAssert.equal( + readSpeakableAnswer({ type: "agentMessage", text: "All green.", phase: "future_phase" }), + "All green.", + ); }); - it("keeps commentary private", () => { + it("keeps commentary and partial answers private", () => { + NodeAssert.equal( + readSpeakableAnswer({ + type: "agentMessage", + text: "Still working.", + phase: "partial_answer", + }), + undefined, + ); NodeAssert.equal( readSpeakableAnswer({ type: "agentMessage", text: "Looking…", phase: "commentary" }), undefined, @@ -465,6 +482,56 @@ describe("makeCodexVoice", () => { }), ); + it.effect("keeps in-flight speech with its owner before replacing a realtime session", () => + Effect.scoped( + Effect.gen(function* () { + const fake = yield* makeFakeClient; + const voice = yield* makeCodexVoice(fake.client, yield* Effect.scope); + const start = () => + voice + .start({ providerThreadId: PROVIDER_THREAD, offerSdp: "offer" }) + .pipe(Stream.runCollect, Effect.forkChild({ startImmediately: true })); + const first = yield* start(); + yield* Deferred.await(fake.startRequested); + const speechEntered = yield* Deferred.make(); + const releaseSpeech = yield* Deferred.make(); + fake.setSpeech( + Deferred.succeed(speechEntered, undefined).pipe( + Effect.andThen(Deferred.await(releaseSpeech)), + ), + ); + yield* voice.expectSpokenTurn("voice-message"); + yield* voice.registerTurn("voice-message", "delegated"); + yield* fake.emit("item/completed", { + threadId: PROVIDER_THREAD, + turnId: "delegated", + item: { type: "agentMessage", text: "Old answer.", phase: "final_answer" }, + }); + yield* fake.emit("turn/completed", { + threadId: PROVIDER_THREAD, + turn: { id: "delegated", status: "completed" }, + }); + yield* Deferred.await(speechEntered); + const replacement = yield* start(); + const whileSpeaking = fake.requests.map(({ method }) => method); + yield* Deferred.succeed(releaseSpeech, undefined); + NodeAssert.deepEqual(whileSpeaking, ["thread/realtime/start"]); + yield* Deferred.await(fake.secondStartRequested); + NodeAssert.deepEqual( + fake.requests.map(({ method }) => method), + [ + "thread/realtime/start", + "thread/realtime/appendSpeech", + "thread/realtime/stop", + "thread/realtime/start", + ], + ); + yield* Fiber.join(first); + yield* Fiber.interrupt(replacement); + }), + ), + ); + it.effect("speaks the answer of a handoff the host resubmitted as its own turn", () => Effect.gen(function* () { const fake = yield* makeFakeClient; @@ -525,8 +592,8 @@ describe("makeCodexVoice", () => { turn: { id: "resubmitted", status: "completed" }, }); - NodeAssert.equal(fake.scheduledSpeechCount, 1); yield* Deferred.await(fake.speechRequested); + NodeAssert.equal(fake.scheduledSpeechCount, 1); yield* voice.registerTurn("voice-message", "after-success"); yield* fake.emit("item/completed", { threadId: PROVIDER_THREAD, diff --git a/apps/server/src/provider/CodexVoice.ts b/apps/server/src/provider/CodexVoice.ts index c59a39a91060a..472b946bf597c 100644 --- a/apps/server/src/provider/CodexVoice.ts +++ b/apps/server/src/provider/CodexVoice.ts @@ -50,7 +50,11 @@ export function readSpeakableAnswer(item: unknown): string | undefined { if (!item || typeof item !== "object") return undefined; const record = item as { type?: unknown; text?: unknown; phase?: unknown; questions?: unknown }; if (record.type !== "agentMessage" || typeof record.text !== "string") return undefined; - if (record.phase === "commentary" || (Array.isArray(record.questions) && record.questions.length)) + if ( + record.phase === "commentary" || + record.phase === "partial_answer" || + (Array.isArray(record.questions) && record.questions.length) + ) return undefined; const text = record.text.trim(); if ( @@ -199,11 +203,19 @@ export const makeCodexVoice = Effect.fn("makeCodexVoice")(function* ( } const answer = delegatedTurn?.answer; if (answer === undefined) return Effect.void; - return client.raw - .request("thread/realtime/appendSpeech", { - threadId: active.providerThreadId, - text: answer, - }) + return transition + .withPermit( + Ref.get(activeRef).pipe( + Effect.flatMap((current) => + current === active && !active.closed + ? client.raw.request("thread/realtime/appendSpeech", { + threadId: active.providerThreadId, + text: answer, + }) + : Effect.void, + ), + ), + ) .pipe( Effect.catch((cause) => Effect.logWarning("Codex voice could not speak the delegated answer", { cause }), From b99414ca4a07dc2e8d48f0a66d01072adda4343c Mon Sep 17 00:00:00 2001 From: Jake Leventhal Date: Mon, 2 Nov 2026 13:07:28 -0500 Subject: [PATCH 13/19] fix(voice): bound speech and cleanup acknowledgments --- apps/server/src/provider/CodexVoice.test.ts | 110 ++++++++++++++++++-- apps/server/src/provider/CodexVoice.ts | 60 +++++++++-- 2 files changed, 152 insertions(+), 18 deletions(-) diff --git a/apps/server/src/provider/CodexVoice.test.ts b/apps/server/src/provider/CodexVoice.test.ts index 2e641b99471fe..eacee2d182387 100644 --- a/apps/server/src/provider/CodexVoice.test.ts +++ b/apps/server/src/provider/CodexVoice.test.ts @@ -10,6 +10,7 @@ import * as Logger from "effect/Logger"; import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; import type * as CodexClient from "effect-codex-app-server/client"; import * as CodexErrors from "effect-codex-app-server/errors"; @@ -36,6 +37,7 @@ const makeFakeClient = Effect.gen(function* () { let onStart: Effect.Effect = Effect.void; let onStop: Effect.Effect = Effect.void; let onSpeech: Effect.Effect = Effect.void; + let onSpeechResponse: typeof onSpeech = Effect.void; const startRequested = yield* Deferred.make(); const secondStartRequested = yield* Deferred.make(); const thirdStartRequested = yield* Deferred.make(); @@ -65,6 +67,7 @@ const makeFakeClient = Effect.gen(function* () { } if (method === "thread/realtime/appendSpeech") { yield* Deferred.succeed(speechRequested, undefined); + yield* onSpeechResponse; } if (method === "thread/realtime/stop") { yield* Deferred.succeed(stopRequested, undefined); @@ -104,6 +107,9 @@ const makeFakeClient = Effect.gen(function* () { setSpeech: (effect: typeof onSpeech) => { onSpeech = effect; }, + setSpeechResponse: (effect: typeof onSpeechResponse) => { + onSpeechResponse = effect; + }, stopRequested, closePreviousOnStop: () => { closeOnStop = true; @@ -455,15 +461,105 @@ describe("makeCodexVoice", () => { ), ); - it.effect("stops the realtime session when the client unsubscribes", () => + it.effect("stops on unsubscribe when only the closed notification confirms cleanup", () => Effect.gen(function* () { const voice = yield* startVoice; - yield* Fiber.interrupt(voice.fiber); - yield* Deferred.await(voice.stopRequested); - NodeAssert.deepEqual(voice.requests.at(-1), { - method: "thread/realtime/stop", - params: { threadId: PROVIDER_THREAD }, - }); + const releaseStop = yield* Deferred.make(); + const stopInterrupted = yield* Deferred.make(); + voice.setStop( + Deferred.await(releaseStop).pipe( + Effect.onInterrupt(() => Deferred.succeed(stopInterrupted, undefined)), + ), + ); + yield* Effect.gen(function* () { + const cleanup = yield* Fiber.interrupt(voice.fiber).pipe(Effect.forkChild); + yield* Deferred.await(voice.stopRequested); + yield* voice.emit("thread/realtime/closed", { threadId: PROVIDER_THREAD }); + yield* TestClock.adjust("9 seconds"); + NodeAssert.equal(yield* Deferred.isDone(stopInterrupted), true); + NodeAssert.ok(cleanup.pollUnsafe()); + yield* Fiber.join(cleanup); + NodeAssert.deepEqual(voice.requests.at(-1), { + method: "thread/realtime/stop", + params: { threadId: PROVIDER_THREAD }, + }); + }).pipe(Effect.ensuring(Deferred.succeed(releaseStop, undefined))); + }), + ); + + it.effect("bounds unanswered speech and stop without replacing an unconfirmed session", () => + Effect.gen(function* () { + const releaseSpeech = yield* Deferred.make(); + const releaseStop = yield* Deferred.make(); + yield* Effect.scoped( + Effect.gen(function* () { + const fake = yield* makeFakeClient; + const voice = yield* makeCodexVoice(fake.client, yield* Effect.scope); + const speechInterrupted = yield* Deferred.make(); + fake.setSpeechResponse( + Deferred.await(releaseSpeech).pipe( + Effect.onInterrupt(() => Deferred.succeed(speechInterrupted, undefined)), + ), + ); + fake.setStop(Deferred.await(releaseStop)); + const first = yield* voice + .start({ providerThreadId: PROVIDER_THREAD, offerSdp: "offer" }) + .pipe(Stream.runCollect, Effect.forkChild); + yield* Deferred.await(fake.startRequested); + yield* voice.expectSpokenTurn("voice-message"); + yield* voice.registerTurn("voice-message", "delegated"); + yield* fake.emit("item/completed", { + threadId: PROVIDER_THREAD, + turnId: "delegated", + item: { type: "agentMessage", text: "Old answer.", phase: "final_answer" }, + }); + yield* fake.emit("turn/completed", { + threadId: PROVIDER_THREAD, + turn: { id: "delegated", status: "completed" }, + }); + yield* Deferred.await(fake.speechRequested); + yield* TestClock.adjust("10 seconds"); + NodeAssert.equal(yield* Deferred.isDone(speechInterrupted), true); + NodeAssert.deepEqual( + fake.requests.map(({ method }) => method), + ["thread/realtime/start", "thread/realtime/appendSpeech", "thread/realtime/stop"], + ); + yield* TestClock.adjust("10 seconds"); + NodeAssert.ok(first.pollUnsafe()); + NodeAssert.ok(Exit.isFailure(yield* Fiber.await(first))); + fake.setStop( + Effect.fail( + new CodexErrors.CodexAppServerRequestError({ + code: -32603, + errorMessage: "stop still unconfirmed", + }), + ), + ); + const replacement = yield* voice + .start({ providerThreadId: PROVIDER_THREAD, offerSdp: "replacement" }) + .pipe(Stream.runCollect, Effect.exit); + NodeAssert.ok(Exit.isFailure(replacement)); + NodeAssert.equal( + fake.requests.filter(({ method }) => method.endsWith("/start")).length, + 1, + ); + yield* fake.emit("thread/realtime/closed", { threadId: PROVIDER_THREAD }); + const recovered = yield* voice + .start({ providerThreadId: PROVIDER_THREAD, offerSdp: "recovered" }) + .pipe(Stream.runCollect, Effect.forkChild); + yield* Deferred.await(fake.secondStartRequested); + yield* Deferred.succeed(releaseSpeech, undefined); + NodeAssert.equal( + fake.requests.filter(({ method }) => method.endsWith("/appendSpeech")).length, + 1, + ); + yield* fake.emit("thread/realtime/closed", { threadId: PROVIDER_THREAD }); + yield* Fiber.join(recovered); + }).pipe( + Effect.ensuring(Deferred.succeed(releaseSpeech, undefined)), + Effect.ensuring(Deferred.succeed(releaseStop, undefined)), + ), + ); }), ); diff --git a/apps/server/src/provider/CodexVoice.ts b/apps/server/src/provider/CodexVoice.ts index 472b946bf597c..1c74a38fb4530 100644 --- a/apps/server/src/provider/CodexVoice.ts +++ b/apps/server/src/provider/CodexVoice.ts @@ -2,6 +2,7 @@ * Client handoffs start Codex turns, then append their final answer to live speech. */ import type { ProviderVoiceSessionEvent } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; @@ -87,6 +88,7 @@ type VoiceQueue = Queue.Queue; readonly delegatedTurns: Map; /** Handoff message identities survive native retries until successful completion. */ readonly expectedHandoffs: Set; @@ -161,12 +163,16 @@ export const makeCodexVoice = Effect.fn("makeCodexVoice")(function* ( Effect.flatMap(Ref.get(activeRef), (active) => { if (!active || active.providerThreadId !== payload.threadId) return Effect.void; active.stopped = true; - return active.closed - ? Effect.void - : finish(active, { - type: "closed", - ...(payload.reason ? { reason: payload.reason } : {}), - }); + return Deferred.succeed(active.stopConfirmed, undefined).pipe( + Effect.andThen( + active.closed + ? Effect.void + : finish(active, { + type: "closed", + ...(payload.reason ? { reason: payload.reason } : {}), + }), + ), + ); }), ); @@ -208,10 +214,27 @@ export const makeCodexVoice = Effect.fn("makeCodexVoice")(function* ( Ref.get(activeRef).pipe( Effect.flatMap((current) => current === active && !active.closed - ? client.raw.request("thread/realtime/appendSpeech", { - threadId: active.providerThreadId, - text: answer, - }) + ? client.raw + .request("thread/realtime/appendSpeech", { + threadId: active.providerThreadId, + text: answer, + }) + .pipe( + Effect.timeoutOrElse({ + duration: "10 seconds", + orElse: () => + Effect.suspend(() => { + active.closed = true; + return Queue.fail( + active.queue, + new CodexVoiceSessionError({ + detail: + "Codex did not acknowledge speech; ending the voice session.", + }), + ).pipe(Effect.asVoid); + }), + }), + ) : Effect.void, ), ), @@ -231,7 +254,21 @@ export const makeCodexVoice = Effect.fn("makeCodexVoice")(function* ( if ((yield* Ref.get(activeRef)) !== active) return; active.closed = true; if (!active.stopped) { - yield* client.raw.request("thread/realtime/stop", { threadId: active.providerThreadId }); + // A timed-out append may already be queued; stop confirmation is the replacement barrier. + yield* Effect.raceFirst( + client.raw.request("thread/realtime/stop", { threadId: active.providerThreadId }), + Deferred.await(active.stopConfirmed), + ).pipe( + Effect.timeoutOrElse({ + duration: "10 seconds", + orElse: () => + Effect.fail( + new CodexVoiceSessionError({ + detail: "Codex did not confirm voice cleanup; retry after the session closes.", + }), + ), + }), + ); active.stopped = true; } yield* Ref.set(activeRef, undefined); @@ -257,6 +294,7 @@ export const makeCodexVoice = Effect.fn("makeCodexVoice")(function* ( const active: ActiveVoice = { providerThreadId: input.providerThreadId, queue, + stopConfirmed: yield* Deferred.make(), delegatedTurns: new Map(), expectedHandoffs: new Set(), closed: false, From a4c50d20efde035f40a4edf7f4ea76edaa0680aa Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Tue, 3 Nov 2026 23:18:15 +0000 Subject: [PATCH 14/19] fix(voice): preserve steering, permissions, and audio ownership --- .../features/voice-mode/voiceAudio.test.ts | 22 ++++- .../src/features/voice-mode/voiceAudio.ts | 40 ++++----- .../src/features/voice-mode/voiceMode.ts | 2 +- .../Adapters/CodexAdapterV2.test.ts | 86 ++++++++++++++++++- .../Adapters/CodexAdapterV2.ts | 55 ++++++++++++ apps/web/src/components/CommandPalette.tsx | 5 ++ apps/web/src/components/chat/ChatComposer.tsx | 1 + apps/web/src/voice/voiceAvailability.test.ts | 3 + apps/web/src/voice/voiceAvailability.ts | 6 ++ docs/internals/voice-conversations.md | 2 +- 10 files changed, 189 insertions(+), 33 deletions(-) diff --git a/apps/mobile/src/features/voice-mode/voiceAudio.test.ts b/apps/mobile/src/features/voice-mode/voiceAudio.test.ts index 1af4c349f31d0..d7d05ae40918d 100644 --- a/apps/mobile/src/features/voice-mode/voiceAudio.test.ts +++ b/apps/mobile/src/features/voice-mode/voiceAudio.test.ts @@ -4,23 +4,38 @@ import type { AudioMode } from "expo-audio"; const audio = vi.hoisted(() => ({ setMode: vi.fn<(mode: Partial) => Promise>(), })); -vi.mock("expo-audio", () => ({ setAudioModeAsync: audio.setMode })); +vi.mock("expo-audio", () => ({ + setAudioModeAsync: audio.setMode, + setIsAudioActiveAsync: async () => {}, + AudioModule: { setAudioSessionOwner: () => {} }, +})); + +import { createAudioPlaybackSession } from "../../lib/audioSession"; import { setVoiceAudioActive } from "./voiceAudio"; it("finishes an old release before activating its successor", async () => { let recording = true; + let holdRelease = false; let currentMode: Partial = {}; const releaseStarted = Promise.withResolvers(); const finishRelease = Promise.withResolvers(); audio.setMode.mockImplementation(async (mode) => { - if (mode.allowsRecording === false) { + if (mode.allowsRecording === false && holdRelease) { releaseStarted.resolve(); await finishRelease.promise; } recording = mode.allowsRecording === true; currentMode = { ...currentMode, ...mode }; }); + await setVoiceAudioActive(true); + const player = { play: vi.fn(), pause: vi.fn() }; + const playback = createAudioPlaybackSession(player); + await playback.play(); + expect(player.play).not.toHaveBeenCalled(); + playback.pause(); + expect(recording).toBe(true); + holdRelease = true; const release = setVoiceAudioActive(false); await releaseStarted.promise; const successor = setVoiceAudioActive(true); @@ -35,8 +50,7 @@ it("finishes an old release before activating its successor", async () => { expect(currentMode).toEqual({ allowsRecording: false, playsInSilentMode: true, - interruptionMode: "mixWithOthers", + interruptionMode: "doNotMix", shouldPlayInBackground: false, - shouldRouteThroughEarpiece: false, }); }); diff --git a/apps/mobile/src/features/voice-mode/voiceAudio.ts b/apps/mobile/src/features/voice-mode/voiceAudio.ts index cb0235293175a..b34d65d9abe1e 100644 --- a/apps/mobile/src/features/voice-mode/voiceAudio.ts +++ b/apps/mobile/src/features/voice-mode/voiceAudio.ts @@ -1,28 +1,18 @@ -import { setAudioModeAsync } from "expo-audio"; +import { + acquireAudioSession, + activateAudioSession, + releaseAudioSession, +} from "../../lib/audioSession"; -// Expo audio-mode changes must finish before a successor configures the shared native session. -let pending = Promise.resolve(); +let owner: ReturnType | null = null; -export function setVoiceAudioActive(active: boolean): Promise { - const result = pending.then(() => - setAudioModeAsync( - active - ? { - allowsRecording: true, - playsInSilentMode: true, - interruptionMode: "doNotMix", - shouldPlayInBackground: false, - shouldRouteThroughEarpiece: false, - } - : { - allowsRecording: false, - playsInSilentMode: true, - interruptionMode: "mixWithOthers", - shouldPlayInBackground: false, - shouldRouteThroughEarpiece: false, - }, - ), - ); - pending = result.catch(() => {}); - return result; +export async function setVoiceAudioActive(active: boolean, onCancel?: () => void): Promise { + if (active) { + owner ??= acquireAudioSession("recording", onCancel); + await activateAudioSession(owner); + } else { + const previous = owner; + owner = null; + if (previous) await releaseAudioSession(previous); + } } diff --git a/apps/mobile/src/features/voice-mode/voiceMode.ts b/apps/mobile/src/features/voice-mode/voiceMode.ts index 8d0795864d883..d1be636e2804d 100644 --- a/apps/mobile/src/features/voice-mode/voiceMode.ts +++ b/apps/mobile/src/features/voice-mode/voiceMode.ts @@ -71,7 +71,7 @@ voiceMode.subscribe(({ phase }) => { if (phase === "active" && !routedAudio) { routedAudio = true; // WebRTC resets iOS routing when its audio unit starts, so route after activation. - setVoiceAudioActive(true).catch(() => undefined); + setVoiceAudioActive(true, () => voiceMode.stop()).catch(() => undefined); } if (phase !== "idle" && !keepAwake) { // A hands-free conversation must not auto-lock: backgrounding ends it. diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts index d53e26f53c7cc..e516314bb3026 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts @@ -1481,7 +1481,12 @@ describe("CodexAdapterV2 post-settle continuation", () => { } }).pipe( Effect.provide( - Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), ), ), ), @@ -1605,9 +1610,86 @@ describe("CodexAdapterV2 post-settle continuation", () => { assert.equal(request?.delivery, "message_text"); assert.equal(request?.detail, "run the tests"); assert.equal(request?.providerThreadId, harness.providerThread.id); + const preamble = codexReplayPreamble({ + nativeThreadId, + nativeTurnId: "active", + prompt: "work", + }); + const voiceStarted = yield* Deferred.make(); + const activeHarness = yield* makeCodexReplayHarness( + makeCodexReplayTranscript({ + scenario: "voice-active-steer", + entries: [ + ...preamble.slice(0, 5), + { + type: "expect_outbound", + label: "thread/realtime/start", + frame: { + id: 3, + method: "thread/realtime/start", + params: { + threadId: nativeThreadId, + outputModality: "audio", + transport: { type: "webrtc", sdp: "v=offer\r\n" }, + version: "v3", + clientManagedHandoffs: true, + includeStartupContext: false, + }, + }, + }, + { + type: "emit_inbound", + label: "thread/realtime/start", + frame: { id: 3, result: {} }, + }, + ...preamble.slice(5).map((entry) => withReplayRequestId(entry, 4)), + ...voiceTurn("active", "also check types", 5, "completed").filter( + (entry) => + entry.label === "item/started/active" || + entry.label === "turn/completed/active", + ), + ], + }), + undefined, + undefined, + undefined, + (entry) => + entry.label === "thread/realtime/start" + ? Deferred.succeed(voiceStarted, undefined) + : Effect.void, + ); + yield* activeHarness.runtime.startVoiceSession!({ + providerThread: activeHarness.providerThread, + offerSdp: "v=offer\r\n", + }).pipe(Stream.runDrain, Effect.forkScoped); + yield* Deferred.await(voiceStarted); + const activeInput = makeCodexTestTurnInput({ + threadId: activeHarness.threadId, + providerThread: activeHarness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("voice-active"), + text: "work", + }); + yield* activeHarness.runtime.startTurn(activeInput); + yield* activeHarness.firstTerminal; + const spoken = activeHarness.events.find( + (event) => + event.type === "message.updated" && event.message.text === "also check types", + ); + assert.equal(spoken?.type, "message.updated"); + if (spoken?.type === "message.updated") { + assert.equal(spoken.message.runId, activeInput.runId); + assert.equal(spoken.message.createdBy, "user"); + } + assert.equal(activeHarness.continuationRequests.length, 0); }).pipe( Effect.provide( - Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), ), ), ), diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index ec10722e7c8de..db07c99cc3b10 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -4879,6 +4879,61 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( } if (payload.item.type === "userMessage") { + const handoff = readRealtimeDelegationFromItem(payload.item); + const voiceThread = yield* Ref.get(voiceProviderThread); + if ( + handoff !== undefined && + context.subagent === null && + voiceThread?.nativeThreadRef?.nativeId === payload.threadId + ) { + const { ordinal, startedAt } = yield* resolveItemPosition( + context, + payload.item.id, + DateTime.makeUnsafe(payload.startedAtMs), + ); + const artifacts = makeSubagentConversationArtifacts({ + messageId: idAllocator.derive.messageFromProviderItem({ + driver: CODEX_PROVIDER, + nativeItemId: payload.item.id, + }), + turnItemId: idAllocator.derive.turnItemFromProviderItem({ + driver: CODEX_PROVIDER, + nativeItemId: payload.item.id, + }), + threadId: context.projectionThreadId, + rootNodeId: context.providerNodeId, + providerThreadId: context.providerThread.id, + providerTurnId: context.providerTurnId, + nativeItemRef: codexNativeItemRef(payload.item.id), + role: "user", + text: handoff, + ordinal, + now: startedAt, + }); + yield* emitProviderEvent({ + type: "message.updated", + driver: CODEX_PROVIDER, + message: { + ...artifacts.message, + runId: context.projectionRunId, + createdBy: "user", + }, + }); + if (artifacts.turnItem.type === "user_message") + yield* emitProviderEvent({ + type: "turn_item.updated", + driver: CODEX_PROVIDER, + turnItem: { + ...artifacts.turnItem, + runId: context.projectionRunId, + createdBy: "user", + inputIntent: "steer", + }, + }); + yield* voice.expectSpokenTurn(context.input.message.messageId); + yield* voice.registerTurn(context.input.message.messageId, payload.turnId); + return; + } yield* emitSubagentUserMessage( context, payload.item, diff --git a/apps/web/src/components/CommandPalette.tsx b/apps/web/src/components/CommandPalette.tsx index bc26900eeeae7..0e593dbe93eab 100644 --- a/apps/web/src/components/CommandPalette.tsx +++ b/apps/web/src/components/CommandPalette.tsx @@ -1459,6 +1459,10 @@ function OpenCommandPaletteDialog(props: { }); } + const canStartVoice = useEnvironmentScope( + activeThread?.environmentId ?? null, + AuthOrchestrationOperateScope, + ); const voiceEngaged = voiceState.phase !== "idle"; const voiceTarget = activeThread ? scopeThreadRef(activeThread.environmentId, activeThread.id) @@ -1468,6 +1472,7 @@ function OpenCommandPaletteDialog(props: { const voiceAvailability = resolveThreadVoiceAvailability( activeThread, providersByEnvironment.get(activeThread.environmentId) ?? [], + canStartVoice, ); if (voiceAvailability.kind !== "unsupported") { actionItems.push({ diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index fbdf34792a45c..6a2e3dce1488b 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -958,6 +958,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) provider: providerStatuses.find((provider) => provider.instanceId === voiceProviderInstanceId), hasMessages: voiceThread?.latestUserMessageAt != null, hasSession: voiceThread?.activeProviderThreadId != null, + canOperateThread, }); const voicePhase = useVoicePhaseFor(routeThreadRef); const voiceEngaged = voicePhase !== "idle"; diff --git a/apps/web/src/voice/voiceAvailability.test.ts b/apps/web/src/voice/voiceAvailability.test.ts index 0d5b0ff85e1f5..cab78a3df3d9a 100644 --- a/apps/web/src/voice/voiceAvailability.test.ts +++ b/apps/web/src/voice/voiceAvailability.test.ts @@ -36,5 +36,8 @@ describe("resolveThreadVoiceAvailability", () => { expect( resolveThreadVoiceAvailability(thread({ provider: codex, messages: true }), providers), ).toEqual({ kind: "ready" }); + expect( + resolveThreadVoiceAvailability(thread({ provider: codex, messages: true }), providers, false), + ).toEqual({ kind: "unavailable", reason: "This connection cannot operate threads" }); }); }); diff --git a/apps/web/src/voice/voiceAvailability.ts b/apps/web/src/voice/voiceAvailability.ts index 1e919f3b8d203..1e3ef32821599 100644 --- a/apps/web/src/voice/voiceAvailability.ts +++ b/apps/web/src/voice/voiceAvailability.ts @@ -16,8 +16,12 @@ export function resolveVoiceAvailability(input: { readonly provider: Pick | null | undefined; readonly hasMessages: boolean; readonly hasSession: boolean; + readonly canOperateThread?: boolean; }): VoiceAvailability { if (input.provider?.supportsVoice !== true) return VOICE_UNSUPPORTED; + if (input.canOperateThread === false) { + return { kind: "unavailable", reason: "This connection cannot operate threads" }; + } // The server hands voice requests to the thread's bound provider session, // which only exists once a turn has been sent. if (!input.hasMessages || !input.hasSession) { @@ -37,10 +41,12 @@ export interface VoiceThread { export function resolveThreadVoiceAvailability( thread: VoiceThread, providers: ReadonlyArray, + canOperateThread = true, ): VoiceAvailability { return resolveVoiceAvailability({ provider: providers.find((provider) => provider.instanceId === thread.providerInstanceId), hasMessages: thread.latestUserMessageAt !== null, hasSession: thread.activeProviderThreadId !== null, + canOperateThread, }); } diff --git a/docs/internals/voice-conversations.md b/docs/internals/voice-conversations.md index 2603e448228b1..e1d8544280765 100644 --- a/docs/internals/voice-conversations.md +++ b/docs/internals/voice-conversations.md @@ -23,7 +23,7 @@ keeps commentary and reasoning unspoken. Codex starts or steers the delegated turn itself, with a user message wrapped in ``, and it cannot be told not to. Steering an active run -needs nothing special. An idle thread is different: V2 runs are +keeps its native delivery and records the spoken request in the active run. An idle thread is different: V2 runs are orchestrator-owned and there is no provider-initiated run yet, so [CodexAdapterV2](../../apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts) interrupts that turn and, once it settles, offers a `voice` continuation. The From 2ee5719ac7848684e12c2bef95f7327835a2aa17 Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Tue, 3 Nov 2026 23:44:46 +0000 Subject: [PATCH 15/19] fix(voice): release failed audio sessions and respect composition --- .../src/features/voice-mode/voiceAudio.test.ts | 11 +++++++++-- .../src/features/voice-mode/voiceAudio.ts | 18 ++++++++++++++++-- .../src/features/voice-mode/voiceMode.ts | 11 +++++++++-- apps/mobile/src/lib/audioSession.ts | 3 ++- apps/web/src/components/CommandPalette.tsx | 5 ++++- apps/web/src/components/chat/ChatComposer.tsx | 2 +- 6 files changed, 41 insertions(+), 9 deletions(-) diff --git a/apps/mobile/src/features/voice-mode/voiceAudio.test.ts b/apps/mobile/src/features/voice-mode/voiceAudio.test.ts index d7d05ae40918d..0cdc1b5fbe9d4 100644 --- a/apps/mobile/src/features/voice-mode/voiceAudio.test.ts +++ b/apps/mobile/src/features/voice-mode/voiceAudio.test.ts @@ -47,10 +47,17 @@ it("finishes an old release before activating its successor", async () => { await setVoiceAudioActive(true); expect(recording).toBe(true); await setVoiceAudioActive(false); - expect(currentMode).toEqual({ + expect.soft(currentMode).toEqual({ allowsRecording: false, playsInSilentMode: true, - interruptionMode: "doNotMix", + interruptionMode: "mixWithOthers", shouldPlayInBackground: false, }); + const onCancel = vi.fn(); + audio.setMode.mockRejectedValueOnce(new Error("activation failed")); + await expect(setVoiceAudioActive(true, onCancel)).rejects.toThrow("activation failed"); + expect.soft(onCancel).toHaveBeenCalled(); + await playback.play(); + expect.soft(player.play).toHaveBeenCalledOnce(); + playback.pause(); }); diff --git a/apps/mobile/src/features/voice-mode/voiceAudio.ts b/apps/mobile/src/features/voice-mode/voiceAudio.ts index b34d65d9abe1e..2061d8cb3830c 100644 --- a/apps/mobile/src/features/voice-mode/voiceAudio.ts +++ b/apps/mobile/src/features/voice-mode/voiceAudio.ts @@ -6,10 +6,24 @@ import { let owner: ReturnType | null = null; -export async function setVoiceAudioActive(active: boolean, onCancel?: () => void): Promise { +export async function setVoiceAudioActive( + active: boolean, + onCancel?: (error?: unknown) => void, +): Promise { if (active) { owner ??= acquireAudioSession("recording", onCancel); - await activateAudioSession(owner); + const current = owner; + try { + await activateAudioSession(current); + } catch (error) { + if (owner === current) { + owner = null; + const release = releaseAudioSession(current); + onCancel?.(error); + await release.catch(() => undefined); + } + throw error; + } } else { const previous = owner; owner = null; diff --git a/apps/mobile/src/features/voice-mode/voiceMode.ts b/apps/mobile/src/features/voice-mode/voiceMode.ts index d1be636e2804d..54adf820b959c 100644 --- a/apps/mobile/src/features/voice-mode/voiceMode.ts +++ b/apps/mobile/src/features/voice-mode/voiceMode.ts @@ -9,7 +9,7 @@ import { import * as Effect from "effect/Effect"; import { AtomRegistry } from "effect/reactivity"; import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake"; -import { AppState, NativeModules } from "react-native"; +import { Alert, AppState, NativeModules } from "react-native"; import type * as WebRTC from "react-native-webrtc"; import { connectionAtomRuntime } from "../../connection/runtime"; @@ -71,7 +71,14 @@ voiceMode.subscribe(({ phase }) => { if (phase === "active" && !routedAudio) { routedAudio = true; // WebRTC resets iOS routing when its audio unit starts, so route after activation. - setVoiceAudioActive(true, () => voiceMode.stop()).catch(() => undefined); + setVoiceAudioActive(true, (error) => { + voiceMode.stop(); + if (error !== undefined) + Alert.alert( + "Voice audio failed", + error instanceof Error ? error.message : "Could not activate audio.", + ); + }).catch(() => undefined); } if (phase !== "idle" && !keepAwake) { // A hands-free conversation must not auto-lock: backgrounding ends it. diff --git a/apps/mobile/src/lib/audioSession.ts b/apps/mobile/src/lib/audioSession.ts index 7ebcd0bf726ec..d61548b25879c 100644 --- a/apps/mobile/src/lib/audioSession.ts +++ b/apps/mobile/src/lib/audioSession.ts @@ -85,7 +85,8 @@ export function releaseAudioSession(token: AudioSessionToken): Promise { return enqueue(async () => { if (owner !== token) return; try { - if (token.kind === "recording") await setAudioModeAsync({ allowsRecording: false }); + if (token.kind === "recording") + await setAudioModeAsync({ allowsRecording: false, interruptionMode: "mixWithOthers" }); } finally { if (owner === token) { if (token.kind === "recording") await setIsAudioActiveAsync(false); diff --git a/apps/web/src/components/CommandPalette.tsx b/apps/web/src/components/CommandPalette.tsx index 0e593dbe93eab..897c7a05460e3 100644 --- a/apps/web/src/components/CommandPalette.tsx +++ b/apps/web/src/components/CommandPalette.tsx @@ -2524,7 +2524,10 @@ function OpenCommandPaletteDialog(props: { command === "chat.newLocal" || command === "chat.newWithoutProject"); const matchingItem = - isJump || isNewThread || command === "voice.toggle" || command === "voice.toggleMute" + isJump || + isNewThread || + (!(event.nativeEvent.isComposing || event.keyCode === 229) && + (command === "voice.toggle" || command === "voice.toggleMute")) ? displayedGroups .flatMap((group) => group.items) .find((item) => item.shortcutCommand === command) diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index 6a2e3dce1488b..39dac533fcc1f 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -1141,7 +1141,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) useEffect(() => { if (backgroundThread) return; const handler = (event: globalThis.KeyboardEvent) => { - if (isCommandPaletteOpen()) return; + if (isCommandPaletteOpen() || event.isComposing || event.keyCode === 229) return; const command = resolveShortcutCommand(event, keybindings, { context: { terminalFocus: getTerminalFocusOwner() !== null, From 08b6a7aac24e243843bab2a5bdf3a91dd79d6744 Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Fri, 6 Nov 2026 07:58:11 +0000 Subject: [PATCH 16/19] chore(deps): refresh lockfile after merging main --- pnpm-lock.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2497c7c825f73..6b335127e8c7c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -13514,7 +13514,7 @@ snapshots: '@config-plugins/react-native-webrtc@15.0.2(expo@58.0.14)': dependencies: - expo: 58.0.14(277a83826bb26ba15256633fefeb0bc2) + expo: 58.0.14(ecd3689b2bda1172f8915f190f5c7383) '@connectrpc/connect-web@1.7.0(@bufbuild/protobuf@1.10.0)(@connectrpc/connect@1.7.0(@bufbuild/protobuf@1.10.0))': dependencies: From d999753767d26955dd7597f40cd63b7fabe5784f Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Fri, 6 Nov 2026 08:02:05 +0000 Subject: [PATCH 17/19] fix(voice): use a schema error for unavailable voice sessions --- packages/client-runtime/src/voice-mode/session.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/client-runtime/src/voice-mode/session.ts b/packages/client-runtime/src/voice-mode/session.ts index ae12202941794..9b41e0ed27f32 100644 --- a/packages/client-runtime/src/voice-mode/session.ts +++ b/packages/client-runtime/src/voice-mode/session.ts @@ -5,18 +5,19 @@ import { ORCHESTRATION_V2_WS_METHODS, } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; -import * as Data from "effect/Data"; import * as Effect from "effect/Effect"; import type * as Fiber from "effect/Fiber"; +import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; import { EnvironmentRegistry } from "../connection/registry.ts"; import { request, runStream } from "../rpc/client.ts"; import type { VoiceModePlatform } from "./controller.ts"; -class VoiceSessionUnavailable extends Data.TaggedError("VoiceSessionUnavailable")<{ - readonly message: string; -}> {} +class VoiceSessionUnavailable extends Schema.TaggedError()( + "VoiceSessionUnavailable", + { message: Schema.String }, +) {} function failureMessage(cause: Cause.Cause): string { const error = Cause.squash(cause); From 470f5a528fe277236ed5a8dd1244a20b294bc05b Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Tue, 10 Nov 2026 03:45:35 +0000 Subject: [PATCH 18/19] fix(web): pass the thread shell to the composer for voice availability --- apps/web/src/components/ChatView.tsx | 1 + apps/web/src/components/chat/ChatComposer.tsx | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 6daf189f7d6df..9afd67999afdb 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -3575,6 +3575,7 @@ export default function ChatView(props: ChatViewProps) { draftId={draftId} activeThreadId={activeThreadId} activeThread={activeThread} + activeThreadShell={activeThreadShell} promptHistoryEntries={timelineEntries} forceExpandedOnMobile={ forceExpandedMobileComposer && isDraftHeroState diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index b1bd5101c19bb..67907bc8884b1 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -240,7 +240,7 @@ import { useManagedPolicy } from "../../state/managedPolicy"; import { hasProviderSetup } from "./ProviderStatusBanner"; import { type AppModelOption, getAppModelOptionsForInstance } from "../../modelSelection"; import type { UnifiedSettings } from "@t3tools/contracts/settings"; -import { isVideoAttachment, type SessionPhase, type Thread } from "../../types"; +import { isVideoAttachment, type SessionPhase, type Thread, type ThreadShell } from "../../types"; import { type ComposerPromptHistoryPosition } from "./composerPromptHistory"; import type { PendingUserInputDraftAnswer } from "@t3tools/client-runtime/state/pending-user-input"; import type { PendingApproval, PendingUserInput, TimelineEntry } from "../../session-logic"; @@ -359,6 +359,7 @@ export interface ChatComposerProps { activeThreadId: ThreadId | null; activeThread: Thread | undefined; + activeThreadShell: ThreadShell | null; promptHistoryEntries: ReadonlyArray; forceExpandedOnMobile: boolean; projectSelectionRequired: boolean; From d62563571e550e0b54e708b11c4c3c4fd7e616b0 Mon Sep 17 00:00:00 2001 From: Romil Bijarnia Date: Sat, 14 Nov 2026 00:36:11 +1100 Subject: [PATCH 19/19] fix(codex): import completed voice handoffs without rerunning work --- .../Adapters/CodexAdapterV2.test.ts | 340 +++++++++++++++--- .../Adapters/CodexAdapterV2.ts | 274 +++++++++++--- .../ProviderContinuationService.test.ts | 68 ++-- .../ProviderContinuationService.ts | 2 +- 4 files changed, 553 insertions(+), 131 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts index 65e39b49dc6b8..6cd05e4d62c85 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts @@ -1560,7 +1560,7 @@ describe("CodexAdapterV2 post-settle continuation", () => { ); it.effect( - "resubmits a voice handoff Codex started only once its turn is interrupted or failed", + "adopts completed voice handoffs without rerunning them and resubmits interrupted work", () => Effect.scoped( Effect.gen(function* () { @@ -1611,6 +1611,133 @@ describe("CodexAdapterV2 post-settle continuation", () => { label: `turn/interrupt/${turnId}`, frame: { id, result: {} }, }, + ...(turnId === "voice-raced" + ? [ + { + type: "emit_inbound" as const, + label: "voice child spawned", + frame: { + method: "item/completed", + params: { + threadId: nativeThreadId, + turnId, + completedAtMs: 1782622441000, + item: { + type: "collabAgentToolCall", + id: "voice-child-spawn", + tool: "spawnAgent", + status: "completed", + senderThreadId: nativeThreadId, + receiverThreadIds: ["voice-raced-child-thread"], + prompt: "Report the child result.", + model: "gpt-5.4", + reasoningEffort: "low", + agentsStates: { + ["voice-raced-child-thread"]: { status: "running", message: null }, + }, + }, + }, + }, + }, + { + type: "emit_inbound" as const, + label: "voice child turn started", + frame: { + method: "turn/started", + params: { + threadId: "voice-raced-child-thread", + turn: { + ...makeCodexReplayTurn({ + id: "voice-raced-child-turn", + status: "inProgress", + }), + startedAt: 1782622441, + }, + }, + }, + }, + { + type: "emit_inbound" as const, + label: "voice child answer completed", + frame: { + method: "item/completed", + params: { + threadId: "voice-raced-child-thread", + turnId: "voice-raced-child-turn", + completedAtMs: 1782622441500, + item: { + type: "agentMessage", + id: "voice-child-answer", + text: "VOICE_CHILD_DONE", + phase: "final_answer", + memoryCitation: null, + }, + }, + }, + }, + { + type: "emit_inbound" as const, + label: "voice child turn completed", + frame: { + method: "turn/completed", + params: { + threadId: "voice-raced-child-thread", + turn: { + ...makeCodexReplayTurn({ + id: "voice-raced-child-turn", + status: "completed", + }), + startedAt: 1782622441, + completedAt: 1782622442, + }, + }, + }, + }, + { + type: "emit_inbound" as const, + label: "voice command completed", + frame: { + method: "item/completed", + params: { + threadId: nativeThreadId, + turnId, + completedAtMs: 1782622441000, + item: { + type: "commandExecution", + id: "voice-command", + command: "pwd", + cwd: "/workspace", + processId: "42", + source: "unifiedExecStartup", + status: "completed", + commandActions: [{ type: "unknown", command: "pwd" }], + aggregatedOutput: "/workspace\n", + exitCode: 0, + durationMs: 10, + }, + }, + }, + }, + { + type: "emit_inbound" as const, + label: "voice answer completed", + frame: { + method: "item/completed", + params: { + threadId: nativeThreadId, + turnId, + completedAtMs: 1782622442000, + item: { + type: "agentMessage", + id: "voice-answer", + text: "All green.", + phase: "final_answer", + }, + }, + }, + }, + ] + : []), { type: "emit_inbound" as const, label: `turn/completed/${turnId}`, @@ -1623,60 +1750,182 @@ describe("CodexAdapterV2 post-settle continuation", () => { }, }, ]; - const harness = yield* makeCodexReplayHarness( - makeCodexReplayTranscript({ - scenario: "voice-handoff-resubmit", - entries: [ - // Initialization and thread/start only; Codex, not T3, starts these turns. - ...codexReplayPreamble({ - nativeThreadId, - nativeTurnId: "unused", - prompt: "", - }).slice(0, -3), - { - type: "expect_outbound", - label: "thread/realtime/start", - frame: { - id: 3, - method: "thread/realtime/start", - params: { - threadId: nativeThreadId, - outputModality: "audio", - transport: { type: "webrtc", sdp: "v=offer\r\n" }, - version: "v3", - clientManagedHandoffs: true, - includeStartupContext: false, - }, + const handoffTranscript = makeCodexReplayTranscript({ + scenario: "voice-handoff-resubmit", + entries: [ + // Initialization and thread/start only; Codex, not T3, starts these turns. + ...codexReplayPreamble({ + nativeThreadId, + nativeTurnId: "unused", + prompt: "", + }).slice(0, -3), + { + type: "expect_outbound", + label: "thread/realtime/start", + frame: { + id: 3, + method: "thread/realtime/start", + params: { + threadId: nativeThreadId, + outputModality: "audio", + transport: { type: "webrtc", sdp: "v=offer\r\n" }, + version: "v3", + clientManagedHandoffs: true, + includeStartupContext: false, }, }, - { - type: "emit_inbound", - label: "thread/realtime/start", - frame: { id: 3, result: {} }, + }, + { + type: "emit_inbound", + label: "thread/realtime/start", + frame: { id: 3, result: {} }, + }, + // The interrupt lost the race: Codex finished, so it must not run twice. + ...voiceTurn("voice-raced", "already done", 4, "completed"), + { + type: "expect_outbound", + label: "speak completed native answer", + frame: { + id: 5, + method: "thread/realtime/appendSpeech", + params: { threadId: nativeThreadId, text: "All green." }, }, - // The interrupt lost the race: Codex finished, so it must not run twice. - ...voiceTurn("voice-raced", "already done", 4, "completed"), - ...voiceTurn("voice-stopped", "run the tests", 5, "interrupted"), - // A failed turn never did the work either, so its request is kept. - ...voiceTurn("voice-failed", "open a pull request", 6, "failed"), - ], - }), - ); + }, + { + type: "emit_inbound", + label: "speech acknowledged", + frame: { id: 5, result: {} }, + }, + ...voiceTurn("voice-stopped", "run the tests", 6, "interrupted"), + // A failed turn never did the work either, so its request is kept. + ...voiceTurn("voice-failed", "open a pull request", 7, "failed"), + ], + }); + const harness = yield* makeCodexReplayHarness(handoffTranscript); yield* harness.runtime.startVoiceSession!({ providerThread: harness.providerThread, offerSdp: "v=offer\r\n", }).pipe(Stream.runDrain, Effect.forkScoped); - yield* harness.until(() => harness.continuationRequests.length === 2); + yield* harness.until(() => + harness.continuationRequests.some((entry) => entry.detail === "open a pull request"), + ); assert.deepEqual( harness.continuationRequests.map((entry) => entry.detail), - ["run the tests", "open a pull request"], + ["already done", "run the tests", "open a pull request"], ); - const request = harness.continuationRequests[0]; + const completed = harness.continuationRequests[0]; + assert.equal(completed?.delivery, "adapter_buffered"); + const completedInput = makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("voice-adopted"), + text: completed!.detail!, + }); + yield* harness.runtime.startTurn({ + ...completedInput, + message: { ...completedInput.message, messageId: completed!.messageId! }, + }); + yield* harness.firstTerminal; + const answer = harness.events.find( + (event) => event.type === "message.updated" && event.message.text === "All green.", + ); + assert.equal(answer?.type, "message.updated"); + if (answer?.type === "message.updated") + assert.equal(answer.message.runId, completedInput.runId); + assert.isTrue( + harness.events.some( + (event) => + event.type === "turn_item.updated" && + event.turnItem.type === "command_execution" && + event.turnItem.status === "completed", + ), + ); + assert.equal(harness.terminalEvents()[0]?.status, "completed"); + assert.isTrue( + harness.events.some( + (event) => + event.type === "message.updated" && event.message.text === "VOICE_CHILD_DONE", + ), + ); + const child = harness.subagentUpdates().at(-1)?.subagent; + assert.equal(child?.status, "completed"); + assert.equal(child?.result, "VOICE_CHILD_DONE"); + assert.isFalse(yield* harness.hasPendingBackgroundWork); + const duplicate = yield* Effect.flip( + harness.runtime.startTurn({ + ...completedInput, + message: { ...completedInput.message, messageId: completed!.messageId! }, + }), + ); + assert.include(String(duplicate.cause), "already been claimed"); + const request = harness.continuationRequests[1]; assert.equal(request?.origin, "voice"); assert.equal(request?.delivery, "message_text"); assert.equal(request?.detail, "run the tests"); assert.equal(request?.providerThreadId, harness.providerThread.id); + const callClosed = yield* Deferred.make(); + const closedEntries = voiceTurn("voice-raced", "already done", 4, "completed"); + const terminalEntry = closedEntries.pop()!; + const closedHarness = yield* makeCodexReplayHarness( + makeCodexReplayTranscript({ + scenario: "voice-completes-after-call-closed", + entries: [ + ...handoffTranscript.entries.slice( + 0, + handoffTranscript.entries.findIndex( + (entry) => "label" in entry && entry.label === "turn/started/voice-raced", + ), + ), + ...closedEntries, + { + type: "emit_inbound", + label: "voice closed before native turn completes", + frame: { + method: "thread/realtime/closed", + params: { threadId: nativeThreadId }, + }, + }, + terminalEntry, + ], + }), + undefined, + undefined, + undefined, + (entry) => + entry.label === "turn/completed/voice-raced" + ? Deferred.await(callClosed) + : Effect.void, + ); + yield* closedHarness.runtime.startVoiceSession!({ + providerThread: closedHarness.providerThread, + offerSdp: "v=offer\r\n", + }).pipe( + Stream.runDrain, + Effect.ensuring(Deferred.succeed(callClosed, undefined)), + Effect.forkScoped, + ); + yield* closedHarness.until(() => closedHarness.continuationRequests.length === 1); + const closedRequest = closedHarness.continuationRequests[0]!; + const closedInput = makeCodexTestTurnInput({ + threadId: closedHarness.threadId, + providerThread: closedHarness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("voice-adopted-after-close"), + text: closedRequest.detail!, + }); + yield* closedHarness.runtime.startTurn({ + ...closedInput, + message: { ...closedInput.message, messageId: closedRequest.messageId! }, + }); + yield* closedHarness.firstTerminal; + assert.isTrue( + closedHarness.events.some( + (event) => event.type === "message.updated" && event.message.text === "All green.", + ), + ); + assert.equal(closedHarness.terminalEvents()[0]?.status, "completed"); const preamble = codexReplayPreamble({ nativeThreadId, nativeTurnId: "active", @@ -1749,6 +1998,17 @@ describe("CodexAdapterV2 post-settle continuation", () => { assert.equal(spoken.message.createdBy, "user"); } assert.equal(activeHarness.continuationRequests.length, 0); + const missingHistory = yield* Effect.flip( + activeHarness.runtime.startTurn({ + ...activeInput, + message: { + ...activeInput.message, + messageId: completed!.messageId!, + creationSource: "provider", + }, + }), + ); + assert.include(String(missingHistory.cause), "work was not run again"); }).pipe( Effect.provide( Layer.mergeAll( diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index 3d6243d6d151d..80b352da776b3 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -78,10 +78,12 @@ import type { ProviderTurnId, ProviderInstanceId, RuntimeMode, + RunId, ThreadId, } from "@t3tools/contracts"; import * as CodexClient from "effect-codex-app-server/client"; import * as CodexErrors from "effect-codex-app-server/errors"; +import type * as CodexRpc from "effect-codex-app-server/rpc"; import * as CodexSchema from "effect-codex-app-server/schema"; import * as Cause from "effect/Cause"; import * as Context from "effect/Context"; @@ -1986,16 +1988,82 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( ); /** Native turns Codex started for a voice handoff -> the request and its thread. */ const voiceSession = yield* Ref.make(undefined); - const voiceResubmits = yield* Ref.make( - new Map< - string, - { - readonly text: string; - readonly providerThread: OrchestrationV2ProviderThread; - readonly session: symbol | undefined; - } - >(), - ); + interface CapturedVoiceHandoff { + readonly text: string; + readonly providerThread: OrchestrationV2ProviderThread; + readonly session: symbol | undefined; + readonly nativeTurnId: string; + readonly delegationItemId: string; + readonly nativeThreadIds: Set; + readonly notifications: Array>; + adoptionRunId?: RunId; + } + const voiceResubmits = yield* Ref.make(new Map()); + const completedVoiceHandoffs = new Map< + MessageId, + { + readonly handoff: CapturedVoiceHandoff; + readonly turn: CodexSchema.V2TurnCompletedNotification["turn"]; + claimed: boolean; + } + >(); + // Native voice output arrives before the orchestrator allocates its run. + const handleServerNotification = ( + method: M, + handler: Parameters>[1], + ) => + client.handleServerNotification(method, (payload) => + Effect.gen(function* () { + const handoffs = yield* Ref.get(voiceResubmits); + if (handoffs.size === 0) return yield* handler(payload); + const threadId = "threadId" in payload ? payload.threadId : undefined; + const turnId = + method === "turn/started" || method === "turn/completed" + ? (payload as CodexRpc.ServerNotificationParamsByMethod["turn/completed"]).turn.id + : "turnId" in payload + ? payload.turnId + : undefined; + const ownedTurn = + typeof turnId === "string" + ? ((yield* Ref.get(activeTurns)).get(turnId) ?? + (yield* Ref.get(settledTurns)).get(turnId)) + : undefined; + const captured = yield* Ref.modify(voiceResubmits, (current) => { + const pending = [...current.values()].find( + (entry) => + typeof threadId === "string" && + entry.nativeThreadIds.has(threadId) && + (ownedTurn === undefined || + ownedTurn.projectionRunId === entry.adoptionRunId) && + (threadId !== entry.providerThread.nativeThreadRef?.nativeId || + turnId === entry.nativeTurnId), + ); + if ( + pending !== undefined && + !(method === "turn/completed" && turnId === pending.nativeTurnId) + ) { + if (method === "item/started" || method === "item/completed") { + const item = ( + payload as CodexRpc.ServerNotificationParamsByMethod["item/completed"] + ).item; + if (item.type === "userMessage" && item.id === pending.delegationItemId) + return [true, current] as const; + if (item.type === "subAgentActivity" && item.kind === "started") { + pending.nativeThreadIds.add(item.agentThreadId); + } + if (item.type === "collabAgentToolCall" && item.tool === "spawnAgent") { + for (const id of item.receiverThreadIds) pending.nativeThreadIds.add(id); + } + } + pending.notifications.push(handler(payload)); + return [true, current] as const; + } + return [false, current] as const; + }); + if (captured) return; + yield* handler(payload); + }), + ); const finalAnswerItemIdsByTurn = yield* Ref.make(new Map>()); const completedFinalAnswerTextsByTurn = yield* Ref.make(new Map>()); const imageTurns = new Map< @@ -4463,7 +4531,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( return { node, request, turnItem }; }); - yield* client.handleServerNotification("item/agentMessage/delta", (payload) => + yield* handleServerNotification("item/agentMessage/delta", (payload) => Effect.gen(function* () { const context = (yield* Ref.get(activeTurns)).get(payload.turnId); if (context !== undefined) { @@ -4477,7 +4545,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }), ); - yield* client.handleServerNotification("item/commandExecution/outputDelta", (payload) => + yield* handleServerNotification("item/commandExecution/outputDelta", (payload) => Effect.gen(function* () { // Background commands emit deltas after settlement; item/started already registered their turn. const context = @@ -4497,14 +4565,14 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }), ); - yield* client.handleServerNotification("item/reasoning/summaryTextDelta", (payload) => + yield* handleServerNotification("item/reasoning/summaryTextDelta", (payload) => appendReasoning(payload, "summary", payload.summaryIndex), ); - yield* client.handleServerNotification("item/reasoning/textDelta", (payload) => + yield* handleServerNotification("item/reasoning/textDelta", (payload) => appendReasoning(payload, "content", payload.contentIndex), ); - yield* client.handleServerNotification("item/plan/delta", (payload) => + yield* handleServerNotification("item/plan/delta", (payload) => Effect.gen(function* () { const context = yield* awaitActiveTurn(payload.turnId); if (context === undefined) { @@ -4519,7 +4587,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }).pipe(Effect.orDie), ); - yield* client.handleServerNotification("turn/plan/updated", (payload) => + yield* handleServerNotification("turn/plan/updated", (payload) => Effect.gen(function* () { const context = yield* awaitActiveTurn(payload.turnId); if (context === undefined) { @@ -4557,7 +4625,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }).pipe(Effect.orDie), ); - yield* client.handleServerNotification("account/rateLimits/updated", (payload) => + yield* handleServerNotification("account/rateLimits/updated", (payload) => Effect.gen(function* () { const previous = yield* Ref.getAndUpdate(rateLimitSnapshot, (previous) => mergeCodexRateLimits(previous, payload.rateLimits), @@ -4588,7 +4656,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }).pipe(Effect.orDie), ); - yield* client.handleServerNotification("thread/tokenUsage/updated", (payload) => + yield* handleServerNotification("thread/tokenUsage/updated", (payload) => Effect.gen(function* () { accumulateCodexTurnTokenUsage( usageStateForThread(payload.threadId), @@ -4627,7 +4695,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }).pipe(Effect.orDie), ); - yield* client.handleServerNotification("thread/settings/updated", (payload) => + yield* handleServerNotification("thread/settings/updated", (payload) => Effect.sync(() => reroutedThreads.delete(payload.threadId)).pipe( Effect.andThen( updateSubagentSelection( @@ -4639,14 +4707,14 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( ), ), ); - yield* client.handleServerNotification("model/rerouted", (payload) => + yield* handleServerNotification("model/rerouted", (payload) => Effect.sync(() => { reroutedTurnModels.set(payload.turnId, payload.toModel); reroutedThreads.add(payload.threadId); }).pipe(Effect.andThen(updateSubagentSelection(payload.threadId, payload.toModel))), ); - yield* client.handleServerNotification("hook/completed", (payload) => + yield* handleServerNotification("hook/completed", (payload) => Effect.gen(function* () { if (payload.turnId == null || payload.run.status !== "stopped") return; const context = (yield* Ref.get(activeTurns)).get(payload.turnId); @@ -4662,7 +4730,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }), ); - yield* client.handleServerNotification("turn/started", (payload) => + yield* handleServerNotification("turn/started", (payload) => Effect.gen(function* () { const context = (yield* Ref.get(activeTurns)).get(payload.turn.id); if (context !== undefined) { @@ -4748,14 +4816,14 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }).pipe(Effect.orDie, turnTerminalizationPermit.withPermits(1)), ); - yield* client.handleServerNotification("thread/goal/updated", (payload) => + yield* handleServerNotification("thread/goal/updated", (payload) => Effect.gen(function* () { goalsByNativeThread.set(payload.threadId, providerGoalFromCodex(payload.goal)); yield* emitGoalUpdate(payload.threadId); if (payload.goal.status !== "active") yield* releaseGoalHold(payload.threadId); }).pipe(turnTerminalizationPermit.withPermits(1)), ); - yield* client.handleServerNotification("thread/goal/cleared", (payload) => + yield* handleServerNotification("thread/goal/cleared", (payload) => Effect.gen(function* () { goalsByNativeThread.set(payload.threadId, null); yield* emitGoalUpdate(payload.threadId); @@ -4763,7 +4831,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }).pipe(turnTerminalizationPermit.withPermits(1)), ); - yield* client.handleServerNotification("error", (payload) => + yield* handleServerNotification("error", (payload) => Effect.gen(function* () { const context = yield* awaitActiveTurn(payload.turnId); if (context === undefined) { @@ -4887,7 +4955,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( // Unloading an idle thread kills its terminals without completing // their command items, while the shared provider session stays alive. - yield* client.handleServerNotification("thread/closed", (payload) => + yield* handleServerNotification("thread/closed", (payload) => Effect.gen(function* () { const completedAt = yield* DateTime.now; for (const context of (yield* Ref.get(settledTurns)).values()) { @@ -4914,7 +4982,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }).pipe(Effect.orDie, turnTerminalizationPermit.withPermits(1)), ); - yield* client.handleServerNotification("item/started", (payload) => + yield* handleServerNotification("item/started", (payload) => Effect.gen(function* () { if (payload.item.type === "userMessage") { const handoff = readRealtimeDelegationFromItem(payload.item); @@ -4937,6 +5005,10 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( text: handoff, providerThread: voiceThread, session: voiceOwner, + nativeTurnId: payload.turnId, + delegationItemId: payload.item.id, + nativeThreadIds: new Set([payload.threadId]), + notifications: [], }), ); yield* client @@ -5139,7 +5211,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }).pipe(Effect.orDie, turnTerminalizationPermit.withPermits(1)), ); - yield* client.handleServerNotification("item/completed", (payload) => + yield* handleServerNotification("item/completed", (payload) => Effect.gen(function* () { if (payload.item.type === "contextCompaction") // Fork requests off the notification reader so it can read their responses. @@ -6735,26 +6807,43 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( ); }); - yield* client.handleServerNotification("turn/completed", (payload) => + yield* handleServerNotification("turn/completed", (payload) => Effect.gen(function* () { const voiceResubmit = (yield* Ref.get(voiceResubmits)).get(payload.turn.id); if (voiceResubmit !== undefined) { + const messageId = MessageId.make( + `${payload.turn.status === "completed" ? "voice-completed" : "voice"}:${payload.threadId}:${payload.turn.id}`, + ); + if (payload.turn.status === "completed" && continuationRequests !== undefined) { + if (completedVoiceHandoffs.has(messageId)) return; + completedVoiceHandoffs.set(messageId, { + handoff: voiceResubmit, + turn: payload.turn, + claimed: false, + }); + yield* continuationRequests.offer({ + messageId, + threadId: voiceResubmit.providerThread.appThreadId ?? input.threadId, + providerThreadId: voiceResubmit.providerThread.id, + driver: CODEX_PROVIDER, + detail: voiceResubmit.text, + delivery: "adapter_buffered", + origin: "voice", + }); + return; + } yield* Ref.update(voiceResubmits, (current) => { const updated = new Map(current); updated.delete(payload.turn.id); return updated; }); - // Resubmit a turn Codex stopped or failed, so the request isn't - // lost. If the interrupt lost the race and the turn completed, - // Codex already did the work (and the voice speaks its answer), - // so running it again would repeat it. + // Only unfinished work needs a new native turn. if ( (payload.turn.status === "interrupted" || payload.turn.status === "failed") && continuationRequests !== undefined && voiceResubmit.session !== undefined && voiceResubmit.session === (yield* Ref.get(voiceSession)) ) { - const messageId = MessageId.make(`voice:${payload.threadId}:${payload.turn.id}`); yield* voice.expectSpokenTurn(messageId); yield* continuationRequests.offer({ messageId, @@ -7440,32 +7529,104 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }), ), ), - startTurn: (turnInput) => { - const goalCommand = - turnInput.restartContinuationOfRunId === undefined - ? parseGoalCommand(turnInput.message) - : null; - if (goalCommand !== null) return runGoalCommand(turnInput, goalCommand); - return Effect.gen(function* () { - yield* startNativeTurn( - turnInput, - turnInput.restartContinuationOfRunId === undefined - ? yield* toCodexInput(turnInput) - : [], - ); - }).pipe( - Effect.mapError( - (cause) => + startTurn: Effect.fn("CodexAdapterV2.startTurn")( + (turnInput: ProviderAdapter.ProviderAdapterV2TurnInput) => { + const completedVoice = completedVoiceHandoffs.get(turnInput.message.messageId); + if (completedVoice !== undefined) { + return Effect.gen(function* () { + if (completedVoice.handoff.providerThread.id !== turnInput.providerThread.id) { + return yield* new ProviderAdapter.ProviderAdapterProtocolError({ + driver: CODEX_PROVIDER, + detail: "The completed voice turn belongs to another provider thread.", + }); + } + if (completedVoice.claimed) { + return yield* new ProviderAdapter.ProviderAdapterProtocolError({ + driver: CODEX_PROVIDER, + detail: "The completed voice turn has already been claimed for import.", + }); + } + completedVoice.claimed = true; + completedVoice.handoff.adoptionRunId = turnInput.runId; + const context = yield* registerRootTurn({ + turnInput, + nativeTurnId: completedVoice.turn.id, + startedAt: codexTimestamp(completedVoice.turn.startedAt), + }); + while (true) { + const notification = yield* Ref.modify(voiceResubmits, (current) => { + const nextNotification = completedVoice.handoff.notifications.shift(); + if (nextNotification !== undefined) + return [nextNotification, current] as const; + const next = new Map(current); + next.delete(completedVoice.turn.id); + return [undefined, next] as const; + }); + if (notification === undefined) break; + yield* notification; + } + yield* finalizeCodexTurn({ + context, + nativeTurnId: completedVoice.turn.id, + status: "completed", + completedAt: codexTimestamp(completedVoice.turn.completedAt), + }); + }).pipe( + Effect.uninterruptible, + Effect.mapError( + (cause) => + new ProviderAdapter.ProviderAdapterTurnStartError({ + driver: CODEX_PROVIDER, + threadId: turnInput.threadId, + providerThreadId: turnInput.providerThread.id, + runId: turnInput.runId, + cause, + }), + ), + ); + } + // Persisted imports must never execute their already-completed request after a restart. + if (turnInput.message.messageId.startsWith("voice-completed:")) { + return Effect.fail( new ProviderAdapter.ProviderAdapterTurnStartError({ driver: CODEX_PROVIDER, threadId: turnInput.threadId, providerThreadId: turnInput.providerThread.id, runId: turnInput.runId, - cause, + cause: new ProviderAdapter.ProviderAdapterProtocolError({ + driver: CODEX_PROVIDER, + detail: + "Completed voice history is unavailable after reconnecting; the work was not run again.", + }), }), - ), - ); - }, + ); + } + const goalCommand = + turnInput.restartContinuationOfRunId === undefined + ? parseGoalCommand(turnInput.message) + : null; + if (goalCommand !== null) return runGoalCommand(turnInput, goalCommand); + return Effect.gen(function* () { + yield* startNativeTurn( + turnInput, + turnInput.restartContinuationOfRunId === undefined + ? yield* toCodexInput(turnInput) + : [], + ); + }).pipe( + Effect.mapError( + (cause) => + new ProviderAdapter.ProviderAdapterTurnStartError({ + driver: CODEX_PROVIDER, + threadId: turnInput.threadId, + providerThreadId: turnInput.providerThread.id, + runId: turnInput.runId, + cause, + }), + ), + ); + }, + ), steerTurn: Effect.fn("CodexAdapterV2.steerTurn")( function* (turnInput) { const threadId = yield* getNativeThreadId(turnInput.providerThread); @@ -8039,11 +8200,6 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( // A replaced conversation must not clear its successor's thread. Stream.ensuring( Effect.gen(function* () { - yield* Ref.update( - voiceResubmits, - (current) => - new Map([...current].filter(([, entry]) => entry.session !== owner)), - ); if ((yield* Ref.get(voiceSession)) === owner) { yield* Ref.set(voiceSession, undefined); yield* Ref.set(voiceProviderThread, undefined); diff --git a/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts b/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts index da182e7bdd2a1..efef407cd7573 100644 --- a/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts @@ -195,37 +195,43 @@ describe("ProviderContinuationService", () => { }); }); - it.effect("resubmits a spoken voice request as the user's own message", () => { - return Effect.gen(function* () { - const dispatched = yield* Queue.unbounded(); - yield* Effect.gen(function* () { - const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests; - yield* requests.offer({ - threadId, - providerThreadId, - driver, - detail: "run the tests", - delivery: "message_text", - origin: "voice", - }); - const command = (yield* Queue.take(dispatched)) as { - readonly createdBy: string; - readonly creationSource: string; - readonly text: string; - readonly notification?: OrchestrationV2Notification; - }; - assert.equal(command.createdBy, "user"); - assert.equal(command.creationSource, "server"); - assert.equal(command.text, "run the tests"); - assert.isUndefined(command.notification); - }).pipe( - Effect.provide( - layerTest({ dispatched, getThreadRecords: () => Effect.succeed(projection) }), - ), - Effect.scoped, - ); - }); - }); + it.effect.each(["message_text", "adapter_buffered"] as const)( + "dispatches a %s voice request as the user's own message", + (delivery) => { + return Effect.gen(function* () { + const dispatched = yield* Queue.unbounded(); + yield* Effect.gen(function* () { + const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests; + yield* requests.offer({ + threadId, + providerThreadId, + driver, + detail: "run the tests", + delivery, + origin: "voice", + }); + const command = (yield* Queue.take(dispatched)) as { + readonly createdBy: string; + readonly creationSource: string; + readonly text: string; + readonly notification?: OrchestrationV2Notification; + }; + assert.equal(command.createdBy, "user"); + assert.equal( + command.creationSource, + delivery === "adapter_buffered" ? "provider" : "server", + ); + assert.equal(command.text, "run the tests"); + assert.isUndefined(command.notification); + }).pipe( + Effect.provide( + layerTest({ dispatched, getThreadRecords: () => Effect.succeed(projection) }), + ), + Effect.scoped, + ); + }); + }, + ); it.effect("dispatches a current delegated completion as one server-owned queued message", () => { return Effect.gen(function* () { diff --git a/apps/server/src/orchestration-v2/ProviderContinuationService.ts b/apps/server/src/orchestration-v2/ProviderContinuationService.ts index 9f73083370117..fd733200becf5 100644 --- a/apps/server/src/orchestration-v2/ProviderContinuationService.ts +++ b/apps/server/src/orchestration-v2/ProviderContinuationService.ts @@ -197,7 +197,7 @@ export const layer = Layer.effectDiscard( attachments: [], dispatchMode: { type: "queue_after_active" }, createdBy: "user", - creationSource: "server", + creationSource: request.delivery === "adapter_buffered" ? "provider" : "server", } : { type: "message.dispatch",