From ee82f1c3a4a9371d324b5c70c9e4e155e1985c76 Mon Sep 17 00:00:00 2001 From: Rodrigo Date: Tue, 6 Oct 2026 01:46:05 -0700 Subject: [PATCH 1/2] feat(mobile): choose the language for iPhone voice input --- apps/mobile/src/Stack.tsx | 6 ++ .../features/settings/SettingsRouteScreen.tsx | 3 + .../SettingsVoiceInputRouteScreen.tsx | 56 +++++++++++++++++++ .../components/settings-sheet-targets.ts | 1 + .../voice-input/VoiceInputProvider.tsx | 12 +++- .../src/native/voiceTranscription.ios.test.ts | 10 ++++ .../src/native/voiceTranscription.ios.ts | 4 +- apps/mobile/src/native/voiceTranscription.ts | 2 +- .../src/persistence/mobile-preferences.ts | 6 ++ docs/user/composer.md | 1 + .../src/voice-input/controller.test.ts | 11 ++++ .../src/voice-input/controller.ts | 4 +- 12 files changed, 110 insertions(+), 6 deletions(-) create mode 100644 apps/mobile/src/features/settings/SettingsVoiceInputRouteScreen.tsx diff --git a/apps/mobile/src/Stack.tsx b/apps/mobile/src/Stack.tsx index ec4b2565e1749..aaaea63825093 100644 --- a/apps/mobile/src/Stack.tsx +++ b/apps/mobile/src/Stack.tsx @@ -101,6 +101,7 @@ import { } from "./features/settings/ScheduledTaskPickerScreens"; import { ScheduledTaskEditorProvider } from "./features/settings/scheduled-task-editor"; import { SettingsKeyboardRouteScreen } from "./features/settings/SettingsKeyboardRouteScreen"; +import { SettingsVoiceInputRouteScreen } from "./features/settings/SettingsVoiceInputRouteScreen"; import { SettingsLegalRouteScreen } from "./features/settings/SettingsLegalRouteScreen"; import { SettingsOpenSourceLicenseRouteScreen, @@ -310,6 +311,11 @@ const SettingsContentStack = createV5SheetStackNavigator({ title: "Keyboard", }, }), + SettingsVoiceInput: createNativeStackScreen({ + screen: SettingsVoiceInputRouteScreen, + linking: "voice-input", + options: { title: "Voice input" }, + }), SettingsFollowUp: createNativeStackScreen({ screen: SettingsFollowUpRouteScreen, linking: "follow-ups", diff --git a/apps/mobile/src/features/settings/SettingsRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsRouteScreen.tsx index 1f474a9d56b85..4366d10f8abd8 100644 --- a/apps/mobile/src/features/settings/SettingsRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsRouteScreen.tsx @@ -156,6 +156,9 @@ function SettingsIndexSections() { {Platform.OS === "ios" ? ( ) : null} + {Platform.OS === "ios" ? ( + + ) : null} diff --git a/apps/mobile/src/features/settings/SettingsVoiceInputRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsVoiceInputRouteScreen.tsx new file mode 100644 index 0000000000000..fcab9d91ecc8d --- /dev/null +++ b/apps/mobile/src/features/settings/SettingsVoiceInputRouteScreen.tsx @@ -0,0 +1,56 @@ +import { useAtomSet, useAtomValue } from "@effect/atom-react"; +import { AsyncResult } from "effect/reactivity"; +import { ScrollView } from "react-native"; +import { useSafeAreaInsets } from "react-native-safe-area-context"; + +import { mobilePreferencesAtom, updateMobilePreferencesAtom } from "../../state/preferences"; +import { SettingsChoiceRow } from "./components/SettingsChoiceRow"; +import { SettingsSection } from "./components/SettingsSection"; + +// Locales Apple's on-device transcriber supports; its native module does not expose the list. +const LANGUAGES = [ + { locale: undefined, label: "Automatic", description: "Uses the app's language." }, + { locale: "en-US", label: "English (US)", description: "English (United States)" }, + { locale: "en-GB", label: "English (UK)", description: "English (United Kingdom)" }, + { locale: "es-ES", label: "Español (España)", description: "Spanish (Spain)" }, + { locale: "es-MX", label: "Español (México)", description: "Spanish (Mexico)" }, + { locale: "fr-FR", label: "Français", description: "French" }, + { locale: "de-DE", label: "Deutsch", description: "German" }, + { locale: "it-IT", label: "Italiano", description: "Italian" }, + { locale: "pt-BR", label: "Português (Brasil)", description: "Portuguese (Brazil)" }, + { locale: "ja-JP", label: "日本語", description: "Japanese" }, + { locale: "ko-KR", label: "한국어", description: "Korean" }, + { locale: "zh-CN", label: "中文(普通话)", description: "Chinese (Mandarin)" }, +] as const; + +export function SettingsVoiceInputRouteScreen() { + const insets = useSafeAreaInsets(); + const savePreferences = useAtomSet(updateMobilePreferencesAtom); + const selectedLocale = useAtomValue(mobilePreferencesAtom, (result) => + AsyncResult.isSuccess(result) ? result.value.voiceInputLanguage : null, + ); + + return ( + + + {LANGUAGES.map((option, index) => ( + 0} + disabled={selectedLocale === null} + onPress={() => savePreferences({ voiceInputLanguage: option.locale })} + /> + ))} + + + ); +} diff --git a/apps/mobile/src/features/settings/components/settings-sheet-targets.ts b/apps/mobile/src/features/settings/components/settings-sheet-targets.ts index ad0197d634d85..4b7c5e9b83168 100644 --- a/apps/mobile/src/features/settings/components/settings-sheet-targets.ts +++ b/apps/mobile/src/features/settings/components/settings-sheet-targets.ts @@ -13,6 +13,7 @@ export type SettingsSheetTarget = | "SettingsEnvironmentMaintenance" | "SettingsProviderAccounts" | "SettingsKeyboard" + | "SettingsVoiceInput" | "SettingsFollowUp" | "SettingsScheduledTasks" | "SettingsProjectGrouping" diff --git a/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx b/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx index ebd628e4bad4a..23e9d99a5d475 100644 --- a/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx +++ b/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx @@ -1,3 +1,4 @@ +import { useAtomValue } from "@effect/atom-react"; import { AudioModule, RecordingPresets, @@ -7,6 +8,7 @@ import { type RecorderState, type RecordingStatus, } from "expo-audio"; +import { AsyncResult } from "effect/reactivity"; import { File } from "expo-file-system"; import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake"; import { @@ -23,6 +25,7 @@ import { AppState, Platform } from "react-native"; import { useSharedValue } from "react-native-reanimated"; import { getLocalVoiceTranscriber } from "../../native/voiceTranscription"; +import { mobilePreferencesAtom } from "../../state/preferences"; import { getNativeShowcaseScene } from "../showcase/nativeShowcaseScene"; import { VOICE_RECORDING_LIMIT_SECONDS, @@ -112,6 +115,13 @@ function useVoiceInputRuntime() { const audioLevelsRef = useRef(Array(VOICE_WAVEFORM_SAMPLE_COUNT).fill(0)); const audioLevels = useSharedValue(audioLevelsRef.current); const sessionRef = useRef(null); + const language = useAtomValue(mobilePreferencesAtom, (result) => + AsyncResult.isSuccess(result) ? result.value.voiceInputLanguage : undefined, + ); + const languageRef = useRef(language); + useEffect(() => { + languageRef.current = language; + }, [language]); const recorderRef = useRef | null>(null); if (!sessionRef.current || !recorderRef.current) { @@ -130,7 +140,7 @@ function useVoiceInputRuntime() { recorderRef.current = recorder; sessionRef.current = new VoiceInputSession({ recorder, - getTranscriber: getLocalVoiceTranscriber, + getTranscriber: () => getLocalVoiceTranscriber(languageRef.current), requestPermission: async () => { const permission = await requestRecordingPermissionsAsync(); return { granted: permission.granted, canAskAgain: permission.canAskAgain }; diff --git a/apps/mobile/src/native/voiceTranscription.ios.test.ts b/apps/mobile/src/native/voiceTranscription.ios.test.ts index b08e32cdb6f65..f60b43bce93f4 100644 --- a/apps/mobile/src/native/voiceTranscription.ios.test.ts +++ b/apps/mobile/src/native/voiceTranscription.ios.test.ts @@ -74,6 +74,16 @@ describe("getLocalVoiceTranscriber", () => { expect(mocks.transcribe).toHaveBeenCalledWith(audio, "sv-SE"); }); + it("uses the chosen language instead of the app's locale", async () => { + mocks.prepare.mockResolvedValue("es-ES"); + const prepared = await getLocalVoiceTranscriber("es-ES")!.prepare({ + signal: new AbortController().signal, + }); + + expect(mocks.prepare).toHaveBeenCalledWith("es-ES"); + expect(prepared.locale).toBe("es-ES"); + }); + it("does not start native transcription after cancellation during a file read", async () => { const enteredRead = deferred(); const readResult = deferred(); diff --git a/apps/mobile/src/native/voiceTranscription.ios.ts b/apps/mobile/src/native/voiceTranscription.ios.ts index 216b9e958dd66..e8d22674a1481 100644 --- a/apps/mobile/src/native/voiceTranscription.ios.ts +++ b/apps/mobile/src/native/voiceTranscription.ios.ts @@ -33,8 +33,8 @@ function getNativeErrorCode(error: unknown): string | undefined { return typeof error.code === "string" ? error.code : undefined; } -export function getLocalVoiceTranscriber(): VoiceTranscriber | null { - const locale = getDeviceLocale(); +export function getLocalVoiceTranscriber(language?: string): VoiceTranscriber | null { + const locale = language ?? getDeviceLocale(); if (!AppleTranscription.isAvailable(locale)) return null; return { prepare: (options) => prepareVoiceTranscription(locale, options) }; } diff --git a/apps/mobile/src/native/voiceTranscription.ts b/apps/mobile/src/native/voiceTranscription.ts index e003064ae3f84..47dd2df817d38 100644 --- a/apps/mobile/src/native/voiceTranscription.ts +++ b/apps/mobile/src/native/voiceTranscription.ts @@ -1,5 +1,5 @@ import type { VoiceTranscriber } from "@t3tools/client-runtime/voice-input"; -export function getLocalVoiceTranscriber(): VoiceTranscriber | null { +export function getLocalVoiceTranscriber(_language?: string): VoiceTranscriber | null { return null; } diff --git a/apps/mobile/src/persistence/mobile-preferences.ts b/apps/mobile/src/persistence/mobile-preferences.ts index ee96bce2d5cec..fee633d2285ab 100644 --- a/apps/mobile/src/persistence/mobile-preferences.ts +++ b/apps/mobile/src/persistence/mobile-preferences.ts @@ -36,6 +36,8 @@ export interface Preferences { * message sent during a running turn queues behind it or steers it. */ readonly followUpBehavior?: FollowUpBehavior; + /** Locale for on-device voice input. Unset follows the app's locale. iOS only. */ + readonly voiceInputLanguage?: string; /** @deprecated Kept temporarily so older OTA bundles retain the selected mode. */ readonly projectGroupingEnabled?: boolean; readonly projectGroupingMode?: SidebarProjectGroupingMode; @@ -107,6 +109,7 @@ function sanitizePreferences(parsed: Preferences): Preferences { collapsedProjectGroups?: readonly string[]; composerEnterBehavior?: ComposerEnterBehavior; followUpBehavior?: FollowUpBehavior; + voiceInputLanguage?: string; projectGroupingEnabled?: boolean; projectGroupingMode?: SidebarProjectGroupingMode; planModeEnabled?: boolean; @@ -172,6 +175,9 @@ function sanitizePreferences(parsed: Preferences): Preferences { if (parsed.followUpBehavior === "queue" || parsed.followUpBehavior === "steer") { preferences.followUpBehavior = parsed.followUpBehavior; } + if (typeof parsed.voiceInputLanguage === "string") { + preferences.voiceInputLanguage = parsed.voiceInputLanguage; + } if (typeof parsed.projectGroupingEnabled === "boolean") { preferences.projectGroupingEnabled = parsed.projectGroupingEnabled; } diff --git a/docs/user/composer.md b/docs/user/composer.md index ba8d26fd81c29..17c6ae4e6ba0f 100644 --- a/docs/user/composer.md +++ b/docs/user/composer.md @@ -157,6 +157,7 @@ and use **Attach again** or remove the missing file before sending. On supported iPhones with iOS 26 or later, use the composer's microphone to record, then confirm to transcribe. Text is inserted where your selection was when recording started, ready for you to review and edit before sending. +To dictate in a language other than the app's, choose it in **Settings → Voice input**. The first use may download Apple's speech model and needs a network connection. Later transcription works offline for that language. Recordings can be up to five diff --git a/packages/client-runtime/src/voice-input/controller.test.ts b/packages/client-runtime/src/voice-input/controller.test.ts index d73c8e7505baf..7a9cffec10ed1 100644 --- a/packages/client-runtime/src/voice-input/controller.test.ts +++ b/packages/client-runtime/src/voice-input/controller.test.ts @@ -134,6 +134,17 @@ describe("resolveTranscriptCommit", () => { }); }); + it("adds word spacing for other space-delimited languages", () => { + const atEnd = draft({ + text: "Revisá la caché.", + selection: { start: "Revisá la caché.".length, end: "Revisá la caché.".length }, + }); + expect(resolveTranscriptCommit(atEnd, atEnd, "Ésta también.", "es-ES")).toMatchObject({ + kind: "commit", + text: "Revisá la caché. Ésta también.", + }); + }); + it("does not add English boundary spaces to CJK or selected inline text", () => { const cjk = draft({ text: "修正キャッシュ", selection: { start: 8, end: 8 } }); expect(resolveTranscriptCommit(cjk, cjk, "テストも", "ja-JP")).toMatchObject({ diff --git a/packages/client-runtime/src/voice-input/controller.ts b/packages/client-runtime/src/voice-input/controller.ts index 80a8a528f335f..47cca1d0a36aa 100644 --- a/packages/client-runtime/src/voice-input/controller.ts +++ b/packages/client-runtime/src/voice-input/controller.ts @@ -92,9 +92,9 @@ export function resolveTranscriptCommit( const isEmptySelection = captured.selection.start === captured.selection.end; const normalizedLocale = locale.replaceAll("_", "-").toLowerCase(); - const usesEnglishSpacing = normalizedLocale === "en" || normalizedLocale.startsWith("en-"); + const usesWordSpacing = !/^(ja|zh|yue|th|lo|km|my)(-|$)/.test(normalizedLocale); let insertion = replacement; - if (isEmptySelection && usesEnglishSpacing) { + if (isEmptySelection && usesWordSpacing) { const left = captured.text[captured.selection.start - 1]; const right = captured.text[captured.selection.start]; const leftNeedsBoundary = From 891eb9eadc6a93fb6ca1deff1543c720ef344919 Mon Sep 17 00:00:00 2001 From: Rodrigo Date: Fri, 9 Oct 2026 19:32:19 -0700 Subject: [PATCH 2/2] fix(mobile): accented letters count as word boundaries in voice transcripts --- .../src/voice-input/controller.test.ts | 14 ++++++++++++++ .../client-runtime/src/voice-input/controller.ts | 4 ++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/packages/client-runtime/src/voice-input/controller.test.ts b/packages/client-runtime/src/voice-input/controller.test.ts index 7a9cffec10ed1..ae637b0d9748e 100644 --- a/packages/client-runtime/src/voice-input/controller.test.ts +++ b/packages/client-runtime/src/voice-input/controller.test.ts @@ -145,6 +145,20 @@ describe("resolveTranscriptCommit", () => { }); }); + it("treats accented letters as word boundaries", () => { + const afterAccent = draft({ text: "café", selection: { start: 4, end: 4 } }); + expect(resolveTranscriptCommit(afterAccent, afterAccent, "también", "es-ES")).toMatchObject({ + kind: "commit", + text: "café también", + }); + + const beforeAccent = draft({ text: "él", selection: { start: 0, end: 0 } }); + expect(resolveTranscriptCommit(beforeAccent, beforeAccent, "Dijo", "es-ES")).toMatchObject({ + kind: "commit", + text: "Dijo él", + }); + }); + it("does not add English boundary spaces to CJK or selected inline text", () => { const cjk = draft({ text: "修正キャッシュ", selection: { start: 8, end: 8 } }); expect(resolveTranscriptCommit(cjk, cjk, "テストも", "ja-JP")).toMatchObject({ diff --git a/packages/client-runtime/src/voice-input/controller.ts b/packages/client-runtime/src/voice-input/controller.ts index 47cca1d0a36aa..c2d07898968e0 100644 --- a/packages/client-runtime/src/voice-input/controller.ts +++ b/packages/client-runtime/src/voice-input/controller.ts @@ -99,11 +99,11 @@ export function resolveTranscriptCommit( const right = captured.text[captured.selection.start]; const leftNeedsBoundary = left !== undefined && - /[A-Za-z0-9.!?,:;)\]}'"]/.test(left) && + /[\p{L}\p{N}.!?,:;)\]}'"]/u.test(left) && (right === undefined || /\s/.test(right)); const rightNeedsBoundary = right !== undefined && - /[A-Za-z0-9([{'"]/.test(right) && + /[\p{L}\p{N}([{'"]/u.test(right) && (left === undefined || /\s/.test(left)); insertion = `${leftNeedsBoundary ? " " : ""}${replacement}${rightNeedsBoundary ? " " : ""}`; }