diff --git a/apps/mobile/src/Stack.tsx b/apps/mobile/src/Stack.tsx index ec4b2565e1749..aaaea63825093 100644 --- a/apps/mobile/src/Stack.tsx +++ b/apps/mobile/src/Stack.tsx @@ -101,6 +101,7 @@ import { } from "./features/settings/ScheduledTaskPickerScreens"; import { ScheduledTaskEditorProvider } from "./features/settings/scheduled-task-editor"; import { SettingsKeyboardRouteScreen } from "./features/settings/SettingsKeyboardRouteScreen"; +import { SettingsVoiceInputRouteScreen } from "./features/settings/SettingsVoiceInputRouteScreen"; import { SettingsLegalRouteScreen } from "./features/settings/SettingsLegalRouteScreen"; import { SettingsOpenSourceLicenseRouteScreen, @@ -310,6 +311,11 @@ const SettingsContentStack = createV5SheetStackNavigator({ title: "Keyboard", }, }), + SettingsVoiceInput: createNativeStackScreen({ + screen: SettingsVoiceInputRouteScreen, + linking: "voice-input", + options: { title: "Voice input" }, + }), SettingsFollowUp: createNativeStackScreen({ screen: SettingsFollowUpRouteScreen, linking: "follow-ups", diff --git a/apps/mobile/src/features/settings/SettingsRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsRouteScreen.tsx index 1f474a9d56b85..4366d10f8abd8 100644 --- a/apps/mobile/src/features/settings/SettingsRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsRouteScreen.tsx @@ -156,6 +156,9 @@ function SettingsIndexSections() { {Platform.OS === "ios" ? ( ) : null} + {Platform.OS === "ios" ? ( + + ) : null} diff --git a/apps/mobile/src/features/settings/SettingsVoiceInputRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsVoiceInputRouteScreen.tsx new file mode 100644 index 0000000000000..fcab9d91ecc8d --- /dev/null +++ b/apps/mobile/src/features/settings/SettingsVoiceInputRouteScreen.tsx @@ -0,0 +1,56 @@ +import { useAtomSet, useAtomValue } from "@effect/atom-react"; +import { AsyncResult } from "effect/reactivity"; +import { ScrollView } from "react-native"; +import { useSafeAreaInsets } from "react-native-safe-area-context"; + +import { mobilePreferencesAtom, updateMobilePreferencesAtom } from "../../state/preferences"; +import { SettingsChoiceRow } from "./components/SettingsChoiceRow"; +import { SettingsSection } from "./components/SettingsSection"; + +// Locales Apple's on-device transcriber supports; its native module does not expose the list. +const LANGUAGES = [ + { locale: undefined, label: "Automatic", description: "Uses the app's language." }, + { locale: "en-US", label: "English (US)", description: "English (United States)" }, + { locale: "en-GB", label: "English (UK)", description: "English (United Kingdom)" }, + { locale: "es-ES", label: "Español (España)", description: "Spanish (Spain)" }, + { locale: "es-MX", label: "Español (México)", description: "Spanish (Mexico)" }, + { locale: "fr-FR", label: "Français", description: "French" }, + { locale: "de-DE", label: "Deutsch", description: "German" }, + { locale: "it-IT", label: "Italiano", description: "Italian" }, + { locale: "pt-BR", label: "Português (Brasil)", description: "Portuguese (Brazil)" }, + { locale: "ja-JP", label: "日本語", description: "Japanese" }, + { locale: "ko-KR", label: "한국어", description: "Korean" }, + { locale: "zh-CN", label: "中文(普通话)", description: "Chinese (Mandarin)" }, +] as const; + +export function SettingsVoiceInputRouteScreen() { + const insets = useSafeAreaInsets(); + const savePreferences = useAtomSet(updateMobilePreferencesAtom); + const selectedLocale = useAtomValue(mobilePreferencesAtom, (result) => + AsyncResult.isSuccess(result) ? result.value.voiceInputLanguage : null, + ); + + return ( + + + {LANGUAGES.map((option, index) => ( + 0} + disabled={selectedLocale === null} + onPress={() => savePreferences({ voiceInputLanguage: option.locale })} + /> + ))} + + + ); +} diff --git a/apps/mobile/src/features/settings/components/settings-sheet-targets.ts b/apps/mobile/src/features/settings/components/settings-sheet-targets.ts index ad0197d634d85..4b7c5e9b83168 100644 --- a/apps/mobile/src/features/settings/components/settings-sheet-targets.ts +++ b/apps/mobile/src/features/settings/components/settings-sheet-targets.ts @@ -13,6 +13,7 @@ export type SettingsSheetTarget = | "SettingsEnvironmentMaintenance" | "SettingsProviderAccounts" | "SettingsKeyboard" + | "SettingsVoiceInput" | "SettingsFollowUp" | "SettingsScheduledTasks" | "SettingsProjectGrouping" diff --git a/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx b/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx index ebd628e4bad4a..23e9d99a5d475 100644 --- a/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx +++ b/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx @@ -1,3 +1,4 @@ +import { useAtomValue } from "@effect/atom-react"; import { AudioModule, RecordingPresets, @@ -7,6 +8,7 @@ import { type RecorderState, type RecordingStatus, } from "expo-audio"; +import { AsyncResult } from "effect/reactivity"; import { File } from "expo-file-system"; import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake"; import { @@ -23,6 +25,7 @@ import { AppState, Platform } from "react-native"; import { useSharedValue } from "react-native-reanimated"; import { getLocalVoiceTranscriber } from "../../native/voiceTranscription"; +import { mobilePreferencesAtom } from "../../state/preferences"; import { getNativeShowcaseScene } from "../showcase/nativeShowcaseScene"; import { VOICE_RECORDING_LIMIT_SECONDS, @@ -112,6 +115,13 @@ function useVoiceInputRuntime() { const audioLevelsRef = useRef(Array(VOICE_WAVEFORM_SAMPLE_COUNT).fill(0)); const audioLevels = useSharedValue(audioLevelsRef.current); const sessionRef = useRef(null); + const language = useAtomValue(mobilePreferencesAtom, (result) => + AsyncResult.isSuccess(result) ? result.value.voiceInputLanguage : undefined, + ); + const languageRef = useRef(language); + useEffect(() => { + languageRef.current = language; + }, [language]); const recorderRef = useRef | null>(null); if (!sessionRef.current || !recorderRef.current) { @@ -130,7 +140,7 @@ function useVoiceInputRuntime() { recorderRef.current = recorder; sessionRef.current = new VoiceInputSession({ recorder, - getTranscriber: getLocalVoiceTranscriber, + getTranscriber: () => getLocalVoiceTranscriber(languageRef.current), requestPermission: async () => { const permission = await requestRecordingPermissionsAsync(); return { granted: permission.granted, canAskAgain: permission.canAskAgain }; diff --git a/apps/mobile/src/native/voiceTranscription.ios.test.ts b/apps/mobile/src/native/voiceTranscription.ios.test.ts index b08e32cdb6f65..f60b43bce93f4 100644 --- a/apps/mobile/src/native/voiceTranscription.ios.test.ts +++ b/apps/mobile/src/native/voiceTranscription.ios.test.ts @@ -74,6 +74,16 @@ describe("getLocalVoiceTranscriber", () => { expect(mocks.transcribe).toHaveBeenCalledWith(audio, "sv-SE"); }); + it("uses the chosen language instead of the app's locale", async () => { + mocks.prepare.mockResolvedValue("es-ES"); + const prepared = await getLocalVoiceTranscriber("es-ES")!.prepare({ + signal: new AbortController().signal, + }); + + expect(mocks.prepare).toHaveBeenCalledWith("es-ES"); + expect(prepared.locale).toBe("es-ES"); + }); + it("does not start native transcription after cancellation during a file read", async () => { const enteredRead = deferred(); const readResult = deferred(); diff --git a/apps/mobile/src/native/voiceTranscription.ios.ts b/apps/mobile/src/native/voiceTranscription.ios.ts index 216b9e958dd66..e8d22674a1481 100644 --- a/apps/mobile/src/native/voiceTranscription.ios.ts +++ b/apps/mobile/src/native/voiceTranscription.ios.ts @@ -33,8 +33,8 @@ function getNativeErrorCode(error: unknown): string | undefined { return typeof error.code === "string" ? error.code : undefined; } -export function getLocalVoiceTranscriber(): VoiceTranscriber | null { - const locale = getDeviceLocale(); +export function getLocalVoiceTranscriber(language?: string): VoiceTranscriber | null { + const locale = language ?? getDeviceLocale(); if (!AppleTranscription.isAvailable(locale)) return null; return { prepare: (options) => prepareVoiceTranscription(locale, options) }; } diff --git a/apps/mobile/src/native/voiceTranscription.ts b/apps/mobile/src/native/voiceTranscription.ts index e003064ae3f84..47dd2df817d38 100644 --- a/apps/mobile/src/native/voiceTranscription.ts +++ b/apps/mobile/src/native/voiceTranscription.ts @@ -1,5 +1,5 @@ import type { VoiceTranscriber } from "@t3tools/client-runtime/voice-input"; -export function getLocalVoiceTranscriber(): VoiceTranscriber | null { +export function getLocalVoiceTranscriber(_language?: string): VoiceTranscriber | null { return null; } diff --git a/apps/mobile/src/persistence/mobile-preferences.ts b/apps/mobile/src/persistence/mobile-preferences.ts index ee96bce2d5cec..fee633d2285ab 100644 --- a/apps/mobile/src/persistence/mobile-preferences.ts +++ b/apps/mobile/src/persistence/mobile-preferences.ts @@ -36,6 +36,8 @@ export interface Preferences { * message sent during a running turn queues behind it or steers it. */ readonly followUpBehavior?: FollowUpBehavior; + /** Locale for on-device voice input. Unset follows the app's locale. iOS only. */ + readonly voiceInputLanguage?: string; /** @deprecated Kept temporarily so older OTA bundles retain the selected mode. */ readonly projectGroupingEnabled?: boolean; readonly projectGroupingMode?: SidebarProjectGroupingMode; @@ -107,6 +109,7 @@ function sanitizePreferences(parsed: Preferences): Preferences { collapsedProjectGroups?: readonly string[]; composerEnterBehavior?: ComposerEnterBehavior; followUpBehavior?: FollowUpBehavior; + voiceInputLanguage?: string; projectGroupingEnabled?: boolean; projectGroupingMode?: SidebarProjectGroupingMode; planModeEnabled?: boolean; @@ -172,6 +175,9 @@ function sanitizePreferences(parsed: Preferences): Preferences { if (parsed.followUpBehavior === "queue" || parsed.followUpBehavior === "steer") { preferences.followUpBehavior = parsed.followUpBehavior; } + if (typeof parsed.voiceInputLanguage === "string") { + preferences.voiceInputLanguage = parsed.voiceInputLanguage; + } if (typeof parsed.projectGroupingEnabled === "boolean") { preferences.projectGroupingEnabled = parsed.projectGroupingEnabled; } diff --git a/docs/user/composer.md b/docs/user/composer.md index ba8d26fd81c29..17c6ae4e6ba0f 100644 --- a/docs/user/composer.md +++ b/docs/user/composer.md @@ -157,6 +157,7 @@ and use **Attach again** or remove the missing file before sending. On supported iPhones with iOS 26 or later, use the composer's microphone to record, then confirm to transcribe. Text is inserted where your selection was when recording started, ready for you to review and edit before sending. +To dictate in a language other than the app's, choose it in **Settings → Voice input**. The first use may download Apple's speech model and needs a network connection. Later transcription works offline for that language. Recordings can be up to five diff --git a/packages/client-runtime/src/voice-input/controller.test.ts b/packages/client-runtime/src/voice-input/controller.test.ts index d73c8e7505baf..ae637b0d9748e 100644 --- a/packages/client-runtime/src/voice-input/controller.test.ts +++ b/packages/client-runtime/src/voice-input/controller.test.ts @@ -134,6 +134,31 @@ describe("resolveTranscriptCommit", () => { }); }); + it("adds word spacing for other space-delimited languages", () => { + const atEnd = draft({ + text: "Revisá la caché.", + selection: { start: "Revisá la caché.".length, end: "Revisá la caché.".length }, + }); + expect(resolveTranscriptCommit(atEnd, atEnd, "Ésta también.", "es-ES")).toMatchObject({ + kind: "commit", + text: "Revisá la caché. Ésta también.", + }); + }); + + it("treats accented letters as word boundaries", () => { + const afterAccent = draft({ text: "café", selection: { start: 4, end: 4 } }); + expect(resolveTranscriptCommit(afterAccent, afterAccent, "también", "es-ES")).toMatchObject({ + kind: "commit", + text: "café también", + }); + + const beforeAccent = draft({ text: "él", selection: { start: 0, end: 0 } }); + expect(resolveTranscriptCommit(beforeAccent, beforeAccent, "Dijo", "es-ES")).toMatchObject({ + kind: "commit", + text: "Dijo él", + }); + }); + it("does not add English boundary spaces to CJK or selected inline text", () => { const cjk = draft({ text: "修正キャッシュ", selection: { start: 8, end: 8 } }); expect(resolveTranscriptCommit(cjk, cjk, "テストも", "ja-JP")).toMatchObject({ diff --git a/packages/client-runtime/src/voice-input/controller.ts b/packages/client-runtime/src/voice-input/controller.ts index 80a8a528f335f..c2d07898968e0 100644 --- a/packages/client-runtime/src/voice-input/controller.ts +++ b/packages/client-runtime/src/voice-input/controller.ts @@ -92,18 +92,18 @@ export function resolveTranscriptCommit( const isEmptySelection = captured.selection.start === captured.selection.end; const normalizedLocale = locale.replaceAll("_", "-").toLowerCase(); - const usesEnglishSpacing = normalizedLocale === "en" || normalizedLocale.startsWith("en-"); + const usesWordSpacing = !/^(ja|zh|yue|th|lo|km|my)(-|$)/.test(normalizedLocale); let insertion = replacement; - if (isEmptySelection && usesEnglishSpacing) { + if (isEmptySelection && usesWordSpacing) { const left = captured.text[captured.selection.start - 1]; const right = captured.text[captured.selection.start]; const leftNeedsBoundary = left !== undefined && - /[A-Za-z0-9.!?,:;)\]}'"]/.test(left) && + /[\p{L}\p{N}.!?,:;)\]}'"]/u.test(left) && (right === undefined || /\s/.test(right)); const rightNeedsBoundary = right !== undefined && - /[A-Za-z0-9([{'"]/.test(right) && + /[\p{L}\p{N}([{'"]/u.test(right) && (left === undefined || /\s/.test(left)); insertion = `${leftNeedsBoundary ? " " : ""}${replacement}${rightNeedsBoundary ? " " : ""}`; }