diff --git a/apps/mobile/src/native/voiceTranscription.ios.test.ts b/apps/mobile/src/native/voiceTranscription.ios.test.ts index b08e32cdb6f6..b855b1bf2bb4 100644 --- a/apps/mobile/src/native/voiceTranscription.ios.test.ts +++ b/apps/mobile/src/native/voiceTranscription.ios.test.ts @@ -8,8 +8,11 @@ const mocks = vi.hoisted(() => ({ prepare: vi.fn<(locale: string) => Promise>(), transcribe: vi.fn<(audio: ArrayBufferLike, locale: string) => Promise>(), readAudio: vi.fn<() => Promise>(), + getSetting: vi.fn<(key: string) => unknown>(), })); +vi.mock("react-native", () => ({ Settings: { get: mocks.getSetting } })); + vi.mock("@react-native-ai/apple/src/NativeAppleTranscription", () => ({ default: { isAvailable: mocks.isAvailable, @@ -74,6 +77,39 @@ describe("getLocalVoiceTranscriber", () => { expect(mocks.transcribe).toHaveBeenCalledWith(audio, "sv-SE"); }); + it("prefers the user's languages over the English-only app locale", async () => { + const resolvedOptions = Intl.DateTimeFormat().resolvedOptions(); + vi.spyOn(Intl.DateTimeFormat.prototype, "resolvedOptions").mockReturnValue({ + ...resolvedOptions, + locale: "en-DE", + }); + mocks.getSetting.mockReturnValue(["de-DE", "en-US"]); + mocks.prepare.mockResolvedValue("de_DE"); + + const prepared = await getLocalVoiceTranscriber()!.prepare({ + signal: new AbortController().signal, + }); + + expect(mocks.prepare.mock.calls).toEqual([["de-DE"]]); + expect(prepared.locale).toBe("de_DE"); + }); + + it("falls back to the next language when one is unsupported", async () => { + mocks.getSetting.mockReturnValue(["gsw-CH", "de-CH"]); + mocks.prepare + .mockRejectedValueOnce( + Object.assign(new Error(), { code: "AppleTranscriptionUnsupportedLocale" }), + ) + .mockResolvedValueOnce("de_CH"); + + const prepared = await getLocalVoiceTranscriber()!.prepare({ + signal: new AbortController().signal, + }); + + expect(mocks.prepare.mock.calls).toEqual([["gsw-CH"], ["de-CH"]]); + expect(prepared.locale).toBe("de_CH"); + }); + it("does not start native transcription after cancellation during a file read", async () => { const enteredRead = deferred(); const readResult = deferred(); diff --git a/apps/mobile/src/native/voiceTranscription.ios.ts b/apps/mobile/src/native/voiceTranscription.ios.ts index 216b9e958dd6..4363965dd214 100644 --- a/apps/mobile/src/native/voiceTranscription.ios.ts +++ b/apps/mobile/src/native/voiceTranscription.ios.ts @@ -1,5 +1,6 @@ import AppleTranscription from "@react-native-ai/apple/src/NativeAppleTranscription"; import { File } from "expo-file-system"; +import { Settings } from "react-native"; import { VoiceTranscriptionError, @@ -9,8 +10,13 @@ import { type VoiceTranscriptionOptions, } from "@t3tools/client-runtime/voice-input"; -function getDeviceLocale(): string { - return Intl.DateTimeFormat().resolvedOptions().locale; +// The English-only app locale can differ from the user's preferred spoken languages. +function getPreferredLocales(): string[] { + const languages = Settings.get("AppleLanguages"); + const preferred = Array.isArray(languages) + ? languages.filter((language): language is string => typeof language === "string") + : []; + return [...new Set([...preferred, Intl.DateTimeFormat().resolvedOptions().locale])]; } function wrapError( @@ -34,12 +40,28 @@ function getNativeErrorCode(error: unknown): string | undefined { } export function getLocalVoiceTranscriber(): VoiceTranscriber | null { - const locale = getDeviceLocale(); - if (!AppleTranscription.isAvailable(locale)) return null; - return { prepare: (options) => prepareVoiceTranscription(locale, options) }; + const locales = getPreferredLocales(); + if (!AppleTranscription.isAvailable(locales[0]!)) return null; + return { prepare: (options) => prepareVoiceTranscription(locales, options) }; } async function prepareVoiceTranscription( + locales: readonly string[], + options: VoiceTranscriptionOptions, +): Promise { + for (const locale of locales.slice(0, -1)) { + try { + return await prepareLocale(locale, options); + } catch (error) { + if (!(error instanceof VoiceTranscriptionError && error.code === "unsupported-locale")) { + throw error; + } + } + } + return prepareLocale(locales.at(-1)!, options); +} + +async function prepareLocale( locale: string, { signal }: VoiceTranscriptionOptions, ): Promise { diff --git a/docs/user/composer.md b/docs/user/composer.md index 5c5335350e5e..a933dc83eaf2 100644 --- a/docs/user/composer.md +++ b/docs/user/composer.md @@ -178,6 +178,10 @@ On supported iPhones with iOS 26 or later, use the composer's microphone to reco then confirm to transcribe. Text is inserted where your selection was when recording started, ready for you to review and edit before sending. +Transcription tries your iPhone's preferred languages in order +(Settings > General > Language & Region), then the app's language. If Apple's speech +model supports none of them, voice input reports that the language is unavailable. + The first use may download Apple's speech model and needs a network connection. Later transcription works offline for that language. Recordings can be up to five minutes long. Canceling, leaving the screen, or an audio interruption discards the diff --git a/packages/client-runtime/src/voice-input/controller.test.ts b/packages/client-runtime/src/voice-input/controller.test.ts index d73c8e7505ba..9bd7fe67c9c5 100644 --- a/packages/client-runtime/src/voice-input/controller.test.ts +++ b/packages/client-runtime/src/voice-input/controller.test.ts @@ -134,13 +134,34 @@ describe("resolveTranscriptCommit", () => { }); }); - it("does not add English boundary spaces to CJK or selected inline text", () => { - const cjk = draft({ text: "修正キャッシュ", selection: { start: 8, end: 8 } }); + it("adds word spacing for other space-delimited languages", () => { + const atStart = draft({ text: "Änderungen", selection: { start: 0, end: 0 } }); + expect(resolveTranscriptCommit(atStart, atStart, "Weitere", "de_DE")).toMatchObject({ + kind: "commit", + text: "Weitere Änderungen", + }); + + const combiningMark = draft({ text: "नमस्ते", selection: { start: 6, end: 6 } }); + expect(resolveTranscriptCommit(combiningMark, combiningMark, "दुनिया", "hi-IN")).toMatchObject({ + kind: "commit", + text: "नमस्ते दुनिया", + }); + }); + + it("preserves spaceless languages and selected inline text", () => { + const text = "修正キャッシュ"; + const cjk = draft({ text, selection: { start: text.length, end: text.length } }); expect(resolveTranscriptCommit(cjk, cjk, "テストも", "ja-JP")).toMatchObject({ kind: "commit", text: "修正キャッシュテストも", }); + const thai = draft({ text: "โลก", selection: { start: 0, end: 0 } }); + expect(resolveTranscriptCommit(thai, thai, "สวัสดี", "th-TH")).toMatchObject({ + kind: "commit", + text: "สวัสดีโลก", + }); + const selected = draft({ text: "one $skill two", selection: { start: 4, end: 10 } }); expect(resolveTranscriptCommit(selected, selected, "new", "en-US")).toMatchObject({ kind: "commit", diff --git a/packages/client-runtime/src/voice-input/controller.ts b/packages/client-runtime/src/voice-input/controller.ts index 39dfd8542d88..039635b3049b 100644 --- a/packages/client-runtime/src/voice-input/controller.ts +++ b/packages/client-runtime/src/voice-input/controller.ts @@ -70,6 +70,8 @@ type TranscriptCommitResult = | { readonly kind: "stale" } | { readonly kind: "empty" }; +const WORD_SPACELESS_LANGUAGES = new Set(["ja", "zh", "yue", "th", "lo", "km", "my"]); + export function resolveTranscriptCommit( captured: VoiceDraftSnapshot, current: VoiceDraftSnapshot | null, @@ -91,19 +93,19 @@ export function resolveTranscriptCommit( } const isEmptySelection = captured.selection.start === captured.selection.end; - const normalizedLocale = locale.replaceAll("_", "-").toLowerCase(); - const usesEnglishSpacing = normalizedLocale === "en" || normalizedLocale.startsWith("en-"); + const language = locale.split(/[-_]/)[0]!.toLowerCase(); + const usesWordSpacing = !WORD_SPACELESS_LANGUAGES.has(language); let insertion = replacement; - if (isEmptySelection && usesEnglishSpacing) { + if (isEmptySelection && usesWordSpacing) { const left = captured.text[captured.selection.start - 1]; const right = captured.text[captured.selection.start]; const leftNeedsBoundary = left !== undefined && - /[A-Za-z0-9.!?,:;)\]}'"]/.test(left) && + /[\p{L}\p{M}\p{N}.!?,:;)\]}'"]/u.test(left) && (right === undefined || /\s/.test(right)); const rightNeedsBoundary = right !== undefined && - /[A-Za-z0-9([{'"]/.test(right) && + /[\p{L}\p{N}([{'"]/u.test(right) && (left === undefined || /\s/.test(left)); insertion = `${leftNeedsBoundary ? " " : ""}${replacement}${rightNeedsBoundary ? " " : ""}`; }