From 196d8e9c5e05a1a4c17e0d4e4b8de66e315f4c8f Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 26 Oct 2026 23:48:29 -0700 Subject: [PATCH 1/5] fix(mobile): voice input hears the first words, in the user's language Start the recorder while Apple's speech model prepares instead of after, so speech right after tapping the mic is kept. Prepare with the user's preferred iOS languages before the English-only app locale, and add word spacing for every space-delimited language. Co-authored-by: nichtlegacy Co-Authored-By: Claude Opus 5.5 --- .../src/native/voiceTranscription.ios.test.ts | 36 +++++++++++ .../src/native/voiceTranscription.ios.ts | 32 ++++++++-- docs/user/composer.md | 4 ++ .../src/voice-input/controller.test.ts | 64 +++++++++++++++++++ .../src/voice-input/controller.ts | 51 ++++++++++----- 5 files changed, 166 insertions(+), 21 deletions(-) diff --git a/apps/mobile/src/native/voiceTranscription.ios.test.ts b/apps/mobile/src/native/voiceTranscription.ios.test.ts index d3ee19c262de6..38d196c7dcb39 100644 --- a/apps/mobile/src/native/voiceTranscription.ios.test.ts +++ b/apps/mobile/src/native/voiceTranscription.ios.test.ts @@ -8,8 +8,11 @@ const mocks = vi.hoisted(() => ({ prepare: vi.fn<(locale: string) => Promise>(), transcribe: vi.fn<(audio: ArrayBufferLike, locale: string) => Promise>(), readAudio: vi.fn<() => Promise>(), + getSetting: vi.fn<(key: string) => unknown>(), })); +vi.mock("react-native", () => ({ Settings: { get: mocks.getSetting } })); + vi.mock("@react-native-ai/apple/src/NativeAppleTranscription", () => ({ default: { isAvailable: mocks.isAvailable, @@ -56,6 +59,39 @@ afterEach(() => { }); describe("getLocalVoiceTranscriber", () => { + it("prefers the user's languages over the English-only app locale", async () => { + const resolvedOptions = Intl.DateTimeFormat().resolvedOptions(); + vi.spyOn(Intl.DateTimeFormat.prototype, "resolvedOptions").mockReturnValue({ + ...resolvedOptions, + locale: "en-KR", + }); + mocks.getSetting.mockReturnValue(["ko-KR", "en-US"]); + mocks.prepare.mockResolvedValue("ko_KR"); + + const prepared = await getLocalVoiceTranscriber()!.prepare({ + signal: new AbortController().signal, + }); + + expect(mocks.prepare.mock.calls).toEqual([["ko-KR"]]); + expect(prepared.locale).toBe("ko_KR"); + }); + + it("falls back to the next language when one is unsupported", async () => { + mocks.getSetting.mockReturnValue(["gsw-CH", "de-CH"]); + mocks.prepare + .mockRejectedValueOnce( + Object.assign(new Error(), { code: "AppleTranscriptionUnsupportedLocale" }), + ) + .mockResolvedValueOnce("de_CH"); + + const prepared = await getLocalVoiceTranscriber()!.prepare({ + signal: new AbortController().signal, + }); + + expect(mocks.prepare.mock.calls).toEqual([["gsw-CH"], ["de-CH"]]); + expect(prepared.locale).toBe("de_CH"); + }); + it("does not start native transcription after cancellation during a file read", async () => { const enteredRead = deferred(); const readResult = deferred(); diff --git a/apps/mobile/src/native/voiceTranscription.ios.ts b/apps/mobile/src/native/voiceTranscription.ios.ts index 216b9e958dd66..4363965dd214a 100644 --- a/apps/mobile/src/native/voiceTranscription.ios.ts +++ b/apps/mobile/src/native/voiceTranscription.ios.ts @@ -1,5 +1,6 @@ import AppleTranscription from "@react-native-ai/apple/src/NativeAppleTranscription"; import { File } from "expo-file-system"; +import { Settings } from "react-native"; import { VoiceTranscriptionError, @@ -9,8 +10,13 @@ import { type VoiceTranscriptionOptions, } from "@t3tools/client-runtime/voice-input"; -function getDeviceLocale(): string { - return Intl.DateTimeFormat().resolvedOptions().locale; +// The English-only app locale can differ from the user's preferred spoken languages. +function getPreferredLocales(): string[] { + const languages = Settings.get("AppleLanguages"); + const preferred = Array.isArray(languages) + ? languages.filter((language): language is string => typeof language === "string") + : []; + return [...new Set([...preferred, Intl.DateTimeFormat().resolvedOptions().locale])]; } function wrapError( @@ -34,12 +40,28 @@ function getNativeErrorCode(error: unknown): string | undefined { } export function getLocalVoiceTranscriber(): VoiceTranscriber | null { - const locale = getDeviceLocale(); - if (!AppleTranscription.isAvailable(locale)) return null; - return { prepare: (options) => prepareVoiceTranscription(locale, options) }; + const locales = getPreferredLocales(); + if (!AppleTranscription.isAvailable(locales[0]!)) return null; + return { prepare: (options) => prepareVoiceTranscription(locales, options) }; } async function prepareVoiceTranscription( + locales: readonly string[], + options: VoiceTranscriptionOptions, +): Promise { + for (const locale of locales.slice(0, -1)) { + try { + return await prepareLocale(locale, options); + } catch (error) { + if (!(error instanceof VoiceTranscriptionError && error.code === "unsupported-locale")) { + throw error; + } + } + } + return prepareLocale(locales.at(-1)!, options); +} + +async function prepareLocale( locale: string, { signal }: VoiceTranscriptionOptions, ): Promise { diff --git a/docs/user/composer.md b/docs/user/composer.md index 35b425faf31d1..57865809996a3 100644 --- a/docs/user/composer.md +++ b/docs/user/composer.md @@ -179,6 +179,10 @@ On supported iPhones with iOS 26 or later, use the composer's microphone to reco then confirm to transcribe. Text is inserted where your selection was when recording started, ready for you to review and edit before sending. +Transcription tries your iPhone's preferred languages in order +(Settings > General > Language & Region), then the app's language. If Apple's speech +model supports none of them, voice input reports that the language is unavailable. + The first use may download Apple's speech model and needs a network connection. Later transcription works offline for that language. Recordings can be up to five minutes long. Canceling, leaving the screen, or an audio interruption discards the diff --git a/packages/client-runtime/src/voice-input/controller.test.ts b/packages/client-runtime/src/voice-input/controller.test.ts index d433748e5bdbb..25547955f9404 100644 --- a/packages/client-runtime/src/voice-input/controller.test.ts +++ b/packages/client-runtime/src/voice-input/controller.test.ts @@ -105,6 +105,29 @@ describe("resolveTranscriptCommit", () => { kind: "stale", }); }); + + it("adds word spacing for other space-delimited languages", () => { + const korean = draft({ text: "코드를", selection: { start: 3, end: 3 } }); + expect(resolveTranscriptCommit(korean, korean, "검토해 주세요", "ko_KR")).toMatchObject({ + kind: "commit", + text: "코드를 검토해 주세요", + }); + + const combiningMark = draft({ text: "नमस्ते", selection: { start: 6, end: 6 } }); + expect(resolveTranscriptCommit(combiningMark, combiningMark, "दुनिया", "hi-IN")).toMatchObject({ + kind: "commit", + text: "नमस्ते दुनिया", + }); + }); + + it("keeps spaceless languages spaceless", () => { + const text = "修正キャッシュ"; + const cjk = draft({ text, selection: { start: text.length, end: text.length } }); + expect(resolveTranscriptCommit(cjk, cjk, "テストも", "ja-JP")).toMatchObject({ + kind: "commit", + text: "修正キャッシュテストも", + }); + }); }); describe("VoiceInputController", () => { @@ -303,6 +326,47 @@ describe("VoiceInputController", () => { }, ); + it("records while the speech model prepares, then transcribes once it is ready", async () => { + const preparation = deferred(); + const harness = createHarness({ + getTranscriber: () => ({ prepare: () => preparation.promise }), + }); + + await harness.controller.start(); + expect(harness.controller.currentState.phase).toBe("recording"); + expect(harness.recorder.record).toHaveBeenCalledTimes(1); + + const stopping = harness.controller.stop(); + expect(harness.controller.currentState.phase).toBe("transcribing"); + preparation.resolve(preparedTranscription()); + await stopping; + + expect(harness.commits.map((commit) => commit.text)).toEqual(["hello new text"]); + }); + + it("discards the recording when the speech model fails to prepare", async () => { + const preparation = deferred(); + const deleted = deferred(); + const harness = createHarness({ + getTranscriber: () => ({ prepare: () => preparation.promise }), + deleteRecording: deleted.resolve, + }); + await harness.controller.start(); + + preparation.reject(Object.assign(new Error("unsupported"), { code: "unsupported-locale" })); + + expect(await deleted.promise).toBe("file:///voice.m4a"); + expect(harness.recorder.stop).toHaveBeenCalledTimes(1); + expect(harness.controller.currentState).toMatchObject({ + phase: "error", + error: "Voice transcription is not available for this language.", + }); + const next = createHarness(); + await next.controller.start(); + expect(next.controller.currentState.phase).toBe("recording"); + await next.controller.interruptRecording(); + }); + it("releases the microphone before transcription starts", async () => { const events: string[] = []; const harness = createHarness({ diff --git a/packages/client-runtime/src/voice-input/controller.ts b/packages/client-runtime/src/voice-input/controller.ts index b9d0f54f67672..369f5f6fe0108 100644 --- a/packages/client-runtime/src/voice-input/controller.ts +++ b/packages/client-runtime/src/voice-input/controller.ts @@ -66,6 +66,8 @@ type TranscriptCommitResult = | { readonly kind: "stale" } | { readonly kind: "empty" }; +const WORD_SPACELESS_LANGUAGES = new Set(["ja", "zh", "yue", "th", "lo", "km", "my"]); + export function resolveTranscriptCommit( captured: VoiceDraftSnapshot, current: VoiceDraftSnapshot | null, @@ -87,19 +89,19 @@ export function resolveTranscriptCommit( } const isEmptySelection = captured.selection.start === captured.selection.end; - const normalizedLocale = locale.replaceAll("_", "-").toLowerCase(); - const usesEnglishSpacing = normalizedLocale === "en" || normalizedLocale.startsWith("en-"); + const language = locale.split(/[-_]/)[0]!.toLowerCase(); + const usesWordSpacing = !WORD_SPACELESS_LANGUAGES.has(language); let insertion = replacement; - if (isEmptySelection && usesEnglishSpacing) { + if (isEmptySelection && usesWordSpacing) { const left = captured.text[captured.selection.start - 1]; const right = captured.text[captured.selection.start]; const leftNeedsBoundary = left !== undefined && - /[A-Za-z0-9.!?,:;)\]}'"]/.test(left) && + /[\p{L}\p{M}\p{N}.!?,:;)\]}'"]/u.test(left) && (right === undefined || /\s/.test(right)); const rightNeedsBoundary = right !== undefined && - /[A-Za-z0-9([{'"]/.test(right) && + /[\p{L}\p{N}([{'"]/u.test(right) && (left === undefined || /\s/.test(left)); insertion = `${leftNeedsBoundary ? " " : ""}${replacement}${rightNeedsBoundary ? " " : ""}`; } @@ -184,7 +186,7 @@ export class VoiceInputController { private state: VoiceInputState = IDLE_STATE; private operationToken = 0; private sessionToken: VoiceInputSession | null = null; - private transcription: PreparedVoiceTranscription | null = null; + private transcription: Promise | null = null; private transcriptionAbortController: AbortController | null = null; private capturedDraft: VoiceDraftSnapshot | null = null; private recordingUri: string | null = null; @@ -245,15 +247,21 @@ export class VoiceInputController { return; } - try { - this.transcription = await runTranscriptionOperation(() => - transcriber.prepare({ signal: abortController.signal }), - ); - } catch (error) { - if (this.isCurrent(operationToken)) this.setError(preparationErrorMessage(error), "retry"); - return; - } - if (!this.isCurrent(operationToken)) return; + // Record while the speech model prepares so the first words are not lost. + const transcription = runTranscriptionOperation(() => + transcriber.prepare({ signal: abortController.signal }), + ); + this.transcription = transcription; + transcription.catch((error: unknown) => { + if (!this.isCurrent(operationToken)) return; + const message = preparationErrorMessage(error); + if (this.state.phase === "recording") { + void this.discardRecording(message); + } else if (this.state.phase === "preparing") { + this.invalidateOperation(); + this.setError(message, "retry"); + } + }); await this.dependencies.configureRecording(); this.recordingConfigured = true; @@ -388,9 +396,18 @@ export class VoiceInputController { } const recordingUri = this.recordingUri; - const transcription = this.transcription; const signal = this.transcriptionAbortController.signal; const capturedDraft = this.capturedDraft; + let transcription: PreparedVoiceTranscription; + try { + transcription = await this.transcription; + } catch (error) { + if (this.isCurrent(operationToken)) { + this.setError(preparationErrorMessage(error), "retry"); + } + return; + } + if (!this.isCurrent(operationToken)) return; let transcript: string; try { transcript = await runTranscriptionOperation(() => @@ -462,6 +479,8 @@ export class VoiceInputController { } this.ownedRecordingUris.clear(); await this.releaseAudioSession(); + // Preparation cancels cooperatively; the next session must not overlap it. + await this.transcription?.catch(() => {}); releaseSession(this.sessionToken); this.sessionToken = null; this.capturedDraft = null; From 159df307d91503ddb2dfb76f4016570a135eff51 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 26 Oct 2026 23:48:30 -0700 Subject: [PATCH 2/5] fix(mobile): edits after dictation reach the draft The editor is hidden with display: none while dictating, which recreates the iOS view and resets its event count. The JS wrapper kept the old count and dropped every later edit as stale, so send used the raw transcript. Remount the wrapper with the view, as the hidden composer already does. Co-Authored-By: Claude Opus 5.5 --- apps/mobile/src/features/threads/ThreadComposer.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/mobile/src/features/threads/ThreadComposer.tsx b/apps/mobile/src/features/threads/ThreadComposer.tsx index df5aca19dec1a..55b374fb51177 100644 --- a/apps/mobile/src/features/threads/ThreadComposer.tsx +++ b/apps/mobile/src/features/threads/ThreadComposer.tsx @@ -967,7 +967,8 @@ export const ThreadComposer = function ThreadComposer(props: ThreadComposerProps ]} >