From 1ea9d0312e61518ec9ba36ecc740d6f330c3166c Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Thu, 5 Nov 2026 19:44:34 -0800 Subject: [PATCH] fix(mobile): keep the opening words of local iPhone dictation Start local iPhone capture while Apple prepares the user's speech model. The existing start hook can now return null to request transcription of the completed file. A model failure discards the clip. Stopping waits for the model. Cancelling does not: it releases the file and session so another recording can start before the download finishes. Each recording asks Apple whether its own model is ready; a cancelled request cannot block a later recording from transcribing. Android, web and environment transcribers keep their own preparation and consent checks before capture. Co-Authored-By: Claude Fable 5.1 --- .../native/voiceTranscription.android.test.ts | 4 +- .../src/native/voiceTranscription.ios.test.ts | 95 ++++++++++++++++++- .../src/native/voiceTranscription.ios.ts | 45 +++++++-- .../src/voice-input/controller.test.ts | 4 + .../src/voice-input/controller.ts | 16 ++-- .../src/voice-input/transcription.ts | 7 +- 6 files changed, 151 insertions(+), 20 deletions(-) diff --git a/apps/mobile/src/native/voiceTranscription.android.test.ts b/apps/mobile/src/native/voiceTranscription.android.test.ts index e171cf9cb500a..e69998f8b15e1 100644 --- a/apps/mobile/src/native/voiceTranscription.android.test.ts +++ b/apps/mobile/src/native/voiceTranscription.android.test.ts @@ -78,10 +78,11 @@ it.each([ selection: { start: 16, end: 16 }, revision: 1, }; + const prepareToRecordAsync = vi.fn(async () => {}); const controller = new VoiceInputController({ recorder: { uri: "file:///acceptance.m4a", - prepareToRecordAsync: async () => {}, + prepareToRecordAsync, record: () => {}, stop: async () => {}, }, @@ -101,4 +102,5 @@ it.each([ expect(commit).not.toHaveBeenCalled(); expect(draft.text).toBe("Preserved draft."); expect(controller.currentState.error).toMatch(expected); + expect(prepareToRecordAsync).toHaveBeenCalledTimes(phase === "preparation" ? 0 : 1); }); diff --git a/apps/mobile/src/native/voiceTranscription.ios.test.ts b/apps/mobile/src/native/voiceTranscription.ios.test.ts index 4703fd6071f79..e23711626acf6 100644 --- a/apps/mobile/src/native/voiceTranscription.ios.test.ts +++ b/apps/mobile/src/native/voiceTranscription.ios.test.ts @@ -1,7 +1,8 @@ import type { TranscriptionResult } from "@react-native-ai/apple/src/NativeAppleTranscription"; import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; -import { VoiceTranscriptionError } from "@t3tools/client-runtime/voice-input"; +import { VoiceInputController, VoiceTranscriptionError } from "@t3tools/client-runtime/voice-input"; +import { resetVoiceInputGlobalsForTests } from "../../../../packages/client-runtime/src/voice-input/controller"; const mocks = vi.hoisted(() => ({ isAvailable: vi.fn<(locale: string) => boolean>(), @@ -65,8 +66,43 @@ async function transcribeOnce(signal = new AbortController().signal) { .catch((error: unknown) => error); } +/** Drives one on-device dictation through the controller the composer uses. */ +function createDictation() { + const recorder = { + uri: "file:///voice.m4a", + prepareToRecordAsync: async () => {}, + record: vi.fn(), + stop: vi.fn(async () => {}), + }; + const commits: string[] = []; + const deleted: string[] = []; + const ended = Promise.withResolvers(); + const controller = new VoiceInputController({ + recorder, + getTranscriber: () => ({ + prepare: async (options) => (await getLocalVoiceTranscriber())!.prepare(options), + }), + requestPermission: async () => ({ granted: true, canAskAgain: true }), + configureRecording: async () => {}, + releaseRecording: async () => {}, + deleteRecording: (uri) => deleted.push(uri), + readDraft: () => ({ + ownerKey: "environment:thread", + text: "", + selection: { start: 0, end: 0 }, + revision: 1, + }), + commitDraft: (text) => commits.push(text), + onStateChange: (state) => { + if (state.phase === "idle" || state.phase === "error") ended.resolve(); + }, + }); + return { controller, recorder, commits, deleted, ended: ended.promise }; +} + beforeEach(() => { vi.resetAllMocks(); + resetVoiceInputGlobalsForTests(); useLanguages([], "sv-SE"); mocks.isAvailable.mockReturnValue(true); mocks.prepare.mockResolvedValue("sv-SE"); @@ -147,6 +183,63 @@ describe("getLocalVoiceTranscriber", () => { }, ); + it("records while Apple prepares its speech model and transcribes once it is ready", async () => { + const nativePrepare = Promise.withResolvers(); + mocks.prepare.mockReturnValue(nativePrepare.promise); + const dictation = createDictation(); + + await dictation.controller.start(); + expect(dictation.recorder.record).toHaveBeenCalledOnce(); + expect(dictation.controller.currentState.phase).toBe("recording"); + + const stopping = dictation.controller.stop(); + nativePrepare.resolve("sv-SE"); + await stopping; + + expect(mocks.transcribe).toHaveBeenCalledOnce(); + expect(dictation.commits).toEqual(["Hej världen."]); + expect(dictation.deleted).toEqual(["file:///voice.m4a"]); + }); + + it.each(["cancel", "cancel after stop", "model failure", "model failure after stop"] as const)( + "discards early audio without a late commit after %s", + async (ending) => { + const nativePrepare = Promise.withResolvers(); + mocks.prepare.mockReturnValue(nativePrepare.promise); + const dictation = createDictation(); + await dictation.controller.start(); + + let finishing: Promise | undefined; + const cancelled = ending === "cancel" || ending === "cancel after stop"; + if (ending.endsWith("after stop")) finishing = dictation.controller.stop(); + if (cancelled) { + dictation.controller.cancel(); + } else { + nativePrepare.reject(Object.assign(new Error("download failed"), { code: "Network" })); + } + await finishing; + await dictation.ended; + + const next = createDictation(); + mocks.prepare.mockResolvedValue("sv-SE"); + await next.controller.start(); + expect(next.controller.currentState.phase).toBe("recording"); + expect(dictation.deleted).toEqual(["file:///voice.m4a"]); + expect(mocks.transcribe).not.toHaveBeenCalled(); + await next.controller.stop(); + expect(next.commits).toEqual(["Hej världen."]); + nativePrepare.resolve("sv-SE"); + + expect(dictation.commits).toEqual([]); + expect(dictation.recorder.stop).toHaveBeenCalled(); + expect(dictation.controller.currentState).toMatchObject( + cancelled + ? { phase: "idle" } + : { phase: "error", error: "Voice transcription could not prepare this language." }, + ); + }, + ); + it("does not start native transcription after cancellation during a file read", async () => { const enteredRead = Promise.withResolvers(); const readResult = Promise.withResolvers(); diff --git a/apps/mobile/src/native/voiceTranscription.ios.ts b/apps/mobile/src/native/voiceTranscription.ios.ts index 910a17c77fd15..8f5822afb011b 100644 --- a/apps/mobile/src/native/voiceTranscription.ios.ts +++ b/apps/mobile/src/native/voiceTranscription.ios.ts @@ -63,6 +63,41 @@ async function prepareVoiceTranscription( }); } + let locale = locales[0]!; + const model = prepareLocales(locales, signal).then((supported) => { + locale = supported; + }); + // Recorder setup can fail before either consumer observes the model. + void model.catch(() => {}); + + async function ready(signal: AbortSignal) { + throwIfVoiceTranscriptionAborted(signal); + const cancelled = Promise.withResolvers(); + const cancel = () => cancelled.resolve(); + signal.addEventListener("abort", cancel, { once: true }); + try { + await Promise.race([model, cancelled.promise]); + throwIfVoiceTranscriptionAborted(signal); + return locale; + } finally { + signal.removeEventListener("abort", cancel); + } + } + return { + get locale() { + return locale; + }, + start: async (_uri, { signal }) => { + await ready(signal); + return null; + }, + transcribe: async (uri, options) => + transcribeVoiceRecording(uri, await ready(options.signal), options), + }; +} + +async function prepareLocales(locales: readonly string[], signal: AbortSignal): Promise { + throwIfVoiceTranscriptionAborted(signal); let unsupported: VoiceTranscriptionError | undefined; for (const locale of locales) { try { @@ -75,17 +110,11 @@ async function prepareVoiceTranscription( throw unsupported!; } -async function prepareLocale( - locale: string, - signal: AbortSignal, -): Promise { +async function prepareLocale(locale: string, signal: AbortSignal): Promise { try { const supportedLocale = await AppleTranscription.prepare(locale); throwIfVoiceTranscriptionAborted(signal); - return { - locale: supportedLocale, - transcribe: (uri, options) => transcribeVoiceRecording(uri, supportedLocale, options), - }; + return supportedLocale; } catch (error) { throwIfVoiceTranscriptionAborted(signal); if (getNativeErrorCode(error) === "AppleTranscriptionUnsupportedLocale") { diff --git a/packages/client-runtime/src/voice-input/controller.test.ts b/packages/client-runtime/src/voice-input/controller.test.ts index 265a284e20fe3..41e56933ce104 100644 --- a/packages/client-runtime/src/voice-input/controller.test.ts +++ b/packages/client-runtime/src/voice-input/controller.test.ts @@ -309,7 +309,9 @@ describe("VoiceInputController", () => { code: "unavailable", message: "This device requires iOS 26 or later.", }); + const configureRecording = vi.fn(async () => undefined); const unavailable = createHarness({ + configureRecording, getTranscriber: () => ({ prepare: async () => { throw reason; @@ -318,6 +320,8 @@ describe("VoiceInputController", () => { }); await unavailable.controller.start(); expect(unavailable.controller.currentState.error).toBe(reason.message); + // Environment and web transcribers gate capture on consent checked during preparation. + expect(configureRecording).not.toHaveBeenCalled(); expect(unavailable.recorder.record).not.toHaveBeenCalled(); const denied = createHarness({ diff --git a/packages/client-runtime/src/voice-input/controller.ts b/packages/client-runtime/src/voice-input/controller.ts index c67287dd74d36..668705546a489 100644 --- a/packages/client-runtime/src/voice-input/controller.ts +++ b/packages/client-runtime/src/voice-input/controller.ts @@ -236,7 +236,7 @@ export class VoiceInputController { private operationToken = 0; private sessionToken: VoiceInputSession | null = null; private transcription: PreparedVoiceTranscription | null = null; - private liveTranscription: Promise<{ text: string } | { error: unknown }> | null = null; + private liveTranscription: Promise<{ text: string | null } | { error: unknown }> | null = null; private transcriptionAbortController: AbortController | null = null; private postProcessingAbortController: AbortController | null = null; private skipPostProcessingRequested = false; @@ -504,17 +504,15 @@ export class VoiceInputController { const capturedDraft = this.capturedDraft; let transcript: string; try { - if (this.liveTranscription) { - const result = await this.liveTranscription; - if ("error" in result) throw result.error; - transcript = result.text; - } else { - transcript = await runTranscriptionOperation(() => + const live = this.liveTranscription ? await this.liveTranscription : null; + if (live && "error" in live) throw live.error; + transcript = + live?.text ?? + (await runTranscriptionOperation(() => transcription.streaming ? transcription.streaming.finish({ signal }) : transcription.transcribe(recordingUri!, { signal }), - ); - } + )); } catch (error) { if (this.isCurrent(operationToken)) { if ( diff --git a/packages/client-runtime/src/voice-input/transcription.ts b/packages/client-runtime/src/voice-input/transcription.ts index 7af1196c8c0a0..4c674ee9c1c2c 100644 --- a/packages/client-runtime/src/voice-input/transcription.ts +++ b/packages/client-runtime/src/voice-input/transcription.ts @@ -11,7 +11,12 @@ export type VoiceStreamingSession = { export type PreparedVoiceTranscription = { readonly locale: string; - readonly start?: (uri: string, options: VoiceTranscriptionOptions) => Promise; + /** + * Runs alongside capture once recording starts. Resolves with the transcript when it + * transcribed live, or null when `transcribe` still runs after the recording stops. + * A rejection stops the recording. + */ + readonly start?: (uri: string, options: VoiceTranscriptionOptions) => Promise; readonly transcribe: (uri: string, options: VoiceTranscriptionOptions) => Promise; /** Present when audio streams during capture; `finish` then replaces `transcribe`. */ readonly streaming?: VoiceStreamingSession;