diff --git a/apps/mobile/src/native/voiceTranscription.android.test.ts b/apps/mobile/src/native/voiceTranscription.android.test.ts index e171cf9cb500a..e69998f8b15e1 100644 --- a/apps/mobile/src/native/voiceTranscription.android.test.ts +++ b/apps/mobile/src/native/voiceTranscription.android.test.ts @@ -78,10 +78,11 @@ it.each([ selection: { start: 16, end: 16 }, revision: 1, }; + const prepareToRecordAsync = vi.fn(async () => {}); const controller = new VoiceInputController({ recorder: { uri: "file:///acceptance.m4a", - prepareToRecordAsync: async () => {}, + prepareToRecordAsync, record: () => {}, stop: async () => {}, }, @@ -101,4 +102,5 @@ it.each([ expect(commit).not.toHaveBeenCalled(); expect(draft.text).toBe("Preserved draft."); expect(controller.currentState.error).toMatch(expected); + expect(prepareToRecordAsync).toHaveBeenCalledTimes(phase === "preparation" ? 0 : 1); }); diff --git a/apps/mobile/src/native/voiceTranscription.ios.test.ts b/apps/mobile/src/native/voiceTranscription.ios.test.ts index 4703fd6071f79..e23711626acf6 100644 --- a/apps/mobile/src/native/voiceTranscription.ios.test.ts +++ b/apps/mobile/src/native/voiceTranscription.ios.test.ts @@ -1,7 +1,8 @@ import type { TranscriptionResult } from "@react-native-ai/apple/src/NativeAppleTranscription"; import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; -import { VoiceTranscriptionError } from "@t3tools/client-runtime/voice-input"; +import { VoiceInputController, VoiceTranscriptionError } from "@t3tools/client-runtime/voice-input"; +import { resetVoiceInputGlobalsForTests } from "../../../../packages/client-runtime/src/voice-input/controller"; const mocks = vi.hoisted(() => ({ isAvailable: vi.fn<(locale: string) => boolean>(), @@ -65,8 +66,43 @@ async function transcribeOnce(signal = new AbortController().signal) { .catch((error: unknown) => error); } +/** Drives one on-device dictation through the controller the composer uses. */ +function createDictation() { + const recorder = { + uri: "file:///voice.m4a", + prepareToRecordAsync: async () => {}, + record: vi.fn(), + stop: vi.fn(async () => {}), + }; + const commits: string[] = []; + const deleted: string[] = []; + const ended = Promise.withResolvers(); + const controller = new VoiceInputController({ + recorder, + getTranscriber: () => ({ + prepare: async (options) => (await getLocalVoiceTranscriber())!.prepare(options), + }), + requestPermission: async () => ({ granted: true, canAskAgain: true }), + configureRecording: async () => {}, + releaseRecording: async () => {}, + deleteRecording: (uri) => deleted.push(uri), + readDraft: () => ({ + ownerKey: "environment:thread", + text: "", + selection: { start: 0, end: 0 }, + revision: 1, + }), + commitDraft: (text) => commits.push(text), + onStateChange: (state) => { + if (state.phase === "idle" || state.phase === "error") ended.resolve(); + }, + }); + return { controller, recorder, commits, deleted, ended: ended.promise }; +} + beforeEach(() => { vi.resetAllMocks(); + resetVoiceInputGlobalsForTests(); useLanguages([], "sv-SE"); mocks.isAvailable.mockReturnValue(true); mocks.prepare.mockResolvedValue("sv-SE"); @@ -147,6 +183,63 @@ describe("getLocalVoiceTranscriber", () => { }, ); + it("records while Apple prepares its speech model and transcribes once it is ready", async () => { + const nativePrepare = Promise.withResolvers(); + mocks.prepare.mockReturnValue(nativePrepare.promise); + const dictation = createDictation(); + + await dictation.controller.start(); + expect(dictation.recorder.record).toHaveBeenCalledOnce(); + expect(dictation.controller.currentState.phase).toBe("recording"); + + const stopping = dictation.controller.stop(); + nativePrepare.resolve("sv-SE"); + await stopping; + + expect(mocks.transcribe).toHaveBeenCalledOnce(); + expect(dictation.commits).toEqual(["Hej världen."]); + expect(dictation.deleted).toEqual(["file:///voice.m4a"]); + }); + + it.each(["cancel", "cancel after stop", "model failure", "model failure after stop"] as const)( + "discards early audio without a late commit after %s", + async (ending) => { + const nativePrepare = Promise.withResolvers(); + mocks.prepare.mockReturnValue(nativePrepare.promise); + const dictation = createDictation(); + await dictation.controller.start(); + + let finishing: Promise | undefined; + const cancelled = ending === "cancel" || ending === "cancel after stop"; + if (ending.endsWith("after stop")) finishing = dictation.controller.stop(); + if (cancelled) { + dictation.controller.cancel(); + } else { + nativePrepare.reject(Object.assign(new Error("download failed"), { code: "Network" })); + } + await finishing; + await dictation.ended; + + const next = createDictation(); + mocks.prepare.mockResolvedValue("sv-SE"); + await next.controller.start(); + expect(next.controller.currentState.phase).toBe("recording"); + expect(dictation.deleted).toEqual(["file:///voice.m4a"]); + expect(mocks.transcribe).not.toHaveBeenCalled(); + await next.controller.stop(); + expect(next.commits).toEqual(["Hej världen."]); + nativePrepare.resolve("sv-SE"); + + expect(dictation.commits).toEqual([]); + expect(dictation.recorder.stop).toHaveBeenCalled(); + expect(dictation.controller.currentState).toMatchObject( + cancelled + ? { phase: "idle" } + : { phase: "error", error: "Voice transcription could not prepare this language." }, + ); + }, + ); + it("does not start native transcription after cancellation during a file read", async () => { const enteredRead = Promise.withResolvers(); const readResult = Promise.withResolvers(); diff --git a/apps/mobile/src/native/voiceTranscription.ios.ts b/apps/mobile/src/native/voiceTranscription.ios.ts index 910a17c77fd15..8f5822afb011b 100644 --- a/apps/mobile/src/native/voiceTranscription.ios.ts +++ b/apps/mobile/src/native/voiceTranscription.ios.ts @@ -63,6 +63,41 @@ async function prepareVoiceTranscription( }); } + let locale = locales[0]!; + const model = prepareLocales(locales, signal).then((supported) => { + locale = supported; + }); + // Recorder setup can fail before either consumer observes the model. + void model.catch(() => {}); + + async function ready(signal: AbortSignal) { + throwIfVoiceTranscriptionAborted(signal); + const cancelled = Promise.withResolvers(); + const cancel = () => cancelled.resolve(); + signal.addEventListener("abort", cancel, { once: true }); + try { + await Promise.race([model, cancelled.promise]); + throwIfVoiceTranscriptionAborted(signal); + return locale; + } finally { + signal.removeEventListener("abort", cancel); + } + } + return { + get locale() { + return locale; + }, + start: async (_uri, { signal }) => { + await ready(signal); + return null; + }, + transcribe: async (uri, options) => + transcribeVoiceRecording(uri, await ready(options.signal), options), + }; +} + +async function prepareLocales(locales: readonly string[], signal: AbortSignal): Promise { + throwIfVoiceTranscriptionAborted(signal); let unsupported: VoiceTranscriptionError | undefined; for (const locale of locales) { try { @@ -75,17 +110,11 @@ async function prepareVoiceTranscription( throw unsupported!; } -async function prepareLocale( - locale: string, - signal: AbortSignal, -): Promise { +async function prepareLocale(locale: string, signal: AbortSignal): Promise { try { const supportedLocale = await AppleTranscription.prepare(locale); throwIfVoiceTranscriptionAborted(signal); - return { - locale: supportedLocale, - transcribe: (uri, options) => transcribeVoiceRecording(uri, supportedLocale, options), - }; + return supportedLocale; } catch (error) { throwIfVoiceTranscriptionAborted(signal); if (getNativeErrorCode(error) === "AppleTranscriptionUnsupportedLocale") { diff --git a/packages/client-runtime/src/voice-input/controller.test.ts b/packages/client-runtime/src/voice-input/controller.test.ts index 265a284e20fe3..41e56933ce104 100644 --- a/packages/client-runtime/src/voice-input/controller.test.ts +++ b/packages/client-runtime/src/voice-input/controller.test.ts @@ -309,7 +309,9 @@ describe("VoiceInputController", () => { code: "unavailable", message: "This device requires iOS 26 or later.", }); + const configureRecording = vi.fn(async () => undefined); const unavailable = createHarness({ + configureRecording, getTranscriber: () => ({ prepare: async () => { throw reason; @@ -318,6 +320,8 @@ describe("VoiceInputController", () => { }); await unavailable.controller.start(); expect(unavailable.controller.currentState.error).toBe(reason.message); + // Environment and web transcribers gate capture on consent checked during preparation. + expect(configureRecording).not.toHaveBeenCalled(); expect(unavailable.recorder.record).not.toHaveBeenCalled(); const denied = createHarness({ diff --git a/packages/client-runtime/src/voice-input/controller.ts b/packages/client-runtime/src/voice-input/controller.ts index c67287dd74d36..668705546a489 100644 --- a/packages/client-runtime/src/voice-input/controller.ts +++ b/packages/client-runtime/src/voice-input/controller.ts @@ -236,7 +236,7 @@ export class VoiceInputController { private operationToken = 0; private sessionToken: VoiceInputSession | null = null; private transcription: PreparedVoiceTranscription | null = null; - private liveTranscription: Promise<{ text: string } | { error: unknown }> | null = null; + private liveTranscription: Promise<{ text: string | null } | { error: unknown }> | null = null; private transcriptionAbortController: AbortController | null = null; private postProcessingAbortController: AbortController | null = null; private skipPostProcessingRequested = false; @@ -504,17 +504,15 @@ export class VoiceInputController { const capturedDraft = this.capturedDraft; let transcript: string; try { - if (this.liveTranscription) { - const result = await this.liveTranscription; - if ("error" in result) throw result.error; - transcript = result.text; - } else { - transcript = await runTranscriptionOperation(() => + const live = this.liveTranscription ? await this.liveTranscription : null; + if (live && "error" in live) throw live.error; + transcript = + live?.text ?? + (await runTranscriptionOperation(() => transcription.streaming ? transcription.streaming.finish({ signal }) : transcription.transcribe(recordingUri!, { signal }), - ); - } + )); } catch (error) { if (this.isCurrent(operationToken)) { if ( diff --git a/packages/client-runtime/src/voice-input/transcription.ts b/packages/client-runtime/src/voice-input/transcription.ts index 7af1196c8c0a0..4c674ee9c1c2c 100644 --- a/packages/client-runtime/src/voice-input/transcription.ts +++ b/packages/client-runtime/src/voice-input/transcription.ts @@ -11,7 +11,12 @@ export type VoiceStreamingSession = { export type PreparedVoiceTranscription = { readonly locale: string; - readonly start?: (uri: string, options: VoiceTranscriptionOptions) => Promise; + /** + * Runs alongside capture once recording starts. Resolves with the transcript when it + * transcribed live, or null when `transcribe` still runs after the recording stops. + * A rejection stops the recording. + */ + readonly start?: (uri: string, options: VoiceTranscriptionOptions) => Promise; readonly transcribe: (uri: string, options: VoiceTranscriptionOptions) => Promise; /** Present when audio streams during capture; `finish` then replaces `transcribe`. */ readonly streaming?: VoiceStreamingSession;