From 52641debe35416cb99f099554336f15acb771813 Mon Sep 17 00:00:00 2001 From: RVZO6 <66844656+RVZO6@users.noreply.github.com> Date: Tue, 10 Nov 2026 19:12:51 +0000 Subject: [PATCH 01/12] feat(desktop): dictate on this Mac with Apple Speech macOS desktop users had to download a transcription model to an environment, while iOS already dictates with Apple's on-device SpeechAnalyzer. Add "On this Mac -> Apple Speech" as a per-device transcription destination in the desktop app. A small Swift helper, adapted from the mobile Apple transcription code and built on the macOS release runners, transcribes the recording locally through Electron IPC. Environment models, the server, and mobile do not change. Co-Authored-By: Claude Opus 5.5 --- apps/desktop/scripts/build-apple-speech.mjs | 74 +++++++ apps/desktop/scripts/dev-electron.mjs | 5 + apps/desktop/scripts/start-electron.mjs | 5 + apps/desktop/src/app/DesktopApp.test.ts | 2 + apps/desktop/src/ipc/DesktopIpcHandlers.ts | 8 + apps/desktop/src/ipc/channels.ts | 3 + apps/desktop/src/ipc/methods/appleSpeech.ts | 37 ++++ apps/desktop/src/main.ts | 2 + apps/desktop/src/preload.ts | 7 + apps/desktop/src/speech/AppleSpeech.test.ts | 130 ++++++++++++ apps/desktop/src/speech/AppleSpeech.ts | 189 ++++++++++++++++++ apps/desktop/vite.config.ts | 6 +- .../settings/VoiceSettingsPanel.tsx | 46 ++++- apps/web/src/speech/appleSpeech.ts | 34 ++++ apps/web/src/speech/browserVoiceInput.ts | 51 +++++ ...useEnvironmentSpeechInput.browser.test.tsx | 102 ++++++---- .../src/speech/useEnvironmentSpeechInput.ts | 64 ++++-- apps/web/src/speech/voiceConsent.ts | 10 +- docs/internals/voice-input.md | 6 + docs/user/composer.md | 5 +- docs/user/faq.md | 2 +- docs/user/voice-input.md | 6 + native/apple-speech/AppleSpeech.swift | 125 ++++++++++++ packages/contracts/src/ipc.ts | 23 +++ scripts/build-desktop-artifact.ts | 40 ++++ 25 files changed, 909 insertions(+), 73 deletions(-) create mode 100644 apps/desktop/scripts/build-apple-speech.mjs create mode 100644 apps/desktop/src/ipc/methods/appleSpeech.ts create mode 100644 apps/desktop/src/speech/AppleSpeech.test.ts create mode 100644 apps/desktop/src/speech/AppleSpeech.ts create mode 100644 apps/web/src/speech/appleSpeech.ts create mode 100644 native/apple-speech/AppleSpeech.swift diff --git a/apps/desktop/scripts/build-apple-speech.mjs b/apps/desktop/scripts/build-apple-speech.mjs new file mode 100644 index 0000000000000..efef5a452190f --- /dev/null +++ b/apps/desktop/scripts/build-apple-speech.mjs @@ -0,0 +1,74 @@ +import * as NodeChildProcess from "node:child_process"; +import * as NodeFS from "node:fs"; +import * as NodePath from "node:path"; +import * as NodeURL from "node:url"; +import * as NodeUtil from "node:util"; + +// oxlint-disable-next-line t3code/no-global-process-runtime -- The Swift compiler targets the actual host; this script has no Effect runtime. +const hostArch = process.arch; +// oxlint-disable-next-line t3code/no-global-process-runtime -- The Speech framework exists only in the macOS SDK. +const hostPlatform = process.platform; + +const { values } = NodeUtil.parseArgs({ + options: { output: { type: "string" }, arch: { type: "string", default: hostArch } }, +}); +// Packaging passes --output and must not ship without the helper; dev builds only lose the option. +const required = values.output !== undefined; + +if (hostPlatform !== "darwin") { + if (required) throw new Error("The Apple Speech helper can only be built on macOS."); +} else { + const targets = { arm64: ["arm64"], x64: ["x86_64"], universal: ["arm64", "x86_64"] }[ + values.arch + ]; + if (targets === undefined) throw new Error(`Unsupported macOS architecture: ${values.arch}`); + const root = NodeURL.fileURLToPath(new URL("../../../native/apple-speech/", import.meta.url)); + const source = NodePath.resolve(root, "AppleSpeech.swift"); + const output = values.output ?? NodePath.resolve(root, "build", values.arch, "t3-apple-speech"); + let current = false; + try { + current = + NodeFS.statSync(output).mtimeMs >= + Math.max( + NodeFS.statSync(source).mtimeMs, + NodeFS.statSync(NodeURL.fileURLToPath(import.meta.url)).mtimeMs, + ); + } catch {} + if (!current) { + NodeFS.mkdirSync(NodePath.dirname(output), { recursive: true }); + const temporary = `${output}.${process.pid}.tmp`; + const slices = targets.map((target) => `${temporary}.${target}`); + try { + targets.forEach((target, index) => + NodeChildProcess.execFileSync( + "xcrun", + [ + "swiftc", + "-O", + "-swift-version", + "5", + "-parse-as-library", + "-target", + `${target}-apple-macos26.0`, + source, + "-o", + slices[index], + ], + { stdio: "inherit" }, + ), + ); + NodeChildProcess.execFileSync("xcrun", ["lipo", "-create", ...slices, "-output", temporary], { + stdio: "inherit", + }); + NodeFS.renameSync(temporary, output); + } catch (cause) { + if (required) + throw new Error("Building the Apple Speech helper requires Xcode 26 or later.", { cause }); + console.warn( + "Skipping the Apple Speech helper (needs Xcode 26 or later); on-device Mac dictation is unavailable in this build.", + ); + } finally { + for (const file of [temporary, ...slices]) NodeFS.rmSync(file, { force: true }); + } + } +} diff --git a/apps/desktop/scripts/dev-electron.mjs b/apps/desktop/scripts/dev-electron.mjs index 07578297f06d7..f8bdfccf2027e 100644 --- a/apps/desktop/scripts/dev-electron.mjs +++ b/apps/desktop/scripts/dev-electron.mjs @@ -58,6 +58,11 @@ NodeChildProcess.execFileSync( [NodePath.join(desktopDir, "scripts/build-browser-secret.mjs")], { stdio: "inherit" }, ); +NodeChildProcess.execFileSync( + process.execPath, + [NodePath.join(desktopDir, "scripts/build-apple-speech.mjs")], + { stdio: "inherit" }, +); await waitForResources({ baseDir: desktopDir, diff --git a/apps/desktop/scripts/start-electron.mjs b/apps/desktop/scripts/start-electron.mjs index 5dde034121b8e..8cabe3e5069fe 100644 --- a/apps/desktop/scripts/start-electron.mjs +++ b/apps/desktop/scripts/start-electron.mjs @@ -8,6 +8,11 @@ NodeChildProcess.execFileSync( [NodePath.join(desktopDir, "scripts/build-browser-secret.mjs")], { stdio: "inherit" }, ); +NodeChildProcess.execFileSync( + process.execPath, + [NodePath.join(desktopDir, "scripts/build-apple-speech.mjs")], + { stdio: "inherit" }, +); const childEnv = { ...process.env }; delete childEnv.ELECTRON_RUN_AS_NODE; diff --git a/apps/desktop/src/app/DesktopApp.test.ts b/apps/desktop/src/app/DesktopApp.test.ts index 6cb00baf897d1..a929253bc5b10 100644 --- a/apps/desktop/src/app/DesktopApp.test.ts +++ b/apps/desktop/src/app/DesktopApp.test.ts @@ -69,6 +69,7 @@ import * as ElectronTheme from "../electron/ElectronTheme.ts"; import * as ElectronWindow from "../electron/ElectronWindow.ts"; import * as MacPermissions from "../permissions/MacPermissions.ts"; import * as PreviewManager from "../preview/Manager.ts"; +import * as AppleSpeech from "../speech/AppleSpeech.ts"; it.effect.each([false, true])( "stops backends together before flushing history and cancels Linux registration, stop defect=%s", @@ -230,6 +231,7 @@ it.effect.each([false, true])( Layer.mock(ElectronTheme.ElectronTheme, {}), Layer.mock(ElectronWindow.ElectronWindow, {}), Layer.mock(MacPermissions.MacPermissions, {}), + Layer.mock(AppleSpeech.AppleSpeech, {}), Layer.mock(PreviewManager.PreviewManager, { isBrowserPartition: () => { throw new Error("unexpected browser partition lookup"); diff --git a/apps/desktop/src/ipc/DesktopIpcHandlers.ts b/apps/desktop/src/ipc/DesktopIpcHandlers.ts index e3136977098f2..049bfc1c2c792 100644 --- a/apps/desktop/src/ipc/DesktopIpcHandlers.ts +++ b/apps/desktop/src/ipc/DesktopIpcHandlers.ts @@ -1,6 +1,11 @@ import * as Effect from "effect/Effect"; import { receiveProviderAuthCallback, cancelProviderAuthCallback } from "./methods/providerAuth.ts"; +import { + appleSpeechAvailable, + appleSpeechPrepare, + appleSpeechTranscribe, +} from "./methods/appleSpeech.ts"; import * as DesktopIpc from "./DesktopIpc.ts"; import { installNotificationBadge } from "./methods/notificationBadge.ts"; import { getClientSettings, setClientSettings } from "./methods/clientSettings.ts"; @@ -152,6 +157,9 @@ export const installDesktopIpcHandlers = Effect.fn("desktop.ipc.installHandlers" yield* ipc.handle(cancelProviderAuthCallback); yield* ipc.handle(openSystemSettings); yield* ipc.handle(checkSystemPermission); + yield* ipc.handle(appleSpeechAvailable); + yield* ipc.handle(appleSpeechPrepare); + yield* ipc.handle(appleSpeechTranscribe); yield* ipc.handle(pasteAsText); yield* ipc.handle(revealWindow); yield* ipc.handle(probeRemoteEditors); diff --git a/apps/desktop/src/ipc/channels.ts b/apps/desktop/src/ipc/channels.ts index 4f5117c21cded..4a7f0c5ccfe2b 100644 --- a/apps/desktop/src/ipc/channels.ts +++ b/apps/desktop/src/ipc/channels.ts @@ -121,6 +121,9 @@ export const PREVIEW_OPEN_LINK_CHANNEL = "desktop:preview-open-link"; export const MAC_PERMISSION_HELPER_CHANNEL = "desktop:mac-permission-helper"; export const CHECK_SYSTEM_PERMISSION_CHANNEL = "desktop:check-system-permission"; +export const APPLE_SPEECH_AVAILABLE_CHANNEL = "desktop:apple-speech-available"; +export const APPLE_SPEECH_PREPARE_CHANNEL = "desktop:apple-speech-prepare"; +export const APPLE_SPEECH_TRANSCRIBE_CHANNEL = "desktop:apple-speech-transcribe"; export const PREVIEW_RECORDING_INPUT_CHANNEL = "desktop:preview-recording-input"; diff --git a/apps/desktop/src/ipc/methods/appleSpeech.ts b/apps/desktop/src/ipc/methods/appleSpeech.ts new file mode 100644 index 0000000000000..8396db6f29b3d --- /dev/null +++ b/apps/desktop/src/ipc/methods/appleSpeech.ts @@ -0,0 +1,37 @@ +import { + DesktopAppleSpeechResultSchema, + DesktopAppleSpeechTranscribeInputSchema, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; + +import * as AppleSpeech from "../../speech/AppleSpeech.ts"; +import * as IpcChannels from "../channels.ts"; +import * as DesktopIpc from "../DesktopIpc.ts"; + +export const appleSpeechAvailable = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.APPLE_SPEECH_AVAILABLE_CHANNEL, + payload: Schema.Void, + result: Schema.Boolean, + handler: Effect.fn("desktop.ipc.appleSpeech.available")(function* () { + return yield* (yield* AppleSpeech.AppleSpeech).isAvailable; + }), +}); + +export const appleSpeechPrepare = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.APPLE_SPEECH_PREPARE_CHANNEL, + payload: Schema.String.check(Schema.isMaxLength(64)), + result: DesktopAppleSpeechResultSchema, + handler: Effect.fn("desktop.ipc.appleSpeech.prepare")(function* (locale) { + return yield* (yield* AppleSpeech.AppleSpeech).prepare(locale); + }), +}); + +export const appleSpeechTranscribe = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.APPLE_SPEECH_TRANSCRIBE_CHANNEL, + payload: DesktopAppleSpeechTranscribeInputSchema, + result: DesktopAppleSpeechResultSchema, + handler: Effect.fn("desktop.ipc.appleSpeech.transcribe")(function* ({ locale, pcm }) { + return yield* (yield* AppleSpeech.AppleSpeech).transcribe(locale, pcm); + }), +}); diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.ts index d109518be15e4..c1d080d756d9d 100644 --- a/apps/desktop/src/main.ts +++ b/apps/desktop/src/main.ts @@ -71,6 +71,7 @@ import * as DesktopUpdatePolicy from "./updates/DesktopUpdatePolicy.ts"; import * as DesktopUpdates from "./updates/DesktopUpdates.ts"; import * as BrowserImport from "./preview/BrowserImport/BrowserImport.ts"; import * as LinuxBrowserSecret from "./preview/BrowserImport/LinuxBrowserSecret.ts"; +import * as AppleSpeech from "./speech/AppleSpeech.ts"; import * as BrowserSession from "./preview/BrowserSession.ts"; import * as DesktopBrowserHost from "./preview/DesktopBrowserHost.ts"; import * as PreviewManager from "./preview/Manager.ts"; @@ -154,6 +155,7 @@ const layerDesktopFoundation = Layer.mergeAll( DesktopAssets.layer, DesktopObservability.layer, DesktopRendererHistory.layer, + AppleSpeech.layer, ).pipe(Layer.provideMerge(layerDesktopEnvironment)); const layerDesktopSsh = layerDesktopSshEnvironment.pipe( diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index 5d14ba07140a5..8c989db71d8e9 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -213,6 +213,13 @@ contextBridge.exposeInMainWorld("desktopBridge", { readMedia: (url: string) => ipcRenderer.invoke(IpcChannels.READ_MEDIA_CHANNEL, url), checkSystemPermission: (pane: string) => ipcRenderer.invoke(IpcChannels.CHECK_SYSTEM_PERMISSION_CHANNEL, pane), + appleSpeech: { + isAvailable: () => ipcRenderer.invoke(IpcChannels.APPLE_SPEECH_AVAILABLE_CHANNEL, undefined), + prepare: (locale: string) => + ipcRenderer.invoke(IpcChannels.APPLE_SPEECH_PREPARE_CHANNEL, locale), + transcribe: (input: { locale: string; pcm: Uint8Array }) => + ipcRenderer.invoke(IpcChannels.APPLE_SPEECH_TRANSCRIBE_CHANNEL, input), + }, openSystemSettings: (pane: string) => ipcRenderer.invoke(IpcChannels.OPEN_SYSTEM_SETTINGS_CHANNEL, pane), probeRemoteEditors: () => ipcRenderer.invoke(IpcChannels.PROBE_REMOTE_EDITORS_CHANNEL, undefined), diff --git a/apps/desktop/src/speech/AppleSpeech.test.ts b/apps/desktop/src/speech/AppleSpeech.test.ts new file mode 100644 index 0000000000000..069017d80e5cd --- /dev/null +++ b/apps/desktop/src/speech/AppleSpeech.test.ts @@ -0,0 +1,130 @@ +import { assert, describe, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Sink from "effect/Sink"; +import * as Stream from "effect/Stream"; +import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; + +import * as DesktopEnvironment from "../app/DesktopEnvironment.ts"; +import * as AppleSpeech from "./AppleSpeech.ts"; + +const encoder = new TextEncoder(); +const text = (value: string) => (value ? Stream.make(encoder.encode(value)) : Stream.empty); + +function makeProcess(result: { stdout?: string; stderr?: string; exitCode: number }) { + return ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(1), + stdout: text(result.stdout ?? ""), + stderr: text(result.stderr ?? ""), + all: Stream.empty, + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(result.exitCode)), + isRunning: Effect.succeed(false), + kill: () => Effect.void, + stdin: Sink.drain, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + unref: Effect.succeed(Effect.void), + }); +} + +function setup(result: { stdout?: string; stderr?: string; exitCode: number }) { + const commands: Array> = []; + const written: Uint8Array[] = []; + const layer = AppleSpeech.layer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.succeed(DesktopEnvironment.DesktopEnvironment, { + platform: "darwin", + isPackaged: true, + resourcesPath: "/app/Resources", + path: { join: (...parts: string[]) => parts.join("/") }, + } as unknown as DesktopEnvironment.DesktopEnvironment["Service"]), + FileSystem.layerNoop({ + exists: () => Effect.succeed(true), + makeTempFileScoped: () => Effect.succeed("/tmp/recording.wav"), + writeFile: (_path, data) => Effect.sync(() => void written.push(data)), + }), + Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make((command) => { + const { command: executable, args } = command as unknown as { + command: string; + args: ReadonlyArray; + }; + commands.push([executable, ...args]); + return Effect.succeed(makeProcess(result)); + }), + ), + ), + ), + ); + return { commands, written, layer }; +} + +describe("AppleSpeech", () => { + it.effect.each([ + [ + { stdout: '{"value":"en-US"}', exitCode: 0 }, + { ok: true, value: "en-US" }, + ], + [ + { stderr: "Locale not supported: xx\n", exitCode: 2 }, + { ok: false, code: "unsupported-locale", message: "Locale not supported: xx" }, + ], + [{ exitCode: 3 }, { ok: false, code: "unavailable" }], + [ + { stderr: "boom", exitCode: 1 }, + { ok: false, code: "failed", message: "boom" }, + ], + [ + { stdout: "not json", exitCode: 0 }, + { ok: false, code: "failed" }, + ], + ] as const)("maps helper output %j to %j", ([output, expected]) => { + const { layer, commands } = setup(output); + return Effect.gen(function* () { + const result = yield* (yield* AppleSpeech.AppleSpeech).prepare("en-US"); + assert.deepInclude(result, expected); + assert.deepEqual(commands, [ + ["/app/Resources/apple-speech/t3-apple-speech", "prepare", "en-US"], + ]); + }).pipe(Effect.provide(layer)); + }); + + it.effect("transcribes Float32 PCM through a 16-bit WAV file", () => { + const { layer, commands, written } = setup({ stdout: '{"value":"hello"}', exitCode: 0 }); + const pcm = new Uint8Array(new Float32Array([0, 1, -1, 2]).buffer); + return Effect.gen(function* () { + const result = yield* (yield* AppleSpeech.AppleSpeech).transcribe("en-US", pcm); + assert.deepEqual(result, { ok: true, value: "hello" }); + assert.deepEqual(commands, [ + [ + "/app/Resources/apple-speech/t3-apple-speech", + "transcribe", + "en-US", + "/tmp/recording.wav", + ], + ]); + const wav = new DataView(written[0]!.buffer); + assert.equal(new TextDecoder().decode(written[0]!.subarray(0, 4)), "RIFF"); + assert.equal(wav.getUint16(20, true), 1); + assert.equal(wav.getUint32(24, true), 16_000); + assert.equal(wav.getUint16(34, true), 16); + assert.equal(wav.getUint32(40, true), 8); + assert.deepEqual( + [0, 1, 2, 3].map((index) => wav.getInt16(44 + index * 2, true)), + [0, 32767, -32767, 32767], + ); + }).pipe(Effect.provide(layer)); + }); + + it.effect("rejects PCM that is not whole Float32 samples without running the helper", () => { + const { layer, commands } = setup({ exitCode: 0 }); + return Effect.gen(function* () { + const result = yield* (yield* AppleSpeech.AppleSpeech).transcribe("en-US", new Uint8Array(3)); + assert.deepInclude(result, { ok: false, code: "failed" }); + assert.lengthOf(commands, 0); + }).pipe(Effect.provide(layer)); + }); +}); diff --git a/apps/desktop/src/speech/AppleSpeech.ts b/apps/desktop/src/speech/AppleSpeech.ts new file mode 100644 index 0000000000000..cf79b8b4b4f0b --- /dev/null +++ b/apps/desktop/src/speech/AppleSpeech.ts @@ -0,0 +1,189 @@ +import type { DesktopAppleSpeechResult } from "@t3tools/contracts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Context from "effect/Context"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; + +import * as DesktopEnvironment from "../app/DesktopEnvironment.ts"; + +const SAMPLE_RATE = 16_000; +const MAX_PCM_BYTES = SAMPLE_RATE * Float32Array.BYTES_PER_ELEMENT * 5 * 60; +// The first prepare for a language can download Apple's model. +const PREPARE_TIMEOUT = Duration.minutes(10); +const TRANSCRIBE_TIMEOUT = Duration.minutes(2); + +const decodeOutput = Schema.decodeUnknownEffect( + Schema.fromJsonString(Schema.Struct({ value: Schema.String })), +); + +/** 16 kHz mono Float32 PCM to a 16-bit WAV that AVAudioFile reads. */ +function encodeSpeechWav(pcmBytes: Uint8Array): Uint8Array { + const samples = new Float32Array( + pcmBytes.buffer, + pcmBytes.byteOffset, + pcmBytes.byteLength / Float32Array.BYTES_PER_ELEMENT, + ); + const wav = new Uint8Array(44 + samples.length * 2); + const view = new DataView(wav.buffer); + const ascii = (offset: number, text: string) => { + for (let index = 0; index < text.length; index++) + view.setUint8(offset + index, text.charCodeAt(index)); + }; + ascii(0, "RIFF"); + view.setUint32(4, 36 + samples.length * 2, true); + ascii(8, "WAVE"); + ascii(12, "fmt "); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); + view.setUint16(22, 1, true); + view.setUint32(24, SAMPLE_RATE, true); + view.setUint32(28, SAMPLE_RATE * 2, true); + view.setUint16(32, 2, true); + view.setUint16(34, 16, true); + ascii(36, "data"); + view.setUint32(40, samples.length * 2, true); + samples.forEach((sample, index) => { + const clamped = Number.isFinite(sample) ? Math.max(-1, Math.min(1, sample)) : 0; + view.setInt16(44 + index * 2, Math.round(clamped * 0x7fff), true); + }); + return wav; +} + +export class AppleSpeech extends Context.Service< + AppleSpeech, + { + readonly isAvailable: Effect.Effect; + readonly prepare: (locale: string) => Effect.Effect; + readonly transcribe: ( + locale: string, + pcm: Uint8Array, + ) => Effect.Effect; + } +>()("@t3tools/desktop/speech/AppleSpeech") {} + +const unavailable: DesktopAppleSpeechResult = { + ok: false, + code: "unavailable", + message: "Apple Speech needs the T3 Code desktop app on macOS 26 or later.", +}; + +export const layer = Layer.effect( + AppleSpeech, + Effect.gen(function* () { + const environment = yield* DesktopEnvironment.DesktopEnvironment; + const fileSystem = yield* FileSystem.FileSystem; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const hostEnvironment = yield* HostProcess.Environment; + + let helper: string | undefined; + if (environment.platform === "darwin") { + const relative = environment.path.join("apple-speech", "t3-apple-speech"); + const candidates = environment.isPackaged + ? [environment.path.join(environment.resourcesPath, relative)] + : [ + environment.path.join( + environment.rootDir, + "native", + "apple-speech", + "build", + environment.processArch, + "t3-apple-speech", + ), + ]; + for (const candidate of candidates) { + if (yield* fileSystem.exists(candidate).pipe(Effect.orElseSucceed(() => false))) { + helper = candidate; + break; + } + } + } + + const run = (args: readonly string[], timeout: Duration.Input) => + Effect.scoped( + Effect.gen(function* () { + if (helper === undefined) return unavailable; + const handle = yield* spawner.spawn( + ChildProcess.make(helper, args, { stdin: "ignore", env: hostEnvironment }), + ); + const [stdout, stderr, exitCode] = yield* Effect.all( + [ + handle.stdout.pipe(Stream.decodeText(), Stream.mkString), + handle.stderr.pipe(Stream.decodeText(), Stream.mkString), + handle.exitCode, + ], + { concurrency: "unbounded" }, + ); + const message = stderr.trim() || "Apple Speech failed."; + switch (Number(exitCode)) { + case 0: + return { ok: true, value: (yield* decodeOutput(stdout)).value } as const; + case 2: + return { ok: false, code: "unsupported-locale", message } as const; + case 3: + return unavailable; + default: + return { ok: false, code: "failed", message } as const; + } + }), + ).pipe( + // Leaving the scope kills a helper that outlives its timeout. + Effect.timeoutOrElse({ + duration: timeout, + orElse: () => + Effect.succeed({ + ok: false, + code: "failed", + message: "Apple Speech did not respond.", + }), + }), + Effect.catch((cause) => + Effect.succeed({ + ok: false, + code: "failed", + message: cause instanceof Error ? cause.message : "Apple Speech failed.", + }), + ), + ); + + const isAvailable = yield* Effect.cached( + run(["available"], Duration.seconds(10)).pipe(Effect.map((result) => result.ok)), + ); + + return AppleSpeech.of({ + isAvailable, + prepare: Effect.fn("AppleSpeech.prepare")(function* (locale) { + return yield* run(["prepare", locale], PREPARE_TIMEOUT); + }), + transcribe: Effect.fn("AppleSpeech.transcribe")(function* (locale, pcm) { + if ( + pcm.byteLength === 0 || + pcm.byteLength > MAX_PCM_BYTES || + pcm.byteLength % Float32Array.BYTES_PER_ELEMENT !== 0 + ) + return { + ok: false, + code: "failed", + message: "The recording is empty or too long.", + } as const; + return yield* Effect.scoped( + Effect.gen(function* () { + const file = yield* fileSystem.makeTempFileScoped({ suffix: ".wav" }); + yield* fileSystem.writeFile(file, encodeSpeechWav(pcm)); + return yield* run(["transcribe", locale, file], TRANSCRIBE_TIMEOUT); + }), + ).pipe( + Effect.orElseSucceed((): DesktopAppleSpeechResult => ({ + ok: false, + code: "failed", + message: "Could not save the recording for Apple Speech.", + })), + ); + }), + }); + }), +); diff --git a/apps/desktop/vite.config.ts b/apps/desktop/vite.config.ts index ba37576815097..3ee1985d24af2 100644 --- a/apps/desktop/vite.config.ts +++ b/apps/desktop/vite.config.ts @@ -21,7 +21,7 @@ export default defineConfig({ tasks: { build: { command: - "node scripts/build-browser-secret.mjs && node scripts/build-preview-annotation-css.mjs && vp pack", + "node scripts/build-browser-secret.mjs && node scripts/build-apple-speech.mjs && node scripts/build-preview-annotation-css.mjs && vp pack", dependsOn: ["t3#build"], cache: false, }, @@ -33,13 +33,13 @@ export default defineConfig({ }, dev: { command: - "node scripts/build-browser-secret.mjs && node scripts/build-preview-annotation-css.mjs && cross-env T3CODE_DESKTOP_DEV=1 vp pack --watch", + "node scripts/build-browser-secret.mjs && node scripts/build-apple-speech.mjs && node scripts/build-preview-annotation-css.mjs && cross-env T3CODE_DESKTOP_DEV=1 vp pack --watch", dependsOn: ["t3#build"], cache: false, }, "dev:bundle": { command: - "node scripts/build-browser-secret.mjs && node scripts/build-preview-annotation-css.mjs && vp pack --watch", + "node scripts/build-browser-secret.mjs && node scripts/build-apple-speech.mjs && node scripts/build-preview-annotation-css.mjs && vp pack --watch", cache: false, }, "dev:electron": { diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.tsx index f26ead5df71ac..e3a3735d40807 100644 --- a/apps/web/src/components/settings/VoiceSettingsPanel.tsx +++ b/apps/web/src/components/settings/VoiceSettingsPanel.tsx @@ -34,6 +34,7 @@ import { DownloadIcon, LanguagesIcon, KeyRoundIcon, + LaptopIcon, MonitorIcon, MicOffIcon, MicIcon, @@ -46,6 +47,7 @@ import { import { useEffect, useRef, useState, type ReactNode } from "react"; import { useStableCallback } from "~/hooks/useStableCallback"; +import { useAppleSpeechAvailable } from "../../speech/appleSpeech"; import { listBrowserMicrophones, type BrowserMicrophone } from "../../speech/browserMicrophones"; import { preferredSpeechLanguages } from "../../speech/preferredSpeechLanguages"; import { @@ -120,6 +122,7 @@ import { searchableSetting } from "./settingsSearch"; import { useOptionalSettingsScope, useSettingsScope } from "./SettingsScopeContext"; import { useScopedModelDisabledReason } from "./useScopedModelAvailability"; import { useScopedSettings, useUpdateScopedSettings } from "./useScopedSettings"; +const DEVICE = "device"; const OFF = "off"; function CloudSpeechLogo({ provider }: { provider: CloudSpeechProvider }) { @@ -1095,6 +1098,7 @@ export function VoiceSettingsPanel() { const settings = useClientSettings(); const settingsScope = useOptionalSettingsScope(); const destination = useVoiceTranscriptionDestination(); + const appleSpeechAvailable = useAppleSpeechAvailable(); const canManageKey = useEnvironmentScope(destination.environmentId, AuthSettingsWriteScope); const [pendingCloud, setPendingCloud] = useState<{ provider: CloudSpeechProvider; @@ -1152,11 +1156,13 @@ export function VoiceSettingsPanel() { })), })); const cloudChoices = cloudGroups.flatMap((group) => group.choices); - const destinationValue = destination.cloudProvider - ? JSON.stringify(["cloud", destination.environmentId, destination.cloudProvider]) - : destination.environmentId - ? JSON.stringify(["environment", destination.environmentId]) - : OFF; + const destinationValue = destination.device + ? DEVICE + : destination.cloudProvider + ? JSON.stringify(["cloud", destination.environmentId, destination.cloudProvider]) + : destination.environmentId + ? JSON.stringify(["environment", destination.environmentId]) + : OFF; const cloudEnvironment = environments.find( (env) => env.environmentId === pendingCloud?.environmentId, ); @@ -1288,6 +1294,7 @@ export function VoiceSettingsPanel() { setPendingCloud(null); const chosen = environmentChoices.find((choice) => choice.value === value); if (value === OFF) chooseAudioDestination(null); + else if (value === DEVICE) chooseAudioDestination({ kind: "device" }); else if (chosen) chooseAudioDestination({ kind: "environment", @@ -1302,7 +1309,9 @@ export function VoiceSettingsPanel() { > - {destination.cloudProvider ? ( + {destination.device ? ( + + ) : destination.cloudProvider ? ( ) : destination.environmentId ? ( @@ -1310,11 +1319,15 @@ export function VoiceSettingsPanel() { )} - {destination.cloudProvider - ? CLOUD_SPEECH_LABELS[destination.cloudProvider] - : destination.environmentId - ? (chosenEnvironment?.label ?? "Chosen environment (unavailable)") - : "Off"} + {destination.device + ? appleSpeechAvailable + ? "Apple Speech" + : "Apple Speech (unavailable)" + : destination.cloudProvider + ? CLOUD_SPEECH_LABELS[destination.cloudProvider] + : destination.environmentId + ? (chosenEnvironment?.label ?? "Chosen environment (unavailable)") + : "Off"} {destination.cloudProvider && ( {" via "} @@ -1332,6 +1345,17 @@ export function VoiceSettingsPanel() { Off + {(appleSpeechAvailable || destination.device) && ( + + On this Mac + + + + {appleSpeechAvailable ? "Apple Speech" : "Apple Speech (unavailable)"} + + + + )} {environmentChoices.length > 0 && ( On an environment diff --git a/apps/web/src/speech/appleSpeech.ts b/apps/web/src/speech/appleSpeech.ts new file mode 100644 index 0000000000000..60475500e50fe --- /dev/null +++ b/apps/web/src/speech/appleSpeech.ts @@ -0,0 +1,34 @@ +import { useSyncExternalStore } from "react"; + +let available = false; +let probe: Promise | undefined; +const listeners = new Set<() => void>(); + +function subscribe(listener: () => void) { + listeners.add(listener); + probe ??= (window.desktopBridge?.appleSpeech?.isAvailable() ?? Promise.resolve(false)) + .catch(() => false) + .then((value) => { + available = value; + listeners.forEach((notify) => notify()); + }); + return () => listeners.delete(listener); +} + +/** Apple Speech runs only in the macOS 26 desktop app; the main process probes it once. */ +export function useAppleSpeechAvailable() { + return useSyncExternalStore( + subscribe, + () => available, + () => false, + ); +} + +/** The packaged app pins `navigator.language` to en-US, so "auto" asks the OS. */ +export function appleSpeechLocale(speechLanguage: string) { + if (speechLanguage !== "auto") return speechLanguage; + const bridge = window.desktopBridge; + return ( + bridge?.getPreferredSystemLanguages?.()[0] ?? bridge?.getSystemLocale?.() ?? navigator.language + ); +} diff --git a/apps/web/src/speech/browserVoiceInput.ts b/apps/web/src/speech/browserVoiceInput.ts index 345ebf78f2bb4..2fbacc1be5dee 100644 --- a/apps/web/src/speech/browserVoiceInput.ts +++ b/apps/web/src/speech/browserVoiceInput.ts @@ -15,6 +15,7 @@ import type { SpeechTranscriptionOptions, SpeechStreamText } from "@t3tools/cont import type { CloudSpeechModelId, CloudSpeechProvider } from "@t3tools/contracts"; import * as Schema from "effect/Schema"; +import { appleSpeechLocale } from "./appleSpeech"; import { createBrowserPcmCapture } from "./browserPcmCapture"; import { runEnvironmentSpeech } from "./runEnvironmentSpeech"; @@ -125,3 +126,53 @@ export function createBrowserVoiceInputPlatform(input: { }; }); } + +/** Records like the environment path, then transcribes in the desktop main process; audio stays on this Mac. */ +export function createAppleSpeechVoiceInputPlatform(input: { + readonly hasAudioConsent: () => boolean; + readonly getTranscriptionOptions: () => SpeechTranscriptionOptions; + readonly getMicrophoneId: () => string; + readonly onLevel: (level: number) => void; + readonly onDurationLimit: () => void; + readonly onError: (message: string) => void; +}) { + return createBrowserPcmCapture(input, async ({ signal }) => { + const appleSpeech = window.desktopBridge?.appleSpeech; + if (!appleSpeech) + throw new VoiceTranscriptionError({ + code: "unavailable", + message: "Apple Speech needs the T3 Code desktop app on macOS 26 or later.", + }); + const prepared = await appleSpeech.prepare( + appleSpeechLocale(input.getTranscriptionOptions().speechLanguage), + ); + throwIfVoiceTranscriptionAborted(signal); + if (!prepared.ok) + throw new VoiceTranscriptionError({ + code: prepared.code === "failed" ? "preparation-failed" : prepared.code, + message: prepared.message, + }); + const locale = prepared.value; + return { + locale, + transcribe: async (uri, { signal: transcriptionSignal }) => { + const response = await fetch(uri, { signal: transcriptionSignal }); + const pcm = new Uint8Array(await response.arrayBuffer()); + throwIfVoiceTranscriptionAborted(transcriptionSignal); + if (!input.hasAudioConsent()) + throw new VoiceTranscriptionError({ + code: "cancelled", + message: "Voice transcription was cancelled.", + }); + const result = await appleSpeech.transcribe({ locale, pcm }); + throwIfVoiceTranscriptionAborted(transcriptionSignal); + if (!result.ok) + throw new VoiceTranscriptionError({ + code: "transcription-failed", + message: result.message, + }); + return result.value; + }, + }; + }); +} diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx index 508ef9aa7b605..e75d272df16ac 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx +++ b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx @@ -58,6 +58,7 @@ const mocks = vi.hoisted(() => ({ transcripts: [] as Array, committed: vi.fn(), draft: "", + appleSpeech: false, })); vi.mock("../state/session", () => ({ @@ -161,44 +162,51 @@ vi.mock("@t3tools/client-runtime/voice-input", async (importOriginal) => ({ return { text: `Cleaned ${transcript}` }; }), })); +vi.mock("./appleSpeech", () => ({ useAppleSpeechAvailable: () => mocks.appleSpeech })); vi.mock("./browserVoiceInput", () => ({ - createBrowserVoiceInputPlatform: ( - input: Parameters[0], - ) => { - const environmentId = input.prepared.environmentId; - mocks.platforms.push(environmentId); - return { - recorder: { - uri: `blob:${environmentId}`, - prepareToRecordAsync: async () => { - mocks.microphoneRequests.push(environmentId); - }, - record() {}, - stop: async () => {}, + createAppleSpeechVoiceInputPlatform: ( + input: Omit[0], "prepared">, + ) => mockPlatform("apple-speech", input), + createBrowserVoiceInputPlatform: (input: Parameters[0]) => + mockPlatform(input.prepared.environmentId, input), +})); + +function mockPlatform( + environmentId: string, + input: Omit[0], "prepared">, +) { + mocks.platforms.push(environmentId); + return { + recorder: { + uri: `blob:${environmentId}`, + prepareToRecordAsync: async () => { + mocks.microphoneRequests.push(environmentId); }, - transcriber: { - prepare: async () => { - mocks.recordingOptions.push(input.getTranscriptionOptions()); - return { - locale: "en", - transcribe: async () => { - mocks.transcribeCalls.push( - input.cloudProvider - ? `${[input.cloudProvider, (input as { cloudModel?: string }).cloudModel].filter(Boolean).join(":")} via ${environmentId}` - : environmentId, - ); - const next = mocks.transcripts.shift() ?? "dictated"; - if (next instanceof Error) throw next; - return next; - }, - }; - }, + record() {}, + stop: async () => {}, + }, + transcriber: { + prepare: async () => { + mocks.recordingOptions.push(input.getTranscriptionOptions()); + return { + locale: "en", + transcribe: async () => { + mocks.transcribeCalls.push( + input.cloudProvider + ? `${[input.cloudProvider, (input as { cloudModel?: string }).cloudModel].filter(Boolean).join(":")} via ${environmentId}` + : environmentId, + ); + const next = mocks.transcripts.shift() ?? "dictated"; + if (next instanceof Error) throw next; + return next; + }, + }; }, - cancelRecording() {}, - deleteRecording() {}, - }; - }, -})); + }, + cancelRecording() {}, + deleteRecording() {}, + }; +} const THREAD_ENV = "thread-env"; @@ -233,6 +241,7 @@ function setup(options: { readonly destination?: string | null } = {}) { mocks.transcripts = []; mocks.committed.mockClear(); mocks.draft = ""; + mocks.appleSpeech = false; } function setDestination(environmentId: string | null) { @@ -361,6 +370,29 @@ it.each([ expect(mocks.committed).not.toHaveBeenCalled(); }); +it.each([true, false])( + "transcribes on this Mac with Apple Speech, never an environment, when available is %s", + async (available) => { + setup(); + mocks.appleSpeech = available; + chooseAudioDestination({ kind: "device" }); + await using _screen = await render(); + expect(mocks.voice.transcriptionEnvironmentId).toBeNull(); + expect(mocks.voice.available).toBe(available); + await act(() => mocks.voice.start()); + if (!available) { + expectNoAudioSent(); + return; + } + await act(() => mocks.voice.stop()); + expect(new Set([...mocks.platforms, ...mocks.transcribeCalls])).toEqual( + new Set(["apple-speech"]), + ); + expect(mocks.statusRequests).toEqual([]); + expect(mocks.committed).toHaveBeenCalledExactlyOnceWith("dictated", expect.anything()); + }, +); + it("retries a failed transcription on the same chosen environment", async () => { setup({ destination: "env-a" }); mocks.transcripts = [new Error("inference failed"), "second try"]; diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.ts b/apps/web/src/speech/useEnvironmentSpeechInput.ts index c569c14500be6..74d9496a358cc 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.ts +++ b/apps/web/src/speech/useEnvironmentSpeechInput.ts @@ -37,7 +37,11 @@ import { } from "../hooks/useSettings"; import { useEnvironment } from "../state/environments"; import { useEnvironmentScope, usePreparedConnection } from "../state/session"; -import { createBrowserVoiceInputPlatform } from "./browserVoiceInput"; +import { useAppleSpeechAvailable } from "./appleSpeech"; +import { + createAppleSpeechVoiceInputPlatform, + createBrowserVoiceInputPlatform, +} from "./browserVoiceInput"; import { preferredSpeechLanguages } from "./preferredSpeechLanguages"; import { runEnvironmentSpeech } from "./runEnvironmentSpeech"; import { bindAudioGrant, readCleanupGrant, useAudioChoice } from "./voiceConsent"; @@ -111,12 +115,18 @@ type HookInput = { * Microphone audio leaves this device only for the environment the person chose in * Voice settings. Nothing is inferred from the primary or the draft's environment, and * the recording is abandoned the moment that choice changes or its connection drops. + * Apple Speech ("device") keeps audio on this Mac and needs no environment. */ export function useVoiceTranscriptionDestination() { const hydrated = useClientSettingsHydrated(); const choice = useAudioChoice(); const grant = choice === "off" ? null : choice; - const environmentId = hydrated && grant ? grant.destination.environmentId : null; + const appleSpeechAvailable = useAppleSpeechAvailable(); + const device = hydrated && grant?.destination.kind === "device"; + const environmentId = + hydrated && grant && grant.destination.kind !== "device" + ? grant.destination.environmentId + : null; const environment = useEnvironment(environmentId); const connection = Option.getOrNull(usePreparedConnection(environmentId)); const canDictate = useEnvironmentScope(environmentId, AuthOrchestrationOperateScope); @@ -130,11 +140,13 @@ export function useVoiceTranscriptionDestination() { return { ...(hydrated && choice === null ? { offerFirstRunSetup: true } : {}), environmentId, + ...(device ? { device: true, onDevice: appleSpeechAvailable } : {}), ...(provider ? { cloudProvider: provider } : {}), ...(model ? { cloudModel: model } : {}), grantId: grant?.id ?? null, - label: - provider && environment + label: device + ? "Apple Speech" + : provider && environment ? `${CLOUD_SPEECH_LABELS[provider]} via ${environment.label}` : (environment?.label ?? null), prepared: supported && canDictate ? connection : null, @@ -145,6 +157,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { const navigate = useNavigate(); const destination = useVoiceTranscriptionDestination(); const prepared = destination.prepared; + const onDevice = destination.onDevice === true; const grantId = destination.grantId; const cloudProvider = destination.cloudProvider; const cloudModel = destination.cloudModel; @@ -226,7 +239,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- owns the recording controller bound to the consented destination; it is disposed with the consent useEffect(() => { - if (!prepared || !grantId || typeof navigator === "undefined") return; + if ((!prepared && !onDevice) || !grantId || typeof navigator === "undefined") return; const hasAudioConsent = bindAudioGrant(grantId); let disposed = false; let controller: VoiceInputController; @@ -256,10 +269,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { revision: draftRevisionRef.current.revision, }; }; - const platform = createBrowserVoiceInputPlatform({ - prepared, - cloudProvider, - cloudModel, + const platformInput = { hasAudioConsent, getTranscriptionOptions: (): SpeechTranscriptionOptions => { const { @@ -299,13 +309,22 @@ export function useEnvironmentSpeechInput(input: HookInput) { getMicrophoneId: () => latestRef.current.clientSettings.voiceMicrophone, onLevel: setLevel, onDurationLimit: () => void controller.stop(), - onText: (text) => { - if (!disposed) setPreview(text); - }, - onError: (message) => { + onError: (message: string) => { if (!disposed) void controller.interruptRecording(message); }, - }); + }; + const platform = + prepared && !onDevice + ? createBrowserVoiceInputPlatform({ + ...platformInput, + prepared, + cloudProvider, + cloudModel, + onText: (text) => { + if (!disposed) setPreview(text); + }, + }) + : createAppleSpeechVoiceInputPlatform(platformInput); const postProcess: NonNullable = ( transcript, options, @@ -373,7 +392,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { controller.dispose(); if (controllerRef.current === controller) controllerRef.current = null; }; - }, [prepared, grantId, cloudProvider, cloudModel]); + }, [prepared, onDevice, grantId, cloudProvider, cloudModel]); // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- probes the chosen environment's speech status once its connection is available useEffect(() => { @@ -485,7 +504,13 @@ export function useEnvironmentSpeechInput(input: HookInput) { const request = ++startRequestRef.current; const controller = controllerRef.current; const owner = latestRef.current.input.ownerKey; - if (!controller || !prepared) return; + if (!controller) return; + if (onDevice) { + setLevel(0); + await controller.start(); + return; + } + if (!prepared) return; if (cloudProvider) { setLevel(0); await controller.start(); @@ -593,9 +618,10 @@ export function useEnvironmentSpeechInput(input: HookInput) { const destinationReady = grantId === null ? Boolean(destination.offerFirstRunSetup) && input.offerSetup !== false - : prepared !== null && - (cloudProvider !== undefined || - (status?.prepared === prepared && status.value?.supported !== false)); + : onDevice || + (prepared !== null && + (cloudProvider !== undefined || + (status?.prepared === prepared && status.value?.supported !== false))); return { available: destinationReady && diff --git a/apps/web/src/speech/voiceConsent.ts b/apps/web/src/speech/voiceConsent.ts index 7c897fbc37ea5..2e7809b066de3 100644 --- a/apps/web/src/speech/voiceConsent.ts +++ b/apps/web/src/speech/voiceConsent.ts @@ -13,7 +13,13 @@ import { // Separate keys: a stale window saving ordinary preferences must not restore either grant. const AUDIO_KEY = "t3code:voice-audio-grant:v1"; const CLEANUP_KEY = "t3code:voice-cleanup-grant:v1"; -const Grant = Schema.Struct({ id: Schema.String, destination: VoiceTranscriptionDestination }); +// "device" is Apple Speech in the macOS desktop app. Audio stays on this device, so it stays out of the shared contract. +const AudioDestination = Schema.Union([ + VoiceTranscriptionDestination, + Schema.Struct({ kind: Schema.Literal("device") }), +]); +export type AudioDestination = typeof AudioDestination.Type; +const Grant = Schema.Struct({ id: Schema.String, destination: AudioDestination }); const AudioGrant = Schema.NullOr(Grant); const sameGrant = Schema.toEquivalence(AudioGrant); const SavedChoice = Schema.Union([AudioGrant, Schema.Literal("off")]); @@ -62,7 +68,7 @@ function replaceChoice(choice: AudioChoice, previous = readChoice()) { window.localStorage.removeItem(savedKey(old.id)); } -export function chooseAudioDestination(destination: VoiceTranscriptionDestination | null) { +export function chooseAudioDestination(destination: AudioDestination | null) { replaceChoice(destination ? { id: randomUUID(), destination } : "off"); } diff --git a/docs/internals/voice-input.md b/docs/internals/voice-input.md index 2bf73af0ab36d..6b18bec16450b 100644 --- a/docs/internals/voice-input.md +++ b/docs/internals/voice-input.md @@ -21,6 +21,12 @@ transcription share a recorder and upload only finished recordings. The client keeps the grant in device-bound secure storage, outside preference backups, and rechecks it after native decoding and before upload. +The macOS desktop app adds a `device` destination, Apple Speech. It exists only in +the web grant schema, not in `VoiceTranscriptionDestination`, because no environment +or other client can act on it. The desktop main process runs a +[Swift helper](../../native/apple-speech/AppleSpeech.swift) that the macOS release +runners compile; the Speech framework exists only in the macOS SDK. + The [web grant](../../apps/web/src/speech/voiceConsent.ts) lives outside the preferences snapshot so another window saving unrelated settings cannot resurrect a revoked grant. Each new choice gets a distinct grant, even if the destination was previously selected. The diff --git a/docs/user/composer.md b/docs/user/composer.md index 5365b97e27210..93b707c1b07d7 100644 --- a/docs/user/composer.md +++ b/docs/user/composer.md @@ -197,8 +197,9 @@ and use **Attach again** or remove the missing file before sending. ## Voice input on web and desktop Use the composer's microphone or `mod+shift+d` to dictate into your draft. Audio -is transcribed on an environment you choose in **Settings → Voice**; nothing is -sent until you choose one. See [Voice input](./voice-input.md). +is transcribed on an environment you choose in **Settings → Voice**, or on the Mac +with Apple Speech in the macOS desktop app; nothing is sent until you choose one. +See [Voice input](./voice-input.md). ## Voice input on mobile diff --git a/docs/user/faq.md b/docs/user/faq.md index 8d34de5e2ea45..48a5267a87d5c 100644 --- a/docs/user/faq.md +++ b/docs/user/faq.md @@ -77,7 +77,7 @@ Open **Settings → Worktrees → Cleanup** and turn on automatic worktree clean ## Is there voice input? -Yes. On iPhone and Android, dictation uses on-device speech by default; you can opt in to a paired environment instead. Web and desktop require you to choose an environment first. The transcript goes into your draft for review. See [Voice input](./voice-input.md). +Yes. On iPhone and Android, dictation uses on-device speech by default; you can opt in to a paired environment instead. Web and desktop require you to choose an environment first; on macOS 26 or later, the desktop app can also use Apple Speech on the Mac. The transcript goes into your draft for review. See [Voice input](./voice-input.md). ## Does it work on Windows, Mac, and Linux? diff --git a/docs/user/voice-input.md b/docs/user/voice-input.md index c116c9e67a395..15901e3221977 100644 --- a/docs/user/voice-input.md +++ b/docs/user/voice-input.md @@ -24,6 +24,12 @@ Set it back to **Off** at any time; a recording in progress is discarded. If the chosen environment goes offline, voice input waits for it. T3 Code never substitutes another environment. +In the desktop app on macOS 26 or later, you can instead choose **Apple Speech** +under **On this Mac**. Audio stays on the Mac and macOS transcribes it. The first +recording in a language can take longer while macOS downloads its speech model. +Silence filtering, filler-word removal, and the dictionary apply only to +environment and cloud transcription. + For cloud dictation, choose **OpenAI** or **Grok Voice Transcribe** via the environment you want to use. If it needs a key, enter one and choose **Save and enable**. You can also choose a [secret provider](secrets.md) instead of **Value** and enter a diff --git a/native/apple-speech/AppleSpeech.swift b/native/apple-speech/AppleSpeech.swift new file mode 100644 index 0000000000000..a315a189fa9ba --- /dev/null +++ b/native/apple-speech/AppleSpeech.swift @@ -0,0 +1,125 @@ +// Adapted from AppleTranscriptionImpl.swift in @react-native-ai/apple (MIT, Callstack), +// including this repo's patch, so the Mac and iPhone transcribe the same way. +// +// t3-apple-speech available -> {"value":""} +// t3-apple-speech prepare -> {"value":"en-US"} (resolved locale) +// t3-apple-speech transcribe -> {"value":"..."} (transcript) +// Exit codes are consumed by AppleSpeech.ts; failures print one message line to stderr. + +import AVFoundation +import Foundation +import Speech + +private enum ExitCode: Int32 { + case failure = 1 + case unsupportedLocale = 2 + case unavailable = 3 + case usage = 64 +} + +private struct Failure: Error { + let code: ExitCode + let message: String +} + +@main +struct AppleSpeech { + static func main() async { + let arguments = Array(CommandLine.arguments.dropFirst()) + do { + switch (arguments.first, arguments.count) { + case ("available", 1): + try requireAvailable() + try write("") + case ("prepare", 2): + try write(try await prepare(arguments[1])) + case ("transcribe", 3): + try write(try await transcribe(arguments[1], path: arguments[2])) + default: + throw Failure(code: .usage, message: "Unknown command.") + } + } catch let failure as Failure { + fail(failure.code, failure.message) + } catch { + fail(.failure, error.localizedDescription) + } + } + + private static func requireAvailable() throws { + guard SpeechTranscriber.isAvailable else { + throw Failure(code: .unavailable, message: "Speech transcription is not available on this Mac.") + } + } + + private static func makeTranscriber(for locale: Locale) -> SpeechTranscriber { + let preset = SpeechTranscriber.Preset.timeIndexedTranscriptionWithAlternatives + return SpeechTranscriber( + locale: locale, + transcriptionOptions: preset.transcriptionOptions, + reportingOptions: preset.reportingOptions.subtracting([.alternativeTranscriptions]), + attributeOptions: preset.attributeOptions + ) + } + + private static func prepare(_ identifier: String) async throws -> String { + try requireAvailable() + guard + let locale = await SpeechTranscriber.supportedLocale( + equivalentTo: Locale(identifier: identifier)) + else { + throw Failure(code: .unsupportedLocale, message: "Locale not supported: \(identifier)") + } + let transcriber = makeTranscriber(for: locale) + switch await AssetInventory.status(forModules: [transcriber]) { + case .installed: + break + case .supported, .downloading: + if let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) { + try await request.downloadAndInstall() + } + case .unsupported: + throw Failure( + code: .unsupportedLocale, message: "Assets not supported for locale: \(identifier)") + @unknown default: + throw Failure(code: .failure, message: "Unknown asset inventory status.") + } + return locale.identifier(.bcp47) + } + + private static func transcribe(_ identifier: String, path: String) async throws -> String { + try requireAvailable() + let audioFile = try AVAudioFile(forReading: URL(fileURLWithPath: path)) + let transcriber = makeTranscriber(for: Locale(identifier: identifier)) + let analyzer = SpeechAnalyzer(modules: [transcriber]) + let collector = Task { () throws -> [String] in + var texts: [String] = [] + for try await result in transcriber.results where result.isFinal { + let text = String(result.text.characters).trimmingCharacters(in: .whitespacesAndNewlines) + if !text.isEmpty { texts.append(text) } + } + return texts + } + do { + if let lastSampleTime = try await analyzer.analyzeSequence(from: audioFile) { + try await analyzer.finalizeAndFinish(through: lastSampleTime) + } else { + await analyzer.cancelAndFinishNow() + } + return try await collector.value.joined(separator: " ") + } catch { + collector.cancel() + await analyzer.cancelAndFinishNow() + _ = try? await collector.value + throw error + } + } + + private static func write(_ value: String) throws { + FileHandle.standardOutput.write(try JSONSerialization.data(withJSONObject: ["value": value])) + } + + private static func fail(_ code: ExitCode, _ message: String) -> Never { + FileHandle.standardError.write(Data((message + "\n").utf8)) + exit(code.rawValue) + } +} diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index 29be3d6d0219f..5efb9e2c083ba 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -1026,6 +1026,23 @@ export const DesktopMediaReadSchema = Schema.Struct({ }); export type DesktopMediaRead = typeof DesktopMediaReadSchema.Type; +/** `value` is the resolved locale from `prepare` and the transcript from `transcribe`. */ +export const DesktopAppleSpeechResultSchema = Schema.Union([ + Schema.Struct({ ok: Schema.Literal(true), value: Schema.String }), + Schema.Struct({ + ok: Schema.Literal(false), + code: Schema.Literals(["unavailable", "unsupported-locale", "failed"]), + message: Schema.String, + }), +]); +export type DesktopAppleSpeechResult = typeof DesktopAppleSpeechResultSchema.Type; + +export const DesktopAppleSpeechTranscribeInputSchema = Schema.Struct({ + locale: Schema.String.check(Schema.isMaxLength(64)), + // 16 kHz mono Float32 PCM, at most five minutes. + pcm: Schema.Uint8Array, +}); + export const SystemSettingsPaneSchema = Schema.Literals(["full-disk-access"]); export type SystemSettingsPane = typeof SystemSettingsPaneSchema.Type; @@ -1156,6 +1173,12 @@ export interface DesktopBridge { onRequest: (listener: (request: DesktopAppActivationRequest) => void) => () => void; }; preview: DesktopPreviewBridge; + /** On-device dictation through macOS 26 SpeechAnalyzer; absent in older desktop builds. */ + appleSpeech?: { + isAvailable: () => Promise; + prepare: (locale: string) => Promise; + transcribe: (input: { locale: string; pcm: Uint8Array }) => Promise; + }; } /** Renderer callback invoked by Electron with a fresh user gesture before display-media capture. */ diff --git a/scripts/build-desktop-artifact.ts b/scripts/build-desktop-artifact.ts index 2fff4919c40d3..3c2251d65e51d 100644 --- a/scripts/build-desktop-artifact.ts +++ b/scripts/build-desktop-artifact.ts @@ -996,6 +996,8 @@ export const DESKTOP_FILE_EXCLUSIONS = [ "!apps/desktop/resources/browser-secret/**/*", "!apps/desktop/prod-resources/browser-secret", "!apps/desktop/prod-resources/browser-secret/**/*", + "!apps/desktop/prod-resources/apple-speech", + "!apps/desktop/prod-resources/apple-speech/**/*", "!apps/desktop/prod-resources/windows-server", "!apps/desktop/prod-resources/windows-server/**/*", "!apps/desktop/prod-resources/wsl-runtime.tar.gz", @@ -1118,6 +1120,9 @@ export const LINUX_CAPTURE_EXTRA_RESOURCES = [ export const LINUX_BROWSER_SECRET_EXTRA_RESOURCES = [ { from: "apps/desktop/prod-resources/browser-secret", to: "browser-secret" }, ] as const; +export const MAC_APPLE_SPEECH_EXTRA_RESOURCES = [ + { from: "apps/desktop/prod-resources/apple-speech", to: "apple-speech" }, +] as const; export interface MacPasskeySigningConfiguration { readonly appId: string; @@ -2460,6 +2465,31 @@ export const stageBrowserSecret = Effect.fn("stageBrowserSecret")(function* (inp ); }); +export const stageAppleSpeech = Effect.fn("stageAppleSpeech")(function* (input: { + readonly repoRoot: string; + readonly stageResourcesDir: string; + readonly platform: typeof BuildPlatform.Type; + readonly arch: typeof BuildArch.Type; + readonly verbose: boolean; +}) { + if (input.platform !== "mac") return; + const path = yield* Path.Path; + yield* runCommand( + ChildProcess.make( + "node", + [ + path.join(input.repoRoot, "apps/desktop/scripts/build-apple-speech.mjs"), + "--arch", + input.arch, + "--output", + path.join(input.stageResourcesDir, "apple-speech", "t3-apple-speech"), + ], + { cwd: input.repoRoot }, + ), + { label: "build macOS Apple Speech helper", verbose: input.verbose }, + ); +}); + const generateMacIconSet = Effect.fn("build-desktop-artifact.generateMacIconSet")(function* ( sourcePng: string, targetIcns: string, @@ -2804,6 +2834,7 @@ export const createBuildConfig = Effect.fn("createBuildConfig")(function* ( ...DESKTOP_EXTRA_RESOURCES, ...(platform === "linux" ? LINUX_CAPTURE_EXTRA_RESOURCES : []), ...(platform === "linux" ? LINUX_BROWSER_SECRET_EXTRA_RESOURCES : []), + ...(platform === "mac" ? MAC_APPLE_SPEECH_EXTRA_RESOURCES : []), ...(platform === "win" ? WINDOWS_SERVER_EXTRA_RESOURCES : []), ...(platform === "win" && wslRuntimeBundled ? WSL_RUNTIME_EXTRA_RESOURCES : []), ], @@ -2849,6 +2880,8 @@ export const createBuildConfig = Effect.fn("createBuildConfig")(function* ( "T3 Code lets commands run by terminals and coding agents use the camera.", NSMicrophoneUsageDescription: "T3 Code lets commands run by terminals and coding agents use the microphone.", + NSSpeechRecognitionUsageDescription: + "T3 Code transcribes your dictation on this Mac when you choose Apple Speech.", // macOS lists an app under Default web browser only when it opens web // pages as documents as well as http and https links (see protocols). CFBundleDocumentTypes: [ @@ -3795,6 +3828,13 @@ const buildDesktopArtifactUnlocked = Effect.fn("buildDesktopArtifactUnlocked")(f arch: options.arch, verbose: options.verbose, }); + yield* stageAppleSpeech({ + repoRoot, + stageResourcesDir, + platform: options.platform, + arch: options.arch, + verbose: options.verbose, + }); yield* assertPlatformBuildResources( options.platform, From 95275742c38f99b19a99e361b56e23732716ee57 Mon Sep 17 00:00:00 2001 From: RVZO6 <66844656+RVZO6@users.noreply.github.com> Date: Wed, 11 Nov 2026 06:32:35 +0000 Subject: [PATCH 02/12] fix(desktop): prepare Apple Speech once per language and retry failed probes Each dictation start awaited a new helper spawn before the microphone opened, and a slow first availability probe hid Apple Speech until restart. Share one in-flight prepare per locale and keep successful results, so starts after the first open the microphone at once and concurrent first downloads share one helper. Keep only definite availability answers; a timed-out or failed probe runs again. Co-Authored-By: Claude Opus 5.5 --- apps/desktop/src/speech/AppleSpeech.test.ts | 40 +++++++++++++++------ apps/desktop/src/speech/AppleSpeech.ts | 35 ++++++++++++++++-- apps/web/src/speech/appleSpeech.ts | 5 ++- 3 files changed, 66 insertions(+), 14 deletions(-) diff --git a/apps/desktop/src/speech/AppleSpeech.test.ts b/apps/desktop/src/speech/AppleSpeech.test.ts index 069017d80e5cd..fa3aea03754d9 100644 --- a/apps/desktop/src/speech/AppleSpeech.test.ts +++ b/apps/desktop/src/speech/AppleSpeech.test.ts @@ -81,16 +81,36 @@ describe("AppleSpeech", () => { { stdout: "not json", exitCode: 0 }, { ok: false, code: "failed" }, ], - ] as const)("maps helper output %j to %j", ([output, expected]) => { - const { layer, commands } = setup(output); - return Effect.gen(function* () { - const result = yield* (yield* AppleSpeech.AppleSpeech).prepare("en-US"); - assert.deepInclude(result, expected); - assert.deepEqual(commands, [ - ["/app/Resources/apple-speech/t3-apple-speech", "prepare", "en-US"], - ]); - }).pipe(Effect.provide(layer)); - }); + ] as const)( + "maps helper output %j to %j, preparing again only after a failure", + ([output, expected]) => { + const { layer, commands } = setup(output); + return Effect.gen(function* () { + const speech = yield* AppleSpeech.AppleSpeech; + assert.deepInclude(yield* speech.prepare("en-US"), expected); + assert.deepInclude(yield* speech.prepare("en-US"), expected); + const command = ["/app/Resources/apple-speech/t3-apple-speech", "prepare", "en-US"]; + assert.deepEqual(commands, expected.ok ? [command] : [command, command]); + }).pipe(Effect.provide(layer)); + }, + ); + + it.effect.each([ + [0, true, 1], + [3, false, 1], + [1, false, 2], + ] as const)( + "isAvailable with helper exit %i is %s and probes %i time(s) for two calls", + ([exitCode, expected, probes]) => { + const { layer, commands } = setup({ stdout: '{"value":""}', exitCode }); + return Effect.gen(function* () { + const speech = yield* AppleSpeech.AppleSpeech; + assert.equal(yield* speech.isAvailable, expected); + assert.equal(yield* speech.isAvailable, expected); + assert.lengthOf(commands, probes); + }).pipe(Effect.provide(layer)); + }, + ); it.effect("transcribes Float32 PCM through a 16-bit WAV file", () => { const { layer, commands, written } = setup({ stdout: '{"value":"hello"}', exitCode: 0 }); diff --git a/apps/desktop/src/speech/AppleSpeech.ts b/apps/desktop/src/speech/AppleSpeech.ts index cf79b8b4b4f0b..1dfd2c7dd70e5 100644 --- a/apps/desktop/src/speech/AppleSpeech.ts +++ b/apps/desktop/src/speech/AppleSpeech.ts @@ -150,14 +150,43 @@ export const layer = Layer.effect( ), ); - const isAvailable = yield* Effect.cached( - run(["available"], Duration.seconds(10)).pipe(Effect.map((result) => result.ok)), + // Only a definite answer is kept; a slow or failed probe is asked again next time. + let available: boolean | undefined; + const isAvailable = Effect.suspend(() => + available === undefined + ? run(["available"], Duration.seconds(10)).pipe( + Effect.tap((result) => + Effect.sync(() => { + if (result.ok || result.code === "unavailable") available = result.ok; + }), + ), + Effect.map((result) => result.ok), + ) + : Effect.succeed(available), ); + // Each dictation start awaits prepare before the microphone opens, so a locale is + // prepared once. Callers share an in-flight prepare (one asset download), and a + // failure is dropped so the next start retries it. + const prepared = new Map>(); + return AppleSpeech.of({ isAvailable, prepare: Effect.fn("AppleSpeech.prepare")(function* (locale) { - return yield* run(["prepare", locale], PREPARE_TIMEOUT); + let shared = prepared.get(locale); + if (shared === undefined) { + shared = yield* Effect.cached( + run(["prepare", locale], PREPARE_TIMEOUT).pipe( + Effect.tap((result) => + Effect.sync(() => { + if (!result.ok) prepared.delete(locale); + }), + ), + ), + ); + prepared.set(locale, shared); + } + return yield* shared; }), transcribe: Effect.fn("AppleSpeech.transcribe")(function* (locale, pcm) { if ( diff --git a/apps/web/src/speech/appleSpeech.ts b/apps/web/src/speech/appleSpeech.ts index 60475500e50fe..d5b1b19f39063 100644 --- a/apps/web/src/speech/appleSpeech.ts +++ b/apps/web/src/speech/appleSpeech.ts @@ -10,12 +10,15 @@ function subscribe(listener: () => void) { .catch(() => false) .then((value) => { available = value; + // The main process keeps a definite answer, so asking again after "no" is cheap + // and recovers from a probe that timed out. + if (!value && window.desktopBridge?.appleSpeech) probe = undefined; listeners.forEach((notify) => notify()); }); return () => listeners.delete(listener); } -/** Apple Speech runs only in the macOS 26 desktop app; the main process probes it once. */ +/** Apple Speech runs only in the macOS 26 desktop app; the main process caches the answer. */ export function useAppleSpeechAvailable() { return useSyncExternalStore( subscribe, From db3ee10b83bb536b9d76dcc6503b96c86fdfaa3b Mon Sep 17 00:00:00 2001 From: RVZO6 <66844656+RVZO6@users.noreply.github.com> Date: Thu, 12 Nov 2026 07:00:45 +0000 Subject: [PATCH 03/12] fix(desktop): Apple Speech review fixes for timeouts, caching and settings - Claim the per-locale prepare under a permit, so concurrent first starts always share one helper, and test it with a pending helper. - Let transcription run as long as the longest accepted recording. - Retry a "not available" probe for mounted views, up to three times. - Show the Language setting for Apple Speech and turn off silence filtering and filler-word removal, which it does not use. Co-Authored-By: Claude Opus 5.5 --- apps/desktop/src/speech/AppleSpeech.test.ts | 37 ++++++++++++++++-- apps/desktop/src/speech/AppleSpeech.ts | 35 ++++++++++------- .../settings/VoiceSettingsPanel.tsx | 39 ++++++++++++------- apps/web/src/speech/appleSpeech.ts | 24 +++++++++--- docs/user/voice-input.md | 5 ++- 5 files changed, 101 insertions(+), 39 deletions(-) diff --git a/apps/desktop/src/speech/AppleSpeech.test.ts b/apps/desktop/src/speech/AppleSpeech.test.ts index fa3aea03754d9..7cc3f23b1040b 100644 --- a/apps/desktop/src/speech/AppleSpeech.test.ts +++ b/apps/desktop/src/speech/AppleSpeech.test.ts @@ -1,5 +1,7 @@ import { assert, describe, it } from "@effect/vitest"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Sink from "effect/Sink"; @@ -12,13 +14,22 @@ import * as AppleSpeech from "./AppleSpeech.ts"; const encoder = new TextEncoder(); const text = (value: string) => (value ? Stream.make(encoder.encode(value)) : Stream.empty); -function makeProcess(result: { stdout?: string; stderr?: string; exitCode: number }) { +type HelperOutput = { + stdout?: string; + stderr?: string; + exitCode: number; + exit?: Deferred.Deferred; +}; + +function makeProcess(result: HelperOutput) { return ChildProcessSpawner.makeHandle({ pid: ChildProcessSpawner.ProcessId(1), stdout: text(result.stdout ?? ""), stderr: text(result.stderr ?? ""), all: Stream.empty, - exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(result.exitCode)), + exitCode: (result.exit ? Deferred.await(result.exit) : Effect.void).pipe( + Effect.as(ChildProcessSpawner.ExitCode(result.exitCode)), + ), isRunning: Effect.succeed(false), kill: () => Effect.void, stdin: Sink.drain, @@ -28,7 +39,7 @@ function makeProcess(result: { stdout?: string; stderr?: string; exitCode: numbe }); } -function setup(result: { stdout?: string; stderr?: string; exitCode: number }) { +function setup(result: HelperOutput) { const commands: Array> = []; const written: Uint8Array[] = []; const layer = AppleSpeech.layer.pipe( @@ -95,6 +106,26 @@ describe("AppleSpeech", () => { }, ); + it.effect("shares one helper between prepares that start while the first is running", () => + Effect.gen(function* () { + const exit = yield* Deferred.make(); + const { layer, commands } = setup({ stdout: '{"value":"en-US"}', exitCode: 0, exit }); + return yield* Effect.gen(function* () { + const speech = yield* AppleSpeech.AppleSpeech; + const first = yield* speech + .prepare("en-US") + .pipe(Effect.forkChild({ startImmediately: true })); + const second = yield* speech + .prepare("en-US") + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Deferred.succeed(exit, undefined); + assert.deepEqual(yield* Fiber.join(first), { ok: true, value: "en-US" }); + assert.deepEqual(yield* Fiber.join(second), { ok: true, value: "en-US" }); + assert.lengthOf(commands, 1); + }).pipe(Effect.provide(layer)); + }), + ); + it.effect.each([ [0, true, 1], [3, false, 1], diff --git a/apps/desktop/src/speech/AppleSpeech.ts b/apps/desktop/src/speech/AppleSpeech.ts index 1dfd2c7dd70e5..7357802c42be9 100644 --- a/apps/desktop/src/speech/AppleSpeech.ts +++ b/apps/desktop/src/speech/AppleSpeech.ts @@ -6,6 +6,7 @@ import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; @@ -15,7 +16,8 @@ const SAMPLE_RATE = 16_000; const MAX_PCM_BYTES = SAMPLE_RATE * Float32Array.BYTES_PER_ELEMENT * 5 * 60; // The first prepare for a language can download Apple's model. const PREPARE_TIMEOUT = Duration.minutes(10); -const TRANSCRIBE_TIMEOUT = Duration.minutes(2); +// Matches the longest accepted recording, so analysis can never outlast its deadline. +const TRANSCRIBE_TIMEOUT = Duration.minutes(5); const decodeOutput = Schema.decodeUnknownEffect( Schema.fromJsonString(Schema.Struct({ value: Schema.String })), @@ -169,24 +171,27 @@ export const layer = Layer.effect( // prepared once. Callers share an in-flight prepare (one asset download), and a // failure is dropped so the next start retries it. const prepared = new Map>(); + const claimPermit = yield* Semaphore.make(1); + const claimPrepare = Effect.fn("AppleSpeech.claimPrepare")(function* (locale: string) { + const existing = prepared.get(locale); + if (existing !== undefined) return existing; + const shared = yield* Effect.cached( + run(["prepare", locale], PREPARE_TIMEOUT).pipe( + Effect.tap((result) => + Effect.sync(() => { + if (!result.ok) prepared.delete(locale); + }), + ), + ), + ); + prepared.set(locale, shared); + return shared; + }, claimPermit.withPermits(1)); return AppleSpeech.of({ isAvailable, prepare: Effect.fn("AppleSpeech.prepare")(function* (locale) { - let shared = prepared.get(locale); - if (shared === undefined) { - shared = yield* Effect.cached( - run(["prepare", locale], PREPARE_TIMEOUT).pipe( - Effect.tap((result) => - Effect.sync(() => { - if (!result.ok) prepared.delete(locale); - }), - ), - ), - ); - prepared.set(locale, shared); - } - return yield* shared; + return yield* yield* claimPrepare(locale); }), transcribe: Effect.fn("AppleSpeech.transcribe")(function* (locale, pcm) { if ( diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.tsx index 378f6faf14bf7..57de26a44b346 100644 --- a/apps/web/src/components/settings/VoiceSettingsPanel.tsx +++ b/apps/web/src/components/settings/VoiceSettingsPanel.tsx @@ -1278,7 +1278,9 @@ export function VoiceSettingsPanel() { setPendingCloud(null)} /> )} - {(model || destination.cloudProvider) && ( + {(model || destination.cloudProvider || destination.device) && ( } /> @@ -1562,14 +1568,16 @@ export function VoiceSettingsPanel() { {...searchableSetting("speech-silence-filtering")} title="Silence filtering" description={ - destination.cloudProvider === "openai" - ? "OpenAI handles silence itself." - : "Skip silence and background noise before transcribing." + destination.device + ? "Apple Speech handles silence itself." + : destination.cloudProvider === "openai" + ? "OpenAI handles silence itself." + : "Skip silence and background noise before transcribing." } control={ void updateClientSettings({ speechSilenceFiltering: checked }) @@ -1580,10 +1588,15 @@ export function VoiceSettingsPanel() { void updateClientSettings({ speechRemoveFillerWords: checked }) diff --git a/apps/web/src/speech/appleSpeech.ts b/apps/web/src/speech/appleSpeech.ts index d5b1b19f39063..dd1b737044c80 100644 --- a/apps/web/src/speech/appleSpeech.ts +++ b/apps/web/src/speech/appleSpeech.ts @@ -2,19 +2,31 @@ import { useSyncExternalStore } from "react"; let available = false; let probe: Promise | undefined; +let retries = 0; const listeners = new Set<() => void>(); +// The main process keeps only a definite answer, so a "no" may be a slow first probe. +const MAX_RETRIES = 3; +const RETRY_DELAY_MS = 15_000; -function subscribe(listener: () => void) { - listeners.add(listener); - probe ??= (window.desktopBridge?.appleSpeech?.isAvailable() ?? Promise.resolve(false)) +function startProbe() { + const bridge = window.desktopBridge?.appleSpeech; + probe = (bridge?.isAvailable() ?? Promise.resolve(false)) .catch(() => false) .then((value) => { available = value; - // The main process keeps a definite answer, so asking again after "no" is cheap - // and recovers from a probe that timed out. - if (!value && window.desktopBridge?.appleSpeech) probe = undefined; listeners.forEach((notify) => notify()); + if (value || !bridge || retries >= MAX_RETRIES) return; + retries++; + setTimeout(() => { + probe = undefined; + if (listeners.size > 0) startProbe(); + }, RETRY_DELAY_MS); }); +} + +function subscribe(listener: () => void) { + listeners.add(listener); + if (!probe) startProbe(); return () => listeners.delete(listener); } diff --git a/docs/user/voice-input.md b/docs/user/voice-input.md index 516f3f3eaee56..50620408ecd41 100644 --- a/docs/user/voice-input.md +++ b/docs/user/voice-input.md @@ -27,8 +27,9 @@ substitutes another environment. In the desktop app on macOS 26 or later, you can instead choose **Apple Speech** under **On this Mac**. Audio stays on the Mac and macOS transcribes it. The first recording in a language can take longer while macOS downloads its speech model. -Silence filtering, filler-word removal, and the dictionary apply only to -environment and cloud transcription. +**Language** set to automatic uses your Mac's language. Silence filtering and +filler-word removal are not available; the dictionary applies only when transcript +cleanup is on. For cloud dictation, choose **OpenAI** or **Grok Voice Transcribe** via the environment you want to use. If it needs a key, enter one and choose **Save and enable**. From d33c82222a423621ba0c199a721c5d29d950b505 Mon Sep 17 00:00:00 2001 From: RVZO6 <66844656+RVZO6@users.noreply.github.com> Date: Thu, 12 Nov 2026 07:25:18 +0000 Subject: [PATCH 04/12] fix(desktop): Apple Speech accepts unaligned PCM and runs one analysis at a time - Copy a PCM view whose offset is not 4-byte aligned before reading it as Float32, instead of throwing a RangeError. - Serialize transcriptions in the main process, so overlapping IPC calls never run several native analyzers at once. - Test that a mounted view probes availability again after "not available". Co-Authored-By: Claude Opus 5.5 --- apps/desktop/src/speech/AppleSpeech.test.ts | 9 +++++-- apps/desktop/src/speech/AppleSpeech.ts | 12 ++++++--- .../src/speech/appleSpeech.browser.test.tsx | 25 +++++++++++++++++++ 3 files changed, 41 insertions(+), 5 deletions(-) create mode 100644 apps/web/src/speech/appleSpeech.browser.test.tsx diff --git a/apps/desktop/src/speech/AppleSpeech.test.ts b/apps/desktop/src/speech/AppleSpeech.test.ts index 7cc3f23b1040b..b9090307bab1e 100644 --- a/apps/desktop/src/speech/AppleSpeech.test.ts +++ b/apps/desktop/src/speech/AppleSpeech.test.ts @@ -145,9 +145,14 @@ describe("AppleSpeech", () => { it.effect("transcribes Float32 PCM through a 16-bit WAV file", () => { const { layer, commands, written } = setup({ stdout: '{"value":"hello"}', exitCode: 0 }); - const pcm = new Uint8Array(new Float32Array([0, 1, -1, 2]).buffer); + // An unaligned view, as a structured-clone or subarray caller may pass. + const pcm = new Uint8Array(17); + new DataView(pcm.buffer).setFloat32(1, 0, true); + new DataView(pcm.buffer).setFloat32(5, 1, true); + new DataView(pcm.buffer).setFloat32(9, -1, true); + new DataView(pcm.buffer).setFloat32(13, 2, true); return Effect.gen(function* () { - const result = yield* (yield* AppleSpeech.AppleSpeech).transcribe("en-US", pcm); + const result = yield* (yield* AppleSpeech.AppleSpeech).transcribe("en-US", pcm.subarray(1)); assert.deepEqual(result, { ok: true, value: "hello" }); assert.deepEqual(commands, [ [ diff --git a/apps/desktop/src/speech/AppleSpeech.ts b/apps/desktop/src/speech/AppleSpeech.ts index 7357802c42be9..14db0f05992c6 100644 --- a/apps/desktop/src/speech/AppleSpeech.ts +++ b/apps/desktop/src/speech/AppleSpeech.ts @@ -25,10 +25,13 @@ const decodeOutput = Schema.decodeUnknownEffect( /** 16 kHz mono Float32 PCM to a 16-bit WAV that AVAudioFile reads. */ function encodeSpeechWav(pcmBytes: Uint8Array): Uint8Array { + // Float32Array needs a 4-byte-aligned offset; copy a view that is not. + const aligned = + pcmBytes.byteOffset % Float32Array.BYTES_PER_ELEMENT === 0 ? pcmBytes : pcmBytes.slice(); const samples = new Float32Array( - pcmBytes.buffer, - pcmBytes.byteOffset, - pcmBytes.byteLength / Float32Array.BYTES_PER_ELEMENT, + aligned.buffer, + aligned.byteOffset, + aligned.byteLength / Float32Array.BYTES_PER_ELEMENT, ); const wav = new Uint8Array(44 + samples.length * 2); const view = new DataView(wav.buffer); @@ -172,6 +175,8 @@ export const layer = Layer.effect( // failure is dropped so the next start retries it. const prepared = new Map>(); const claimPermit = yield* Semaphore.make(1); + // One native analysis at a time, even if IPC callers overlap. + const transcribePermit = yield* Semaphore.make(1); const claimPrepare = Effect.fn("AppleSpeech.claimPrepare")(function* (locale: string) { const existing = prepared.get(locale); if (existing !== undefined) return existing; @@ -216,6 +221,7 @@ export const layer = Layer.effect( code: "failed", message: "Could not save the recording for Apple Speech.", })), + transcribePermit.withPermits(1), ); }), }); diff --git a/apps/web/src/speech/appleSpeech.browser.test.tsx b/apps/web/src/speech/appleSpeech.browser.test.tsx new file mode 100644 index 0000000000000..6f5068799d4d5 --- /dev/null +++ b/apps/web/src/speech/appleSpeech.browser.test.tsx @@ -0,0 +1,25 @@ +import { expect, it, vi } from "vite-plus/test"; + +import { act, fakeTimers, render, stubGlobal } from "../../test/render"; +import { useAppleSpeechAvailable } from "./appleSpeech"; + +function Availability() { + return {useAppleSpeechAvailable() ? "available" : "unavailable"}; +} + +it("probes Apple Speech again for a mounted view after a 'not available' answer", async () => { + let answerFirstProbe!: (available: boolean) => void; + const isAvailable = vi + .fn<() => Promise>() + .mockReturnValueOnce(new Promise((resolve) => (answerFirstProbe = resolve))) + .mockResolvedValue(true); + using _bridge = stubGlobal("desktopBridge", { appleSpeech: { isAvailable } }); + // The hook reads a module store and the desktop bridge, not routes. + await using screen = await render(, { router: false }); + using _timers = fakeTimers(); + await act(() => answerFirstProbe(false)); + await expect.element(screen.getByText("unavailable")).toBeInTheDocument(); + await act(() => vi.advanceTimersByTime(15_000)); + await expect.element(screen.getByText("available", { exact: true })).toBeInTheDocument(); + expect(isAvailable).toHaveBeenCalledTimes(2); +}); From b3ca461a0e52d55a9f75153e558102a7b8a93cd3 Mon Sep 17 00:00:00 2001 From: RVZO6 <66844656+RVZO6@users.noreply.github.com> Date: Thu, 12 Nov 2026 07:36:16 +0000 Subject: [PATCH 05/12] test(web): check the Apple Speech availability retry as a unit test The retry lives in the availability store, not in React, so test the store directly instead of loading a browser test file. Co-Authored-By: Claude Opus 5.5 --- .../src/speech/appleSpeech.browser.test.tsx | 25 -------------- apps/web/src/speech/appleSpeech.test.ts | 33 +++++++++++++++++++ apps/web/src/speech/appleSpeech.ts | 18 ++++++---- 3 files changed, 44 insertions(+), 32 deletions(-) delete mode 100644 apps/web/src/speech/appleSpeech.browser.test.tsx create mode 100644 apps/web/src/speech/appleSpeech.test.ts diff --git a/apps/web/src/speech/appleSpeech.browser.test.tsx b/apps/web/src/speech/appleSpeech.browser.test.tsx deleted file mode 100644 index 6f5068799d4d5..0000000000000 --- a/apps/web/src/speech/appleSpeech.browser.test.tsx +++ /dev/null @@ -1,25 +0,0 @@ -import { expect, it, vi } from "vite-plus/test"; - -import { act, fakeTimers, render, stubGlobal } from "../../test/render"; -import { useAppleSpeechAvailable } from "./appleSpeech"; - -function Availability() { - return {useAppleSpeechAvailable() ? "available" : "unavailable"}; -} - -it("probes Apple Speech again for a mounted view after a 'not available' answer", async () => { - let answerFirstProbe!: (available: boolean) => void; - const isAvailable = vi - .fn<() => Promise>() - .mockReturnValueOnce(new Promise((resolve) => (answerFirstProbe = resolve))) - .mockResolvedValue(true); - using _bridge = stubGlobal("desktopBridge", { appleSpeech: { isAvailable } }); - // The hook reads a module store and the desktop bridge, not routes. - await using screen = await render(, { router: false }); - using _timers = fakeTimers(); - await act(() => answerFirstProbe(false)); - await expect.element(screen.getByText("unavailable")).toBeInTheDocument(); - await act(() => vi.advanceTimersByTime(15_000)); - await expect.element(screen.getByText("available", { exact: true })).toBeInTheDocument(); - expect(isAvailable).toHaveBeenCalledTimes(2); -}); diff --git a/apps/web/src/speech/appleSpeech.test.ts b/apps/web/src/speech/appleSpeech.test.ts new file mode 100644 index 0000000000000..b8f7fcfc2e124 --- /dev/null +++ b/apps/web/src/speech/appleSpeech.test.ts @@ -0,0 +1,33 @@ +import { expect, it, vi } from "vite-plus/test"; + +it("probes Apple Speech again for a subscriber after a 'not available' answer", async () => { + const original = Object.getOwnPropertyDescriptor(globalThis, "window"); + const isAvailable = vi + .fn<() => Promise>() + .mockResolvedValueOnce(false) + .mockResolvedValue(true); + Object.defineProperty(globalThis, "window", { + value: { desktopBridge: { appleSpeech: { isAvailable } } }, + configurable: true, + }); + vi.useFakeTimers(); + let unsubscribe = () => {}; + try { + const { appleSpeechAvailability } = await import("./appleSpeech"); + const notify = vi.fn(); + unsubscribe = appleSpeechAvailability.subscribe(notify); + + await vi.advanceTimersByTimeAsync(0); + expect(notify).toHaveBeenCalledTimes(1); + expect(appleSpeechAvailability.getSnapshot()).toBe(false); + await vi.advanceTimersByTimeAsync(15_000); + expect(isAvailable).toHaveBeenCalledTimes(2); + expect(notify).toHaveBeenCalledTimes(2); + expect(appleSpeechAvailability.getSnapshot()).toBe(true); + } finally { + unsubscribe(); + vi.useRealTimers(); + if (original) Object.defineProperty(globalThis, "window", original); + else Reflect.deleteProperty(globalThis, "window"); + } +}); diff --git a/apps/web/src/speech/appleSpeech.ts b/apps/web/src/speech/appleSpeech.ts index dd1b737044c80..5d592a5d9d185 100644 --- a/apps/web/src/speech/appleSpeech.ts +++ b/apps/web/src/speech/appleSpeech.ts @@ -24,17 +24,21 @@ function startProbe() { }); } -function subscribe(listener: () => void) { - listeners.add(listener); - if (!probe) startProbe(); - return () => listeners.delete(listener); -} +/** The store behind `useAppleSpeechAvailable`. */ +export const appleSpeechAvailability = { + subscribe(listener: () => void) { + listeners.add(listener); + if (!probe) startProbe(); + return () => listeners.delete(listener); + }, + getSnapshot: () => available, +}; /** Apple Speech runs only in the macOS 26 desktop app; the main process caches the answer. */ export function useAppleSpeechAvailable() { return useSyncExternalStore( - subscribe, - () => available, + appleSpeechAvailability.subscribe, + appleSpeechAvailability.getSnapshot, () => false, ); } From cfe62366f6aa38be2bef7a12285337616d1046bd Mon Sep 17 00:00:00 2001 From: RVZO6 <66844656+RVZO6@users.noreply.github.com> Date: Thu, 12 Nov 2026 07:42:08 +0000 Subject: [PATCH 06/12] fix(desktop): Apple Speech copies unaligned Buffer audio Buffer.slice returns a view, so an unaligned Buffer still reached Float32Array and threw. Copy with new Uint8Array, and test with an unaligned Buffer. Co-Authored-By: Claude Opus 5.5 --- apps/desktop/src/speech/AppleSpeech.test.ts | 7 +++++-- apps/desktop/src/speech/AppleSpeech.ts | 6 ++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/apps/desktop/src/speech/AppleSpeech.test.ts b/apps/desktop/src/speech/AppleSpeech.test.ts index b9090307bab1e..54eba0706e7af 100644 --- a/apps/desktop/src/speech/AppleSpeech.test.ts +++ b/apps/desktop/src/speech/AppleSpeech.test.ts @@ -145,14 +145,17 @@ describe("AppleSpeech", () => { it.effect("transcribes Float32 PCM through a 16-bit WAV file", () => { const { layer, commands, written } = setup({ stdout: '{"value":"hello"}', exitCode: 0 }); - // An unaligned view, as a structured-clone or subarray caller may pass. + // An unaligned Buffer view: Buffer.slice and subarray do not copy. const pcm = new Uint8Array(17); new DataView(pcm.buffer).setFloat32(1, 0, true); new DataView(pcm.buffer).setFloat32(5, 1, true); new DataView(pcm.buffer).setFloat32(9, -1, true); new DataView(pcm.buffer).setFloat32(13, 2, true); return Effect.gen(function* () { - const result = yield* (yield* AppleSpeech.AppleSpeech).transcribe("en-US", pcm.subarray(1)); + const result = yield* (yield* AppleSpeech.AppleSpeech).transcribe( + "en-US", + Buffer.from(pcm.buffer).subarray(1), + ); assert.deepEqual(result, { ok: true, value: "hello" }); assert.deepEqual(commands, [ [ diff --git a/apps/desktop/src/speech/AppleSpeech.ts b/apps/desktop/src/speech/AppleSpeech.ts index 14db0f05992c6..b8ab710be3fff 100644 --- a/apps/desktop/src/speech/AppleSpeech.ts +++ b/apps/desktop/src/speech/AppleSpeech.ts @@ -25,9 +25,11 @@ const decodeOutput = Schema.decodeUnknownEffect( /** 16 kHz mono Float32 PCM to a 16-bit WAV that AVAudioFile reads. */ function encodeSpeechWav(pcmBytes: Uint8Array): Uint8Array { - // Float32Array needs a 4-byte-aligned offset; copy a view that is not. + // Float32Array needs a 4-byte-aligned offset; copy a view that is not (Buffer.slice is a view). const aligned = - pcmBytes.byteOffset % Float32Array.BYTES_PER_ELEMENT === 0 ? pcmBytes : pcmBytes.slice(); + pcmBytes.byteOffset % Float32Array.BYTES_PER_ELEMENT === 0 + ? pcmBytes + : new Uint8Array(pcmBytes); const samples = new Float32Array( aligned.buffer, aligned.byteOffset, From 7451539a65518f9e82eb5009a84db10ed1746ce5 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 17 Nov 2026 03:16:21 +0000 Subject: [PATCH 07/12] refactor(desktop): resolve the Apple Speech helper path directly Co-Authored-By: Claude Opus 5.5 --- apps/desktop/src/speech/AppleSpeech.ts | 31 +++++++------------ .../settings/VoiceSettingsPanel.tsx | 2 +- 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/apps/desktop/src/speech/AppleSpeech.ts b/apps/desktop/src/speech/AppleSpeech.ts index b8ab710be3fff..54eb5f81b6b98 100644 --- a/apps/desktop/src/speech/AppleSpeech.ts +++ b/apps/desktop/src/speech/AppleSpeech.ts @@ -89,25 +89,18 @@ export const layer = Layer.effect( let helper: string | undefined; if (environment.platform === "darwin") { - const relative = environment.path.join("apple-speech", "t3-apple-speech"); - const candidates = environment.isPackaged - ? [environment.path.join(environment.resourcesPath, relative)] - : [ - environment.path.join( - environment.rootDir, - "native", - "apple-speech", - "build", - environment.processArch, - "t3-apple-speech", - ), - ]; - for (const candidate of candidates) { - if (yield* fileSystem.exists(candidate).pipe(Effect.orElseSucceed(() => false))) { - helper = candidate; - break; - } - } + const candidate = environment.isPackaged + ? environment.path.join(environment.resourcesPath, "apple-speech", "t3-apple-speech") + : environment.path.join( + environment.rootDir, + "native", + "apple-speech", + "build", + environment.processArch, + "t3-apple-speech", + ); + if (yield* fileSystem.exists(candidate).pipe(Effect.orElseSucceed(() => false))) + helper = candidate; } const run = (args: readonly string[], timeout: Duration.Input) => diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.tsx index 2c4c45600be8a..ac123fc9383b9 100644 --- a/apps/web/src/components/settings/VoiceSettingsPanel.tsx +++ b/apps/web/src/components/settings/VoiceSettingsPanel.tsx @@ -139,9 +139,9 @@ import { searchableSetting } from "./settingsSearch"; import { useOptionalSettingsScope, useSettingsScope } from "./SettingsScopeContext"; import { useScopedModelDisabledReason } from "./useScopedModelAvailability"; import { useScopedSettings, useUpdateScopedSettings } from "./useScopedSettings"; -const DEVICE = "device"; const OFF = "off"; +const DEVICE = "device"; function CloudSpeechLogo({ provider }: { provider: CloudSpeechProvider }) { if (provider === "openrouter") return ; From b6814984d6c78a2b72a519d9c04790c5e0ce6f84 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 20 Nov 2026 16:49:20 -0800 Subject: [PATCH 08/12] fix(desktop): handle Apple Speech cancellation and supported locales --- apps/desktop/src/ipc/methods/appleSpeech.ts | 4 +- apps/desktop/src/preload.ts | 3 +- apps/desktop/src/speech/AppleSpeech.test.ts | 41 ++++++++++++--- apps/desktop/src/speech/AppleSpeech.ts | 43 ++++++++++------ .../chat/GlobalVoiceInputIndicator.tsx | 3 +- .../settings/VoiceSettingsPanel.tsx | 11 ++-- apps/web/src/speech/appleSpeech.ts | 44 ++++++++++------ .../speech/browserVoiceInput.browser.test.ts | 50 ++++++++++++++++++- apps/web/src/speech/browserVoiceInput.ts | 28 +++++++++-- ...useEnvironmentSpeechInput.browser.test.tsx | 17 +++++-- .../src/speech/useEnvironmentSpeechInput.ts | 4 +- native/apple-speech/AppleSpeech.swift | 5 +- packages/contracts/src/ipc.ts | 2 +- 13 files changed, 196 insertions(+), 59 deletions(-) diff --git a/apps/desktop/src/ipc/methods/appleSpeech.ts b/apps/desktop/src/ipc/methods/appleSpeech.ts index 8396db6f29b3d..c2d631a00814e 100644 --- a/apps/desktop/src/ipc/methods/appleSpeech.ts +++ b/apps/desktop/src/ipc/methods/appleSpeech.ts @@ -12,9 +12,9 @@ import * as DesktopIpc from "../DesktopIpc.ts"; export const appleSpeechAvailable = DesktopIpc.makeIpcMethod({ channel: IpcChannels.APPLE_SPEECH_AVAILABLE_CHANNEL, payload: Schema.Void, - result: Schema.Boolean, + result: Schema.Array(Schema.String), handler: Effect.fn("desktop.ipc.appleSpeech.available")(function* () { - return yield* (yield* AppleSpeech.AppleSpeech).isAvailable; + return yield* (yield* AppleSpeech.AppleSpeech).supportedLocales; }), }); diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index a042718b3f6d0..ac52fbc73d868 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -263,7 +263,8 @@ contextBridge.exposeInMainWorld("desktopBridge", { checkSystemPermission: (pane: string) => ipcRenderer.invoke(IpcChannels.CHECK_SYSTEM_PERMISSION_CHANNEL, pane), appleSpeech: { - isAvailable: () => ipcRenderer.invoke(IpcChannels.APPLE_SPEECH_AVAILABLE_CHANNEL, undefined), + supportedLocales: () => + ipcRenderer.invoke(IpcChannels.APPLE_SPEECH_AVAILABLE_CHANNEL, undefined), prepare: (locale: string) => ipcRenderer.invoke(IpcChannels.APPLE_SPEECH_PREPARE_CHANNEL, locale), transcribe: (input: { locale: string; pcm: Uint8Array }) => diff --git a/apps/desktop/src/speech/AppleSpeech.test.ts b/apps/desktop/src/speech/AppleSpeech.test.ts index 0a6cf839f113a..a4f1a161b706d 100644 --- a/apps/desktop/src/speech/AppleSpeech.test.ts +++ b/apps/desktop/src/speech/AppleSpeech.test.ts @@ -1,5 +1,7 @@ import { assert, describe, it } from "@effect/vitest"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Sink from "effect/Sink"; @@ -16,6 +18,7 @@ type HelperOutput = { stdout?: string; stderr?: string; exitCode: number; + exit?: Deferred.Deferred; }; function makeProcess(result: HelperOutput) { @@ -24,7 +27,9 @@ function makeProcess(result: HelperOutput) { stdout: text(result.stdout ?? ""), stderr: text(result.stderr ?? ""), all: Stream.empty, - exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(result.exitCode)), + exitCode: (result.exit ? Deferred.await(result.exit) : Effect.void).pipe( + Effect.as(ChildProcessSpawner.ExitCode(result.exitCode)), + ), isRunning: Effect.succeed(false), kill: () => Effect.void, stdin: Sink.drain, @@ -94,18 +99,38 @@ describe("AppleSpeech", () => { }).pipe(Effect.provide(layer)); }); + it.effect("shares one helper between prepares that start while the first is running", () => + Effect.gen(function* () { + const exit = yield* Deferred.make(); + const { layer, commands } = setup({ stdout: '{"value":"en-US"}', exitCode: 0, exit }); + return yield* Effect.gen(function* () { + const speech = yield* AppleSpeech.AppleSpeech; + const first = yield* speech + .prepare("en-US") + .pipe(Effect.forkChild({ startImmediately: true })); + const second = yield* speech + .prepare("en-US") + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Deferred.succeed(exit, undefined); + assert.deepEqual(yield* Fiber.join(first), { ok: true, value: "en-US" }); + assert.deepEqual(yield* Fiber.join(second), { ok: true, value: "en-US" }); + assert.lengthOf(commands, 1); + }).pipe(Effect.provide(layer)); + }), + ); + it.effect.each([ - [0, true, 1], - [3, false, 1], - [1, false, 2], + [0, ["en-US", "fr-FR"], 1], + [3, [], 1], + [1, [], 2], ] as const)( - "isAvailable with helper exit %i is %s and probes %i time(s) for two calls", + "resolves locales on helper exit %i to %j with %i probe(s) across two calls", ([exitCode, expected, probes]) => { - const { layer, commands } = setup({ stdout: '{"value":""}', exitCode }); + const { layer, commands } = setup({ stdout: '{"value":"en-US,fr-FR"}', exitCode }); return Effect.gen(function* () { const speech = yield* AppleSpeech.AppleSpeech; - assert.equal(yield* speech.isAvailable, expected); - assert.equal(yield* speech.isAvailable, expected); + assert.deepEqual(yield* speech.supportedLocales, expected); + assert.deepEqual(yield* speech.supportedLocales, expected); assert.lengthOf(commands, probes); }).pipe(Effect.provide(layer)); }, diff --git a/apps/desktop/src/speech/AppleSpeech.ts b/apps/desktop/src/speech/AppleSpeech.ts index 39759d8a1d96b..f9b74f8aa501c 100644 --- a/apps/desktop/src/speech/AppleSpeech.ts +++ b/apps/desktop/src/speech/AppleSpeech.ts @@ -6,6 +6,7 @@ import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; @@ -15,7 +16,6 @@ const SAMPLE_RATE = 16_000; const MAX_PCM_BYTES = SAMPLE_RATE * Float32Array.BYTES_PER_ELEMENT * 5 * 60; // Preparing a language for the first time downloads Apple's model. const PREPARE_TIMEOUT = Duration.minutes(10); -// Matches the longest accepted recording, so analysis can never outlast its deadline. const TRANSCRIBE_TIMEOUT = Duration.minutes(5); const decodeOutput = Schema.decodeUnknownEffect( @@ -63,7 +63,7 @@ function encodeSpeechWav(pcmBytes: Uint8Array): Uint8Array { export class AppleSpeech extends Context.Service< AppleSpeech, { - readonly isAvailable: Effect.Effect; + readonly supportedLocales: Effect.Effect; readonly prepare: (locale: string) => Effect.Effect; readonly transcribe: ( locale: string, @@ -135,24 +135,39 @@ export const layer = Layer.effect( ); // Only a definite answer is kept; a slow or failed probe is asked again next time. - let available: boolean | undefined; - const isAvailable = Effect.suspend(() => - available === undefined + let locales: readonly string[] | undefined; + const supportedLocales = Effect.suspend(() => + locales === undefined ? run(["available"], Duration.seconds(10)).pipe( - Effect.tap((result) => - Effect.sync(() => { - if (result.ok || result.code === "unavailable") available = result.ok; - }), - ), - Effect.map((result) => result.ok), + Effect.map((result) => { + if (result.ok) locales = result.value.split(",").filter(Boolean); + else if (result.code === "unavailable") locales = []; + return locales ?? []; + }), ) - : Effect.succeed(available), + : Effect.succeed(locales), ); + const pending = new Map>(); + const preparePermit = yield* Semaphore.make(1); + const getPendingPrepare = Effect.fn("AppleSpeech.getPendingPrepare")(function* ( + locale: string, + ) { + const existing = pending.get(locale); + if (existing) return existing; + const preparation = yield* Effect.cached( + run(["prepare", locale], PREPARE_TIMEOUT).pipe( + Effect.ensuring(Effect.sync(() => pending.delete(locale))), + ), + ); + pending.set(locale, preparation); + return preparation; + }, preparePermit.withPermits(1)); + return AppleSpeech.of({ - isAvailable, + supportedLocales, prepare: Effect.fn("AppleSpeech.prepare")(function* (locale) { - return yield* run(["prepare", locale], PREPARE_TIMEOUT); + return yield* yield* getPendingPrepare(locale); }), transcribe: Effect.fn("AppleSpeech.transcribe")(function* (locale, pcm) { if ( diff --git a/apps/web/src/components/chat/GlobalVoiceInputIndicator.tsx b/apps/web/src/components/chat/GlobalVoiceInputIndicator.tsx index dbe0453df2e9a..b756e1beab9ad 100644 --- a/apps/web/src/components/chat/GlobalVoiceInputIndicator.tsx +++ b/apps/web/src/components/chat/GlobalVoiceInputIndicator.tsx @@ -15,8 +15,7 @@ import { ComposerSpeechRecordingPill } from "./ComposerSpeechButton"; */ export function GlobalVoiceInputIndicator({ hidden = false }: { hidden?: boolean }) { const gt = useGT(); - const { prepared, grantId, cloudProvider, cloudModel } = useVoiceTranscriptionDestination(); - const session = useRetainedVoiceInputSession({ prepared, grantId, cloudProvider, cloudModel }); + const session = useRetainedVoiceInputSession(useVoiceTranscriptionDestination()); const { state, target, recordingStartedAt } = useVoiceInputSessionSnapshot(session); const presented = useVoiceOwnerPresented(target?.ownerKey ?? null); const { level } = useVoiceInputSessionActivity( diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.tsx index d4d7d40a097a3..bd6d38b3c84ab 100644 --- a/apps/web/src/components/settings/VoiceSettingsPanel.tsx +++ b/apps/web/src/components/settings/VoiceSettingsPanel.tsx @@ -60,7 +60,7 @@ import { useEffect, useRef, useState, type ReactNode } from "react"; import { msg, T, useGT, useLocale, useMessages, Var } from "gt-react"; import { useStableCallback } from "~/hooks/useStableCallback"; -import { useAppleSpeechAvailable } from "../../speech/appleSpeech"; +import { useAppleSpeechLocales } from "../../speech/appleSpeech"; import { cn } from "~/lib/utils"; import { listAudioInputs, @@ -1325,7 +1325,8 @@ export function VoiceSettingsPanel() { const settings = useClientSettings(); const settingsScope = useOptionalSettingsScope(); const destination = useVoiceTranscriptionDestination(); - const appleSpeechAvailable = useAppleSpeechAvailable(); + const appleSpeechLocales = useAppleSpeechLocales(); + const appleSpeechAvailable = appleSpeechLocales.length > 0; const appleSpeechLabel = appleSpeechAvailable ? APPLE_SPEECH_LABEL : gt("Apple Speech (unavailable)"); @@ -1693,7 +1694,11 @@ export function VoiceSettingsPanel() { control={ } diff --git a/apps/web/src/speech/appleSpeech.ts b/apps/web/src/speech/appleSpeech.ts index 9d8e2f5899a20..ba84629c61cd6 100644 --- a/apps/web/src/speech/appleSpeech.ts +++ b/apps/web/src/speech/appleSpeech.ts @@ -1,25 +1,41 @@ import { useSyncExternalStore } from "react"; -let available = false; +const emptyLocales: readonly string[] = []; +let locales = emptyLocales; + +export function readAppleSpeechLocales() { + return locales; +} let probe: Promise | undefined; const listeners = new Set<() => void>(); +function probeLocales() { + if (locales.length === 0 && !probe && window.desktopBridge?.appleSpeech) { + probe = window.desktopBridge.appleSpeech + .supportedLocales() + .then((value) => { + if (value.length === 0) return; + locales = value; + listeners.forEach((notify) => notify()); + }) + .catch(() => {}) + .finally(() => { + probe = undefined; + }); + } +} + function subscribe(listener: () => void) { + if (listeners.size === 0) window.addEventListener("focus", probeLocales); listeners.add(listener); - probe ??= (window.desktopBridge?.appleSpeech?.isAvailable() ?? Promise.resolve(false)) - .catch(() => false) - .then((value) => { - available = value; - listeners.forEach((notify) => notify()); - }); - return () => listeners.delete(listener); + probeLocales(); + return () => { + listeners.delete(listener); + if (listeners.size === 0) window.removeEventListener("focus", probeLocales); + }; } /** Apple Speech runs only in the desktop app on macOS 26 or later. */ -export function useAppleSpeechAvailable() { - return useSyncExternalStore( - subscribe, - () => available, - () => false, - ); +export function useAppleSpeechLocales() { + return useSyncExternalStore(subscribe, readAppleSpeechLocales, () => emptyLocales); } diff --git a/apps/web/src/speech/browserVoiceInput.browser.test.ts b/apps/web/src/speech/browserVoiceInput.browser.test.ts index 663f2d81896b9..7acf17f96aa81 100644 --- a/apps/web/src/speech/browserVoiceInput.browser.test.ts +++ b/apps/web/src/speech/browserVoiceInput.browser.test.ts @@ -10,7 +10,10 @@ import { expect, it, vi } from "vite-plus/test"; import { stubGlobal } from "../../test/render"; import { createBrowserPcmCapture } from "./browserPcmCapture"; -import { createBrowserVoiceInputPlatform } from "./browserVoiceInput"; +import { + createAppleSpeechVoiceInputPlatform, + createBrowserVoiceInputPlatform, +} from "./browserVoiceInput"; vi.mock("../connection/runtime", async () => { const [{ Atom }, { layerRemoteHttpClient }] = await Promise.all([ @@ -194,6 +197,51 @@ it("retains all PCM until stop, retries batch audio, and captures preferences be expect(platform.recorder.uri).toBeNull(); }); +it.each(["prepare", "transcribe"] as const)( + "cancels Apple Speech while native %s is pending", + async (stage) => { + const entered = Promise.withResolvers(); + const pending = Promise.withResolvers<{ ok: true; value: string }>(); + using _bridge = stubGlobal("desktopBridge", { + appleSpeech: { + prepare: async () => { + if (stage !== "prepare") return { ok: true, value: "en-US" }; + entered.resolve(); + return pending.promise; + }, + transcribe: async () => { + entered.resolve(); + return pending.promise; + }, + }, + }); + const platform = createAppleSpeechVoiceInputPlatform({ + hasAudioConsent: () => true, + getTranscriptionOptions: () => DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, + getMicrophoneId: () => "", + onLevel() {}, + onDurationLimit() {}, + onError() {}, + }); + const abort = new AbortController(); + using _cleanup = { + [Symbol.dispose]: () => { + abort.abort(); + pending.resolve({ ok: true, value: "late" }); + platform.cancelRecording(); + }, + }; + const options = { signal: abort.signal }; + const transcription = await platform.transcriber.prepare(options); + const uri = URL.createObjectURL(new Blob([new Float32Array([0.5])])); + using _blob = { [Symbol.dispose]: () => URL.revokeObjectURL(uri) }; + const result = transcription.transcribe(uri, options); + await entered.promise; + abort.abort(); + await expect(result).rejects.toThrow(); + }, +); + it("streams the opening audio in order when the destination becomes ready during capture", async () => { using microphone = capture(); const feed = vi.fn(); diff --git a/apps/web/src/speech/browserVoiceInput.ts b/apps/web/src/speech/browserVoiceInput.ts index 6af0579a3cab7..994a543e0c933 100644 --- a/apps/web/src/speech/browserVoiceInput.ts +++ b/apps/web/src/speech/browserVoiceInput.ts @@ -20,6 +20,7 @@ import type { import type { CloudSpeechModelId, CloudSpeechProvider } from "@t3tools/contracts"; import * as Schema from "effect/Schema"; +import { readAppleSpeechLocales } from "./appleSpeech"; import { createBrowserPcmCapture } from "./browserPcmCapture"; import { runEnvironmentSpeech } from "./runEnvironmentSpeech"; import { msg } from "gt-react"; @@ -146,6 +147,18 @@ export function createBrowserVoiceInputPlatform(input: { }); } +async function waitForAppleSpeech(request: Promise, signal: AbortSignal): Promise { + const aborted = Promise.withResolvers(); + const onAbort = () => aborted.reject(signal.reason); + signal.addEventListener("abort", onAbort, { once: true }); + if (signal.aborted) onAbort(); + try { + return await Promise.race([request, aborted.promise]); + } finally { + signal.removeEventListener("abort", onAbort); + } +} + /** Records like the environment path, then transcribes in the desktop main process; audio stays on this Mac. */ export function createAppleSpeechVoiceInputPlatform(input: { readonly hasAudioConsent: () => boolean; @@ -160,11 +173,15 @@ export function createAppleSpeechVoiceInputPlatform(input: { const { speechLanguage } = input.getTranscriptionOptions(); // The packaged app pins `navigator.language` to en-US, so "auto" asks the OS. const system = window.desktopBridge?.getPreferredSystemLanguages?.()[0] ?? navigator.language; - const prepared = await appleSpeech?.prepare( - speechLanguage === "auto" ? system : speechLanguage, + if (!appleSpeech) throw new Error("Apple Speech is not available."); + signal.throwIfAborted(); + const prepared = await waitForAppleSpeech( + appleSpeech.prepare( + readAppleSpeechLocales().includes(speechLanguage) ? speechLanguage : system, + ), + signal, ); throwIfVoiceTranscriptionAborted(signal); - if (!appleSpeech || !prepared) throw new Error("Apple Speech is not available."); if (!prepared.ok) { if (prepared.code === "failed") throw new Error("Apple Speech could not prepare."); throw new VoiceTranscriptionError({ @@ -187,7 +204,10 @@ export function createAppleSpeechVoiceInputPlatform(input: { code: "cancelled", message: m(msg("Voice transcription was cancelled.")), }); - const result = await appleSpeech.transcribe({ locale, pcm }); + const result = await waitForAppleSpeech( + appleSpeech.transcribe({ locale, pcm }), + transcriptionSignal, + ); throwIfVoiceTranscriptionAborted(transcriptionSignal); if (!result.ok) throw new Error("Apple Speech could not transcribe."); return result.value; diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx index 1ffe2b44a08d7..8519d6a1ce0cc 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx +++ b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx @@ -176,7 +176,9 @@ vi.mock("@t3tools/client-runtime/voice-input", async (importOriginal) => ({ return { text: `Cleaned ${transcript}` }; }), })); -vi.mock("./appleSpeech", () => ({ useAppleSpeechAvailable: () => mocks.appleSpeech })); +vi.mock("./appleSpeech", () => ({ + useAppleSpeechLocales: () => (mocks.appleSpeech ? ["en-US"] : []), +})); vi.mock("./browserVoiceInput", () => ({ createAppleSpeechVoiceInputPlatform: ( input: Omit[0], "prepared">, @@ -433,10 +435,15 @@ it("retries a failed transcription on the same chosen environment", async () => }); it.each([ - ["a stored thread draft keeps recording into it", true], - ["an editor-only draft is abandoned", false], -])("when the editor switches owner mid-recording, %s", async (_name, persistent) => { + ["a stored thread draft keeps recording into it", true, false], + ["an editor-only draft is abandoned", false, false], + ["Apple Speech keeps recording into a stored thread draft", true, true], +])("when the editor switches owner mid-recording, %s", async (_name, persistent, onDevice) => { setup({ destination: "env-a" }); + if (onDevice) { + mocks.appleSpeech = true; + chooseAudioDestination({ kind: "device" }); + } mocks.storedDraft = "Earlier text"; const view = (ownerKey: string) => ( <> @@ -464,7 +471,7 @@ it.each([ await page.getByRole("button", { name: "Finish voice input" }).click(); await vi.waitFor(() => expect(mocks.storedDraft).toBe("Earlier text dictated")); - expect(mocks.transcribeCalls).toEqual(["env-a"]); + expect(mocks.transcribeCalls).toEqual([onDevice ? "apple-speech" : "env-a"]); expect(mocks.committed).not.toHaveBeenCalled(); }); diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.ts b/apps/web/src/speech/useEnvironmentSpeechInput.ts index 54798ce5fafbc..0ad3c4b2054e4 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.ts +++ b/apps/web/src/speech/useEnvironmentSpeechInput.ts @@ -33,7 +33,7 @@ import { } from "../hooks/useSettings"; import { useEnvironment } from "../state/environments"; import { useEnvironmentScope, usePreparedConnection } from "../state/session"; -import { useAppleSpeechAvailable } from "./appleSpeech"; +import { useAppleSpeechLocales } from "./appleSpeech"; import { preferredSpeechLanguages } from "./preferredSpeechLanguages"; import { runEnvironmentSpeech } from "./runEnvironmentSpeech"; import { useAudioChoice } from "./voiceConsent"; @@ -125,7 +125,7 @@ export function useVoiceTranscriptionDestination() { const hydrated = useClientSettingsHydrated(); const choice = useAudioChoice(); const grant = choice === "off" ? null : choice; - const appleSpeechAvailable = useAppleSpeechAvailable(); + const appleSpeechAvailable = useAppleSpeechLocales().length > 0; const device = hydrated && grant?.destination.kind === "device"; const environmentId = hydrated && grant && grant.destination.kind !== "device" diff --git a/native/apple-speech/AppleSpeech.swift b/native/apple-speech/AppleSpeech.swift index a315a189fa9ba..3121f55e33dc3 100644 --- a/native/apple-speech/AppleSpeech.swift +++ b/native/apple-speech/AppleSpeech.swift @@ -1,7 +1,7 @@ // Adapted from AppleTranscriptionImpl.swift in @react-native-ai/apple (MIT, Callstack), // including this repo's patch, so the Mac and iPhone transcribe the same way. // -// t3-apple-speech available -> {"value":""} +// t3-apple-speech available -> {"value":"en-US,fr-FR,..."} // t3-apple-speech prepare -> {"value":"en-US"} (resolved locale) // t3-apple-speech transcribe -> {"value":"..."} (transcript) // Exit codes are consumed by AppleSpeech.ts; failures print one message line to stderr. @@ -30,7 +30,8 @@ struct AppleSpeech { switch (arguments.first, arguments.count) { case ("available", 1): try requireAvailable() - try write("") + let locales = await SpeechTranscriber.supportedLocales + try write(locales.map { $0.identifier(.bcp47) }.joined(separator: ",")) case ("prepare", 2): try write(try await prepare(arguments[1])) case ("transcribe", 3): diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index a33b1b5ee18e3..962cfd089dd2b 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -1243,7 +1243,7 @@ export interface DesktopBridge { preview: DesktopPreviewBridge; /** On-device dictation through macOS 26 SpeechAnalyzer; absent in older desktop builds. */ appleSpeech?: { - isAvailable: () => Promise; + supportedLocales: () => Promise; prepare: (locale: string) => Promise; transcribe: (input: { locale: string; pcm: Uint8Array }) => Promise; }; From 47559ea1379028a0da5de039370ce985a0ced721 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 20 Nov 2026 16:52:50 -0800 Subject: [PATCH 09/12] fix(desktop): honor saved Apple Speech language preferences --- .../settings/SpeechLanguageSelect.tsx | 11 +++++++++-- .../components/settings/VoiceSettingsPanel.tsx | 14 ++++++++++++-- apps/web/src/speech/appleSpeech.ts | 18 ++++++++++++++++-- .../speech/browserVoiceInput.browser.test.ts | 11 +++++++++++ apps/web/src/speech/browserVoiceInput.ts | 7 ++++--- 5 files changed, 52 insertions(+), 9 deletions(-) diff --git a/apps/web/src/components/settings/SpeechLanguageSelect.tsx b/apps/web/src/components/settings/SpeechLanguageSelect.tsx index 4ddff5b92a4d5..21ca449655dc5 100644 --- a/apps/web/src/components/settings/SpeechLanguageSelect.tsx +++ b/apps/web/src/components/settings/SpeechLanguageSelect.tsx @@ -19,7 +19,13 @@ type ModelLanguages = Pick< "languages" | "supportsLanguageDetection" >; -export function SpeechLanguageSelect({ model }: { readonly model: ModelLanguages | null }) { +export function SpeechLanguageSelect({ + model, + automaticLabel, +}: { + readonly model: ModelLanguages | null; + readonly automaticLabel?: string | undefined; +}) { const gt = useGT(); const locale = useLocale(); const preference = useClientSettings((settings) => settings.speechLanguage); @@ -30,7 +36,8 @@ export function SpeechLanguageSelect({ model }: { readonly model: ModelLanguages preference, ) : preference; - const effectiveLabel = resolved === "auto" ? gt("Auto-detect") : languageLabel(resolved, locale); + const effectiveLabel = + resolved === "auto" ? (automaticLabel ?? gt("Auto-detect")) : languageLabel(resolved, locale); const unavailable = preference !== "auto" && preference !== resolved ? gt("{language} unavailable", { language: languageLabel(preference, locale) }) diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.tsx index bd6d38b3c84ab..84a76f373c8be 100644 --- a/apps/web/src/components/settings/VoiceSettingsPanel.tsx +++ b/apps/web/src/components/settings/VoiceSettingsPanel.tsx @@ -60,7 +60,7 @@ import { useEffect, useRef, useState, type ReactNode } from "react"; import { msg, T, useGT, useLocale, useMessages, Var } from "gt-react"; import { useStableCallback } from "~/hooks/useStableCallback"; -import { useAppleSpeechLocales } from "../../speech/appleSpeech"; +import { supportsAppleSpeechLocale, useAppleSpeechLocales } from "../../speech/appleSpeech"; import { cn } from "~/lib/utils"; import { listAudioInputs, @@ -1326,6 +1326,11 @@ export function VoiceSettingsPanel() { const settingsScope = useOptionalSettingsScope(); const destination = useVoiceTranscriptionDestination(); const appleSpeechLocales = useAppleSpeechLocales(); + const appleSpeechLanguages = + !appleSpeechLocales.includes(settings.speechLanguage) && + supportsAppleSpeechLocale(appleSpeechLocales, settings.speechLanguage) + ? [...appleSpeechLocales, settings.speechLanguage] + : appleSpeechLocales; const appleSpeechAvailable = appleSpeechLocales.length > 0; const appleSpeechLabel = appleSpeechAvailable ? APPLE_SPEECH_LABEL @@ -1693,9 +1698,14 @@ export function VoiceSettingsPanel() { } control={ | undefined; +const listeners = new Set<() => void>(); export function readAppleSpeechLocales() { return locales; } -let probe: Promise | undefined; -const listeners = new Set<() => void>(); + +export function supportsAppleSpeechLocale(available: readonly string[], locale: string) { + const language = locale.toLowerCase().split("-")[0]; + return available.some((candidate) => candidate.toLowerCase().split("-")[0] === language); +} + +export function resolveAppleSpeechLocale( + available: readonly string[], + preference: string, + system: readonly string[], +) { + if (supportsAppleSpeechLocale(available, preference)) return preference; + return system.find((locale) => supportsAppleSpeechLocale(available, locale)) ?? system[0]; +} function probeLocales() { if (locales.length === 0 && !probe && window.desktopBridge?.appleSpeech) { diff --git a/apps/web/src/speech/browserVoiceInput.browser.test.ts b/apps/web/src/speech/browserVoiceInput.browser.test.ts index 7acf17f96aa81..5f3023a635240 100644 --- a/apps/web/src/speech/browserVoiceInput.browser.test.ts +++ b/apps/web/src/speech/browserVoiceInput.browser.test.ts @@ -9,6 +9,7 @@ import { decodeSpeechPcmRequest } from "@t3tools/shared/speech"; import { expect, it, vi } from "vite-plus/test"; import { stubGlobal } from "../../test/render"; +import { resolveAppleSpeechLocale } from "./appleSpeech"; import { createBrowserPcmCapture } from "./browserPcmCapture"; import { createAppleSpeechVoiceInputPlatform, @@ -197,6 +198,16 @@ it("retains all PCM until stop, retries batch audio, and captures preferences be expect(platform.recorder.uri).toBeNull(); }); +it.each([ + ["fr", ["en-US"], "fr"], + ["auto", ["nl-NL", "en-US"], "en-US"], +] as const)( + "resolves Apple Speech language %s with Mac preferences %j", + (preference, system, expected) => { + expect(resolveAppleSpeechLocale(["en-US", "fr-FR"], preference, system)).toBe(expected); + }, +); + it.each(["prepare", "transcribe"] as const)( "cancels Apple Speech while native %s is pending", async (stage) => { diff --git a/apps/web/src/speech/browserVoiceInput.ts b/apps/web/src/speech/browserVoiceInput.ts index 994a543e0c933..2d03019ce4ca8 100644 --- a/apps/web/src/speech/browserVoiceInput.ts +++ b/apps/web/src/speech/browserVoiceInput.ts @@ -20,7 +20,7 @@ import type { import type { CloudSpeechModelId, CloudSpeechProvider } from "@t3tools/contracts"; import * as Schema from "effect/Schema"; -import { readAppleSpeechLocales } from "./appleSpeech"; +import { readAppleSpeechLocales, resolveAppleSpeechLocale } from "./appleSpeech"; import { createBrowserPcmCapture } from "./browserPcmCapture"; import { runEnvironmentSpeech } from "./runEnvironmentSpeech"; import { msg } from "gt-react"; @@ -172,12 +172,13 @@ export function createAppleSpeechVoiceInputPlatform(input: { const appleSpeech = window.desktopBridge?.appleSpeech; const { speechLanguage } = input.getTranscriptionOptions(); // The packaged app pins `navigator.language` to en-US, so "auto" asks the OS. - const system = window.desktopBridge?.getPreferredSystemLanguages?.()[0] ?? navigator.language; + const system = window.desktopBridge?.getPreferredSystemLanguages?.() ?? []; if (!appleSpeech) throw new Error("Apple Speech is not available."); signal.throwIfAborted(); const prepared = await waitForAppleSpeech( appleSpeech.prepare( - readAppleSpeechLocales().includes(speechLanguage) ? speechLanguage : system, + resolveAppleSpeechLocale(readAppleSpeechLocales(), speechLanguage, system) ?? + navigator.language, ), signal, ); From e3ccc92eb7d4fbc33eac9f7dd0b932f843cf69c1 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 20 Nov 2026 16:55:49 -0800 Subject: [PATCH 10/12] fix(desktop): gate Apple Speech by macOS version and credit upstream --- apps/desktop/src/speech/AppleSpeech.test.ts | 17 ++++++++++------- apps/desktop/src/speech/AppleSpeech.ts | 4 +++- third-party-licenses.config.json | 15 +++++++++++++++ 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/apps/desktop/src/speech/AppleSpeech.test.ts b/apps/desktop/src/speech/AppleSpeech.test.ts index a4f1a161b706d..0e831fab8d15b 100644 --- a/apps/desktop/src/speech/AppleSpeech.test.ts +++ b/apps/desktop/src/speech/AppleSpeech.test.ts @@ -1,4 +1,5 @@ import { assert, describe, it } from "@effect/vitest"; +import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; @@ -39,12 +40,13 @@ function makeProcess(result: HelperOutput) { }); } -function setup(result: HelperOutput) { +function setup(result: HelperOutput, kernelRelease = "25.0.0") { const commands: Array> = []; const written: Uint8Array[] = []; const layer = AppleSpeech.layer.pipe( Layer.provide( Layer.mergeAll( + Layer.succeed(HostProcess.KernelRelease, kernelRelease), Layer.succeed(DesktopEnvironment.DesktopEnvironment, { platform: "darwin", isPackaged: true, @@ -120,13 +122,14 @@ describe("AppleSpeech", () => { ); it.effect.each([ - [0, ["en-US", "fr-FR"], 1], - [3, [], 1], - [1, [], 2], + ["25.0.0", 0, ["en-US", "fr-FR"], 1], + ["25.0.0", 3, [], 1], + ["25.0.0", 1, [], 2], + ["24.6.0", 0, [], 0], ] as const)( - "resolves locales on helper exit %i to %j with %i probe(s) across two calls", - ([exitCode, expected, probes]) => { - const { layer, commands } = setup({ stdout: '{"value":"en-US,fr-FR"}', exitCode }); + "on Darwin %s resolves locales on helper exit %i to %j with %i probe(s) across two calls", + ([release, exitCode, expected, probes]) => { + const { layer, commands } = setup({ stdout: '{"value":"en-US,fr-FR"}', exitCode }, release); return Effect.gen(function* () { const speech = yield* AppleSpeech.AppleSpeech; assert.deepEqual(yield* speech.supportedLocales, expected); diff --git a/apps/desktop/src/speech/AppleSpeech.ts b/apps/desktop/src/speech/AppleSpeech.ts index f9b74f8aa501c..a2f58d4f949a9 100644 --- a/apps/desktop/src/speech/AppleSpeech.ts +++ b/apps/desktop/src/speech/AppleSpeech.ts @@ -82,9 +82,11 @@ export const layer = Layer.effect( const fileSystem = yield* FileSystem.FileSystem; const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const hostEnvironment = yield* HostProcess.Environment; + const kernelRelease = yield* HostProcess.KernelRelease; let helper: string | undefined; - if (environment.platform === "darwin") { + // macOS 26 is Darwin 25, the helper's minimum deployment target. + if (environment.platform === "darwin" && Number.parseInt(kernelRelease, 10) >= 25) { const candidate = environment.isPackaged ? environment.path.join(environment.resourcesPath, "apple-speech", "t3-apple-speech") : environment.path.join( diff --git a/third-party-licenses.config.json b/third-party-licenses.config.json index 2cfb7324d9eae..48f19d112feb1 100644 --- a/third-party-licenses.config.json +++ b/third-party-licenses.config.json @@ -1,5 +1,20 @@ { "customNotices": [ + { + "bundles": ["desktop"], + "license": "MIT", + "name": "Apple Speech helper (adapted from @react-native-ai/apple)", + "sourceUrl": "https://github.com/callstackincubator/ai", + "generatedNotices": [ + { + "licenseId": "MIT", + "copyrights": [ + "Copyright (c) 2024-2025 Szymon Rybczak", + "Copyright (c) 2025-present Callstack" + ] + } + ] + }, { "bundles": ["desktop", "mobile", "web"], "license": "MIT", From 94721e05f4aa08cab937ad55785b29ca850e59e8 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 20 Nov 2026 16:56:39 -0800 Subject: [PATCH 11/12] docs: clarify how to enable Apple Speech dictation --- docs/user/faq.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/faq.md b/docs/user/faq.md index b9719c08ad49f..947344894ff4e 100644 --- a/docs/user/faq.md +++ b/docs/user/faq.md @@ -77,7 +77,7 @@ Open **Settings → Worktrees → Cleanup** and turn on automatic worktree clean ## Is there voice input? -Yes. On iPhone and Android, dictation uses on-device speech by default; you can opt in to a paired environment instead. Web and desktop require you to choose an environment first; on macOS 26 or later, the desktop app can also use Apple Speech on the Mac. The transcript goes into your draft for review. See [Voice input](./voice-input.md). +Yes. On iPhone and Android, dictation uses on-device speech by default, with the option to use a paired environment. On macOS 26 or later, desktop users can choose Apple Speech directly under **On this Mac** or choose an environment. Web and other desktop users can enable dictation by choosing an environment. The transcript goes into your draft for review. See [Voice input](./voice-input.md). ## Does it work on Windows, Mac, and Linux? From 063726be1a395535fac5e6874a74fd1ea87588c4 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 20 Nov 2026 17:10:35 -0800 Subject: [PATCH 12/12] refactor(voice): share Apple Speech transcription across iOS and macOS --- apps/desktop/scripts/build-apple-speech.mjs | 8 +- apps/mobile/fingerprint.config.js | 10 +- .../ios/AppleSpeechEngine.swift | 1 + .../ios/T3VoiceTranscriptionModule.swift | 19 ++ apps/mobile/package.json | 1 - .../src/native/voiceTranscription.ios.test.ts | 72 +++---- .../src/native/voiceTranscription.ios.ts | 74 +++---- native/apple-speech/AppleSpeech.swift | 97 +-------- native/apple-speech/AppleSpeechEngine.swift | 98 +++++++++ patches/@react-native-ai__apple@0.12.0.patch | 194 ------------------ pnpm-lock.yaml | 39 ---- pnpm-workspace.yaml | 2 - third-party-licenses.config.json | 15 +- 13 files changed, 207 insertions(+), 423 deletions(-) create mode 120000 apps/mobile/modules/t3-voice-transcription/ios/AppleSpeechEngine.swift create mode 100644 native/apple-speech/AppleSpeechEngine.swift delete mode 100644 patches/@react-native-ai__apple@0.12.0.patch diff --git a/apps/desktop/scripts/build-apple-speech.mjs b/apps/desktop/scripts/build-apple-speech.mjs index efef5a452190f..6334100ea386f 100644 --- a/apps/desktop/scripts/build-apple-speech.mjs +++ b/apps/desktop/scripts/build-apple-speech.mjs @@ -23,14 +23,16 @@ if (hostPlatform !== "darwin") { ]; if (targets === undefined) throw new Error(`Unsupported macOS architecture: ${values.arch}`); const root = NodeURL.fileURLToPath(new URL("../../../native/apple-speech/", import.meta.url)); - const source = NodePath.resolve(root, "AppleSpeech.swift"); + const sources = ["AppleSpeech.swift", "AppleSpeechEngine.swift"].map((file) => + NodePath.resolve(root, file), + ); const output = values.output ?? NodePath.resolve(root, "build", values.arch, "t3-apple-speech"); let current = false; try { current = NodeFS.statSync(output).mtimeMs >= Math.max( - NodeFS.statSync(source).mtimeMs, + ...sources.map((source) => NodeFS.statSync(source).mtimeMs), NodeFS.statSync(NodeURL.fileURLToPath(import.meta.url)).mtimeMs, ); } catch {} @@ -50,7 +52,7 @@ if (hostPlatform !== "darwin") { "-parse-as-library", "-target", `${target}-apple-macos26.0`, - source, + ...sources, "-o", slices[index], ], diff --git a/apps/mobile/fingerprint.config.js b/apps/mobile/fingerprint.config.js index 7e684229320a2..e62be49dd5a33 100644 --- a/apps/mobile/fingerprint.config.js +++ b/apps/mobile/fingerprint.config.js @@ -14,5 +14,13 @@ if (!majorVersion) { module.exports = { nativeModuleSourceType: "files", - extraSources: [{ type: "contents", id: "appMajorVersion", contents: majorVersion }], + extraSources: [ + { type: "contents", id: "appMajorVersion", contents: majorVersion }, + // Expo skips symlinks when hashing local modules. + { + type: "file", + filePath: "../../native/apple-speech/AppleSpeechEngine.swift", + reasons: ["appleSpeech"], + }, + ], }; diff --git a/apps/mobile/modules/t3-voice-transcription/ios/AppleSpeechEngine.swift b/apps/mobile/modules/t3-voice-transcription/ios/AppleSpeechEngine.swift new file mode 120000 index 0000000000000..8c54186f2ac73 --- /dev/null +++ b/apps/mobile/modules/t3-voice-transcription/ios/AppleSpeechEngine.swift @@ -0,0 +1 @@ +../../../../../native/apple-speech/AppleSpeechEngine.swift \ No newline at end of file diff --git a/apps/mobile/modules/t3-voice-transcription/ios/T3VoiceTranscriptionModule.swift b/apps/mobile/modules/t3-voice-transcription/ios/T3VoiceTranscriptionModule.swift index 8c02fbd84a9d5..f975854669dfc 100644 --- a/apps/mobile/modules/t3-voice-transcription/ios/T3VoiceTranscriptionModule.swift +++ b/apps/mobile/modules/t3-voice-transcription/ios/T3VoiceTranscriptionModule.swift @@ -6,6 +6,25 @@ public class T3VoiceTranscriptionModule: Module { public func definition() -> ModuleDefinition { Name("T3VoiceTranscription") + Function("isAppleSpeechAvailable") { + if #available(iOS 26, *) { return AppleSpeechEngine.isAvailable } + return false + } + + AsyncFunction("prepareAppleSpeech") { (locale: String) async throws -> String? in + guard #available(iOS 26, *) else { throw CocoaError(.featureUnsupported) } + do { + return try await AppleSpeechEngine.prepare(locale) + } catch AppleSpeechEngine.Failure.unsupportedLocale { + return nil + } + } + + AsyncFunction("transcribeWithAppleSpeech") { (uri: URL, locale: String) async throws -> String in + guard #available(iOS 26, *) else { throw CocoaError(.featureUnsupported) } + return try await AppleSpeechEngine.transcribe(locale, file: uri) + } + AsyncFunction("decodeRecording") { (uri: URL) throws -> Data in guard uri.isFileURL else { throw CocoaError(.fileReadUnsupportedScheme) } var opened: ExtAudioFileRef? diff --git a/apps/mobile/package.json b/apps/mobile/package.json index b6e3c57d83606..d86ac2320328b 100644 --- a/apps/mobile/package.json +++ b/apps/mobile/package.json @@ -68,7 +68,6 @@ "@material/material-color-utilities": "0.3.0", "@pierre/diffs": "catalog:", "@pierre/highlights": "catalog:", - "@react-native-ai/apple": "0.12.0", "@react-native-menu/menu": "^2.0.0", "@react-navigation/elements": "2.9.45", "@react-navigation/native": "7.5.1", diff --git a/apps/mobile/src/native/voiceTranscription.ios.test.ts b/apps/mobile/src/native/voiceTranscription.ios.test.ts index e23711626acf6..5d08ecc9aa244 100644 --- a/apps/mobile/src/native/voiceTranscription.ios.test.ts +++ b/apps/mobile/src/native/voiceTranscription.ios.test.ts @@ -1,14 +1,12 @@ -import type { TranscriptionResult } from "@react-native-ai/apple/src/NativeAppleTranscription"; import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; import { VoiceInputController, VoiceTranscriptionError } from "@t3tools/client-runtime/voice-input"; import { resetVoiceInputGlobalsForTests } from "../../../../packages/client-runtime/src/voice-input/controller"; const mocks = vi.hoisted(() => ({ - isAvailable: vi.fn<(locale: string) => boolean>(), - prepare: vi.fn<(locale: string) => Promise>(), - transcribe: vi.fn<(audio: ArrayBufferLike, locale: string) => Promise>(), - readAudio: vi.fn<() => Promise>(), + isAvailable: vi.fn<() => boolean>(), + prepare: vi.fn<(locale: string) => Promise>(), + transcribe: vi.fn<(uri: string, locale: string) => Promise>(), preferredLanguages: [] as unknown, })); @@ -17,34 +15,16 @@ vi.mock("react-native", () => ({ Settings: { get: (key: string) => (key === "AppleLanguages" ? mocks.preferredLanguages : null) }, })); -vi.mock("@react-native-ai/apple/src/NativeAppleTranscription", () => ({ - default: { - isAvailable: mocks.isAvailable, - prepare: mocks.prepare, - transcribe: mocks.transcribe, - }, -})); - -vi.mock("expo-file-system", () => ({ - File: class { - arrayBuffer = mocks.readAudio; - }, +vi.mock("expo", () => ({ + requireOptionalNativeModule: () => ({ + isAppleSpeechAvailable: mocks.isAvailable, + prepareAppleSpeech: mocks.prepare, + transcribeWithAppleSpeech: mocks.transcribe, + }), })); import { getLocalVoiceTranscriber } from "./voiceTranscription.ios"; -const audio = new ArrayBuffer(4); -const nativeTranscript: TranscriptionResult = { - duration: 2, - segments: [ - { text: " Hej", startSecond: 0, endSecond: 1 }, - { text: "världen. ", startSecond: 1, endSecond: 2 }, - ], -}; - -const unsupportedLocale = () => - Object.assign(new Error("unsupported"), { code: "AppleTranscriptionUnsupportedLocale" }); - function useLanguages(preferred: ReadonlyArray, appLocale: string) { mocks.preferredLanguages = preferred; const resolvedOptions = Intl.DateTimeFormat().resolvedOptions(); @@ -106,8 +86,7 @@ beforeEach(() => { useLanguages([], "sv-SE"); mocks.isAvailable.mockReturnValue(true); mocks.prepare.mockResolvedValue("sv-SE"); - mocks.readAudio.mockResolvedValue(audio); - mocks.transcribe.mockResolvedValue(nativeTranscript); + mocks.transcribe.mockResolvedValue("Hej världen."); }); afterEach(() => { @@ -137,19 +116,19 @@ describe("getLocalVoiceTranscriber", () => { ])("prepares $case", async ({ preferred, supported, tried }) => { useLanguages(preferred, "en-US"); mocks.prepare.mockImplementation(async (candidate) => { - if (!supported.includes(candidate)) throw unsupportedLocale(); + if (!supported.includes(candidate)) return null; return candidate; }); const locale = tried.at(-1)!; expect(await transcribeOnce()).toEqual({ text: "Hej världen.", locale }); expect(mocks.prepare.mock.calls.map(([candidate]) => candidate)).toEqual(tried); - expect(mocks.transcribe).toHaveBeenCalledWith(audio, locale); + expect(mocks.transcribe).toHaveBeenCalledWith("file:///voice.m4a", locale); }); it("reports an unsupported language when neither preferred nor app locales are supported", async () => { useLanguages(["gsw-CH"], "yi"); - mocks.prepare.mockRejectedValue(unsupportedLocale()); + mocks.prepare.mockResolvedValue(null); expect(await transcribeOnce()).toMatchObject({ code: "unsupported-locale" }); expect(mocks.prepare.mock.calls.map(([candidate]) => candidate)).toEqual(["gsw-CH", "yi"]); @@ -160,7 +139,7 @@ describe("getLocalVoiceTranscriber", () => { async (failure) => { useLanguages(["ko-KR", "ja-JP"], "en-US"); const entered = Promise.withResolvers(); - const nativePrepare = Promise.withResolvers(); + const nativePrepare = Promise.withResolvers(); mocks.prepare.mockImplementation(() => { entered.resolve(); return nativePrepare.promise; @@ -171,7 +150,7 @@ describe("getLocalVoiceTranscriber", () => { await entered.promise; if (failure === "cancellation") { controller.abort(); - nativePrepare.reject(unsupportedLocale()); + nativePrepare.resolve(null); } else { nativePrepare.reject(Object.assign(new Error("download failed"), { code: "Network" })); } @@ -184,7 +163,7 @@ describe("getLocalVoiceTranscriber", () => { ); it("records while Apple prepares its speech model and transcribes once it is ready", async () => { - const nativePrepare = Promise.withResolvers(); + const nativePrepare = Promise.withResolvers(); mocks.prepare.mockReturnValue(nativePrepare.promise); const dictation = createDictation(); @@ -204,7 +183,7 @@ describe("getLocalVoiceTranscriber", () => { it.each(["cancel", "cancel after stop", "model failure", "model failure after stop"] as const)( "discards early audio without a late commit after %s", async (ending) => { - const nativePrepare = Promise.withResolvers(); + const nativePrepare = Promise.withResolvers(); mocks.prepare.mockReturnValue(nativePrepare.promise); const dictation = createDictation(); await dictation.controller.start(); @@ -240,12 +219,12 @@ describe("getLocalVoiceTranscriber", () => { }, ); - it("does not start native transcription after cancellation during a file read", async () => { - const enteredRead = Promise.withResolvers(); - const readResult = Promise.withResolvers(); - mocks.readAudio.mockImplementation(() => { - enteredRead.resolve(); - return readResult.promise; + it("discards transcription if cancelled while the native module reads the recording", async () => { + const entered = Promise.withResolvers(); + const transcription = Promise.withResolvers(); + mocks.transcribe.mockImplementation(() => { + entered.resolve(); + return transcription.promise; }); const controller = new AbortController(); const options = { signal: controller.signal }; @@ -254,13 +233,12 @@ describe("getLocalVoiceTranscriber", () => { .transcribe("file:///voice.m4a", options) .catch((error: unknown) => error); - await enteredRead.promise; + await entered.promise; controller.abort(); - readResult.resolve(audio); + transcription.resolve("Hej världen."); const error = await result; expect(error).toBeInstanceOf(VoiceTranscriptionError); expect(error).toMatchObject({ code: "cancelled" }); - expect(mocks.transcribe).not.toHaveBeenCalled(); }); }); diff --git a/apps/mobile/src/native/voiceTranscription.ios.ts b/apps/mobile/src/native/voiceTranscription.ios.ts index e59c6349dfcd4..9701036e5ac94 100644 --- a/apps/mobile/src/native/voiceTranscription.ios.ts +++ b/apps/mobile/src/native/voiceTranscription.ios.ts @@ -1,6 +1,5 @@ import * as Schema from "effect/Schema"; -import AppleTranscription from "@react-native-ai/apple/src/NativeAppleTranscription"; -import { File } from "expo-file-system"; +import { requireOptionalNativeModule } from "expo"; import { msg } from "gt-react-native"; import { Settings } from "react-native"; import { m } from "../i18n/messages"; @@ -13,6 +12,13 @@ import { type VoiceTranscriptionOptions, } from "@t3tools/client-runtime/voice-input"; +interface AppleSpeechModule { + isAppleSpeechAvailable?: () => boolean; + prepareAppleSpeech: (locale: string) => Promise; + transcribeWithAppleSpeech: (uri: string, locale: string) => Promise; +} + +const native = requireOptionalNativeModule("T3VoiceTranscription"); const isVoiceTranscriptionError = Schema.is(VoiceTranscriptionError); /** The user's ordered Language & Region list, then the app locale; the app ships in English only. */ @@ -38,27 +44,20 @@ function wrapError( return new VoiceTranscriptionError({ code, message, cause }); } -function getNativeErrorCode(error: unknown): string | undefined { - if (typeof error !== "object" || error === null || !("code" in error)) { - return undefined; - } - - return typeof error.code === "string" ? error.code : undefined; -} - export async function getLocalVoiceTranscriber(): Promise { const locales = getPreferredLocales(); // Availability is a device and OS check; which languages have a model only `prepare` knows. - if (!AppleTranscription.isAvailable(locales[0]!)) return null; - return { prepare: (options) => prepareVoiceTranscription(locales, options) }; + if (!native?.isAppleSpeechAvailable?.()) return null; + return { prepare: (options) => prepareVoiceTranscription(native, locales, options) }; } async function prepareVoiceTranscription( + native: AppleSpeechModule, locales: readonly string[], { signal }: VoiceTranscriptionOptions, ): Promise { throwIfVoiceTranscriptionAborted(signal); - if (!AppleTranscription.isAvailable(locales[0]!)) { + if (!native?.isAppleSpeechAvailable?.()) { throw new VoiceTranscriptionError({ code: "unavailable", message: m( @@ -70,7 +69,7 @@ async function prepareVoiceTranscription( } let locale = locales[0]!; - const model = prepareLocales(locales, signal).then((supported) => { + const model = prepareLocales(native, locales, signal).then((supported) => { locale = supported; }); // Recorder setup can fail before either consumer observes the model. @@ -98,16 +97,20 @@ async function prepareVoiceTranscription( return null; }, transcribe: async (uri, options) => - transcribeVoiceRecording(uri, await ready(options.signal), options), + transcribeVoiceRecording(native, uri, await ready(options.signal), options), }; } -async function prepareLocales(locales: readonly string[], signal: AbortSignal): Promise { +async function prepareLocales( + native: AppleSpeechModule, + locales: readonly string[], + signal: AbortSignal, +): Promise { throwIfVoiceTranscriptionAborted(signal); let unsupported: VoiceTranscriptionError | undefined; for (const locale of locales) { try { - return await prepareLocale(locale, signal); + return await prepareLocale(native, locale, signal); } catch (error) { if (!isVoiceTranscriptionError(error) || error.code !== "unsupported-locale") throw error; unsupported = error; @@ -116,44 +119,43 @@ async function prepareLocales(locales: readonly string[], signal: AbortSignal): throw unsupported!; } -async function prepareLocale(locale: string, signal: AbortSignal): Promise { +async function prepareLocale( + native: AppleSpeechModule, + locale: string, + signal: AbortSignal, +): Promise { + let supportedLocale: string | null; try { - const supportedLocale = await AppleTranscription.prepare(locale); - throwIfVoiceTranscriptionAborted(signal); - return supportedLocale; + supportedLocale = await native.prepareAppleSpeech(locale); } catch (error) { throwIfVoiceTranscriptionAborted(signal); - if (getNativeErrorCode(error) === "AppleTranscriptionUnsupportedLocale") { - throw new VoiceTranscriptionError({ - code: "unsupported-locale", - message: m(msg("Voice transcription does not support any of your languages.")), - cause: error, - }); - } - throw wrapError( "preparation-failed", m(msg("Voice transcription could not prepare this language.")), error, ); } + throwIfVoiceTranscriptionAborted(signal); + if (supportedLocale === null) { + throw new VoiceTranscriptionError({ + code: "unsupported-locale", + message: m(msg("Voice transcription does not support any of your languages.")), + }); + } + return supportedLocale; } async function transcribeVoiceRecording( + native: AppleSpeechModule, uri: string, locale: string, { signal }: VoiceTranscriptionOptions, ): Promise { try { throwIfVoiceTranscriptionAborted(signal); - const audio = await new File(uri).arrayBuffer(); - throwIfVoiceTranscriptionAborted(signal); - const result = await AppleTranscription.transcribe(audio, locale); + const text = await native.transcribeWithAppleSpeech(uri, locale); throwIfVoiceTranscriptionAborted(signal); - return result.segments - .map((segment) => segment.text) - .join(" ") - .trim(); + return text; } catch (error) { throwIfVoiceTranscriptionAborted(signal); throw wrapError("transcription-failed", m(msg("Voice transcription failed.")), error); diff --git a/native/apple-speech/AppleSpeech.swift b/native/apple-speech/AppleSpeech.swift index 3121f55e33dc3..25dce03560c57 100644 --- a/native/apple-speech/AppleSpeech.swift +++ b/native/apple-speech/AppleSpeech.swift @@ -1,14 +1,9 @@ -// Adapted from AppleTranscriptionImpl.swift in @react-native-ai/apple (MIT, Callstack), -// including this repo's patch, so the Mac and iPhone transcribe the same way. -// // t3-apple-speech available -> {"value":"en-US,fr-FR,..."} // t3-apple-speech prepare -> {"value":"en-US"} (resolved locale) // t3-apple-speech transcribe -> {"value":"..."} (transcript) // Exit codes are consumed by AppleSpeech.ts; failures print one message line to stderr. -import AVFoundation import Foundation -import Speech private enum ExitCode: Int32 { case failure = 1 @@ -17,11 +12,6 @@ private enum ExitCode: Int32 { case usage = 64 } -private struct Failure: Error { - let code: ExitCode - let message: String -} - @main struct AppleSpeech { static func main() async { @@ -29,89 +19,22 @@ struct AppleSpeech { do { switch (arguments.first, arguments.count) { case ("available", 1): - try requireAvailable() - let locales = await SpeechTranscriber.supportedLocales - try write(locales.map { $0.identifier(.bcp47) }.joined(separator: ",")) + try write(try await AppleSpeechEngine.supportedLocales().joined(separator: ",")) case ("prepare", 2): - try write(try await prepare(arguments[1])) + try write(try await AppleSpeechEngine.prepare(arguments[1])) case ("transcribe", 3): - try write(try await transcribe(arguments[1], path: arguments[2])) + try write(try await AppleSpeechEngine.transcribe(arguments[1], file: URL(fileURLWithPath: arguments[2]))) default: - throw Failure(code: .usage, message: "Unknown command.") + fail(.usage, "Unknown command.") } - } catch let failure as Failure { - fail(failure.code, failure.message) - } catch { - fail(.failure, error.localizedDescription) - } - } - - private static func requireAvailable() throws { - guard SpeechTranscriber.isAvailable else { - throw Failure(code: .unavailable, message: "Speech transcription is not available on this Mac.") - } - } - - private static func makeTranscriber(for locale: Locale) -> SpeechTranscriber { - let preset = SpeechTranscriber.Preset.timeIndexedTranscriptionWithAlternatives - return SpeechTranscriber( - locale: locale, - transcriptionOptions: preset.transcriptionOptions, - reportingOptions: preset.reportingOptions.subtracting([.alternativeTranscriptions]), - attributeOptions: preset.attributeOptions - ) - } - - private static func prepare(_ identifier: String) async throws -> String { - try requireAvailable() - guard - let locale = await SpeechTranscriber.supportedLocale( - equivalentTo: Locale(identifier: identifier)) - else { - throw Failure(code: .unsupportedLocale, message: "Locale not supported: \(identifier)") - } - let transcriber = makeTranscriber(for: locale) - switch await AssetInventory.status(forModules: [transcriber]) { - case .installed: - break - case .supported, .downloading: - if let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) { - try await request.downloadAndInstall() - } - case .unsupported: - throw Failure( - code: .unsupportedLocale, message: "Assets not supported for locale: \(identifier)") - @unknown default: - throw Failure(code: .failure, message: "Unknown asset inventory status.") - } - return locale.identifier(.bcp47) - } - - private static func transcribe(_ identifier: String, path: String) async throws -> String { - try requireAvailable() - let audioFile = try AVAudioFile(forReading: URL(fileURLWithPath: path)) - let transcriber = makeTranscriber(for: Locale(identifier: identifier)) - let analyzer = SpeechAnalyzer(modules: [transcriber]) - let collector = Task { () throws -> [String] in - var texts: [String] = [] - for try await result in transcriber.results where result.isFinal { - let text = String(result.text.characters).trimmingCharacters(in: .whitespacesAndNewlines) - if !text.isEmpty { texts.append(text) } + } catch let failure as AppleSpeechEngine.Failure { + switch failure { + case .unsupportedLocale: fail(.unsupportedLocale, failure.localizedDescription) + case .unavailable: fail(.unavailable, failure.localizedDescription) + case .unknownAssetStatus: fail(.failure, failure.localizedDescription) } - return texts - } - do { - if let lastSampleTime = try await analyzer.analyzeSequence(from: audioFile) { - try await analyzer.finalizeAndFinish(through: lastSampleTime) - } else { - await analyzer.cancelAndFinishNow() - } - return try await collector.value.joined(separator: " ") } catch { - collector.cancel() - await analyzer.cancelAndFinishNow() - _ = try? await collector.value - throw error + fail(.failure, error.localizedDescription) } } diff --git a/native/apple-speech/AppleSpeechEngine.swift b/native/apple-speech/AppleSpeechEngine.swift new file mode 100644 index 0000000000000..cf140384a4030 --- /dev/null +++ b/native/apple-speech/AppleSpeechEngine.swift @@ -0,0 +1,98 @@ +// Adapted from AppleTranscriptionImpl.swift in @react-native-ai/apple (MIT, Callstack). +import AVFoundation +import Foundation +import Speech + +@available(iOS 26, macOS 26, *) +enum AppleSpeechEngine { + enum Failure: LocalizedError { + case unavailable + case unsupportedLocale(String) + case unknownAssetStatus + + var errorDescription: String? { + switch self { + case .unavailable: return "Speech transcription is not available." + case .unsupportedLocale(let locale): return "Locale not supported: \(locale)" + case .unknownAssetStatus: return "Unknown asset inventory status." + } + } + } + + static var isAvailable: Bool { SpeechTranscriber.isAvailable } + + static func supportedLocales() async throws -> [String] { + try requireAvailable() + return await SpeechTranscriber.supportedLocales.map { $0.identifier(.bcp47) } + } + + private static func requireAvailable() throws { + guard isAvailable else { + throw Failure.unavailable + } + } + + private static func makeTranscriber(for locale: Locale) -> SpeechTranscriber { + let preset = SpeechTranscriber.Preset.timeIndexedTranscriptionWithAlternatives + return SpeechTranscriber( + locale: locale, + transcriptionOptions: preset.transcriptionOptions, + reportingOptions: preset.reportingOptions.subtracting([.alternativeTranscriptions]), + attributeOptions: preset.attributeOptions + ) + } + + static func prepare(_ identifier: String) async throws -> String { + try requireAvailable() + guard + let locale = await SpeechTranscriber.supportedLocale( + equivalentTo: Locale(identifier: identifier)) + else { + throw Failure.unsupportedLocale(identifier) + } + let transcriber = makeTranscriber(for: locale) + switch await AssetInventory.status(forModules: [transcriber]) { + case .installed: + break + case .supported, .downloading: + if let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) { + try await request.downloadAndInstall() + } + case .unsupported: + throw Failure.unsupportedLocale(identifier) + @unknown default: + throw Failure.unknownAssetStatus + } + return locale.identifier(.bcp47) + } + + static func transcribe(_ identifier: String, file: URL) async throws -> String { + try requireAvailable() + guard file.isFileURL else { throw CocoaError(.fileReadUnsupportedScheme) } + let audioFile = try AVAudioFile(forReading: file) + let transcriber = makeTranscriber(for: Locale(identifier: identifier)) + let analyzer = SpeechAnalyzer(modules: [transcriber]) + let collector = Task { () throws -> [String] in + var texts: [String] = [] + for try await result in transcriber.results where result.isFinal { + let text = String(result.text.characters).trimmingCharacters(in: .whitespacesAndNewlines) + if !text.isEmpty { texts.append(text) } + } + return texts + } + do { + if let lastSampleTime = try await analyzer.analyzeSequence(from: audioFile) { + try await analyzer.finalizeAndFinish(through: lastSampleTime) + } else { + await analyzer.cancelAndFinishNow() + } + return try await collector.value.joined(separator: " ") + } catch { + collector.cancel() + await analyzer.cancelAndFinishNow() + _ = try? await collector.value + throw error + } + } + +} diff --git a/patches/@react-native-ai__apple@0.12.0.patch b/patches/@react-native-ai__apple@0.12.0.patch deleted file mode 100644 index b2a7aaf229914..0000000000000 --- a/patches/@react-native-ai__apple@0.12.0.patch +++ /dev/null @@ -1,194 +0,0 @@ -diff --git a/ios/transcription/AppleTranscriptionImpl.swift b/ios/transcription/AppleTranscriptionImpl.swift -index 188371a5f55fadae19187108c4a688569d0223e3..3e1cdd77b0bb8ae6c97a40d29c1854200d931eee 100644 ---- a/ios/transcription/AppleTranscriptionImpl.swift -+++ b/ios/transcription/AppleTranscriptionImpl.swift -@@ -12,6 +12,11 @@ import UniformTypeIdentifiers - - @objc - public class AppleTranscriptionImpl: NSObject { -+ private struct CollectedSegment: Sendable { -+ let text: String -+ let startSecond: Double -+ let endSecond: Double -+ } - - @available(iOS 26, *) - private func createTranscriber(for locale: Locale) -> SpeechTranscriber { -@@ -41,7 +46,7 @@ public class AppleTranscriptionImpl: NSObject { - let locale = Locale(identifier: language) - - guard let supportedLocale = await SpeechTranscriber.supportedLocale(equivalentTo: locale) else { -- reject("AppleTranscription", "Locale not supported: \(language)", nil) -+ reject("AppleTranscriptionUnsupportedLocale", "Locale not supported: \(language)", nil) - return - } - -@@ -51,16 +56,14 @@ public class AppleTranscriptionImpl: NSObject { - - switch status { - case .installed: -- resolve(nil) -+ resolve(supportedLocale.identifier) - case .supported, .downloading: -- if let request = try? await AssetInventory.assetInstallationRequest(supporting: [transcriber]) { -+ if let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) { - try await request.downloadAndInstall() -- resolve(nil) -- } else { -- resolve(nil) - } -+ resolve(supportedLocale.identifier) - case .unsupported: -- reject("AppleTranscription", "Assets not supported for locale: \(supportedLocale.identifier)", nil) -+ reject("AppleTranscriptionUnsupportedLocale", "Assets not supported for locale: \(supportedLocale.identifier)", nil) - @unknown default: - reject ("AppleTranscription", "Unknown asset inventory status", nil) - } -@@ -83,57 +86,77 @@ public class AppleTranscriptionImpl: NSObject { - do { - try audioData.write(to: fileURL) - -- guard let audioFile = try? AVAudioFile(forReading: fileURL) else { -- reject("AppleTranscription", "Invalid audio data", nil) -+ let audioFile: AVAudioFile -+ do { -+ audioFile = try AVAudioFile(forReading: fileURL) -+ } catch { -+ try? FileManager.default.removeItem(at: fileURL) -+ reject("AppleTranscription", "Invalid audio data", error) - return - } - - Task { -+ defer { -+ try? FileManager.default.removeItem(at: fileURL) -+ } -+ - do { - let transcriber = createTranscriber(for: Locale(identifier: language)) - - let analyzer = SpeechAnalyzer(modules: [transcriber]) -- -- defer { -- try? FileManager.default.removeItem(at: fileURL) -- } -- -- var segments: [[String: Any]] = [] -- -- Task { -+ -+ let collectorTask = Task { () throws -> [CollectedSegment] in -+ var segments: [CollectedSegment] = [] -+ - for try await result in transcriber.results { - if result.isFinal { -- let segment: [String: Any] = [ -- "text": String(result.text.characters), -- "startSecond": CMTimeGetSeconds(result.range.start), -- "endSecond": CMTimeGetSeconds(CMTimeRangeGetEnd(result.range)) -- ] -- segments.append(segment) -+ segments.append( -+ CollectedSegment( -+ text: String(result.text.characters), -+ startSecond: CMTimeGetSeconds(result.range.start), -+ endSecond: CMTimeGetSeconds(CMTimeRangeGetEnd(result.range)) -+ ) -+ ) - } - } -+ -+ return segments - } -- -- let lastSampleTime = try await analyzer.analyzeSequence(from: audioFile) -- -- if let lastSampleTime { -- try await analyzer.finalizeAndFinish(through: lastSampleTime) -- } else { -+ -+ do { -+ let lastSampleTime = try await analyzer.analyzeSequence(from: audioFile) -+ -+ if let lastSampleTime { -+ try await analyzer.finalizeAndFinish(through: lastSampleTime) -+ } else { -+ await analyzer.cancelAndFinishNow() -+ } -+ -+ let segments: [[String: Any]] = try await collectorTask.value.map { segment in -+ [ -+ "text": segment.text, -+ "startSecond": segment.startSecond, -+ "endSecond": segment.endSecond -+ ] -+ } -+ let totalDuration = if let lastSampleTime { CMTimeGetSeconds(lastSampleTime) } else { 0.0 } -+ -+ resolve([ -+ "segments": segments, -+ "duration": totalDuration -+ ]) -+ } catch { -+ collectorTask.cancel() - await analyzer.cancelAndFinishNow() -+ _ = try? await collectorTask.value -+ throw error - } -- -- let totalDuration = if let lastSampleTime { CMTimeGetSeconds(lastSampleTime) } else { 0.0 } -- -- let result: [String: Any] = [ -- "segments": segments, -- "duration": totalDuration -- ] -- -- resolve(result) - } catch { - reject("AppleTranscription", "Transcription failed: \(error.localizedDescription)", error) - } - } - } catch { -+ try? FileManager.default.removeItem(at: fileURL) - reject("AppleTranscription", "Failed to write audio data: \(error.localizedDescription)", error) - } - } else { -@@ -141,4 +164,3 @@ public class AppleTranscriptionImpl: NSObject { - } - } - } -- -diff --git a/lib/typescript/NativeAppleTranscription.d.ts b/lib/typescript/NativeAppleTranscription.d.ts -index 985b6b3593a884c41d689346be9d73d86c55eae1..86e7936f23a8673d099f7fe18af9a31abae5a13a 100644 ---- a/lib/typescript/NativeAppleTranscription.d.ts -+++ b/lib/typescript/NativeAppleTranscription.d.ts -@@ -10,14 +10,14 @@ export interface TranscriptionResult { - } - export interface Spec extends TurboModule { - isAvailable(language: string): boolean; -- prepare(language: string): Promise; -+ prepare(language: string): Promise; - } - declare global { - function __apple__llm__transcribe__(data: ArrayBufferLike, language: string): Promise; - } - declare const _default: { - transcribe: (data: ArrayBufferLike, language: string) => Promise; -- prepare: (language: string) => Promise; -+ prepare: (language: string) => Promise; - isAvailable: (language: string) => boolean; - }; - export default _default; -diff --git a/src/NativeAppleTranscription.ts b/src/NativeAppleTranscription.ts -index 13332a0176000b60b6f6f043de681db55a7b4bef..5389fb4b3a53de48721c307d2145b6770624c852 100644 ---- a/src/NativeAppleTranscription.ts -+++ b/src/NativeAppleTranscription.ts -@@ -14,7 +14,7 @@ export interface TranscriptionResult { - - export interface Spec extends TurboModule { - isAvailable(language: string): boolean -- prepare(language: string): Promise -+ prepare(language: string): Promise - } - - declare global { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2f1143e5ea109..1ec44ebabe361 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -286,7 +286,6 @@ overrides: '@ff-labs/fff-node': 0.11.0 '@legendapp/list': 3.7.3 '@pierre/diffs': 1.7.0-beta.4 - '@react-native-ai/apple': 0.12.0 '@react-native-menu/menu': 2.0.0 '@react-navigation/native-stack': 7.21.0 dbus-next: 0.10.2 @@ -372,7 +371,6 @@ patchedDependencies: '@ff-labs/fff-node@0.11.0': 93f62a76399b435a76588b2bef237db0a1e85880c1ac0d0759c416e9cc579252 '@muse-code/sdk@1.4.6': 7b495b607fc5d76b444c7c4d58821e839726a30e6f3c53564f0d59f2d5b42101 '@pierre/diffs@1.7.0-beta.4': 7edcdd484a99c7ec0b2667499c4cb21d113ec9bea1f8570c0b0a030d33bbf1cd - '@react-native-ai/apple@0.12.0': 2d09870c2848d185cb05b53ed823a46e12dba519324d8dd8e584e28731990f9d '@react-native-menu/menu@2.0.0': 6872ba71b9847c5a0210b1b896923dadc3860faedfe28d9f062bcedfe786ef1d '@react-navigation/native-stack@7.21.0': 990d575444c4af2ce8218b3b7a8280fe244c03abbdb32b822a2e862f791dbf42 alchemy@2.0.0-beta.85: 07cc8c7e14b3b9af143ddd40a059f117c27e55139fbf20bd5dde2eb30509cedd @@ -585,9 +583,6 @@ importers: '@pierre/highlights': specifier: 'catalog:' version: 0.5.0 - '@react-native-ai/apple': - specifier: 0.12.0 - version: 0.12.0(patch_hash=2d09870c2848d185cb05b53ed823a46e12dba519324d8dd8e584e28731990f9d)(react-native@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6)) '@react-native-menu/menu': specifier: 2.0.0 version: 2.0.0(patch_hash=6872ba71b9847c5a0210b1b896923dadc3860faedfe28d9f062bcedfe786ef1d)(react-native@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0) @@ -2660,16 +2655,6 @@ packages: typescript: optional: true - '@ai-sdk/provider-utils@4.0.57': - resolution: {integrity: sha512-89a7sPvZqXsP3PigcAQLWntu/duIVEcN/RyChAB+j6p3C7NwW767sVttvL8r0+BetTg63uAXQoYHCZZueQKBSw==} - engines: {node: '>=18.17'} - peerDependencies: - zod: ^3.25.76 || ^4.1.8 - - '@ai-sdk/provider@3.0.18': - resolution: {integrity: sha512-IpefF5ssZVOZCD4Wu7zxt6PwZoGT2HLVjxpJ10Bjx6EAmRfLYJPjyrfxb0RYDlwvXnDb9nkQ7Do/oJVDuRFzew==} - engines: {node: '>=18'} - '@ai-sdk/provider@4.0.36': resolution: {integrity: sha512-GqvWZSZdBelCGughlBdOoupT258rgM3YMbaLAS9WSZJ0vlt8c9vm3My+ZT31mYhiReMCMCF2qF2axnGbILpk1A==} engines: {node: '>=22'} @@ -6488,11 +6473,6 @@ packages: resolution: {integrity: sha512-gMDYY2rw6OWajCcDlXSIgs2LC432YJXSb3Lm5yM187uhRgBYddoEVULi36h+IolX3r7jSb3ew7vn9FfI8NSo0A==} hasBin: true - '@react-native-ai/apple@0.12.0': - resolution: {integrity: sha512-BC/kEDbCZprv1xcQCWsgapXm/WEsj5lieBDvU1vJXStrN+BG+hWM01zoqDGTD/j71zvt9DEfHcMvEqbrGY2uAA==} - peerDependencies: - react-native: '>=0.76.0' - '@react-native-masked-view/masked-view@0.3.2': resolution: {integrity: sha512-XwuQoW7/GEgWRMovOQtX3A4PrXhyaZm0lVUiY8qJDvdngjLms9Cpdck6SmGAUNqQwcj2EadHC1HwL0bEyoa/SQ==} peerDependencies: @@ -13147,18 +13127,6 @@ snapshots: optionalDependencies: typescript: 6.0.3 - '@ai-sdk/provider-utils@4.0.57(zod@4.6.7)': - dependencies: - '@ai-sdk/provider': 3.0.18 - '@standard-schema/spec': 1.1.0 - eventsource-parser: 3.1.0 - undici: 6.29.1 - zod: 4.6.7 - - '@ai-sdk/provider@3.0.18': - dependencies: - json-schema: 0.4.0 - '@ai-sdk/provider@4.0.36': dependencies: json-schema: 0.4.0 @@ -16563,13 +16531,6 @@ snapshots: prompts: 2.4.2 tinyexec: 1.3.1 - '@react-native-ai/apple@0.12.0(patch_hash=2d09870c2848d185cb05b53ed823a46e12dba519324d8dd8e584e28731990f9d)(react-native@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))': - dependencies: - '@ai-sdk/provider': 3.0.18 - '@ai-sdk/provider-utils': 4.0.57(zod@4.6.7) - react-native: 0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6) - zod: 4.6.7 - '@react-native-masked-view/masked-view@0.3.2(react-native@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)': dependencies: react: 19.3.0 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index fbc816311fd65..a1b23decf627b 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -117,7 +117,6 @@ overrides: "@ff-labs/fff-node": 0.11.0 "@legendapp/list": 3.7.3 "@pierre/diffs": 1.7.0-beta.4 - "@react-native-ai/apple": 0.12.0 "@react-native-menu/menu": 2.0.0 "@react-navigation/native-stack": 7.21.0 dbus-next: 0.10.2 @@ -252,7 +251,6 @@ patchedDependencies: # editor carets when the code font changes, stops the find bar reclaiming focus from the file, # and keeps drag selections intact across virtualized scroll renders. "@pierre/diffs@1.7.0-beta.4": patches/@pierre__diffs@1.7.0-beta.4.patch - "@react-native-ai/apple@0.12.0": patches/@react-native-ai__apple@0.12.0.patch "@react-native-menu/menu@2.0.0": patches/@react-native-menu__menu@2.0.0.patch "@react-navigation/native-stack@7.21.0": patches/@react-navigation%2Fnative-stack@7.21.0.patch dbus-next@0.10.2: patches/dbus-next@0.10.2.patch diff --git a/third-party-licenses.config.json b/third-party-licenses.config.json index 48f19d112feb1..3f5ed2ff2c4de 100644 --- a/third-party-licenses.config.json +++ b/third-party-licenses.config.json @@ -1,9 +1,9 @@ { "customNotices": [ { - "bundles": ["desktop"], + "bundles": ["desktop", "mobile"], "license": "MIT", - "name": "Apple Speech helper (adapted from @react-native-ai/apple)", + "name": "Apple Speech engine (adapted from @react-native-ai/apple)", "sourceUrl": "https://github.com/callstackincubator/ai", "generatedNotices": [ { @@ -623,17 +623,6 @@ "copyrights": ["Copyright (c) Meta Platforms, Inc. and affiliates."] } }, - { - "name": "@react-native-ai/apple", - "sourceUrl": "https://github.com/callstackincubator/ai", - "generatedNotice": { - "licenseId": "MIT", - "copyrights": [ - "Copyright (c) 2024-2025 Szymon Rybczak", - "Copyright (c) 2025-present Callstack" - ] - } - }, { "name": "react-remove-scroll-bar", "generatedNotice": {