Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
52641de
feat(desktop): dictate on this Mac with Apple Speech
RVZO6 Nov 10, 2026
9527574
fix(desktop): prepare Apple Speech once per language and retry failed…
RVZO6 Nov 11, 2026
025d845
Merge remote-tracking branch 'origin/main' into feat/desktop-apple-sp…
RVZO6 Nov 12, 2026
db3ee10
fix(desktop): Apple Speech review fixes for timeouts, caching and set…
RVZO6 Nov 12, 2026
d33c822
fix(desktop): Apple Speech accepts unaligned PCM and runs one analysi…
RVZO6 Nov 12, 2026
b3ca461
test(web): check the Apple Speech availability retry as a unit test
RVZO6 Nov 12, 2026
cfe6236
fix(desktop): Apple Speech copies unaligned Buffer audio
RVZO6 Nov 12, 2026
da53240
Merge remote-tracking branch 'origin/main' into feat/desktop-apple-sp…
juliusmarminge Nov 17, 2026
7451539
refactor(desktop): resolve the Apple Speech helper path directly
juliusmarminge Nov 17, 2026
32268b5
Merge main and rebuild macOS Apple Speech dictation
juliusmarminge Nov 21, 2026
a54030d
Merge remote-tracking branch 'origin/main' into t3/takeover-34027
juliusmarminge Nov 21, 2026
b681498
fix(desktop): handle Apple Speech cancellation and supported locales
juliusmarminge Nov 21, 2026
47559ea
fix(desktop): honor saved Apple Speech language preferences
juliusmarminge Nov 21, 2026
e3ccc92
fix(desktop): gate Apple Speech by macOS version and credit upstream
juliusmarminge Nov 21, 2026
94721e0
docs: clarify how to enable Apple Speech dictation
juliusmarminge Nov 21, 2026
063726b
refactor(voice): share Apple Speech transcription across iOS and macOS
juliusmarminge Nov 21, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 76 additions & 0 deletions apps/desktop/scripts/build-apple-speech.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
import * as NodeChildProcess from "node:child_process";
import * as NodeFS from "node:fs";
import * as NodePath from "node:path";
import * as NodeURL from "node:url";
import * as NodeUtil from "node:util";

// oxlint-disable-next-line t3code/no-global-process-runtime -- The Swift compiler targets the actual host; this script has no Effect runtime.
const hostArch = process.arch;
// oxlint-disable-next-line t3code/no-global-process-runtime -- The Speech framework exists only in the macOS SDK.
const hostPlatform = process.platform;

const { values } = NodeUtil.parseArgs({
options: { output: { type: "string" }, arch: { type: "string", default: hostArch } },
});
// Packaging passes --output and must not ship without the helper; dev builds only lose the option.
const required = values.output !== undefined;

if (hostPlatform !== "darwin") {
if (required) throw new Error("The Apple Speech helper can only be built on macOS.");
} else {
const targets = { arm64: ["arm64"], x64: ["x86_64"], universal: ["arm64", "x86_64"] }[
values.arch
];
if (targets === undefined) throw new Error(`Unsupported macOS architecture: ${values.arch}`);
const root = NodeURL.fileURLToPath(new URL("../../../native/apple-speech/", import.meta.url));
const sources = ["AppleSpeech.swift", "AppleSpeechEngine.swift"].map((file) =>
NodePath.resolve(root, file),
);
const output = values.output ?? NodePath.resolve(root, "build", values.arch, "t3-apple-speech");
let current = false;
try {
current =
NodeFS.statSync(output).mtimeMs >=
Math.max(
...sources.map((source) => NodeFS.statSync(source).mtimeMs),
NodeFS.statSync(NodeURL.fileURLToPath(import.meta.url)).mtimeMs,
);
} catch {}
if (!current) {
NodeFS.mkdirSync(NodePath.dirname(output), { recursive: true });
const temporary = `${output}.${process.pid}.tmp`;
const slices = targets.map((target) => `${temporary}.${target}`);
try {
targets.forEach((target, index) =>
NodeChildProcess.execFileSync(
"xcrun",
[
"swiftc",
"-O",
"-swift-version",
"5",
"-parse-as-library",
"-target",
`${target}-apple-macos26.0`,
...sources,
"-o",
slices[index],
],
{ stdio: "inherit" },
),
);
NodeChildProcess.execFileSync("xcrun", ["lipo", "-create", ...slices, "-output", temporary], {
stdio: "inherit",
});
NodeFS.renameSync(temporary, output);
} catch (cause) {
if (required)
throw new Error("Building the Apple Speech helper requires Xcode 26 or later.", { cause });
console.warn(
"Skipping the Apple Speech helper (needs Xcode 26 or later); on-device Mac dictation is unavailable in this build.",
);
} finally {
for (const file of [temporary, ...slices]) NodeFS.rmSync(file, { force: true });
}
}
}
5 changes: 5 additions & 0 deletions apps/desktop/scripts/dev-electron.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,11 @@ NodeChildProcess.execFileSync(
[NodePath.join(desktopDir, "scripts/build-browser-secret.mjs")],
{ stdio: "inherit" },
);
NodeChildProcess.execFileSync(
process.execPath,
[NodePath.join(desktopDir, "scripts/build-apple-speech.mjs")],
{ stdio: "inherit" },
);

await waitForResources({
baseDir: desktopDir,
Expand Down
5 changes: 5 additions & 0 deletions apps/desktop/scripts/start-electron.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,11 @@ NodeChildProcess.execFileSync(
[NodePath.join(desktopDir, "scripts/build-browser-secret.mjs")],
{ stdio: "inherit" },
);
NodeChildProcess.execFileSync(
process.execPath,
[NodePath.join(desktopDir, "scripts/build-apple-speech.mjs")],
{ stdio: "inherit" },
);

const childEnv = { ...process.env };
delete childEnv.ELECTRON_RUN_AS_NODE;
Expand Down
2 changes: 2 additions & 0 deletions apps/desktop/src/app/DesktopApp.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ import * as ElectronWindow from "../electron/ElectronWindow.ts";
import * as MacPermissions from "../permissions/MacPermissions.ts";
import * as LoopbackTunnel from "../preview/LoopbackTunnel.ts";
import * as PreviewManager from "../preview/Manager.ts";
import * as AppleSpeech from "../speech/AppleSpeech.ts";

interface StubbedDesktop {
readonly localEnvironmentEnabled: boolean;
Expand Down Expand Up @@ -233,6 +234,7 @@ const runStubbedDesktop = (stub: StubbedDesktop) =>
Layer.mock(ElectronTheme.ElectronTheme, {}),
Layer.mock(ElectronWindow.ElectronWindow, {}),
Layer.mock(MacPermissions.MacPermissions, {}),
Layer.mock(AppleSpeech.AppleSpeech, {}),
Layer.mock(LoopbackTunnel.LoopbackTunnel, {}),
Layer.mock(PreviewManager.PreviewManager, {
isBrowserPartition: () => {
Expand Down
8 changes: 8 additions & 0 deletions apps/desktop/src/ipc/DesktopIpcHandlers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,11 @@ import { startFileDrag } from "./methods/fileDrag.ts";
import * as Effect from "effect/Effect";

import { receiveProviderAuthCallback, cancelProviderAuthCallback } from "./methods/providerAuth.ts";
import {
appleSpeechAvailable,
appleSpeechPrepare,
appleSpeechTranscribe,
} from "./methods/appleSpeech.ts";
import * as DesktopIpc from "./DesktopIpc.ts";
import { installNotificationBadge } from "./methods/notificationBadge.ts";
import { getClientSettings, setClientSettings } from "./methods/clientSettings.ts";
Expand Down Expand Up @@ -161,6 +166,9 @@ export const installDesktopIpcHandlers = Effect.fn("desktop.ipc.installHandlers"
yield* ipc.handle(cancelProviderAuthCallback);
yield* ipc.handle(openSystemSettings);
yield* ipc.handle(checkSystemPermission);
yield* ipc.handle(appleSpeechAvailable);
yield* ipc.handle(appleSpeechPrepare);
yield* ipc.handle(appleSpeechTranscribe);
yield* ipc.handle(pasteAsText);
yield* ipc.handle(revealWindow);
yield* ipc.handle(syncHtmlFullscreen);
Expand Down
3 changes: 3 additions & 0 deletions apps/desktop/src/ipc/channels.ts
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,9 @@ export const PORT_FORWARD_SHARES_LOOPBACK_CHANNEL = "desktop:port-forward-shares
export const MAC_PERMISSION_HELPER_CHANNEL = "desktop:mac-permission-helper";

export const CHECK_SYSTEM_PERMISSION_CHANNEL = "desktop:check-system-permission";
export const APPLE_SPEECH_AVAILABLE_CHANNEL = "desktop:apple-speech-available";
export const APPLE_SPEECH_PREPARE_CHANNEL = "desktop:apple-speech-prepare";
export const APPLE_SPEECH_TRANSCRIBE_CHANNEL = "desktop:apple-speech-transcribe";

export const PREVIEW_RECORDING_INPUT_CHANNEL = "desktop:preview-recording-input";

Expand Down
37 changes: 37 additions & 0 deletions apps/desktop/src/ipc/methods/appleSpeech.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
import {
DesktopAppleSpeechResultSchema,
DesktopAppleSpeechTranscribeInputSchema,
} from "@t3tools/contracts";
import * as Effect from "effect/Effect";
import * as Schema from "effect/Schema";

import * as AppleSpeech from "../../speech/AppleSpeech.ts";
import * as IpcChannels from "../channels.ts";
import * as DesktopIpc from "../DesktopIpc.ts";

export const appleSpeechAvailable = DesktopIpc.makeIpcMethod({
channel: IpcChannels.APPLE_SPEECH_AVAILABLE_CHANNEL,
payload: Schema.Void,
result: Schema.Array(Schema.String),
handler: Effect.fn("desktop.ipc.appleSpeech.available")(function* () {
return yield* (yield* AppleSpeech.AppleSpeech).supportedLocales;
}),
});

export const appleSpeechPrepare = DesktopIpc.makeIpcMethod({
channel: IpcChannels.APPLE_SPEECH_PREPARE_CHANNEL,
payload: Schema.String.check(Schema.isMaxLength(64)),
result: DesktopAppleSpeechResultSchema,
handler: Effect.fn("desktop.ipc.appleSpeech.prepare")(function* (locale) {
return yield* (yield* AppleSpeech.AppleSpeech).prepare(locale);
}),
});

export const appleSpeechTranscribe = DesktopIpc.makeIpcMethod({
channel: IpcChannels.APPLE_SPEECH_TRANSCRIBE_CHANNEL,
payload: DesktopAppleSpeechTranscribeInputSchema,
result: DesktopAppleSpeechResultSchema,
handler: Effect.fn("desktop.ipc.appleSpeech.transcribe")(function* ({ locale, pcm }) {
return yield* (yield* AppleSpeech.AppleSpeech).transcribe(locale, pcm);
}),
});
2 changes: 2 additions & 0 deletions apps/desktop/src/main.ts
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ import * as DesktopUpdatePolicy from "./updates/DesktopUpdatePolicy.ts";
import * as DesktopUpdates from "./updates/DesktopUpdates.ts";
import * as BrowserImport from "./preview/BrowserImport/BrowserImport.ts";
import * as LinuxBrowserSecret from "./preview/BrowserImport/LinuxBrowserSecret.ts";
import * as AppleSpeech from "./speech/AppleSpeech.ts";
import * as BrowserSession from "./preview/BrowserSession.ts";
import * as DesktopBrowserHost from "./preview/DesktopBrowserHost.ts";
import * as LoopbackTunnel from "./preview/LoopbackTunnel.ts";
Expand Down Expand Up @@ -156,6 +157,7 @@ const layerDesktopFoundation = Layer.mergeAll(
DesktopPrimaryPaste.layer,
DesktopObservability.layer,
DesktopRendererHistory.layer,
AppleSpeech.layer,
).pipe(Layer.provideMerge(layerDesktopEnvironment));

const layerDesktopSsh = layerDesktopSshEnvironment.pipe(
Expand Down
8 changes: 8 additions & 0 deletions apps/desktop/src/preload.ts
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,14 @@ contextBridge.exposeInMainWorld("desktopBridge", {
readMedia: (url: string) => ipcRenderer.invoke(IpcChannels.READ_MEDIA_CHANNEL, url),
checkSystemPermission: (pane: string) =>
ipcRenderer.invoke(IpcChannels.CHECK_SYSTEM_PERMISSION_CHANNEL, pane),
appleSpeech: {
supportedLocales: () =>
ipcRenderer.invoke(IpcChannels.APPLE_SPEECH_AVAILABLE_CHANNEL, undefined),
prepare: (locale: string) =>
ipcRenderer.invoke(IpcChannels.APPLE_SPEECH_PREPARE_CHANNEL, locale),
transcribe: (input: { locale: string; pcm: Uint8Array }) =>
ipcRenderer.invoke(IpcChannels.APPLE_SPEECH_TRANSCRIBE_CHANNEL, input),
},
openSystemSettings: (pane: string) =>
ipcRenderer.invoke(IpcChannels.OPEN_SYSTEM_SETTINGS_CHANNEL, pane),
probeRemoteEditors: () => ipcRenderer.invoke(IpcChannels.PROBE_REMOTE_EDITORS_CHANNEL, undefined),
Expand Down
176 changes: 176 additions & 0 deletions apps/desktop/src/speech/AppleSpeech.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,176 @@
import { assert, describe, it } from "@effect/vitest";
import * as HostProcess from "@t3tools/shared/HostProcess";
import * as Deferred from "effect/Deferred";
import * as Effect from "effect/Effect";
import * as Fiber from "effect/Fiber";
import * as FileSystem from "effect/FileSystem";
import * as Layer from "effect/Layer";
import * as Sink from "effect/Sink";
import * as Stream from "effect/Stream";
import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";

import * as DesktopEnvironment from "../app/DesktopEnvironment.ts";
import * as AppleSpeech from "./AppleSpeech.ts";

const encoder = new TextEncoder();
const text = (value: string) => (value ? Stream.make(encoder.encode(value)) : Stream.empty);

type HelperOutput = {
stdout?: string;
stderr?: string;
exitCode: number;
exit?: Deferred.Deferred<void>;
};

function makeProcess(result: HelperOutput) {
return ChildProcessSpawner.makeHandle({
pid: ChildProcessSpawner.ProcessId(1),
stdout: text(result.stdout ?? ""),
stderr: text(result.stderr ?? ""),
all: Stream.empty,
exitCode: (result.exit ? Deferred.await(result.exit) : Effect.void).pipe(
Effect.as(ChildProcessSpawner.ExitCode(result.exitCode)),
),
isRunning: Effect.succeed(false),
kill: () => Effect.void,
stdin: Sink.drain,
getInputFd: () => Sink.drain,
getOutputFd: () => Stream.empty,
unref: Effect.succeed(Effect.void),
});
}

function setup(result: HelperOutput, kernelRelease = "25.0.0") {
const commands: Array<ReadonlyArray<string>> = [];
const written: Uint8Array[] = [];
const layer = AppleSpeech.layer.pipe(
Layer.provide(
Layer.mergeAll(
Layer.succeed(HostProcess.KernelRelease, kernelRelease),
Layer.succeed(DesktopEnvironment.DesktopEnvironment, {
platform: "darwin",
isPackaged: true,
resourcesPath: "/app/Resources",
path: { join: (...parts: string[]) => parts.join("/") },
} as unknown as DesktopEnvironment.DesktopEnvironment["Service"]),
FileSystem.layerNoop({
exists: () => Effect.succeed(true),
makeTempFileScoped: () => Effect.succeed("/tmp/recording.wav"),
writeFile: (_path, data) => Effect.sync(() => void written.push(data)),
}),
Layer.succeed(
ChildProcessSpawner.ChildProcessSpawner,
ChildProcessSpawner.make((command) => {
const { command: executable, args } = command as unknown as {
command: string;
args: ReadonlyArray<string>;
};
commands.push([executable, ...args]);
return Effect.succeed(makeProcess(result));
}),
),
),
),
);
return { commands, written, layer };
}

describe("AppleSpeech", () => {
it.effect.each([
[
{ stdout: '{"value":"en-US"}', exitCode: 0 },
{ ok: true, value: "en-US" },
],
[
{ stderr: "Locale not supported: xx\n", exitCode: 2 },
{ ok: false, code: "unsupported-locale" },
],
[{ exitCode: 3 }, { ok: false, code: "unavailable" }],
[
{ stderr: "boom", exitCode: 1 },
{ ok: false, code: "failed" },
],
] as const)("maps helper output %j to %j", ([output, expected]) => {
const { layer, commands } = setup(output);
return Effect.gen(function* () {
const speech = yield* AppleSpeech.AppleSpeech;
assert.deepEqual(yield* speech.prepare("en-US"), expected);
assert.deepEqual(commands, [
["/app/Resources/apple-speech/t3-apple-speech", "prepare", "en-US"],
]);
}).pipe(Effect.provide(layer));
});

it.effect("shares one helper between prepares that start while the first is running", () =>
Effect.gen(function* () {
const exit = yield* Deferred.make<void>();
const { layer, commands } = setup({ stdout: '{"value":"en-US"}', exitCode: 0, exit });
return yield* Effect.gen(function* () {
const speech = yield* AppleSpeech.AppleSpeech;
const first = yield* speech
.prepare("en-US")
.pipe(Effect.forkChild({ startImmediately: true }));
const second = yield* speech
.prepare("en-US")
.pipe(Effect.forkChild({ startImmediately: true }));
yield* Deferred.succeed(exit, undefined);
assert.deepEqual(yield* Fiber.join(first), { ok: true, value: "en-US" });
assert.deepEqual(yield* Fiber.join(second), { ok: true, value: "en-US" });
assert.lengthOf(commands, 1);
}).pipe(Effect.provide(layer));
}),
);

it.effect.each([
["25.0.0", 0, ["en-US", "fr-FR"], 1],
["25.0.0", 3, [], 1],
["25.0.0", 1, [], 2],
["24.6.0", 0, [], 0],
] as const)(
"on Darwin %s resolves locales on helper exit %i to %j with %i probe(s) across two calls",
([release, exitCode, expected, probes]) => {
const { layer, commands } = setup({ stdout: '{"value":"en-US,fr-FR"}', exitCode }, release);
return Effect.gen(function* () {
const speech = yield* AppleSpeech.AppleSpeech;
assert.deepEqual(yield* speech.supportedLocales, expected);
assert.deepEqual(yield* speech.supportedLocales, expected);
assert.lengthOf(commands, probes);
}).pipe(Effect.provide(layer));
},
);

it.effect("transcribes Float32 PCM through a 16-bit WAV file", () => {
const { layer, commands, written } = setup({ stdout: '{"value":"hello"}', exitCode: 0 });
// An unaligned Buffer view: Buffer.slice and subarray do not copy.
const pcm = new Uint8Array(17);
new DataView(pcm.buffer).setFloat32(1, 0, true);
new DataView(pcm.buffer).setFloat32(5, 1, true);
new DataView(pcm.buffer).setFloat32(9, -1, true);
new DataView(pcm.buffer).setFloat32(13, 2, true);
return Effect.gen(function* () {
const result = yield* (yield* AppleSpeech.AppleSpeech).transcribe(
"en-US",
Buffer.from(pcm.buffer).subarray(1),
);
assert.deepEqual(result, { ok: true, value: "hello" });
assert.deepEqual(commands, [
[
"/app/Resources/apple-speech/t3-apple-speech",
"transcribe",
"en-US",
"/tmp/recording.wav",
],
]);
const wav = new DataView(written[0]!.buffer);
assert.equal(new TextDecoder().decode(written[0]!.subarray(0, 4)), "RIFF");
assert.equal(wav.getUint16(20, true), 1);
assert.equal(wav.getUint32(24, true), 16_000);
assert.equal(wav.getUint16(34, true), 16);
assert.equal(wav.getUint32(40, true), 8);
assert.deepEqual(
[0, 1, 2, 3].map((index) => wav.getInt16(44 + index * 2, true)),
[0, 32767, -32767, 32767],
);
}).pipe(Effect.provide(layer));
});
});
Loading
Loading