Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -18,22 +18,18 @@ export function SettingsDragHandle(props: {
const onEnd = useEffectEvent((translation: number, cancelled: boolean) =>
props.onEnd(translation, cancelled),
);
const [gesture] = useState(() => {
let translation = 0;
return Gesture.Pan()
// Gesture callbacks become worklets that each copy their closure, so state
// shared between them through a local variable never updates; the final
// event carries the translation instead.
const [gesture] = useState(() =>
Gesture.Pan()
.minDistance(0)
.shouldCancelWhenOutside(false)
.runOnJS(true)
.onStart(() => {
translation = 0;
onStart();
})
.onUpdate((event) => {
translation = event.translationY;
onMove(event.translationY);
})
.onFinalize((_, success) => onEnd(translation, !success));
});
.onStart(() => onStart())
.onUpdate((event) => onMove(event.translationY))
.onFinalize((event, success) => onEnd(event.translationY, !success)),
);
return (
<GestureDetector gesture={gesture}>
<View
Expand Down
3 changes: 2 additions & 1 deletion apps/mobile/src/features/threads/ThreadComposer.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -967,7 +967,8 @@ export const ThreadComposer = function ThreadComposer(props: ThreadComposerProps
]}
>
<ComposerEditor
hidden={props.hidden}
// display: none recreates the iOS view, so its revision history must reset too.
hidden={props.hidden || showsCompactDictation}
draftKey={composerDraftKey}
environmentId={props.environmentId}
onOpenMention={(path) => {
Expand Down
36 changes: 36 additions & 0 deletions apps/mobile/src/native/voiceTranscription.ios.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,11 @@ const mocks = vi.hoisted(() => ({
prepare: vi.fn<(locale: string) => Promise<string>>(),
transcribe: vi.fn<(audio: ArrayBufferLike, locale: string) => Promise<TranscriptionResult>>(),
readAudio: vi.fn<() => Promise<ArrayBuffer>>(),
getSetting: vi.fn<(key: string) => unknown>(),
}));

vi.mock("react-native", () => ({ Settings: { get: mocks.getSetting } }));

vi.mock("@react-native-ai/apple/src/NativeAppleTranscription", () => ({
default: {
isAvailable: mocks.isAvailable,
Expand Down Expand Up @@ -56,6 +59,39 @@ afterEach(() => {
});

describe("getLocalVoiceTranscriber", () => {
it("prefers the user's languages over the English-only app locale", async () => {
const resolvedOptions = Intl.DateTimeFormat().resolvedOptions();
vi.spyOn(Intl.DateTimeFormat.prototype, "resolvedOptions").mockReturnValue({
...resolvedOptions,
locale: "en-KR",
});
mocks.getSetting.mockReturnValue(["ko-KR", "en-US"]);
mocks.prepare.mockResolvedValue("ko_KR");

const prepared = await getLocalVoiceTranscriber()!.prepare({
signal: new AbortController().signal,
});

expect(mocks.prepare.mock.calls).toEqual([["ko-KR"]]);
expect(prepared.locale).toBe("ko_KR");
});

it("falls back to the next language when one is unsupported", async () => {
mocks.getSetting.mockReturnValue(["gsw-CH", "de-CH"]);
mocks.prepare
.mockRejectedValueOnce(
Object.assign(new Error(), { code: "AppleTranscriptionUnsupportedLocale" }),
)
.mockResolvedValueOnce("de_CH");

const prepared = await getLocalVoiceTranscriber()!.prepare({
signal: new AbortController().signal,
});

expect(mocks.prepare.mock.calls).toEqual([["gsw-CH"], ["de-CH"]]);
expect(prepared.locale).toBe("de_CH");
});

it("does not start native transcription after cancellation during a file read", async () => {
const enteredRead = deferred<void>();
const readResult = deferred<ArrayBuffer>();
Expand Down
32 changes: 27 additions & 5 deletions apps/mobile/src/native/voiceTranscription.ios.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import AppleTranscription from "@react-native-ai/apple/src/NativeAppleTranscription";
import { File } from "expo-file-system";
import { Settings } from "react-native";

import {
VoiceTranscriptionError,
Expand All @@ -9,8 +10,13 @@ import {
type VoiceTranscriptionOptions,
} from "@t3tools/client-runtime/voice-input";

function getDeviceLocale(): string {
return Intl.DateTimeFormat().resolvedOptions().locale;
// The English-only app locale can differ from the user's preferred spoken languages.
function getPreferredLocales(): string[] {
const languages = Settings.get("AppleLanguages");
const preferred = Array.isArray(languages)
? languages.filter((language): language is string => typeof language === "string")
: [];
return [...new Set([...preferred, Intl.DateTimeFormat().resolvedOptions().locale])];
}

function wrapError(
Expand All @@ -34,12 +40,28 @@ function getNativeErrorCode(error: unknown): string | undefined {
}

export function getLocalVoiceTranscriber(): VoiceTranscriber | null {
const locale = getDeviceLocale();
if (!AppleTranscription.isAvailable(locale)) return null;
return { prepare: (options) => prepareVoiceTranscription(locale, options) };
const locales = getPreferredLocales();
if (!AppleTranscription.isAvailable(locales[0]!)) return null;
return { prepare: (options) => prepareVoiceTranscription(locales, options) };
}

async function prepareVoiceTranscription(
locales: readonly string[],
options: VoiceTranscriptionOptions,
): Promise<PreparedVoiceTranscription> {
for (const locale of locales.slice(0, -1)) {
try {
return await prepareLocale(locale, options);
} catch (error) {
if (!(error instanceof VoiceTranscriptionError && error.code === "unsupported-locale")) {
throw error;
}
}
}
return prepareLocale(locales.at(-1)!, options);
}

async function prepareLocale(
locale: string,
{ signal }: VoiceTranscriptionOptions,
): Promise<PreparedVoiceTranscription> {
Expand Down
4 changes: 4 additions & 0 deletions docs/user/composer.md
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,10 @@ On supported iPhones with iOS 26 or later, use the composer's microphone to reco
then confirm to transcribe. Text is inserted where your selection was when
recording started, ready for you to review and edit before sending.

Transcription tries your iPhone's preferred languages in order
(Settings > General > Language & Region), then the app's language. If Apple's speech
model supports none of them, voice input reports that the language is unavailable.

The first use may download Apple's speech model and needs a network connection.
Later transcription works offline for that language. Recordings can be up to five
minutes long. Canceling, leaving the screen, or an audio interruption discards the
Expand Down
95 changes: 95 additions & 0 deletions packages/client-runtime/src/voice-input/controller.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,29 @@ describe("resolveTranscriptCommit", () => {
kind: "stale",
});
});

it("adds word spacing for other space-delimited languages", () => {
const korean = draft({ text: "코드를", selection: { start: 3, end: 3 } });
expect(resolveTranscriptCommit(korean, korean, "검토해 주세요", "ko_KR")).toMatchObject({
kind: "commit",
text: "코드를 검토해 주세요",
});

const combiningMark = draft({ text: "नमस्ते", selection: { start: 6, end: 6 } });
expect(resolveTranscriptCommit(combiningMark, combiningMark, "दुनिया", "hi-IN")).toMatchObject({
kind: "commit",
text: "नमस्ते दुनिया",
});
});

it("keeps spaceless languages spaceless", () => {
const text = "修正キャッシュ";
const cjk = draft({ text, selection: { start: text.length, end: text.length } });
expect(resolveTranscriptCommit(cjk, cjk, "テストも", "ja-JP")).toMatchObject({
kind: "commit",
text: "修正キャッシュテストも",
});
});
});

describe("VoiceInputController", () => {
Expand Down Expand Up @@ -303,6 +326,78 @@ describe("VoiceInputController", () => {
},
);

it("records while the speech model prepares, then transcribes once it is ready", async () => {
const preparation = deferred<PreparedVoiceTranscription>();
const harness = createHarness({
getTranscriber: () => ({ prepare: () => preparation.promise }),
});

await harness.controller.start();
expect(harness.controller.currentState.phase).toBe("recording");
expect(harness.recorder.record).toHaveBeenCalledTimes(1);

const stopping = harness.controller.stop();
expect(harness.controller.currentState.phase).toBe("transcribing");
preparation.resolve(preparedTranscription());
await stopping;

expect(harness.commits.map((commit) => commit.text)).toEqual(["hello new text"]);
});

it("discards the recording when the speech model fails to prepare", async () => {
const preparation = deferred<PreparedVoiceTranscription>();
const deleted = deferred<string>();
const harness = createHarness({
getTranscriber: () => ({ prepare: () => preparation.promise }),
deleteRecording: deleted.resolve,
});
await harness.controller.start();

preparation.reject(Object.assign(new Error("unsupported"), { code: "unsupported-locale" }));

expect(await deleted.promise).toBe("file:///voice.m4a");
expect(harness.recorder.stop).toHaveBeenCalledTimes(1);
expect(harness.controller.currentState).toMatchObject({
phase: "error",
error: "Voice transcription is not available for this language.",
});
const next = createHarness();
await next.controller.start();
expect(next.controller.currentState.phase).toBe("recording");
await next.controller.interruptRecording();
});

it.each(["start", "stop"] as const)(
"cancels speech model preparation when the recorder fails to %s",
async (failure) => {
const cancellablePrepare: VoiceTranscriber["prepare"] = ({ signal }) =>
new Promise((_resolve, reject) => {
signal.addEventListener("abort", () => reject(new Error("cancelled")));
});
const harness = createHarness({
getTranscriber: () => ({ prepare: cancellablePrepare }),
});
if (failure === "start") {
harness.recorder.prepareToRecordAsync.mockRejectedValueOnce(new Error("recorder busy"));
} else {
harness.recorder.stop.mockRejectedValueOnce(new Error("recorder busy"));
}

await harness.controller.start();
if (failure === "stop") await harness.controller.stop();
expect(harness.controller.currentState.error).toBe(
failure === "start"
? "Could not start voice recording."
: "Could not finish voice recording.",
);

const next = createHarness();
await next.controller.start();
expect(next.controller.currentState.phase).toBe("recording");
await next.controller.interruptRecording();
},
);

it("releases the microphone before transcription starts", async () => {
const events: string[] = [];
const harness = createHarness({
Expand Down
53 changes: 37 additions & 16 deletions packages/client-runtime/src/voice-input/controller.ts
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,8 @@ type TranscriptCommitResult =
| { readonly kind: "stale" }
| { readonly kind: "empty" };

const WORD_SPACELESS_LANGUAGES = new Set(["ja", "zh", "yue", "th", "lo", "km", "my"]);

export function resolveTranscriptCommit(
captured: VoiceDraftSnapshot,
current: VoiceDraftSnapshot | null,
Expand All @@ -87,19 +89,19 @@ export function resolveTranscriptCommit(
}

const isEmptySelection = captured.selection.start === captured.selection.end;
const normalizedLocale = locale.replaceAll("_", "-").toLowerCase();
const usesEnglishSpacing = normalizedLocale === "en" || normalizedLocale.startsWith("en-");
const language = locale.split(/[-_]/)[0]!.toLowerCase();
const usesWordSpacing = !WORD_SPACELESS_LANGUAGES.has(language);
let insertion = replacement;
if (isEmptySelection && usesEnglishSpacing) {
if (isEmptySelection && usesWordSpacing) {
const left = captured.text[captured.selection.start - 1];
const right = captured.text[captured.selection.start];
const leftNeedsBoundary =
left !== undefined &&
/[A-Za-z0-9.!?,:;)\]}'"]/.test(left) &&
/[\p{L}\p{M}\p{N}.!?,:;)\]}'"]/u.test(left) &&
(right === undefined || /\s/.test(right));
const rightNeedsBoundary =
right !== undefined &&
/[A-Za-z0-9([{'"]/.test(right) &&
/[\p{L}\p{N}([{'"]/u.test(right) &&
(left === undefined || /\s/.test(left));
insertion = `${leftNeedsBoundary ? " " : ""}${replacement}${rightNeedsBoundary ? " " : ""}`;
}
Expand Down Expand Up @@ -184,7 +186,7 @@ export class VoiceInputController {
private state: VoiceInputState = IDLE_STATE;
private operationToken = 0;
private sessionToken: VoiceInputSession | null = null;
private transcription: PreparedVoiceTranscription | null = null;
private transcription: Promise<PreparedVoiceTranscription> | null = null;
private transcriptionAbortController: AbortController | null = null;
private capturedDraft: VoiceDraftSnapshot | null = null;
private recordingUri: string | null = null;
Expand Down Expand Up @@ -245,15 +247,21 @@ export class VoiceInputController {
return;
}

try {
this.transcription = await runTranscriptionOperation(() =>
transcriber.prepare({ signal: abortController.signal }),
);
} catch (error) {
if (this.isCurrent(operationToken)) this.setError(preparationErrorMessage(error), "retry");
return;
}
if (!this.isCurrent(operationToken)) return;
// Record while the speech model prepares so the first words are not lost.
const transcription = runTranscriptionOperation(() =>
transcriber.prepare({ signal: abortController.signal }),
);
this.transcription = transcription;
transcription.catch((error: unknown) => {
if (!this.isCurrent(operationToken)) return;
const message = preparationErrorMessage(error);
if (this.state.phase === "recording") {
void this.discardRecording(message);
} else if (this.state.phase === "preparing") {
this.invalidateOperation();
this.setError(message, "retry");
}
});

await this.dependencies.configureRecording();
this.recordingConfigured = true;
Expand Down Expand Up @@ -388,9 +396,18 @@ export class VoiceInputController {
}

const recordingUri = this.recordingUri;
const transcription = this.transcription;
const signal = this.transcriptionAbortController.signal;
const capturedDraft = this.capturedDraft;
let transcription: PreparedVoiceTranscription;
try {
transcription = await this.transcription;
} catch (error) {
if (this.isCurrent(operationToken)) {
this.setError(preparationErrorMessage(error), "retry");
}
return;
}
if (!this.isCurrent(operationToken)) return;
let transcript: string;
try {
transcript = await runTranscriptionOperation(() =>
Expand Down Expand Up @@ -462,6 +479,10 @@ export class VoiceInputController {
}
this.ownedRecordingUris.clear();
await this.releaseAudioSession();
// Preparation cancels cooperatively; the next session must not overlap it.
if (this.sessionToken) this.sessionToken.abandoned = true;
this.transcriptionAbortController?.abort();
await this.transcription?.catch(() => {});
releaseSession(this.sessionToken);
this.sessionToken = null;
this.capturedDraft = null;
Expand Down
Loading