Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions apps/mobile/src/Stack.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ import {
} from "./features/settings/ScheduledTaskPickerScreens";
import { ScheduledTaskEditorProvider } from "./features/settings/scheduled-task-editor";
import { SettingsKeyboardRouteScreen } from "./features/settings/SettingsKeyboardRouteScreen";
import { SettingsVoiceInputRouteScreen } from "./features/settings/SettingsVoiceInputRouteScreen";
import { SettingsLegalRouteScreen } from "./features/settings/SettingsLegalRouteScreen";
import {
SettingsOpenSourceLicenseRouteScreen,
Expand Down Expand Up @@ -310,6 +311,11 @@ const SettingsContentStack = createV5SheetStackNavigator({
title: "Keyboard",
},
}),
SettingsVoiceInput: createNativeStackScreen({
screen: SettingsVoiceInputRouteScreen,
linking: "voice-input",
options: { title: "Voice input" },
}),
SettingsFollowUp: createNativeStackScreen({
screen: SettingsFollowUpRouteScreen,
linking: "follow-ups",
Expand Down
3 changes: 3 additions & 0 deletions apps/mobile/src/features/settings/SettingsRouteScreen.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,9 @@ function SettingsIndexSections() {
{Platform.OS === "ios" ? (
<SettingsRow icon="keyboard" label="Keyboard" target="SettingsKeyboard" />
) : null}
{Platform.OS === "ios" ? (
<SettingsRow icon="mic" label="Voice input" target="SettingsVoiceInput" />
) : null}
</SettingsSection>

<SettingsSection title="Automations">
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
import { useAtomSet, useAtomValue } from "@effect/atom-react";
import { AsyncResult } from "effect/reactivity";
import { ScrollView } from "react-native";
import { useSafeAreaInsets } from "react-native-safe-area-context";

import { mobilePreferencesAtom, updateMobilePreferencesAtom } from "../../state/preferences";
import { SettingsChoiceRow } from "./components/SettingsChoiceRow";
import { SettingsSection } from "./components/SettingsSection";

// Locales Apple's on-device transcriber supports; its native module does not expose the list.
const LANGUAGES = [
{ locale: undefined, label: "Automatic", description: "Uses the app's language." },
{ locale: "en-US", label: "English (US)", description: "English (United States)" },
{ locale: "en-GB", label: "English (UK)", description: "English (United Kingdom)" },
{ locale: "es-ES", label: "Español (España)", description: "Spanish (Spain)" },
{ locale: "es-MX", label: "Español (México)", description: "Spanish (Mexico)" },
{ locale: "fr-FR", label: "Français", description: "French" },
{ locale: "de-DE", label: "Deutsch", description: "German" },
{ locale: "it-IT", label: "Italiano", description: "Italian" },
{ locale: "pt-BR", label: "Português (Brasil)", description: "Portuguese (Brazil)" },
{ locale: "ja-JP", label: "日本語", description: "Japanese" },
{ locale: "ko-KR", label: "한국어", description: "Korean" },
{ locale: "zh-CN", label: "中文(普通话)", description: "Chinese (Mandarin)" },
] as const;

export function SettingsVoiceInputRouteScreen() {
const insets = useSafeAreaInsets();
const savePreferences = useAtomSet(updateMobilePreferencesAtom);
const selectedLocale = useAtomValue(mobilePreferencesAtom, (result) =>
AsyncResult.isSuccess(result) ? result.value.voiceInputLanguage : null,
);

return (
<ScrollView
contentInsetAdjustmentBehavior="automatic"
showsVerticalScrollIndicator={false}
className="flex-1 bg-sheet"
contentContainerClassName="gap-3 px-5 pt-4"
contentContainerStyle={{ paddingBottom: Math.max(insets.bottom, 18) + 18 }}
>
<SettingsSection title="Language">
{LANGUAGES.map((option, index) => (
<SettingsChoiceRow
key={option.label}
label={option.label}
description={option.description}
selected={selectedLocale === option.locale}
separated={index > 0}
disabled={selectedLocale === null}
onPress={() => savePreferences({ voiceInputLanguage: option.locale })}
/>
))}
</SettingsSection>
</ScrollView>
);
}
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ export type SettingsSheetTarget =
| "SettingsEnvironmentMaintenance"
| "SettingsProviderAccounts"
| "SettingsKeyboard"
| "SettingsVoiceInput"
| "SettingsFollowUp"
| "SettingsScheduledTasks"
| "SettingsProjectGrouping"
Expand Down
12 changes: 11 additions & 1 deletion apps/mobile/src/features/voice-input/VoiceInputProvider.tsx
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { useAtomValue } from "@effect/atom-react";
import {
AudioModule,
RecordingPresets,
Expand All @@ -7,6 +8,7 @@ import {
type RecorderState,
type RecordingStatus,
} from "expo-audio";
import { AsyncResult } from "effect/reactivity";
import { File } from "expo-file-system";
import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake";
import {
Expand All @@ -23,6 +25,7 @@ import { AppState, Platform } from "react-native";
import { useSharedValue } from "react-native-reanimated";

import { getLocalVoiceTranscriber } from "../../native/voiceTranscription";
import { mobilePreferencesAtom } from "../../state/preferences";
import { getNativeShowcaseScene } from "../showcase/nativeShowcaseScene";
import {
VOICE_RECORDING_LIMIT_SECONDS,
Expand Down Expand Up @@ -112,6 +115,13 @@ function useVoiceInputRuntime() {
const audioLevelsRef = useRef(Array<number>(VOICE_WAVEFORM_SAMPLE_COUNT).fill(0));
const audioLevels = useSharedValue(audioLevelsRef.current);
const sessionRef = useRef<VoiceInputSession | null>(null);
const language = useAtomValue(mobilePreferencesAtom, (result) =>
AsyncResult.isSuccess(result) ? result.value.voiceInputLanguage : undefined,
);
const languageRef = useRef(language);
useEffect(() => {
languageRef.current = language;
}, [language]);
const recorderRef = useRef<LazyVoiceRecorder<RecorderState> | null>(null);

if (!sessionRef.current || !recorderRef.current) {
Expand All @@ -130,7 +140,7 @@ function useVoiceInputRuntime() {
recorderRef.current = recorder;
sessionRef.current = new VoiceInputSession({
recorder,
getTranscriber: getLocalVoiceTranscriber,
getTranscriber: () => getLocalVoiceTranscriber(languageRef.current),
requestPermission: async () => {
const permission = await requestRecordingPermissionsAsync();
return { granted: permission.granted, canAskAgain: permission.canAskAgain };
Expand Down
10 changes: 10 additions & 0 deletions apps/mobile/src/native/voiceTranscription.ios.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,16 @@ describe("getLocalVoiceTranscriber", () => {
expect(mocks.transcribe).toHaveBeenCalledWith(audio, "sv-SE");
});

it("uses the chosen language instead of the app's locale", async () => {
mocks.prepare.mockResolvedValue("es-ES");
const prepared = await getLocalVoiceTranscriber("es-ES")!.prepare({
signal: new AbortController().signal,
});

expect(mocks.prepare).toHaveBeenCalledWith("es-ES");
expect(prepared.locale).toBe("es-ES");
});

it("does not start native transcription after cancellation during a file read", async () => {
const enteredRead = deferred<void>();
const readResult = deferred<ArrayBuffer>();
Expand Down
4 changes: 2 additions & 2 deletions apps/mobile/src/native/voiceTranscription.ios.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@ function getNativeErrorCode(error: unknown): string | undefined {
return typeof error.code === "string" ? error.code : undefined;
}

export function getLocalVoiceTranscriber(): VoiceTranscriber | null {
const locale = getDeviceLocale();
export function getLocalVoiceTranscriber(language?: string): VoiceTranscriber | null {
const locale = language ?? getDeviceLocale();
if (!AppleTranscription.isAvailable(locale)) return null;
return { prepare: (options) => prepareVoiceTranscription(locale, options) };
}
Expand Down
2 changes: 1 addition & 1 deletion apps/mobile/src/native/voiceTranscription.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import type { VoiceTranscriber } from "@t3tools/client-runtime/voice-input";

export function getLocalVoiceTranscriber(): VoiceTranscriber | null {
export function getLocalVoiceTranscriber(_language?: string): VoiceTranscriber | null {
return null;
}
6 changes: 6 additions & 0 deletions apps/mobile/src/persistence/mobile-preferences.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ export interface Preferences {
* message sent during a running turn queues behind it or steers it.
*/
readonly followUpBehavior?: FollowUpBehavior;
/** Locale for on-device voice input. Unset follows the app's locale. iOS only. */
readonly voiceInputLanguage?: string;
/** @deprecated Kept temporarily so older OTA bundles retain the selected mode. */
readonly projectGroupingEnabled?: boolean;
readonly projectGroupingMode?: SidebarProjectGroupingMode;
Expand Down Expand Up @@ -107,6 +109,7 @@ function sanitizePreferences(parsed: Preferences): Preferences {
collapsedProjectGroups?: readonly string[];
composerEnterBehavior?: ComposerEnterBehavior;
followUpBehavior?: FollowUpBehavior;
voiceInputLanguage?: string;
projectGroupingEnabled?: boolean;
projectGroupingMode?: SidebarProjectGroupingMode;
planModeEnabled?: boolean;
Expand Down Expand Up @@ -172,6 +175,9 @@ function sanitizePreferences(parsed: Preferences): Preferences {
if (parsed.followUpBehavior === "queue" || parsed.followUpBehavior === "steer") {
preferences.followUpBehavior = parsed.followUpBehavior;
}
if (typeof parsed.voiceInputLanguage === "string") {
preferences.voiceInputLanguage = parsed.voiceInputLanguage;
}
if (typeof parsed.projectGroupingEnabled === "boolean") {
preferences.projectGroupingEnabled = parsed.projectGroupingEnabled;
}
Expand Down
1 change: 1 addition & 0 deletions docs/user/composer.md
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,7 @@ and use **Attach again** or remove the missing file before sending.
On supported iPhones with iOS 26 or later, use the composer's microphone to record,
then confirm to transcribe. Text is inserted where your selection was when
recording started, ready for you to review and edit before sending.
To dictate in a language other than the app's, choose it in **Settings → Voice input**.

The first use may download Apple's speech model and needs a network connection.
Later transcription works offline for that language. Recordings can be up to five
Expand Down
25 changes: 25 additions & 0 deletions packages/client-runtime/src/voice-input/controller.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,31 @@ describe("resolveTranscriptCommit", () => {
});
});

it("adds word spacing for other space-delimited languages", () => {
const atEnd = draft({
text: "Revisá la caché.",
selection: { start: "Revisá la caché.".length, end: "Revisá la caché.".length },
});
expect(resolveTranscriptCommit(atEnd, atEnd, "Ésta también.", "es-ES")).toMatchObject({
kind: "commit",
text: "Revisá la caché. Ésta también.",
});
});

it("treats accented letters as word boundaries", () => {
const afterAccent = draft({ text: "café", selection: { start: 4, end: 4 } });
expect(resolveTranscriptCommit(afterAccent, afterAccent, "también", "es-ES")).toMatchObject({
kind: "commit",
text: "café también",
});

const beforeAccent = draft({ text: "él", selection: { start: 0, end: 0 } });
expect(resolveTranscriptCommit(beforeAccent, beforeAccent, "Dijo", "es-ES")).toMatchObject({
kind: "commit",
text: "Dijo él",
});
});

it("does not add English boundary spaces to CJK or selected inline text", () => {
const cjk = draft({ text: "修正キャッシュ", selection: { start: 8, end: 8 } });
expect(resolveTranscriptCommit(cjk, cjk, "テストも", "ja-JP")).toMatchObject({
Expand Down
8 changes: 4 additions & 4 deletions packages/client-runtime/src/voice-input/controller.ts
Comment thread
macroscopeapp[bot] marked this conversation as resolved.
Resolved
Original file line number Diff line number Diff line change
Expand Up @@ -92,18 +92,18 @@ export function resolveTranscriptCommit(

const isEmptySelection = captured.selection.start === captured.selection.end;
const normalizedLocale = locale.replaceAll("_", "-").toLowerCase();
const usesEnglishSpacing = normalizedLocale === "en" || normalizedLocale.startsWith("en-");
const usesWordSpacing = !/^(ja|zh|yue|th|lo|km|my)(-|$)/.test(normalizedLocale);
let insertion = replacement;
if (isEmptySelection && usesEnglishSpacing) {
if (isEmptySelection && usesWordSpacing) {
const left = captured.text[captured.selection.start - 1];
const right = captured.text[captured.selection.start];
const leftNeedsBoundary =
left !== undefined &&
/[A-Za-z0-9.!?,:;)\]}'"]/.test(left) &&
/[\p{L}\p{N}.!?,:;)\]}'"]/u.test(left) &&
(right === undefined || /\s/.test(right));
const rightNeedsBoundary =
right !== undefined &&
/[A-Za-z0-9([{'"]/.test(right) &&
/[\p{L}\p{N}([{'"]/u.test(right) &&
(left === undefined || /\s/.test(left));
insertion = `${leftNeedsBoundary ? " " : ""}${replacement}${rightNeedsBoundary ? " " : ""}`;
}
Expand Down
Loading