Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions apps/mobile/src/Stack.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ import { SettingsProviderAccountsRouteScreen } from "./features/settings/Setting
import { SettingsAuthRouteScreen } from "./features/settings/SettingsAuthRouteScreen";
import { SettingsEnvironmentDetailRouteScreen } from "./features/settings/SettingsEnvironmentDetailRouteScreen";
import { SettingsEnvironmentsRouteScreen } from "./features/settings/SettingsEnvironmentsRouteScreen";
import { SettingsDictationLanguageRouteScreen } from "./features/settings/SettingsDictationLanguageRouteScreen";
import { SettingsFollowUpRouteScreen } from "./features/settings/SettingsFollowUpRouteScreen";
import {
SettingsEnvironmentMaintenanceRouteScreen,
Expand Down Expand Up @@ -373,6 +374,14 @@ const SETTINGS_SCREENS = {
title: "Microphone",
},
}),
SettingsDictationLanguage: createNativeStackScreen({
screen: SettingsDictationLanguageRouteScreen,
linking: "settings/dictation-language",
options: {
...SETTINGS_PAGE_OPTIONS,
title: "Dictation language",
},
}),
SettingsFollowUp: createNativeStackScreen({
screen: SettingsFollowUpRouteScreen,
linking: "settings/follow-ups",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
import { ScreenScrollView as ScrollView } from "../../components/ui/ScreenScrollView";
import { useAtomSet, useAtomValue } from "@effect/atom-react";
import { AsyncResult } from "effect/reactivity";
import { View } from "react-native";
import { useSafeAreaInsets } from "react-native-safe-area-context";

import {
DEVICE_DICTATION_LANGUAGE_LABEL,
useDictationLanguages,
} from "../../lib/dictationLanguages";
import { mobilePreferencesAtom, updateMobilePreferencesAtom } from "../../state/preferences";
import { SettingsChoiceRow } from "./components/SettingsChoiceRow";
import { SettingsSection } from "./components/SettingsSection";

export function SettingsDictationLanguageRouteScreen() {
const insets = useSafeAreaInsets();
const preferencesResult = useAtomValue(mobilePreferencesAtom);
const savePreferences = useAtomSet(updateMobilePreferencesAtom);
const preferencesReady = AsyncResult.isSuccess(preferencesResult) && !preferencesResult.waiting;
const selectedId = AsyncResult.isSuccess(preferencesResult)
? (preferencesResult.value.voiceInputLocale ?? null)
: undefined;
const languages = useDictationLanguages();
const options = [{ id: null, label: DEVICE_DICTATION_LANGUAGE_LABEL }, ...(languages ?? [])];

return (
<View collapsable={false} className="flex-1 bg-sheet">
<ScrollView
contentInsetAdjustmentBehavior="automatic"
showsVerticalScrollIndicator={false}
className="flex-1"
contentContainerClassName="gap-3 px-5 pt-4"
contentContainerStyle={{ paddingBottom: Math.max(insets.bottom, 18) + 18 }}
>
<SettingsSection
footer={
languages?.length === 0
? "This device has no dictation languages to choose from."
: "The language you speak when dictating."
}
>
{options.map((option, index) => (
<SettingsChoiceRow
key={option.id ?? "device"}
label={option.label}
selected={selectedId === option.id}
separated={index > 0}
disabled={!preferencesReady}
onPress={() => savePreferences({ voiceInputLocale: option.id })}
/>
))}
</SettingsSection>
</ScrollView>
</View>
);
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import { useSafeAreaInsets } from "react-native-safe-area-context";
import { type AppSymbolName, SymbolView } from "../../components/ui/AppSymbol";
import { AppText as Text } from "../../components/ui/AppText";
import { cn } from "../../lib/cn";
import { dictationLanguageLabel, useDictationLanguages } from "../../lib/dictationLanguages";
import { makePermissionStatusAtom } from "../../lib/permissionStatusAtom";
import {
isMicrophoneDevice,
Expand All @@ -23,6 +24,7 @@ import { mobilePreferencesAtom, updateMobilePreferencesAtom } from "../../state/
import { ListRowLayout } from "../../components/ui/ListRowLayout";
import { SettingsActionRow } from "./components/SettingsActionRow";
import { SettingsDragHandle } from "./components/SettingsDragHandle";
import { SettingsRow } from "./components/SettingsRow";
import { SettingsSection } from "./components/SettingsSection";

const REMOVE_SIZE = 20;
Expand Down Expand Up @@ -94,6 +96,7 @@ export function SettingsMicrophoneRouteScreen() {
const insets = useSafeAreaInsets();
const preferencesResult = useAtomValue(mobilePreferencesAtom);
const savePreferences = useAtomSet(updateMobilePreferencesAtom);
const dictationLanguages = useDictationLanguages();
const ready = AsyncResult.isSuccess(preferencesResult) && !preferencesResult.waiting;
const entries = microphoneOrder(
AsyncResult.isSuccess(preferencesResult) ? preferencesResult.value.microphones : undefined,
Expand Down Expand Up @@ -217,6 +220,19 @@ export function SettingsMicrophoneRouteScreen() {
devices are added by name after you use voice input with them connected; until then, the
Other row ranks them.
</Text>
<SettingsSection title="Language">
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
<SettingsRow
icon="globe"
label="Dictation language"
value={dictationLanguageLabel(
dictationLanguages,
AsyncResult.isSuccess(preferencesResult)
? preferencesResult.value.voiceInputLocale
: undefined,
)}
target="SettingsDictationLanguage"
/>
</SettingsSection>
</ScrollView>
</View>
);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ export const SETTINGS_PAGE_TARGETS = [
"SettingsProviderAccounts",
"SettingsKeyboard",
"SettingsMicrophone",
"SettingsDictationLanguage",
"SettingsFollowUp",
"SettingsScheduledTasks",
"SettingsProjectGrouping",
Expand Down
9 changes: 8 additions & 1 deletion apps/mobile/src/features/voice-input/VoiceInputProvider.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,14 @@ const voiceInputAtom = Atom.make((get) => {
});
const session: VoiceInputSession = new VoiceInputSession({
recorder,
getTranscriber: getLocalVoiceTranscriber,
getTranscriber: () => {
const preferences = registry.get(mobilePreferencesAtom);
return getLocalVoiceTranscriber(
AsyncResult.isSuccess(preferences)
? (preferences.value.voiceInputLocale ?? undefined)
: undefined,
);
},
requestPermission: async () => {
const permission = await requestRecordingPermissionsAsync();
return { granted: permission.granted, canAskAgain: permission.canAskAgain };
Expand Down
21 changes: 21 additions & 0 deletions apps/mobile/src/lib/dictationLanguages.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
import { describe, expect, it } from "vite-plus/test";

import { dictationLanguageLabel } from "./dictationLanguages";

const languages = [{ id: "pt-BR", label: "Português (Brasil)" }];

describe("dictationLanguageLabel", () => {
it("names the chosen language from the loaded list", () => {
expect(dictationLanguageLabel(languages, "pt-BR")).toBe("Português (Brasil)");
});

it("shows the raw tag until the list loads or when the list lacks it", () => {
expect(dictationLanguageLabel(undefined, "pt-BR")).toBe("pt-BR");
expect(dictationLanguageLabel(languages, "sv-SE")).toBe("sv-SE");
});

it("names the device language when nothing is chosen", () => {
expect(dictationLanguageLabel(languages, null)).toBe("Device language");
expect(dictationLanguageLabel(undefined, undefined)).toBe("Device language");
});
});
35 changes: 35 additions & 0 deletions apps/mobile/src/lib/dictationLanguages.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
import { useEffect, useState } from "react";

import { getDictationLanguages } from "../native/voiceTranscription";
import type { DictationLanguage } from "../native/voiceTranscription.types";

export const DEVICE_DICTATION_LANGUAGE_LABEL = "Device language";

/** The languages this device can transcribe, or undefined while they load. */
export function useDictationLanguages(): ReadonlyArray<DictationLanguage> | undefined {
const [languages, setLanguages] = useState<ReadonlyArray<DictationLanguage>>();
// oxlint-disable-next-line t3code/react-effect-budget -- Load the native device capabilities when this screen mounts.
useEffect(() => {
let cancelled = false;
getDictationLanguages().then(
(loaded) => {
if (!cancelled) setLanguages(loaded);
},
() => {
if (!cancelled) setLanguages([]);
},
);
return () => {
cancelled = true;
};
}, []);
return languages;
}

export function dictationLanguageLabel(
languages: ReadonlyArray<DictationLanguage> | undefined,
id: string | null | undefined,
): string {
if (id == null) return DEVICE_DICTATION_LANGUAGE_LABEL;
return languages?.find((language) => language.id === id)?.label ?? id;
}
61 changes: 60 additions & 1 deletion apps/mobile/src/native/voiceTranscription.ios.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import { VoiceTranscriptionError } from "@t3tools/client-runtime/voice-input";
const mocks = vi.hoisted(() => ({
isAvailable: vi.fn<(locale: string) => boolean>(),
prepare: vi.fn<(locale: string) => Promise<string>>(),
supportedLanguages:
vi.fn<() => Promise<ReadonlyArray<{ readonly id: string; readonly label: string }>>>(),
transcribe: vi.fn<(audio: ArrayBufferLike, locale: string) => Promise<TranscriptionResult>>(),
readAudio: vi.fn<() => Promise<ArrayBuffer>>(),
}));
Expand All @@ -14,6 +16,7 @@ vi.mock("@react-native-ai/apple/src/NativeAppleTranscription", () => ({
default: {
isAvailable: mocks.isAvailable,
prepare: mocks.prepare,
supportedLanguages: mocks.supportedLanguages,
transcribe: mocks.transcribe,
},
}));
Expand All @@ -24,7 +27,7 @@ vi.mock("expo-file-system", () => ({
},
}));

import { getLocalVoiceTranscriber } from "./voiceTranscription.ios";
import { getDictationLanguages, getLocalVoiceTranscriber } from "./voiceTranscription.ios";

const audio = new ArrayBuffer(4);
const nativeTranscript: TranscriptionResult = {
Expand Down Expand Up @@ -56,6 +59,40 @@ afterEach(() => {
});

describe("getLocalVoiceTranscriber", () => {
it("keeps the selected language and Apple's resolved locale when the device language changes", async () => {
const resolvedOptions = Intl.DateTimeFormat().resolvedOptions();
const deviceLocale = vi
.spyOn(Intl.DateTimeFormat.prototype, "resolvedOptions")
.mockReturnValue({ ...resolvedOptions, locale: "sv-FI" });
const transcriber = getLocalVoiceTranscriber()!;
const options = { signal: new AbortController().signal };

deviceLocale.mockReturnValue({ ...resolvedOptions, locale: "de-DE" });
const prepared = await transcriber.prepare(options);
deviceLocale.mockReturnValue({ ...resolvedOptions, locale: "en-US" });

await expect(prepared.transcribe("file:///voice.m4a", options)).resolves.toBe("Hej världen.");
expect(mocks.prepare).toHaveBeenCalledWith("sv-FI");
expect(prepared.locale).toBe("sv-SE");
expect(mocks.transcribe).toHaveBeenCalledWith(audio, "sv-SE");
});

it("prepares the chosen dictation language instead of the device language", async () => {
const resolvedOptions = Intl.DateTimeFormat().resolvedOptions();
vi.spyOn(Intl.DateTimeFormat.prototype, "resolvedOptions").mockReturnValue({
...resolvedOptions,
locale: "en-US",
});
mocks.prepare.mockResolvedValue("pt_BR");

const prepared = await getLocalVoiceTranscriber("pt-BR")!.prepare({
signal: new AbortController().signal,
});

expect(mocks.prepare).toHaveBeenCalledWith("pt-BR");
expect(prepared.locale).toBe("pt_BR");
});

it("does not start native transcription after cancellation during a file read", async () => {
const enteredRead = deferred<void>();
const readResult = deferred<ArrayBuffer>();
Expand All @@ -80,3 +117,25 @@ describe("getLocalVoiceTranscriber", () => {
expect(mocks.transcribe).not.toHaveBeenCalled();
});
});

describe("getDictationLanguages", () => {
it("lists what the device supports, sorted by name", async () => {
mocks.supportedLanguages.mockResolvedValue([
{ id: "pt-BR", label: "Português (Brasil)" },
{ id: "de-DE", label: "Deutsch (Deutschland)" },
{ id: "en-US", label: "English (US)" },
]);

expect(await getDictationLanguages()).toEqual([
{ id: "de-DE", label: "Deutsch (Deutschland)" },
{ id: "en-US", label: "English (US)" },
{ id: "pt-BR", label: "Português (Brasil)" },
]);
});

it("is empty where the device cannot transcribe", async () => {
mocks.supportedLanguages.mockResolvedValue([]);

expect(await getDictationLanguages()).toEqual([]);
});
});
11 changes: 9 additions & 2 deletions apps/mobile/src/native/voiceTranscription.ios.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import {
type VoiceTranscriptionOptions,
} from "@t3tools/client-runtime/voice-input";

import type { DictationLanguage } from "./voiceTranscription.types";

function getDeviceLocale(): string {
return Intl.DateTimeFormat().resolvedOptions().locale;
}
Expand All @@ -33,12 +35,17 @@ function getNativeErrorCode(error: unknown): string | undefined {
return typeof error.code === "string" ? error.code : undefined;
}

export function getLocalVoiceTranscriber(): VoiceTranscriber | null {
const locale = getDeviceLocale();
export function getLocalVoiceTranscriber(preferredLocale?: string): VoiceTranscriber | null {
const locale = preferredLocale ?? getDeviceLocale();
if (!AppleTranscription.isAvailable(locale)) return null;
return { prepare: (options) => prepareVoiceTranscription(locale, options) };
}

export async function getDictationLanguages(): Promise<ReadonlyArray<DictationLanguage>> {
const languages = await AppleTranscription.supportedLanguages();
return [...languages].sort((a, b) => a.label.localeCompare(b.label));
}

async function prepareVoiceTranscription(
locale: string,
{ signal }: VoiceTranscriptionOptions,
Expand Down
8 changes: 7 additions & 1 deletion apps/mobile/src/native/voiceTranscription.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
import type { VoiceTranscriber } from "@t3tools/client-runtime/voice-input";

export function getLocalVoiceTranscriber(): VoiceTranscriber | null {
import type { DictationLanguage } from "./voiceTranscription.types";

export function getLocalVoiceTranscriber(_preferredLocale?: string): VoiceTranscriber | null {
return null;
}

export function getDictationLanguages(): Promise<ReadonlyArray<DictationLanguage>> {
return Promise.resolve([]);
}
4 changes: 4 additions & 0 deletions apps/mobile/src/native/voiceTranscription.types.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
export interface DictationLanguage {
readonly id: string;
readonly label: string;
}
14 changes: 14 additions & 0 deletions apps/mobile/src/persistence/mobile-preferences.ts
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,8 @@ export interface Preferences {
readonly composerEnterBehavior?: ComposerEnterBehavior;
readonly followUpBehavior?: FollowUpBehavior;
readonly microphones?: ReadonlyArray<MicrophoneEntry>;
/** A BCP 47 tag the device can transcribe, or null for the device language. */
readonly voiceInputLocale?: string | null;
readonly projectGroupingEnabled?: boolean;
readonly projectGroupingMode?: SidebarProjectGroupingMode;
readonly planModeEnabled?: boolean;
Expand Down Expand Up @@ -165,6 +167,14 @@ export class MobilePreferencesStore extends Context.Service<
}
>()("@t3tools/mobile/persistence/MobilePreferencesStore") {}

// Shape only; whether the device can transcribe the tag is checked when preparing voice input.
const LANGUAGE_TAG =
/^[A-Za-z]{2,8}(?:-[A-Za-z0-9]{2,8})*(?:-[0-9A-WY-Za-wy-z](?:-[A-Za-z0-9]{2,8})+)*(?:-[xX](?:-[A-Za-z0-9]{1,8})+)?$/;

function isLanguageTag(value: unknown): value is string {
return typeof value === "string" && LANGUAGE_TAG.test(value);
}

function sanitizePreferences(parsed: Preferences): Preferences {
const preferences: {
navigationTabOrder?: readonly string[];
Expand All @@ -189,6 +199,7 @@ function sanitizePreferences(parsed: Preferences): Preferences {
composerEnterBehavior?: ComposerEnterBehavior;
followUpBehavior?: FollowUpBehavior;
microphones?: ReadonlyArray<MicrophoneEntry>;
voiceInputLocale?: string | null;
projectGroupingEnabled?: boolean;
projectGroupingMode?: SidebarProjectGroupingMode;
planModeEnabled?: boolean;
Expand Down Expand Up @@ -302,6 +313,9 @@ function sanitizePreferences(parsed: Preferences): Preferences {
: [];
});
}
if (parsed.voiceInputLocale === null || isLanguageTag(parsed.voiceInputLocale)) {
preferences.voiceInputLocale = parsed.voiceInputLocale;
}
if (typeof parsed.projectGroupingEnabled === "boolean") {
preferences.projectGroupingEnabled = parsed.projectGroupingEnabled;
}
Expand Down
Loading
Loading