From 806a1c8552126f4279572c3df8d0bd089b352d9f Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Thu, 5 Nov 2026 17:57:22 -0800 Subject: [PATCH 1/9] feat(voice): opt into cloud dictation with OpenAI or Grok --- apps/mobile/app.config.ts | 3 + .../features/settings/CloudVoiceSection.tsx | 260 ++++++++++++++++ .../settings/SettingsVoiceRouteScreen.tsx | 28 +- .../voice-input/voiceDestination.test.ts | 46 ++- .../features/voice-input/voiceDestination.ts | 82 +++-- .../src/environment/ServerEnvironment.ts | 1 + apps/server/src/server.ts | 2 + apps/server/src/serverSettings.test.ts | 14 + apps/server/src/settingsSecrets.ts | 7 + .../sourceControlHostSecrets.test.ts | 3 + .../sourceControl/sourceControlHostSecrets.ts | 7 +- apps/server/src/speech/CloudSpeech.test.ts | 106 +++++++ apps/server/src/speech/CloudSpeech.ts | 187 +++++++++++ apps/server/src/speech/http.test.ts | 124 ++++++-- apps/server/src/speech/http.ts | 37 +++ .../CloudVoiceConfiguration.browser.test.tsx | 132 ++++++++ .../settings/CloudVoiceConfiguration.tsx | 272 ++++++++++++++++ .../settings/SecretsSettings.logic.test.ts | 2 + .../settings/SecretsSettings.logic.ts | 14 +- .../VoiceSettingsPanel.browser.test.tsx | 4 +- .../settings/VoiceSettingsPanel.tsx | 294 ++++++++++++++---- .../src/components/settings/settingsSearch.ts | 6 +- apps/web/src/speech/browserVoiceInput.ts | 73 +++-- .../src/speech/useEnvironmentSpeechInput.ts | 33 +- docs/internals/voice-input.md | 9 +- docs/user/composer.md | 2 + docs/user/voice-input.md | 20 +- packages/client-runtime/src/secretRequest.ts | 6 + .../src/voice-input/environment.ts | 82 ++++- .../client-runtime/src/voice-input/index.ts | 4 + packages/contracts/src/environment.ts | 1 + packages/contracts/src/environmentHttp.ts | 40 +++ packages/contracts/src/settings.ts | 30 +- packages/contracts/src/speech.ts | 47 +++ packages/shared/src/speech.ts | 9 + 35 files changed, 1834 insertions(+), 153 deletions(-) create mode 100644 apps/mobile/src/features/settings/CloudVoiceSection.tsx create mode 100644 apps/server/src/speech/CloudSpeech.test.ts create mode 100644 apps/server/src/speech/CloudSpeech.ts create mode 100644 apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx create mode 100644 apps/web/src/components/settings/CloudVoiceConfiguration.tsx diff --git a/apps/mobile/app.config.ts b/apps/mobile/app.config.ts index 784668082bb6f..97eb9b0cadcab 100644 --- a/apps/mobile/app.config.ts +++ b/apps/mobile/app.config.ts @@ -458,6 +458,9 @@ const config: ExpoConfig = { ], extra: { appVariant: APP_VARIANT, + secretsFeatureEnabled: ["1", "true"].includes( + process.env.T3CODE_FEATURE_SECRETS?.trim().toLowerCase() ?? "", + ), iosPersonalTeamBuild: isIosPersonalTeamBuild, relay: { url: repoEnv.T3CODE_RELAY_URL ?? null, diff --git a/apps/mobile/src/features/settings/CloudVoiceSection.tsx b/apps/mobile/src/features/settings/CloudVoiceSection.tsx new file mode 100644 index 0000000000000..89f0967268380 --- /dev/null +++ b/apps/mobile/src/features/settings/CloudVoiceSection.tsx @@ -0,0 +1,260 @@ +import { useAtomRefresh, useAtomValue } from "@effect/atom-react"; +import { setCloudSpeechKey } from "@t3tools/client-runtime/voice-input"; +import { + configuredSecretSources, + secretRequestReference, +} from "@t3tools/client-runtime/secret-request"; +import { + AuthSettingsWriteScope, + CLOUD_SPEECH_LABELS, + CLOUD_SPEECH_PROVIDER_NAMES, + CloudSpeechProvider, + CloudSpeechApiKey, + isSecretReference, + type SecretProviderId, + type SettingsSecretValue, +} from "@t3tools/contracts"; +import * as Schema from "effect/Schema"; +import Constants from "expo-constants"; +import { AsyncResult } from "effect/reactivity"; +import { useState } from "react"; +import { TextInput, View } from "react-native"; +import * as Dialog from "../../components/ui/Dialog"; +import { ProviderIcon } from "../../components/ProviderIcon"; +import { ListActionRow } from "../../components/ui/ListActionRow"; +import { ListChoiceRow } from "../../components/ui/ListChoiceRow"; +import { Chip } from "../../components/ui/Chip"; +import type { EnvironmentPresentation } from "../../state/environments"; +import { useEnvironmentScope } from "../../state/session"; +import { + chooseVoiceDestination, + cloudVoiceConnectionAtom, + cloudVoiceKeysAtom, + runSpeech, +} from "../voice-input/voiceDestination"; +import { SettingsSection } from "./components/SettingsSection"; + +type Props = { environment: EnvironmentPresentation; active: CloudSpeechProvider | null }; +const isApiKey = Schema.is(CloudSpeechApiKey); + +export function CloudVoiceSection({ environment, active }: Props) { + const [open, setOpen] = useState(active !== null); + return ( + + setOpen(!open)} + /> + {open && } + + ); +} + +function CloudChoices({ environment, active }: Props) { + const id = environment.environmentId; + const connection = useAtomValue(cloudVoiceConnectionAtom(id)); + const keys = useAtomValue(cloudVoiceKeysAtom(id)); + const reload = useAtomRefresh(cloudVoiceKeysAtom(id)); + const canManage = useEnvironmentScope(id, AuthSettingsWriteScope); + const [editing, setEditing] = useState<{ + provider: CloudSpeechProvider; + enable: boolean; + } | null>(null); + const [draft, setDraft] = useState(""); + const [sourceId, setSourceId] = useState(null); + const sources = + Constants.expoConfig?.extra?.secretsFeatureEnabled === true && + environment.serverConfig?.secretReferences + ? configuredSecretSources(environment.serverConfig.settings.secretProviders) + : []; + const source = sources.find((candidate) => candidate.id === sourceId); + const credential = + sourceId !== null + ? source && secretRequestReference(source, draft) + : isApiKey(draft.trim()) + ? draft.trim() + : undefined; + const [saving, setSaving] = useState(false); + const enable = (provider: CloudSpeechProvider) => { + try { + chooseVoiceDestination(id, provider); + } catch { + Dialog.alert( + "Could not enable cloud dictation", + "Check device storage and choose the provider again.", + ); + } + }; + const editKey = (provider: CloudSpeechProvider, enable: boolean) => { + const stored = environment.serverConfig?.settings.cloudSpeechCredentials?.[provider]; + const ref = + isSecretReference(stored) && sources.some((candidate) => candidate.id === stored.provider) + ? stored + : null; + setSourceId(ref?.provider ?? null); + setDraft(ref?.reference ?? ""); + setEditing({ provider, enable }); + }; + const changeKey = async (provider: CloudSpeechProvider, value: SettingsSecretValue | null) => { + if (!connection || !canManage || saving) return; + setSaving(true); + try { + await runSpeech(setCloudSpeechKey(connection, provider, value)); + } catch { + setSaving(false); + Dialog.alert( + "Could not update the API key", + "Check the environment connection and your settings permissions.", + ); + return; + } + setSaving(false); + if (value !== null && editing?.enable) enable(provider); + setDraft(""); + setEditing(null); + reload(); + }; + if (!AsyncResult.isSuccess(keys) || !keys.value || !connection) + return ( + + ); + return CloudSpeechProvider.literals.map((provider) => { + const label = CLOUD_SPEECH_LABELS[provider]; + const company = CLOUD_SPEECH_PROVIDER_NAMES[provider]; + const disclosure = `After you stop recording, ${environment.label} sends your audio and vocabulary hints, including project names, to ${company}.`; + const saved = keys.value![provider]; + return ( + + } + selected={active === provider} + disabled={(!saved && !canManage) || saving} + description={ + active === provider && saved && canManage + ? "Tap to manage key" + : saved + ? "API key saved on this environment" + : canManage + ? "Add an API key to use this provider" + : "Ask an environment administrator to add an API key" + } + onPress={() => { + if (!saved) return editKey(provider, true); + if (active === provider) { + if (canManage) editKey(provider, false); + return; + } + Dialog.confirm({ + title: `Use ${label} for dictation?`, + message: `${disclosure} Select This device to stop sharing.`, + confirmText: `Use ${label}`, + onConfirm: () => enable(provider), + }); + }} + /> + {canManage && editing?.provider === provider && ( + <> + {sources.length > 0 && ( + + { + setDraft(""); + setSourceId(null); + }} + /> + {sources.map((candidate) => ( + { + setDraft(""); + setSourceId(candidate.id); + }} + /> + ))} + + )} + { + if (credential) void changeKey(provider, credential); + }} + value={draft} + onChangeText={setDraft} + className="mx-4 rounded-lg bg-background px-3 py-2 text-foreground" + /> + { + if (credential) void changeKey(provider, credential); + }} + /> + {saved && ( + + Dialog.confirm({ + title: `Remove the ${label} key from ${environment.label}?`, + message: "Everyone using it for dictation will lose access.", + confirmText: "Remove key", + destructive: true, + onConfirm: () => void changeKey(provider, null), + }) + } + /> + )} + { + setEditing(null); + setDraft(""); + }} + /> + + )} + + ); + }); +} diff --git a/apps/mobile/src/features/settings/SettingsVoiceRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsVoiceRouteScreen.tsx index 57ca2cfe5dd40..01dbb5494fe65 100644 --- a/apps/mobile/src/features/settings/SettingsVoiceRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsVoiceRouteScreen.tsx @@ -30,6 +30,7 @@ import { hasEnvironmentVoiceCapture, } from "../voice-input/voiceDestination"; import { VoiceModelSection } from "./VoiceModelSection"; +import { CloudVoiceSection } from "./CloudVoiceSection"; import type { EnvironmentId } from "@t3tools/contracts"; import { ThreadSwipeable } from "../home/thread-swipe-actions"; import { ListActionRow as SettingsActionRow } from "../../components/ui/ListActionRow"; @@ -241,7 +242,10 @@ export function SettingsVoiceRouteScreen() { ))} {isReady && grant && !grantEnvironment ? ( @@ -255,7 +259,27 @@ export function SettingsVoiceRouteScreen() { /> ) : null} - {grantEnvironment ? ( + {hasEnvironmentVoiceCapture && + environments + .filter( + (env) => + env.serverConfig?.environment.capabilities.cloudVoiceTranscription || + (grant?.destination.kind === "cloud" && + grant.destination.environmentId === env.environmentId), + ) + .map((environment) => ( + + ))} + {grantEnvironment && grant?.destination.kind === "environment" ? ( ({ mocks.probe(id); return Atom.make({ connection: { phase: id === "offline" ? "offline" : "connected" }, - serverConfig: { environment: { capabilities: { voiceTranscription: true } } }, + serverConfig: { + environment: { + capabilities: { voiceTranscription: true, cloudVoiceTranscription: true }, + }, + }, }); }), }, @@ -159,6 +163,46 @@ it("keeps local as the default and never reuses consent for an offline or resele ["https://my-server.test/api/voice/status", "error"], ["https://my-server.test/api/voice/transcribe", "error"], ]); + chooseVoiceDestination(host, "openai"); + const cloud = appAtomRegistry.get(voiceRouteAtom)!.transcriber!; + mocks.fetch.mockResolvedValueOnce(Response.json({ openai: false, xai: true })); + await expect(cloud.prepare(options)).rejects.toMatchObject({ + code: "unavailable", + message: expect.stringContaining("Add your OpenAI API key"), + }); + mocks.fetch.mockResolvedValueOnce(Response.json({ openai: true, xai: true })); + const recorded = await cloud.prepare(options); + chooseVoiceDestination(host, "xai"); + await expect(recorded.transcribe("file:///voice.m4a", options)).rejects.toThrow( + "Your transcription choice changed.", + ); + mocks.fetch + .mockResolvedValueOnce(Response.json({ openai: true, xai: true })) + .mockResolvedValueOnce(Response.json({ text: "cloud" })); + const next = await appAtomRegistry.get(voiceRouteAtom)!.transcriber!.prepare(options); + await expect(next.transcribe("file:///voice.m4a", options)).resolves.toBe("cloud"); + mocks.fetch.mockResolvedValueOnce( + Response.json( + { + _tag: "CloudSpeechError", + provider: "xai", + reason: "rejected-key", + }, + { status: 422 }, + ), + ); + await expect(next.transcribe("file:///voice.m4a", options)).rejects.toMatchObject({ + _tag: "VoiceTranscriptionError", + code: "transcription-failed", + message: expect.stringContaining("rejected this API key"), + }); + expect(mocks.fetch.mock.calls.slice(2).map(([url]) => new URL(String(url)).pathname)).toEqual([ + "/api/voice/cloud/keys", + "/api/voice/cloud/keys", + "/api/voice/cloud/keys", + "/api/voice/cloud/xai/transcribe", + "/api/voice/cloud/xai/transcribe", + ]); const failWrite = () => { throw new Error("Device storage is unavailable"); }; diff --git a/apps/mobile/src/features/voice-input/voiceDestination.ts b/apps/mobile/src/features/voice-input/voiceDestination.ts index 6f30769c2d3a2..c75940ca2c5c6 100644 --- a/apps/mobile/src/features/voice-input/voiceDestination.ts +++ b/apps/mobile/src/features/voice-input/voiceDestination.ts @@ -1,10 +1,14 @@ import { AuthOrchestrationOperateScope, + type CloudSpeechProvider, type EnvironmentId, DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, } from "@t3tools/contracts"; import { getEnvironmentSpeechModels, + getCloudSpeechKeys, + requireCloudSpeechKey, + transcribeCloudPcm, getEnvironmentSpeechStatus, transcribeEnvironmentPcm, VoiceTranscriptionError, @@ -34,7 +38,10 @@ export const voiceGrantAtom = Atom.make((get) => { return AsyncResult.getOrElse(get(storedGrant), () => null); }); -export function chooseVoiceDestination(environmentId: EnvironmentId | null) { +export function chooseVoiceDestination( + environmentId: EnvironmentId | null, + provider?: CloudSpeechProvider, +) { // A failed change must not leave the old destination armed in this session. appAtomRegistry.set(choice, null); const grant: VoiceGrant | null = @@ -42,7 +49,9 @@ export function chooseVoiceDestination(environmentId: EnvironmentId | null) { ? null : { id: uuidv4(), - destination: { kind: "environment", environmentId }, + destination: provider + ? { kind: "cloud", environmentId, provider } + : { kind: "environment", environmentId }, }; writeVoiceGrant(grant); appAtomRegistry.set(choice, grant); @@ -53,19 +62,25 @@ const native = requireOptionalNativeModule<{ }>("T3VoiceTranscription"); export const hasEnvironmentVoiceCapture = typeof native?.decodeRecording === "function"; -export const voiceConnectionAtom = Atom.family((id: EnvironmentId) => - Atom.make((get) => { - const environment = get(environmentPresentations.presentationAtom(id)); - if ( - !hasEnvironmentVoiceCapture || - environment?.connection.phase !== "connected" || - !environment.serverConfig?.environment.capabilities.voiceTranscription || - !sessionHasScope(get(environmentSession.sessionStateAtom(id)), AuthOrchestrationOperateScope) - ) - return null; - return Option.getOrNull(get(environmentSession.preparedConnectionValueAtom(id))); - }), -); +const voiceConnections = (capability: "voiceTranscription" | "cloudVoiceTranscription") => + Atom.family((id: EnvironmentId) => + Atom.make((get) => { + const environment = get(environmentPresentations.presentationAtom(id)); + if ( + !hasEnvironmentVoiceCapture || + environment?.connection.phase !== "connected" || + !environment.serverConfig?.environment.capabilities[capability] || + !sessionHasScope( + get(environmentSession.sessionStateAtom(id)), + AuthOrchestrationOperateScope, + ) + ) + return null; + return Option.getOrNull(get(environmentSession.preparedConnectionValueAtom(id))); + }), + ); +export const voiceConnectionAtom = voiceConnections("voiceTranscription"); +export const cloudVoiceConnectionAtom = voiceConnections("cloudVoiceTranscription"); // React Native's XMLHttpRequest fetch follows redirects. Expo's native transport honors redirect: "error". export function runSpeech( @@ -92,10 +107,18 @@ export function runSpeech( { signal }, ); } -function environmentTranscriber(prepared: PreparedConnection, canSend: () => boolean) { +function environmentTranscriber( + prepared: PreparedConnection, + canSend: () => boolean, + provider?: CloudSpeechProvider, +) { return createEnvironmentTranscriber({ canSend, prepare: async (signal) => { + if (provider) { + await runSpeech(requireCloudSpeechKey(prepared, provider), signal, canSend); + return "auto"; + } const status = await runSpeech(getEnvironmentSpeechStatus(prepared), signal, canSend); if (!status.supported) throw new VoiceTranscriptionError({ code: "unavailable", message: status.reason }); @@ -126,7 +149,20 @@ function environmentTranscriber(prepared: PreparedConnection, canSend: () => boo upload: async (pcm, signal) => ( await runSpeech( - transcribeEnvironmentPcm(prepared, pcm, DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, canSend), + provider + ? transcribeCloudPcm( + prepared, + provider, + pcm, + DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, + canSend, + ) + : transcribeEnvironmentPcm( + prepared, + pcm, + DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, + canSend, + ), signal, canSend, ) @@ -137,9 +173,10 @@ function environmentTranscriber(prepared: PreparedConnection, canSend: () => boo const speechAtom = ( id: EnvironmentId, read: (prepared: PreparedConnection) => Effect.Effect, + connectionAtom = voiceConnectionAtom, ) => Atom.make((get) => { - const prepared = get(voiceConnectionAtom(id)); + const prepared = get(connectionAtom(id)); return prepared ? Effect.tryPromise({ try: (signal) => runSpeech(read(prepared), signal), @@ -155,6 +192,9 @@ const speechAtom = ( export const voiceSpeechStatusAtom = Atom.family((id: EnvironmentId) => speechAtom(id, getEnvironmentSpeechStatus), ); +export const cloudVoiceKeysAtom = Atom.family((id: EnvironmentId) => + speechAtom(id, getCloudSpeechKeys, cloudVoiceConnectionAtom), +); export const voiceSpeechModelsAtom = Atom.family((id: EnvironmentId) => speechAtom(id, (prepared) => getEnvironmentSpeechModels(prepared)), ); @@ -167,7 +207,10 @@ export const voiceSpeechBenchmarksAtom = Atom.family((id: EnvironmentId) => export const voiceRouteAtom = Atom.make((get) => { const grant = get(voiceGrantAtom); if (!grant) return null; - const connectionAtom = voiceConnectionAtom(grant.destination.environmentId); + const provider = grant.destination.kind === "cloud" ? grant.destination.provider : undefined; + const connectionAtom = (provider ? cloudVoiceConnectionAtom : voiceConnectionAtom)( + grant.destination.environmentId, + ); const connection = get(connectionAtom); const registry = get.registry; return { @@ -176,6 +219,7 @@ export const voiceRouteAtom = Atom.make((get) => { environmentTranscriber( connection, () => registry.get(voiceGrantAtom) === grant && registry.get(connectionAtom) === connection, + provider, ), }; }); diff --git a/apps/server/src/environment/ServerEnvironment.ts b/apps/server/src/environment/ServerEnvironment.ts index 3e780298d70a6..6d2c30db16130 100644 --- a/apps/server/src/environment/ServerEnvironment.ts +++ b/apps/server/src/environment/ServerEnvironment.ts @@ -274,6 +274,7 @@ export const make = Effect.gen(function* () { scheduledTaskThreadModes: true, npmReleaseLookup: true, voiceTranscription: isSpeechPlatformSupported(hostPlatform, hostArchitecture, hostLibc), + cloudVoiceTranscription: true, ...(serverSelfUpdate === null ? {} : { serverSelfUpdate }), ...(serverInstallation === null ? {} : { serverInstallation }), ...(serverSelfUpdate === "boot-service" || desktopAppUpdate diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 5c9ab9c433ffc..e7baa0a91164e 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -43,6 +43,7 @@ import * as NodePtyAdapter from "./terminal/NodePtyAdapter.ts"; import * as PullRequestHttp from "./pullRequest/http.ts"; import * as SpeechModelStore from "./speech/SpeechModelStore.ts"; import * as SpeechService from "./speech/SpeechService.ts"; +import * as CloudSpeech from "./speech/CloudSpeech.ts"; import { speechHttpApiLayer } from "./speech/http.ts"; import { speechStreamRouteLayer } from "./speech/stream.ts"; import * as PullRequestProviderRegistry from "./pullRequest/PullRequestProviderRegistry.ts"; @@ -823,6 +824,7 @@ const layerMakeRoutes = Layer.mergeAll( Layer.provide(SpeechModelStore.layer), ), ), + Layer.provide(CloudSpeech.layer.pipe(Layer.provide(SecretResolver.layer))), Layer.provide(layerServerSettings), Layer.provide(PreviewAutomationBroker.layer), Layer.provide(ServerSelfUpdate.layer.pipe(Layer.provide(layerDesktopAppUpdate))), diff --git a/apps/server/src/serverSettings.test.ts b/apps/server/src/serverSettings.test.ts index caba93a788b7f..21a7ed4fae1c8 100644 --- a/apps/server/src/serverSettings.test.ts +++ b/apps/server/src/serverSettings.test.ts @@ -1642,6 +1642,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" const openCode = ProviderInstanceId.make("opencode"); const hub = UsageLimitSourceId.make("hub"); const withUsage = yield* serverSettings.updateSettings({ + cloudSpeechCredentials: { openai: usageReference, xai: "literal-fixture" }, usageLimitSources: { [hub]: { kind: "cliproxy", @@ -1687,6 +1688,10 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" secretReferences: false, }); assert.notInclude(JSON.stringify(legacy), "op://"); + assert.deepEqual(legacy.cloudSpeechCredentials, { + openai: SECRET_REDACTED, + xai: SECRET_REDACTED, + }); assert.equal(legacy.sourceControlHosts.bitbucket?.accessToken, SECRET_REDACTED); assert.equal(legacy.usageLimitSources[hub]?.managementKey, SECRET_REDACTED); assert.equal(legacy.easDeviceHosts[0]?.accessToken, SECRET_REDACTED); @@ -1696,6 +1701,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" assert.equal(configOf(legacy, antigravity)?.apiKey, SECRET_REDACTED); assert.equal(configOf(legacy, openCode)?.serverPassword, SECRET_REDACTED); const echoed = yield* serverSettings.updateSettings({ + cloudSpeechCredentials: legacy.cloudSpeechCredentials, sourceControlHosts: legacy.sourceControlHosts, usageLimitSources: legacy.usageLimitSources, easDeviceHosts: legacy.easDeviceHosts, @@ -1708,6 +1714,10 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" "ghe.acme.test": "ghp_ghe", }); assert.deepEqual(echoed.usageLimitSources[hub]?.managementKey, usageReference); + assert.deepEqual(echoed.cloudSpeechCredentials, { + openai: usageReference, + xai: "literal-fixture", + }); assert.deepEqual(echoed.easDeviceHosts[0]?.accessToken, easReference); assert.deepEqual(echoed.simlockDeviceHosts[0]?.agentToken, simlockReference); assert.deepEqual(configOf(echoed, antigravity)?.apiKey, antigravityReference); @@ -1747,6 +1757,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" bitbucket: { accessToken: personal("Bitbucket/token") }, github: { tokens: { "github.com": personal("GitHub/token"), "ghe.acme.test": work } }, }, + cloudSpeechCredentials: { openai: personal("Voice/key"), xai: work }, usageLimitSources: { [hub]: { kind: "cliproxy", @@ -1791,6 +1802,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" "ghe.acme.test": work, }); assert.equal(removed.usageLimitSources[hub]?.managementKey, ""); + assert.deepEqual(removed.cloudSpeechCredentials, { openai: "", xai: work }); assert.equal(removed.easDeviceHosts[0]?.accessToken, ""); assert.deepEqual(Object.keys(removed.secretProviders), ["work"]); }).pipe(Effect.provide(layerServerSettingsWithSecrets())), @@ -1800,6 +1812,8 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" Effect.sync(() => { // Shipped clients decode these as strings. Each must be covered by the older-client case above. assert.deepEqual(strictSecretReferencePaths(ServerSettings.ast), [ + "cloudSpeechCredentials.openai", + "cloudSpeechCredentials.xai", "easDeviceHosts.[].accessToken", "simlockDeviceHosts.[].agentToken", "usageLimitSources.*.managementKey", diff --git a/apps/server/src/settingsSecrets.ts b/apps/server/src/settingsSecrets.ts index ce93b51a8ebaf..ae69a61f1f5dd 100644 --- a/apps/server/src/settingsSecrets.ts +++ b/apps/server/src/settingsSecrets.ts @@ -1,6 +1,7 @@ import { AntigravitySettings, ClaudeSettings, + CloudSpeechCredentials, CodexSettings, EasDeviceHostConfig, type ProviderInstanceConfig, @@ -45,6 +46,7 @@ const providerSecretFields = new Map([ ), ]); const usageLimitSecretFields = secretSettingsFields(UsageLimitSourceConfig); +const voiceSecretFields = secretSettingsFields(CloudSpeechCredentials); const easDeviceHostSecretFields = secretSettingsFields(EasDeviceHostConfig); const simlockDeviceHostSecretFields = secretSettingsFields(SimlockDeviceHostConfig); const encode = (value: string) => Buffer.from(value, "utf8").toString("base64url"); @@ -232,6 +234,11 @@ function mapSecretFieldGroups( return { ...settings, + cloudSpeechCredentials: mapFields( + settings.cloudSpeechCredentials, + voiceSecretFields, + (provider) => `voice-cloud-${provider}`, + ) as ServerSettings["cloudSpeechCredentials"], providerInstances, sourceControlHosts, usageLimitSources, diff --git a/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts b/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts index 10a37b631ecf5..c84ed1f58d763 100644 --- a/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts +++ b/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts @@ -164,6 +164,7 @@ it.effect("reads a collected reference again once it is invalidated", () => }, ]; cache.set("cliproxy#key", "stale"); + cache.set("voice#key", "stale"); cache.set("expo#token", "stale"); cache.set("simlock#token", "stale"); assert.deepEqual((yield* read()).bitbucket, { @@ -175,12 +176,14 @@ it.effect("reads a collected reference again once it is invalidated", () => resolver, Effect.succeed({ sourceControlHosts: hosts, + cloudSpeechCredentials: { openai: source("voice#key"), xai: "" }, usageLimitSources, easDeviceHosts, simlockDeviceHosts, }), ); assert.isFalse(cache.has("cliproxy#key")); + assert.isFalse(cache.has("voice#key")); assert.isFalse(cache.has("expo#token")); assert.isFalse(cache.has("simlock#token")); assert.deepEqual((yield* read()).bitbucket, { diff --git a/apps/server/src/sourceControl/sourceControlHostSecrets.ts b/apps/server/src/sourceControl/sourceControlHostSecrets.ts index ba5a868e0799d..94eec45d7c9ea 100644 --- a/apps/server/src/sourceControl/sourceControlHostSecrets.ts +++ b/apps/server/src/sourceControl/sourceControlHostSecrets.ts @@ -118,7 +118,11 @@ export const invalidateSettingsSecrets = Effect.fn("SettingsSecrets.invalidate") settings: Effect.Effect< Pick< ServerSettings, - "sourceControlHosts" | "usageLimitSources" | "easDeviceHosts" | "simlockDeviceHosts" + | "sourceControlHosts" + | "usageLimitSources" + | "easDeviceHosts" + | "simlockDeviceHosts" + | "cloudSpeechCredentials" >, E >, @@ -128,6 +132,7 @@ export const invalidateSettingsSecrets = Effect.fn("SettingsSecrets.invalidate") ...collectSourceControlHostSecretReferences(current.sourceControlHosts), ...collectUsageLimitSecretReferences(current.usageLimitSources), ...collectDeviceHostSecretReferences(current), + ...Object.values(current.cloudSpeechCredentials).filter(isSecretReference), ]), Effect.orElseSucceed(() => undefined), ); diff --git a/apps/server/src/speech/CloudSpeech.test.ts b/apps/server/src/speech/CloudSpeech.test.ts new file mode 100644 index 0000000000000..263a73dcd692a --- /dev/null +++ b/apps/server/src/speech/CloudSpeech.test.ts @@ -0,0 +1,106 @@ +import { it } from "@effect/vitest"; +import { + CloudSpeechProvider, + DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, + SecretProviderId, +} from "@t3tools/contracts"; +import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Redacted from "effect/Redacted"; +import * as FetchHttpClient from "effect/http/FetchHttpClient"; +import { expect, vi } from "vite-plus/test"; +import * as ServerSettings from "../serverSettings.ts"; +import * as CloudSpeech from "./CloudSpeech.ts"; + +it.effect.each(CloudSpeechProvider.literals)( + "sends only to %s using its own saved key", + (provider) => + Effect.gen(function* () { + const reference = { + provider: SecretProviderId.make("work"), + reference: "op://Work/Voice/key", + }; + let resolved: string | undefined; + let cached: string | undefined; + const read = vi.fn(() => Effect.sync(() => [(cached ??= resolved)])); + const upstream = vi.fn(async () => Response.json({ text: "hello" })); + const service = yield* CloudSpeech.CloudSpeech.pipe( + Effect.provide( + CloudSpeech.layer.pipe( + Layer.provide(FetchHttpClient.layer), + Layer.provide(ServerSettings.ServerSettingsService.layerTest()), + Layer.provide( + Layer.mock(SecretResolver.SecretResolver)({ + read, + label: () => Effect.succeed("Work"), + invalidate: () => + Effect.sync(() => { + cached = undefined; + }), + }), + ), + ), + ), + ); + const transcribe = (pcm = new Uint8Array(new Float32Array([0, 0.5]).buffer)) => + service + .transcribe(provider, pcm, { + ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, + speechLanguage: "en", + speechCustomWords: [{ term: "T3 Code", aliases: [] }], + }) + .pipe(Effect.provideService(FetchHttpClient.Fetch, upstream)); + yield* service.setKey(provider === "openai" ? "xai" : "openai", Redacted.make("other")); + expect((yield* Effect.flip(transcribe()))._tag).toBe("CloudSpeechError"); + yield* service.setKey(provider, Redacted.make("chosen")); + expect(yield* service.configured).toEqual({ openai: true, xai: true }); + yield* Effect.flip(transcribe(new Uint8Array(3))); + expect(upstream).not.toHaveBeenCalled(); + expect(yield* transcribe()).toBe("hello"); + const [url, init] = upstream.mock.calls[0]!; + expect(String(url)).toBe( + { + openai: "https://api.openai.com/v1/audio/transcriptions", + xai: "https://api.x.ai/v1/stt", + }[provider], + ); + expect(init?.redirect).toBe("error"); + expect(new Headers(init?.headers).get("authorization")).toBe("Bearer chosen"); + const form = init?.body as FormData; + expect(form.get("model")).toBe( + provider === "openai" ? "gpt-transcribe" : "grok-voice-transcribe-2.0", + ); + expect(form.getAll(provider === "openai" ? "keywords[]" : "keyterm")).toEqual(["T3 Code"]); + expect(form.get(provider === "openai" ? "languages[]" : "language")).toBe("en"); + expect([...form.keys()].at(-1)).toBe("file"); + const wav = new DataView( + yield* Effect.promise(() => (form.get("file") as Blob).arrayBuffer()), + ); + expect(wav.getUint32(24, true)).toBe(16_000); + expect(wav.getInt16(46, true)).toBe(16384); + upstream.mockResolvedValueOnce( + new Response("sensitive upstream diagnostic", { status: 401 }), + ); + expect(yield* Effect.flip(transcribe())).toMatchObject({ provider, reason: "rejected-key" }); + yield* service.setKey(provider, null); + expect((yield* service.configured)[provider]).toBe(false); + yield* Effect.flip(transcribe()); + expect(upstream).toHaveBeenCalledTimes(2); + yield* service.setKey(provider, reference); + expect((yield* service.configured)[provider]).toBe(true); + expect(read).not.toHaveBeenCalled(); + resolved = "resolved-fixture"; + expect(yield* transcribe()).toBe("hello"); + expect(read).toHaveBeenCalledWith([reference]); + expect(new Headers(upstream.mock.calls[2]?.[1]?.headers).get("authorization")).toBe( + "Bearer resolved-fixture", + ); + resolved = undefined; + expect(yield* Effect.flip(transcribe())).toMatchObject({ + provider, + reason: "unreadable-key", + }); + expect(upstream).toHaveBeenCalledTimes(3); + }), +); diff --git a/apps/server/src/speech/CloudSpeech.ts b/apps/server/src/speech/CloudSpeech.ts new file mode 100644 index 0000000000000..4dd60d614db54 --- /dev/null +++ b/apps/server/src/speech/CloudSpeech.ts @@ -0,0 +1,187 @@ +import { + CloudSpeechApiKey, + CloudSpeechError, + isSecretReference, + type SecretReference, + type ServerSettingsError, + type CloudSpeechKeys, + type CloudSpeechProvider, + EnvironmentSpeechTranscriptionResult, + type SpeechTranscriptionOptions, +} from "@t3tools/contracts"; +import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; +import { cloudSpeechLanguages, effectiveSpeechLanguage } from "@t3tools/shared/speech"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Redacted from "effect/Redacted"; +import * as Schema from "effect/Schema"; +import { FetchHttpClient, HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http"; +import * as ServerSettings from "../serverSettings.ts"; +import { requireSettingsSecret } from "../secrets/settingsSecretValue.ts"; +import { applySpeechAliases } from "./customWords.ts"; +import { removeSpeechFillerWords } from "./fillerWords.ts"; +import { decodeSpeechPcm, SpeechInvalidAudioError } from "./SpeechService.ts"; + +const endpoints = { + openai: { url: "https://api.openai.com/v1/audio/transcriptions", model: "gpt-transcribe" }, + xai: { url: "https://api.x.ai/v1/stt", model: "grok-voice-transcribe-2.0" }, +}; +const isApiKey = Schema.is(CloudSpeechApiKey); + +function wave(samples: Float32Array) { + const buffer = new ArrayBuffer(44 + samples.length * 2); + const view = new DataView(buffer); + const tag = (at: number, value: string) => + new Uint8Array(buffer, at, 4).set(new TextEncoder().encode(value)); + tag(0, "RIFF"); + view.setUint32(4, buffer.byteLength - 8, true); + tag(8, "WAVE"); + tag(12, "fmt "); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); + view.setUint16(22, 1, true); + view.setUint32(24, 16_000, true); + view.setUint32(28, 32_000, true); + view.setUint16(32, 2, true); + view.setUint16(34, 16, true); + tag(36, "data"); + view.setUint32(40, buffer.byteLength - 44, true); + for (let index = 0; index < samples.length; index++) { + const sample = samples[index]!; + view.setInt16(44 + index * 2, Math.round(sample * (sample < 0 ? 32768 : 32767)), true); + } + return new Blob([buffer], { type: "audio/wav" }); +} + +function audioBody( + provider: CloudSpeechProvider, + audio: Blob, + language: string, + options: SpeechTranscriptionOptions, +) { + const form = new FormData(); + form.set("model", endpoints[provider].model); + if (language !== "auto") form.set(provider === "openai" ? "languages[]" : "language", language); + for (const { term } of options.speechCustomWords) { + if (provider === "openai" && /[<>\r\n]/.test(term)) continue; + form.append(provider === "openai" ? "keywords[]" : "keyterm", term); + } + if (provider === "xai") { + if (language !== "auto") form.set("format", "true"); + form.set( + "filler_words", + String(!options.speechRemoveFillerWords || Boolean(options.speechCorrectionWord)), + ); + if (!options.speechSilenceFiltering) form.set("vad_threshold", "0"); + } + // xAI ignores options that follow the file part. + form.set("file", audio, "dictation.wav"); + return HttpClientRequest.bodyFormData(form); +} + +export class CloudSpeech extends Context.Service< + CloudSpeech, + { + readonly configured: Effect.Effect; + readonly setKey: ( + provider: CloudSpeechProvider, + key: Redacted.Redacted | SecretReference | null, + ) => Effect.Effect; + readonly transcribe: ( + provider: CloudSpeechProvider, + pcm: Uint8Array, + options: SpeechTranscriptionOptions, + ) => Effect.Effect; + } +>()("t3/speech/CloudSpeech") {} + +export const layer = Layer.effect( + CloudSpeech, + Effect.gen(function* () { + const settings = yield* ServerSettings.ServerSettingsService; + const resolver = yield* SecretResolver.SecretResolver; + const http = yield* HttpClient.HttpClient; + const credentials = settings.getSettings.pipe( + Effect.map((value) => value.cloudSpeechCredentials), + ); + const configured = credentials.pipe( + Effect.map(({ openai, xai }) => ({ openai: Boolean(openai), xai: Boolean(xai) })), + ); + const setKey = Effect.fn("CloudSpeech.setKey")(function* ( + provider: CloudSpeechProvider, + key: Redacted.Redacted | SecretReference | null, + ) { + yield* settings.updateSettings({ + cloudSpeechCredentials: { + [provider]: Redacted.isRedacted(key) ? Redacted.value(key) : (key ?? ""), + }, + }); + }); + const transcribe = Effect.fn("CloudSpeech.transcribe")( + function* ( + provider: CloudSpeechProvider, + pcm: Uint8Array, + options: SpeechTranscriptionOptions, + ) { + const source = (yield* credentials)[provider]; + if (!source) return yield* new CloudSpeechError({ provider, reason: "missing-key" }); + const samples = yield* Effect.try({ + try: () => decodeSpeechPcm(pcm), + catch: () => + new SpeechInvalidAudioError({ + byteLength: pcm.byteLength, + message: "Invalid PCM audio.", + }), + }); + const language = effectiveSpeechLanguage(cloudSpeechLanguages, options.speechLanguage); + // A vault that locked since the last recording must not authorize a new upload. + if (isSecretReference(source)) yield* resolver.invalidate([source]); + const unreadable = () => new CloudSpeechError({ provider, reason: "unreadable-key" }); + const key = yield* requireSettingsSecret(resolver, source).pipe( + Effect.mapError(unreadable), + ); + if (!isApiKey(key)) return yield* unreadable(); + const request = HttpClientRequest.post(endpoints[provider].url).pipe( + HttpClientRequest.bearerToken(Redacted.make(key)), + audioBody(provider, wave(samples), language, options), + ); + const unavailable = () => new CloudSpeechError({ provider, reason: "unavailable" }); + const response = yield* http + .execute(request) + .pipe( + Effect.provideService(FetchHttpClient.RequestInit, { redirect: "error" }), + Effect.provideService(HttpClient.TracerPropagationEnabled, false), + Effect.mapError(unavailable), + ); + if (response.status < 200 || response.status >= 300) + return yield* new CloudSpeechError({ + provider, + reason: + response.status === 429 + ? "rate-limit" + : [401, 402, 403].includes(response.status) + ? "rejected-key" + : "unavailable", + }); + const { text } = yield* HttpClientResponse.schemaBodyJson( + EnvironmentSpeechTranscriptionResult, + )(response).pipe(Effect.mapError(unavailable)); + const corrected = applySpeechAliases(text, options.speechCustomWords); + return ( + options.speechRemoveFillerWords + ? removeSpeechFillerWords(corrected, language, options.speechCorrectionWord) + : corrected + ).trim(); + }, + (effect, provider) => + effect.pipe( + Effect.timeoutOrElse({ + duration: "2 minutes", + orElse: () => Effect.fail(new CloudSpeechError({ provider, reason: "unavailable" })), + }), + ), + ); + return CloudSpeech.of({ configured, setKey, transcribe }); + }), +); diff --git a/apps/server/src/speech/http.test.ts b/apps/server/src/speech/http.test.ts index 169a1e893cef8..78d01e2ee8a70 100644 --- a/apps/server/src/speech/http.test.ts +++ b/apps/server/src/speech/http.test.ts @@ -7,6 +7,9 @@ import { expect, vi } from "vite-plus/test"; import { AuthSessionId, AuthOrchestrationOperateScope, + AuthSettingsWriteScope, + type AuthEnvironmentScope, + CloudSpeechError, DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, SPEECH_MAX_OPTIONS_BYTES, DEFAULT_SERVER_SETTINGS, @@ -16,6 +19,7 @@ import { } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Redacted from "effect/Redacted"; import * as Schema from "effect/Schema"; import * as HttpRouter from "effect/http/HttpRouter"; import * as HttpServer from "effect/http/HttpServer"; @@ -23,6 +27,7 @@ import * as HttpApi from "effect/http-api/HttpApi"; import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder"; import type * as NetAddress from "effect/net/NetAddress"; import * as SpeechService from "./SpeechService.ts"; +import * as CloudSpeech from "./CloudSpeech.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as TextGeneration from "../textGeneration/TextGeneration.ts"; import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; @@ -30,7 +35,20 @@ import { speechHttpApiLayer } from "./http.ts"; const decodeSessionId = Schema.decodeUnknownSync(AuthSessionId); +const principal = (...scopes: AuthEnvironmentScope[]) => + Layer.succeed(EnvironmentAuthenticatedAuth, (effect) => + effect.pipe( + Effect.provideService(EnvironmentAuthenticatedPrincipal, { + sessionId: decodeSessionId("voice-test"), + subject: "test", + method: "bearer-access-token", + scopes: new Set(scopes), + }), + ), + ); + const dependencies = Layer.mergeAll( + Layer.mock(CloudSpeech.CloudSpeech)({}), Layer.mock(ProviderRegistry.ProviderRegistry)({ getProviders: Effect.succeed([]) }), Layer.mock(ServerSettings.ServerSettingsService)({ getSettings: Effect.succeed(DEFAULT_SERVER_SETTINGS), @@ -51,21 +69,11 @@ const dependencies = Layer.mergeAll( effectIt.live("rejects oversized chunked audio without invoking transcription", () => Effect.gen(function* () { const transcribe = vi.fn(() => Effect.succeed("unexpected")); - const auth = Layer.succeed(EnvironmentAuthenticatedAuth, (effect) => - effect.pipe( - Effect.provideService(EnvironmentAuthenticatedPrincipal, { - sessionId: decodeSessionId("voice-test"), - subject: "test", - method: "bearer-access-token", - scopes: new Set([AuthOrchestrationOperateScope]), - }), - ), - ); const routes = HttpApiBuilder.layer( HttpApi.make("environment").add(EnvironmentVoiceHttpApi), ).pipe( Layer.provide(speechHttpApiLayer), - Layer.provide(auth), + Layer.provide(principal(AuthOrchestrationOperateScope)), Layer.provide( Layer.succeed(SpeechService.SpeechService, { status: Effect.succeed({ supported: false as const, reason: "test" }), @@ -97,6 +105,14 @@ effectIt.live("rejects oversized chunked audio without invoking transcription", }), ); expect(select.status).toBe(403); + const changeKey = yield* Effect.promise(() => + fetch(new URL("cloud/xai/key", url), { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ key: "fixture" }), + }), + ); + expect(changeKey.status).toBe(403); yield* Effect.promise(() => expect( fetch(url, { @@ -120,6 +136,80 @@ effectIt.live("rejects oversized chunked audio without invoking transcription", }), ); +effectIt.effect( + "keeps keys write-only and returns provider failures without an auth challenge", + () => + Effect.gen(function* () { + let stored = false; + const reference = { provider: "work", reference: "op://Work/Voice/key" }; + const save = vi.fn(); + const cloud = Layer.mock(CloudSpeech.CloudSpeech)({ + configured: Effect.sync(() => ({ openai: stored, xai: false })), + setKey: (provider, key) => + Effect.sync(() => { + save(provider, key); + stored = + provider === "openai" && + Redacted.isRedacted(key) && + Redacted.value(key) === "fixture"; + }), + transcribe: (provider) => + Effect.fail(new CloudSpeechError({ provider, reason: "rejected-key" })), + }); + const { handler, dispose } = HttpRouter.toWebHandler( + HttpApiBuilder.layer(HttpApi.make("environment").add(EnvironmentVoiceHttpApi)).pipe( + Layer.provide(speechHttpApiLayer), + Layer.provide(principal(AuthOrchestrationOperateScope, AuthSettingsWriteScope)), + Layer.provide(Layer.mock(SpeechService.SpeechService)({})), + Layer.provide(cloud), + Layer.provide(dependencies), + Layer.provide(HttpServer.layerServices), + Layer.provide(NodeServices.layer), + ), + { disableLogger: true }, + ); + yield* Effect.addFinalizer(() => Effect.promise(dispose)); + const context = yield* Layer.build(NodeServices.layer); + const call = (path: string, init?: RequestInit) => + Effect.promise(() => + handler(new Request(`http://localhost/api/voice/cloud/${path}`, init), context), + ); + const saved = yield* call("openai/key", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ key: "fixture" }), + }); + expect(saved.status).toBe(204); + expect(yield* Effect.promise(() => saved.text())).toBe(""); + const keys = yield* call("keys"); + expect(yield* Effect.promise(() => keys.json())).toEqual({ + openai: true, + xai: false, + }); + const setReference = (key: unknown) => + call("xai/key", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ key }), + }); + expect((yield* setReference(reference)).status).toBe(204); + expect(save).toHaveBeenLastCalledWith("xai", reference); + expect((yield* setReference(reference.reference)).status).toBe(400); + expect(save).toHaveBeenCalledTimes(2); + const failed = yield* call("openai/transcribe", { + method: "POST", + headers: { "content-type": "application/octet-stream" }, + body: encodeSpeechPcmRequest(new Uint8Array(4), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS), + }); + expect(failed.status).toBe(422); + expect(yield* Effect.promise(() => failed.json())).toEqual({ + _tag: "CloudSpeechError", + provider: "openai", + reason: "rejected-key", + }); + }).pipe(Effect.scoped), +); + effectIt.effect.each([ { error: new SpeechService.SpeechInvalidAudioError({ byteLength: 3, message: "invalid PCM" }), @@ -151,16 +241,6 @@ effectIt.effect.each([ ({ error, reason, ...options }) => Effect.gen(function* () { // Only authentication and the native service are replaced; exercise the actual HTTP handlers and codecs. - const auth = Layer.succeed(EnvironmentAuthenticatedAuth, (effect) => - effect.pipe( - Effect.provideService(EnvironmentAuthenticatedPrincipal, { - sessionId: decodeSessionId("voice-test"), - subject: "test", - method: "bearer-access-token", - scopes: new Set([AuthOrchestrationOperateScope]), - }), - ), - ); const transcribe = vi.fn(() => Effect.fail(error)); const service = Layer.succeed(SpeechService.SpeechService, { status: Effect.succeed({ supported: false as const, reason: "test" }), @@ -179,7 +259,7 @@ effectIt.effect.each([ const { handler, dispose } = HttpRouter.toWebHandler( HttpApiBuilder.layer(api).pipe( Layer.provide(speechHttpApiLayer), - Layer.provide(auth), + Layer.provide(principal(AuthOrchestrationOperateScope)), Layer.provide(service), Layer.provide(HttpServer.layerServices), Layer.provide(dependencies), diff --git a/apps/server/src/speech/http.ts b/apps/server/src/speech/http.ts index 2e1d1ae71fced..d1c2fc58a07f1 100644 --- a/apps/server/src/speech/http.ts +++ b/apps/server/src/speech/http.ts @@ -21,6 +21,7 @@ import { requireEnvironmentScope, } from "../auth/http.ts"; import * as SpeechPostProcessing from "./postProcessing.ts"; +import * as CloudSpeech from "./CloudSpeech.ts"; import * as SpeechService from "./SpeechService.ts"; const MAX_TRANSCRIBE_BODY_BYTES = SpeechService.MAX_SPEECH_BYTES + SPEECH_MAX_OPTIONS_BYTES + 4; @@ -57,6 +58,7 @@ export const speechHttpApiLayer = HttpApiBuilder.group( "voice", Effect.fnUntraced(function* (handlers) { const speech = yield* SpeechService.SpeechService; + const cloud = yield* CloudSpeech.CloudSpeech; const postProcessing = yield* SpeechPostProcessing.SpeechPostProcessing; const guard = Effect.fnUntraced(function* ( endpoint: string, @@ -66,6 +68,41 @@ export const speechHttpApiLayer = HttpApiBuilder.group( yield* requireEnvironmentScope(scope); }); return handlers + .handle( + "cloudKeys", + Effect.fn("environment.voice.cloudKeys")(function* (args) { + yield* guard(args.endpoint.name); + return yield* cloud.configured.pipe( + Effect.catch((error) => failEnvironmentInternal("internal_error", error)), + ); + }), + ) + .handle( + "setCloudKey", + Effect.fn("environment.voice.setCloudKey")(function* (args) { + yield* guard(args.endpoint.name, AuthSettingsWriteScope); + yield* cloud + .setKey(args.params.provider, args.payload.key) + .pipe(Effect.catch((error) => failEnvironmentInternal("internal_error", error))); + }), + ) + .handle( + "transcribeCloud", + Effect.fn("environment.voice.transcribeCloud")(function* (args) { + yield* guard(args.endpoint.name); + const decoded = parseAudio(args.payload); + if (Option.isNone(decoded)) return yield* failEnvironmentInvalidRequest("invalid_audio"); + const text = yield* cloud + .transcribe(args.params.provider, decoded.value.pcm, decoded.value.options) + .pipe( + Effect.catchTags({ + ServerSettingsError: (error) => failEnvironmentInternal("internal_error", error), + SpeechInvalidAudioError: () => failEnvironmentInvalidRequest("invalid_audio"), + }), + ); + return { text }; + }), + ) .handle( "status", Effect.fn("environment.voice.status")(function* (args) { diff --git a/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx b/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx new file mode 100644 index 0000000000000..fe4bd59164a80 --- /dev/null +++ b/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx @@ -0,0 +1,132 @@ +import { PrimaryConnectionTarget } from "@t3tools/client-runtime/connection"; +import { EnvironmentId } from "@t3tools/contracts"; +import * as Option from "effect/Option"; +import { expect, it, vi } from "vite-plus/test"; +import { page, userEvent } from "vite-plus/test/browser"; +import { render } from "../../../test/render"; +import "~/index.css"; +import { chooseAudioDestination } from "../../speech/voiceConsent"; +import { useEnvironmentScope, usePreparedConnection } from "../../state/session"; +import { VoiceSettingsPanel } from "./VoiceSettingsPanel"; +import { SettingsScopeProvider } from "./SettingsScopeContext"; + +vi.mock(import("../../state/session"), { spy: true }); +vi.mock("../../env", async (original) => ({ + ...(await original()), + isSecretsFeatureEnabled: true, +})); +vi.mock("../../state/environments", async (importOriginal) => ({ + ...(await importOriginal()), + useEnvironments: () => ({ + environments: [ + { + environmentId: "voice-host", + label: "Laptop", + connection: { phase: "available" }, + serverConfig: { + secretReferences: true, + settings: { + secretProviders: { + work: { kind: "one_password", label: "Work vault", config: { account: "work" } }, + }, + }, + environment: { + platform: { os: "linux", arch: "x64" }, + capabilities: { voiceTranscription: true, cloudVoiceTranscription: true }, + }, + }, + }, + ], + }), +})); + +it("enables the chosen provider only after a successful setup and keeps its grant when editing the key", async () => { + const id = EnvironmentId.make("voice-host"); + const target = new PrimaryConnectionTarget({ + environmentId: id, + label: "Laptop", + httpBaseUrl: window.location.origin, + wsBaseUrl: "ws://localhost", + }); + const prepared = Option.some({ + ...target, + target, + socketUrl: target.wsBaseUrl, + httpAuthorization: null, + }); + using cleanup = new DisposableStack(); + cleanup.defer(() => chooseAudioDestination(null)); + chooseAudioDestination({ kind: "environment", environmentId: id }); + const writes: unknown[] = []; + let rejectNextWrite = true; + const fetch = globalThis.fetch; + using _network = vi.spyOn(globalThis, "fetch").mockImplementation((request, init) => { + const url = new URL(request instanceof Request ? request.url : request, window.location.href); + if (url.pathname === "/api/voice/cloud/keys") + return Promise.resolve(Response.json({ openai: writes.length > 0, xai: true })); + if (url.pathname === "/api/voice/cloud/openai/key") { + if (rejectNextWrite) { + rejectNextWrite = false; + return Promise.resolve(new Response(null, { status: 403 })); + } + return new Request(request, init).json().then((body) => { + writes.push(body); + return new Response(null, { status: 204 }); + }); + } + return fetch(request, init); + }); + await vi.mocked(useEnvironmentScope).withImplementation( + () => true, + async () => { + await vi.mocked(usePreparedConnection).withImplementation( + () => prepared, + async () => { + await using screen = await render( + {}}> + + , + ); + const destination = screen.getByRole("combobox", { name: "Transcribe with" }); + await expect.element(destination).toHaveTextContent("Laptop"); + await destination.click(); + await page.getByRole("option", { name: "OpenAI via Laptop" }).click(); + const dialog = page.getByRole("dialog"); + await dialog.getByRole("button", { name: "Cancel" }).click(); + await expect.element(destination).toHaveTextContent("Laptop"); + expect(writes).toEqual([]); + await destination.click(); + await page.getByRole("option", { name: "OpenAI via Laptop" }).click(); + await dialog.getByRole("combobox", { name: "OpenAI API key source" }).click(); + await page.getByRole("option", { name: "Work vault" }).click(); + await dialog.getByLabelText("OpenAI API key reference").fill("op://Work/Voice/key"); + await userEvent.keyboard("{Enter}"); + await expect.element(dialog.getByRole("alert")).toBeVisible(); + expect(writes).toEqual([]); + await expect + .element(dialog.getByRole("button", { name: "Save and enable" })) + .toBeEnabled(); + await dialog.getByLabelText("OpenAI API key reference").click(); + await userEvent.keyboard("{Enter}"); + await expect.element(dialog).not.toBeInTheDocument(); + await expect.element(destination).toHaveTextContent("OpenAI via Laptop"); + expect(writes).toEqual([{ key: { provider: "work", reference: "op://Work/Voice/key" } }]); + const grant = window.localStorage.getItem("t3code:voice-audio-grant:v1"); + await screen.getByRole("button", { name: "Manage key" }).click(); + await dialog.getByLabelText("OpenAI API key", { exact: true }).fill("replacement"); + await userEvent.keyboard("{Enter}"); + await expect.element(dialog).not.toBeInTheDocument(); + expect(writes.at(-1)).toEqual({ key: "replacement" }); + expect(window.localStorage.getItem("t3code:voice-audio-grant:v1")).toBe(grant); + await destination.click(); + await page.getByRole("option", { name: "Grok Voice Transcribe via Laptop" }).click(); + await dialog.getByRole("button", { name: "Use Grok Voice Transcribe" }).click(); + await expect.element(destination).toHaveTextContent("Grok Voice Transcribe via Laptop"); + await destination.click(); + await page.getByRole("option", { name: "Off", exact: true }).click(); + await expect.element(destination).toHaveTextContent("Off"); + }, + ); + }, + ); +}); diff --git a/apps/web/src/components/settings/CloudVoiceConfiguration.tsx b/apps/web/src/components/settings/CloudVoiceConfiguration.tsx new file mode 100644 index 0000000000000..23e472a36f7fd --- /dev/null +++ b/apps/web/src/components/settings/CloudVoiceConfiguration.tsx @@ -0,0 +1,272 @@ +import { useAtomRefresh, useAtomValue } from "@effect/atom-react"; +import type { PreparedConnection } from "@t3tools/client-runtime/connection"; +import { + configuredSecretSources, + secretRequestReference, +} from "@t3tools/client-runtime/secret-request"; +import { getCloudSpeechKeys, setCloudSpeechKey } from "@t3tools/client-runtime/voice-input"; +import { + AuthOrchestrationOperateScope, + AuthSettingsWriteScope, + CloudSpeechApiKey, + CLOUD_SPEECH_LABELS, + CLOUD_SPEECH_PROVIDER_NAMES, + isSecretReference, + type CloudSpeechProvider, + type SecretProviderId, + type SettingsSecretValue, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import { Atom, AsyncResult } from "effect/reactivity"; +import { useId, useState } from "react"; +import { isSecretsFeatureEnabled } from "../../env"; +import { layer, runtime } from "../../lib/runtime"; +import { chooseAudioDestination } from "../../speech/voiceConsent"; +import type { EnvironmentPresentation } from "../../state/environments"; +import { useEnvironmentScope, usePreparedConnection } from "../../state/session"; +import { Button } from "../ui/button"; +import { + Dialog, + DialogDescription, + DialogFooter, + DialogHeader, + DialogPanel, + DialogPopup, + DialogTitle, +} from "../ui/dialog"; +import { Input } from "../ui/input"; +import { Label } from "../ui/label"; +import { SecretInput } from "./SecretInput"; +import { SecretSourceSelect } from "./SecretSourceInput"; + +const cloudKeysAtom = Atom.family((prepared: PreparedConnection | null) => + Atom.make( + prepared ? getCloudSpeechKeys(prepared).pipe(Effect.provide(layer)) : Effect.succeed(null), + ), +); +const isApiKey = Schema.is(CloudSpeechApiKey); + +export function CloudVoiceConfiguration({ + environment, + provider, + active, + close, +}: { + environment: Pick; + provider: CloudSpeechProvider; + active: boolean; + close: () => void; +}) { + const id = environment.environmentId; + const name = CLOUD_SPEECH_LABELS[provider]; + const company = CLOUD_SPEECH_PROVIDER_NAMES[provider]; + const formId = useId(); + const prepared = Option.getOrNull(usePreparedConnection(id)); + const canUse = useEnvironmentScope(id, AuthOrchestrationOperateScope); + const canManage = useEnvironmentScope(id, AuthSettingsWriteScope); + const keys = cloudKeysAtom(canUse ? prepared : null); + const presence = useAtomValue(keys); + const refresh = useAtomRefresh(keys); + const [remove, setRemove] = useState(false); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(""); + const saved = AsyncResult.isSuccess(presence) ? presence.value?.[provider] : undefined; + const existing = environment.serverConfig?.settings?.cloudSpeechCredentials?.[provider]; + const sources = + isSecretsFeatureEnabled && environment.serverConfig?.secretReferences + ? configuredSecretSources(environment.serverConfig.settings?.secretProviders ?? {}) + : []; + const reference = + isSecretReference(existing) && sources.some((s) => s.id === existing.provider) + ? existing + : null; + const [sourceId, setSourceId] = useState(reference?.provider ?? null); + const [draft, setDraft] = useState(reference?.reference ?? ""); + const source = sources.find((s) => s.id === sourceId); + const credential = + sourceId !== null + ? source && secretRequestReference(source, draft) + : isApiKey(draft.trim()) + ? draft.trim() + : undefined; + const editing = canManage && saved !== undefined && (active || !saved); + const connected = canUse && prepared !== null; + const enable = () => chooseAudioDestination({ kind: "cloud", provider, environmentId: id }); + const action = async (value: SettingsSecretValue | null) => { + if (!connected || !canManage || busy) return; + setError(""); + setBusy(true); + try { + await runtime.runPromise(setCloudSpeechKey(prepared, provider, value)); + refresh(); + if (!active && value !== null) enable(); + close(); + } catch { + setError("Couldn’t save this change. Check your connection and try again."); + } + setBusy(false); + }; + + return ( + { + if (!open && !busy) close(); + }} + > + + + + {remove + ? `Remove the ${company} key?` + : active + ? `Manage ${company} key` + : `Use ${name} for dictation?`} + + + {remove + ? `Everyone using ${name} for dictation on ${environment.label} will lose access.` + : active + ? `Anyone with dictation access to ${environment.label} can use this API key.` + : `When you stop recording, ${environment.label} sends your audio and vocabulary hints, including project names, to ${company}.`} + + + {!remove && ( + + {!connected ? ( +

Connect to {environment.label} with dictation access to continue.

+ ) : AsyncResult.isFailure(presence) && saved === undefined ? ( + + ) : saved === undefined ? ( +

Checking for an API key…

+ ) : editing ? ( +
{ + event.preventDefault(); + if (credential) void action(credential); + }} + > + +
+ {sources.length > 0 && ( + { + setSourceId(next); + setDraft(""); + setError(""); + }} + /> + )} +
+ {sourceId !== null ? ( + setDraft(event.target.value)} + /> + ) : ( + setDraft(event.target.value)} + /> + )} +
+
+

+ {draft.trim() && !credential + ? (source?.invalidReferenceMessage ?? + "Enter an API key, not a URL or secret reference.") + : active + ? "Updating the key keeps dictation enabled." + : `Anyone with dictation access to ${environment.label} can bill this key.`} +

+
+ ) : !saved ? ( +

Ask an administrator on {environment.label} to add an API key.

+ ) : ( +

+ This choice applies only to this device. Choose Off to stop sharing. +

+ )} +
+ )} + {error && ( +

+ {error} +

+ )} + + {connected && canManage && saved && active && !remove && ( +
+ +
+ )} + + {connected && + (remove ? ( + + ) : editing ? ( + + ) : saved && !active ? ( + + ) : null)} +
+
+
+ ); +} diff --git a/apps/web/src/components/settings/SecretsSettings.logic.test.ts b/apps/web/src/components/settings/SecretsSettings.logic.test.ts index 1a0ff29e11cfb..1480844f58baf 100644 --- a/apps/web/src/components/settings/SecretsSettings.logic.test.ts +++ b/apps/web/src/components/settings/SecretsSettings.logic.test.ts @@ -31,6 +31,7 @@ const settings = Schema.decodeSync(ServerSettings)({ easDeviceHosts: [ { kind: "eas", id: "eas", label: "EAS Simulator", accessToken: personal("op://expo") }, ], + cloudSpeechCredentials: { openai: personal("op://voice"), xai: "" }, }); it.each([ @@ -42,6 +43,7 @@ it.each([ "GitHub · github.com", "https://hub.test · Management Key", "EAS Simulator · Access Token", + "Voice · OpenAI", ], ], ["work", ["Antigravity Work · OTHER_TOKEN", "GitHub · ghe.acme.test"]], diff --git a/apps/web/src/components/settings/SecretsSettings.logic.ts b/apps/web/src/components/settings/SecretsSettings.logic.ts index 3ae528aed162a..a85bc9e330143 100644 --- a/apps/web/src/components/settings/SecretsSettings.logic.ts +++ b/apps/web/src/components/settings/SecretsSettings.logic.ts @@ -1,4 +1,6 @@ import { + CLOUD_SPEECH_LABELS, + CloudSpeechProvider, EasDeviceHostConfig, isSecretReference, secretReferencesInSettings, @@ -12,7 +14,11 @@ import { deriveProviderSettingsFields, type SettingsFormDefinition } from "./Pro type SecretUseSettings = Pick< ServerSettings, - "providerInstances" | "sourceControlHosts" | "usageLimitSources" | "easDeviceHosts" + | "providerInstances" + | "sourceControlHosts" + | "usageLimitSources" + | "easDeviceHosts" + | "cloudSpeechCredentials" >; interface SecretProviderUse { @@ -86,5 +92,11 @@ export function secretProviderUses( ...settings.easDeviceHosts.flatMap((host) => uses({ settingsSchema: EasDeviceHostConfig }, `eas:${host.id}`, host.label, host), ), + ...CloudSpeechProvider.literals.flatMap((provider) => { + const credential = settings.cloudSpeechCredentials[provider]; + return isSecretReference(credential) && credential.provider === providerId + ? [{ id: `voice:${provider}`, label: `Voice · ${CLOUD_SPEECH_LABELS[provider]}` }] + : []; + }), ]; } diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx index e62e7180fe073..c8e81874ce4b2 100644 --- a/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx +++ b/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx @@ -375,7 +375,7 @@ it.each([ ); await vi.waitFor(() => expect(document.activeElement?.id).toBe(prerequisite)); await expect - .element(screen.getByRole("combobox", { name: "Send microphone audio to" })) + .element(screen.getByRole("combobox", { name: "Transcribe with" })) .toHaveTextContent("Off"); await expect .element(screen.getByRole("switch", { name: "Clean up transcripts with a provider" })) @@ -394,7 +394,7 @@ it("keeps a missing destination selected without claiming it cannot transcribe", , ); await expect - .element(screen.getByRole("combobox", { name: "Send microphone audio to" })) + .element(screen.getByRole("combobox", { name: "Transcribe with" })) .toHaveTextContent("Chosen environment (unavailable)"); await expect.element(screen.getByText(/can’t transcribe audio/)).not.toBeInTheDocument(); }); diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.tsx index fc3517831268e..dd4d6318761c7 100644 --- a/apps/web/src/components/settings/VoiceSettingsPanel.tsx +++ b/apps/web/src/components/settings/VoiceSettingsPanel.tsx @@ -11,11 +11,14 @@ import { import type { PreparedConnection } from "@t3tools/client-runtime/connection"; import { AuthSettingsWriteScope, + CLOUD_SPEECH_LABELS, + CloudSpeechProvider, DEFAULT_SPEECH_POST_PROCESSING_PROMPT, type EnvironmentId, type EnvironmentSpeechModel, type EnvironmentSpeechStatus, type ProjectId, + ProviderDriverKind, type SpeechAcceleration, type SpeechCustomWords, type SpeechModelUnloadTimeout, @@ -23,11 +26,15 @@ import { } from "@t3tools/contracts"; import { createModelSelection } from "@t3tools/shared/model"; import { resolveSpeechPostProcessingModelSelection } from "@t3tools/shared/serverSettings"; +import { cloudSpeechLanguages } from "@t3tools/shared/speech"; import { AudioLinesIcon, CheckIcon, DownloadIcon, LanguagesIcon, + KeyRoundIcon, + MonitorIcon, + MicOffIcon, MicIcon, RefreshCwIcon, StarIcon, @@ -60,6 +67,7 @@ import { } from "../../speech/useEnvironmentSpeechInput"; import { ComposerSpeechRecordingPill } from "../chat/ComposerSpeechButton"; import { ProviderModelPicker } from "../chat/ProviderModelPicker"; +import { ProviderInstanceIcon } from "../chat/ProviderInstanceIcon"; import { TraitsPicker } from "../chat/TraitsPicker"; import { VoiceInputSetup } from "../chat/VoiceInputSetup"; import { Badge } from "../ui/badge"; @@ -79,11 +87,20 @@ import { MenuSeparator, MenuTrigger, } from "../ui/menu"; -import { Select, SelectItem, SelectPopup, SelectTrigger, SelectValue } from "../ui/select"; +import { + Select, + SelectGroup, + SelectGroupLabel, + SelectItem, + SelectPopup, + SelectTrigger, + SelectValue, +} from "../ui/select"; import { Switch } from "../ui/switch"; import { Textarea } from "../ui/textarea"; import { toastManager } from "../ui/toast"; import { MicrophoneTest } from "./MicrophoneTest"; +import { CloudVoiceConfiguration } from "./CloudVoiceConfiguration"; import { LanguageRow, ModelBenchmarks, orderSpeechLanguages } from "./SpeechModelComparison"; import { languageLabel, SpeechLanguageSelect } from "./SpeechLanguageSelect"; import { filterSpeechModels, type SpeechModelSort } from "./speechModelBrowser"; @@ -102,6 +119,15 @@ import { useScopedModelDisabledReason } from "./useScopedModelAvailability"; import { useScopedSettings, useUpdateScopedSettings } from "./useScopedSettings"; const OFF = "off"; +function CloudSpeechLogo({ provider }: { provider: CloudSpeechProvider }) { + return ( + + ); +} const SYSTEM_DEFAULT = "system-default"; const formatSize = (bytes: number) => `${Math.round(bytes / 1024 / 1024)} MB`; const SHORTCUT_MODE_LABELS: Record = { @@ -1063,9 +1089,14 @@ export function VoiceSettingsPanel() { const settings = useClientSettings(); const settingsScope = useOptionalSettingsScope(); const destination = useVoiceTranscriptionDestination(); + const canManageKey = useEnvironmentScope(destination.environmentId, AuthSettingsWriteScope); + const [pendingCloud, setPendingCloud] = useState<{ + provider: CloudSpeechProvider; + environmentId: EnvironmentId; + } | null>(null); const { environments } = useEnvironments(); const [microphones, setMicrophones] = useState([]); - const prepared = destination.prepared; + const prepared = destination.cloudProvider ? null : destination.prepared; const catalog = useVoiceModels(prepared); const refreshMicrophones = useStableCallback(async () => { @@ -1098,11 +1129,36 @@ export function VoiceSettingsPanel() { ? (microphones.find((device) => device.deviceId === selectedMicrophone)?.label ?? "Selected microphone (unavailable)") : "System default"; - const destinationValue = destination.environmentId ?? OFF; + const environmentChoices = environments.map((environment) => ({ + environment, + value: JSON.stringify(["environment", environment.environmentId]), + })); + const cloudGroups = environments + .filter( + (environment) => environment.serverConfig?.environment.capabilities.cloudVoiceTranscription, + ) + .map((environment) => ({ + environment, + choices: CloudSpeechProvider.literals.map((provider) => ({ + environment, + provider, + value: JSON.stringify(["cloud", environment.environmentId, provider]), + })), + })); + const cloudChoices = cloudGroups.flatMap((group) => group.choices); + const destinationValue = destination.cloudProvider + ? JSON.stringify(["cloud", destination.environmentId, destination.cloudProvider]) + : destination.environmentId + ? JSON.stringify(["environment", destination.environmentId]) + : OFF; + const cloudEnvironment = environments.find( + (env) => env.environmentId === pendingCloud?.environmentId, + ); const chosenEnvironment = environments.find( (environment) => environment.environmentId === destination.environmentId, ); const chosenUnsupported = + !destination.cloudProvider && chosenEnvironment?.serverConfig != null && chosenEnvironment.serverConfig.environment.capabilities.voiceTranscription !== true; const model = prepared && modelStatus?.supported ? modelStatus : null; @@ -1110,6 +1166,8 @@ export function VoiceSettingsPanel() { return ( - - Microphone audio is sent to this environment to transcribe, then discarded. Choose Off - to turn voice input off on this device. This choice is saved on this device only. - - } - control={ - - } + className="@container/destination px-3 py-3 sm:px-4" > +
+

+ Transcribe with + Use +

+
+ + {destination.cloudProvider && chosenEnvironment && canManageKey && ( + + { + if (destination.cloudProvider) + setPendingCloud({ + provider: destination.cloudProvider, + environmentId: chosenEnvironment.environmentId, + }); + }} + > + + + } + /> + Manage key + + )} +
+
{destination.environmentId && chosenUnsupported ? (

- {chosenEnvironment.label} can’t transcribe audio. Pick an environment that can to use - voice input. + {chosenEnvironment.label} can’t transcribe audio locally. Choose a different + destination.

) : null} -
+ - } - /> + {pendingCloud && cloudEnvironment && ( + setPendingCloud(null)} + /> + )} + {(model || destination.cloudProvider) && ( + + } + /> + )} {model?.supportsTranslation ? ( void updateClientSettings({ speechSilenceFiltering: checked }) @@ -1341,7 +1515,7 @@ export function VoiceSettingsPanel() { boolean; readonly getTranscriptionOptions: () => SpeechTranscriptionOptions; readonly getMicrophoneId: () => string; @@ -26,25 +33,6 @@ export function createBrowserVoiceInputPlatform(input: { }) { return createBrowserPcmCapture(input, async (options) => { const recordingOptions = input.getTranscriptionOptions(); - const status = await runEnvironmentSpeech(getEnvironmentSpeechStatus(input.prepared), options); - throwIfVoiceTranscriptionAborted(options.signal); - if (!status.supported) - throw new VoiceTranscriptionError({ code: "unavailable", message: status.reason }); - const language = status.languages - ? effectiveSpeechLanguage( - { - languages: status.languages, - supportsLanguageDetection: status.supportsLanguageDetection ?? false, - }, - recordingOptions.speechLanguage, - ) - : "auto"; - const locale = - !status.supportsStreaming && - status.supportsTranslation && - recordingOptions.speechTranslateToEnglish - ? "en" - : language; const transcribeRecording = async ( uri: string, { signal: transcriptionSignal }: VoiceTranscriptionOptions, @@ -53,7 +41,20 @@ export function createBrowserVoiceInputPlatform(input: { const response = await fetch(uri, { signal: transcriptionSignal }); const pcm = new Uint8Array(await response.arrayBuffer()); const result = await runEnvironmentSpeech( - transcribeEnvironmentPcm(input.prepared, pcm, recordingOptions, input.hasAudioConsent), + input.cloudProvider + ? transcribeCloudPcm( + input.prepared, + input.cloudProvider, + pcm, + recordingOptions, + input.hasAudioConsent, + ) + : transcribeEnvironmentPcm( + input.prepared, + pcm, + recordingOptions, + input.hasAudioConsent, + ), { signal: transcriptionSignal, }, @@ -62,13 +63,41 @@ export function createBrowserVoiceInputPlatform(input: { return result.text; } catch (cause) { throwIfVoiceTranscriptionAborted(transcriptionSignal); + if (isVoiceTranscriptionError(cause)) throw cause; throw new VoiceTranscriptionError({ code: "transcription-failed", - message: "Voice transcription on this environment failed.", + message: "Voice transcription failed. Check the destination in Settings → Voice.", cause, }); } }; + if (input.cloudProvider) { + await runEnvironmentSpeech(requireCloudSpeechKey(input.prepared, input.cloudProvider), options); + throwIfVoiceTranscriptionAborted(options.signal); + return { + locale: effectiveSpeechLanguage(cloudSpeechLanguages, recordingOptions.speechLanguage), + transcribe: transcribeRecording, + }; + } + const status = await runEnvironmentSpeech(getEnvironmentSpeechStatus(input.prepared), options); + throwIfVoiceTranscriptionAborted(options.signal); + if (!status.supported) + throw new VoiceTranscriptionError({ code: "unavailable", message: status.reason }); + const language = status.languages + ? effectiveSpeechLanguage( + { + languages: status.languages, + supportsLanguageDetection: status.supportsLanguageDetection ?? false, + }, + recordingOptions.speechLanguage, + ) + : "auto"; + const locale = + !status.supportsStreaming && + status.supportsTranslation && + recordingOptions.speechTranslateToEnglish + ? "en" + : language; if (!status.supportsStreaming && !status.supportsBufferedStreaming) { return { locale, transcribe: transcribeRecording }; } diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.ts b/apps/web/src/speech/useEnvironmentSpeechInput.ts index 0d1853a5dcc89..f2ab4c3a92301 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.ts +++ b/apps/web/src/speech/useEnvironmentSpeechInput.ts @@ -11,7 +11,7 @@ import { type VoiceInputState, } from "@t3tools/client-runtime/voice-input"; import type { PreparedConnection } from "@t3tools/client-runtime/connection"; -import { AuthOrchestrationOperateScope } from "@t3tools/contracts"; +import { AuthOrchestrationOperateScope, CLOUD_SPEECH_LABELS } from "@t3tools/contracts"; import type { EnvironmentId, EnvironmentSpeechModel, @@ -109,11 +109,19 @@ export function useVoiceTranscriptionDestination() { const environment = useEnvironment(environmentId); const connection = Option.getOrNull(usePreparedConnection(environmentId)); const canDictate = useEnvironmentScope(environmentId, AuthOrchestrationOperateScope); - const supported = environment?.serverConfig?.environment.capabilities.voiceTranscription === true; + const provider = grant?.destination.kind === "cloud" ? grant.destination.provider : undefined; + const supported = + environment?.serverConfig?.environment.capabilities[ + provider ? "cloudVoiceTranscription" : "voiceTranscription" + ] === true; return { environmentId, + ...(provider ? { cloudProvider: provider } : {}), grantId: grant?.id ?? null, - label: environment?.label ?? null, + label: + provider && environment + ? `${CLOUD_SPEECH_LABELS[provider]} via ${environment.label}` + : (environment?.label ?? null), prepared: supported && canDictate ? connection : null, }; } @@ -122,6 +130,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { const destination = useVoiceTranscriptionDestination(); const prepared = destination.prepared; const grantId = destination.grantId; + const cloudProvider = destination.cloudProvider; const originatingEnvironment = useEnvironment(input.environmentId); const originatingConnection = Option.getOrNull(usePreparedConnection(input.environmentId)); const canCleanUp = useEnvironmentScope(input.environmentId, AuthOrchestrationOperateScope); @@ -231,6 +240,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { }; const platform = createBrowserVoiceInputPlatform({ prepared, + cloudProvider, hasAudioConsent, getTranscriptionOptions: (): SpeechTranscriptionOptions => { const { @@ -344,11 +354,11 @@ export function useEnvironmentSpeechInput(input: HookInput) { controller.dispose(); if (controllerRef.current === controller) controllerRef.current = null; }; - }, [prepared, grantId]); + }, [prepared, grantId, cloudProvider]); // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- probes the chosen environment's speech status once its connection is available useEffect(() => { - if (!prepared) return; + if (!prepared || cloudProvider) return; let disposed = false; void probeSpeechStatus(prepared) .then((value) => { @@ -360,8 +370,8 @@ export function useEnvironmentSpeechInput(input: HookInput) { return () => { disposed = true; }; - }, [prepared]); - const currentStatus = status?.prepared === prepared ? status.value : null; + }, [prepared, cloudProvider]); + const currentStatus = !cloudProvider && status?.prepared === prepared ? status.value : null; // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- polls download progress while the setup dialog owns an active model download useEffect(() => { @@ -425,6 +435,11 @@ export function useEnvironmentSpeechInput(input: HookInput) { const controller = controllerRef.current; const owner = latestRef.current.input.ownerKey; if (latestRef.current.input.disabled || !controller || !prepared) return; + if (cloudProvider) { + setLevel(0); + await controller.start(); + return; + } const stillCurrent = () => request === startRequestRef.current && controllerRef.current === controller && @@ -511,8 +526,8 @@ export function useEnvironmentSpeechInput(input: HookInput) { return { available: prepared !== null && - status?.prepared === prepared && - status.value?.supported !== false && + (cloudProvider !== undefined || + (status?.prepared === prepared && status.value?.supported !== false)) && typeof navigator !== "undefined" && Boolean(navigator.mediaDevices?.getUserMedia) && typeof AudioWorkletNode !== "undefined", diff --git a/docs/internals/voice-input.md b/docs/internals/voice-input.md index fb84458a03a67..61aab264dfcf3 100644 --- a/docs/internals/voice-input.md +++ b/docs/internals/voice-input.md @@ -34,9 +34,12 @@ refuses redirects so credentials and audio cannot follow one to another host. Transcript cleanup requires its own grant, off by default. It sends text, never audio, to the text generation provider of the draft's -own environment, because that is where the provider credentials live. A later -backend adds its own destination kind, disclosure, and transcriber; capture and -the controller stay as they are. +own environment, because that is where the provider credentials live. + +A cloud destination binds both the forwarding environment and the named provider. +It uses a separate, batch-only endpoint: changing a server key cannot upgrade +an environment-local grant, and a cloud recording never uses the streaming +socket or falls back to another provider or the environment's native model. ## Server side diff --git a/docs/user/composer.md b/docs/user/composer.md index 13f451e2caa73..22b499ebe7b0c 100644 --- a/docs/user/composer.md +++ b/docs/user/composer.md @@ -203,6 +203,8 @@ choose the environment, and confirm. Finished recordings go only to that environ even when a draft belongs to another one. This also works on phones without local speech recognition. If the chosen environment disconnects, dictation stops; it never switches hosts. Choose **This device** to revoke access and discard pending audio. +For [cloud dictation](./voice-input.md#choose-where-audio-goes), open **Cloud providers** +under the environment in **Settings → Voice** and confirm which provider may receive recordings. On Android, keep T3 Code open while recording and transcribing. On-device dictation may need a network connection to download a speech model the diff --git a/docs/user/voice-input.md b/docs/user/voice-input.md index e204b28660626..5526af1bab015 100644 --- a/docs/user/voice-input.md +++ b/docs/user/voice-input.md @@ -12,9 +12,9 @@ to a named environment on that phone, open **Settings → Voice → Transcribe o ## Choose where audio goes -On web and desktop, transcription runs on one of your T3 Code environments. +On web and desktop, you choose where transcription runs. Nothing is sent until you choose one: open **Settings → Voice** and set -**Send microphone audio to** the environment that should transcribe for you. +**Transcribe with** to the environment that should transcribe for you. Microphone audio recorded on this device is sent to that environment over your authenticated connection, transcribed there, and discarded. The choice is saved on this device only, so every computer and browser you use asks for it once. @@ -23,10 +23,22 @@ Set it back to **Off** at any time; a recording in progress is discarded. If the chosen environment goes offline, voice input waits for it. T3 Code never substitutes another environment. +For cloud dictation, choose **OpenAI** or **Grok Voice Transcribe** via the environment +you want to use. If it needs a key, enter one and choose **Save and enable**. +You can also choose a [secret provider](secrets.md) instead of **Value** and enter a +reference. Unlock the vault on that environment before dictating. Anyone with +dictation access to the environment can use the key, but only this device opts in. +If a key is already saved, just confirm the provider. After +you stop, the environment forwards your recording and any vocabulary hints to +the selected provider, which bills its API account. No local model download or chat +subscription is needed. Switch back to **This device** on mobile or **Off** on +web and desktop to stop sharing. Removing the key stops that provider for everyone +on the environment. + ## Set up a model -The first time you dictate, T3 Code offers to download the recommended English -model to the chosen environment. In **Settings → Voice → Transcription**, select +If you choose environment transcription, T3 Code offers to download the recommended +English model to that environment. In **Settings → Voice → Transcription**, select the languages you dictate in to compare models that support all of them. Accuracy shows how often the model got words or characters wrong in each language; speed is the time to transcribe 30 seconds on a reference GPU. Choose diff --git a/packages/client-runtime/src/secretRequest.ts b/packages/client-runtime/src/secretRequest.ts index d9612411db09d..26a32e735bcfb 100644 --- a/packages/client-runtime/src/secretRequest.ts +++ b/packages/client-runtime/src/secretRequest.ts @@ -95,6 +95,12 @@ export function secretRequestSources( providers: ServerSettings["secretProviders"], ): ReadonlyArray { if (config?.secretRequestReferences !== true) return NO_SOURCES; + return configuredSecretSources(providers); +} + +export function configuredSecretSources( + providers: ServerSettings["secretProviders"], +): ReadonlyArray { return (Object.keys(providers) as Array).flatMap((id) => { const provider = providers[id]; const definition = diff --git a/packages/client-runtime/src/voice-input/environment.ts b/packages/client-runtime/src/voice-input/environment.ts index 21166d8b9a084..1fddab00ae717 100644 --- a/packages/client-runtime/src/voice-input/environment.ts +++ b/packages/client-runtime/src/voice-input/environment.ts @@ -1,8 +1,11 @@ import { encodeSpeechPcmRequest } from "@t3tools/shared/speech"; import * as Effect from "effect/Effect"; import * as Redacted from "effect/Redacted"; -import { SPEECH_STREAM_PATH } from "@t3tools/contracts"; +import * as Schema from "effect/Schema"; +import { CloudSpeechError, SPEECH_STREAM_PATH } from "@t3tools/contracts"; import type { + CloudSpeechProvider, + SettingsSecretValue, SpeechAcceleration, SpeechModelUnloadTimeout, SpeechTranscriptionOptions, @@ -17,8 +20,10 @@ import { makeEnvironmentHttpApiUrlBuilder, } from "../rpc/http.ts"; import { executeAuthenticatedEnvironmentHttpRequest } from "../state/environmentHttpAuth.ts"; +import { VoiceTranscriptionError } from "./transcription.ts"; const INFERENCE_OR_DOWNLOAD_TIMEOUT_MS = 30 * 60_000; +const isCloudSpeechError = Schema.is(CloudSpeechError); type EnvironmentApiClient = Effect.Success< ReturnType> >; @@ -65,6 +70,81 @@ export const getEnvironmentSpeechStatus = (prepared: PreparedConnection) => run: ({ client, headers }) => client.status({ headers }), }); +export const getCloudSpeechKeys = (prepared: PreparedConnection) => + request({ + group: "voice", + prepared, + method: "GET", + path: (urls) => urls.cloudKeys(), + run: ({ client, headers }) => client.cloudKeys({ headers }), + }); + +export const requireCloudSpeechKey = ( + prepared: PreparedConnection, + provider: CloudSpeechProvider, +) => + getCloudSpeechKeys(prepared).pipe( + Effect.flatMap((keys) => + keys[provider] + ? Effect.void + : Effect.fail( + new VoiceTranscriptionError({ + code: "unavailable", + message: new CloudSpeechError({ provider, reason: "missing-key" }).message, + }), + ), + ), + ); + +export const setCloudSpeechKey = ( + prepared: PreparedConnection, + provider: CloudSpeechProvider, + key: SettingsSecretValue | null, +) => + request({ + group: "voice", + prepared, + method: "POST", + path: (urls) => urls.setCloudKey({ params: { provider } }), + run: ({ client, headers }) => + client.setCloudKey({ + headers, + params: { provider }, + payload: { key: typeof key === "string" ? Redacted.make(key) : key }, + }), + }); + +export const transcribeCloudPcm = ( + prepared: PreparedConnection, + provider: CloudSpeechProvider, + pcm: Uint8Array, + options: SpeechTranscriptionOptions, + canSend: () => boolean, +) => + request({ + group: "voice", + prepared, + method: "POST", + canSend, + timeoutMs: 130_000, + path: (urls) => urls.transcribeCloud({ params: { provider } }), + run: ({ client, headers }) => + client.transcribeCloud({ + headers, + params: { provider }, + payload: encodeSpeechPcmRequest(pcm, options), + }), + }).pipe( + Effect.mapError((error) => + error._tag === "RemoteEnvironmentAuthFetchError" && isCloudSpeechError(error.cause) + ? new VoiceTranscriptionError({ + code: "transcription-failed", + message: error.cause.message, + }) + : error, + ), + ); + export const getEnvironmentSpeechStreamUrl = (prepared: PreparedConnection) => { let endpoint = prepared.httpBaseUrl; return request({ diff --git a/packages/client-runtime/src/voice-input/index.ts b/packages/client-runtime/src/voice-input/index.ts index 06a5399d6813c..44a851fe14e69 100644 --- a/packages/client-runtime/src/voice-input/index.ts +++ b/packages/client-runtime/src/voice-input/index.ts @@ -14,6 +14,10 @@ export { cancelEnvironmentSpeechModelDownload, downloadEnvironmentSpeechModel, getEnvironmentSpeechModels, + getCloudSpeechKeys, + requireCloudSpeechKey, + setCloudSpeechKey, + transcribeCloudPcm, getEnvironmentSpeechStatus, getEnvironmentSpeechStreamUrl, postProcessEnvironmentTranscript, diff --git a/packages/contracts/src/environment.ts b/packages/contracts/src/environment.ts index 627b9529a6cc5..96b4537478ecc 100644 --- a/packages/contracts/src/environment.ts +++ b/packages/contracts/src/environment.ts @@ -174,6 +174,7 @@ export const ExecutionEnvironmentCapabilities = Schema.Struct({ scheduledTaskThreadModes: Schema.optionalKey(Schema.Boolean), /** Server can transcribe 16 kHz mono PCM audio on the environment machine. */ voiceTranscription: Schema.optionalKey(Schema.Boolean), + cloudVoiceTranscription: Schema.optionalKey(Schema.Boolean), }); export type ExecutionEnvironmentCapabilities = typeof ExecutionEnvironmentCapabilities.Type; diff --git a/packages/contracts/src/environmentHttp.ts b/packages/contracts/src/environmentHttp.ts index 8dd67640a0bc5..77ea7fb4eb034 100644 --- a/packages/contracts/src/environmentHttp.ts +++ b/packages/contracts/src/environmentHttp.ts @@ -49,6 +49,10 @@ import { TrimmedNonEmptyString, } from "./baseSchemas.ts"; import { + CloudSpeechApiKey, + CloudSpeechError, + CloudSpeechKeys, + CloudSpeechProvider, EnvironmentSpeechAccelerationRequest, EnvironmentSpeechModelRequest, EnvironmentSpeechModels, @@ -58,6 +62,7 @@ import { EnvironmentSpeechStatus, EnvironmentSpeechTranscriptionResult, } from "./speech.ts"; +import { SecretReference } from "./secretReference.ts"; import { OrchestrationV2ShellSnapshot, OrchestrationV2ThreadBoundedSnapshot, @@ -775,6 +780,41 @@ export class EnvironmentVoiceBodyLimit extends HttpApiMiddleware.Service()( + "CloudSpeechError", + { + provider: CloudSpeechProvider, + reason: Schema.Literals([ + "missing-key", + "unreadable-key", + "rejected-key", + "rate-limit", + "unavailable", + ]), + }, + { httpApiStatus: 422 }, +) { + override get message() { + const name = CLOUD_SPEECH_PROVIDER_NAMES[this.provider]; + switch (this.reason) { + case "missing-key": + return `Add your ${name} API key in Settings → Voice.`; + case "unreadable-key": + return `Couldn’t read the ${name} API key. Unlock the vault on the selected environment or update the key in Settings → Voice. No audio was sent to ${name}.`; + case "rejected-key": + return `${name} rejected this API key. Check its permissions and billing in Settings → Voice.`; + case "rate-limit": + return `${name} reached a usage limit. Check the account's quota or try again later.`; + case "unavailable": + return `${name} couldn’t transcribe this recording. Try again later.`; + } + } +} export const SpeechModelId = Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(160)); export type SpeechModelId = typeof SpeechModelId.Type; diff --git a/packages/shared/src/speech.ts b/packages/shared/src/speech.ts index fcf74f10187b8..73749cb1d2fc8 100644 --- a/packages/shared/src/speech.ts +++ b/packages/shared/src/speech.ts @@ -8,6 +8,15 @@ import * as Schema from "effect/Schema"; const decodeOptions = Schema.decodeUnknownSync(Schema.fromJsonString(SpeechTranscriptionOptions)); const encodeOptions = Schema.encodeSync(Schema.fromJsonString(SpeechTranscriptionOptions)); +// Only offer explicit hints understood by all cloud APIs; other languages can still be detected. +export const cloudSpeechLanguages = { + supportsLanguageDetection: true, + languages: + "ar bg ca cs da de el en es fa fi fr hi hr hu id it ja ko mk ms nl pl pt ro ru sk sv th tr uk ur vi zh".split( + " ", + ), +}; + /** Resolve the language the model can produce from the requested language. */ export function effectiveSpeechLanguage( model: { readonly languages: readonly string[]; readonly supportsLanguageDetection: boolean }, From b15841a5da67985aa93224e92a3612f7f8dfeed8 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Thu, 5 Nov 2026 18:22:20 -0800 Subject: [PATCH 2/9] fix(voice): keep the latest destination when a key save finishes --- .../features/settings/CloudVoiceSection.tsx | 11 +-- .../voice-input/voiceDestination.test.ts | 31 +++++++- .../features/voice-input/voiceDestination.ts | 10 +++ .../settings/CloudVoiceConfiguration.tsx | 8 +- ...useEnvironmentSpeechInput.browser.test.tsx | 17 ++++- apps/web/src/speech/voiceConsent.ts | 74 +++++++++++++++---- 6 files changed, 128 insertions(+), 23 deletions(-) diff --git a/apps/mobile/src/features/settings/CloudVoiceSection.tsx b/apps/mobile/src/features/settings/CloudVoiceSection.tsx index 89f0967268380..8dce5981a0cab 100644 --- a/apps/mobile/src/features/settings/CloudVoiceSection.tsx +++ b/apps/mobile/src/features/settings/CloudVoiceSection.tsx @@ -27,7 +27,7 @@ import { Chip } from "../../components/ui/Chip"; import type { EnvironmentPresentation } from "../../state/environments"; import { useEnvironmentScope } from "../../state/session"; import { - chooseVoiceDestination, + pendingCloudVoiceChoice, cloudVoiceConnectionAtom, cloudVoiceKeysAtom, runSpeech, @@ -78,9 +78,9 @@ function CloudChoices({ environment, active }: Props) { ? draft.trim() : undefined; const [saving, setSaving] = useState(false); - const enable = (provider: CloudSpeechProvider) => { + const enable = (commit: () => void) => { try { - chooseVoiceDestination(id, provider); + commit(); } catch { Dialog.alert( "Could not enable cloud dictation", @@ -100,6 +100,7 @@ function CloudChoices({ environment, active }: Props) { }; const changeKey = async (provider: CloudSpeechProvider, value: SettingsSecretValue | null) => { if (!connection || !canManage || saving) return; + const commit = value !== null && editing?.enable ? pendingCloudVoiceChoice(id, provider) : null; setSaving(true); try { await runSpeech(setCloudSpeechKey(connection, provider, value)); @@ -112,7 +113,7 @@ function CloudChoices({ environment, active }: Props) { return; } setSaving(false); - if (value !== null && editing?.enable) enable(provider); + if (commit) enable(commit); setDraft(""); setEditing(null); reload(); @@ -162,7 +163,7 @@ function CloudChoices({ environment, active }: Props) { title: `Use ${label} for dictation?`, message: `${disclosure} Select This device to stop sharing.`, confirmText: `Use ${label}`, - onConfirm: () => enable(provider), + onConfirm: () => enable(pendingCloudVoiceChoice(id, provider)), }); }} /> diff --git a/apps/mobile/src/features/voice-input/voiceDestination.test.ts b/apps/mobile/src/features/voice-input/voiceDestination.test.ts index ac78beca40581..9a23c9aebca40 100644 --- a/apps/mobile/src/features/voice-input/voiceDestination.test.ts +++ b/apps/mobile/src/features/voice-input/voiceDestination.test.ts @@ -2,7 +2,12 @@ import { EnvironmentId } from "@t3tools/contracts"; import { Atom } from "effect/reactivity"; import { expect, it, vi } from "vite-plus/test"; import { appAtomRegistry } from "../../state/atom-registry"; -import { chooseVoiceDestination, voiceGrantAtom, voiceRouteAtom } from "./voiceDestination"; +import { + chooseVoiceDestination, + pendingCloudVoiceChoice, + voiceGrantAtom, + voiceRouteAtom, +} from "./voiceDestination"; import { readVoiceGrant } from "./voiceGrantStorage"; const mocks = vi.hoisted(() => ({ @@ -216,3 +221,27 @@ it("keeps local as the default and never reuses consent for an offline or resele appAtomRegistry.reset(); } }); + +it("a late cloud key save cannot override a newer destination choice", () => { + const host = EnvironmentId.make("my-server"); + try { + chooseVoiceDestination(null); + const first = pendingCloudVoiceChoice(host, "openai"); + chooseVoiceDestination(null); + first(); + expect(appAtomRegistry.get(voiceRouteAtom)).toBeNull(); + expect(JSON.parse(mocks.write.mock.lastCall![1])).toBeNull(); + + const older = pendingCloudVoiceChoice(host, "openai"); + const newer = pendingCloudVoiceChoice(host, "xai"); + older(); + expect(appAtomRegistry.get(voiceRouteAtom)).toBeNull(); + newer(); + const grant = appAtomRegistry.get(voiceGrantAtom); + older(); + expect(appAtomRegistry.get(voiceGrantAtom)).toBe(grant); + expect(grant?.destination).toEqual({ kind: "cloud", environmentId: host, provider: "xai" }); + } finally { + appAtomRegistry.reset(); + } +}); diff --git a/apps/mobile/src/features/voice-input/voiceDestination.ts b/apps/mobile/src/features/voice-input/voiceDestination.ts index c75940ca2c5c6..b51a70ffdbd41 100644 --- a/apps/mobile/src/features/voice-input/voiceDestination.ts +++ b/apps/mobile/src/features/voice-input/voiceDestination.ts @@ -38,10 +38,13 @@ export const voiceGrantAtom = Atom.make((get) => { return AsyncResult.getOrElse(get(storedGrant), () => null); }); +let choiceRevision = 0; + export function chooseVoiceDestination( environmentId: EnvironmentId | null, provider?: CloudSpeechProvider, ) { + choiceRevision++; // A failed change must not leave the old destination armed in this session. appAtomRegistry.set(choice, null); const grant: VoiceGrant | null = @@ -57,6 +60,13 @@ export function chooseVoiceDestination( appAtomRegistry.set(choice, grant); } +export function pendingCloudVoiceChoice(id: EnvironmentId, provider: CloudSpeechProvider) { + const revision = ++choiceRevision; + return () => { + if (revision === choiceRevision) chooseVoiceDestination(id, provider); + }; +} + const native = requireOptionalNativeModule<{ decodeRecording?: (uri: string) => Promise; }>("T3VoiceTranscription"); diff --git a/apps/web/src/components/settings/CloudVoiceConfiguration.tsx b/apps/web/src/components/settings/CloudVoiceConfiguration.tsx index 23e472a36f7fd..ba8845ab496d8 100644 --- a/apps/web/src/components/settings/CloudVoiceConfiguration.tsx +++ b/apps/web/src/components/settings/CloudVoiceConfiguration.tsx @@ -23,7 +23,7 @@ import { Atom, AsyncResult } from "effect/reactivity"; import { useId, useState } from "react"; import { isSecretsFeatureEnabled } from "../../env"; import { layer, runtime } from "../../lib/runtime"; -import { chooseAudioDestination } from "../../speech/voiceConsent"; +import { chooseAudioDestination, pendingCloudAudioChoice } from "../../speech/voiceConsent"; import type { EnvironmentPresentation } from "../../state/environments"; import { useEnvironmentScope, usePreparedConnection } from "../../state/session"; import { Button } from "../ui/button"; @@ -99,9 +99,13 @@ export function CloudVoiceConfiguration({ setError(""); setBusy(true); try { + const commit = + !active && value !== null + ? pendingCloudAudioChoice({ kind: "cloud", provider, environmentId: id }) + : null; await runtime.runPromise(setCloudSpeechKey(prepared, provider, value)); refresh(); - if (!active && value !== null) enable(); + commit?.(); close(); } catch { setError("Couldn’t save this change. Check your connection and try again."); diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx index 04534c3e0cfdc..4695ed50bb54e 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx +++ b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx @@ -6,7 +6,7 @@ import type { PreparedConnection } from "@t3tools/client-runtime/connection"; import { DEFAULT_CLIENT_SETTINGS, DEFAULT_SERVER_SETTINGS, - type EnvironmentId, + EnvironmentId, type ProjectId, type SpeechCustomWords, type SpeechPostProcessingOptions, @@ -15,7 +15,12 @@ import { import { act, render } from "../../test/render"; import { useEnvironmentSpeechInput } from "./useEnvironmentSpeechInput"; -import { chooseAudioDestination, chooseTranscriptCleanup, bindAudioGrant } from "./voiceConsent"; +import { + chooseAudioDestination, + chooseTranscriptCleanup, + bindAudioGrant, + pendingCloudAudioChoice, +} from "./voiceConsent"; import { writeBrowserClientSettings } from "../clientPersistenceStorage"; const mocks = vi.hoisted(() => ({ @@ -367,7 +372,15 @@ it("revokes across windows even before the storage event and survives stale pref const grant = JSON.parse(storage.getItem(key)!) as { id: string }; const canSend = bindAudioGrant(grant.id); expect(canSend()).toBe(true); + const saved = pendingCloudAudioChoice({ + kind: "cloud", + provider: "openai", + environmentId: EnvironmentId.make("env-a"), + }); + expect(canSend()).toBe(true); storage.removeItem(key); + saved(); + expect(storage.getItem(key)).toBeNull(); expect(canSend()).toBe(false); writeBrowserClientSettings({ ...DEFAULT_CLIENT_SETTINGS, wordWrap: false }); await vi.waitFor(() => expect(mocks.voice.available).toBe(false)); diff --git a/apps/web/src/speech/voiceConsent.ts b/apps/web/src/speech/voiceConsent.ts index 81da7cd6adb3d..6e9825145124a 100644 --- a/apps/web/src/speech/voiceConsent.ts +++ b/apps/web/src/speech/voiceConsent.ts @@ -1,8 +1,11 @@ import { VoiceTranscriptionDestination } from "@t3tools/contracts"; import * as Schema from "effect/Schema"; +import { useSyncExternalStore } from "react"; import { randomUUID } from "../lib/utils"; import { + getLocalStorageItem, readLocalStorageValue, + subscribeLocalStorageKey, useLocalStorage, writeLocalStorageValue, } from "../hooks/useLocalStorage"; @@ -10,33 +13,78 @@ import { // Separate keys: a stale window saving ordinary preferences must not restore either grant. const AUDIO_KEY = "t3code:voice-audio-grant:v1"; const CLEANUP_KEY = "t3code:voice-cleanup-grant:v1"; -const AudioGrant = Schema.NullOr( - Schema.Struct({ id: Schema.String, destination: VoiceTranscriptionDestination }), +const Grant = Schema.Struct({ id: Schema.String, destination: VoiceTranscriptionDestination }); +const AudioGrant = Schema.NullOr(Grant); +const sameGrant = Schema.toEquivalence(AudioGrant); +const AudioChoice = Schema.NullOr( + Schema.Union([Grant, Schema.Struct({ pending: Grant, previous: AudioGrant })]), ); +type AudioChoice = typeof AudioChoice.Type; +const savedKey = (id: string) => `${AUDIO_KEY}:saved:${id}`; +const pendingGrant = (choice: AudioChoice) => + choice && "pending" in choice ? choice.pending : null; +const activeGrant = (choice: AudioChoice, saved: boolean) => + choice && "pending" in choice ? (saved ? choice.pending : choice.previous) : choice; const CleanupGrant = Schema.NullOr(Schema.String); -const decodeAudioGrant = Schema.decodeSync(Schema.fromJsonString(AudioGrant)); +const decodeChoice = Schema.decodeSync(Schema.fromJsonString(AudioChoice)); + +function isSaved(id: string | undefined) { + return !!id && window.localStorage.getItem(savedKey(id)) === "true"; +} export function useAudioGrant() { - return useLocalStorage(AUDIO_KEY, null, AudioGrant)[0]; + const [choice] = useLocalStorage(AUDIO_KEY, null, AudioChoice); + const id = pendingGrant(choice)?.id; + const saved = useSyncExternalStore( + (notify) => (id ? subscribeLocalStorageKey(savedKey(id), notify) : () => {}), + () => isSaved(id), + () => false, + ); + return activeGrant(choice, saved); +} + +function replaceChoice(choice: AudioChoice) { + const old = pendingGrant(readLocalStorageValue(AUDIO_KEY, null, AudioChoice)); + writeLocalStorageValue(AUDIO_KEY, choice, null, AudioChoice); + if (old && pendingGrant(getLocalStorageItem(AUDIO_KEY, AudioChoice))?.id !== old.id) + window.localStorage.removeItem(savedKey(old.id)); } export function chooseAudioDestination(destination: VoiceTranscriptionDestination | null) { - writeLocalStorageValue( - AUDIO_KEY, - destination && { id: randomUUID(), destination }, - null, - AudioGrant, - ); + replaceChoice(destination && { id: randomUUID(), destination }); +} + +export function pendingCloudAudioChoice( + destination: Extract, +) { + const choice = getLocalStorageItem(AUDIO_KEY, AudioChoice); + const pending = { id: randomUUID(), destination }; + replaceChoice({ pending, previous: activeGrant(choice, isSaved(pendingGrant(choice)?.id)) }); + // A late response can acknowledge only its own choice, never write the destination another tab picked. + return () => { + if (pendingGrant(getLocalStorageItem(AUDIO_KEY, AudioChoice))?.id === pending.id) + writeLocalStorageValue(savedKey(pending.id), true, false, Schema.Boolean); + }; } // Bind this recording to this act of consent, even if the user later reselects the same destination. export function bindAudioGrant(id: string) { try { - const raw = window.localStorage.getItem(AUDIO_KEY); - if (!raw || decodeAudioGrant(raw)?.id !== id) return () => false; + let cachedRaw: string | null = null; + let choice: AudioChoice = null; + const current = () => { + const raw = window.localStorage.getItem(AUDIO_KEY); + if (raw !== cachedRaw) { + choice = raw ? decodeChoice(raw) : null; + cachedRaw = raw; + } + return activeGrant(choice, isSaved(pendingGrant(choice)?.id)); + }; + const bound = current(); + if (bound?.id !== id) return () => false; return () => { try { - return window.localStorage.getItem(AUDIO_KEY) === raw; + return sameGrant(current(), bound); } catch { return false; } From a4574f422f7f38d7e67778b4d7682475961762f0 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Thu, 5 Nov 2026 18:31:13 -0800 Subject: [PATCH 3/9] docs(voice): clarify where a dictation grant can send audio --- docs/internals/voice-input.md | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/docs/internals/voice-input.md b/docs/internals/voice-input.md index 61aab264dfcf3..7b7fb7cfa6fa7 100644 --- a/docs/internals/voice-input.md +++ b/docs/internals/voice-input.md @@ -2,8 +2,9 @@ Transcription edits a composer draft. It does not submit an agent turn. Audio is temporary client input, and only normal message submission sends the resulting -text. Mobile defaults to on-device transcription; any client that sends audio off -the device needs a grant for one environment. +text to the agent. Mobile defaults to on-device transcription. Sending audio off +the device requires an explicit grant naming one environment and, for cloud +transcription, one provider. The [shared controller](../../packages/client-runtime/src/voice-input/controller.ts) owns the operation while the client supplies capture and transcription. Preparation @@ -15,20 +16,21 @@ capture and `finish` replaces the file-based `transcribe`. ## Where audio may go -Only an explicit audio grant authorizes off-device transcription. On mobile the -grant is stored in device-bound secure storage, outside preference backups. The -recorder is the same for both destinations; only a finished recording can be -uploaded, and its grant is checked again after native decoding and before dispatch. +On mobile, on-device transcription needs no audio grant. Environment and cloud +transcription share a recorder and upload only finished recordings. The client +keeps the grant in device-bound secure storage, outside preference backups, and +rechecks it after native decoding and before upload. + The [web grant](../../apps/web/src/speech/voiceConsent.ts) lives outside the preferences snapshot so another -window saving unrelated settings cannot resurrect a revoked grant. Each new -choice gets a distinct grant, even if the destination was previously selected. The -[web hook](../../apps/web/src/speech/useEnvironmentSpeechInput.ts) binds a -recording controller to the prepared connection of that destination and to -nothing else. When the destination changes, disconnects, or stops advertising the -`voiceTranscription` capability, the controller is disposed: a recording in -progress is discarded, a failed recording kept for retry is deleted, and nothing -falls back to another environment. Every sender in the web app, including the -settings page's transcription test, goes through that hook. The microphone loopback +window saving unrelated settings cannot resurrect a revoked grant. Each new choice +gets a distinct grant, even if the destination was previously selected. The +[web hook](../../apps/web/src/speech/useEnvironmentSpeechInput.ts) binds the +controller to that grant and the selected environment's connection. It requires +`voiceTranscription` to use the environment's model or `cloudVoiceTranscription` +to forward audio to the selected provider. Changing the grant or losing the +connection, permission, or required capability discards both the active recording +and any recording retained for retry. It never chooses a fallback destination. +The settings page's transcription test uses the same hook. The microphone loopback test captures separately for local playback and sends nothing. The transport refuses redirects so credentials and audio cannot follow one to another host. From b1e7d5e19c68e80bffadea167ac421dbdbfcb85b Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Thu, 5 Nov 2026 18:38:22 -0800 Subject: [PATCH 4/9] fix(voice): keep audio off when browser consent is unreadable --- ...useEnvironmentSpeechInput.browser.test.tsx | 41 +++++++++++++++++++ apps/web/src/speech/voiceConsent.ts | 41 ++++++++++++------- 2 files changed, 68 insertions(+), 14 deletions(-) diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx index 4695ed50bb54e..3ec98437eb43f 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx +++ b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx @@ -394,6 +394,47 @@ it("revokes across windows even before the storage event and survives stale pref } }); +it("recovers from a corrupt choice without enabling cloud dictation before the save completes", async () => { + setup(); + window.localStorage.setItem("t3code:voice-audio-grant:v1", "not-json"); + const save = pendingCloudAudioChoice({ + kind: "cloud", + environmentId: EnvironmentId.make("env-b"), + provider: "openai", + }); + await using _screen = await render(); + expect(mocks.voice.transcriptionEnvironmentId).toBeNull(); + await act(save); + expect(mocks.voice.transcriptionEnvironmentId).toBe("env-b"); +}); + +it("uses neither destination when the cloud save marker cannot be read", async () => { + setup({ destination: "env-a" }); + const key = "t3code:voice-audio-grant:v1"; + const previous = JSON.parse(window.localStorage.getItem(key)!) as { id: string }; + const canSendToPrevious = bindAudioGrant(previous.id); + const save = pendingCloudAudioChoice({ + kind: "cloud", + environmentId: EnvironmentId.make("env-b"), + provider: "openai", + }); + save(); + const getItem = Storage.prototype.getItem; + using _blocked = vi.spyOn(Storage.prototype, "getItem").mockImplementation(function ( + this: Storage, + k, + ) { + if (k.startsWith(`${key}:saved:`)) + throw new DOMException("Storage is blocked", "SecurityError"); + return getItem.call(this, k); + }); + await using _screen = await render(); + expect(mocks.voice.transcriptionEnvironmentId).toBeNull(); + expect(canSendToPrevious()).toBe(false); + await act(() => mocks.voice.start()); + expectNoAudioSent(); +}); + it("cleans the transcript only after cleanup is separately enabled", async () => { setup({ destination: "env-a" }); chooseTranscriptCleanup(true); diff --git a/apps/web/src/speech/voiceConsent.ts b/apps/web/src/speech/voiceConsent.ts index 6e9825145124a..726717a583ebe 100644 --- a/apps/web/src/speech/voiceConsent.ts +++ b/apps/web/src/speech/voiceConsent.ts @@ -23,28 +23,38 @@ type AudioChoice = typeof AudioChoice.Type; const savedKey = (id: string) => `${AUDIO_KEY}:saved:${id}`; const pendingGrant = (choice: AudioChoice) => choice && "pending" in choice ? choice.pending : null; -const activeGrant = (choice: AudioChoice, saved: boolean) => - choice && "pending" in choice ? (saved ? choice.pending : choice.previous) : choice; const CleanupGrant = Schema.NullOr(Schema.String); const decodeChoice = Schema.decodeSync(Schema.fromJsonString(AudioChoice)); +const readChoice = () => readLocalStorageValue(AUDIO_KEY, null, AudioChoice); -function isSaved(id: string | undefined) { - return !!id && window.localStorage.getItem(savedKey(id)) === "true"; +function readSaveMarker(id: string | undefined) { + try { + return id ? window.localStorage.getItem(savedKey(id)) : null; + } catch { + return undefined; + } +} + +function activeGrant(choice: AudioChoice, marker: string | null | undefined) { + if (!choice || !("pending" in choice)) return choice; + // Only a confirmed missing marker means the previous grant still applies. + if (marker === null) return choice.previous; + return marker === "true" ? choice.pending : null; } export function useAudioGrant() { const [choice] = useLocalStorage(AUDIO_KEY, null, AudioChoice); const id = pendingGrant(choice)?.id; - const saved = useSyncExternalStore( + const marker = useSyncExternalStore( (notify) => (id ? subscribeLocalStorageKey(savedKey(id), notify) : () => {}), - () => isSaved(id), - () => false, + () => readSaveMarker(id), + () => undefined, ); - return activeGrant(choice, saved); + return activeGrant(choice, marker); } -function replaceChoice(choice: AudioChoice) { - const old = pendingGrant(readLocalStorageValue(AUDIO_KEY, null, AudioChoice)); +function replaceChoice(choice: AudioChoice, previous = readChoice()) { + const old = pendingGrant(previous); writeLocalStorageValue(AUDIO_KEY, choice, null, AudioChoice); if (old && pendingGrant(getLocalStorageItem(AUDIO_KEY, AudioChoice))?.id !== old.id) window.localStorage.removeItem(savedKey(old.id)); @@ -57,12 +67,15 @@ export function chooseAudioDestination(destination: VoiceTranscriptionDestinatio export function pendingCloudAudioChoice( destination: Extract, ) { - const choice = getLocalStorageItem(AUDIO_KEY, AudioChoice); + const choice = readChoice(); const pending = { id: randomUUID(), destination }; - replaceChoice({ pending, previous: activeGrant(choice, isSaved(pendingGrant(choice)?.id)) }); + replaceChoice( + { pending, previous: activeGrant(choice, readSaveMarker(pendingGrant(choice)?.id)) }, + choice, + ); // A late response can acknowledge only its own choice, never write the destination another tab picked. return () => { - if (pendingGrant(getLocalStorageItem(AUDIO_KEY, AudioChoice))?.id === pending.id) + if (pendingGrant(readChoice())?.id === pending.id) writeLocalStorageValue(savedKey(pending.id), true, false, Schema.Boolean); }; } @@ -78,7 +91,7 @@ export function bindAudioGrant(id: string) { choice = raw ? decodeChoice(raw) : null; cachedRaw = raw; } - return activeGrant(choice, isSaved(pendingGrant(choice)?.id)); + return activeGrant(choice, readSaveMarker(pendingGrant(choice)?.id)); }; const bound = current(); if (bound?.id !== id) return () => false; From 343a3cc9d1d39d6162fc5869b9ff85971203b232 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Thu, 5 Nov 2026 18:45:27 -0800 Subject: [PATCH 5/9] fix(voice): allow transcript cleanup on cloud-only environments --- .../useEnvironmentSpeechInput.browser.test.tsx | 15 ++++++++++++--- apps/web/src/speech/useEnvironmentSpeechInput.ts | 5 +++-- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx index 3ec98437eb43f..7bf684d571963 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx +++ b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx @@ -28,6 +28,7 @@ const mocks = vi.hoisted(() => ({ settings: {} as Record, connected: new Set(), voiceCapable: new Set(), + cloudVoiceCapable: new Set(), canDictate: new Set(), prepared: new Map(), primaryEnvironmentId: "primary-env", @@ -68,7 +69,10 @@ vi.mock("../state/environments", () => ({ environmentId, serverConfig: { environment: { - capabilities: { voiceTranscription: mocks.voiceCapable.has(environmentId) }, + capabilities: { + voiceTranscription: mocks.voiceCapable.has(environmentId), + cloudVoiceTranscription: mocks.cloudVoiceCapable.has(environmentId), + }, }, }, } @@ -176,6 +180,7 @@ function setup(options: { readonly destination?: string | null } = {}) { mocks.primaryEnvironmentId = "primary-env"; mocks.connected = new Set(["primary-env", THREAD_ENV, "env-a", "env-b"]); mocks.voiceCapable = new Set(["primary-env", THREAD_ENV, "env-a", "env-b"]); + mocks.cloudVoiceCapable.clear(); mocks.canDictate = new Set(mocks.voiceCapable); mocks.prepared.clear(); mocks.statusRequests = []; @@ -435,15 +440,18 @@ it("uses neither destination when the cloud save marker cannot be read", async ( expectNoAudioSent(); }); -it("cleans the transcript only after cleanup is separately enabled", async () => { +it("cleans text on a cloud-only draft environment only while cleanup is separately enabled", async () => { setup({ destination: "env-a" }); + mocks.voiceCapable.delete(THREAD_ENV); + mocks.cloudVoiceCapable.add(THREAD_ENV); chooseTranscriptCleanup(true); const nearby = "x".repeat(1_024); mocks.draft = "unrelated private paragraph " + nearby; await using screen = await render(); await act(() => mocks.voice.start()); await act(() => mocks.voice.stop()); - expect(mocks.cleanupCalls).toHaveLength(1); + expect(mocks.transcribeCalls).toEqual(["env-a"]); + expect(mocks.cleanupCalls).toEqual([THREAD_ENV]); expect(mocks.cleanupDraft).toHaveBeenCalledExactlyOnceWith({ text: nearby, selection: { start: nearby.length, end: nearby.length }, @@ -468,6 +476,7 @@ it.each([ "keeps a recording without authorized cleanup off the provider: $cleanup / $authorized", async ({ cleanup, authorized }) => { setup({ destination: "env-a" }); + mocks.cloudVoiceCapable.add(THREAD_ENV); chooseTranscriptCleanup(true); mocks.settings.speechCorrectionWord = "sorry"; if (!authorized) mocks.canDictate.delete(THREAD_ENV); diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.ts b/apps/web/src/speech/useEnvironmentSpeechInput.ts index f2ab4c3a92301..bc86f1e5a39c2 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.ts +++ b/apps/web/src/speech/useEnvironmentSpeechInput.ts @@ -131,12 +131,13 @@ export function useEnvironmentSpeechInput(input: HookInput) { const prepared = destination.prepared; const grantId = destination.grantId; const cloudProvider = destination.cloudProvider; - const originatingEnvironment = useEnvironment(input.environmentId); + const draftCapabilities = useEnvironment(input.environmentId)?.serverConfig?.environment + .capabilities; const originatingConnection = Option.getOrNull(usePreparedConnection(input.environmentId)); const canCleanUp = useEnvironmentScope(input.environmentId, AuthOrchestrationOperateScope); const cleanupPrepared = canCleanUp && - originatingEnvironment?.serverConfig?.environment.capabilities.voiceTranscription === true + (draftCapabilities?.voiceTranscription || draftCapabilities?.cloudVoiceTranscription) ? originatingConnection : null; const clientSettings = useClientSettings(); From be3b73f052aaadcc7efc1f00026badf438d80072 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Thu, 5 Nov 2026 18:58:43 -0800 Subject: [PATCH 6/9] test(voice): verify recordings use the selected cloud provider --- ...useEnvironmentSpeechInput.browser.test.tsx | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx index 7bf684d571963..a202fd4013f68 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx +++ b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx @@ -14,6 +14,7 @@ import { } from "@t3tools/contracts"; import { act, render } from "../../test/render"; +import type { createBrowserVoiceInputPlatform } from "./browserVoiceInput"; import { useEnvironmentSpeechInput } from "./useEnvironmentSpeechInput"; import { chooseAudioDestination, @@ -133,12 +134,10 @@ vi.mock("@t3tools/client-runtime/voice-input", async (importOriginal) => ({ return { text: `Cleaned ${transcript}` }; }), })); -// Capture and the wire live behind the platform; its `prepared` is where audio would go. vi.mock("./browserVoiceInput", () => ({ - createBrowserVoiceInputPlatform: (input: { - prepared: PreparedConnection; - getTranscriptionOptions?: () => SpeechTranscriptionOptions; - }) => { + createBrowserVoiceInputPlatform: ( + input: Parameters[0], + ) => { const environmentId = input.prepared.environmentId; mocks.platforms.push(environmentId); return { @@ -152,12 +151,13 @@ vi.mock("./browserVoiceInput", () => ({ }, transcriber: { prepare: async () => { - const options = input.getTranscriptionOptions?.(); - if (options) mocks.recordingOptions.push(options); + mocks.recordingOptions.push(input.getTranscriptionOptions()); return { locale: "en", transcribe: async () => { - mocks.transcribeCalls.push(environmentId); + mocks.transcribeCalls.push( + input.cloudProvider ? `${input.cloudProvider} via ${environmentId}` : environmentId, + ); const next = mocks.transcripts.shift() ?? "dictated"; if (next instanceof Error) throw next; return next; @@ -401,16 +401,23 @@ it("revokes across windows even before the storage event and survives stale pref it("recovers from a corrupt choice without enabling cloud dictation before the save completes", async () => { setup(); + mocks.cloudVoiceCapable.add("env-b"); + mocks.modelState = "missing-model"; window.localStorage.setItem("t3code:voice-audio-grant:v1", "not-json"); const save = pendingCloudAudioChoice({ kind: "cloud", environmentId: EnvironmentId.make("env-b"), - provider: "openai", + provider: "xai", }); await using _screen = await render(); expect(mocks.voice.transcriptionEnvironmentId).toBeNull(); + await act(() => mocks.voice.start()); + expectNoAudioSent(); await act(save); - expect(mocks.voice.transcriptionEnvironmentId).toBe("env-b"); + await act(() => mocks.voice.start()); + await act(() => mocks.voice.stop()); + expect(mocks.transcribeCalls).toEqual(["xai via env-b"]); + expect(mocks.statusRequests).toEqual([]); }); it("uses neither destination when the cloud save marker cannot be read", async () => { From d40a9294f134c3a34a1bd7c84f8116a525d21178 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Fri, 6 Nov 2026 11:53:51 -0800 Subject: [PATCH 7/9] feat(voice): choose an OpenRouter transcription model per device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Discover exact speech-to-text models in the environment and keep key management separate from each device’s choice of where to send a recording. Based on the live-catalog contribution by Aditya Vardhan Sharma. Co-authored-by: Aditya Vardhan Sharma --- .../features/settings/CloudVoiceSection.tsx | 482 +++++++++++++----- .../settings/SettingsVoiceRouteScreen.tsx | 2 +- .../voice-input/voiceDestination.test.ts | 12 + .../features/voice-input/voiceDestination.ts | 52 +- apps/server/src/serverSettings.test.ts | 5 +- .../sourceControlHostSecrets.test.ts | 2 +- apps/server/src/speech/CloudSpeech.test.ts | 234 +++++---- apps/server/src/speech/CloudSpeech.ts | 93 +++- apps/server/src/speech/http.test.ts | 25 +- apps/server/src/speech/http.ts | 14 +- .../CloudVoiceConfiguration.browser.test.tsx | 95 ++++ .../settings/CloudVoiceConfiguration.tsx | 351 ++++++++++--- .../settings/VoiceSettingsPanel.tsx | 61 ++- .../src/components/settings/settingsSearch.ts | 2 +- apps/web/src/speech/browserVoiceInput.ts | 21 +- apps/web/src/speech/runEnvironmentSpeech.ts | 15 +- ...useEnvironmentSpeechInput.browser.test.tsx | 27 +- .../src/speech/useEnvironmentSpeechInput.ts | 9 +- docs/internals/voice-input.md | 10 + docs/user/voice-input.md | 11 + .../src/voice-input/environment.ts | 20 +- .../client-runtime/src/voice-input/index.ts | 1 + packages/contracts/src/environmentHttp.ts | 11 + packages/contracts/src/settings.ts | 19 +- packages/contracts/src/speech.ts | 31 +- packages/shared/src/speech.ts | 8 + 26 files changed, 1219 insertions(+), 394 deletions(-) diff --git a/apps/mobile/src/features/settings/CloudVoiceSection.tsx b/apps/mobile/src/features/settings/CloudVoiceSection.tsx index 8dce5981a0cab..4709671dd428f 100644 --- a/apps/mobile/src/features/settings/CloudVoiceSection.tsx +++ b/apps/mobile/src/features/settings/CloudVoiceSection.tsx @@ -1,4 +1,5 @@ import { useAtomRefresh, useAtomValue } from "@effect/atom-react"; +import { cloudSpeechDisclosure } from "@t3tools/shared/speech"; import { setCloudSpeechKey } from "@t3tools/client-runtime/voice-input"; import { configuredSecretSources, @@ -10,50 +11,136 @@ import { CLOUD_SPEECH_PROVIDER_NAMES, CloudSpeechProvider, CloudSpeechApiKey, + CloudSpeechModelId, isSecretReference, + type CloudSpeechModel, type SecretProviderId, type SettingsSecretValue, + type VoiceTranscriptionDestination, } from "@t3tools/contracts"; import * as Schema from "effect/Schema"; import Constants from "expo-constants"; import { AsyncResult } from "effect/reactivity"; import { useState } from "react"; -import { TextInput, View } from "react-native"; +import { ScrollView, TextInput, View } from "react-native"; +import { Sheet } from "../../components/ui/Sheet"; import * as Dialog from "../../components/ui/Dialog"; import { ProviderIcon } from "../../components/ProviderIcon"; +import { AppText as Text } from "../../components/ui/AppText"; import { ListActionRow } from "../../components/ui/ListActionRow"; import { ListChoiceRow } from "../../components/ui/ListChoiceRow"; import { Chip } from "../../components/ui/Chip"; import type { EnvironmentPresentation } from "../../state/environments"; import { useEnvironmentScope } from "../../state/session"; import { + chooseVoiceDestination, pendingCloudVoiceChoice, cloudVoiceConnectionAtom, cloudVoiceKeysAtom, + cloudVoiceModelsAtom, runSpeech, } from "../voice-input/voiceDestination"; import { SettingsSection } from "./components/SettingsSection"; -type Props = { environment: EnvironmentPresentation; active: CloudSpeechProvider | null }; +type CloudDestination = Extract; +type Props = { environment: EnvironmentPresentation; active: CloudDestination | null }; const isApiKey = Schema.is(CloudSpeechApiKey); +const isModelId = Schema.is(CloudSpeechModelId); +const MODEL_ROWS = 6; export function CloudVoiceSection({ environment, active }: Props) { - const [open, setOpen] = useState(active !== null); + const [open, setOpen] = useState(false); return ( setOpen(!open)} + onPress={() => setOpen(true)} /> - {open && } + {open && ( + setOpen(false)} /> + )} ); } -function CloudChoices({ environment, active }: Props) { +function ModelPicker({ + models, + value, + current, + disabled, + onChange, +}: { + models: ReadonlyArray; + value: string; + /** The grant's model, flagged when the catalog no longer lists it. */ + current: string | undefined; + disabled: boolean; + onChange: (model: CloudSpeechModelId) => void; +}) { + const [query, setQuery] = useState(""); + const [showAll, setShowAll] = useState(false); + const needle = query.trim().toLowerCase(); + const selected = models.find((model) => model.id === value); + const matches = models.filter((model) => + `${model.name} ${model.id}`.toLowerCase().includes(needle), + ); + const shown = showAll ? matches : matches.slice(0, MODEL_ROWS); + const delisted = current !== undefined && !models.some((model) => model.id === current); + return ( + <> + + {delisted ? ( + + OpenRouter no longer lists {current}. Dictation stays off until you choose another model. + + ) : null} + {selected && !shown.some((model) => model.id === selected.id) ? ( + {}} + /> + ) : null} + {shown.map((model, index) => ( + 0} + disabled={disabled} + onPress={() => onChange(model.id)} + /> + ))} + {matches.length === 0 ? ( + No transcription model matches. + ) : matches.length > shown.length ? ( + setShowAll(true)} + /> + ) : null} + + ); +} + +function CloudChoices({ environment, active, onClose }: Props & { onClose: () => void }) { const id = environment.environmentId; const connection = useAtomValue(cloudVoiceConnectionAtom(id)); const keys = useAtomValue(cloudVoiceKeysAtom(id)); @@ -63,7 +150,11 @@ function CloudChoices({ environment, active }: Props) { provider: CloudSpeechProvider; enable: boolean; } | null>(null); + const catalogAtom = cloudVoiceModelsAtom(editing?.provider === "openrouter" ? id : null); + const models = useAtomValue(catalogAtom); + const reloadModels = useAtomRefresh(catalogAtom); const [draft, setDraft] = useState(""); + const [model, setModel] = useState(""); const [sourceId, setSourceId] = useState(null); const sources = Constants.expoConfig?.extra?.secretsFeatureEnabled === true && @@ -78,6 +169,7 @@ function CloudChoices({ environment, active }: Props) { ? draft.trim() : undefined; const [saving, setSaving] = useState(false); + const activeModel = active?.provider === "openrouter" ? active.model : undefined; const enable = (commit: () => void) => { try { commit(); @@ -88,7 +180,7 @@ function CloudChoices({ environment, active }: Props) { ); } }; - const editKey = (provider: CloudSpeechProvider, enable: boolean) => { + const edit = (provider: CloudSpeechProvider, enable: boolean) => { const stored = environment.serverConfig?.settings.cloudSpeechCredentials?.[provider]; const ref = isSecretReference(stored) && sources.some((candidate) => candidate.id === stored.provider) @@ -96,11 +188,24 @@ function CloudChoices({ environment, active }: Props) { : null; setSourceId(ref?.provider ?? null); setDraft(ref?.reference ?? ""); + setModel(provider === "openrouter" ? (activeModel ?? "") : ""); setEditing({ provider, enable }); }; - const changeKey = async (provider: CloudSpeechProvider, value: SettingsSecretValue | null) => { + const reset = () => { + setDraft(""); + setModel(""); + setEditing(null); + }; + const changeKey = async ( + provider: CloudSpeechProvider, + value: SettingsSecretValue | null, + chosen?: CloudSpeechModelId, + ) => { if (!connection || !canManage || saving) return; - const commit = value !== null && editing?.enable ? pendingCloudVoiceChoice(id, provider) : null; + const commit = + value !== null && (editing?.enable || (chosen !== undefined && chosen !== activeModel)) + ? pendingCloudVoiceChoice(id, provider, chosen) + : null; setSaving(true); try { await runSpeech(setCloudSpeechKey(connection, provider, value)); @@ -114,12 +219,12 @@ function CloudChoices({ environment, active }: Props) { } setSaving(false); if (commit) enable(commit); - setDraft(""); - setEditing(null); reload(); + onClose(); }; - if (!AsyncResult.isSuccess(keys) || !keys.value || !connection) - return ( + const catalog = AsyncResult.isSuccess(models) ? (models.value?.models ?? null) : null; + const content = + !AsyncResult.isSuccess(keys) || !keys.value || !connection ? ( - ); - return CloudSpeechProvider.literals.map((provider) => { - const label = CLOUD_SPEECH_LABELS[provider]; - const company = CLOUD_SPEECH_PROVIDER_NAMES[provider]; - const disclosure = `After you stop recording, ${environment.label} sends your audio and vocabulary hints, including project names, to ${company}.`; - const saved = keys.value![provider]; - return ( - - } - selected={active === provider} - disabled={(!saved && !canManage) || saving} - description={ - active === provider && saved && canManage - ? "Tap to manage key" - : saved - ? "API key saved on this environment" - : canManage - ? "Add an API key to use this provider" - : "Ask an environment administrator to add an API key" - } - onPress={() => { - if (!saved) return editKey(provider, true); - if (active === provider) { - if (canManage) editKey(provider, false); - return; - } - Dialog.confirm({ - title: `Use ${label} for dictation?`, - message: `${disclosure} Select This device to stop sharing.`, - confirmText: `Use ${label}`, - onConfirm: () => enable(pendingCloudVoiceChoice(id, provider)), - }); - }} - /> - {canManage && editing?.provider === provider && ( - <> - {sources.length > 0 && ( - - { - setDraft(""); - setSourceId(null); - }} - /> - {sources.map((candidate) => ( - { + if (editing && provider !== editing.provider) return null; + const label = CLOUD_SPEECH_LABELS[provider]; + const company = CLOUD_SPEECH_PROVIDER_NAMES[provider]; + const routed = provider === "openrouter"; + const disclosure = cloudSpeechDisclosure(provider, environment.label); + const saved = keys.value![provider]; + const isActive = active?.provider === provider; + const isEditing = editing?.provider === provider; + const modelChosen = + isModelId(model) && (catalog?.some((entry) => entry.id === model) ?? false); + const modelChanged = model !== (activeModel ?? ""); + const original = environment.serverConfig?.settings.cloudSpeechCredentials?.[provider]; + const keyChanged = + credential !== undefined && + !( + isSecretReference(credential) && + isSecretReference(original) && + original.provider === credential.provider && + original.reference === credential.reference + ); + const routedReady = + modelChosen && + (draft.trim() ? credential !== undefined : Boolean(saved)) && + (!isActive || modelChanged || keyChanged); + const saveRouted = () => { + if (!routedReady || !isModelId(model)) return; + const grantChanges = !isActive || modelChanged; + if (credential !== undefined && keyChanged) + return void changeKey(provider, credential, model); + // Model-only changes never write to the environment. + if (grantChanges) enable(() => chooseVoiceDestination(id, provider, model)); + onClose(); + }; + return ( + + {!isEditing && ( + + } + selected={isActive} + disabled={saved === undefined || (!saved && !canManage) || saving} + description={ + saved === undefined + ? "Update the environment to use this provider" + : isActive && routed + ? `${activeModel} · Tap to change model${canManage ? " or key" : ""}` + : isActive && saved && canManage + ? "Tap to manage key" + : saved + ? routed + ? "API key saved · choose a model to use it" + : "API key saved on this environment" + : canManage + ? "Add an API key to use this provider" + : "Ask an environment administrator to add an API key" + } + onPress={() => { + if (routed) return edit(provider, !isActive); + if (!saved) return edit(provider, true); + if (isActive) { + if (canManage) edit(provider, false); + return; + } + Dialog.confirm({ + title: `Use ${label} for dictation?`, + message: `${disclosure} Select This device to stop sharing.`, + confirmText: `Use ${label}`, + onConfirm: () => enable(pendingCloudVoiceChoice(id, provider)), + }); + }} + /> + )} + {isEditing && routed && ( + <> + {disclosure} + {catalog ? ( + + ) : ( + + )} + + )} + {isEditing && (canManage || routed) && ( + <> + {canManage && sources.length > 0 && ( + + { + setDraft(""); + setSourceId(null); + }} + /> + {sources.map((candidate) => ( + { + setDraft(""); + setSourceId(candidate.id); + }} + /> + ))} + + )} + {canManage && ( + { + if (routed) saveRouted(); + else if (credential) void changeKey(provider, credential); + }} + value={draft} + onChangeText={setDraft} + className="mx-4 rounded-lg bg-background px-3 py-2 text-foreground" + /> + )} + {routed && !saved && !canManage ? ( + {}} + /> + ) : ( + { - setDraft(""); - setSourceId(candidate.id); + if (routed) saveRouted(); + else if (credential) void changeKey(provider, credential); }} /> - ))} - - )} - { - if (credential) void changeKey(provider, credential); - }} - value={draft} - onChangeText={setDraft} - className="mx-4 rounded-lg bg-background px-3 py-2 text-foreground" - /> - { - if (credential) void changeKey(provider, credential); - }} - /> - {saved && ( - - Dialog.confirm({ - title: `Remove the ${label} key from ${environment.label}?`, - message: "Everyone using it for dictation will lose access.", - confirmText: "Remove key", - destructive: true, - onConfirm: () => void changeKey(provider, null), - }) - } - /> + )} + {saved && canManage && ( + + Dialog.confirm({ + title: `Remove the ${label} key from ${environment.label}?`, + message: "Everyone using it for dictation will lose access.", + confirmText: "Remove key", + destructive: true, + onConfirm: () => void changeKey(provider, null), + }) + } + /> + )} + + )} - { - setEditing(null); - setDraft(""); - }} - /> - - )} - + + ); + }) ); - }); + return ( + + + {content} + + + ); } diff --git a/apps/mobile/src/features/settings/SettingsVoiceRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsVoiceRouteScreen.tsx index 01dbb5494fe65..5f7328214cb16 100644 --- a/apps/mobile/src/features/settings/SettingsVoiceRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsVoiceRouteScreen.tsx @@ -274,7 +274,7 @@ export function SettingsVoiceRouteScreen() { active={ grant?.destination.kind === "cloud" && grant.destination.environmentId === environment.environmentId - ? grant.destination.provider + ? grant.destination : null } /> diff --git a/apps/mobile/src/features/voice-input/voiceDestination.test.ts b/apps/mobile/src/features/voice-input/voiceDestination.test.ts index 9a23c9aebca40..dca713c3975c1 100644 --- a/apps/mobile/src/features/voice-input/voiceDestination.test.ts +++ b/apps/mobile/src/features/voice-input/voiceDestination.test.ts @@ -208,6 +208,18 @@ it("keeps local as the default and never reuses consent for an offline or resele "/api/voice/cloud/xai/transcribe", "/api/voice/cloud/xai/transcribe", ]); + // OpenRouter without a model is no destination; with one, the exact id travels with the audio. + chooseVoiceDestination(host, "openrouter"); + expect(appAtomRegistry.get(voiceRouteAtom)).toBeNull(); + chooseVoiceDestination(host, "openrouter", "acme/transcribe-1"); + mocks.fetch + .mockResolvedValueOnce(Response.json({ openai: true, xai: true, openrouter: true })) + .mockResolvedValueOnce(Response.json({ text: "routed" })); + const routed = await appAtomRegistry.get(voiceRouteAtom)!.transcriber!.prepare(options); + await expect(routed.transcribe("file:///voice.m4a", options)).resolves.toBe("routed"); + const upload = new URL(String(mocks.fetch.mock.lastCall![0])); + expect(upload.pathname).toBe("/api/voice/cloud/openrouter/transcribe"); + expect(upload.searchParams.get("model")).toBe("acme/transcribe-1"); const failWrite = () => { throw new Error("Device storage is unavailable"); }; diff --git a/apps/mobile/src/features/voice-input/voiceDestination.ts b/apps/mobile/src/features/voice-input/voiceDestination.ts index b51a70ffdbd41..dd27cca674888 100644 --- a/apps/mobile/src/features/voice-input/voiceDestination.ts +++ b/apps/mobile/src/features/voice-input/voiceDestination.ts @@ -1,12 +1,15 @@ import { AuthOrchestrationOperateScope, + type CloudSpeechModelId, type CloudSpeechProvider, type EnvironmentId, DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, + type VoiceTranscriptionDestination, } from "@t3tools/contracts"; import { getEnvironmentSpeechModels, getCloudSpeechKeys, + getCloudSpeechModels, requireCloudSpeechKey, transcribeCloudPcm, getEnvironmentSpeechStatus, @@ -40,30 +43,45 @@ export const voiceGrantAtom = Atom.make((get) => { let choiceRevision = 0; +type CloudDestination = Extract; +/** OpenRouter has no default model, so a cloud choice without one is no choice at all. */ +function cloudVoiceDestination( + environmentId: EnvironmentId, + provider: CloudSpeechProvider, + model?: CloudSpeechModelId, +): CloudDestination | null { + if (provider === "openrouter") + return model ? { kind: "cloud", environmentId, provider, model } : null; + return { kind: "cloud", environmentId, provider }; +} + export function chooseVoiceDestination( environmentId: EnvironmentId | null, provider?: CloudSpeechProvider, + model?: CloudSpeechModelId, ) { choiceRevision++; // A failed change must not leave the old destination armed in this session. appAtomRegistry.set(choice, null); - const grant: VoiceGrant | null = + const destination: VoiceTranscriptionDestination | null = environmentId === null ? null - : { - id: uuidv4(), - destination: provider - ? { kind: "cloud", environmentId, provider } - : { kind: "environment", environmentId }, - }; + : provider + ? cloudVoiceDestination(environmentId, provider, model) + : { kind: "environment", environmentId }; + const grant: VoiceGrant | null = destination && { id: uuidv4(), destination }; writeVoiceGrant(grant); appAtomRegistry.set(choice, grant); } -export function pendingCloudVoiceChoice(id: EnvironmentId, provider: CloudSpeechProvider) { +export function pendingCloudVoiceChoice( + id: EnvironmentId, + provider: CloudSpeechProvider, + model?: CloudSpeechModelId, +) { const revision = ++choiceRevision; return () => { - if (revision === choiceRevision) chooseVoiceDestination(id, provider); + if (revision === choiceRevision) chooseVoiceDestination(id, provider, model); }; } @@ -120,8 +138,10 @@ export function runSpeech( function environmentTranscriber( prepared: PreparedConnection, canSend: () => boolean, - provider?: CloudSpeechProvider, + cloud?: CloudDestination, ) { + const provider = cloud?.provider; + const model = cloud?.provider === "openrouter" ? cloud.model : undefined; return createEnvironmentTranscriber({ canSend, prepare: async (signal) => { @@ -166,6 +186,7 @@ function environmentTranscriber( pcm, DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, canSend, + model, ) : transcribeEnvironmentPcm( prepared, @@ -205,6 +226,11 @@ export const voiceSpeechStatusAtom = Atom.family((id: EnvironmentId) => export const cloudVoiceKeysAtom = Atom.family((id: EnvironmentId) => speechAtom(id, getCloudSpeechKeys, cloudVoiceConnectionAtom), ); +export const cloudVoiceModelsAtom = Atom.family((id: EnvironmentId | null) => + id === null + ? Atom.make(Effect.succeed(null)) + : speechAtom(id, getCloudSpeechModels, cloudVoiceConnectionAtom), +); export const voiceSpeechModelsAtom = Atom.family((id: EnvironmentId) => speechAtom(id, (prepared) => getEnvironmentSpeechModels(prepared)), ); @@ -217,8 +243,8 @@ export const voiceSpeechBenchmarksAtom = Atom.family((id: EnvironmentId) => export const voiceRouteAtom = Atom.make((get) => { const grant = get(voiceGrantAtom); if (!grant) return null; - const provider = grant.destination.kind === "cloud" ? grant.destination.provider : undefined; - const connectionAtom = (provider ? cloudVoiceConnectionAtom : voiceConnectionAtom)( + const cloud = grant.destination.kind === "cloud" ? grant.destination : undefined; + const connectionAtom = (cloud ? cloudVoiceConnectionAtom : voiceConnectionAtom)( grant.destination.environmentId, ); const connection = get(connectionAtom); @@ -229,7 +255,7 @@ export const voiceRouteAtom = Atom.make((get) => { environmentTranscriber( connection, () => registry.get(voiceGrantAtom) === grant && registry.get(connectionAtom) === connection, - provider, + cloud, ), }; }); diff --git a/apps/server/src/serverSettings.test.ts b/apps/server/src/serverSettings.test.ts index 21a7ed4fae1c8..6c470d1f83f05 100644 --- a/apps/server/src/serverSettings.test.ts +++ b/apps/server/src/serverSettings.test.ts @@ -1691,6 +1691,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" assert.deepEqual(legacy.cloudSpeechCredentials, { openai: SECRET_REDACTED, xai: SECRET_REDACTED, + openrouter: "", }); assert.equal(legacy.sourceControlHosts.bitbucket?.accessToken, SECRET_REDACTED); assert.equal(legacy.usageLimitSources[hub]?.managementKey, SECRET_REDACTED); @@ -1717,6 +1718,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" assert.deepEqual(echoed.cloudSpeechCredentials, { openai: usageReference, xai: "literal-fixture", + openrouter: "", }); assert.deepEqual(echoed.easDeviceHosts[0]?.accessToken, easReference); assert.deepEqual(echoed.simlockDeviceHosts[0]?.agentToken, simlockReference); @@ -1802,7 +1804,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" "ghe.acme.test": work, }); assert.equal(removed.usageLimitSources[hub]?.managementKey, ""); - assert.deepEqual(removed.cloudSpeechCredentials, { openai: "", xai: work }); + assert.deepEqual(removed.cloudSpeechCredentials, { openai: "", xai: work, openrouter: "" }); assert.equal(removed.easDeviceHosts[0]?.accessToken, ""); assert.deepEqual(Object.keys(removed.secretProviders), ["work"]); }).pipe(Effect.provide(layerServerSettingsWithSecrets())), @@ -1814,6 +1816,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" assert.deepEqual(strictSecretReferencePaths(ServerSettings.ast), [ "cloudSpeechCredentials.openai", "cloudSpeechCredentials.xai", + "cloudSpeechCredentials.openrouter", "easDeviceHosts.[].accessToken", "simlockDeviceHosts.[].agentToken", "usageLimitSources.*.managementKey", diff --git a/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts b/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts index c84ed1f58d763..58c84365dfcef 100644 --- a/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts +++ b/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts @@ -176,7 +176,7 @@ it.effect("reads a collected reference again once it is invalidated", () => resolver, Effect.succeed({ sourceControlHosts: hosts, - cloudSpeechCredentials: { openai: source("voice#key"), xai: "" }, + cloudSpeechCredentials: { openai: source("voice#key"), xai: "", openrouter: "" }, usageLimitSources, easDeviceHosts, simlockDeviceHosts, diff --git a/apps/server/src/speech/CloudSpeech.test.ts b/apps/server/src/speech/CloudSpeech.test.ts index 263a73dcd692a..65a42d0b51f54 100644 --- a/apps/server/src/speech/CloudSpeech.test.ts +++ b/apps/server/src/speech/CloudSpeech.test.ts @@ -1,9 +1,5 @@ import { it } from "@effect/vitest"; -import { - CloudSpeechProvider, - DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, - SecretProviderId, -} from "@t3tools/contracts"; +import { DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, SecretProviderId } from "@t3tools/contracts"; import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; @@ -13,94 +9,146 @@ import { expect, vi } from "vite-plus/test"; import * as ServerSettings from "../serverSettings.ts"; import * as CloudSpeech from "./CloudSpeech.ts"; -it.effect.each(CloudSpeechProvider.literals)( - "sends only to %s using its own saved key", - (provider) => - Effect.gen(function* () { - const reference = { - provider: SecretProviderId.make("work"), - reference: "op://Work/Voice/key", - }; - let resolved: string | undefined; - let cached: string | undefined; - const read = vi.fn(() => Effect.sync(() => [(cached ??= resolved)])); - const upstream = vi.fn(async () => Response.json({ text: "hello" })); - const service = yield* CloudSpeech.CloudSpeech.pipe( - Effect.provide( - CloudSpeech.layer.pipe( - Layer.provide(FetchHttpClient.layer), - Layer.provide(ServerSettings.ServerSettingsService.layerTest()), - Layer.provide( - Layer.mock(SecretResolver.SecretResolver)({ - read, - label: () => Effect.succeed("Work"), - invalidate: () => - Effect.sync(() => { - cached = undefined; - }), - }), - ), +it.effect.each(["openai", "xai"] as const)("sends only to %s using its own saved key", (provider) => + Effect.gen(function* () { + const reference = { + provider: SecretProviderId.make("work"), + reference: "op://Work/Voice/key", + }; + let resolved: string | undefined; + let cached: string | undefined; + const read = vi.fn(() => Effect.sync(() => [(cached ??= resolved)])); + const upstream = vi.fn(async () => Response.json({ text: "hello" })); + const service = yield* CloudSpeech.CloudSpeech.pipe( + Effect.provide( + CloudSpeech.layer.pipe( + Layer.provide(FetchHttpClient.layer), + Layer.provide(ServerSettings.ServerSettingsService.layerTest()), + Layer.provide( + Layer.mock(SecretResolver.SecretResolver)({ + read, + label: () => Effect.succeed("Work"), + invalidate: () => + Effect.sync(() => { + cached = undefined; + }), + }), ), ), - ); - const transcribe = (pcm = new Uint8Array(new Float32Array([0, 0.5]).buffer)) => - service - .transcribe(provider, pcm, { - ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, - speechLanguage: "en", - speechCustomWords: [{ term: "T3 Code", aliases: [] }], - }) - .pipe(Effect.provideService(FetchHttpClient.Fetch, upstream)); - yield* service.setKey(provider === "openai" ? "xai" : "openai", Redacted.make("other")); - expect((yield* Effect.flip(transcribe()))._tag).toBe("CloudSpeechError"); - yield* service.setKey(provider, Redacted.make("chosen")); - expect(yield* service.configured).toEqual({ openai: true, xai: true }); - yield* Effect.flip(transcribe(new Uint8Array(3))); - expect(upstream).not.toHaveBeenCalled(); - expect(yield* transcribe()).toBe("hello"); - const [url, init] = upstream.mock.calls[0]!; - expect(String(url)).toBe( - { - openai: "https://api.openai.com/v1/audio/transcriptions", - xai: "https://api.x.ai/v1/stt", - }[provider], - ); - expect(init?.redirect).toBe("error"); - expect(new Headers(init?.headers).get("authorization")).toBe("Bearer chosen"); - const form = init?.body as FormData; - expect(form.get("model")).toBe( - provider === "openai" ? "gpt-transcribe" : "grok-voice-transcribe-2.0", - ); - expect(form.getAll(provider === "openai" ? "keywords[]" : "keyterm")).toEqual(["T3 Code"]); - expect(form.get(provider === "openai" ? "languages[]" : "language")).toBe("en"); - expect([...form.keys()].at(-1)).toBe("file"); - const wav = new DataView( - yield* Effect.promise(() => (form.get("file") as Blob).arrayBuffer()), - ); - expect(wav.getUint32(24, true)).toBe(16_000); - expect(wav.getInt16(46, true)).toBe(16384); - upstream.mockResolvedValueOnce( - new Response("sensitive upstream diagnostic", { status: 401 }), - ); - expect(yield* Effect.flip(transcribe())).toMatchObject({ provider, reason: "rejected-key" }); - yield* service.setKey(provider, null); - expect((yield* service.configured)[provider]).toBe(false); - yield* Effect.flip(transcribe()); - expect(upstream).toHaveBeenCalledTimes(2); - yield* service.setKey(provider, reference); - expect((yield* service.configured)[provider]).toBe(true); - expect(read).not.toHaveBeenCalled(); - resolved = "resolved-fixture"; - expect(yield* transcribe()).toBe("hello"); - expect(read).toHaveBeenCalledWith([reference]); - expect(new Headers(upstream.mock.calls[2]?.[1]?.headers).get("authorization")).toBe( - "Bearer resolved-fixture", - ); - resolved = undefined; - expect(yield* Effect.flip(transcribe())).toMatchObject({ - provider, - reason: "unreadable-key", - }); - expect(upstream).toHaveBeenCalledTimes(3); - }), + ), + ); + const transcribe = (pcm = new Uint8Array(new Float32Array([0, 0.5]).buffer)) => + service + .transcribe(provider, pcm, { + ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, + speechLanguage: "en", + speechCustomWords: [{ term: "T3 Code", aliases: [] }], + }) + .pipe(Effect.provideService(FetchHttpClient.Fetch, upstream)); + yield* service.setKey(provider === "openai" ? "xai" : "openai", Redacted.make("other")); + expect((yield* Effect.flip(transcribe()))._tag).toBe("CloudSpeechError"); + yield* service.setKey(provider, Redacted.make("chosen")); + expect(yield* service.configured).toEqual({ openai: true, xai: true, openrouter: false }); + yield* Effect.flip(transcribe(new Uint8Array(3))); + expect(upstream).not.toHaveBeenCalled(); + expect(yield* transcribe()).toBe("hello"); + const [url, init] = upstream.mock.calls[0]!; + expect(String(url)).toBe( + { + openai: "https://api.openai.com/v1/audio/transcriptions", + xai: "https://api.x.ai/v1/stt", + }[provider], + ); + expect(init?.redirect).toBe("error"); + expect(new Headers(init?.headers).get("authorization")).toBe("Bearer chosen"); + const form = init?.body as FormData; + expect(form.get("model")).toBe( + provider === "openai" ? "gpt-transcribe" : "grok-voice-transcribe-2.0", + ); + expect(form.getAll(provider === "openai" ? "keywords[]" : "keyterm")).toEqual(["T3 Code"]); + expect(form.get(provider === "openai" ? "languages[]" : "language")).toBe("en"); + expect([...form.keys()].at(-1)).toBe("file"); + const wav = new DataView(yield* Effect.promise(() => (form.get("file") as Blob).arrayBuffer())); + expect(wav.getUint32(24, true)).toBe(16_000); + expect(wav.getInt16(46, true)).toBe(16384); + upstream.mockResolvedValueOnce(new Response("sensitive upstream diagnostic", { status: 401 })); + expect(yield* Effect.flip(transcribe())).toMatchObject({ provider, reason: "rejected-key" }); + yield* service.setKey(provider, null); + expect((yield* service.configured)[provider]).toBe(false); + yield* Effect.flip(transcribe()); + expect(upstream).toHaveBeenCalledTimes(2); + yield* service.setKey(provider, reference); + expect((yield* service.configured)[provider]).toBe(true); + expect(read).not.toHaveBeenCalled(); + resolved = "resolved-fixture"; + expect(yield* transcribe()).toBe("hello"); + expect(read).toHaveBeenCalledWith([reference]); + expect(new Headers(upstream.mock.calls[2]?.[1]?.headers).get("authorization")).toBe( + "Bearer resolved-fixture", + ); + resolved = undefined; + expect(yield* Effect.flip(transcribe())).toMatchObject({ + provider, + reason: "unreadable-key", + }); + expect(upstream).toHaveBeenCalledTimes(3); + }), +); + +it.effect("uploads to OpenRouter only for a listed exact model, as zero-retention JSON WAV", () => + Effect.gen(function* () { + const upstream = vi.fn(async (input) => + new URL(String(input)).pathname.endsWith("/models") + ? Response.json({ data: [{ id: "acme/transcribe-1" }, { id: "acme/transcribe-2" }] }) + : Response.json({ text: "hello" }), + ); + const service = yield* CloudSpeech.CloudSpeech.pipe( + Effect.provide( + CloudSpeech.layer.pipe( + Layer.provide(FetchHttpClient.layer), + Layer.provide(ServerSettings.ServerSettingsService.layerTest()), + Layer.provide(Layer.mock(SecretResolver.SecretResolver)({})), + ), + ), + ); + const uploads = () => + upstream.mock.calls.filter(([url]) => String(url).endsWith("/audio/transcriptions")); + const transcribe = (model: string | undefined) => + service + .transcribe( + "openrouter", + new Uint8Array(new Float32Array([0, 0.5]).buffer), + { ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, speechLanguage: "en" }, + model, + ) + .pipe(Effect.provideService(FetchHttpClient.Fetch, upstream)); + yield* service.setKey("openrouter", Redacted.make("router-key")); + expect(yield* Effect.flip(transcribe(undefined))).toMatchObject({ provider: "openrouter" }); + expect(yield* Effect.flip(transcribe("acme/delisted"))).toMatchObject({ + provider: "openrouter", + }); + expect(uploads()).toEqual([]); + expect(yield* transcribe("acme/transcribe-2")).toBe("hello"); + const [[url, init]] = uploads() as [[URL, RequestInit]]; + expect(String(url)).toBe("https://openrouter.ai/api/v1/audio/transcriptions"); + expect(init.redirect).toBe("error"); + const headers = new Headers(init.headers); + expect(headers.get("authorization")).toBe("Bearer router-key"); + expect(headers.get("content-type")).toContain("application/json"); + const body = JSON.parse(yield* Effect.promise(() => new Response(init.body).text())) as { + model: string; + models?: unknown; + provider: { zdr: boolean }; + input_audio: { format: string; data: string }; + }; + expect(body).toMatchObject({ + model: "acme/transcribe-2", + provider: { zdr: true }, + input_audio: { format: "wav" }, + }); + expect(body.models).toBeUndefined(); + const wav = Buffer.from(body.input_audio.data, "base64"); + expect(wav.toString("ascii", 0, 4)).toBe("RIFF"); + expect(wav.readInt16LE(46)).toBe(16384); + }), ); diff --git a/apps/server/src/speech/CloudSpeech.ts b/apps/server/src/speech/CloudSpeech.ts index 4dd60d614db54..048c120f3b386 100644 --- a/apps/server/src/speech/CloudSpeech.ts +++ b/apps/server/src/speech/CloudSpeech.ts @@ -1,6 +1,8 @@ import { CloudSpeechApiKey, CloudSpeechError, + CloudSpeechModel, + CloudSpeechModelId, isSecretReference, type SecretReference, type ServerSettingsError, @@ -26,8 +28,15 @@ import { decodeSpeechPcm, SpeechInvalidAudioError } from "./SpeechService.ts"; const endpoints = { openai: { url: "https://api.openai.com/v1/audio/transcriptions", model: "gpt-transcribe" }, xai: { url: "https://api.x.ai/v1/stt", model: "grok-voice-transcribe-2.0" }, + openrouter: { url: "https://openrouter.ai/api/v1/audio/transcriptions" }, }; +const OPENROUTER_CATALOG_URL = + "https://openrouter.ai/api/v1/models?output_modalities=transcription"; const isApiKey = Schema.is(CloudSpeechApiKey); +const isModelId = Schema.is(CloudSpeechModelId); +const OpenRouterCatalog = Schema.Struct({ + data: Schema.Array(Schema.Struct({ id: Schema.String, name: Schema.optionalKey(Schema.String) })), +}); function wave(samples: Float32Array) { const buffer = new ArrayBuffer(44 + samples.length * 2); @@ -51,12 +60,12 @@ function wave(samples: Float32Array) { const sample = samples[index]!; view.setInt16(44 + index * 2, Math.round(sample * (sample < 0 ? 32768 : 32767)), true); } - return new Blob([buffer], { type: "audio/wav" }); + return buffer; } function audioBody( - provider: CloudSpeechProvider, - audio: Blob, + provider: "openai" | "xai", + audio: ArrayBuffer, language: string, options: SpeechTranscriptionOptions, ) { @@ -76,14 +85,25 @@ function audioBody( if (!options.speechSilenceFiltering) form.set("vad_threshold", "0"); } // xAI ignores options that follow the file part. - form.set("file", audio, "dictation.wav"); + form.set("file", new Blob([audio], { type: "audio/wav" }), "dictation.wav"); return HttpClientRequest.bodyFormData(form); } +function openRouterBody(model: string, audio: ArrayBuffer, language: string) { + return HttpClientRequest.bodyJsonUnsafe({ + model, + input_audio: { data: Buffer.from(audio).toString("base64"), format: "wav" }, + ...(language !== "auto" ? { language } : {}), + provider: { zdr: true }, + }); +} + export class CloudSpeech extends Context.Service< CloudSpeech, { readonly configured: Effect.Effect; + /** OpenRouter's public transcription catalog, cached briefly on this server. */ + readonly models: Effect.Effect, CloudSpeechError>; readonly setKey: ( provider: CloudSpeechProvider, key: Redacted.Redacted | SecretReference | null, @@ -92,6 +112,7 @@ export class CloudSpeech extends Context.Service< provider: CloudSpeechProvider, pcm: Uint8Array, options: SpeechTranscriptionOptions, + model?: string | undefined, ) => Effect.Effect; } >()("t3/speech/CloudSpeech") {} @@ -106,7 +127,37 @@ export const layer = Layer.effect( Effect.map((value) => value.cloudSpeechCredentials), ); const configured = credentials.pipe( - Effect.map(({ openai, xai }) => ({ openai: Boolean(openai), xai: Boolean(xai) })), + Effect.map(({ openai, xai, openrouter }) => ({ + openai: Boolean(openai), + xai: Boolean(xai), + openrouter: Boolean(openrouter), + })), + ); + const execute = (request: HttpClientRequest.HttpClientRequest) => + http + .execute(request) + .pipe( + Effect.provideService(FetchHttpClient.RequestInit, { redirect: "error" }), + Effect.provideService(HttpClient.TracerPropagationEnabled, false), + ); + const fetchCatalog = Effect.fn("CloudSpeech.catalog")(function* () { + const unavailable = () => + new CloudSpeechError({ provider: "openrouter", reason: "unavailable" }); + const response = yield* execute(HttpClientRequest.get(OPENROUTER_CATALOG_URL)).pipe( + Effect.mapError(unavailable), + ); + if (response.status < 200 || response.status >= 300) return yield* unavailable(); + const { data } = yield* HttpClientResponse.schemaBodyJson(OpenRouterCatalog)(response).pipe( + Effect.mapError(unavailable), + ); + return data.flatMap(({ id, name }) => + isModelId(id) ? [{ id, name: name?.trim() || id }] : [], + ); + }); + // The catalog is public, so the cache holds no credential or consent state. + const [models, invalidateModels] = yield* Effect.cachedInvalidateWithTTL( + fetchCatalog(), + "10 minutes", ); const setKey = Effect.fn("CloudSpeech.setKey")(function* ( provider: CloudSpeechProvider, @@ -118,14 +169,25 @@ export const layer = Layer.effect( }, }); }); + const listedModel = Effect.fn("CloudSpeech.listedModel")(function* (model: string) { + const listed = (catalog: ReadonlyArray) => + catalog.some((entry) => entry.id === model); + if (listed(yield* models)) return true; + // A model added since the last refresh should work; a delisted one must stop. + yield* invalidateModels; + return listed(yield* models); + }); const transcribe = Effect.fn("CloudSpeech.transcribe")( function* ( provider: CloudSpeechProvider, pcm: Uint8Array, options: SpeechTranscriptionOptions, + model?: string, ) { const source = (yield* credentials)[provider]; if (!source) return yield* new CloudSpeechError({ provider, reason: "missing-key" }); + if (provider === "openrouter" && (!model || !isModelId(model))) + return yield* new CloudSpeechError({ provider, reason: "model-unavailable" }); const samples = yield* Effect.try({ try: () => decodeSpeechPcm(pcm), catch: () => @@ -135,6 +197,8 @@ export const layer = Layer.effect( }), }); const language = effectiveSpeechLanguage(cloudSpeechLanguages, options.speechLanguage); + if (provider === "openrouter" && !(yield* listedModel(model!))) + return yield* new CloudSpeechError({ provider, reason: "model-unavailable" }); // A vault that locked since the last recording must not authorize a new upload. if (isSecretReference(source)) yield* resolver.invalidate([source]); const unreadable = () => new CloudSpeechError({ provider, reason: "unreadable-key" }); @@ -142,18 +206,15 @@ export const layer = Layer.effect( Effect.mapError(unreadable), ); if (!isApiKey(key)) return yield* unreadable(); + const audio = wave(samples); const request = HttpClientRequest.post(endpoints[provider].url).pipe( HttpClientRequest.bearerToken(Redacted.make(key)), - audioBody(provider, wave(samples), language, options), + provider === "openrouter" + ? openRouterBody(model!, audio, language) + : audioBody(provider, audio, language, options), ); const unavailable = () => new CloudSpeechError({ provider, reason: "unavailable" }); - const response = yield* http - .execute(request) - .pipe( - Effect.provideService(FetchHttpClient.RequestInit, { redirect: "error" }), - Effect.provideService(HttpClient.TracerPropagationEnabled, false), - Effect.mapError(unavailable), - ); + const response = yield* execute(request).pipe(Effect.mapError(unavailable)); if (response.status < 200 || response.status >= 300) return yield* new CloudSpeechError({ provider, @@ -162,7 +223,9 @@ export const layer = Layer.effect( ? "rate-limit" : [401, 402, 403].includes(response.status) ? "rejected-key" - : "unavailable", + : response.status === 404 && provider === "openrouter" + ? "model-unavailable" + : "unavailable", }); const { text } = yield* HttpClientResponse.schemaBodyJson( EnvironmentSpeechTranscriptionResult, @@ -182,6 +245,6 @@ export const layer = Layer.effect( }), ), ); - return CloudSpeech.of({ configured, setKey, transcribe }); + return CloudSpeech.of({ configured, models, setKey, transcribe }); }), ); diff --git a/apps/server/src/speech/http.test.ts b/apps/server/src/speech/http.test.ts index 78d01e2ee8a70..f67129be60423 100644 --- a/apps/server/src/speech/http.test.ts +++ b/apps/server/src/speech/http.test.ts @@ -143,8 +143,10 @@ effectIt.effect( let stored = false; const reference = { provider: "work", reference: "op://Work/Voice/key" }; const save = vi.fn(); + const transcribe = vi.fn(); const cloud = Layer.mock(CloudSpeech.CloudSpeech)({ configured: Effect.sync(() => ({ openai: stored, xai: false })), + models: Effect.succeed([{ id: "acme/transcribe-1", name: "Acme Transcribe" }]), setKey: (provider, key) => Effect.sync(() => { save(provider, key); @@ -153,8 +155,10 @@ effectIt.effect( Redacted.isRedacted(key) && Redacted.value(key) === "fixture"; }), - transcribe: (provider) => - Effect.fail(new CloudSpeechError({ provider, reason: "rejected-key" })), + transcribe: (provider, _pcm, _options, model) => { + transcribe(provider, model); + return Effect.fail(new CloudSpeechError({ provider, reason: "rejected-key" })); + }, }); const { handler, dispose } = HttpRouter.toWebHandler( HttpApiBuilder.layer(HttpApi.make("environment").add(EnvironmentVoiceHttpApi)).pipe( @@ -207,6 +211,23 @@ effectIt.effect( provider: "openai", reason: "rejected-key", }); + expect(transcribe).toHaveBeenLastCalledWith("openai", undefined); + const catalog = yield* call("openrouter/models"); + expect(yield* Effect.promise(() => catalog.json())).toEqual({ + models: [{ id: "acme/transcribe-1", name: "Acme Transcribe" }], + }); + const audio = { + method: "POST", + headers: { "content-type": "application/octet-stream" }, + body: encodeSpeechPcmRequest(new Uint8Array(4), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS), + }; + expect((yield* call("openrouter/transcribe?model=openrouter%2Fauto", audio)).status).toBe( + 400, + ); + expect((yield* call("openrouter/transcribe?model=acme%2Ftranscribe-1", audio)).status).toBe( + 422, + ); + expect(transcribe).toHaveBeenLastCalledWith("openrouter", "acme/transcribe-1"); }).pipe(Effect.scoped), ); diff --git a/apps/server/src/speech/http.ts b/apps/server/src/speech/http.ts index d1c2fc58a07f1..eb4a7a01b88fb 100644 --- a/apps/server/src/speech/http.ts +++ b/apps/server/src/speech/http.ts @@ -86,6 +86,13 @@ export const speechHttpApiLayer = HttpApiBuilder.group( .pipe(Effect.catch((error) => failEnvironmentInternal("internal_error", error))); }), ) + .handle( + "cloudModels", + Effect.fn("environment.voice.cloudModels")(function* (args) { + yield* guard(args.endpoint.name); + return { models: yield* cloud.models }; + }), + ) .handle( "transcribeCloud", Effect.fn("environment.voice.transcribeCloud")(function* (args) { @@ -93,7 +100,12 @@ export const speechHttpApiLayer = HttpApiBuilder.group( const decoded = parseAudio(args.payload); if (Option.isNone(decoded)) return yield* failEnvironmentInvalidRequest("invalid_audio"); const text = yield* cloud - .transcribe(args.params.provider, decoded.value.pcm, decoded.value.options) + .transcribe( + args.params.provider, + decoded.value.pcm, + decoded.value.options, + args.query.model, + ) .pipe( Effect.catchTags({ ServerSettingsError: (error) => failEnvironmentInternal("internal_error", error), diff --git a/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx b/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx index fe4bd59164a80..0295603ed2038 100644 --- a/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx +++ b/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx @@ -130,3 +130,98 @@ it("enables the chosen provider only after a successful setup and keeps its gran }, ); }); + +it("OpenRouter needs an exact model, and changing it writes no credential", async () => { + const id = EnvironmentId.make("voice-host"); + const target = new PrimaryConnectionTarget({ + environmentId: id, + label: "Laptop", + httpBaseUrl: window.location.origin, + wsBaseUrl: "ws://localhost", + }); + const prepared = Option.some({ + ...target, + target, + socketUrl: target.wsBaseUrl, + httpAuthorization: null, + }); + using cleanup = new DisposableStack(); + cleanup.defer(() => chooseAudioDestination(null)); + chooseAudioDestination(null); + const writes: unknown[] = []; + const fetch = globalThis.fetch; + using _network = vi.spyOn(globalThis, "fetch").mockImplementation((request, init) => { + const url = new URL(request instanceof Request ? request.url : request, window.location.href); + if (url.pathname === "/api/voice/cloud/keys") + return Promise.resolve(Response.json({ openai: false, xai: false, openrouter: true })); + if (url.pathname === "/api/voice/cloud/openrouter/models") + return Promise.resolve( + Response.json({ + models: [ + { id: "acme/transcribe-1", name: "Acme: Transcribe 1" }, + { id: "other/listen-2", name: "Other: Listen 2" }, + ], + }), + ); + if (url.pathname === "/api/voice/cloud/openrouter/key") + return new Request(request, init).json().then((body) => { + writes.push(body); + return new Response(null, { status: 204 }); + }); + return fetch(request, init); + }); + const grantKey = "t3code:voice-audio-grant:v1"; + const grantModel = () => { + const raw = window.localStorage.getItem(grantKey); + const parsed = raw ? (JSON.parse(raw) as { destination?: { model?: string } }) : null; + return parsed?.destination?.model ?? null; + }; + await vi.mocked(useEnvironmentScope).withImplementation( + () => true, + async () => { + await vi.mocked(usePreparedConnection).withImplementation( + () => prepared, + async () => { + await using screen = await render( + {}}> + + , + ); + const destination = screen.getByRole("combobox", { name: "Transcribe with" }); + await destination.click(); + await page.getByRole("option", { name: "OpenRouter via Laptop" }).click(); + const dialog = page.getByRole("dialog"); + const save = dialog.getByRole("button", { name: "Save and enable" }); + await expect.element(save).toBeDisabled(); + const picker = dialog.getByRole("combobox", { name: "OpenRouter model" }); + await expect.element(picker).toBeVisible(); + await picker.click(); + await userEvent.keyboard("listen"); + await expect.element(page.getByRole("option", { name: /Other: Listen 2/ })).toBeVisible(); + await page.getByRole("option", { name: /Other: Listen 2/ }).click(); + await expect.element(save).toBeEnabled(); + const key = dialog.getByLabelText("OpenRouter API key", { exact: true }); + await key.fill("op://not-a-literal-key"); + await expect.element(save).toBeDisabled(); + await key.fill(""); + await save.click(); + await expect.element(dialog).not.toBeInTheDocument(); + await expect.element(destination).toHaveTextContent("OpenRouter via Laptop"); + expect(grantModel()).toBe("other/listen-2"); + expect(writes).toEqual([]); + await screen.getByRole("button", { name: "Manage OpenRouter" }).click(); + const current = dialog.getByRole("combobox", { name: "OpenRouter model" }); + await expect.element(current).toHaveValue("Other: Listen 2"); + await userEvent.clear(current); + await userEvent.type(current, "acme"); + await page.getByRole("option", { name: /Acme: Transcribe 1/ }).click(); + await dialog.getByRole("button", { name: "Save" }).click(); + await expect.element(dialog).not.toBeInTheDocument(); + await expect.element(destination).toHaveTextContent("OpenRouter via Laptop"); + expect(grantModel()).toBe("acme/transcribe-1"); + expect(writes).toEqual([]); + }, + ); + }, + ); +}); diff --git a/apps/web/src/components/settings/CloudVoiceConfiguration.tsx b/apps/web/src/components/settings/CloudVoiceConfiguration.tsx index ba8845ab496d8..eafde3d65bbea 100644 --- a/apps/web/src/components/settings/CloudVoiceConfiguration.tsx +++ b/apps/web/src/components/settings/CloudVoiceConfiguration.tsx @@ -1,32 +1,57 @@ import { useAtomRefresh, useAtomValue } from "@effect/atom-react"; +import { cloudSpeechDisclosure } from "@t3tools/shared/speech"; import type { PreparedConnection } from "@t3tools/client-runtime/connection"; import { configuredSecretSources, secretRequestReference, } from "@t3tools/client-runtime/secret-request"; -import { getCloudSpeechKeys, setCloudSpeechKey } from "@t3tools/client-runtime/voice-input"; +import { + getCloudSpeechKeys, + getCloudSpeechModels, + setCloudSpeechKey, +} from "@t3tools/client-runtime/voice-input"; import { AuthOrchestrationOperateScope, AuthSettingsWriteScope, CloudSpeechApiKey, + CloudSpeechModelId, CLOUD_SPEECH_LABELS, CLOUD_SPEECH_PROVIDER_NAMES, isSecretReference, + type CloudSpeechModel, type CloudSpeechProvider, type SecretProviderId, type SettingsSecretValue, + type VoiceTranscriptionDestination, } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; +import type { HttpClient } from "effect/http"; import { Atom, AsyncResult } from "effect/reactivity"; import { useId, useState } from "react"; import { isSecretsFeatureEnabled } from "../../env"; -import { layer, runtime } from "../../lib/runtime"; -import { chooseAudioDestination, pendingCloudAudioChoice } from "../../speech/voiceConsent"; +import { + inEnvironmentSpeechRuntime, + runEnvironmentSpeech, +} from "../../speech/runEnvironmentSpeech"; +import { + chooseAudioDestination, + pendingCloudAudioChoice, + useAudioGrant, +} from "../../speech/voiceConsent"; import type { EnvironmentPresentation } from "../../state/environments"; import { useEnvironmentScope, usePreparedConnection } from "../../state/session"; import { Button } from "../ui/button"; +import { + Combobox, + ComboboxEmpty, + ComboboxInput, + ComboboxItem, + ComboboxList, + ComboboxPopup, + useComboboxFilter, +} from "../ui/combobox"; import { Dialog, DialogDescription, @@ -41,12 +66,87 @@ import { Label } from "../ui/label"; import { SecretInput } from "./SecretInput"; import { SecretSourceSelect } from "./SecretSourceInput"; -const cloudKeysAtom = Atom.family((prepared: PreparedConnection | null) => - Atom.make( - prepared ? getCloudSpeechKeys(prepared).pipe(Effect.provide(layer)) : Effect.succeed(null), - ), -); +// Per dialog instance, not an Atom.family: key presence must be read fresh each time it opens. +function useCloudRead( + prepared: PreparedConnection | null, + read: (prepared: PreparedConnection) => Effect.Effect, +) { + const make = (connection: PreparedConnection | null) => ({ + prepared: connection, + atom: Atom.make( + connection ? inEnvironmentSpeechRuntime(read(connection)) : Effect.succeed(null), + ), + }); + const [current, setCurrent] = useState(() => make(prepared)); + if (current.prepared !== prepared) setCurrent(make(prepared)); + return [useAtomValue(current.atom), useAtomRefresh(current.atom)] as const; +} const isApiKey = Schema.is(CloudSpeechApiKey); +const isModelId = Schema.is(CloudSpeechModelId); +type CloudDestination = Extract; + +function ModelPicker({ + models, + value, + onChange, + disabled, + current, +}: { + models: ReadonlyArray; + value: string; + onChange: (next: string) => void; + disabled: boolean; + /** The grant's model, flagged when the catalog no longer lists it. */ + current: string | undefined; +}) { + const { contains } = useComboboxFilter(); + const selected = models.find((model) => model.id === value) ?? null; + const delisted = current !== undefined && !models.some((model) => model.id === current); + return ( +
+ model.name} + itemToStringValue={(model: CloudSpeechModel) => model.id} + filter={(model: CloudSpeechModel, query) => + contains(model.name, query) || contains(model.id, query) + } + value={selected} + onValueChange={(model) => onChange(model?.id ?? "")} + autoHighlight + disabled={disabled} + > + + + No transcription model matches. + + {(model: CloudSpeechModel) => ( + + + {model.name} + {model.id} + + + )} + + + + {delisted ? ( +

+ OpenRouter no longer lists {current}. Dictation stays off until you choose another model. +

+ ) : selected ? ( +

{selected.id}

+ ) : null} +
+ ); +} export function CloudVoiceConfiguration({ environment, @@ -62,16 +162,25 @@ export function CloudVoiceConfiguration({ const id = environment.environmentId; const name = CLOUD_SPEECH_LABELS[provider]; const company = CLOUD_SPEECH_PROVIDER_NAMES[provider]; + const routed = provider === "openrouter"; const formId = useId(); const prepared = Option.getOrNull(usePreparedConnection(id)); const canUse = useEnvironmentScope(id, AuthOrchestrationOperateScope); const canManage = useEnvironmentScope(id, AuthSettingsWriteScope); - const keys = cloudKeysAtom(canUse ? prepared : null); - const presence = useAtomValue(keys); - const refresh = useAtomRefresh(keys); + const [presence, refresh] = useCloudRead(canUse ? prepared : null, getCloudSpeechKeys); + const [models, refreshModels] = useCloudRead( + routed && canUse ? prepared : null, + getCloudSpeechModels, + ); + const grant = useAudioGrant(); + const currentModel = + active && grant?.destination.kind === "cloud" && grant.destination.provider === "openrouter" + ? grant.destination.model + : undefined; const [remove, setRemove] = useState(false); const [busy, setBusy] = useState(false); const [error, setError] = useState(""); + const [model, setModel] = useState(currentModel ?? ""); const saved = AsyncResult.isSuccess(presence) ? presence.value?.[provider] : undefined; const existing = environment.serverConfig?.settings?.cloudSpeechCredentials?.[provider]; const sources = @@ -91,19 +200,36 @@ export function CloudVoiceConfiguration({ : isApiKey(draft.trim()) ? draft.trim() : undefined; - const editing = canManage && saved !== undefined && (active || !saved); + const keyChanged = + credential !== undefined && + !( + isSecretReference(credential) && + reference?.provider === credential.provider && + reference.reference === credential.reference + ); const connected = canUse && prepared !== null; - const enable = () => chooseAudioDestination({ kind: "cloud", provider, environmentId: id }); - const action = async (value: SettingsSecretValue | null) => { + // Fixed providers edit the key only when nothing is saved or the device already uses them. + const editingKey = canManage && saved !== undefined && (routed || active || !saved); + const listedModels = AsyncResult.isSuccess(models) ? (models.value?.models ?? null) : null; + const modelChosen = isModelId(model) && (listedModels?.some((m) => m.id === model) ?? false); + const keyReady = draft.trim() ? credential !== undefined : Boolean(saved); + const modelChanged = model !== (currentModel ?? ""); + const destination = (): CloudDestination => + routed && isModelId(model) + ? { kind: "cloud", provider: "openrouter", environmentId: id, model } + : { kind: "cloud", provider: provider as "openai" | "xai", environmentId: id }; + const enable = () => chooseAudioDestination(destination()); + const canSubmit = routed + ? modelChosen && keyReady && (!active || modelChanged || keyChanged) + : credential !== undefined; + + const saveKey = async (value: SettingsSecretValue | null, thenEnable: boolean) => { if (!connected || !canManage || busy) return; setError(""); setBusy(true); try { - const commit = - !active && value !== null - ? pendingCloudAudioChoice({ kind: "cloud", provider, environmentId: id }) - : null; - await runtime.runPromise(setCloudSpeechKey(prepared, provider, value)); + const commit = thenEnable ? pendingCloudAudioChoice(destination()) : null; + await runEnvironmentSpeech(setCloudSpeechKey(prepared, provider, value)); refresh(); commit?.(); close(); @@ -112,6 +238,75 @@ export function CloudVoiceConfiguration({ } setBusy(false); }; + const submit = () => { + if (!canSubmit || busy) return; + const grantChanges = !active || modelChanged; + if (credential !== undefined && (!routed || keyChanged)) + return void saveKey(credential, grantChanges); + // Model-only changes never write to the environment. + if (grantChanges) enable(); + close(); + }; + + const keyField = ( + <> + +
+ {sources.length > 0 && ( + { + setSourceId(next); + setDraft(""); + setError(""); + }} + /> + )} +
+ {sourceId !== null ? ( + setDraft(event.target.value)} + /> + ) : ( + setDraft(event.target.value)} + /> + )} +
+
+

+ {draft.trim() && !credential + ? (source?.invalidReferenceMessage ?? "Enter an API key, not a URL or secret reference.") + : active && !routed + ? "Updating the key keeps dictation enabled." + : routed && saved + ? "Leave blank to keep the saved key." + : `Anyone with dictation access to ${environment.label} can bill this key.`} +

+ + ); return ( {remove ? `Everyone using ${name} for dictation on ${environment.label} will lose access.` - : active + : active && !routed ? `Anyone with dictation access to ${environment.label} can use this API key.` - : `When you stop recording, ${environment.label} sends your audio and vocabulary hints, including project names, to ${company}.`} + : cloudSpeechDisclosure(provider, environment.label)} {!remove && ( @@ -145,71 +342,63 @@ export function CloudVoiceConfiguration({ + ) : routed && + AsyncResult.isSuccess(presence) && + presence.value && + saved === undefined ? ( +

Update {environment.label} to use OpenRouter dictation.

) : saved === undefined ? (

Checking for an API key…

- ) : editing ? ( + ) : routed ? (
{ event.preventDefault(); - if (credential) void action(credential); + submit(); }} > - -
- {sources.length > 0 && ( - + + {listedModels ? ( + { - setSourceId(next); - setDraft(""); - setError(""); - }} + current={currentModel} /> + ) : AsyncResult.isFailure(models) ? ( + + ) : ( +

+ Loading transcription models… +

)} -
- {sourceId !== null ? ( - setDraft(event.target.value)} - /> - ) : ( - setDraft(event.target.value)} - /> - )} -
-

- {draft.trim() && !credential - ? (source?.invalidReferenceMessage ?? - "Enter an API key, not a URL or secret reference.") - : active - ? "Updating the key keeps dictation enabled." - : `Anyone with dictation access to ${environment.label} can bill this key.`} -

+ {editingKey ? ( +
{keyField}
+ ) : !saved ? ( +

Ask an administrator on {environment.label} to add an API key.

+ ) : ( +

+ Uses the API key saved on {environment.label}. This choice applies only to this + device. +

+ )} +
+ ) : editingKey ? ( +
{ + event.preventDefault(); + submit(); + }} + > + {keyField}
) : !saved ? (

Ask an administrator on {environment.label} to add an API key.

@@ -226,7 +415,7 @@ export function CloudVoiceConfiguration({

)} - {connected && canManage && saved && active && !remove && ( + {connected && canManage && saved && (active || routed) && !remove && (
- ) : editing ? ( + ) : routed ? ( + saved !== undefined && (saved || canManage) ? ( + + ) : null + ) : editingKey ? ( diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.tsx index dd4d6318761c7..6fb3b66281b1f 100644 --- a/apps/web/src/components/settings/VoiceSettingsPanel.tsx +++ b/apps/web/src/components/settings/VoiceSettingsPanel.tsx @@ -36,6 +36,7 @@ import { MonitorIcon, MicOffIcon, MicIcon, + WaypointsIcon, RefreshCwIcon, StarIcon, SlidersHorizontalIcon, @@ -120,6 +121,7 @@ import { useScopedSettings, useUpdateScopedSettings } from "./useScopedSettings" const OFF = "off"; function CloudSpeechLogo({ provider }: { provider: CloudSpeechProvider }) { + if (provider === "openrouter") return ; return ( env.environmentId === pendingCloud?.environmentId, ); + // OpenRouter users without settings access still manage their own model choice. + const manageLabel = + destination.cloudProvider === "openrouter" ? "Manage OpenRouter" : "Manage key"; const chosenEnvironment = environments.find( (environment) => environment.environmentId === destination.environmentId, ); @@ -1308,8 +1313,8 @@ export function VoiceSettingsPanel() { : "Off"} {destination.cloudProvider && ( - {" "} - via {chosenEnvironment?.label ?? "unavailable environment"} + {" via "} + {chosenEnvironment?.label ?? "unavailable environment"} )} @@ -1363,30 +1368,32 @@ export function VoiceSettingsPanel() { ) : null} - {destination.cloudProvider && chosenEnvironment && canManageKey && ( - - { - if (destination.cloudProvider) - setPendingCloud({ - provider: destination.cloudProvider, - environmentId: chosenEnvironment.environmentId, - }); - }} - > - - - } - /> - Manage key - - )} + {destination.cloudProvider && + chosenEnvironment && + (canManageKey || destination.cloudProvider === "openrouter") && ( + + { + if (destination.cloudProvider) + setPendingCloud({ + provider: destination.cloudProvider, + environmentId: chosenEnvironment.environmentId, + }); + }} + > + + + } + /> + {manageLabel} + + )}
{destination.environmentId && chosenUnsupported ? ( @@ -1515,7 +1522,7 @@ export function VoiceSettingsPanel() { boolean; readonly getTranscriptionOptions: () => SpeechTranscriptionOptions; readonly getMicrophoneId: () => string; @@ -48,6 +50,7 @@ export function createBrowserVoiceInputPlatform(input: { pcm, recordingOptions, input.hasAudioConsent, + input.cloudModel, ) : transcribeEnvironmentPcm( input.prepared, @@ -55,9 +58,7 @@ export function createBrowserVoiceInputPlatform(input: { recordingOptions, input.hasAudioConsent, ), - { - signal: transcriptionSignal, - }, + { signal: transcriptionSignal }, ); throwIfVoiceTranscriptionAborted(transcriptionSignal); return result.text; @@ -72,7 +73,15 @@ export function createBrowserVoiceInputPlatform(input: { } }; if (input.cloudProvider) { - await runEnvironmentSpeech(requireCloudSpeechKey(input.prepared, input.cloudProvider), options); + if (input.cloudProvider === "openrouter" && !input.cloudModel) + throw new VoiceTranscriptionError({ + code: "unavailable", + message: "Choose an OpenRouter model in Settings → Voice.", + }); + await runEnvironmentSpeech( + requireCloudSpeechKey(input.prepared, input.cloudProvider), + options, + ); throwIfVoiceTranscriptionAborted(options.signal); return { locale: effectiveSpeechLanguage(cloudSpeechLanguages, recordingOptions.speechLanguage), diff --git a/apps/web/src/speech/runEnvironmentSpeech.ts b/apps/web/src/speech/runEnvironmentSpeech.ts index bc6d1db747c78..f9635774d030e 100644 --- a/apps/web/src/speech/runEnvironmentSpeech.ts +++ b/apps/web/src/speech/runEnvironmentSpeech.ts @@ -6,14 +6,17 @@ import { connectionAtomRuntime } from "../connection/runtime"; import { appAtomRegistry } from "../rpc/atomRegistry"; // Relay (DPoP) environments authorize through services only the connection runtime provides. +export function inEnvironmentSpeechRuntime( + effect: Effect.Effect, +) { + return AtomRegistry.getResult(appAtomRegistry, connectionAtomRuntime).pipe( + Effect.flatMap((context) => Effect.provide(effect, context)), + ); +} + export function runEnvironmentSpeech( effect: Effect.Effect, options?: Effect.RunOptions, ): Promise { - return Effect.runPromise( - AtomRegistry.getResult(appAtomRegistry, connectionAtomRuntime).pipe( - Effect.flatMap((context) => Effect.provide(effect, context)), - ), - options, - ); + return Effect.runPromise(inEnvironmentSpeechRuntime(effect), options); } diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx index a202fd4013f68..8fb4722282747 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx +++ b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx @@ -156,7 +156,9 @@ vi.mock("./browserVoiceInput", () => ({ locale: "en", transcribe: async () => { mocks.transcribeCalls.push( - input.cloudProvider ? `${input.cloudProvider} via ${environmentId}` : environmentId, + input.cloudProvider + ? `${[input.cloudProvider, (input as { cloudModel?: string }).cloudModel].filter(Boolean).join(":")} via ${environmentId}` + : environmentId, ); const next = mocks.transcripts.shift() ?? "dictated"; if (next instanceof Error) throw next; @@ -420,6 +422,29 @@ it("recovers from a corrupt choice without enabling cloud dictation before the s expect(mocks.statusRequests).toEqual([]); }); +it("sends audio to OpenRouter only under a device grant naming the exact model", async () => { + setup(); + mocks.cloudVoiceCapable.add("env-b"); + const key = "t3code:voice-audio-grant:v1"; + const destination = { kind: "cloud", environmentId: "env-b", provider: "openrouter" }; + window.localStorage.setItem(key, JSON.stringify({ id: "no-model", destination })); + { + await using _screen = await render(); + await act(() => mocks.voice.start()); + expectNoAudioSent(); + } + const model = "acme/transcribe-1"; + window.localStorage.setItem( + key, + JSON.stringify({ id: "model", destination: { ...destination, model } }), + ); + await using _screen = await render(); + await act(() => mocks.voice.start()); + await act(() => mocks.voice.stop()); + expect(mocks.transcribeCalls).toEqual([`openrouter:${model} via env-b`]); + expect(mocks.statusRequests).toEqual([]); +}); + it("uses neither destination when the cloud save marker cannot be read", async () => { setup({ destination: "env-a" }); const key = "t3code:voice-audio-grant:v1"; diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.ts b/apps/web/src/speech/useEnvironmentSpeechInput.ts index bc86f1e5a39c2..279bfcfa8e88d 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.ts +++ b/apps/web/src/speech/useEnvironmentSpeechInput.ts @@ -109,7 +109,9 @@ export function useVoiceTranscriptionDestination() { const environment = useEnvironment(environmentId); const connection = Option.getOrNull(usePreparedConnection(environmentId)); const canDictate = useEnvironmentScope(environmentId, AuthOrchestrationOperateScope); - const provider = grant?.destination.kind === "cloud" ? grant.destination.provider : undefined; + const cloud = grant?.destination.kind === "cloud" ? grant.destination : undefined; + const provider = cloud?.provider; + const model = cloud?.provider === "openrouter" ? cloud.model : undefined; const supported = environment?.serverConfig?.environment.capabilities[ provider ? "cloudVoiceTranscription" : "voiceTranscription" @@ -117,6 +119,7 @@ export function useVoiceTranscriptionDestination() { return { environmentId, ...(provider ? { cloudProvider: provider } : {}), + ...(model ? { cloudModel: model } : {}), grantId: grant?.id ?? null, label: provider && environment @@ -131,6 +134,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { const prepared = destination.prepared; const grantId = destination.grantId; const cloudProvider = destination.cloudProvider; + const cloudModel = destination.cloudModel; const draftCapabilities = useEnvironment(input.environmentId)?.serverConfig?.environment .capabilities; const originatingConnection = Option.getOrNull(usePreparedConnection(input.environmentId)); @@ -242,6 +246,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { const platform = createBrowserVoiceInputPlatform({ prepared, cloudProvider, + cloudModel, hasAudioConsent, getTranscriptionOptions: (): SpeechTranscriptionOptions => { const { @@ -355,7 +360,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { controller.dispose(); if (controllerRef.current === controller) controllerRef.current = null; }; - }, [prepared, grantId, cloudProvider]); + }, [prepared, grantId, cloudProvider, cloudModel]); // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- probes the chosen environment's speech status once its connection is available useEffect(() => { diff --git a/docs/internals/voice-input.md b/docs/internals/voice-input.md index 7b7fb7cfa6fa7..f26e584cd1e39 100644 --- a/docs/internals/voice-input.md +++ b/docs/internals/voice-input.md @@ -43,6 +43,16 @@ It uses a separate, batch-only endpoint: changing a server key cannot upgrade an environment-local grant, and a cloud recording never uses the streaming socket or falls back to another provider or the environment's native model. +OpenRouter routes to many upstreams, so its grant also names the exact model and +the model travels with every upload as a query parameter. The server accepts only +ids present in OpenRouter's public transcription catalog (`output_modalities=transcription`), +which it caches for ten minutes and refreshes once before rejecting an unknown id; +router aliases such as `openrouter/auto` or `:nitro` fail schema validation before +that. Requests set `provider.zdr` and carry no `models` fallback list. OpenRouter +applies the account's safety-retention policy when choosing an endpoint. The catalog +holds no credential or consent state, which is why it may be cached while key +presence is read fresh on every request. + ## Server side The [speech service](../../apps/server/src/speech/SpeechService.ts) runs diff --git a/docs/user/voice-input.md b/docs/user/voice-input.md index 5526af1bab015..877a1a38a20f0 100644 --- a/docs/user/voice-input.md +++ b/docs/user/voice-input.md @@ -35,6 +35,17 @@ subscription is needed. Switch back to **This device** on mobile or **Off** on web and desktop to stop sharing. Removing the key stops that provider for everyone on the environment. +**OpenRouter** works the same way but has no default model: pick one from its live +speech-to-text catalog when you enable it, or later under **Manage OpenRouter**. +The model choice is yours alone and needs no settings access, so anyone with +dictation access can pick a different model without touching the saved key. +OpenRouter chooses who runs the model. T3 requests zero data retention; your +OpenRouter account can also allow safety retention. If no endpoint qualifies, +dictation fails instead of switching models. +Vocabulary hints are not sent to OpenRouter; dictionary spellings are applied +after transcription. If OpenRouter stops listing your model, dictation stops +until you choose another. + ## Set up a model If you choose environment transcription, T3 Code offers to download the recommended diff --git a/packages/client-runtime/src/voice-input/environment.ts b/packages/client-runtime/src/voice-input/environment.ts index 1fddab00ae717..52fa5a0d1fe5b 100644 --- a/packages/client-runtime/src/voice-input/environment.ts +++ b/packages/client-runtime/src/voice-input/environment.ts @@ -4,6 +4,7 @@ import * as Redacted from "effect/Redacted"; import * as Schema from "effect/Schema"; import { CloudSpeechError, SPEECH_STREAM_PATH } from "@t3tools/contracts"; import type { + CloudSpeechModelId, CloudSpeechProvider, SettingsSecretValue, SpeechAcceleration, @@ -79,6 +80,15 @@ export const getCloudSpeechKeys = (prepared: PreparedConnection) => run: ({ client, headers }) => client.cloudKeys({ headers }), }); +export const getCloudSpeechModels = (prepared: PreparedConnection) => + request({ + group: "voice", + prepared, + method: "GET", + path: (urls) => urls.cloudModels(), + run: ({ client, headers }) => client.cloudModels({ headers }), + }); + export const requireCloudSpeechKey = ( prepared: PreparedConnection, provider: CloudSpeechProvider, @@ -120,18 +130,21 @@ export const transcribeCloudPcm = ( pcm: Uint8Array, options: SpeechTranscriptionOptions, canSend: () => boolean, -) => - request({ + model?: CloudSpeechModelId, +) => { + const query = model === undefined ? {} : { model }; + return request({ group: "voice", prepared, method: "POST", canSend, timeoutMs: 130_000, - path: (urls) => urls.transcribeCloud({ params: { provider } }), + path: (urls) => urls.transcribeCloud({ params: { provider }, query }), run: ({ client, headers }) => client.transcribeCloud({ headers, params: { provider }, + query, payload: encodeSpeechPcmRequest(pcm, options), }), }).pipe( @@ -144,6 +157,7 @@ export const transcribeCloudPcm = ( : error, ), ); +}; export const getEnvironmentSpeechStreamUrl = (prepared: PreparedConnection) => { let endpoint = prepared.httpBaseUrl; diff --git a/packages/client-runtime/src/voice-input/index.ts b/packages/client-runtime/src/voice-input/index.ts index 44a851fe14e69..5843267800f67 100644 --- a/packages/client-runtime/src/voice-input/index.ts +++ b/packages/client-runtime/src/voice-input/index.ts @@ -15,6 +15,7 @@ export { downloadEnvironmentSpeechModel, getEnvironmentSpeechModels, getCloudSpeechKeys, + getCloudSpeechModels, requireCloudSpeechKey, setCloudSpeechKey, transcribeCloudPcm, diff --git a/packages/contracts/src/environmentHttp.ts b/packages/contracts/src/environmentHttp.ts index 77ea7fb4eb034..46dfcde197ac2 100644 --- a/packages/contracts/src/environmentHttp.ts +++ b/packages/contracts/src/environmentHttp.ts @@ -52,6 +52,8 @@ import { CloudSpeechApiKey, CloudSpeechError, CloudSpeechKeys, + CloudSpeechModelId, + CloudSpeechModels, CloudSpeechProvider, EnvironmentSpeechAccelerationRequest, EnvironmentSpeechModelRequest, @@ -800,10 +802,19 @@ export class EnvironmentVoiceHttpApi extends HttpApiGroup.make("voice") error: EnvironmentScopedOperationErrors, }).middleware(EnvironmentAuthenticatedAuth), ) + .add( + // Public catalog; carries no credential and no audio. + HttpApiEndpoint.get("cloudModels", "/api/voice/cloud/openrouter/models", { + headers: OptionalBearerHeaders, + success: CloudSpeechModels, + error: [...EnvironmentScopedOperationErrors, CloudSpeechError], + }).middleware(EnvironmentAuthenticatedAuth), + ) .add( HttpApiEndpoint.post("transcribeCloud", "/api/voice/cloud/:provider/transcribe", { headers: OptionalBearerHeaders, params: Schema.Struct({ provider: CloudSpeechProvider }), + query: Schema.Struct({ model: Schema.optionalKey(CloudSpeechModelId) }), payload: Schema.Uint8Array.pipe(HttpApiSchema.asUint8Array()), success: EnvironmentSpeechTranscriptionResult, error: [ diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 50cf8a50d5117..4d1aed1d2b25e 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -53,7 +53,7 @@ import { } from "./providerInstance.ts"; import { PullRequestMergeMethod } from "./pullRequest.ts"; import { - CloudSpeechProvider, + CloudSpeechModelId, SpeechAcceleration, SpeechCorrectionCue, SpeechCustomWords, @@ -366,13 +366,22 @@ export type ModelPreferences = { export const VoiceShortcutMode = Schema.Literals(["auto", "hold", "toggle"]); export type VoiceShortcutMode = typeof VoiceShortcutMode.Type; -/** Where this device sends microphone audio. A later backend adds its own kind here. */ +/** + * Where this device sends microphone audio. A cloud grant names the forwarding environment and + * the provider; an OpenRouter grant also names the exact model, since OpenRouter has no default. + */ export const VoiceTranscriptionDestination = Schema.Union([ Schema.Struct({ kind: Schema.Literal("environment"), environmentId: EnvironmentId }), Schema.Struct({ kind: Schema.Literal("cloud"), environmentId: EnvironmentId, - provider: CloudSpeechProvider, + provider: Schema.Literals(["openai", "xai"]), + }), + Schema.Struct({ + kind: Schema.Literal("cloud"), + environmentId: EnvironmentId, + provider: Schema.Literal("openrouter"), + model: CloudSpeechModelId, }), ]); export type VoiceTranscriptionDestination = typeof VoiceTranscriptionDestination.Type; @@ -1235,6 +1244,7 @@ const CloudSpeechCredential = SettingsSecretValue.pipe( export const CloudSpeechCredentials = Schema.Struct({ openai: CloudSpeechCredential, xai: CloudSpeechCredential, + openrouter: CloudSpeechCredential, }); export const ServerSettings = Schema.Struct({ @@ -1263,7 +1273,7 @@ export const ServerSettings = Schema.Struct({ enableAgentBrowserAccess: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))), cloudSpeechCredentials: CloudSpeechCredentials.pipe( - Schema.withDecodingDefault(Effect.succeed({ openai: "", xai: "" })), + Schema.withDecodingDefault(Effect.succeed({ openai: "", xai: "", openrouter: "" })), ), speechModelId: Schema.String.pipe( Schema.withDecodingDefault(Effect.succeed("handy-computer/parakeet-unified-en-0.6b-gguf")), @@ -1688,6 +1698,7 @@ export const ServerSettingsPatch = Schema.Struct({ Schema.Struct({ openai: Schema.optionalKey(SettingsSecretValue), xai: Schema.optionalKey(SettingsSecretValue), + openrouter: Schema.optionalKey(SettingsSecretValue), }), ), speechModelId: Schema.optionalKey(Schema.String), diff --git a/packages/contracts/src/speech.ts b/packages/contracts/src/speech.ts index d9398ba532b94..5276bb7312d9a 100644 --- a/packages/contracts/src/speech.ts +++ b/packages/contracts/src/speech.ts @@ -7,16 +7,38 @@ export const CloudSpeechApiKey = TrimmedNonEmptyString.check( Schema.isPattern(/^(?![a-z][a-z\d+.-]*:\/\/)[!-~]+$/i), ); -export const CloudSpeechProvider = Schema.Literals(["openai", "xai"]); +export const CloudSpeechProvider = Schema.Literals(["openai", "xai", "openrouter"]); export type CloudSpeechProvider = typeof CloudSpeechProvider.Type; export const CLOUD_SPEECH_LABELS = { openai: "OpenAI", xai: "Grok Voice Transcribe", + openrouter: "OpenRouter", } as const; -export const CLOUD_SPEECH_PROVIDER_NAMES = { openai: "OpenAI", xai: "xAI" } as const; -export const CloudSpeechKeys = Schema.Record(CloudSpeechProvider, Schema.Boolean); +export const CLOUD_SPEECH_PROVIDER_NAMES = { + openai: "OpenAI", + xai: "xAI", + openrouter: "OpenRouter", +} as const; +/** Servers that predate OpenRouter report only the first two providers. */ +export const CloudSpeechKeys = Schema.Struct({ + openai: Schema.Boolean, + xai: Schema.Boolean, + openrouter: Schema.optionalKey(Schema.Boolean), +}); export type CloudSpeechKeys = typeof CloudSpeechKeys.Type; +/** An exact `author/model` slug. Router shortcuts such as `:nitro` or `openrouter/auto` are not models. */ +export const CloudSpeechModelId = Schema.String.check( + Schema.isMinLength(3), + Schema.isMaxLength(160), + Schema.isPattern(/^(?!openrouter\/)[a-z\d][a-z\d.-]*\/[a-z\d][a-z\d._-]*$/i), +); +export type CloudSpeechModelId = typeof CloudSpeechModelId.Type; +export const CloudSpeechModel = Schema.Struct({ id: CloudSpeechModelId, name: Schema.String }); +export type CloudSpeechModel = typeof CloudSpeechModel.Type; +export const CloudSpeechModels = Schema.Struct({ models: Schema.Array(CloudSpeechModel) }); +export type CloudSpeechModels = typeof CloudSpeechModels.Type; + export class CloudSpeechError extends Schema.TaggedError()( "CloudSpeechError", { @@ -26,6 +48,7 @@ export class CloudSpeechError extends Schema.TaggedError()( "unreadable-key", "rejected-key", "rate-limit", + "model-unavailable", "unavailable", ]), }, @@ -42,6 +65,8 @@ export class CloudSpeechError extends Schema.TaggedError()( return `${name} rejected this API key. Check its permissions and billing in Settings → Voice.`; case "rate-limit": return `${name} reached a usage limit. Check the account's quota or try again later.`; + case "model-unavailable": + return `${name} doesn’t offer the selected transcription model. Choose another model in Settings → Voice. No audio was sent to ${name}.`; case "unavailable": return `${name} couldn’t transcribe this recording. Try again later.`; } diff --git a/packages/shared/src/speech.ts b/packages/shared/src/speech.ts index 73749cb1d2fc8..0fb077452bb3a 100644 --- a/packages/shared/src/speech.ts +++ b/packages/shared/src/speech.ts @@ -1,4 +1,6 @@ import { + CLOUD_SPEECH_PROVIDER_NAMES, + type CloudSpeechProvider, SPEECH_MAX_OPTIONS_BYTES, type SpeechCustomWords, SpeechTranscriptionOptions, @@ -8,6 +10,12 @@ import * as Schema from "effect/Schema"; const decodeOptions = Schema.decodeUnknownSync(Schema.fromJsonString(SpeechTranscriptionOptions)); const encodeOptions = Schema.encodeSync(Schema.fromJsonString(SpeechTranscriptionOptions)); +export function cloudSpeechDisclosure(provider: CloudSpeechProvider, environment: string) { + return provider === "openrouter" + ? `After you stop, ${environment} sends the recording to OpenRouter. OpenRouter chooses who runs your model. T3 requests zero data retention; your OpenRouter account may allow safety retention.` + : `After you stop, ${environment} sends your recording and vocabulary hints, including project names, to ${CLOUD_SPEECH_PROVIDER_NAMES[provider]}.`; +} + // Only offer explicit hints understood by all cloud APIs; other languages can still be detected. export const cloudSpeechLanguages = { supportsLanguageDetection: true, From 72382cbd2d1ca7b83791c969ad6fd23d89785497 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Fri, 6 Nov 2026 12:25:56 -0800 Subject: [PATCH 8/9] fix(voice): recheck cloud credentials before sending audio Retry temporary catalog failures without waiting for the cache and list only models eligible for the routing policy. A missing upstream route does not imply that the broker never received the request. --- apps/server/src/speech/CloudSpeech.test.ts | 46 ++++++++++++++++++++++ apps/server/src/speech/CloudSpeech.ts | 34 ++++++++++------ docs/internals/voice-input.md | 4 +- packages/contracts/src/speech.ts | 5 ++- 4 files changed, 73 insertions(+), 16 deletions(-) diff --git a/apps/server/src/speech/CloudSpeech.test.ts b/apps/server/src/speech/CloudSpeech.test.ts index 65a42d0b51f54..112c7a88d9b3b 100644 --- a/apps/server/src/speech/CloudSpeech.test.ts +++ b/apps/server/src/speech/CloudSpeech.test.ts @@ -2,6 +2,7 @@ import { it } from "@effect/vitest"; import { DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, SecretProviderId } from "@t3tools/contracts"; import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Redacted from "effect/Redacted"; import * as FetchHttpClient from "effect/http/FetchHttpClient"; @@ -124,12 +125,17 @@ it.effect("uploads to OpenRouter only for a listed exact model, as zero-retentio .pipe(Effect.provideService(FetchHttpClient.Fetch, upstream)); yield* service.setKey("openrouter", Redacted.make("router-key")); expect(yield* Effect.flip(transcribe(undefined))).toMatchObject({ provider: "openrouter" }); + upstream.mockResolvedValueOnce(new Response(null, { status: 503 })); + expect(yield* Effect.flip(transcribe("acme/transcribe-2"))).toMatchObject({ + reason: "unavailable", + }); expect(yield* Effect.flip(transcribe("acme/delisted"))).toMatchObject({ provider: "openrouter", }); expect(uploads()).toEqual([]); expect(yield* transcribe("acme/transcribe-2")).toBe("hello"); const [[url, init]] = uploads() as [[URL, RequestInit]]; + expect(new URL(String(upstream.mock.calls[0]![0])).searchParams.get("zdr")).toBe("true"); expect(String(url)).toBe("https://openrouter.ai/api/v1/audio/transcriptions"); expect(init.redirect).toBe("error"); const headers = new Headers(init.headers); @@ -152,3 +158,43 @@ it.effect("uploads to OpenRouter only for a listed exact model, as zero-retentio expect(wav.readInt16LE(46)).toBe(16384); }), ); + +it.effect("removing a key while a catalog read is pending prevents the upload", () => + Effect.gen(function* () { + const service = yield* CloudSpeech.CloudSpeech.pipe( + Effect.provide( + CloudSpeech.layer.pipe( + Layer.provide(FetchHttpClient.layer), + Layer.provide(ServerSettings.ServerSettingsService.layerTest()), + Layer.provide(Layer.mock(SecretResolver.SecretResolver)({})), + ), + ), + ); + const catalogStarted = Promise.withResolvers(); + const catalog = Promise.withResolvers(); + const uploads: string[] = []; + const upstream: typeof fetch = async (input) => { + if (new URL(String(input)).pathname.endsWith("/models")) { + catalogStarted.resolve(); + return catalog.promise; + } + uploads.push(String(input)); + return Response.json({ text: "unexpected" }); + }; + yield* service.setKey("openrouter", Redacted.make("old-key")); + const job = yield* service + .transcribe( + "openrouter", + new Uint8Array(4), + DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, + "acme/voice", + ) + .pipe(Effect.provideService(FetchHttpClient.Fetch, upstream), Effect.forkChild); + yield* Effect.promise(() => catalogStarted.promise); + yield* service.setKey("openrouter", null); + catalog.resolve(Response.json({ data: [{ id: "acme/voice" }] })); + const exit = yield* Fiber.await(job); + expect(uploads).toEqual([]); + expect(exit._tag).toBe("Failure"); + }), +); diff --git a/apps/server/src/speech/CloudSpeech.ts b/apps/server/src/speech/CloudSpeech.ts index 048c120f3b386..3b7eab6b5242b 100644 --- a/apps/server/src/speech/CloudSpeech.ts +++ b/apps/server/src/speech/CloudSpeech.ts @@ -3,6 +3,7 @@ import { CloudSpeechError, CloudSpeechModel, CloudSpeechModelId, + SettingsSecretValue, isSecretReference, type SecretReference, type ServerSettingsError, @@ -15,6 +16,7 @@ import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretReso import { cloudSpeechLanguages, effectiveSpeechLanguage } from "@t3tools/shared/speech"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; import * as Layer from "effect/Layer"; import * as Redacted from "effect/Redacted"; import * as Schema from "effect/Schema"; @@ -31,9 +33,10 @@ const endpoints = { openrouter: { url: "https://openrouter.ai/api/v1/audio/transcriptions" }, }; const OPENROUTER_CATALOG_URL = - "https://openrouter.ai/api/v1/models?output_modalities=transcription"; + "https://openrouter.ai/api/v1/models?output_modalities=transcription&zdr=true"; const isApiKey = Schema.is(CloudSpeechApiKey); const isModelId = Schema.is(CloudSpeechModelId); +const sameCredential = Schema.toEquivalence(SettingsSecretValue); const OpenRouterCatalog = Schema.Struct({ data: Schema.Array(Schema.Struct({ id: Schema.String, name: Schema.optionalKey(Schema.String) })), }); @@ -155,9 +158,15 @@ export const layer = Layer.effect( ); }); // The catalog is public, so the cache holds no credential or consent state. - const [models, invalidateModels] = yield* Effect.cachedInvalidateWithTTL( - fetchCatalog(), - "10 minutes", + const models = yield* Effect.cachedWithTTL( + fetchCatalog().pipe( + Effect.timeoutOrElse({ + duration: "10 seconds", + orElse: () => + Effect.fail(new CloudSpeechError({ provider: "openrouter", reason: "unavailable" })), + }), + ), + (exit) => (Exit.isSuccess(exit) ? "10 minutes" : 0), ); const setKey = Effect.fn("CloudSpeech.setKey")(function* ( provider: CloudSpeechProvider, @@ -169,14 +178,6 @@ export const layer = Layer.effect( }, }); }); - const listedModel = Effect.fn("CloudSpeech.listedModel")(function* (model: string) { - const listed = (catalog: ReadonlyArray) => - catalog.some((entry) => entry.id === model); - if (listed(yield* models)) return true; - // A model added since the last refresh should work; a delisted one must stop. - yield* invalidateModels; - return listed(yield* models); - }); const transcribe = Effect.fn("CloudSpeech.transcribe")( function* ( provider: CloudSpeechProvider, @@ -197,7 +198,7 @@ export const layer = Layer.effect( }), }); const language = effectiveSpeechLanguage(cloudSpeechLanguages, options.speechLanguage); - if (provider === "openrouter" && !(yield* listedModel(model!))) + if (provider === "openrouter" && !(yield* models).some((entry) => entry.id === model)) return yield* new CloudSpeechError({ provider, reason: "model-unavailable" }); // A vault that locked since the last recording must not authorize a new upload. if (isSecretReference(source)) yield* resolver.invalidate([source]); @@ -214,6 +215,13 @@ export const layer = Layer.effect( : audioBody(provider, audio, language, options), ); const unavailable = () => new CloudSpeechError({ provider, reason: "unavailable" }); + // A catalog fetch or vault lookup may have outlived the credential it started with. + const current = (yield* credentials)[provider]; + if (!current || !sameCredential(source, current)) + return yield* new CloudSpeechError({ + provider, + reason: current ? "key-changed" : "missing-key", + }); const response = yield* execute(request).pipe(Effect.mapError(unavailable)); if (response.status < 200 || response.status >= 300) return yield* new CloudSpeechError({ diff --git a/docs/internals/voice-input.md b/docs/internals/voice-input.md index f26e584cd1e39..2bf73af0ab36d 100644 --- a/docs/internals/voice-input.md +++ b/docs/internals/voice-input.md @@ -45,8 +45,8 @@ socket or falls back to another provider or the environment's native model. OpenRouter routes to many upstreams, so its grant also names the exact model and the model travels with every upload as a query parameter. The server accepts only -ids present in OpenRouter's public transcription catalog (`output_modalities=transcription`), -which it caches for ten minutes and refreshes once before rejecting an unknown id; +ids present in OpenRouter's public ZDR-eligible transcription catalog, +which it caches for ten minutes after a successful read; router aliases such as `openrouter/auto` or `:nitro` fail schema validation before that. Requests set `provider.zdr` and carry no `models` fallback list. OpenRouter applies the account's safety-retention policy when choosing an endpoint. The catalog diff --git a/packages/contracts/src/speech.ts b/packages/contracts/src/speech.ts index 5276bb7312d9a..e6e4d3ddbc51e 100644 --- a/packages/contracts/src/speech.ts +++ b/packages/contracts/src/speech.ts @@ -45,6 +45,7 @@ export class CloudSpeechError extends Schema.TaggedError()( provider: CloudSpeechProvider, reason: Schema.Literals([ "missing-key", + "key-changed", "unreadable-key", "rejected-key", "rate-limit", @@ -59,6 +60,8 @@ export class CloudSpeechError extends Schema.TaggedError()( switch (this.reason) { case "missing-key": return `Add your ${name} API key in Settings → Voice.`; + case "key-changed": + return `The ${name} key changed before your recording was sent. Try again.`; case "unreadable-key": return `Couldn’t read the ${name} API key. Unlock the vault on the selected environment or update the key in Settings → Voice. No audio was sent to ${name}.`; case "rejected-key": @@ -66,7 +69,7 @@ export class CloudSpeechError extends Schema.TaggedError()( case "rate-limit": return `${name} reached a usage limit. Check the account's quota or try again later.`; case "model-unavailable": - return `${name} doesn’t offer the selected transcription model. Choose another model in Settings → Voice. No audio was sent to ${name}.`; + return `The chosen model isn’t available on ${name}. Choose another model in Settings → Voice.`; case "unavailable": return `${name} couldn’t transcribe this recording. Try again later.`; } From 9747603b30ba609a27de4f013db91b77bb2f725e Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Fri, 6 Nov 2026 19:02:06 -0800 Subject: [PATCH 9/9] fix(voice): show the official OpenRouter glyph in the destination picker --- apps/mobile/src/components/ProviderIcon.tsx | 5 +++-- .../src/features/settings/CloudVoiceSection.tsx | 4 +++- .../src/components/settings/VoiceSettingsPanel.tsx | 6 ++++-- packages/client-runtime/src/providerClients.ts | 13 ++++++++++++- 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/apps/mobile/src/components/ProviderIcon.tsx b/apps/mobile/src/components/ProviderIcon.tsx index cd4574b6f84e5..c5d9acacba238 100644 --- a/apps/mobile/src/components/ProviderIcon.tsx +++ b/apps/mobile/src/components/ProviderIcon.tsx @@ -11,7 +11,7 @@ import { resolveOfficialAcpRegistryIconUrl } from "@t3tools/contracts"; import { useAppearancePreferences } from "../features/settings/appearance/AppearancePreferencesProvider"; import { AppText as Text } from "./ui/AppText"; import { cn } from "../lib/cn"; -import { getProviderClient } from "@t3tools/client-runtime/provider-clients"; +import { getProviderClient, openRouterIcon } from "@t3tools/client-runtime/provider-clients"; type ProviderIconProps = { readonly provider: string | null | undefined; @@ -99,7 +99,8 @@ export function ProviderIcon(props: ProviderIconProps) { ); } - const packageIcon = getProviderClient(props.provider)?.icon; + const packageIcon = + props.provider === "openrouter" ? openRouterIcon : getProviderClient(props.provider)?.icon; if (packageIcon) { const themeFill = (fill: { readonly light: string; readonly dark: string }) => isDarkMode ? fill.dark : fill.light; diff --git a/apps/mobile/src/features/settings/CloudVoiceSection.tsx b/apps/mobile/src/features/settings/CloudVoiceSection.tsx index 4709671dd428f..684f8dbc83769 100644 --- a/apps/mobile/src/features/settings/CloudVoiceSection.tsx +++ b/apps/mobile/src/features/settings/CloudVoiceSection.tsx @@ -278,7 +278,9 @@ function CloudChoices({ environment, active, onClose }: Props & { onClose: () => label={label} leading={ } diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.tsx index 6fb3b66281b1f..5d2a8fbed227e 100644 --- a/apps/web/src/components/settings/VoiceSettingsPanel.tsx +++ b/apps/web/src/components/settings/VoiceSettingsPanel.tsx @@ -9,6 +9,7 @@ import { updateEnvironmentSpeechModelUnloadTimeout, } from "@t3tools/client-runtime/voice-input"; import type { PreparedConnection } from "@t3tools/client-runtime/connection"; +import { openRouterIcon } from "@t3tools/client-runtime/provider-clients"; import { AuthSettingsWriteScope, CLOUD_SPEECH_LABELS, @@ -36,7 +37,6 @@ import { MonitorIcon, MicOffIcon, MicIcon, - WaypointsIcon, RefreshCwIcon, StarIcon, SlidersHorizontalIcon, @@ -69,6 +69,7 @@ import { import { ComposerSpeechRecordingPill } from "../chat/ComposerSpeechButton"; import { ProviderModelPicker } from "../chat/ProviderModelPicker"; import { ProviderInstanceIcon } from "../chat/ProviderInstanceIcon"; +import { ProviderPackageIcon } from "../chat/ProviderPackageIcon"; import { TraitsPicker } from "../chat/TraitsPicker"; import { VoiceInputSetup } from "../chat/VoiceInputSetup"; import { Badge } from "../ui/badge"; @@ -121,7 +122,8 @@ import { useScopedSettings, useUpdateScopedSettings } from "./useScopedSettings" const OFF = "off"; function CloudSpeechLogo({ provider }: { provider: CloudSpeechProvider }) { - if (provider === "openrouter") return ; + if (provider === "openrouter") + return ; return (