From f22a1e24a2fbabe2f4bd1d8e8dc2fc61842663af Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 20 Nov 2026 02:47:23 +0000 Subject: [PATCH] fix(web): keep a dictation while its destination reconnects The shared dictation session was keyed on the connection object, which the supervisor clears and replaces on every reconnect, so a brief drop disposed the session and discarded the recording. The session is now keyed on the chosen destination and sends on whatever connection it has when the recording stops; only a changed choice, a denied scope or a missing capability releases it. A live stream that drops no longer ends the recording: audio is buffered alongside the stream and uploaded when the recording stops. The composer no longer abandons a recording while the thread's environment is unavailable; that only blocks starting, and the transcript still lands. Fixes pingdotgg/t3code#37199 Co-authored-by: flipcyde101 Co-Authored-By: Claude Fable 5.2 --- apps/web/src/components/chat/ChatComposer.tsx | 11 ++- .../chat/GlobalVoiceInputIndicator.tsx | 3 +- .../VoiceSettingsPanel.browser.test.tsx | 18 +++- apps/web/src/speech/browserPcmCapture.ts | 7 +- .../speech/browserVoiceInput.browser.test.ts | 99 +++++++++++++++---- apps/web/src/speech/browserVoiceInput.ts | 62 +++++++----- ...useEnvironmentSpeechInput.browser.test.tsx | 89 ++++++++++++++--- .../src/speech/useEnvironmentSpeechInput.ts | 67 ++++++++----- apps/web/src/speech/voiceInputSession.ts | 31 +++--- apps/web/src/state/session.ts | 18 +++- docs/internals/voice-input.md | 11 ++- 11 files changed, 306 insertions(+), 110 deletions(-) diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index 603edc067dba8..f9a9ac3968cb4 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -1850,19 +1850,22 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) }; const readComposerSnapshotStable = useStableCallback(readComposerSnapshot); - const dictationDisabled = - !canOperateThread || - environmentUnavailable !== null || + // The draft outlives its environment reconnecting, so that only blocks starting a recording. + const dictationStartDisabled = !canOperateThread || environmentUnavailable !== null; + // Another surface has replaced the editor, so a transcript would have nowhere to land. + const dictationDisplaced = projectSelectionRequired || isComposerApprovalState || activePendingIsResponding || activePendingProgress?.activeQuestion?.allowCustomAnswer === false; + const dictationDisabled = dictationStartDisabled || dictationDisplaced; const speechInput = useEnvironmentSpeechInput({ environmentId, projectId: activeProjectId, projectName, ownerKey: questionAttachmentTarget ?? composerDraftTargetKey, - disabled: dictationDisabled, + disabled: dictationDisplaced, + startDisabled: dictationStartDisabled, label: activeThread?.title ?? null, presented: !hiddenThread, // A question's answer lives only in this composer, so only the thread draft outlives it. diff --git a/apps/web/src/components/chat/GlobalVoiceInputIndicator.tsx b/apps/web/src/components/chat/GlobalVoiceInputIndicator.tsx index dbe0453df2e9a..b756e1beab9ad 100644 --- a/apps/web/src/components/chat/GlobalVoiceInputIndicator.tsx +++ b/apps/web/src/components/chat/GlobalVoiceInputIndicator.tsx @@ -15,8 +15,7 @@ import { ComposerSpeechRecordingPill } from "./ComposerSpeechButton"; */ export function GlobalVoiceInputIndicator({ hidden = false }: { hidden?: boolean }) { const gt = useGT(); - const { prepared, grantId, cloudProvider, cloudModel } = useVoiceTranscriptionDestination(); - const session = useRetainedVoiceInputSession({ prepared, grantId, cloudProvider, cloudModel }); + const session = useRetainedVoiceInputSession(useVoiceTranscriptionDestination()); const { state, target, recordingStartedAt } = useVoiceInputSessionSnapshot(session); const presented = useVoiceOwnerPresented(target?.ownerKey ?? null); const { level } = useVoiceInputSessionActivity( diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx index 5c918f9430086..cb60238998f04 100644 --- a/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx +++ b/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx @@ -170,6 +170,7 @@ it.each([{ cpu: null, accelerated: 2.1 }, { cpu: 5, accelerated: null }, { accel label: prepared.label, grantId: "test", prepared, + usable: true, }), async () => { const previous = getClientSettings(); @@ -303,7 +304,13 @@ it("tests the selected model after download and preserves its transcript across const transcript = "Keep this transcript."; using _scope = vi.mocked(useEnvironmentScope).mockReturnValue(true); await vi.mocked(useVoiceTranscriptionDestination).withImplementation( - () => ({ environmentId: target.environmentId, label: target.label, grantId: "test", prepared }), + () => ({ + environmentId: target.environmentId, + label: target.label, + grantId: "test", + prepared, + usable: true, + }), async () => { await vi.mocked(useEnvironmentSpeechInput).withImplementation( (input) => ({ @@ -486,6 +493,7 @@ it.each([ label: target.label, grantId: "test", prepared, + usable: true, })); await using screen = await render( {}}> @@ -562,7 +570,13 @@ it("recovers from a failed download progress poll without notifying, but reports }); using notify = vi.spyOn(toastManager, "add").mockReturnValue("test"); await vi.mocked(useVoiceTranscriptionDestination).withImplementation( - () => ({ environmentId: target.environmentId, label: target.label, grantId: "test", prepared }), + () => ({ + environmentId: target.environmentId, + label: target.label, + grantId: "test", + prepared, + usable: true, + }), async () => { const panel = (key: string) => ( {}}> diff --git a/apps/web/src/speech/browserPcmCapture.ts b/apps/web/src/speech/browserPcmCapture.ts index 5060b9c39ce3a..ddfa07f6addb3 100644 --- a/apps/web/src/speech/browserPcmCapture.ts +++ b/apps/web/src/speech/browserPcmCapture.ts @@ -109,8 +109,9 @@ export function createBrowserPcmCapture( input.onLevel(data); return; } - if (feed) feed(data); - else chunks.push(data); + // Kept while streaming too, so a stream that drops can still upload the recording. + chunks.push(data); + feed?.(data); }; node.onprocessorerror = () => { stopped?.reject(new Error(m(msg("Microphone processing failed.")))); @@ -156,7 +157,7 @@ export function createBrowserPcmCapture( worklet.port.postMessage("stop"); try { await pending; - if (!feed && chunks.length) { + if (chunks.length) { const blob = new Blob(chunks, { type: "application/octet-stream" }); recordingUri = URL.createObjectURL(blob); } diff --git a/apps/web/src/speech/browserVoiceInput.browser.test.ts b/apps/web/src/speech/browserVoiceInput.browser.test.ts index 93e11ad6d5801..c8e3f4782376d 100644 --- a/apps/web/src/speech/browserVoiceInput.browser.test.ts +++ b/apps/web/src/speech/browserVoiceInput.browser.test.ts @@ -100,6 +100,29 @@ const status = { benchmarkTier: "cpu", } satisfies EnvironmentSpeechStatus; +function controllerFor( + platform: ReturnType, + commitDraft: (text: string) => void, +) { + return new VoiceInputController({ + recorder: platform.recorder, + retryFailedRecording: true, + getTranscriber: () => platform.transcriber, + requestPermission: async () => ({ granted: true, canAskAgain: true }), + configureRecording: async () => {}, + releaseRecording: async () => platform.cancelRecording(), + deleteRecording: platform.deleteRecording, + readDraft: () => ({ + ownerKey: "draft", + text: "", + selection: { start: 0, end: 0 }, + revision: 0, + }), + commitDraft, + onStateChange() {}, + }); +} + it("retains all PCM until stop, retries batch audio, and captures preferences before status resolves", async () => { using microphone = capture(); let streaming = false; @@ -125,7 +148,7 @@ it("retains all PCM until stop, retries batch audio, and captures preferences be }); const onError = vi.fn(); const platform = createBrowserVoiceInputPlatform({ - prepared, + getPrepared: () => prepared, hasAudioConsent: () => true, getStreamingEnabled: () => streaming, getTranscriptionOptions: () => options, @@ -136,23 +159,7 @@ it("retains all PCM until stop, retries batch audio, and captures preferences be onError, }); const commitDraft = vi.fn(); - const controller = new VoiceInputController({ - recorder: platform.recorder, - retryFailedRecording: true, - getTranscriber: () => platform.transcriber, - requestPermission: async () => ({ granted: true, canAskAgain: true }), - configureRecording: async () => {}, - releaseRecording: async () => platform.cancelRecording(), - deleteRecording: platform.deleteRecording, - readDraft: () => ({ - ownerKey: "draft", - text: "", - selection: { start: 0, end: 0 }, - revision: 0, - }), - commitDraft, - onStateChange() {}, - }); + const controller = controllerFor(platform, commitDraft); using _controller = { [Symbol.dispose]: () => controller.dispose() }; const start = controller.start(); await vi.waitFor(() => expect(_network).toHaveBeenCalled()); @@ -191,6 +198,60 @@ it("retains all PCM until stop, retries batch audio, and captures preferences be expect(platform.recorder.uri).toBeNull(); }); +it("keeps recording when the live stream drops and uploads the buffered audio on stop", async () => { + using microphone = capture(); + const sockets: EventTarget[] = []; + using _socket = stubGlobal( + "WebSocket", + class extends EventTarget { + constructor() { + super(); + sockets.push(this); + } + send() {} + close() {} + }, + ); + const uploads: number[][] = []; + const fetch = globalThis.fetch; + using _network = vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => { + const request = new Request(input, init); + if (request.url.endsWith("/api/voice/status")) return Response.json(status); + if (request.url.includes("/api/auth/websocket-ticket/")) + return Response.json({ ticket: "stream-ticket", expiresAt: "2030-01-01T00:00:00.000Z" }); + if (request.url.endsWith("/api/voice/transcribe")) { + const { pcm } = decodeSpeechPcmRequest(new Uint8Array(await request.arrayBuffer())); + uploads.push([...new Float32Array(pcm.slice().buffer)]); + return Response.json({ text: "bonjour" }); + } + return fetch(input, init); + }); + const onError = vi.fn(); + const platform = createBrowserVoiceInputPlatform({ + getPrepared: () => prepared, + hasAudioConsent: () => true, + getStreamingEnabled: () => true, + getTranscriptionOptions: () => DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, + getMicrophoneId: () => "", + onLevel() {}, + onDurationLimit() {}, + onText() {}, + onError, + }); + const commitDraft = vi.fn(); + const controller = controllerFor(platform, commitDraft); + using _controller = { [Symbol.dispose]: () => controller.dispose() }; + await controller.start(); + microphone.feed(new Float32Array([0.1])); + sockets[0]!.dispatchEvent(new Event("close")); + microphone.feed(new Float32Array([0.2])); + expect(controller.currentState.phase).toBe("recording"); + expect(onError).not.toHaveBeenCalled(); + await controller.stop(); + expect(uploads).toEqual([[Math.fround(0.1), Math.fround(0.2)]]); + expect(commitDraft).toHaveBeenCalledWith("bonjour", { start: 7, end: 7 }); +}); + it.each([ { native: true, enabled: true, buffered: true, live: true }, { native: true, enabled: false, buffered: true, live: false }, @@ -226,7 +287,7 @@ it.each([ const abort = new AbortController(); using _abort = { [Symbol.dispose]: () => abort.abort() }; const platform = createBrowserVoiceInputPlatform({ - prepared, + getPrepared: () => prepared, hasAudioConsent: () => true, getStreamingEnabled: () => enabled, getTranscriptionOptions: () => ({ diff --git a/apps/web/src/speech/browserVoiceInput.ts b/apps/web/src/speech/browserVoiceInput.ts index d52f49f23fe07..1b405ac15dcff 100644 --- a/apps/web/src/speech/browserVoiceInput.ts +++ b/apps/web/src/speech/browserVoiceInput.ts @@ -24,7 +24,8 @@ import { m } from "../i18n/messages"; const isVoiceTranscriptionError = Schema.is(VoiceTranscriptionError); export function createBrowserVoiceInputPlatform(input: { - readonly prepared: PreparedConnection; + /** The destination's connection right now; null while it reconnects. */ + readonly getPrepared: () => PreparedConnection | null; readonly cloudProvider?: CloudSpeechProvider | undefined; /** The exact OpenRouter model the grant names; required with that provider. */ readonly cloudModel?: CloudSpeechModelId | undefined; @@ -37,7 +38,16 @@ export function createBrowserVoiceInputPlatform(input: { readonly onText: (text: SpeechStreamText) => void; readonly onError: (message: string, code?: VoiceInputErrorCode) => void; }) { + const requirePrepared = () => { + const prepared = input.getPrepared(); + if (prepared) return prepared; + throw new VoiceTranscriptionError({ + code: "unavailable", + message: m(msg("Voice transcription failed. Check the destination in Settings → Voice.")), + }); + }; return createBrowserPcmCapture(input, async (options) => { + const prepared = requirePrepared(); const recordingOptions = input.getTranscriptionOptions(); const streamingEnabled = input.getStreamingEnabled(); const transcribeRecording = async ( @@ -47,22 +57,19 @@ export function createBrowserVoiceInputPlatform(input: { try { const response = await fetch(uri, { signal: transcriptionSignal }); const pcm = new Uint8Array(await response.arrayBuffer()); + // A recording can outlive the connection it started on, so it uploads on the current one. + const current = requirePrepared(); const result = await runEnvironmentSpeech( input.cloudProvider ? transcribeCloudPcm( - input.prepared, + current, input.cloudProvider, pcm, recordingOptions, input.hasAudioConsent, input.cloudModel, ) - : transcribeEnvironmentPcm( - input.prepared, - pcm, - recordingOptions, - input.hasAudioConsent, - ), + : transcribeEnvironmentPcm(current, pcm, recordingOptions, input.hasAudioConsent), { signal: transcriptionSignal }, ); throwIfVoiceTranscriptionAborted(transcriptionSignal); @@ -83,17 +90,14 @@ export function createBrowserVoiceInputPlatform(input: { code: "unavailable", message: m(msg("Choose an OpenRouter model in Settings → Voice.")), }); - await runEnvironmentSpeech( - requireCloudSpeechKey(input.prepared, input.cloudProvider), - options, - ); + await runEnvironmentSpeech(requireCloudSpeechKey(prepared, input.cloudProvider), options); throwIfVoiceTranscriptionAborted(options.signal); return { locale: effectiveSpeechLanguage(cloudSpeechLanguages, recordingOptions.speechLanguage), transcribe: transcribeRecording, }; } - const status = await runEnvironmentSpeech(getEnvironmentSpeechStatus(input.prepared), options); + const status = await runEnvironmentSpeech(getEnvironmentSpeechStatus(prepared), options); throwIfVoiceTranscriptionAborted(options.signal); if (!status.supported) throw new VoiceTranscriptionError({ code: "unavailable", message: status.reason }); @@ -114,18 +118,24 @@ export function createBrowserVoiceInputPlatform(input: { if (status.supportsStreaming ? !nativeStreaming : !status.supportsBufferedStreaming) { return { locale, transcribe: transcribeRecording }; } - const url = await runEnvironmentSpeech(getEnvironmentSpeechStreamUrl(input.prepared), options); - return { - locale, - transcribe: transcribeRecording, - streaming: openSpeechStream({ - url, - canSend: input.hasAudioConsent, - options: recordingOptions, - signal: options.signal, - onText: input.onText, - onError: (error) => input.onError(error.message), - }), - }; + const url = await runEnvironmentSpeech(getEnvironmentSpeechStreamUrl(prepared), options); + const transcription: { + readonly locale: string; + readonly transcribe: typeof transcribeRecording; + streaming?: ReturnType; + } = { locale, transcribe: transcribeRecording }; + transcription.streaming = openSpeechStream({ + url, + canSend: input.hasAudioConsent, + options: recordingOptions, + signal: options.signal, + onText: input.onText, + onError: (error) => { + // A lost stream keeps recording; stopping then uploads the buffered audio instead. + if (input.hasAudioConsent()) delete transcription.streaming; + else input.onError(error.message); + }, + }); + return transcription; }); } diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx index cc40350d3aa11..580eaa95aeedc 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx +++ b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx @@ -38,6 +38,7 @@ const mocks = vi.hoisted(() => ({ voiceCapable: new Set(), cloudVoiceCapable: new Set(), canDictate: new Set(), + denied: new Set(), prepared: new Map(), primaryEnvironmentId: "primary-env", statusRequests: [] as string[], @@ -67,9 +68,8 @@ const mocks = vi.hoisted(() => ({ vi.mock("../components/sidebar/mainAppLocation", () => ({ useCloseAppModal: () => () => Promise.resolve(), })); -vi.mock("../state/session", () => ({ - useEnvironmentScope: (id: string) => mocks.canDictate.has(id), - usePreparedConnection: (environmentId: string | null) => { +vi.mock("../state/session", () => { + const usePreparedConnection = (environmentId: string | null) => { if (!environmentId || !mocks.connected.has(environmentId)) return Option.none(); let prepared = mocks.prepared.get(environmentId); if (!prepared) { @@ -77,8 +77,14 @@ vi.mock("../state/session", () => ({ mocks.prepared.set(environmentId, prepared); } return Option.some(prepared); - }, -})); + }; + return { + useEnvironmentScope: (id: string) => mocks.canDictate.has(id), + useEnvironmentScopeDenied: (id: string) => mocks.denied.has(id), + usePreparedConnection, + readPreparedConnection: (id: string) => Option.getOrNull(usePreparedConnection(id)), + }; +}); vi.mock("../state/environments", () => ({ usePrimaryEnvironmentId: () => mocks.primaryEnvironmentId, useEnvironment: (environmentId: string | null) => @@ -179,23 +185,29 @@ vi.mock("./browserVoiceInput", () => ({ createBrowserVoiceInputPlatform: ( input: Parameters[0], ) => { - const environmentId = input.prepared.environmentId; - mocks.platforms.push(environmentId); + // Throws while the destination is disconnected, as the real platform's requests do. + const connected = () => { + const prepared = input.getPrepared(); + if (!prepared) throw new Error("The destination is disconnected."); + return prepared.environmentId; + }; return { recorder: { - uri: `blob:${environmentId}`, + uri: "blob:recording", prepareToRecordAsync: async () => { - mocks.microphoneRequests.push(environmentId); + mocks.microphoneRequests.push(connected()); }, record() {}, stop: async () => {}, }, transcriber: { prepare: async () => { + mocks.platforms.push(connected()); mocks.recordingOptions.push(input.getTranscriptionOptions()); return { locale: "en", transcribe: async () => { + const environmentId = connected(); mocks.transcribeCalls.push( input.cloudProvider ? `${[input.cloudProvider, (input as { cloudModel?: string }).cloudModel].filter(Boolean).join(":")} via ${environmentId}` @@ -226,6 +238,7 @@ function setup(options: { readonly destination?: string | null } = {}) { mocks.voiceCapable = new Set(["primary-env", THREAD_ENV, "env-a", "env-b"]); mocks.cloudVoiceCapable.clear(); mocks.canDictate = new Set(mocks.voiceCapable); + mocks.denied.clear(); mocks.prepared.clear(); mocks.statusRequests = []; mocks.statusUnavailable = false; @@ -264,6 +277,8 @@ function Probe(props: { readonly showWizard?: boolean; readonly ownerKey?: string; readonly persistent?: boolean; + readonly disabled?: boolean; + readonly startDisabled?: boolean; }) { const value = useEnvironmentSpeechInput({ environmentId: THREAD_ENV as EnvironmentId, @@ -271,6 +286,8 @@ function Probe(props: { projectName: props.projectName, cleanup: props.cleanup, ownerKey: props.ownerKey ?? "draft", + disabled: props.disabled, + startDisabled: props.startDisabled, persistentDraft: props.persistent ? { readStoredText: () => mocks.storedDraft, @@ -370,23 +387,65 @@ it.each([ ); it.each([ - ["revoked", null], - ["moved to another environment", "env-b"], - ["renewed for the same environment", "env-a"], -])("abandons an in-progress recording when consent is %s", async (_, next) => { + ["consent is revoked", () => setDestination(null)], + ["consent is moved to another environment", () => setDestination("env-b")], + ["consent is renewed for the same environment", () => setDestination("env-a")], + ["the environment denies dictation", () => mocks.denied.add("env-a")], + ["the environment stops supporting transcription", () => mocks.voiceCapable.delete("env-a")], +])("abandons an in-progress recording when %s", async (name, change) => { setup({ destination: "env-a" }); await using screen = await render(); await act(() => mocks.voice.start()); expect(mocks.voice.state.phase).toBe("recording"); - setDestination(next); + change(); await screen.rerender(); - if (next === null) expect(mocks.voice.available).toBe(false); + if (name === "consent is revoked") expect(mocks.voice.available).toBe(false); expect(mocks.voice.state.phase).toBe("idle"); await act(() => mocks.voice.stop()); expect(mocks.transcribeCalls).toEqual([]); expect(mocks.committed).not.toHaveBeenCalled(); }); +it("keeps a recording while its destination reconnects and sends it on the new connection", async () => { + setup({ destination: "env-a" }); + await using screen = await render(); + await act(() => mocks.voice.start()); + mocks.connected.delete("env-a"); + await screen.rerender(); + expect(mocks.voice.state.phase).toBe("recording"); + expect(mocks.voice.available).toBe(true); + // Stopped while still disconnected: the recording waits for a retry instead of being lost. + await act(() => mocks.voice.stop()); + expect(mocks.voice.state).toMatchObject({ phase: "error", errorAction: "retry" }); + expect(mocks.committed).not.toHaveBeenCalled(); + + // A reconnect hands out a new connection object. + mocks.prepared.delete("env-a"); + mocks.connected.add("env-a"); + await screen.rerender(); + await act(() => mocks.voice.start()); + expect(mocks.transcribeCalls).toEqual(["env-a"]); + expect(mocks.microphoneRequests).toEqual(["env-a"]); + expect(mocks.committed).toHaveBeenCalledExactlyOnceWith("dictated", expect.anything()); +}); + +it.each([ + ["cannot start right now, the transcript still lands", { startDisabled: true }, true], + ["can no longer take a transcript, the recording is abandoned", { disabled: true }, false], +])("when the editor %s", async (_name, block, kept) => { + setup({ destination: "env-a" }); + await using screen = await render(); + await act(() => mocks.voice.start()); + await screen.rerender(); + expect(mocks.voice.state.phase).toBe(kept ? "recording" : "idle"); + await act(() => mocks.voice.stop()); + expect(mocks.transcribeCalls).toEqual(kept ? ["env-a"] : []); + expect(mocks.committed).toHaveBeenCalledTimes(kept ? 1 : 0); + // Neither block lets a new recording start. + await act(() => mocks.voice.start()); + expect(mocks.voice.state.phase).toBe("idle"); +}); + it("retries a failed transcription on the same chosen environment", async () => { setup({ destination: "env-a" }); mocks.transcripts = [new Error("inference failed"), "second try"]; diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.ts b/apps/web/src/speech/useEnvironmentSpeechInput.ts index b3eb968096c99..5effa2803562f 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.ts +++ b/apps/web/src/speech/useEnvironmentSpeechInput.ts @@ -33,7 +33,11 @@ import { useEnvironmentSettings, } from "../hooks/useSettings"; import { useEnvironment } from "../state/environments"; -import { useEnvironmentScope, usePreparedConnection } from "../state/session"; +import { + useEnvironmentScope, + useEnvironmentScopeDenied, + usePreparedConnection, +} from "../state/session"; import { preferredSpeechLanguages } from "./preferredSpeechLanguages"; import { runEnvironmentSpeech } from "./runEnvironmentSpeech"; import { useAudioChoice } from "./voiceConsent"; @@ -82,7 +86,13 @@ type HookInput = { readonly projectId?: ProjectId | null | undefined; readonly projectName?: string | null | undefined; readonly ownerKey: string; + /** The editor cannot take a transcript: blocks starting and abandons a running recording. */ readonly disabled?: boolean | undefined; + /** + * Blocks starting only. A running recording continues and its transcript still lands, for a + * block the draft outlives, such as its environment reconnecting. + */ + readonly startDisabled?: boolean | undefined; /** * Until a destination is chosen, voice input is offered and its first gesture opens Voice * settings. False hides it instead, for editors whose draft would not survive the navigation. @@ -115,7 +125,8 @@ type HookInput = { /** * Microphone audio leaves this device only for the environment the person chose in * Voice settings. Nothing is inferred from the primary or the draft's environment, and - * the recording is abandoned the moment that choice changes or its connection drops. + * the recording is abandoned the moment that choice changes or the environment refuses it. + * A dropped connection is neither: the recording stays on this device until it can be sent. */ export function useVoiceTranscriptionDestination() { const gt = useGT(); @@ -126,13 +137,13 @@ export function useVoiceTranscriptionDestination() { const environment = useEnvironment(environmentId); const connection = Option.getOrNull(usePreparedConnection(environmentId)); const canDictate = useEnvironmentScope(environmentId, AuthOrchestrationOperateScope); + const denied = useEnvironmentScopeDenied(environmentId, AuthOrchestrationOperateScope); const cloud = grant?.destination.kind === "cloud" ? grant.destination : undefined; const provider = cloud?.provider; const model = cloud?.provider === "openrouter" ? cloud.model : undefined; + const capabilities = environment?.serverConfig?.environment.capabilities; const supported = - environment?.serverConfig?.environment.capabilities[ - provider ? "cloudVoiceTranscription" : "voiceTranscription" - ] === true; + capabilities?.[provider ? "cloudVoiceTranscription" : "voiceTranscription"] === true; return { ...(hydrated && choice === null ? { offerFirstRunSetup: true } : {}), environmentId, @@ -147,16 +158,19 @@ export function useVoiceTranscriptionDestination() { }) : (environment?.label ?? null), prepared: supported && canDictate ? connection : null, + // Only the environment saying no ends a recording; an unreachable one has not answered. + usable: environmentId !== null && !denied && (supported || capabilities === undefined), }; } +const startBlocked = (input: HookInput) => Boolean(input.disabled || input.startDisabled); + export function useEnvironmentSpeechInput(input: HookInput) { const navigate = useNavigate(); const destination = useVoiceTranscriptionDestination(); const prepared = destination.prepared; const grantId = destination.grantId; const cloudProvider = destination.cloudProvider; - const cloudModel = destination.cloudModel; const draftCapabilities = useEnvironment(input.environmentId)?.serverConfig?.environment .capabilities; const originatingConnection = Option.getOrNull(usePreparedConnection(input.environmentId)); @@ -179,22 +193,20 @@ export function useEnvironmentSpeechInput(input: HookInput) { const [queuedStart, setQueuedStart] = useState(null); const [setup, setSetup] = useState(INITIAL_SETUP); const [previousDestination, setPreviousDestination] = useState({ prepared, grantId }); - const [previousOwner, setPreviousOwner] = useState({ - key: input.ownerKey, - disabled: input.disabled, - }); + const blocked = startBlocked(input); + const [previousOwner, setPreviousOwner] = useState({ key: input.ownerKey, blocked }); if (previousDestination.prepared !== prepared || previousDestination.grantId !== grantId) { setPreviousDestination({ prepared, grantId }); setSetup(INITIAL_SETUP); setQueuedStart(null); } - if (previousOwner.key !== input.ownerKey || previousOwner.disabled !== input.disabled) { - setPreviousOwner({ key: input.ownerKey, disabled: input.disabled }); + if (previousOwner.key !== input.ownerKey || previousOwner.blocked !== blocked) { + setPreviousOwner({ key: input.ownerKey, blocked }); setQueuedStart(null); setSetup(INITIAL_SETUP); } - const session = useRetainedVoiceInputSession({ prepared, grantId, cloudProvider, cloudModel }); + const session = useRetainedVoiceInputSession(destination); const snapshot = useVoiceInputSessionSnapshot(session); const owns = snapshot.target?.ownerKey === input.ownerKey; const activity = useVoiceInputSessionActivity(session, input.ownerKey); @@ -225,6 +237,13 @@ export function useEnvironmentSpeechInput(input: HookInput) { session?.ownerChanged(latestRef.current.input.ownerKey); }; }, [session]); + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- a start or setup step awaiting one connection must not finish on its replacement + useEffect(() => { + if (!prepared) return; + return () => { + startRequestRef.current += 1; + }; + }, [prepared]); // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- tells the global indicator this draft is on screen useEffect( () => (input.presented === false ? undefined : presentVoiceOwner(input.ownerKey)), @@ -296,12 +315,12 @@ export function useEnvironmentSpeechInput(input: HookInput) { if (state.phase === "error") pendingSubmissionRef.current = null; if (state.phase !== "idle" || !pending) return; pendingSubmissionRef.current = null; - if (!input.disabled && pending.ownerKey === input.ownerKey && pending.text !== undefined) + if (!blocked && pending.ownerKey === input.ownerKey && pending.text !== undefined) pending.submit(pending.text); - }, [state, input.ownerKey, input.disabled]); + }, [state, input.ownerKey, blocked]); const submitAfterDictation = useStableCallback((submit: (text: string) => void) => { - if (latestRef.current.input.disabled || state.phase === "preparing") return; + if (startBlocked(latestRef.current.input) || state.phase === "preparing") return; if (!session || !voiceInputBlocksSubmission(state)) { submit(latestRef.current.input.readDraft().text); return; @@ -395,7 +414,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { ); const start = useStableCallback(async () => { - if (latestRef.current.input.disabled || (setup.open && setup.downloading)) return; + if (startBlocked(latestRef.current.input) || (setup.open && setup.downloading)) return; if (!grantId) { if (destination.offerFirstRunSetup && latestRef.current.input.offerSetup !== false) await navigate({ to: "/settings/voice", hash: "voice-transcription-destination" }); @@ -412,7 +431,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { request === startRequestRef.current && latestRef.current.session === session && latestRef.current.input.ownerKey === owner && - !latestRef.current.input.disabled; + !startBlocked(latestRef.current.input); setSetup(INITIAL_SETUP); setQueuedStart(prepared); let latestStatus: EnvironmentSpeechStatus; @@ -519,12 +538,14 @@ export function useEnvironmentSpeechInput(input: HookInput) { (cloudProvider !== undefined || (status?.prepared === prepared && status.value?.supported !== false)); return { + // A recording this editor owns keeps its controls while the destination reconnects. available: - destinationReady && - !busyElsewhere && - typeof navigator !== "undefined" && - Boolean(navigator.mediaDevices?.getUserMedia) && - typeof AudioWorkletNode !== "undefined", + (owns && snapshot.state.phase !== "idle") || + (destinationReady && + !busyElsewhere && + typeof navigator !== "undefined" && + Boolean(navigator.mediaDevices?.getUserMedia) && + typeof AudioWorkletNode !== "undefined"), status: currentStatus, transcriptionEnvironmentId: destination.environmentId, transcriptionEnvironmentLabel: destination.label, diff --git a/apps/web/src/speech/voiceInputSession.ts b/apps/web/src/speech/voiceInputSession.ts index 40e6450bd2ce1..8127a02d5434c 100644 --- a/apps/web/src/speech/voiceInputSession.ts +++ b/apps/web/src/speech/voiceInputSession.ts @@ -24,6 +24,7 @@ import { useEffect, useSyncExternalStore } from "react"; import { toastManager } from "../components/ui/toast"; import { chooseRecordingMicrophone } from "./browserMicrophones"; import { createBrowserVoiceInputPlatform } from "./browserVoiceInput"; +import { readPreparedConnection } from "../state/session"; import { runEnvironmentSpeech } from "./runEnvironmentSpeech"; import { bindAudioGrant, readCleanupGrant } from "./voiceConsent"; import { msg } from "gt-react"; @@ -71,7 +72,7 @@ export type VoiceInputSessionActivity = { }; type SessionKey = { - readonly prepared: PreparedConnection; + readonly environmentId: EnvironmentId; readonly grantId: string; readonly cloudProvider: CloudSpeechProvider | undefined; readonly cloudModel: CloudSpeechModelId | undefined; @@ -93,7 +94,8 @@ function cleanupContext({ text, selection }: DraftInput): DraftInput { /** * The browser has one microphone, so every editor shares one dictation per destination. The - * recording belongs to the target that started it, not to whichever editor is mounted now. + * recording belongs to the target that started it, not to whichever editor is mounted now, and + * to the destination, not to one connection: it outlives a reconnect and sends on the new one. */ export class VoiceInputSession { readonly key: SessionKey; @@ -120,7 +122,7 @@ export class VoiceInputSession { } | undefined; const platform = createBrowserVoiceInputPlatform({ - prepared: key.prepared, + getPrepared: () => readPreparedConnection(key.environmentId), cloudProvider: key.cloudProvider, cloudModel: key.cloudModel, hasAudioConsent, @@ -314,7 +316,7 @@ const registryListeners = new Set<() => void>(); function sameKey(session: VoiceInputSession, key: SessionKey): boolean { return ( - session.key.prepared === key.prepared && + session.key.environmentId === key.environmentId && session.key.grantId === key.grantId && session.key.cloudProvider === key.cloudProvider && session.key.cloudModel === key.cloudModel @@ -351,21 +353,28 @@ function subscribeRegistry(listener: () => void): () => void { return () => registryListeners.delete(listener); } -/** Holds the destination's shared session while mounted; null until it is retained. */ +/** + * Holds the destination's shared session while mounted and `usable`; null until it is retained. + * The connection is not part of it, so the session and its recording survive a reconnect. + */ export function useRetainedVoiceInputSession(input: { - readonly prepared: PreparedConnection | null; + readonly environmentId: EnvironmentId | null; readonly grantId: string | null; + readonly usable: boolean; readonly cloudProvider?: CloudSpeechProvider | undefined; readonly cloudModel?: CloudSpeechModelId | undefined; }): VoiceInputSession | null { - const { prepared, grantId, cloudProvider, cloudModel } = input; + const { grantId, cloudProvider, cloudModel } = input; + const environmentId = input.usable ? input.environmentId : null; // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- retains the shared recording session for this destination while mounted useEffect(() => { - if (!prepared || !grantId || typeof navigator === "undefined") return; - return retainVoiceInputSession({ prepared, grantId, cloudProvider, cloudModel }); - }, [prepared, grantId, cloudProvider, cloudModel]); + if (!environmentId || !grantId || typeof navigator === "undefined") return; + return retainVoiceInputSession({ environmentId, grantId, cloudProvider, cloudModel }); + }, [environmentId, grantId, cloudProvider, cloudModel]); return useSyncExternalStore(subscribeRegistry, () => - prepared && grantId ? findSession({ prepared, grantId, cloudProvider, cloudModel }) : null, + environmentId && grantId + ? findSession({ environmentId, grantId, cloudProvider, cloudModel }) + : null, ); } diff --git a/apps/web/src/state/session.ts b/apps/web/src/state/session.ts index f53bb3dfa9684..f2c7411cdf9af 100644 --- a/apps/web/src/state/session.ts +++ b/apps/web/src/state/session.ts @@ -1,5 +1,8 @@ import { useAtomValue } from "@effect/atom-react"; -import { createEnvironmentSessionAtoms } from "@t3tools/client-runtime/state/session"; +import { + createEnvironmentSessionAtoms, + sessionDeniesScope, +} from "@t3tools/client-runtime/state/session"; import { type AuthEnvironmentScope, type AuthSessionState, @@ -35,6 +38,19 @@ export function useEnvironmentScope( ); } +/** Unlike `!useEnvironmentScope`, stays false until the host has answered without the scope. */ +export function useEnvironmentScopeDenied( + environmentId: EnvironmentId | null, + scope: AuthEnvironmentScope, +): boolean { + return useAtomValue( + environmentId === null + ? EMPTY_SESSION_STATE_ATOM + : environmentSession.sessionStateAtom(environmentId), + (result) => sessionDeniesScope(result, scope), + ); +} + export function useEnvironmentCanConfigureRelay(environmentId: EnvironmentId | null): boolean { return useAtomValue( environmentId === null diff --git a/docs/internals/voice-input.md b/docs/internals/voice-input.md index 00964ab4ae332..ab5c79a8fcc65 100644 --- a/docs/internals/voice-input.md +++ b/docs/internals/voice-input.md @@ -25,11 +25,14 @@ The [web grant](../../apps/web/src/speech/voiceConsent.ts) lives outside the pre window saving unrelated settings cannot resurrect a revoked grant. Each new choice gets a distinct grant, even if the destination was previously selected. The [web hook](../../apps/web/src/speech/useEnvironmentSpeechInput.ts) binds the -controller to that grant and the selected environment's connection. It requires +controller to that grant and the selected environment, not to one connection. It requires `voiceTranscription` to use the environment's model or `cloudVoiceTranscription` -to forward audio to the selected provider. Changing the grant or losing the -connection, permission, or required capability discards both the active recording -and any recording retained for retry. It never chooses a fallback destination. +to forward audio to the selected provider. Changing the grant, or the environment +denying the permission or reporting the capability missing, discards both the active +recording and any recording retained for retry. A dropped connection is not a refusal: +the recording stays on the device and is sent on the environment's next connection, +and a live stream that drops falls back to uploading the buffered audio. +It never chooses a fallback destination. The settings page's transcription test uses the same hook. The microphone loopback test captures separately for local playback and sends nothing. The transport refuses redirects so credentials and audio cannot follow one to another host.