From 9463a3d28eee40d1eeda36c14609d280c79d8911 Mon Sep 17 00:00:00 2001 From: Flipcyde Date: Thu, 19 Nov 2026 18:34:13 -0700 Subject: [PATCH 1/7] fix(web): retain desktop dictation during reconnect --- .../chat/GlobalVoiceInputIndicator.tsx | 12 +- .../VoiceSettingsPanel.browser.test.tsx | 18 ++- .../speech/browserVoiceInput.browser.test.ts | 115 +++++++++++++++++ apps/web/src/speech/browserVoiceInput.ts | 38 ++++-- ...useEnvironmentSpeechInput.browser.test.tsx | 70 ++++++++++ .../src/speech/useEnvironmentSpeechInput.ts | 25 +++- apps/web/src/speech/voiceInputSession.ts | 121 ++++++++++++++---- apps/web/src/state/session.ts | 20 +++ 8 files changed, 379 insertions(+), 40 deletions(-) diff --git a/apps/web/src/components/chat/GlobalVoiceInputIndicator.tsx b/apps/web/src/components/chat/GlobalVoiceInputIndicator.tsx index dbe0453df2e9a..53dedde9bb5dd 100644 --- a/apps/web/src/components/chat/GlobalVoiceInputIndicator.tsx +++ b/apps/web/src/components/chat/GlobalVoiceInputIndicator.tsx @@ -15,8 +15,16 @@ import { ComposerSpeechRecordingPill } from "./ComposerSpeechButton"; */ export function GlobalVoiceInputIndicator({ hidden = false }: { hidden?: boolean }) { const gt = useGT(); - const { prepared, grantId, cloudProvider, cloudModel } = useVoiceTranscriptionDestination(); - const session = useRetainedVoiceInputSession({ prepared, grantId, cloudProvider, cloudModel }); + const { environmentId, prepared, grantId, retainOnDisconnect, cloudProvider, cloudModel } = + useVoiceTranscriptionDestination(); + const session = useRetainedVoiceInputSession({ + environmentId, + prepared, + grantId, + retainOnDisconnect, + cloudProvider, + cloudModel, + }); const { state, target, recordingStartedAt } = useVoiceInputSessionSnapshot(session); const presented = useVoiceOwnerPresented(target?.ownerKey ?? null); const { level } = useVoiceInputSessionActivity( diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx index 5c918f9430086..c51952d380363 100644 --- a/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx +++ b/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx @@ -170,6 +170,7 @@ it.each([{ cpu: null, accelerated: 2.1 }, { cpu: 5, accelerated: null }, { accel label: prepared.label, grantId: "test", prepared, + retainOnDisconnect: true, }), async () => { const previous = getClientSettings(); @@ -303,7 +304,13 @@ it("tests the selected model after download and preserves its transcript across const transcript = "Keep this transcript."; using _scope = vi.mocked(useEnvironmentScope).mockReturnValue(true); await vi.mocked(useVoiceTranscriptionDestination).withImplementation( - () => ({ environmentId: target.environmentId, label: target.label, grantId: "test", prepared }), + () => ({ + environmentId: target.environmentId, + label: target.label, + grantId: "test", + prepared, + retainOnDisconnect: true, + }), async () => { await vi.mocked(useEnvironmentSpeechInput).withImplementation( (input) => ({ @@ -486,6 +493,7 @@ it.each([ label: target.label, grantId: "test", prepared, + retainOnDisconnect: true, })); await using screen = await render( {}}> @@ -562,7 +570,13 @@ it("recovers from a failed download progress poll without notifying, but reports }); using notify = vi.spyOn(toastManager, "add").mockReturnValue("test"); await vi.mocked(useVoiceTranscriptionDestination).withImplementation( - () => ({ environmentId: target.environmentId, label: target.label, grantId: "test", prepared }), + () => ({ + environmentId: target.environmentId, + label: target.label, + grantId: "test", + prepared, + retainOnDisconnect: true, + }), async () => { const panel = (key: string) => ( {}}> diff --git a/apps/web/src/speech/browserVoiceInput.browser.test.ts b/apps/web/src/speech/browserVoiceInput.browser.test.ts index 93e11ad6d5801..a6fee1f03e0e8 100644 --- a/apps/web/src/speech/browserVoiceInput.browser.test.ts +++ b/apps/web/src/speech/browserVoiceInput.browser.test.ts @@ -248,3 +248,118 @@ it.each([ ); }, ); + +it.each(["status", "stream ticket"] as const)( + "does not open a stream after authorization is lost during %s preparation", + async (boundary) => { + const sockets: string[] = []; + using _socket = stubGlobal( + "WebSocket", + class extends EventTarget { + constructor(url: string) { + super(); + sockets.push(url); + } + close() {} + }, + ); + let currentPrepared: typeof prepared | null = prepared; + let resolveStatus!: (response: Response) => void; + let resolveTicket!: (response: Response) => void; + const pendingStatus = new Promise((resolve) => { + resolveStatus = resolve; + }); + const pendingTicket = new Promise((resolve) => { + resolveTicket = resolve; + }); + using _network = vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => { + const request = new Request(input, init); + if (request.url.endsWith("/api/voice/status")) return (await pendingStatus).clone(); + if (request.url.includes("/api/auth/websocket-ticket/")) return (await pendingTicket).clone(); + throw new Error(`Unexpected request: ${request.url}`); + }); + const abort = new AbortController(); + using _abort = { [Symbol.dispose]: () => abort.abort() }; + const platform = createBrowserVoiceInputPlatform({ + prepared, + getPrepared: () => currentPrepared, + hasAudioConsent: () => true, + getStreamingEnabled: () => true, + getTranscriptionOptions: () => DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, + getMicrophoneId: () => "", + onLevel() {}, + onDurationLimit() {}, + onText() {}, + onError() {}, + }); + const preparation = platform.transcriber.prepare({ signal: abort.signal }); + await vi.waitFor(() => expect(_network).toHaveBeenCalled()); + if (boundary === "status") { + currentPrepared = null; + resolveStatus(Response.json(status)); + } else { + resolveStatus(Response.json(status)); + await vi.waitFor(() => + expect( + _network.mock.calls.some(([input]) => String(input).includes("websocket-ticket")), + ).toBe(true), + ); + currentPrepared = null; + resolveTicket( + Response.json({ ticket: "stale-ticket", expiresAt: "2030-01-01T00:00:00.000Z" }), + ); + } + await expect(preparation).rejects.toMatchObject({ code: "unavailable" }); + expect(sockets).toEqual([]); + }, +); + +it("refreshes the stream ticket for a same-destination reconnect between awaits", async () => { + const sockets: string[] = []; + using _socket = stubGlobal( + "WebSocket", + class extends EventTarget { + constructor(url: string) { + super(); + sockets.push(url); + } + close() {} + }, + ); + const reconnected = { ...prepared, httpBaseUrl: "https://reconnected.example.test" }; + let currentPrepared: typeof prepared = prepared; + let resolveStatus!: (response: Response) => void; + const pendingStatus = new Promise((resolve) => { + resolveStatus = resolve; + }); + using _network = vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => { + const request = new Request(input, init); + if (request.url === `${prepared.httpBaseUrl}/api/voice/status`) + return (await pendingStatus).clone(); + if (request.url === `${reconnected.httpBaseUrl}/api/voice/status`) return Response.json(status); + if (request.url.includes("/api/auth/websocket-ticket/")) + return Response.json({ ticket: "reconnected-ticket", expiresAt: "2030-01-01T00:00:00.000Z" }); + throw new Error(`Unexpected request: ${request.url}`); + }); + const abort = new AbortController(); + using _abort = { [Symbol.dispose]: () => abort.abort() }; + const platform = createBrowserVoiceInputPlatform({ + prepared, + getPrepared: () => currentPrepared, + hasAudioConsent: () => true, + getStreamingEnabled: () => true, + getTranscriptionOptions: () => DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, + getMicrophoneId: () => "", + onLevel() {}, + onDurationLimit() {}, + onText() {}, + onError() {}, + }); + const preparation = platform.transcriber.prepare({ signal: abort.signal }); + await vi.waitFor(() => expect(_network).toHaveBeenCalled()); + currentPrepared = reconnected; + resolveStatus(Response.json(status)); + const transcription = await preparation; + expect(Boolean(transcription.streaming)).toBe(true); + expect(sockets).toEqual(["wss://reconnected.example.test/ws/voice?wsTicket=reconnected-ticket"]); +}); diff --git a/apps/web/src/speech/browserVoiceInput.ts b/apps/web/src/speech/browserVoiceInput.ts index d52f49f23fe07..92de200bc0962 100644 --- a/apps/web/src/speech/browserVoiceInput.ts +++ b/apps/web/src/speech/browserVoiceInput.ts @@ -25,6 +25,7 @@ const isVoiceTranscriptionError = Schema.is(VoiceTranscriptionError); export function createBrowserVoiceInputPlatform(input: { readonly prepared: PreparedConnection; + readonly getPrepared?: (() => PreparedConnection | null) | undefined; readonly cloudProvider?: CloudSpeechProvider | undefined; /** The exact OpenRouter model the grant names; required with that provider. */ readonly cloudModel?: CloudSpeechModelId | undefined; @@ -37,6 +38,15 @@ export function createBrowserVoiceInputPlatform(input: { readonly onText: (text: SpeechStreamText) => void; readonly onError: (message: string, code?: VoiceInputErrorCode) => void; }) { + const getPrepared = input.getPrepared ?? (() => input.prepared); + const requirePrepared = () => { + const prepared = getPrepared(); + if (prepared) return prepared; + throw new VoiceTranscriptionError({ + code: "unavailable", + message: m(msg("Voice transcription failed. Check the destination in Settings → Voice.")), + }); + }; return createBrowserPcmCapture(input, async (options) => { const recordingOptions = input.getTranscriptionOptions(); const streamingEnabled = input.getStreamingEnabled(); @@ -47,22 +57,18 @@ export function createBrowserVoiceInputPlatform(input: { try { const response = await fetch(uri, { signal: transcriptionSignal }); const pcm = new Uint8Array(await response.arrayBuffer()); + const prepared = requirePrepared(); const result = await runEnvironmentSpeech( input.cloudProvider ? transcribeCloudPcm( - input.prepared, + prepared, input.cloudProvider, pcm, recordingOptions, input.hasAudioConsent, input.cloudModel, ) - : transcribeEnvironmentPcm( - input.prepared, - pcm, - recordingOptions, - input.hasAudioConsent, - ), + : transcribeEnvironmentPcm(prepared, pcm, recordingOptions, input.hasAudioConsent), { signal: transcriptionSignal }, ); throwIfVoiceTranscriptionAborted(transcriptionSignal); @@ -84,7 +90,7 @@ export function createBrowserVoiceInputPlatform(input: { message: m(msg("Choose an OpenRouter model in Settings → Voice.")), }); await runEnvironmentSpeech( - requireCloudSpeechKey(input.prepared, input.cloudProvider), + requireCloudSpeechKey(requirePrepared(), input.cloudProvider), options, ); throwIfVoiceTranscriptionAborted(options.signal); @@ -93,7 +99,8 @@ export function createBrowserVoiceInputPlatform(input: { transcribe: transcribeRecording, }; } - const status = await runEnvironmentSpeech(getEnvironmentSpeechStatus(input.prepared), options); + const prepared = requirePrepared(); + const status = await runEnvironmentSpeech(getEnvironmentSpeechStatus(prepared), options); throwIfVoiceTranscriptionAborted(options.signal); if (!status.supported) throw new VoiceTranscriptionError({ code: "unavailable", message: status.reason }); @@ -114,7 +121,18 @@ export function createBrowserVoiceInputPlatform(input: { if (status.supportsStreaming ? !nativeStreaming : !status.supportsBufferedStreaming) { return { locale, transcribe: transcribeRecording }; } - const url = await runEnvironmentSpeech(getEnvironmentSpeechStreamUrl(input.prepared), options); + let streamPrepared = requirePrepared(); + let url = await runEnvironmentSpeech(getEnvironmentSpeechStreamUrl(streamPrepared), options); + const currentPrepared = requirePrepared(); + if (currentPrepared !== streamPrepared) { + streamPrepared = currentPrepared; + url = await runEnvironmentSpeech(getEnvironmentSpeechStreamUrl(streamPrepared), options); + if (requirePrepared() !== streamPrepared) + throw new VoiceTranscriptionError({ + code: "unavailable", + message: m(msg("Voice transcription failed. Check the destination in Settings → Voice.")), + }); + } return { locale, transcribe: transcribeRecording, diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx index cc40350d3aa11..0b3c412968144 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx +++ b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx @@ -69,6 +69,7 @@ vi.mock("../components/sidebar/mainAppLocation", () => ({ })); vi.mock("../state/session", () => ({ useEnvironmentScope: (id: string) => mocks.canDictate.has(id), + useEnvironmentScopeDenied: () => false, usePreparedConnection: (environmentId: string | null) => { if (!environmentId || !mocks.connected.has(environmentId)) return Option.none(); let prepared = mocks.prepared.get(environmentId); @@ -180,6 +181,7 @@ vi.mock("./browserVoiceInput", () => ({ input: Parameters[0], ) => { const environmentId = input.prepared.environmentId; + const getPrepared = input.getPrepared ?? (() => input.prepared); mocks.platforms.push(environmentId); return { recorder: { @@ -196,6 +198,7 @@ vi.mock("./browserVoiceInput", () => ({ return { locale: "en", transcribe: async () => { + if (!getPrepared()) throw new Error("destination disconnected"); mocks.transcribeCalls.push( input.cloudProvider ? `${[input.cloudProvider, (input as { cloudModel?: string }).cloudModel].filter(Boolean).join(":")} via ${environmentId}` @@ -400,6 +403,73 @@ it("retries a failed transcription on the same chosen environment", async () => expect(mocks.committed).toHaveBeenCalledExactlyOnceWith("second try", expect.anything()); }); +it("retains a recording through a transient destination disconnect and reconnect", async () => { + setup({ destination: "env-a" }); + await using screen = await render(); + await act(() => mocks.voice.start()); + expect(mocks.voice.state.phase).toBe("recording"); + + mocks.connected.delete("env-a"); + mocks.prepared.delete("env-a"); + await screen.rerender(); + expect(mocks.voice.state.phase).toBe("recording"); + + mocks.connected.add("env-a"); + await screen.rerender(); + await act(() => mocks.voice.stop()); + expect(mocks.transcribeCalls).toEqual(["env-a"]); + expect(mocks.committed).toHaveBeenCalledExactlyOnceWith("dictated", expect.anything()); +}); + +it("retains an offline-first destination holder until the same destination reconnects", async () => { + setup({ destination: "env-a" }); + mocks.connected.delete("env-a"); + mocks.prepared.delete("env-a"); + await using screen = await render(); + expect(mocks.voice.state.phase).toBe("idle"); + expectNoAudioSent(); + + mocks.connected.add("env-a"); + await screen.rerender(); + await act(() => mocks.voice.start()); + expect(mocks.voice.state.phase).toBe("recording"); + await act(() => mocks.voice.stop()); + expect(mocks.transcribeCalls).toEqual(["env-a"]); +}); + +it("keeps a stopped recording local while disconnected and retries it on reconnect", async () => { + setup({ destination: "env-a" }); + await using screen = await render(); + await act(() => mocks.voice.start()); + mocks.connected.delete("env-a"); + mocks.prepared.delete("env-a"); + await screen.rerender(); + await act(() => mocks.voice.stop()); + expect(mocks.transcribeCalls).toEqual([]); + expect(mocks.voice.state).toMatchObject({ phase: "error", errorAction: "retry" }); + + mocks.connected.add("env-a"); + await screen.rerender(); + await act(() => mocks.voice.start()); + expect(mocks.transcribeCalls).toEqual(["env-a"]); + expect(mocks.committed).toHaveBeenCalledExactlyOnceWith("dictated", expect.anything()); +}); + +it.each(["authorization", "capability"] as const)( + "abandons a recording when connected destination %s is lost", + async (loss) => { + setup({ destination: "env-a" }); + await using screen = await render(); + await act(() => mocks.voice.start()); + if (loss === "authorization") mocks.canDictate.delete("env-a"); + else mocks.voiceCapable.delete("env-a"); + await screen.rerender(); + expect(mocks.voice.state.phase).toBe("idle"); + await act(() => mocks.voice.stop()); + expect(mocks.transcribeCalls).toEqual([]); + }, +); + it.each([ ["a stored thread draft keeps recording into it", true], ["an editor-only draft is abandoned", false], diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.ts b/apps/web/src/speech/useEnvironmentSpeechInput.ts index b3eb968096c99..69bf36694b187 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.ts +++ b/apps/web/src/speech/useEnvironmentSpeechInput.ts @@ -33,7 +33,11 @@ import { useEnvironmentSettings, } from "../hooks/useSettings"; import { useEnvironment } from "../state/environments"; -import { useEnvironmentScope, usePreparedConnection } from "../state/session"; +import { + useEnvironmentScope, + useEnvironmentScopeDenied, + usePreparedConnection, +} from "../state/session"; import { preferredSpeechLanguages } from "./preferredSpeechLanguages"; import { runEnvironmentSpeech } from "./runEnvironmentSpeech"; import { useAudioChoice } from "./voiceConsent"; @@ -114,8 +118,9 @@ type HookInput = { /** * Microphone audio leaves this device only for the environment the person chose in - * Voice settings. Nothing is inferred from the primary or the draft's environment, and - * the recording is abandoned the moment that choice changes or its connection drops. + * Voice settings. Nothing is inferred from the primary or the draft's environment. A transient + * connection loss keeps the local recording recoverable, but a destination, authorization, or + * capability change abandons it. */ export function useVoiceTranscriptionDestination() { const gt = useGT(); @@ -126,6 +131,7 @@ export function useVoiceTranscriptionDestination() { const environment = useEnvironment(environmentId); const connection = Option.getOrNull(usePreparedConnection(environmentId)); const canDictate = useEnvironmentScope(environmentId, AuthOrchestrationOperateScope); + const scopeDenied = useEnvironmentScopeDenied(environmentId, AuthOrchestrationOperateScope); const cloud = grant?.destination.kind === "cloud" ? grant.destination : undefined; const provider = cloud?.provider; const model = cloud?.provider === "openrouter" ? cloud.model : undefined; @@ -147,6 +153,9 @@ export function useVoiceTranscriptionDestination() { }) : (environment?.label ?? null), prepared: supported && canDictate ? connection : null, + retainOnDisconnect: Boolean( + environmentId && grant && (connection ? supported && canDictate : !scopeDenied), + ), }; } @@ -157,6 +166,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { const grantId = destination.grantId; const cloudProvider = destination.cloudProvider; const cloudModel = destination.cloudModel; + const retainOnDisconnect = destination.retainOnDisconnect; const draftCapabilities = useEnvironment(input.environmentId)?.serverConfig?.environment .capabilities; const originatingConnection = Option.getOrNull(usePreparedConnection(input.environmentId)); @@ -194,7 +204,14 @@ export function useEnvironmentSpeechInput(input: HookInput) { setSetup(INITIAL_SETUP); } - const session = useRetainedVoiceInputSession({ prepared, grantId, cloudProvider, cloudModel }); + const session = useRetainedVoiceInputSession({ + environmentId: destination.environmentId, + prepared, + grantId, + retainOnDisconnect, + cloudProvider, + cloudModel, + }); const snapshot = useVoiceInputSessionSnapshot(session); const owns = snapshot.target?.ownerKey === input.ownerKey; const activity = useVoiceInputSessionActivity(session, input.ownerKey); diff --git a/apps/web/src/speech/voiceInputSession.ts b/apps/web/src/speech/voiceInputSession.ts index 40e6450bd2ce1..b313e340c2358 100644 --- a/apps/web/src/speech/voiceInputSession.ts +++ b/apps/web/src/speech/voiceInputSession.ts @@ -19,7 +19,7 @@ import type { } from "@t3tools/contracts"; import { DEFAULT_CLIENT_SETTINGS } from "@t3tools/contracts"; import { resolveSpeechVocabulary } from "@t3tools/shared/speech"; -import { useEffect, useSyncExternalStore } from "react"; +import { useEffect, useRef, useSyncExternalStore } from "react"; import { toastManager } from "../components/ui/toast"; import { chooseRecordingMicrophone } from "./browserMicrophones"; @@ -71,6 +71,7 @@ export type VoiceInputSessionActivity = { }; type SessionKey = { + readonly environmentId: EnvironmentId; readonly prepared: PreparedConnection; readonly grantId: string; readonly cloudProvider: CloudSpeechProvider | undefined; @@ -97,6 +98,7 @@ function cleanupContext({ text, selection }: DraftInput): DraftInput { */ export class VoiceInputSession { readonly key: SessionKey; + private prepared: PreparedConnection | null; private readonly controller: VoiceInputController; private target: VoiceInputTarget | null = null; private snapshot: VoiceInputSessionSnapshot = { @@ -111,6 +113,7 @@ export class VoiceInputSession { constructor(key: SessionKey) { this.key = key; + this.prepared = key.prepared; const hasAudioConsent = bindAudioGrant(key.grantId); let recordingPreferences: | { @@ -121,6 +124,7 @@ export class VoiceInputSession { | undefined; const platform = createBrowserVoiceInputPlatform({ prepared: key.prepared, + getPrepared: () => this.prepared, cloudProvider: key.cloudProvider, cloudModel: key.cloudModel, hasAudioConsent, @@ -233,6 +237,11 @@ export class VoiceInputSession { return this.controller.currentState; } + updatePrepared(prepared: PreparedConnection | null): void { + if (prepared === null || prepared.environmentId === this.key.environmentId) + this.prepared = prepared; + } + get ownerKey(): string | null { return this.target?.ownerKey ?? null; } @@ -312,37 +321,51 @@ export class VoiceInputSession { const sessions = new Map(); const registryListeners = new Set<() => void>(); -function sameKey(session: VoiceInputSession, key: SessionKey): boolean { +function sameKey( + session: VoiceInputSession, + key: Pick, +): boolean { return ( - session.key.prepared === key.prepared && + session.key.environmentId === key.environmentId && session.key.grantId === key.grantId && session.key.cloudProvider === key.cloudProvider && session.key.cloudModel === key.cloudModel ); } -function findSession(key: SessionKey): VoiceInputSession | null { +type SessionLookupKey = Omit & { + readonly prepared?: PreparedConnection | null; +}; + +function findSession(key: SessionLookupKey): VoiceInputSession | null { for (const session of sessions.keys()) if (sameKey(session, key)) return session; return null; } /** Every holder of a destination shares one session; the last release disposes it. */ -function retainVoiceInputSession(key: SessionKey): () => void { +function retainVoiceInputSession(key: SessionKey): { + readonly session: VoiceInputSession; + readonly release: () => void; +} { const session = findSession(key) ?? new VoiceInputSession(key); + session.updatePrepared(key.prepared); sessions.set(session, (sessions.get(session) ?? 0) + 1); for (const listener of registryListeners) listener(); let released = false; - return () => { - if (released) return; - released = true; - const count = (sessions.get(session) ?? 1) - 1; - if (count > 0) { - sessions.set(session, count); - return; - } - sessions.delete(session); - session.dispose(); - for (const listener of registryListeners) listener(); + return { + session, + release: () => { + if (released) return; + released = true; + const count = (sessions.get(session) ?? 1) - 1; + if (count > 0) { + sessions.set(session, count); + return; + } + sessions.delete(session); + session.dispose(); + for (const listener of registryListeners) listener(); + }, }; } @@ -353,20 +376,74 @@ function subscribeRegistry(listener: () => void): () => void { /** Holds the destination's shared session while mounted; null until it is retained. */ export function useRetainedVoiceInputSession(input: { + readonly environmentId: EnvironmentId | null; readonly prepared: PreparedConnection | null; readonly grantId: string | null; + readonly retainOnDisconnect: boolean; readonly cloudProvider?: CloudSpeechProvider | undefined; readonly cloudModel?: CloudSpeechModelId | undefined; }): VoiceInputSession | null { - const { prepared, grantId, cloudProvider, cloudModel } = input; + const { environmentId, prepared, grantId, retainOnDisconnect, cloudProvider, cloudModel } = input; + const leaseRef = useRef<{ + readonly session: VoiceInputSession; + readonly release: () => void; + } | null>(null); // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- retains the shared recording session for this destination while mounted useEffect(() => { - if (!prepared || !grantId || typeof navigator === "undefined") return; - return retainVoiceInputSession({ prepared, grantId, cloudProvider, cloudModel }); - }, [prepared, grantId, cloudProvider, cloudModel]); - return useSyncExternalStore(subscribeRegistry, () => - prepared && grantId ? findSession({ prepared, grantId, cloudProvider, cloudModel }) : null, + return () => { + leaseRef.current?.release(); + leaseRef.current = null; + }; + }, []); + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- acquires or refreshes the external recording lease when the current transport changes + useEffect(() => { + const currentKey = leaseRef.current?.session.key; + if ( + currentKey && + (currentKey.environmentId !== environmentId || + currentKey.grantId !== grantId || + currentKey.cloudProvider !== cloudProvider || + currentKey.cloudModel !== cloudModel) + ) { + leaseRef.current?.release(); + leaseRef.current = null; + } + if (!retainOnDisconnect) { + leaseRef.current?.release(); + leaseRef.current = null; + return; + } + if ( + !environmentId || + !prepared || + !grantId || + !retainOnDisconnect || + typeof navigator === "undefined" + ) + return; + if (!leaseRef.current) { + const lease = retainVoiceInputSession({ + environmentId, + prepared, + grantId, + cloudProvider, + cloudModel, + }); + leaseRef.current = lease; + } else { + leaseRef.current.session.updatePrepared(prepared); + } + }, [environmentId, grantId, prepared, retainOnDisconnect, cloudProvider, cloudModel]); + const session = useSyncExternalStore(subscribeRegistry, () => + environmentId && grantId + ? findSession({ environmentId, grantId, cloudProvider, cloudModel, prepared }) + : null, ); + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- updates the retained session transport after the chosen environment reconnects + useEffect(() => { + if (session) session.updatePrepared(prepared); + }, [prepared, session]); + return session; } const IDLE_SNAPSHOT: VoiceInputSessionSnapshot = { diff --git a/apps/web/src/state/session.ts b/apps/web/src/state/session.ts index f53bb3dfa9684..379de12873c60 100644 --- a/apps/web/src/state/session.ts +++ b/apps/web/src/state/session.ts @@ -35,6 +35,18 @@ export function useEnvironmentScope( ); } +export function useEnvironmentScopeDenied( + environmentId: EnvironmentId | null, + scope: AuthEnvironmentScope, +): boolean { + return useAtomValue( + environmentId === null + ? EMPTY_SESSION_STATE_ATOM + : environmentSession.sessionStateAtom(environmentId), + (result) => sessionDeniesScope(result, scope), + ); +} + export function useEnvironmentCanConfigureRelay(environmentId: EnvironmentId | null): boolean { return useAtomValue( environmentId === null @@ -57,6 +69,14 @@ export function sessionHasScope( return result._tag !== "Failure" && session !== null && sessionGrantsScope(session, scope); } +function sessionDeniesScope( + result: AsyncResult.AsyncResult, + scope: AuthEnvironmentScope, +): boolean { + const session = AsyncResult.getOrElse(result, () => null); + return result._tag !== "Failure" && session !== null && !sessionGrantsScope(session, scope); +} + const environmentsWithScopeAtom = Atom.family((key: string) => { const [scope, idsKey = ""] = key.split(/\n(.*)/s) as [AuthEnvironmentScope, string?]; const ids = idsKey === "" ? [] : (idsKey.split("\n") as EnvironmentId[]); From e1f9c7009563ce849c4a42a5cee7f7ed7f4910b0 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 20 Nov 2026 01:43:16 +0000 Subject: [PATCH 2/7] refactor(web): fold voice session lease into one effect The session key no longer carries the connection; the hook acquires on a live connection, refreshes the transport on reconnect, and releases only when the destination changes or stops being retainable. Drops a test the retry case already covers. Co-Authored-By: Claude Opus 5.5 --- ...useEnvironmentSpeechInput.browser.test.tsx | 18 --- apps/web/src/speech/voiceInputSession.ts | 108 ++++++------------ 2 files changed, 34 insertions(+), 92 deletions(-) diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx index 0b3c412968144..97dba92a968b4 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx +++ b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx @@ -403,24 +403,6 @@ it("retries a failed transcription on the same chosen environment", async () => expect(mocks.committed).toHaveBeenCalledExactlyOnceWith("second try", expect.anything()); }); -it("retains a recording through a transient destination disconnect and reconnect", async () => { - setup({ destination: "env-a" }); - await using screen = await render(); - await act(() => mocks.voice.start()); - expect(mocks.voice.state.phase).toBe("recording"); - - mocks.connected.delete("env-a"); - mocks.prepared.delete("env-a"); - await screen.rerender(); - expect(mocks.voice.state.phase).toBe("recording"); - - mocks.connected.add("env-a"); - await screen.rerender(); - await act(() => mocks.voice.stop()); - expect(mocks.transcribeCalls).toEqual(["env-a"]); - expect(mocks.committed).toHaveBeenCalledExactlyOnceWith("dictated", expect.anything()); -}); - it("retains an offline-first destination holder until the same destination reconnects", async () => { setup({ destination: "env-a" }); mocks.connected.delete("env-a"); diff --git a/apps/web/src/speech/voiceInputSession.ts b/apps/web/src/speech/voiceInputSession.ts index b313e340c2358..703e26fa63788 100644 --- a/apps/web/src/speech/voiceInputSession.ts +++ b/apps/web/src/speech/voiceInputSession.ts @@ -72,7 +72,6 @@ export type VoiceInputSessionActivity = { type SessionKey = { readonly environmentId: EnvironmentId; - readonly prepared: PreparedConnection; readonly grantId: string; readonly cloudProvider: CloudSpeechProvider | undefined; readonly cloudModel: CloudSpeechModelId | undefined; @@ -111,9 +110,9 @@ export class VoiceInputSession { private readonly activityListeners = new Set<() => void>(); private disposed = false; - constructor(key: SessionKey) { + constructor(key: SessionKey, prepared: PreparedConnection) { this.key = key; - this.prepared = key.prepared; + this.prepared = prepared; const hasAudioConsent = bindAudioGrant(key.grantId); let recordingPreferences: | { @@ -123,7 +122,7 @@ export class VoiceInputSession { } | undefined; const platform = createBrowserVoiceInputPlatform({ - prepared: key.prepared, + prepared, getPrepared: () => this.prepared, cloudProvider: key.cloudProvider, cloudModel: key.cloudModel, @@ -237,9 +236,9 @@ export class VoiceInputSession { return this.controller.currentState; } + /** A reconnect to the same destination replaces the transport; null while it is down. */ updatePrepared(prepared: PreparedConnection | null): void { - if (prepared === null || prepared.environmentId === this.key.environmentId) - this.prepared = prepared; + this.prepared = prepared; } get ownerKey(): string | null { @@ -321,10 +320,7 @@ export class VoiceInputSession { const sessions = new Map(); const registryListeners = new Set<() => void>(); -function sameKey( - session: VoiceInputSession, - key: Pick, -): boolean { +function sameKey(session: VoiceInputSession, key: SessionKey): boolean { return ( session.key.environmentId === key.environmentId && session.key.grantId === key.grantId && @@ -333,22 +329,15 @@ function sameKey( ); } -type SessionLookupKey = Omit & { - readonly prepared?: PreparedConnection | null; -}; - -function findSession(key: SessionLookupKey): VoiceInputSession | null { +function findSession(key: SessionKey): VoiceInputSession | null { for (const session of sessions.keys()) if (sameKey(session, key)) return session; return null; } /** Every holder of a destination shares one session; the last release disposes it. */ -function retainVoiceInputSession(key: SessionKey): { - readonly session: VoiceInputSession; - readonly release: () => void; -} { - const session = findSession(key) ?? new VoiceInputSession(key); - session.updatePrepared(key.prepared); +function retainVoiceInputSession(key: SessionKey, prepared: PreparedConnection) { + const session = findSession(key) ?? new VoiceInputSession(key, prepared); + session.updatePrepared(prepared); sessions.set(session, (sessions.get(session) ?? 0) + 1); for (const listener of registryListeners) listener(); let released = false; @@ -374,7 +363,11 @@ function subscribeRegistry(listener: () => void): () => void { return () => registryListeners.delete(listener); } -/** Holds the destination's shared session while mounted; null until it is retained. */ +/** + * Holds the destination's shared session while mounted; null until it is retained. The + * session is first retained on a live connection and then survives it dropping for as long + * as `retainOnDisconnect` holds, so a reconnect resumes the same recording. + */ export function useRetainedVoiceInputSession(input: { readonly environmentId: EnvironmentId | null; readonly prepared: PreparedConnection | null; @@ -384,66 +377,33 @@ export function useRetainedVoiceInputSession(input: { readonly cloudModel?: CloudSpeechModelId | undefined; }): VoiceInputSession | null { const { environmentId, prepared, grantId, retainOnDisconnect, cloudProvider, cloudModel } = input; - const leaseRef = useRef<{ - readonly session: VoiceInputSession; - readonly release: () => void; - } | null>(null); - // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- retains the shared recording session for this destination while mounted - useEffect(() => { - return () => { + const leaseRef = useRef | null>(null); + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- releases the shared recording session on unmount + useEffect( + () => () => { leaseRef.current?.release(); leaseRef.current = null; - }; - }, []); - // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- acquires or refreshes the external recording lease when the current transport changes + }, + [], + ); + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- acquires, refreshes, or releases the shared recording session as the destination and its transport change useEffect(() => { - const currentKey = leaseRef.current?.session.key; - if ( - currentKey && - (currentKey.environmentId !== environmentId || - currentKey.grantId !== grantId || - currentKey.cloudProvider !== cloudProvider || - currentKey.cloudModel !== cloudModel) - ) { - leaseRef.current?.release(); - leaseRef.current = null; - } - if (!retainOnDisconnect) { - leaseRef.current?.release(); + const key = + environmentId && grantId ? { environmentId, grantId, cloudProvider, cloudModel } : null; + const lease = leaseRef.current; + if (lease && (!key || !retainOnDisconnect || !sameKey(lease.session, key))) { + lease.release(); leaseRef.current = null; - return; - } - if ( - !environmentId || - !prepared || - !grantId || - !retainOnDisconnect || - typeof navigator === "undefined" - ) - return; - if (!leaseRef.current) { - const lease = retainVoiceInputSession({ - environmentId, - prepared, - grantId, - cloudProvider, - cloudModel, - }); - leaseRef.current = lease; - } else { - leaseRef.current.session.updatePrepared(prepared); } - }, [environmentId, grantId, prepared, retainOnDisconnect, cloudProvider, cloudModel]); - const session = useSyncExternalStore(subscribeRegistry, () => + if (leaseRef.current) leaseRef.current.session.updatePrepared(prepared); + else if (key && prepared && retainOnDisconnect && typeof navigator !== "undefined") + leaseRef.current = retainVoiceInputSession(key, prepared); + }, [environmentId, grantId, cloudProvider, cloudModel, prepared, retainOnDisconnect]); + return useSyncExternalStore(subscribeRegistry, () => environmentId && grantId - ? findSession({ environmentId, grantId, cloudProvider, cloudModel, prepared }) + ? findSession({ environmentId, grantId, cloudProvider, cloudModel }) : null, ); - // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- updates the retained session transport after the chosen environment reconnects - useEffect(() => { - if (session) session.updatePrepared(prepared); - }, [prepared, session]); - return session; } const IDLE_SNAPSHOT: VoiceInputSessionSnapshot = { From fe0179ec32ecfbec8887e48449ccbca0ff194638 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 20 Nov 2026 01:53:53 +0000 Subject: [PATCH 3/7] fix(web): let offline holders join a retained voice session A holder that mounts while the destination is down now joins the existing session instead of skipping it, so the session survives its last connected holder unmounting. Scope denial uses the shared client-runtime predicate, which counts explicit 401/403 rejections as denials. Co-Authored-By: Claude Opus 5.5 --- ...useEnvironmentSpeechInput.browser.test.tsx | 23 +++++++++++++++++++ apps/web/src/speech/voiceInputSession.ts | 14 +++++++---- apps/web/src/state/session.ts | 13 ++++------- 3 files changed, 36 insertions(+), 14 deletions(-) diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx index 97dba92a968b4..d39695875e7be 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx +++ b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx @@ -437,6 +437,29 @@ it("keeps a stopped recording local while disconnected and retries it on reconne expect(mocks.committed).toHaveBeenCalledExactlyOnceWith("dictated", expect.anything()); }); +it("keeps a recording alive when its last connected holder unmounts while offline", async () => { + setup({ destination: "env-a" }); + mocks.storedDraft = ""; + const view = (editor: boolean, indicator: boolean) => ( + <> + {editor ? : null} + {indicator ? : null} + + ); + await using screen = await render(view(true, false)); + await act(() => mocks.voice.start()); + mocks.connected.delete("env-a"); + mocks.prepared.delete("env-a"); + // The indicator mounts offline, then the recording's editor unmounts. + await screen.rerender(view(true, true)); + await screen.rerender(view(false, true)); + mocks.connected.add("env-a"); + await screen.rerender(view(false, true)); + await page.getByRole("button", { name: "Finish voice input" }).click(); + await vi.waitFor(() => expect(mocks.storedDraft).toBe("dictated")); + expect(mocks.transcribeCalls).toEqual(["env-a"]); +}); + it.each(["authorization", "capability"] as const)( "abandons a recording when connected destination %s is lost", async (loss) => { diff --git a/apps/web/src/speech/voiceInputSession.ts b/apps/web/src/speech/voiceInputSession.ts index 703e26fa63788..a0736001bd908 100644 --- a/apps/web/src/speech/voiceInputSession.ts +++ b/apps/web/src/speech/voiceInputSession.ts @@ -334,9 +334,13 @@ function findSession(key: SessionKey): VoiceInputSession | null { return null; } -/** Every holder of a destination shares one session; the last release disposes it. */ -function retainVoiceInputSession(key: SessionKey, prepared: PreparedConnection) { - const session = findSession(key) ?? new VoiceInputSession(key, prepared); +/** + * Every holder of a destination shares one session; the last release disposes it. Offline, + * a holder joins an existing session but never creates one. + */ +function retainVoiceInputSession(key: SessionKey, prepared: PreparedConnection | null) { + const session = findSession(key) ?? (prepared ? new VoiceInputSession(key, prepared) : null); + if (!session) return null; session.updatePrepared(prepared); sessions.set(session, (sessions.get(session) ?? 0) + 1); for (const listener of registryListeners) listener(); @@ -377,7 +381,7 @@ export function useRetainedVoiceInputSession(input: { readonly cloudModel?: CloudSpeechModelId | undefined; }): VoiceInputSession | null { const { environmentId, prepared, grantId, retainOnDisconnect, cloudProvider, cloudModel } = input; - const leaseRef = useRef | null>(null); + const leaseRef = useRef>(null); // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- releases the shared recording session on unmount useEffect( () => () => { @@ -396,7 +400,7 @@ export function useRetainedVoiceInputSession(input: { leaseRef.current = null; } if (leaseRef.current) leaseRef.current.session.updatePrepared(prepared); - else if (key && prepared && retainOnDisconnect && typeof navigator !== "undefined") + else if (key && retainOnDisconnect && typeof navigator !== "undefined") leaseRef.current = retainVoiceInputSession(key, prepared); }, [environmentId, grantId, cloudProvider, cloudModel, prepared, retainOnDisconnect]); return useSyncExternalStore(subscribeRegistry, () => diff --git a/apps/web/src/state/session.ts b/apps/web/src/state/session.ts index 379de12873c60..f3cbd81605c77 100644 --- a/apps/web/src/state/session.ts +++ b/apps/web/src/state/session.ts @@ -1,5 +1,8 @@ import { useAtomValue } from "@effect/atom-react"; -import { createEnvironmentSessionAtoms } from "@t3tools/client-runtime/state/session"; +import { + createEnvironmentSessionAtoms, + sessionDeniesScope, +} from "@t3tools/client-runtime/state/session"; import { type AuthEnvironmentScope, type AuthSessionState, @@ -69,14 +72,6 @@ export function sessionHasScope( return result._tag !== "Failure" && session !== null && sessionGrantsScope(session, scope); } -function sessionDeniesScope( - result: AsyncResult.AsyncResult, - scope: AuthEnvironmentScope, -): boolean { - const session = AsyncResult.getOrElse(result, () => null); - return result._tag !== "Failure" && session !== null && !sessionGrantsScope(session, scope); -} - const environmentsWithScopeAtom = Atom.family((key: string) => { const [scope, idsKey = ""] = key.split(/\n(.*)/s) as [AuthEnvironmentScope, string?]; const ids = idsKey === "" ? [] : (idsKey.split("\n") as EnvironmentId[]); From 601291c7936f2431529814781ce1177ca853ce7c Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 20 Nov 2026 02:07:12 +0000 Subject: [PATCH 4/7] fix(web): keep dictation through a dropped stream and offline controls Streaming dictation now keeps the captured audio locally. When the live speech socket fails mid-recording (consent still granted), the recording continues and is uploaded as a buffered recording on stop, retryable after a reconnect. An editor that owns a live recording keeps its voice controls while its destination is down, so stop and cancel stay reachable. Co-Authored-By: Claude Opus 5.5 --- apps/web/src/speech/browserPcmCapture.ts | 7 +- .../speech/browserVoiceInput.browser.test.ts | 79 +++++++++++++++++++ apps/web/src/speech/browserVoiceInput.ts | 30 ++++--- ...useEnvironmentSpeechInput.browser.test.tsx | 2 + .../src/speech/useEnvironmentSpeechInput.ts | 12 +-- 5 files changed, 110 insertions(+), 20 deletions(-) diff --git a/apps/web/src/speech/browserPcmCapture.ts b/apps/web/src/speech/browserPcmCapture.ts index 5060b9c39ce3a..5ab1dba2399bb 100644 --- a/apps/web/src/speech/browserPcmCapture.ts +++ b/apps/web/src/speech/browserPcmCapture.ts @@ -109,8 +109,9 @@ export function createBrowserPcmCapture( input.onLevel(data); return; } - if (feed) feed(data); - else chunks.push(data); + // Kept even while streaming, so a dropped stream can still upload the recording. + chunks.push(data); + feed?.(data); }; node.onprocessorerror = () => { stopped?.reject(new Error(m(msg("Microphone processing failed.")))); @@ -156,7 +157,7 @@ export function createBrowserPcmCapture( worklet.port.postMessage("stop"); try { await pending; - if (!feed && chunks.length) { + if (chunks.length) { const blob = new Blob(chunks, { type: "application/octet-stream" }); recordingUri = URL.createObjectURL(blob); } diff --git a/apps/web/src/speech/browserVoiceInput.browser.test.ts b/apps/web/src/speech/browserVoiceInput.browser.test.ts index a6fee1f03e0e8..e181eacfa67e5 100644 --- a/apps/web/src/speech/browserVoiceInput.browser.test.ts +++ b/apps/web/src/speech/browserVoiceInput.browser.test.ts @@ -363,3 +363,82 @@ it("refreshes the stream ticket for a same-destination reconnect between awaits" expect(Boolean(transcription.streaming)).toBe(true); expect(sockets).toEqual(["wss://reconnected.example.test/ws/voice?wsTicket=reconnected-ticket"]); }); + +it("keeps a streamed recording when its stream drops and uploads it after reconnecting", async () => { + using microphone = capture(); + const sockets: EventTarget[] = []; + using _socket = stubGlobal( + "WebSocket", + class extends EventTarget { + constructor() { + super(); + sockets.push(this); + } + send() {} + close() {} + }, + ); + const uploads: Uint8Array[] = []; + const fetch = globalThis.fetch; + using _network = vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => { + const request = new Request(input, init); + if (request.url.endsWith("/api/voice/status")) return Response.json(status); + if (request.url.includes("/api/auth/websocket-ticket/")) + return Response.json({ ticket: "stream-ticket", expiresAt: "2030-01-01T00:00:00.000Z" }); + if (request.url.endsWith("/api/voice/transcribe")) { + uploads.push(decodeSpeechPcmRequest(new Uint8Array(await request.arrayBuffer())).pcm); + return Response.json({ text: "recovered" }); + } + return fetch(input, init); + }); + let currentPrepared: typeof prepared | null = prepared; + const onError = vi.fn(); + const platform = createBrowserVoiceInputPlatform({ + prepared, + getPrepared: () => currentPrepared, + hasAudioConsent: () => true, + getStreamingEnabled: () => true, + getTranscriptionOptions: () => DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, + getMicrophoneId: () => "", + onLevel() {}, + onDurationLimit() {}, + onText() {}, + onError, + }); + const commitDraft = vi.fn(); + const controller = new VoiceInputController({ + recorder: platform.recorder, + retryFailedRecording: true, + getTranscriber: () => platform.transcriber, + requestPermission: async () => ({ granted: true, canAskAgain: true }), + configureRecording: async () => {}, + releaseRecording: async () => platform.cancelRecording(), + deleteRecording: platform.deleteRecording, + readDraft: () => ({ + ownerKey: "draft", + text: "", + selection: { start: 0, end: 0 }, + revision: 0, + }), + commitDraft, + onStateChange() {}, + }); + using _controller = { [Symbol.dispose]: () => controller.dispose() }; + await controller.start(); + microphone.feed(new Float32Array([0.1])); + currentPrepared = null; + sockets[0]!.dispatchEvent(new Event("close")); + microphone.feed(new Float32Array([0.2])); + expect(controller.currentState.phase).toBe("recording"); + expect(onError).not.toHaveBeenCalled(); + + await controller.stop(); + expect(controller.currentState).toMatchObject({ phase: "error", errorAction: "retry" }); + currentPrepared = prepared; + await controller.start(); + expect(commitDraft).toHaveBeenCalledWith("recovered", expect.anything()); + expect(uploads.map((pcm) => [...new Float32Array(pcm.slice().buffer)])).toEqual([ + [Math.fround(0.1), Math.fround(0.2)], + ]); + expect(sockets).toHaveLength(1); +}); diff --git a/apps/web/src/speech/browserVoiceInput.ts b/apps/web/src/speech/browserVoiceInput.ts index 92de200bc0962..09ccedaba2eaa 100644 --- a/apps/web/src/speech/browserVoiceInput.ts +++ b/apps/web/src/speech/browserVoiceInput.ts @@ -133,17 +133,23 @@ export function createBrowserVoiceInputPlatform(input: { message: m(msg("Voice transcription failed. Check the destination in Settings → Voice.")), }); } - return { - locale, - transcribe: transcribeRecording, - streaming: openSpeechStream({ - url, - canSend: input.hasAudioConsent, - options: recordingOptions, - signal: options.signal, - onText: input.onText, - onError: (error) => input.onError(error.message), - }), - }; + const transcription: { + readonly locale: string; + readonly transcribe: typeof transcribeRecording; + streaming?: ReturnType; + } = { locale, transcribe: transcribeRecording }; + transcription.streaming = openSpeechStream({ + url, + canSend: input.hasAudioConsent, + options: recordingOptions, + signal: options.signal, + onText: input.onText, + onError: (error) => { + // A dropped stream keeps recording; the controller then uploads the local recording. + if (input.hasAudioConsent()) delete transcription.streaming; + else input.onError(error.message); + }, + }); + return transcription; }); } diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx index d39695875e7be..eb356b86bd717 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx +++ b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx @@ -426,6 +426,8 @@ it("keeps a stopped recording local while disconnected and retries it on reconne mocks.connected.delete("env-a"); mocks.prepared.delete("env-a"); await screen.rerender(); + // The editor keeps its stop and cancel controls while the destination is down. + expect(mocks.voice.available).toBe(true); await act(() => mocks.voice.stop()); expect(mocks.transcribeCalls).toEqual([]); expect(mocks.voice.state).toMatchObject({ phase: "error", errorAction: "retry" }); diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.ts b/apps/web/src/speech/useEnvironmentSpeechInput.ts index 69bf36694b187..b0916763786e1 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.ts +++ b/apps/web/src/speech/useEnvironmentSpeechInput.ts @@ -536,12 +536,14 @@ export function useEnvironmentSpeechInput(input: HookInput) { (cloudProvider !== undefined || (status?.prepared === prepared && status.value?.supported !== false)); return { + // A recording this editor owns keeps its controls while the destination reconnects. available: - destinationReady && - !busyElsewhere && - typeof navigator !== "undefined" && - Boolean(navigator.mediaDevices?.getUserMedia) && - typeof AudioWorkletNode !== "undefined", + (owns && snapshot.state.phase !== "idle") || + (destinationReady && + !busyElsewhere && + typeof navigator !== "undefined" && + Boolean(navigator.mediaDevices?.getUserMedia) && + typeof AudioWorkletNode !== "undefined"), status: currentStatus, transcriptionEnvironmentId: destination.environmentId, transcriptionEnvironmentLabel: destination.label, From fe804017de9e3c5040f7bf4ac6c576b6984a5de9 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 20 Nov 2026 02:10:58 +0000 Subject: [PATCH 5/7] fix(web): keep dictation when a connected session lookup fails While connected, the retained recording now ends only on an explicit scope denial or lost capability, matching the offline rule, so a transient auth session lookup failure no longer discards it. Co-Authored-By: Claude Opus 5.5 --- ...useEnvironmentSpeechInput.browser.test.tsx | 36 +++++++++++-------- .../src/speech/useEnvironmentSpeechInput.ts | 3 +- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx index eb356b86bd717..b1c0ea717bb79 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx +++ b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx @@ -38,6 +38,7 @@ const mocks = vi.hoisted(() => ({ voiceCapable: new Set(), cloudVoiceCapable: new Set(), canDictate: new Set(), + scopeLookupFailed: new Set(), prepared: new Map(), primaryEnvironmentId: "primary-env", statusRequests: [] as string[], @@ -69,7 +70,8 @@ vi.mock("../components/sidebar/mainAppLocation", () => ({ })); vi.mock("../state/session", () => ({ useEnvironmentScope: (id: string) => mocks.canDictate.has(id), - useEnvironmentScopeDenied: () => false, + useEnvironmentScopeDenied: (id: string) => + !mocks.canDictate.has(id) && !mocks.scopeLookupFailed.has(id), usePreparedConnection: (environmentId: string | null) => { if (!environmentId || !mocks.connected.has(environmentId)) return Option.none(); let prepared = mocks.prepared.get(environmentId); @@ -229,6 +231,7 @@ function setup(options: { readonly destination?: string | null } = {}) { mocks.voiceCapable = new Set(["primary-env", THREAD_ENV, "env-a", "env-b"]); mocks.cloudVoiceCapable.clear(); mocks.canDictate = new Set(mocks.voiceCapable); + mocks.scopeLookupFailed = new Set(); mocks.prepared.clear(); mocks.statusRequests = []; mocks.statusUnavailable = false; @@ -462,20 +465,23 @@ it("keeps a recording alive when its last connected holder unmounts while offlin expect(mocks.transcribeCalls).toEqual(["env-a"]); }); -it.each(["authorization", "capability"] as const)( - "abandons a recording when connected destination %s is lost", - async (loss) => { - setup({ destination: "env-a" }); - await using screen = await render(); - await act(() => mocks.voice.start()); - if (loss === "authorization") mocks.canDictate.delete("env-a"); - else mocks.voiceCapable.delete("env-a"); - await screen.rerender(); - expect(mocks.voice.state.phase).toBe("idle"); - await act(() => mocks.voice.stop()); - expect(mocks.transcribeCalls).toEqual([]); - }, -); +it.each([ + ["authorization is denied", "idle"], + ["capability is lost", "idle"], + ["the session lookup fails", "recording"], +] as const)("when a connected destination's %s, the recording ends up %s", async (loss, phase) => { + setup({ destination: "env-a" }); + await using screen = await render(); + await act(() => mocks.voice.start()); + if (loss === "capability is lost") mocks.voiceCapable.delete("env-a"); + else mocks.canDictate.delete("env-a"); + if (loss === "the session lookup fails") mocks.scopeLookupFailed.add("env-a"); + await screen.rerender(); + expect(mocks.voice.state.phase).toBe(phase); + if (phase === "recording") return; + await act(() => mocks.voice.stop()); + expect(mocks.transcribeCalls).toEqual([]); +}); it.each([ ["a stored thread draft keeps recording into it", true], diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.ts b/apps/web/src/speech/useEnvironmentSpeechInput.ts index b0916763786e1..0e6598aec750a 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.ts +++ b/apps/web/src/speech/useEnvironmentSpeechInput.ts @@ -153,8 +153,9 @@ export function useVoiceTranscriptionDestination() { }) : (environment?.label ?? null), prepared: supported && canDictate ? connection : null, + // Only an explicit denial or lost capability releases a recording; a failed lookup keeps it. retainOnDisconnect: Boolean( - environmentId && grant && (connection ? supported && canDictate : !scopeDenied), + environmentId && grant && !scopeDenied && (connection === null || supported), ), }; } From 0bd08b6b4df619982246e8a10ed80b11d63f50f4 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 20 Nov 2026 02:21:43 +0000 Subject: [PATCH 6/7] fix(web): composer keeps dictation while its environment is disconnected The chat composer's dictation is disabled while its environment is unavailable or cannot be operated, and the hook cancelled the recording on any disable. The hook now takes `discardsRecording`, so the composer only abandons a recording when another surface takes it over (approval, pending question, project selection); a disconnect keeps it. Co-Authored-By: Claude Opus 5.5 --- apps/web/src/components/chat/ChatComposer.tsx | 8 +++++--- ...useEnvironmentSpeechInput.browser.test.tsx | 20 +++++++++++++++++++ .../src/speech/useEnvironmentSpeechInput.ts | 10 ++++++++-- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index 603edc067dba8..79d8ab184c779 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -1850,19 +1850,21 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) }; const readComposerSnapshotStable = useStableCallback(readComposerSnapshot); - const dictationDisabled = - !canOperateThread || - environmentUnavailable !== null || + // Another surface owns the composer; a recording into it would land nowhere. + const dictationDisplaced = projectSelectionRequired || isComposerApprovalState || activePendingIsResponding || activePendingProgress?.activeQuestion?.allowCustomAnswer === false; + const dictationDisabled = + !canOperateThread || environmentUnavailable !== null || dictationDisplaced; const speechInput = useEnvironmentSpeechInput({ environmentId, projectId: activeProjectId, projectName, ownerKey: questionAttachmentTarget ?? composerDraftTargetKey, disabled: dictationDisabled, + discardsRecording: dictationDisplaced, label: activeThread?.title ?? null, presented: !hiddenThread, // A question's answer lives only in this composer, so only the thread draft outlives it. diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx index b1c0ea717bb79..6f9a3465da27b 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx +++ b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx @@ -270,6 +270,8 @@ function Probe(props: { readonly showWizard?: boolean; readonly ownerKey?: string; readonly persistent?: boolean; + readonly disabled?: boolean; + readonly discardsRecording?: boolean; }) { const value = useEnvironmentSpeechInput({ environmentId: THREAD_ENV as EnvironmentId, @@ -277,6 +279,8 @@ function Probe(props: { projectName: props.projectName, cleanup: props.cleanup, ownerKey: props.ownerKey ?? "draft", + disabled: props.disabled, + discardsRecording: props.discardsRecording, persistentDraft: props.persistent ? { readStoredText: () => mocks.storedDraft, @@ -422,6 +426,22 @@ it("retains an offline-first destination holder until the same destination recon expect(mocks.transcribeCalls).toEqual(["env-a"]); }); +it.each([ + ["a disconnect keeps it", false, "recording"], + ["a displaced composer discards it", true, "idle"], +] as const)("when the editor is disabled mid-recording, %s", async (_name, discards, phase) => { + setup({ destination: "env-a" }); + await using screen = await render(); + await act(() => mocks.voice.start()); + if (!discards) { + mocks.connected.delete("env-a"); + mocks.prepared.delete("env-a"); + } + // The composer's props when its environment drops, or when an approval takes it over. + await screen.rerender(); + expect(mocks.voice.state.phase).toBe(phase); +}); + it("keeps a stopped recording local while disconnected and retries it on reconnect", async () => { setup({ destination: "env-a" }); await using screen = await render(); diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.ts b/apps/web/src/speech/useEnvironmentSpeechInput.ts index 0e6598aec750a..a3ac9a6a71eb9 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.ts +++ b/apps/web/src/speech/useEnvironmentSpeechInput.ts @@ -87,6 +87,11 @@ type HookInput = { readonly projectName?: string | null | undefined; readonly ownerKey: string; readonly disabled?: boolean | undefined; + /** + * Whether `disabled` also abandons this editor's live recording; defaults to `disabled`. False + * keeps it through a temporary block such as a disconnected environment. + */ + readonly discardsRecording?: boolean | undefined; /** * Until a destination is chosen, voice input is offered and its first gesture opens Voice * settings. False hides it instead, for editors whose draft would not survive the navigation. @@ -297,6 +302,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { }, [prepared, setup.downloading, setup.open, setupModelId]); const previousOwnerRef = useRef(input.ownerKey); + const discardsRecording = input.discardsRecording ?? input.disabled; // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- abandons any queued submission, and a recording that cannot outlive its editor, when the draft's owner changes useEffect(() => { const previous = previousOwnerRef.current; @@ -305,8 +311,8 @@ export function useEnvironmentSpeechInput(input: HookInput) { pendingSubmissionRef.current = null; startRequestRef.current += 1; if (previous !== input.ownerKey) session?.ownerChanged(previous); - else if (session?.ownerKey === input.ownerKey) session.cancel(); - }, [input.ownerKey, input.disabled, session]); + else if (discardsRecording && session?.ownerKey === input.ownerKey) session.cancel(); + }, [input.ownerKey, input.disabled, discardsRecording, session]); // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- delivers a queued submission only after the transcript has committed to the draft useEffect(() => { From 0c2d467c2ef1bb4a5839146ee072bb71abdadd2f Mon Sep 17 00:00:00 2001 From: flipcyde101 <169501444+flipcyde101@users.noreply.github.com> Date: Thu, 19 Nov 2026 19:37:48 -0700 Subject: [PATCH 7/7] fix(web): ignore superseded dictation setup results --- ...useEnvironmentSpeechInput.browser.test.tsx | 33 ++++++++++++++++++- .../src/speech/useEnvironmentSpeechInput.ts | 14 ++++++-- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx index 6f9a3465da27b..66447f8df60e1 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx +++ b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx @@ -42,6 +42,8 @@ const mocks = vi.hoisted(() => ({ prepared: new Map(), primaryEnvironmentId: "primary-env", statusRequests: [] as string[], + statusStarted: 0, + statusDelay: null as Promise | null, statusUnavailable: false, statusTier: "accelerated" as "cpu" | "accelerated", modelState: "ready" as "ready" | "missing-model", @@ -126,7 +128,9 @@ vi.mock("../components/ui/toast", () => ({ toastManager: { add: vi.fn() } })); vi.mock("@t3tools/client-runtime/voice-input", async (importOriginal) => ({ ...(await importOriginal()), getEnvironmentSpeechStatus: (prepared: PreparedConnection) => - Effect.sync(() => { + Effect.promise(async () => { + mocks.statusStarted += 1; + if (mocks.statusDelay) await mocks.statusDelay; mocks.statusRequests.push(prepared.environmentId); if (mocks.statusUnavailable) throw new Error("Temporarily offline"); return { @@ -234,6 +238,8 @@ function setup(options: { readonly destination?: string | null } = {}) { mocks.scopeLookupFailed = new Set(); mocks.prepared.clear(); mocks.statusRequests = []; + mocks.statusStarted = 0; + mocks.statusDelay = null; mocks.statusUnavailable = false; mocks.modelState = "ready"; mocks.statusTier = "accelerated"; @@ -397,6 +403,31 @@ it.each([ expect(mocks.committed).not.toHaveBeenCalled(); }); +it("does not reopen setup from a missing-model status returned for a replaced connection", async () => { + setup({ destination: "env-a" }); + let resolveStatus!: () => void; + mocks.statusDelay = new Promise((resolve) => { + resolveStatus = resolve; + }); + mocks.modelState = "missing-model"; + await using screen = await render(); + const startPromise = mocks.voice.start(); + await vi.waitFor(() => expect(mocks.statusStarted).toBeGreaterThanOrEqual(2)); + + mocks.prepared.set("env-a", { environmentId: "env-a", label: "env-a-reconnected" }); + await screen.rerender(); + resolveStatus(); + await startPromise; + + const reopened = await vi + .waitFor(() => expect(mocks.voice.setup.open).toBe(true), { timeout: 100 }) + .then(() => true, () => false); + expect(reopened).toBe(false); + expect(mocks.voice.setup.open).toBe(false); + expect(mocks.voice.state.phase).toBe("idle"); + expect(mocks.transcribeCalls).toEqual([]); +}); + it("retries a failed transcription on the same chosen environment", async () => { setup({ destination: "env-a" }); mocks.transcripts = [new Error("inference failed"), "second try"]; diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.ts b/apps/web/src/speech/useEnvironmentSpeechInput.ts index a3ac9a6a71eb9..c2da48f61e408 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.ts +++ b/apps/web/src/speech/useEnvironmentSpeechInput.ts @@ -232,10 +232,17 @@ export function useEnvironmentSpeechInput(input: HookInput) { submit: (text: string) => void; text?: string; } | null>(null); - const latestRef = useRef({ input, clientSettings, cleanupPrepared, projectWords, session }); + const latestRef = useRef({ + input, + clientSettings, + cleanupPrepared, + projectWords, + session, + prepared, + }); // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- publishes committed props before an async completion can read them through the session useLayoutEffect(() => { - latestRef.current = { input, clientSettings, cleanupPrepared, projectWords, session }; + latestRef.current = { input, clientSettings, cleanupPrepared, projectWords, session, prepared }; }); const mountedRef = useRef(false); // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- abandons this editor's pending work, and a recording that cannot outlive it, on unmount or a destination change @@ -393,7 +400,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { const catalog = await runEnvironmentSpeech( getEnvironmentSpeechModels(connection, true), ).catch(() => null); - if (request !== startRequestRef.current) return; + if (request !== startRequestRef.current || latestRef.current.prepared !== connection) return; const choice = chooseSetupSpeechModel(catalog?.models ?? [], { selectedModelId, intent: latestRef.current.clientSettings.speechLanguage, @@ -435,6 +442,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { const stillCurrent = () => request === startRequestRef.current && latestRef.current.session === session && + latestRef.current.prepared === prepared && latestRef.current.input.ownerKey === owner && !latestRef.current.input.disabled; setSetup(INITIAL_SETUP);