diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index 603edc067dba8..79d8ab184c779 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -1850,19 +1850,21 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) }; const readComposerSnapshotStable = useStableCallback(readComposerSnapshot); - const dictationDisabled = - !canOperateThread || - environmentUnavailable !== null || + // Another surface owns the composer; a recording into it would land nowhere. + const dictationDisplaced = projectSelectionRequired || isComposerApprovalState || activePendingIsResponding || activePendingProgress?.activeQuestion?.allowCustomAnswer === false; + const dictationDisabled = + !canOperateThread || environmentUnavailable !== null || dictationDisplaced; const speechInput = useEnvironmentSpeechInput({ environmentId, projectId: activeProjectId, projectName, ownerKey: questionAttachmentTarget ?? composerDraftTargetKey, disabled: dictationDisabled, + discardsRecording: dictationDisplaced, label: activeThread?.title ?? null, presented: !hiddenThread, // A question's answer lives only in this composer, so only the thread draft outlives it. diff --git a/apps/web/src/components/chat/GlobalVoiceInputIndicator.tsx b/apps/web/src/components/chat/GlobalVoiceInputIndicator.tsx index dbe0453df2e9a..53dedde9bb5dd 100644 --- a/apps/web/src/components/chat/GlobalVoiceInputIndicator.tsx +++ b/apps/web/src/components/chat/GlobalVoiceInputIndicator.tsx @@ -15,8 +15,16 @@ import { ComposerSpeechRecordingPill } from "./ComposerSpeechButton"; */ export function GlobalVoiceInputIndicator({ hidden = false }: { hidden?: boolean }) { const gt = useGT(); - const { prepared, grantId, cloudProvider, cloudModel } = useVoiceTranscriptionDestination(); - const session = useRetainedVoiceInputSession({ prepared, grantId, cloudProvider, cloudModel }); + const { environmentId, prepared, grantId, retainOnDisconnect, cloudProvider, cloudModel } = + useVoiceTranscriptionDestination(); + const session = useRetainedVoiceInputSession({ + environmentId, + prepared, + grantId, + retainOnDisconnect, + cloudProvider, + cloudModel, + }); const { state, target, recordingStartedAt } = useVoiceInputSessionSnapshot(session); const presented = useVoiceOwnerPresented(target?.ownerKey ?? null); const { level } = useVoiceInputSessionActivity( diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx index 5c918f9430086..c51952d380363 100644 --- a/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx +++ b/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx @@ -170,6 +170,7 @@ it.each([{ cpu: null, accelerated: 2.1 }, { cpu: 5, accelerated: null }, { accel label: prepared.label, grantId: "test", prepared, + retainOnDisconnect: true, }), async () => { const previous = getClientSettings(); @@ -303,7 +304,13 @@ it("tests the selected model after download and preserves its transcript across const transcript = "Keep this transcript."; using _scope = vi.mocked(useEnvironmentScope).mockReturnValue(true); await vi.mocked(useVoiceTranscriptionDestination).withImplementation( - () => ({ environmentId: target.environmentId, label: target.label, grantId: "test", prepared }), + () => ({ + environmentId: target.environmentId, + label: target.label, + grantId: "test", + prepared, + retainOnDisconnect: true, + }), async () => { await vi.mocked(useEnvironmentSpeechInput).withImplementation( (input) => ({ @@ -486,6 +493,7 @@ it.each([ label: target.label, grantId: "test", prepared, + retainOnDisconnect: true, })); await using screen = await render( {}}> @@ -562,7 +570,13 @@ it("recovers from a failed download progress poll without notifying, but reports }); using notify = vi.spyOn(toastManager, "add").mockReturnValue("test"); await vi.mocked(useVoiceTranscriptionDestination).withImplementation( - () => ({ environmentId: target.environmentId, label: target.label, grantId: "test", prepared }), + () => ({ + environmentId: target.environmentId, + label: target.label, + grantId: "test", + prepared, + retainOnDisconnect: true, + }), async () => { const panel = (key: string) => ( {}}> diff --git a/apps/web/src/speech/browserPcmCapture.ts b/apps/web/src/speech/browserPcmCapture.ts index 5060b9c39ce3a..5ab1dba2399bb 100644 --- a/apps/web/src/speech/browserPcmCapture.ts +++ b/apps/web/src/speech/browserPcmCapture.ts @@ -109,8 +109,9 @@ export function createBrowserPcmCapture( input.onLevel(data); return; } - if (feed) feed(data); - else chunks.push(data); + // Kept even while streaming, so a dropped stream can still upload the recording. + chunks.push(data); + feed?.(data); }; node.onprocessorerror = () => { stopped?.reject(new Error(m(msg("Microphone processing failed.")))); @@ -156,7 +157,7 @@ export function createBrowserPcmCapture( worklet.port.postMessage("stop"); try { await pending; - if (!feed && chunks.length) { + if (chunks.length) { const blob = new Blob(chunks, { type: "application/octet-stream" }); recordingUri = URL.createObjectURL(blob); } diff --git a/apps/web/src/speech/browserVoiceInput.browser.test.ts b/apps/web/src/speech/browserVoiceInput.browser.test.ts index 93e11ad6d5801..e181eacfa67e5 100644 --- a/apps/web/src/speech/browserVoiceInput.browser.test.ts +++ b/apps/web/src/speech/browserVoiceInput.browser.test.ts @@ -248,3 +248,197 @@ it.each([ ); }, ); + +it.each(["status", "stream ticket"] as const)( + "does not open a stream after authorization is lost during %s preparation", + async (boundary) => { + const sockets: string[] = []; + using _socket = stubGlobal( + "WebSocket", + class extends EventTarget { + constructor(url: string) { + super(); + sockets.push(url); + } + close() {} + }, + ); + let currentPrepared: typeof prepared | null = prepared; + let resolveStatus!: (response: Response) => void; + let resolveTicket!: (response: Response) => void; + const pendingStatus = new Promise((resolve) => { + resolveStatus = resolve; + }); + const pendingTicket = new Promise((resolve) => { + resolveTicket = resolve; + }); + using _network = vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => { + const request = new Request(input, init); + if (request.url.endsWith("/api/voice/status")) return (await pendingStatus).clone(); + if (request.url.includes("/api/auth/websocket-ticket/")) return (await pendingTicket).clone(); + throw new Error(`Unexpected request: ${request.url}`); + }); + const abort = new AbortController(); + using _abort = { [Symbol.dispose]: () => abort.abort() }; + const platform = createBrowserVoiceInputPlatform({ + prepared, + getPrepared: () => currentPrepared, + hasAudioConsent: () => true, + getStreamingEnabled: () => true, + getTranscriptionOptions: () => DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, + getMicrophoneId: () => "", + onLevel() {}, + onDurationLimit() {}, + onText() {}, + onError() {}, + }); + const preparation = platform.transcriber.prepare({ signal: abort.signal }); + await vi.waitFor(() => expect(_network).toHaveBeenCalled()); + if (boundary === "status") { + currentPrepared = null; + resolveStatus(Response.json(status)); + } else { + resolveStatus(Response.json(status)); + await vi.waitFor(() => + expect( + _network.mock.calls.some(([input]) => String(input).includes("websocket-ticket")), + ).toBe(true), + ); + currentPrepared = null; + resolveTicket( + Response.json({ ticket: "stale-ticket", expiresAt: "2030-01-01T00:00:00.000Z" }), + ); + } + await expect(preparation).rejects.toMatchObject({ code: "unavailable" }); + expect(sockets).toEqual([]); + }, +); + +it("refreshes the stream ticket for a same-destination reconnect between awaits", async () => { + const sockets: string[] = []; + using _socket = stubGlobal( + "WebSocket", + class extends EventTarget { + constructor(url: string) { + super(); + sockets.push(url); + } + close() {} + }, + ); + const reconnected = { ...prepared, httpBaseUrl: "https://reconnected.example.test" }; + let currentPrepared: typeof prepared = prepared; + let resolveStatus!: (response: Response) => void; + const pendingStatus = new Promise((resolve) => { + resolveStatus = resolve; + }); + using _network = vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => { + const request = new Request(input, init); + if (request.url === `${prepared.httpBaseUrl}/api/voice/status`) + return (await pendingStatus).clone(); + if (request.url === `${reconnected.httpBaseUrl}/api/voice/status`) return Response.json(status); + if (request.url.includes("/api/auth/websocket-ticket/")) + return Response.json({ ticket: "reconnected-ticket", expiresAt: "2030-01-01T00:00:00.000Z" }); + throw new Error(`Unexpected request: ${request.url}`); + }); + const abort = new AbortController(); + using _abort = { [Symbol.dispose]: () => abort.abort() }; + const platform = createBrowserVoiceInputPlatform({ + prepared, + getPrepared: () => currentPrepared, + hasAudioConsent: () => true, + getStreamingEnabled: () => true, + getTranscriptionOptions: () => DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, + getMicrophoneId: () => "", + onLevel() {}, + onDurationLimit() {}, + onText() {}, + onError() {}, + }); + const preparation = platform.transcriber.prepare({ signal: abort.signal }); + await vi.waitFor(() => expect(_network).toHaveBeenCalled()); + currentPrepared = reconnected; + resolveStatus(Response.json(status)); + const transcription = await preparation; + expect(Boolean(transcription.streaming)).toBe(true); + expect(sockets).toEqual(["wss://reconnected.example.test/ws/voice?wsTicket=reconnected-ticket"]); +}); + +it("keeps a streamed recording when its stream drops and uploads it after reconnecting", async () => { + using microphone = capture(); + const sockets: EventTarget[] = []; + using _socket = stubGlobal( + "WebSocket", + class extends EventTarget { + constructor() { + super(); + sockets.push(this); + } + send() {} + close() {} + }, + ); + const uploads: Uint8Array[] = []; + const fetch = globalThis.fetch; + using _network = vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => { + const request = new Request(input, init); + if (request.url.endsWith("/api/voice/status")) return Response.json(status); + if (request.url.includes("/api/auth/websocket-ticket/")) + return Response.json({ ticket: "stream-ticket", expiresAt: "2030-01-01T00:00:00.000Z" }); + if (request.url.endsWith("/api/voice/transcribe")) { + uploads.push(decodeSpeechPcmRequest(new Uint8Array(await request.arrayBuffer())).pcm); + return Response.json({ text: "recovered" }); + } + return fetch(input, init); + }); + let currentPrepared: typeof prepared | null = prepared; + const onError = vi.fn(); + const platform = createBrowserVoiceInputPlatform({ + prepared, + getPrepared: () => currentPrepared, + hasAudioConsent: () => true, + getStreamingEnabled: () => true, + getTranscriptionOptions: () => DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, + getMicrophoneId: () => "", + onLevel() {}, + onDurationLimit() {}, + onText() {}, + onError, + }); + const commitDraft = vi.fn(); + const controller = new VoiceInputController({ + recorder: platform.recorder, + retryFailedRecording: true, + getTranscriber: () => platform.transcriber, + requestPermission: async () => ({ granted: true, canAskAgain: true }), + configureRecording: async () => {}, + releaseRecording: async () => platform.cancelRecording(), + deleteRecording: platform.deleteRecording, + readDraft: () => ({ + ownerKey: "draft", + text: "", + selection: { start: 0, end: 0 }, + revision: 0, + }), + commitDraft, + onStateChange() {}, + }); + using _controller = { [Symbol.dispose]: () => controller.dispose() }; + await controller.start(); + microphone.feed(new Float32Array([0.1])); + currentPrepared = null; + sockets[0]!.dispatchEvent(new Event("close")); + microphone.feed(new Float32Array([0.2])); + expect(controller.currentState.phase).toBe("recording"); + expect(onError).not.toHaveBeenCalled(); + + await controller.stop(); + expect(controller.currentState).toMatchObject({ phase: "error", errorAction: "retry" }); + currentPrepared = prepared; + await controller.start(); + expect(commitDraft).toHaveBeenCalledWith("recovered", expect.anything()); + expect(uploads.map((pcm) => [...new Float32Array(pcm.slice().buffer)])).toEqual([ + [Math.fround(0.1), Math.fround(0.2)], + ]); + expect(sockets).toHaveLength(1); +}); diff --git a/apps/web/src/speech/browserVoiceInput.ts b/apps/web/src/speech/browserVoiceInput.ts index d52f49f23fe07..09ccedaba2eaa 100644 --- a/apps/web/src/speech/browserVoiceInput.ts +++ b/apps/web/src/speech/browserVoiceInput.ts @@ -25,6 +25,7 @@ const isVoiceTranscriptionError = Schema.is(VoiceTranscriptionError); export function createBrowserVoiceInputPlatform(input: { readonly prepared: PreparedConnection; + readonly getPrepared?: (() => PreparedConnection | null) | undefined; readonly cloudProvider?: CloudSpeechProvider | undefined; /** The exact OpenRouter model the grant names; required with that provider. */ readonly cloudModel?: CloudSpeechModelId | undefined; @@ -37,6 +38,15 @@ export function createBrowserVoiceInputPlatform(input: { readonly onText: (text: SpeechStreamText) => void; readonly onError: (message: string, code?: VoiceInputErrorCode) => void; }) { + const getPrepared = input.getPrepared ?? (() => input.prepared); + const requirePrepared = () => { + const prepared = getPrepared(); + if (prepared) return prepared; + throw new VoiceTranscriptionError({ + code: "unavailable", + message: m(msg("Voice transcription failed. Check the destination in Settings → Voice.")), + }); + }; return createBrowserPcmCapture(input, async (options) => { const recordingOptions = input.getTranscriptionOptions(); const streamingEnabled = input.getStreamingEnabled(); @@ -47,22 +57,18 @@ export function createBrowserVoiceInputPlatform(input: { try { const response = await fetch(uri, { signal: transcriptionSignal }); const pcm = new Uint8Array(await response.arrayBuffer()); + const prepared = requirePrepared(); const result = await runEnvironmentSpeech( input.cloudProvider ? transcribeCloudPcm( - input.prepared, + prepared, input.cloudProvider, pcm, recordingOptions, input.hasAudioConsent, input.cloudModel, ) - : transcribeEnvironmentPcm( - input.prepared, - pcm, - recordingOptions, - input.hasAudioConsent, - ), + : transcribeEnvironmentPcm(prepared, pcm, recordingOptions, input.hasAudioConsent), { signal: transcriptionSignal }, ); throwIfVoiceTranscriptionAborted(transcriptionSignal); @@ -84,7 +90,7 @@ export function createBrowserVoiceInputPlatform(input: { message: m(msg("Choose an OpenRouter model in Settings → Voice.")), }); await runEnvironmentSpeech( - requireCloudSpeechKey(input.prepared, input.cloudProvider), + requireCloudSpeechKey(requirePrepared(), input.cloudProvider), options, ); throwIfVoiceTranscriptionAborted(options.signal); @@ -93,7 +99,8 @@ export function createBrowserVoiceInputPlatform(input: { transcribe: transcribeRecording, }; } - const status = await runEnvironmentSpeech(getEnvironmentSpeechStatus(input.prepared), options); + const prepared = requirePrepared(); + const status = await runEnvironmentSpeech(getEnvironmentSpeechStatus(prepared), options); throwIfVoiceTranscriptionAborted(options.signal); if (!status.supported) throw new VoiceTranscriptionError({ code: "unavailable", message: status.reason }); @@ -114,18 +121,35 @@ export function createBrowserVoiceInputPlatform(input: { if (status.supportsStreaming ? !nativeStreaming : !status.supportsBufferedStreaming) { return { locale, transcribe: transcribeRecording }; } - const url = await runEnvironmentSpeech(getEnvironmentSpeechStreamUrl(input.prepared), options); - return { - locale, - transcribe: transcribeRecording, - streaming: openSpeechStream({ - url, - canSend: input.hasAudioConsent, - options: recordingOptions, - signal: options.signal, - onText: input.onText, - onError: (error) => input.onError(error.message), - }), - }; + let streamPrepared = requirePrepared(); + let url = await runEnvironmentSpeech(getEnvironmentSpeechStreamUrl(streamPrepared), options); + const currentPrepared = requirePrepared(); + if (currentPrepared !== streamPrepared) { + streamPrepared = currentPrepared; + url = await runEnvironmentSpeech(getEnvironmentSpeechStreamUrl(streamPrepared), options); + if (requirePrepared() !== streamPrepared) + throw new VoiceTranscriptionError({ + code: "unavailable", + message: m(msg("Voice transcription failed. Check the destination in Settings → Voice.")), + }); + } + const transcription: { + readonly locale: string; + readonly transcribe: typeof transcribeRecording; + streaming?: ReturnType; + } = { locale, transcribe: transcribeRecording }; + transcription.streaming = openSpeechStream({ + url, + canSend: input.hasAudioConsent, + options: recordingOptions, + signal: options.signal, + onText: input.onText, + onError: (error) => { + // A dropped stream keeps recording; the controller then uploads the local recording. + if (input.hasAudioConsent()) delete transcription.streaming; + else input.onError(error.message); + }, + }); + return transcription; }); } diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx index cc40350d3aa11..66447f8df60e1 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx +++ b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx @@ -38,9 +38,12 @@ const mocks = vi.hoisted(() => ({ voiceCapable: new Set(), cloudVoiceCapable: new Set(), canDictate: new Set(), + scopeLookupFailed: new Set(), prepared: new Map(), primaryEnvironmentId: "primary-env", statusRequests: [] as string[], + statusStarted: 0, + statusDelay: null as Promise | null, statusUnavailable: false, statusTier: "accelerated" as "cpu" | "accelerated", modelState: "ready" as "ready" | "missing-model", @@ -69,6 +72,8 @@ vi.mock("../components/sidebar/mainAppLocation", () => ({ })); vi.mock("../state/session", () => ({ useEnvironmentScope: (id: string) => mocks.canDictate.has(id), + useEnvironmentScopeDenied: (id: string) => + !mocks.canDictate.has(id) && !mocks.scopeLookupFailed.has(id), usePreparedConnection: (environmentId: string | null) => { if (!environmentId || !mocks.connected.has(environmentId)) return Option.none(); let prepared = mocks.prepared.get(environmentId); @@ -123,7 +128,9 @@ vi.mock("../components/ui/toast", () => ({ toastManager: { add: vi.fn() } })); vi.mock("@t3tools/client-runtime/voice-input", async (importOriginal) => ({ ...(await importOriginal()), getEnvironmentSpeechStatus: (prepared: PreparedConnection) => - Effect.sync(() => { + Effect.promise(async () => { + mocks.statusStarted += 1; + if (mocks.statusDelay) await mocks.statusDelay; mocks.statusRequests.push(prepared.environmentId); if (mocks.statusUnavailable) throw new Error("Temporarily offline"); return { @@ -180,6 +187,7 @@ vi.mock("./browserVoiceInput", () => ({ input: Parameters[0], ) => { const environmentId = input.prepared.environmentId; + const getPrepared = input.getPrepared ?? (() => input.prepared); mocks.platforms.push(environmentId); return { recorder: { @@ -196,6 +204,7 @@ vi.mock("./browserVoiceInput", () => ({ return { locale: "en", transcribe: async () => { + if (!getPrepared()) throw new Error("destination disconnected"); mocks.transcribeCalls.push( input.cloudProvider ? `${[input.cloudProvider, (input as { cloudModel?: string }).cloudModel].filter(Boolean).join(":")} via ${environmentId}` @@ -226,8 +235,11 @@ function setup(options: { readonly destination?: string | null } = {}) { mocks.voiceCapable = new Set(["primary-env", THREAD_ENV, "env-a", "env-b"]); mocks.cloudVoiceCapable.clear(); mocks.canDictate = new Set(mocks.voiceCapable); + mocks.scopeLookupFailed = new Set(); mocks.prepared.clear(); mocks.statusRequests = []; + mocks.statusStarted = 0; + mocks.statusDelay = null; mocks.statusUnavailable = false; mocks.modelState = "ready"; mocks.statusTier = "accelerated"; @@ -264,6 +276,8 @@ function Probe(props: { readonly showWizard?: boolean; readonly ownerKey?: string; readonly persistent?: boolean; + readonly disabled?: boolean; + readonly discardsRecording?: boolean; }) { const value = useEnvironmentSpeechInput({ environmentId: THREAD_ENV as EnvironmentId, @@ -271,6 +285,8 @@ function Probe(props: { projectName: props.projectName, cleanup: props.cleanup, ownerKey: props.ownerKey ?? "draft", + disabled: props.disabled, + discardsRecording: props.discardsRecording, persistentDraft: props.persistent ? { readStoredText: () => mocks.storedDraft, @@ -387,6 +403,31 @@ it.each([ expect(mocks.committed).not.toHaveBeenCalled(); }); +it("does not reopen setup from a missing-model status returned for a replaced connection", async () => { + setup({ destination: "env-a" }); + let resolveStatus!: () => void; + mocks.statusDelay = new Promise((resolve) => { + resolveStatus = resolve; + }); + mocks.modelState = "missing-model"; + await using screen = await render(); + const startPromise = mocks.voice.start(); + await vi.waitFor(() => expect(mocks.statusStarted).toBeGreaterThanOrEqual(2)); + + mocks.prepared.set("env-a", { environmentId: "env-a", label: "env-a-reconnected" }); + await screen.rerender(); + resolveStatus(); + await startPromise; + + const reopened = await vi + .waitFor(() => expect(mocks.voice.setup.open).toBe(true), { timeout: 100 }) + .then(() => true, () => false); + expect(reopened).toBe(false); + expect(mocks.voice.setup.open).toBe(false); + expect(mocks.voice.state.phase).toBe("idle"); + expect(mocks.transcribeCalls).toEqual([]); +}); + it("retries a failed transcription on the same chosen environment", async () => { setup({ destination: "env-a" }); mocks.transcripts = [new Error("inference failed"), "second try"]; @@ -400,6 +441,99 @@ it("retries a failed transcription on the same chosen environment", async () => expect(mocks.committed).toHaveBeenCalledExactlyOnceWith("second try", expect.anything()); }); +it("retains an offline-first destination holder until the same destination reconnects", async () => { + setup({ destination: "env-a" }); + mocks.connected.delete("env-a"); + mocks.prepared.delete("env-a"); + await using screen = await render(); + expect(mocks.voice.state.phase).toBe("idle"); + expectNoAudioSent(); + + mocks.connected.add("env-a"); + await screen.rerender(); + await act(() => mocks.voice.start()); + expect(mocks.voice.state.phase).toBe("recording"); + await act(() => mocks.voice.stop()); + expect(mocks.transcribeCalls).toEqual(["env-a"]); +}); + +it.each([ + ["a disconnect keeps it", false, "recording"], + ["a displaced composer discards it", true, "idle"], +] as const)("when the editor is disabled mid-recording, %s", async (_name, discards, phase) => { + setup({ destination: "env-a" }); + await using screen = await render(); + await act(() => mocks.voice.start()); + if (!discards) { + mocks.connected.delete("env-a"); + mocks.prepared.delete("env-a"); + } + // The composer's props when its environment drops, or when an approval takes it over. + await screen.rerender(); + expect(mocks.voice.state.phase).toBe(phase); +}); + +it("keeps a stopped recording local while disconnected and retries it on reconnect", async () => { + setup({ destination: "env-a" }); + await using screen = await render(); + await act(() => mocks.voice.start()); + mocks.connected.delete("env-a"); + mocks.prepared.delete("env-a"); + await screen.rerender(); + // The editor keeps its stop and cancel controls while the destination is down. + expect(mocks.voice.available).toBe(true); + await act(() => mocks.voice.stop()); + expect(mocks.transcribeCalls).toEqual([]); + expect(mocks.voice.state).toMatchObject({ phase: "error", errorAction: "retry" }); + + mocks.connected.add("env-a"); + await screen.rerender(); + await act(() => mocks.voice.start()); + expect(mocks.transcribeCalls).toEqual(["env-a"]); + expect(mocks.committed).toHaveBeenCalledExactlyOnceWith("dictated", expect.anything()); +}); + +it("keeps a recording alive when its last connected holder unmounts while offline", async () => { + setup({ destination: "env-a" }); + mocks.storedDraft = ""; + const view = (editor: boolean, indicator: boolean) => ( + <> + {editor ? : null} + {indicator ? : null} + + ); + await using screen = await render(view(true, false)); + await act(() => mocks.voice.start()); + mocks.connected.delete("env-a"); + mocks.prepared.delete("env-a"); + // The indicator mounts offline, then the recording's editor unmounts. + await screen.rerender(view(true, true)); + await screen.rerender(view(false, true)); + mocks.connected.add("env-a"); + await screen.rerender(view(false, true)); + await page.getByRole("button", { name: "Finish voice input" }).click(); + await vi.waitFor(() => expect(mocks.storedDraft).toBe("dictated")); + expect(mocks.transcribeCalls).toEqual(["env-a"]); +}); + +it.each([ + ["authorization is denied", "idle"], + ["capability is lost", "idle"], + ["the session lookup fails", "recording"], +] as const)("when a connected destination's %s, the recording ends up %s", async (loss, phase) => { + setup({ destination: "env-a" }); + await using screen = await render(); + await act(() => mocks.voice.start()); + if (loss === "capability is lost") mocks.voiceCapable.delete("env-a"); + else mocks.canDictate.delete("env-a"); + if (loss === "the session lookup fails") mocks.scopeLookupFailed.add("env-a"); + await screen.rerender(); + expect(mocks.voice.state.phase).toBe(phase); + if (phase === "recording") return; + await act(() => mocks.voice.stop()); + expect(mocks.transcribeCalls).toEqual([]); +}); + it.each([ ["a stored thread draft keeps recording into it", true], ["an editor-only draft is abandoned", false], diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.ts b/apps/web/src/speech/useEnvironmentSpeechInput.ts index b3eb968096c99..c2da48f61e408 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.ts +++ b/apps/web/src/speech/useEnvironmentSpeechInput.ts @@ -33,7 +33,11 @@ import { useEnvironmentSettings, } from "../hooks/useSettings"; import { useEnvironment } from "../state/environments"; -import { useEnvironmentScope, usePreparedConnection } from "../state/session"; +import { + useEnvironmentScope, + useEnvironmentScopeDenied, + usePreparedConnection, +} from "../state/session"; import { preferredSpeechLanguages } from "./preferredSpeechLanguages"; import { runEnvironmentSpeech } from "./runEnvironmentSpeech"; import { useAudioChoice } from "./voiceConsent"; @@ -83,6 +87,11 @@ type HookInput = { readonly projectName?: string | null | undefined; readonly ownerKey: string; readonly disabled?: boolean | undefined; + /** + * Whether `disabled` also abandons this editor's live recording; defaults to `disabled`. False + * keeps it through a temporary block such as a disconnected environment. + */ + readonly discardsRecording?: boolean | undefined; /** * Until a destination is chosen, voice input is offered and its first gesture opens Voice * settings. False hides it instead, for editors whose draft would not survive the navigation. @@ -114,8 +123,9 @@ type HookInput = { /** * Microphone audio leaves this device only for the environment the person chose in - * Voice settings. Nothing is inferred from the primary or the draft's environment, and - * the recording is abandoned the moment that choice changes or its connection drops. + * Voice settings. Nothing is inferred from the primary or the draft's environment. A transient + * connection loss keeps the local recording recoverable, but a destination, authorization, or + * capability change abandons it. */ export function useVoiceTranscriptionDestination() { const gt = useGT(); @@ -126,6 +136,7 @@ export function useVoiceTranscriptionDestination() { const environment = useEnvironment(environmentId); const connection = Option.getOrNull(usePreparedConnection(environmentId)); const canDictate = useEnvironmentScope(environmentId, AuthOrchestrationOperateScope); + const scopeDenied = useEnvironmentScopeDenied(environmentId, AuthOrchestrationOperateScope); const cloud = grant?.destination.kind === "cloud" ? grant.destination : undefined; const provider = cloud?.provider; const model = cloud?.provider === "openrouter" ? cloud.model : undefined; @@ -147,6 +158,10 @@ export function useVoiceTranscriptionDestination() { }) : (environment?.label ?? null), prepared: supported && canDictate ? connection : null, + // Only an explicit denial or lost capability releases a recording; a failed lookup keeps it. + retainOnDisconnect: Boolean( + environmentId && grant && !scopeDenied && (connection === null || supported), + ), }; } @@ -157,6 +172,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { const grantId = destination.grantId; const cloudProvider = destination.cloudProvider; const cloudModel = destination.cloudModel; + const retainOnDisconnect = destination.retainOnDisconnect; const draftCapabilities = useEnvironment(input.environmentId)?.serverConfig?.environment .capabilities; const originatingConnection = Option.getOrNull(usePreparedConnection(input.environmentId)); @@ -194,7 +210,14 @@ export function useEnvironmentSpeechInput(input: HookInput) { setSetup(INITIAL_SETUP); } - const session = useRetainedVoiceInputSession({ prepared, grantId, cloudProvider, cloudModel }); + const session = useRetainedVoiceInputSession({ + environmentId: destination.environmentId, + prepared, + grantId, + retainOnDisconnect, + cloudProvider, + cloudModel, + }); const snapshot = useVoiceInputSessionSnapshot(session); const owns = snapshot.target?.ownerKey === input.ownerKey; const activity = useVoiceInputSessionActivity(session, input.ownerKey); @@ -209,10 +232,17 @@ export function useEnvironmentSpeechInput(input: HookInput) { submit: (text: string) => void; text?: string; } | null>(null); - const latestRef = useRef({ input, clientSettings, cleanupPrepared, projectWords, session }); + const latestRef = useRef({ + input, + clientSettings, + cleanupPrepared, + projectWords, + session, + prepared, + }); // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- publishes committed props before an async completion can read them through the session useLayoutEffect(() => { - latestRef.current = { input, clientSettings, cleanupPrepared, projectWords, session }; + latestRef.current = { input, clientSettings, cleanupPrepared, projectWords, session, prepared }; }); const mountedRef = useRef(false); // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- abandons this editor's pending work, and a recording that cannot outlive it, on unmount or a destination change @@ -279,6 +309,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { }, [prepared, setup.downloading, setup.open, setupModelId]); const previousOwnerRef = useRef(input.ownerKey); + const discardsRecording = input.discardsRecording ?? input.disabled; // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- abandons any queued submission, and a recording that cannot outlive its editor, when the draft's owner changes useEffect(() => { const previous = previousOwnerRef.current; @@ -287,8 +318,8 @@ export function useEnvironmentSpeechInput(input: HookInput) { pendingSubmissionRef.current = null; startRequestRef.current += 1; if (previous !== input.ownerKey) session?.ownerChanged(previous); - else if (session?.ownerKey === input.ownerKey) session.cancel(); - }, [input.ownerKey, input.disabled, session]); + else if (discardsRecording && session?.ownerKey === input.ownerKey) session.cancel(); + }, [input.ownerKey, input.disabled, discardsRecording, session]); // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- delivers a queued submission only after the transcript has committed to the draft useEffect(() => { @@ -369,7 +400,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { const catalog = await runEnvironmentSpeech( getEnvironmentSpeechModels(connection, true), ).catch(() => null); - if (request !== startRequestRef.current) return; + if (request !== startRequestRef.current || latestRef.current.prepared !== connection) return; const choice = chooseSetupSpeechModel(catalog?.models ?? [], { selectedModelId, intent: latestRef.current.clientSettings.speechLanguage, @@ -411,6 +442,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { const stillCurrent = () => request === startRequestRef.current && latestRef.current.session === session && + latestRef.current.prepared === prepared && latestRef.current.input.ownerKey === owner && !latestRef.current.input.disabled; setSetup(INITIAL_SETUP); @@ -519,12 +551,14 @@ export function useEnvironmentSpeechInput(input: HookInput) { (cloudProvider !== undefined || (status?.prepared === prepared && status.value?.supported !== false)); return { + // A recording this editor owns keeps its controls while the destination reconnects. available: - destinationReady && - !busyElsewhere && - typeof navigator !== "undefined" && - Boolean(navigator.mediaDevices?.getUserMedia) && - typeof AudioWorkletNode !== "undefined", + (owns && snapshot.state.phase !== "idle") || + (destinationReady && + !busyElsewhere && + typeof navigator !== "undefined" && + Boolean(navigator.mediaDevices?.getUserMedia) && + typeof AudioWorkletNode !== "undefined"), status: currentStatus, transcriptionEnvironmentId: destination.environmentId, transcriptionEnvironmentLabel: destination.label, diff --git a/apps/web/src/speech/voiceInputSession.ts b/apps/web/src/speech/voiceInputSession.ts index 40e6450bd2ce1..a0736001bd908 100644 --- a/apps/web/src/speech/voiceInputSession.ts +++ b/apps/web/src/speech/voiceInputSession.ts @@ -19,7 +19,7 @@ import type { } from "@t3tools/contracts"; import { DEFAULT_CLIENT_SETTINGS } from "@t3tools/contracts"; import { resolveSpeechVocabulary } from "@t3tools/shared/speech"; -import { useEffect, useSyncExternalStore } from "react"; +import { useEffect, useRef, useSyncExternalStore } from "react"; import { toastManager } from "../components/ui/toast"; import { chooseRecordingMicrophone } from "./browserMicrophones"; @@ -71,7 +71,7 @@ export type VoiceInputSessionActivity = { }; type SessionKey = { - readonly prepared: PreparedConnection; + readonly environmentId: EnvironmentId; readonly grantId: string; readonly cloudProvider: CloudSpeechProvider | undefined; readonly cloudModel: CloudSpeechModelId | undefined; @@ -97,6 +97,7 @@ function cleanupContext({ text, selection }: DraftInput): DraftInput { */ export class VoiceInputSession { readonly key: SessionKey; + private prepared: PreparedConnection | null; private readonly controller: VoiceInputController; private target: VoiceInputTarget | null = null; private snapshot: VoiceInputSessionSnapshot = { @@ -109,8 +110,9 @@ export class VoiceInputSession { private readonly activityListeners = new Set<() => void>(); private disposed = false; - constructor(key: SessionKey) { + constructor(key: SessionKey, prepared: PreparedConnection) { this.key = key; + this.prepared = prepared; const hasAudioConsent = bindAudioGrant(key.grantId); let recordingPreferences: | { @@ -120,7 +122,8 @@ export class VoiceInputSession { } | undefined; const platform = createBrowserVoiceInputPlatform({ - prepared: key.prepared, + prepared, + getPrepared: () => this.prepared, cloudProvider: key.cloudProvider, cloudModel: key.cloudModel, hasAudioConsent, @@ -233,6 +236,11 @@ export class VoiceInputSession { return this.controller.currentState; } + /** A reconnect to the same destination replaces the transport; null while it is down. */ + updatePrepared(prepared: PreparedConnection | null): void { + this.prepared = prepared; + } + get ownerKey(): string | null { return this.target?.ownerKey ?? null; } @@ -314,7 +322,7 @@ const registryListeners = new Set<() => void>(); function sameKey(session: VoiceInputSession, key: SessionKey): boolean { return ( - session.key.prepared === key.prepared && + session.key.environmentId === key.environmentId && session.key.grantId === key.grantId && session.key.cloudProvider === key.cloudProvider && session.key.cloudModel === key.cloudModel @@ -326,23 +334,31 @@ function findSession(key: SessionKey): VoiceInputSession | null { return null; } -/** Every holder of a destination shares one session; the last release disposes it. */ -function retainVoiceInputSession(key: SessionKey): () => void { - const session = findSession(key) ?? new VoiceInputSession(key); +/** + * Every holder of a destination shares one session; the last release disposes it. Offline, + * a holder joins an existing session but never creates one. + */ +function retainVoiceInputSession(key: SessionKey, prepared: PreparedConnection | null) { + const session = findSession(key) ?? (prepared ? new VoiceInputSession(key, prepared) : null); + if (!session) return null; + session.updatePrepared(prepared); sessions.set(session, (sessions.get(session) ?? 0) + 1); for (const listener of registryListeners) listener(); let released = false; - return () => { - if (released) return; - released = true; - const count = (sessions.get(session) ?? 1) - 1; - if (count > 0) { - sessions.set(session, count); - return; - } - sessions.delete(session); - session.dispose(); - for (const listener of registryListeners) listener(); + return { + session, + release: () => { + if (released) return; + released = true; + const count = (sessions.get(session) ?? 1) - 1; + if (count > 0) { + sessions.set(session, count); + return; + } + sessions.delete(session); + session.dispose(); + for (const listener of registryListeners) listener(); + }, }; } @@ -351,21 +367,46 @@ function subscribeRegistry(listener: () => void): () => void { return () => registryListeners.delete(listener); } -/** Holds the destination's shared session while mounted; null until it is retained. */ +/** + * Holds the destination's shared session while mounted; null until it is retained. The + * session is first retained on a live connection and then survives it dropping for as long + * as `retainOnDisconnect` holds, so a reconnect resumes the same recording. + */ export function useRetainedVoiceInputSession(input: { + readonly environmentId: EnvironmentId | null; readonly prepared: PreparedConnection | null; readonly grantId: string | null; + readonly retainOnDisconnect: boolean; readonly cloudProvider?: CloudSpeechProvider | undefined; readonly cloudModel?: CloudSpeechModelId | undefined; }): VoiceInputSession | null { - const { prepared, grantId, cloudProvider, cloudModel } = input; - // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- retains the shared recording session for this destination while mounted + const { environmentId, prepared, grantId, retainOnDisconnect, cloudProvider, cloudModel } = input; + const leaseRef = useRef>(null); + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- releases the shared recording session on unmount + useEffect( + () => () => { + leaseRef.current?.release(); + leaseRef.current = null; + }, + [], + ); + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- acquires, refreshes, or releases the shared recording session as the destination and its transport change useEffect(() => { - if (!prepared || !grantId || typeof navigator === "undefined") return; - return retainVoiceInputSession({ prepared, grantId, cloudProvider, cloudModel }); - }, [prepared, grantId, cloudProvider, cloudModel]); + const key = + environmentId && grantId ? { environmentId, grantId, cloudProvider, cloudModel } : null; + const lease = leaseRef.current; + if (lease && (!key || !retainOnDisconnect || !sameKey(lease.session, key))) { + lease.release(); + leaseRef.current = null; + } + if (leaseRef.current) leaseRef.current.session.updatePrepared(prepared); + else if (key && retainOnDisconnect && typeof navigator !== "undefined") + leaseRef.current = retainVoiceInputSession(key, prepared); + }, [environmentId, grantId, cloudProvider, cloudModel, prepared, retainOnDisconnect]); return useSyncExternalStore(subscribeRegistry, () => - prepared && grantId ? findSession({ prepared, grantId, cloudProvider, cloudModel }) : null, + environmentId && grantId + ? findSession({ environmentId, grantId, cloudProvider, cloudModel }) + : null, ); } diff --git a/apps/web/src/state/session.ts b/apps/web/src/state/session.ts index f53bb3dfa9684..f3cbd81605c77 100644 --- a/apps/web/src/state/session.ts +++ b/apps/web/src/state/session.ts @@ -1,5 +1,8 @@ import { useAtomValue } from "@effect/atom-react"; -import { createEnvironmentSessionAtoms } from "@t3tools/client-runtime/state/session"; +import { + createEnvironmentSessionAtoms, + sessionDeniesScope, +} from "@t3tools/client-runtime/state/session"; import { type AuthEnvironmentScope, type AuthSessionState, @@ -35,6 +38,18 @@ export function useEnvironmentScope( ); } +export function useEnvironmentScopeDenied( + environmentId: EnvironmentId | null, + scope: AuthEnvironmentScope, +): boolean { + return useAtomValue( + environmentId === null + ? EMPTY_SESSION_STATE_ATOM + : environmentSession.sessionStateAtom(environmentId), + (result) => sessionDeniesScope(result, scope), + ); +} + export function useEnvironmentCanConfigureRelay(environmentId: EnvironmentId | null): boolean { return useAtomValue( environmentId === null