Skip to content
8 changes: 5 additions & 3 deletions apps/web/src/components/chat/ChatComposer.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -1850,19 +1850,21 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
};
const readComposerSnapshotStable = useStableCallback(readComposerSnapshot);

const dictationDisabled =
!canOperateThread ||
environmentUnavailable !== null ||
// Another surface owns the composer; a recording into it would land nowhere.
const dictationDisplaced =
projectSelectionRequired ||
isComposerApprovalState ||
activePendingIsResponding ||
activePendingProgress?.activeQuestion?.allowCustomAnswer === false;
const dictationDisabled =
!canOperateThread || environmentUnavailable !== null || dictationDisplaced;
const speechInput = useEnvironmentSpeechInput({
environmentId,
projectId: activeProjectId,
projectName,
ownerKey: questionAttachmentTarget ?? composerDraftTargetKey,
disabled: dictationDisabled,
discardsRecording: dictationDisplaced,
label: activeThread?.title ?? null,
presented: !hiddenThread,
// A question's answer lives only in this composer, so only the thread draft outlives it.
Expand Down
12 changes: 10 additions & 2 deletions apps/web/src/components/chat/GlobalVoiceInputIndicator.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,16 @@ import { ComposerSpeechRecordingPill } from "./ComposerSpeechButton";
*/
export function GlobalVoiceInputIndicator({ hidden = false }: { hidden?: boolean }) {
const gt = useGT();
const { prepared, grantId, cloudProvider, cloudModel } = useVoiceTranscriptionDestination();
const session = useRetainedVoiceInputSession({ prepared, grantId, cloudProvider, cloudModel });
const { environmentId, prepared, grantId, retainOnDisconnect, cloudProvider, cloudModel } =
useVoiceTranscriptionDestination();
const session = useRetainedVoiceInputSession({
environmentId,
prepared,
grantId,
retainOnDisconnect,
cloudProvider,
cloudModel,
});
const { state, target, recordingStartedAt } = useVoiceInputSessionSnapshot(session);
const presented = useVoiceOwnerPresented(target?.ownerKey ?? null);
const { level } = useVoiceInputSessionActivity(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,7 @@ it.each([{ cpu: null, accelerated: 2.1 }, { cpu: 5, accelerated: null }, { accel
label: prepared.label,
grantId: "test",
prepared,
retainOnDisconnect: true,
}),
async () => {
const previous = getClientSettings();
Expand Down Expand Up @@ -303,7 +304,13 @@ it("tests the selected model after download and preserves its transcript across
const transcript = "Keep this transcript.";
using _scope = vi.mocked(useEnvironmentScope).mockReturnValue(true);
await vi.mocked(useVoiceTranscriptionDestination).withImplementation(
() => ({ environmentId: target.environmentId, label: target.label, grantId: "test", prepared }),
() => ({
environmentId: target.environmentId,
label: target.label,
grantId: "test",
prepared,
retainOnDisconnect: true,
}),
async () => {
await vi.mocked(useEnvironmentSpeechInput).withImplementation(
(input) => ({
Expand Down Expand Up @@ -486,6 +493,7 @@ it.each([
label: target.label,
grantId: "test",
prepared,
retainOnDisconnect: true,
}));
await using screen = await render(
<SettingsScopeProvider search={{}} onChange={() => {}}>
Expand Down Expand Up @@ -562,7 +570,13 @@ it("recovers from a failed download progress poll without notifying, but reports
});
using notify = vi.spyOn(toastManager, "add").mockReturnValue("test");
await vi.mocked(useVoiceTranscriptionDestination).withImplementation(
() => ({ environmentId: target.environmentId, label: target.label, grantId: "test", prepared }),
() => ({
environmentId: target.environmentId,
label: target.label,
grantId: "test",
prepared,
retainOnDisconnect: true,
}),
async () => {
const panel = (key: string) => (
<SettingsScopeProvider search={{}} onChange={() => {}}>
Expand Down
7 changes: 4 additions & 3 deletions apps/web/src/speech/browserPcmCapture.ts
Original file line number Diff line number Diff line change
Expand Up @@ -109,8 +109,9 @@ export function createBrowserPcmCapture(
input.onLevel(data);
return;
}
if (feed) feed(data);
else chunks.push(data);
// Kept even while streaming, so a dropped stream can still upload the recording.
chunks.push(data);
feed?.(data);
};
node.onprocessorerror = () => {
stopped?.reject(new Error(m(msg("Microphone processing failed."))));
Expand Down Expand Up @@ -156,7 +157,7 @@ export function createBrowserPcmCapture(
worklet.port.postMessage("stop");
try {
await pending;
if (!feed && chunks.length) {
if (chunks.length) {
const blob = new Blob(chunks, { type: "application/octet-stream" });
recordingUri = URL.createObjectURL(blob);
}
Expand Down
194 changes: 194 additions & 0 deletions apps/web/src/speech/browserVoiceInput.browser.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -248,3 +248,197 @@ it.each([
);
},
);

it.each(["status", "stream ticket"] as const)(
"does not open a stream after authorization is lost during %s preparation",
async (boundary) => {
const sockets: string[] = [];
using _socket = stubGlobal(
"WebSocket",
class extends EventTarget {
constructor(url: string) {
super();
sockets.push(url);
}
close() {}
},
);
let currentPrepared: typeof prepared | null = prepared;
let resolveStatus!: (response: Response) => void;
let resolveTicket!: (response: Response) => void;
const pendingStatus = new Promise<Response>((resolve) => {
resolveStatus = resolve;
});
const pendingTicket = new Promise<Response>((resolve) => {
resolveTicket = resolve;
});
using _network = vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => {
const request = new Request(input, init);
if (request.url.endsWith("/api/voice/status")) return (await pendingStatus).clone();
if (request.url.includes("/api/auth/websocket-ticket/")) return (await pendingTicket).clone();
throw new Error(`Unexpected request: ${request.url}`);
});
const abort = new AbortController();
using _abort = { [Symbol.dispose]: () => abort.abort() };
const platform = createBrowserVoiceInputPlatform({
prepared,
getPrepared: () => currentPrepared,
hasAudioConsent: () => true,
getStreamingEnabled: () => true,
getTranscriptionOptions: () => DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
getMicrophoneId: () => "",
onLevel() {},
onDurationLimit() {},
onText() {},
onError() {},
});
const preparation = platform.transcriber.prepare({ signal: abort.signal });
await vi.waitFor(() => expect(_network).toHaveBeenCalled());
if (boundary === "status") {
currentPrepared = null;
resolveStatus(Response.json(status));
} else {
resolveStatus(Response.json(status));
await vi.waitFor(() =>
expect(
_network.mock.calls.some(([input]) => String(input).includes("websocket-ticket")),
).toBe(true),
);
currentPrepared = null;
resolveTicket(
Response.json({ ticket: "stale-ticket", expiresAt: "2030-01-01T00:00:00.000Z" }),
);
}
await expect(preparation).rejects.toMatchObject({ code: "unavailable" });
expect(sockets).toEqual([]);
},
);

it("refreshes the stream ticket for a same-destination reconnect between awaits", async () => {
const sockets: string[] = [];
using _socket = stubGlobal(
"WebSocket",
class extends EventTarget {
constructor(url: string) {
super();
sockets.push(url);
}
close() {}
},
);
const reconnected = { ...prepared, httpBaseUrl: "https://reconnected.example.test" };
let currentPrepared: typeof prepared = prepared;
let resolveStatus!: (response: Response) => void;
const pendingStatus = new Promise<Response>((resolve) => {
resolveStatus = resolve;
});
using _network = vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => {
const request = new Request(input, init);
if (request.url === `${prepared.httpBaseUrl}/api/voice/status`)
return (await pendingStatus).clone();
if (request.url === `${reconnected.httpBaseUrl}/api/voice/status`) return Response.json(status);
if (request.url.includes("/api/auth/websocket-ticket/"))
return Response.json({ ticket: "reconnected-ticket", expiresAt: "2030-01-01T00:00:00.000Z" });
throw new Error(`Unexpected request: ${request.url}`);
});
const abort = new AbortController();
using _abort = { [Symbol.dispose]: () => abort.abort() };
const platform = createBrowserVoiceInputPlatform({
prepared,
getPrepared: () => currentPrepared,
hasAudioConsent: () => true,
getStreamingEnabled: () => true,
getTranscriptionOptions: () => DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
getMicrophoneId: () => "",
onLevel() {},
onDurationLimit() {},
onText() {},
onError() {},
});
const preparation = platform.transcriber.prepare({ signal: abort.signal });
await vi.waitFor(() => expect(_network).toHaveBeenCalled());
currentPrepared = reconnected;
resolveStatus(Response.json(status));
const transcription = await preparation;
expect(Boolean(transcription.streaming)).toBe(true);
expect(sockets).toEqual(["wss://reconnected.example.test/ws/voice?wsTicket=reconnected-ticket"]);
});

it("keeps a streamed recording when its stream drops and uploads it after reconnecting", async () => {
using microphone = capture();
const sockets: EventTarget[] = [];
using _socket = stubGlobal(
"WebSocket",
class extends EventTarget {
constructor() {
super();
sockets.push(this);
}
send() {}
close() {}
},
);
const uploads: Uint8Array[] = [];
const fetch = globalThis.fetch;
using _network = vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => {
const request = new Request(input, init);
if (request.url.endsWith("/api/voice/status")) return Response.json(status);
if (request.url.includes("/api/auth/websocket-ticket/"))
return Response.json({ ticket: "stream-ticket", expiresAt: "2030-01-01T00:00:00.000Z" });
if (request.url.endsWith("/api/voice/transcribe")) {
uploads.push(decodeSpeechPcmRequest(new Uint8Array(await request.arrayBuffer())).pcm);
return Response.json({ text: "recovered" });
}
return fetch(input, init);
});
let currentPrepared: typeof prepared | null = prepared;
const onError = vi.fn();
const platform = createBrowserVoiceInputPlatform({
prepared,
getPrepared: () => currentPrepared,
hasAudioConsent: () => true,
getStreamingEnabled: () => true,
getTranscriptionOptions: () => DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
getMicrophoneId: () => "",
onLevel() {},
onDurationLimit() {},
onText() {},
onError,
});
const commitDraft = vi.fn();
const controller = new VoiceInputController({
recorder: platform.recorder,
retryFailedRecording: true,
getTranscriber: () => platform.transcriber,
requestPermission: async () => ({ granted: true, canAskAgain: true }),
configureRecording: async () => {},
releaseRecording: async () => platform.cancelRecording(),
deleteRecording: platform.deleteRecording,
readDraft: () => ({
ownerKey: "draft",
text: "",
selection: { start: 0, end: 0 },
revision: 0,
}),
commitDraft,
onStateChange() {},
});
using _controller = { [Symbol.dispose]: () => controller.dispose() };
await controller.start();
microphone.feed(new Float32Array([0.1]));
currentPrepared = null;
sockets[0]!.dispatchEvent(new Event("close"));
microphone.feed(new Float32Array([0.2]));
expect(controller.currentState.phase).toBe("recording");
expect(onError).not.toHaveBeenCalled();

await controller.stop();
expect(controller.currentState).toMatchObject({ phase: "error", errorAction: "retry" });
currentPrepared = prepared;
await controller.start();
expect(commitDraft).toHaveBeenCalledWith("recovered", expect.anything());
expect(uploads.map((pcm) => [...new Float32Array(pcm.slice().buffer)])).toEqual([
[Math.fround(0.1), Math.fround(0.2)],
]);
expect(sockets).toHaveLength(1);
});
Loading
Loading