Skip to content
Open
39 changes: 13 additions & 26 deletions apps/web/src/components/chat/VoiceInputSetup.tsx
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import { T, Var, msg, useGT, useLocale } from "gt-react";
import { m } from "~/i18n/messages";
import { useStableCallback } from "~/hooks/useStableCallback";
import {
listBrowserMicrophones,
labelBrowserMicrophones,
updateMicrophoneOrder,
useMicrophoneRefresh,
type BrowserMicrophone,
} from "../../speech/browserMicrophones";
import type {
Expand All @@ -13,7 +13,7 @@ import type {
} from "@t3tools/contracts";
import { useNavigate } from "@tanstack/react-router";
import { CircleAlertIcon, MicIcon, RefreshCwIcon } from "lucide-react";
import { useEffect, useRef, useState } from "react";
import { useState } from "react";

import { useClientSettings } from "../../hooks/useSettings";
import {
Expand Down Expand Up @@ -58,30 +58,17 @@ export function VoiceInputSetup(props: {
const selectedMicrophone = preferredMicrophoneId(order, microphones);
const [loadingMicrophones, setLoadingMicrophones] = useState(false);
const [microphoneError, setMicrophoneError] = useState<string | null>(null);
const microphoneRefreshId = useRef(0);
const refreshMicrophones = useStableCallback(async () => {
if (!navigator.mediaDevices?.enumerateDevices) return;
const requestId = ++microphoneRefreshId.current;
setLoadingMicrophones(true);
const devices = await listBrowserMicrophones().catch(() => null);
if (requestId !== microphoneRefreshId.current) return;

if (devices === null) {
setMicrophoneError(m(msg("Couldn’t list microphones. The system default still works.")));
} else {
setMicrophones(devices);
setMicrophoneError(null);
}
setLoadingMicrophones(false);
const refreshMicrophones = useMicrophoneRefresh({
enabled: props.open && props.step === 1,
onStart: () => setLoadingMicrophones(true),
onResult: (devices) => {
if (devices) setMicrophones(labelBrowserMicrophones(devices));
setMicrophoneError(
devices ? null : m(msg("Couldn’t list microphones. The system default still works.")),
);
setLoadingMicrophones(false);
},
});
// oxlint-disable-next-line t3code/no-unreasoned-state-sync -- subscribes to native microphone changes while the device setup step is visible
useEffect(() => {
if (!props.open || props.step !== 1) return;
queueMicrotask(() => void refreshMicrophones());
const mediaDevices = navigator.mediaDevices;
mediaDevices?.addEventListener?.("devicechange", refreshMicrophones);
return () => mediaDevices?.removeEventListener?.("devicechange", refreshMicrophones);
}, [props.open, props.step, refreshMicrophones]);
const selectedIsUnavailable = Boolean(
selectedMicrophone && !microphones.some((device) => device.deviceId === selectedMicrophone),
);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import {
} from "@t3tools/contracts";
import { expect, it, vi } from "vite-plus/test";
import { userEvent } from "vite-plus/test/browser";
import { render } from "../../../test/render";
import { act, render } from "../../../test/render";
import { __setClientSettingsForTests, getClientSettings } from "../../hooks/useSettings";
import { readBrowserClientSettings } from "../../clientPersistenceStorage";
import { chooseRecordingMicrophone } from "../../speech/browserMicrophones";
Expand Down Expand Up @@ -746,3 +746,41 @@ it("records from the first connected microphone in the saved order", async () =>
.element(screen.getByRole("button", { name: "Forget System default" }))
.not.toBeInTheDocument();
});

it("keeps an unplugged microphone disconnected when an older scan finishes last", async () => {
const previous = getClientSettings();
using cleanup = new DisposableStack();
cleanup.defer(() => __setClientSettingsForTests(previous));
__setClientSettingsForTests({
...previous,
voiceMicrophones: [{ kind: "device", deviceId: "usb", label: "USB test microphone" }],
});
const microphone = (deviceId: string, label: string): MediaDeviceInfo => ({
kind: "audioinput",
deviceId,
groupId: deviceId,
label,
toJSON: () => ({}),
});
const internal = microphone("internal", "Built-in microphone");
const usb = microphone("usb", "USB test microphone");
const olderScan = deferred<MediaDeviceInfo[]>();
using _enumerate = vi
.spyOn(navigator.mediaDevices, "enumerateDevices")
.mockReturnValueOnce(olderScan.promise)
.mockResolvedValue([internal]);
await using screen = await render(
<SettingsScopeProvider search={{}} onChange={() => {}}>
<VoiceSettingsPanel />
</SettingsScopeProvider>,
);

const microphones = screen.getByRole("list", { name: "Microphones, preferred first" });
const usbRow = microphones.getByRole("listitem").filter({ hasText: "USB test microphone" });
await userEvent.click(screen.getByRole("button", { name: "Refresh microphones" }));
await expect.element(microphones.getByText("Built-in microphone")).toBeVisible();
expect(usbRow.element().textContent).toContain("Not connected");

await act(() => olderScan.resolve([internal, usb]));
expect(usbRow.element().textContent).toContain("Not connected");
});
26 changes: 7 additions & 19 deletions apps/web/src/components/settings/VoiceSettingsPanel.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,8 @@ import { msg, T, useGT, useLocale, useMessages, Var } from "gt-react";
import { useStableCallback } from "~/hooks/useStableCallback";
import { cn } from "~/lib/utils";
import {
listAudioInputs,
updateMicrophoneOrder,
useMicrophoneRefresh,
type BrowserMicrophone,
} from "../../speech/browserMicrophones";
import {
Expand Down Expand Up @@ -1331,25 +1331,13 @@ export function VoiceSettingsPanel() {
const prepared = destination.cloudProvider ? null : destination.prepared;
const catalog = useVoiceModels(prepared);

const refreshMicrophones = useStableCallback(async () => {
if (!navigator.mediaDevices?.enumerateDevices) return;
try {
const devices = await listAudioInputs();
setMicrophones(devices);
updateMicrophoneOrder((order) => rememberMicrophones(order, devices));
} catch {
setMicrophones([]);
}
const refreshMicrophones = useMicrophoneRefresh({
enabled: true,
onResult: (devices) => {
setMicrophones(devices ?? []);
if (devices) updateMicrophoneOrder((order) => rememberMicrophones(order, devices));
},
});
// oxlint-disable-next-line t3code/no-unreasoned-state-sync -- subscribes to microphone device changes while settings are open
useEffect(() => {
queueMicrotask(() => void refreshMicrophones());
const mediaDevices = navigator.mediaDevices;
if (!mediaDevices?.addEventListener) return;
const onChange = () => void refreshMicrophones();
mediaDevices.addEventListener("devicechange", onChange);
return () => mediaDevices.removeEventListener("devicechange", onChange);
}, [refreshMicrophones]);
const modelStatus = catalog.status;
const target = settingsScope?.target;
const members = settingsScope?.scope.members;
Expand Down
51 changes: 51 additions & 0 deletions apps/web/src/speech/browserMicrophones.browser.test.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
import { startTransition, useLayoutEffect, useState } from "react";
import { expect, it, vi } from "vite-plus/test";
import { render } from "../../test/render";
import { useMicrophoneRefresh } from "./browserMicrophones";

it("drops a microphone scan resolved during the commit that removes its view", async () => {
const scan = Promise.withResolvers<MediaDeviceInfo[]>();
const started = Promise.withResolvers<void>();
const onResult = vi.fn();
using _enumerate = vi.spyOn(navigator.mediaDevices, "enumerateDevices").mockImplementation(() => {
started.resolve();
return scan.promise;
});

function MicrophoneView() {
useMicrophoneRefresh({ enabled: true, onResult });
return null;
}

function Replacement() {
useLayoutEffect(() => {
scan.resolve([]);
}, []);
return <p>Microphone view closed</p>;
}

function App() {
const [open, setOpen] = useState(true);
return open ? (
<>
<button type="button" onClick={() => startTransition(() => setOpen(false))}>
Close microphone view
</button>
<MicrophoneView />
</>
) : (
<Replacement />
);
}

// A plain root keeps the replacement commit independent of the router.
await using screen = await render(<App />, { router: false });
await started.promise;
// act would flush passive cleanup before the scan can complete.
screen
.getByRole("button", { name: "Close microphone view" })
.element()
.dispatchEvent(new MouseEvent("click", { bubbles: true }));
await expect.element(screen.getByText("Microphone view closed")).toBeVisible();
expect(onResult).not.toHaveBeenCalled();
});
48 changes: 45 additions & 3 deletions apps/web/src/speech/browserMicrophones.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import { useLayoutEffect, useRef } from "react";
import { useStableCallback } from "../hooks/useStableCallback";
import {
getClientSettings,
isClientSettingsHydrated,
Expand All @@ -15,20 +17,60 @@ import { m } from "../i18n/messages";
export type BrowserMicrophone = Pick<MediaDeviceInfo, "deviceId" | "label">;

/** Audio inputs as the browser reports them. Labels stay empty until microphone permission. */
export async function listAudioInputs(): Promise<BrowserMicrophone[]> {
async function listAudioInputs(): Promise<BrowserMicrophone[]> {
const devices = await navigator.mediaDevices.enumerateDevices();
return devices
.filter((device) => device.kind === "audioinput")
.map(({ deviceId, label }) => ({ deviceId, label }));
}

export async function listBrowserMicrophones(): Promise<BrowserMicrophone[]> {
return (await listAudioInputs()).map(({ deviceId, label }, index) => ({
export function labelBrowserMicrophones(devices: BrowserMicrophone[]): BrowserMicrophone[] {
return devices.map(({ deviceId, label }, index) => ({
deviceId,
label: label || m(msg("Microphone {number}", { number: index + 1 })),
}));
}

/** Delivers the latest scan's result, or null if it fails, while the view is enabled. */
export function useMicrophoneRefresh({
enabled,
onStart,
onResult,
}: {
enabled: boolean;
onStart?: () => void;
onResult: (devices: BrowserMicrophone[] | null) => void;
}) {
// null while the view is closed; otherwise the scan whose result may still be delivered.
const latest = useRef<symbol | null>(null);
const refresh = useStableCallback(async () => {
if (latest.current === null || !navigator.mediaDevices?.enumerateDevices) return;
const scan = Symbol();
latest.current = scan;
onStart?.();
const devices = await listAudioInputs().catch(() => null);
if (latest.current === scan) onResult(devices);
});

// oxlint-disable-next-line t3code/no-unreasoned-state-sync -- observes browser device changes; layout cleanup drops pending scans in the commit that closes the view
useLayoutEffect(() => {
if (!enabled) return;
const opened = Symbol();
latest.current = opened;
queueMicrotask(() => {
if (latest.current === opened) void refresh();
});
const devices = navigator.mediaDevices;
devices?.addEventListener?.("devicechange", refresh);
return () => {
latest.current = null;
devices?.removeEventListener?.("devicechange", refresh);
};
}, [enabled, refresh]);

return refresh;
}

/** Applies a change to the saved order. Skipped before hydration so defaults never replace a saved order. */
export function updateMicrophoneOrder(
change: (order: ReadonlyArray<MicrophoneEntry>) => ReadonlyArray<MicrophoneEntry>,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
/// <reference lib="dom" />
import { expect } from "e2e";

import { test, WEB } from "./t3.ts";

declare global {
interface Window {
microphoneEnumeration?: {
holdOldScan(): void;
readonly oldScanStarted: Promise<void>;
disconnectUsbMicrophone(): void;
releaseOldScan(): void;
};
}
}

test(
"A slow microphone scan cannot bring back a disconnected microphone",
WEB,
async ({ t3, app, browser, screen }) => {
await t3.project("demo");
await browser.addInitScript(() => {
const devices = navigator.mediaDevices;
const internalMic = {
kind: "audioinput",
deviceId: "internal",
label: "Internal test microphone",
};
const usbMic = { kind: "audioinput", deviceId: "usb", label: "USB test microphone" };
let connected = [internalMic, usbMic];
let holdNext = false;
const started = Promise.withResolvers<void>();
const release = Promise.withResolvers<void>();
// Emulate device snapshots when each enumeration starts and vary their delivery order.
Object.defineProperty(devices, "enumerateDevices", {
value: async () => {
const snapshot = connected;
if (holdNext) {
holdNext = false;
started.resolve();
await release.promise;
}
return snapshot;
},
});
window.microphoneEnumeration = {
holdOldScan: () => {
holdNext = true;
},
oldScanStarted: started.promise,
disconnectUsbMicrophone: () => {
connected = [internalMic];
devices.dispatchEvent(new Event("devicechange"));
},
releaseOldScan: release.resolve,
};
});

await t3.open("/settings/voice");
const moveUsbUp = screen.getByRole("button", "Move USB test microphone up", { exact: true });
await moveUsbUp.waitFor();
while (await moveUsbUp.isEnabled()) await moveUsbUp.tap();

const list = screen.getByRole("list", "Microphones, preferred first", { exact: true });
const usbRow = list.getByRole("listitem").filter({ hasText: "USB test microphone" });
await usbRow.getByText("In use", { exact: true }).waitFor();

await browser.evaluate(() => {
window.microphoneEnumeration!.holdOldScan();
return null;
});
await screen.getByRole("button", "Refresh microphones", { exact: true }).tap();
await browser.evaluate(async () => {
await window.microphoneEnumeration!.oldScanStarted;
window.microphoneEnumeration!.disconnectUsbMicrophone();
return null;
});
const disconnected = usbRow.getByText("Not connected", { exact: true });
await disconnected.waitFor();

await browser.evaluate(async () => {
window.microphoneEnumeration!.releaseOldScan();
// Sample after the delivered device list has had a chance to paint.
await new Promise<void>((resolve) =>
requestAnimationFrame(() => requestAnimationFrame(() => resolve())),
);
return null;
});
await app.screenshot("after-stale-microphone-scan");
expect(
await disconnected.isVisible(),
"The unplugged microphone remains marked Not connected",
).toBe(true);
},
);
Loading