diff --git a/apps/server/src/speech/SpeechModelStore.test.ts b/apps/server/src/speech/SpeechModelStore.test.ts
index dda2f3cad4e2..e08a4cf022f4 100644
--- a/apps/server/src/speech/SpeechModelStore.test.ts
+++ b/apps/server/src/speech/SpeechModelStore.test.ts
@@ -13,7 +13,6 @@ import * as Stream from "effect/Stream";
import { HttpClient, HttpClientError, HttpClientRequest, HttpClientResponse } from "effect/http";
import * as ServerConfig from "../config.ts";
-import { getSpeechModel, type SpeechModel } from "./model.ts";
import * as SpeechModelStore from "./SpeechModelStore.ts";
const bytes = Uint8Array.from({ length: 64 }, (_, index) => (index * 7) % 256);
@@ -21,8 +20,9 @@ const SPLIT = 20;
const harness = Effect.gen(function* () {
const digest = yield* Digest.Digest;
- const model: SpeechModel = {
- ...getSpeechModel("handy-computer/whisper-tiny-gguf")!,
+ const model = {
+ id: "fixture/speech-model",
+ revision: "a".repeat(40),
filename: "model.gguf",
size: bytes.length,
sha256: Hex.encode(yield* digest.digest(Digest.sha256, bytes).pipe(Effect.orDie)),
@@ -121,10 +121,8 @@ it.effect("resumes an interrupted transfer, replaces a corrupt same-size file, a
undefined,
`bytes=${SPLIT}-`,
]);
- expect(h.requests.map((request) => request.url)).toEqual([
- "https://huggingface.co/handy-computer/whisper-tiny-gguf/resolve/6687f30c99641ee265df421e582354adbc8848fc/model.gguf",
- "https://huggingface.co/handy-computer/whisper-tiny-gguf/resolve/6687f30c99641ee265df421e582354adbc8848fc/model.gguf",
- ]);
+ const pinnedUrl = `https://huggingface.co/${h.model.id}/resolve/${h.model.revision}/${h.model.filename}`;
+ expect(h.requests.map((request) => request.url)).toEqual([pinnedUrl, pinnedUrl]);
expect(progress.at(0)).toBe(0);
expect(progress.at(-1)).toBe(bytes.length);
expect([...(yield* h.fs.readFile(installed))]).toEqual([...bytes]);
@@ -134,6 +132,9 @@ it.effect("resumes an interrupted transfer, replaces a corrupt same-size file, a
]);
expect(yield* h.store.isReady(h.model)).toBe(true);
expect(yield* h.store.installedPath(h.model)).toBe(installed);
+ const requestsBeforeReuse = h.requests.length;
+ expect(yield* h.store.download(h.model)).toBe(installed);
+ expect(h.requests).toHaveLength(requestsBeforeReuse);
}).pipe(Effect.provide(dependencies)),
);
diff --git a/apps/server/src/speech/SpeechModelStore.ts b/apps/server/src/speech/SpeechModelStore.ts
index b85d5b47f1fb..99f3fe6ca5bb 100644
--- a/apps/server/src/speech/SpeechModelStore.ts
+++ b/apps/server/src/speech/SpeechModelStore.ts
@@ -17,6 +17,8 @@ import * as Stream from "effect/Stream";
import * as ServerConfig from "../config.ts";
import type { SpeechModel } from "./model.ts";
+type ModelFile = Pick;
+
class SpeechModelDownloadError extends Schema.TaggedError()(
"SpeechModelDownloadError",
{
@@ -39,17 +41,17 @@ export class SpeechModelStore extends Context.Service<
SpeechModelStore,
{
/** Cheap size check for status polls; never hashes the file. */
- readonly isReady: (model: SpeechModel) => Effect.Effect;
+ readonly isReady: (model: ModelFile) => Effect.Effect;
/** The installed file once its SHA-256 has been verified, hashing it on first use. */
- readonly installedPath: (model: SpeechModel) => Effect.Effect;
+ readonly installedPath: (model: ModelFile) => Effect.Effect;
readonly download: (
- model: SpeechModel,
+ model: ModelFile,
progress?: SpeechModelDownloadProgress,
) => Effect.Effect<
string,
SpeechModelDownloadError | PlatformError | HttpClientError.HttpClientError
>;
- readonly remove: (model: SpeechModel) => Effect.Effect;
+ readonly remove: (model: ModelFile) => Effect.Effect;
}
>()("t3/speech/SpeechModelStore") {}
@@ -70,11 +72,11 @@ const make = Effect.gen(function* () {
yield* fs.remove(path.join(directory, name), { force: true }).pipe(Effect.ignore);
}
}
- const modelPath = (model: SpeechModel) => path.join(directory, model.filename);
+ const modelPath = (model: ModelFile) => path.join(directory, model.filename);
// Files whose digest failed, by identity, so status polls stop calling them ready.
const corrupt = new Map();
- const identity = (model: SpeechModel, info: FileSystem.File.Info) =>
+ const identity = (model: ModelFile, info: FileSystem.File.Info) =>
[
model.sha256,
info.size,
@@ -85,7 +87,7 @@ const make = Effect.gen(function* () {
const statFile = (file: string) => fs.stat(file).pipe(Effect.orElseSucceed(() => undefined));
- const expectedFile = (model: SpeechModel, file: string) =>
+ const expectedFile = (model: ModelFile, file: string) =>
statFile(file).pipe(
Effect.map((info) =>
info?.type === "File" && Number(info.size) === model.size ? info : undefined,
@@ -93,7 +95,7 @@ const make = Effect.gen(function* () {
);
const digestMatches = Effect.fn("SpeechModelStore.digest")(
- function* (model: SpeechModel, file: string) {
+ function* (model: ModelFile, file: string) {
const hash = NodeCrypto.createHash("sha256");
yield* fs
.stream(file)
@@ -105,7 +107,7 @@ const make = Effect.gen(function* () {
// A verified file is trusted while its size, mtime, device and inode are unchanged.
const verifiedPath = Effect.fn("SpeechModelStore.verifiedPath")(function* (
- model: SpeechModel,
+ model: ModelFile,
file: string,
onVerificationStart?: () => void,
) {
@@ -127,7 +129,7 @@ const make = Effect.gen(function* () {
});
const fetchInto = Effect.fn("SpeechModelStore.fetch")(function* (
- model: SpeechModel,
+ model: ModelFile,
url: string,
partialPath: string,
offset: number,
diff --git a/apps/server/src/speech/SpeechService.test.ts b/apps/server/src/speech/SpeechService.test.ts
index 3c93ba62f31f..972c401ce309 100644
--- a/apps/server/src/speech/SpeechService.test.ts
+++ b/apps/server/src/speech/SpeechService.test.ts
@@ -10,19 +10,22 @@ import * as Result from "effect/Result";
import * as Scheduler from "effect/Scheduler";
import * as Scope from "effect/Scope";
import * as Exit from "effect/Exit";
+import * as PlatformError from "effect/PlatformError";
import * as HostProcess from "@t3tools/shared/HostProcess";
import { DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS } from "@t3tools/contracts";
import * as ServerSettings from "../serverSettings.ts";
import { listNativeSpeechGpuDevices } from "./native.ts";
import * as SpeechModelStore from "./SpeechModelStore.ts";
import * as SpeechService from "./SpeechService.ts";
+import { SILERO_VAD } from "./silero.ts";
const native = vi.hoisted(() => ({
backend: "Vulkan0",
supportsStreaming: true,
+ supportsLongForm: false,
supportsTranslation: true,
supportsVocabulary: false,
- begin: vi.fn(async (_language?: string, _vocabulary?: readonly string[]) => {}),
+ begin: vi.fn(async (_options?: unknown, _filterSilence?: boolean, _vadPath?: string) => {}),
feed: vi.fn(async (_pcm: Float32Array) => ({
revision: 1,
text: { committed: "", tentative: "hello" },
@@ -66,13 +69,13 @@ const modelDefinitions = vi.hoisted(() => [
name: "fallback",
description: "fallback",
size: 8,
- languages: ["en"],
+ languages: ["en", "fr"],
accuracy: 2,
speed: 2,
recommended: false,
supportsStreaming: false,
supportsLanguageDetection: false,
- supportsTranslation: false,
+ supportsTranslation: true,
},
]);
vi.mock("./native.ts", () => ({
@@ -112,8 +115,10 @@ const pcm = () => new Uint8Array(new Float32Array([0.25]).buffer);
beforeEach(() => {
vi.clearAllMocks();
loadNative.mockReset();
+ native.finish.mockReset();
native.backend = "Vulkan0";
native.supportsVocabulary = false;
+ native.supportsStreaming = true;
readyModels.clear();
readyModels.add("test-model");
readyModels.add("fallback-model");
@@ -146,6 +151,21 @@ it.effect("prepares a batch model before audio arrives and reuses it for transcr
yield* speech.prepareModel;
expect(loadNative).toHaveBeenCalledTimes(1);
expect(native.transcribe).not.toHaveBeenCalled();
+ native.supportsStreaming = false;
+ yield* Effect.gen(function* () {
+ const stream = yield* speech.startStream({
+ ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
+ speechLanguage: "fr",
+ speechTranslateToEnglish: true,
+ speechSilenceFiltering: false,
+ });
+ expect(native.begin).toHaveBeenLastCalledWith(
+ { timestamps: "none", language: "fr", task: "translate", targetLanguage: "en" },
+ false,
+ "test.gguf",
+ );
+ expect(yield* stream.finish).toBe("hello");
+ }).pipe(Effect.scoped);
expect(yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)).toBe("hello");
expect(loadNative).toHaveBeenCalledTimes(1);
}).pipe(Effect.provide(layer)),
@@ -223,28 +243,112 @@ it.effect("keeps a detached download exclusive until the transfer finishes", ()
const speech = yield* SpeechService.SpeechService;
const started = yield* Deferred.make();
const finish = yield* Deferred.make();
- download.mockImplementationOnce(() =>
- Deferred.succeed(started, undefined).pipe(
- Effect.andThen(Deferred.await(finish)),
- Effect.as("test.gguf"),
- ),
- );
+ const helperStarted = yield* Deferred.make();
+ let helperStartedDuringInstall = false;
+ let helperInterrupted = false;
+ download
+ .mockImplementationOnce(() =>
+ Deferred.succeed(started, undefined).pipe(
+ Effect.andThen(Deferred.await(finish)),
+ Effect.as("test.gguf"),
+ ),
+ )
+ .mockImplementationOnce(() =>
+ Deferred.isDone(finish).pipe(
+ Effect.tap((done) =>
+ Effect.sync(() => {
+ helperStartedDuringInstall = !done;
+ }),
+ ),
+ Effect.andThen(Deferred.succeed(helperStarted, undefined)),
+ Effect.andThen(Effect.never),
+ Effect.onInterrupt(() =>
+ Effect.sync(() => {
+ helperInterrupted = true;
+ }),
+ ),
+ ),
+ );
const request = yield* speech.downloadModel("test-model").pipe(Effect.forkChild);
yield* Deferred.await(started);
+ yield* Deferred.await(helperStarted);
yield* Fiber.interrupt(request);
+ yield* Deferred.succeed(finish, undefined);
+ yield* TestClock.adjust("16 seconds");
+ expect(helperInterrupted).toBe(false);
const removed = yield* Effect.result(speech.removeModel("test-model"));
expect(Result.isFailure(removed) && removed.failure).toMatchObject({ _tag: "SpeechBusyError" });
const join = yield* speech
.downloadModel("test-model")
.pipe(Effect.forkChild({ startImmediately: true }));
- yield* Deferred.succeed(finish, undefined);
+ yield* TestClock.adjust("2 minutes");
+ expect(helperInterrupted).toBe(true);
yield* Fiber.join(join);
- expect(download).toHaveBeenCalledTimes(1);
+ expect(download.mock.calls.map(([model]) => model.id)).toEqual(["test-model", SILERO_VAD.id]);
+ expect(helperStartedDuringInstall).toBe(true);
yield* speech.removeModel("test-model");
expect(readyModels.has("test-model")).toBe(false);
}).pipe(Effect.provide(layer)),
);
+it.effect("bounds missing Silero recovery during recording and continues with raw audio", () =>
+ Effect.gen(function* () {
+ const helperStarted = yield* Deferred.make();
+ const helperInterrupted = yield* Deferred.make();
+ download.mockImplementationOnce(() =>
+ Deferred.succeed(helperStarted, undefined).pipe(
+ Effect.andThen(Effect.never),
+ Effect.onInterrupt(() => Deferred.succeed(helperInterrupted, undefined)),
+ ),
+ );
+ const speech = yield* SpeechService.SpeechService;
+ const recording = yield* speech
+ .startStream(DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)
+ .pipe(Effect.forkChild);
+ yield* Deferred.await(helperStarted);
+ yield* TestClock.adjust("14999 millis");
+ expect(native.begin).not.toHaveBeenCalled();
+ yield* TestClock.adjust("1 millis");
+ yield* Deferred.await(helperInterrupted);
+ const stream = yield* Fiber.join(recording);
+ expect(native.begin).toHaveBeenLastCalledWith(expect.anything(), true, undefined);
+ yield* stream.feed(pcm());
+ expect(yield* stream.finish).toBe("hello");
+ }).pipe(Effect.scoped, Effect.provide(layer)),
+);
+
+it.effect("keeps the installed speech model usable when optional Silero acquisition fails", () =>
+ Effect.gen(function* () {
+ download
+ .mockImplementationOnce(() => Effect.succeed("test.gguf"))
+ .mockImplementationOnce(() =>
+ Effect.fail(
+ new PlatformError.PlatformError(
+ new PlatformError.SystemError({
+ _tag: "NotFound",
+ module: "FileSystem",
+ method: "open",
+ }),
+ ),
+ ),
+ );
+ const speech = yield* SpeechService.SpeechService;
+ const installed = yield* speech.downloadModel("test-model");
+ expect(installed.models.find((model) => model.id === "test-model")?.state).toBe("installed");
+ expect(yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)).toBe("hello");
+ yield* Effect.scoped(
+ Effect.gen(function* () {
+ const stream = yield* speech.startStream(DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS);
+ expect(yield* stream.finish).toBe("hello");
+ }),
+ );
+ expect(download).toHaveBeenCalledTimes(2);
+ yield* TestClock.adjust("10 minutes");
+ expect(yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)).toBe("hello");
+ expect(download).toHaveBeenCalledTimes(3);
+ }).pipe(Effect.provide(layer)),
+);
+
it.effect(
"allows another model operation after an already-installed download completes immediately",
() =>
@@ -333,7 +437,11 @@ it.effect("uses vocabulary only for supported transcription and always applies a
text: { committed: vocabulary ? "git hub" : "GitHub", tentative: "Bob" },
});
expect(yield* stream.finish).toBe(vocabulary ? "git hub and Bob" : "GitHub and Bob");
- expect(native.begin.mock.lastCall).toEqual(["fr", vocabulary]);
+ expect(native.begin.mock.lastCall).toEqual([
+ { timestamps: "none", language: "fr", ...(vocabulary ? { vocabulary } : {}) },
+ true,
+ "test.gguf",
+ ]);
}).pipe(Effect.scoped, Effect.provide(layer));
}
}
@@ -354,6 +462,58 @@ it.effect("unloads after the chosen idle period and reloads on the next recordin
}).pipe(Effect.provide(layer)),
);
+it.effect.each([
+ { cue: "um", remove: true, text: "Make it red, um, blue.", expected: "Make it red, um, blue." },
+ {
+ cue: "uh no",
+ remove: true,
+ text: "Make it red, uh no, blue.",
+ expected: "Make it red, uh no, blue.",
+ },
+ {
+ cue: "uh no",
+ remove: true,
+ text: "Uh, Make it red, UH NO, blue. Uh nope.",
+ expected: "Make it red, UH NO, blue. nope.",
+ },
+ {
+ cue: " UM ",
+ remove: true,
+ text: "Uh, Make it red, um, blue.",
+ expected: "Make it red, um, blue.",
+ },
+ {
+ cue: undefined,
+ remove: true,
+ text: "Uh, Make it red, um, blue.",
+ expected: "Make it red, blue.",
+ },
+ {
+ cue: "um",
+ remove: false,
+ text: "Uh, Make it red, um, blue.",
+ expected: "Uh, Make it red, um, blue.",
+ },
+])(
+ "preserves only an explicit correction cue during batch and stream filler removal: $cue / $remove",
+ ({ cue, remove, text, expected }) =>
+ Effect.gen(function* () {
+ const speech = yield* SpeechService.SpeechService;
+ const options = {
+ ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
+ speechLanguage: "en",
+ speechRemoveFillerWords: remove,
+ speechCustomWords: [{ term: "um", aliases: [] }],
+ ...(cue === undefined ? {} : { speechCorrectionWord: cue }),
+ };
+ native.transcribe.mockResolvedValueOnce({ text });
+ native.finish.mockResolvedValueOnce(text);
+ expect(yield* speech.transcribe(pcm(), options)).toBe(expected);
+ const stream = yield* speech.startStream(options);
+ expect(yield* stream.finish).toBe(expected);
+ }).pipe(Effect.scoped, Effect.provide(layer)),
+);
+
it.effect("holds model ownership until the stream scope closes and preserves silence", () =>
Effect.gen(function* () {
const speech = yield* SpeechService.SpeechService;
@@ -560,9 +720,13 @@ it.effect("replays streamed audio on CPU after an Auto-placed feed crashes, once
};
expect(yield* stream.feed(pcm())).toEqual({ revision: 5, text: recovered });
expect(loadNative.mock.calls.slice(-2).map((call) => call[3])).toEqual(["auto", "cpu"]);
- expect(native.begin.mock.calls.slice(-2)).toEqual([
- ["en", ["GitHub", "Bob"]],
- ["en", cpuSupportsVocabulary ? ["GitHub", "Bob"] : undefined],
+ expect(native.begin.mock.calls.slice(-2).map(([options]) => options)).toEqual([
+ { timestamps: "none", language: "en", vocabulary: ["GitHub", "Bob"] },
+ {
+ timestamps: "none",
+ language: "en",
+ ...(cpuSupportsVocabulary ? { vocabulary: ["GitHub", "Bob"] } : {}),
+ },
]);
expect(native.feed).toHaveBeenCalledTimes(feedCalls + 4);
expect(yield* stream.feed(pcm())).toEqual({ revision: 6, text: recovered });
@@ -583,7 +747,7 @@ it.effect.each([false, true])(
const cpu = {
...native,
backend: "CPU",
- begin: vi.fn(async (_language?: string) => {}),
+ begin: vi.fn(async (_options?: unknown) => {}),
feed: vi.fn(async (_pcm: Float32Array) => ({
revision: 1,
text: { committed: "", tentative: "" },
@@ -611,7 +775,11 @@ it.effect.each([false, true])(
expect(cpu.feed.mock.calls.map(([chunk]) => [...chunk])).toEqual(
chunks.map((chunk) => [...chunk]),
);
- expect(cpu.begin).toHaveBeenCalledWith("fr", undefined);
+ expect(cpu.begin).toHaveBeenCalledWith(
+ { timestamps: "none", language: "fr" },
+ true,
+ "test.gguf",
+ );
expect(gpu.dispose).toHaveBeenCalledOnce();
expect(cpu.dispose).not.toHaveBeenCalled();
const busy = yield* Effect.result(speech.selectModel("fallback-model"));
diff --git a/apps/server/src/speech/SpeechService.ts b/apps/server/src/speech/SpeechService.ts
index d1ad3b22a945..c1f762f26089 100644
--- a/apps/server/src/speech/SpeechService.ts
+++ b/apps/server/src/speech/SpeechService.ts
@@ -32,6 +32,7 @@ import {
type SpeechModel,
} from "./model.ts";
import { listNativeSpeechGpuDevices, loadNativeSpeechModel } from "./native.ts";
+import { SILERO_VAD } from "./silero.ts";
import { isSpeechPlatformSupported } from "./platform.ts";
import * as SpeechModelStore from "./SpeechModelStore.ts";
@@ -39,14 +40,13 @@ const SAMPLE_RATE = 16_000;
const MAX_SPEECH_DURATION_SECONDS = 5 * 60;
export const MAX_SPEECH_BYTES =
SAMPLE_RATE * Float32Array.BYTES_PER_ELEMENT * MAX_SPEECH_DURATION_SECONDS;
-const MIN_CAPTURE_RMS = 0.0005;
export class SpeechInvalidAudioError extends Schema.TaggedError()(
"SpeechInvalidAudioError",
{ byteLength: Schema.Number, message: Schema.String },
) {}
-export function decodeSpeechPcm(pcmBytes: Uint8Array, preserveSilence = false): Float32Array {
+export function decodeSpeechPcm(pcmBytes: Uint8Array): Float32Array {
if (
pcmBytes.byteLength === 0 ||
pcmBytes.byteLength > MAX_SPEECH_BYTES ||
@@ -60,7 +60,6 @@ export function decodeSpeechPcm(pcmBytes: Uint8Array, preserveSilence = false):
// WebSocket frames can be pooled Node Buffers whose backing storage is larger
// than the frame. Copy the view so only the received bytes are decoded.
const pcm = new Float32Array(Uint8Array.from(pcmBytes).buffer);
- let energy = 0;
for (const sample of pcm) {
if (!Number.isFinite(sample) || sample < -1 || sample > 1) {
throw new SpeechInvalidAudioError({
@@ -68,11 +67,8 @@ export function decodeSpeechPcm(pcmBytes: Uint8Array, preserveSilence = false):
message: "audio contains an invalid PCM sample",
});
}
- energy += sample * sample;
}
- return !preserveSilence && Math.sqrt(energy / pcm.length) < MIN_CAPTURE_RMS
- ? new Float32Array()
- : pcm;
+ return pcm;
}
export class SpeechOperationError extends Schema.TaggedError()(
@@ -281,6 +277,23 @@ const make = Effect.gen(function* () {
const selectedModel = (settings: SettingsSnapshot): SpeechModel =>
getSpeechModel(settings.speechModelId) ?? getSpeechModel(DEFAULT_SPEECH_MODEL_ID)!;
+ let silenceRetryAt = 0;
+ const acquireSilenceModel = Effect.fn("SpeechService.acquireSilenceModel")(function* (
+ purpose: "installation" | "recording",
+ ) {
+ if (purpose === "recording" && now() < silenceRetryAt) return undefined;
+ const path = yield* store.download(SILERO_VAD).pipe(
+ Effect.timeout(purpose === "recording" ? "15 seconds" : "2 minutes"),
+ Effect.orElseSucceed(() => undefined),
+ );
+ if (path !== undefined) {
+ silenceRetryAt = 0;
+ return path;
+ }
+ silenceRetryAt = now() + 10 * 60_000;
+ yield* Effect.logWarning("Silero unavailable; using raw audio with bounded ASR processing.");
+ });
+
// Publish the reservation before forking: a cached transfer can finish before forkIn returns.
const downloadFiber = Effect.fn("SpeechService.downloadFiber")(function* (modelId: string) {
const definition = getSpeechModel(modelId);
@@ -306,6 +319,9 @@ const make = Effect.gen(function* () {
},
})
.pipe(
+ (modelDownload) =>
+ Effect.all([modelDownload, acquireSilenceModel("installation")], { concurrency: 2 }),
+ Effect.map(([modelPath]) => modelPath),
Effect.mapError((cause) => speechError("model download", cause)),
Effect.forkIn(scope),
);
@@ -442,6 +458,7 @@ const make = Effect.gen(function* () {
model: definition.name,
size: definition.size,
supportsStreaming: definition.supportsStreaming,
+ supportsBufferedStreaming: true,
supportsTranslation: definition.supportsTranslation,
languages: definition.languages,
supportsLanguageDetection: definition.supportsLanguageDetection,
@@ -512,7 +529,9 @@ const make = Effect.gen(function* () {
language: string | undefined,
) => {
return (
- options.speechRemoveFillerWords ? removeSpeechFillerWords(text, language) : text
+ options.speechRemoveFillerWords
+ ? removeSpeechFillerWords(text, language, options.speechCorrectionWord)
+ : text
).trim();
};
@@ -543,6 +562,15 @@ const make = Effect.gen(function* () {
),
);
+ const silenceModelPath = Effect.fn("SpeechService.silenceModelPath")(function* (
+ loaded: LoadedModel,
+ options: SpeechTranscriptionOptions,
+ ) {
+ if (!options.speechSilenceFiltering && (loaded.supportsStreaming || loaded.supportsLongForm))
+ return undefined;
+ return yield* acquireSilenceModel("recording");
+ });
+
const transcribe: SpeechService["Service"]["transcribe"] = Effect.fn("SpeechService.transcribe")(
function* (pcmBytes, options) {
const settings = yield* readSettings("transcription");
@@ -574,6 +602,7 @@ const make = Effect.gen(function* () {
),
catch: (cause) => speechError("model preparation", cause),
});
+ const vadPath = yield* silenceModelPath(loaded, options);
const dictionary = normalizeSpeechCustomWords(options.speechCustomWords);
const customWords = dictionary.map(({ term }) => term);
// The vocabulary capability guarantees plain transcription, not translation.
@@ -584,14 +613,19 @@ const make = Effect.gen(function* () {
try: (signal) => {
// Interruption kills the isolated process; the next request reloads the model.
signal.addEventListener("abort", () => dropModel(inference), { once: true });
- return inference.transcribe(pcm, {
- timestamps: "none",
- ...(language === "auto" ? {} : { language }),
- ...(translateToEnglish
- ? { task: "translate" as const, targetLanguage: "en" as const }
- : {}),
- ...(vocabularyFor(inference).length > 0 ? { vocabulary: customWords } : {}),
- });
+ return inference.transcribe(
+ pcm,
+ {
+ timestamps: "none",
+ ...(language === "auto" ? {} : { language }),
+ ...(translateToEnglish
+ ? { task: "translate" as const, targetLanguage: "en" as const }
+ : {}),
+ ...(vocabularyFor(inference).length > 0 ? { vocabulary: customWords } : {}),
+ },
+ options.speechSilenceFiltering,
+ vadPath,
+ );
},
catch: (cause) => {
dropModel(inference);
@@ -703,12 +737,28 @@ const make = Effect.gen(function* () {
loadModel(modelPath, definition, signal, settings.speechAcceleration),
);
loaded = streamModel;
+ const vadPath = yield* silenceModelPath(streamModel, options);
+ let translateToEnglish = false;
const beginStream = async () => {
- if (!streamModel.supportsStreaming)
- throw new Error("The selected model does not support streaming.");
+ translateToEnglish =
+ options.speechTranslateToEnglish &&
+ !streamModel.supportsStreaming &&
+ streamModel.supportsTranslation &&
+ language !== "en";
const vocabulary =
- streamModel.supportsVocabulary && customWords.length > 0 ? customWords : undefined;
- await streamModel.begin(language === "auto" ? undefined : language, vocabulary);
+ streamModel.supportsVocabulary && !translateToEnglish && customWords.length
+ ? customWords
+ : undefined;
+ await streamModel.begin(
+ {
+ timestamps: "none",
+ ...(language === "auto" ? {} : { language }),
+ ...(translateToEnglish ? { task: "translate", targetLanguage: "en" } : {}),
+ ...(vocabulary ? { vocabulary } : {}),
+ },
+ options.speechSilenceFiltering,
+ vadPath,
+ );
correct = makeSpeechTextCorrector(vocabulary ? [] : customWords, dictionary);
started = true;
};
@@ -768,7 +818,7 @@ const make = Effect.gen(function* () {
byteLength: bytes.byteLength,
message: "Speech stream audio limit exceeded.",
});
- const pcm = decodeSpeechPcm(bytes, true);
+ const pcm = decodeSpeechPcm(bytes);
byteLength += bytes.byteLength;
received.push(pcm);
let update: Awaited> | undefined;
@@ -806,7 +856,7 @@ const make = Effect.gen(function* () {
return finalizeTranscript(
correct(text),
options,
- language === "auto" ? undefined : language,
+ translateToEnglish ? "en" : language === "auto" ? undefined : language,
);
}),
};
diff --git a/apps/server/src/speech/audio.test.ts b/apps/server/src/speech/audio.test.ts
new file mode 100644
index 000000000000..d049f8da51cb
--- /dev/null
+++ b/apps/server/src/speech/audio.test.ts
@@ -0,0 +1,92 @@
+import { expect, it } from "vite-plus/test";
+import { createSpeechAudio } from "./audio.ts";
+
+const frame = 512;
+const make = (overrides = {}) =>
+ createSpeechAudio({
+ frameSamples: frame,
+ filterSilence: true,
+ nativeStreaming: false,
+ longForm: false,
+ maxAudioMs: 0,
+ ...overrides,
+ });
+const frames = (count: number) =>
+ Float32Array.from({ length: count * frame }, (_, i) => (Math.floor(i / frame) + 1) / 1000);
+const seconds = (n: number) => Math.floor((n * 16000) / frame);
+
+it("retains pre-roll, onset and offline/streaming tails without duplicating edges", () => {
+ for (const nativeStreaming of [false, true]) {
+ const audio = make({ nativeStreaming });
+ const pcm = frames(100);
+ const chunks: Float32Array[] = [];
+ for (let i = 0; i < 100; i++) {
+ audio.append(pcm.subarray(i * frame, (i + 1) * frame));
+ audio.score(i, [i >= 20 && i < 24 ? 0.9 : 0]);
+ chunks.push(...audio.take());
+ }
+ chunks.push(...audio.take(true));
+ const retained = chunks.flatMap((chunk) => [...chunk]);
+ expect(retained).toEqual([
+ ...pcm.subarray(6 * frame, (24 + (nativeStreaming ? 52 : 15)) * frame),
+ ]);
+ }
+});
+
+it("handles silence, short speech, a partial last frame and disabled filtering", () => {
+ const silence = make();
+ silence.append(new Float32Array(frame * 4 + 7));
+ silence.score(0, [0, 0, 0, 0, 0]);
+ expect(silence.take(true)).toEqual([]);
+ const short = make();
+ short.append(new Float32Array([0.2, 0.1]));
+ short.score(0, [0.8]);
+ expect([...short.take(true)[0]!]).toEqual([...new Float32Array([0.2, 0.1])]);
+ const raw = make({ filterSilence: false });
+ const pcm = frames(50).subarray(0, 50 * frame - 3);
+ raw.append(pcm);
+ raw.score(0, new Array(50).fill(0));
+ expect(raw.take(true)[0]).toEqual(pcm);
+});
+
+it("plans at pauses while recording and preserves ordered audio and final tails", () => {
+ const audio = make({ filterSilence: false });
+ const pcm = frames(seconds(100));
+ const chunks: Float32Array[] = [];
+ for (let start = 0; start < pcm.length; start += frame * 7) {
+ const piece = pcm.subarray(start, start + frame * 7);
+ audio.append(piece);
+ audio.score(
+ start / frame,
+ Array.from({ length: piece.length / frame }, (_, i) => {
+ const t = ((start / frame + i) * frame) / 16000;
+ return (t >= 28 && t < 29) || (t >= 55 && t < 56) ? 0 : 1;
+ }),
+ );
+ chunks.push(...audio.take());
+ }
+ expect(chunks.length).toBeGreaterThan(0);
+ expect(chunks[0]!.length / 16000).toBeCloseTo(28.5, 1);
+ chunks.push(...audio.take(true));
+ expect(Float32Array.from(chunks.flatMap((chunk) => [...chunk]))).toEqual(pcm);
+ expect(chunks.every((chunk) => chunk.length <= 40 * 16000)).toBe(true);
+ expect(chunks.at(-1)!.length).toBeGreaterThanOrEqual(5 * 16000);
+});
+
+it("respects smaller engine limits, hard cuts, and native/internal-long-form eligibility", () => {
+ for (const config of [{}, { maxAudioMs: 4000 }, { nativeStreaming: true }, { longForm: true }]) {
+ const audio = make({ filterSilence: false, ...config });
+ const pcm = frames(seconds(84));
+ audio.append(pcm);
+ audio.score(0, new Array(pcm.length / frame).fill(1));
+ const live = audio.take();
+ const chunks = [...live, ...audio.take(true)];
+ expect(chunks.reduce((n, chunk) => n + chunk.length, 0)).toBe(pcm.length);
+ if ("nativeStreaming" in config || "longForm" in config) expect(chunks).toHaveLength(1);
+ else
+ expect(
+ chunks.every((chunk) => chunk.length <= ("maxAudioMs" in config ? 32000 : 640000)),
+ ).toBe(true);
+ if ("longForm" in config) expect(live).toEqual([]);
+ }
+});
diff --git a/apps/server/src/speech/audio.ts b/apps/server/src/speech/audio.ts
new file mode 100644
index 000000000000..62f31f182c8a
--- /dev/null
+++ b/apps/server/src/speech/audio.ts
@@ -0,0 +1,174 @@
+// Self-contained because the native worker embeds this factory in Node, Electron and SEA.
+export function createSpeechAudio(config: {
+ frameSamples: number;
+ filterSilence: boolean;
+ nativeStreaming: boolean;
+ longForm: boolean;
+ maxAudioMs: number;
+}) {
+ const rate = 16_000;
+ const frame = config.frameSamples;
+ const prefill = Math.ceil((450 * rate) / 1000 / frame);
+ const onset = Math.ceil((60 * rate) / 1000 / frame);
+ const tail = Math.ceil(((config.nativeStreaming ? 1650 : 450) * rate) / 1000 / frame);
+ const eligible = !config.nativeStreaming && !config.longForm;
+ // Preserve the existing output-limit headroom and bounded offline retries.
+ const maximum = Math.min(
+ 40 * rate,
+ config.maxAudioMs > 0 ? Math.max(1, Math.floor((config.maxAudioMs * rate) / 2000)) : 40 * rate,
+ );
+ const minimum = Math.min(5 * rate, Math.floor(maximum / 4));
+ const target = Math.min(30 * rate, maximum * 0.75);
+ const blocks: { start: number; pcm: Float32Array }[] = [];
+ const probs: number[] = [];
+ const keep: boolean[] = [];
+ let length = 0;
+ let start = 0;
+ let streamAt = 0;
+ let inSpeech = false;
+ let onsetCount = 0;
+ let hangover = 0;
+
+ const raw = (from: number, to: number) => {
+ const output = new Float32Array(to - from);
+ for (const block of blocks) {
+ const left = Math.max(from, block.start);
+ const right = Math.min(to, block.start + block.pcm.length);
+ if (left < right)
+ output.set(block.pcm.subarray(left - block.start, right - block.start), left - from);
+ }
+ return output;
+ };
+ const read = (from: number, to: number) => {
+ const pcm = raw(from, to);
+ if (!config.filterSilence) return pcm;
+ let count = 0;
+ for (let at = from; at < to;) {
+ const end = Math.min(to, (Math.floor(at / frame) + 1) * frame);
+ if (keep[Math.floor(at / frame)]) {
+ pcm.copyWithin(count, at - from, end - from);
+ count += end - at;
+ }
+ at = end;
+ }
+ return pcm.subarray(0, count);
+ };
+ // Silero timestamp hysteresis is separate from the capture filter's 0.3 threshold.
+ const pauses = (samples: number, final: boolean) => {
+ const segments: { start: number; end: number }[] = [];
+ let speechStart: number | undefined;
+ let silenceStart: number | undefined;
+ for (let i = 0; i < probs.length; i++) {
+ const at = i * frame;
+ if (probs[i]! >= 0.5) {
+ silenceStart = undefined;
+ speechStart ??= at;
+ } else if (probs[i]! < 0.35 && speechStart !== undefined) {
+ silenceStart ??= at;
+ if (at - silenceStart >= 0.1 * rate) {
+ if (silenceStart - speechStart > 0.25 * rate)
+ segments.push({ start: speechStart, end: silenceStart });
+ speechStart = silenceStart = undefined;
+ }
+ }
+ }
+ if (speechStart !== undefined && samples - speechStart > 0.25 * rate)
+ segments.push({ start: speechStart, end: samples });
+ const candidates: { cut: number; gap: number }[] = [];
+ let end = 0;
+ for (const segment of segments) {
+ if (segment.start > end)
+ candidates.push({
+ cut: Math.floor((end + segment.start) / 2),
+ gap: (segment.start - end) / rate,
+ });
+ end = segment.end;
+ }
+ // An open pause's midpoint can still move, so only final audio uses it.
+ if (final && samples > end)
+ candidates.push({ cut: Math.floor((end + samples) / 2), gap: (samples - end) / rate });
+ return candidates;
+ };
+
+ return {
+ append(pcm: Float32Array) {
+ blocks.push({ start: length, pcm });
+ length += pcm.length;
+ },
+ score(firstFrame: number, scores: readonly number[]) {
+ if (firstFrame !== probs.length || scores.some((p) => !Number.isFinite(p) || p < 0 || p > 1))
+ throw new Error("Invalid Silero frame sequence.");
+ for (const probability of scores) {
+ const i = probs.length;
+ probs.push(probability);
+ keep.push(false);
+ const voiced = probability > 0.3;
+ if (!inSpeech && voiced) {
+ if (++onsetCount >= onset) {
+ inSpeech = true;
+ hangover = tail;
+ onsetCount = 0;
+ for (let j = Math.max(0, i - prefill); j <= i; j++) keep[j] = true;
+ }
+ } else if (inSpeech && voiced) {
+ hangover = tail;
+ keep[i] = true;
+ } else if (inSpeech && hangover > 0) {
+ hangover--;
+ keep[i] = true;
+ } else {
+ inSpeech = false;
+ onsetCount = 0;
+ }
+ }
+ },
+ audio: () => raw(0, length),
+ samples: () => length,
+ take(final = false): Float32Array[] {
+ const samples = Math.min(length, probs.length * frame);
+ // Preserve a very short utterance or syllable stopped before onset confirms.
+ if (final && onsetCount > 0)
+ for (let i = Math.max(0, probs.length - 1 - prefill); i < probs.length; i++) keep[i] = true;
+ if (config.nativeStreaming) {
+ const output = read(streamAt, samples);
+ let lastKept = streamAt;
+ for (let i = Math.floor(streamAt / frame); i < probs.length; i++)
+ if (!config.filterSilence || keep[i]) lastKept = Math.min(samples, (i + 1) * frame);
+ streamAt = Math.max(lastKept, Math.max(0, samples - prefill * frame));
+ return output.length ? [output] : [];
+ }
+ const chunks: Float32Array[] = [];
+ const lookahead = Math.max(minimum, (prefill + onset) * frame);
+ if (eligible && samples - start > maximum + (final ? 0 : lookahead)) {
+ const candidates = pauses(samples, final);
+ while (samples - start > maximum + (final ? 0 : lookahead)) {
+ let best: { cut: number; score: number } | undefined;
+ let fallback: typeof best;
+ for (const candidate of candidates) {
+ const position = candidate.cut - start;
+ if (position < minimum || position > maximum) continue;
+ const offTarget = (position - target) / rate;
+ const score =
+ Math.min(candidate.gap, 1) - Math.abs(offTarget) * (offTarget > 0 ? 0.1 : 0.05);
+ if (!fallback || score > fallback.score) fallback = { cut: candidate.cut, score };
+ if (samples - candidate.cut >= minimum && (!best || score > best.score))
+ best = { cut: candidate.cut, score };
+ }
+ const hard =
+ start +
+ (samples - start < maximum + minimum ? Math.floor((samples - start) / 2) : maximum);
+ const cut = (best ?? fallback)?.cut ?? hard;
+ const chunk = read(start, cut);
+ if (chunk.length) chunks.push(chunk);
+ start = cut;
+ }
+ }
+ if (final && samples > start) {
+ const chunk = read(start, samples);
+ if (chunk.length) chunks.push(chunk);
+ start = samples;
+ }
+ return chunks;
+ },
+ };
+}
diff --git a/apps/server/src/speech/fillerWords.ts b/apps/server/src/speech/fillerWords.ts
index 0181db1bf916..5061341833c0 100644
--- a/apps/server/src/speech/fillerWords.ts
+++ b/apps/server/src/speech/fillerWords.ts
@@ -22,19 +22,21 @@ const LANGUAGE_FILLER_WORDS: Readonly> = {
const escapeRegex = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-export function removeSpeechFillerWords(text: string, language?: string): string {
+export function removeSpeechFillerWords(
+ text: string,
+ language?: string,
+ correctionCue?: string,
+): string {
const baseLanguage = language?.split(/[-_]/, 1)[0]?.toLowerCase();
const words = [
...UNIVERSAL_FILLER_WORDS,
...(baseLanguage ? (LANGUAGE_FILLER_WORDS[baseLanguage] ?? []) : []),
];
- let filtered = text;
- for (const word of words) {
- const pattern = new RegExp(
- `(? cue ?? "");
return filtered.replace(/\s{2,}/g, " ").trim();
}
diff --git a/apps/server/src/speech/native.test.ts b/apps/server/src/speech/native.test.ts
index da92597d06e7..97aa1b566020 100644
--- a/apps/server/src/speech/native.test.ts
+++ b/apps/server/src/speech/native.test.ts
@@ -39,7 +39,11 @@ it("returns transcription from an isolated process and disposes it", async () =>
);
try {
expect(
- await model.transcribe(new Float32Array([0.25]), { timestamps: "none", language: "en" }),
+ await model.transcribe(
+ new Float32Array([0.25]),
+ { timestamps: "none", language: "en" },
+ false,
+ ),
).toEqual({ text: "0.25" });
} finally {
await model.dispose();
@@ -60,7 +64,7 @@ it("probes generic vocabulary support and delivers ordered terms to offline infe
language: "en",
...(supported ? { vocabulary: ["T3 Code", "GitHub"] } : {}),
};
- const result = await model.transcribe(new Float32Array([0.25]), options);
+ const result = await model.transcribe(new Float32Array([0.25]), options, false);
expect(JSON.parse(result.text)).toEqual(options);
} finally {
await model.dispose();
@@ -76,7 +80,7 @@ it("ignores IPC messages that do not belong to the speech protocol", async () =>
);
try {
await expect(
- model.transcribe(new Float32Array([0.25]), { timestamps: "none", language: "en" }),
+ model.transcribe(new Float32Array([0.25]), { timestamps: "none", language: "en" }, false),
).resolves.toEqual({ text: "hello" });
} finally {
await model.dispose();
@@ -90,10 +94,14 @@ it("terminates hung native inference when the service shuts down", async () => {
controller.signal,
fixture("async () => { while (true) {} }"),
);
- const pending = model.transcribe(new Float32Array([0.25]), {
- timestamps: "none",
- language: "en",
- });
+ const pending = model.transcribe(
+ new Float32Array([0.25]),
+ {
+ timestamps: "none",
+ language: "en",
+ },
+ false,
+ );
const rejected = expect(pending).rejects.toThrow();
controller.abort();
await model.dispose();
@@ -111,17 +119,22 @@ it("returns streaming previews and final text from an isolated process", async (
);
try {
expect(model.supportsStreaming).toBe(true);
- await model.begin();
- expect(await model.feed(new Float32Array([0.25]))).toEqual({
+ await model.begin({ timestamps: "none" }, false);
+ expect(await model.feed(new Float32Array(512).fill(0.25))).toEqual({
revision: 1,
text: { committed: "hello ", tentative: "world" },
});
expect(await model.finish()).toBe("hello world");
- await model.begin("fr");
+ await model.begin({ timestamps: "none", language: "fr" }, false);
await model.reset();
- await model.begin("fr");
+ await model.begin({ timestamps: "none", language: "fr" }, false);
+ await model.feed(new Float32Array(512));
expect(await model.finish()).toBe("fr");
- await model.begin("fr", ["T3 Code", "GitHub"]);
+ await model.begin(
+ { timestamps: "none", language: "fr", vocabulary: ["T3 Code", "GitHub"] },
+ false,
+ );
+ await model.feed(new Float32Array(512).fill(0.25));
expect(await model.finish()).toBe("T3 Code / GitHub");
} finally {
await model.dispose();
@@ -135,9 +148,170 @@ it("kills a hung streaming feed without waiting for native cleanup", async () =>
controller.signal,
streamingFixture("async () => { while (true) {} }"),
);
- await model.begin();
- const rejected = expect(model.feed(new Float32Array([0.25]))).rejects.toThrow();
+ await model.begin({ timestamps: "none" }, false);
+ const rejected = expect(model.feed(new Float32Array(512).fill(0.25))).rejects.toThrow();
controller.abort();
await model.dispose();
await rejected;
});
+const vadFixture = ({
+ streaming = false,
+ failVad = false,
+ failVadLoad = false,
+ failAsr = false,
+ maxAudioMs = 4000,
+} = {}) =>
+ `data:text/javascript,${encodeURIComponent(`
+ const summarize = pcm => String(pcm.length) + ":" + pcm[0].toFixed(2) + ":" + pcm[pcm.length - 1].toFixed(2);
+ export const TranscribeModel = { load: async (path, options) => {
+ if (path === "vad.gguf") {
+ if (${failVadLoad}) throw new Error("VAD unavailable");
+ if (options.backend !== "cpu") throw new Error("VAD must use CPU");
+ return { vadInfo: { sampleRate: 16000, frameSamples: 512 }, dispose() {},
+ createVadSession: ({ nThreads }) => {
+ if (nThreads !== 1) throw new Error("VAD thread count");
+ let pending = []; let at = 0;
+ const score = final => {
+ const firstFrame = at;
+ const probs = [];
+ while (pending.length >= 512 || final && pending.length) {
+ probs.push(pending[0] > 0.5 ? 0.9 : 0);
+ pending.splice(0, 512); at++;
+ }
+ return { firstFrame, probs };
+ };
+ return { dispose() {}, streamReset: async () => { pending = []; at = 0; },
+ streamFeed: async pcm => {
+ if (${failVad} && pcm[0] < 0) throw new Error("VAD fixture failed");
+ pending.push(...pcm); return score(false);
+ },
+ streamFlush: async () => score(true),
+ };
+ },
+ };
+ }
+ return { backend: "CPU", supports: () => false,
+ capabilities: { supportsStreaming: ${streaming}, maxAudioMs: ${maxAudioMs} },
+ transcribe: async pcm => { if (${failAsr}) throw new Error("ASR fixture failed"); return { text: summarize(pcm) }; },
+ createSession: () => {
+ let all = [];
+ return { dispose() {}, stream: async () => ({
+ feed: async pcm => { all.push(...pcm); return { committedChanged: true }; },
+ finalize: async () => {}, reset() { all = []; },
+ get text() { const full = summarize(all); return { committed: full, tentative: "", full }; },
+ }) };
+ },
+ };
+ }};
+ `)}`;
+
+it("filters silence and preserves partial speech through the production worker and resets between recordings", async () => {
+ for (const streaming of [false, true]) {
+ const model = await loadNativeSpeechModel(
+ "asr.gguf",
+ new AbortController().signal,
+ vadFixture({ streaming }),
+ "cpu",
+ );
+ try {
+ expect(
+ await model.transcribe(new Float32Array(2000), { timestamps: "none" }, true, "vad.gguf"),
+ ).toEqual({ text: "" });
+ expect(
+ await model.transcribe(
+ new Float32Array([0.8, 0.6]),
+ { timestamps: "none" },
+ true,
+ "vad.gguf",
+ ),
+ ).toEqual({ text: "2:0.80:0.60" });
+ expect(
+ await model.transcribe(new Float32Array(2000), { timestamps: "none" }, false, "vad.gguf"),
+ ).toEqual({ text: "2000:0.00:0.00" });
+ for (const cancel of [true, false]) {
+ await model.begin({ timestamps: "none" }, true, "vad.gguf");
+ await model.feed(new Float32Array(600));
+ expect(await model.finish()).toBe("");
+ await model.begin({ timestamps: "none" }, true, "vad.gguf");
+ await model.feed(new Float32Array(511).fill(0.8));
+ if (cancel) await model.reset();
+ else expect(await model.finish()).toBe("511:0.80:0.80");
+ }
+ } finally {
+ await model.dispose();
+ }
+ }
+});
+
+it("decodes complete stable chunks before Stop and returns the ordered final tail", async () => {
+ const model = await loadNativeSpeechModel(
+ "asr.gguf",
+ new AbortController().signal,
+ vadFixture(),
+ "cpu",
+ );
+ try {
+ await model.begin({ timestamps: "none" }, false, "vad.gguf");
+ await model.feed(new Float32Array(16000).fill(0.1));
+ await model.feed(new Float32Array(16000).fill(0.2));
+ const update = await model.feed(new Float32Array(16000).fill(0.3));
+ expect(update.text).toEqual({ committed: "32000:0.10:0.20", tentative: "" });
+ await model.feed(new Float32Array([0.4, 0.5]));
+ expect(await model.finish()).toBe("32000:0.10:0.20 16002:0.30:0.50");
+ } finally {
+ await model.dispose();
+ }
+});
+
+it("preserves withheld audio on VAD failure while keeping ASR failures distinct", async () => {
+ for (const streaming of [false, true]) {
+ const model = await loadNativeSpeechModel(
+ "asr.gguf",
+ new AbortController().signal,
+ vadFixture({ streaming, failVad: true }),
+ "cpu",
+ );
+ try {
+ await model.begin({ timestamps: "none" }, true, "vad.gguf");
+ await model.feed(new Float32Array(512).fill(0.1));
+ await model.feed(new Float32Array(512).fill(-0.2));
+ expect(await model.finish()).toBe("1024:0.10:-0.20");
+ await model.begin({ timestamps: "none" }, true, "vad.gguf");
+ await model.feed(new Float32Array(512));
+ expect(await model.finish()).toBe("");
+ } finally {
+ await model.dispose();
+ }
+ }
+ const unavailable = await loadNativeSpeechModel(
+ "asr.gguf",
+ new AbortController().signal,
+ vadFixture({ failVadLoad: true }),
+ "cpu",
+ );
+ try {
+ expect(
+ await unavailable.transcribe(
+ new Float32Array(2000).fill(0.1),
+ { timestamps: "none" },
+ true,
+ "vad.gguf",
+ ),
+ ).toEqual({ text: "2000:0.10:0.10" });
+ } finally {
+ await unavailable.dispose();
+ }
+ const model = await loadNativeSpeechModel(
+ "asr.gguf",
+ new AbortController().signal,
+ vadFixture({ failAsr: true }),
+ "cpu",
+ );
+ try {
+ await expect(
+ model.transcribe(new Float32Array(512).fill(0.8), { timestamps: "none" }, true, "vad.gguf"),
+ ).rejects.toThrow("ASR fixture failed");
+ } finally {
+ await model.dispose();
+ }
+});
diff --git a/apps/server/src/speech/native.ts b/apps/server/src/speech/native.ts
index 88809ab97663..fae308333b24 100644
--- a/apps/server/src/speech/native.ts
+++ b/apps/server/src/speech/native.ts
@@ -2,16 +2,142 @@
import * as NodeChildProcess from "node:child_process";
import * as NodeSea from "node:sea";
import type { SpeechStreamText } from "@t3tools/contracts";
+import { createSpeechAudio } from "./audio.ts";
// Inline the small child entry so the packaged CLI does not need a separate worker artifact.
const entry = `
+const createSpeechAudio = ${createSpeechAudio.toString()};
+let speech;
let model;
+let vadModel;
+let vad;
let session;
let stream;
+let audio;
+let options;
+let nativeStreaming;
+let useVad;
+let scored;
+let revision;
+let texts;
+let heardAudio;
+const warnVad = () => console.warn("Silero silence filtering unavailable; preserving raw recording and using bounded ASR chunks.");
+const resetAsr = () => {
+ if (stream) stream.reset();
+ if (session) session.dispose();
+ stream = session = undefined;
+};
+const newAudio = (filterSilence) => createSpeechAudio({
+ frameSamples: 512, filterSilence, nativeStreaming: model.capabilities.supportsStreaming,
+ longForm: model.supports("long_form"), maxAudioMs: model.capabilities.maxAudioMs,
+});
+const openAsr = async () => {
+ if (nativeStreaming) {
+ session = model.createSession();
+ stream = await session.stream(options);
+ }
+};
+const begin = async (message, live) => {
+ resetAsr();
+ options = message.options;
+ nativeStreaming = live && model.capabilities.supportsStreaming;
+ texts = [];
+ scored = revision = 0;
+ heardAudio = false;
+ useVad = Boolean(message.vadPath) && (message.filterSilence ||
+ (!model.capabilities.supportsStreaming && !model.supports("long_form")));
+ if (useVad) {
+ try {
+ if (!vadModel) {
+ vadModel = await speech.TranscribeModel.load(message.vadPath, { backend: "cpu" });
+ if (vadModel.vadInfo.sampleRate !== 16000 || vadModel.vadInfo.frameSamples !== 512)
+ throw new Error("Unsupported Silero model frames.");
+ vad = vadModel.createVadSession({ nThreads: 1 });
+ }
+ await vad.streamReset();
+ } catch {
+ vad?.dispose();
+ vadModel?.dispose();
+ vad = vadModel = undefined;
+ useVad = false;
+ warnVad();
+ }
+ }
+ audio = newAudio(useVad && message.filterSilence);
+ await openAsr();
+};
+const transcribe = async (pcm) => {
+ try {
+ const result = await model.transcribe(pcm, options);
+ texts.push(result.text);
+ } catch (error) {
+ if ((error?.constructor?.name !== "OutputTruncated" &&
+ error?.constructor?.name !== "InputTooLong") || pcm.length <= 4000) throw error;
+ const middle = Math.floor(pcm.length / 2);
+ await transcribe(pcm.subarray(0, middle));
+ await transcribe(pcm.subarray(middle));
+ }
+};
+const decode = async (final) => {
+ let preview = null;
+ for (const pcm of audio.take(final)) {
+ if (!pcm.length) continue;
+ heardAudio = true;
+ if (nativeStreaming) {
+ const update = await stream.feed(pcm);
+ if (update.committedChanged || update.tentativeChanged) {
+ const text = stream.text;
+ preview = { committed: text.committed, tentative: text.tentative };
+ }
+ } else {
+ await transcribe(pcm);
+ preview = { committed: texts.join(" ").trim(), tentative: "" };
+ }
+ }
+ return preview;
+};
+const feed = async (pcm, final = false) => {
+ if (pcm) audio.append(pcm);
+ try {
+ if (useVad) {
+ if (pcm) {
+ const result = await vad.streamFeed(pcm);
+ audio.score(result.firstFrame, result.probs);
+ if (result.firstFrame + result.probs.length !== Math.floor(audio.samples() / 512))
+ throw new Error("Incomplete Silero frames.");
+ }
+ if (final) {
+ const result = await vad.streamFlush();
+ audio.score(result.firstFrame, result.probs);
+ if (result.firstFrame + result.probs.length !== Math.ceil(audio.samples() / 512))
+ throw new Error("Incomplete Silero tail.");
+ }
+ } else {
+ const count = final ? Math.ceil(audio.samples() / 512) : Math.floor(audio.samples() / 512);
+ audio.score(scored, new Array(count - scored).fill(1));
+ scored = count;
+ }
+ } catch {
+ // Replay on optional VAD failure, including audio it had withheld; ASR errors stay separate.
+ const raw = audio.audio();
+ useVad = false;
+ warnVad();
+ resetAsr();
+ texts = [];
+ heardAudio = false;
+ audio = newAudio(false);
+ audio.append(raw);
+ scored = final ? Math.ceil(raw.length / 512) : Math.floor(raw.length / 512);
+ audio.score(0, new Array(scored).fill(1));
+ await openAsr();
+ }
+ return await decode(final);
+};
process.on("message", async (message) => {
try {
if (message.kind === "load") {
- const { TranscribeModel, getAvailableBackends } = await loadSpeechModule(message.moduleUrl);
+ speech = await loadSpeechModule(message.moduleUrl);
+ const { TranscribeModel, getAvailableBackends } = speech;
const device = message.acceleration.startsWith("gpu:")
? getAvailableBackends().find((item) =>
(item.deviceType === "gpu" || item.deviceType === "igpu") &&
@@ -19,80 +145,40 @@ process.on("message", async (message) => {
: undefined;
if (message.acceleration.startsWith("gpu:") && !device)
throw new Error("The selected speech GPU is unavailable.");
- model = await TranscribeModel.load(message.path, device
- ? { device }
- : { backend: message.acceleration });
+ model = await TranscribeModel.load(message.path, device ? { device } : { backend: message.acceleration });
process.send({ type: "t3-speech-reply", ok: true, backend: model.backend,
supportsStreaming: model.capabilities.supportsStreaming,
+ supportsLongForm: model.supports("long_form"),
supportsTranslation: model.capabilities.supportsTranslate,
supportsVocabulary: model.supports("vocabulary") });
} else if (message.kind === "begin") {
- session = model.createSession();
- stream = await session.stream({ timestamps: "none", language: message.language,
- vocabulary: message.vocabulary });
+ await begin(message, true);
process.send({ type: "t3-speech-reply", ok: true });
} else if (message.kind === "feed") {
- const update = await stream.feed(message.pcm);
- const text = update.committedChanged || update.tentativeChanged ? stream.text : null;
- process.send({ type: "t3-speech-reply", ok: true, revision: update.revision,
- preview: text ? { committed: text.committed, tentative: text.tentative } : null });
+ const preview = await feed(message.pcm);
+ process.send({ type: "t3-speech-reply", ok: true, revision: ++revision, preview });
} else if (message.kind === "finish") {
- await stream.finalize();
- const text = stream.text.full;
- stream.reset();
- session.dispose();
- stream = session = undefined;
+ await feed(undefined, true);
+ if (nativeStreaming && heardAudio) await stream.finalize();
+ const text = nativeStreaming ? (heardAudio ? stream.text.full : "") : texts.join(" ").trim();
+ resetAsr();
+ audio = undefined;
process.send({ type: "t3-speech-reply", ok: true, text });
} else if (message.kind === "reset") {
- stream.reset();
- session.dispose();
- stream = session = undefined;
+ resetAsr();
+ if (vad) await vad.streamReset();
+ audio = undefined;
process.send({ type: "t3-speech-reply", ok: true });
} else if (message.kind === "transcribe") {
- // Leave unbounded models alone. Use headroom for output-bound models:
- // maxAudioMs is an estimate, so dense speech may still need smaller pieces.
- const limit = model.capabilities.maxAudioMs;
- const chunkSamples = limit > 0 ? Math.max(4000, Math.floor(limit * 16 / 2)) : message.pcm.length;
- const texts = [];
- const transcribe = async (pcm) => {
- try {
- const result = await model.transcribe(pcm, message.options);
- texts.push(result.text);
- } catch (error) {
- if ((error?.constructor?.name !== "OutputTruncated" &&
- error?.constructor?.name !== "InputTooLong") || pcm.length <= 4000) throw error;
- const middle = Math.floor(pcm.length / 2);
- await transcribe(pcm.subarray(0, middle));
- await transcribe(pcm.subarray(middle));
- }
- };
- let offset = 0;
- while (offset < message.pcm.length) {
- let end = Math.min(offset + chunkSamples, message.pcm.length);
- if (end < message.pcm.length) {
- // Prefer the quietest 20 ms frame in the final quarter of the window.
- let quietest = Infinity;
- const windowEnd = end;
- for (let start = offset + Math.floor(chunkSamples * 0.75); start + 320 <= windowEnd; start += 320) {
- let energy = 0;
- for (let i = start; i < start + 320; i++) energy += message.pcm[i] ** 2;
- if (energy < quietest) {
- quietest = energy;
- end = start + 160;
- }
- }
- }
- await transcribe(message.pcm.subarray(offset, end));
- offset = end;
- }
- process.send({ type: "t3-speech-reply", ok: true, text: texts.join(" ").trim() });
+ await begin(message, false);
+ await feed(message.pcm, true);
+ const text = texts.join(" ").trim();
+ audio = undefined;
+ process.send({ type: "t3-speech-reply", ok: true, text });
}
} catch (error) {
- process.send({
- type: "t3-speech-reply",
- ok: false,
- error: error instanceof Error ? error.stack ?? error.message : String(error),
- });
+ process.send({ type: "t3-speech-reply", ok: false,
+ error: error instanceof Error ? error.stack ?? error.message : String(error) });
}
});
process.on("disconnect", () => process.exit(0));
@@ -104,6 +190,7 @@ type Reply = {
readonly text?: string;
readonly error?: string;
readonly supportsStreaming?: boolean;
+ readonly supportsLongForm?: boolean;
readonly supportsTranslation?: boolean;
readonly supportsVocabulary?: boolean;
readonly backend?: string;
@@ -305,10 +392,11 @@ async function loadNativeSpeechModelOnce(
return {
backend: loaded.backend ?? "unknown",
supportsStreaming: loaded.supportsStreaming === true,
+ supportsLongForm: loaded.supportsLongForm === true,
supportsTranslation: loaded.supportsTranslation === true,
supportsVocabulary: loaded.supportsVocabulary === true,
- begin: async (language?: string, vocabulary?: readonly string[]) => {
- await send({ kind: "begin", language, vocabulary });
+ begin: async (options: NativeSpeechOptions, filterSilence: boolean, vadPath?: string) => {
+ await send({ kind: "begin", options, filterSilence, vadPath });
},
feed: async (pcm: Float32Array) => {
const reply = await send({ kind: "feed", pcm });
@@ -343,18 +431,22 @@ async function loadNativeSpeechModelOnce(
},
transcribe: async (
pcm: Float32Array,
- options: {
- readonly timestamps: "none";
- readonly task?: "translate";
- readonly targetLanguage?: "en";
- readonly language?: string;
- readonly vocabulary?: readonly string[];
- },
+ options: NativeSpeechOptions,
+ filterSilence: boolean,
+ vadPath?: string,
) => {
- const result = await send({ kind: "transcribe", pcm, options });
+ const result = await send({ kind: "transcribe", pcm, options, filterSilence, vadPath });
if (typeof result.text !== "string") throw new Error("Invalid speech process response.");
return { text: result.text };
},
dispose,
};
}
+
+type NativeSpeechOptions = {
+ readonly timestamps: "none";
+ readonly task?: "translate";
+ readonly targetLanguage?: "en";
+ readonly language?: string;
+ readonly vocabulary?: readonly string[];
+};
diff --git a/apps/server/src/speech/silero.ts b/apps/server/src/speech/silero.ts
new file mode 100644
index 000000000000..dceddb71dde5
--- /dev/null
+++ b/apps/server/src/speech/silero.ts
@@ -0,0 +1,8 @@
+// MIT-licensed internal helper; acquired and verified by the existing model store.
+export const SILERO_VAD = {
+ id: "handy-computer/silero-vad-v6.2-gguf",
+ revision: "97bbd75179f535e4708bccdf61e9278b8b60dfc7",
+ filename: "silero-vad-v6.2-F32.gguf",
+ size: 1_241_056,
+ sha256: "0d871a800c420fdd5d1b17a3aea4296d1c39608f0ff4a9295eb76457eb1c88e6",
+};
diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx
index 4408bee38ad0..8e4183244480 100644
--- a/apps/web/src/components/ChatView.tsx
+++ b/apps/web/src/components/ChatView.tsx
@@ -3245,6 +3245,7 @@ export default function ChatView(props: ChatViewProps) {
: undefined,
}}
files={{
+ projectId: activeProject?.id,
projectName: activeProject?.title ?? null,
workspaceRoot: activeWorkspaceRoot,
availableEditors,
@@ -3694,6 +3695,7 @@ export default function ChatView(props: ChatViewProps) {
}
activeThreadModelSelection={activeThread?.modelSelection}
activeProjectId={activeProject?.id ?? null}
+ projectName={activeProject?.title}
contextWindowEntries={serverVisibleTurnItems}
contextWindowLiveUsage={activeThreadLiveTokenUsage}
contextWindowProviderThread={activeProviderThread}
diff --git a/apps/web/src/components/DiffPanel.browser.test.tsx b/apps/web/src/components/DiffPanel.browser.test.tsx
index 3157c3e5da2a..fb2fc81bae84 100644
--- a/apps/web/src/components/DiffPanel.browser.test.tsx
+++ b/apps/web/src/components/DiffPanel.browser.test.tsx
@@ -84,6 +84,7 @@ vi.mock("~/state/use-atom-command", () => ({ useAtomCommand: () => async () => {
vi.mock("~/editorPreferences", () => ({ useOpenInPreferredEditor: () => async () => {} }));
vi.mock("~/fileContextMenu", () => ({ useFileContextMenuHandler: () => () => {} }));
vi.mock("~/browser/useOpenLink", () => ({ useOpenLink: () => async () => {} }));
+vi.mock("./diffs/DiffCommentAnnotation", () => ({ DiffCommentAnnotation: () => null }));
vi.mock("./diffs/DiffFileTree", () => ({ DiffFileTree: () => null }));
vi.mock("./DiffFilePathCopyButton", () => ({ DiffFilePathCopyButton: () => null }));
@@ -250,7 +251,7 @@ it.each([false, true])(
const toggle = screen.getByRole("button", { name: `Expand ${path}`, exact: true });
await expect.element(toggle).toHaveAttribute("aria-expanded", "false");
await userEvent.click(toggle);
- act(() =>
+ await act(() =>
context.registry.set(
context.patch,
AsyncResult.success({
diff --git a/apps/web/src/components/DiffPanel.tsx b/apps/web/src/components/DiffPanel.tsx
index f40760f847ef..e7e6d614ef77 100644
--- a/apps/web/src/components/DiffPanel.tsx
+++ b/apps/web/src/components/DiffPanel.tsx
@@ -1425,6 +1425,8 @@ export default function DiffPanel({
>
(null);
const commentPopupRef = useRef(null);
const draftCommentRef = useRef(null);
+ const [voiceSetupOpen, setVoiceSetupOpen] = useState(false);
const [unavailableSourceAnchor, setUnavailableSourceAnchor] =
useState(null);
const commentOpen = commentEditor?.open ?? false;
@@ -167,7 +168,7 @@ export function AssistantCitationChip({
{composerSourceLink}
{commentEditor ? (
{
if (!open && !settleDraftOnClose(eventDetails.reason)) {
eventDetails.cancel();
@@ -185,6 +186,8 @@ export function AssistantCitationChip({
{commentEditor.open ? (
{
draftCommentRef.current = comment;
commentEditor.onDraftChange?.(comment);
diff --git a/apps/web/src/components/chat/AssistantCitationCommentEditor.tsx b/apps/web/src/components/chat/AssistantCitationCommentEditor.tsx
index 0a8357bb9bab..cdd8e1249cfc 100644
--- a/apps/web/src/components/chat/AssistantCitationCommentEditor.tsx
+++ b/apps/web/src/components/chat/AssistantCitationCommentEditor.tsx
@@ -1,7 +1,10 @@
import { ASSISTANT_CITATION_MAX_COMMENT_LENGTH, type AssistantCitation } from "@t3tools/contracts";
-import { useState, type Ref } from "react";
+import { use, useEffect, useRef, useState, type RefObject } from "react";
+
+import { TextFieldSpeechControls, useTextFieldSpeech } from "~/speech/useTextFieldSpeech";
import { textDirection } from "~/textDirection";
+import { ComposerContextActionsContext } from "../composerContextActions";
import { Button } from "../ui/button";
export function AssistantCitationCommentEditor({
@@ -11,26 +14,55 @@ export function AssistantCitationCommentEditor({
onSubmitAndSend,
onCancel,
onDraftChange,
+ onVoiceSetupOpenChange,
}: {
citation: AssistantCitation;
- inputRef?: Ref;
+ inputRef?: RefObject;
onSubmit: (comment: string) => boolean;
onSubmitAndSend?: (comment: string) => boolean;
onCancel: () => void;
onDraftChange?: (comment: string) => void;
+ onVoiceSetupOpenChange?: (open: boolean) => void;
}) {
const [comment, setComment] = useState(citation.comment ?? "");
+ const localInputRef = useRef(null);
+ const textareaRef = inputRef ?? localInputRef;
+ const { environmentId, projectId } = use(ComposerContextActionsContext);
+ const updateComment = (text: string) => {
+ setComment(text);
+ onDraftChange?.(text);
+ };
+ const speech = useTextFieldSpeech({
+ environmentId: environmentId ?? citation.environmentId,
+ projectId,
+ ownerKey: JSON.stringify([
+ "citation",
+ environmentId,
+ citation.threadId,
+ citation.messageId,
+ citation.start,
+ citation.end,
+ ]),
+ text: comment,
+ textareaRef,
+ onTextChange: updateComment,
+ });
+ const setupOpen = speech.setup.open;
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- keeps the citation popover open while the voice setup dialog owns focus
+ useEffect(() => {
+ onVoiceSetupOpenChange?.(setupOpen);
+ return () => onVoiceSetupOpenChange?.(false);
+ }, [onVoiceSetupOpenChange, setupOpen]);
const commentTooLong = comment.length > ASSISTANT_CITATION_MAX_COMMENT_LENGTH;
const submit = () => {
- if (!commentTooLong) onSubmit(comment);
+ speech.submitAfterDictation((text) => {
+ if (text.length <= ASSISTANT_CITATION_MAX_COMMENT_LENGTH) onSubmit(text);
+ });
};
const submitAndSend = () => {
- if (commentTooLong) return;
- if (onSubmitAndSend) {
- onSubmitAndSend(comment);
- } else {
- onSubmit(comment);
- }
+ speech.submitAfterDictation((text) => {
+ if (text.length <= ASSISTANT_CITATION_MAX_COMMENT_LENGTH) (onSubmitAndSend ?? onSubmit)(text);
+ });
};
return (
@@ -41,12 +73,14 @@ export function AssistantCitationCommentEditor({
if (event.nativeEvent.isComposing || event.keyCode === 229) return;
if (event.key === "Escape") {
event.preventDefault();
- onCancel();
+ if (speech.state.phase !== "idle") speech.cancel();
+ else onCancel();
}
}}
>
) : null}
-
+
+
event.preventDefault()}
onClick={submit}
>
diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx
index e4515f759fd7..69022a93fbee 100644
--- a/apps/web/src/components/chat/ChatComposer.tsx
+++ b/apps/web/src/components/chat/ChatComposer.tsx
@@ -1,3 +1,4 @@
+import { VoiceInputSetup } from "./VoiceInputSetup";
import { useComposerTypingGuard } from "./useComposerTypingGuard";
import { insertAttachmentReferencesForDraft } from "./composerAttachmentReferences";
import { DESKTOP_PASTE_AS_TEXT_EVENT } from "../../lib/desktopPasteAsText";
@@ -71,6 +72,7 @@ import {
type ComposerSubmissionIntent,
collapseExpandedComposerCursor,
composerRequiresModifier,
+ composerSubmissionIntentForKey,
composerStateAtPromptEnd,
detectComposerTrigger,
expandCollapsedComposerCursor,
@@ -135,6 +137,10 @@ import {
useAttachmentUploadStore,
} from "../../lib/attachmentUploadQueue";
import { attachmentUploadBlockReason } from "../../lib/attachmentUploadState";
+import { useDictationShortcut } from "../../speech/useDictationShortcut";
+import { useDictationSubmissionShortcut } from "../../speech/useDictationSubmissionShortcut";
+import { useEnvironmentSpeechInput } from "../../speech/useEnvironmentSpeechInput";
+import { ComposerSpeechButton, ComposerSpeechRecordingPill } from "./ComposerSpeechButton";
import type { AssistantCitationSourceAnchor } from "~/lib/assistantTextSelection";
import { shortcutLabelForCommand } from "../../keybindings";
import {
@@ -427,6 +433,7 @@ export interface ChatComposerProps {
activeThreadModelSelection: ModelSelection | null | undefined;
reportedModelSelection?: ModelSelection | null;
activeProjectId: ProjectId | null;
+ projectName?: string | undefined;
contextWindowEntries: Parameters[0];
contextWindowLiveUsage: Parameters[1];
@@ -554,6 +561,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
activeThreadModelSelection,
reportedModelSelection,
activeProjectId,
+ projectName,
contextWindowEntries,
contextWindowLiveUsage,
contextWindowProviderThread,
@@ -728,6 +736,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
const composerContextActions = {
environmentId,
cwd: gitCwd ?? undefined,
+ projectId: activeProjectId ?? undefined,
expandImage: (imageId: string) => {
const preview = buildExpandedImagePreview(composerImages, imageId);
if (preview) onExpandImage(preview);
@@ -1833,6 +1842,54 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
};
const readComposerSnapshotStable = useStableCallback(readComposerSnapshot);
+ const dictationDisabled =
+ !canOperateThread ||
+ environmentUnavailable !== null ||
+ projectSelectionRequired ||
+ isComposerApprovalState ||
+ activePendingIsResponding ||
+ activePendingProgress?.activeQuestion?.allowCustomAnswer === false;
+ const speechInput = useEnvironmentSpeechInput({
+ environmentId,
+ projectId: activeProjectId,
+ projectName,
+ ownerKey: questionAttachmentTarget ?? composerDraftTargetKey,
+ draftText: prompt,
+ disabled: dictationDisabled,
+ readDraft: () => {
+ const snapshot = readComposerSnapshot();
+ return {
+ text: snapshot.value,
+ selection: composerEditorRef.current?.readSelectionRange() ?? {
+ start: snapshot.expandedCursor,
+ end: snapshot.expandedCursor,
+ },
+ };
+ },
+ commitDraft: (text, selection) => {
+ const snapshot = readComposerSnapshot();
+ const applied = applyPromptReplacement(0, snapshot.value.length, text, {
+ expectedText: snapshot.value,
+ focusEditorAfterReplace: false,
+ });
+ if (!applied) return;
+ const cursor = collapseExpandedComposerCursor(text, selection.start, isLiteralPendingAnswer);
+ setComposerCursor(cursor);
+ window.requestAnimationFrame(() => composerEditorRef.current?.focusAt(cursor));
+ },
+ });
+ const { blocksSubmission: voiceInputActive, submitAfterDictation } = speechInput;
+ const voiceSendDisabledReason =
+ sendDisabledReason ?? (speechInput.preparing ? "Preparing voice input" : null);
+ const dictationShortcutLabel = useDictationShortcut({
+ keybindings,
+ speech: speechInput,
+ disabled: dictationDisabled,
+ terminalOpen,
+ modelPickerOpen: isComposerModelPickerOpen,
+ ...(backgroundThread ? { targetRef: composerFormRef } : {}),
+ });
+
const resolveActiveComposerTrigger = (): {
snapshot: { value: string; cursor: number; expandedCursor: number };
trigger: ComposerTrigger | null;
@@ -2076,11 +2133,19 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
return showPlanFollowUpPrompt || composerSendState.hasSendableContent;
};
+ const submitComposerRef = useRef<
+ (dispatchMode?: ComposerDispatchMode, submissionIntent?: ComposerSubmissionIntent) => void
+ >(() => {});
const submitComposer = (
event?: { preventDefault: () => void },
dispatchMode?: ComposerDispatchMode,
submissionIntent?: ComposerSubmissionIntent,
) => {
+ if (voiceInputActive) {
+ event?.preventDefault();
+ submitAfterDictation(() => submitComposerRef.current(dispatchMode, submissionIntent));
+ return;
+ }
if (
noProviderAvailable ||
isSendDisabled ||
@@ -2138,6 +2203,38 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
}
};
const submitComposerStable = useStableCallback(submitComposer);
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- publishes the committed submission handler to the asynchronous dictation completion
+ useLayoutEffect(() => {
+ submitComposerRef.current = (mode, intent) => submitComposerStable(undefined, mode, intent);
+ }, [submitComposerStable]);
+ useDictationSubmissionShortcut({
+ enabled: voiceInputActive && !speechInput.preparing,
+ formRef: composerFormRef,
+ onKeyDown: (event) => {
+ const intent = composerSubmissionIntentForKey({
+ event,
+ keybindings,
+ isSoftwareKeyboardOpen,
+ isDraftThread: routeKind === "draft",
+ isRunning: phase === "running",
+ sendShortcut: settings.sendShortcut,
+ prompt: promptRef.current,
+ });
+ if (!intent) return;
+ event.preventDefault();
+ event.stopImmediatePropagation();
+ submitComposer(
+ undefined,
+ resolveComposerDispatchMode({
+ running: phase === "running",
+ alternateModifier: intent === "alternate",
+ canSteer: !isCompacting,
+ activeTurnDefault: followUpBehavior,
+ }),
+ intent,
+ );
+ },
+ });
const handleSubmitMessage = (event: React.MouseEvent) => {
event.preventDefault();
submitComposer(
@@ -3444,7 +3541,8 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
disabled={
projectSelectionRequired ||
isChoiceOnlyPendingQuestion ||
- activePendingIsResponding
+ activePendingIsResponding ||
+ speechInput.freezesEditor
}
/>
@@ -3462,7 +3560,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
showPlanFollowUpPrompt={false}
promptHasText={false}
isSendBusy={isSendBusy}
- sendDisabledReason={sendDisabledReason}
+ sendDisabledReason={voiceSendDisabledReason}
isEnvironmentUnavailable={
environmentUnavailable !== null ||
noProviderAvailable ||
@@ -3484,13 +3582,25 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
message={providerInputSubmissionError ?? composerSubmissionError}
/>
- {isComposerCollapsedMobile ? null : (
+ {speechInput.preview ? (
+
+ {speechInput.preview.committed}
+ {speechInput.preview.tentative}
+
+ ) : null}
+
+ {isComposerCollapsedMobile && speechInput.state.phase === "idle" ? null : (
0 && "pt-2",
- showMobilePendingAnswerActions && "hidden sm:flex",
+ showMobilePendingAnswerActions &&
+ speechInput.state.phase === "idle" &&
+ "hidden sm:flex",
)}
>
{composerAttachButton ?
{composerAttachButton}
: null}
@@ -3541,16 +3651,17 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
showPlanFollowUpPrompt={
pendingUserInputs.length === 0 && showPlanFollowUpPrompt
}
- promptHasText={prompt.trim().length > 0}
+ promptHasText={prompt.trim().length > 0 || voiceInputActive}
+ voiceInputActive={voiceInputActive}
isSendBusy={isSendBusy}
- sendDisabledReason={sendDisabledReason}
+ sendDisabledReason={voiceSendDisabledReason}
isEnvironmentUnavailable={
environmentUnavailable !== null ||
noProviderAvailable ||
projectSelectionRequired
}
isPreparingWorktree={isPreparingWorktree}
- hasSendableContent={composerSendState.hasSendableContent}
+ hasSendableContent={composerSendState.hasSendableContent || voiceInputActive}
canResume={showResumeAction}
preserveComposerFocusOnPointerDown={isMobileViewport}
isEditingQueuedMessage={isEditingQueuedMessage}
@@ -3567,7 +3678,45 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
{...(compactCommandAvailable
? { onCompactContext: compactThreadContextStable }
: {})}
- />
+ >
+ {speechInput.available ? (
+ <>
+
void speechInput.start()}
+ onStop={() => void speechInput.stop()}
+ onCancel={() => speechInput.cancel()}
+ onSkipPostProcessing={() => speechInput.skipPostProcessing()}
+ />
+ void speechInput.start()}
+ onCancel={() => speechInput.cancel()}
+ />
+ >
+ ) : null}
+ void speechInput.setup.download()}
+ onCancelDownload={() => void speechInput.setup.cancelDownload()}
+ onStartRecording={() => void speechInput.setup.startRecording()}
+ />
+
)}
diff --git a/apps/web/src/components/chat/ComposerFooterControls.tsx b/apps/web/src/components/chat/ComposerFooterControls.tsx
index 6ac13a514287..0a11e90a4cba 100644
--- a/apps/web/src/components/chat/ComposerFooterControls.tsx
+++ b/apps/web/src/components/chat/ComposerFooterControls.tsx
@@ -246,6 +246,7 @@ export function ComposerPermissionsSection(props: {
}
export const ComposerFooterPrimaryActions = memo(function ComposerFooterPrimaryActions(props: {
+ children?: React.ReactNode;
compact: boolean;
canOperateThread: boolean;
activeContextWindow: ContextWindowSnapshot | null;
@@ -266,6 +267,7 @@ export const ComposerFooterPrimaryActions = memo(function ComposerFooterPrimaryA
alternateShortcutLabel: string | null;
showPlanFollowUpPrompt: boolean;
promptHasText: boolean;
+ voiceInputActive?: boolean;
isSendBusy: boolean;
sendDisabledReason: string | null;
isEnvironmentUnavailable: boolean;
@@ -298,6 +300,7 @@ export const ComposerFooterPrimaryActions = memo(function ComposerFooterPrimaryA
) : props.reserveContextWindowMeter ? (
) : null}
+ {props.children}
{formatPendingPrimaryActionLabel({
diff --git a/apps/web/src/components/chat/ComposerSpeechButton.tsx b/apps/web/src/components/chat/ComposerSpeechButton.tsx
new file mode 100644
index 000000000000..c0047b255ed7
--- /dev/null
+++ b/apps/web/src/components/chat/ComposerSpeechButton.tsx
@@ -0,0 +1,302 @@
+import type { VoiceInputState } from "@t3tools/client-runtime/voice-input";
+import { CheckIcon, MicIcon, XIcon } from "lucide-react";
+import { useEffect, useState } from "react";
+
+import { VoiceInputErrorPill } from "./VoiceInputErrorPill";
+import { VoiceInputPill, VoiceWaveform } from "./VoiceInputPill";
+import { cn } from "~/lib/utils";
+import { Button } from "../ui/button";
+import { Spinner } from "../ui/spinner";
+import { Tooltip, TooltipPopup, TooltipTrigger } from "../ui/tooltip";
+
+type SpeechPresentation = {
+ status: string | null;
+ showsCancel: boolean;
+ showsConfirm: boolean;
+ confirmEnabled: boolean;
+ showsSend: boolean;
+};
+
+function resolveSpeechPresentation(
+ state: VoiceInputState,
+ progress: { downloaded: number; total: number } | null,
+): SpeechPresentation {
+ switch (state.phase) {
+ case "idle":
+ return {
+ status: null,
+ showsCancel: false,
+ showsConfirm: false,
+ confirmEnabled: false,
+ showsSend: true,
+ };
+ case "error":
+ return {
+ status: state.error ?? "Voice input failed",
+ showsCancel: false,
+ showsConfirm: false,
+ confirmEnabled: false,
+ showsSend: true,
+ };
+ case "preparing":
+ return {
+ status: progress
+ ? `Downloading speech model ${Math.round((progress.downloaded / Math.max(1, progress.total)) * 100)}%`
+ : "Preparing",
+ showsCancel: true,
+ showsConfirm: true,
+ confirmEnabled: false,
+ showsSend: true,
+ };
+ case "recording":
+ return {
+ status: "Recording",
+ showsCancel: true,
+ showsConfirm: true,
+ confirmEnabled: true,
+ showsSend: true,
+ };
+ case "transcribing":
+ return {
+ status: "Transcribing",
+ showsCancel: true,
+ showsConfirm: true,
+ confirmEnabled: false,
+ showsSend: true,
+ };
+ case "post-processing":
+ return {
+ status: "Post-processing",
+ showsCancel: true,
+ showsConfirm: true,
+ confirmEnabled: false,
+ showsSend: true,
+ };
+ }
+}
+
+function formatElapsed(seconds: number) {
+ return `${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, "0")}`;
+}
+
+function RecordingStatus(props: { level: number; recordingLimitSeconds?: number | undefined }) {
+ const [elapsedSeconds, setElapsedSeconds] = useState(0);
+
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- owns the timer displaying the native recording duration
+ useEffect(() => {
+ const startedAt = Date.now();
+ const timer = window.setInterval(
+ () => setElapsedSeconds(Math.floor((Date.now() - startedAt) / 1_000)),
+ 250,
+ );
+ return () => window.clearInterval(timer);
+ }, []);
+
+ return (
+ <>
+
+
+
+ {formatElapsed(elapsedSeconds)}
+ {props.recordingLimitSeconds !== undefined
+ ? ` / ${formatElapsed(props.recordingLimitSeconds)}`
+ : ""}
+
+ >
+ );
+}
+
+function ComposerSpeechStatus(props: {
+ state: VoiceInputState;
+ progress: { downloaded: number; total: number } | null;
+ level: number;
+ recordingLimitSeconds?: number | undefined;
+}) {
+ const presentation = resolveSpeechPresentation(props.state, props.progress);
+
+ if (!presentation.status) return null;
+ const isRecording = props.state.phase === "recording";
+ const isError = props.state.phase === "error";
+
+ return (
+
+ {isRecording ? (
+
+ ) : (
+
+ {presentation.status}
+
+ )}
+
+ );
+}
+
+function ComposerSpeechCancelButton(props: {
+ state: VoiceInputState;
+ shortcutLabel?: string | null;
+ onCancel(): void;
+}) {
+ if (props.state.phase === "idle") return null;
+ const label = props.state.phase === "error" ? "Dismiss voice input error" : "Cancel voice input";
+
+ return (
+
+ event.preventDefault()}
+ onClick={props.onCancel}
+ className="shrink-0"
+ >
+
+
+ }
+ />
+
+ {label}
+ {props.shortcutLabel ? ` (${props.shortcutLabel})` : ""}
+
+
+ );
+}
+
+export function ComposerSpeechButton(props: {
+ state: VoiceInputState;
+ progress: { downloaded: number; total: number } | null;
+ shortcutLabel?: string | null;
+ disabled?: boolean;
+ onStart(): void;
+ onCancel(): void;
+}) {
+ const presentation = resolveSpeechPresentation(props.state, props.progress);
+ const label = "Start voice input";
+
+ if (presentation.showsConfirm || props.state.phase === "error") return null;
+
+ return (
+
+ event.preventDefault()}
+ onClick={() => {
+ if (props.disabled) return;
+ props.onStart();
+ }}
+ className="relative shrink-0"
+ >
+
+
+ }
+ />
+
+ {label}
+ {props.shortcutLabel ? ` (${props.shortcutLabel})` : ""}
+
+
+ );
+}
+
+export function ComposerSpeechRecordingPill(props: {
+ state: VoiceInputState;
+ progress: { downloaded: number; total: number } | null;
+ finishShortcutLabel?: string | null;
+ cancelShortcutLabel?: string | null;
+ level: number;
+ recordingLimitSeconds?: number | undefined;
+ onStart(): void;
+ disabled?: boolean;
+ onStop(): void;
+ onCancel(): void;
+ onSkipPostProcessing(): void;
+}) {
+ const presentation = resolveSpeechPresentation(props.state, props.progress);
+ if (props.state.phase === "error") {
+ return (
+
+ );
+ }
+ if (!presentation.showsConfirm) return null;
+
+ const label = presentation.confirmEnabled
+ ? "Finish voice input"
+ : (presentation.status ?? "Voice input is busy");
+
+ return (
+
+
+
+ {props.state.phase === "post-processing" ? (
+ event.preventDefault()}
+ onClick={props.onSkipPostProcessing}
+ className="shrink-0"
+ >
+ Skip
+
+ ) : (
+
+ event.preventDefault()}
+ onClick={props.onStop}
+ className="shrink-0"
+ >
+ {presentation.confirmEnabled ? : }
+
+ }
+ />
+
+ {label}
+ {presentation.confirmEnabled && props.finishShortcutLabel
+ ? ` (${props.finishShortcutLabel})`
+ : ""}
+
+
+ )}
+
+ );
+}
diff --git a/apps/web/src/components/chat/RightPanelSurfaceContent.tsx b/apps/web/src/components/chat/RightPanelSurfaceContent.tsx
index b703e5d05da9..f0d8afa5f319 100644
--- a/apps/web/src/components/chat/RightPanelSurfaceContent.tsx
+++ b/apps/web/src/components/chat/RightPanelSurfaceContent.tsx
@@ -50,6 +50,7 @@ interface RightPanelPullRequestProps {
}
interface RightPanelFileProps {
+ readonly projectId?: ProjectId | undefined;
readonly projectName: string | null;
readonly workspaceRoot: string | undefined;
readonly availableEditors: ReadonlyArray;
@@ -209,6 +210,7 @@ export function RightPanelSurfaceContent(props: {
environmentId={props.environmentId}
cwd={files.workspaceRoot ?? ""}
projectName={files.projectName ?? ""}
+ projectId={files.projectId}
threadRef={threadRef}
composerDraftTarget={props.composerDraftTarget}
keybindings={props.keybindings}
diff --git a/apps/web/src/components/chat/VoiceInputErrorPill.tsx b/apps/web/src/components/chat/VoiceInputErrorPill.tsx
new file mode 100644
index 000000000000..26f4895e1b3f
--- /dev/null
+++ b/apps/web/src/components/chat/VoiceInputErrorPill.tsx
@@ -0,0 +1,115 @@
+import { MICROPHONE_PERMISSION_ERROR } from "@t3tools/client-runtime/voice-input";
+import { useNavigate } from "@tanstack/react-router";
+import { RotateCcwIcon, SettingsIcon, XIcon } from "lucide-react";
+
+import { Button } from "../ui/button";
+import { Tooltip, TooltipPopup, TooltipTrigger } from "../ui/tooltip";
+import { VoiceInputPill } from "./VoiceInputPill";
+
+const VOICE_ERROR_LABELS: Record = {
+ [MICROPHONE_PERMISSION_ERROR]: "Microphone access needed",
+ "Microphone access is required for voice input.": "Microphone access needed",
+ "This draft is no longer available.": "Draft unavailable",
+ "Another voice recording is already active.": "Another recording active",
+ "Voice transcription is not available.": "Voice input unavailable",
+ "Voice transcription is still finishing. Try again shortly.": "Transcription still busy",
+ "Voice transcription is not available for this language.": "Language unsupported",
+ "Could not prepare voice transcription.": "Couldn’t prepare dictation",
+ "Could not start voice recording.": "Couldn’t start recording",
+ "Voice recording was interrupted.": "Recording interrupted",
+ "Could not finish voice recording.": "Couldn’t finish recording",
+ "Could not transcribe this recording.": "Transcription failed",
+ "Voice transcription on this environment failed.": "Transcription failed",
+ "The draft changed while voice input was running. The transcript was not added.": "Draft changed",
+ "No speech was detected.": "No speech detected",
+ "Microphone processing failed.": "Microphone processing failed",
+ "The microphone disconnected.": "Microphone disconnected",
+};
+
+function voiceErrorLabel(error: string | null): string {
+ return (error && VOICE_ERROR_LABELS[error]) || "Voice input failed";
+}
+
+export function VoiceInputErrorPill(props: {
+ error: string | null;
+ errorAction: "retry" | "settings" | null;
+ disabled?: boolean;
+ cancelShortcutLabel?: string | null;
+ onRetry(): void;
+ onDismiss(): void;
+}) {
+ const navigate = useNavigate();
+ const openSettings = props.errorAction === "settings";
+ const actionLabel = openSettings
+ ? "Open voice settings"
+ : props.error === "Could not transcribe this recording." ||
+ props.error === "Voice transcription on this environment failed."
+ ? "Retry transcription"
+ : "Record again";
+ return (
+
+
+ event.preventDefault()}
+ onClick={props.onDismiss}
+ >
+
+
+ }
+ />
+
+ Dismiss voice input error
+ {props.cancelShortcutLabel ? ` (${props.cancelShortcutLabel})` : ""}
+
+
+
+
+ }
+ >
+ {voiceErrorLabel(props.error)}
+
+ {props.error ?? "Voice input failed"}
+
+ {props.errorAction ? (
+
+ event.preventDefault()}
+ onClick={() => {
+ if (openSettings) {
+ props.onDismiss();
+ void navigate({ to: "/settings/voice" });
+ } else {
+ props.onRetry();
+ }
+ }}
+ >
+ {openSettings ? : }
+
+ }
+ />
+ {actionLabel}
+
+ ) : null}
+
+ );
+}
diff --git a/apps/web/src/components/chat/VoiceInputPill.tsx b/apps/web/src/components/chat/VoiceInputPill.tsx
new file mode 100644
index 000000000000..a08567dbd673
--- /dev/null
+++ b/apps/web/src/components/chat/VoiceInputPill.tsx
@@ -0,0 +1,51 @@
+import { memo, useEffect, useRef, type ReactNode } from "react";
+import { useMediaQuery } from "~/hooks/useMediaQuery";
+
+const WAVEFORM_BAR_COUNT = 28;
+const WAVEFORM_BAR_IDS = Array.from({ length: WAVEFORM_BAR_COUNT }, (_, index) => String(index));
+
+export const VoiceWaveform = memo(function VoiceWaveform(props: { level: number }) {
+ const prefersReducedMotion = useMediaQuery("(prefers-reduced-motion: reduce)");
+ const barsRef = useRef>([]);
+ const levelsRef = useRef(Array(WAVEFORM_BAR_COUNT).fill(0));
+
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- updates native waveform bar styles when microphone levels arrive
+ useEffect(() => {
+ const levels = levelsRef.current;
+ levels.copyWithin(0, 1);
+ levels[levels.length - 1] = props.level;
+ barsRef.current.forEach((bar, index) => {
+ if (!bar) return;
+ const level = levels[index] ?? 0;
+ bar.style.opacity = String(0.22 + level * 0.78);
+ bar.style.transform = `scaleY(${prefersReducedMotion ? 0.35 : Math.max(0.08, level)})`;
+ });
+ }, [prefersReducedMotion, props.level]);
+
+ return (
+
+ {WAVEFORM_BAR_IDS.map((id, index) => (
+ {
+ barsRef.current[index] = bar;
+ }}
+ className="h-full w-0.5 shrink-0 origin-center rounded-full bg-primary opacity-25 transition-[transform,opacity] duration-100 ease-out motion-reduce:transition-none"
+ style={{ transform: `scaleY(${prefersReducedMotion ? 0.35 : 0.08})` }}
+ />
+ ))}
+
+ );
+});
+
+export function VoiceInputPill({ children }: { children: ReactNode }) {
+ return (
+ // Same height as the toolbar's icon buttons, so starting a recording never resizes the composer.
+
+ {children}
+
+ );
+}
diff --git a/apps/web/src/components/chat/VoiceInputSetup.tsx b/apps/web/src/components/chat/VoiceInputSetup.tsx
new file mode 100644
index 000000000000..7b30783cf21d
--- /dev/null
+++ b/apps/web/src/components/chat/VoiceInputSetup.tsx
@@ -0,0 +1,243 @@
+import { useStableCallback } from "~/hooks/useStableCallback";
+import { listBrowserMicrophones, type BrowserMicrophone } from "../../speech/browserMicrophones";
+import type {
+ EnvironmentId,
+ EnvironmentSpeechModel,
+ EnvironmentSpeechStatus,
+} from "@t3tools/contracts";
+import { useNavigate } from "@tanstack/react-router";
+import { RefreshCwIcon } from "lucide-react";
+import { useEffect, useState } from "react";
+
+import { useClientSettings, updateClientSettings } from "../../hooks/useSettings";
+import { useEnvironment } from "../../state/environments";
+import { AuthSettingsWriteScope } from "@t3tools/contracts";
+import { useEnvironmentScope } from "../../state/session";
+import { Button } from "../ui/button";
+import { Dialog } from "../ui/dialog";
+import { Select, SelectItem, SelectPopup, SelectTrigger, SelectValue } from "../ui/select";
+import { WizardFooter, WizardHeader, WizardPanel, WizardPopup, WizardSteps } from "../ui/wizard";
+
+export function VoiceInputSetup(props: {
+ environmentId: EnvironmentId | null;
+ open: boolean;
+ step: number;
+ status: EnvironmentSpeechStatus | null;
+ model: EnvironmentSpeechModel | null;
+ downloading: boolean;
+ error: string | null;
+ onOpenChange(open: boolean): void;
+ onDownload(): void;
+ onCancelDownload(): void;
+ onStartRecording(): void;
+}) {
+ const navigate = useNavigate();
+ const environment = useEnvironment(props.environmentId);
+ const canManage = useEnvironmentScope(props.environmentId, AuthSettingsWriteScope);
+ const selectedMicrophone = useClientSettings((settings) => settings.voiceMicrophone);
+ const [microphones, setMicrophones] = useState([]);
+ const [loadingMicrophones, setLoadingMicrophones] = useState(false);
+ const [microphoneError, setMicrophoneError] = useState(null);
+ const refreshMicrophones = useStableCallback(async () => {
+ if (!navigator.mediaDevices?.enumerateDevices) return;
+ setLoadingMicrophones(true);
+ try {
+ setMicrophones(await listBrowserMicrophones());
+ setMicrophoneError(null);
+ } catch {
+ setMicrophoneError("Could not list microphones. You can still use the system default.");
+ }
+ setLoadingMicrophones(false);
+ });
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- subscribes to native microphone changes while the device setup step is visible
+ useEffect(() => {
+ if (!props.open || props.step !== 1) return;
+ queueMicrotask(() => void refreshMicrophones());
+ const mediaDevices = navigator.mediaDevices;
+ mediaDevices?.addEventListener?.("devicechange", refreshMicrophones);
+ return () => mediaDevices?.removeEventListener?.("devicechange", refreshMicrophones);
+ }, [props.open, props.step, refreshMicrophones]);
+ const selectedIsUnavailable = Boolean(
+ selectedMicrophone && !microphones.some((device) => device.deviceId === selectedMicrophone),
+ );
+ const selectedMicrophoneLabel = selectedMicrophone
+ ? (microphones.find((device) => device.deviceId === selectedMicrophone)?.label ??
+ "Selected microphone (Unavailable)")
+ : "System default";
+ const status = props.status?.supported ? props.status : null;
+ const model = props.model;
+ const progress =
+ model?.downloaded === undefined ? null : Math.round((model.downloaded / model.size) * 100);
+ const openSettings = (hash: string) => {
+ props.onOpenChange(false);
+ void navigate({ to: "/settings/voice", hash });
+ };
+
+ return (
+
+
+
+
+
+
+ {props.step === 0 ? (
+
+
+
+
{status?.model ?? "English transcription model"}
+ {status?.modelId === "handy-computer/parakeet-unified-en-0.6b-gguf" ? (
+
+ Recommended for English
+
+ ) : null}
+
+
+ Download the {status ? Math.round(status.size / 1024 / 1024) : 697} MB model to{" "}
+ {environment?.label ?? "your transcription environment"}. Microphone audio is sent
+ there for transcription, as chosen in Voice settings.
+
+
+ Silero (1.2 MB), used for silence filtering and pause detection, downloads
+ alongside the speech model. An installed copy is reused.
+
+ {props.downloading ? (
+
+
+
+ {model?.state === "verifying"
+ ? "Verifying download…"
+ : progress === null
+ ? "Starting download…"
+ : `${progress}% downloaded`}
+
+
+ ) : null}
+
+ {props.error ? (
+
+ {props.error}
+
+ ) : null}
+
+ ) : (
+
+
+
+ Microphone
+
+
+ {
+ if (value)
+ void updateClientSettings({
+ voiceMicrophone:
+ value === "system-default" ? "" : value.slice("device:".length),
+ });
+ }}
+ >
+
+ {selectedMicrophoneLabel}
+
+
+ System default
+ {selectedIsUnavailable ? (
+
+ Selected microphone (Unavailable)
+
+ ) : null}
+ {microphones.map((device) => (
+
+ {device.label}
+
+ ))}
+
+
+ void refreshMicrophones()}
+ >
+
+
+
+ {selectedIsUnavailable ? (
+
+ The selected microphone is unavailable. Choose another or use the system
+ default.
+
+ ) : null}
+ {microphoneError ? (
+
+ {microphoneError}
+
+ ) : null}
+
+
+ Speak and transcribe in multiple languages or translate into English, and get names
+ and technical terms right with the dictionary in Voice settings.
+
+
+ )}
+
+
+ {props.step === 0 ? (
+ <>
+ openSettings("speech-models")}>
+ Choose another model or language
+
+ {props.downloading ? (
+
+ Cancel download
+
+ ) : null}
+
+ {props.downloading
+ ? "Downloading…"
+ : props.error
+ ? "Retry download"
+ : canManage
+ ? "Download model"
+ : "Ask an environment admin to download"}
+
+ >
+ ) : (
+ <>
+ openSettings("")}>
+ Customize in Voice settings
+
+ Start dictating
+ >
+ )}
+
+
+
+ );
+}
diff --git a/apps/web/src/components/composerContextActions.ts b/apps/web/src/components/composerContextActions.ts
new file mode 100644
index 000000000000..b5df31ea2b26
--- /dev/null
+++ b/apps/web/src/components/composerContextActions.ts
@@ -0,0 +1,22 @@
+import type { EnvironmentId, ProjectId } from "@t3tools/contracts";
+import { createContext, type MouseEvent } from "react";
+
+export interface ComposerContextActions {
+ environmentId: EnvironmentId | null;
+ cwd?: string | undefined;
+ projectId?: ProjectId | undefined;
+ expandImage: (imageId: string) => void;
+ expandVideo: (fileId: string) => void;
+ openFile: (fileId: string) => void;
+ openMention: (path: string) => void;
+ openPullRequest: (event: MouseEvent, url: string) => void;
+}
+
+export const ComposerContextActionsContext = createContext({
+ environmentId: null,
+ expandImage: () => {},
+ expandVideo: () => {},
+ openFile: () => {},
+ openMention: () => {},
+ openPullRequest: () => {},
+});
diff --git a/apps/web/src/components/composerContextPresentation.tsx b/apps/web/src/components/composerContextPresentation.tsx
index f8ac8222177f..845e921725b5 100644
--- a/apps/web/src/components/composerContextPresentation.tsx
+++ b/apps/web/src/components/composerContextPresentation.tsx
@@ -9,8 +9,8 @@ import type {
import { formatAttachmentSize } from "@t3tools/client-runtime/state/attachments";
import { videoMimeType } from "@t3tools/shared/video";
import { MessageCircleIcon, MousePointerClickIcon } from "lucide-react";
-import { createContext, type MouseEvent, type ReactElement, type ReactNode, use } from "react";
-import type { EnvironmentId } from "@t3tools/contracts";
+import { createContext, type ReactElement, type ReactNode, use } from "react";
+import { ComposerContextActionsContext } from "./composerContextActions";
import type { ComposerFileAttachment, ComposerImageAttachment } from "~/composerDraftStore";
import { composerFileNeedsReattach } from "~/composerDraftStore";
@@ -68,24 +68,10 @@ export type ComposerDraftContextRecord =
| { kind: "thread"; record: ThreadContextRecord }
| { kind: "environment"; record: EnvironmentContextRecord };
-export interface ComposerContextActions {
- environmentId: EnvironmentId | null;
- cwd?: string | undefined;
- expandImage: (imageId: string) => void;
- expandVideo: (fileId: string) => void;
- openFile: (fileId: string) => void;
- openMention: (path: string) => void;
- openPullRequest: (event: MouseEvent, url: string) => void;
-}
-
-export const ComposerContextActionsContext = createContext({
- environmentId: null,
- expandImage: () => {},
- expandVideo: () => {},
- openFile: () => {},
- openMention: () => {},
- openPullRequest: () => {},
-});
+export {
+ ComposerContextActionsContext,
+ type ComposerContextActions,
+} from "./composerContextActions";
export type ComposerDraftContextRecords = ReadonlyMap;
diff --git a/apps/web/src/components/diffs/AnnotatableCodeView.tsx b/apps/web/src/components/diffs/AnnotatableCodeView.tsx
index 81bcba75aa79..fdaea2df124b 100644
--- a/apps/web/src/components/diffs/AnnotatableCodeView.tsx
+++ b/apps/web/src/components/diffs/AnnotatableCodeView.tsx
@@ -7,7 +7,7 @@ import type {
SelectedLineRange,
} from "@pierre/diffs";
import type { CodeViewHandle } from "@pierre/diffs/react";
-import type { ScopedThreadRef } from "@t3tools/contracts";
+import type { EnvironmentId, ProjectId, ScopedThreadRef } from "@t3tools/contracts";
import { useState, type ReactNode, type Ref } from "react";
import { type DraftId, useComposerDraftStore } from "~/composerDraftStore";
@@ -72,6 +72,8 @@ function appendAnnotationEntry(
}
interface AnnotatableCodeViewProps {
+ environmentId?: EnvironmentId | undefined;
+ projectId?: ProjectId | null | undefined;
codeViewKey: string;
files: ReadonlyArray<{
fileDiff: FileDiffMetadata;
@@ -104,6 +106,8 @@ interface DiffSelectionContext {
}
export function AnnotatableCodeView({
+ environmentId,
+ projectId,
codeViewKey,
files,
pendingFiles,
@@ -292,6 +296,14 @@ export function AnnotatableCodeView({
{annotation.metadata.entries.map((entry) => (
void,
+ allowEmpty = false,
+ ) => {
+ const finish = (value: string) => {
+ if (!pending && (allowEmpty || value.trim())) action(value.trim());
+ };
+ if (speech) speech.submitAfterDictation(finish);
+ else finish(displayedText);
+ };
+
// Pierre hides native carets and selection; textarea comments need their own visible feedback.
- return (
+ const renderDraft = (speech?: TextFieldSpeech) => (