diff --git a/apps/server/src/speech/SpeechModelStore.test.ts b/apps/server/src/speech/SpeechModelStore.test.ts index dda2f3cad4e2..e08a4cf022f4 100644 --- a/apps/server/src/speech/SpeechModelStore.test.ts +++ b/apps/server/src/speech/SpeechModelStore.test.ts @@ -13,7 +13,6 @@ import * as Stream from "effect/Stream"; import { HttpClient, HttpClientError, HttpClientRequest, HttpClientResponse } from "effect/http"; import * as ServerConfig from "../config.ts"; -import { getSpeechModel, type SpeechModel } from "./model.ts"; import * as SpeechModelStore from "./SpeechModelStore.ts"; const bytes = Uint8Array.from({ length: 64 }, (_, index) => (index * 7) % 256); @@ -21,8 +20,9 @@ const SPLIT = 20; const harness = Effect.gen(function* () { const digest = yield* Digest.Digest; - const model: SpeechModel = { - ...getSpeechModel("handy-computer/whisper-tiny-gguf")!, + const model = { + id: "fixture/speech-model", + revision: "a".repeat(40), filename: "model.gguf", size: bytes.length, sha256: Hex.encode(yield* digest.digest(Digest.sha256, bytes).pipe(Effect.orDie)), @@ -121,10 +121,8 @@ it.effect("resumes an interrupted transfer, replaces a corrupt same-size file, a undefined, `bytes=${SPLIT}-`, ]); - expect(h.requests.map((request) => request.url)).toEqual([ - "https://huggingface.co/handy-computer/whisper-tiny-gguf/resolve/6687f30c99641ee265df421e582354adbc8848fc/model.gguf", - "https://huggingface.co/handy-computer/whisper-tiny-gguf/resolve/6687f30c99641ee265df421e582354adbc8848fc/model.gguf", - ]); + const pinnedUrl = `https://huggingface.co/${h.model.id}/resolve/${h.model.revision}/${h.model.filename}`; + expect(h.requests.map((request) => request.url)).toEqual([pinnedUrl, pinnedUrl]); expect(progress.at(0)).toBe(0); expect(progress.at(-1)).toBe(bytes.length); expect([...(yield* h.fs.readFile(installed))]).toEqual([...bytes]); @@ -134,6 +132,9 @@ it.effect("resumes an interrupted transfer, replaces a corrupt same-size file, a ]); expect(yield* h.store.isReady(h.model)).toBe(true); expect(yield* h.store.installedPath(h.model)).toBe(installed); + const requestsBeforeReuse = h.requests.length; + expect(yield* h.store.download(h.model)).toBe(installed); + expect(h.requests).toHaveLength(requestsBeforeReuse); }).pipe(Effect.provide(dependencies)), ); diff --git a/apps/server/src/speech/SpeechModelStore.ts b/apps/server/src/speech/SpeechModelStore.ts index b85d5b47f1fb..99f3fe6ca5bb 100644 --- a/apps/server/src/speech/SpeechModelStore.ts +++ b/apps/server/src/speech/SpeechModelStore.ts @@ -17,6 +17,8 @@ import * as Stream from "effect/Stream"; import * as ServerConfig from "../config.ts"; import type { SpeechModel } from "./model.ts"; +type ModelFile = Pick; + class SpeechModelDownloadError extends Schema.TaggedError()( "SpeechModelDownloadError", { @@ -39,17 +41,17 @@ export class SpeechModelStore extends Context.Service< SpeechModelStore, { /** Cheap size check for status polls; never hashes the file. */ - readonly isReady: (model: SpeechModel) => Effect.Effect; + readonly isReady: (model: ModelFile) => Effect.Effect; /** The installed file once its SHA-256 has been verified, hashing it on first use. */ - readonly installedPath: (model: SpeechModel) => Effect.Effect; + readonly installedPath: (model: ModelFile) => Effect.Effect; readonly download: ( - model: SpeechModel, + model: ModelFile, progress?: SpeechModelDownloadProgress, ) => Effect.Effect< string, SpeechModelDownloadError | PlatformError | HttpClientError.HttpClientError >; - readonly remove: (model: SpeechModel) => Effect.Effect; + readonly remove: (model: ModelFile) => Effect.Effect; } >()("t3/speech/SpeechModelStore") {} @@ -70,11 +72,11 @@ const make = Effect.gen(function* () { yield* fs.remove(path.join(directory, name), { force: true }).pipe(Effect.ignore); } } - const modelPath = (model: SpeechModel) => path.join(directory, model.filename); + const modelPath = (model: ModelFile) => path.join(directory, model.filename); // Files whose digest failed, by identity, so status polls stop calling them ready. const corrupt = new Map(); - const identity = (model: SpeechModel, info: FileSystem.File.Info) => + const identity = (model: ModelFile, info: FileSystem.File.Info) => [ model.sha256, info.size, @@ -85,7 +87,7 @@ const make = Effect.gen(function* () { const statFile = (file: string) => fs.stat(file).pipe(Effect.orElseSucceed(() => undefined)); - const expectedFile = (model: SpeechModel, file: string) => + const expectedFile = (model: ModelFile, file: string) => statFile(file).pipe( Effect.map((info) => info?.type === "File" && Number(info.size) === model.size ? info : undefined, @@ -93,7 +95,7 @@ const make = Effect.gen(function* () { ); const digestMatches = Effect.fn("SpeechModelStore.digest")( - function* (model: SpeechModel, file: string) { + function* (model: ModelFile, file: string) { const hash = NodeCrypto.createHash("sha256"); yield* fs .stream(file) @@ -105,7 +107,7 @@ const make = Effect.gen(function* () { // A verified file is trusted while its size, mtime, device and inode are unchanged. const verifiedPath = Effect.fn("SpeechModelStore.verifiedPath")(function* ( - model: SpeechModel, + model: ModelFile, file: string, onVerificationStart?: () => void, ) { @@ -127,7 +129,7 @@ const make = Effect.gen(function* () { }); const fetchInto = Effect.fn("SpeechModelStore.fetch")(function* ( - model: SpeechModel, + model: ModelFile, url: string, partialPath: string, offset: number, diff --git a/apps/server/src/speech/SpeechService.test.ts b/apps/server/src/speech/SpeechService.test.ts index 3c93ba62f31f..972c401ce309 100644 --- a/apps/server/src/speech/SpeechService.test.ts +++ b/apps/server/src/speech/SpeechService.test.ts @@ -10,19 +10,22 @@ import * as Result from "effect/Result"; import * as Scheduler from "effect/Scheduler"; import * as Scope from "effect/Scope"; import * as Exit from "effect/Exit"; +import * as PlatformError from "effect/PlatformError"; import * as HostProcess from "@t3tools/shared/HostProcess"; import { DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS } from "@t3tools/contracts"; import * as ServerSettings from "../serverSettings.ts"; import { listNativeSpeechGpuDevices } from "./native.ts"; import * as SpeechModelStore from "./SpeechModelStore.ts"; import * as SpeechService from "./SpeechService.ts"; +import { SILERO_VAD } from "./silero.ts"; const native = vi.hoisted(() => ({ backend: "Vulkan0", supportsStreaming: true, + supportsLongForm: false, supportsTranslation: true, supportsVocabulary: false, - begin: vi.fn(async (_language?: string, _vocabulary?: readonly string[]) => {}), + begin: vi.fn(async (_options?: unknown, _filterSilence?: boolean, _vadPath?: string) => {}), feed: vi.fn(async (_pcm: Float32Array) => ({ revision: 1, text: { committed: "", tentative: "hello" }, @@ -66,13 +69,13 @@ const modelDefinitions = vi.hoisted(() => [ name: "fallback", description: "fallback", size: 8, - languages: ["en"], + languages: ["en", "fr"], accuracy: 2, speed: 2, recommended: false, supportsStreaming: false, supportsLanguageDetection: false, - supportsTranslation: false, + supportsTranslation: true, }, ]); vi.mock("./native.ts", () => ({ @@ -112,8 +115,10 @@ const pcm = () => new Uint8Array(new Float32Array([0.25]).buffer); beforeEach(() => { vi.clearAllMocks(); loadNative.mockReset(); + native.finish.mockReset(); native.backend = "Vulkan0"; native.supportsVocabulary = false; + native.supportsStreaming = true; readyModels.clear(); readyModels.add("test-model"); readyModels.add("fallback-model"); @@ -146,6 +151,21 @@ it.effect("prepares a batch model before audio arrives and reuses it for transcr yield* speech.prepareModel; expect(loadNative).toHaveBeenCalledTimes(1); expect(native.transcribe).not.toHaveBeenCalled(); + native.supportsStreaming = false; + yield* Effect.gen(function* () { + const stream = yield* speech.startStream({ + ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, + speechLanguage: "fr", + speechTranslateToEnglish: true, + speechSilenceFiltering: false, + }); + expect(native.begin).toHaveBeenLastCalledWith( + { timestamps: "none", language: "fr", task: "translate", targetLanguage: "en" }, + false, + "test.gguf", + ); + expect(yield* stream.finish).toBe("hello"); + }).pipe(Effect.scoped); expect(yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)).toBe("hello"); expect(loadNative).toHaveBeenCalledTimes(1); }).pipe(Effect.provide(layer)), @@ -223,28 +243,112 @@ it.effect("keeps a detached download exclusive until the transfer finishes", () const speech = yield* SpeechService.SpeechService; const started = yield* Deferred.make(); const finish = yield* Deferred.make(); - download.mockImplementationOnce(() => - Deferred.succeed(started, undefined).pipe( - Effect.andThen(Deferred.await(finish)), - Effect.as("test.gguf"), - ), - ); + const helperStarted = yield* Deferred.make(); + let helperStartedDuringInstall = false; + let helperInterrupted = false; + download + .mockImplementationOnce(() => + Deferred.succeed(started, undefined).pipe( + Effect.andThen(Deferred.await(finish)), + Effect.as("test.gguf"), + ), + ) + .mockImplementationOnce(() => + Deferred.isDone(finish).pipe( + Effect.tap((done) => + Effect.sync(() => { + helperStartedDuringInstall = !done; + }), + ), + Effect.andThen(Deferred.succeed(helperStarted, undefined)), + Effect.andThen(Effect.never), + Effect.onInterrupt(() => + Effect.sync(() => { + helperInterrupted = true; + }), + ), + ), + ); const request = yield* speech.downloadModel("test-model").pipe(Effect.forkChild); yield* Deferred.await(started); + yield* Deferred.await(helperStarted); yield* Fiber.interrupt(request); + yield* Deferred.succeed(finish, undefined); + yield* TestClock.adjust("16 seconds"); + expect(helperInterrupted).toBe(false); const removed = yield* Effect.result(speech.removeModel("test-model")); expect(Result.isFailure(removed) && removed.failure).toMatchObject({ _tag: "SpeechBusyError" }); const join = yield* speech .downloadModel("test-model") .pipe(Effect.forkChild({ startImmediately: true })); - yield* Deferred.succeed(finish, undefined); + yield* TestClock.adjust("2 minutes"); + expect(helperInterrupted).toBe(true); yield* Fiber.join(join); - expect(download).toHaveBeenCalledTimes(1); + expect(download.mock.calls.map(([model]) => model.id)).toEqual(["test-model", SILERO_VAD.id]); + expect(helperStartedDuringInstall).toBe(true); yield* speech.removeModel("test-model"); expect(readyModels.has("test-model")).toBe(false); }).pipe(Effect.provide(layer)), ); +it.effect("bounds missing Silero recovery during recording and continues with raw audio", () => + Effect.gen(function* () { + const helperStarted = yield* Deferred.make(); + const helperInterrupted = yield* Deferred.make(); + download.mockImplementationOnce(() => + Deferred.succeed(helperStarted, undefined).pipe( + Effect.andThen(Effect.never), + Effect.onInterrupt(() => Deferred.succeed(helperInterrupted, undefined)), + ), + ); + const speech = yield* SpeechService.SpeechService; + const recording = yield* speech + .startStream(DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS) + .pipe(Effect.forkChild); + yield* Deferred.await(helperStarted); + yield* TestClock.adjust("14999 millis"); + expect(native.begin).not.toHaveBeenCalled(); + yield* TestClock.adjust("1 millis"); + yield* Deferred.await(helperInterrupted); + const stream = yield* Fiber.join(recording); + expect(native.begin).toHaveBeenLastCalledWith(expect.anything(), true, undefined); + yield* stream.feed(pcm()); + expect(yield* stream.finish).toBe("hello"); + }).pipe(Effect.scoped, Effect.provide(layer)), +); + +it.effect("keeps the installed speech model usable when optional Silero acquisition fails", () => + Effect.gen(function* () { + download + .mockImplementationOnce(() => Effect.succeed("test.gguf")) + .mockImplementationOnce(() => + Effect.fail( + new PlatformError.PlatformError( + new PlatformError.SystemError({ + _tag: "NotFound", + module: "FileSystem", + method: "open", + }), + ), + ), + ); + const speech = yield* SpeechService.SpeechService; + const installed = yield* speech.downloadModel("test-model"); + expect(installed.models.find((model) => model.id === "test-model")?.state).toBe("installed"); + expect(yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)).toBe("hello"); + yield* Effect.scoped( + Effect.gen(function* () { + const stream = yield* speech.startStream(DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS); + expect(yield* stream.finish).toBe("hello"); + }), + ); + expect(download).toHaveBeenCalledTimes(2); + yield* TestClock.adjust("10 minutes"); + expect(yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)).toBe("hello"); + expect(download).toHaveBeenCalledTimes(3); + }).pipe(Effect.provide(layer)), +); + it.effect( "allows another model operation after an already-installed download completes immediately", () => @@ -333,7 +437,11 @@ it.effect("uses vocabulary only for supported transcription and always applies a text: { committed: vocabulary ? "git hub" : "GitHub", tentative: "Bob" }, }); expect(yield* stream.finish).toBe(vocabulary ? "git hub and Bob" : "GitHub and Bob"); - expect(native.begin.mock.lastCall).toEqual(["fr", vocabulary]); + expect(native.begin.mock.lastCall).toEqual([ + { timestamps: "none", language: "fr", ...(vocabulary ? { vocabulary } : {}) }, + true, + "test.gguf", + ]); }).pipe(Effect.scoped, Effect.provide(layer)); } } @@ -354,6 +462,58 @@ it.effect("unloads after the chosen idle period and reloads on the next recordin }).pipe(Effect.provide(layer)), ); +it.effect.each([ + { cue: "um", remove: true, text: "Make it red, um, blue.", expected: "Make it red, um, blue." }, + { + cue: "uh no", + remove: true, + text: "Make it red, uh no, blue.", + expected: "Make it red, uh no, blue.", + }, + { + cue: "uh no", + remove: true, + text: "Uh, Make it red, UH NO, blue. Uh nope.", + expected: "Make it red, UH NO, blue. nope.", + }, + { + cue: " UM ", + remove: true, + text: "Uh, Make it red, um, blue.", + expected: "Make it red, um, blue.", + }, + { + cue: undefined, + remove: true, + text: "Uh, Make it red, um, blue.", + expected: "Make it red, blue.", + }, + { + cue: "um", + remove: false, + text: "Uh, Make it red, um, blue.", + expected: "Uh, Make it red, um, blue.", + }, +])( + "preserves only an explicit correction cue during batch and stream filler removal: $cue / $remove", + ({ cue, remove, text, expected }) => + Effect.gen(function* () { + const speech = yield* SpeechService.SpeechService; + const options = { + ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, + speechLanguage: "en", + speechRemoveFillerWords: remove, + speechCustomWords: [{ term: "um", aliases: [] }], + ...(cue === undefined ? {} : { speechCorrectionWord: cue }), + }; + native.transcribe.mockResolvedValueOnce({ text }); + native.finish.mockResolvedValueOnce(text); + expect(yield* speech.transcribe(pcm(), options)).toBe(expected); + const stream = yield* speech.startStream(options); + expect(yield* stream.finish).toBe(expected); + }).pipe(Effect.scoped, Effect.provide(layer)), +); + it.effect("holds model ownership until the stream scope closes and preserves silence", () => Effect.gen(function* () { const speech = yield* SpeechService.SpeechService; @@ -560,9 +720,13 @@ it.effect("replays streamed audio on CPU after an Auto-placed feed crashes, once }; expect(yield* stream.feed(pcm())).toEqual({ revision: 5, text: recovered }); expect(loadNative.mock.calls.slice(-2).map((call) => call[3])).toEqual(["auto", "cpu"]); - expect(native.begin.mock.calls.slice(-2)).toEqual([ - ["en", ["GitHub", "Bob"]], - ["en", cpuSupportsVocabulary ? ["GitHub", "Bob"] : undefined], + expect(native.begin.mock.calls.slice(-2).map(([options]) => options)).toEqual([ + { timestamps: "none", language: "en", vocabulary: ["GitHub", "Bob"] }, + { + timestamps: "none", + language: "en", + ...(cpuSupportsVocabulary ? { vocabulary: ["GitHub", "Bob"] } : {}), + }, ]); expect(native.feed).toHaveBeenCalledTimes(feedCalls + 4); expect(yield* stream.feed(pcm())).toEqual({ revision: 6, text: recovered }); @@ -583,7 +747,7 @@ it.effect.each([false, true])( const cpu = { ...native, backend: "CPU", - begin: vi.fn(async (_language?: string) => {}), + begin: vi.fn(async (_options?: unknown) => {}), feed: vi.fn(async (_pcm: Float32Array) => ({ revision: 1, text: { committed: "", tentative: "" }, @@ -611,7 +775,11 @@ it.effect.each([false, true])( expect(cpu.feed.mock.calls.map(([chunk]) => [...chunk])).toEqual( chunks.map((chunk) => [...chunk]), ); - expect(cpu.begin).toHaveBeenCalledWith("fr", undefined); + expect(cpu.begin).toHaveBeenCalledWith( + { timestamps: "none", language: "fr" }, + true, + "test.gguf", + ); expect(gpu.dispose).toHaveBeenCalledOnce(); expect(cpu.dispose).not.toHaveBeenCalled(); const busy = yield* Effect.result(speech.selectModel("fallback-model")); diff --git a/apps/server/src/speech/SpeechService.ts b/apps/server/src/speech/SpeechService.ts index d1ad3b22a945..c1f762f26089 100644 --- a/apps/server/src/speech/SpeechService.ts +++ b/apps/server/src/speech/SpeechService.ts @@ -32,6 +32,7 @@ import { type SpeechModel, } from "./model.ts"; import { listNativeSpeechGpuDevices, loadNativeSpeechModel } from "./native.ts"; +import { SILERO_VAD } from "./silero.ts"; import { isSpeechPlatformSupported } from "./platform.ts"; import * as SpeechModelStore from "./SpeechModelStore.ts"; @@ -39,14 +40,13 @@ const SAMPLE_RATE = 16_000; const MAX_SPEECH_DURATION_SECONDS = 5 * 60; export const MAX_SPEECH_BYTES = SAMPLE_RATE * Float32Array.BYTES_PER_ELEMENT * MAX_SPEECH_DURATION_SECONDS; -const MIN_CAPTURE_RMS = 0.0005; export class SpeechInvalidAudioError extends Schema.TaggedError()( "SpeechInvalidAudioError", { byteLength: Schema.Number, message: Schema.String }, ) {} -export function decodeSpeechPcm(pcmBytes: Uint8Array, preserveSilence = false): Float32Array { +export function decodeSpeechPcm(pcmBytes: Uint8Array): Float32Array { if ( pcmBytes.byteLength === 0 || pcmBytes.byteLength > MAX_SPEECH_BYTES || @@ -60,7 +60,6 @@ export function decodeSpeechPcm(pcmBytes: Uint8Array, preserveSilence = false): // WebSocket frames can be pooled Node Buffers whose backing storage is larger // than the frame. Copy the view so only the received bytes are decoded. const pcm = new Float32Array(Uint8Array.from(pcmBytes).buffer); - let energy = 0; for (const sample of pcm) { if (!Number.isFinite(sample) || sample < -1 || sample > 1) { throw new SpeechInvalidAudioError({ @@ -68,11 +67,8 @@ export function decodeSpeechPcm(pcmBytes: Uint8Array, preserveSilence = false): message: "audio contains an invalid PCM sample", }); } - energy += sample * sample; } - return !preserveSilence && Math.sqrt(energy / pcm.length) < MIN_CAPTURE_RMS - ? new Float32Array() - : pcm; + return pcm; } export class SpeechOperationError extends Schema.TaggedError()( @@ -281,6 +277,23 @@ const make = Effect.gen(function* () { const selectedModel = (settings: SettingsSnapshot): SpeechModel => getSpeechModel(settings.speechModelId) ?? getSpeechModel(DEFAULT_SPEECH_MODEL_ID)!; + let silenceRetryAt = 0; + const acquireSilenceModel = Effect.fn("SpeechService.acquireSilenceModel")(function* ( + purpose: "installation" | "recording", + ) { + if (purpose === "recording" && now() < silenceRetryAt) return undefined; + const path = yield* store.download(SILERO_VAD).pipe( + Effect.timeout(purpose === "recording" ? "15 seconds" : "2 minutes"), + Effect.orElseSucceed(() => undefined), + ); + if (path !== undefined) { + silenceRetryAt = 0; + return path; + } + silenceRetryAt = now() + 10 * 60_000; + yield* Effect.logWarning("Silero unavailable; using raw audio with bounded ASR processing."); + }); + // Publish the reservation before forking: a cached transfer can finish before forkIn returns. const downloadFiber = Effect.fn("SpeechService.downloadFiber")(function* (modelId: string) { const definition = getSpeechModel(modelId); @@ -306,6 +319,9 @@ const make = Effect.gen(function* () { }, }) .pipe( + (modelDownload) => + Effect.all([modelDownload, acquireSilenceModel("installation")], { concurrency: 2 }), + Effect.map(([modelPath]) => modelPath), Effect.mapError((cause) => speechError("model download", cause)), Effect.forkIn(scope), ); @@ -442,6 +458,7 @@ const make = Effect.gen(function* () { model: definition.name, size: definition.size, supportsStreaming: definition.supportsStreaming, + supportsBufferedStreaming: true, supportsTranslation: definition.supportsTranslation, languages: definition.languages, supportsLanguageDetection: definition.supportsLanguageDetection, @@ -512,7 +529,9 @@ const make = Effect.gen(function* () { language: string | undefined, ) => { return ( - options.speechRemoveFillerWords ? removeSpeechFillerWords(text, language) : text + options.speechRemoveFillerWords + ? removeSpeechFillerWords(text, language, options.speechCorrectionWord) + : text ).trim(); }; @@ -543,6 +562,15 @@ const make = Effect.gen(function* () { ), ); + const silenceModelPath = Effect.fn("SpeechService.silenceModelPath")(function* ( + loaded: LoadedModel, + options: SpeechTranscriptionOptions, + ) { + if (!options.speechSilenceFiltering && (loaded.supportsStreaming || loaded.supportsLongForm)) + return undefined; + return yield* acquireSilenceModel("recording"); + }); + const transcribe: SpeechService["Service"]["transcribe"] = Effect.fn("SpeechService.transcribe")( function* (pcmBytes, options) { const settings = yield* readSettings("transcription"); @@ -574,6 +602,7 @@ const make = Effect.gen(function* () { ), catch: (cause) => speechError("model preparation", cause), }); + const vadPath = yield* silenceModelPath(loaded, options); const dictionary = normalizeSpeechCustomWords(options.speechCustomWords); const customWords = dictionary.map(({ term }) => term); // The vocabulary capability guarantees plain transcription, not translation. @@ -584,14 +613,19 @@ const make = Effect.gen(function* () { try: (signal) => { // Interruption kills the isolated process; the next request reloads the model. signal.addEventListener("abort", () => dropModel(inference), { once: true }); - return inference.transcribe(pcm, { - timestamps: "none", - ...(language === "auto" ? {} : { language }), - ...(translateToEnglish - ? { task: "translate" as const, targetLanguage: "en" as const } - : {}), - ...(vocabularyFor(inference).length > 0 ? { vocabulary: customWords } : {}), - }); + return inference.transcribe( + pcm, + { + timestamps: "none", + ...(language === "auto" ? {} : { language }), + ...(translateToEnglish + ? { task: "translate" as const, targetLanguage: "en" as const } + : {}), + ...(vocabularyFor(inference).length > 0 ? { vocabulary: customWords } : {}), + }, + options.speechSilenceFiltering, + vadPath, + ); }, catch: (cause) => { dropModel(inference); @@ -703,12 +737,28 @@ const make = Effect.gen(function* () { loadModel(modelPath, definition, signal, settings.speechAcceleration), ); loaded = streamModel; + const vadPath = yield* silenceModelPath(streamModel, options); + let translateToEnglish = false; const beginStream = async () => { - if (!streamModel.supportsStreaming) - throw new Error("The selected model does not support streaming."); + translateToEnglish = + options.speechTranslateToEnglish && + !streamModel.supportsStreaming && + streamModel.supportsTranslation && + language !== "en"; const vocabulary = - streamModel.supportsVocabulary && customWords.length > 0 ? customWords : undefined; - await streamModel.begin(language === "auto" ? undefined : language, vocabulary); + streamModel.supportsVocabulary && !translateToEnglish && customWords.length + ? customWords + : undefined; + await streamModel.begin( + { + timestamps: "none", + ...(language === "auto" ? {} : { language }), + ...(translateToEnglish ? { task: "translate", targetLanguage: "en" } : {}), + ...(vocabulary ? { vocabulary } : {}), + }, + options.speechSilenceFiltering, + vadPath, + ); correct = makeSpeechTextCorrector(vocabulary ? [] : customWords, dictionary); started = true; }; @@ -768,7 +818,7 @@ const make = Effect.gen(function* () { byteLength: bytes.byteLength, message: "Speech stream audio limit exceeded.", }); - const pcm = decodeSpeechPcm(bytes, true); + const pcm = decodeSpeechPcm(bytes); byteLength += bytes.byteLength; received.push(pcm); let update: Awaited> | undefined; @@ -806,7 +856,7 @@ const make = Effect.gen(function* () { return finalizeTranscript( correct(text), options, - language === "auto" ? undefined : language, + translateToEnglish ? "en" : language === "auto" ? undefined : language, ); }), }; diff --git a/apps/server/src/speech/audio.test.ts b/apps/server/src/speech/audio.test.ts new file mode 100644 index 000000000000..d049f8da51cb --- /dev/null +++ b/apps/server/src/speech/audio.test.ts @@ -0,0 +1,92 @@ +import { expect, it } from "vite-plus/test"; +import { createSpeechAudio } from "./audio.ts"; + +const frame = 512; +const make = (overrides = {}) => + createSpeechAudio({ + frameSamples: frame, + filterSilence: true, + nativeStreaming: false, + longForm: false, + maxAudioMs: 0, + ...overrides, + }); +const frames = (count: number) => + Float32Array.from({ length: count * frame }, (_, i) => (Math.floor(i / frame) + 1) / 1000); +const seconds = (n: number) => Math.floor((n * 16000) / frame); + +it("retains pre-roll, onset and offline/streaming tails without duplicating edges", () => { + for (const nativeStreaming of [false, true]) { + const audio = make({ nativeStreaming }); + const pcm = frames(100); + const chunks: Float32Array[] = []; + for (let i = 0; i < 100; i++) { + audio.append(pcm.subarray(i * frame, (i + 1) * frame)); + audio.score(i, [i >= 20 && i < 24 ? 0.9 : 0]); + chunks.push(...audio.take()); + } + chunks.push(...audio.take(true)); + const retained = chunks.flatMap((chunk) => [...chunk]); + expect(retained).toEqual([ + ...pcm.subarray(6 * frame, (24 + (nativeStreaming ? 52 : 15)) * frame), + ]); + } +}); + +it("handles silence, short speech, a partial last frame and disabled filtering", () => { + const silence = make(); + silence.append(new Float32Array(frame * 4 + 7)); + silence.score(0, [0, 0, 0, 0, 0]); + expect(silence.take(true)).toEqual([]); + const short = make(); + short.append(new Float32Array([0.2, 0.1])); + short.score(0, [0.8]); + expect([...short.take(true)[0]!]).toEqual([...new Float32Array([0.2, 0.1])]); + const raw = make({ filterSilence: false }); + const pcm = frames(50).subarray(0, 50 * frame - 3); + raw.append(pcm); + raw.score(0, new Array(50).fill(0)); + expect(raw.take(true)[0]).toEqual(pcm); +}); + +it("plans at pauses while recording and preserves ordered audio and final tails", () => { + const audio = make({ filterSilence: false }); + const pcm = frames(seconds(100)); + const chunks: Float32Array[] = []; + for (let start = 0; start < pcm.length; start += frame * 7) { + const piece = pcm.subarray(start, start + frame * 7); + audio.append(piece); + audio.score( + start / frame, + Array.from({ length: piece.length / frame }, (_, i) => { + const t = ((start / frame + i) * frame) / 16000; + return (t >= 28 && t < 29) || (t >= 55 && t < 56) ? 0 : 1; + }), + ); + chunks.push(...audio.take()); + } + expect(chunks.length).toBeGreaterThan(0); + expect(chunks[0]!.length / 16000).toBeCloseTo(28.5, 1); + chunks.push(...audio.take(true)); + expect(Float32Array.from(chunks.flatMap((chunk) => [...chunk]))).toEqual(pcm); + expect(chunks.every((chunk) => chunk.length <= 40 * 16000)).toBe(true); + expect(chunks.at(-1)!.length).toBeGreaterThanOrEqual(5 * 16000); +}); + +it("respects smaller engine limits, hard cuts, and native/internal-long-form eligibility", () => { + for (const config of [{}, { maxAudioMs: 4000 }, { nativeStreaming: true }, { longForm: true }]) { + const audio = make({ filterSilence: false, ...config }); + const pcm = frames(seconds(84)); + audio.append(pcm); + audio.score(0, new Array(pcm.length / frame).fill(1)); + const live = audio.take(); + const chunks = [...live, ...audio.take(true)]; + expect(chunks.reduce((n, chunk) => n + chunk.length, 0)).toBe(pcm.length); + if ("nativeStreaming" in config || "longForm" in config) expect(chunks).toHaveLength(1); + else + expect( + chunks.every((chunk) => chunk.length <= ("maxAudioMs" in config ? 32000 : 640000)), + ).toBe(true); + if ("longForm" in config) expect(live).toEqual([]); + } +}); diff --git a/apps/server/src/speech/audio.ts b/apps/server/src/speech/audio.ts new file mode 100644 index 000000000000..62f31f182c8a --- /dev/null +++ b/apps/server/src/speech/audio.ts @@ -0,0 +1,174 @@ +// Self-contained because the native worker embeds this factory in Node, Electron and SEA. +export function createSpeechAudio(config: { + frameSamples: number; + filterSilence: boolean; + nativeStreaming: boolean; + longForm: boolean; + maxAudioMs: number; +}) { + const rate = 16_000; + const frame = config.frameSamples; + const prefill = Math.ceil((450 * rate) / 1000 / frame); + const onset = Math.ceil((60 * rate) / 1000 / frame); + const tail = Math.ceil(((config.nativeStreaming ? 1650 : 450) * rate) / 1000 / frame); + const eligible = !config.nativeStreaming && !config.longForm; + // Preserve the existing output-limit headroom and bounded offline retries. + const maximum = Math.min( + 40 * rate, + config.maxAudioMs > 0 ? Math.max(1, Math.floor((config.maxAudioMs * rate) / 2000)) : 40 * rate, + ); + const minimum = Math.min(5 * rate, Math.floor(maximum / 4)); + const target = Math.min(30 * rate, maximum * 0.75); + const blocks: { start: number; pcm: Float32Array }[] = []; + const probs: number[] = []; + const keep: boolean[] = []; + let length = 0; + let start = 0; + let streamAt = 0; + let inSpeech = false; + let onsetCount = 0; + let hangover = 0; + + const raw = (from: number, to: number) => { + const output = new Float32Array(to - from); + for (const block of blocks) { + const left = Math.max(from, block.start); + const right = Math.min(to, block.start + block.pcm.length); + if (left < right) + output.set(block.pcm.subarray(left - block.start, right - block.start), left - from); + } + return output; + }; + const read = (from: number, to: number) => { + const pcm = raw(from, to); + if (!config.filterSilence) return pcm; + let count = 0; + for (let at = from; at < to;) { + const end = Math.min(to, (Math.floor(at / frame) + 1) * frame); + if (keep[Math.floor(at / frame)]) { + pcm.copyWithin(count, at - from, end - from); + count += end - at; + } + at = end; + } + return pcm.subarray(0, count); + }; + // Silero timestamp hysteresis is separate from the capture filter's 0.3 threshold. + const pauses = (samples: number, final: boolean) => { + const segments: { start: number; end: number }[] = []; + let speechStart: number | undefined; + let silenceStart: number | undefined; + for (let i = 0; i < probs.length; i++) { + const at = i * frame; + if (probs[i]! >= 0.5) { + silenceStart = undefined; + speechStart ??= at; + } else if (probs[i]! < 0.35 && speechStart !== undefined) { + silenceStart ??= at; + if (at - silenceStart >= 0.1 * rate) { + if (silenceStart - speechStart > 0.25 * rate) + segments.push({ start: speechStart, end: silenceStart }); + speechStart = silenceStart = undefined; + } + } + } + if (speechStart !== undefined && samples - speechStart > 0.25 * rate) + segments.push({ start: speechStart, end: samples }); + const candidates: { cut: number; gap: number }[] = []; + let end = 0; + for (const segment of segments) { + if (segment.start > end) + candidates.push({ + cut: Math.floor((end + segment.start) / 2), + gap: (segment.start - end) / rate, + }); + end = segment.end; + } + // An open pause's midpoint can still move, so only final audio uses it. + if (final && samples > end) + candidates.push({ cut: Math.floor((end + samples) / 2), gap: (samples - end) / rate }); + return candidates; + }; + + return { + append(pcm: Float32Array) { + blocks.push({ start: length, pcm }); + length += pcm.length; + }, + score(firstFrame: number, scores: readonly number[]) { + if (firstFrame !== probs.length || scores.some((p) => !Number.isFinite(p) || p < 0 || p > 1)) + throw new Error("Invalid Silero frame sequence."); + for (const probability of scores) { + const i = probs.length; + probs.push(probability); + keep.push(false); + const voiced = probability > 0.3; + if (!inSpeech && voiced) { + if (++onsetCount >= onset) { + inSpeech = true; + hangover = tail; + onsetCount = 0; + for (let j = Math.max(0, i - prefill); j <= i; j++) keep[j] = true; + } + } else if (inSpeech && voiced) { + hangover = tail; + keep[i] = true; + } else if (inSpeech && hangover > 0) { + hangover--; + keep[i] = true; + } else { + inSpeech = false; + onsetCount = 0; + } + } + }, + audio: () => raw(0, length), + samples: () => length, + take(final = false): Float32Array[] { + const samples = Math.min(length, probs.length * frame); + // Preserve a very short utterance or syllable stopped before onset confirms. + if (final && onsetCount > 0) + for (let i = Math.max(0, probs.length - 1 - prefill); i < probs.length; i++) keep[i] = true; + if (config.nativeStreaming) { + const output = read(streamAt, samples); + let lastKept = streamAt; + for (let i = Math.floor(streamAt / frame); i < probs.length; i++) + if (!config.filterSilence || keep[i]) lastKept = Math.min(samples, (i + 1) * frame); + streamAt = Math.max(lastKept, Math.max(0, samples - prefill * frame)); + return output.length ? [output] : []; + } + const chunks: Float32Array[] = []; + const lookahead = Math.max(minimum, (prefill + onset) * frame); + if (eligible && samples - start > maximum + (final ? 0 : lookahead)) { + const candidates = pauses(samples, final); + while (samples - start > maximum + (final ? 0 : lookahead)) { + let best: { cut: number; score: number } | undefined; + let fallback: typeof best; + for (const candidate of candidates) { + const position = candidate.cut - start; + if (position < minimum || position > maximum) continue; + const offTarget = (position - target) / rate; + const score = + Math.min(candidate.gap, 1) - Math.abs(offTarget) * (offTarget > 0 ? 0.1 : 0.05); + if (!fallback || score > fallback.score) fallback = { cut: candidate.cut, score }; + if (samples - candidate.cut >= minimum && (!best || score > best.score)) + best = { cut: candidate.cut, score }; + } + const hard = + start + + (samples - start < maximum + minimum ? Math.floor((samples - start) / 2) : maximum); + const cut = (best ?? fallback)?.cut ?? hard; + const chunk = read(start, cut); + if (chunk.length) chunks.push(chunk); + start = cut; + } + } + if (final && samples > start) { + const chunk = read(start, samples); + if (chunk.length) chunks.push(chunk); + start = samples; + } + return chunks; + }, + }; +} diff --git a/apps/server/src/speech/fillerWords.ts b/apps/server/src/speech/fillerWords.ts index 0181db1bf916..5061341833c0 100644 --- a/apps/server/src/speech/fillerWords.ts +++ b/apps/server/src/speech/fillerWords.ts @@ -22,19 +22,21 @@ const LANGUAGE_FILLER_WORDS: Readonly> = { const escapeRegex = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); -export function removeSpeechFillerWords(text: string, language?: string): string { +export function removeSpeechFillerWords( + text: string, + language?: string, + correctionCue?: string, +): string { const baseLanguage = language?.split(/[-_]/, 1)[0]?.toLowerCase(); const words = [ ...UNIVERSAL_FILLER_WORDS, ...(baseLanguage ? (LANGUAGE_FILLER_WORDS[baseLanguage] ?? []) : []), ]; - let filtered = text; - for (const word of words) { - const pattern = new RegExp( - `(? cue ?? ""); return filtered.replace(/\s{2,}/g, " ").trim(); } diff --git a/apps/server/src/speech/native.test.ts b/apps/server/src/speech/native.test.ts index da92597d06e7..97aa1b566020 100644 --- a/apps/server/src/speech/native.test.ts +++ b/apps/server/src/speech/native.test.ts @@ -39,7 +39,11 @@ it("returns transcription from an isolated process and disposes it", async () => ); try { expect( - await model.transcribe(new Float32Array([0.25]), { timestamps: "none", language: "en" }), + await model.transcribe( + new Float32Array([0.25]), + { timestamps: "none", language: "en" }, + false, + ), ).toEqual({ text: "0.25" }); } finally { await model.dispose(); @@ -60,7 +64,7 @@ it("probes generic vocabulary support and delivers ordered terms to offline infe language: "en", ...(supported ? { vocabulary: ["T3 Code", "GitHub"] } : {}), }; - const result = await model.transcribe(new Float32Array([0.25]), options); + const result = await model.transcribe(new Float32Array([0.25]), options, false); expect(JSON.parse(result.text)).toEqual(options); } finally { await model.dispose(); @@ -76,7 +80,7 @@ it("ignores IPC messages that do not belong to the speech protocol", async () => ); try { await expect( - model.transcribe(new Float32Array([0.25]), { timestamps: "none", language: "en" }), + model.transcribe(new Float32Array([0.25]), { timestamps: "none", language: "en" }, false), ).resolves.toEqual({ text: "hello" }); } finally { await model.dispose(); @@ -90,10 +94,14 @@ it("terminates hung native inference when the service shuts down", async () => { controller.signal, fixture("async () => { while (true) {} }"), ); - const pending = model.transcribe(new Float32Array([0.25]), { - timestamps: "none", - language: "en", - }); + const pending = model.transcribe( + new Float32Array([0.25]), + { + timestamps: "none", + language: "en", + }, + false, + ); const rejected = expect(pending).rejects.toThrow(); controller.abort(); await model.dispose(); @@ -111,17 +119,22 @@ it("returns streaming previews and final text from an isolated process", async ( ); try { expect(model.supportsStreaming).toBe(true); - await model.begin(); - expect(await model.feed(new Float32Array([0.25]))).toEqual({ + await model.begin({ timestamps: "none" }, false); + expect(await model.feed(new Float32Array(512).fill(0.25))).toEqual({ revision: 1, text: { committed: "hello ", tentative: "world" }, }); expect(await model.finish()).toBe("hello world"); - await model.begin("fr"); + await model.begin({ timestamps: "none", language: "fr" }, false); await model.reset(); - await model.begin("fr"); + await model.begin({ timestamps: "none", language: "fr" }, false); + await model.feed(new Float32Array(512)); expect(await model.finish()).toBe("fr"); - await model.begin("fr", ["T3 Code", "GitHub"]); + await model.begin( + { timestamps: "none", language: "fr", vocabulary: ["T3 Code", "GitHub"] }, + false, + ); + await model.feed(new Float32Array(512).fill(0.25)); expect(await model.finish()).toBe("T3 Code / GitHub"); } finally { await model.dispose(); @@ -135,9 +148,170 @@ it("kills a hung streaming feed without waiting for native cleanup", async () => controller.signal, streamingFixture("async () => { while (true) {} }"), ); - await model.begin(); - const rejected = expect(model.feed(new Float32Array([0.25]))).rejects.toThrow(); + await model.begin({ timestamps: "none" }, false); + const rejected = expect(model.feed(new Float32Array(512).fill(0.25))).rejects.toThrow(); controller.abort(); await model.dispose(); await rejected; }); +const vadFixture = ({ + streaming = false, + failVad = false, + failVadLoad = false, + failAsr = false, + maxAudioMs = 4000, +} = {}) => + `data:text/javascript,${encodeURIComponent(` + const summarize = pcm => String(pcm.length) + ":" + pcm[0].toFixed(2) + ":" + pcm[pcm.length - 1].toFixed(2); + export const TranscribeModel = { load: async (path, options) => { + if (path === "vad.gguf") { + if (${failVadLoad}) throw new Error("VAD unavailable"); + if (options.backend !== "cpu") throw new Error("VAD must use CPU"); + return { vadInfo: { sampleRate: 16000, frameSamples: 512 }, dispose() {}, + createVadSession: ({ nThreads }) => { + if (nThreads !== 1) throw new Error("VAD thread count"); + let pending = []; let at = 0; + const score = final => { + const firstFrame = at; + const probs = []; + while (pending.length >= 512 || final && pending.length) { + probs.push(pending[0] > 0.5 ? 0.9 : 0); + pending.splice(0, 512); at++; + } + return { firstFrame, probs }; + }; + return { dispose() {}, streamReset: async () => { pending = []; at = 0; }, + streamFeed: async pcm => { + if (${failVad} && pcm[0] < 0) throw new Error("VAD fixture failed"); + pending.push(...pcm); return score(false); + }, + streamFlush: async () => score(true), + }; + }, + }; + } + return { backend: "CPU", supports: () => false, + capabilities: { supportsStreaming: ${streaming}, maxAudioMs: ${maxAudioMs} }, + transcribe: async pcm => { if (${failAsr}) throw new Error("ASR fixture failed"); return { text: summarize(pcm) }; }, + createSession: () => { + let all = []; + return { dispose() {}, stream: async () => ({ + feed: async pcm => { all.push(...pcm); return { committedChanged: true }; }, + finalize: async () => {}, reset() { all = []; }, + get text() { const full = summarize(all); return { committed: full, tentative: "", full }; }, + }) }; + }, + }; + }}; + `)}`; + +it("filters silence and preserves partial speech through the production worker and resets between recordings", async () => { + for (const streaming of [false, true]) { + const model = await loadNativeSpeechModel( + "asr.gguf", + new AbortController().signal, + vadFixture({ streaming }), + "cpu", + ); + try { + expect( + await model.transcribe(new Float32Array(2000), { timestamps: "none" }, true, "vad.gguf"), + ).toEqual({ text: "" }); + expect( + await model.transcribe( + new Float32Array([0.8, 0.6]), + { timestamps: "none" }, + true, + "vad.gguf", + ), + ).toEqual({ text: "2:0.80:0.60" }); + expect( + await model.transcribe(new Float32Array(2000), { timestamps: "none" }, false, "vad.gguf"), + ).toEqual({ text: "2000:0.00:0.00" }); + for (const cancel of [true, false]) { + await model.begin({ timestamps: "none" }, true, "vad.gguf"); + await model.feed(new Float32Array(600)); + expect(await model.finish()).toBe(""); + await model.begin({ timestamps: "none" }, true, "vad.gguf"); + await model.feed(new Float32Array(511).fill(0.8)); + if (cancel) await model.reset(); + else expect(await model.finish()).toBe("511:0.80:0.80"); + } + } finally { + await model.dispose(); + } + } +}); + +it("decodes complete stable chunks before Stop and returns the ordered final tail", async () => { + const model = await loadNativeSpeechModel( + "asr.gguf", + new AbortController().signal, + vadFixture(), + "cpu", + ); + try { + await model.begin({ timestamps: "none" }, false, "vad.gguf"); + await model.feed(new Float32Array(16000).fill(0.1)); + await model.feed(new Float32Array(16000).fill(0.2)); + const update = await model.feed(new Float32Array(16000).fill(0.3)); + expect(update.text).toEqual({ committed: "32000:0.10:0.20", tentative: "" }); + await model.feed(new Float32Array([0.4, 0.5])); + expect(await model.finish()).toBe("32000:0.10:0.20 16002:0.30:0.50"); + } finally { + await model.dispose(); + } +}); + +it("preserves withheld audio on VAD failure while keeping ASR failures distinct", async () => { + for (const streaming of [false, true]) { + const model = await loadNativeSpeechModel( + "asr.gguf", + new AbortController().signal, + vadFixture({ streaming, failVad: true }), + "cpu", + ); + try { + await model.begin({ timestamps: "none" }, true, "vad.gguf"); + await model.feed(new Float32Array(512).fill(0.1)); + await model.feed(new Float32Array(512).fill(-0.2)); + expect(await model.finish()).toBe("1024:0.10:-0.20"); + await model.begin({ timestamps: "none" }, true, "vad.gguf"); + await model.feed(new Float32Array(512)); + expect(await model.finish()).toBe(""); + } finally { + await model.dispose(); + } + } + const unavailable = await loadNativeSpeechModel( + "asr.gguf", + new AbortController().signal, + vadFixture({ failVadLoad: true }), + "cpu", + ); + try { + expect( + await unavailable.transcribe( + new Float32Array(2000).fill(0.1), + { timestamps: "none" }, + true, + "vad.gguf", + ), + ).toEqual({ text: "2000:0.10:0.10" }); + } finally { + await unavailable.dispose(); + } + const model = await loadNativeSpeechModel( + "asr.gguf", + new AbortController().signal, + vadFixture({ failAsr: true }), + "cpu", + ); + try { + await expect( + model.transcribe(new Float32Array(512).fill(0.8), { timestamps: "none" }, true, "vad.gguf"), + ).rejects.toThrow("ASR fixture failed"); + } finally { + await model.dispose(); + } +}); diff --git a/apps/server/src/speech/native.ts b/apps/server/src/speech/native.ts index 88809ab97663..fae308333b24 100644 --- a/apps/server/src/speech/native.ts +++ b/apps/server/src/speech/native.ts @@ -2,16 +2,142 @@ import * as NodeChildProcess from "node:child_process"; import * as NodeSea from "node:sea"; import type { SpeechStreamText } from "@t3tools/contracts"; +import { createSpeechAudio } from "./audio.ts"; // Inline the small child entry so the packaged CLI does not need a separate worker artifact. const entry = ` +const createSpeechAudio = ${createSpeechAudio.toString()}; +let speech; let model; +let vadModel; +let vad; let session; let stream; +let audio; +let options; +let nativeStreaming; +let useVad; +let scored; +let revision; +let texts; +let heardAudio; +const warnVad = () => console.warn("Silero silence filtering unavailable; preserving raw recording and using bounded ASR chunks."); +const resetAsr = () => { + if (stream) stream.reset(); + if (session) session.dispose(); + stream = session = undefined; +}; +const newAudio = (filterSilence) => createSpeechAudio({ + frameSamples: 512, filterSilence, nativeStreaming: model.capabilities.supportsStreaming, + longForm: model.supports("long_form"), maxAudioMs: model.capabilities.maxAudioMs, +}); +const openAsr = async () => { + if (nativeStreaming) { + session = model.createSession(); + stream = await session.stream(options); + } +}; +const begin = async (message, live) => { + resetAsr(); + options = message.options; + nativeStreaming = live && model.capabilities.supportsStreaming; + texts = []; + scored = revision = 0; + heardAudio = false; + useVad = Boolean(message.vadPath) && (message.filterSilence || + (!model.capabilities.supportsStreaming && !model.supports("long_form"))); + if (useVad) { + try { + if (!vadModel) { + vadModel = await speech.TranscribeModel.load(message.vadPath, { backend: "cpu" }); + if (vadModel.vadInfo.sampleRate !== 16000 || vadModel.vadInfo.frameSamples !== 512) + throw new Error("Unsupported Silero model frames."); + vad = vadModel.createVadSession({ nThreads: 1 }); + } + await vad.streamReset(); + } catch { + vad?.dispose(); + vadModel?.dispose(); + vad = vadModel = undefined; + useVad = false; + warnVad(); + } + } + audio = newAudio(useVad && message.filterSilence); + await openAsr(); +}; +const transcribe = async (pcm) => { + try { + const result = await model.transcribe(pcm, options); + texts.push(result.text); + } catch (error) { + if ((error?.constructor?.name !== "OutputTruncated" && + error?.constructor?.name !== "InputTooLong") || pcm.length <= 4000) throw error; + const middle = Math.floor(pcm.length / 2); + await transcribe(pcm.subarray(0, middle)); + await transcribe(pcm.subarray(middle)); + } +}; +const decode = async (final) => { + let preview = null; + for (const pcm of audio.take(final)) { + if (!pcm.length) continue; + heardAudio = true; + if (nativeStreaming) { + const update = await stream.feed(pcm); + if (update.committedChanged || update.tentativeChanged) { + const text = stream.text; + preview = { committed: text.committed, tentative: text.tentative }; + } + } else { + await transcribe(pcm); + preview = { committed: texts.join(" ").trim(), tentative: "" }; + } + } + return preview; +}; +const feed = async (pcm, final = false) => { + if (pcm) audio.append(pcm); + try { + if (useVad) { + if (pcm) { + const result = await vad.streamFeed(pcm); + audio.score(result.firstFrame, result.probs); + if (result.firstFrame + result.probs.length !== Math.floor(audio.samples() / 512)) + throw new Error("Incomplete Silero frames."); + } + if (final) { + const result = await vad.streamFlush(); + audio.score(result.firstFrame, result.probs); + if (result.firstFrame + result.probs.length !== Math.ceil(audio.samples() / 512)) + throw new Error("Incomplete Silero tail."); + } + } else { + const count = final ? Math.ceil(audio.samples() / 512) : Math.floor(audio.samples() / 512); + audio.score(scored, new Array(count - scored).fill(1)); + scored = count; + } + } catch { + // Replay on optional VAD failure, including audio it had withheld; ASR errors stay separate. + const raw = audio.audio(); + useVad = false; + warnVad(); + resetAsr(); + texts = []; + heardAudio = false; + audio = newAudio(false); + audio.append(raw); + scored = final ? Math.ceil(raw.length / 512) : Math.floor(raw.length / 512); + audio.score(0, new Array(scored).fill(1)); + await openAsr(); + } + return await decode(final); +}; process.on("message", async (message) => { try { if (message.kind === "load") { - const { TranscribeModel, getAvailableBackends } = await loadSpeechModule(message.moduleUrl); + speech = await loadSpeechModule(message.moduleUrl); + const { TranscribeModel, getAvailableBackends } = speech; const device = message.acceleration.startsWith("gpu:") ? getAvailableBackends().find((item) => (item.deviceType === "gpu" || item.deviceType === "igpu") && @@ -19,80 +145,40 @@ process.on("message", async (message) => { : undefined; if (message.acceleration.startsWith("gpu:") && !device) throw new Error("The selected speech GPU is unavailable."); - model = await TranscribeModel.load(message.path, device - ? { device } - : { backend: message.acceleration }); + model = await TranscribeModel.load(message.path, device ? { device } : { backend: message.acceleration }); process.send({ type: "t3-speech-reply", ok: true, backend: model.backend, supportsStreaming: model.capabilities.supportsStreaming, + supportsLongForm: model.supports("long_form"), supportsTranslation: model.capabilities.supportsTranslate, supportsVocabulary: model.supports("vocabulary") }); } else if (message.kind === "begin") { - session = model.createSession(); - stream = await session.stream({ timestamps: "none", language: message.language, - vocabulary: message.vocabulary }); + await begin(message, true); process.send({ type: "t3-speech-reply", ok: true }); } else if (message.kind === "feed") { - const update = await stream.feed(message.pcm); - const text = update.committedChanged || update.tentativeChanged ? stream.text : null; - process.send({ type: "t3-speech-reply", ok: true, revision: update.revision, - preview: text ? { committed: text.committed, tentative: text.tentative } : null }); + const preview = await feed(message.pcm); + process.send({ type: "t3-speech-reply", ok: true, revision: ++revision, preview }); } else if (message.kind === "finish") { - await stream.finalize(); - const text = stream.text.full; - stream.reset(); - session.dispose(); - stream = session = undefined; + await feed(undefined, true); + if (nativeStreaming && heardAudio) await stream.finalize(); + const text = nativeStreaming ? (heardAudio ? stream.text.full : "") : texts.join(" ").trim(); + resetAsr(); + audio = undefined; process.send({ type: "t3-speech-reply", ok: true, text }); } else if (message.kind === "reset") { - stream.reset(); - session.dispose(); - stream = session = undefined; + resetAsr(); + if (vad) await vad.streamReset(); + audio = undefined; process.send({ type: "t3-speech-reply", ok: true }); } else if (message.kind === "transcribe") { - // Leave unbounded models alone. Use headroom for output-bound models: - // maxAudioMs is an estimate, so dense speech may still need smaller pieces. - const limit = model.capabilities.maxAudioMs; - const chunkSamples = limit > 0 ? Math.max(4000, Math.floor(limit * 16 / 2)) : message.pcm.length; - const texts = []; - const transcribe = async (pcm) => { - try { - const result = await model.transcribe(pcm, message.options); - texts.push(result.text); - } catch (error) { - if ((error?.constructor?.name !== "OutputTruncated" && - error?.constructor?.name !== "InputTooLong") || pcm.length <= 4000) throw error; - const middle = Math.floor(pcm.length / 2); - await transcribe(pcm.subarray(0, middle)); - await transcribe(pcm.subarray(middle)); - } - }; - let offset = 0; - while (offset < message.pcm.length) { - let end = Math.min(offset + chunkSamples, message.pcm.length); - if (end < message.pcm.length) { - // Prefer the quietest 20 ms frame in the final quarter of the window. - let quietest = Infinity; - const windowEnd = end; - for (let start = offset + Math.floor(chunkSamples * 0.75); start + 320 <= windowEnd; start += 320) { - let energy = 0; - for (let i = start; i < start + 320; i++) energy += message.pcm[i] ** 2; - if (energy < quietest) { - quietest = energy; - end = start + 160; - } - } - } - await transcribe(message.pcm.subarray(offset, end)); - offset = end; - } - process.send({ type: "t3-speech-reply", ok: true, text: texts.join(" ").trim() }); + await begin(message, false); + await feed(message.pcm, true); + const text = texts.join(" ").trim(); + audio = undefined; + process.send({ type: "t3-speech-reply", ok: true, text }); } } catch (error) { - process.send({ - type: "t3-speech-reply", - ok: false, - error: error instanceof Error ? error.stack ?? error.message : String(error), - }); + process.send({ type: "t3-speech-reply", ok: false, + error: error instanceof Error ? error.stack ?? error.message : String(error) }); } }); process.on("disconnect", () => process.exit(0)); @@ -104,6 +190,7 @@ type Reply = { readonly text?: string; readonly error?: string; readonly supportsStreaming?: boolean; + readonly supportsLongForm?: boolean; readonly supportsTranslation?: boolean; readonly supportsVocabulary?: boolean; readonly backend?: string; @@ -305,10 +392,11 @@ async function loadNativeSpeechModelOnce( return { backend: loaded.backend ?? "unknown", supportsStreaming: loaded.supportsStreaming === true, + supportsLongForm: loaded.supportsLongForm === true, supportsTranslation: loaded.supportsTranslation === true, supportsVocabulary: loaded.supportsVocabulary === true, - begin: async (language?: string, vocabulary?: readonly string[]) => { - await send({ kind: "begin", language, vocabulary }); + begin: async (options: NativeSpeechOptions, filterSilence: boolean, vadPath?: string) => { + await send({ kind: "begin", options, filterSilence, vadPath }); }, feed: async (pcm: Float32Array) => { const reply = await send({ kind: "feed", pcm }); @@ -343,18 +431,22 @@ async function loadNativeSpeechModelOnce( }, transcribe: async ( pcm: Float32Array, - options: { - readonly timestamps: "none"; - readonly task?: "translate"; - readonly targetLanguage?: "en"; - readonly language?: string; - readonly vocabulary?: readonly string[]; - }, + options: NativeSpeechOptions, + filterSilence: boolean, + vadPath?: string, ) => { - const result = await send({ kind: "transcribe", pcm, options }); + const result = await send({ kind: "transcribe", pcm, options, filterSilence, vadPath }); if (typeof result.text !== "string") throw new Error("Invalid speech process response."); return { text: result.text }; }, dispose, }; } + +type NativeSpeechOptions = { + readonly timestamps: "none"; + readonly task?: "translate"; + readonly targetLanguage?: "en"; + readonly language?: string; + readonly vocabulary?: readonly string[]; +}; diff --git a/apps/server/src/speech/silero.ts b/apps/server/src/speech/silero.ts new file mode 100644 index 000000000000..dceddb71dde5 --- /dev/null +++ b/apps/server/src/speech/silero.ts @@ -0,0 +1,8 @@ +// MIT-licensed internal helper; acquired and verified by the existing model store. +export const SILERO_VAD = { + id: "handy-computer/silero-vad-v6.2-gguf", + revision: "97bbd75179f535e4708bccdf61e9278b8b60dfc7", + filename: "silero-vad-v6.2-F32.gguf", + size: 1_241_056, + sha256: "0d871a800c420fdd5d1b17a3aea4296d1c39608f0ff4a9295eb76457eb1c88e6", +}; diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 4408bee38ad0..8e4183244480 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -3245,6 +3245,7 @@ export default function ChatView(props: ChatViewProps) { : undefined, }} files={{ + projectId: activeProject?.id, projectName: activeProject?.title ?? null, workspaceRoot: activeWorkspaceRoot, availableEditors, @@ -3694,6 +3695,7 @@ export default function ChatView(props: ChatViewProps) { } activeThreadModelSelection={activeThread?.modelSelection} activeProjectId={activeProject?.id ?? null} + projectName={activeProject?.title} contextWindowEntries={serverVisibleTurnItems} contextWindowLiveUsage={activeThreadLiveTokenUsage} contextWindowProviderThread={activeProviderThread} diff --git a/apps/web/src/components/DiffPanel.browser.test.tsx b/apps/web/src/components/DiffPanel.browser.test.tsx index 3157c3e5da2a..fb2fc81bae84 100644 --- a/apps/web/src/components/DiffPanel.browser.test.tsx +++ b/apps/web/src/components/DiffPanel.browser.test.tsx @@ -84,6 +84,7 @@ vi.mock("~/state/use-atom-command", () => ({ useAtomCommand: () => async () => { vi.mock("~/editorPreferences", () => ({ useOpenInPreferredEditor: () => async () => {} })); vi.mock("~/fileContextMenu", () => ({ useFileContextMenuHandler: () => () => {} })); vi.mock("~/browser/useOpenLink", () => ({ useOpenLink: () => async () => {} })); +vi.mock("./diffs/DiffCommentAnnotation", () => ({ DiffCommentAnnotation: () => null })); vi.mock("./diffs/DiffFileTree", () => ({ DiffFileTree: () => null })); vi.mock("./DiffFilePathCopyButton", () => ({ DiffFilePathCopyButton: () => null })); @@ -250,7 +251,7 @@ it.each([false, true])( const toggle = screen.getByRole("button", { name: `Expand ${path}`, exact: true }); await expect.element(toggle).toHaveAttribute("aria-expanded", "false"); await userEvent.click(toggle); - act(() => + await act(() => context.registry.set( context.patch, AsyncResult.success({ diff --git a/apps/web/src/components/DiffPanel.tsx b/apps/web/src/components/DiffPanel.tsx index f40760f847ef..e7e6d614ef77 100644 --- a/apps/web/src/components/DiffPanel.tsx +++ b/apps/web/src/components/DiffPanel.tsx @@ -1425,6 +1425,8 @@ export default function DiffPanel({ > (null); const commentPopupRef = useRef(null); const draftCommentRef = useRef(null); + const [voiceSetupOpen, setVoiceSetupOpen] = useState(false); const [unavailableSourceAnchor, setUnavailableSourceAnchor] = useState(null); const commentOpen = commentEditor?.open ?? false; @@ -167,7 +168,7 @@ export function AssistantCitationChip({ {composerSourceLink} {commentEditor ? ( { if (!open && !settleDraftOnClose(eventDetails.reason)) { eventDetails.cancel(); @@ -185,6 +186,8 @@ export function AssistantCitationChip({ {commentEditor.open ? ( { draftCommentRef.current = comment; commentEditor.onDraftChange?.(comment); diff --git a/apps/web/src/components/chat/AssistantCitationCommentEditor.tsx b/apps/web/src/components/chat/AssistantCitationCommentEditor.tsx index 0a8357bb9bab..cdd8e1249cfc 100644 --- a/apps/web/src/components/chat/AssistantCitationCommentEditor.tsx +++ b/apps/web/src/components/chat/AssistantCitationCommentEditor.tsx @@ -1,7 +1,10 @@ import { ASSISTANT_CITATION_MAX_COMMENT_LENGTH, type AssistantCitation } from "@t3tools/contracts"; -import { useState, type Ref } from "react"; +import { use, useEffect, useRef, useState, type RefObject } from "react"; + +import { TextFieldSpeechControls, useTextFieldSpeech } from "~/speech/useTextFieldSpeech"; import { textDirection } from "~/textDirection"; +import { ComposerContextActionsContext } from "../composerContextActions"; import { Button } from "../ui/button"; export function AssistantCitationCommentEditor({ @@ -11,26 +14,55 @@ export function AssistantCitationCommentEditor({ onSubmitAndSend, onCancel, onDraftChange, + onVoiceSetupOpenChange, }: { citation: AssistantCitation; - inputRef?: Ref; + inputRef?: RefObject; onSubmit: (comment: string) => boolean; onSubmitAndSend?: (comment: string) => boolean; onCancel: () => void; onDraftChange?: (comment: string) => void; + onVoiceSetupOpenChange?: (open: boolean) => void; }) { const [comment, setComment] = useState(citation.comment ?? ""); + const localInputRef = useRef(null); + const textareaRef = inputRef ?? localInputRef; + const { environmentId, projectId } = use(ComposerContextActionsContext); + const updateComment = (text: string) => { + setComment(text); + onDraftChange?.(text); + }; + const speech = useTextFieldSpeech({ + environmentId: environmentId ?? citation.environmentId, + projectId, + ownerKey: JSON.stringify([ + "citation", + environmentId, + citation.threadId, + citation.messageId, + citation.start, + citation.end, + ]), + text: comment, + textareaRef, + onTextChange: updateComment, + }); + const setupOpen = speech.setup.open; + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- keeps the citation popover open while the voice setup dialog owns focus + useEffect(() => { + onVoiceSetupOpenChange?.(setupOpen); + return () => onVoiceSetupOpenChange?.(false); + }, [onVoiceSetupOpenChange, setupOpen]); const commentTooLong = comment.length > ASSISTANT_CITATION_MAX_COMMENT_LENGTH; const submit = () => { - if (!commentTooLong) onSubmit(comment); + speech.submitAfterDictation((text) => { + if (text.length <= ASSISTANT_CITATION_MAX_COMMENT_LENGTH) onSubmit(text); + }); }; const submitAndSend = () => { - if (commentTooLong) return; - if (onSubmitAndSend) { - onSubmitAndSend(comment); - } else { - onSubmit(comment); - } + speech.submitAfterDictation((text) => { + if (text.length <= ASSISTANT_CITATION_MAX_COMMENT_LENGTH) (onSubmitAndSend ?? onSubmit)(text); + }); }; return ( @@ -41,12 +73,14 @@ export function AssistantCitationCommentEditor({ if (event.nativeEvent.isComposing || event.keyCode === 229) return; if (event.key === "Escape") { event.preventDefault(); - onCancel(); + if (speech.state.phase !== "idle") speech.cancel(); + else onCancel(); } }} >