diff --git a/.github/workflows/release-tauri.yml b/.github/workflows/release-tauri.yml index d55d9d8fd..feecb025a 100644 --- a/.github/workflows/release-tauri.yml +++ b/.github/workflows/release-tauri.yml @@ -324,6 +324,9 @@ jobs: env: TAURI_SIGNING_PRIVATE_KEY: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY }} TAURI_SIGNING_PRIVATE_KEY_PASSWORD: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY_PASSWORD }} + # [DEBUG-VOICE-1003] Matching Windows symbols for native crash diagnosis. + CARGO_PROFILE_RELEASE_DEBUG: line-tables-only + CARGO_PROFILE_RELEASE_STRIP: none run: | set +e # WiX/MSI only accepts a numeric prerelease identifier. Manual @@ -449,15 +452,23 @@ jobs: # Tests are tooling, not application payload. A targeted repair checks out # immutable app source above, but uses the reviewed workflow commit's smoke # script so fixes to test semantics do not require moving a release tag. - - name: Checkout installer smoke tooling + - name: Fetch installer smoke tooling if: matrix.platform == 'windows-latest' - uses: actions/checkout@v4 - with: - ref: ${{ github.workflow_sha }} - path: release-tooling - sparse-checkout: openless-all/app/scripts/windows-ime-install-smoke.ps1 - sparse-checkout-cone-mode: false - persist-credentials: false + shell: pwsh + env: + GH_TOKEN: ${{ github.token }} + TOOLING_SHA: ${{ github.workflow_sha }} + run: | + # Fetch only the pinned script: a sparse checkout still visits unrelated + # gitlinks during auth cleanup, including optional vendor runtimes. + $scriptPath = 'openless-all/app/scripts/windows-ime-install-smoke.ps1' + $encoded = gh api "repos/$env:GITHUB_REPOSITORY/contents/$scriptPath" --method GET -f "ref=$env:TOOLING_SHA" --jq '.content' + if ($LASTEXITCODE -ne 0) { throw 'Failed to fetch pinned installer smoke tooling.' } + $bytes = [Convert]::FromBase64String(($encoded -join '')) + if ($bytes.Length -eq 0) { throw 'Pinned installer smoke tooling is empty.' } + $destination = Join-Path $env:GITHUB_WORKSPACE "release-tooling/$scriptPath" + New-Item -ItemType Directory -Force -Path (Split-Path $destination -Parent) | Out-Null + [System.IO.File]::WriteAllBytes($destination, $bytes) - name: Verify Windows installers register TSF IME if: matrix.platform == 'windows-latest' @@ -608,6 +619,17 @@ jobs: openless-all/app/src-tauri/target/release/bundle/latest-darwin-${{ matrix.updater-arch }}*.json if-no-files-found: error + # [DEBUG-VOICE-1003] Verify both matching files exist before publishing diagnostics. + - name: Verify Windows diagnostic symbols + if: matrix.platform == 'windows-latest' + shell: pwsh + run: | + $files = @('openless-all/app/src-tauri/target/release/openless.exe', 'openless-all/app/src-tauri/target/release/openless.pdb') + foreach ($file in $files) { + if (-not (Test-Path $file)) { throw 'Missing Windows diagnostic binary or symbols' } + Get-FileHash -Algorithm SHA256 $file + } + - name: Upload Windows artifacts if: matrix.platform == 'windows-latest' uses: actions/upload-artifact@v4 @@ -616,6 +638,8 @@ jobs: path: | openless-all/app/src-tauri/target/release/bundle/nsis/*.exe openless-all/app/src-tauri/target/release/bundle/msi/*.msi + openless-all/app/src-tauri/target/release/openless.exe + openless-all/app/src-tauri/target/release/openless.pdb openless-all/app/src-tauri/target/release/bundle/provenance/openless-provenance.json if-no-files-found: error diff --git a/docs/research/issue-1003.md b/docs/research/issue-1003.md new file mode 100644 index 000000000..2925ccbb2 --- /dev/null +++ b/docs/research/issue-1003.md @@ -0,0 +1,197 @@ +# Issue #1003 语音激活输入法研究报告 + +> 调研日期：2026-10-18 +> +> 研究范围：GitHub Issue #1003、PR #1219、PR 关联提交，以及 `beta` 与 `feat/issue-1003-voice-activation` 的源码。本文只记录可追溯的主源证据；没有把 PR 的声明当作本次本地运行结果。 + +## 1. 结论摘要 + +| 问题 | 结论 | 证据 | +| --- | --- | --- | +| 功能是否已经进入当前 `beta` | **没有。** Issue 仍 OPEN；PR #1219 仍 OPEN、Draft、`REVIEW_REQUIRED`、`BLOCKED`，没有 merge commit。当前 `beta` 是 `de072a88`，而 PR head 是 fork `HKLHaoBin/openless` 的 `9270b43c`。 | [Issue #1003](https://github.com/Open-Less/openless/issues/1003)、[PR #1219](https://github.com/Open-Less/openless/pull/1219) | +| 分支落后情况 | 以当前 `beta` `de072a88373a222280a8c22948092a260880196c` 和功能分支 `9270b43c7e79ad9d462ad8bec78250de5f499093` 比较：功能分支 **领先 8、落后 3**，两边分叉，合并基点为 `96278f40d48b8ed02e3ce22ddc96b694c863a6bc`。落后的 3 个提交是当前 beta 新增的 `af8adf7e`、`5ab6ed43`、`de072a88`。 | [PR base/head 元数据](https://github.com/Open-Less/openless/pull/1219)、[功能分支提交](https://github.com/HKLHaoBin/openless/commits/feat/issue-1003-voice-activation)；本地 `git rev-list beta..feat/issue-1003-voice-activation` 与反向计数为 8/3 | +| 重启后持久化 `voiceActivationEnabled` 是否自启动 | **条件性地是。** `voice_activation_enabled` 以偏好字段持久化；应用第一次调用 `get_startup_snapshot` 时，Core `start()` 读取当前偏好并调用 `configure_voice_activation()`。模型已安装、权限可用且麦克风未占用时会进入等待唤醒；模型缺失、权限拒绝或麦克风忙时只会报告失败状态。 | [`config.rs:395-465`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/config.rs#L395-L465)、[`App.tsx:84-96`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src/App.tsx#L84-L96)、[`shared.ts:45-57`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src/lib/ipc/shared.ts#L45-L57)、[`api.rs:4530-4609`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/api.rs#L4530-L4609) | +| 连续监听是否有 VAD/CPU gate | **没有找到。** 空闲 warm capture 的每个 PCM chunk 都会直接交给 activation consumer；Sherpa online worker 对每个收到的 PCM 调用 `accept_pcm_chunk`。现有环形缓冲和录音队列是容量/线程安全机制，不是静音跳过或 CPU 门控。 | [`recorder.rs:594-645`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/recorder.rs#L594-L645)、[`sherpa_provider.rs:244-255`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs#L244-L255)、[`sherpa_provider.rs:292-343`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs#L292-L343) | +| 结束词带标点是否清理 | **已在功能分支实现。** NFKC、空白和终端/中英文标点会参与归一化；匹配仅接受文本后缀；从原始 transcript 尾部逆向剥离结束词时跳过标点和空白，保留正文本身的标点。测试覆盖 `完成，语音录入。`、全角句号、正文标点和中间出现结束词的反例。 | [`voice_activation.rs:49-115`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/voice_activation.rs#L49-L115)、[`voice_activation.rs:390-445`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/voice_activation.rs#L390-L445)、[`api.rs:2264-2321`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/api.rs#L2264-L2321) | + +**一句话判断：** PR 已完成“本地 Windows 语音唤醒/结束词”的主体链路以及结束词标点修复，但尚未合并到当前 beta；持续监听仍把静音 PCM 全部送入 Zipformer，CPU/VAD 问题是合并前必须明确验收或拆分处理的风险。 + +## 2. 主源与需求演进 + +### 2.1 原始需求 + +Issue 标题为“功能建议：用语音替代按键来激活输入法”。原始描述提出：说“开始语音录入”，录入正文，再说“完成语音录入”（或等待超时）；使用较书面的词以降低日常口语误激活，并希望命令可自定义、识别尽可能本地化。[Issue #1003](https://github.com/Open-Less/openless/issues/1003) + +### 2.2 讨论中新增的约束 + +1. **按键替代场景。** 评论说明右 Ctrl/快捷键会干扰肌肉记忆，也有人用 ScrLock 或鼠标侧键折衷。[评论 5436896181](https://github.com/Open-Less/openless/issues/1003#issuecomment-5436896181)、[评论 5441322107](https://github.com/Open-Less/openless/issues/1003#issuecomment-5441322107) +2. **常驻监听的系统成本。** 评论明确提出全天开麦可能耗电、持续占用 CPU，并可能占用 Windows 麦克风导致其他应用难以使用。[评论 5596617615](https://github.com/Open-Less/openless/issues/1003#issuecomment-5596617615) +3. **本地模型引导。** 维护者在 2026-10-14 发布模型缺失引导及 Windows 测试步骤，要求模型未安装时给出跳转按钮，安装后重新配置/回到设置确认进入等待唤醒。[评论 6170844277](https://github.com/Open-Less/openless/issues/1003#issuecomment-6170844277) +4. **结束词的 ASR 断句。** 用户报告“完成，语音录入。”会被混入正文；随后 PR 提交 `37e4665d` 宣称修复该问题并增加回归测试。[评论 6216314465](https://github.com/Open-Less/openless/issues/1003#issuecomment-6216314465)、[提交 37e4665d](https://github.com/HKLHaoBin/openless/commit/37e4665d78a39011431401393264002a4879c197) +5. **VAD/CPU 争议。** 用户建议无人在说话时不要把音频交给 Zipformer；维护者先认为额外人声模型可能增加 CPU，之后在讨论中接受这是应解决的优化，并表示可另行拆分以保持 PR 可维护。[评论 6218184776](https://github.com/Open-Less/openless/issues/1003#issuecomment-6218184776)、[评论 6219721968](https://github.com/Open-Less/openless/issues/1003#issuecomment-6219721968)、[评论 6221936954](https://github.com/Open-Less/openless/issues/1003#issuecomment-6221936954)、[评论 6231986920](https://github.com/Open-Less/openless/issues/1003#issuecomment-6231986920)、[评论 6236232646](https://github.com/Open-Less/openless/issues/1003#issuecomment-6236232646) + +### 2.3 PR 及修复提交序列 + +- `b0196a9e`：加入可配置 voice activation 的主体实现。[提交 b0196a9e](https://github.com/HKLHaoBin/openless/commit/b0196a9ed9bb1358c4d4bb1fb9b7cbf8ca4c66b3) +- `65afa352`：补充模型缺失引导。[提交 65afa352](https://github.com/HKLHaoBin/openless/commit/65afa3521be38cf1c143f3ddff1635fc18fa99d3) +- `37e4665d`：容忍结束词中间插入标点，并清理 transcript。[提交 37e4665d](https://github.com/HKLHaoBin/openless/commit/37e4665d78a39011431401393264002a4879c197) +- `d66c07c4`、`3e56d3a3`：功能分支合入当时的 beta；提交信息显示合并时曾处理多处冲突。[提交 d66c07c4](https://github.com/HKLHaoBin/openless/commit/d66c07c4f4a60d6fb5af4acf14aef3c94b2676cd)、[提交 3e56d3a3](https://github.com/HKLHaoBin/openless/commit/3e56d3a3c214a474cff10df34810c18660c6bf46) +- `9270b43c`：更新 warm capture 测试以适配 preroll 标志，当前 PR head。[提交 9270b43c](https://github.com/HKLHaoBin/openless/commit/9270b43c7e79ad9d462ad8bec78250de5f499093) + +PR #1219 的 body 写明 `Closes #1003`，并列出 Core 测试、格式检查、Bugbot 复审和三个 Actions 运行，但 PR 状态本身仍是 Draft/待 Review。[PR #1219](https://github.com/Open-Less/openless/pull/1219) + +## 3. 当前 beta 与 PR 状态 + +### 3.1 合并状态 + +截至调研日期，GitHub PR 元数据为： + +- base：`Open-Less/openless:beta`，PR 记录的 base SHA 为 `96278f40`； +- head：`HKLHaoBin/openless:feat/issue-1003-voice-activation`，head SHA 为 `9270b43c`； +- `state: OPEN`、`isDraft: true`、`reviewDecision: REVIEW_REQUIRED`、`mergeStateStatus: BLOCKED`； +- `mergeCommit: null`、`mergedAt: null`。 + +这些字段来自 [PR #1219](https://github.com/Open-Less/openless/pull/1219)。因此，功能分支曾把 beta 合入自身，**不能反推 beta 已经合入功能分支**；当前 beta 仍未包含该 PR。当前 beta 的本地 ref 为 `de072a88`，且功能新增文件 `openless-all/app/crates/openless-core/src/voice_activation.rs` 在当前 beta 中不存在。 + +### 3.2 落后/领先关系 + +以本地已存在的两个 ref 做纯 Git 图比较： + +```text +beta = de072a88373a222280a8c22948092a260880196c +feature = 9270b43c7e79ad9d462ad8bec78250de5f499093 +feature ahead of beta = 8 +feature behind beta = 3 +merge-base = 96278f40d48b8ed02e3ce22ddc96b694c863a6bc +``` + +功能分支落后的三个提交分别是： + +- `af8adf7e fix(dictation): keep local startup stoppable and bound inference resources`； +- `5ab6ed43 feat(capsule): add persisted multi-display modes and prepare 2.0.2 Beta.1`； +- `de072a88 docs(release): verified 2.0.2 Beta.1 publication evidence [skip ci]`。 + +PR 的 base/head 和当前分支提交可在 [PR #1219](https://github.com/Open-Less/openless/pull/1219) 与 [功能分支提交列表](https://github.com/HKLHaoBin/openless/commits/feat/issue-1003-voice-activation) 复核。合并前应以当前 beta 重新合并/变基并重新解决冲突，不能直接使用 PR 创建时的 base 作为最新基线。 + +### 3.3 检查状态 + +PR 当前列出的检查均为通过，包括 Android cargo check、Linux Core/egui、Windows checks、macOS checks、macOS Rust 1.88 MSRV、UI motion 和 Changed areas。[PR checks](https://github.com/Open-Less/openless/pull/1219/checks) + +这些是 CI/PR 的已有证据，不是本次调研重新运行的结果；本次没有修改业务代码、没有运行构建，也没有宣称已经在 Windows 实机复现。 + +## 4. 代码影响范围 + +PR 相对其 base 的 GitHub 统计为 **41 个文件、+3227/-201**。[PR #1219 changed files](https://github.com/Open-Less/openless/pull/1219/files) + +| 层次 | 主要影响 | 关键源码证据 | +| --- | --- | --- | +| Core policy/state | 新增 phrase matcher、生命周期/代际、超时、状态信号、端口；`OpenLessBackend` 增加配置、唤醒 handoff、结束和恢复。 | [`voice_activation.rs`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/voice_activation.rs)、[`api.rs:3093-3285`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/api.rs#L3093-L3285)、[`ports.rs`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/ports.rs) | +| 偏好/持久化 | 增加启用、唤醒/结束开关、两条 phrase、超时和麦克风字段，并保留旧配置默认值。 | [`config.rs:395-465`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/config.rs#L395-L465)、[`config.rs:978-1028`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/config.rs#L978-L1028) | +| Windows host/音频 | Tauri 运行时加载 Zipformer、创建在线 Sherpa session、接入 warm capture、处理 mic/permission/model 错误和 generation。 | [`core_adapters.rs:1492-1851`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/core_adapters.rs#L1492-L1851)、[`recorder.rs:332-672`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/recorder.rs#L332-L672) | +| 本地 ASR | Sherpa provider/runtime 支持在线 chunk、token callback 和结束 flush。 | [`sherpa_provider.rs:78-255`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs#L78-L255)、[`sherpa_provider.rs:274-343`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs#L274-L343) | +| UI/IPC/本地化 | Recording Input 增加 Windows-only voice activation 设置、状态和模型缺失跳转；同步 mock/types/i18n。 | [`RecordingInputSection.tsx:146-200`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src/pages/settings/RecordingInputSection.tsx#L146-L200)、[`RecordingInputSection.tsx:590-660`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src/pages/settings/RecordingInputSection.tsx#L590-L660)、[`voiceActivation.ts:1-30`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src/lib/voiceActivation.ts#L1-L30) | +| 测试/构建面 | Core phrase/lifecycle 测试、UI model-missing 测试、host contract、workflow 依赖变更。 | [`voice_activation.rs:390-503`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/voice_activation.rs#L390-L503)、[`voiceActivation.test.ts`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src/lib/voiceActivation.test.ts) | + +## 5. 已实现能力与行为链路 + +### 5.1 持久化与启动 + +`UserPreferences` 将 `voice_activation_enabled` 及相关字段纳入 camelCase wire contract，并给新增字段提供默认值。[`config.rs:395-465`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/config.rs#L395-L465) + +启动链路是： + +```text +App.tsx useEffect + -> getStartupSnapshot() + -> invoke('get_startup_snapshot') + -> commands::get_startup_snapshot() + -> core.start() + -> get_preferences() + -> configure_voice_activation() + -> model check / Sherpa online listener / ListeningForWake +``` + +前端启动握手见 [`App.tsx:84-96`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src/App.tsx#L84-L96) 与 [`shared.ts:45-57`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src/lib/ipc/shared.ts#L45-L57)；命令和 Core 的调用见 [`commands/dictation.rs:3-8`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/commands/dictation.rs#L3-L8)、[`api.rs:4530-4609`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/api.rs#L4530-L4609)。因此，重启后的“自启动”不是偏好字段自身执行，而是应用首个 startup handshake 触发；若模型/权限/麦克风条件不满足，启动会发出失败状态而不会进入监听。 + +设置保存时，启用状态、phrase、超时和麦克风字段的变化会再次调用 `configure_voice_activation()`。[`commands/settings.rs:237-265`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/commands/settings.rs#L237-L265)、[`commands/settings.rs:284-317`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/commands/settings.rs#L284-L317) + +### 5.2 唤醒、交接、超时和恢复 + +Windows runtime 检查固定的 `zipformer-bilingual-zh-en-streaming`，模型存在后创建在线 Sherpa ASR，并把 token callback 接到 `VoiceActivationEmitter`。[`core_adapters.rs:1701-1765`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/core_adapters.rs#L1701-L1765) + +唤醒匹配后，Core 通过 generation 校验、warm-capture handoff 建立普通 dictation session，再保存 `VoiceActivationSession`；结束词或超时调用 `finish_voice_activation`，结束后按启用偏好重新配置监听。[`api.rs:3292-3528`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/api.rs#L3292-L3528)、[`api.rs:3565-3619`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/api.rs#L3565-L3619)。配置注释和停止旧 handle 的单麦克风约束见 [`api.rs:3093-3125`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/api.rs#L3093-L3125)。 + +### 5.3 结束词标点清理 + +实现分两层： + +1. `normalize_phrase` 用 NFKC、去空白和标点处理 ASR phrase；`VoicePhraseMatcher` 只认文本后缀，避免正文中间出现 phrase 就误停。[`voice_activation.rs:49-60`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/voice_activation.rs#L49-L60)、[`voice_activation.rs:201-229`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/voice_activation.rs#L201-L229) +2. `strip_phrase_suffix` 从原始 transcript 尾部反向扫描，跳过空白/中英文终端标点，仅切掉结束词；Core 在每个 `TranscriptDelta` 累积 raw transcript 后，将清理后的文字写回 state 并发布实时事件。[`voice_activation.rs:62-115`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/voice_activation.rs#L62-L115)、[`api.rs:2264-2321`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/api.rs#L2264-L2321) + +结束录音后，完成路径还会再次对 `engine_result.raw_text`、`engine_result.polished_text` 和可选 `engine_result.asr_transcript` 调用同一 suffix cleaner，再构造 `DictationResult`；对应实现见 [`api.rs:7291-7317`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/api.rs#L7291-L7317)。这些清理后的字段随后用于完成事件和历史持久化，历史写入字段见 [`api.rs:7470-7479`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/api.rs#L7470-L7479)、[`api.rs:7665-7752`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/api.rs#L7665-L7752)。该结论仍应通过 Windows 实机验证，因为不同 ASR 的 delta/revision 时序不等于单元测试输入。 + +### 5.4 模型缺失引导 + +runtime 在配置时检查模型是否安装，缺失返回 `ModelMissing`；UI 监听 `voice-activation:state-changed`，只对精确的模型缺失错误显示跳转模型页，避免把权限拒绝或麦克风占用误导成下载问题。[`core_adapters.rs:1718-1734`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/core_adapters.rs#L1718-L1734)、[`RecordingInputSection.tsx:146-200`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src/pages/settings/RecordingInputSection.tsx#L146-L200)、[`voiceActivation.ts:21-30`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src/lib/voiceActivation.ts#L21-L30) + +## 6. 可复现缺口与风险 + +### 6.1 明确缺口：连续监听没有 VAD/CPU gate + +代码路径是无条件转发： + +- `WarmCaptureSink::consume_pcm_chunk` 先写 ring，然后在满足 activation 条件时直接 `activation.consume_pcm_chunk(pcm)`，没有能量阈值、VAD 状态或静音短路。[`recorder.rs:594-645`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/recorder.rs#L594-L645) +- `SherpaOnnxAsr::consume_pcm_chunk` 对 online worker 直接 `send_pcm(pcm)`；worker 收到每个 `Pcm` 后立即调用 `session.accept_pcm_chunk(&pcm, &emit)`。[`sherpa_provider.rs:244-255`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs#L244-L255)、[`sherpa_provider.rs:292-310`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs#L292-L310) +- `send_pcm` 使用标准无界 `mpsc::channel`，只检查空 chunk/取消，不检查音量或丢弃静音。[`sherpa_provider.rs:274-343`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs#L274-L343) + +**复现方法：** Windows 安装并准备 Zipformer，开启语音激活，在安静房间保持 60 秒不说话；记录进程 CPU、Sherpa worker 消费量和 microphone 独占状态，再以相同条件说唤醒词。预期若有 gate，静音段应显著减少 inference；当前静态路径预期仍逐块送入模型。该实验尚未由本次调研运行。 + +### 6.2 模型下载后自动恢复未形成闭环的风险 + +当前明确调用 `configure_voice_activation()` 的位置是 startup、偏好保存、warm fallback 释放和运行时失败恢复；本地模型下载命令本身只调用 local-ASR service 的 `start_download`。[`api.rs:4530-4609`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/api.rs#L4530-L4609)、[`commands/settings.rs:284-317`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/commands/settings.rs#L284-L317)、[`commands/local_asr.rs:329-344`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/commands/local_asr.rs#L329-L344)。因此“下载完成后当前 listener 自动恢复”没有直接源码证据；Issue 的测试说明也要求返回录音设置或重新配置。[评论 6170844277](https://github.com/Open-Less/openless/issues/1003#issuecomment-6170844277) + +这不一定是已确认 bug：下载流程可能由前端刷新或用户切换设置间接触发，需实机确认。若产品要求无刷新自动恢复，应添加下载完成事件到 voice activation reconfigure 的显式连接及回归测试。 + +### 6.3 麦克风占用、权限和模型缺失是启动条件，不是静默降级 + +配置 runtime 会检查模型；arm warm capture 失败时映射为权限拒绝、无输入设备、mic busy 或 runtime failure。[`core_adapters.rs:1498-1509`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/core_adapters.rs#L1498-L1509)、[`core_adapters.rs:1723-1795`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/core_adapters.rs#L1723-L1795)。应验证失败后热键录音仍可正常工作，且其他应用占用麦克风时 UI 不显示错误的“下载模型”建议；现有 UI 的错误分类测试只覆盖状态判别，未覆盖 Windows 设备竞争。[`voiceActivation.test.ts:12-42`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src/lib/voiceActivation.test.ts#L12-L42) + +### 6.4 分支基线漂移与合并风险 + +功能分支已通过两个 merge commit 合入旧 beta，但当前 beta 又前进 3 个提交；且 PR 仍 blocked/review required。合并前必须用当前 beta 重新整合并复跑全部桌面、Android、Core/UI 检查，尤其关注 PR 曾经修改的 `api.rs`、`core_adapters.rs`、`recorder.rs`、settings UI 和 workflow 文件。[PR #1219](https://github.com/Open-Less/openless/pull/1219)、[PR changed files](https://github.com/Open-Less/openless/pull/1219/files) + +## 7. 下一步实现建议 + +1. **先同步当前 beta，再决定 PR 边界。** 以 `de072a88` 为基线整合功能，解决冲突后重新跑 PR 全套检查；不要把“功能分支合入旧 beta”当作“beta 已合并”。 +2. **为静音成本设定可测验收。** 在不改变唤醒召回率的前提下加入轻量、可解释的能量/频域 gate 或 sherpa 自带 VAD（若评估后成本可接受）；记录静音/有人声两类 CPU、延迟、漏唤醒和误唤醒指标。Issue 已有 VAD 建议和后续接受该优化方向的讨论。[评论 6218184776](https://github.com/Open-Less/openless/issues/1003#issuecomment-6218184776)、[评论 6236232646](https://github.com/Open-Less/openless/issues/1003#issuecomment-6236232646) +3. **避免无界积压。** 目前在线 PCM 使用无界 `mpsc::channel`；若推理速度低于采集速度，建议增加有界队列、明确丢帧/降采样策略和故障遥测，再用长时间静音及低性能 Windows 机器验证。[`sherpa_provider.rs:274-343`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs#L274-L343) +4. **闭合模型生命周期。** 模型下载成功后应触发一次明确的 voice activation reconfigure，或在 UI 中明确要求重新配置；覆盖“启用已保存 + 模型缺失 → 下载 → 无需重启即监听”的测试。 +5. **补充真实设备验收。** 覆盖重启自动启动、权限拒绝、麦克风被其他应用占用、模型缺失/下载、热键与语音入口互斥、结束词断句、超时后自动恢复监听。 +6. **把结束词清理保持在 raw/source 层验证。** 继续保留正文自身标点，测试普通逗号、全角标点、空格/换行、ASR revision、结束词出现在正文中间等组合，并核对实时 transcript、最终插入、history raw/asr/final 三套字段。[`voice_activation.rs:394-445`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/voice_activation.rs#L394-L445)、[`api.rs:2264-2321`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/api.rs#L2264-L2321) + +## 8. 验证清单 + +### 8.1 静态/CI + +- [x] `gh issue view 1003`：Issue 状态、原始需求和评论已核对。[Issue #1003](https://github.com/Open-Less/openless/issues/1003) +- [x] `gh pr view 1219`：base/head、Draft、Review、merge 字段已核对。[PR #1219](https://github.com/Open-Less/openless/pull/1219) +- [x] Git 图：当前 beta `de072a88` 与 head `9270b43c` 的 ahead/behind 为 8/3。 +- [x] PR 已有检查记录均为 pass。[PR checks](https://github.com/Open-Less/openless/pull/1219/checks) +- [x] 静态阅读确认启动、持久化、标点清理和每块 PCM 转发路径。 +- [ ] 以当前 beta 重新整合功能分支并重跑 CI（本次调研未执行）。 +- [ ] 运行 `cargo test -p openless-core`、voice activation 定向测试、`cargo fmt --all -- --check`（本次调研未执行；PR body 声称曾通过）。 + +### 8.2 Windows 实机 + +- [ ] 在设置 → 录音输入启用语音激活，确认安装模型后状态为等待唤醒。 +- [ ] 关闭并重启应用，确认已持久化的 `voiceActivationEnabled` 在 startup handshake 后重新进入等待唤醒；分别测试模型存在、模型缺失、权限拒绝、麦克风忙。 +- [ ] 说“开始语音录入”→正文→“完成语音录入”，确认唤醒词/结束词不进入正文。 +- [ ] 重复测试“完成，语音录入。”、“完成语音录入。”、句末全角标点、词间空格/换行；确认正文原有标点保留。 +- [ ] 语音激活开启后保持 60 秒静音，记录 CPU、内存、Sherpa inference 调用/队列量；与有人声的同长度样本比较，确认是否存在 VAD/CPU gate。 +- [ ] 下载模型后不重启，仅按产品预期操作（返回设置或等待事件），确认 listener 是否自动恢复；若不能，记录为模型生命周期缺口。 +- [ ] 连续完成两轮唤醒/正文/结束，验证结束后 listener 自动恢复；超时也应恢复。 +- [ ] 其他应用占用麦克风时，错误应归类为 mic busy/permission，不应显示错误的模型下载按钮；关闭语音激活后原热键入口仍可用。 + +## 9. 证据边界 + +本报告的“已实现”结论来自功能分支 `9270b43c` 的源码和测试；“当前 beta 未合并”来自 GitHub PR 状态、base/head 元数据和本地 Git 图；“VAD/CPU gate 缺失”是对上述 PCM 调用链的静态审计结论，不是 CPU profiler 结果。Issue 评论中的 Windows 测试包和 Actions 链接属于第一方项目记录，但本次没有下载或运行该产物。 diff --git a/openless-all/app/crates/openless-core/src/api.rs b/openless-all/app/crates/openless-core/src/api.rs index 660aa1ec5..670275f71 100644 --- a/openless-all/app/crates/openless-core/src/api.rs +++ b/openless-all/app/crates/openless-core/src/api.rs @@ -39,7 +39,7 @@ use crate::style_packs::StylePack; use crate::types::{ CorrectionRule, DictationPhase, DictationResult, DictationSession, DictationStateSnapshot, DictionaryEntry, HistoryChange, HistoryInsertStatus, HistorySource, PreferencesChange, - SessionId, StylePackChange, VocabPresetStore, VocabularyChange, + SessionId, StylePackChange, TranscriptAccumulator, VocabPresetStore, VocabularyChange, }; use crate::vocabulary::DictionaryStore; use crate::{ActivityStore, CorrectionRuleStore, HistoryStore, PreferencesStore, StylePackStore}; @@ -128,12 +128,18 @@ struct DictationReservation { pending_capture: Option>, } +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +enum PendingAudioCaptureKey { + Hotkey(u64), + VoiceActivation(u64), +} + /// Removes a provisional capture if the async hotkey dispatch is aborted before /// it transfers ownership to a session reservation. struct PendingAudioCaptureAbortGuard { - captures: Arc>>>, + captures: Arc>>>, task_spawner: Arc, - press_id: u64, + key: PendingAudioCaptureKey, } impl Drop for PendingAudioCaptureAbortGuard { @@ -142,7 +148,7 @@ impl Drop for PendingAudioCaptureAbortGuard { .captures .lock() .expect("pending audio capture lock poisoned") - .remove(&self.press_id); + .remove(&self.key); let Some(capture) = capture else { return; }; @@ -2308,6 +2314,7 @@ struct BackendEngineProgress { state: Arc>, phase_changed: Arc, text_insertions: Arc>>, + raw_transcripts: Mutex>, } impl EngineProgressSink for BackendEngineProgress { @@ -2431,7 +2438,7 @@ impl EngineProgressSink for BackendEngineProgress { self.phase_changed.notify_waiters(); } } - EngineProgress::TranscriptDelta(delta) => { + EngineProgress::TranscriptDelta(mut delta) => { let mut state = self.state.write().expect("backend state lock poisoned"); ensure_active_session(&state, session_id)?; if !matches!( @@ -2445,6 +2452,41 @@ impl EngineProgressSink for BackendEngineProgress { "transcript delta arrived after transcription completed", )); } + // Traditional final text was cleaned before correction and polishing. + let activation_phrases = state + .dictation_context + .as_ref() + .filter(|context| { + !delta.is_final + || context.pipeline_mode + == crate::shared_types::PipelineMode::Multimodal + }) + .and_then(|context| context.voice_activation.clone()); + if let Some(commands) = activation_phrases { + let mut raw_transcripts = self + .raw_transcripts + .lock() + .expect("raw transcript lock poisoned"); + let raw_transcript = raw_transcripts.entry(session_id).or_insert_with(|| { + let mut transcript = TranscriptAccumulator::default(); + if let Some(existing) = state.transcripts.get(&session_id) { + let seed = crate::types::TranscriptDelta { + text: existing.text().to_string(), + offset: 0, + is_final: false, + }; + let _ = transcript.apply(&seed); + } + transcript + }); + raw_transcript.apply(&delta)?; + let cleaned = commands.clean(raw_transcript.text()); + delta = crate::types::TranscriptDelta { + text: cleaned, + offset: 0, + is_final: delta.is_final, + }; + } let transcript = { let accumulator = state.transcripts.entry(session_id).or_default(); accumulator.apply(&delta)?; @@ -2539,7 +2581,8 @@ pub struct OpenLessBackend { phase_changed: Arc, hotkey: Mutex, hotkey_dispatch_gate: tokio::sync::Mutex<()>, - pending_audio_captures: Arc>>>, + pending_audio_captures: + Arc>>>, less_computer_hotkey_press_at: Mutex>, vocabulary: Arc, correction_rules: Arc, @@ -2564,6 +2607,27 @@ pub struct OpenLessBackend { voice_sessions: Arc, runtime_start_work: Arc, less_computer_voice_controls: Arc>>>, + voice_activation_handle: Arc>>>, + voice_activation_generation: Arc, + voice_activation_handoff_generation: AtomicU64, + voice_activation_configure_gate: tokio::sync::Mutex<()>, + backend_lifecycle_gate: tokio::sync::Mutex<()>, + backend_shutdown_requested: AtomicBool, + voice_activation_session: Arc>>, +} + +struct VoiceActivationHandoffGuard { + backend: Arc, + generation: u64, +} + +impl Drop for VoiceActivationHandoffGuard { + fn drop(&mut self) { + let _ = self + .backend + .voice_activation_handoff_generation + .compare_exchange(self.generation, 0, Ordering::AcqRel, Ordering::Acquire); + } } #[path = "cloud_sync_e2ee/api.rs"] @@ -2737,6 +2801,91 @@ impl HistoryProviderAttribution { } } +struct CoreVoiceActivationSink { + events: Arc, +} + +impl crate::ports::VoiceActivationSink for CoreVoiceActivationSink { + fn publish( + &self, + signal: crate::voice_activation::VoiceActivationSignal, + ) -> Result<(), BackendError> { + use crate::voice_activation::{VoiceActivationPhase, VoiceActivationSignal}; + let (generation, phase, session_id) = match signal { + VoiceActivationSignal::WakeDetected { + activation_generation, + } => ( + activation_generation, + VoiceActivationPhase::WakeDetected, + None, + ), + VoiceActivationSignal::FinishDetected { + activation_generation, + session_id, + } => ( + activation_generation, + VoiceActivationPhase::Finishing, + Some(session_id), + ), + VoiceActivationSignal::RuntimeStateChanged { + activation_generation, + phase, + } => (activation_generation, phase, None), + }; + self.events.publish( + None, + BackendEventKind::VoiceActivationStateChanged( + crate::voice_activation::VoiceActivationStateSnapshot { + phase, + activation_generation: generation, + session_id, + message: None, + }, + ), + ); + Ok(()) + } +} + +fn map_voice_activation_error(error: crate::ports::VoiceActivationError) -> BackendError { + let (code, message) = match error { + crate::ports::VoiceActivationError::Unsupported => ( + BackendErrorCode::Unsupported, + "voice activation is unsupported".into(), + ), + crate::ports::VoiceActivationError::ModelMissing => ( + BackendErrorCode::InvalidState, + "voice activation model is not installed".into(), + ), + crate::ports::VoiceActivationError::PermissionDenied => ( + BackendErrorCode::PermissionDenied, + "microphone permission was denied".into(), + ), + crate::ports::VoiceActivationError::MicBusy => ( + BackendErrorCode::Busy, + "microphone is already in use".into(), + ), + crate::ports::VoiceActivationError::InvalidConfig(message) + | crate::ports::VoiceActivationError::Runtime(message) => { + (BackendErrorCode::InvalidState, message) + } + }; + BackendError::new(code, message) +} + +fn voice_activation_failure_phase( + code: BackendErrorCode, +) -> crate::voice_activation::VoiceActivationPhase { + match code { + BackendErrorCode::Unsupported => crate::voice_activation::VoiceActivationPhase::Unavailable, + BackendErrorCode::PermissionDenied => { + crate::voice_activation::VoiceActivationPhase::PermissionDenied + } + BackendErrorCode::Busy => crate::voice_activation::VoiceActivationPhase::MicBusy, + _ => crate::voice_activation::VoiceActivationPhase::Failed, + } +} + impl OpenLessBackend { /// Construct an explicitly unavailable Core so the native shell can show /// its startup error instead of panicking before any window is visible. @@ -3207,6 +3356,13 @@ impl OpenLessBackend { voice_sessions, runtime_start_work, less_computer_voice_controls: Arc::new(Mutex::new(HashMap::new())), + voice_activation_handle: Arc::new(Mutex::new(None)), + voice_activation_generation: Arc::new(AtomicU64::new(0)), + voice_activation_handoff_generation: AtomicU64::new(0), + voice_activation_configure_gate: tokio::sync::Mutex::new(()), + backend_lifecycle_gate: tokio::sync::Mutex::new(()), + backend_shutdown_requested: AtomicBool::new(false), + voice_activation_session: Arc::new(Mutex::new(None)), }) } @@ -3224,8 +3380,655 @@ impl OpenLessBackend { BackendErrorCode::Busy, "encrypted sync restore must finish before starting runtime work", ) - })?; - Ok(()) + })?; + Ok(()) + } + + /// Configure the opt-in local voice activation listener from the current + /// preference snapshot. Reconfiguration replaces the previous native + /// handle only after it has been stopped, so one process owns one mic. + pub async fn configure_voice_activation(&self) -> Result<(), BackendError> { + let _configure_gate = self.voice_activation_configure_gate.lock().await; + let preferences = self.get_preferences(); + if self + .voice_activation_session + .lock() + .expect("voice activation session lock poisoned") + .is_some() + && preferences.voice_activation_enabled + { + return Ok(()); + } + // Invalidate callbacks before awaiting the old runtime. Sherpa may have + // queued a final callback while its cancellation is being observed. + let generation = self + .voice_activation_generation + .fetch_add(1, Ordering::AcqRel) + + 1; + self.voice_activation_handoff_generation + .store(0, Ordering::Release); + self.voice_activation_session + .lock() + .expect("voice activation session lock poisoned") + .take(); + let previous = self + .voice_activation_handle + .lock() + .expect("voice activation handle lock poisoned") + .take(); + let had_previous_handle = previous.is_some(); + if let Some(previous) = previous { + let _ = previous + .stop(crate::voice_activation::VoiceActivationStopReason::Disabled) + .await; + } + if !preferences.voice_activation_enabled + || self.backend_shutdown_requested.load(Ordering::Acquire) + || !self.snapshot().running + { + // Do not add a synthetic event to the first startup transition when + // activation was never enabled. Existing subscribers expect + // BackendStarted to be the first startup event; later disable/error + // transitions still publish Disabled below when a visible state exists. + let should_publish_disabled = had_previous_handle + || self + .events + .latest_voice_activation_state() + .map(|state| { + state.phase != crate::voice_activation::VoiceActivationPhase::Disabled + }) + .unwrap_or(false); + if should_publish_disabled { + self.events.publish( + None, + BackendEventKind::VoiceActivationStateChanged( + crate::voice_activation::VoiceActivationStateSnapshot { + phase: crate::voice_activation::VoiceActivationPhase::Disabled, + activation_generation: generation, + session_id: None, + message: None, + }, + ), + ); + } + return Ok(()); + } + let Some(runtime) = self.deps.voice_activation_runtime.as_ref() else { + self.events.publish( + None, + BackendEventKind::VoiceActivationStateChanged( + crate::voice_activation::VoiceActivationStateSnapshot { + phase: crate::voice_activation::VoiceActivationPhase::Unavailable, + activation_generation: generation, + session_id: None, + message: Some("voice activation host runtime is unavailable".into()), + }, + ), + ); + return if preferences.voice_activation_enabled { + Err(BackendError::new( + BackendErrorCode::Unsupported, + "voice activation host runtime is unavailable", + )) + } else { + Ok(()) + }; + }; + + let config = crate::voice_activation::VoiceActivationConfig { + enabled: preferences.voice_activation_enabled, + wake_enabled: preferences.voice_activation_wake_enabled, + finish_enabled: preferences.voice_activation_finish_enabled, + wake_phrase: preferences.voice_activation_wake_phrase, + finish_phrase: preferences.voice_activation_finish_phrase, + timeout_enabled: preferences.voice_activation_timeout_enabled, + timeout_seconds: preferences.voice_activation_timeout_seconds, + microphone_device_name: (!preferences.microphone_device_name.trim().is_empty()) + .then_some(preferences.microphone_device_name), + activation_generation: generation, + } + .normalized() + .map_err(|error| BackendError::new(BackendErrorCode::InvalidArgument, error.to_string()))?; + self.events.publish( + None, + BackendEventKind::VoiceActivationStateChanged( + crate::voice_activation::VoiceActivationStateSnapshot { + phase: crate::voice_activation::VoiceActivationPhase::Starting, + activation_generation: generation, + session_id: None, + message: None, + }, + ), + ); + let sink: Arc = Arc::new(CoreVoiceActivationSink { + events: Arc::clone(&self.events), + }); + match runtime.configure(config, sink).await { + Ok(handle) => { + let stale_handle = { + let mut slot = self + .voice_activation_handle + .lock() + .expect("voice activation handle lock poisoned"); + if self.voice_activation_generation.load(Ordering::Acquire) != generation { + Some(handle) + } else { + *slot = Some(handle.into()); + None + } + }; + if let Some(handle) = stale_handle { + let _ = handle + .stop(crate::voice_activation::VoiceActivationStopReason::Cancelled) + .await; + return Err(BackendError::new( + BackendErrorCode::Cancelled, + "voice activation configuration became stale", + )); + } + self.events.publish( + None, + BackendEventKind::VoiceActivationStateChanged( + crate::voice_activation::VoiceActivationStateSnapshot { + phase: crate::voice_activation::VoiceActivationPhase::ListeningForWake, + activation_generation: generation, + session_id: None, + message: None, + }, + ), + ); + Ok(()) + } + Err(error) => { + let error = map_voice_activation_error(error); + self.events.publish( + None, + BackendEventKind::VoiceActivationStateChanged( + crate::voice_activation::VoiceActivationStateSnapshot { + phase: voice_activation_failure_phase(error.code), + activation_generation: generation, + session_id: None, + message: Some(error.message.clone()), + }, + ), + ); + Err(error) + } + } + } + + pub async fn stop_voice_activation( + &self, + reason: crate::voice_activation::VoiceActivationStopReason, + ) -> Result<(), BackendError> { + let _configure_gate = self.voice_activation_configure_gate.lock().await; + let handle = self + .voice_activation_handle + .lock() + .expect("voice activation handle lock poisoned") + .take(); + self.voice_activation_session + .lock() + .expect("voice activation session lock poisoned") + .take(); + let next_generation = self + .voice_activation_generation + .fetch_add(1, Ordering::AcqRel) + + 1; + self.voice_activation_handoff_generation + .store(0, Ordering::Release); + let should_publish_disabled = handle.is_some() + || self + .events + .latest_voice_activation_state() + .map(|state| state.phase != crate::voice_activation::VoiceActivationPhase::Disabled) + .unwrap_or(false); + if let Some(handle) = handle { + handle + .stop(reason) + .await + .map_err(map_voice_activation_error)?; + } + if should_publish_disabled { + self.events.publish( + None, + BackendEventKind::VoiceActivationStateChanged( + crate::voice_activation::VoiceActivationStateSnapshot { + phase: crate::voice_activation::VoiceActivationPhase::Disabled, + activation_generation: next_generation, + session_id: None, + message: None, + }, + ), + ); + } + Ok(()) + } + + pub async fn stop_voice_activation_if_current( + &self, + generation: u64, + reason: crate::voice_activation::VoiceActivationStopReason, + ) -> Result { + let _configure_gate = self.voice_activation_configure_gate.lock().await; + if self.voice_activation_generation.load(Ordering::Acquire) != generation { + return Ok(false); + } + let handle = self + .voice_activation_handle + .lock() + .expect("voice activation handle lock poisoned") + .take(); + self.voice_activation_session + .lock() + .expect("voice activation session lock poisoned") + .take(); + let next_generation = self + .voice_activation_generation + .fetch_add(1, Ordering::AcqRel) + + 1; + self.voice_activation_handoff_generation + .store(0, Ordering::Release); + let should_publish_disabled = handle.is_some() + || self + .events + .latest_voice_activation_state() + .map(|state| state.phase != crate::voice_activation::VoiceActivationPhase::Disabled) + .unwrap_or(false); + if let Some(handle) = handle { + handle + .stop(reason) + .await + .map_err(map_voice_activation_error)?; + } + if should_publish_disabled { + self.events.publish( + None, + BackendEventKind::VoiceActivationStateChanged( + crate::voice_activation::VoiceActivationStateSnapshot { + phase: crate::voice_activation::VoiceActivationPhase::Disabled, + activation_generation: next_generation, + session_id: None, + message: None, + }, + ), + ); + } + Ok(true) + } + + pub fn voice_activation_generation_is_current(&self, generation: u64) -> bool { + self.voice_activation_generation.load(Ordering::Acquire) == generation + } + + /// Called by the host runtime after it recognizes a semantic command. The + /// host callback is synchronous, so the actual handoff/finalization is + /// owned by Core's task spawner and survives a dropped IPC request. + pub fn handle_voice_activation_signal( + self: &Arc, + signal: crate::voice_activation::VoiceActivationSignal, + ) { + use crate::voice_activation::VoiceActivationSignal; + let generation = self.voice_activation_generation.load(Ordering::Acquire); + let signal_generation = match signal { + VoiceActivationSignal::WakeDetected { + activation_generation, + } + | VoiceActivationSignal::FinishDetected { + activation_generation, + .. + } + | VoiceActivationSignal::RuntimeStateChanged { + activation_generation, + .. + } => activation_generation, + }; + if signal_generation != generation { + return; + } + match signal { + VoiceActivationSignal::WakeDetected { .. } => { + if self.snapshot().dictation.session_id.is_some() + || self + .voice_activation_session + .lock() + .expect("voice activation session lock poisoned") + .is_some() + { + return; + } + let handle = self + .voice_activation_handle + .lock() + .expect("voice activation handle lock poisoned") + .clone(); + let Some(handle) = handle else { + return; + }; + if self + .voice_activation_handoff_generation + .compare_exchange(0, signal_generation, Ordering::AcqRel, Ordering::Acquire) + .is_err() + { + return; + } + let backend = Arc::clone(self); + self.deps.task_spawner.spawn(Box::pin(async move { + let _handoff_guard = VoiceActivationHandoffGuard { + backend: Arc::clone(&backend), + generation: signal_generation, + }; + if backend + .voice_activation_generation + .load(Ordering::Acquire) + != signal_generation + { + return; + } + let handoff = match handle + .handoff_to_dictation( + crate::ports::VoiceDictationHandoffRequest { + activation_generation: signal_generation, + }, + ) + .await + { + Ok(handoff) => handoff, + Err(error) => { + log::warn!("voice activation handoff failed: {error}"); + match backend + .stop_voice_activation_if_current( + signal_generation, + crate::voice_activation::VoiceActivationStopReason::RuntimeFailure, + ) + .await + { + Ok(true) if backend.get_preferences().voice_activation_enabled => { + let _ = backend.configure_voice_activation().await; + } + Ok(_) => {} + Err(error) => { + log::warn!("voice activation recovery stop failed: {error}"); + } + } + return; + } + }; + if backend + .voice_activation_generation + .load(Ordering::Acquire) + != signal_generation + { + let _ = handoff.capture.discard().await; + return; + } + match backend + .start_voice_dictation_with_capture(handoff.capture, signal_generation) + .await + { + Ok(session_id) => { + if backend + .voice_activation_generation + .load(Ordering::Acquire) + != signal_generation + { + let _ = backend.cancel_dictation(Some(session_id)).await; + return; + } + let committed = { + let _configure_gate = + backend.voice_activation_configure_gate.lock().await; + let session_is_current = backend + .state + .read() + .expect("backend state lock poisoned") + .dictation + .session_id + == Some(session_id); + if backend.voice_activation_generation.load(Ordering::Acquire) + != signal_generation + || !session_is_current + { + false + } else { + handle.bind_dictation_session(session_id); + true + } + }; + if !committed { + let _ = backend.cancel_dictation(Some(session_id)).await; + return; + } + backend.events.publish( + None, + BackendEventKind::VoiceActivationStateChanged( + crate::voice_activation::VoiceActivationStateSnapshot { + phase: crate::voice_activation::VoiceActivationPhase::ListeningForFinish, + activation_generation: signal_generation, + session_id: Some(session_id), + message: None, + }, + ), + ); + } + Err(error) => { + log::warn!("voice dictation start failed: {error}"); + match backend + .stop_voice_activation_if_current( + signal_generation, + crate::voice_activation::VoiceActivationStopReason::RuntimeFailure, + ) + .await + { + Ok(true) if backend.get_preferences().voice_activation_enabled => { + let _ = backend.configure_voice_activation().await; + } + Ok(_) => {} + Err(error) => { + log::warn!("voice activation recovery stop failed: {error}"); + } + } + } + } + })); + } + VoiceActivationSignal::FinishDetected { + session_id, + activation_generation, + } => { + let backend = Arc::clone(self); + self.deps.task_spawner.spawn(Box::pin(async move { + backend + .finish_voice_activation( + activation_generation, + session_id, + crate::voice_activation::VoiceActivationStopReason::FinishPhrase, + ) + .await; + })); + } + VoiceActivationSignal::RuntimeStateChanged { + activation_generation, + phase: crate::voice_activation::VoiceActivationPhase::Failed, + } => { + let backend = Arc::clone(self); + self.deps.task_spawner.spawn(Box::pin(async move { + match backend + .stop_voice_activation_if_current( + activation_generation, + crate::voice_activation::VoiceActivationStopReason::RuntimeFailure, + ) + .await + { + Ok(true) if backend.get_preferences().voice_activation_enabled => { + let _ = backend.configure_voice_activation().await; + } + Ok(_) => {} + Err(error) => { + log::warn!("voice activation recovery stop failed: {error}"); + } + } + })); + } + VoiceActivationSignal::RuntimeStateChanged { .. } => {} + } + } + + async fn start_voice_dictation_with_capture( + &self, + capture: Box, + generation: u64, + ) -> Result { + if self.voice_activation_generation.load(Ordering::Acquire) != generation { + let _ = capture.discard().await; + return Err(BackendError::new( + BackendErrorCode::Cancelled, + "voice activation generation is stale", + )); + } + let preferences = self.get_preferences(); + let activation_config = match (crate::voice_activation::VoiceActivationConfig { + wake_phrase: preferences.voice_activation_wake_phrase.clone(), + finish_enabled: preferences.voice_activation_finish_enabled, + finish_phrase: preferences.voice_activation_finish_phrase.clone(), + timeout_enabled: preferences.voice_activation_timeout_enabled, + timeout_seconds: preferences.voice_activation_timeout_seconds, + ..crate::voice_activation::VoiceActivationConfig::default() + }) + .normalized() + { + Ok(config) => config, + Err(error) => { + let _ = capture.discard().await; + return Err(BackendError::new( + BackendErrorCode::InvalidArgument, + error.to_string(), + )); + } + }; + self.pending_audio_captures + .lock() + .expect("pending audio capture lock poisoned") + .insert(PendingAudioCaptureKey::VoiceActivation(generation), capture); + let mut reservation = match self.reserve_dictation_session( + true, + DictationOutputTarget::ForegroundApp, + Some(PendingAudioCaptureKey::VoiceActivation(generation)), + ) { + Ok(reservation) => reservation, + Err(error) => { + self.discard_pending_audio_capture(PendingAudioCaptureKey::VoiceActivation( + generation, + )) + .await?; + return Err(error); + } + }; + let session_id = reservation.session_id; + { + let _configure_gate = self.voice_activation_configure_gate.lock().await; + if self.voice_activation_generation.load(Ordering::Acquire) != generation { + if let Some(capture) = reservation.pending_capture.take() { + if let Err(error) = capture.discard().await { + log::warn!("discard stale voice capture failed: {error}"); + } + } + self.reset_dictation_session(session_id); + return Err(BackendError::new( + BackendErrorCode::Cancelled, + "voice activation generation became stale while starting dictation", + )); + } + *self + .voice_activation_session + .lock() + .expect("voice activation session lock poisoned") = Some( + crate::voice_activation::VoiceActivationSession { + generation: crate::voice_activation::VoiceActivationGeneration(generation), + session_id, + timeout: activation_config.timeout(), + commands: crate::voice_activation::VoiceActivationTextPolicy { + wake_phrase: activation_config.wake_phrase.clone(), + finish_phrase: activation_config + .finish_enabled + .then(|| activation_config.finish_phrase.clone()), + }, + }); + } + match self + .start_reserved_dictation(reservation, DictationStartOptions::default()) + .await + { + Ok(session_id) => Ok(session_id), + Err(error) => { + let mut session = self + .voice_activation_session + .lock() + .expect("voice activation session lock poisoned"); + if session.as_ref().is_some_and(|session| { + session.generation.0 == generation && session.session_id == session_id + }) { + session.take(); + } + Err(error) + } + } + } + + async fn cleanup_voice_activation_after_dictation(&self, session_id: SessionId) { + let is_voice_session = self + .voice_activation_session + .lock() + .expect("voice activation session lock poisoned") + .as_ref() + .is_some_and(|session| session.session_id == session_id); + if !is_voice_session { + return; + } + let enabled = self.get_preferences().voice_activation_enabled; + let _ = self + .stop_voice_activation(crate::voice_activation::VoiceActivationStopReason::Cancelled) + .await; + if enabled { + let _ = self.configure_voice_activation().await; + } + } + + async fn finish_voice_activation( + &self, + generation: u64, + session_id: SessionId, + reason: crate::voice_activation::VoiceActivationStopReason, + ) { + let accepted = self + .voice_activation_session + .lock() + .expect("voice activation session lock poisoned") + .as_ref() + .is_some_and(|session| { + session.accepts( + crate::voice_activation::VoiceActivationGeneration(generation), + session_id, + ) + }); + if !accepted { + return; + } + if self.stop_dictation_session(session_id).await.is_err() { + return; + } + let still_voice_session = self + .voice_activation_session + .lock() + .expect("voice activation session lock poisoned") + .as_ref() + .is_some_and(|session| session.session_id == session_id); + if still_voice_session { + let _ = self.stop_voice_activation(reason).await; + if self.get_preferences().voice_activation_enabled { + let _ = self.configure_voice_activation().await; + } + } } pub fn repositories(&self) -> BackendRepositories { @@ -4132,10 +4935,13 @@ impl OpenLessBackend { state: Arc::clone(&self.state), phase_changed: Arc::clone(&self.phase_changed), text_insertions: Arc::clone(&self.text_insertions), + raw_transcripts: Mutex::new(HashMap::new()), }) } pub async fn start(&self) -> Result { + let _lifecycle_gate = self.backend_lifecycle_gate.lock().await; + self.backend_shutdown_requested.store(false, Ordering::Release); if let Some(error) = &self.startup_error { return Err(error.clone()); } @@ -4183,10 +4989,26 @@ impl OpenLessBackend { } } }; - let mut state = self.state.write().expect("backend state lock poisoned"); - state.credentials = credentials; - if state.running { - return Ok(StartupSnapshot { + let snapshot = { + let mut state = self.state.write().expect("backend state lock poisoned"); + state.credentials = credentials; + if state.running { + return Ok(StartupSnapshot { + contract_version: crate::BACKEND_CONTRACT_VERSION.to_string(), + backend: BackendSnapshot { + running: true, + dictation: state.dictation.clone(), + vocabulary_revision: self.vocabulary_revision.load(Ordering::Acquire), + history_revision: self.history_revision.load(Ordering::Acquire), + style_pack_revision: self.style_pack_revision.load(Ordering::Acquire), + preferences_revision: self.preferences_revision.load(Ordering::Acquire), + credentials: state.credentials.clone(), + }, + }); + } + state.running = true; + self.events.publish(None, BackendEventKind::BackendStarted); + StartupSnapshot { contract_version: crate::BACKEND_CONTRACT_VERSION.to_string(), backend: BackendSnapshot { running: true, @@ -4197,25 +5019,20 @@ impl OpenLessBackend { preferences_revision: self.preferences_revision.load(Ordering::Acquire), credentials: state.credentials.clone(), }, - }); + } + }; + if let Err(error) = self.configure_voice_activation().await { + log::warn!("[voice-activation] startup configuration failed: {error}"); } - state.running = true; - self.events.publish(None, BackendEventKind::BackendStarted); - Ok(StartupSnapshot { - contract_version: crate::BACKEND_CONTRACT_VERSION.to_string(), - backend: BackendSnapshot { - running: true, - dictation: state.dictation.clone(), - vocabulary_revision: self.vocabulary_revision.load(Ordering::Acquire), - history_revision: self.history_revision.load(Ordering::Acquire), - style_pack_revision: self.style_pack_revision.load(Ordering::Acquire), - preferences_revision: self.preferences_revision.load(Ordering::Acquire), - credentials: state.credentials.clone(), - }, - }) + Ok(snapshot) } pub async fn shutdown(&self) -> Result<(), BackendError> { + let _lifecycle_gate = self.backend_lifecycle_gate.lock().await; + self.backend_shutdown_requested.store(true, Ordering::Release); + let _ = self + .stop_voice_activation(crate::voice_activation::VoiceActivationStopReason::Disabled) + .await; if let Some(sync) = &self.encrypted_sync { sync.shutdown().await; } @@ -4385,6 +5202,12 @@ impl OpenLessBackend { } } + pub fn latest_voice_activation_state( + &self, + ) -> Option { + self.events.latest_voice_activation_state() + } + pub fn dictation_output_target(&self) -> Option { let state = self.state.read().expect("backend state lock poisoned"); state @@ -4473,7 +5296,7 @@ impl OpenLessBackend { .pending_audio_captures .lock() .expect("pending audio capture lock poisoned") - .insert(press_id, capture); + .insert(PendingAudioCaptureKey::Hotkey(press_id), capture); if let Some(replaced) = replaced { self.deps.task_spawner.spawn(Box::pin(async move { if let Err(error) = replaced.discard().await { @@ -4491,15 +5314,21 @@ impl OpenLessBackend { } } - fn take_pending_audio_capture(&self, press_id: u64) -> Option> { + fn take_pending_audio_capture( + &self, + key: PendingAudioCaptureKey, + ) -> Option> { self.pending_audio_captures .lock() .expect("pending audio capture lock poisoned") - .remove(&press_id) + .remove(&key) } - async fn discard_pending_audio_capture(&self, press_id: u64) -> Result<(), BackendError> { - let capture = self.take_pending_audio_capture(press_id); + async fn discard_pending_audio_capture( + &self, + key: PendingAudioCaptureKey, + ) -> Result<(), BackendError> { + let capture = self.take_pending_audio_capture(key); match capture { Some(capture) => capture.discard().await, None => Ok(()), @@ -4614,7 +5443,7 @@ impl OpenLessBackend { Some(PendingAudioCaptureAbortGuard { captures: Arc::clone(&self.pending_audio_captures), task_spawner: Arc::clone(&self.deps.task_spawner), - press_id, + key: PendingAudioCaptureKey::Hotkey(press_id), }) } else { None @@ -4623,7 +5452,9 @@ impl OpenLessBackend { None }; if let DictationHotkeyEdge::Combined { press_id, .. } = edge { - let _ = self.discard_pending_audio_capture(press_id).await; + let _ = self + .discard_pending_audio_capture(PendingAudioCaptureKey::Hotkey(press_id)) + .await; } let (intent, reservation) = if let HotkeyIntent::WaitForModifierGrace { press_id } = intent { @@ -4643,7 +5474,7 @@ impl OpenLessBackend { Some(self.reserve_dictation_session( options.start.insert_text, options.start.output_target, - Some(press_id), + Some(PendingAudioCaptureKey::Hotkey(press_id)), )) } else { None @@ -4651,7 +5482,9 @@ impl OpenLessBackend { (intent, reservation) }; if !matches!(intent, HotkeyIntent::Start { .. }) { - let _ = self.discard_pending_audio_capture(press_id).await; + let _ = self + .discard_pending_audio_capture(PendingAudioCaptureKey::Hotkey(press_id)) + .await; } (intent, reservation) } else { @@ -4666,14 +5499,16 @@ impl OpenLessBackend { self.reserve_dictation_session( options.start.insert_text, options.start.output_target, - Some(press_id), + Some(PendingAudioCaptureKey::Hotkey(press_id)), ) }) }; if reservation.is_some() { (intent, reservation) } else { - let _ = self.discard_pending_audio_capture(press_id).await; + let _ = self + .discard_pending_audio_capture(PendingAudioCaptureKey::Hotkey(press_id)) + .await; self.hotkey .lock() .expect("hotkey interpreter lock poisoned") @@ -4696,7 +5531,9 @@ impl OpenLessBackend { .await } Err(error) => { - let _ = self.discard_pending_audio_capture(press_id).await; + let _ = self + .discard_pending_audio_capture(PendingAudioCaptureKey::Hotkey(press_id)) + .await; Err(error) } }; @@ -5670,6 +6507,25 @@ impl OpenLessBackend { } }; + let voice_activation_config = crate::voice_activation::VoiceActivationConfig { + enabled: preferences.voice_activation_enabled, + wake_enabled: preferences.voice_activation_wake_enabled, + finish_enabled: preferences.voice_activation_finish_enabled, + wake_phrase: preferences.voice_activation_wake_phrase.clone(), + finish_phrase: preferences.voice_activation_finish_phrase.clone(), + timeout_enabled: preferences.voice_activation_timeout_enabled, + timeout_seconds: preferences.voice_activation_timeout_seconds, + microphone_device_name: (!preferences.microphone_device_name.trim().is_empty()) + .then(|| preferences.microphone_device_name.clone()), + activation_generation: 0, + }; + voice_activation_config.normalized().map_err(|error| { + BackendError::new( + BackendErrorCode::InvalidArgument, + format!("invalid voice activation settings: {error}"), + ) + })?; + let effects = crate::SettingsEffectPlan::between(&previous, &preferences); let mut receipt = match runtime.prepare(&effects) { Ok(receipt) => receipt, @@ -6515,7 +7371,7 @@ impl OpenLessBackend { &self, insert_text: bool, output_target: DictationOutputTarget, - capture_generation: Option, + capture_key: Option, ) -> Result { { let state = self.state.read().expect("backend state lock poisoned"); @@ -6573,11 +7429,14 @@ impl OpenLessBackend { ); self.phase_changed.notify_waiters(); } - let pending_capture = - capture_generation.and_then(|generation| self.take_pending_audio_capture(generation)); + let capture_generation = capture_key.map(|key| match key { + PendingAudioCaptureKey::Hotkey(generation) + | PendingAudioCaptureKey::VoiceActivation(generation) => generation, + }); + let pending_capture = capture_key.and_then(|key| self.take_pending_audio_capture(key)); let capture_generation = pending_capture .as_ref() - .map(|_| capture_generation.expect("capture generation was checked")); + .map(|_| capture_generation.expect("capture key was checked")); Ok(DictationReservation { session_id, resources: starting_resources, @@ -6633,6 +7492,29 @@ impl OpenLessBackend { return Err(error); } }; + let activation = self + .voice_activation_session + .lock() + .expect("voice activation session lock poisoned") + .as_ref() + .filter(|session| session.session_id == session_id) + .cloned(); + let context = if let Some(activation) = activation { + let mut context = (*context).clone(); + context.voice_activation = Some(activation.commands); + if let Some(timeout) = activation.timeout { + let timeout_ms = timeout.as_millis() as u64; + context.recording.silence_after_ms = Some( + context + .recording + .silence_after_ms + .map_or(timeout_ms, |general| general.min(timeout_ms)), + ); + } + Arc::new(context) + } else { + context + }; let context = { let mut state = self.state.write().expect("backend state lock poisoned"); if let Err(error) = ensure_running(&state) { @@ -6659,10 +7541,17 @@ impl OpenLessBackend { SilenceMonitor { session_id, started_at, - detector: crate::silence_auto_stop::SilenceAutoStop::new( - std::time::Duration::from_millis(silence_ms), - started_at, - ), + detector: if context.voice_activation.is_some() { + crate::silence_auto_stop::SilenceAutoStop::after_speech( + std::time::Duration::from_millis(silence_ms), + started_at, + ) + } else { + crate::silence_auto_stop::SilenceAutoStop::new( + std::time::Duration::from_millis(silence_ms), + started_at, + ) + }, } }); let context = match state.dictation_start_output_target { @@ -7030,6 +7919,8 @@ impl OpenLessBackend { let _ = self.cancel_session_adapters(session_id).await; let _ = self.hide_dictation_feedback(session_id); self.reset_dictation_session(session_id); + self.cleanup_voice_activation_after_dictation(session_id) + .await; return Err(error); } } @@ -7097,10 +7988,22 @@ impl OpenLessBackend { let _ = self.cancel_session_adapters(session_id).await; let _ = self.hide_dictation_feedback(session_id); self.reset_dictation_session(session_id); + self.cleanup_voice_activation_after_dictation(session_id) + .await; return Err(error); } }; + if context.pipeline_mode == crate::shared_types::PipelineMode::Multimodal { + if let Some(commands) = context.voice_activation.as_ref() { + engine_result.raw_text = commands.clean(&engine_result.raw_text); + engine_result.polished_text = commands.clean(&engine_result.polished_text); + if let Some(text) = engine_result.asr_transcript.as_mut() { + *text = commands.clean(text); + } + } + } + if context.output_target == DictationOutputTarget::CloudNote && engine_result.polish_failed { let error = @@ -7108,6 +8011,8 @@ impl OpenLessBackend { self.mark_dictation_failed(session_id, &error); let _ = self.hide_dictation_feedback(session_id); self.reset_dictation_session(session_id); + self.cleanup_voice_activation_after_dictation(session_id) + .await; self.cleanup_cloud_note(session_id, true)?; return Err(error); } @@ -7136,6 +8041,8 @@ impl OpenLessBackend { let _ = self.cancel_text_insertion(session_id).await; let _ = self.hide_dictation_feedback(session_id); self.reset_dictation_session(session_id); + self.cleanup_voice_activation_after_dictation(session_id) + .await; return Err(error); } @@ -7167,19 +8074,21 @@ impl OpenLessBackend { // Cancellation may happen while ASR/LLM work is in flight. Never // insert a result after the session has been cancelled or replaced. - { + let cancelled_before_insertion = { let state = self.state.read().expect("backend state lock poisoned"); - if state.dictation.session_id != Some(session_id) + state.dictation.session_id != Some(session_id) || !matches!( state.dictation.phase, DictationPhase::Transcribing | DictationPhase::Polishing ) - { - return Err(BackendError::new( - BackendErrorCode::Cancelled, - "dictation session was cancelled before insertion", - )); - } + }; + if cancelled_before_insertion { + self.cleanup_voice_activation_after_dictation(session_id) + .await; + return Err(BackendError::new( + BackendErrorCode::Cancelled, + "dictation session was cancelled before insertion", + )); } { @@ -7233,6 +8142,8 @@ impl OpenLessBackend { let _ = self.cancel_text_insertion(session_id).await; let _ = self.hide_dictation_feedback(session_id); self.reset_dictation_session(session_id); + self.cleanup_voice_activation_after_dictation(session_id) + .await; return Err(error); } } @@ -7257,46 +8168,52 @@ impl OpenLessBackend { .unwrap_or(crate::types::InsertStatus::NotRequested), }; - let mut state = self.state.write().expect("backend state lock poisoned"); - if state.dictation.session_id != Some(session_id) - || state.dictation.phase - != if context.insertion.enabled { - DictationPhase::Inserting - } else { - DictationPhase::Completed + let session_replaced = { + let mut state = self.state.write().expect("backend state lock poisoned"); + let replaced = state.dictation.session_id != Some(session_id) + || state.dictation.phase + != if context.insertion.enabled { + DictationPhase::Inserting + } else { + DictationPhase::Completed + }; + if !replaced { + state.dictation.phase = DictationPhase::Completed; + state.dictation.message = Some(match insert_outcome { + Some(InsertOutcome::Inserted) => "inserted".to_string(), + Some(InsertOutcome::PasteSent) => "paste_sent".to_string(), + Some(InsertOutcome::CopiedFallback) => "copied_fallback".to_string(), + None => "insertion_not_requested".to_string(), + }); + if matches!(insert_outcome, Some(InsertOutcome::CopiedFallback)) { + self.events.publish( + Some(session_id), + BackendEventKind::InsertFallback(crate::types::InsertFallbackPayload { + reason: "clipboard_fallback".to_string(), + copied_text: Some(engine_result.polished_text.clone()), + }), + ); } - { + self.events.publish( + Some(session_id), + BackendEventKind::DictationCompleted(result.clone()), + ); + self.events.publish( + Some(session_id), + BackendEventKind::DictationStateChanged(state.dictation.clone()), + ); + self.phase_changed.notify_waiters(); + } + replaced + }; + if session_replaced { + self.cleanup_voice_activation_after_dictation(session_id) + .await; return Err(BackendError::new( BackendErrorCode::Cancelled, "dictation session was replaced before completion", )); } - state.dictation.phase = DictationPhase::Completed; - state.dictation.message = Some(match insert_outcome { - Some(InsertOutcome::Inserted) => "inserted".to_string(), - Some(InsertOutcome::PasteSent) => "paste_sent".to_string(), - Some(InsertOutcome::CopiedFallback) => "copied_fallback".to_string(), - None => "insertion_not_requested".to_string(), - }); - if matches!(insert_outcome, Some(InsertOutcome::CopiedFallback)) { - self.events.publish( - Some(session_id), - BackendEventKind::InsertFallback(crate::types::InsertFallbackPayload { - reason: "clipboard_fallback".to_string(), - copied_text: Some(engine_result.polished_text.clone()), - }), - ); - } - self.events.publish( - Some(session_id), - BackendEventKind::DictationCompleted(result.clone()), - ); - self.events.publish( - Some(session_id), - BackendEventKind::DictationStateChanged(state.dictation.clone()), - ); - self.phase_changed.notify_waiters(); - drop(state); self.arm_edit_observation( session_id, context.insertion.observe_edits, @@ -7308,6 +8225,8 @@ impl OpenLessBackend { // delayed successful completion must not clear its successor's state, // transcript, silence detector or physical-hotkey generation. self.reset_dictation_session(session_id); + self.cleanup_voice_activation_after_dictation(session_id) + .await; self.persist_completed_dictation(&context, &result, insert_outcome, &engine_result); self.cleanup_cloud_note(session_id, true)?; host_result?; @@ -7872,6 +8791,8 @@ impl OpenLessBackend { let cancel_result = self.cancel_session_adapters(session_id).await; let _ = self.hide_dictation_feedback(session_id); self.reset_dictation_session(session_id); + self.cleanup_voice_activation_after_dictation(session_id) + .await; cancel_result } @@ -7932,6 +8853,7 @@ impl OpenLessBackend { Ok(()) }; let cancel_result = self.cancel_session_adapters(active).await; + self.cleanup_voice_activation_after_dictation(active).await; // The state can already display cancellation, but native audio/input // cleanup still owns the shared resource. Reject new capture until that // cleanup finishes, including on its error path. @@ -8202,6 +9124,60 @@ mod tests { fn assert_send_sync() {} + #[tokio::test] + async fn voice_activation_cancelled_handoff_releases_reserved_session() { + struct Capture(Arc, bool); + impl PendingAudioCapture for Capture { + fn commit( + self: Box, + _session_id: SessionId, + _context: Arc, + _consumer: Arc, + _progress: Arc, + ) -> BoxFuture<'static, Result, BackendError>> + { + panic!("stale handoff must not commit"); + } + fn discard(self: Box) -> BoxFuture<'static, Result<(), BackendError>> { + Box::pin(async move { + self.0.fetch_add(1, Ordering::SeqCst); + if self.1 { + Err(BackendError::new( + BackendErrorCode::Platform, + "discard failed", + )) + } else { + Ok(()) + } + }) + } + } + for discard_fails in [false, true] { + let (backend, _) = backend(); + backend.start().await.unwrap(); + let generation = backend.voice_activation_generation.load(Ordering::Acquire); + let discarded = Arc::new(AtomicU64::new(0)); + let gate = backend.voice_activation_configure_gate.lock().await; + let mut start = Box::pin(backend.start_voice_dictation_with_capture( + Box::new(Capture(Arc::clone(&discarded), discard_fails)), + generation, + )); + assert!(futures_util::poll!(&mut start).is_pending()); + assert_eq!(backend.snapshot().dictation.phase, DictationPhase::Starting); + backend + .voice_activation_generation + .fetch_add(1, Ordering::AcqRel); + drop(gate); + assert_eq!(start.await.unwrap_err().code, BackendErrorCode::Cancelled); + assert!(backend.snapshot().dictation.session_id.is_none()); + assert_eq!(discarded.load(Ordering::SeqCst), 1); + let next = backend + .reserve_dictation_session(false, DictationOutputTarget::ForegroundApp, None) + .unwrap(); + backend.reset_dictation_session(next.session_id); + } + } + #[test] fn vocabulary_learning_settings_migrate_normalize_and_roundtrip() { let old: UserPreferences = serde_json::from_str("{}").unwrap(); @@ -8726,6 +9702,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -9428,6 +10405,7 @@ mod tests { let runtime = Arc::new(LessComputerCaptureRuntime::default()); let host = Arc::new(crate::testing::RecordingHostActions::default()); let dependencies = BackendDependencies { + voice_activation_runtime: None, host_actions: host.clone(), dictation_engine: Arc::new(VoiceOnlyEngine(Arc::clone(&transcription))), ..BackendDependencies::unsupported() @@ -9589,6 +10567,7 @@ mod tests { }); let runtime = Arc::new(LessComputerCaptureRuntime::default()); let dependencies = BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(crate::testing::RecordingHostActions::default()), dictation_engine: Arc::new(DictationOnlyEngine(Arc::clone(&transcription))), ..BackendDependencies::unsupported() @@ -9884,6 +10863,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, dictation_engine: Arc::new(engine), ..BackendDependencies::unsupported() }, @@ -9957,6 +10937,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, dictation_engine: Arc::new(engine), ..BackendDependencies::unsupported() }, @@ -10431,6 +11412,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, dictation_engine: Arc::new(engine), ..BackendDependencies::unsupported() }, @@ -10528,6 +11510,7 @@ mod tests { credential_store: llm_configured_credentials(), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -10688,6 +11671,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -10814,6 +11798,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -10947,6 +11932,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -11097,6 +12083,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -11502,6 +12489,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -11650,6 +12638,7 @@ mod tests { credential_store: credential_store.clone(), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -11719,6 +12708,7 @@ mod tests { credential_store, services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -11828,6 +12818,7 @@ mod tests { credential_store: Arc::new(InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -12008,6 +12999,7 @@ mod tests { credential_store: Arc::new(PersistenceOnlyCredentialStore), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -12037,6 +13029,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, services, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(FakeInserter), @@ -12215,6 +13208,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(FakeInserter), dictation_engine: Arc::new(FakeEngine), @@ -12346,6 +13340,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: host.clone(), text_inserter: Arc::new(FakeInserter), dictation_engine: Arc::new(FakeEngine), @@ -12454,6 +13449,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -12844,6 +13840,265 @@ mod tests { backend.shutdown().await.unwrap(); } + async fn start_voice_test_session( + backend: &OpenLessBackend, + timeout: Option, + insert_text: bool, + ) -> SessionId { + let reservation = backend + .reserve_dictation_session(insert_text, DictationOutputTarget::ForegroundApp, None) + .unwrap(); + let id = reservation.session_id; + *backend.voice_activation_session.lock().unwrap() = + Some(crate::voice_activation::VoiceActivationSession { + generation: crate::voice_activation::VoiceActivationGeneration( + backend.voice_activation_generation.load(Ordering::Acquire), + ), + session_id: id, + timeout, + commands: crate::voice_activation::VoiceActivationTextPolicy { + wake_phrase: "开始语音录入".into(), + finish_phrase: Some("完成语音录入".into()), + }, + }); + backend + .start_reserved_dictation( + reservation, + DictationStartOptions { + insert_text, + ..Default::default() + }, + ) + .await + .unwrap() + } + + #[tokio::test] + async fn voice_activation_silence_uses_earliest_enabled_timeout_and_rejects_old_session() { + for (voice_timeout, general_timeout, expected) in [ + (Some(15), None, Some(15)), + (None, None, None), + (Some(15), Some(3), Some(3)), + (None, Some(3), Some(3)), + ] { + let (backend, _) = backend(); + let mut preferences = backend.get_preferences(); + preferences.silence_auto_stop_enabled = general_timeout.is_some(); + preferences.silence_auto_stop_seconds = general_timeout.unwrap_or(3) as f32; + preferences.hotkey.mode = crate::shared_types::HotkeyMode::Toggle; + backend.set_preferences(preferences).unwrap(); + backend.start().await.unwrap(); + let id = start_voice_test_session( + &backend, + voice_timeout.map(std::time::Duration::from_secs), + false, + ) + .await; + let context = backend + .state + .read() + .unwrap() + .dictation_context + .clone() + .unwrap(); + assert_eq!( + context.recording.silence_after_ms, + expected.map(|seconds| seconds * 1000) + ); + let commands = context.voice_activation.clone(); + assert!(commands.is_some()); + assert_eq!( + context.with_translation_requested(true).voice_activation, + commands + ); + assert_eq!( + context + .with_output_target(DictationOutputTarget::QuickNote) + .voice_activation, + commands + ); + let progress = backend.engine_progress_sink(); + for elapsed_ms in (0..=30_000).step_by(100) { + progress + .publish( + id, + EngineProgress::RecordingLevel { + elapsed_ms, + level: 0.1, + }, + ) + .unwrap(); + } + let count_stops = || { + backend.replay_events_after(0).events.iter().filter(|event| + matches!(&event.kind, BackendEventKind::RecordingControlRequested(request) if request.session_id == id) + ).count() + }; + assert_eq!(count_stops(), 0); + let deadline = 30_000 + expected.unwrap_or(300) * 1000; + progress + .publish( + id, + EngineProgress::RecordingLevel { + elapsed_ms: deadline - 1, + level: 0.0, + }, + ) + .unwrap(); + assert_eq!(count_stops(), 0); + progress + .publish( + id, + EngineProgress::RecordingLevel { + elapsed_ms: deadline, + level: 0.0, + }, + ) + .unwrap(); + assert_eq!(count_stops(), usize::from(expected.is_some())); + progress + .publish( + id, + EngineProgress::RecordingLevel { + elapsed_ms: deadline + 100, + level: 0.0, + }, + ) + .unwrap(); + assert_eq!(count_stops(), usize::from(expected.is_some())); + let generation = backend.voice_activation_generation.load(Ordering::Acquire); + backend.cancel_dictation(Some(id)).await.unwrap(); + let next = backend.start_dictation().await.unwrap(); + assert!(progress + .publish( + id, + EngineProgress::RecordingLevel { + elapsed_ms: deadline + 200, + level: 0.0 + } + ) + .is_err()); + backend + .finish_voice_activation( + generation, + id, + crate::voice_activation::VoiceActivationStopReason::Timeout, + ) + .await; + assert_eq!(backend.snapshot().dictation.session_id, Some(next)); + backend.cancel_dictation(Some(next)).await.unwrap(); + } + } + + #[tokio::test] + async fn voice_activation_persists_clean_body_and_never_inserts_command_only_audio() { + use crate::testing::{ + FixtureAudioRecorder, FixtureInsertionAction, FixtureTextInserter, FixtureTextPolisher, + FixtureTranscriptionEngine, + }; + for body in ["正文", "开始语音录入正文", ""] { + let dir = TestDataDir::new("voice-command-history"); + let polisher = Arc::new(FixtureTextPolisher::successful("完整正文")); + let inserter = Arc::new(FixtureTextInserter::with_outcome(InsertOutcome::Inserted)); + let engine = crate::PipelineDictationEngine::new( + Arc::new(FixtureAudioRecorder::new(vec![vec![1, 0, 2, 0]], vec![])), + Arc::new(FixtureTranscriptionEngine::successful( + format!("开始语音录入 {body} 完成语音录入"), + 1000, + )), + polisher.clone(), + ); + let backend = OpenLessBackend::new( + BackendConfig { + data_dir: dir.path().to_path_buf(), + ..Default::default() + }, + BackendDependencies { + host_actions: Arc::new(FakeHost::default()), + text_inserter: inserter.clone(), + dictation_engine: Arc::new(engine), + credential_store: llm_configured_credentials(), + ..BackendDependencies::unsupported() + }, + ) + .unwrap(); + let mut prefs = backend.get_preferences(); + prefs.streaming_insert = false; + backend.set_preferences(prefs).unwrap(); + backend.start().await.unwrap(); + let id = start_voice_test_session(&backend, None, true).await; + let result = backend.stop_dictation_session(id).await; + let history = backend.list_history().unwrap(); + assert_eq!(history.len(), 1); + assert_eq!(history[0].raw_transcript, body); + let final_transcript = + backend + .replay_events_after(0) + .events + .into_iter() + .find_map(|event| match event.kind { + BackendEventKind::TranscriptDelta(delta) if delta.is_final => { + Some(delta.text) + } + _ => None, + }); + assert_eq!(final_transcript.as_deref(), Some(body)); + if body.is_empty() { + assert!(result.is_err()); + assert!(history[0].final_text.is_empty()); + assert!(polisher.inputs().is_empty()); + assert!(!inserter.actions().iter().any(|action| matches!( + action, + FixtureInsertionAction::Insert { .. } | FixtureInsertionAction::Write { .. } + ))); + } else { + assert_eq!(result.unwrap().polished_text, "完整正文"); + assert_eq!(history[0].final_text, "完整正文"); + assert_eq!(polisher.inputs(), vec![body]); + assert!(inserter.actions().iter().any(|action| matches!(action, + FixtureInsertionAction::Insert { text, .. } if text == "完整正文"))); + } + assert!(backend.snapshot().dictation.session_id.is_none()); + } + } + + #[tokio::test] + async fn voice_activation_transcript_revisions_keep_raw_offsets() { + let (backend, _) = backend(); + backend.start().await.unwrap(); + let id = start_voice_test_session(&backend, None, false).await; + let progress = backend.engine_progress_sink(); + for (text, offset, expected) in [ + ("开始语音录入你好", 0, "你好"), + ("世界完成语音录入", 8, "你好世界"), + ("你好，世界完成语音录入", 6, "你好，世界"), + ] { + progress + .publish( + id, + EngineProgress::TranscriptDelta(crate::TranscriptDelta { + text: text.into(), + offset, + is_final: false, + }), + ) + .unwrap(); + assert_eq!( + backend.state.read().unwrap().transcripts[&id].text(), + expected + ); + if offset == 0 { + backend + .stop_voice_activation( + crate::voice_activation::VoiceActivationStopReason::Disabled, + ) + .await + .unwrap(); + } + } + backend.cancel_dictation(Some(id)).await.unwrap(); + } + #[tokio::test] async fn recording_readiness_waits_for_the_first_pcm_even_when_its_meter_is_zero() { let (backend, _) = backend(); @@ -13345,6 +14600,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -13486,6 +14742,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -13566,6 +14823,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -13664,6 +14922,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -13730,6 +14989,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -13796,6 +15056,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -13869,6 +15130,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -14107,6 +15369,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -14159,6 +15422,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -14551,6 +15815,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -14592,6 +15857,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -14656,6 +15922,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -14765,6 +16032,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -14826,6 +16094,7 @@ mod tests { credential_store: llm_configured_credentials(), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -15193,6 +16462,7 @@ mod tests { credential_store: llm_configured_credentials(), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -15623,6 +16893,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(FakeInserter), dictation_engine: Arc::new(FakeEngine), @@ -15756,6 +17027,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -15814,6 +17086,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: host, text_inserter: Arc::new(crate::testing::FixtureTextInserter::with_outcome( outcome, @@ -16208,6 +17481,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -16252,6 +17526,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -16293,6 +17568,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -16336,6 +17612,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -16421,6 +17698,7 @@ mod tests { state: Arc::clone(&backend.state), phase_changed: Arc::clone(&backend.phase_changed), text_insertions: Arc::clone(&backend.text_insertions), + raw_transcripts: Mutex::new(HashMap::new()), }; backend.cancel_dictation(Some(session)).await.unwrap(); @@ -16449,6 +17727,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(FakeInserter), dictation_engine: Arc::new(BlockingStartEngine { @@ -16630,6 +17909,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -16700,6 +17980,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(FakeInserter), dictation_engine: Arc::new(BlockingStartEngine { diff --git a/openless-all/app/crates/openless-core/src/cloud_sync_e2ee_documents/registry.rs b/openless-all/app/crates/openless-core/src/cloud_sync_e2ee_documents/registry.rs index b32bdba5e..67cc380e5 100644 --- a/openless-all/app/crates/openless-core/src/cloud_sync_e2ee_documents/registry.rs +++ b/openless-all/app/crates/openless-core/src/cloud_sync_e2ee_documents/registry.rs @@ -193,6 +193,55 @@ pub const PREFERENCE_FIELDS: &[PreferenceField] = &[ shape: PreferenceShape::Boolean, reason: "local_only_microphone_capture", }, + PreferenceField { + rust_name: "voice_activation_enabled", + key: "voiceActivationEnabled", + class: PreferenceClass::Excluded, + shape: PreferenceShape::Boolean, + reason: "local_only_microphone_capture", + }, + PreferenceField { + rust_name: "voice_activation_wake_enabled", + key: "voiceActivationWakeEnabled", + class: PreferenceClass::Excluded, + shape: PreferenceShape::Boolean, + reason: "local_only_microphone_capture", + }, + PreferenceField { + rust_name: "voice_activation_finish_enabled", + key: "voiceActivationFinishEnabled", + class: PreferenceClass::Excluded, + shape: PreferenceShape::Boolean, + reason: "local_only_microphone_capture", + }, + PreferenceField { + rust_name: "voice_activation_wake_phrase", + key: "voiceActivationWakePhrase", + class: PreferenceClass::Excluded, + shape: PreferenceShape::Text, + reason: "local_only_microphone_capture", + }, + PreferenceField { + rust_name: "voice_activation_finish_phrase", + key: "voiceActivationFinishPhrase", + class: PreferenceClass::Excluded, + shape: PreferenceShape::Text, + reason: "local_only_microphone_capture", + }, + PreferenceField { + rust_name: "voice_activation_timeout_enabled", + key: "voiceActivationTimeoutEnabled", + class: PreferenceClass::Excluded, + shape: PreferenceShape::Boolean, + reason: "local_only_microphone_capture", + }, + PreferenceField { + rust_name: "voice_activation_timeout_seconds", + key: "voiceActivationTimeoutSeconds", + class: PreferenceClass::Excluded, + shape: PreferenceShape::Unsigned32, + reason: "local_only_microphone_capture", + }, PreferenceField { rust_name: "active_asr_provider", key: "activeAsrProvider", diff --git a/openless-all/app/crates/openless-core/src/config.rs b/openless-all/app/crates/openless-core/src/config.rs index 096dc9fd2..75527de1b 100644 --- a/openless-all/app/crates/openless-core/src/config.rs +++ b/openless-all/app/crates/openless-core/src/config.rs @@ -72,6 +72,7 @@ pub struct BackendDependencies { pub dictation_engine: Arc, pub task_spawner: Arc, pub credential_store: Arc, + pub voice_activation_runtime: Option>, pub services: BackendServices, pub local_asr_runtime: Option>, pub marketplace_config: Option, @@ -91,6 +92,7 @@ impl BackendDependencies { dictation_engine: Arc::new(UnsupportedDictationEngine), task_spawner: Arc::new(TokioTaskSpawner), credential_store: Arc::new(crate::credentials::UnsupportedCredentialStore), + voice_activation_runtime: None, services: BackendServices::unsupported(), local_asr_runtime: None, marketplace_config: None, diff --git a/openless-all/app/crates/openless-core/src/dictation_context.rs b/openless-all/app/crates/openless-core/src/dictation_context.rs index c06c3ef74..9a9cb977a 100644 --- a/openless-all/app/crates/openless-core/src/dictation_context.rs +++ b/openless-all/app/crates/openless-core/src/dictation_context.rs @@ -198,6 +198,8 @@ pub struct DictationContext { pub recording: RecordingPlan, pub pipeline_mode: PipelineMode, pub correction_rules: Vec, + /// Runtime-only commands, captured only for voice-started dictation. + pub voice_activation: Option, pub asr: ProviderInvocation, /// Local model a failed cloud ASR call is retried with, frozen at session /// start (see local_asr_fallback()); None when the fallback doesn't apply. @@ -340,6 +342,7 @@ impl DictationContext { preferences.pipeline_mode, ), correction_rules: Vec::new(), + voice_activation: None, asr, asr_fallback, llm, diff --git a/openless-all/app/crates/openless-core/src/dictation_engine.rs b/openless-all/app/crates/openless-core/src/dictation_engine.rs index 9e140803f..01c777f60 100644 --- a/openless-all/app/crates/openless-core/src/dictation_engine.rs +++ b/openless-all/app/crates/openless-core/src/dictation_engine.rs @@ -644,7 +644,9 @@ impl DictationEngine for PipelineDictationEngine { ) }); let race = match standby { - Some(standby) => race_cloud_with_local_standby(transcription.finish(), standby).await, + Some(standby) => { + race_cloud_with_local_standby(transcription.finish(), standby).await + } None => AsrRace::Cloud { result: transcription.finish().await, standby: None, @@ -825,6 +827,9 @@ impl DictationEngine for PipelineDictationEngine { ) .into()); } + if let Some(commands) = context.voice_activation.as_ref() { + transcript.text = commands.clean(&transcript.text); + } let original_asr_text = transcript.text.clone(); transcript.text = crate::correction::apply_correction_rules( &transcript.text, @@ -869,6 +874,7 @@ impl DictationEngine for PipelineDictationEngine { // Optional polish may be skipped after fallback; an explicit translation is still required. let translation_active = context.polish.translation_active; let uses_polisher = context.uses_llm_polisher() + && !(context.voice_activation.is_some() && transcript.text.trim().is_empty()) && !skips_polish_for_short_text && (!local_fallback_used || translation_active); log::info!( @@ -1230,7 +1236,10 @@ struct ReplayPlan { backoff: bool, } -const CLOUD_RETRY: ReplayPlan = ReplayPlan { attempts: 2, backoff: true }; +const CLOUD_RETRY: ReplayPlan = ReplayPlan { + attempts: 2, + backoff: true, +}; type AsrAttempt = ( crate::ports::TranscriptOutput, @@ -3464,6 +3473,128 @@ mod tests { assert_eq!(result.polished_text, "raw text"); } + #[tokio::test] + async fn voice_activation_commands_are_cleaned_before_correction_and_polishing() { + use crate::testing::{ + FixtureAudioRecorder, FixtureTextPolisher, FixtureTranscriptionEngine, + }; + for (input, voice, finish, expected, original) in [ + ( + "开始，文字录入。 错字 完成，文字录入。", + true, + true, + "正文", + Some("错字"), + ), + ( + "开始文字录入 正文完成文字录入之前 完成文字录入", + true, + true, + "正文完成文字录入之前", + None, + ), + ( + "开始文字录入 正文完成文字录入", + true, + false, + "正文完成文字录入", + None, + ), + ("开始文字录入 完成文字录入", true, true, "", None), + ( + "开始文字录入 正文 完成文字录入", + false, + true, + "开始文字录入 正文 完成文字录入", + None, + ), + ] { + let polisher = Arc::new(FixtureTextPolisher::successful("润色结果")); + let engine = PipelineDictationEngine::new( + Arc::new(FixtureAudioRecorder::new(vec![vec![1, 0, 2, 0]], vec![])), + Arc::new(FixtureTranscriptionEngine::successful(input, 1_000)), + polisher.clone(), + ); + let mut context = DictationContext::default(); + context.polish.polish_skip_max_chars = 0; + context.voice_activation = + voice.then(|| crate::voice_activation::VoiceActivationTextPolicy { + wake_phrase: "开始文字录入".into(), + finish_phrase: finish.then(|| "完成文字录入".into()), + }); + context.correction_rules.push(crate::types::CorrectionRule { + id: "voice-cleanup".into(), + pattern: "错字".into(), + replacement: "正文".into(), + enabled: true, + created_at: String::new(), + source: Default::default(), + }); + let session_id = SessionId::new(); + let progress = Arc::new(RecordingProgress::default()); + engine + .start(session_id, Arc::new(context), progress.clone()) + .await + .unwrap(); + let result = engine.finish(session_id, progress.clone()).await.unwrap(); + assert_eq!(result.raw_text, expected); + assert_eq!(result.asr_transcript.as_deref(), original); + if expected.is_empty() { + assert!(polisher.inputs().is_empty()); + assert!(result.polished_text.is_empty()); + } else { + assert_eq!(polisher.inputs(), vec![expected]); + assert_eq!(result.polished_text, "润色结果"); + } + assert!(progress.events.lock().unwrap().iter().any(|event| matches!(event, + EngineProgress::TranscriptDelta(delta) if delta.is_final && delta.text == expected))); + } + } + + #[tokio::test] + async fn voice_activation_polish_failure_retains_only_spoken_body() { + use crate::testing::{ + FixtureAudioRecorder, FixtureTextPolisher, FixtureTranscriptionEngine, + }; + for policy in [PolishFailurePolicy::Fail, PolishFailurePolicy::UseRawText] { + let polisher = Arc::new(FixtureTextPolisher::failing(BackendError::new( + BackendErrorCode::Provider, + "fixture", + ))); + let engine = PipelineDictationEngine::new( + Arc::new(FixtureAudioRecorder::new(vec![vec![1, 0, 2, 0]], vec![])), + Arc::new(FixtureTranscriptionEngine::successful( + "开始语音录入 正文 完成语音录入", + 1000, + )), + polisher.clone(), + ) + .with_polish_failure_policy(policy); + let context = DictationContext { + voice_activation: Some(crate::voice_activation::VoiceActivationTextPolicy { + wake_phrase: "开始语音录入".into(), + finish_phrase: Some("完成语音录入".into()), + }), + ..Default::default() + }; + let id = SessionId::new(); + let progress = Arc::new(RecordingProgress::default()); + engine + .start(id, Arc::new(context), progress.clone()) + .await + .unwrap(); + let result = engine.finish(id, progress).await; + assert_eq!(polisher.inputs(), vec!["正文"]); + match result { + Ok(output) => { + assert_eq!(output.raw_text, "正文"); + assert_eq!(output.polished_text, "正文"); + } + Err(failure) => assert_eq!(failure.raw_text.as_deref(), Some("正文")), + } + } + } + #[tokio::test] async fn short_transcript_under_threshold_skips_polish() { // Issue #1032: "raw text" has 7 non-whitespace characters, within a limit of 8. @@ -3755,7 +3886,9 @@ mod tests { /// started with, and records that provider — so a test can tell the cloud /// call, its retries and the local fallback apart. struct RoutedTranscriber { - outputs: Mutex>>>, + outputs: Mutex< + std::collections::HashMap>>, + >, started: Arc>>, /// How long each provider's finish takes; providers not listed answer at once. delays: std::collections::HashMap, @@ -3828,7 +3961,11 @@ mod tests { fn fallback_test_engine( cloud: Vec>, local: Vec>, - ) -> (PipelineDictationEngine, Arc, Arc>>) { + ) -> ( + PipelineDictationEngine, + Arc, + Arc>>, + ) { let (engine, progress, started, _) = fallback_polish_test_engine( cloud, local, @@ -3896,14 +4033,21 @@ mod tests { contexts: Arc::new(Mutex::new(Vec::new())), }), ); - (engine, Arc::new(RecordingProgress::default()), started, polish_calls) + ( + engine, + Arc::new(RecordingProgress::default()), + started, + polish_calls, + ) } /// A polishing session (not raw mode) on CLOUD with the LOCAL fallback. fn polishing_fallback_context() -> Arc { let mut context = DictationContext { asr: crate::dictation_context::ProviderInvocation::new(CLOUD, CLOUD), - asr_fallback: Some(crate::dictation_context::ProviderInvocation::new(LOCAL, LOCAL)), + asr_fallback: Some(crate::dictation_context::ProviderInvocation::new( + LOCAL, LOCAL, + )), ..DictationContext::default() }; assert!(context.uses_llm_polisher(), "fixture must exercise the polisher"); @@ -4008,7 +4152,11 @@ mod tests { #[tokio::test] async fn a_local_fallback_skips_polish_and_reports_no_failure() { let (engine, progress, started, polish_calls) = fallback_polish_test_engine( - vec![Err(BackendError::new(BackendErrorCode::Provider, "offline").retryable(true))], + vec![Err(BackendError::new( + BackendErrorCode::Provider, + "offline", + ) + .retryable(true))], vec![transcript("本地识别")], Err(BackendError::new(BackendErrorCode::Provider, "offline")), ); @@ -4037,7 +4185,10 @@ mod tests { let (engine, progress, _, polish_calls) = fallback_polish_test_engine( vec![], vec![transcript("本地识别")], - Err(BackendError::new(BackendErrorCode::Provider, "invalid api key")), + Err(BackendError::new( + BackendErrorCode::Provider, + "invalid api key", + )), ); let session_id = SessionId::new(); engine.start(session_id, local_polishing_context(), progress.clone()).await.unwrap(); @@ -4090,18 +4241,27 @@ mod tests { fn fallback_context() -> Arc { let mut context = (*raw_dictation_context()).clone(); context.asr = crate::dictation_context::ProviderInvocation::new(CLOUD, CLOUD); - context.asr_fallback = Some(crate::dictation_context::ProviderInvocation::new(LOCAL, LOCAL)); + context.asr_fallback = Some(crate::dictation_context::ProviderInvocation::new( + LOCAL, LOCAL, + )); Arc::new(context) } fn transcript(text: &str) -> Result { - Ok(TranscriptOutput { text: text.into(), duration_ms: 25 }) + Ok(TranscriptOutput { + text: text.into(), + duration_ms: 25, + }) } #[tokio::test] async fn failed_cloud_asr_goes_straight_to_the_local_fallback() { let (engine, progress, started) = fallback_test_engine( - vec![Err(BackendError::new(BackendErrorCode::Provider, "offline").retryable(true))], + vec![Err(BackendError::new( + BackendErrorCode::Provider, + "offline", + ) + .retryable(true))], vec![transcript("本地识别")], ); let session_id = SessionId::new(); @@ -4117,7 +4277,10 @@ mod tests { #[tokio::test] async fn a_non_retryable_cloud_error_also_falls_back() { let (engine, progress, started) = fallback_test_engine( - vec![Err(BackendError::new(BackendErrorCode::Provider, "quota exhausted"))], + vec![Err(BackendError::new( + BackendErrorCode::Provider, + "quota exhausted", + ))], vec![transcript("本地识别")], ); let session_id = SessionId::new(); @@ -4135,7 +4298,10 @@ mod tests { || Err(BackendError::new(BackendErrorCode::Provider, "offline").retryable(true)); let (engine, progress, started) = fallback_test_engine( vec![temporary(), temporary(), temporary()], - vec![Err(BackendError::new(BackendErrorCode::Unsupported, "model not downloaded"))], + vec![Err(BackendError::new( + BackendErrorCode::Unsupported, + "model not downloaded", + ))], ); let session_id = SessionId::new(); engine.start(session_id, fallback_context(), progress.clone()).await.unwrap(); @@ -4206,7 +4372,10 @@ mod tests { let (engine, progress, _, _) = racing_test_engine( &[(CLOUD, 200)], vec![transcript("云端识别")], - vec![Err(BackendError::new(BackendErrorCode::Unsupported, "model not downloaded"))], + vec![Err(BackendError::new( + BackendErrorCode::Unsupported, + "model not downloaded", + ))], Ok(crate::ports::PolishOutput::text("unused")), ); let session_id = SessionId::new(); diff --git a/openless-all/app/crates/openless-core/src/events.rs b/openless-all/app/crates/openless-core/src/events.rs index b118992d7..f5c75272c 100644 --- a/openless-all/app/crates/openless-core/src/events.rs +++ b/openless-all/app/crates/openless-core/src/events.rs @@ -14,6 +14,7 @@ use crate::types::{ InsertFallbackPayload, NotificationPayload, PermissionSnapshot, PolishDelta, PreferencesChange, SessionId, StylePackChange, TranscriptDelta, VocabularyChange, }; +use crate::voice_activation::VoiceActivationStateSnapshot; #[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] #[serde(rename_all = "camelCase")] @@ -407,6 +408,7 @@ pub enum BackendEventKind { BackendStarted, BackendStopping, DictationStateChanged(DictationStateSnapshot), + VoiceActivationStateChanged(crate::voice_activation::VoiceActivationStateSnapshot), TranscriptDelta(TranscriptDelta), PolishDelta(PolishDelta), DictationCompleted(DictationResult), @@ -458,6 +460,9 @@ pub struct EventBus { sequence: AtomicU64, sender: broadcast::Sender, backlog: Mutex>, + // Retain the latest activation state independently of the bounded replay so a host can + // recover the microphone listener after a subscription lag. + latest_voice_activation: Mutex>, // Retain one presentation snapshot independently of the bounded replay so // a reopened host can recover a long-running transcription. latest_less_voice: Mutex>, @@ -470,6 +475,7 @@ impl EventBus { sequence: AtomicU64::new(0), sender, backlog: Mutex::new(VecDeque::with_capacity(EVENT_REPLAY_CAPACITY)), + latest_voice_activation: Mutex::new(None), latest_less_voice: Mutex::new(None), } } @@ -490,6 +496,18 @@ impl EventBus { session_id, kind, }; + if let BackendEventKind::VoiceActivationStateChanged(state) = &event.kind { + let mut latest = self + .latest_voice_activation + .lock() + .expect("voice activation projection lock poisoned"); + if latest + .as_ref() + .is_none_or(|previous| state.activation_generation >= previous.activation_generation) + { + *latest = Some(state.clone()); + } + } if let BackendEventKind::LessComputerEvent(voice) = &event.kind { if let LessComputerEventKind::VoiceState { session_id, phase, .. @@ -556,6 +574,13 @@ impl EventBus { } } + pub fn latest_voice_activation_state(&self) -> Option { + self.latest_voice_activation + .lock() + .expect("voice activation projection lock poisoned") + .clone() + } + pub fn latest_less_computer_voice_state(&self) -> Option { self.latest_less_voice .lock() @@ -588,6 +613,10 @@ impl BackendEventPublisher { self.bus.replay_after(sequence) } + pub fn latest_voice_activation_state(&self) -> Option { + self.bus.latest_voice_activation_state() + } + pub fn latest_less_computer_voice_state(&self) -> Option { self.bus.latest_less_computer_voice_state() } @@ -665,6 +694,42 @@ mod tests { assert_eq!(event.unwrap().sequence, 2); } + #[test] + fn latest_voice_activation_state_ignores_stale_generations() { + let bus = EventBus::new(4); + bus.publish( + None, + BackendEventKind::VoiceActivationStateChanged(VoiceActivationStateSnapshot { + phase: crate::voice_activation::VoiceActivationPhase::ListeningForWake, + activation_generation: 2, + session_id: None, + message: None, + }), + ); + bus.publish( + None, + BackendEventKind::VoiceActivationStateChanged(VoiceActivationStateSnapshot { + phase: crate::voice_activation::VoiceActivationPhase::Failed, + activation_generation: 1, + session_id: None, + message: Some("stale".into()), + }), + ); + assert_eq!( + bus.latest_voice_activation_state() + .expect("latest activation state") + .activation_generation, + 2 + ); + assert_eq!( + BackendEventPublisher::new(Arc::new(bus)) + .latest_voice_activation_state() + .expect("publisher activation state") + .activation_generation, + 2 + ); + } + #[test] fn native_publishers_keep_live_and_replay_sequences_in_the_same_order() { const THREADS: usize = 8; diff --git a/openless-all/app/crates/openless-core/src/lib.rs b/openless-all/app/crates/openless-core/src/lib.rs index bdf7f9ed1..92964d687 100644 --- a/openless-all/app/crates/openless-core/src/lib.rs +++ b/openless-all/app/crates/openless-core/src/lib.rs @@ -76,6 +76,7 @@ pub mod style_packs; pub mod testing; pub mod types; pub mod vocabulary; +pub mod voice_activation; pub mod voice_edit_session; mod voice_session; @@ -303,7 +304,9 @@ pub use ports::{ PendingAudioCapture, PolishOutput, RecordingArchive, RecordingControlSink, RecordingEvent, RecordingProgressSink, ResourceResolver, TextInserter, TextInsertionSession, TextPolisher, TextStreamChunk, TextStreamSink, TranscriptOutput, TranscriptionEngine, TranscriptionSession, - UnsupportedTextInserter, VoiceCapture, + UnsupportedTextInserter, UnsupportedVoiceActivationRuntime, VoiceActivationError, + VoiceActivationHandle, VoiceActivationRuntime, VoiceActivationSink, VoiceCapture, + VoiceDictationHandoff, VoiceDictationHandoffRequest, }; pub use preferences::PreferencesStore; pub use prompt_compose::{ @@ -377,6 +380,15 @@ pub use vocabulary::{ builtin_vocab_presets, list_vocab_presets, resolve_vocab_presets, save_vocab_presets, DictionaryStore, }; +pub use voice_activation::{ + normalize_phrase, validate_phrase, PhraseValidationError, VoiceActivationCommand, + VoiceActivationConfig, VoiceActivationDecision, VoiceActivationGeneration, + VoiceActivationPhase, VoiceActivationSession, VoiceActivationSignal, + VoiceActivationStateSnapshot, VoiceActivationStopReason, VoiceActivationTextPolicy, + VoiceCommandMatch, VoiceCommandRecognizer, VoicePhraseMatcher, DEFAULT_FINISH_PHRASE, + DEFAULT_TIMEOUT_SECONDS, DEFAULT_WAKE_PHRASE, MAX_PHRASE_CHARS, MAX_TIMEOUT_SECONDS, + MIN_TIMEOUT_SECONDS, +}; pub use voice_edit_session::{ TextSelection, VoiceEditCommit, VoiceEditContext, VoiceEditError, VoiceEditPhase, VoiceEditSession, VoiceEditSnapshot, VoiceEditTarget, VoiceEditTurn, diff --git a/openless-all/app/crates/openless-core/src/ports.rs b/openless-all/app/crates/openless-core/src/ports.rs index 5874fb54e..3ab20b2a4 100644 --- a/openless-all/app/crates/openless-core/src/ports.rs +++ b/openless-all/app/crates/openless-core/src/ports.rs @@ -8,6 +8,7 @@ use crate::audio::AudioProcessingConfig; use crate::dictation_context::DictationContext; use crate::errors::{BackendError, BackendErrorCode}; use crate::types::{InsertStatus, PolishDelta, SessionId, TranscriptDelta}; +use crate::voice_activation::{VoiceActivationConfig, VoiceActivationSignal}; #[derive(Debug, Clone, PartialEq, Eq)] pub enum HostAction { @@ -73,6 +74,85 @@ impl HostActions for NoopHostActions { } } +/// Host-owned sink for semantic voice activation signals. Standby recognition +/// must only publish wake/finish commands and runtime state; it must never +/// publish the recognized standby text as a normal transcript delta. +pub trait VoiceActivationSink: Send + Sync { + fn publish(&self, signal: VoiceActivationSignal) -> Result<(), BackendError>; +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum VoiceActivationError { + Unsupported, + ModelMissing, + PermissionDenied, + MicBusy, + InvalidConfig(String), + Runtime(String), +} + +impl std::fmt::Display for VoiceActivationError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Unsupported => formatter.write_str("voice activation is unsupported"), + Self::ModelMissing => formatter.write_str("voice activation model is not installed"), + Self::PermissionDenied => formatter.write_str("microphone permission was denied"), + Self::MicBusy => formatter.write_str("microphone is already in use"), + Self::InvalidConfig(message) | Self::Runtime(message) => formatter.write_str(message), + } + } +} + +impl std::error::Error for VoiceActivationError {} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VoiceDictationHandoffRequest { + pub activation_generation: u64, +} + +pub struct VoiceDictationHandoff { + pub capture: Box, +} + +/// A single host-owned microphone/runtime seam for voice activation. The Core +/// decides lifecycle and stale-event policy; the host decides how local ASR and +/// the native capture stream are implemented. +pub trait VoiceActivationRuntime: Send + Sync + 'static { + fn configure( + &self, + config: VoiceActivationConfig, + sink: Arc, + ) -> BoxFuture<'static, Result, VoiceActivationError>>; +} + +pub trait VoiceActivationHandle: Send + Sync { + /// Bind the Core-owned dictation session so finish signals can carry the + /// same session identity even though the host performs recognition. + fn bind_dictation_session(&self, _session_id: SessionId) {} + + fn handoff_to_dictation( + &self, + request: VoiceDictationHandoffRequest, + ) -> BoxFuture<'static, Result>; + + fn stop( + &self, + reason: crate::voice_activation::VoiceActivationStopReason, + ) -> BoxFuture<'static, Result<(), VoiceActivationError>>; +} + +pub struct UnsupportedVoiceActivationRuntime; + +impl VoiceActivationRuntime for UnsupportedVoiceActivationRuntime { + fn configure( + &self, + _config: VoiceActivationConfig, + _sink: Arc, + ) -> BoxFuture<'static, Result, VoiceActivationError>> { + Box::pin(async { Err(VoiceActivationError::Unsupported) }) + } +} + /// Resolve a packaged resource without exposing a framework-specific resource /// directory object to the core or UI. pub trait ResourceResolver: Send + Sync { diff --git a/openless-all/app/crates/openless-core/src/shared_types.rs b/openless-all/app/crates/openless-core/src/shared_types.rs index baf4bb000..a133a6ff4 100644 --- a/openless-all/app/crates/openless-core/src/shared_types.rs +++ b/openless-all/app/crates/openless-core/src/shared_types.rs @@ -360,6 +360,18 @@ fn default_silence_auto_stop_seconds() -> f32 { 3.0 } +fn default_voice_activation_wake_phrase() -> String { + crate::voice_activation::DEFAULT_WAKE_PHRASE.into() +} + +fn default_voice_activation_finish_phrase() -> String { + crate::voice_activation::DEFAULT_FINISH_PHRASE.into() +} + +fn default_voice_activation_timeout_seconds() -> u32 { + crate::voice_activation::DEFAULT_TIMEOUT_SECONDS +} + fn resolve_windows_insertion_mode( mode: WindowsInsertionMode, legacy_sendinput_only: bool, @@ -445,6 +457,21 @@ pub struct UserPreferences { /// Recording input device name. Empty string = system default microphone. #[serde(default)] pub microphone_device_name: String, + /// Opt-in local voice command listener. Standby audio is never sent to a cloud provider. + #[serde(default)] + pub voice_activation_enabled: bool, + #[serde(default = "default_true")] + pub voice_activation_wake_enabled: bool, + #[serde(default = "default_true")] + pub voice_activation_finish_enabled: bool, + #[serde(default = "default_voice_activation_wake_phrase")] + pub voice_activation_wake_phrase: String, + #[serde(default = "default_voice_activation_finish_phrase")] + pub voice_activation_finish_phrase: String, + #[serde(default = "default_true")] + pub voice_activation_timeout_enabled: bool, + #[serde(default = "default_voice_activation_timeout_seconds")] + pub voice_activation_timeout_seconds: u32, pub active_asr_provider: String, // "volcengine" | "apple-speech" | ... pub active_llm_provider: String, // "ark" | "openai" | ... /// Recognition pipeline mode (experimental, issue #902). With @@ -1009,6 +1036,20 @@ struct UserPreferencesWire { silence_auto_stop_seconds: f32, #[serde(default)] microphone_device_name: String, + #[serde(default)] + voice_activation_enabled: bool, + #[serde(default = "default_true")] + voice_activation_wake_enabled: bool, + #[serde(default = "default_true")] + voice_activation_finish_enabled: bool, + #[serde(default = "default_voice_activation_wake_phrase")] + voice_activation_wake_phrase: String, + #[serde(default = "default_voice_activation_finish_phrase")] + voice_activation_finish_phrase: String, + #[serde(default = "default_true")] + voice_activation_timeout_enabled: bool, + #[serde(default = "default_voice_activation_timeout_seconds")] + voice_activation_timeout_seconds: u32, active_asr_provider: String, active_llm_provider: String, #[serde(default = "default_pipeline_mode")] @@ -1271,6 +1312,13 @@ impl Default for UserPreferencesWire { silence_auto_stop_enabled: prefs.silence_auto_stop_enabled, silence_auto_stop_seconds: prefs.silence_auto_stop_seconds, microphone_device_name: prefs.microphone_device_name, + voice_activation_enabled: prefs.voice_activation_enabled, + voice_activation_wake_enabled: prefs.voice_activation_wake_enabled, + voice_activation_finish_enabled: prefs.voice_activation_finish_enabled, + voice_activation_wake_phrase: prefs.voice_activation_wake_phrase, + voice_activation_finish_phrase: prefs.voice_activation_finish_phrase, + voice_activation_timeout_enabled: prefs.voice_activation_timeout_enabled, + voice_activation_timeout_seconds: prefs.voice_activation_timeout_seconds, active_asr_provider: prefs.active_asr_provider, active_llm_provider: prefs.active_llm_provider, pipeline_mode: prefs.pipeline_mode, @@ -1483,6 +1531,16 @@ impl<'de> Deserialize<'de> for UserPreferences { silence_auto_stop_enabled: wire.silence_auto_stop_enabled, silence_auto_stop_seconds: wire.silence_auto_stop_seconds, microphone_device_name: wire.microphone_device_name, + voice_activation_enabled: wire.voice_activation_enabled, + voice_activation_wake_enabled: wire.voice_activation_wake_enabled, + voice_activation_finish_enabled: wire.voice_activation_finish_enabled, + voice_activation_wake_phrase: wire.voice_activation_wake_phrase, + voice_activation_finish_phrase: wire.voice_activation_finish_phrase, + voice_activation_timeout_enabled: wire.voice_activation_timeout_enabled, + voice_activation_timeout_seconds: wire.voice_activation_timeout_seconds.clamp( + crate::voice_activation::MIN_TIMEOUT_SECONDS, + crate::voice_activation::MAX_TIMEOUT_SECONDS, + ), active_asr_provider: wire.active_asr_provider, active_llm_provider: wire.active_llm_provider, pipeline_mode: wire.pipeline_mode, @@ -1868,6 +1926,13 @@ impl Default for UserPreferences { silence_auto_stop_enabled: false, silence_auto_stop_seconds: default_silence_auto_stop_seconds(), microphone_device_name: String::new(), + voice_activation_enabled: false, + voice_activation_wake_enabled: true, + voice_activation_finish_enabled: true, + voice_activation_wake_phrase: default_voice_activation_wake_phrase(), + voice_activation_finish_phrase: default_voice_activation_finish_phrase(), + voice_activation_timeout_enabled: true, + voice_activation_timeout_seconds: default_voice_activation_timeout_seconds(), active_asr_provider: default_active_asr_provider(), active_llm_provider: "ark".into(), pipeline_mode: PipelineMode::Traditional, diff --git a/openless-all/app/crates/openless-core/src/silence_auto_stop.rs b/openless-all/app/crates/openless-core/src/silence_auto_stop.rs index ebd19ae72..21f27ad0b 100644 --- a/openless-all/app/crates/openless-core/src/silence_auto_stop.rs +++ b/openless-all/app/crates/openless-core/src/silence_auto_stop.rs @@ -31,6 +31,15 @@ impl SilenceAutoStop { } } + /// A wake command already established speech before formal capture began. + pub fn after_speech(silence_after_speech: Duration, started_at: Instant) -> Self { + Self { + speech_detected: true, + last_speech_at: Some(started_at), + ..Self::new(silence_after_speech, started_at) + } + } + pub fn on_level(&mut self, level: f32, now: Instant) -> Option { if self.decided { return None; @@ -72,6 +81,35 @@ mod tests { .find_map(|(level, offset)| detector.on_level(*level, base + *offset)) } + #[test] + fn voice_activation_timeout_tracks_speech_instead_of_total_duration() { + let base = Instant::now(); + let mut detector = SilenceAutoStop::after_speech(Duration::from_secs(15), base); + for seconds in 0..=30 { + assert_eq!( + detector.on_level(0.1, base + Duration::from_secs(seconds)), + None + ); + } + assert_eq!(detector.on_level(0.0, base + Duration::from_secs(44)), None); + assert_eq!( + detector.on_level(0.0, base + Duration::from_secs(45)), + Some(SilenceDecision::Stop) + ); + assert_eq!(detector.on_level(0.0, base + Duration::from_secs(60)), None); + } + + #[test] + fn voice_activation_without_body_uses_configured_silence_timeout() { + let base = Instant::now(); + let mut detector = SilenceAutoStop::after_speech(Duration::from_secs(15), base); + assert_eq!(detector.on_level(0.0, base + Duration::from_secs(10)), None); + assert_eq!( + detector.on_level(0.0, base + Duration::from_secs(15)), + Some(SilenceDecision::Stop) + ); + } + #[test] fn speech_then_silence_stops_once() { let base = Instant::now(); diff --git a/openless-all/app/crates/openless-core/src/voice_activation.rs b/openless-all/app/crates/openless-core/src/voice_activation.rs new file mode 100644 index 000000000..1faf28ded --- /dev/null +++ b/openless-all/app/crates/openless-core/src/voice_activation.rs @@ -0,0 +1,565 @@ +//! Pure policy and matching primitives for voice-activated dictation. +//! +//! The host owns the microphone and local recognizer. This module deliberately +//! only consumes revised recognition text and produces command decisions, so a +//! standby listener can never accidentally publish normal transcript events. + +use std::time::Duration; + +use unicode_normalization::UnicodeNormalization; + +use crate::types::SessionId; + +pub const DEFAULT_WAKE_PHRASE: &str = "开始语音录入"; +pub const DEFAULT_FINISH_PHRASE: &str = "完成语音录入"; +pub const DEFAULT_TIMEOUT_SECONDS: u32 = 30; +pub const MIN_TIMEOUT_SECONDS: u32 = 5; +pub const MAX_TIMEOUT_SECONDS: u32 = 300; +pub const MAX_PHRASE_CHARS: usize = 64; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PhraseValidationError { + Empty, + TooLong { max_chars: usize }, + ControlCharacter, +} + +impl std::fmt::Display for PhraseValidationError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Empty => write!(f, "voice activation phrase must not be empty"), + Self::TooLong { max_chars } => { + write!( + f, + "voice activation phrase must be at most {max_chars} characters" + ) + } + Self::ControlCharacter => { + write!( + f, + "voice activation phrase must not contain control characters" + ) + } + } + } +} + +impl std::error::Error for PhraseValidationError {} + +/// Normalize ASR output and user phrases before suffix matching. +/// +/// NFKC handles full-width punctuation and compatibility forms. Whitespace +/// and punctuation are intentionally removed because streaming ASR may insert +/// them between Chinese command characters while English recognition may emit +/// multiple spaces or hyphens. +pub fn normalize_phrase(text: &str) -> String { + text.nfkc() + .filter(|character| !character.is_whitespace() && !is_terminal_punctuation(*character)) + .flat_map(char::to_lowercase) + .collect() +} + +/// Remove a recognized command from the end of a transcript while preserving +/// the spacing and punctuation in the spoken body. +pub fn strip_phrase_suffix(text: &str, phrase: &str) -> Option { + let normalized_phrase = validate_phrase(phrase).ok()?; + let phrase_chars = normalized_phrase.chars().rev(); + let mut cursor = text.len(); + let mut cut_at = cursor; + for expected in phrase_chars { + while cursor > 0 { + let previous = text[..cursor].char_indices().next_back()?; + if previous.1.is_whitespace() || is_terminal_punctuation(previous.1) { + cursor = previous.0; + continue; + } + break; + } + let (start, character) = text[..cursor].char_indices().next_back()?; + if normalize_phrase(&character.to_string()) != expected.to_string() { + return None; + } + cut_at = start; + cursor = start; + } + Some(text[..cut_at].trim_end().to_string()) +} + +/// Remove a recognized command from the beginning of a transcript while preserving +/// the spoken body. Only a true prefix match is accepted. +pub fn strip_phrase_prefix(text: &str, phrase: &str) -> Option { + let normalized_phrase = validate_phrase(phrase).ok()?; + let mut cursor = 0; + for expected in normalized_phrase.chars() { + loop { + let (_, character) = text[cursor..].char_indices().next()?; + cursor += character.len_utf8(); + if character.is_whitespace() || is_terminal_punctuation(character) { + continue; + } + if normalize_phrase(&character.to_string()) != expected.to_string() { + return None; + } + break; + } + } + while cursor < text.len() { + let character = text[cursor..].chars().next()?; + if !character.is_whitespace() && !is_terminal_punctuation(character) { + break; + } + cursor += character.len_utf8(); + } + Some(text[cursor..].to_string()) +} + +fn is_terminal_punctuation(character: char) -> bool { + character.to_string().nfkc().any(|normalized| { + normalized.is_ascii_punctuation() + || matches!( + normalized, + '。' | '！' + | '？' + | '，' + | '、' + | '；' + | '：' + | '“' + | '”' + | '‘' + | '’' + | '（' + | '）' + | '【' + | '】' + | '《' + | '》' + | '…' + | '～' + | '—' + | '–' + ) + }) +} + +pub fn validate_phrase(text: &str) -> Result { + let normalized = normalize_phrase(text); + if normalized.is_empty() { + return Err(PhraseValidationError::Empty); + } + if normalized.chars().any(char::is_control) { + return Err(PhraseValidationError::ControlCharacter); + } + if normalized.chars().count() > MAX_PHRASE_CHARS { + return Err(PhraseValidationError::TooLong { + max_chars: MAX_PHRASE_CHARS, + }); + } + Ok(normalized) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VoiceActivationConfig { + pub enabled: bool, + pub wake_enabled: bool, + pub finish_enabled: bool, + pub wake_phrase: String, + pub finish_phrase: String, + pub timeout_enabled: bool, + pub timeout_seconds: u32, + /// Runtime-only selected input; never serialized as a user preference field. + pub microphone_device_name: Option, + /// Runtime-owned generation; this is not persisted in preferences. + pub activation_generation: u64, +} + +impl Default for VoiceActivationConfig { + fn default() -> Self { + Self { + enabled: false, + wake_enabled: true, + finish_enabled: true, + wake_phrase: DEFAULT_WAKE_PHRASE.into(), + finish_phrase: DEFAULT_FINISH_PHRASE.into(), + timeout_enabled: true, + timeout_seconds: DEFAULT_TIMEOUT_SECONDS, + microphone_device_name: None, + activation_generation: 0, + } + } +} + +impl VoiceActivationConfig { + pub fn normalized(&self) -> Result { + let wake_phrase = validate_phrase(&self.wake_phrase)?; + let finish_phrase = validate_phrase(&self.finish_phrase)?; + Ok(Self { + enabled: self.enabled, + wake_enabled: self.wake_enabled, + finish_enabled: self.finish_enabled, + wake_phrase, + finish_phrase, + timeout_enabled: self.timeout_enabled, + timeout_seconds: self + .timeout_seconds + .clamp(MIN_TIMEOUT_SECONDS, MAX_TIMEOUT_SECONDS), + microphone_device_name: self.microphone_device_name.clone(), + activation_generation: self.activation_generation, + }) + } + + pub fn timeout(&self) -> Option { + self.timeout_enabled + .then(|| Duration::from_secs(u64::from(self.timeout_seconds))) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VoiceActivationCommand { + Wake, + Finish, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VoiceCommandMatch { + pub command: VoiceActivationCommand, + pub text_without_command: String, +} + +/// Matches only the end of the revised ASR text. A phrase in the middle of a +/// sentence must never trigger activation or stop an active dictation session. +#[derive(Debug, Clone)] +pub struct VoicePhraseMatcher { + normalized_phrase: String, + command: VoiceActivationCommand, +} + +impl VoicePhraseMatcher { + pub fn new( + phrase: impl AsRef, + command: VoiceActivationCommand, + ) -> Result { + Ok(Self { + normalized_phrase: validate_phrase(phrase.as_ref())?, + command, + }) + } + + pub fn matches_suffix(&self, text: &str) -> bool { + normalize_phrase(text).ends_with(&self.normalized_phrase) + } + + pub fn strip_suffix(&self, text: &str) -> Option { + let normalized = normalize_phrase(text); + normalized + .strip_suffix(&self.normalized_phrase) + .map(str::to_string) + } + + pub fn command(&self) -> VoiceActivationCommand { + self.command + } +} + +/// Handles ASR revisions. A revised result may replace a previous result, +/// therefore a command is reported at most once for each observed text value. +#[derive(Debug, Clone)] +pub struct VoiceCommandRecognizer { + wake: VoicePhraseMatcher, + finish: VoicePhraseMatcher, + last_text: String, + wake_reported: bool, + finish_reported: bool, +} + +impl VoiceCommandRecognizer { + pub fn new(config: &VoiceActivationConfig) -> Result { + let config = config.normalized()?; + Ok(Self { + wake: VoicePhraseMatcher::new(config.wake_phrase, VoiceActivationCommand::Wake)?, + finish: VoicePhraseMatcher::new(config.finish_phrase, VoiceActivationCommand::Finish)?, + last_text: String::new(), + wake_reported: false, + finish_reported: false, + }) + } + + pub fn observe( + &mut self, + text: &str, + command: VoiceActivationCommand, + ) -> Option { + let normalized = normalize_phrase(text); + if normalized == self.last_text { + return None; + } + self.last_text = normalized.clone(); + let matcher = match command { + VoiceActivationCommand::Wake => &self.wake, + VoiceActivationCommand::Finish => &self.finish, + }; + let already_reported = match command { + VoiceActivationCommand::Wake => self.wake_reported, + VoiceActivationCommand::Finish => self.finish_reported, + }; + if already_reported || !normalized.ends_with(&matcher.normalized_phrase) { + return None; + } + match command { + VoiceActivationCommand::Wake => self.wake_reported = true, + VoiceActivationCommand::Finish => self.finish_reported = true, + } + Some(VoiceCommandMatch { + command, + text_without_command: normalized + .strip_suffix(&matcher.normalized_phrase) + .unwrap_or_default() + .to_string(), + }) + } + + pub fn reset(&mut self) { + self.last_text.clear(); + self.wake_reported = false; + self.finish_reported = false; + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum VoiceActivationSignal { + WakeDetected { + activation_generation: u64, + }, + FinishDetected { + activation_generation: u64, + session_id: SessionId, + }, + RuntimeStateChanged { + activation_generation: u64, + phase: VoiceActivationPhase, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum VoiceActivationPhase { + Disabled, + Starting, + ListeningForWake, + WakeDetected, + HandingOff, + ListeningForFinish, + Finishing, + Unavailable, + PermissionDenied, + MicBusy, + Failed, +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct VoiceActivationStateSnapshot { + pub phase: VoiceActivationPhase, + pub activation_generation: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub session_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub message: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VoiceActivationStopReason { + Disabled, + DictationStarted, + FinishPhrase, + Timeout, + Cancelled, + RuntimeFailure, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct VoiceActivationGeneration(pub u64); + +impl VoiceActivationGeneration { + pub fn next(self) -> Self { + Self(self.0.saturating_add(1)) + } +} + +/// Commands frozen for a voice-started session; ordinary dictation has no policy. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VoiceActivationTextPolicy { + pub wake_phrase: String, + pub finish_phrase: Option, +} + +impl VoiceActivationTextPolicy { + pub fn clean(&self, text: &str) -> String { + let body = strip_phrase_prefix(text, &self.wake_phrase).unwrap_or_else(|| text.to_owned()); + self.finish_phrase + .as_ref() + .and_then(|phrase| strip_phrase_suffix(&body, phrase)) + .unwrap_or(body) + } +} + +#[derive(Debug, Clone)] +pub struct VoiceActivationSession { + pub generation: VoiceActivationGeneration, + pub session_id: SessionId, + pub timeout: Option, + pub commands: VoiceActivationTextPolicy, +} + +impl VoiceActivationSession { + pub fn accepts(&self, generation: VoiceActivationGeneration, session_id: SessionId) -> bool { + self.generation == generation && self.session_id == session_id + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VoiceActivationDecision { + Ignore, + StartDictation, + FinishDictation, + StopAfterTimeout, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn normalization_handles_spaces_width_case_and_punctuation() { + assert_eq!(normalize_phrase(" Ａ 开始 语音录入 "), "a开始语音录入"); + assert_eq!(normalize_phrase("完成\n语音\t录入"), DEFAULT_FINISH_PHRASE); + assert_eq!(normalize_phrase("完成，语音录入。"), DEFAULT_FINISH_PHRASE); + } + + #[test] + fn strip_phrase_prefix_preserves_body_and_requires_prefix() { + assert_eq!( + strip_phrase_prefix("开始语音录入，你好，世界", DEFAULT_WAKE_PHRASE), + Some("你好，世界".into()) + ); + assert_eq!( + strip_phrase_prefix("开始，语音录入 hello", DEFAULT_WAKE_PHRASE), + Some("hello".into()) + ); + assert!(strip_phrase_prefix("你好，开始语音录入", DEFAULT_WAKE_PHRASE).is_none()); + } + + #[test] + fn strip_phrase_prefix_handles_nfkc_and_empty_body() { + assert_eq!(strip_phrase_prefix("開始語音錄入。", "开始语音录入"), None); + assert_eq!( + strip_phrase_prefix("开始语音录入。", DEFAULT_WAKE_PHRASE), + Some(String::new()) + ); + } + #[test] + fn strip_phrase_suffix_preserves_spoken_body() { + assert_eq!( + strip_phrase_suffix("你好，世界 完成 语音录入", DEFAULT_FINISH_PHRASE), + Some("你好，世界".into()) + ); + assert_eq!( + strip_phrase_suffix("你好，世界 完成，语音录入。", DEFAULT_FINISH_PHRASE), + Some("你好，世界".into()) + ); + assert_eq!( + strip_phrase_suffix("你好，世界 完成语音录入．", DEFAULT_FINISH_PHRASE), + Some("你好，世界".into()) + ); + assert!(strip_phrase_suffix("完成语音录入之前", DEFAULT_FINISH_PHRASE).is_none()); + } + + #[test] + fn phrase_validation_rejects_empty_control_and_long_values() { + assert_eq!(validate_phrase(" \n"), Err(PhraseValidationError::Empty)); + assert_eq!( + validate_phrase("开始\u{0000}录入"), + Err(PhraseValidationError::ControlCharacter) + ); + assert!(matches!( + validate_phrase(&"a".repeat(MAX_PHRASE_CHARS + 1)), + Err(PhraseValidationError::TooLong { .. }) + )); + } + + #[test] + fn matcher_only_accepts_suffix_and_strips_command() { + let matcher = + VoicePhraseMatcher::new(DEFAULT_FINISH_PHRASE, VoiceActivationCommand::Finish).unwrap(); + assert!(!matcher.matches_suffix("完成语音录入之前的内容")); + assert!(matcher.matches_suffix("正文 完成，语音录入。")); + assert_eq!( + matcher.strip_suffix("正文 完成语音录入").as_deref(), + Some( + "正文完成语音录入" + .strip_suffix(DEFAULT_FINISH_PHRASE) + .unwrap() + ) + ); + } + + #[test] + fn recognizer_does_not_repeat_a_command_for_asr_revisions() { + let config = VoiceActivationConfig::default(); + let mut recognizer = VoiceCommandRecognizer::new(&config).unwrap(); + assert!(recognizer + .observe("开始语音", VoiceActivationCommand::Wake) + .is_none()); + assert!(recognizer + .observe("开始语音录入", VoiceActivationCommand::Wake) + .is_some()); + assert!(recognizer + .observe("开始语音录入", VoiceActivationCommand::Wake) + .is_none()); + recognizer.reset(); + assert!(recognizer + .observe("开始语音录入", VoiceActivationCommand::Wake) + .is_some()); + } + + #[test] + fn generation_and_session_guards_reject_stale_events() { + let session_id = SessionId::new(); + let session = VoiceActivationSession { + generation: VoiceActivationGeneration(7), + session_id, + timeout: Some(Duration::from_secs(1)), + commands: VoiceActivationTextPolicy { + wake_phrase: DEFAULT_WAKE_PHRASE.into(), + finish_phrase: Some(DEFAULT_FINISH_PHRASE.into()), + }, + }; + assert!(session.accepts(VoiceActivationGeneration(7), session_id)); + assert!(!session.accepts(VoiceActivationGeneration(6), session_id)); + assert!(!session.accepts(VoiceActivationGeneration(7), SessionId::new())); + } + + #[test] + fn timeout_is_bounded_and_only_applies_after_the_voice_session_starts() { + let config = VoiceActivationConfig { + timeout_seconds: 1, + ..VoiceActivationConfig::default() + }; + assert_eq!( + config.normalized().unwrap().timeout_seconds, + MIN_TIMEOUT_SECONDS + ); + assert_eq!( + config.normalized().unwrap().timeout(), + Some(Duration::from_secs(5)) + ); + assert_eq!( + VoiceActivationConfig { + timeout_enabled: false, + ..config + } + .timeout(), + None + ); + } +} diff --git a/openless-all/app/e2e/settings-regressions.spec.ts b/openless-all/app/e2e/settings-regressions.spec.ts index 4503c2993..1664d4229 100644 --- a/openless-all/app/e2e/settings-regressions.spec.ts +++ b/openless-all/app/e2e/settings-regressions.spec.ts @@ -227,3 +227,61 @@ test('conditional notices animate actual layout downward and retain content duri await expect(page.getByTestId('notice')).toHaveText('Working status'); await expect(page.getByTestId('notice')).toBeHidden(); }); + +test('voice activation opens model setup without leaving the multimodal pipeline', async ({ + page, +}, testInfo) => { + await page.addInitScript(() => { + Object.defineProperty(navigator, 'platform', { get: () => 'Win32' }); + Object.defineProperty(navigator, 'userAgent', { get: () => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)' }); + }); + await page.goto('http://127.0.0.1:5326/?os=win'); + await page.getByRole('button', { name: '设置', exact: true }).click(); + const dialog = page.getByRole('dialog', { name: '设置', exact: true }); + const categories = dialog.getByRole('navigation', { name: '设置分类' }); + await categories.getByRole('button', { name: 'AI 服务与模型', exact: true }).click(); + await dialog.getByRole('button', { name: '多模态模式', exact: true }).click(); + const models = dialog + .locator('.ol-service-views') + .getByRole('button', { name: /^本地模型/ }); + await expect(models).toBeDisabled(); + await categories.getByRole('button', { name: '录音与输入', exact: true }).click(); + await dialog.getByRole('switch', { name: '语音唤醒听写', exact: true }).click(); + const download = dialog.getByRole('button', { + name: '前往 AI 服务与模型下载 Zipformer', + exact: true, + }); + await expect(download).toBeVisible(); + const providerState = () => + page.evaluate(async () => { + const { mockSettings } = await import('/src/lib/ipc/mock-data.ts'); + return [ + mockSettings.pipelineMode, + mockSettings.activeAsrProvider, + mockSettings.activeOmniProvider, + ]; + }); + const before = await providerState(); + await download.focus(); + await page.keyboard.press('Enter'); + await expect(models).toHaveAttribute('aria-pressed', 'true'); + await expect(dialog.getByRole('button', { name: '多模态模式', exact: true })).toHaveAttribute( + 'aria-pressed', + 'true', + ); + await expect( + dialog.getByText('此处准备语音唤醒模型，听写仍使用当前多模态服务。', { exact: true }), + ).toBeVisible(); + await expect( + dialog.locator('.ol-service-views').getByRole('button', { name: /^语言模型/ }), + ).toBeDisabled(); + await expect( + dialog.locator('.ol-service-views').getByRole('button', { name: /^语音识别/ }), + ).toBeDisabled(); + expect(await providerState()).toEqual(before); + await page.screenshot({ path: testInfo.outputPath('voice-activation-model-setup.png') }); + await categories.getByRole('button', { name: '录音与输入', exact: true }).click(); + await dialog.getByRole('switch', { name: '语音唤醒听写', exact: true }).click(); + await categories.getByRole('button', { name: 'AI 服务与模型', exact: true }).click(); + await expect(models).toBeDisabled(); +}); diff --git a/openless-all/app/linux-egui/examples/headless_host.rs b/openless-all/app/linux-egui/examples/headless_host.rs index 2ada52eb1..809bc08b6 100644 --- a/openless-all/app/linux-egui/examples/headless_host.rs +++ b/openless-all/app/linux-egui/examples/headless_host.rs @@ -74,6 +74,7 @@ async fn main() -> Result<(), BackendError> { )), task_spawner: Arc::new(TokioTaskSpawner), credential_store: Arc::new(InMemoryCredentialStore::default()), + voice_activation_runtime: None, services: BackendServices::unsupported(), local_asr_runtime: None, marketplace_config: None, diff --git a/openless-all/app/linux-egui/src/backend.rs b/openless-all/app/linux-egui/src/backend.rs index 84b6f73ae..aa9a28e06 100644 --- a/openless-all/app/linux-egui/src/backend.rs +++ b/openless-all/app/linux-egui/src/backend.rs @@ -298,6 +298,7 @@ impl LinuxBackendBuilder { dictation_engine, task_spawner, credential_store, + voice_activation_runtime: None, services, // Linux ships no local inference runtime. Leaving this unset // makes every local-ASR call resolve to Core's unsupported diff --git a/openless-all/app/linux-egui/src/runtime.rs b/openless-all/app/linux-egui/src/runtime.rs index 0b31c73a1..ca4fbaf20 100644 --- a/openless-all/app/linux-egui/src/runtime.rs +++ b/openless-all/app/linux-egui/src/runtime.rs @@ -257,6 +257,7 @@ mod tests { )), task_spawner: Arc::new(TokioTaskSpawner), credential_store: Arc::new(InMemoryCredentialStore::default()), + voice_activation_runtime: None, services, local_asr_runtime: None, marketplace_config: None, diff --git a/openless-all/app/linux-egui/tests/host_contract.rs b/openless-all/app/linux-egui/tests/host_contract.rs index f1c6cf384..d049227ad 100644 --- a/openless-all/app/linux-egui/tests/host_contract.rs +++ b/openless-all/app/linux-egui/tests/host_contract.rs @@ -66,6 +66,7 @@ async fn linux_host_exposes_snapshot_and_non_blocking_event_subscription() { dictation_engine: Arc::new(FixtureDictationEngine::successful("raw", "polished")), task_spawner: Arc::new(TokioTaskSpawner), credential_store: Arc::new(InMemoryCredentialStore::default()), + voice_activation_runtime: None, services: BackendServices::unsupported(), local_asr_runtime: None, marketplace_config: None, @@ -168,6 +169,7 @@ async fn forwarded_launch_intents_use_core_state_and_semantic_host_actions() { dictation_engine: Arc::new(engine.clone()), task_spawner: Arc::new(TokioTaskSpawner), credential_store: Arc::new(InMemoryCredentialStore::default()), + voice_activation_runtime: None, services: BackendServices::unsupported(), local_asr_runtime: None, marketplace_config: None, diff --git a/openless-all/app/scripts/test-recorder-module.py b/openless-all/app/scripts/test-recorder-module.py index 8e1e828de..2fac17a0e 100644 --- a/openless-all/app/scripts/test-recorder-module.py +++ b/openless-all/app/scripts/test-recorder-module.py @@ -22,6 +22,7 @@ cpal = "=0.15.3" parking_lot = "0.12" serde = { version = "1", features = ["derive"] } +serde_json = "1" thiserror = "1" log = "0.4" """) diff --git a/openless-all/app/scripts/windows-msi-release-contract.test.mjs b/openless-all/app/scripts/windows-msi-release-contract.test.mjs index d3fec672e..b926339d8 100644 --- a/openless-all/app/scripts/windows-msi-release-contract.test.mjs +++ b/openless-all/app/scripts/windows-msi-release-contract.test.mjs @@ -85,10 +85,22 @@ const publisher = steps.find((s) => s.uses?.startsWith('softprops/action-gh-rele assert.equal(publisher.with.tag_name, '${{ env.OPENLESS_RELEASE_TAG }}'); assert.equal(publisher.with.draft, true); assert.ok(steps.find((s) => s.name === 'Confirm repair target remains draft')); -const smokeTooling = steps.find((s) => s.name === 'Checkout installer smoke tooling'); -assert.equal(smokeTooling.with.ref, '${{ github.workflow_sha }}'); -assert.equal(smokeTooling.with['sparse-checkout'], 'openless-all/app/scripts/windows-ime-install-smoke.ps1'); -assert.equal(smokeTooling.with['persist-credentials'], false); +const smokeTooling = steps.find((s) => s.name === 'Fetch installer smoke tooling'); +assert.ok(smokeTooling, 'Windows smoke tooling must be fetched without checking out unrelated gitlinks'); +assert.equal(smokeTooling.if, "matrix.platform == 'windows-latest'"); +assert.equal(smokeTooling.shell, 'pwsh'); +assert.equal(smokeTooling.env.TOOLING_SHA, '${{ github.workflow_sha }}'); +assert.equal(smokeTooling.env.GH_TOKEN, '${{ github.token }}'); +assert.equal(smokeTooling.uses, undefined); +assert.match(smokeTooling.run, /\$scriptPath = 'openless-all\/app\/scripts\/windows-ime-install-smoke\.ps1'/); +assert.match(smokeTooling.run, /gh api "repos\/\$env:GITHUB_REPOSITORY\/contents\/\$scriptPath"/); +assert.match(smokeTooling.run, /--method GET -f "ref=\$env:TOOLING_SHA" --jq '\.content'/); +assert.match(smokeTooling.run, /if \(\$LASTEXITCODE -ne 0\) \{ throw/); +assert.match(smokeTooling.run, /FromBase64String\(\(\$encoded -join ''\)\)/); +assert.match(smokeTooling.run, /if \(\$bytes\.Length -eq 0\) \{ throw/); +assert.match(smokeTooling.run, /Join-Path \$env:GITHUB_WORKSPACE "release-tooling\/\$scriptPath"/); +assert.match(smokeTooling.run, /WriteAllBytes\(\$destination, \$bytes\)/); +assert.doesNotMatch(smokeTooling.run, /git checkout|git submodule|continue-on-error/); const smoke = steps.find((s) => s.name === 'Verify Windows installers register TSF IME'); assert.match(smoke.run, /-File \$smokeScript/); assert.match( diff --git a/openless-all/app/src-tauri/backend-tests/tests/core_contract.rs b/openless-all/app/src-tauri/backend-tests/tests/core_contract.rs index 148966681..222b9852c 100644 --- a/openless-all/app/src-tauri/backend-tests/tests/core_contract.rs +++ b/openless-all/app/src-tauri/backend-tests/tests/core_contract.rs @@ -28,6 +28,7 @@ async fn backend_tests_can_use_the_framework_independent_core_contract() { dictation_engine: Arc::new(FixtureDictationEngine::successful("raw", "polished")), task_spawner: Arc::new(TokioTaskSpawner), credential_store: Arc::new(InMemoryCredentialStore::default()), + voice_activation_runtime: None, services: openless_core::BackendServices::unsupported(), local_asr_runtime: None, selection_runtime: None, diff --git a/openless-all/app/src-tauri/backend-tests/tests/voice_edit_host.rs b/openless-all/app/src-tauri/backend-tests/tests/voice_edit_host.rs index ee91c9cf8..2d8d8ed5f 100644 --- a/openless-all/app/src-tauri/backend-tests/tests/voice_edit_host.rs +++ b/openless-all/app/src-tauri/backend-tests/tests/voice_edit_host.rs @@ -237,6 +237,7 @@ fn fixture_with_transcript(transcript: &str) -> (tempfile::TempDir, Arc, dictation_engine: engine.clone(), task_spawner: Arc::new(TokioTaskSpawner), credential_store: Arc::new(InMemoryCredentialStore::default()), + voice_activation_runtime: None, services: openless_core::BackendServices::unsupported(), local_asr_runtime: None, selection_runtime: None, diff --git a/openless-all/app/src-tauri/src/android/native_bridge.rs b/openless-all/app/src-tauri/src/android/native_bridge.rs index d9f86bec3..83eb45647 100644 --- a/openless-all/app/src-tauri/src/android/native_bridge.rs +++ b/openless-all/app/src-tauri/src/android/native_bridge.rs @@ -1867,6 +1867,7 @@ mod tests { credential_store: Arc::new(InMemoryCredentialStore::default()), services: BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, selection_runtime: None, selection_polisher: None, qa_runtime: None, diff --git a/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs b/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs index 3d0004956..e0db6c0e4 100644 --- a/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs +++ b/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs @@ -24,7 +24,7 @@ use parking_lot::Mutex; use crate::asr::RawTranscript; -use super::sherpa_runtime::{SherpaOnlineSession, SherpaOnnxRuntime}; +use super::sherpa_runtime::{SherpaOnlineSession, SherpaOnnxRuntime, SherpaTokenEmission}; pub struct SherpaOnnxAsr { runtime: Arc, @@ -50,11 +50,12 @@ struct OnlineWorker { enum OnlineWorkerMessage { Pcm(Vec), + FlushSegment, Finish, Cancel, } -pub type SherpaTokenHandler = Arc; +pub type SherpaTokenHandler = Arc; impl SherpaOnnxAsr { pub fn new( @@ -252,6 +253,14 @@ impl crate::recorder::AudioConsumer for SherpaOnnxAsr { } } } + + fn finish_segment(&self) { + if let SherpaProviderMode::Online { worker } = &self.mode { + if let Some(worker) = worker.lock().as_ref() { + worker.flush_segment(); + } + } + } } fn pcm_duration_ms(pcm: &[u8]) -> u64 { @@ -281,12 +290,14 @@ impl OnlineWorker { let join_handle = std::thread::Builder::new() .name(format!("openless-sherpa-online-{alias}")) .spawn(move || { - let emit = |piece: &str| { - if piece.is_empty() || worker_cancelled.load(Ordering::SeqCst) { + // [DEBUG-VOICE-1003] P2 / H1,H2,H4: worker control-message lifecycle. + crate::recorder::voice_diagnostic("P2", &["H1", "H2", "H4"], "worker", "started", serde_json::json!({})); + let emit = |emission: &SherpaTokenEmission| { + if emission.text.is_empty() || worker_cancelled.load(Ordering::SeqCst) { return; } if let Some(handler) = token_handler.as_ref() { - handler(piece.to_string()); + handler(emission.clone()); } }; let result = loop { @@ -302,7 +313,17 @@ impl OnlineWorker { break Err(anyhow::anyhow!("sherpa-onnx streaming cancelled")); } } + Ok(OnlineWorkerMessage::FlushSegment) => { + crate::recorder::voice_diagnostic("P2", &["H1", "H2", "H4"], "flush", "received", serde_json::json!({"cancelled": worker_cancelled.load(Ordering::SeqCst)})); + if worker_cancelled.load(Ordering::SeqCst) { + break Err(anyhow::anyhow!("sherpa-onnx streaming cancelled")); + } + if let Err(error) = session.flush_segment(&emit) { + break Err(error); + } + } Ok(OnlineWorkerMessage::Finish) => { + crate::recorder::voice_diagnostic("P2", &["H1", "H2", "H4"], "finish", "received", serde_json::json!({"cancelled": worker_cancelled.load(Ordering::SeqCst)})); if worker_cancelled.load(Ordering::SeqCst) { break Err(anyhow::anyhow!("sherpa-onnx streaming cancelled")); } @@ -314,6 +335,7 @@ impl OnlineWorker { } } }; + crate::recorder::voice_diagnostic("P2", &["H1", "H2", "H4"], "worker", "exiting", serde_json::json!({"ok": result.is_ok(), "cancelled": worker_cancelled.load(Ordering::SeqCst)})); let _ = result_tx.send(result); }) .expect("spawn sherpa online worker"); @@ -342,6 +364,15 @@ impl OnlineWorker { } } + fn flush_segment(&self) { + if self.cancelled.load(Ordering::SeqCst) { + return; + } + if self.tx.send(OnlineWorkerMessage::FlushSegment).is_err() { + log::warn!("[sherpa-asr] online worker is not accepting segment flush"); + } + } + async fn finish(self, audio_timeout: Duration) -> Result { let _ = self.tx.send(OnlineWorkerMessage::Finish); let result_rx = self @@ -375,11 +406,13 @@ impl OnlineWorker { } fn cancel(self) { + crate::recorder::voice_diagnostic("P2", &["H1", "H2", "H4"], "cancel", "entered", serde_json::json!({})); self.cancelled.store(true, Ordering::SeqCst); let _ = self.tx.send(OnlineWorkerMessage::Cancel); if let Some(join_handle) = self.join_handle.lock().take() { let _ = join_handle.join(); } + crate::recorder::voice_diagnostic("P2", &["H1", "H2", "H4"], "cancel", "returned", serde_json::json!({})); } } diff --git a/openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs b/openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs index 58f4eba0f..88d2e5944 100644 --- a/openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs +++ b/openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs @@ -26,6 +26,13 @@ use crate::asr::local::sherpa::{ }; const QWEN3_ASR_CHUNK_LIMIT_MS: u64 = 30_000; +#[derive(Clone, Debug)] +pub struct SherpaTokenEmission { + pub text: String, + pub is_delta: bool, + pub is_new_segment: bool, +} + const QWEN3_ASR_MAX_NEW_TOKENS: i32 = 256; #[cfg(local_asr_sherpa)] @@ -49,6 +56,9 @@ struct LoadedOfflineModel { /// Online model load state. Each dictation session creates its own `OnlineStream` from the recognizer. #[derive(Clone)] struct LoadedOnlineModel { + // [DEBUG-VOICE-1003] Observe overlap without serializing native calls. + #[cfg(local_asr_sherpa)] + diagnostic_active: Arc, alias: String, #[cfg(local_asr_sherpa)] recognizer: Arc, @@ -326,6 +336,13 @@ impl SherpaOnnxRuntime { let _lifecycle = self.lifecycle.lock().await; let mut state = self.state.lock(); state.offline_loaded = None; + if let Some(loaded) = state.online_loaded.as_ref() { + #[cfg(local_asr_sherpa)] + crate::recorder::voice_diagnostic("P5", &["H4"], "cache_owner", "releasing", + serde_json::json!({"count": Arc::strong_count(&loaded.recognizer), + "active": loaded.diagnostic_active.load(Ordering::Relaxed)}), + ); + } state.online_loaded = None; Ok(()) } @@ -344,6 +361,13 @@ impl SherpaOnnxRuntime { } let mut state = self.state.lock(); state.offline_loaded = None; + if let Some(loaded) = state.online_loaded.as_ref() { + #[cfg(local_asr_sherpa)] + crate::recorder::voice_diagnostic("P5", &["H4"], "cache_owner", "releasing", + serde_json::json!({"count": Arc::strong_count(&loaded.recognizer), + "active": loaded.diagnostic_active.load(Ordering::Relaxed)}), + ); + } state.online_loaded = None; Ok(true) } @@ -474,6 +498,7 @@ async fn load_model(alias: &str, dir: &Path) -> Result { Some(openless_core::LocalAsrExecutionMode::Online) => { let recognizer = create_online_recognizer(&alias, &dir)?; Ok(LoadedModel::Online(LoadedOnlineModel { + diagnostic_active: Arc::new(std::sync::atomic::AtomicUsize::new(0)), alias, recognizer: Arc::new(recognizer), })) @@ -641,6 +666,8 @@ async fn transcribe_loaded_model( } pub struct SherpaOnlineSession { + #[cfg(local_asr_sherpa)] + diagnostic_active: Arc, alias: String, #[cfg(local_asr_sherpa)] recognizer: Arc, @@ -649,6 +676,55 @@ pub struct SherpaOnlineSession { committed_text: String, last_partial_text: String, last_emitted_text: String, + pending_new_segment: bool, +} + +// [DEBUG-VOICE-1003] P3 / H2,H4,H6: native entry/return and per-recognizer overlap. +#[cfg(local_asr_sherpa)] +struct VoiceNativeProbe<'a> { + active: &'a std::sync::atomic::AtomicUsize, + stage: &'static str, +} + +#[cfg(local_asr_sherpa)] +impl<'a> VoiceNativeProbe<'a> { + fn enter(active: &'a std::sync::atomic::AtomicUsize, stage: &'static str) -> Self { + let count = active.fetch_add(1, Ordering::Relaxed) + 1; + crate::recorder::voice_diagnostic( + "P3", + &["H2", "H4", "H6"], + "native", + "entered", + serde_json::json!({"stage": stage, "count": count}), + ); + Self { active, stage } + } +} + +#[cfg(local_asr_sherpa)] +impl Drop for VoiceNativeProbe<'_> { + fn drop(&mut self) { + let count = self.active.fetch_sub(1, Ordering::Relaxed) - 1; + crate::recorder::voice_diagnostic( + "P3", + &["H2", "H4", "H6"], + "native", + "returned", + serde_json::json!({"stage": self.stage, "count": count}), + ); + } +} + +// [DEBUG-VOICE-1003] P5 / H4: observe ownership before existing field destruction. +// No fields are moved, reordered or manually dropped here. +#[cfg(local_asr_sherpa)] +impl Drop for SherpaOnlineSession { + fn drop(&mut self) { + crate::recorder::voice_diagnostic("P5", &["H4"], "session_owner", "dropping", + serde_json::json!({"count": Arc::strong_count(&self.recognizer), + "active": self.diagnostic_active.load(Ordering::Relaxed)}), + ); + } } impl SherpaOnlineSession { @@ -658,7 +734,7 @@ impl SherpaOnlineSession { pub fn accept_pcm_chunk(&mut self, pcm: &[u8], on_token: &F) -> Result<()> where - F: Fn(&str), + F: Fn(&SherpaTokenEmission), { if pcm.is_empty() { return Ok(()); @@ -669,17 +745,29 @@ impl SherpaOnlineSession { pub fn finish(&mut self, on_token: &F) -> Result where - F: Fn(&str), + F: Fn(&SherpaTokenEmission), { self.finish_inner(on_token) } + /// Flush the current VAD segment while keeping this online session alive for + /// the next speech segment. + pub fn flush_segment(&mut self, on_token: &F) -> Result<()> + where + F: Fn(&SherpaTokenEmission), + { + self.flush_segment_inner(on_token) + } + #[cfg(local_asr_sherpa)] fn accept_samples(&mut self, samples: &[f32], on_token: &F) -> Result<()> where - F: Fn(&str), + F: Fn(&SherpaTokenEmission), { - self.stream.accept_waveform(16_000, samples); + { + let _probe = VoiceNativeProbe::enter(&self.diagnostic_active, "accept"); + self.stream.accept_waveform(16_000, samples); + } self.drain_ready(on_token); Ok(()) } @@ -687,7 +775,7 @@ impl SherpaOnlineSession { #[cfg(not(local_asr_sherpa))] fn accept_samples(&mut self, _samples: &[f32], _on_token: &F) -> Result<()> where - F: Fn(&str), + F: Fn(&SherpaTokenEmission), { Ok(()) } @@ -695,9 +783,18 @@ impl SherpaOnlineSession { #[cfg(local_asr_sherpa)] fn finish_inner(&mut self, on_token: &F) -> Result where - F: Fn(&str), + F: Fn(&SherpaTokenEmission), { - self.stream.input_finished(); + finish_online_segment_input( + |tail| { + let _probe = VoiceNativeProbe::enter(&self.diagnostic_active, "tail_padding"); + self.stream.accept_waveform(16_000, tail); + }, + || { + let _probe = VoiceNativeProbe::enter(&self.diagnostic_active, "input_finished"); + self.stream.input_finished(); + }, + ); self.drain_ready(on_token); self.capture_result(on_token, false); Ok(self.final_text()) @@ -706,28 +803,93 @@ impl SherpaOnlineSession { #[cfg(not(local_asr_sherpa))] fn finish_inner(&mut self, _on_token: &F) -> Result where - F: Fn(&str), + F: Fn(&SherpaTokenEmission), { Ok(self.final_text()) } + #[cfg(local_asr_sherpa)] + fn flush_segment_inner(&mut self, on_token: &F) -> Result<()> + where + F: Fn(&SherpaTokenEmission), + { + finish_online_segment_input( + |tail| { + let _probe = VoiceNativeProbe::enter(&self.diagnostic_active, "tail_padding"); + self.stream.accept_waveform(16_000, tail); + }, + || { + let _probe = VoiceNativeProbe::enter(&self.diagnostic_active, "input_finished"); + self.stream.input_finished(); + }, + ); + self.drain_ready_without_endpoint_reset(on_token); + self.capture_result(on_token, false); + if !self.last_partial_text.is_empty() { + append_segment(&mut self.committed_text, &self.last_partial_text); + } + self.last_partial_text.clear(); + self.last_emitted_text.clear(); + self.pending_new_segment = true; + { + // EOF also closes feature extraction; recognizer reset only resets decoding. + let _probe = VoiceNativeProbe::enter(&self.diagnostic_active, "new_stream"); + self.stream = self.recognizer.create_stream(); + } + Ok(()) + } + + #[cfg(not(local_asr_sherpa))] + fn flush_segment_inner(&mut self, _on_token: &F) -> Result<()> + where + F: Fn(&SherpaTokenEmission), + { + Ok(()) + } + #[cfg(local_asr_sherpa)] fn drain_ready(&mut self, on_token: &F) where - F: Fn(&str), + F: Fn(&SherpaTokenEmission), { while self.recognizer.is_ready(&self.stream) { - self.recognizer.decode(&self.stream); + { + let _probe = VoiceNativeProbe::enter(&self.diagnostic_active, "decode"); + self.recognizer.decode(&self.stream); + } self.capture_result(on_token, true); } } + #[cfg(local_asr_sherpa)] + fn drain_ready_without_endpoint_reset(&mut self, on_token: &F) + where + F: Fn(&SherpaTokenEmission), + { + while self.recognizer.is_ready(&self.stream) { + { + let _probe = VoiceNativeProbe::enter(&self.diagnostic_active, "decode"); + self.recognizer.decode(&self.stream); + } + self.capture_result(on_token, false); + } + } + #[cfg(local_asr_sherpa)] fn capture_result(&mut self, on_token: &F, allow_endpoint_reset: bool) where - F: Fn(&str), + F: Fn(&SherpaTokenEmission), { - let Some(result) = self.recognizer.get_result(&self.stream) else { + let result = { + let _probe = VoiceNativeProbe::enter(&self.diagnostic_active, "result"); + self.recognizer.get_result(&self.stream) + }; + crate::recorder::voice_diagnostic("P3", &["H2", "H4", "H6"], "native_result", "observed", + serde_json::json!({"present": result.is_some(), + "count": result.as_ref().map(|result| result.text.chars().count()).unwrap_or(0), + "stage": if allow_endpoint_reset { "streaming" } else { "flush" }}), + ); + let Some(result) = result else { return; }; let text = result.text.trim().to_string(); @@ -740,24 +902,38 @@ impl SherpaOnlineSession { append_segment(&mut self.committed_text, &text); self.last_partial_text.clear(); self.last_emitted_text.clear(); - self.recognizer.reset(&self.stream); + self.pending_new_segment = true; + { + let _probe = VoiceNativeProbe::enter(&self.diagnostic_active, "reset"); + self.recognizer.reset(&self.stream); + } } } fn emit_delta(&mut self, text: &str, on_token: &F) where - F: Fn(&str), + F: Fn(&SherpaTokenEmission), { if text == self.last_emitted_text { return; } + let is_new_segment = self.pending_new_segment; if let Some(delta) = text.strip_prefix(&self.last_emitted_text) { if !delta.is_empty() { - on_token(delta); + on_token(&SherpaTokenEmission { + text: delta.to_string(), + is_delta: true, + is_new_segment, + }); } } else { - on_token(text); + on_token(&SherpaTokenEmission { + text: text.to_string(), + is_delta: false, + is_new_segment, + }); } + self.pending_new_segment = false; self.last_emitted_text = text.to_string(); } @@ -770,6 +946,15 @@ impl SherpaOnlineSession { } } +// Sherpa 1.13.2's online-decode-files.py supplies 0.66 s of zero samples before +// input_finished(): marking EOF alone does not provide the model's trailing context. +// This is bounded model input, not a wall-clock wait or continuous idle inference. +fn finish_online_segment_input(accept_tail: impl FnOnce(&[f32]), input_finished: impl FnOnce()) { + const TAIL: [f32; 16_000 * 66 / 100] = [0.0; 16_000 * 66 / 100]; + accept_tail(&TAIL); + input_finished(); +} + fn append_segment(text: &mut String, segment: &str) { let segment = segment.trim(); if segment.is_empty() { @@ -784,13 +969,22 @@ fn append_segment(text: &mut String, segment: &str) { #[cfg(local_asr_sherpa)] fn create_online_session_from_loaded(loaded: LoadedOnlineModel) -> Result { let stream = loaded.recognizer.create_stream(); + crate::recorder::voice_diagnostic( + "P5", + &["H4"], + "session_owner", + "created", + serde_json::json!({"count": Arc::strong_count(&loaded.recognizer)}), + ); Ok(SherpaOnlineSession { + diagnostic_active: loaded.diagnostic_active, alias: loaded.alias, recognizer: loaded.recognizer, stream, committed_text: String::new(), last_partial_text: String::new(), last_emitted_text: String::new(), + pending_new_segment: false, }) } @@ -801,6 +995,7 @@ fn create_online_session_from_loaded(loaded: LoadedOnlineModel) -> Result Result<(), String> { let warm_hub = crate::recorder::warm_capture_hub(); + if !warm_hub.begin_preview() { + return Err("dictation is using the microphone".into()); + } + if let Some(backend) = app.try_state::>() { + if let Err(error) = backend + .stop_voice_activation(openless_core::VoiceActivationStopReason::Disabled) + .await + { + warm_hub.end_preview(); + return Err(error.to_string()); + } + } let warm_hub_for_error = Arc::clone(&warm_hub); let app_for_error = app.clone(); + let backend_for_resume = app + .try_state::>() + .map(|state| Arc::clone(&*state)); let result = match tauri::async_runtime::spawn_blocking(move || { let state = app.state::(); let _operation = state.operation.lock(); - if !warm_hub.begin_preview() { - return Err("dictation is using the microphone".into()); - } if let Some(existing) = state.recorder.lock().take() { existing.stop(); } @@ -176,9 +188,14 @@ pub async fn start_microphone_level_monitor( if state.recorder.lock().is_none() { warm_hub_for_error.end_preview(); } - return Err(format!("start microphone monitor task failed: {error}")); + Err(format!("start microphone monitor task failed: {error}")) } }; + if result.is_err() { + if let Some(backend) = backend_for_resume { + let _ = backend.configure_voice_activation().await; + } + } result } @@ -196,6 +213,10 @@ pub async fn stop_microphone_level_monitor(app: AppHandle) { #[cfg(not(mobile))] let app_for_error = app.clone(); #[cfg(not(mobile))] + let backend_for_resume = app + .try_state::>() + .map(|state| Arc::clone(&*state)); + #[cfg(not(mobile))] let result = tauri::async_runtime::spawn_blocking(move || { let state = app.state::(); let _operation = state.operation.lock(); @@ -216,6 +237,10 @@ pub async fn stop_microphone_level_monitor(app: AppHandle) { } #[cfg(not(mobile))] let _ = result; + #[cfg(not(mobile))] + if let Some(backend) = backend_for_resume { + let _ = backend.configure_voice_activation().await; + } } /// Copy the current session's openless.log to a user-chosen location (the frontend obtains diff --git a/openless-all/app/src-tauri/src/commands/mod.rs b/openless-all/app/src-tauri/src/commands/mod.rs index c73831c8c..6c060e01f 100644 --- a/openless-all/app/src-tauri/src/commands/mod.rs +++ b/openless-all/app/src-tauri/src/commands/mod.rs @@ -113,6 +113,7 @@ pub use selection_polish_preview::*; #[cfg(all(not(mobile), target_os = "windows"))] pub use selection_voice::*; pub use settings::*; +pub(crate) use settings::sync_low_latency_capture_preferences; #[cfg(any(not(mobile), local_asr_sherpa))] // Sherpa commands are registered on Windows and arm64 Android; other desktop // targets keep an empty re-export. diff --git a/openless-all/app/src-tauri/src/commands/settings.rs b/openless-all/app/src-tauri/src/commands/settings.rs index 76949221f..e3c989a03 100644 --- a/openless-all/app/src-tauri/src/commands/settings.rs +++ b/openless-all/app/src-tauri/src/commands/settings.rs @@ -219,9 +219,14 @@ pub(crate) fn persist_strict_settings( } #[cfg(target_os = "windows")] -fn sync_low_latency_capture_preferences(prefs: &UserPreferences) { - crate::recorder::warm_capture_hub().configure( - prefs.low_latency_dictation_enabled && !crate::desktop_resources::active(), +pub(crate) fn sync_low_latency_capture_preferences(prefs: &UserPreferences) { + let warm_hub = crate::recorder::warm_capture_hub(); + warm_hub.configure( + crate::recorder::warm_capture_enabled( + prefs.low_latency_dictation_enabled, + crate::desktop_resources::active(), + warm_hub.activation_is_active(), + ), (!prefs.microphone_device_name.trim().is_empty()) .then(|| prefs.microphone_device_name.clone()), openless_core::AudioProcessingConfig { @@ -231,7 +236,38 @@ fn sync_low_latency_capture_preferences(prefs: &UserPreferences) { } #[cfg(not(target_os = "windows"))] -fn sync_low_latency_capture_preferences(_prefs: &UserPreferences) {} +pub(crate) fn sync_low_latency_capture_preferences(_prefs: &UserPreferences) {} + +fn voice_activation_preferences_changed(previous: &UserPreferences, next: &UserPreferences) -> bool { + previous.voice_activation_enabled != next.voice_activation_enabled + || previous.voice_activation_wake_enabled != next.voice_activation_wake_enabled + || previous.voice_activation_finish_enabled != next.voice_activation_finish_enabled + || previous.voice_activation_wake_phrase != next.voice_activation_wake_phrase + || previous.voice_activation_finish_phrase != next.voice_activation_finish_phrase + || previous.voice_activation_timeout_enabled != next.voice_activation_timeout_enabled + || previous.voice_activation_timeout_seconds != next.voice_activation_timeout_seconds + || previous.microphone_device_name != next.microphone_device_name +} + +fn validate_voice_activation_preferences( + prefs: &UserPreferences, +) -> Result<(), String> { + openless_core::VoiceActivationConfig { + enabled: prefs.voice_activation_enabled, + wake_enabled: prefs.voice_activation_wake_enabled, + finish_enabled: prefs.voice_activation_finish_enabled, + wake_phrase: prefs.voice_activation_wake_phrase.clone(), + finish_phrase: prefs.voice_activation_finish_phrase.clone(), + timeout_enabled: prefs.voice_activation_timeout_enabled, + timeout_seconds: prefs.voice_activation_timeout_seconds, + microphone_device_name: (!prefs.microphone_device_name.trim().is_empty()) + .then(|| prefs.microphone_device_name.clone()), + activation_generation: 0, + } + .normalized() + .map(|_| ()) + .map_err(|error| format!("invalid voice activation settings: {error}")) +} async fn invalidate_llm_tests_if_thinking_changed( coord: &Coordinator, @@ -269,6 +305,7 @@ pub async fn set_settings( .map_err(|e| e.to_string())?; sync_style_pack_preferences(&mut prefs, &packs); prefs.android_overlay_trigger = prefs.android_overlay_trigger.normalized(); + validate_voice_activation_preferences(&prefs)?; invalidate_llm_tests_if_thinking_changed(&coord, &remote_prev, &prefs).await?; // Persist changed fields and notify every WebView through the shared settings path. if let Some(edits) = edits { @@ -278,6 +315,11 @@ pub async fn set_settings( } let prefs = coord.backend().get_preferences(); sync_low_latency_capture_preferences(&prefs); + if voice_activation_preferences_changed(&remote_prev, &prefs) { + if let Err(error) = coord.backend().configure_voice_activation().await { + log::warn!("[voice-activation] settings reconfiguration failed: {error}"); + } + } // Sync the capsule-style atom on save: the next recording's entrance frame carries the // new style instead of depending on emit_capsule's ~30Hz main-thread closure sync (a // congested Windows main thread delays the closure → the whole session shows the old @@ -345,6 +387,7 @@ pub async fn set_settings( .map_err(|e| e.to_string())?; sync_style_pack_preferences(&mut prefs, &packs); prefs.android_overlay_trigger = prefs.android_overlay_trigger.normalized(); + validate_voice_activation_preferences(&prefs)?; invalidate_llm_tests_if_thinking_changed(&coord, &previous, &prefs).await?; if let Some(edits) = edits { persist_setting_fields(&coord, &edits)?; @@ -353,8 +396,12 @@ pub async fn set_settings( } let prefs = coord.backend().get_preferences(); sync_low_latency_capture_preferences(&prefs); - // Sync the capsule-style atom on save (same source as the Android notification capsule - // payload; see emit_capsule). + if voice_activation_preferences_changed(&previous, &prefs) { + if let Err(error) = coord.backend().configure_voice_activation().await { + log::warn!("[voice-activation] settings reconfiguration failed: {error}"); + } + } + // Sync the capsule-style atom on save (same source as the Android notification capsule payload; see emit_capsule). coord.sync_capsule_style_from_preferences(); // Rebuild the client connection pool immediately when the system-proxy switch changes (issue #869). if previous.use_system_proxy != prefs.use_system_proxy { diff --git a/openless-all/app/src-tauri/src/commands/sherpa_asr.rs b/openless-all/app/src-tauri/src/commands/sherpa_asr.rs index 877cb9cc2..a3f0bc220 100644 --- a/openless-all/app/src-tauri/src/commands/sherpa_asr.rs +++ b/openless-all/app/src-tauri/src/commands/sherpa_asr.rs @@ -208,12 +208,24 @@ pub async fn sherpa_onnx_asr_download_model( model_alias: String, mirror: Option, ) -> Result<(), String> { + let target = sherpa_target(model_alias)?; + #[cfg(target_os = "windows")] + let voice_activation_model = target.model_id() == "zipformer-bilingual-zh-en-streaming"; backend .services() .local_asr - .start_download(sherpa_target(model_alias)?, parse_mirror(mirror)) + .start_download(target, parse_mirror(mirror)) .await - .map_err(core_error) + .map_err(core_error)?; + #[cfg(target_os = "windows")] + if voice_activation_model && backend.get_preferences().voice_activation_enabled { + if let Err(error) = backend.configure_voice_activation().await { + log::warn!( + "[voice-activation] Sherpa model download completed but runtime refresh failed: {error}" + ); + } + } + Ok(()) } #[tauri::command] diff --git a/openless-all/app/src-tauri/src/core_adapters.rs b/openless-all/app/src-tauri/src/core_adapters.rs index a667f1d42..98af79c56 100644 --- a/openless-all/app/src-tauri/src/core_adapters.rs +++ b/openless-all/app/src-tauri/src/core_adapters.rs @@ -15,12 +15,16 @@ use openless_core::{ RecordingArchive, RecordingProgressSink, SessionId, TextInserter as CoreTextInserter, TextInsertionSession, TextPolisher, TextStreamChunk, TextStreamSink, TranscriptOutput, TranscriptionEngine, TranscriptionSession, + VoiceActivationCommand, VoiceActivationConfig, VoiceActivationError, VoiceActivationPhase, + VoiceActivationSignal, VoiceActivationStopReason, VoiceCommandRecognizer, + VoiceDictationHandoff, VoiceDictationHandoffRequest, }; use parking_lot::Mutex; use tauri::{AppHandle, Emitter, Manager}; use crate::recorder::{ - warm_capture_hub, AudioConsumer as LegacyAudioConsumer, Recorder, RecorderError, + warm_capture_hub, AudioConsumer as LegacyAudioConsumer, AudioConsumerFanout, Recorder, + RecorderError, StagedRecorder, WarmActiveCapture, WarmCaptureHub, WarmStagedRecorder, STAGED_CAPTURE_MAX_BYTES, }; @@ -125,12 +129,23 @@ pub(crate) fn backend_dependencies( ); let warm_hub = warm_capture_hub(); #[cfg(target_os = "windows")] + let voice_activation_runtime = Arc::new(TauriVoiceActivationRuntime::new( + Arc::clone(&app), + Arc::clone(&backend), + Arc::clone(&native_asr_dependencies.sherpa), + model_store.clone(), + Arc::clone(&warm_hub), + Arc::clone(&preferences), + )); + #[cfg(target_os = "windows")] { let prefs = preferences.get(); warm_hub.configure( - prefs.low_latency_dictation_enabled - && !prefs.low_memory_background_enabled - && !crate::desktop_resources::active(), + crate::recorder::warm_capture_enabled( + prefs.low_latency_dictation_enabled, + prefs.low_memory_background_enabled || crate::desktop_resources::active(), + false, + ), (!prefs.microphone_device_name.trim().is_empty()) .then(|| prefs.microphone_device_name.clone()), openless_core::AudioProcessingConfig { @@ -287,9 +302,14 @@ pub(crate) fn backend_dependencies( .expect("built-in Omni provider ids are non-empty"); } let mut dependencies = openless_core::BackendDependencies::unsupported(); - if let Some(model_store) = model_store { + if let Some(model_store) = model_store.clone() { dependencies.services.configure_model_store(model_store); } + #[cfg(target_os = "windows")] + { + dependencies.voice_activation_runtime = + Some(voice_activation_runtime as Arc); + } dependencies .services .configure_auxiliary_runtime(auxiliary_polisher, auxiliary_transcription); @@ -1678,6 +1698,580 @@ impl openless_core::PlatformApi for TauriPlatformApi { } } +#[cfg(target_os = "windows")] +const VOICE_ACTIVATION_MODEL_ALIAS: &str = "zipformer-bilingual-zh-en-streaming"; + +#[cfg(target_os = "windows")] +const MAX_VOICE_RECOGNIZER_CONTEXT_CHARS: usize = 256; + +#[cfg(target_os = "windows")] +fn map_voice_activation_recorder_error(error: RecorderError) -> VoiceActivationError { + match error { + RecorderError::PermissionDenied => VoiceActivationError::PermissionDenied, + RecorderError::NoInputDevice => { + VoiceActivationError::Runtime("no microphone input device detected".into()) + } + RecorderError::EngineFailed(message) + if message.to_ascii_lowercase().contains("busy") => VoiceActivationError::MicBusy, + RecorderError::EngineFailed(message) => VoiceActivationError::Runtime(message), + } +} + +#[cfg(target_os = "windows")] +struct VoiceActivationEmitter { + backend: BackendSlot, + sink: Arc, + generation: u64, + command: VoiceActivationCommand, + session_id: Arc>>, + binding_gate: Mutex<()>, + state: Mutex, +} + +#[cfg(target_os = "windows")] +struct VoiceActivationEmitterState { + recognizer: VoiceCommandRecognizer, + text: String, + segment_text: String, + pending_finish: bool, + pending_finish_match: bool, +} + +#[cfg(target_os = "windows")] +impl VoiceActivationEmitter { + fn new( + backend: BackendSlot, + sink: Arc, + generation: u64, + command: VoiceActivationCommand, + config: &VoiceActivationConfig, + session_id: Arc>>, + ) -> Result { + let recognizer = VoiceCommandRecognizer::new(config) + .map_err(|error| VoiceActivationError::InvalidConfig(error.to_string()))?; + Ok(Self { + backend, + sink, + generation, + command, + session_id, + binding_gate: Mutex::new(()), + state: Mutex::new(VoiceActivationEmitterState { + recognizer, + text: String::new(), + segment_text: String::new(), + pending_finish: false, + pending_finish_match: false, + }), + }) + } + + fn observe(&self, piece: String, is_delta: bool, is_new_segment: bool) { + if piece.trim().is_empty() { + return; + } + let _binding = self.binding_gate.lock(); + let bound_session_id = if self.command == VoiceActivationCommand::Finish { + *self.session_id.lock() + } else { + None + }; + let signal = { + let mut state = self.state.lock(); + if is_new_segment { + state.text.push_str(&piece); + state.segment_text = piece; + } else if is_delta { + state.text.push_str(&piece); + state.segment_text.push_str(&piece); + } else { + let prefix_len = state + .text + .chars() + .count() + .saturating_sub(state.segment_text.chars().count()); + state.text = state.text.chars().take(prefix_len).collect(); + state.text.push_str(&piece); + state.segment_text = piece; + } + if state.text.chars().count() > MAX_VOICE_RECOGNIZER_CONTEXT_CHARS { + state.text = state + .text + .chars() + .rev() + .take(MAX_VOICE_RECOGNIZER_CONTEXT_CHARS) + .collect::() + .chars() + .rev() + .collect(); + state.segment_text = state.text.clone(); + state.recognizer.reset(); + } + let text = state.text.clone(); + let matched = state.recognizer.observe(&text, self.command); + // [DEBUG-VOICE-1003] P4 / H2,H3: no transcript content or phrases. + crate::recorder::voice_diagnostic("P4", &["H2", "H3"], "command_match", "observed", + serde_json::json!({"matched": matched.is_some(), "bound": bound_session_id.is_some(), + "mode": if self.command == VoiceActivationCommand::Wake { "wake" } else { "finish" }, + "count": text.chars().count(), "new_segment": is_new_segment, "delta": is_delta})); + if self.command == VoiceActivationCommand::Finish && bound_session_id.is_none() { + state.pending_finish = true; + state.pending_finish_match |= matched.is_some(); + return; + } + let Some(_) = matched else { + state.pending_finish = false; + return; + }; + state.pending_finish = false; + state.pending_finish_match = false; + match self.command { + VoiceActivationCommand::Wake => { + VoiceActivationSignal::WakeDetected { + activation_generation: self.generation, + } + } + VoiceActivationCommand::Finish => VoiceActivationSignal::FinishDetected { + activation_generation: self.generation, + session_id: bound_session_id.expect("finish session was checked above"), + }, + } + }; + let _ = self.sink.publish(signal.clone()); + if let Some(backend) = self.backend.lock().as_ref().and_then(std::sync::Weak::upgrade) { + // [DEBUG-VOICE-1003] P6 / H3: dispatch guard facts, without identifiers. + crate::recorder::voice_diagnostic("P6", &["H3"], "command_dispatch", "entered", + serde_json::json!({"current": backend.voice_activation_generation_is_current(self.generation), + "present": backend.snapshot().dictation.session_id.is_some(), + "mode": if self.command == VoiceActivationCommand::Wake { "wake" } else { "finish" }})); + backend.handle_voice_activation_signal(signal); + crate::recorder::voice_diagnostic("P6", &["H3"], "command_dispatch", "returned", serde_json::json!({})); + } + } + + fn bind_session(&self, session_id: SessionId) { + { + let _binding = self.binding_gate.lock(); + *self.session_id.lock() = Some(session_id); + } + self.retry_pending(); + } + + fn retry_pending(&self) { + let (text, pending_match) = { + let _binding = self.binding_gate.lock(); + let mut state = self.state.lock(); + if !state.pending_finish { + return; + } + let pending_match = state.pending_finish_match; + state.pending_finish = false; + state.pending_finish_match = false; + (state.text.clone(), pending_match) + }; + if pending_match { + let Some(session_id) = *self.session_id.lock() else { + return; + }; + let signal = VoiceActivationSignal::FinishDetected { + activation_generation: self.generation, + session_id, + }; + let _ = self.sink.publish(signal.clone()); + if let Some(backend) = self.backend.lock().as_ref().and_then(std::sync::Weak::upgrade) { + // [DEBUG-VOICE-1003] P6 / H3: dispatch guard facts, without identifiers. + crate::recorder::voice_diagnostic("P6", &["H3"], "command_dispatch", "entered", + serde_json::json!({"current": backend.voice_activation_generation_is_current(self.generation), + "present": backend.snapshot().dictation.session_id.is_some(), + "mode": if self.command == VoiceActivationCommand::Wake { "wake" } else { "finish" }})); + backend.handle_voice_activation_signal(signal); + crate::recorder::voice_diagnostic("P6", &["H3"], "command_dispatch", "returned", serde_json::json!({})); + } + } else if !text.is_empty() { + self.observe(text, false, false); + } + } +} + +#[cfg(target_os = "windows")] +struct TauriVoiceActivationRuntime { + app: AppHandleSlot, + backend: BackendSlot, + sherpa: Arc, + model_store: Option>, + warm_hub: Arc, + preferences: Arc, +} + +#[cfg(target_os = "windows")] +impl TauriVoiceActivationRuntime { + fn new( + app: AppHandleSlot, + backend: BackendSlot, + sherpa: Arc, + model_store: Option>, + warm_hub: Arc, + preferences: Arc, + ) -> Self { + Self { + app, + backend, + sherpa, + model_store, + warm_hub, + preferences, + } + } +} + +#[cfg(target_os = "windows")] +impl openless_core::VoiceActivationRuntime for TauriVoiceActivationRuntime { + fn configure( + &self, + config: VoiceActivationConfig, + sink: Arc, + ) -> BoxFuture<'static, Result, VoiceActivationError>> { + let app = Arc::clone(&self.app); + let backend = Arc::clone(&self.backend); + let sherpa = Arc::clone(&self.sherpa); + let model_store = self.model_store.clone(); + let warm_hub = Arc::clone(&self.warm_hub); + let preferences = Arc::clone(&self.preferences); + Box::pin(async move { + let config = config + .normalized() + .map_err(|error| VoiceActivationError::InvalidConfig(error.to_string()))?; + let target = openless_core::LocalAsrTarget::parse( + openless_core::LocalAsrRuntime::SherpaOnnx, + VOICE_ACTIVATION_MODEL_ALIAS, + ) + .map_err(|error| VoiceActivationError::InvalidConfig(error.to_string()))?; + let Some(model_store) = model_store else { + return Err(VoiceActivationError::ModelMissing); + }; + if !model_store + .is_installed(&target) + .map_err(|error| VoiceActivationError::Runtime(error.to_string()))? + { + return Err(VoiceActivationError::ModelMissing); + } + let model_dir = model_store + .runtime_model_dir(&target) + .map_err(|error| VoiceActivationError::Runtime(error.to_string()))?; + let session_id = Arc::new(Mutex::new(None)); + let emitter = Arc::new(VoiceActivationEmitter::new( + Arc::clone(&backend), + Arc::clone(&sink), + config.activation_generation, + VoiceActivationCommand::Wake, + &config, + Arc::clone(&session_id), + )?); + let wake_asr = if config.wake_enabled { + let emitter_for_handler = Arc::clone(&emitter); + let token_handler = Arc::new( + move |emission: crate::asr::local::sherpa_runtime::SherpaTokenEmission| { + emitter_for_handler.observe( + emission.text, + emission.is_delta, + emission.is_new_segment, + ) + }, + ); + Some(Arc::new( + crate::asr::local::SherpaOnnxAsr::new_for_model( + Arc::clone(&sherpa), + VOICE_ACTIVATION_MODEL_ALIAS.to_string(), + model_dir.clone(), + Some("zh".into()), + Some(token_handler), + ) + .await + .map_err(|error| VoiceActivationError::Runtime(error.to_string()))?, + )) + } else { + None + }; + let preferences_snapshot = preferences.get(); + let fallback_warm_capture_enabled = crate::recorder::warm_capture_enabled( + preferences_snapshot.low_latency_dictation_enabled, + crate::desktop_resources::active(), + false, + ); + let audio_processing = openless_core::AudioProcessingConfig { + enhancement_enabled: preferences_snapshot.audio_enhancement_enabled, + }; + if let Some(wake_asr) = wake_asr.as_ref() { + warm_hub.set_activation_generation(config.activation_generation); + if !warm_hub.set_activation_consumer_for_generation( + config.activation_generation, + Some(Arc::clone(wake_asr) as Arc), + false, + ) { + return Err(VoiceActivationError::Runtime( + "voice activation generation became stale".into(), + )); + } + let fault_backend = Arc::clone(&backend); + let fault_sink = Arc::clone(&sink); + let fault_generation = config.activation_generation; + warm_hub.set_activation_fault_handler( + fault_generation, + Some(Arc::new(move |error| { + log::warn!("[voice-activation] warm microphone restart failed: {error}"); + let signal = VoiceActivationSignal::RuntimeStateChanged { + activation_generation: fault_generation, + phase: VoiceActivationPhase::Failed, + }; + let _ = fault_sink.publish(signal.clone()); + if let Some(backend) = fault_backend + .lock() + .as_ref() + .and_then(std::sync::Weak::upgrade) + { + backend.handle_voice_activation_signal(signal); + } + })), + ); + } + warm_hub.configure( + wake_asr.is_some() || fallback_warm_capture_enabled, + config.microphone_device_name.clone(), + audio_processing, + ); + if wake_asr.is_some() && !warm_hub.is_cold_capture_active() { + let probe = match warm_hub.arm_without_preroll( + config.activation_generation, + config.microphone_device_name.clone(), + ) { + Ok(probe) => probe, + Err(error) => { + warm_hub.clear_activation_fault_handler_if(config.activation_generation); + warm_hub.clear_activation_generation_if(config.activation_generation); + warm_hub.set_activation_consumer(None); + warm_hub.configure( + fallback_warm_capture_enabled, + config.microphone_device_name.clone(), + audio_processing, + ); + return Err(map_voice_activation_recorder_error(error)); + } + }; + drop(probe); + } + let handle = TauriVoiceActivationHandle { + app, + backend, + sherpa, + model_dir, + model_alias: VOICE_ACTIVATION_MODEL_ALIAS.to_string(), + warm_hub, + session_id, + sink: Arc::clone(&sink), + preferences, + config: config.clone(), + wake_asr: Arc::new(Mutex::new(wake_asr)), + finish_asr: Arc::new(Mutex::new(None)), + finish_emitter: Arc::new(Mutex::new(None)), + }; + let _ = sink.publish(VoiceActivationSignal::RuntimeStateChanged { + activation_generation: config.activation_generation, + phase: VoiceActivationPhase::ListeningForWake, + }); + Ok(Box::new(handle) as Box) + }) + } +} + +#[cfg(target_os = "windows")] +struct TauriVoiceActivationHandle { + app: AppHandleSlot, + backend: BackendSlot, + sherpa: Arc, + model_dir: PathBuf, + model_alias: String, + warm_hub: Arc, + session_id: Arc>>, + sink: Arc, + preferences: Arc, + config: VoiceActivationConfig, + wake_asr: Arc>>>, + finish_asr: Arc>>>, + finish_emitter: Arc>>>, +} + +#[cfg(target_os = "windows")] +impl openless_core::VoiceActivationHandle for TauriVoiceActivationHandle { + fn bind_dictation_session(&self, session_id: SessionId) { + if let Some(emitter) = self.finish_emitter.lock().clone() { + emitter.bind_session(session_id); + } else { + *self.session_id.lock() = Some(session_id); + } + } + + fn handoff_to_dictation( + &self, + request: VoiceDictationHandoffRequest, + ) -> BoxFuture<'static, Result> { + let app = Arc::clone(&self.app); + let backend = Arc::clone(&self.backend); + let backend_for_generation = Arc::clone(&backend); + let requested_generation = request.activation_generation; + let sherpa = Arc::clone(&self.sherpa); + let model_dir = self.model_dir.clone(); + let model_alias = self.model_alias.clone(); + let warm_hub = Arc::clone(&self.warm_hub); + let session_id = Arc::clone(&self.session_id); + let sink = Arc::clone(&self.sink); + let config = self.config.clone(); + let wake_asr = Arc::clone(&self.wake_asr); + let finish_asr = Arc::clone(&self.finish_asr); + let finish_emitter_slot = Arc::clone(&self.finish_emitter); + Box::pin(async move { + let generation_is_current = || { + backend_for_generation + .lock() + .as_ref() + .and_then(std::sync::Weak::upgrade) + .is_some_and(|backend| { + backend.voice_activation_generation_is_current(requested_generation) + }) + }; + if !generation_is_current() { + return Err(VoiceActivationError::Runtime( + "voice activation generation is stale".into(), + )); + } + let staged = warm_hub + .arm_with_preroll( + config.activation_generation, + config.microphone_device_name.clone(), + ) + .map_err(map_voice_activation_recorder_error)?; + if !generation_is_current() { + drop(staged); + return Err(VoiceActivationError::Runtime( + "voice activation generation is stale".into(), + )); + } + if !warm_hub.set_activation_consumer_for_generation( + requested_generation, + None, + false, + ) { + drop(staged); + return Err(VoiceActivationError::Runtime( + "voice activation generation is stale".into(), + )); + } + if let Some(wake_asr) = wake_asr.lock().take() { + wake_asr.cancel(); + } + if config.finish_enabled { + let emitter = Arc::new(VoiceActivationEmitter::new( + Arc::clone(&backend), + sink, + config.activation_generation, + VoiceActivationCommand::Finish, + &config, + Arc::clone(&session_id), + )?); + let emitter_for_handler = Arc::clone(&emitter); + let token_handler = Arc::new( + move |emission: crate::asr::local::sherpa_runtime::SherpaTokenEmission| { + emitter_for_handler.observe( + emission.text, + emission.is_delta, + emission.is_new_segment, + ) + }, + ); + let asr = Arc::new( + crate::asr::local::SherpaOnnxAsr::new_for_model( + Arc::clone(&sherpa), + model_alias, + model_dir, + Some("zh".into()), + Some(token_handler), + ) + .await + .map_err(|error| VoiceActivationError::Runtime(error.to_string()))?, + ); + if !generation_is_current() { + asr.cancel(); + drop(staged); + return Err(VoiceActivationError::Runtime( + "voice activation generation is stale".into(), + )); + } + if !warm_hub.set_activation_consumer_for_generation( + requested_generation, + Some(Arc::clone(&asr) as Arc), + true, + ) { + asr.cancel(); + drop(staged); + return Err(VoiceActivationError::Runtime( + "voice activation generation is stale".into(), + )); + } + *finish_asr.lock() = Some(asr); + *finish_emitter_slot.lock() = Some(emitter); + } + Ok(VoiceDictationHandoff { + capture: Box::new(TauriPendingAudioCapture { + app, + backend, + warm_hub, + staged: Some(TauriStagedCapture::Warm(staged)), + cold_capture_reservation: None, + }), + }) + }) + } + + fn stop( + &self, + _reason: VoiceActivationStopReason, + ) -> BoxFuture<'static, Result<(), VoiceActivationError>> { + let warm_hub = Arc::clone(&self.warm_hub); + let wake_asr = Arc::clone(&self.wake_asr); + let finish_asr = Arc::clone(&self.finish_asr); + let preferences = Arc::clone(&self.preferences); + let activation_generation = self.config.activation_generation; + Box::pin(async move { + let owns_activation = warm_hub.clear_activation_generation_if(activation_generation); + if owns_activation { + warm_hub.clear_activation_fault_handler_if(activation_generation); + warm_hub.set_activation_consumer(None); + } + if let Some(asr) = wake_asr.lock().take() { + asr.cancel(); + } + if let Some(asr) = finish_asr.lock().take() { + asr.cancel(); + } + if owns_activation { + let prefs = preferences.get(); + warm_hub.configure( + crate::recorder::warm_capture_enabled( + prefs.low_latency_dictation_enabled, + crate::desktop_resources::active(), + false, + ), + (!prefs.microphone_device_name.trim().is_empty()) + .then(|| prefs.microphone_device_name.clone()), + openless_core::AudioProcessingConfig { + enhancement_enabled: prefs.audio_enhancement_enabled, + }, + ); + } + Ok(()) + }) + } +} + #[cfg(local_asr_sherpa)] struct TauriNativeTranscriptionEngine { dependencies: TauriNativeAsrDependencies, @@ -1903,17 +2497,31 @@ impl TranscriptionEngine for TauriNativeTranscriptionEngine { }); let token_sink = Arc::clone(&partials); let token_offset = Arc::new(AtomicU64::new(0)); + let segment_start = Arc::new(AtomicU64::new(0)); let handler_offset = Arc::clone(&token_offset); - let token_handler = Arc::new(move |piece: String| { - let offset = - handler_offset.fetch_add(piece.chars().count() as u64, Ordering::AcqRel); - if let Err(error) = token_sink.publish(TextStreamChunk { - text: piece, - offset, - }) { - log::warn!("[core-adapter] publish sherpa partial failed: {error}"); - } - }); + let handler_segment_start = Arc::clone(&segment_start); + let token_handler = Arc::new( + move |emission: crate::asr::local::sherpa_runtime::SherpaTokenEmission| { + let text_len = emission.text.chars().count() as u64; + let offset = if emission.is_new_segment { + let offset = handler_offset.load(Ordering::Acquire); + handler_segment_start.store(offset, Ordering::Release); + handler_offset.fetch_add(text_len, Ordering::AcqRel) + } else if emission.is_delta { + handler_offset.fetch_add(text_len, Ordering::AcqRel) + } else { + let offset = handler_segment_start.load(Ordering::Acquire); + handler_offset.store(offset + text_len, Ordering::Release); + offset + }; + if let Err(error) = token_sink.publish(TextStreamChunk { + text: emission.text, + offset, + }) { + log::warn!("[core-adapter] publish sherpa partial failed: {error}"); + } + }, + ); let target = openless_core::LocalAsrTarget::parse( openless_core::LocalAsrRuntime::SherpaOnnx, model.clone(), @@ -2286,11 +2894,54 @@ impl TauriStagedCapture { } } +fn release_cold_capture_and_resume(warm_hub: &WarmCaptureHub) { + warm_hub.end_cold_capture(); + warm_hub.resume_after_cold_capture(); +} + +struct ColdCaptureReservation { + warm_hub: Arc, + resume_warm_hub: bool, + active: bool, +} + +impl ColdCaptureReservation { + fn new(warm_hub: Arc, resume_warm_hub: bool) -> Self { + Self { + warm_hub, + resume_warm_hub, + active: true, + } + } + + fn release(&mut self) { + if !self.active { + return; + } + self.active = false; + self.warm_hub.end_cold_capture(); + if self.resume_warm_hub { + self.warm_hub.resume_after_cold_capture(); + } + } + + fn disarm(&mut self) { + self.active = false; + } +} + +impl Drop for ColdCaptureReservation { + fn drop(&mut self) { + self.release(); + } +} + enum TauriCommittedCapture { Cold { recorder: Recorder, runtime_errors: std::sync::mpsc::Receiver, archive_active: bool, + reconfigure_voice_activation: bool, }, Warm { capture: WarmActiveCapture, @@ -2315,6 +2966,7 @@ impl TauriStagedCapture { recorder, runtime_errors, archive_active, + reconfigure_voice_activation: false, }) } Self::Warm(staged) => { @@ -2338,8 +2990,8 @@ impl TauriStagedCapture { if message == "warm provisional capture was not armed" ) => { - fallback_hub.begin_cold_capture()?; - fallback_hub.suspend_for_cold_capture(); + let finish_consumer = fallback_hub.activation_consumer_for_capture(); + fallback_hub.begin_cold_capture_and_suspend()?; let cold = match Recorder::start_staged_with_processing( fallback_device, STAGED_CAPTURE_MAX_BYTES, @@ -2347,18 +2999,25 @@ impl TauriStagedCapture { ) { Ok(cold) => cold, Err(error) => { - fallback_hub.end_cold_capture(); + release_cold_capture_and_resume(&fallback_hub); return Err(error); } }; let (recorder, runtime_errors, archive_active) = match cold.commit( - fallback_consumer, + finish_consumer + .map(|finish| { + AudioConsumerFanout::new(vec![ + Arc::clone(&fallback_consumer), + finish, + ]) as Arc + }) + .unwrap_or(fallback_consumer), fallback_level_handler, fallback_archive_path, ) { Ok(committed) => committed, Err(error) => { - fallback_hub.end_cold_capture(); + release_cold_capture_and_resume(&fallback_hub); return Err(error); } }; @@ -2366,6 +3025,7 @@ impl TauriStagedCapture { recorder, runtime_errors, archive_active, + reconfigure_voice_activation: true, }) } Err(error) => Err(error), @@ -2387,6 +3047,7 @@ struct TauriPendingAudioCapture { backend: BackendSlot, warm_hub: Arc, staged: Option, + cold_capture_reservation: Option, } struct TauriActiveRecording { @@ -2394,6 +3055,8 @@ struct TauriActiveRecording { warm_capture: Option, cold_capture_hub: Option>, resume_warm_hub: Option>, + backend: BackendSlot, + reconfigure_voice_activation: bool, archive: Option>, _mute: Option, } @@ -2411,6 +3074,7 @@ impl PendingAudioCapture for TauriPendingAudioCapture { backend, warm_hub, staged, + cold_capture_reservation, } = *self; let Some(staged) = staged else { return Box::pin(async { @@ -2420,6 +3084,8 @@ impl PendingAudioCapture for TauriPendingAudioCapture { )) }); }; + let cold_capture = staged.is_cold(); + let has_cold_capture_reservation = cold_capture_reservation.is_some(); Box::pin(async move { let permanent_archive = matches!( context.output_target, @@ -2444,7 +3110,9 @@ impl PendingAudioCapture for TauriPendingAudioCapture { #[cfg(not(mobile))] let monitor_app = app.lock().clone(); let warm_hub_for_join = Arc::clone(&warm_hub); + let backend_for_recording = Arc::clone(&backend); let joined = tauri::async_runtime::spawn_blocking(move || { + let mut cold_capture_reservation = cold_capture_reservation; let archive_path = archive_path .map_err(|error| { log::warn!("[recordings] archive unavailable; capture continues: {error}") @@ -2505,7 +3173,9 @@ impl PendingAudioCapture for TauriPendingAudioCapture { Ok(committed) => committed, Err(error) => { drop(mute); - warm_hub.end_cold_capture(); + if let Some(reservation) = cold_capture_reservation.as_mut() { + reservation.release(); + } return Err(error); } }; @@ -2516,11 +3186,13 @@ impl PendingAudioCapture for TauriPendingAudioCapture { runtime_errors, archive_active, resume_warm_hub, + reconfigure_voice_activation, ) = match committed { TauriCommittedCapture::Cold { recorder, runtime_errors, archive_active, + reconfigure_voice_activation, } => ( Some(recorder), None, @@ -2528,6 +3200,7 @@ impl PendingAudioCapture for TauriPendingAudioCapture { Some(runtime_errors), archive_active, Some(Arc::clone(&warm_hub)), + reconfigure_voice_activation, ), TauriCommittedCapture::Warm { capture, @@ -2540,21 +3213,21 @@ impl PendingAudioCapture for TauriPendingAudioCapture { Some(runtime_errors), archive_active, None, + false, ), }; if recording_plan.archive_required && !archive_active { - let has_warm_capture = warm_capture.is_some(); if let Some(recorder) = recorder { recorder.stop(); } if let Some(capture) = warm_capture { capture.stop(); } - if let Some(cold_capture_hub) = cold_capture_hub.as_ref() { + if let Some(reservation) = cold_capture_reservation.as_mut() { + reservation.release(); + } else if let Some(cold_capture_hub) = cold_capture_hub.as_ref() { cold_capture_hub.end_cold_capture(); - } - if !has_warm_capture { - if let Some(warm_hub) = resume_warm_hub { + if let Some(warm_hub) = resume_warm_hub.as_ref() { warm_hub.resume_after_cold_capture(); } } @@ -2566,6 +3239,9 @@ impl PendingAudioCapture for TauriPendingAudioCapture { "速记录音文件无法创建，已阻止开始录音以避免丢失内容", )); } + if let Some(reservation) = cold_capture_reservation.as_mut() { + reservation.disarm(); + } if recording_plan.archive_enabled { if let Err(error) = crate::persistence::prune_recordings( recording_plan.retention_days, @@ -2579,6 +3255,8 @@ impl PendingAudioCapture for TauriPendingAudioCapture { warm_capture, cold_capture_hub, resume_warm_hub, + backend: Arc::clone(&backend_for_recording), + reconfigure_voice_activation, archive: archive_path .map(|path| Arc::new(TauriRecordingArchive::new(path, archive_active))), _mute: mute, @@ -2589,7 +3267,9 @@ impl PendingAudioCapture for TauriPendingAudioCapture { Ok(Ok(result)) => result, Ok(Err(error)) => return Err(error), Err(error) => { - warm_hub_for_join.end_cold_capture(); + if cold_capture && !has_cold_capture_reservation { + release_cold_capture_and_resume(&warm_hub_for_join); + } return Err(BackendError::new( BackendErrorCode::Internal, format!("join promoted Tauri recorder task: {error}"), @@ -2627,26 +3307,32 @@ impl PendingAudioCapture for TauriPendingAudioCapture { fn discard(self: Box) -> BoxFuture<'static, Result<(), BackendError>> { let TauriPendingAudioCapture { - warm_hub, staged, .. + staged, + cold_capture_reservation, + .. } = *self; Box::pin(async move { let Some(staged) = staged else { + if let Some(reservation) = cold_capture_reservation { + drop(reservation); + } return Ok(()); }; - let cold_capture = staged.is_cold(); - let result = tauri::async_runtime::spawn_blocking(move || { - staged.discard().map_err(map_recorder_error) + let result = match tauri::async_runtime::spawn_blocking(move || { + let result = staged.discard().map_err(map_recorder_error); + if let Some(mut reservation) = cold_capture_reservation { + reservation.release(); + } + result }) .await - .map_err(|error| { - BackendError::new( + { + Ok(result) => result, + Err(error) => Err(BackendError::new( BackendErrorCode::Internal, format!("join provisional Tauri recorder discard task: {error}"), - ) - })?; - if cold_capture { - warm_hub.end_cold_capture(); - } + )), + }; result }) } @@ -2800,13 +3486,15 @@ impl ActiveRecording for TauriActiveRecording { let warm_capture = self.warm_capture.take(); let cold_capture_hub = self.cold_capture_hub.take(); let resume_warm_hub = self.resume_warm_hub.take(); + let backend = Arc::clone(&self.backend); + let reconfigure_voice_activation = self.reconfigure_voice_activation; if recorder.is_none() && warm_capture.is_none() { return Err(BackendError::new( BackendErrorCode::InvalidState, "Tauri recorder was already stopped", )); } - tauri::async_runtime::spawn_blocking(move || { + let stop_result = tauri::async_runtime::spawn_blocking(move || { if let Some(recorder) = recorder { recorder.stop(); } @@ -2830,7 +3518,21 @@ impl ActiveRecording for TauriActiveRecording { BackendErrorCode::Internal, format!("join Tauri recorder stop task: {error}"), ) - })? + })?; + if reconfigure_voice_activation { + let backend = { + let slot = backend.lock(); + slot.as_ref().and_then(std::sync::Weak::upgrade) + }; + if let Some(backend) = backend { + if let Err(error) = backend.configure_voice_activation().await { + log::warn!( + "[voice-activation] reconfigure after warm fallback release failed: {error}" + ); + } + } + } + stop_result }) } } @@ -2876,19 +3578,23 @@ impl TauriAudioRecorder { backend: Arc::clone(&self.backend), warm_hub: Arc::clone(&self.warm_hub), staged: Some(TauriStagedCapture::Warm(staged)), + cold_capture_reservation: None, })); } log::warn!("[recorder] warm capture unavailable; falling back to press-time staging"); } if !request.low_latency { - self.warm_hub.stop(); + self.warm_hub + .begin_cold_capture_and_suspend() + .map_err(map_recorder_error)?; + } else { + self.warm_hub + .begin_cold_capture() + .map_err(map_recorder_error)?; } let max_buffer_bytes = (request.pre_roll_ms as usize) .saturating_mul(32) .clamp(2, 16_000); - self.warm_hub - .begin_cold_capture() - .map_err(map_recorder_error)?; let staged = match Recorder::start_staged_with_processing( request.microphone_device_name, max_buffer_bytes, @@ -2896,7 +3602,7 @@ impl TauriAudioRecorder { ) { Ok(staged) => staged, Err(error) => { - self.warm_hub.end_cold_capture(); + release_cold_capture_and_resume(&self.warm_hub); return Err(map_recorder_error(error)); } }; @@ -2905,6 +3611,10 @@ impl TauriAudioRecorder { backend: Arc::clone(&self.backend), warm_hub: Arc::clone(&self.warm_hub), staged: Some(TauriStagedCapture::Cold(staged)), + cold_capture_reservation: Some(ColdCaptureReservation::new( + Arc::clone(&self.warm_hub), + !request.low_latency, + )), })) } } @@ -2991,6 +3701,7 @@ impl AudioRecorder for TauriAudioRecorder { || context.output_target == openless_core::DictationOutputTarget::Undecided); let fault_progress = Arc::clone(&progress); let worker_resume_warm_hub = Arc::clone(&resume_warm_hub); + let backend_for_recording = Arc::clone(&backend); let joined = tauri::async_runtime::spawn_blocking(move || { if prune_recordings_before_capture { if let Err(error) = crate::persistence::prune_recordings( @@ -3052,6 +3763,8 @@ impl AudioRecorder for TauriAudioRecorder { warm_capture: None, cold_capture_hub: Some(Arc::clone(&warm_hub)), resume_warm_hub: Some(worker_resume_warm_hub), + backend: Arc::clone(&backend_for_recording), + reconfigure_voice_activation: true, archive: archive_path .map(|path| Arc::new(TauriRecordingArchive::new(path, archive_active))), _mute: mute, diff --git a/openless-all/app/src-tauri/src/lib.rs b/openless-all/app/src-tauri/src/lib.rs index db5fc801c..e86153098 100644 --- a/openless-all/app/src-tauri/src/lib.rs +++ b/openless-all/app/src-tauri/src/lib.rs @@ -1591,13 +1591,21 @@ fn handle_microphone_tray_menu_event(app: &AppHandle, id: &str) { }; let coord = app.state::>(); - if let Err(err) = coord - .backend() - .select_microphone_device(selected.device_name.clone()) - { + let backend = coord.backend(); + if let Err(err) = backend.select_microphone_device(selected.device_name.clone()) { log::warn!("[tray] save microphone preference failed: {err}"); return; } + commands::sync_low_latency_capture_preferences(&backend.get_preferences()); + let selected_device_name = selected.device_name.clone(); + let backend_for_reconfigure = Arc::clone(&backend); + tauri::async_runtime::spawn(async move { + if let Err(err) = backend_for_reconfigure.configure_voice_activation().await { + log::warn!( + "[tray] reconfigure microphone-dependent voice runtime for {selected_device_name:?} failed: {err}" + ); + } + }); commands::sync_tray_microphone_selection(&items, &selected.device_name); } diff --git a/openless-all/app/src-tauri/src/recorder.rs b/openless-all/app/src-tauri/src/recorder.rs index eb436f175..d5acb8839 100644 --- a/openless-all/app/src-tauri/src/recorder.rs +++ b/openless-all/app/src-tauri/src/recorder.rs @@ -39,12 +39,120 @@ const AUDIO_PROCESSING_QUEUE_CAPACITY: usize = 64; const WAV_ARCHIVE_QUEUE_CAPACITY: usize = 256; /// The provisional capture cap is 500 ms at the canonical 16 kHz mono i16 format. pub(crate) const STAGED_CAPTURE_MAX_BYTES: usize = TARGET_SAMPLE_RATE as usize; +/// Voice handoff staging keeps up to 30 seconds while the formal dictation session is prepared. +const VOICE_STAGED_CAPTURE_MAX_BYTES: usize = TARGET_SAMPLE_RATE as usize * 30 * 2; +/// Voice activation wake/finish inference is fed only after this cheap RMS gate passes. +/// The level uses the same normalized 0..1 scale as `pcm_rms_level`. +const ACTIVATION_VAD_RMS_THRESHOLD: f32 = 0.015; +/// Retain a short silence pre-roll so the first consonant is not lost at the gate boundary. +const ACTIVATION_VAD_PRE_ROLL_BYTES: usize = TARGET_SAMPLE_RATE as usize * 2 / 5; +/// Keep sending a few quiet chunks after speech so phrase tails reach the recognizer. +const ACTIVATION_VAD_HANGOVER_CHUNKS: usize = 8; +const VOICE_CAPTURE_KEY_PREFIX: u64 = 1 << 63; + +// [DEBUG-VOICE-1003] Temporary P1-P6 diagnostics; remove only after user confirmation. +// Audio/model work never waits for disk I/O. Only the dedicated NDJSON writer opens a file. +pub(crate) fn voice_diagnostic( + probe_id: &'static str, + hypothesis_ids: &'static [&'static str], + event: &'static str, + outcome: &'static str, + safe: serde_json::Value, +) { + #[cfg(all(target_os = "windows", not(test)))] + { + use std::io::Write; + static WRITER: OnceLock>> = OnceLock::new(); + static SEQUENCE: AtomicU64 = AtomicU64::new(0); + static DROPPED: AtomicU64 = AtomicU64::new(0); + static RUN: OnceLock = OnceLock::new(); + let run = RUN.get_or_init(|| { + format!( + "r{:x}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() + ) + }); + let writer = WRITER.get_or_init(|| { + let (tx, rx) = sync_channel::(4096); + std::thread::Builder::new().name("voice-1003-diagnostics".into()).spawn(move || { + let path = std::env::temp_dir().join("openless-1003-3c02ffc1.ndjson"); + let Ok(mut file) = std::fs::OpenOptions::new().create(true).append(true).open(path) else { + return; + }; + for record in rx { + let Ok(mut line) = serde_json::to_vec(&record) else { continue; }; + line.push(b'\n'); + if file.write_all(&line).is_err() { break; } + } + }).ok().map(|_| tx) + }); + if let Some(writer) = writer { + let record = serde_json::json!({ + "schema": "diagnosing-bugs.v1", "session_id": "voice-1003-3c02ffc1", + "attempt_id": "A2-terminal-tail", "change_id": "C2-terminal-padding", + "run_id": run, "probe_id": probe_id, "hypothesis_ids": hypothesis_ids, + "event": event, "outcome": outcome, "safe": safe, + "seq": SEQUENCE.fetch_add(1, Ordering::Relaxed), + "dropped": DROPPED.load(Ordering::Relaxed), + "ts": std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default().as_millis().to_string(), + }); + if writer.try_send(record).is_err() { DROPPED.fetch_add(1, Ordering::Relaxed); } + } + } + #[cfg(any(not(target_os = "windows"), test))] + let _ = (probe_id, hypothesis_ids, event, outcome, safe); +} + +/// Low-memory mode suppresses optional prewarming, not an explicitly active voice listener. +pub(crate) fn warm_capture_enabled( + low_latency: bool, + low_memory: bool, + voice_activation: bool, +) -> bool { + voice_activation || (low_latency && !low_memory) +} + +fn voice_capture_key(generation: u64) -> u64 { + generation | VOICE_CAPTURE_KEY_PREFIX +} /// Downstream that receives resampled Int16 PCM bytes (little-endian). pub trait AudioConsumer: Send + Sync { /// Each delivery is a little-endian byte sequence of Int16 samples; /// the length is always a multiple of 2. fn consume_pcm_chunk(&self, pcm: &[u8]); + + /// Mark the end of the current VAD-gated segment without stopping the consumer. + /// Consumers that do not maintain streaming endpoint state can ignore this boundary. + fn finish_segment(&self) {} +} + +pub(crate) struct AudioConsumerFanout { + consumers: Vec>, +} + +impl AudioConsumerFanout { + pub(crate) fn new(consumers: Vec>) -> Arc { + Arc::new(Self { consumers }) + } +} + +impl AudioConsumer for AudioConsumerFanout { + fn consume_pcm_chunk(&self, pcm: &[u8]) { + for consumer in &self.consumers { + consumer.consume_pcm_chunk(pcm); + } + } + + fn finish_segment(&self) { + for consumer in &self.consumers { + consumer.finish_segment(); + } + } } /// Compatibility bridge for legacy recorder call sites while cloud ASR @@ -295,9 +403,85 @@ impl CaptureSink { struct WarmCaptureSink { state: Mutex, delivery_gate: Mutex<()>, + activation_consumer: Mutex>>, + activation_consumer_during_capture: AtomicBool, + activation_generation: Mutex>, + activation_fault_handler: Mutex)>>, + activation_vad: Mutex, max_bytes: usize, } +struct ActivationVadOutput { + pcm: Option>, + end_of_speech: bool, +} + +struct ActivationVad { + pre_roll: VecDeque, + speech_active: bool, + hangover_chunks: usize, +} + +impl Default for ActivationVad { + fn default() -> Self { + Self { + pre_roll: VecDeque::with_capacity(ACTIVATION_VAD_PRE_ROLL_BYTES), + speech_active: false, + hangover_chunks: 0, + } + } +} + +impl ActivationVad { + fn reset(&mut self) { + self.pre_roll.clear(); + self.speech_active = false; + self.hangover_chunks = 0; + } + + /// Return gated PCM and report a single speech-to-silence boundary. + /// This deliberately uses one RMS pass and a bounded queue rather than a second model. + fn gate(&mut self, pcm: &[u8]) -> ActivationVadOutput { + if pcm.is_empty() { + return ActivationVadOutput { + pcm: None, + end_of_speech: false, + }; + } + let speech = pcm_rms_level(pcm) >= ACTIVATION_VAD_RMS_THRESHOLD; + if speech { + let mut output = if self.speech_active { + Vec::new() + } else { + self.pre_roll.iter().copied().collect() + }; + // The current block can be a complete handoff buffer, not just a device chunk. + output.extend_from_slice(pcm); + self.pre_roll.clear(); + self.speech_active = true; + self.hangover_chunks = ACTIVATION_VAD_HANGOVER_CHUNKS; + return ActivationVadOutput { + pcm: Some(output), + end_of_speech: false, + }; + } + if self.speech_active && self.hangover_chunks > 0 { + self.hangover_chunks -= 1; + return ActivationVadOutput { + pcm: Some(pcm.to_vec()), + end_of_speech: false, + }; + } + WarmCaptureSink::append_bounded(&mut self.pre_roll, pcm, ACTIVATION_VAD_PRE_ROLL_BYTES); + let end_of_speech = self.speech_active; + self.speech_active = false; + ActivationVadOutput { + pcm: None, + end_of_speech, + } + } +} + enum WarmCaptureSinkState { Ring { pcm: VecDeque, @@ -309,6 +493,8 @@ enum WarmCaptureClaim { Staging { pcm: VecDeque, pre_roll_bytes: usize, + allow_postroll_eviction: bool, + max_bytes: usize, max_level: f32, }, Promoted(ForwardTarget), @@ -323,10 +509,66 @@ impl WarmCaptureSink { claims: HashMap::new(), }), delivery_gate: Mutex::new(()), + activation_consumer: Mutex::new(None), + activation_consumer_during_capture: AtomicBool::new(false), + activation_generation: Mutex::new(None), + activation_fault_handler: Mutex::new(None), + activation_vad: Mutex::new(ActivationVad::default()), max_bytes, }) } + fn activation_consumer_during_capture(&self) -> Option> { + self.activation_consumer_during_capture + .load(Ordering::Acquire) + .then(|| self.activation_consumer.lock().clone()) + .flatten() + } + + fn set_activation_consumer(&self, consumer: Option>) { + self.activation_consumer_during_capture.store(false, Ordering::Release); + *self.activation_consumer.lock() = consumer; + self.activation_vad.lock().reset(); + } + + fn set_activation_consumer_during_capture(&self, consumer: Option>) { + self.activation_consumer_during_capture + .store(consumer.is_some(), Ordering::Release); + *self.activation_consumer.lock() = consumer; + self.activation_vad.lock().reset(); + } + + fn deliver_activation(&self, consumer: &Arc, pcm: &[u8]) { + let (gated, was_active, active, hangover) = { + let mut vad = self.activation_vad.lock(); + let was_active = vad.speech_active; + let gated = vad.gate(pcm); + (gated, was_active, vad.speech_active, vad.hangover_chunks) + }; + // [DEBUG-VOICE-1003] P1 / H1: transitions plus bounded coverage of sustained noise/silence. + static BLOCKS: AtomicU64 = AtomicU64::new(0); + let blocks = BLOCKS.fetch_add(1, Ordering::Relaxed); + if was_active != active || gated.end_of_speech || blocks % 64 == 0 { + voice_diagnostic( + "P1", + &["H1"], + "vad_gate", + "observed", + serde_json::json!({ + "bytes": pcm.len(), "active": active, "hangover": hangover, + "forwarded": gated.pcm.is_some(), "boundary": gated.end_of_speech, + "count": blocks, + }), + ); + } + if let Some(pcm) = gated.pcm { + consumer.consume_pcm_chunk(&pcm); + } + if gated.end_of_speech { + consumer.finish_segment(); + } + } + fn append_bounded(buffer: &mut VecDeque, pcm: &[u8], max_bytes: usize) { let even_len = pcm.len() & !1; buffer.extend(pcm[..even_len].iter().copied()); @@ -339,26 +581,42 @@ impl WarmCaptureSink { fn append_staged( buffer: &mut VecDeque, pre_roll_bytes: &mut usize, + allow_postroll_eviction: bool, pcm: &[u8], max_bytes: usize, ) { let even_len = pcm.len() & !1; let incoming = &pcm[..even_len]; - let excess = buffer - .len() - .saturating_add(incoming.len()) - .saturating_sub(max_bytes); - let evictable = excess.min(*pre_roll_bytes); - for _ in 0..(evictable / 2) { + let excess = buffer.len().saturating_add(incoming.len()).saturating_sub(max_bytes); + let evictable_preroll = excess.min(*pre_roll_bytes); + for _ in 0..(evictable_preroll / 2) { + let _ = buffer.pop_front(); + let _ = buffer.pop_front(); + } + *pre_roll_bytes -= evictable_preroll; + let evictable_postroll = if allow_postroll_eviction { + excess.saturating_sub(evictable_preroll) + } else { + 0 + }; + for _ in 0..(evictable_postroll / 2) { let _ = buffer.pop_front(); let _ = buffer.pop_front(); } - *pre_roll_bytes -= evictable; let available = max_bytes.saturating_sub(buffer.len()) & !1; buffer.extend(incoming[..incoming.len().min(available)].iter().copied()); } - fn arm(&self, press_id: u64) -> Result<(), RecorderError> { + fn arm(&self, press_id: u64, include_preroll: bool) -> Result<(), RecorderError> { + self.arm_with_limit(press_id, include_preroll, self.max_bytes) + } + + fn arm_with_limit( + &self, + press_id: u64, + include_preroll: bool, + max_bytes: usize, + ) -> Result<(), RecorderError> { let _delivery = self.delivery_gate.lock(); let mut state = self.state.lock(); let WarmCaptureSinkState::Ring { pcm, claims } = &mut *state; @@ -367,11 +625,15 @@ impl WarmCaptureSink { "warm capture generation is already armed".into(), )); } + let max_bytes = max_bytes.max(2) & !1; + let staged = include_preroll.then(|| pcm.clone()).unwrap_or_default(); claims.insert( press_id, WarmCaptureClaim::Staging { - pcm: pcm.clone(), - pre_roll_bytes: pcm.len(), + pre_roll_bytes: staged.len(), + allow_postroll_eviction: !include_preroll, + max_bytes, + pcm: staged, max_level: 0.0, }, ); @@ -418,8 +680,18 @@ impl WarmCaptureSink { Some(WarmCaptureClaim::Staging { .. }) | None => None, } }; + let activation = self + .activation_consumer_during_capture + .load(Ordering::Acquire) + .then(|| self.activation_consumer.lock().clone()) + .flatten(); if let Some(target) = target { - CaptureSink::deliver(&target, &prefix); + if !prefix.is_empty() { + CaptureSink::deliver(&target, &prefix); + if let Some(activation) = activation { + self.deliver_activation(&activation, &prefix); + } + } (target.level_handler)(staged_level); } Ok(()) @@ -512,27 +784,52 @@ impl AudioConsumer for WarmCaptureSink { return; } let _delivery = self.delivery_gate.lock(); - let mut targets = Vec::new(); - { + let (targets, has_promoted_claim, has_staging_claim) = { let mut state = self.state.lock(); let WarmCaptureSinkState::Ring { pcm: ring, claims } = &mut *state; Self::append_bounded(ring, pcm, self.max_bytes); + let mut targets = Vec::new(); + let mut has_promoted_claim = false; + let mut has_staging_claim = false; for claim in claims.values_mut() { match claim { WarmCaptureClaim::Staging { pcm: staged, pre_roll_bytes, + allow_postroll_eviction, + max_bytes, .. } => { - Self::append_staged(staged, pre_roll_bytes, pcm, self.max_bytes); + has_staging_claim = true; + Self::append_staged( + staged, + pre_roll_bytes, + *allow_postroll_eviction, + pcm, + *max_bytes, + ); + } + WarmCaptureClaim::Promoted(target) => { + has_promoted_claim = true; + targets.push(target.clone()); } - WarmCaptureClaim::Promoted(target) => targets.push(target.clone()), } } - } + (targets, has_promoted_claim, has_staging_claim) + }; + let activation = ((!has_promoted_claim && !has_staging_claim) + || (has_promoted_claim + && self + .activation_consumer_during_capture + .load(Ordering::Acquire))) + .then(|| self.activation_consumer.lock().clone()) + .flatten(); for target in targets { CaptureSink::deliver(&target, pcm); } + if let Some(activation) = activation { + self.deliver_activation(&activation, pcm); + } } } @@ -701,6 +998,76 @@ impl WarmCaptureHub { } } + pub(crate) fn activation_consumer_for_capture(&self) -> Option> { + self.sink.activation_consumer_during_capture() + } + + pub(crate) fn set_activation_consumer(&self, consumer: Option>) { + self.sink.set_activation_consumer(consumer); + } + + pub(crate) fn set_activation_consumer_during_capture( + &self, + consumer: Option>, + ) { + self.sink.set_activation_consumer_during_capture(consumer); + } + + pub(crate) fn set_activation_consumer_for_generation( + &self, + generation: u64, + consumer: Option>, + during_capture: bool, + ) -> bool { + let active_generation = self.sink.activation_generation.lock(); + if *active_generation != Some(generation) { + return false; + } + if during_capture { + self.sink.set_activation_consumer_during_capture(consumer); + } else { + self.sink.set_activation_consumer(consumer); + } + true + } + + pub(crate) fn set_activation_generation(&self, generation: u64) { + *self.sink.activation_generation.lock() = Some(generation); + } + + pub(crate) fn clear_activation_generation_if(&self, generation: u64) -> bool { + let mut active_generation = self.sink.activation_generation.lock(); + if *active_generation == Some(generation) { + *active_generation = None; + true + } else { + false + } + } + + pub(crate) fn activation_is_active(&self) -> bool { + self.sink.activation_generation.lock().is_some() + } + + pub(crate) fn is_cold_capture_active(&self) -> bool { + self.cold_capture_active.load(Ordering::Acquire) + } + + pub(crate) fn set_activation_fault_handler( + &self, + generation: u64, + handler: Option>, + ) { + *self.sink.activation_fault_handler.lock() = handler.map(|handler| (generation, handler)); + } + + pub(crate) fn clear_activation_fault_handler_if(&self, generation: u64) { + let mut handler = self.sink.activation_fault_handler.lock(); + if handler.as_ref().is_some_and(|(current, _)| *current == generation) { + *handler = None; + } + } + pub(crate) fn configure( &self, enabled: bool, @@ -829,10 +1196,29 @@ impl WarmCaptureHub { let staging_claims = self.sink.staging_claim_ids(); self.runtime_fallback_claims.lock().extend(staging_claims); let archives = self.sink.clear(); - self.notify_runtime_error(error); + self.notify_runtime_error(error.clone()); + let active_activation_generation = *self.sink.activation_generation.lock(); + let activation_handler = { + let mut handler = self.sink.activation_fault_handler.lock(); + match handler.take() { + Some((handler_generation, callback)) + if active_activation_generation == Some(handler_generation) => + { + Some(callback) + } + Some(other) => { + *handler = Some(other); + None + } + None => None, + } + }; for archive in archives { archive.finish(); } + if let Some(handler) = activation_handler { + handler(error); + } } fn promote( @@ -874,22 +1260,79 @@ impl WarmCaptureHub { press_id: u64, device_name: Option, processing: openless_core::AudioProcessingConfig, + ) -> Result { + self.arm_internal( + press_id, + device_name, + true, + STAGED_CAPTURE_MAX_BYTES, + Some(processing), + ) + } + + pub(crate) fn arm_with_preroll( + self: &Arc, + press_id: u64, + device_name: Option, + ) -> Result { + self.arm_internal( + voice_capture_key(press_id), + device_name, + true, + VOICE_STAGED_CAPTURE_MAX_BYTES, + None, + ) + } + + pub(crate) fn arm_without_preroll( + self: &Arc, + press_id: u64, + device_name: Option, + ) -> Result { + self.arm_internal( + voice_capture_key(press_id), + device_name, + false, + VOICE_STAGED_CAPTURE_MAX_BYTES, + None, + ) + } + + fn arm_internal( + self: &Arc, + press_id: u64, + device_name: Option, + include_preroll: bool, + staged_max_bytes: usize, + processing: Option, ) -> Result { if !self.enabled.load(Ordering::Acquire) { return Err(RecorderError::EngineFailed( "warm capture is disabled".into(), )); } + if self.cold_capture_active.load(Ordering::Acquire) { + return Err(RecorderError::EngineFailed( + "warm capture is unavailable during cold capture".into(), + )); + } loop { let lifecycle = self.lifecycle_gate.lock(); + let desired_processing = processing.unwrap_or(*self.processing_config.lock()); if *self.device_name.lock() != device_name - || *self.processing_config.lock() != processing + || *self.processing_config.lock() != desired_processing { + if self.sink.has_promoted_claims() { + return Err(RecorderError::EngineFailed( + "warm capture device change is deferred while capture is active".into(), + )); + } drop(lifecycle); - self.configure(true, device_name.clone(), processing); + self.configure(true, device_name.clone(), desired_processing); continue; } - self.sink.arm(press_id)?; + self.sink + .arm_with_limit(press_id, include_preroll, staged_max_bytes)?; if let Err(error) = self.ensure_started_locked(device_name.clone()) { self.sink.discard(press_id); return Err(error); @@ -928,6 +1371,22 @@ impl WarmCaptureHub { pub(crate) fn begin_cold_capture(&self) -> Result<(), RecorderError> { let _lifecycle = self.lifecycle_gate.lock(); + self.begin_cold_capture_locked() + } + + /// Atomically reserve the microphone for a cold recorder and suspend the warm stream. + /// + /// The reservation is acquired before the stream is stopped while holding the same lifecycle + /// gate, so a concurrent warm arm cannot slip into the interval between admission and + /// suspension. + pub(crate) fn begin_cold_capture_and_suspend(&self) -> Result<(), RecorderError> { + let _lifecycle = self.lifecycle_gate.lock(); + self.begin_cold_capture_locked()?; + self.suspend_for_cold_capture_locked(); + Ok(()) + } + + fn begin_cold_capture_locked(&self) -> Result<(), RecorderError> { if self.cold_capture_active.swap(true, Ordering::AcqRel) { return Err(RecorderError::EngineFailed( "another microphone capture is already active".into(), @@ -1000,6 +1459,10 @@ impl WarmCaptureHub { /// preference enabled so the stream can be rebuilt when that recorder stops. pub(crate) fn suspend_for_cold_capture(&self) { let _lifecycle = self.lifecycle_gate.lock(); + self.suspend_for_cold_capture_locked(); + } + + fn suspend_for_cold_capture_locked(&self) { self.stream_generation.fetch_add(1, Ordering::AcqRel); self.stop_recorder(); let archives = self.sink.clear(); @@ -2232,12 +2695,17 @@ mod tests { #[derive(Default)] struct RecordingConsumer { chunks: StdMutex>>, + segment_flushes: StdMutex, } impl AudioConsumer for RecordingConsumer { fn consume_pcm_chunk(&self, pcm: &[u8]) { self.chunks.lock().unwrap().push(pcm.to_vec()); } + + fn finish_segment(&self) { + *self.segment_flushes.lock().unwrap() += 1; + } } #[test] @@ -2317,6 +2785,160 @@ mod tests { .collect() } + #[test] + fn warm_promotion_replays_staged_prefix_to_finish_consumer() { + let sink = WarmCaptureSink::new(STAGED_CAPTURE_MAX_BYTES); + let formal = Arc::new(RecordingConsumer::default()); + let finish = Arc::new(RecordingConsumer::default()); + sink.arm_with_limit(7, false, VOICE_STAGED_CAPTURE_MAX_BYTES) + .unwrap(); + sink.set_activation_consumer_during_capture(Some( + Arc::clone(&finish) as Arc + )); + let speech = [0x00, 0x40].repeat(16_000); + sink.consume_pcm_chunk(&speech); + + sink.promote( + 7, + Arc::clone(&formal) as Arc, + Arc::new(|_| {}), + None, + ) + .unwrap(); + + assert_eq!(formal.chunks.lock().unwrap().as_slice(), &[speech.to_vec()]); + assert_eq!(finish.chunks.lock().unwrap().as_slice(), &[speech.to_vec()]); + } + + #[test] + fn warm_promotion_replays_bounded_preroll_to_formal_consumer() { + let sink = WarmCaptureSink::new(STAGED_CAPTURE_MAX_BYTES); + let formal = Arc::new(RecordingConsumer::default()); + let prefix = vec![1, 2, 3, 4]; + let speech = vec![0x00, 0x40, 0x00, 0x40]; + sink.consume_pcm_chunk(&prefix); + sink.arm_with_limit(8, true, STAGED_CAPTURE_MAX_BYTES) + .unwrap(); + sink.consume_pcm_chunk(&speech); + + sink.promote( + 8, + Arc::clone(&formal) as Arc, + Arc::new(|_| {}), + None, + ) + .unwrap(); + + assert_eq!( + formal.chunks.lock().unwrap().as_slice(), + &[vec![1, 2, 3, 4, 0x00, 0x40, 0x00, 0x40]] + ); + } + #[test] + fn low_memory_disables_only_optional_warm_capture() { + for low_latency in [false, true] { + for low_memory in [false, true] { + assert!(warm_capture_enabled(low_latency, low_memory, true)); + } + } + assert!(warm_capture_enabled(true, false, false)); + assert!(!warm_capture_enabled(true, true, false)); + assert!(!warm_capture_enabled(false, false, false)); + assert!(!warm_capture_enabled(false, true, false)); + } + + #[test] + fn activation_vad_skips_silence_and_preserves_speech_onset() { + let sink = WarmCaptureSink::new(STAGED_CAPTURE_MAX_BYTES); + let activation = Arc::new(RecordingConsumer::default()); + sink.set_activation_consumer(Some(Arc::clone(&activation) as Arc)); + let silence = vec![0; 320]; + sink.consume_pcm_chunk(&silence); + assert!(activation.chunks.lock().unwrap().is_empty()); + + let speech = vec![0x00, 0x40, 0x00, 0x40]; + sink.consume_pcm_chunk(&speech); + let chunks = activation.chunks.lock().unwrap(); + assert_eq!(chunks.len(), 1); + assert!(chunks[0].ends_with(&speech)); + } + + #[test] + fn activation_vad_preserves_large_blocks_and_does_not_replay_forwarded_audio() { + let mut vad = ActivationVad::default(); + let silence = vec![0; ACTIVATION_VAD_PRE_ROLL_BYTES + 320]; + assert!(vad.gate(&silence).pcm.is_none()); + let speech = [0x00, 0x40].repeat(16_000); + let first = vad.gate(&speech).pcm.unwrap(); + assert_eq!(first.len(), ACTIVATION_VAD_PRE_ROLL_BYTES + speech.len()); + assert_eq!( + &first[..ACTIVATION_VAD_PRE_ROLL_BYTES], + &silence[..ACTIVATION_VAD_PRE_ROLL_BYTES] + ); + assert_eq!(&first[ACTIVATION_VAD_PRE_ROLL_BYTES..], &speech); + assert_eq!(vad.gate(&speech[..320]).pcm.unwrap(), speech[..320]); + for _ in 0..ACTIVATION_VAD_HANGOVER_CHUNKS { + assert_eq!(vad.gate(&[0; 320]).pcm.unwrap(), [0; 320]); + } + assert!(vad.gate(&[0; 320]).end_of_speech); + let next = vad.gate(&speech[..320]).pcm.unwrap(); + assert_eq!(next.len(), 640); + assert_eq!(&next[320..], &speech[..320]); + } + + #[test] + fn activation_vad_hangover_is_bounded() { + let mut vad = ActivationVad::default(); + let speech = vec![0x00, 0x40, 0x00, 0x40]; + assert!(vad.gate(&speech).pcm.is_some()); + for _ in 0..ACTIVATION_VAD_HANGOVER_CHUNKS { + let output = vad.gate(&[0, 0, 0, 0]); + assert!(output.pcm.is_some()); + assert!(!output.end_of_speech); + } + let output = vad.gate(&[0, 0, 0, 0]); + assert!(output.pcm.is_none()); + assert!(output.end_of_speech); + let output = vad.gate(&[0, 0, 0, 0]); + assert!(output.pcm.is_none()); + assert!(!output.end_of_speech); + } + + #[test] + fn activation_vad_flushes_after_hangover_without_flushing_sustained_silence() { + let sink = WarmCaptureSink::new(STAGED_CAPTURE_MAX_BYTES); + let activation = Arc::new(RecordingConsumer::default()); + sink.set_activation_consumer(Some(Arc::clone(&activation) as Arc)); + let speech = vec![0x00, 0x40, 0x00, 0x40]; + sink.consume_pcm_chunk(&speech); + for _ in 0..=ACTIVATION_VAD_HANGOVER_CHUNKS { + sink.consume_pcm_chunk(&[0, 0, 0, 0]); + } + sink.consume_pcm_chunk(&[0, 0, 0, 0]); + + assert_eq!(*activation.segment_flushes.lock().unwrap(), 1); + assert!(activation.chunks.lock().unwrap().len() >= 1); + } + #[test] + fn activation_fault_handler_uses_voice_generation_not_stream_generation() { + let hub = WarmCaptureHub::new(); + let called = Arc::new(AtomicBool::new(false)); + let called_for_handler = Arc::clone(&called); + hub.set_activation_generation(41); + hub.set_activation_fault_handler( + 41, + Some(Arc::new(move |_| { + called_for_handler.store(true, Ordering::Release); + })), + ); + + // The warm stream generation starts at zero; it is intentionally not the + // voice-activation generation owned by Core. + hub.stop_after_runtime_error(0, RecorderError::EngineFailed("stream failed".into())); + + assert!(called.load(Ordering::Acquire)); + } + #[test] fn native_alternatives_preserve_channels_format_and_advertised_rates() { let range = cpal::SupportedStreamConfigRange::new( @@ -2566,7 +3188,7 @@ mod tests { fn warm_sink_captures_bounded_preroll_and_post_press_audio_in_order() { let sink = WarmCaptureSink::new(8); sink.consume_pcm_chunk(&[1, 2, 3, 4, 5, 6, 7, 8]); - sink.arm(17).unwrap(); + sink.arm(17, true).unwrap(); sink.consume_pcm_chunk(&[9, 10]); sink.publish_level(0.75); @@ -2591,10 +3213,34 @@ mod tests { } #[test] - fn warm_sink_preserves_earliest_post_press_audio_after_preroll_eviction() { + fn warm_sink_activation_consumer_receives_standby_pcm_without_claiming_archive() { + let sink = WarmCaptureSink::new(16); + let consumer = Arc::new(RecordingConsumer::default()); + sink.set_activation_consumer(Some(consumer.clone() as Arc)); + sink.consume_pcm_chunk(&[1, 2, 3, 4]); + sink.set_activation_consumer(None); + sink.consume_pcm_chunk(&[5, 6]); + + assert_eq!(consumer.chunks.lock().unwrap().as_slice(), &[vec![1, 2, 3, 4]]); + assert!(matches!(&*sink.state.lock(), WarmCaptureSinkState::Ring { .. })); + } + + #[test] + fn warm_sink_activation_arm_excludes_wake_preroll() { + let sink = WarmCaptureSink::new(16); + sink.consume_pcm_chunk(&[1, 2, 3, 4]); + sink.arm(19, false).unwrap(); + sink.consume_pcm_chunk(&[5, 6]); + let consumer = Arc::new(RecordingConsumer::default()); + sink.promote(19, consumer.clone(), Arc::new(|_| {}), None) + .unwrap(); + assert_eq!(consumer.chunks.lock().unwrap().as_slice(), &[vec![5, 6]]); + } + #[test] + fn warm_sink_staging_is_bounded_to_the_capture_capacity() { let sink = WarmCaptureSink::new(8); sink.consume_pcm_chunk(&[1, 2, 3, 4, 5, 6, 7, 8]); - sink.arm(17).unwrap(); + sink.arm(17, true).unwrap(); sink.consume_pcm_chunk(&[9, 10, 11, 12, 13, 14, 15, 16]); sink.consume_pcm_chunk(&[17, 18, 19, 20, 21, 22]); @@ -2608,12 +3254,30 @@ mod tests { ); } + #[test] + fn warm_sink_without_preroll_keeps_latest_post_wake_audio() { + let sink = WarmCaptureSink::new(8); + sink.arm(17, false).unwrap(); + sink.consume_pcm_chunk(&[1, 2, 3, 4, 5, 6, 7, 8]); + sink.consume_pcm_chunk(&[9, 10, 11, 12, 13, 14, 15, 16]); + sink.consume_pcm_chunk(&[17, 18, 19, 20, 21, 22]); + + let consumer = Arc::new(RecordingConsumer::default()); + sink.promote(17, consumer.clone(), Arc::new(|_| {}), None) + .unwrap(); + + assert_eq!( + consumer.chunks.lock().unwrap().as_slice(), + &[vec![15, 16, 17, 18, 19, 20, 21, 22]] + ); + } + #[test] fn warm_sink_discard_removes_only_matching_generation() { let sink = WarmCaptureSink::new(16); sink.consume_pcm_chunk(&[1, 2]); - sink.arm(17).unwrap(); - sink.arm(18).unwrap(); + sink.arm(17, true).unwrap(); + sink.arm(18, true).unwrap(); sink.discard(17); sink.consume_pcm_chunk(&[3, 4]); @@ -2670,7 +3334,7 @@ mod tests { fn warm_config_change_keeps_staged_claim_on_cold_fallback() { let hub = WarmCaptureHub::new(); hub.enabled.store(true, Ordering::Release); - hub.sink.arm(42).unwrap(); + hub.sink.arm(42, true).unwrap(); hub.configure( false, @@ -2690,7 +3354,7 @@ mod tests { let hub = WarmCaptureHub::new(); hub.enabled.store(true, Ordering::Release); for press_id in 1..=claim_count { - hub.sink.arm(press_id).unwrap(); + hub.sink.arm(press_id, true).unwrap(); hub.sink .promote( press_id, diff --git a/openless-all/app/src-tauri/src/tauri_events.rs b/openless-all/app/src-tauri/src/tauri_events.rs index 3d0ca270b..f40dd3851 100644 --- a/openless-all/app/src-tauri/src/tauri_events.rs +++ b/openless-all/app/src-tauri/src/tauri_events.rs @@ -186,6 +186,9 @@ async fn forward_legacy_event( } emit_dictation_state(app, backend, snapshot) } + BackendEventKind::VoiceActivationStateChanged(state) => { + let _ = app.emit("voice-activation:state-changed", state); + } BackendEventKind::TranscriptDelta(_) => {} BackendEventKind::DictationCompleted(result) => { capsule_owners.transcription_notice = None; @@ -807,6 +810,9 @@ async fn emit_resync(app: &AppHandle, backend: &OpenLessBackend, owners: &mut Ca } let _ = app.emit("credentials:changed", snapshot.credentials); let _ = app.emit("vocab:updated", ()); + if let Some(state) = backend.latest_voice_activation_state() { + let _ = app.emit("voice-activation:state-changed", state); + } let qa = match backend.services().qa.snapshot().await { Ok(snapshot) => Some(snapshot), diff --git a/openless-all/app/src/components/SettingsModal.tsx b/openless-all/app/src/components/SettingsModal.tsx index bedba0643..42fa2a625 100644 --- a/openless-all/app/src/components/SettingsModal.tsx +++ b/openless-all/app/src/components/SettingsModal.tsx @@ -31,6 +31,7 @@ import { visibleAdvancedPages, isTestingFeaturePage, type AdvancedPageId, + type ServiceViewId, type SettingsSectionId, } from '../pages/settings/navigation'; import { ChannelEditorHostContext } from '../pages/settings/ChannelEditorHostContext'; @@ -73,6 +74,7 @@ export function SettingsModal({ initialSettingsSection ?? 'general', ); const [requestedAdvancedPage, setAdvancedPage] = useState(null); + const [requestedServiceView, setRequestedServiceView] = useState(); const [query, setQuery] = useState(''); const [platformCaps, setPlatformCaps] = useState(getCachedPlatformCapabilities); const experimentalAsr = useExperimentalAsr(platformCaps?.platform === 'desktop' && os === 'mac'); @@ -237,16 +239,23 @@ export function SettingsModal({ }); }; - const selectSection = (next: SettingsSectionId, fromSearch = false) => { + const selectSection = ( + next: SettingsSectionId, + fromSearch = false, + serviceView?: ServiceViewId, + ) => { void providerForm.finish(() => { channelCloseRef.current?.(); setSection(next); + setRequestedServiceView(next === 'services' ? serviceView : undefined); setAdvancedPage(null); setQuery(''); if (fromSearch) window.requestAnimationFrame(() => headingRef.current?.focus()); }); }; + const openLocalModels = () => selectSection('services', false, 'models'); + const handleKeyDown = (event: KeyboardEvent) => { // Nested selectors and shortcut recorders own their keys before the dialog does. if (event.defaultPrevented || event.nativeEvent.isComposing) return; @@ -743,11 +752,16 @@ export function SettingsModal({ gap: 16, }} > - {section === 'general' && } + {section === 'general' && } {section === 'inputMethod' && } {section === 'shortcuts' && } {section === 'appearance' && } - {section === 'services' && } + {section === 'services' && ( + + )} {section === 'privacy' && } {section === 'advanced' && ( { await invoke('plugin:autostart|disable'); } -export function RecordingInputSection({ forcedOs }: { forcedOs?: OS } = {}) { +export function RecordingInputSection({ + forcedOs, + onOpenLocalModels, +}: { + forcedOs?: OS; + onOpenLocalModels?: () => void; +} = {}) { const { t } = useTranslation(); const os = forcedOs ?? detectOS(); const { prefs, capability, updatePrefs: savePrefs, refresh } = useHotkeySettings(); @@ -69,6 +81,8 @@ export function RecordingInputSection({ forcedOs }: { forcedOs?: OS } = {}) { const [microphoneDevices, setMicrophoneDevices] = useState([]); const [microphoneDevicesLoaded, setMicrophoneDevicesLoaded] = useState(false); const [microphoneDevicesError, setMicrophoneDevicesError] = useState(null); + const [voiceActivationModelMissing, setVoiceActivationModelMissing] = useState(false); + const voiceActivationStateObservedRef = useRef(false); const [displayModeRestartPending, setDisplayModeRestartPending] = useState(false); const [displayModeRestarting, setDisplayModeRestarting] = useState(false); const displayModeRestartTitleId = useId(); @@ -131,6 +145,62 @@ export function RecordingInputSection({ forcedOs }: { forcedOs?: OS } = {}) { }; }, [loadMicrophoneDevices]); + useEffect(() => { + if (!isTauri || os !== 'win') { + setVoiceActivationModelMissing(false); + return; + } + let cancelled = false; + let unlisten: (() => void) | undefined; + async function listenForVoiceActivationChanges() { + const { listen } = await import('@tauri-apps/api/event'); + if (cancelled) return; + const stopListening = await listen( + 'voice-activation:state-changed', + (event) => { + voiceActivationStateObservedRef.current = true; + setVoiceActivationModelMissing(isVoiceActivationModelMissing(event.payload)); + }, + ); + if (cancelled) { + stopListening(); + return; + } + unlisten = stopListening; + } + void listenForVoiceActivationChanges(); + return () => { + cancelled = true; + unlisten?.(); + }; + }, [os]); + + useEffect(() => { + if (os !== 'win' || !prefs?.voiceActivationEnabled) { + voiceActivationStateObservedRef.current = false; + setVoiceActivationModelMissing(false); + return; + } + voiceActivationStateObservedRef.current = false; + let cancelled = false; + void getSherpaOnnxAsrCatalog() + .then((catalog) => { + if (cancelled || voiceActivationStateObservedRef.current) return; + const model = catalog.find(({ alias }) => alias === VOICE_ACTIVATION_MODEL_ALIAS); + setVoiceActivationModelMissing(model?.cached === false); + }) + .catch(() => { + // The runtime event remains authoritative when the catalog cannot be read. + }); + return () => { + cancelled = true; + }; + }, [os, prefs?.voiceActivationEnabled]); + + useEffect(() => { + if (!prefs?.voiceActivationEnabled) setVoiceActivationModelMissing(false); + }, [prefs?.voiceActivationEnabled]); + const saveKeyboardListAffectingPrefs = useCallback( async (nextPrefs: UserPreferences) => { try { @@ -580,15 +650,133 @@ export function RecordingInputSection({ forcedOs }: { forcedOs?: OS } = {}) { /> {showDesktopHotkey && os === 'win' && ( - - - + <> + + + + + savePrefs({ ...prefs, voiceActivationEnabled: next })} + /> + + {prefs.voiceActivationEnabled && ( +
+
+ {t('settings.recording.voiceActivationLocalOnly')} +
+ {voiceActivationModelMissing && ( +
+ + {t('settings.recording.voiceActivationModelMissing')} + + {onOpenLocalModels && ( + + {t('settings.recording.voiceActivationDownloadZipformer')} + + )} +
+ )} + + savePrefs({ ...prefs, voiceActivationWakeEnabled: next })} + /> + + {prefs.voiceActivationWakeEnabled && ( + + savePrefs({ ...prefs, voiceActivationWakePhrase: event.currentTarget.value }) + } + style={{ ...inputStyle, width: '100%', margin: '2px 0 8px' }} + /> + )} + + savePrefs({ ...prefs, voiceActivationFinishEnabled: next })} + /> + + {prefs.voiceActivationFinishEnabled && ( + + savePrefs({ + ...prefs, + voiceActivationFinishPhrase: event.currentTarget.value, + }) + } + style={{ ...inputStyle, width: '100%', margin: '2px 0 8px' }} + /> + )} + + + savePrefs({ ...prefs, voiceActivationTimeoutEnabled: next }) + } + /> + + {prefs.voiceActivationTimeoutEnabled && ( + + savePrefs({ + ...prefs, + voiceActivationTimeoutSeconds: Math.max( + 5, + Math.min(300, Number(event.currentTarget.value) || 30), + ), + }) + } + style={{ ...inputStyle, width: 90, marginTop: 2 }} + /> + )} +
+ )} + )} item.id === 'inputMethod'), false, 'Harmony wiring does not add an Android input-method navigation entry'); -assert.deepEqual(visibleAdvancedPages('harmony', 'harmony').map((item) => item.id), ['testingFeatures', 'vocabularyLearning', 'customSyncServer', 'debug'], 'Harmony retains shared utility settings without desktop coding-agent controls'); +assert.equal( + visibleSettingsSections(false, 'harmony').some((item) => item.id === 'inputMethod'), + false, + 'Harmony wiring does not add an Android input-method navigation entry', +); +assert.deepEqual( + visibleAdvancedPages('harmony', 'harmony').map((item) => item.id), + ['testingFeatures', 'vocabularyLearning', 'customSyncServer', 'debug'], + 'Harmony retains shared utility settings without desktop coding-agent controls', +); for (const os of ['mac', 'win', 'android'] as const) { assert.equal( visibleAdvancedPages('desktop', os).some((item) => @@ -230,3 +238,23 @@ assert.deepEqual( ['lessComputer', 'claudeConsole'], 'macOS exposes both testing features after opt-in', ); + +assert.equal( + resolveServiceView('models', serviceViews, true, 'win', true), + 'models', + 'voice activation can prepare models without changing the multimodal pipeline', +); +assert.equal( + isServiceViewInactive('models', true, 'win', true), + false, + 'the model button and resolved view agree for voice activation', +); +for (const os of ['mac', 'harmony', 'android'] as const) { + assert.equal( + isServiceViewInactive('models', true, os, true), + true, + 'voice activation does not unlock models on other platforms', + ); +} +assert.equal(isServiceViewInactive('llm', true, 'win', true), true, 'LLM settings remain inactive'); +assert.equal(isServiceViewInactive('asr', true, 'win', true), true, 'ASR settings remain inactive'); diff --git a/openless-all/app/src/pages/settings/navigation.ts b/openless-all/app/src/pages/settings/navigation.ts index 2e0794501..6be04caf6 100644 --- a/openless-all/app/src/pages/settings/navigation.ts +++ b/openless-all/app/src/pages/settings/navigation.ts @@ -107,7 +107,13 @@ export function availableServiceViews(localModels: boolean, os?: OS): ServiceVie ]; } -export function isServiceViewInactive(view: ServiceViewId, multimodal: boolean, os?: OS): boolean { +export function isServiceViewInactive( + view: ServiceViewId, + multimodal: boolean, + os?: OS, + voiceActivationModels = false, +): boolean { + if (view === 'models' && os === 'win' && voiceActivationModels) return false; if (view === 'models' && os === 'mac') return true; return multimodal ? view !== 'omni' && view !== 'connections' : view === 'omni'; } @@ -117,7 +123,10 @@ export function resolveServiceView( available: ServiceViewId[], multimodal: boolean, os?: OS, + voiceActivationModels = false, ): ServiceViewId { - const enabled = available.filter((view) => !isServiceViewInactive(view, multimodal, os)); + const enabled = available.filter( + (view) => !isServiceViewInactive(view, multimodal, os, voiceActivationModels), + ); return enabled.includes(requested) ? requested : enabled[0]; } diff --git a/openless-all/app/src/pages/settings/tabs.tsx b/openless-all/app/src/pages/settings/tabs.tsx index 725f3cff7..77ae3ef56 100644 --- a/openless-all/app/src/pages/settings/tabs.tsx +++ b/openless-all/app/src/pages/settings/tabs.tsx @@ -64,13 +64,16 @@ function usePlatformCaps(): PlatformCapabilities | null { } // Recording & input: from recording to text insertion, plus phone input. -export function GeneralTab({ os }: { os?: OS } = {}) { +export function GeneralTab({ + os, + onOpenLocalModels, +}: { os?: OS; onOpenLocalModels?: () => void } = {}) { const platformCaps = usePlatformCaps(); const showRemoteInput = platformCaps?.platform === 'desktop'; return ( <> - + {showRemoteInput && } @@ -118,19 +121,31 @@ export function AppearanceTab() { } // AI services & models: organizes channels, local models, and network settings by current pipeline and platform capabilities. -export function ServicesTab({ asrNavigation = 0 }: { asrNavigation?: number } = {}) { +export function ServicesTab({ + asrNavigation = 0, + initialView, +}: { asrNavigation?: number; initialView?: ServiceViewId } = {}) { const { t } = useTranslation(); const { prefs, updatePrefs } = useHotkeySettings(); const platformCaps = usePlatformCaps(); const showLocalModel = detectOS() !== 'mac' && platformCaps?.supportsLocalAsr === true; const multimodal = prefs?.multimodalPipelineEnabled === true && prefs.pipelineMode === 'multimodal'; - const [view, setView] = useState(asrNavigation ? 'asr' : 'llm'); + const [view, setView] = useState(initialView ?? (asrNavigation ? 'asr' : 'llm')); useEffect(() => { - if (asrNavigation) setView('asr'); - }, [asrNavigation]); + if (initialView) setView(initialView); + else if (asrNavigation) setView('asr'); + }, [initialView, asrNavigation]); + const voiceActivationModels = + showLocalModel && detectOS() === 'win' && prefs?.voiceActivationEnabled === true; const views = availableServiceViews(showLocalModel, detectOS()); - const requestedView = resolveServiceView(view, views, multimodal, detectOS()); + const requestedView = resolveServiceView( + view, + views, + multimodal, + detectOS(), + voiceActivationModels, + ); const contentRef = useRef(null); const selectedView = usePageTransition(requestedView, contentRef, false); const pipelineRef = useRef(null); @@ -261,7 +276,12 @@ export function ServicesTab({ asrNavigation = 0 }: { asrNavigation?: number } = ); } - const inactive = isServiceViewInactive(id, multimodal, detectOS()); + const inactive = isServiceViewInactive( + id, + multimodal, + detectOS(), + voiceActivationModels, + ); const required = !inactive && (id === 'llm' || id === 'asr'); const configured = id === 'llm' ? requiredConfigured.llm : id === 'asr' ? requiredConfigured.asr : false; @@ -315,12 +335,25 @@ export function ServicesTab({ asrNavigation = 0 }: { asrNavigation?: number } =
{/* The LocalModelPicker inside the channel editor jumps to this view through this context. */} setView('models') : null} + value={ + showLocalModel && (!multimodal || voiceActivationModels) + ? () => setView('models') + : null + } > {selectedView === 'llm' && } {selectedView === 'asr' && } {selectedView === 'omni' && } - {showLocalModel && selectedView === 'models' && } + {showLocalModel && selectedView === 'models' && ( + <> + {multimodal && voiceActivationModels && ( +

+ {t('settings.recording.voiceActivationMultimodalModels')} +

+ )} + + + )} {selectedView === 'connections' && ( <>