From e705f4aded380becd08d8d1bd3f70a0e8ad5dfdb Mon Sep 17 00:00:00 2001 From: HKLHaoBin Date: Sun, 18 Oct 2026 08:51:04 +0800 Subject: [PATCH 01/12] feat: add Windows voice activation dictation Co-Authored-By: Claude Opus 5 --- docs/research/issue-1003.md | 197 +++ .../app/crates/openless-core/src/api.rs | 1122 +++++++++++++++-- .../src/cloud_sync_e2ee_documents/registry.rs | 49 + .../app/crates/openless-core/src/config.rs | 2 + .../app/crates/openless-core/src/events.rs | 65 + .../app/crates/openless-core/src/lib.rs | 13 +- .../app/crates/openless-core/src/ports.rs | 80 ++ .../crates/openless-core/src/shared_types.rs | 65 + .../openless-core/src/voice_activation.rs | 503 ++++++++ .../app/linux-egui/examples/headless_host.rs | 1 + openless-all/app/linux-egui/src/backend.rs | 1 + openless-all/app/linux-egui/src/runtime.rs | 1 + .../app/linux-egui/tests/host_contract.rs | 2 + .../backend-tests/tests/core_contract.rs | 1 + .../backend-tests/tests/voice_edit_host.rs | 1 + .../src-tauri/src/android/native_bridge.rs | 1 + .../src/asr/local/sherpa_provider.rs | 10 +- .../src-tauri/src/asr/local/sherpa_runtime.rs | 43 +- .../app/src-tauri/src/commands/misc.rs | 33 +- .../app/src-tauri/src/commands/mod.rs | 1 + .../app/src-tauri/src/commands/settings.rs | 55 +- .../app/src-tauri/src/commands/sherpa_asr.rs | 16 +- .../app/src-tauri/src/core_adapters.rs | 770 ++++++++++- openless-all/app/src-tauri/src/lib.rs | 16 +- openless-all/app/src-tauri/src/recorder.rs | 503 +++++++- .../app/src-tauri/src/tauri_events.rs | 6 + .../app/src/components/SettingsModal.tsx | 20 +- openless-all/app/src/i18n/de.ts | 12 + openless-all/app/src/i18n/en.ts | 12 + openless-all/app/src/i18n/es.ts | 12 + openless-all/app/src/i18n/fr.ts | 12 + openless-all/app/src/i18n/ja.ts | 12 + openless-all/app/src/i18n/ko.ts | 12 + openless-all/app/src/i18n/zh-CN.ts | 14 +- openless-all/app/src/i18n/zh-TW.ts | 11 + openless-all/app/src/lib/ipc/mock-data.ts | 7 + openless-all/app/src/lib/types.ts | 8 + .../app/src/lib/voiceActivation.test.ts | 43 + openless-all/app/src/lib/voiceActivation.ts | 30 + .../pages/settings/RecordingInputSection.tsx | 208 ++- openless-all/app/src/pages/settings/tabs.tsx | 18 +- 41 files changed, 3781 insertions(+), 207 deletions(-) create mode 100644 docs/research/issue-1003.md create mode 100644 openless-all/app/crates/openless-core/src/voice_activation.rs create mode 100644 openless-all/app/src/lib/voiceActivation.test.ts create mode 100644 openless-all/app/src/lib/voiceActivation.ts diff --git a/docs/research/issue-1003.md b/docs/research/issue-1003.md new file mode 100644 index 000000000..2925ccbb2 --- /dev/null +++ b/docs/research/issue-1003.md @@ -0,0 +1,197 @@ +# Issue #1003 语音激活输入法研究报告 + +> 调研日期：2026-10-18 +> +> 研究范围：GitHub Issue #1003、PR #1219、PR 关联提交，以及 `beta` 与 `feat/issue-1003-voice-activation` 的源码。本文只记录可追溯的主源证据；没有把 PR 的声明当作本次本地运行结果。 + +## 1. 结论摘要 + +| 问题 | 结论 | 证据 | +| --- | --- | --- | +| 功能是否已经进入当前 `beta` | **没有。** Issue 仍 OPEN；PR #1219 仍 OPEN、Draft、`REVIEW_REQUIRED`、`BLOCKED`，没有 merge commit。当前 `beta` 是 `de072a88`，而 PR head 是 fork `HKLHaoBin/openless` 的 `9270b43c`。 | [Issue #1003](https://github.com/Open-Less/openless/issues/1003)、[PR #1219](https://github.com/Open-Less/openless/pull/1219) | +| 分支落后情况 | 以当前 `beta` `de072a88373a222280a8c22948092a260880196c` 和功能分支 `9270b43c7e79ad9d462ad8bec78250de5f499093` 比较：功能分支 **领先 8、落后 3**，两边分叉，合并基点为 `96278f40d48b8ed02e3ce22ddc96b694c863a6bc`。落后的 3 个提交是当前 beta 新增的 `af8adf7e`、`5ab6ed43`、`de072a88`。 | [PR base/head 元数据](https://github.com/Open-Less/openless/pull/1219)、[功能分支提交](https://github.com/HKLHaoBin/openless/commits/feat/issue-1003-voice-activation)；本地 `git rev-list beta..feat/issue-1003-voice-activation` 与反向计数为 8/3 | +| 重启后持久化 `voiceActivationEnabled` 是否自启动 | **条件性地是。** `voice_activation_enabled` 以偏好字段持久化；应用第一次调用 `get_startup_snapshot` 时，Core `start()` 读取当前偏好并调用 `configure_voice_activation()`。模型已安装、权限可用且麦克风未占用时会进入等待唤醒；模型缺失、权限拒绝或麦克风忙时只会报告失败状态。 | [`config.rs:395-465`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/config.rs#L395-L465)、[`App.tsx:84-96`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src/App.tsx#L84-L96)、[`shared.ts:45-57`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src/lib/ipc/shared.ts#L45-L57)、[`api.rs:4530-4609`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/api.rs#L4530-L4609) | +| 连续监听是否有 VAD/CPU gate | **没有找到。** 空闲 warm capture 的每个 PCM chunk 都会直接交给 activation consumer；Sherpa online worker 对每个收到的 PCM 调用 `accept_pcm_chunk`。现有环形缓冲和录音队列是容量/线程安全机制，不是静音跳过或 CPU 门控。 | [`recorder.rs:594-645`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/recorder.rs#L594-L645)、[`sherpa_provider.rs:244-255`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs#L244-L255)、[`sherpa_provider.rs:292-343`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs#L292-L343) | +| 结束词带标点是否清理 | **已在功能分支实现。** NFKC、空白和终端/中英文标点会参与归一化；匹配仅接受文本后缀；从原始 transcript 尾部逆向剥离结束词时跳过标点和空白，保留正文本身的标点。测试覆盖 `完成，语音录入。`、全角句号、正文标点和中间出现结束词的反例。 | [`voice_activation.rs:49-115`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/voice_activation.rs#L49-L115)、[`voice_activation.rs:390-445`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/voice_activation.rs#L390-L445)、[`api.rs:2264-2321`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/api.rs#L2264-L2321) | + +**一句话判断：** PR 已完成“本地 Windows 语音唤醒/结束词”的主体链路以及结束词标点修复，但尚未合并到当前 beta；持续监听仍把静音 PCM 全部送入 Zipformer，CPU/VAD 问题是合并前必须明确验收或拆分处理的风险。 + +## 2. 主源与需求演进 + +### 2.1 原始需求 + +Issue 标题为“功能建议：用语音替代按键来激活输入法”。原始描述提出：说“开始语音录入”，录入正文，再说“完成语音录入”（或等待超时）；使用较书面的词以降低日常口语误激活，并希望命令可自定义、识别尽可能本地化。[Issue #1003](https://github.com/Open-Less/openless/issues/1003) + +### 2.2 讨论中新增的约束 + +1. **按键替代场景。** 评论说明右 Ctrl/快捷键会干扰肌肉记忆，也有人用 ScrLock 或鼠标侧键折衷。[评论 5436896181](https://github.com/Open-Less/openless/issues/1003#issuecomment-5436896181)、[评论 5441322107](https://github.com/Open-Less/openless/issues/1003#issuecomment-5441322107) +2. **常驻监听的系统成本。** 评论明确提出全天开麦可能耗电、持续占用 CPU，并可能占用 Windows 麦克风导致其他应用难以使用。[评论 5596617615](https://github.com/Open-Less/openless/issues/1003#issuecomment-5596617615) +3. **本地模型引导。** 维护者在 2026-10-14 发布模型缺失引导及 Windows 测试步骤，要求模型未安装时给出跳转按钮，安装后重新配置/回到设置确认进入等待唤醒。[评论 6170844277](https://github.com/Open-Less/openless/issues/1003#issuecomment-6170844277) +4. **结束词的 ASR 断句。** 用户报告“完成，语音录入。”会被混入正文；随后 PR 提交 `37e4665d` 宣称修复该问题并增加回归测试。[评论 6216314465](https://github.com/Open-Less/openless/issues/1003#issuecomment-6216314465)、[提交 37e4665d](https://github.com/HKLHaoBin/openless/commit/37e4665d78a39011431401393264002a4879c197) +5. **VAD/CPU 争议。** 用户建议无人在说话时不要把音频交给 Zipformer；维护者先认为额外人声模型可能增加 CPU，之后在讨论中接受这是应解决的优化，并表示可另行拆分以保持 PR 可维护。[评论 6218184776](https://github.com/Open-Less/openless/issues/1003#issuecomment-6218184776)、[评论 6219721968](https://github.com/Open-Less/openless/issues/1003#issuecomment-6219721968)、[评论 6221936954](https://github.com/Open-Less/openless/issues/1003#issuecomment-6221936954)、[评论 6231986920](https://github.com/Open-Less/openless/issues/1003#issuecomment-6231986920)、[评论 6236232646](https://github.com/Open-Less/openless/issues/1003#issuecomment-6236232646) + +### 2.3 PR 及修复提交序列 + +- `b0196a9e`：加入可配置 voice activation 的主体实现。[提交 b0196a9e](https://github.com/HKLHaoBin/openless/commit/b0196a9ed9bb1358c4d4bb1fb9b7cbf8ca4c66b3) +- `65afa352`：补充模型缺失引导。[提交 65afa352](https://github.com/HKLHaoBin/openless/commit/65afa3521be38cf1c143f3ddff1635fc18fa99d3) +- `37e4665d`：容忍结束词中间插入标点，并清理 transcript。[提交 37e4665d](https://github.com/HKLHaoBin/openless/commit/37e4665d78a39011431401393264002a4879c197) +- `d66c07c4`、`3e56d3a3`：功能分支合入当时的 beta；提交信息显示合并时曾处理多处冲突。[提交 d66c07c4](https://github.com/HKLHaoBin/openless/commit/d66c07c4f4a60d6fb5af4acf14aef3c94b2676cd)、[提交 3e56d3a3](https://github.com/HKLHaoBin/openless/commit/3e56d3a3c214a474cff10df34810c18660c6bf46) +- `9270b43c`：更新 warm capture 测试以适配 preroll 标志，当前 PR head。[提交 9270b43c](https://github.com/HKLHaoBin/openless/commit/9270b43c7e79ad9d462ad8bec78250de5f499093) + +PR #1219 的 body 写明 `Closes #1003`，并列出 Core 测试、格式检查、Bugbot 复审和三个 Actions 运行，但 PR 状态本身仍是 Draft/待 Review。[PR #1219](https://github.com/Open-Less/openless/pull/1219) + +## 3. 当前 beta 与 PR 状态 + +### 3.1 合并状态 + +截至调研日期，GitHub PR 元数据为： + +- base：`Open-Less/openless:beta`，PR 记录的 base SHA 为 `96278f40`； +- head：`HKLHaoBin/openless:feat/issue-1003-voice-activation`，head SHA 为 `9270b43c`； +- `state: OPEN`、`isDraft: true`、`reviewDecision: REVIEW_REQUIRED`、`mergeStateStatus: BLOCKED`； +- `mergeCommit: null`、`mergedAt: null`。 + +这些字段来自 [PR #1219](https://github.com/Open-Less/openless/pull/1219)。因此，功能分支曾把 beta 合入自身，**不能反推 beta 已经合入功能分支**；当前 beta 仍未包含该 PR。当前 beta 的本地 ref 为 `de072a88`，且功能新增文件 `openless-all/app/crates/openless-core/src/voice_activation.rs` 在当前 beta 中不存在。 + +### 3.2 落后/领先关系 + +以本地已存在的两个 ref 做纯 Git 图比较： + +```text +beta = de072a88373a222280a8c22948092a260880196c +feature = 9270b43c7e79ad9d462ad8bec78250de5f499093 +feature ahead of beta = 8 +feature behind beta = 3 +merge-base = 96278f40d48b8ed02e3ce22ddc96b694c863a6bc +``` + +功能分支落后的三个提交分别是： + +- `af8adf7e fix(dictation): keep local startup stoppable and bound inference resources`； +- `5ab6ed43 feat(capsule): add persisted multi-display modes and prepare 2.0.2 Beta.1`； +- `de072a88 docs(release): verified 2.0.2 Beta.1 publication evidence [skip ci]`。 + +PR 的 base/head 和当前分支提交可在 [PR #1219](https://github.com/Open-Less/openless/pull/1219) 与 [功能分支提交列表](https://github.com/HKLHaoBin/openless/commits/feat/issue-1003-voice-activation) 复核。合并前应以当前 beta 重新合并/变基并重新解决冲突，不能直接使用 PR 创建时的 base 作为最新基线。 + +### 3.3 检查状态 + +PR 当前列出的检查均为通过，包括 Android cargo check、Linux Core/egui、Windows checks、macOS checks、macOS Rust 1.88 MSRV、UI motion 和 Changed areas。[PR checks](https://github.com/Open-Less/openless/pull/1219/checks) + +这些是 CI/PR 的已有证据，不是本次调研重新运行的结果；本次没有修改业务代码、没有运行构建，也没有宣称已经在 Windows 实机复现。 + +## 4. 代码影响范围 + +PR 相对其 base 的 GitHub 统计为 **41 个文件、+3227/-201**。[PR #1219 changed files](https://github.com/Open-Less/openless/pull/1219/files) + +| 层次 | 主要影响 | 关键源码证据 | +| --- | --- | --- | +| Core policy/state | 新增 phrase matcher、生命周期/代际、超时、状态信号、端口；`OpenLessBackend` 增加配置、唤醒 handoff、结束和恢复。 | [`voice_activation.rs`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/voice_activation.rs)、[`api.rs:3093-3285`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/api.rs#L3093-L3285)、[`ports.rs`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/ports.rs) | +| 偏好/持久化 | 增加启用、唤醒/结束开关、两条 phrase、超时和麦克风字段，并保留旧配置默认值。 | [`config.rs:395-465`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/config.rs#L395-L465)、[`config.rs:978-1028`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/config.rs#L978-L1028) | +| Windows host/音频 | Tauri 运行时加载 Zipformer、创建在线 Sherpa session、接入 warm capture、处理 mic/permission/model 错误和 generation。 | [`core_adapters.rs:1492-1851`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/core_adapters.rs#L1492-L1851)、[`recorder.rs:332-672`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/recorder.rs#L332-L672) | +| 本地 ASR | Sherpa provider/runtime 支持在线 chunk、token callback 和结束 flush。 | [`sherpa_provider.rs:78-255`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs#L78-L255)、[`sherpa_provider.rs:274-343`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs#L274-L343) | +| UI/IPC/本地化 | Recording Input 增加 Windows-only voice activation 设置、状态和模型缺失跳转；同步 mock/types/i18n。 | [`RecordingInputSection.tsx:146-200`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src/pages/settings/RecordingInputSection.tsx#L146-L200)、[`RecordingInputSection.tsx:590-660`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src/pages/settings/RecordingInputSection.tsx#L590-L660)、[`voiceActivation.ts:1-30`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src/lib/voiceActivation.ts#L1-L30) | +| 测试/构建面 | Core phrase/lifecycle 测试、UI model-missing 测试、host contract、workflow 依赖变更。 | [`voice_activation.rs:390-503`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/voice_activation.rs#L390-L503)、[`voiceActivation.test.ts`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src/lib/voiceActivation.test.ts) | + +## 5. 已实现能力与行为链路 + +### 5.1 持久化与启动 + +`UserPreferences` 将 `voice_activation_enabled` 及相关字段纳入 camelCase wire contract，并给新增字段提供默认值。[`config.rs:395-465`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/config.rs#L395-L465) + +启动链路是： + +```text +App.tsx useEffect + -> getStartupSnapshot() + -> invoke('get_startup_snapshot') + -> commands::get_startup_snapshot() + -> core.start() + -> get_preferences() + -> configure_voice_activation() + -> model check / Sherpa online listener / ListeningForWake +``` + +前端启动握手见 [`App.tsx:84-96`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src/App.tsx#L84-L96) 与 [`shared.ts:45-57`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src/lib/ipc/shared.ts#L45-L57)；命令和 Core 的调用见 [`commands/dictation.rs:3-8`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/commands/dictation.rs#L3-L8)、[`api.rs:4530-4609`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/api.rs#L4530-L4609)。因此，重启后的“自启动”不是偏好字段自身执行，而是应用首个 startup handshake 触发；若模型/权限/麦克风条件不满足，启动会发出失败状态而不会进入监听。 + +设置保存时，启用状态、phrase、超时和麦克风字段的变化会再次调用 `configure_voice_activation()`。[`commands/settings.rs:237-265`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/commands/settings.rs#L237-L265)、[`commands/settings.rs:284-317`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/commands/settings.rs#L284-L317) + +### 5.2 唤醒、交接、超时和恢复 + +Windows runtime 检查固定的 `zipformer-bilingual-zh-en-streaming`，模型存在后创建在线 Sherpa ASR，并把 token callback 接到 `VoiceActivationEmitter`。[`core_adapters.rs:1701-1765`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/core_adapters.rs#L1701-L1765) + +唤醒匹配后，Core 通过 generation 校验、warm-capture handoff 建立普通 dictation session，再保存 `VoiceActivationSession`；结束词或超时调用 `finish_voice_activation`，结束后按启用偏好重新配置监听。[`api.rs:3292-3528`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/api.rs#L3292-L3528)、[`api.rs:3565-3619`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/api.rs#L3565-L3619)。配置注释和停止旧 handle 的单麦克风约束见 [`api.rs:3093-3125`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/api.rs#L3093-L3125)。 + +### 5.3 结束词标点清理 + +实现分两层： + +1. `normalize_phrase` 用 NFKC、去空白和标点处理 ASR phrase；`VoicePhraseMatcher` 只认文本后缀，避免正文中间出现 phrase 就误停。[`voice_activation.rs:49-60`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/voice_activation.rs#L49-L60)、[`voice_activation.rs:201-229`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/voice_activation.rs#L201-L229) +2. `strip_phrase_suffix` 从原始 transcript 尾部反向扫描，跳过空白/中英文终端标点，仅切掉结束词；Core 在每个 `TranscriptDelta` 累积 raw transcript 后，将清理后的文字写回 state 并发布实时事件。[`voice_activation.rs:62-115`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/voice_activation.rs#L62-L115)、[`api.rs:2264-2321`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/api.rs#L2264-L2321) + +结束录音后，完成路径还会再次对 `engine_result.raw_text`、`engine_result.polished_text` 和可选 `engine_result.asr_transcript` 调用同一 suffix cleaner，再构造 `DictationResult`；对应实现见 [`api.rs:7291-7317`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/api.rs#L7291-L7317)。这些清理后的字段随后用于完成事件和历史持久化，历史写入字段见 [`api.rs:7470-7479`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/api.rs#L7470-L7479)、[`api.rs:7665-7752`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/api.rs#L7665-L7752)。该结论仍应通过 Windows 实机验证，因为不同 ASR 的 delta/revision 时序不等于单元测试输入。 + +### 5.4 模型缺失引导 + +runtime 在配置时检查模型是否安装，缺失返回 `ModelMissing`；UI 监听 `voice-activation:state-changed`，只对精确的模型缺失错误显示跳转模型页，避免把权限拒绝或麦克风占用误导成下载问题。[`core_adapters.rs:1718-1734`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/core_adapters.rs#L1718-L1734)、[`RecordingInputSection.tsx:146-200`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src/pages/settings/RecordingInputSection.tsx#L146-L200)、[`voiceActivation.ts:21-30`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src/lib/voiceActivation.ts#L21-L30) + +## 6. 可复现缺口与风险 + +### 6.1 明确缺口：连续监听没有 VAD/CPU gate + +代码路径是无条件转发： + +- `WarmCaptureSink::consume_pcm_chunk` 先写 ring，然后在满足 activation 条件时直接 `activation.consume_pcm_chunk(pcm)`，没有能量阈值、VAD 状态或静音短路。[`recorder.rs:594-645`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/recorder.rs#L594-L645) +- `SherpaOnnxAsr::consume_pcm_chunk` 对 online worker 直接 `send_pcm(pcm)`；worker 收到每个 `Pcm` 后立即调用 `session.accept_pcm_chunk(&pcm, &emit)`。[`sherpa_provider.rs:244-255`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs#L244-L255)、[`sherpa_provider.rs:292-310`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs#L292-L310) +- `send_pcm` 使用标准无界 `mpsc::channel`，只检查空 chunk/取消，不检查音量或丢弃静音。[`sherpa_provider.rs:274-343`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs#L274-L343) + +**复现方法：** Windows 安装并准备 Zipformer，开启语音激活，在安静房间保持 60 秒不说话；记录进程 CPU、Sherpa worker 消费量和 microphone 独占状态，再以相同条件说唤醒词。预期若有 gate，静音段应显著减少 inference；当前静态路径预期仍逐块送入模型。该实验尚未由本次调研运行。 + +### 6.2 模型下载后自动恢复未形成闭环的风险 + +当前明确调用 `configure_voice_activation()` 的位置是 startup、偏好保存、warm fallback 释放和运行时失败恢复；本地模型下载命令本身只调用 local-ASR service 的 `start_download`。[`api.rs:4530-4609`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/api.rs#L4530-L4609)、[`commands/settings.rs:284-317`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/commands/settings.rs#L284-L317)、[`commands/local_asr.rs:329-344`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/commands/local_asr.rs#L329-L344)。因此“下载完成后当前 listener 自动恢复”没有直接源码证据；Issue 的测试说明也要求返回录音设置或重新配置。[评论 6170844277](https://github.com/Open-Less/openless/issues/1003#issuecomment-6170844277) + +这不一定是已确认 bug：下载流程可能由前端刷新或用户切换设置间接触发，需实机确认。若产品要求无刷新自动恢复，应添加下载完成事件到 voice activation reconfigure 的显式连接及回归测试。 + +### 6.3 麦克风占用、权限和模型缺失是启动条件，不是静默降级 + +配置 runtime 会检查模型；arm warm capture 失败时映射为权限拒绝、无输入设备、mic busy 或 runtime failure。[`core_adapters.rs:1498-1509`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/core_adapters.rs#L1498-L1509)、[`core_adapters.rs:1723-1795`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/core_adapters.rs#L1723-L1795)。应验证失败后热键录音仍可正常工作，且其他应用占用麦克风时 UI 不显示错误的“下载模型”建议；现有 UI 的错误分类测试只覆盖状态判别，未覆盖 Windows 设备竞争。[`voiceActivation.test.ts:12-42`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src/lib/voiceActivation.test.ts#L12-L42) + +### 6.4 分支基线漂移与合并风险 + +功能分支已通过两个 merge commit 合入旧 beta，但当前 beta 又前进 3 个提交；且 PR 仍 blocked/review required。合并前必须用当前 beta 重新整合并复跑全部桌面、Android、Core/UI 检查，尤其关注 PR 曾经修改的 `api.rs`、`core_adapters.rs`、`recorder.rs`、settings UI 和 workflow 文件。[PR #1219](https://github.com/Open-Less/openless/pull/1219)、[PR changed files](https://github.com/Open-Less/openless/pull/1219/files) + +## 7. 下一步实现建议 + +1. **先同步当前 beta，再决定 PR 边界。** 以 `de072a88` 为基线整合功能，解决冲突后重新跑 PR 全套检查；不要把“功能分支合入旧 beta”当作“beta 已合并”。 +2. **为静音成本设定可测验收。** 在不改变唤醒召回率的前提下加入轻量、可解释的能量/频域 gate 或 sherpa 自带 VAD（若评估后成本可接受）；记录静音/有人声两类 CPU、延迟、漏唤醒和误唤醒指标。Issue 已有 VAD 建议和后续接受该优化方向的讨论。[评论 6218184776](https://github.com/Open-Less/openless/issues/1003#issuecomment-6218184776)、[评论 6236232646](https://github.com/Open-Less/openless/issues/1003#issuecomment-6236232646) +3. **避免无界积压。** 目前在线 PCM 使用无界 `mpsc::channel`；若推理速度低于采集速度，建议增加有界队列、明确丢帧/降采样策略和故障遥测，再用长时间静音及低性能 Windows 机器验证。[`sherpa_provider.rs:274-343`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs#L274-L343) +4. **闭合模型生命周期。** 模型下载成功后应触发一次明确的 voice activation reconfigure，或在 UI 中明确要求重新配置；覆盖“启用已保存 + 模型缺失 → 下载 → 无需重启即监听”的测试。 +5. **补充真实设备验收。** 覆盖重启自动启动、权限拒绝、麦克风被其他应用占用、模型缺失/下载、热键与语音入口互斥、结束词断句、超时后自动恢复监听。 +6. **把结束词清理保持在 raw/source 层验证。** 继续保留正文自身标点，测试普通逗号、全角标点、空格/换行、ASR revision、结束词出现在正文中间等组合，并核对实时 transcript、最终插入、history raw/asr/final 三套字段。[`voice_activation.rs:394-445`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/voice_activation.rs#L394-L445)、[`api.rs:2264-2321`](https://github.com/HKLHaoBin/openless/blob/9270b43c7e79ad9d462ad8bec78250de5f499093/openless-all/app/crates/openless-core/src/api.rs#L2264-L2321) + +## 8. 验证清单 + +### 8.1 静态/CI + +- [x] `gh issue view 1003`：Issue 状态、原始需求和评论已核对。[Issue #1003](https://github.com/Open-Less/openless/issues/1003) +- [x] `gh pr view 1219`：base/head、Draft、Review、merge 字段已核对。[PR #1219](https://github.com/Open-Less/openless/pull/1219) +- [x] Git 图：当前 beta `de072a88` 与 head `9270b43c` 的 ahead/behind 为 8/3。 +- [x] PR 已有检查记录均为 pass。[PR checks](https://github.com/Open-Less/openless/pull/1219/checks) +- [x] 静态阅读确认启动、持久化、标点清理和每块 PCM 转发路径。 +- [ ] 以当前 beta 重新整合功能分支并重跑 CI（本次调研未执行）。 +- [ ] 运行 `cargo test -p openless-core`、voice activation 定向测试、`cargo fmt --all -- --check`（本次调研未执行；PR body 声称曾通过）。 + +### 8.2 Windows 实机 + +- [ ] 在设置 → 录音输入启用语音激活，确认安装模型后状态为等待唤醒。 +- [ ] 关闭并重启应用，确认已持久化的 `voiceActivationEnabled` 在 startup handshake 后重新进入等待唤醒；分别测试模型存在、模型缺失、权限拒绝、麦克风忙。 +- [ ] 说“开始语音录入”→正文→“完成语音录入”，确认唤醒词/结束词不进入正文。 +- [ ] 重复测试“完成，语音录入。”、“完成语音录入。”、句末全角标点、词间空格/换行；确认正文原有标点保留。 +- [ ] 语音激活开启后保持 60 秒静音，记录 CPU、内存、Sherpa inference 调用/队列量；与有人声的同长度样本比较，确认是否存在 VAD/CPU gate。 +- [ ] 下载模型后不重启，仅按产品预期操作（返回设置或等待事件），确认 listener 是否自动恢复；若不能，记录为模型生命周期缺口。 +- [ ] 连续完成两轮唤醒/正文/结束，验证结束后 listener 自动恢复；超时也应恢复。 +- [ ] 其他应用占用麦克风时，错误应归类为 mic busy/permission，不应显示错误的模型下载按钮；关闭语音激活后原热键入口仍可用。 + +## 9. 证据边界 + +本报告的“已实现”结论来自功能分支 `9270b43c` 的源码和测试；“当前 beta 未合并”来自 GitHub PR 状态、base/head 元数据和本地 Git 图；“VAD/CPU gate 缺失”是对上述 PCM 调用链的静态审计结论，不是 CPU profiler 结果。Issue 评论中的 Windows 测试包和 Actions 链接属于第一方项目记录，但本次没有下载或运行该产物。 diff --git a/openless-all/app/crates/openless-core/src/api.rs b/openless-all/app/crates/openless-core/src/api.rs index ee9124023..25395f749 100644 --- a/openless-all/app/crates/openless-core/src/api.rs +++ b/openless-all/app/crates/openless-core/src/api.rs @@ -1,7 +1,7 @@ mod cloud_note; use futures_util::future::BoxFuture; use std::collections::HashMap; -use std::sync::atomic::{AtomicU64, AtomicU8, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicU8, Ordering}; use std::sync::{Arc, Mutex, RwLock}; use crate::activity::ActivityDay; @@ -39,7 +39,7 @@ use crate::style_packs::StylePack; use crate::types::{ CorrectionRule, DictationPhase, DictationResult, DictationSession, DictationStateSnapshot, DictionaryEntry, HistoryChange, HistoryInsertStatus, HistorySource, PreferencesChange, - SessionId, StylePackChange, VocabPresetStore, VocabularyChange, + SessionId, StylePackChange, TranscriptAccumulator, VocabPresetStore, VocabularyChange, }; use crate::vocabulary::DictionaryStore; use crate::{ActivityStore, CorrectionRuleStore, HistoryStore, PreferencesStore, StylePackStore}; @@ -128,12 +128,18 @@ struct DictationReservation { pending_capture: Option>, } +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +enum PendingAudioCaptureKey { + Hotkey(u64), + VoiceActivation(u64), +} + /// Removes a provisional capture if the async hotkey dispatch is aborted before /// it transfers ownership to a session reservation. struct PendingAudioCaptureAbortGuard { - captures: Arc>>>, + captures: Arc>>>, task_spawner: Arc, - press_id: u64, + key: PendingAudioCaptureKey, } impl Drop for PendingAudioCaptureAbortGuard { @@ -142,7 +148,7 @@ impl Drop for PendingAudioCaptureAbortGuard { .captures .lock() .expect("pending audio capture lock poisoned") - .remove(&self.press_id); + .remove(&self.key); let Some(capture) = capture else { return; }; @@ -2130,6 +2136,8 @@ struct BackendEngineProgress { state: Arc>, phase_changed: Arc, text_insertions: Arc>>, + voice_activation_session: Arc>>, + raw_transcripts: Mutex>, } impl EngineProgressSink for BackendEngineProgress { @@ -2253,7 +2261,14 @@ impl EngineProgressSink for BackendEngineProgress { self.phase_changed.notify_waiters(); } } - EngineProgress::TranscriptDelta(delta) => { + EngineProgress::TranscriptDelta(mut delta) => { + let finish_phrase = self + .voice_activation_session + .lock() + .expect("voice activation session lock poisoned") + .as_ref() + .filter(|session| session.finish_enabled && session.session_id == session_id) + .map(|session| session.finish_phrase.clone()); let mut state = self.state.write().expect("backend state lock poisoned"); ensure_active_session(&state, session_id)?; if !matches!( @@ -2267,6 +2282,34 @@ impl EngineProgressSink for BackendEngineProgress { "transcript delta arrived after transcription completed", )); } + if let Some(finish_phrase) = finish_phrase { + let mut raw_transcripts = self + .raw_transcripts + .lock() + .expect("raw transcript lock poisoned"); + let raw_transcript = raw_transcripts.entry(session_id).or_insert_with(|| { + let mut transcript = TranscriptAccumulator::default(); + if let Some(existing) = state.transcripts.get(&session_id) { + let seed = crate::types::TranscriptDelta { + text: existing.text().to_string(), + offset: 0, + is_final: false, + }; + let _ = transcript.apply(&seed); + } + transcript + }); + raw_transcript.apply(&delta)?; + delta = crate::types::TranscriptDelta { + text: crate::voice_activation::strip_phrase_suffix( + raw_transcript.text(), + &finish_phrase, + ) + .unwrap_or_else(|| raw_transcript.text().to_string()), + offset: 0, + is_final: delta.is_final, + }; + } state .transcripts .entry(session_id) @@ -2315,7 +2358,8 @@ pub struct OpenLessBackend { phase_changed: Arc, hotkey: Mutex, hotkey_dispatch_gate: tokio::sync::Mutex<()>, - pending_audio_captures: Arc>>>, + pending_audio_captures: + Arc>>>, less_computer_hotkey_press_at: Mutex>, vocabulary: Arc, correction_rules: Arc, @@ -2339,6 +2383,27 @@ pub struct OpenLessBackend { voice_sessions: Arc, runtime_start_work: Arc, less_computer_voice_controls: Arc>>>, + voice_activation_handle: Arc>>>, + voice_activation_generation: Arc, + voice_activation_handoff_generation: AtomicU64, + voice_activation_configure_gate: tokio::sync::Mutex<()>, + backend_lifecycle_gate: tokio::sync::Mutex<()>, + backend_shutdown_requested: AtomicBool, + voice_activation_session: Arc>>, +} + +struct VoiceActivationHandoffGuard { + backend: Arc, + generation: u64, +} + +impl Drop for VoiceActivationHandoffGuard { + fn drop(&mut self) { + let _ = self + .backend + .voice_activation_handoff_generation + .compare_exchange(self.generation, 0, Ordering::AcqRel, Ordering::Acquire); + } } #[path = "cloud_sync_e2ee/api.rs"] @@ -2504,6 +2569,93 @@ impl HistoryProviderAttribution { } } +struct CoreVoiceActivationSink { + events: Arc, +} + +impl crate::ports::VoiceActivationSink for CoreVoiceActivationSink { + fn publish( + &self, + signal: crate::voice_activation::VoiceActivationSignal, + ) -> Result<(), BackendError> { + use crate::voice_activation::{VoiceActivationPhase, VoiceActivationSignal}; + let (generation, phase, session_id) = match signal { + VoiceActivationSignal::WakeDetected { + activation_generation, + } => ( + activation_generation, + VoiceActivationPhase::WakeDetected, + None, + ), + VoiceActivationSignal::FinishDetected { + activation_generation, + session_id, + } => ( + activation_generation, + VoiceActivationPhase::Finishing, + Some(session_id), + ), + VoiceActivationSignal::RuntimeStateChanged { + activation_generation, + phase, + } => (activation_generation, phase, None), + }; + self.events.publish( + None, + BackendEventKind::VoiceActivationStateChanged( + crate::voice_activation::VoiceActivationStateSnapshot { + phase, + activation_generation: generation, + session_id, + message: None, + }, + ), + ); + Ok(()) + } +} + +fn map_voice_activation_error(error: crate::ports::VoiceActivationError) -> BackendError { + let (code, message) = match error { + crate::ports::VoiceActivationError::Unsupported => ( + BackendErrorCode::Unsupported, + "voice activation is unsupported".into(), + ), + crate::ports::VoiceActivationError::ModelMissing => ( + BackendErrorCode::InvalidState, + "voice activation model is not installed".into(), + ), + crate::ports::VoiceActivationError::PermissionDenied => ( + BackendErrorCode::PermissionDenied, + "microphone permission was denied".into(), + ), + crate::ports::VoiceActivationError::MicBusy => ( + BackendErrorCode::Busy, + "microphone is already in use".into(), + ), + crate::ports::VoiceActivationError::InvalidConfig(message) + | crate::ports::VoiceActivationError::Runtime(message) => { + (BackendErrorCode::InvalidState, message) + } + }; + BackendError::new(code, message) +} + +fn voice_activation_failure_phase( + code: BackendErrorCode, +) -> crate::voice_activation::VoiceActivationPhase { + match code { + BackendErrorCode::Unsupported => { + crate::voice_activation::VoiceActivationPhase::Unavailable + } + BackendErrorCode::PermissionDenied => { + crate::voice_activation::VoiceActivationPhase::PermissionDenied + } + BackendErrorCode::Busy => crate::voice_activation::VoiceActivationPhase::MicBusy, + _ => crate::voice_activation::VoiceActivationPhase::Failed, + } +} + impl OpenLessBackend { /// Construct an explicitly unavailable Core so the native shell can show /// its startup error instead of panicking before any window is visible. @@ -2945,6 +3097,13 @@ impl OpenLessBackend { voice_sessions, runtime_start_work, less_computer_voice_controls: Arc::new(Mutex::new(HashMap::new())), + voice_activation_handle: Arc::new(Mutex::new(None)), + voice_activation_generation: Arc::new(AtomicU64::new(0)), + voice_activation_handoff_generation: AtomicU64::new(0), + voice_activation_configure_gate: tokio::sync::Mutex::new(()), + backend_lifecycle_gate: tokio::sync::Mutex::new(()), + backend_shutdown_requested: AtomicBool::new(false), + voice_activation_session: Arc::new(Mutex::new(None)), }) } @@ -2966,6 +3125,646 @@ impl OpenLessBackend { Ok(()) } + /// Configure the opt-in local voice activation listener from the current + /// preference snapshot. Reconfiguration replaces the previous native + /// handle only after it has been stopped, so one process owns one mic. + pub async fn configure_voice_activation(&self) -> Result<(), BackendError> { + let _configure_gate = self.voice_activation_configure_gate.lock().await; + let preferences = self.get_preferences(); + if self + .voice_activation_session + .lock() + .expect("voice activation session lock poisoned") + .is_some() + && preferences.voice_activation_enabled + { + return Ok(()); + } + // Invalidate callbacks before awaiting the old runtime. Sherpa may have + // queued a final callback while its cancellation is being observed. + let generation = self + .voice_activation_generation + .fetch_add(1, Ordering::AcqRel) + + 1; + self.voice_activation_handoff_generation + .store(0, Ordering::Release); + self.voice_activation_session + .lock() + .expect("voice activation session lock poisoned") + .take(); + let previous = self + .voice_activation_handle + .lock() + .expect("voice activation handle lock poisoned") + .take(); + if let Some(previous) = previous { + let _ = previous + .stop(crate::voice_activation::VoiceActivationStopReason::Disabled) + .await; + } + if !preferences.voice_activation_enabled + || self.backend_shutdown_requested.load(Ordering::Acquire) + || !self.snapshot().running + { + self.events.publish( + None, + BackendEventKind::VoiceActivationStateChanged( + crate::voice_activation::VoiceActivationStateSnapshot { + phase: crate::voice_activation::VoiceActivationPhase::Disabled, + activation_generation: generation, + session_id: None, + message: None, + }, + ), + ); + return Ok(()); + } + let Some(runtime) = self.deps.voice_activation_runtime.as_ref() else { + self.events.publish( + None, + BackendEventKind::VoiceActivationStateChanged( + crate::voice_activation::VoiceActivationStateSnapshot { + phase: crate::voice_activation::VoiceActivationPhase::Unavailable, + activation_generation: generation, + session_id: None, + message: Some("voice activation host runtime is unavailable".into()), + }, + ), + ); + return if preferences.voice_activation_enabled { + Err(BackendError::new( + BackendErrorCode::Unsupported, + "voice activation host runtime is unavailable", + )) + } else { + Ok(()) + }; + }; + + let config = crate::voice_activation::VoiceActivationConfig { + enabled: preferences.voice_activation_enabled, + wake_enabled: preferences.voice_activation_wake_enabled, + finish_enabled: preferences.voice_activation_finish_enabled, + wake_phrase: preferences.voice_activation_wake_phrase, + finish_phrase: preferences.voice_activation_finish_phrase, + timeout_enabled: preferences.voice_activation_timeout_enabled, + timeout_seconds: preferences.voice_activation_timeout_seconds, + microphone_device_name: (!preferences.microphone_device_name.trim().is_empty()) + .then_some(preferences.microphone_device_name), + activation_generation: generation, + } + .normalized() + .map_err(|error| BackendError::new(BackendErrorCode::InvalidArgument, error.to_string()))?; + self.events.publish( + None, + BackendEventKind::VoiceActivationStateChanged( + crate::voice_activation::VoiceActivationStateSnapshot { + phase: crate::voice_activation::VoiceActivationPhase::Starting, + activation_generation: generation, + session_id: None, + message: None, + }, + ), + ); + let sink: Arc = Arc::new(CoreVoiceActivationSink { + events: Arc::clone(&self.events), + }); + match runtime.configure(config, sink).await { + Ok(handle) => { + let stale_handle = { + let mut slot = self + .voice_activation_handle + .lock() + .expect("voice activation handle lock poisoned"); + if self.voice_activation_generation.load(Ordering::Acquire) != generation { + Some(handle) + } else { + *slot = Some(handle.into()); + None + } + }; + if let Some(handle) = stale_handle { + let _ = handle + .stop(crate::voice_activation::VoiceActivationStopReason::Cancelled) + .await; + return Err(BackendError::new( + BackendErrorCode::Cancelled, + "voice activation configuration became stale", + )); + } + self.events.publish( + None, + BackendEventKind::VoiceActivationStateChanged( + crate::voice_activation::VoiceActivationStateSnapshot { + phase: crate::voice_activation::VoiceActivationPhase::ListeningForWake, + activation_generation: generation, + session_id: None, + message: None, + }, + ), + ); + Ok(()) + } + Err(error) => { + let error = map_voice_activation_error(error); + self.events.publish( + None, + BackendEventKind::VoiceActivationStateChanged( + crate::voice_activation::VoiceActivationStateSnapshot { + phase: voice_activation_failure_phase(error.code), + activation_generation: generation, + session_id: None, + message: Some(error.message.clone()), + }, + ), + ); + Err(error) + } + } + } + + pub async fn stop_voice_activation( + &self, + reason: crate::voice_activation::VoiceActivationStopReason, + ) -> Result<(), BackendError> { + let _configure_gate = self.voice_activation_configure_gate.lock().await; + let handle = self + .voice_activation_handle + .lock() + .expect("voice activation handle lock poisoned") + .take(); + self.voice_activation_session + .lock() + .expect("voice activation session lock poisoned") + .take(); + let next_generation = self + .voice_activation_generation + .fetch_add(1, Ordering::AcqRel) + + 1; + self.voice_activation_handoff_generation + .store(0, Ordering::Release); + if let Some(handle) = handle { + handle + .stop(reason) + .await + .map_err(map_voice_activation_error)?; + } + self.events.publish( + None, + BackendEventKind::VoiceActivationStateChanged( + crate::voice_activation::VoiceActivationStateSnapshot { + phase: crate::voice_activation::VoiceActivationPhase::Disabled, + activation_generation: next_generation, + session_id: None, + message: None, + }, + ), + ); + Ok(()) + } + + pub async fn stop_voice_activation_if_current( + &self, + generation: u64, + reason: crate::voice_activation::VoiceActivationStopReason, + ) -> Result { + let _configure_gate = self.voice_activation_configure_gate.lock().await; + if self.voice_activation_generation.load(Ordering::Acquire) != generation { + return Ok(false); + } + let handle = self + .voice_activation_handle + .lock() + .expect("voice activation handle lock poisoned") + .take(); + self.voice_activation_session + .lock() + .expect("voice activation session lock poisoned") + .take(); + let next_generation = self + .voice_activation_generation + .fetch_add(1, Ordering::AcqRel) + + 1; + self.voice_activation_handoff_generation + .store(0, Ordering::Release); + if let Some(handle) = handle { + handle + .stop(reason) + .await + .map_err(map_voice_activation_error)?; + } + self.events.publish( + None, + BackendEventKind::VoiceActivationStateChanged( + crate::voice_activation::VoiceActivationStateSnapshot { + phase: crate::voice_activation::VoiceActivationPhase::Disabled, + activation_generation: next_generation, + session_id: None, + message: None, + }, + ), + ); + Ok(true) + } + + pub fn voice_activation_generation_is_current(&self, generation: u64) -> bool { + self.voice_activation_generation.load(Ordering::Acquire) == generation + } + + /// Called by the host runtime after it recognizes a semantic command. The + /// host callback is synchronous, so the actual handoff/finalization is + /// owned by Core's task spawner and survives a dropped IPC request. + pub fn handle_voice_activation_signal( + self: &Arc, + signal: crate::voice_activation::VoiceActivationSignal, + ) { + use crate::voice_activation::VoiceActivationSignal; + let generation = self.voice_activation_generation.load(Ordering::Acquire); + let signal_generation = match signal { + VoiceActivationSignal::WakeDetected { + activation_generation, + } + | VoiceActivationSignal::FinishDetected { + activation_generation, + .. + } + | VoiceActivationSignal::RuntimeStateChanged { + activation_generation, + .. + } => activation_generation, + }; + if signal_generation != generation { + return; + } + match signal { + VoiceActivationSignal::WakeDetected { .. } => { + if self.snapshot().dictation.session_id.is_some() + || self + .voice_activation_session + .lock() + .expect("voice activation session lock poisoned") + .is_some() + { + return; + } + let handle = self + .voice_activation_handle + .lock() + .expect("voice activation handle lock poisoned") + .clone(); + let Some(handle) = handle else { + return; + }; + if self + .voice_activation_handoff_generation + .compare_exchange(0, signal_generation, Ordering::AcqRel, Ordering::Acquire) + .is_err() + { + return; + } + let backend = Arc::clone(self); + self.deps.task_spawner.spawn(Box::pin(async move { + let _handoff_guard = VoiceActivationHandoffGuard { + backend: Arc::clone(&backend), + generation: signal_generation, + }; + if backend + .voice_activation_generation + .load(Ordering::Acquire) + != signal_generation + { + return; + } + let handoff = match handle + .handoff_to_dictation( + crate::ports::VoiceDictationHandoffRequest { + activation_generation: signal_generation, + }, + ) + .await + { + Ok(handoff) => handoff, + Err(error) => { + log::warn!("voice activation handoff failed: {error}"); + match backend + .stop_voice_activation_if_current( + signal_generation, + crate::voice_activation::VoiceActivationStopReason::RuntimeFailure, + ) + .await + { + Ok(true) if backend.get_preferences().voice_activation_enabled => { + let _ = backend.configure_voice_activation().await; + } + Ok(_) => {} + Err(error) => { + log::warn!("voice activation recovery stop failed: {error}"); + } + } + return; + } + }; + if backend + .voice_activation_generation + .load(Ordering::Acquire) + != signal_generation + { + let _ = handoff.capture.discard().await; + return; + } + match backend + .start_voice_dictation_with_capture(handoff.capture, signal_generation) + .await + { + Ok(session_id) => { + if backend + .voice_activation_generation + .load(Ordering::Acquire) + != signal_generation + { + let _ = backend.cancel_dictation(Some(session_id)).await; + return; + } + let preferences = backend.get_preferences(); + let timeout = crate::voice_activation::VoiceActivationConfig { + timeout_enabled: preferences.voice_activation_timeout_enabled, + timeout_seconds: preferences.voice_activation_timeout_seconds, + ..crate::voice_activation::VoiceActivationConfig::default() + } + .normalized() + .ok() + .and_then(|config| config.timeout()); + let voice_session = crate::voice_activation::VoiceActivationSession { + generation: crate::voice_activation::VoiceActivationGeneration( + signal_generation, + ), + session_id, + started_at: std::time::Instant::now(), + timeout, + finish_enabled: preferences.voice_activation_finish_enabled, + finish_phrase: preferences.voice_activation_finish_phrase, + }; + let committed = { + let _configure_gate = + backend.voice_activation_configure_gate.lock().await; + let session_is_current = backend + .state + .read() + .expect("backend state lock poisoned") + .dictation + .session_id + == Some(session_id); + if backend.voice_activation_generation.load(Ordering::Acquire) + != signal_generation + || !session_is_current + { + false + } else { + *backend + .voice_activation_session + .lock() + .expect("voice activation session lock poisoned") = + Some(voice_session); + handle.bind_dictation_session(session_id); + true + } + }; + if !committed { + let _ = backend.cancel_dictation(Some(session_id)).await; + return; + } + backend.events.publish( + None, + BackendEventKind::VoiceActivationStateChanged( + crate::voice_activation::VoiceActivationStateSnapshot { + phase: crate::voice_activation::VoiceActivationPhase::ListeningForFinish, + activation_generation: signal_generation, + session_id: Some(session_id), + message: None, + }, + ), + ); + if let Some(timeout) = timeout { + let timeout_backend = Arc::clone(&backend); + backend.deps.task_spawner.spawn(Box::pin(async move { + tokio::time::sleep(timeout).await; + timeout_backend + .finish_voice_activation( + signal_generation, + session_id, + crate::voice_activation::VoiceActivationStopReason::Timeout, + ) + .await; + })); + } + } + Err(error) => { + log::warn!("voice dictation start failed: {error}"); + match backend + .stop_voice_activation_if_current( + signal_generation, + crate::voice_activation::VoiceActivationStopReason::RuntimeFailure, + ) + .await + { + Ok(true) if backend.get_preferences().voice_activation_enabled => { + let _ = backend.configure_voice_activation().await; + } + Ok(_) => {} + Err(error) => { + log::warn!("voice activation recovery stop failed: {error}"); + } + } + } + } + })); + } + VoiceActivationSignal::FinishDetected { + session_id, + activation_generation, + } => { + let backend = Arc::clone(self); + self.deps.task_spawner.spawn(Box::pin(async move { + backend + .finish_voice_activation( + activation_generation, + session_id, + crate::voice_activation::VoiceActivationStopReason::FinishPhrase, + ) + .await; + })); + } + VoiceActivationSignal::RuntimeStateChanged { + activation_generation, + phase: crate::voice_activation::VoiceActivationPhase::Failed, + } => { + let backend = Arc::clone(self); + self.deps.task_spawner.spawn(Box::pin(async move { + match backend + .stop_voice_activation_if_current( + activation_generation, + crate::voice_activation::VoiceActivationStopReason::RuntimeFailure, + ) + .await + { + Ok(true) if backend.get_preferences().voice_activation_enabled => { + let _ = backend.configure_voice_activation().await; + } + Ok(_) => {} + Err(error) => { + log::warn!("voice activation recovery stop failed: {error}"); + } + } + })); + } + VoiceActivationSignal::RuntimeStateChanged { .. } => {} + } + } + + async fn start_voice_dictation_with_capture( + &self, + capture: Box, + generation: u64, + ) -> Result { + if self.voice_activation_generation.load(Ordering::Acquire) != generation { + let _ = capture.discard().await; + return Err(BackendError::new( + BackendErrorCode::Cancelled, + "voice activation generation is stale", + )); + } + self.pending_audio_captures + .lock() + .expect("pending audio capture lock poisoned") + .insert(PendingAudioCaptureKey::VoiceActivation(generation), capture); + let reservation = match self.reserve_dictation_session( + true, + DictationOutputTarget::ForegroundApp, + Some(PendingAudioCaptureKey::VoiceActivation(generation)), + ) { + Ok(reservation) => reservation, + Err(error) => { + self.discard_pending_audio_capture(PendingAudioCaptureKey::VoiceActivation( + generation, + )) + .await?; + return Err(error); + } + }; + let session_id = reservation.session_id; + let preferences = self.get_preferences(); + let activation_config = crate::voice_activation::VoiceActivationConfig { + finish_enabled: preferences.voice_activation_finish_enabled, + finish_phrase: preferences.voice_activation_finish_phrase.clone(), + timeout_enabled: preferences.voice_activation_timeout_enabled, + timeout_seconds: preferences.voice_activation_timeout_seconds, + ..crate::voice_activation::VoiceActivationConfig::default() + } + .normalized() + .map_err(|error| BackendError::new(BackendErrorCode::InvalidArgument, error.to_string()))?; + { + let _configure_gate = self.voice_activation_configure_gate.lock().await; + if self.voice_activation_generation.load(Ordering::Acquire) != generation { + self.voice_sessions.release(session_id); + self.discard_pending_audio_capture(PendingAudioCaptureKey::VoiceActivation( + generation, + )) + .await?; + return Err(BackendError::new( + BackendErrorCode::Cancelled, + "voice activation generation became stale while starting dictation", + )); + } + *self + .voice_activation_session + .lock() + .expect("voice activation session lock poisoned") = Some( + crate::voice_activation::VoiceActivationSession { + generation: crate::voice_activation::VoiceActivationGeneration(generation), + session_id, + started_at: std::time::Instant::now(), + timeout: activation_config.timeout(), + finish_enabled: activation_config.finish_enabled, + finish_phrase: activation_config.finish_phrase.clone(), + }, + ); + } + match self + .start_reserved_dictation(reservation, DictationStartOptions::default()) + .await + { + Ok(session_id) => Ok(session_id), + Err(error) => { + let mut session = self + .voice_activation_session + .lock() + .expect("voice activation session lock poisoned"); + if session.as_ref().is_some_and(|session| { + session.generation.0 == generation && session.session_id == session_id + }) { + session.take(); + } + Err(error) + } + } + } + + async fn cleanup_voice_activation_after_dictation(&self, session_id: SessionId) { + let is_voice_session = self + .voice_activation_session + .lock() + .expect("voice activation session lock poisoned") + .as_ref() + .is_some_and(|session| session.session_id == session_id); + if !is_voice_session { + return; + } + let enabled = self.get_preferences().voice_activation_enabled; + let _ = self + .stop_voice_activation(crate::voice_activation::VoiceActivationStopReason::Cancelled) + .await; + if enabled { + let _ = self.configure_voice_activation().await; + } + } + + async fn finish_voice_activation( + &self, + generation: u64, + session_id: SessionId, + reason: crate::voice_activation::VoiceActivationStopReason, + ) { + let accepted = self + .voice_activation_session + .lock() + .expect("voice activation session lock poisoned") + .as_ref() + .is_some_and(|session| { + session.accepts( + crate::voice_activation::VoiceActivationGeneration(generation), + session_id, + ) + }); + if !accepted { + return; + } + if self.stop_dictation_session(session_id).await.is_err() { + return; + } + let still_voice_session = self + .voice_activation_session + .lock() + .expect("voice activation session lock poisoned") + .as_ref() + .is_some_and(|session| session.session_id == session_id); + if still_voice_session { + let _ = self.stop_voice_activation(reason).await; + if self.get_preferences().voice_activation_enabled { + let _ = self.configure_voice_activation().await; + } + } + } + pub fn repositories(&self) -> BackendRepositories { BackendRepositories { preferences: Arc::clone(&self.preferences), @@ -3870,10 +4669,14 @@ impl OpenLessBackend { state: Arc::clone(&self.state), phase_changed: Arc::clone(&self.phase_changed), text_insertions: Arc::clone(&self.text_insertions), + voice_activation_session: Arc::clone(&self.voice_activation_session), + raw_transcripts: Mutex::new(HashMap::new()), }) } pub async fn start(&self) -> Result { + let _lifecycle_gate = self.backend_lifecycle_gate.lock().await; + self.backend_shutdown_requested.store(false, Ordering::Release); if let Some(error) = &self.startup_error { return Err(error.clone()); } @@ -3917,10 +4720,26 @@ impl OpenLessBackend { } } }; - let mut state = self.state.write().expect("backend state lock poisoned"); - state.credentials = credentials; - if state.running { - return Ok(StartupSnapshot { + let snapshot = { + let mut state = self.state.write().expect("backend state lock poisoned"); + state.credentials = credentials; + if state.running { + return Ok(StartupSnapshot { + contract_version: crate::BACKEND_CONTRACT_VERSION.to_string(), + backend: BackendSnapshot { + running: true, + dictation: state.dictation.clone(), + vocabulary_revision: self.vocabulary_revision.load(Ordering::Acquire), + history_revision: self.history_revision.load(Ordering::Acquire), + style_pack_revision: self.style_pack_revision.load(Ordering::Acquire), + preferences_revision: self.preferences_revision.load(Ordering::Acquire), + credentials: state.credentials.clone(), + }, + }); + } + state.running = true; + self.events.publish(None, BackendEventKind::BackendStarted); + StartupSnapshot { contract_version: crate::BACKEND_CONTRACT_VERSION.to_string(), backend: BackendSnapshot { running: true, @@ -3931,25 +4750,20 @@ impl OpenLessBackend { preferences_revision: self.preferences_revision.load(Ordering::Acquire), credentials: state.credentials.clone(), }, - }); + } + }; + if let Err(error) = self.configure_voice_activation().await { + log::warn!("[voice-activation] startup configuration failed: {error}"); } - state.running = true; - self.events.publish(None, BackendEventKind::BackendStarted); - Ok(StartupSnapshot { - contract_version: crate::BACKEND_CONTRACT_VERSION.to_string(), - backend: BackendSnapshot { - running: true, - dictation: state.dictation.clone(), - vocabulary_revision: self.vocabulary_revision.load(Ordering::Acquire), - history_revision: self.history_revision.load(Ordering::Acquire), - style_pack_revision: self.style_pack_revision.load(Ordering::Acquire), - preferences_revision: self.preferences_revision.load(Ordering::Acquire), - credentials: state.credentials.clone(), - }, - }) + Ok(snapshot) } pub async fn shutdown(&self) -> Result<(), BackendError> { + let _lifecycle_gate = self.backend_lifecycle_gate.lock().await; + self.backend_shutdown_requested.store(true, Ordering::Release); + let _ = self + .stop_voice_activation(crate::voice_activation::VoiceActivationStopReason::Disabled) + .await; if let Some(sync) = &self.encrypted_sync { sync.shutdown().await; } @@ -4119,6 +4933,12 @@ impl OpenLessBackend { } } + pub fn latest_voice_activation_state( + &self, + ) -> Option { + self.events.latest_voice_activation_state() + } + pub fn dictation_output_target(&self) -> Option { let state = self.state.read().expect("backend state lock poisoned"); state @@ -4207,7 +5027,7 @@ impl OpenLessBackend { .pending_audio_captures .lock() .expect("pending audio capture lock poisoned") - .insert(press_id, capture); + .insert(PendingAudioCaptureKey::Hotkey(press_id), capture); if let Some(replaced) = replaced { self.deps.task_spawner.spawn(Box::pin(async move { if let Err(error) = replaced.discard().await { @@ -4225,15 +5045,21 @@ impl OpenLessBackend { } } - fn take_pending_audio_capture(&self, press_id: u64) -> Option> { + fn take_pending_audio_capture( + &self, + key: PendingAudioCaptureKey, + ) -> Option> { self.pending_audio_captures .lock() .expect("pending audio capture lock poisoned") - .remove(&press_id) + .remove(&key) } - async fn discard_pending_audio_capture(&self, press_id: u64) -> Result<(), BackendError> { - let capture = self.take_pending_audio_capture(press_id); + async fn discard_pending_audio_capture( + &self, + key: PendingAudioCaptureKey, + ) -> Result<(), BackendError> { + let capture = self.take_pending_audio_capture(key); match capture { Some(capture) => capture.discard().await, None => Ok(()), @@ -4348,7 +5174,7 @@ impl OpenLessBackend { Some(PendingAudioCaptureAbortGuard { captures: Arc::clone(&self.pending_audio_captures), task_spawner: Arc::clone(&self.deps.task_spawner), - press_id, + key: PendingAudioCaptureKey::Hotkey(press_id), }) } else { None @@ -4357,7 +5183,9 @@ impl OpenLessBackend { None }; if let DictationHotkeyEdge::Combined { press_id, .. } = edge { - let _ = self.discard_pending_audio_capture(press_id).await; + let _ = self + .discard_pending_audio_capture(PendingAudioCaptureKey::Hotkey(press_id)) + .await; } let (intent, reservation) = if let HotkeyIntent::WaitForModifierGrace { press_id } = intent { @@ -4377,7 +5205,7 @@ impl OpenLessBackend { Some(self.reserve_dictation_session( options.start.insert_text, options.start.output_target, - Some(press_id), + Some(PendingAudioCaptureKey::Hotkey(press_id)), )) } else { None @@ -4385,7 +5213,9 @@ impl OpenLessBackend { (intent, reservation) }; if !matches!(intent, HotkeyIntent::Start { .. }) { - let _ = self.discard_pending_audio_capture(press_id).await; + let _ = self + .discard_pending_audio_capture(PendingAudioCaptureKey::Hotkey(press_id)) + .await; } (intent, reservation) } else { @@ -4400,14 +5230,16 @@ impl OpenLessBackend { self.reserve_dictation_session( options.start.insert_text, options.start.output_target, - Some(press_id), + Some(PendingAudioCaptureKey::Hotkey(press_id)), ) }) }; if reservation.is_some() { (intent, reservation) } else { - let _ = self.discard_pending_audio_capture(press_id).await; + let _ = self + .discard_pending_audio_capture(PendingAudioCaptureKey::Hotkey(press_id)) + .await; self.hotkey .lock() .expect("hotkey interpreter lock poisoned") @@ -4430,7 +5262,9 @@ impl OpenLessBackend { .await } Err(error) => { - let _ = self.discard_pending_audio_capture(press_id).await; + let _ = self + .discard_pending_audio_capture(PendingAudioCaptureKey::Hotkey(press_id)) + .await; Err(error) } }; @@ -5151,6 +5985,25 @@ impl OpenLessBackend { } }; + let voice_activation_config = crate::voice_activation::VoiceActivationConfig { + enabled: preferences.voice_activation_enabled, + wake_enabled: preferences.voice_activation_wake_enabled, + finish_enabled: preferences.voice_activation_finish_enabled, + wake_phrase: preferences.voice_activation_wake_phrase.clone(), + finish_phrase: preferences.voice_activation_finish_phrase.clone(), + timeout_enabled: preferences.voice_activation_timeout_enabled, + timeout_seconds: preferences.voice_activation_timeout_seconds, + microphone_device_name: (!preferences.microphone_device_name.trim().is_empty()) + .then(|| preferences.microphone_device_name.clone()), + activation_generation: 0, + }; + voice_activation_config.normalized().map_err(|error| { + BackendError::new( + BackendErrorCode::InvalidArgument, + format!("invalid voice activation settings: {error}"), + ) + })?; + let effects = crate::SettingsEffectPlan::between(&previous, &preferences); let mut receipt = match runtime.prepare(&effects) { Ok(receipt) => receipt, @@ -5996,7 +6849,7 @@ impl OpenLessBackend { &self, insert_text: bool, output_target: DictationOutputTarget, - capture_generation: Option, + capture_key: Option, ) -> Result { { let state = self.state.read().expect("backend state lock poisoned"); @@ -6054,11 +6907,14 @@ impl OpenLessBackend { ); self.phase_changed.notify_waiters(); } - let pending_capture = - capture_generation.and_then(|generation| self.take_pending_audio_capture(generation)); + let capture_generation = capture_key.map(|key| match key { + PendingAudioCaptureKey::Hotkey(generation) + | PendingAudioCaptureKey::VoiceActivation(generation) => generation, + }); + let pending_capture = capture_key.and_then(|key| self.take_pending_audio_capture(key)); let capture_generation = pending_capture .as_ref() - .map(|_| capture_generation.expect("capture generation was checked")); + .map(|_| capture_generation.expect("capture key was checked")); Ok(DictationReservation { session_id, resources: starting_resources, @@ -6514,6 +7370,8 @@ impl OpenLessBackend { let _ = self.cancel_session_adapters(session_id).await; let _ = self.hide_dictation_feedback(session_id); self.reset_dictation_session(session_id); + self.cleanup_voice_activation_after_dictation(session_id) + .await; return Err(error); } } @@ -6581,10 +7439,41 @@ impl OpenLessBackend { let _ = self.cancel_session_adapters(session_id).await; let _ = self.hide_dictation_feedback(session_id); self.reset_dictation_session(session_id); + self.cleanup_voice_activation_after_dictation(session_id) + .await; return Err(error); } }; + let finish_phrase = self + .voice_activation_session + .lock() + .expect("voice activation session lock poisoned") + .as_ref() + .filter(|session| session.finish_enabled && session.session_id == session_id) + .map(|session| session.finish_phrase.clone()); + if let Some(finish_phrase) = finish_phrase { + if let Some(stripped) = crate::voice_activation::strip_phrase_suffix( + &engine_result.raw_text, + &finish_phrase, + ) { + engine_result.raw_text = stripped; + } + if let Some(stripped) = crate::voice_activation::strip_phrase_suffix( + &engine_result.polished_text, + &finish_phrase, + ) { + engine_result.polished_text = stripped; + } + if let Some(asr_transcript) = engine_result.asr_transcript.as_mut() { + if let Some(stripped) = + crate::voice_activation::strip_phrase_suffix(asr_transcript, &finish_phrase) + { + *asr_transcript = stripped; + } + } + } + if context.output_target == DictationOutputTarget::CloudNote && engine_result.polish_failed { let error = @@ -6592,6 +7481,8 @@ impl OpenLessBackend { self.mark_dictation_failed(session_id, &error); let _ = self.hide_dictation_feedback(session_id); self.reset_dictation_session(session_id); + self.cleanup_voice_activation_after_dictation(session_id) + .await; self.cleanup_cloud_note(session_id, true)?; return Err(error); } @@ -6620,6 +7511,8 @@ impl OpenLessBackend { let _ = self.cancel_text_insertion(session_id).await; let _ = self.hide_dictation_feedback(session_id); self.reset_dictation_session(session_id); + self.cleanup_voice_activation_after_dictation(session_id) + .await; return Err(error); } @@ -6651,19 +7544,21 @@ impl OpenLessBackend { // Cancellation may happen while ASR/LLM work is in flight. Never // insert a result after the session has been cancelled or replaced. - { + let cancelled_before_insertion = { let state = self.state.read().expect("backend state lock poisoned"); - if state.dictation.session_id != Some(session_id) + state.dictation.session_id != Some(session_id) || !matches!( state.dictation.phase, DictationPhase::Transcribing | DictationPhase::Polishing ) - { - return Err(BackendError::new( - BackendErrorCode::Cancelled, - "dictation session was cancelled before insertion", - )); - } + }; + if cancelled_before_insertion { + self.cleanup_voice_activation_after_dictation(session_id) + .await; + return Err(BackendError::new( + BackendErrorCode::Cancelled, + "dictation session was cancelled before insertion", + )); } { @@ -6714,6 +7609,8 @@ impl OpenLessBackend { let _ = self.cancel_text_insertion(session_id).await; let _ = self.hide_dictation_feedback(session_id); self.reset_dictation_session(session_id); + self.cleanup_voice_activation_after_dictation(session_id) + .await; return Err(error); } } @@ -6738,46 +7635,52 @@ impl OpenLessBackend { .unwrap_or(crate::types::InsertStatus::NotRequested), }; - let mut state = self.state.write().expect("backend state lock poisoned"); - if state.dictation.session_id != Some(session_id) - || state.dictation.phase - != if context.insertion.enabled { - DictationPhase::Inserting - } else { - DictationPhase::Completed + let session_replaced = { + let mut state = self.state.write().expect("backend state lock poisoned"); + let replaced = state.dictation.session_id != Some(session_id) + || state.dictation.phase + != if context.insertion.enabled { + DictationPhase::Inserting + } else { + DictationPhase::Completed + }; + if !replaced { + state.dictation.phase = DictationPhase::Completed; + state.dictation.message = Some(match insert_outcome { + Some(InsertOutcome::Inserted) => "inserted".to_string(), + Some(InsertOutcome::PasteSent) => "paste_sent".to_string(), + Some(InsertOutcome::CopiedFallback) => "copied_fallback".to_string(), + None => "insertion_not_requested".to_string(), + }); + if matches!(insert_outcome, Some(InsertOutcome::CopiedFallback)) { + self.events.publish( + Some(session_id), + BackendEventKind::InsertFallback(crate::types::InsertFallbackPayload { + reason: "clipboard_fallback".to_string(), + copied_text: Some(engine_result.polished_text.clone()), + }), + ); } - { + self.events.publish( + Some(session_id), + BackendEventKind::DictationCompleted(result.clone()), + ); + self.events.publish( + Some(session_id), + BackendEventKind::DictationStateChanged(state.dictation.clone()), + ); + self.phase_changed.notify_waiters(); + } + replaced + }; + if session_replaced { + self.cleanup_voice_activation_after_dictation(session_id) + .await; return Err(BackendError::new( BackendErrorCode::Cancelled, "dictation session was replaced before completion", )); } - state.dictation.phase = DictationPhase::Completed; - state.dictation.message = Some(match insert_outcome { - Some(InsertOutcome::Inserted) => "inserted".to_string(), - Some(InsertOutcome::PasteSent) => "paste_sent".to_string(), - Some(InsertOutcome::CopiedFallback) => "copied_fallback".to_string(), - None => "insertion_not_requested".to_string(), - }); - if matches!(insert_outcome, Some(InsertOutcome::CopiedFallback)) { - self.events.publish( - Some(session_id), - BackendEventKind::InsertFallback(crate::types::InsertFallbackPayload { - reason: "clipboard_fallback".to_string(), - copied_text: Some(engine_result.polished_text.clone()), - }), - ); - } - self.events.publish( - Some(session_id), - BackendEventKind::DictationCompleted(result.clone()), - ); - self.events.publish( - Some(session_id), - BackendEventKind::DictationStateChanged(state.dictation.clone()), - ); - self.phase_changed.notify_waiters(); - drop(state); self.arm_edit_observation( session_id, context.insertion.observe_edits, @@ -6789,6 +7692,8 @@ impl OpenLessBackend { // delayed successful completion must not clear its successor's state, // transcript, silence detector or physical-hotkey generation. self.reset_dictation_session(session_id); + self.cleanup_voice_activation_after_dictation(session_id) + .await; self.persist_completed_dictation(&context, &result, insert_outcome, &engine_result); self.cleanup_cloud_note(session_id, true)?; host_result?; @@ -7249,6 +8154,8 @@ impl OpenLessBackend { let cancel_result = self.cancel_session_adapters(session_id).await; let _ = self.hide_dictation_feedback(session_id); self.reset_dictation_session(session_id); + self.cleanup_voice_activation_after_dictation(session_id) + .await; cancel_result } @@ -7309,6 +8216,7 @@ impl OpenLessBackend { Ok(()) }; let cancel_result = self.cancel_session_adapters(active).await; + self.cleanup_voice_activation_after_dictation(active).await; // The state can already display cancellation, but native audio/input // cleanup still owns the shared resource. Reject new capture until that // cleanup finishes, including on its error path. @@ -8058,6 +8966,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -8760,6 +9669,7 @@ mod tests { let runtime = Arc::new(LessComputerCaptureRuntime::default()); let host = Arc::new(crate::testing::RecordingHostActions::default()); let dependencies = BackendDependencies { + voice_activation_runtime: None, host_actions: host.clone(), dictation_engine: Arc::new(VoiceOnlyEngine(Arc::clone(&transcription))), ..BackendDependencies::unsupported() @@ -8921,6 +9831,7 @@ mod tests { }); let runtime = Arc::new(LessComputerCaptureRuntime::default()); let dependencies = BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(crate::testing::RecordingHostActions::default()), dictation_engine: Arc::new(DictationOnlyEngine(Arc::clone(&transcription))), ..BackendDependencies::unsupported() @@ -9216,6 +10127,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, dictation_engine: Arc::new(engine), ..BackendDependencies::unsupported() }, @@ -9289,6 +10201,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, dictation_engine: Arc::new(engine), ..BackendDependencies::unsupported() }, @@ -9763,6 +10676,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, dictation_engine: Arc::new(engine), ..BackendDependencies::unsupported() }, @@ -9860,6 +10774,7 @@ mod tests { credential_store: llm_configured_credentials(), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -9917,6 +10832,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -10043,6 +10959,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -10176,6 +11093,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -10326,6 +11244,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -10683,6 +11602,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -10831,6 +11751,7 @@ mod tests { credential_store: credential_store.clone(), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -10900,6 +11821,7 @@ mod tests { credential_store, services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -11009,6 +11931,7 @@ mod tests { credential_store: Arc::new(InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -11189,6 +12112,7 @@ mod tests { credential_store: Arc::new(PersistenceOnlyCredentialStore), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -11218,6 +12142,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, services, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(FakeInserter), @@ -11396,6 +12321,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(FakeInserter), dictation_engine: Arc::new(FakeEngine), @@ -11527,6 +12453,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: host.clone(), text_inserter: Arc::new(FakeInserter), dictation_engine: Arc::new(FakeEngine), @@ -11635,6 +12562,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -12526,6 +13454,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -12667,6 +13596,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -12747,6 +13677,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -12845,6 +13776,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -12911,6 +13843,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -12977,6 +13910,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -13050,6 +13984,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -13288,6 +14223,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -13340,6 +14276,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -13609,6 +14546,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -13650,6 +14588,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -13714,6 +14653,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -13839,6 +14779,7 @@ mod tests { credential_store: llm_configured_credentials(), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -14206,6 +15147,7 @@ mod tests { credential_store: llm_configured_credentials(), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -14364,6 +15306,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(FakeInserter), dictation_engine: Arc::new(FakeEngine), @@ -14497,6 +15440,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -14555,6 +15499,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: host, text_inserter: Arc::new(crate::testing::FixtureTextInserter::with_outcome( outcome, @@ -14949,6 +15894,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -14993,6 +15939,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -15034,6 +15981,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -15077,6 +16025,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -15162,6 +16111,8 @@ mod tests { state: Arc::clone(&backend.state), phase_changed: Arc::clone(&backend.phase_changed), text_insertions: Arc::clone(&backend.text_insertions), + voice_activation_session: Arc::clone(&backend.voice_activation_session), + raw_transcripts: Mutex::new(HashMap::new()), }; backend.cancel_dictation(Some(session)).await.unwrap(); @@ -15190,6 +16141,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(FakeInserter), dictation_engine: Arc::new(BlockingStartEngine { @@ -15371,6 +16323,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, @@ -15441,6 +16394,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(FakeInserter), dictation_engine: Arc::new(BlockingStartEngine { diff --git a/openless-all/app/crates/openless-core/src/cloud_sync_e2ee_documents/registry.rs b/openless-all/app/crates/openless-core/src/cloud_sync_e2ee_documents/registry.rs index 4ac190fef..ac8890d98 100644 --- a/openless-all/app/crates/openless-core/src/cloud_sync_e2ee_documents/registry.rs +++ b/openless-all/app/crates/openless-core/src/cloud_sync_e2ee_documents/registry.rs @@ -179,6 +179,55 @@ pub const PREFERENCE_FIELDS: &[PreferenceField] = &[ shape: PreferenceShape::Boolean, reason: "local_only_microphone_capture", }, + PreferenceField { + rust_name: "voice_activation_enabled", + key: "voiceActivationEnabled", + class: PreferenceClass::Excluded, + shape: PreferenceShape::Boolean, + reason: "local_only_microphone_capture", + }, + PreferenceField { + rust_name: "voice_activation_wake_enabled", + key: "voiceActivationWakeEnabled", + class: PreferenceClass::Excluded, + shape: PreferenceShape::Boolean, + reason: "local_only_microphone_capture", + }, + PreferenceField { + rust_name: "voice_activation_finish_enabled", + key: "voiceActivationFinishEnabled", + class: PreferenceClass::Excluded, + shape: PreferenceShape::Boolean, + reason: "local_only_microphone_capture", + }, + PreferenceField { + rust_name: "voice_activation_wake_phrase", + key: "voiceActivationWakePhrase", + class: PreferenceClass::Excluded, + shape: PreferenceShape::Text, + reason: "local_only_microphone_capture", + }, + PreferenceField { + rust_name: "voice_activation_finish_phrase", + key: "voiceActivationFinishPhrase", + class: PreferenceClass::Excluded, + shape: PreferenceShape::Text, + reason: "local_only_microphone_capture", + }, + PreferenceField { + rust_name: "voice_activation_timeout_enabled", + key: "voiceActivationTimeoutEnabled", + class: PreferenceClass::Excluded, + shape: PreferenceShape::Boolean, + reason: "local_only_microphone_capture", + }, + PreferenceField { + rust_name: "voice_activation_timeout_seconds", + key: "voiceActivationTimeoutSeconds", + class: PreferenceClass::Excluded, + shape: PreferenceShape::Unsigned32, + reason: "local_only_microphone_capture", + }, PreferenceField { rust_name: "active_asr_provider", key: "activeAsrProvider", diff --git a/openless-all/app/crates/openless-core/src/config.rs b/openless-all/app/crates/openless-core/src/config.rs index 096dc9fd2..75527de1b 100644 --- a/openless-all/app/crates/openless-core/src/config.rs +++ b/openless-all/app/crates/openless-core/src/config.rs @@ -72,6 +72,7 @@ pub struct BackendDependencies { pub dictation_engine: Arc, pub task_spawner: Arc, pub credential_store: Arc, + pub voice_activation_runtime: Option>, pub services: BackendServices, pub local_asr_runtime: Option>, pub marketplace_config: Option, @@ -91,6 +92,7 @@ impl BackendDependencies { dictation_engine: Arc::new(UnsupportedDictationEngine), task_spawner: Arc::new(TokioTaskSpawner), credential_store: Arc::new(crate::credentials::UnsupportedCredentialStore), + voice_activation_runtime: None, services: BackendServices::unsupported(), local_asr_runtime: None, marketplace_config: None, diff --git a/openless-all/app/crates/openless-core/src/events.rs b/openless-all/app/crates/openless-core/src/events.rs index b118992d7..f5c75272c 100644 --- a/openless-all/app/crates/openless-core/src/events.rs +++ b/openless-all/app/crates/openless-core/src/events.rs @@ -14,6 +14,7 @@ use crate::types::{ InsertFallbackPayload, NotificationPayload, PermissionSnapshot, PolishDelta, PreferencesChange, SessionId, StylePackChange, TranscriptDelta, VocabularyChange, }; +use crate::voice_activation::VoiceActivationStateSnapshot; #[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] #[serde(rename_all = "camelCase")] @@ -407,6 +408,7 @@ pub enum BackendEventKind { BackendStarted, BackendStopping, DictationStateChanged(DictationStateSnapshot), + VoiceActivationStateChanged(crate::voice_activation::VoiceActivationStateSnapshot), TranscriptDelta(TranscriptDelta), PolishDelta(PolishDelta), DictationCompleted(DictationResult), @@ -458,6 +460,9 @@ pub struct EventBus { sequence: AtomicU64, sender: broadcast::Sender, backlog: Mutex>, + // Retain the latest activation state independently of the bounded replay so a host can + // recover the microphone listener after a subscription lag. + latest_voice_activation: Mutex>, // Retain one presentation snapshot independently of the bounded replay so // a reopened host can recover a long-running transcription. latest_less_voice: Mutex>, @@ -470,6 +475,7 @@ impl EventBus { sequence: AtomicU64::new(0), sender, backlog: Mutex::new(VecDeque::with_capacity(EVENT_REPLAY_CAPACITY)), + latest_voice_activation: Mutex::new(None), latest_less_voice: Mutex::new(None), } } @@ -490,6 +496,18 @@ impl EventBus { session_id, kind, }; + if let BackendEventKind::VoiceActivationStateChanged(state) = &event.kind { + let mut latest = self + .latest_voice_activation + .lock() + .expect("voice activation projection lock poisoned"); + if latest + .as_ref() + .is_none_or(|previous| state.activation_generation >= previous.activation_generation) + { + *latest = Some(state.clone()); + } + } if let BackendEventKind::LessComputerEvent(voice) = &event.kind { if let LessComputerEventKind::VoiceState { session_id, phase, .. @@ -556,6 +574,13 @@ impl EventBus { } } + pub fn latest_voice_activation_state(&self) -> Option { + self.latest_voice_activation + .lock() + .expect("voice activation projection lock poisoned") + .clone() + } + pub fn latest_less_computer_voice_state(&self) -> Option { self.latest_less_voice .lock() @@ -588,6 +613,10 @@ impl BackendEventPublisher { self.bus.replay_after(sequence) } + pub fn latest_voice_activation_state(&self) -> Option { + self.bus.latest_voice_activation_state() + } + pub fn latest_less_computer_voice_state(&self) -> Option { self.bus.latest_less_computer_voice_state() } @@ -665,6 +694,42 @@ mod tests { assert_eq!(event.unwrap().sequence, 2); } + #[test] + fn latest_voice_activation_state_ignores_stale_generations() { + let bus = EventBus::new(4); + bus.publish( + None, + BackendEventKind::VoiceActivationStateChanged(VoiceActivationStateSnapshot { + phase: crate::voice_activation::VoiceActivationPhase::ListeningForWake, + activation_generation: 2, + session_id: None, + message: None, + }), + ); + bus.publish( + None, + BackendEventKind::VoiceActivationStateChanged(VoiceActivationStateSnapshot { + phase: crate::voice_activation::VoiceActivationPhase::Failed, + activation_generation: 1, + session_id: None, + message: Some("stale".into()), + }), + ); + assert_eq!( + bus.latest_voice_activation_state() + .expect("latest activation state") + .activation_generation, + 2 + ); + assert_eq!( + BackendEventPublisher::new(Arc::new(bus)) + .latest_voice_activation_state() + .expect("publisher activation state") + .activation_generation, + 2 + ); + } + #[test] fn native_publishers_keep_live_and_replay_sequences_in_the_same_order() { const THREADS: usize = 8; diff --git a/openless-all/app/crates/openless-core/src/lib.rs b/openless-all/app/crates/openless-core/src/lib.rs index 0e3bca1c7..421344022 100644 --- a/openless-all/app/crates/openless-core/src/lib.rs +++ b/openless-all/app/crates/openless-core/src/lib.rs @@ -74,6 +74,7 @@ pub mod style_packs; pub mod testing; pub mod types; pub mod vocabulary; +pub mod voice_activation; pub mod voice_edit_session; mod voice_session; @@ -301,7 +302,9 @@ pub use ports::{ PendingAudioCapture, PolishOutput, RecordingArchive, RecordingControlSink, RecordingEvent, RecordingProgressSink, ResourceResolver, TextInserter, TextInsertionSession, TextPolisher, TextStreamChunk, TextStreamSink, TranscriptOutput, TranscriptionEngine, TranscriptionSession, - UnsupportedTextInserter, VoiceCapture, + UnsupportedTextInserter, UnsupportedVoiceActivationRuntime, VoiceActivationError, + VoiceActivationHandle, VoiceActivationRuntime, VoiceActivationSink, VoiceCapture, + VoiceDictationHandoff, VoiceDictationHandoffRequest, }; pub use preferences::PreferencesStore; pub use prompt_compose::{ @@ -375,6 +378,14 @@ pub use vocabulary::{ builtin_vocab_presets, list_vocab_presets, resolve_vocab_presets, save_vocab_presets, DictionaryStore, }; +pub use voice_activation::{ + normalize_phrase, validate_phrase, PhraseValidationError, VoiceActivationCommand, + VoiceActivationConfig, VoiceActivationDecision, VoiceActivationGeneration, + VoiceActivationPhase, VoiceActivationSession, VoiceActivationSignal, + VoiceActivationStateSnapshot, VoiceActivationStopReason, VoiceCommandMatch, + VoiceCommandRecognizer, VoicePhraseMatcher, DEFAULT_FINISH_PHRASE, DEFAULT_TIMEOUT_SECONDS, + DEFAULT_WAKE_PHRASE, MAX_PHRASE_CHARS, MAX_TIMEOUT_SECONDS, MIN_TIMEOUT_SECONDS, +}; pub use voice_edit_session::{ TextSelection, VoiceEditCommit, VoiceEditContext, VoiceEditError, VoiceEditPhase, VoiceEditSession, VoiceEditSnapshot, VoiceEditTarget, VoiceEditTurn, diff --git a/openless-all/app/crates/openless-core/src/ports.rs b/openless-all/app/crates/openless-core/src/ports.rs index 8337cf73a..48d7a6d9e 100644 --- a/openless-all/app/crates/openless-core/src/ports.rs +++ b/openless-all/app/crates/openless-core/src/ports.rs @@ -8,6 +8,7 @@ use crate::audio::AudioProcessingConfig; use crate::dictation_context::DictationContext; use crate::errors::{BackendError, BackendErrorCode}; use crate::types::{InsertStatus, PolishDelta, SessionId, TranscriptDelta}; +use crate::voice_activation::{VoiceActivationConfig, VoiceActivationSignal}; #[derive(Debug, Clone, PartialEq, Eq)] pub enum HostAction { @@ -73,6 +74,85 @@ impl HostActions for NoopHostActions { } } +/// Host-owned sink for semantic voice activation signals. Standby recognition +/// must only publish wake/finish commands and runtime state; it must never +/// publish the recognized standby text as a normal transcript delta. +pub trait VoiceActivationSink: Send + Sync { + fn publish(&self, signal: VoiceActivationSignal) -> Result<(), BackendError>; +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum VoiceActivationError { + Unsupported, + ModelMissing, + PermissionDenied, + MicBusy, + InvalidConfig(String), + Runtime(String), +} + +impl std::fmt::Display for VoiceActivationError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Unsupported => formatter.write_str("voice activation is unsupported"), + Self::ModelMissing => formatter.write_str("voice activation model is not installed"), + Self::PermissionDenied => formatter.write_str("microphone permission was denied"), + Self::MicBusy => formatter.write_str("microphone is already in use"), + Self::InvalidConfig(message) | Self::Runtime(message) => formatter.write_str(message), + } + } +} + +impl std::error::Error for VoiceActivationError {} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VoiceDictationHandoffRequest { + pub activation_generation: u64, +} + +pub struct VoiceDictationHandoff { + pub capture: Box, +} + +/// A single host-owned microphone/runtime seam for voice activation. The Core +/// decides lifecycle and stale-event policy; the host decides how local ASR and +/// the native capture stream are implemented. +pub trait VoiceActivationRuntime: Send + Sync + 'static { + fn configure( + &self, + config: VoiceActivationConfig, + sink: Arc, + ) -> BoxFuture<'static, Result, VoiceActivationError>>; +} + +pub trait VoiceActivationHandle: Send + Sync { + /// Bind the Core-owned dictation session so finish signals can carry the + /// same session identity even though the host performs recognition. + fn bind_dictation_session(&self, _session_id: SessionId) {} + + fn handoff_to_dictation( + &self, + request: VoiceDictationHandoffRequest, + ) -> BoxFuture<'static, Result>; + + fn stop( + &self, + reason: crate::voice_activation::VoiceActivationStopReason, + ) -> BoxFuture<'static, Result<(), VoiceActivationError>>; +} + +pub struct UnsupportedVoiceActivationRuntime; + +impl VoiceActivationRuntime for UnsupportedVoiceActivationRuntime { + fn configure( + &self, + _config: VoiceActivationConfig, + _sink: Arc, + ) -> BoxFuture<'static, Result, VoiceActivationError>> { + Box::pin(async { Err(VoiceActivationError::Unsupported) }) + } +} + /// Resolve a packaged resource without exposing a framework-specific resource /// directory object to the core or UI. pub trait ResourceResolver: Send + Sync { diff --git a/openless-all/app/crates/openless-core/src/shared_types.rs b/openless-all/app/crates/openless-core/src/shared_types.rs index 9ff98cb1f..97bf10256 100644 --- a/openless-all/app/crates/openless-core/src/shared_types.rs +++ b/openless-all/app/crates/openless-core/src/shared_types.rs @@ -360,6 +360,18 @@ fn default_silence_auto_stop_seconds() -> f32 { 3.0 } +fn default_voice_activation_wake_phrase() -> String { + crate::voice_activation::DEFAULT_WAKE_PHRASE.into() +} + +fn default_voice_activation_finish_phrase() -> String { + crate::voice_activation::DEFAULT_FINISH_PHRASE.into() +} + +fn default_voice_activation_timeout_seconds() -> u32 { + crate::voice_activation::DEFAULT_TIMEOUT_SECONDS +} + fn resolve_windows_insertion_mode( mode: WindowsInsertionMode, legacy_sendinput_only: bool, @@ -439,6 +451,21 @@ pub struct UserPreferences { /// Recording input device name. Empty string = system default microphone. #[serde(default)] pub microphone_device_name: String, + /// Opt-in local voice command listener. Standby audio is never sent to a cloud provider. + #[serde(default)] + pub voice_activation_enabled: bool, + #[serde(default = "default_true")] + pub voice_activation_wake_enabled: bool, + #[serde(default = "default_true")] + pub voice_activation_finish_enabled: bool, + #[serde(default = "default_voice_activation_wake_phrase")] + pub voice_activation_wake_phrase: String, + #[serde(default = "default_voice_activation_finish_phrase")] + pub voice_activation_finish_phrase: String, + #[serde(default = "default_true")] + pub voice_activation_timeout_enabled: bool, + #[serde(default = "default_voice_activation_timeout_seconds")] + pub voice_activation_timeout_seconds: u32, pub active_asr_provider: String, // "volcengine" | "apple-speech" | ... pub active_llm_provider: String, // "ark" | "openai" | ... /// Recognition pipeline mode (experimental, issue #902). With @@ -990,6 +1017,20 @@ struct UserPreferencesWire { silence_auto_stop_seconds: f32, #[serde(default)] microphone_device_name: String, + #[serde(default)] + voice_activation_enabled: bool, + #[serde(default = "default_true")] + voice_activation_wake_enabled: bool, + #[serde(default = "default_true")] + voice_activation_finish_enabled: bool, + #[serde(default = "default_voice_activation_wake_phrase")] + voice_activation_wake_phrase: String, + #[serde(default = "default_voice_activation_finish_phrase")] + voice_activation_finish_phrase: String, + #[serde(default = "default_true")] + voice_activation_timeout_enabled: bool, + #[serde(default = "default_voice_activation_timeout_seconds")] + voice_activation_timeout_seconds: u32, active_asr_provider: String, active_llm_provider: String, #[serde(default = "default_pipeline_mode")] @@ -1246,6 +1287,13 @@ impl Default for UserPreferencesWire { silence_auto_stop_enabled: prefs.silence_auto_stop_enabled, silence_auto_stop_seconds: prefs.silence_auto_stop_seconds, microphone_device_name: prefs.microphone_device_name, + voice_activation_enabled: prefs.voice_activation_enabled, + voice_activation_wake_enabled: prefs.voice_activation_wake_enabled, + voice_activation_finish_enabled: prefs.voice_activation_finish_enabled, + voice_activation_wake_phrase: prefs.voice_activation_wake_phrase, + voice_activation_finish_phrase: prefs.voice_activation_finish_phrase, + voice_activation_timeout_enabled: prefs.voice_activation_timeout_enabled, + voice_activation_timeout_seconds: prefs.voice_activation_timeout_seconds, active_asr_provider: prefs.active_asr_provider, active_llm_provider: prefs.active_llm_provider, pipeline_mode: prefs.pipeline_mode, @@ -1450,6 +1498,16 @@ impl<'de> Deserialize<'de> for UserPreferences { silence_auto_stop_enabled: wire.silence_auto_stop_enabled, silence_auto_stop_seconds: wire.silence_auto_stop_seconds, microphone_device_name: wire.microphone_device_name, + voice_activation_enabled: wire.voice_activation_enabled, + voice_activation_wake_enabled: wire.voice_activation_wake_enabled, + voice_activation_finish_enabled: wire.voice_activation_finish_enabled, + voice_activation_wake_phrase: wire.voice_activation_wake_phrase, + voice_activation_finish_phrase: wire.voice_activation_finish_phrase, + voice_activation_timeout_enabled: wire.voice_activation_timeout_enabled, + voice_activation_timeout_seconds: wire.voice_activation_timeout_seconds.clamp( + crate::voice_activation::MIN_TIMEOUT_SECONDS, + crate::voice_activation::MAX_TIMEOUT_SECONDS, + ), active_asr_provider: wire.active_asr_provider, active_llm_provider: wire.active_llm_provider, pipeline_mode: wire.pipeline_mode, @@ -1831,6 +1889,13 @@ impl Default for UserPreferences { silence_auto_stop_enabled: false, silence_auto_stop_seconds: default_silence_auto_stop_seconds(), microphone_device_name: String::new(), + voice_activation_enabled: false, + voice_activation_wake_enabled: true, + voice_activation_finish_enabled: true, + voice_activation_wake_phrase: default_voice_activation_wake_phrase(), + voice_activation_finish_phrase: default_voice_activation_finish_phrase(), + voice_activation_timeout_enabled: true, + voice_activation_timeout_seconds: default_voice_activation_timeout_seconds(), active_asr_provider: default_active_asr_provider(), active_llm_provider: "ark".into(), pipeline_mode: PipelineMode::Traditional, diff --git a/openless-all/app/crates/openless-core/src/voice_activation.rs b/openless-all/app/crates/openless-core/src/voice_activation.rs new file mode 100644 index 000000000..8c04983fe --- /dev/null +++ b/openless-all/app/crates/openless-core/src/voice_activation.rs @@ -0,0 +1,503 @@ +//! Pure policy and matching primitives for voice-activated dictation. +//! +//! The host owns the microphone and local recognizer. This module deliberately +//! only consumes revised recognition text and produces command decisions, so a +//! standby listener can never accidentally publish normal transcript events. + +use std::time::{Duration, Instant}; + +use unicode_normalization::UnicodeNormalization; + +use crate::types::SessionId; + +pub const DEFAULT_WAKE_PHRASE: &str = "开始语音录入"; +pub const DEFAULT_FINISH_PHRASE: &str = "完成语音录入"; +pub const DEFAULT_TIMEOUT_SECONDS: u32 = 30; +pub const MIN_TIMEOUT_SECONDS: u32 = 5; +pub const MAX_TIMEOUT_SECONDS: u32 = 300; +pub const MAX_PHRASE_CHARS: usize = 64; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PhraseValidationError { + Empty, + TooLong { max_chars: usize }, + ControlCharacter, +} + +impl std::fmt::Display for PhraseValidationError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Empty => write!(f, "voice activation phrase must not be empty"), + Self::TooLong { max_chars } => { + write!( + f, + "voice activation phrase must be at most {max_chars} characters" + ) + } + Self::ControlCharacter => { + write!( + f, + "voice activation phrase must not contain control characters" + ) + } + } + } +} + +impl std::error::Error for PhraseValidationError {} + +/// Normalize ASR output and user phrases before suffix matching. +/// +/// NFKC handles full-width punctuation and compatibility forms. Whitespace +/// and punctuation are intentionally removed because streaming ASR may insert +/// them between Chinese command characters while English recognition may emit +/// multiple spaces or hyphens. +pub fn normalize_phrase(text: &str) -> String { + text.nfkc() + .filter(|character| !character.is_whitespace() && !is_terminal_punctuation(*character)) + .flat_map(char::to_lowercase) + .collect() +} + +/// Remove a recognized command from the end of a transcript while preserving +/// the spacing and punctuation in the spoken body. +pub fn strip_phrase_suffix(text: &str, phrase: &str) -> Option { + let normalized_phrase = validate_phrase(phrase).ok()?; + let phrase_chars = normalized_phrase.chars().rev(); + let mut cursor = text.len(); + let mut cut_at = cursor; + for expected in phrase_chars { + while cursor > 0 { + let previous = text[..cursor].char_indices().next_back()?; + if previous.1.is_whitespace() || is_terminal_punctuation(previous.1) { + cursor = previous.0; + continue; + } + break; + } + let (start, character) = text[..cursor].char_indices().next_back()?; + if normalize_phrase(&character.to_string()) != expected.to_string() { + return None; + } + cut_at = start; + cursor = start; + } + Some(text[..cut_at].trim_end().to_string()) +} + +fn is_terminal_punctuation(character: char) -> bool { + character.to_string().nfkc().any(|normalized| { + normalized.is_ascii_punctuation() + || matches!( + normalized, + '。' | '！' + | '？' + | '，' + | '、' + | '；' + | '：' + | '“' + | '”' + | '‘' + | '’' + | '（' + | '）' + | '【' + | '】' + | '《' + | '》' + | '…' + | '～' + | '—' + | '–' + ) + }) +} + +pub fn validate_phrase(text: &str) -> Result { + let normalized = normalize_phrase(text); + if normalized.is_empty() { + return Err(PhraseValidationError::Empty); + } + if normalized.chars().any(char::is_control) { + return Err(PhraseValidationError::ControlCharacter); + } + if normalized.chars().count() > MAX_PHRASE_CHARS { + return Err(PhraseValidationError::TooLong { + max_chars: MAX_PHRASE_CHARS, + }); + } + Ok(normalized) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VoiceActivationConfig { + pub enabled: bool, + pub wake_enabled: bool, + pub finish_enabled: bool, + pub wake_phrase: String, + pub finish_phrase: String, + pub timeout_enabled: bool, + pub timeout_seconds: u32, + /// Runtime-only selected input; never serialized as a user preference field. + pub microphone_device_name: Option, + /// Runtime-owned generation; this is not persisted in preferences. + pub activation_generation: u64, +} + +impl Default for VoiceActivationConfig { + fn default() -> Self { + Self { + enabled: false, + wake_enabled: true, + finish_enabled: true, + wake_phrase: DEFAULT_WAKE_PHRASE.into(), + finish_phrase: DEFAULT_FINISH_PHRASE.into(), + timeout_enabled: true, + timeout_seconds: DEFAULT_TIMEOUT_SECONDS, + microphone_device_name: None, + activation_generation: 0, + } + } +} + +impl VoiceActivationConfig { + pub fn normalized(&self) -> Result { + let wake_phrase = validate_phrase(&self.wake_phrase)?; + let finish_phrase = validate_phrase(&self.finish_phrase)?; + Ok(Self { + enabled: self.enabled, + wake_enabled: self.wake_enabled, + finish_enabled: self.finish_enabled, + wake_phrase, + finish_phrase, + timeout_enabled: self.timeout_enabled, + timeout_seconds: self + .timeout_seconds + .clamp(MIN_TIMEOUT_SECONDS, MAX_TIMEOUT_SECONDS), + microphone_device_name: self.microphone_device_name.clone(), + activation_generation: self.activation_generation, + }) + } + + pub fn timeout(&self) -> Option { + self.timeout_enabled + .then(|| Duration::from_secs(u64::from(self.timeout_seconds))) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VoiceActivationCommand { + Wake, + Finish, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VoiceCommandMatch { + pub command: VoiceActivationCommand, + pub text_without_command: String, +} + +/// Matches only the end of the revised ASR text. A phrase in the middle of a +/// sentence must never trigger activation or stop an active dictation session. +#[derive(Debug, Clone)] +pub struct VoicePhraseMatcher { + normalized_phrase: String, + command: VoiceActivationCommand, +} + +impl VoicePhraseMatcher { + pub fn new( + phrase: impl AsRef, + command: VoiceActivationCommand, + ) -> Result { + Ok(Self { + normalized_phrase: validate_phrase(phrase.as_ref())?, + command, + }) + } + + pub fn matches_suffix(&self, text: &str) -> bool { + normalize_phrase(text).ends_with(&self.normalized_phrase) + } + + pub fn strip_suffix(&self, text: &str) -> Option { + let normalized = normalize_phrase(text); + normalized + .strip_suffix(&self.normalized_phrase) + .map(str::to_string) + } + + pub fn command(&self) -> VoiceActivationCommand { + self.command + } +} + +/// Handles ASR revisions. A revised result may replace a previous result, +/// therefore a command is reported at most once for each observed text value. +#[derive(Debug, Clone)] +pub struct VoiceCommandRecognizer { + wake: VoicePhraseMatcher, + finish: VoicePhraseMatcher, + last_text: String, + wake_reported: bool, + finish_reported: bool, +} + +impl VoiceCommandRecognizer { + pub fn new(config: &VoiceActivationConfig) -> Result { + let config = config.normalized()?; + Ok(Self { + wake: VoicePhraseMatcher::new(config.wake_phrase, VoiceActivationCommand::Wake)?, + finish: VoicePhraseMatcher::new(config.finish_phrase, VoiceActivationCommand::Finish)?, + last_text: String::new(), + wake_reported: false, + finish_reported: false, + }) + } + + pub fn observe( + &mut self, + text: &str, + command: VoiceActivationCommand, + ) -> Option { + let normalized = normalize_phrase(text); + if normalized == self.last_text { + return None; + } + self.last_text = normalized.clone(); + let matcher = match command { + VoiceActivationCommand::Wake => &self.wake, + VoiceActivationCommand::Finish => &self.finish, + }; + let already_reported = match command { + VoiceActivationCommand::Wake => self.wake_reported, + VoiceActivationCommand::Finish => self.finish_reported, + }; + if already_reported || !normalized.ends_with(&matcher.normalized_phrase) { + return None; + } + match command { + VoiceActivationCommand::Wake => self.wake_reported = true, + VoiceActivationCommand::Finish => self.finish_reported = true, + } + Some(VoiceCommandMatch { + command, + text_without_command: normalized + .strip_suffix(&matcher.normalized_phrase) + .unwrap_or_default() + .to_string(), + }) + } + + pub fn reset(&mut self) { + self.last_text.clear(); + self.wake_reported = false; + self.finish_reported = false; + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum VoiceActivationSignal { + WakeDetected { + activation_generation: u64, + }, + FinishDetected { + activation_generation: u64, + session_id: SessionId, + }, + RuntimeStateChanged { + activation_generation: u64, + phase: VoiceActivationPhase, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum VoiceActivationPhase { + Disabled, + Starting, + ListeningForWake, + WakeDetected, + HandingOff, + ListeningForFinish, + Finishing, + Unavailable, + PermissionDenied, + MicBusy, + Failed, +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct VoiceActivationStateSnapshot { + pub phase: VoiceActivationPhase, + pub activation_generation: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub session_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub message: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VoiceActivationStopReason { + Disabled, + DictationStarted, + FinishPhrase, + Timeout, + Cancelled, + RuntimeFailure, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct VoiceActivationGeneration(pub u64); + +impl VoiceActivationGeneration { + pub fn next(self) -> Self { + Self(self.0.saturating_add(1)) + } +} + +#[derive(Debug, Clone)] +pub struct VoiceActivationSession { + pub generation: VoiceActivationGeneration, + pub session_id: SessionId, + pub started_at: Instant, + pub timeout: Option, + pub finish_enabled: bool, + pub finish_phrase: String, +} + +impl VoiceActivationSession { + pub fn timed_out_at(&self, now: Instant) -> bool { + self.timeout + .is_some_and(|timeout| now.saturating_duration_since(self.started_at) >= timeout) + } + + pub fn accepts(&self, generation: VoiceActivationGeneration, session_id: SessionId) -> bool { + self.generation == generation && self.session_id == session_id + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VoiceActivationDecision { + Ignore, + StartDictation, + FinishDictation, + StopAfterTimeout, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn normalization_handles_spaces_width_case_and_punctuation() { + assert_eq!(normalize_phrase(" Ａ 开始 语音录入 "), "a开始语音录入"); + assert_eq!(normalize_phrase("完成\n语音\t录入"), DEFAULT_FINISH_PHRASE); + assert_eq!(normalize_phrase("完成，语音录入。"), DEFAULT_FINISH_PHRASE); + } + + #[test] + fn strip_phrase_suffix_preserves_spoken_body() { + assert_eq!( + strip_phrase_suffix("你好，世界 完成 语音录入", DEFAULT_FINISH_PHRASE), + Some("你好，世界".into()) + ); + assert_eq!( + strip_phrase_suffix("你好，世界 完成，语音录入。", DEFAULT_FINISH_PHRASE), + Some("你好，世界".into()) + ); + assert_eq!( + strip_phrase_suffix("你好，世界 完成语音录入．", DEFAULT_FINISH_PHRASE), + Some("你好，世界".into()) + ); + assert!(strip_phrase_suffix("完成语音录入之前", DEFAULT_FINISH_PHRASE).is_none()); + } + + #[test] + fn phrase_validation_rejects_empty_control_and_long_values() { + assert_eq!(validate_phrase(" \n"), Err(PhraseValidationError::Empty)); + assert_eq!( + validate_phrase("开始\u{0000}录入"), + Err(PhraseValidationError::ControlCharacter) + ); + assert!(matches!( + validate_phrase(&"a".repeat(MAX_PHRASE_CHARS + 1)), + Err(PhraseValidationError::TooLong { .. }) + )); + } + + #[test] + fn matcher_only_accepts_suffix_and_strips_command() { + let matcher = + VoicePhraseMatcher::new(DEFAULT_FINISH_PHRASE, VoiceActivationCommand::Finish).unwrap(); + assert!(!matcher.matches_suffix("完成语音录入之前的内容")); + assert!(matcher.matches_suffix("正文 完成，语音录入。")); + assert_eq!( + matcher.strip_suffix("正文 完成语音录入").as_deref(), + Some( + "正文完成语音录入" + .strip_suffix(DEFAULT_FINISH_PHRASE) + .unwrap() + ) + ); + } + + #[test] + fn recognizer_does_not_repeat_a_command_for_asr_revisions() { + let config = VoiceActivationConfig::default(); + let mut recognizer = VoiceCommandRecognizer::new(&config).unwrap(); + assert!(recognizer + .observe("开始语音", VoiceActivationCommand::Wake) + .is_none()); + assert!(recognizer + .observe("开始语音录入", VoiceActivationCommand::Wake) + .is_some()); + assert!(recognizer + .observe("开始语音录入", VoiceActivationCommand::Wake) + .is_none()); + recognizer.reset(); + assert!(recognizer + .observe("开始语音录入", VoiceActivationCommand::Wake) + .is_some()); + } + + #[test] + fn generation_and_session_guards_reject_stale_events() { + let session_id = SessionId::new(); + let session = VoiceActivationSession { + generation: VoiceActivationGeneration(7), + session_id, + started_at: Instant::now(), + timeout: Some(Duration::from_secs(1)), + finish_enabled: true, + finish_phrase: DEFAULT_FINISH_PHRASE.into(), + }; + assert!(session.accepts(VoiceActivationGeneration(7), session_id)); + assert!(!session.accepts(VoiceActivationGeneration(6), session_id)); + assert!(!session.accepts(VoiceActivationGeneration(7), SessionId::new())); + } + + #[test] + fn timeout_is_bounded_and_only_applies_after_the_voice_session_starts() { + let config = VoiceActivationConfig { + timeout_seconds: 1, + ..VoiceActivationConfig::default() + }; + assert_eq!( + config.normalized().unwrap().timeout_seconds, + MIN_TIMEOUT_SECONDS + ); + let session = VoiceActivationSession { + generation: VoiceActivationGeneration(1), + session_id: SessionId::new(), + started_at: Instant::now(), + timeout: Some(Duration::from_secs(1)), + finish_enabled: true, + finish_phrase: DEFAULT_FINISH_PHRASE.into(), + }; + assert!(!session.timed_out_at(Instant::now())); + assert!(session.timed_out_at(session.started_at + Duration::from_secs(1))); + } +} diff --git a/openless-all/app/linux-egui/examples/headless_host.rs b/openless-all/app/linux-egui/examples/headless_host.rs index 2ada52eb1..809bc08b6 100644 --- a/openless-all/app/linux-egui/examples/headless_host.rs +++ b/openless-all/app/linux-egui/examples/headless_host.rs @@ -74,6 +74,7 @@ async fn main() -> Result<(), BackendError> { )), task_spawner: Arc::new(TokioTaskSpawner), credential_store: Arc::new(InMemoryCredentialStore::default()), + voice_activation_runtime: None, services: BackendServices::unsupported(), local_asr_runtime: None, marketplace_config: None, diff --git a/openless-all/app/linux-egui/src/backend.rs b/openless-all/app/linux-egui/src/backend.rs index c41399ff4..83500cbf8 100644 --- a/openless-all/app/linux-egui/src/backend.rs +++ b/openless-all/app/linux-egui/src/backend.rs @@ -284,6 +284,7 @@ impl LinuxBackendBuilder { dictation_engine, task_spawner, credential_store, + voice_activation_runtime: None, services, // Linux ships no local inference runtime. Leaving this unset // makes every local-ASR call resolve to Core's unsupported diff --git a/openless-all/app/linux-egui/src/runtime.rs b/openless-all/app/linux-egui/src/runtime.rs index 1f1fc0ae7..7786f0af6 100644 --- a/openless-all/app/linux-egui/src/runtime.rs +++ b/openless-all/app/linux-egui/src/runtime.rs @@ -245,6 +245,7 @@ mod tests { )), task_spawner: Arc::new(TokioTaskSpawner), credential_store: Arc::new(InMemoryCredentialStore::default()), + voice_activation_runtime: None, services, local_asr_runtime: None, marketplace_config: None, diff --git a/openless-all/app/linux-egui/tests/host_contract.rs b/openless-all/app/linux-egui/tests/host_contract.rs index f1c6cf384..d049227ad 100644 --- a/openless-all/app/linux-egui/tests/host_contract.rs +++ b/openless-all/app/linux-egui/tests/host_contract.rs @@ -66,6 +66,7 @@ async fn linux_host_exposes_snapshot_and_non_blocking_event_subscription() { dictation_engine: Arc::new(FixtureDictationEngine::successful("raw", "polished")), task_spawner: Arc::new(TokioTaskSpawner), credential_store: Arc::new(InMemoryCredentialStore::default()), + voice_activation_runtime: None, services: BackendServices::unsupported(), local_asr_runtime: None, marketplace_config: None, @@ -168,6 +169,7 @@ async fn forwarded_launch_intents_use_core_state_and_semantic_host_actions() { dictation_engine: Arc::new(engine.clone()), task_spawner: Arc::new(TokioTaskSpawner), credential_store: Arc::new(InMemoryCredentialStore::default()), + voice_activation_runtime: None, services: BackendServices::unsupported(), local_asr_runtime: None, marketplace_config: None, diff --git a/openless-all/app/src-tauri/backend-tests/tests/core_contract.rs b/openless-all/app/src-tauri/backend-tests/tests/core_contract.rs index 148966681..222b9852c 100644 --- a/openless-all/app/src-tauri/backend-tests/tests/core_contract.rs +++ b/openless-all/app/src-tauri/backend-tests/tests/core_contract.rs @@ -28,6 +28,7 @@ async fn backend_tests_can_use_the_framework_independent_core_contract() { dictation_engine: Arc::new(FixtureDictationEngine::successful("raw", "polished")), task_spawner: Arc::new(TokioTaskSpawner), credential_store: Arc::new(InMemoryCredentialStore::default()), + voice_activation_runtime: None, services: openless_core::BackendServices::unsupported(), local_asr_runtime: None, selection_runtime: None, diff --git a/openless-all/app/src-tauri/backend-tests/tests/voice_edit_host.rs b/openless-all/app/src-tauri/backend-tests/tests/voice_edit_host.rs index ee91c9cf8..2d8d8ed5f 100644 --- a/openless-all/app/src-tauri/backend-tests/tests/voice_edit_host.rs +++ b/openless-all/app/src-tauri/backend-tests/tests/voice_edit_host.rs @@ -237,6 +237,7 @@ fn fixture_with_transcript(transcript: &str) -> (tempfile::TempDir, Arc, dictation_engine: engine.clone(), task_spawner: Arc::new(TokioTaskSpawner), credential_store: Arc::new(InMemoryCredentialStore::default()), + voice_activation_runtime: None, services: openless_core::BackendServices::unsupported(), local_asr_runtime: None, selection_runtime: None, diff --git a/openless-all/app/src-tauri/src/android/native_bridge.rs b/openless-all/app/src-tauri/src/android/native_bridge.rs index c881f66e0..9248c41aa 100644 --- a/openless-all/app/src-tauri/src/android/native_bridge.rs +++ b/openless-all/app/src-tauri/src/android/native_bridge.rs @@ -1461,6 +1461,7 @@ mod tests { credential_store: Arc::new(InMemoryCredentialStore::default()), services: BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, selection_runtime: None, selection_polisher: None, qa_runtime: None, diff --git a/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs b/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs index 3d0004956..3d85ba37e 100644 --- a/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs +++ b/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs @@ -24,7 +24,7 @@ use parking_lot::Mutex; use crate::asr::RawTranscript; -use super::sherpa_runtime::{SherpaOnlineSession, SherpaOnnxRuntime}; +use super::sherpa_runtime::{SherpaOnlineSession, SherpaOnnxRuntime, SherpaTokenEmission}; pub struct SherpaOnnxAsr { runtime: Arc, @@ -54,7 +54,7 @@ enum OnlineWorkerMessage { Cancel, } -pub type SherpaTokenHandler = Arc; +pub type SherpaTokenHandler = Arc; impl SherpaOnnxAsr { pub fn new( @@ -281,12 +281,12 @@ impl OnlineWorker { let join_handle = std::thread::Builder::new() .name(format!("openless-sherpa-online-{alias}")) .spawn(move || { - let emit = |piece: &str| { - if piece.is_empty() || worker_cancelled.load(Ordering::SeqCst) { + let emit = |emission: &SherpaTokenEmission| { + if emission.text.is_empty() || worker_cancelled.load(Ordering::SeqCst) { return; } if let Some(handler) = token_handler.as_ref() { - handler(piece.to_string()); + handler(emission.clone()); } }; let result = loop { diff --git a/openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs b/openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs index 7de3b00f7..747c09cbc 100644 --- a/openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs +++ b/openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs @@ -26,6 +26,13 @@ use crate::asr::local::sherpa::{ }; const QWEN3_ASR_CHUNK_LIMIT_MS: u64 = 30_000; +#[derive(Clone, Debug)] +pub struct SherpaTokenEmission { + pub text: String, + pub is_delta: bool, + pub is_new_segment: bool, +} + const QWEN3_ASR_MAX_NEW_TOKENS: i32 = 256; #[cfg(target_os = "windows")] @@ -649,6 +656,7 @@ pub struct SherpaOnlineSession { committed_text: String, last_partial_text: String, last_emitted_text: String, + pending_new_segment: bool, } impl SherpaOnlineSession { @@ -658,7 +666,7 @@ impl SherpaOnlineSession { pub fn accept_pcm_chunk(&mut self, pcm: &[u8], on_token: &F) -> Result<()> where - F: Fn(&str), + F: Fn(&SherpaTokenEmission), { if pcm.is_empty() { return Ok(()); @@ -669,7 +677,7 @@ impl SherpaOnlineSession { pub fn finish(&mut self, on_token: &F) -> Result where - F: Fn(&str), + F: Fn(&SherpaTokenEmission), { self.finish_inner(on_token) } @@ -677,7 +685,7 @@ impl SherpaOnlineSession { #[cfg(target_os = "windows")] fn accept_samples(&mut self, samples: &[f32], on_token: &F) -> Result<()> where - F: Fn(&str), + F: Fn(&SherpaTokenEmission), { self.stream.accept_waveform(16_000, samples); self.drain_ready(on_token); @@ -687,7 +695,7 @@ impl SherpaOnlineSession { #[cfg(not(target_os = "windows"))] fn accept_samples(&mut self, _samples: &[f32], _on_token: &F) -> Result<()> where - F: Fn(&str), + F: Fn(&SherpaTokenEmission), { Ok(()) } @@ -695,7 +703,7 @@ impl SherpaOnlineSession { #[cfg(target_os = "windows")] fn finish_inner(&mut self, on_token: &F) -> Result where - F: Fn(&str), + F: Fn(&SherpaTokenEmission), { self.stream.input_finished(); self.drain_ready(on_token); @@ -706,7 +714,7 @@ impl SherpaOnlineSession { #[cfg(not(target_os = "windows"))] fn finish_inner(&mut self, _on_token: &F) -> Result where - F: Fn(&str), + F: Fn(&SherpaTokenEmission), { Ok(self.final_text()) } @@ -714,7 +722,7 @@ impl SherpaOnlineSession { #[cfg(target_os = "windows")] fn drain_ready(&mut self, on_token: &F) where - F: Fn(&str), + F: Fn(&SherpaTokenEmission), { while self.recognizer.is_ready(&self.stream) { self.recognizer.decode(&self.stream); @@ -725,7 +733,7 @@ impl SherpaOnlineSession { #[cfg(target_os = "windows")] fn capture_result(&mut self, on_token: &F, allow_endpoint_reset: bool) where - F: Fn(&str), + F: Fn(&SherpaTokenEmission), { let Some(result) = self.recognizer.get_result(&self.stream) else { return; @@ -740,24 +748,35 @@ impl SherpaOnlineSession { append_segment(&mut self.committed_text, &text); self.last_partial_text.clear(); self.last_emitted_text.clear(); + self.pending_new_segment = true; self.recognizer.reset(&self.stream); } } fn emit_delta(&mut self, text: &str, on_token: &F) where - F: Fn(&str), + F: Fn(&SherpaTokenEmission), { if text == self.last_emitted_text { return; } + let is_new_segment = self.pending_new_segment; if let Some(delta) = text.strip_prefix(&self.last_emitted_text) { if !delta.is_empty() { - on_token(delta); + on_token(&SherpaTokenEmission { + text: delta.to_string(), + is_delta: true, + is_new_segment, + }); } } else { - on_token(text); + on_token(&SherpaTokenEmission { + text: text.to_string(), + is_delta: false, + is_new_segment, + }); } + self.pending_new_segment = false; self.last_emitted_text = text.to_string(); } @@ -791,6 +810,7 @@ fn create_online_session_from_loaded(loaded: LoadedOnlineModel) -> Result Result Result<(), String> { let warm_hub = crate::recorder::warm_capture_hub(); + if !warm_hub.begin_preview() { + return Err("dictation is using the microphone".into()); + } + if let Some(backend) = app.try_state::>() { + if let Err(error) = backend + .stop_voice_activation(openless_core::VoiceActivationStopReason::Disabled) + .await + { + warm_hub.end_preview(); + return Err(error.to_string()); + } + } let warm_hub_for_error = Arc::clone(&warm_hub); let app_for_error = app.clone(); + let backend_for_resume = app + .try_state::>() + .map(|state| Arc::clone(&*state)); let result = match tauri::async_runtime::spawn_blocking(move || { let state = app.state::(); let _operation = state.operation.lock(); - if !warm_hub.begin_preview() { - return Err("dictation is using the microphone".into()); - } if let Some(existing) = state.recorder.lock().take() { existing.stop(); } @@ -176,9 +188,14 @@ pub async fn start_microphone_level_monitor( if state.recorder.lock().is_none() { warm_hub_for_error.end_preview(); } - return Err(format!("start microphone monitor task failed: {error}")); + Err(format!("start microphone monitor task failed: {error}")) } }; + if result.is_err() { + if let Some(backend) = backend_for_resume { + let _ = backend.configure_voice_activation().await; + } + } result } @@ -196,6 +213,10 @@ pub async fn stop_microphone_level_monitor(app: AppHandle) { #[cfg(not(mobile))] let app_for_error = app.clone(); #[cfg(not(mobile))] + let backend_for_resume = app + .try_state::>() + .map(|state| Arc::clone(&*state)); + #[cfg(not(mobile))] let result = tauri::async_runtime::spawn_blocking(move || { let state = app.state::(); let _operation = state.operation.lock(); @@ -216,6 +237,10 @@ pub async fn stop_microphone_level_monitor(app: AppHandle) { } #[cfg(not(mobile))] let _ = result; + #[cfg(not(mobile))] + if let Some(backend) = backend_for_resume { + let _ = backend.configure_voice_activation().await; + } } /// Copy the current session's openless.log to a user-chosen location (the frontend obtains diff --git a/openless-all/app/src-tauri/src/commands/mod.rs b/openless-all/app/src-tauri/src/commands/mod.rs index 00675353f..b3bfe1d48 100644 --- a/openless-all/app/src-tauri/src/commands/mod.rs +++ b/openless-all/app/src-tauri/src/commands/mod.rs @@ -111,6 +111,7 @@ pub use selection_polish_preview::*; #[cfg(all(not(mobile), target_os = "windows"))] pub use selection_voice::*; pub use settings::*; +pub(crate) use settings::sync_low_latency_capture_preferences; #[cfg(not(mobile))] // Sherpa commands are registered on Windows only; other desktop targets keep // an empty re-export. diff --git a/openless-all/app/src-tauri/src/commands/settings.rs b/openless-all/app/src-tauri/src/commands/settings.rs index 770f5aa0b..5c2f198ac 100644 --- a/openless-all/app/src-tauri/src/commands/settings.rs +++ b/openless-all/app/src-tauri/src/commands/settings.rs @@ -219,9 +219,10 @@ pub(crate) fn persist_strict_settings( } #[cfg(target_os = "windows")] -fn sync_low_latency_capture_preferences(prefs: &UserPreferences) { - crate::recorder::warm_capture_hub().configure( - prefs.low_latency_dictation_enabled, +pub(crate) fn sync_low_latency_capture_preferences(prefs: &UserPreferences) { + let warm_hub = crate::recorder::warm_capture_hub(); + warm_hub.configure( + prefs.low_latency_dictation_enabled || warm_hub.activation_is_active(), (!prefs.microphone_device_name.trim().is_empty()) .then(|| prefs.microphone_device_name.clone()), openless_core::AudioProcessingConfig { @@ -231,7 +232,38 @@ fn sync_low_latency_capture_preferences(prefs: &UserPreferences) { } #[cfg(not(target_os = "windows"))] -fn sync_low_latency_capture_preferences(_prefs: &UserPreferences) {} +pub(crate) fn sync_low_latency_capture_preferences(_prefs: &UserPreferences) {} + +fn voice_activation_preferences_changed(previous: &UserPreferences, next: &UserPreferences) -> bool { + previous.voice_activation_enabled != next.voice_activation_enabled + || previous.voice_activation_wake_enabled != next.voice_activation_wake_enabled + || previous.voice_activation_finish_enabled != next.voice_activation_finish_enabled + || previous.voice_activation_wake_phrase != next.voice_activation_wake_phrase + || previous.voice_activation_finish_phrase != next.voice_activation_finish_phrase + || previous.voice_activation_timeout_enabled != next.voice_activation_timeout_enabled + || previous.voice_activation_timeout_seconds != next.voice_activation_timeout_seconds + || previous.microphone_device_name != next.microphone_device_name +} + +fn validate_voice_activation_preferences( + prefs: &UserPreferences, +) -> Result<(), String> { + openless_core::VoiceActivationConfig { + enabled: prefs.voice_activation_enabled, + wake_enabled: prefs.voice_activation_wake_enabled, + finish_enabled: prefs.voice_activation_finish_enabled, + wake_phrase: prefs.voice_activation_wake_phrase.clone(), + finish_phrase: prefs.voice_activation_finish_phrase.clone(), + timeout_enabled: prefs.voice_activation_timeout_enabled, + timeout_seconds: prefs.voice_activation_timeout_seconds, + microphone_device_name: (!prefs.microphone_device_name.trim().is_empty()) + .then(|| prefs.microphone_device_name.clone()), + activation_generation: 0, + } + .normalized() + .map(|_| ()) + .map_err(|error| format!("invalid voice activation settings: {error}")) +} async fn invalidate_llm_tests_if_thinking_changed( coord: &Coordinator, @@ -269,6 +301,7 @@ pub async fn set_settings( .map_err(|e| e.to_string())?; sync_style_pack_preferences(&mut prefs, &packs); prefs.android_overlay_trigger = prefs.android_overlay_trigger.normalized(); + validate_voice_activation_preferences(&prefs)?; invalidate_llm_tests_if_thinking_changed(&coord, &remote_prev, &prefs).await?; // Persist changed fields and notify every WebView through the shared settings path. if let Some(edits) = edits { @@ -278,6 +311,11 @@ pub async fn set_settings( } let prefs = coord.backend().get_preferences(); sync_low_latency_capture_preferences(&prefs); + if voice_activation_preferences_changed(&remote_prev, &prefs) { + if let Err(error) = coord.backend().configure_voice_activation().await { + log::warn!("[voice-activation] settings reconfiguration failed: {error}"); + } + } // Sync the capsule-style atom on save: the next recording's entrance frame carries the // new style instead of depending on emit_capsule's ~30Hz main-thread closure sync (a // congested Windows main thread delays the closure → the whole session shows the old @@ -345,6 +383,7 @@ pub async fn set_settings( .map_err(|e| e.to_string())?; sync_style_pack_preferences(&mut prefs, &packs); prefs.android_overlay_trigger = prefs.android_overlay_trigger.normalized(); + validate_voice_activation_preferences(&prefs)?; invalidate_llm_tests_if_thinking_changed(&coord, &previous, &prefs).await?; if let Some(edits) = edits { persist_setting_fields(&coord, &edits)?; @@ -353,8 +392,12 @@ pub async fn set_settings( } let prefs = coord.backend().get_preferences(); sync_low_latency_capture_preferences(&prefs); - // Sync the capsule-style atom on save (same source as the Android notification capsule - // payload; see emit_capsule). + if voice_activation_preferences_changed(&previous, &prefs) { + if let Err(error) = coord.backend().configure_voice_activation().await { + log::warn!("[voice-activation] settings reconfiguration failed: {error}"); + } + } + // Sync the capsule-style atom on save (same source as the Android notification capsule payload; see emit_capsule). coord.sync_capsule_style_from_preferences(); // Rebuild the client connection pool immediately when the system-proxy switch changes (issue #869). if previous.use_system_proxy != prefs.use_system_proxy { diff --git a/openless-all/app/src-tauri/src/commands/sherpa_asr.rs b/openless-all/app/src-tauri/src/commands/sherpa_asr.rs index 877cb9cc2..a3f0bc220 100644 --- a/openless-all/app/src-tauri/src/commands/sherpa_asr.rs +++ b/openless-all/app/src-tauri/src/commands/sherpa_asr.rs @@ -208,12 +208,24 @@ pub async fn sherpa_onnx_asr_download_model( model_alias: String, mirror: Option, ) -> Result<(), String> { + let target = sherpa_target(model_alias)?; + #[cfg(target_os = "windows")] + let voice_activation_model = target.model_id() == "zipformer-bilingual-zh-en-streaming"; backend .services() .local_asr - .start_download(sherpa_target(model_alias)?, parse_mirror(mirror)) + .start_download(target, parse_mirror(mirror)) .await - .map_err(core_error) + .map_err(core_error)?; + #[cfg(target_os = "windows")] + if voice_activation_model && backend.get_preferences().voice_activation_enabled { + if let Err(error) = backend.configure_voice_activation().await { + log::warn!( + "[voice-activation] Sherpa model download completed but runtime refresh failed: {error}" + ); + } + } + Ok(()) } #[tauri::command] diff --git a/openless-all/app/src-tauri/src/core_adapters.rs b/openless-all/app/src-tauri/src/core_adapters.rs index 2b83bfae2..a44b88afd 100644 --- a/openless-all/app/src-tauri/src/core_adapters.rs +++ b/openless-all/app/src-tauri/src/core_adapters.rs @@ -15,12 +15,16 @@ use openless_core::{ RecordingArchive, RecordingProgressSink, SessionId, TextInserter as CoreTextInserter, TextInsertionSession, TextPolisher, TextStreamChunk, TextStreamSink, TranscriptOutput, TranscriptionEngine, TranscriptionSession, + VoiceActivationCommand, VoiceActivationConfig, VoiceActivationError, VoiceActivationPhase, + VoiceActivationSignal, VoiceActivationStopReason, VoiceCommandRecognizer, + VoiceDictationHandoff, VoiceDictationHandoffRequest, }; use parking_lot::Mutex; use tauri::{AppHandle, Emitter, Manager}; use crate::recorder::{ - warm_capture_hub, AudioConsumer as LegacyAudioConsumer, Recorder, RecorderError, + warm_capture_hub, AudioConsumer as LegacyAudioConsumer, AudioConsumerFanout, Recorder, + RecorderError, StagedRecorder, WarmActiveCapture, WarmCaptureHub, WarmStagedRecorder, STAGED_CAPTURE_MAX_BYTES, }; @@ -125,6 +129,15 @@ pub(crate) fn backend_dependencies( ); let warm_hub = warm_capture_hub(); #[cfg(target_os = "windows")] + let voice_activation_runtime = Arc::new(TauriVoiceActivationRuntime::new( + Arc::clone(&app), + Arc::clone(&backend), + Arc::clone(&native_asr_dependencies.sherpa), + model_store.clone(), + Arc::clone(&warm_hub), + Arc::clone(&preferences), + )); + #[cfg(target_os = "windows")] { let prefs = preferences.get(); warm_hub.configure( @@ -276,9 +289,14 @@ pub(crate) fn backend_dependencies( .expect("built-in Omni provider ids are non-empty"); } let mut dependencies = openless_core::BackendDependencies::unsupported(); - if let Some(model_store) = model_store { + if let Some(model_store) = model_store.clone() { dependencies.services.configure_model_store(model_store); } + #[cfg(target_os = "windows")] + { + dependencies.voice_activation_runtime = + Some(voice_activation_runtime as Arc); + } dependencies .services .configure_auxiliary_runtime(auxiliary_polisher, auxiliary_transcription); @@ -1475,6 +1493,555 @@ impl openless_core::PlatformApi for TauriPlatformApi { } #[cfg(target_os = "windows")] +const VOICE_ACTIVATION_MODEL_ALIAS: &str = "zipformer-bilingual-zh-en-streaming"; + +#[cfg(target_os = "windows")] +const MAX_VOICE_RECOGNIZER_CONTEXT_CHARS: usize = 256; + +#[cfg(target_os = "windows")] +fn map_voice_activation_recorder_error(error: RecorderError) -> VoiceActivationError { + match error { + RecorderError::PermissionDenied => VoiceActivationError::PermissionDenied, + RecorderError::NoInputDevice => { + VoiceActivationError::Runtime("no microphone input device detected".into()) + } + RecorderError::EngineFailed(message) + if message.to_ascii_lowercase().contains("busy") => VoiceActivationError::MicBusy, + RecorderError::EngineFailed(message) => VoiceActivationError::Runtime(message), + } +} + +#[cfg(target_os = "windows")] +struct VoiceActivationEmitter { + backend: BackendSlot, + sink: Arc, + generation: u64, + command: VoiceActivationCommand, + session_id: Arc>>, + binding_gate: Mutex<()>, + state: Mutex, +} + +#[cfg(target_os = "windows")] +struct VoiceActivationEmitterState { + recognizer: VoiceCommandRecognizer, + text: String, + segment_text: String, + pending_finish: bool, + pending_finish_match: bool, +} + +#[cfg(target_os = "windows")] +impl VoiceActivationEmitter { + fn new( + backend: BackendSlot, + sink: Arc, + generation: u64, + command: VoiceActivationCommand, + config: &VoiceActivationConfig, + session_id: Arc>>, + ) -> Result { + let recognizer = VoiceCommandRecognizer::new(config) + .map_err(|error| VoiceActivationError::InvalidConfig(error.to_string()))?; + Ok(Self { + backend, + sink, + generation, + command, + session_id, + binding_gate: Mutex::new(()), + state: Mutex::new(VoiceActivationEmitterState { + recognizer, + text: String::new(), + segment_text: String::new(), + pending_finish: false, + pending_finish_match: false, + }), + }) + } + + fn observe(&self, piece: String, is_delta: bool, is_new_segment: bool) { + if piece.trim().is_empty() { + return; + } + let _binding = self.binding_gate.lock(); + let bound_session_id = if self.command == VoiceActivationCommand::Finish { + *self.session_id.lock() + } else { + None + }; + let signal = { + let mut state = self.state.lock(); + if is_new_segment { + state.text.push_str(&piece); + state.segment_text = piece; + } else if is_delta { + state.text.push_str(&piece); + state.segment_text.push_str(&piece); + } else { + let prefix_len = state + .text + .chars() + .count() + .saturating_sub(state.segment_text.chars().count()); + state.text = state.text.chars().take(prefix_len).collect(); + state.text.push_str(&piece); + state.segment_text = piece; + } + if state.text.chars().count() > MAX_VOICE_RECOGNIZER_CONTEXT_CHARS { + state.text = state + .text + .chars() + .rev() + .take(MAX_VOICE_RECOGNIZER_CONTEXT_CHARS) + .collect::() + .chars() + .rev() + .collect(); + state.segment_text = state.text.clone(); + state.recognizer.reset(); + } + let text = state.text.clone(); + let matched = state.recognizer.observe(&text, self.command); + if self.command == VoiceActivationCommand::Finish && bound_session_id.is_none() { + state.pending_finish = true; + state.pending_finish_match |= matched.is_some(); + return; + } + let Some(_) = matched else { + state.pending_finish = false; + return; + }; + state.pending_finish = false; + state.pending_finish_match = false; + match self.command { + VoiceActivationCommand::Wake => { + VoiceActivationSignal::WakeDetected { + activation_generation: self.generation, + } + } + VoiceActivationCommand::Finish => VoiceActivationSignal::FinishDetected { + activation_generation: self.generation, + session_id: bound_session_id.expect("finish session was checked above"), + }, + } + }; + let _ = self.sink.publish(signal.clone()); + if let Some(backend) = self.backend.lock().as_ref().and_then(std::sync::Weak::upgrade) { + backend.handle_voice_activation_signal(signal); + } + } + + fn bind_session(&self, session_id: SessionId) { + { + let _binding = self.binding_gate.lock(); + *self.session_id.lock() = Some(session_id); + } + self.retry_pending(); + } + + fn retry_pending(&self) { + let (text, pending_match) = { + let _binding = self.binding_gate.lock(); + let mut state = self.state.lock(); + if !state.pending_finish { + return; + } + let pending_match = state.pending_finish_match; + state.pending_finish = false; + state.pending_finish_match = false; + (state.text.clone(), pending_match) + }; + if pending_match { + let Some(session_id) = *self.session_id.lock() else { + return; + }; + let signal = VoiceActivationSignal::FinishDetected { + activation_generation: self.generation, + session_id, + }; + let _ = self.sink.publish(signal.clone()); + if let Some(backend) = self.backend.lock().as_ref().and_then(std::sync::Weak::upgrade) { + backend.handle_voice_activation_signal(signal); + } + } else if !text.is_empty() { + self.observe(text, false, false); + } + } +} + +#[cfg(target_os = "windows")] +struct TauriVoiceActivationRuntime { + app: AppHandleSlot, + backend: BackendSlot, + sherpa: Arc, + model_store: Option>, + warm_hub: Arc, + preferences: Arc, +} + +#[cfg(target_os = "windows")] +impl TauriVoiceActivationRuntime { + fn new( + app: AppHandleSlot, + backend: BackendSlot, + sherpa: Arc, + model_store: Option>, + warm_hub: Arc, + preferences: Arc, + ) -> Self { + Self { + app, + backend, + sherpa, + model_store, + warm_hub, + preferences, + } + } +} + +#[cfg(target_os = "windows")] +impl openless_core::VoiceActivationRuntime for TauriVoiceActivationRuntime { + fn configure( + &self, + config: VoiceActivationConfig, + sink: Arc, + ) -> BoxFuture<'static, Result, VoiceActivationError>> { + let app = Arc::clone(&self.app); + let backend = Arc::clone(&self.backend); + let sherpa = Arc::clone(&self.sherpa); + let model_store = self.model_store.clone(); + let warm_hub = Arc::clone(&self.warm_hub); + let preferences = Arc::clone(&self.preferences); + Box::pin(async move { + let config = config + .normalized() + .map_err(|error| VoiceActivationError::InvalidConfig(error.to_string()))?; + let target = openless_core::LocalAsrTarget::parse( + openless_core::LocalAsrRuntime::SherpaOnnx, + VOICE_ACTIVATION_MODEL_ALIAS, + ) + .map_err(|error| VoiceActivationError::InvalidConfig(error.to_string()))?; + let Some(model_store) = model_store else { + return Err(VoiceActivationError::ModelMissing); + }; + if !model_store + .is_installed(&target) + .map_err(|error| VoiceActivationError::Runtime(error.to_string()))? + { + return Err(VoiceActivationError::ModelMissing); + } + let model_dir = model_store + .runtime_model_dir(&target) + .map_err(|error| VoiceActivationError::Runtime(error.to_string()))?; + let session_id = Arc::new(Mutex::new(None)); + let emitter = Arc::new(VoiceActivationEmitter::new( + Arc::clone(&backend), + Arc::clone(&sink), + config.activation_generation, + VoiceActivationCommand::Wake, + &config, + Arc::clone(&session_id), + )?); + let wake_asr = if config.wake_enabled { + let emitter_for_handler = Arc::clone(&emitter); + let token_handler = Arc::new( + move |emission: crate::asr::local::sherpa_runtime::SherpaTokenEmission| { + emitter_for_handler.observe( + emission.text, + emission.is_delta, + emission.is_new_segment, + ) + }, + ); + Some(Arc::new( + crate::asr::local::SherpaOnnxAsr::new_for_model( + Arc::clone(&sherpa), + VOICE_ACTIVATION_MODEL_ALIAS.to_string(), + model_dir.clone(), + Some("zh".into()), + Some(token_handler), + ) + .await + .map_err(|error| VoiceActivationError::Runtime(error.to_string()))?, + )) + } else { + None + }; + let preferences_snapshot = preferences.get(); + let fallback_warm_capture_enabled = preferences_snapshot.low_latency_dictation_enabled; + let audio_processing = openless_core::AudioProcessingConfig { + enhancement_enabled: preferences_snapshot.audio_enhancement_enabled, + }; + if let Some(wake_asr) = wake_asr.as_ref() { + warm_hub.set_activation_generation(config.activation_generation); + if !warm_hub.set_activation_consumer_for_generation( + config.activation_generation, + Some(Arc::clone(wake_asr) as Arc), + false, + ) { + return Err(VoiceActivationError::Runtime( + "voice activation generation became stale".into(), + )); + } + let fault_backend = Arc::clone(&backend); + let fault_sink = Arc::clone(&sink); + let fault_generation = config.activation_generation; + warm_hub.set_activation_fault_handler( + fault_generation, + Some(Arc::new(move |error| { + log::warn!("[voice-activation] warm microphone restart failed: {error}"); + let signal = VoiceActivationSignal::RuntimeStateChanged { + activation_generation: fault_generation, + phase: VoiceActivationPhase::Failed, + }; + let _ = fault_sink.publish(signal.clone()); + if let Some(backend) = fault_backend + .lock() + .as_ref() + .and_then(std::sync::Weak::upgrade) + { + backend.handle_voice_activation_signal(signal); + } + })), + ); + } + warm_hub.configure( + wake_asr.is_some() || fallback_warm_capture_enabled, + config.microphone_device_name.clone(), + audio_processing, + ); + if wake_asr.is_some() && !warm_hub.is_cold_capture_active() { + let probe = match warm_hub.arm_without_preroll( + config.activation_generation, + config.microphone_device_name.clone(), + ) { + Ok(probe) => probe, + Err(error) => { + warm_hub.clear_activation_fault_handler_if(config.activation_generation); + warm_hub.clear_activation_generation_if(config.activation_generation); + warm_hub.set_activation_consumer(None); + warm_hub.configure( + fallback_warm_capture_enabled, + config.microphone_device_name.clone(), + audio_processing, + ); + return Err(map_voice_activation_recorder_error(error)); + } + }; + drop(probe); + } + let handle = TauriVoiceActivationHandle { + app, + backend, + sherpa, + model_dir, + model_alias: VOICE_ACTIVATION_MODEL_ALIAS.to_string(), + warm_hub, + session_id, + sink: Arc::clone(&sink), + preferences, + config: config.clone(), + wake_asr: Arc::new(Mutex::new(wake_asr)), + finish_asr: Arc::new(Mutex::new(None)), + finish_emitter: Arc::new(Mutex::new(None)), + }; + let _ = sink.publish(VoiceActivationSignal::RuntimeStateChanged { + activation_generation: config.activation_generation, + phase: VoiceActivationPhase::ListeningForWake, + }); + Ok(Box::new(handle) as Box) + }) + } +} + +#[cfg(target_os = "windows")] +struct TauriVoiceActivationHandle { + app: AppHandleSlot, + backend: BackendSlot, + sherpa: Arc, + model_dir: PathBuf, + model_alias: String, + warm_hub: Arc, + session_id: Arc>>, + sink: Arc, + preferences: Arc, + config: VoiceActivationConfig, + wake_asr: Arc>>>, + finish_asr: Arc>>>, + finish_emitter: Arc>>>, +} + +#[cfg(target_os = "windows")] +impl openless_core::VoiceActivationHandle for TauriVoiceActivationHandle { + fn bind_dictation_session(&self, session_id: SessionId) { + if let Some(emitter) = self.finish_emitter.lock().clone() { + emitter.bind_session(session_id); + } else { + *self.session_id.lock() = Some(session_id); + } + } + + fn handoff_to_dictation( + &self, + request: VoiceDictationHandoffRequest, + ) -> BoxFuture<'static, Result> { + let app = Arc::clone(&self.app); + let backend = Arc::clone(&self.backend); + let backend_for_generation = Arc::clone(&backend); + let requested_generation = request.activation_generation; + let sherpa = Arc::clone(&self.sherpa); + let model_dir = self.model_dir.clone(); + let model_alias = self.model_alias.clone(); + let warm_hub = Arc::clone(&self.warm_hub); + let session_id = Arc::clone(&self.session_id); + let sink = Arc::clone(&self.sink); + let config = self.config.clone(); + let wake_asr = Arc::clone(&self.wake_asr); + let finish_asr = Arc::clone(&self.finish_asr); + let finish_emitter_slot = Arc::clone(&self.finish_emitter); + Box::pin(async move { + let generation_is_current = || { + backend_for_generation + .lock() + .as_ref() + .and_then(std::sync::Weak::upgrade) + .is_some_and(|backend| { + backend.voice_activation_generation_is_current(requested_generation) + }) + }; + if !generation_is_current() { + return Err(VoiceActivationError::Runtime( + "voice activation generation is stale".into(), + )); + } + let staged = warm_hub + .arm_without_preroll( + config.activation_generation, + config.microphone_device_name.clone(), + ) + .map_err(map_voice_activation_recorder_error)?; + if !generation_is_current() { + drop(staged); + return Err(VoiceActivationError::Runtime( + "voice activation generation is stale".into(), + )); + } + if !warm_hub.set_activation_consumer_for_generation( + requested_generation, + None, + false, + ) { + drop(staged); + return Err(VoiceActivationError::Runtime( + "voice activation generation is stale".into(), + )); + } + if let Some(wake_asr) = wake_asr.lock().take() { + wake_asr.cancel(); + } + if config.finish_enabled { + let emitter = Arc::new(VoiceActivationEmitter::new( + Arc::clone(&backend), + sink, + config.activation_generation, + VoiceActivationCommand::Finish, + &config, + Arc::clone(&session_id), + )?); + let emitter_for_handler = Arc::clone(&emitter); + let token_handler = Arc::new( + move |emission: crate::asr::local::sherpa_runtime::SherpaTokenEmission| { + emitter_for_handler.observe( + emission.text, + emission.is_delta, + emission.is_new_segment, + ) + }, + ); + let asr = Arc::new( + crate::asr::local::SherpaOnnxAsr::new_for_model( + Arc::clone(&sherpa), + model_alias, + model_dir, + Some("zh".into()), + Some(token_handler), + ) + .await + .map_err(|error| VoiceActivationError::Runtime(error.to_string()))?, + ); + if !generation_is_current() { + asr.cancel(); + drop(staged); + return Err(VoiceActivationError::Runtime( + "voice activation generation is stale".into(), + )); + } + if !warm_hub.set_activation_consumer_for_generation( + requested_generation, + Some(Arc::clone(&asr) as Arc), + true, + ) { + asr.cancel(); + drop(staged); + return Err(VoiceActivationError::Runtime( + "voice activation generation is stale".into(), + )); + } + *finish_asr.lock() = Some(asr); + *finish_emitter_slot.lock() = Some(emitter); + } + Ok(VoiceDictationHandoff { + capture: Box::new(TauriPendingAudioCapture { + app, + backend, + warm_hub, + staged: Some(TauriStagedCapture::Warm(staged)), + cold_capture_reservation: None, + }), + }) + }) + } + + fn stop( + &self, + _reason: VoiceActivationStopReason, + ) -> BoxFuture<'static, Result<(), VoiceActivationError>> { + let warm_hub = Arc::clone(&self.warm_hub); + let wake_asr = Arc::clone(&self.wake_asr); + let finish_asr = Arc::clone(&self.finish_asr); + let preferences = Arc::clone(&self.preferences); + let activation_generation = self.config.activation_generation; + Box::pin(async move { + let owns_activation = warm_hub.clear_activation_generation_if(activation_generation); + if owns_activation { + warm_hub.clear_activation_fault_handler_if(activation_generation); + warm_hub.set_activation_consumer(None); + } + if let Some(asr) = wake_asr.lock().take() { + asr.cancel(); + } + if let Some(asr) = finish_asr.lock().take() { + asr.cancel(); + } + if owns_activation { + let prefs = preferences.get(); + warm_hub.configure( + prefs.low_latency_dictation_enabled, + (!prefs.microphone_device_name.trim().is_empty()) + .then(|| prefs.microphone_device_name.clone()), + openless_core::AudioProcessingConfig { + enhancement_enabled: prefs.audio_enhancement_enabled, + }, + ); + } + Ok(()) + }) + } +} + +#[cfg(any(target_os = "windows", target_os = "macos"))] struct TauriNativeTranscriptionEngine { dependencies: TauriNativeAsrDependencies, model_store: Option>, @@ -1618,17 +2185,31 @@ impl TranscriptionEngine for TauriNativeTranscriptionEngine { }); let token_sink = Arc::clone(&partials); let token_offset = Arc::new(AtomicU64::new(0)); + let segment_start = Arc::new(AtomicU64::new(0)); let handler_offset = Arc::clone(&token_offset); - let token_handler = Arc::new(move |piece: String| { - let offset = - handler_offset.fetch_add(piece.chars().count() as u64, Ordering::AcqRel); - if let Err(error) = token_sink.publish(TextStreamChunk { - text: piece, - offset, - }) { - log::warn!("[core-adapter] publish sherpa partial failed: {error}"); - } - }); + let handler_segment_start = Arc::clone(&segment_start); + let token_handler = Arc::new( + move |emission: crate::asr::local::sherpa_runtime::SherpaTokenEmission| { + let text_len = emission.text.chars().count() as u64; + let offset = if emission.is_new_segment { + let offset = handler_offset.load(Ordering::Acquire); + handler_segment_start.store(offset, Ordering::Release); + handler_offset.fetch_add(text_len, Ordering::AcqRel) + } else if emission.is_delta { + handler_offset.fetch_add(text_len, Ordering::AcqRel) + } else { + let offset = handler_segment_start.load(Ordering::Acquire); + handler_offset.store(offset + text_len, Ordering::Release); + offset + }; + if let Err(error) = token_sink.publish(TextStreamChunk { + text: emission.text, + offset, + }) { + log::warn!("[core-adapter] publish sherpa partial failed: {error}"); + } + }, + ); let target = openless_core::LocalAsrTarget::parse( openless_core::LocalAsrRuntime::SherpaOnnx, model.clone(), @@ -1989,11 +2570,54 @@ impl TauriStagedCapture { } } +fn release_cold_capture_and_resume(warm_hub: &WarmCaptureHub) { + warm_hub.end_cold_capture(); + warm_hub.resume_after_cold_capture(); +} + +struct ColdCaptureReservation { + warm_hub: Arc, + resume_warm_hub: bool, + active: bool, +} + +impl ColdCaptureReservation { + fn new(warm_hub: Arc, resume_warm_hub: bool) -> Self { + Self { + warm_hub, + resume_warm_hub, + active: true, + } + } + + fn release(&mut self) { + if !self.active { + return; + } + self.active = false; + self.warm_hub.end_cold_capture(); + if self.resume_warm_hub { + self.warm_hub.resume_after_cold_capture(); + } + } + + fn disarm(&mut self) { + self.active = false; + } +} + +impl Drop for ColdCaptureReservation { + fn drop(&mut self) { + self.release(); + } +} + enum TauriCommittedCapture { Cold { recorder: Recorder, runtime_errors: std::sync::mpsc::Receiver, archive_active: bool, + reconfigure_voice_activation: bool, }, Warm { capture: WarmActiveCapture, @@ -2018,6 +2642,7 @@ impl TauriStagedCapture { recorder, runtime_errors, archive_active, + reconfigure_voice_activation: false, }) } Self::Warm(staged) => { @@ -2041,8 +2666,8 @@ impl TauriStagedCapture { if message == "warm provisional capture was not armed" ) => { - fallback_hub.begin_cold_capture()?; - fallback_hub.suspend_for_cold_capture(); + let finish_consumer = fallback_hub.activation_consumer_for_capture(); + fallback_hub.begin_cold_capture_and_suspend()?; let cold = match Recorder::start_staged_with_processing( fallback_device, STAGED_CAPTURE_MAX_BYTES, @@ -2050,18 +2675,25 @@ impl TauriStagedCapture { ) { Ok(cold) => cold, Err(error) => { - fallback_hub.end_cold_capture(); + release_cold_capture_and_resume(&fallback_hub); return Err(error); } }; let (recorder, runtime_errors, archive_active) = match cold.commit( - fallback_consumer, + finish_consumer + .map(|finish| { + AudioConsumerFanout::new(vec![ + Arc::clone(&fallback_consumer), + finish, + ]) as Arc + }) + .unwrap_or(fallback_consumer), fallback_level_handler, fallback_archive_path, ) { Ok(committed) => committed, Err(error) => { - fallback_hub.end_cold_capture(); + release_cold_capture_and_resume(&fallback_hub); return Err(error); } }; @@ -2069,6 +2701,7 @@ impl TauriStagedCapture { recorder, runtime_errors, archive_active, + reconfigure_voice_activation: true, }) } Err(error) => Err(error), @@ -2090,6 +2723,7 @@ struct TauriPendingAudioCapture { backend: BackendSlot, warm_hub: Arc, staged: Option, + cold_capture_reservation: Option, } struct TauriActiveRecording { @@ -2097,6 +2731,8 @@ struct TauriActiveRecording { warm_capture: Option, cold_capture_hub: Option>, resume_warm_hub: Option>, + backend: BackendSlot, + reconfigure_voice_activation: bool, archive: Option>, _mute: Option, } @@ -2114,6 +2750,7 @@ impl PendingAudioCapture for TauriPendingAudioCapture { backend, warm_hub, staged, + cold_capture_reservation, } = *self; let Some(staged) = staged else { return Box::pin(async { @@ -2123,6 +2760,8 @@ impl PendingAudioCapture for TauriPendingAudioCapture { )) }); }; + let cold_capture = staged.is_cold(); + let has_cold_capture_reservation = cold_capture_reservation.is_some(); Box::pin(async move { let permanent_archive = matches!( context.output_target, @@ -2147,7 +2786,9 @@ impl PendingAudioCapture for TauriPendingAudioCapture { #[cfg(not(mobile))] let monitor_app = app.lock().clone(); let warm_hub_for_join = Arc::clone(&warm_hub); + let backend_for_recording = Arc::clone(&backend); let joined = tauri::async_runtime::spawn_blocking(move || { + let mut cold_capture_reservation = cold_capture_reservation; let archive_path = archive_path .map_err(|error| { log::warn!("[recordings] archive unavailable; capture continues: {error}") @@ -2208,7 +2849,9 @@ impl PendingAudioCapture for TauriPendingAudioCapture { Ok(committed) => committed, Err(error) => { drop(mute); - warm_hub.end_cold_capture(); + if let Some(reservation) = cold_capture_reservation.as_mut() { + reservation.release(); + } return Err(error); } }; @@ -2219,11 +2862,13 @@ impl PendingAudioCapture for TauriPendingAudioCapture { runtime_errors, archive_active, resume_warm_hub, + reconfigure_voice_activation, ) = match committed { TauriCommittedCapture::Cold { recorder, runtime_errors, archive_active, + reconfigure_voice_activation, } => ( Some(recorder), None, @@ -2231,6 +2876,7 @@ impl PendingAudioCapture for TauriPendingAudioCapture { Some(runtime_errors), archive_active, Some(Arc::clone(&warm_hub)), + reconfigure_voice_activation, ), TauriCommittedCapture::Warm { capture, @@ -2243,21 +2889,21 @@ impl PendingAudioCapture for TauriPendingAudioCapture { Some(runtime_errors), archive_active, None, + false, ), }; if recording_plan.archive_required && !archive_active { - let has_warm_capture = warm_capture.is_some(); if let Some(recorder) = recorder { recorder.stop(); } if let Some(capture) = warm_capture { capture.stop(); } - if let Some(cold_capture_hub) = cold_capture_hub.as_ref() { + if let Some(reservation) = cold_capture_reservation.as_mut() { + reservation.release(); + } else if let Some(cold_capture_hub) = cold_capture_hub.as_ref() { cold_capture_hub.end_cold_capture(); - } - if !has_warm_capture { - if let Some(warm_hub) = resume_warm_hub { + if let Some(warm_hub) = resume_warm_hub.as_ref() { warm_hub.resume_after_cold_capture(); } } @@ -2269,6 +2915,9 @@ impl PendingAudioCapture for TauriPendingAudioCapture { "速记录音文件无法创建，已阻止开始录音以避免丢失内容", )); } + if let Some(reservation) = cold_capture_reservation.as_mut() { + reservation.disarm(); + } if recording_plan.archive_enabled { if let Err(error) = crate::persistence::prune_recordings( recording_plan.retention_days, @@ -2282,6 +2931,8 @@ impl PendingAudioCapture for TauriPendingAudioCapture { warm_capture, cold_capture_hub, resume_warm_hub, + backend: Arc::clone(&backend_for_recording), + reconfigure_voice_activation, archive: archive_path .map(|path| Arc::new(TauriRecordingArchive::new(path, archive_active))), _mute: mute, @@ -2292,7 +2943,9 @@ impl PendingAudioCapture for TauriPendingAudioCapture { Ok(Ok(result)) => result, Ok(Err(error)) => return Err(error), Err(error) => { - warm_hub_for_join.end_cold_capture(); + if cold_capture && !has_cold_capture_reservation { + release_cold_capture_and_resume(&warm_hub_for_join); + } return Err(BackendError::new( BackendErrorCode::Internal, format!("join promoted Tauri recorder task: {error}"), @@ -2330,26 +2983,32 @@ impl PendingAudioCapture for TauriPendingAudioCapture { fn discard(self: Box) -> BoxFuture<'static, Result<(), BackendError>> { let TauriPendingAudioCapture { - warm_hub, staged, .. + staged, + cold_capture_reservation, + .. } = *self; Box::pin(async move { let Some(staged) = staged else { + if let Some(reservation) = cold_capture_reservation { + drop(reservation); + } return Ok(()); }; - let cold_capture = staged.is_cold(); - let result = tauri::async_runtime::spawn_blocking(move || { - staged.discard().map_err(map_recorder_error) + let result = match tauri::async_runtime::spawn_blocking(move || { + let result = staged.discard().map_err(map_recorder_error); + if let Some(mut reservation) = cold_capture_reservation { + reservation.release(); + } + result }) .await - .map_err(|error| { - BackendError::new( + { + Ok(result) => result, + Err(error) => Err(BackendError::new( BackendErrorCode::Internal, format!("join provisional Tauri recorder discard task: {error}"), - ) - })?; - if cold_capture { - warm_hub.end_cold_capture(); - } + )), + }; result }) } @@ -2503,13 +3162,15 @@ impl ActiveRecording for TauriActiveRecording { let warm_capture = self.warm_capture.take(); let cold_capture_hub = self.cold_capture_hub.take(); let resume_warm_hub = self.resume_warm_hub.take(); + let backend = Arc::clone(&self.backend); + let reconfigure_voice_activation = self.reconfigure_voice_activation; if recorder.is_none() && warm_capture.is_none() { return Err(BackendError::new( BackendErrorCode::InvalidState, "Tauri recorder was already stopped", )); } - tauri::async_runtime::spawn_blocking(move || { + let stop_result = tauri::async_runtime::spawn_blocking(move || { if let Some(recorder) = recorder { recorder.stop(); } @@ -2533,7 +3194,21 @@ impl ActiveRecording for TauriActiveRecording { BackendErrorCode::Internal, format!("join Tauri recorder stop task: {error}"), ) - })? + })?; + if reconfigure_voice_activation { + let backend = { + let slot = backend.lock(); + slot.as_ref().and_then(std::sync::Weak::upgrade) + }; + if let Some(backend) = backend { + if let Err(error) = backend.configure_voice_activation().await { + log::warn!( + "[voice-activation] reconfigure after warm fallback release failed: {error}" + ); + } + } + } + stop_result }) } } @@ -2579,19 +3254,23 @@ impl TauriAudioRecorder { backend: Arc::clone(&self.backend), warm_hub: Arc::clone(&self.warm_hub), staged: Some(TauriStagedCapture::Warm(staged)), + cold_capture_reservation: None, })); } log::warn!("[recorder] warm capture unavailable; falling back to press-time staging"); } if !request.low_latency { - self.warm_hub.stop(); + self.warm_hub + .begin_cold_capture_and_suspend() + .map_err(map_recorder_error)?; + } else { + self.warm_hub + .begin_cold_capture() + .map_err(map_recorder_error)?; } let max_buffer_bytes = (request.pre_roll_ms as usize) .saturating_mul(32) .clamp(2, 16_000); - self.warm_hub - .begin_cold_capture() - .map_err(map_recorder_error)?; let staged = match Recorder::start_staged_with_processing( request.microphone_device_name, max_buffer_bytes, @@ -2599,7 +3278,7 @@ impl TauriAudioRecorder { ) { Ok(staged) => staged, Err(error) => { - self.warm_hub.end_cold_capture(); + release_cold_capture_and_resume(&self.warm_hub); return Err(map_recorder_error(error)); } }; @@ -2608,6 +3287,10 @@ impl TauriAudioRecorder { backend: Arc::clone(&self.backend), warm_hub: Arc::clone(&self.warm_hub), staged: Some(TauriStagedCapture::Cold(staged)), + cold_capture_reservation: Some(ColdCaptureReservation::new( + Arc::clone(&self.warm_hub), + !request.low_latency, + )), })) } } @@ -2694,6 +3377,7 @@ impl AudioRecorder for TauriAudioRecorder { || context.output_target == openless_core::DictationOutputTarget::Undecided); let fault_progress = Arc::clone(&progress); let worker_resume_warm_hub = Arc::clone(&resume_warm_hub); + let backend_for_recording = Arc::clone(&backend); let joined = tauri::async_runtime::spawn_blocking(move || { if prune_recordings_before_capture { if let Err(error) = crate::persistence::prune_recordings( @@ -2755,6 +3439,8 @@ impl AudioRecorder for TauriAudioRecorder { warm_capture: None, cold_capture_hub: Some(Arc::clone(&warm_hub)), resume_warm_hub: Some(worker_resume_warm_hub), + backend: Arc::clone(&backend_for_recording), + reconfigure_voice_activation: true, archive: archive_path .map(|path| Arc::new(TauriRecordingArchive::new(path, archive_active))), _mute: mute, diff --git a/openless-all/app/src-tauri/src/lib.rs b/openless-all/app/src-tauri/src/lib.rs index 37239aff6..0ee292dcb 100644 --- a/openless-all/app/src-tauri/src/lib.rs +++ b/openless-all/app/src-tauri/src/lib.rs @@ -1534,13 +1534,21 @@ fn handle_microphone_tray_menu_event(app: &AppHandle, id: &str) { }; let coord = app.state::>(); - if let Err(err) = coord - .backend() - .select_microphone_device(selected.device_name.clone()) - { + let backend = coord.backend(); + if let Err(err) = backend.select_microphone_device(selected.device_name.clone()) { log::warn!("[tray] save microphone preference failed: {err}"); return; } + commands::sync_low_latency_capture_preferences(&backend.get_preferences()); + let selected_device_name = selected.device_name.clone(); + let backend_for_reconfigure = Arc::clone(&backend); + tauri::async_runtime::spawn(async move { + if let Err(err) = backend_for_reconfigure.configure_voice_activation().await { + log::warn!( + "[tray] reconfigure microphone-dependent voice runtime for {selected_device_name:?} failed: {err}" + ); + } + }); commands::sync_tray_microphone_selection(&items, &selected.device_name); } diff --git a/openless-all/app/src-tauri/src/recorder.rs b/openless-all/app/src-tauri/src/recorder.rs index eb436f175..869a8cbbd 100644 --- a/openless-all/app/src-tauri/src/recorder.rs +++ b/openless-all/app/src-tauri/src/recorder.rs @@ -39,6 +39,20 @@ const AUDIO_PROCESSING_QUEUE_CAPACITY: usize = 64; const WAV_ARCHIVE_QUEUE_CAPACITY: usize = 256; /// The provisional capture cap is 500 ms at the canonical 16 kHz mono i16 format. pub(crate) const STAGED_CAPTURE_MAX_BYTES: usize = TARGET_SAMPLE_RATE as usize; +/// Voice handoff staging keeps up to 30 seconds while the formal dictation session is prepared. +const VOICE_STAGED_CAPTURE_MAX_BYTES: usize = TARGET_SAMPLE_RATE as usize * 30 * 2; +/// Voice activation wake/finish inference is fed only after this cheap RMS gate passes. +/// The level uses the same normalized 0..1 scale as `pcm_rms_level`. +const ACTIVATION_VAD_RMS_THRESHOLD: f32 = 0.015; +/// Retain a short silence pre-roll so the first consonant is not lost at the gate boundary. +const ACTIVATION_VAD_PRE_ROLL_BYTES: usize = TARGET_SAMPLE_RATE as usize * 2 / 5; +/// Keep sending a few quiet chunks after speech so phrase tails reach the recognizer. +const ACTIVATION_VAD_HANGOVER_CHUNKS: usize = 8; +const VOICE_CAPTURE_KEY_PREFIX: u64 = 1 << 63; + +fn voice_capture_key(generation: u64) -> u64 { + generation | VOICE_CAPTURE_KEY_PREFIX +} /// Downstream that receives resampled Int16 PCM bytes (little-endian). pub trait AudioConsumer: Send + Sync { @@ -47,6 +61,24 @@ pub trait AudioConsumer: Send + Sync { fn consume_pcm_chunk(&self, pcm: &[u8]); } +pub(crate) struct AudioConsumerFanout { + consumers: Vec>, +} + +impl AudioConsumerFanout { + pub(crate) fn new(consumers: Vec>) -> Arc { + Arc::new(Self { consumers }) + } +} + +impl AudioConsumer for AudioConsumerFanout { + fn consume_pcm_chunk(&self, pcm: &[u8]) { + for consumer in &self.consumers { + consumer.consume_pcm_chunk(pcm); + } + } +} + /// Compatibility bridge for legacy recorder call sites while cloud ASR /// implementations live in the framework-independent core. impl AudioConsumer for T @@ -295,9 +327,73 @@ impl CaptureSink { struct WarmCaptureSink { state: Mutex, delivery_gate: Mutex<()>, + activation_consumer: Mutex>>, + activation_consumer_during_capture: AtomicBool, + activation_generation: Mutex>, + activation_fault_handler: Mutex< + Option<(u64, Arc)>, + >, + activation_vad: Mutex, max_bytes: usize, } +struct ActivationVad { + pre_roll: VecDeque, + speech_active: bool, + hangover_chunks: usize, +} + +impl Default for ActivationVad { + fn default() -> Self { + Self { + pre_roll: VecDeque::with_capacity(ACTIVATION_VAD_PRE_ROLL_BYTES), + speech_active: false, + hangover_chunks: 0, + } + } +} + +impl ActivationVad { + fn reset(&mut self) { + self.pre_roll.clear(); + self.speech_active = false; + self.hangover_chunks = 0; + } + + /// Return PCM for the activation recognizer, or `None` for sustained silence. + /// This deliberately uses one RMS pass and a bounded queue rather than a second model. + fn gate(&mut self, pcm: &[u8]) -> Option> { + if pcm.is_empty() { + return None; + } + WarmCaptureSink::append_bounded( + &mut self.pre_roll, + pcm, + ACTIVATION_VAD_PRE_ROLL_BYTES, + ); + let speech = pcm_rms_level(pcm) >= ACTIVATION_VAD_RMS_THRESHOLD; + if speech { + let mut output = if self.speech_active { + Vec::new() + } else { + self.pre_roll.iter().copied().collect() + }; + if output.is_empty() { + output.extend_from_slice(pcm); + } + self.speech_active = true; + self.hangover_chunks = ACTIVATION_VAD_HANGOVER_CHUNKS; + return Some(output); + } + if self.speech_active && self.hangover_chunks > 0 { + self.hangover_chunks -= 1; + return Some(pcm.to_vec()); + } + self.speech_active = false; + None + } +} + enum WarmCaptureSinkState { Ring { pcm: VecDeque, @@ -309,6 +405,8 @@ enum WarmCaptureClaim { Staging { pcm: VecDeque, pre_roll_bytes: usize, + allow_postroll_eviction: bool, + max_bytes: usize, max_level: f32, }, Promoted(ForwardTarget), @@ -323,10 +421,42 @@ impl WarmCaptureSink { claims: HashMap::new(), }), delivery_gate: Mutex::new(()), + activation_consumer: Mutex::new(None), + activation_consumer_during_capture: AtomicBool::new(false), + activation_generation: Mutex::new(None), + activation_fault_handler: Mutex::new(None), + activation_vad: Mutex::new(ActivationVad::default()), max_bytes, }) } + fn activation_consumer_during_capture(&self) -> Option> { + self.activation_consumer_during_capture + .load(Ordering::Acquire) + .then(|| self.activation_consumer.lock().clone()) + .flatten() + } + + fn set_activation_consumer(&self, consumer: Option>) { + self.activation_consumer_during_capture.store(false, Ordering::Release); + *self.activation_consumer.lock() = consumer; + self.activation_vad.lock().reset(); + } + + fn set_activation_consumer_during_capture(&self, consumer: Option>) { + self.activation_consumer_during_capture + .store(consumer.is_some(), Ordering::Release); + *self.activation_consumer.lock() = consumer; + self.activation_vad.lock().reset(); + } + + fn deliver_activation(&self, consumer: &Arc, pcm: &[u8]) { + let Some(gated) = self.activation_vad.lock().gate(pcm) else { + return; + }; + consumer.consume_pcm_chunk(&gated); + } + fn append_bounded(buffer: &mut VecDeque, pcm: &[u8], max_bytes: usize) { let even_len = pcm.len() & !1; buffer.extend(pcm[..even_len].iter().copied()); @@ -339,26 +469,42 @@ impl WarmCaptureSink { fn append_staged( buffer: &mut VecDeque, pre_roll_bytes: &mut usize, + allow_postroll_eviction: bool, pcm: &[u8], max_bytes: usize, ) { let even_len = pcm.len() & !1; let incoming = &pcm[..even_len]; - let excess = buffer - .len() - .saturating_add(incoming.len()) - .saturating_sub(max_bytes); - let evictable = excess.min(*pre_roll_bytes); - for _ in 0..(evictable / 2) { + let excess = buffer.len().saturating_add(incoming.len()).saturating_sub(max_bytes); + let evictable_preroll = excess.min(*pre_roll_bytes); + for _ in 0..(evictable_preroll / 2) { + let _ = buffer.pop_front(); + let _ = buffer.pop_front(); + } + *pre_roll_bytes -= evictable_preroll; + let evictable_postroll = if allow_postroll_eviction { + excess.saturating_sub(evictable_preroll) + } else { + 0 + }; + for _ in 0..(evictable_postroll / 2) { let _ = buffer.pop_front(); let _ = buffer.pop_front(); } - *pre_roll_bytes -= evictable; let available = max_bytes.saturating_sub(buffer.len()) & !1; buffer.extend(incoming[..incoming.len().min(available)].iter().copied()); } - fn arm(&self, press_id: u64) -> Result<(), RecorderError> { + fn arm(&self, press_id: u64, include_preroll: bool) -> Result<(), RecorderError> { + self.arm_with_limit(press_id, include_preroll, self.max_bytes) + } + + fn arm_with_limit( + &self, + press_id: u64, + include_preroll: bool, + max_bytes: usize, + ) -> Result<(), RecorderError> { let _delivery = self.delivery_gate.lock(); let mut state = self.state.lock(); let WarmCaptureSinkState::Ring { pcm, claims } = &mut *state; @@ -367,11 +513,15 @@ impl WarmCaptureSink { "warm capture generation is already armed".into(), )); } + let max_bytes = max_bytes.max(2) & !1; + let staged = include_preroll.then(|| pcm.clone()).unwrap_or_default(); claims.insert( press_id, WarmCaptureClaim::Staging { - pcm: pcm.clone(), - pre_roll_bytes: pcm.len(), + pre_roll_bytes: staged.len(), + allow_postroll_eviction: !include_preroll, + max_bytes, + pcm: staged, max_level: 0.0, }, ); @@ -418,8 +568,18 @@ impl WarmCaptureSink { Some(WarmCaptureClaim::Staging { .. }) | None => None, } }; + let activation = self + .activation_consumer_during_capture + .load(Ordering::Acquire) + .then(|| self.activation_consumer.lock().clone()) + .flatten(); if let Some(target) = target { - CaptureSink::deliver(&target, &prefix); + if !prefix.is_empty() { + CaptureSink::deliver(&target, &prefix); + if let Some(activation) = activation { + self.deliver_activation(&activation, &prefix); + } + } (target.level_handler)(staged_level); } Ok(()) @@ -512,27 +672,52 @@ impl AudioConsumer for WarmCaptureSink { return; } let _delivery = self.delivery_gate.lock(); - let mut targets = Vec::new(); - { + let (targets, has_promoted_claim, has_staging_claim) = { let mut state = self.state.lock(); let WarmCaptureSinkState::Ring { pcm: ring, claims } = &mut *state; Self::append_bounded(ring, pcm, self.max_bytes); + let mut targets = Vec::new(); + let mut has_promoted_claim = false; + let mut has_staging_claim = false; for claim in claims.values_mut() { match claim { WarmCaptureClaim::Staging { pcm: staged, pre_roll_bytes, + allow_postroll_eviction, + max_bytes, .. } => { - Self::append_staged(staged, pre_roll_bytes, pcm, self.max_bytes); + has_staging_claim = true; + Self::append_staged( + staged, + pre_roll_bytes, + *allow_postroll_eviction, + pcm, + *max_bytes, + ); + } + WarmCaptureClaim::Promoted(target) => { + has_promoted_claim = true; + targets.push(target.clone()); } - WarmCaptureClaim::Promoted(target) => targets.push(target.clone()), } } - } + (targets, has_promoted_claim, has_staging_claim) + }; + let activation = ((!has_promoted_claim && !has_staging_claim) + || (has_promoted_claim + && self + .activation_consumer_during_capture + .load(Ordering::Acquire))) + .then(|| self.activation_consumer.lock().clone()) + .flatten(); for target in targets { CaptureSink::deliver(&target, pcm); } + if let Some(activation) = activation { + self.deliver_activation(&activation, pcm); + } } } @@ -701,6 +886,76 @@ impl WarmCaptureHub { } } + pub(crate) fn activation_consumer_for_capture(&self) -> Option> { + self.sink.activation_consumer_during_capture() + } + + pub(crate) fn set_activation_consumer(&self, consumer: Option>) { + self.sink.set_activation_consumer(consumer); + } + + pub(crate) fn set_activation_consumer_during_capture( + &self, + consumer: Option>, + ) { + self.sink.set_activation_consumer_during_capture(consumer); + } + + pub(crate) fn set_activation_consumer_for_generation( + &self, + generation: u64, + consumer: Option>, + during_capture: bool, + ) -> bool { + let active_generation = self.sink.activation_generation.lock(); + if *active_generation != Some(generation) { + return false; + } + if during_capture { + self.sink.set_activation_consumer_during_capture(consumer); + } else { + self.sink.set_activation_consumer(consumer); + } + true + } + + pub(crate) fn set_activation_generation(&self, generation: u64) { + *self.sink.activation_generation.lock() = Some(generation); + } + + pub(crate) fn clear_activation_generation_if(&self, generation: u64) -> bool { + let mut active_generation = self.sink.activation_generation.lock(); + if *active_generation == Some(generation) { + *active_generation = None; + true + } else { + false + } + } + + pub(crate) fn activation_is_active(&self) -> bool { + self.sink.activation_generation.lock().is_some() + } + + pub(crate) fn is_cold_capture_active(&self) -> bool { + self.cold_capture_active.load(Ordering::Acquire) + } + + pub(crate) fn set_activation_fault_handler( + &self, + generation: u64, + handler: Option>, + ) { + *self.sink.activation_fault_handler.lock() = handler.map(|handler| (generation, handler)); + } + + pub(crate) fn clear_activation_fault_handler_if(&self, generation: u64) { + let mut handler = self.sink.activation_fault_handler.lock(); + if handler.as_ref().is_some_and(|(current, _)| *current == generation) { + *handler = None; + } + } + pub(crate) fn configure( &self, enabled: bool, @@ -829,10 +1084,29 @@ impl WarmCaptureHub { let staging_claims = self.sink.staging_claim_ids(); self.runtime_fallback_claims.lock().extend(staging_claims); let archives = self.sink.clear(); - self.notify_runtime_error(error); + self.notify_runtime_error(error.clone()); + let active_activation_generation = *self.sink.activation_generation.lock(); + let activation_handler = { + let mut handler = self.sink.activation_fault_handler.lock(); + match handler.take() { + Some((handler_generation, callback)) + if active_activation_generation == Some(handler_generation) => + { + Some(callback) + } + Some(other) => { + *handler = Some(other); + None + } + None => None, + } + }; for archive in archives { archive.finish(); } + if let Some(handler) = activation_handler { + handler(error); + } } fn promote( @@ -874,22 +1148,65 @@ impl WarmCaptureHub { press_id: u64, device_name: Option, processing: openless_core::AudioProcessingConfig, + ) -> Result { + self.arm_internal( + press_id, + device_name, + true, + STAGED_CAPTURE_MAX_BYTES, + Some(processing), + ) + } + + pub(crate) fn arm_without_preroll( + self: &Arc, + press_id: u64, + device_name: Option, + ) -> Result { + self.arm_internal( + voice_capture_key(press_id), + device_name, + false, + VOICE_STAGED_CAPTURE_MAX_BYTES, + None, + ) + } + + fn arm_internal( + self: &Arc, + press_id: u64, + device_name: Option, + include_preroll: bool, + staged_max_bytes: usize, + processing: Option, ) -> Result { if !self.enabled.load(Ordering::Acquire) { return Err(RecorderError::EngineFailed( "warm capture is disabled".into(), )); } + if self.cold_capture_active.load(Ordering::Acquire) { + return Err(RecorderError::EngineFailed( + "warm capture is unavailable during cold capture".into(), + )); + } loop { let lifecycle = self.lifecycle_gate.lock(); + let desired_processing = processing.unwrap_or(*self.processing_config.lock()); if *self.device_name.lock() != device_name - || *self.processing_config.lock() != processing + || *self.processing_config.lock() != desired_processing { + if self.sink.has_promoted_claims() { + return Err(RecorderError::EngineFailed( + "warm capture device change is deferred while capture is active".into(), + )); + } drop(lifecycle); - self.configure(true, device_name.clone(), processing); + self.configure(true, device_name.clone(), desired_processing); continue; } - self.sink.arm(press_id)?; + self.sink + .arm_with_limit(press_id, include_preroll, staged_max_bytes)?; if let Err(error) = self.ensure_started_locked(device_name.clone()) { self.sink.discard(press_id); return Err(error); @@ -928,6 +1245,22 @@ impl WarmCaptureHub { pub(crate) fn begin_cold_capture(&self) -> Result<(), RecorderError> { let _lifecycle = self.lifecycle_gate.lock(); + self.begin_cold_capture_locked() + } + + /// Atomically reserve the microphone for a cold recorder and suspend the warm stream. + /// + /// The reservation is acquired before the stream is stopped while holding the same lifecycle + /// gate, so a concurrent warm arm cannot slip into the interval between admission and + /// suspension. + pub(crate) fn begin_cold_capture_and_suspend(&self) -> Result<(), RecorderError> { + let _lifecycle = self.lifecycle_gate.lock(); + self.begin_cold_capture_locked()?; + self.suspend_for_cold_capture_locked(); + Ok(()) + } + + fn begin_cold_capture_locked(&self) -> Result<(), RecorderError> { if self.cold_capture_active.swap(true, Ordering::AcqRel) { return Err(RecorderError::EngineFailed( "another microphone capture is already active".into(), @@ -1000,6 +1333,10 @@ impl WarmCaptureHub { /// preference enabled so the stream can be rebuilt when that recorder stops. pub(crate) fn suspend_for_cold_capture(&self) { let _lifecycle = self.lifecycle_gate.lock(); + self.suspend_for_cold_capture_locked(); + } + + fn suspend_for_cold_capture_locked(&self) { self.stream_generation.fetch_add(1, Ordering::AcqRel); self.stop_recorder(); let archives = self.sink.clear(); @@ -2317,6 +2654,76 @@ mod tests { .collect() } + #[test] + fn warm_promotion_replays_staged_prefix_to_finish_consumer() { + let sink = WarmCaptureSink::new(STAGED_CAPTURE_MAX_BYTES); + let formal = Arc::new(RecordingConsumer::default()); + let finish = Arc::new(RecordingConsumer::default()); + sink.arm(7, false).unwrap(); + sink.set_activation_consumer_during_capture(Some( + Arc::clone(&finish) as Arc, + )); + let speech = [0x00, 0x40, 0x00, 0x40]; + sink.consume_pcm_chunk(&speech); + + sink.promote( + 7, + Arc::clone(&formal) as Arc, + Arc::new(|_| {}), + None, + ) + .unwrap(); + + assert_eq!(formal.chunks.lock().unwrap().as_slice(), &[speech.to_vec()]); + assert_eq!(finish.chunks.lock().unwrap().as_slice(), &[speech.to_vec()]); + } + + #[test] + fn activation_vad_skips_silence_and_preserves_speech_onset() { + let sink = WarmCaptureSink::new(STAGED_CAPTURE_MAX_BYTES); + let activation = Arc::new(RecordingConsumer::default()); + sink.set_activation_consumer(Some(Arc::clone(&activation) as Arc)); + let silence = vec![0; 320]; + sink.consume_pcm_chunk(&silence); + assert!(activation.chunks.lock().unwrap().is_empty()); + + let speech = vec![0x00, 0x40, 0x00, 0x40]; + sink.consume_pcm_chunk(&speech); + let chunks = activation.chunks.lock().unwrap(); + assert_eq!(chunks.len(), 1); + assert!(chunks[0].ends_with(&speech)); + } + + #[test] + fn activation_vad_hangover_is_bounded() { + let mut vad = ActivationVad::default(); + let speech = vec![0x00, 0x40, 0x00, 0x40]; + assert!(vad.gate(&speech).is_some()); + for _ in 0..ACTIVATION_VAD_HANGOVER_CHUNKS { + assert!(vad.gate(&[0, 0, 0, 0]).is_some()); + } + assert!(vad.gate(&[0, 0, 0, 0]).is_none()); + } + #[test] + fn activation_fault_handler_uses_voice_generation_not_stream_generation() { + let hub = WarmCaptureHub::new(); + let called = Arc::new(AtomicBool::new(false)); + let called_for_handler = Arc::clone(&called); + hub.set_activation_generation(41); + hub.set_activation_fault_handler( + 41, + Some(Arc::new(move |_| { + called_for_handler.store(true, Ordering::Release); + })), + ); + + // The warm stream generation starts at zero; it is intentionally not the + // voice-activation generation owned by Core. + hub.stop_after_runtime_error(0, RecorderError::EngineFailed("stream failed".into())); + + assert!(called.load(Ordering::Acquire)); + } + #[test] fn native_alternatives_preserve_channels_format_and_advertised_rates() { let range = cpal::SupportedStreamConfigRange::new( @@ -2566,7 +2973,7 @@ mod tests { fn warm_sink_captures_bounded_preroll_and_post_press_audio_in_order() { let sink = WarmCaptureSink::new(8); sink.consume_pcm_chunk(&[1, 2, 3, 4, 5, 6, 7, 8]); - sink.arm(17).unwrap(); + sink.arm(17, true).unwrap(); sink.consume_pcm_chunk(&[9, 10]); sink.publish_level(0.75); @@ -2591,10 +2998,34 @@ mod tests { } #[test] - fn warm_sink_preserves_earliest_post_press_audio_after_preroll_eviction() { + fn warm_sink_activation_consumer_receives_standby_pcm_without_claiming_archive() { + let sink = WarmCaptureSink::new(16); + let consumer = Arc::new(RecordingConsumer::default()); + sink.set_activation_consumer(Some(consumer.clone() as Arc)); + sink.consume_pcm_chunk(&[1, 2, 3, 4]); + sink.set_activation_consumer(None); + sink.consume_pcm_chunk(&[5, 6]); + + assert_eq!(consumer.chunks.lock().unwrap().as_slice(), &[vec![1, 2, 3, 4]]); + assert!(matches!(&*sink.state.lock(), WarmCaptureSinkState::Ring { .. })); + } + + #[test] + fn warm_sink_activation_arm_excludes_wake_preroll() { + let sink = WarmCaptureSink::new(16); + sink.consume_pcm_chunk(&[1, 2, 3, 4]); + sink.arm(19, false).unwrap(); + sink.consume_pcm_chunk(&[5, 6]); + let consumer = Arc::new(RecordingConsumer::default()); + sink.promote(19, consumer.clone(), Arc::new(|_| {}), None) + .unwrap(); + assert_eq!(consumer.chunks.lock().unwrap().as_slice(), &[vec![5, 6]]); + } + #[test] + fn warm_sink_staging_is_bounded_to_the_capture_capacity() { let sink = WarmCaptureSink::new(8); sink.consume_pcm_chunk(&[1, 2, 3, 4, 5, 6, 7, 8]); - sink.arm(17).unwrap(); + sink.arm(17, true).unwrap(); sink.consume_pcm_chunk(&[9, 10, 11, 12, 13, 14, 15, 16]); sink.consume_pcm_chunk(&[17, 18, 19, 20, 21, 22]); @@ -2608,12 +3039,30 @@ mod tests { ); } + #[test] + fn warm_sink_without_preroll_keeps_latest_post_wake_audio() { + let sink = WarmCaptureSink::new(8); + sink.arm(17, false).unwrap(); + sink.consume_pcm_chunk(&[1, 2, 3, 4, 5, 6, 7, 8]); + sink.consume_pcm_chunk(&[9, 10, 11, 12, 13, 14, 15, 16]); + sink.consume_pcm_chunk(&[17, 18, 19, 20, 21, 22]); + + let consumer = Arc::new(RecordingConsumer::default()); + sink.promote(17, consumer.clone(), Arc::new(|_| {}), None) + .unwrap(); + + assert_eq!( + consumer.chunks.lock().unwrap().as_slice(), + &[vec![15, 16, 17, 18, 19, 20, 21, 22]] + ); + } + #[test] fn warm_sink_discard_removes_only_matching_generation() { let sink = WarmCaptureSink::new(16); sink.consume_pcm_chunk(&[1, 2]); - sink.arm(17).unwrap(); - sink.arm(18).unwrap(); + sink.arm(17, true).unwrap(); + sink.arm(18, true).unwrap(); sink.discard(17); sink.consume_pcm_chunk(&[3, 4]); @@ -2670,7 +3119,7 @@ mod tests { fn warm_config_change_keeps_staged_claim_on_cold_fallback() { let hub = WarmCaptureHub::new(); hub.enabled.store(true, Ordering::Release); - hub.sink.arm(42).unwrap(); + hub.sink.arm(42, true).unwrap(); hub.configure( false, @@ -2690,7 +3139,7 @@ mod tests { let hub = WarmCaptureHub::new(); hub.enabled.store(true, Ordering::Release); for press_id in 1..=claim_count { - hub.sink.arm(press_id).unwrap(); + hub.sink.arm(press_id, true).unwrap(); hub.sink .promote( press_id, diff --git a/openless-all/app/src-tauri/src/tauri_events.rs b/openless-all/app/src-tauri/src/tauri_events.rs index 3d0ca270b..f40dd3851 100644 --- a/openless-all/app/src-tauri/src/tauri_events.rs +++ b/openless-all/app/src-tauri/src/tauri_events.rs @@ -186,6 +186,9 @@ async fn forward_legacy_event( } emit_dictation_state(app, backend, snapshot) } + BackendEventKind::VoiceActivationStateChanged(state) => { + let _ = app.emit("voice-activation:state-changed", state); + } BackendEventKind::TranscriptDelta(_) => {} BackendEventKind::DictationCompleted(result) => { capsule_owners.transcription_notice = None; @@ -807,6 +810,9 @@ async fn emit_resync(app: &AppHandle, backend: &OpenLessBackend, owners: &mut Ca } let _ = app.emit("credentials:changed", snapshot.credentials); let _ = app.emit("vocab:updated", ()); + if let Some(state) = backend.latest_voice_activation_state() { + let _ = app.emit("voice-activation:state-changed", state); + } let qa = match backend.services().qa.snapshot().await { Ok(snapshot) => Some(snapshot), diff --git a/openless-all/app/src/components/SettingsModal.tsx b/openless-all/app/src/components/SettingsModal.tsx index dcde28b09..e74ab10d5 100644 --- a/openless-all/app/src/components/SettingsModal.tsx +++ b/openless-all/app/src/components/SettingsModal.tsx @@ -31,6 +31,7 @@ import { visibleAdvancedPages, isTestingFeaturePage, type AdvancedPageId, + type ServiceViewId, type SettingsSectionId, } from '../pages/settings/navigation'; import { ChannelEditorHostContext } from '../pages/settings/ChannelEditorHostContext'; @@ -73,6 +74,7 @@ export function SettingsModal({ initialSettingsSection ?? 'general', ); const [requestedAdvancedPage, setAdvancedPage] = useState(null); + const [requestedServiceView, setRequestedServiceView] = useState(); const [query, setQuery] = useState(''); const [platformCaps, setPlatformCaps] = useState(getCachedPlatformCapabilities); const experimentalAsr = useExperimentalAsr(platformCaps?.platform === 'desktop' && os === 'mac'); @@ -237,16 +239,23 @@ export function SettingsModal({ }); }; - const selectSection = (next: SettingsSectionId, fromSearch = false) => { + const selectSection = ( + next: SettingsSectionId, + fromSearch = false, + serviceView?: ServiceViewId, + ) => { void providerForm.finish(() => { channelCloseRef.current?.(); setSection(next); + setRequestedServiceView(next === 'services' ? serviceView : undefined); setAdvancedPage(null); setQuery(''); if (fromSearch) window.requestAnimationFrame(() => headingRef.current?.focus()); }); }; + const openLocalModels = () => selectSection('services', false, 'models'); + const handleKeyDown = (event: KeyboardEvent) => { // Nested selectors and shortcut recorders own their keys before the dialog does. if (event.defaultPrevented || event.nativeEvent.isComposing) return; @@ -743,11 +752,16 @@ export function SettingsModal({ gap: 16, }} > - {section === 'general' && } + {section === 'general' && } {section === 'inputMethod' && } {section === 'shortcuts' && } {section === 'appearance' && } - {section === 'services' && } + {section === 'services' && ( + + )} {section === 'privacy' && } {section === 'advanced' && ( { await invoke('plugin:autostart|disable'); } -export function RecordingInputSection() { +export function RecordingInputSection({ + onOpenLocalModels, +}: { + onOpenLocalModels?: () => void; +} = {}) { const { t } = useTranslation(); const os = detectOS(); const { prefs, capability, updatePrefs: savePrefs, refresh } = useHotkeySettings(); @@ -61,6 +71,8 @@ export function RecordingInputSection() { const [microphoneDevices, setMicrophoneDevices] = useState([]); const [microphoneDevicesLoaded, setMicrophoneDevicesLoaded] = useState(false); const [microphoneDevicesError, setMicrophoneDevicesError] = useState(null); + const [voiceActivationModelMissing, setVoiceActivationModelMissing] = useState(false); + const voiceActivationStateObservedRef = useRef(false); useEffect(() => { void getPlatformCapabilities().then(setPlatformCaps); @@ -131,6 +143,62 @@ export function RecordingInputSection() { }; }, [loadMicrophoneDevices]); + useEffect(() => { + if (!isTauri || os !== 'win') { + setVoiceActivationModelMissing(false); + return; + } + let cancelled = false; + let unlisten: (() => void) | undefined; + async function listenForVoiceActivationChanges() { + const { listen } = await import('@tauri-apps/api/event'); + if (cancelled) return; + const stopListening = await listen( + 'voice-activation:state-changed', + (event) => { + voiceActivationStateObservedRef.current = true; + setVoiceActivationModelMissing(isVoiceActivationModelMissing(event.payload)); + }, + ); + if (cancelled) { + stopListening(); + return; + } + unlisten = stopListening; + } + void listenForVoiceActivationChanges(); + return () => { + cancelled = true; + unlisten?.(); + }; + }, [os]); + + useEffect(() => { + if (!isTauri || os !== 'win' || !prefs?.voiceActivationEnabled) { + voiceActivationStateObservedRef.current = false; + setVoiceActivationModelMissing(false); + return; + } + voiceActivationStateObservedRef.current = false; + let cancelled = false; + void getSherpaOnnxAsrCatalog() + .then((catalog) => { + if (cancelled || voiceActivationStateObservedRef.current) return; + const model = catalog.find(({ alias }) => alias === VOICE_ACTIVATION_MODEL_ALIAS); + setVoiceActivationModelMissing(model?.cached === false); + }) + .catch(() => { + // The runtime event remains authoritative when the catalog cannot be read. + }); + return () => { + cancelled = true; + }; + }, [os, prefs?.voiceActivationEnabled]); + + useEffect(() => { + if (!prefs?.voiceActivationEnabled) setVoiceActivationModelMissing(false); + }, [prefs?.voiceActivationEnabled]); + const saveKeyboardListAffectingPrefs = useCallback( async (nextPrefs: UserPreferences) => { try { @@ -531,15 +599,133 @@ export function RecordingInputSection() { /> {showDesktopHotkey && os === 'win' && ( - - - + <> + + + + + savePrefs({ ...prefs, voiceActivationEnabled: next })} + /> + + {prefs.voiceActivationEnabled && ( +
+
+ {t('settings.recording.voiceActivationLocalOnly')} +
+ {voiceActivationModelMissing && ( +
+ + {t('settings.recording.voiceActivationModelMissing')} + + {onOpenLocalModels && ( + + {t('settings.recording.voiceActivationDownloadZipformer')} + + )} +
+ )} + + savePrefs({ ...prefs, voiceActivationWakeEnabled: next })} + /> + + {prefs.voiceActivationWakeEnabled && ( + + savePrefs({ ...prefs, voiceActivationWakePhrase: event.currentTarget.value }) + } + style={{ ...inputStyle, width: '100%', margin: '2px 0 8px' }} + /> + )} + + savePrefs({ ...prefs, voiceActivationFinishEnabled: next })} + /> + + {prefs.voiceActivationFinishEnabled && ( + + savePrefs({ + ...prefs, + voiceActivationFinishPhrase: event.currentTarget.value, + }) + } + style={{ ...inputStyle, width: '100%', margin: '2px 0 8px' }} + /> + )} + + + savePrefs({ ...prefs, voiceActivationTimeoutEnabled: next }) + } + /> + + {prefs.voiceActivationTimeoutEnabled && ( + + savePrefs({ + ...prefs, + voiceActivationTimeoutSeconds: Math.max( + 5, + Math.min(300, Number(event.currentTarget.value) || 30), + ), + }) + } + style={{ ...inputStyle, width: 90, marginTop: 2 }} + /> + )} +
+ )} + )} void } = {}) { const platformCaps = usePlatformCaps(); const showRemoteInput = platformCaps?.platform === 'desktop'; return ( <> - + {showRemoteInput && } @@ -116,17 +116,23 @@ export function AppearanceTab() { } // AI services & models: organizes channels, local models, and network settings by current pipeline and platform capabilities. -export function ServicesTab({ asrNavigation = 0 }: { asrNavigation?: number } = {}) { +export function ServicesTab({ + asrNavigation = 0, + initialView, +}: { asrNavigation?: number; initialView?: ServiceViewId } = {}) { const { t } = useTranslation(); const { prefs, updatePrefs } = useHotkeySettings(); const platformCaps = usePlatformCaps(); const showLocalModel = detectOS() !== 'mac' && platformCaps?.supportsLocalAsr === true; const multimodal = prefs?.multimodalPipelineEnabled === true && prefs.pipelineMode === 'multimodal'; - const [view, setView] = useState(asrNavigation ? 'asr' : 'llm'); + const [view, setView] = useState( + initialView ?? (asrNavigation ? 'asr' : 'llm'), + ); useEffect(() => { - if (asrNavigation) setView('asr'); - }, [asrNavigation]); + if (initialView) setView(initialView); + else if (asrNavigation) setView('asr'); + }, [initialView, asrNavigation]); const views = availableServiceViews(showLocalModel, detectOS()); const requestedView = resolveServiceView(view, views, multimodal, detectOS()); const contentRef = useRef(null); From f2892f7892e996d37d19225c26f818eb8fe565c8 Mon Sep 17 00:00:00 2001 From: HKLHaoBin Date: Sun, 18 Oct 2026 09:38:06 +0800 Subject: [PATCH 02/12] fix: preserve cross-platform activation event contracts Co-Authored-By: Claude Opus 5 --- .../app/crates/openless-core/src/api.rs | 101 ++++++++++++------ .../app/src-tauri/src/core_adapters.rs | 2 +- 2 files changed, 69 insertions(+), 34 deletions(-) diff --git a/openless-all/app/crates/openless-core/src/api.rs b/openless-all/app/crates/openless-core/src/api.rs index 25395f749..1fb722e2c 100644 --- a/openless-all/app/crates/openless-core/src/api.rs +++ b/openless-all/app/crates/openless-core/src/api.rs @@ -3157,6 +3157,7 @@ impl OpenLessBackend { .lock() .expect("voice activation handle lock poisoned") .take(); + let had_previous_handle = previous.is_some(); if let Some(previous) = previous { let _ = previous .stop(crate::voice_activation::VoiceActivationStopReason::Disabled) @@ -3166,17 +3167,31 @@ impl OpenLessBackend { || self.backend_shutdown_requested.load(Ordering::Acquire) || !self.snapshot().running { - self.events.publish( - None, - BackendEventKind::VoiceActivationStateChanged( - crate::voice_activation::VoiceActivationStateSnapshot { - phase: crate::voice_activation::VoiceActivationPhase::Disabled, - activation_generation: generation, - session_id: None, - message: None, - }, - ), - ); + // Do not add a synthetic event to the first startup transition when + // activation was never enabled. Existing subscribers expect + // BackendStarted to be the first startup event; later disable/error + // transitions still publish Disabled below when a visible state exists. + let should_publish_disabled = had_previous_handle + || self + .events + .latest_voice_activation_state() + .map(|state| { + state.phase != crate::voice_activation::VoiceActivationPhase::Disabled + }) + .unwrap_or(false); + if should_publish_disabled { + self.events.publish( + None, + BackendEventKind::VoiceActivationStateChanged( + crate::voice_activation::VoiceActivationStateSnapshot { + phase: crate::voice_activation::VoiceActivationPhase::Disabled, + activation_generation: generation, + session_id: None, + message: None, + }, + ), + ); + } return Ok(()); } let Some(runtime) = self.deps.voice_activation_runtime.as_ref() else { @@ -3303,23 +3318,33 @@ impl OpenLessBackend { + 1; self.voice_activation_handoff_generation .store(0, Ordering::Release); + let should_publish_disabled = handle.is_some() + || self + .events + .latest_voice_activation_state() + .map(|state| { + state.phase != crate::voice_activation::VoiceActivationPhase::Disabled + }) + .unwrap_or(false); if let Some(handle) = handle { handle .stop(reason) .await .map_err(map_voice_activation_error)?; } - self.events.publish( - None, - BackendEventKind::VoiceActivationStateChanged( - crate::voice_activation::VoiceActivationStateSnapshot { - phase: crate::voice_activation::VoiceActivationPhase::Disabled, - activation_generation: next_generation, - session_id: None, - message: None, - }, - ), - ); + if should_publish_disabled { + self.events.publish( + None, + BackendEventKind::VoiceActivationStateChanged( + crate::voice_activation::VoiceActivationStateSnapshot { + phase: crate::voice_activation::VoiceActivationPhase::Disabled, + activation_generation: next_generation, + session_id: None, + message: None, + }, + ), + ); + } Ok(()) } @@ -3347,23 +3372,33 @@ impl OpenLessBackend { + 1; self.voice_activation_handoff_generation .store(0, Ordering::Release); + let should_publish_disabled = handle.is_some() + || self + .events + .latest_voice_activation_state() + .map(|state| { + state.phase != crate::voice_activation::VoiceActivationPhase::Disabled + }) + .unwrap_or(false); if let Some(handle) = handle { handle .stop(reason) .await .map_err(map_voice_activation_error)?; } - self.events.publish( - None, - BackendEventKind::VoiceActivationStateChanged( - crate::voice_activation::VoiceActivationStateSnapshot { - phase: crate::voice_activation::VoiceActivationPhase::Disabled, - activation_generation: next_generation, - session_id: None, - message: None, - }, - ), - ); + if should_publish_disabled { + self.events.publish( + None, + BackendEventKind::VoiceActivationStateChanged( + crate::voice_activation::VoiceActivationStateSnapshot { + phase: crate::voice_activation::VoiceActivationPhase::Disabled, + activation_generation: next_generation, + session_id: None, + message: None, + }, + ), + ); + } Ok(true) } diff --git a/openless-all/app/src-tauri/src/core_adapters.rs b/openless-all/app/src-tauri/src/core_adapters.rs index a44b88afd..90493cbe9 100644 --- a/openless-all/app/src-tauri/src/core_adapters.rs +++ b/openless-all/app/src-tauri/src/core_adapters.rs @@ -2041,7 +2041,7 @@ impl openless_core::VoiceActivationHandle for TauriVoiceActivationHandle { } } -#[cfg(any(target_os = "windows", target_os = "macos"))] +#[cfg(target_os = "windows")] struct TauriNativeTranscriptionEngine { dependencies: TauriNativeAsrDependencies, model_store: Option>, From e40a3c271b8c7ef826f93b7a30d58416169d4a5b Mon Sep 17 00:00:00 2001 From: HKLHaoBin Date: Mon, 19 Oct 2026 02:13:27 +0800 Subject: [PATCH 03/12] fix: flush voice activation segments Co-Authored-By: Claude Opus 5 --- .../app/crates/openless-core/src/api.rs | 88 +++++++++---- .../openless-core/src/voice_activation.rs | 52 ++++++++ .../src/asr/local/sherpa_provider.rs | 26 ++++ .../src-tauri/src/asr/local/sherpa_runtime.rs | 46 +++++++ .../app/src-tauri/src/core_adapters.rs | 2 +- openless-all/app/src-tauri/src/recorder.rs | 123 ++++++++++++++++-- 6 files changed, 298 insertions(+), 39 deletions(-) diff --git a/openless-all/app/crates/openless-core/src/api.rs b/openless-all/app/crates/openless-core/src/api.rs index 1fb722e2c..372578627 100644 --- a/openless-all/app/crates/openless-core/src/api.rs +++ b/openless-all/app/crates/openless-core/src/api.rs @@ -2262,13 +2262,20 @@ impl EngineProgressSink for BackendEngineProgress { } } EngineProgress::TranscriptDelta(mut delta) => { - let finish_phrase = self + let activation_phrases = self .voice_activation_session .lock() .expect("voice activation session lock poisoned") .as_ref() - .filter(|session| session.finish_enabled && session.session_id == session_id) - .map(|session| session.finish_phrase.clone()); + .filter(|session| session.session_id == session_id) + .map(|session| { + ( + session.wake_phrase.clone(), + session + .finish_enabled + .then(|| session.finish_phrase.clone()), + ) + }); let mut state = self.state.write().expect("backend state lock poisoned"); ensure_active_session(&state, session_id)?; if !matches!( @@ -2282,7 +2289,7 @@ impl EngineProgressSink for BackendEngineProgress { "transcript delta arrived after transcription completed", )); } - if let Some(finish_phrase) = finish_phrase { + if let Some((wake_phrase, finish_phrase)) = activation_phrases { let mut raw_transcripts = self .raw_transcripts .lock() @@ -2300,12 +2307,21 @@ impl EngineProgressSink for BackendEngineProgress { transcript }); raw_transcript.apply(&delta)?; + let mut cleaned = raw_transcript.text().to_string(); + if let Some(stripped) = + crate::voice_activation::strip_phrase_prefix(&cleaned, &wake_phrase) + { + cleaned = stripped; + } + if let Some(finish_phrase) = finish_phrase { + if let Some(stripped) = + crate::voice_activation::strip_phrase_suffix(&cleaned, &finish_phrase) + { + cleaned = stripped; + } + } delta = crate::types::TranscriptDelta { - text: crate::voice_activation::strip_phrase_suffix( - raw_transcript.text(), - &finish_phrase, - ) - .unwrap_or_else(|| raw_transcript.text().to_string()), + text: cleaned, offset: 0, is_final: delta.is_final, }; @@ -3536,6 +3552,7 @@ impl OpenLessBackend { session_id, started_at: std::time::Instant::now(), timeout, + wake_phrase: preferences.voice_activation_wake_phrase.clone(), finish_enabled: preferences.voice_activation_finish_enabled, finish_phrase: preferences.voice_activation_finish_phrase, }; @@ -3689,6 +3706,7 @@ impl OpenLessBackend { let session_id = reservation.session_id; let preferences = self.get_preferences(); let activation_config = crate::voice_activation::VoiceActivationConfig { + wake_phrase: preferences.voice_activation_wake_phrase.clone(), finish_enabled: preferences.voice_activation_finish_enabled, finish_phrase: preferences.voice_activation_finish_phrase.clone(), timeout_enabled: preferences.voice_activation_timeout_enabled, @@ -3719,6 +3737,7 @@ impl OpenLessBackend { session_id, started_at: std::time::Instant::now(), timeout: activation_config.timeout(), + wake_phrase: activation_config.wake_phrase.clone(), finish_enabled: activation_config.finish_enabled, finish_phrase: activation_config.finish_phrase.clone(), }, @@ -7480,32 +7499,51 @@ impl OpenLessBackend { } }; - let finish_phrase = self + let activation_phrases = self .voice_activation_session .lock() .expect("voice activation session lock poisoned") .as_ref() - .filter(|session| session.finish_enabled && session.session_id == session_id) - .map(|session| session.finish_phrase.clone()); - if let Some(finish_phrase) = finish_phrase { - if let Some(stripped) = crate::voice_activation::strip_phrase_suffix( - &engine_result.raw_text, - &finish_phrase, - ) { - engine_result.raw_text = stripped; - } - if let Some(stripped) = crate::voice_activation::strip_phrase_suffix( - &engine_result.polished_text, - &finish_phrase, - ) { - engine_result.polished_text = stripped; + .filter(|session| session.session_id == session_id) + .map(|session| { + ( + session.wake_phrase.clone(), + session + .finish_enabled + .then(|| session.finish_phrase.clone()), + ) + }); + if let Some((wake_phrase, finish_phrase)) = activation_phrases { + for text in [ + &mut engine_result.raw_text, + &mut engine_result.polished_text, + ] { + if let Some(stripped) = + crate::voice_activation::strip_phrase_prefix(text, &wake_phrase) + { + *text = stripped; + } + if let Some(finish_phrase) = finish_phrase.as_ref() { + if let Some(stripped) = + crate::voice_activation::strip_phrase_suffix(text, finish_phrase) + { + *text = stripped; + } + } } if let Some(asr_transcript) = engine_result.asr_transcript.as_mut() { if let Some(stripped) = - crate::voice_activation::strip_phrase_suffix(asr_transcript, &finish_phrase) + crate::voice_activation::strip_phrase_prefix(asr_transcript, &wake_phrase) { *asr_transcript = stripped; } + if let Some(finish_phrase) = finish_phrase.as_ref() { + if let Some(stripped) = + crate::voice_activation::strip_phrase_suffix(asr_transcript, finish_phrase) + { + *asr_transcript = stripped; + } + } } } diff --git a/openless-all/app/crates/openless-core/src/voice_activation.rs b/openless-all/app/crates/openless-core/src/voice_activation.rs index 8c04983fe..8f2393fc1 100644 --- a/openless-all/app/crates/openless-core/src/voice_activation.rs +++ b/openless-all/app/crates/openless-core/src/voice_activation.rs @@ -85,6 +85,34 @@ pub fn strip_phrase_suffix(text: &str, phrase: &str) -> Option { Some(text[..cut_at].trim_end().to_string()) } +/// Remove a recognized command from the beginning of a transcript while preserving +/// the spoken body. Only a true prefix match is accepted. +pub fn strip_phrase_prefix(text: &str, phrase: &str) -> Option { + let normalized_phrase = validate_phrase(phrase).ok()?; + let mut cursor = 0; + for expected in normalized_phrase.chars() { + loop { + let (_, character) = text[cursor..].char_indices().next()?; + cursor += character.len_utf8(); + if character.is_whitespace() || is_terminal_punctuation(character) { + continue; + } + if normalize_phrase(&character.to_string()) != expected.to_string() { + return None; + } + break; + } + } + while cursor < text.len() { + let character = text[cursor..].chars().next()?; + if !character.is_whitespace() && !is_terminal_punctuation(character) { + break; + } + cursor += character.len_utf8(); + } + Some(text[cursor..].to_string()) +} + fn is_terminal_punctuation(character: char) -> bool { character.to_string().nfkc().any(|normalized| { normalized.is_ascii_punctuation() @@ -364,6 +392,7 @@ pub struct VoiceActivationSession { pub session_id: SessionId, pub started_at: Instant, pub timeout: Option, + pub wake_phrase: String, pub finish_enabled: bool, pub finish_phrase: String, } @@ -398,6 +427,27 @@ mod tests { assert_eq!(normalize_phrase("完成，语音录入。"), DEFAULT_FINISH_PHRASE); } + #[test] + fn strip_phrase_prefix_preserves_body_and_requires_prefix() { + assert_eq!( + strip_phrase_prefix("开始语音录入，你好，世界", DEFAULT_WAKE_PHRASE), + Some("你好，世界".into()) + ); + assert_eq!( + strip_phrase_prefix("开始，语音录入 hello", DEFAULT_WAKE_PHRASE), + Some("hello".into()) + ); + assert!(strip_phrase_prefix("你好，开始语音录入", DEFAULT_WAKE_PHRASE).is_none()); + } + + #[test] + fn strip_phrase_prefix_handles_nfkc_and_empty_body() { + assert_eq!(strip_phrase_prefix("開始語音錄入。", "开始语音录入"), None); + assert_eq!( + strip_phrase_prefix("开始语音录入。", DEFAULT_WAKE_PHRASE), + Some(String::new()) + ); + } #[test] fn strip_phrase_suffix_preserves_spoken_body() { assert_eq!( @@ -471,6 +521,7 @@ mod tests { session_id, started_at: Instant::now(), timeout: Some(Duration::from_secs(1)), + wake_phrase: DEFAULT_WAKE_PHRASE.into(), finish_enabled: true, finish_phrase: DEFAULT_FINISH_PHRASE.into(), }; @@ -494,6 +545,7 @@ mod tests { session_id: SessionId::new(), started_at: Instant::now(), timeout: Some(Duration::from_secs(1)), + wake_phrase: DEFAULT_WAKE_PHRASE.into(), finish_enabled: true, finish_phrase: DEFAULT_FINISH_PHRASE.into(), }; diff --git a/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs b/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs index 3d85ba37e..b319dd98c 100644 --- a/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs +++ b/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs @@ -50,6 +50,7 @@ struct OnlineWorker { enum OnlineWorkerMessage { Pcm(Vec), + FlushSegment, Finish, Cancel, } @@ -252,6 +253,14 @@ impl crate::recorder::AudioConsumer for SherpaOnnxAsr { } } } + + fn finish_segment(&self) { + if let SherpaProviderMode::Online { worker } = &self.mode { + if let Some(worker) = worker.lock().as_ref() { + worker.flush_segment(); + } + } + } } fn pcm_duration_ms(pcm: &[u8]) -> u64 { @@ -302,6 +311,14 @@ impl OnlineWorker { break Err(anyhow::anyhow!("sherpa-onnx streaming cancelled")); } } + Ok(OnlineWorkerMessage::FlushSegment) => { + if worker_cancelled.load(Ordering::SeqCst) { + break Err(anyhow::anyhow!("sherpa-onnx streaming cancelled")); + } + if let Err(error) = session.flush_segment(&emit) { + break Err(error); + } + } Ok(OnlineWorkerMessage::Finish) => { if worker_cancelled.load(Ordering::SeqCst) { break Err(anyhow::anyhow!("sherpa-onnx streaming cancelled")); @@ -342,6 +359,15 @@ impl OnlineWorker { } } + fn flush_segment(&self) { + if self.cancelled.load(Ordering::SeqCst) { + return; + } + if self.tx.send(OnlineWorkerMessage::FlushSegment).is_err() { + log::warn!("[sherpa-asr] online worker is not accepting segment flush"); + } + } + async fn finish(self, audio_timeout: Duration) -> Result { let _ = self.tx.send(OnlineWorkerMessage::Finish); let result_rx = self diff --git a/openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs b/openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs index 747c09cbc..ca4fff387 100644 --- a/openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs +++ b/openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs @@ -682,6 +682,15 @@ impl SherpaOnlineSession { self.finish_inner(on_token) } + /// Flush the current VAD segment while keeping this online session alive for + /// the next speech segment. + pub fn flush_segment(&mut self, on_token: &F) -> Result<()> + where + F: Fn(&SherpaTokenEmission), + { + self.flush_segment_inner(on_token) + } + #[cfg(target_os = "windows")] fn accept_samples(&mut self, samples: &[f32], on_token: &F) -> Result<()> where @@ -719,6 +728,32 @@ impl SherpaOnlineSession { Ok(self.final_text()) } + #[cfg(target_os = "windows")] + fn flush_segment_inner(&mut self, on_token: &F) -> Result<()> + where + F: Fn(&SherpaTokenEmission), + { + self.stream.input_finished(); + self.drain_ready_without_endpoint_reset(on_token); + self.capture_result(on_token, false); + if !self.last_partial_text.is_empty() { + append_segment(&mut self.committed_text, &self.last_partial_text); + } + self.last_partial_text.clear(); + self.last_emitted_text.clear(); + self.pending_new_segment = true; + self.recognizer.reset(&self.stream); + Ok(()) + } + + #[cfg(not(target_os = "windows"))] + fn flush_segment_inner(&mut self, _on_token: &F) -> Result<()> + where + F: Fn(&SherpaTokenEmission), + { + Ok(()) + } + #[cfg(target_os = "windows")] fn drain_ready(&mut self, on_token: &F) where @@ -730,6 +765,17 @@ impl SherpaOnlineSession { } } + #[cfg(target_os = "windows")] + fn drain_ready_without_endpoint_reset(&mut self, on_token: &F) + where + F: Fn(&SherpaTokenEmission), + { + while self.recognizer.is_ready(&self.stream) { + self.recognizer.decode(&self.stream); + self.capture_result(on_token, false); + } + } + #[cfg(target_os = "windows")] fn capture_result(&mut self, on_token: &F, allow_endpoint_reset: bool) where diff --git a/openless-all/app/src-tauri/src/core_adapters.rs b/openless-all/app/src-tauri/src/core_adapters.rs index 90493cbe9..a6fcaca05 100644 --- a/openless-all/app/src-tauri/src/core_adapters.rs +++ b/openless-all/app/src-tauri/src/core_adapters.rs @@ -1917,7 +1917,7 @@ impl openless_core::VoiceActivationHandle for TauriVoiceActivationHandle { )); } let staged = warm_hub - .arm_without_preroll( + .arm_with_preroll( config.activation_generation, config.microphone_device_name.clone(), ) diff --git a/openless-all/app/src-tauri/src/recorder.rs b/openless-all/app/src-tauri/src/recorder.rs index 869a8cbbd..f86f44c47 100644 --- a/openless-all/app/src-tauri/src/recorder.rs +++ b/openless-all/app/src-tauri/src/recorder.rs @@ -59,6 +59,10 @@ pub trait AudioConsumer: Send + Sync { /// Each delivery is a little-endian byte sequence of Int16 samples; /// the length is always a multiple of 2. fn consume_pcm_chunk(&self, pcm: &[u8]); + + /// Mark the end of the current VAD-gated segment without stopping the consumer. + /// Consumers that do not maintain streaming endpoint state can ignore this boundary. + fn finish_segment(&self) {} } pub(crate) struct AudioConsumerFanout { @@ -77,6 +81,12 @@ impl AudioConsumer for AudioConsumerFanout { consumer.consume_pcm_chunk(pcm); } } + + fn finish_segment(&self) { + for consumer in &self.consumers { + consumer.finish_segment(); + } + } } /// Compatibility bridge for legacy recorder call sites while cloud ASR @@ -337,6 +347,11 @@ struct WarmCaptureSink { max_bytes: usize, } +struct ActivationVadOutput { + pcm: Option>, + end_of_speech: bool, +} + struct ActivationVad { pre_roll: VecDeque, speech_active: bool, @@ -360,11 +375,14 @@ impl ActivationVad { self.hangover_chunks = 0; } - /// Return PCM for the activation recognizer, or `None` for sustained silence. + /// Return gated PCM and report a single speech-to-silence boundary. /// This deliberately uses one RMS pass and a bounded queue rather than a second model. - fn gate(&mut self, pcm: &[u8]) -> Option> { + fn gate(&mut self, pcm: &[u8]) -> ActivationVadOutput { if pcm.is_empty() { - return None; + return ActivationVadOutput { + pcm: None, + end_of_speech: false, + }; } WarmCaptureSink::append_bounded( &mut self.pre_roll, @@ -383,14 +401,24 @@ impl ActivationVad { } self.speech_active = true; self.hangover_chunks = ACTIVATION_VAD_HANGOVER_CHUNKS; - return Some(output); + return ActivationVadOutput { + pcm: Some(output), + end_of_speech: false, + }; } if self.speech_active && self.hangover_chunks > 0 { self.hangover_chunks -= 1; - return Some(pcm.to_vec()); + return ActivationVadOutput { + pcm: Some(pcm.to_vec()), + end_of_speech: false, + }; } + let end_of_speech = self.speech_active; self.speech_active = false; - None + ActivationVadOutput { + pcm: None, + end_of_speech, + } } } @@ -451,10 +479,13 @@ impl WarmCaptureSink { } fn deliver_activation(&self, consumer: &Arc, pcm: &[u8]) { - let Some(gated) = self.activation_vad.lock().gate(pcm) else { - return; - }; - consumer.consume_pcm_chunk(&gated); + let gated = self.activation_vad.lock().gate(pcm); + if let Some(pcm) = gated.pcm { + consumer.consume_pcm_chunk(&pcm); + } + if gated.end_of_speech { + consumer.finish_segment(); + } } fn append_bounded(buffer: &mut VecDeque, pcm: &[u8], max_bytes: usize) { @@ -1158,6 +1189,20 @@ impl WarmCaptureHub { ) } + pub(crate) fn arm_with_preroll( + self: &Arc, + press_id: u64, + device_name: Option, + ) -> Result { + self.arm_internal( + voice_capture_key(press_id), + device_name, + true, + VOICE_STAGED_CAPTURE_MAX_BYTES, + None, + ) + } + pub(crate) fn arm_without_preroll( self: &Arc, press_id: u64, @@ -2569,12 +2614,17 @@ mod tests { #[derive(Default)] struct RecordingConsumer { chunks: StdMutex>>, + segment_flushes: StdMutex, } impl AudioConsumer for RecordingConsumer { fn consume_pcm_chunk(&self, pcm: &[u8]) { self.chunks.lock().unwrap().push(pcm.to_vec()); } + + fn finish_segment(&self) { + *self.segment_flushes.lock().unwrap() += 1; + } } #[test] @@ -2678,6 +2728,30 @@ mod tests { assert_eq!(finish.chunks.lock().unwrap().as_slice(), &[speech.to_vec()]); } + #[test] + fn warm_promotion_replays_bounded_preroll_to_formal_consumer() { + let sink = WarmCaptureSink::new(STAGED_CAPTURE_MAX_BYTES); + let formal = Arc::new(RecordingConsumer::default()); + let prefix = vec![1, 2, 3, 4]; + let speech = vec![0x00, 0x40, 0x00, 0x40]; + sink.consume_pcm_chunk(&prefix); + sink.arm_with_limit(8, true, STAGED_CAPTURE_MAX_BYTES) + .unwrap(); + sink.consume_pcm_chunk(&speech); + + sink.promote( + 8, + Arc::clone(&formal) as Arc, + Arc::new(|_| {}), + None, + ) + .unwrap(); + + assert_eq!( + formal.chunks.lock().unwrap().as_slice(), + &[vec![1, 2, 3, 4, 0x00, 0x40, 0x00, 0x40]] + ); + } #[test] fn activation_vad_skips_silence_and_preserves_speech_onset() { let sink = WarmCaptureSink::new(STAGED_CAPTURE_MAX_BYTES); @@ -2698,11 +2772,34 @@ mod tests { fn activation_vad_hangover_is_bounded() { let mut vad = ActivationVad::default(); let speech = vec![0x00, 0x40, 0x00, 0x40]; - assert!(vad.gate(&speech).is_some()); + assert!(vad.gate(&speech).pcm.is_some()); for _ in 0..ACTIVATION_VAD_HANGOVER_CHUNKS { - assert!(vad.gate(&[0, 0, 0, 0]).is_some()); + let output = vad.gate(&[0, 0, 0, 0]); + assert!(output.pcm.is_some()); + assert!(!output.end_of_speech); + } + let output = vad.gate(&[0, 0, 0, 0]); + assert!(output.pcm.is_none()); + assert!(output.end_of_speech); + let output = vad.gate(&[0, 0, 0, 0]); + assert!(output.pcm.is_none()); + assert!(!output.end_of_speech); + } + + #[test] + fn activation_vad_flushes_after_hangover_without_flushing_sustained_silence() { + let sink = WarmCaptureSink::new(STAGED_CAPTURE_MAX_BYTES); + let activation = Arc::new(RecordingConsumer::default()); + sink.set_activation_consumer(Some(Arc::clone(&activation) as Arc)); + let speech = vec![0x00, 0x40, 0x00, 0x40]; + sink.consume_pcm_chunk(&speech); + for _ in 0..=ACTIVATION_VAD_HANGOVER_CHUNKS { + sink.consume_pcm_chunk(&[0, 0, 0, 0]); } - assert!(vad.gate(&[0, 0, 0, 0]).is_none()); + sink.consume_pcm_chunk(&[0, 0, 0, 0]); + + assert_eq!(*activation.segment_flushes.lock().unwrap(), 1); + assert!(activation.chunks.lock().unwrap().len() >= 1); } #[test] fn activation_fault_handler_uses_voice_generation_not_stream_generation() { From e4660dc24b95d4cefc1d8e0234636b109b716455 Mon Sep 17 00:00:00 2001 From: HKLHaoBin Date: Mon, 19 Oct 2026 13:22:01 +0800 Subject: [PATCH 04/12] chore: instrument voice activation diagnosis Add bounded privacy-safe P1-P6 runtime probes and matching Windows symbols for issue #1003 reproduction. No behavioral fix is included. Co-Authored-By: Claude Fable 5 --- .github/workflows/release-tauri.yml | 16 +++ .../src/asr/local/sherpa_provider.rs | 7 ++ .../src-tauri/src/asr/local/sherpa_runtime.rs | 103 ++++++++++++++++-- .../app/src-tauri/src/core_adapters.rs | 17 +++ openless-all/app/src-tauri/src/recorder.rs | 66 ++++++++++- 5 files changed, 200 insertions(+), 9 deletions(-) diff --git a/.github/workflows/release-tauri.yml b/.github/workflows/release-tauri.yml index 45761ea8b..d64cc0c46 100644 --- a/.github/workflows/release-tauri.yml +++ b/.github/workflows/release-tauri.yml @@ -303,6 +303,9 @@ jobs: env: TAURI_SIGNING_PRIVATE_KEY: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY }} TAURI_SIGNING_PRIVATE_KEY_PASSWORD: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY_PASSWORD }} + # [DEBUG-VOICE-1003] Matching Windows symbols for native crash diagnosis. + CARGO_PROFILE_RELEASE_DEBUG: line-tables-only + CARGO_PROFILE_RELEASE_STRIP: none run: | set +e # WiX/MSI only accepts a numeric prerelease identifier. Manual @@ -557,6 +560,17 @@ jobs: openless-all/app/src-tauri/target/release/bundle/latest-darwin-${{ matrix.updater-arch }}*.json if-no-files-found: error + # [DEBUG-VOICE-1003] Verify both matching files exist before publishing diagnostics. + - name: Verify Windows diagnostic symbols + if: matrix.platform == 'windows-latest' + shell: pwsh + run: | + $files = @('openless-all/app/src-tauri/target/release/openless.exe', 'openless-all/app/src-tauri/target/release/openless.pdb') + foreach ($file in $files) { + if (-not (Test-Path $file)) { throw 'Missing Windows diagnostic binary or symbols' } + Get-FileHash -Algorithm SHA256 $file + } + - name: Upload Windows artifacts if: matrix.platform == 'windows-latest' uses: actions/upload-artifact@v4 @@ -565,6 +579,8 @@ jobs: path: | openless-all/app/src-tauri/target/release/bundle/nsis/*.exe openless-all/app/src-tauri/target/release/bundle/msi/*.msi + openless-all/app/src-tauri/target/release/openless.exe + openless-all/app/src-tauri/target/release/openless.pdb if-no-files-found: error - name: Upload Windows updater artifacts diff --git a/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs b/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs index b319dd98c..e0db6c0e4 100644 --- a/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs +++ b/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs @@ -290,6 +290,8 @@ impl OnlineWorker { let join_handle = std::thread::Builder::new() .name(format!("openless-sherpa-online-{alias}")) .spawn(move || { + // [DEBUG-VOICE-1003] P2 / H1,H2,H4: worker control-message lifecycle. + crate::recorder::voice_diagnostic("P2", &["H1", "H2", "H4"], "worker", "started", serde_json::json!({})); let emit = |emission: &SherpaTokenEmission| { if emission.text.is_empty() || worker_cancelled.load(Ordering::SeqCst) { return; @@ -312,6 +314,7 @@ impl OnlineWorker { } } Ok(OnlineWorkerMessage::FlushSegment) => { + crate::recorder::voice_diagnostic("P2", &["H1", "H2", "H4"], "flush", "received", serde_json::json!({"cancelled": worker_cancelled.load(Ordering::SeqCst)})); if worker_cancelled.load(Ordering::SeqCst) { break Err(anyhow::anyhow!("sherpa-onnx streaming cancelled")); } @@ -320,6 +323,7 @@ impl OnlineWorker { } } Ok(OnlineWorkerMessage::Finish) => { + crate::recorder::voice_diagnostic("P2", &["H1", "H2", "H4"], "finish", "received", serde_json::json!({"cancelled": worker_cancelled.load(Ordering::SeqCst)})); if worker_cancelled.load(Ordering::SeqCst) { break Err(anyhow::anyhow!("sherpa-onnx streaming cancelled")); } @@ -331,6 +335,7 @@ impl OnlineWorker { } } }; + crate::recorder::voice_diagnostic("P2", &["H1", "H2", "H4"], "worker", "exiting", serde_json::json!({"ok": result.is_ok(), "cancelled": worker_cancelled.load(Ordering::SeqCst)})); let _ = result_tx.send(result); }) .expect("spawn sherpa online worker"); @@ -401,11 +406,13 @@ impl OnlineWorker { } fn cancel(self) { + crate::recorder::voice_diagnostic("P2", &["H1", "H2", "H4"], "cancel", "entered", serde_json::json!({})); self.cancelled.store(true, Ordering::SeqCst); let _ = self.tx.send(OnlineWorkerMessage::Cancel); if let Some(join_handle) = self.join_handle.lock().take() { let _ = join_handle.join(); } + crate::recorder::voice_diagnostic("P2", &["H1", "H2", "H4"], "cancel", "returned", serde_json::json!({})); } } diff --git a/openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs b/openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs index ca4fff387..6b65ff95a 100644 --- a/openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs +++ b/openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs @@ -56,6 +56,9 @@ struct LoadedOfflineModel { /// Online model load state. Each dictation session creates its own `OnlineStream` from the recognizer. #[derive(Clone)] struct LoadedOnlineModel { + // [DEBUG-VOICE-1003] Observe overlap without serializing native calls. + #[cfg(target_os = "windows")] + diagnostic_active: Arc, alias: String, #[cfg(target_os = "windows")] recognizer: Arc, @@ -333,6 +336,12 @@ impl SherpaOnnxRuntime { let _lifecycle = self.lifecycle.lock().await; let mut state = self.state.lock(); state.offline_loaded = None; + if let Some(loaded) = state.online_loaded.as_ref() { + #[cfg(target_os = "windows")] + crate::recorder::voice_diagnostic("P5", &["H4"], "cache_owner", "releasing", + serde_json::json!({"count": Arc::strong_count(&loaded.recognizer), + "active": loaded.diagnostic_active.load(Ordering::Relaxed)})); + } state.online_loaded = None; Ok(()) } @@ -351,6 +360,12 @@ impl SherpaOnnxRuntime { } let mut state = self.state.lock(); state.offline_loaded = None; + if let Some(loaded) = state.online_loaded.as_ref() { + #[cfg(target_os = "windows")] + crate::recorder::voice_diagnostic("P5", &["H4"], "cache_owner", "releasing", + serde_json::json!({"count": Arc::strong_count(&loaded.recognizer), + "active": loaded.diagnostic_active.load(Ordering::Relaxed)})); + } state.online_loaded = None; Ok(true) } @@ -481,6 +496,7 @@ async fn load_model(alias: &str, dir: &Path) -> Result { Some(openless_core::LocalAsrExecutionMode::Online) => { let recognizer = create_online_recognizer(&alias, &dir)?; Ok(LoadedModel::Online(LoadedOnlineModel { + diagnostic_active: Arc::new(std::sync::atomic::AtomicUsize::new(0)), alias, recognizer: Arc::new(recognizer), })) @@ -648,6 +664,8 @@ async fn transcribe_loaded_model( } pub struct SherpaOnlineSession { + #[cfg(target_os = "windows")] + diagnostic_active: Arc, alias: String, #[cfg(target_os = "windows")] recognizer: Arc, @@ -659,6 +677,43 @@ pub struct SherpaOnlineSession { pending_new_segment: bool, } +// [DEBUG-VOICE-1003] P3 / H2,H4: native entry/return and per-recognizer overlap. +#[cfg(target_os = "windows")] +struct VoiceNativeProbe<'a> { + active: &'a std::sync::atomic::AtomicUsize, + stage: &'static str, +} + +#[cfg(target_os = "windows")] +impl<'a> VoiceNativeProbe<'a> { + fn enter(active: &'a std::sync::atomic::AtomicUsize, stage: &'static str) -> Self { + let count = active.fetch_add(1, Ordering::Relaxed) + 1; + crate::recorder::voice_diagnostic("P3", &["H2", "H4"], "native", "entered", + serde_json::json!({"stage": stage, "count": count})); + Self { active, stage } + } +} + +#[cfg(target_os = "windows")] +impl Drop for VoiceNativeProbe<'_> { + fn drop(&mut self) { + let count = self.active.fetch_sub(1, Ordering::Relaxed) - 1; + crate::recorder::voice_diagnostic("P3", &["H2", "H4"], "native", "returned", + serde_json::json!({"stage": self.stage, "count": count})); + } +} + +// [DEBUG-VOICE-1003] P5 / H4: observe ownership before existing field destruction. +// No fields are moved, reordered or manually dropped here. +#[cfg(target_os = "windows")] +impl Drop for SherpaOnlineSession { + fn drop(&mut self) { + crate::recorder::voice_diagnostic("P5", &["H4"], "session_owner", "dropping", + serde_json::json!({"count": Arc::strong_count(&self.recognizer), + "active": self.diagnostic_active.load(Ordering::Relaxed)})); + } +} + impl SherpaOnlineSession { pub fn alias(&self) -> &str { &self.alias @@ -696,7 +751,10 @@ impl SherpaOnlineSession { where F: Fn(&SherpaTokenEmission), { - self.stream.accept_waveform(16_000, samples); + { + let _probe = VoiceNativeProbe::enter(&self.diagnostic_active, "accept"); + self.stream.accept_waveform(16_000, samples); + } self.drain_ready(on_token); Ok(()) } @@ -714,7 +772,10 @@ impl SherpaOnlineSession { where F: Fn(&SherpaTokenEmission), { - self.stream.input_finished(); + { + let _probe = VoiceNativeProbe::enter(&self.diagnostic_active, "input_finished"); + self.stream.input_finished(); + } self.drain_ready(on_token); self.capture_result(on_token, false); Ok(self.final_text()) @@ -733,7 +794,10 @@ impl SherpaOnlineSession { where F: Fn(&SherpaTokenEmission), { - self.stream.input_finished(); + { + let _probe = VoiceNativeProbe::enter(&self.diagnostic_active, "input_finished"); + self.stream.input_finished(); + } self.drain_ready_without_endpoint_reset(on_token); self.capture_result(on_token, false); if !self.last_partial_text.is_empty() { @@ -742,7 +806,10 @@ impl SherpaOnlineSession { self.last_partial_text.clear(); self.last_emitted_text.clear(); self.pending_new_segment = true; - self.recognizer.reset(&self.stream); + { + let _probe = VoiceNativeProbe::enter(&self.diagnostic_active, "reset"); + self.recognizer.reset(&self.stream); + } Ok(()) } @@ -760,7 +827,10 @@ impl SherpaOnlineSession { F: Fn(&SherpaTokenEmission), { while self.recognizer.is_ready(&self.stream) { - self.recognizer.decode(&self.stream); + { + let _probe = VoiceNativeProbe::enter(&self.diagnostic_active, "decode"); + self.recognizer.decode(&self.stream); + } self.capture_result(on_token, true); } } @@ -771,7 +841,10 @@ impl SherpaOnlineSession { F: Fn(&SherpaTokenEmission), { while self.recognizer.is_ready(&self.stream) { - self.recognizer.decode(&self.stream); + { + let _probe = VoiceNativeProbe::enter(&self.diagnostic_active, "decode"); + self.recognizer.decode(&self.stream); + } self.capture_result(on_token, false); } } @@ -781,7 +854,15 @@ impl SherpaOnlineSession { where F: Fn(&SherpaTokenEmission), { - let Some(result) = self.recognizer.get_result(&self.stream) else { + let result = { + let _probe = VoiceNativeProbe::enter(&self.diagnostic_active, "result"); + self.recognizer.get_result(&self.stream) + }; + crate::recorder::voice_diagnostic("P3", &["H2", "H4"], "native_result", "observed", + serde_json::json!({"present": result.is_some(), + "count": result.as_ref().map(|result| result.text.chars().count()).unwrap_or(0), + "stage": if allow_endpoint_reset { "streaming" } else { "flush" }})); + let Some(result) = result else { return; }; let text = result.text.trim().to_string(); @@ -795,7 +876,10 @@ impl SherpaOnlineSession { self.last_partial_text.clear(); self.last_emitted_text.clear(); self.pending_new_segment = true; - self.recognizer.reset(&self.stream); + { + let _probe = VoiceNativeProbe::enter(&self.diagnostic_active, "reset"); + self.recognizer.reset(&self.stream); + } } } @@ -849,7 +933,10 @@ fn append_segment(text: &mut String, segment: &str) { #[cfg(target_os = "windows")] fn create_online_session_from_loaded(loaded: LoadedOnlineModel) -> Result { let stream = loaded.recognizer.create_stream(); + crate::recorder::voice_diagnostic("P5", &["H4"], "session_owner", "created", + serde_json::json!({"count": Arc::strong_count(&loaded.recognizer)})); Ok(SherpaOnlineSession { + diagnostic_active: loaded.diagnostic_active, alias: loaded.alias, recognizer: loaded.recognizer, stream, diff --git a/openless-all/app/src-tauri/src/core_adapters.rs b/openless-all/app/src-tauri/src/core_adapters.rs index a6fcaca05..5ddc026f2 100644 --- a/openless-all/app/src-tauri/src/core_adapters.rs +++ b/openless-all/app/src-tauri/src/core_adapters.rs @@ -1603,6 +1603,11 @@ impl VoiceActivationEmitter { } let text = state.text.clone(); let matched = state.recognizer.observe(&text, self.command); + // [DEBUG-VOICE-1003] P4 / H2,H3: no transcript content or phrases. + crate::recorder::voice_diagnostic("P4", &["H2", "H3"], "command_match", "observed", + serde_json::json!({"matched": matched.is_some(), "bound": bound_session_id.is_some(), + "mode": if self.command == VoiceActivationCommand::Wake { "wake" } else { "finish" }, + "count": text.chars().count(), "new_segment": is_new_segment, "delta": is_delta})); if self.command == VoiceActivationCommand::Finish && bound_session_id.is_none() { state.pending_finish = true; state.pending_finish_match |= matched.is_some(); @@ -1628,7 +1633,13 @@ impl VoiceActivationEmitter { }; let _ = self.sink.publish(signal.clone()); if let Some(backend) = self.backend.lock().as_ref().and_then(std::sync::Weak::upgrade) { + // [DEBUG-VOICE-1003] P6 / H3: dispatch guard facts, without identifiers. + crate::recorder::voice_diagnostic("P6", &["H3"], "command_dispatch", "entered", + serde_json::json!({"current": backend.voice_activation_generation_is_current(self.generation), + "present": backend.snapshot().dictation.session_id.is_some(), + "mode": if self.command == VoiceActivationCommand::Wake { "wake" } else { "finish" }})); backend.handle_voice_activation_signal(signal); + crate::recorder::voice_diagnostic("P6", &["H3"], "command_dispatch", "returned", serde_json::json!({})); } } @@ -1662,7 +1673,13 @@ impl VoiceActivationEmitter { }; let _ = self.sink.publish(signal.clone()); if let Some(backend) = self.backend.lock().as_ref().and_then(std::sync::Weak::upgrade) { + // [DEBUG-VOICE-1003] P6 / H3: dispatch guard facts, without identifiers. + crate::recorder::voice_diagnostic("P6", &["H3"], "command_dispatch", "entered", + serde_json::json!({"current": backend.voice_activation_generation_is_current(self.generation), + "present": backend.snapshot().dictation.session_id.is_some(), + "mode": if self.command == VoiceActivationCommand::Wake { "wake" } else { "finish" }})); backend.handle_voice_activation_signal(signal); + crate::recorder::voice_diagnostic("P6", &["H3"], "command_dispatch", "returned", serde_json::json!({})); } } else if !text.is_empty() { self.observe(text, false, false); diff --git a/openless-all/app/src-tauri/src/recorder.rs b/openless-all/app/src-tauri/src/recorder.rs index f86f44c47..b3f2bbc56 100644 --- a/openless-all/app/src-tauri/src/recorder.rs +++ b/openless-all/app/src-tauri/src/recorder.rs @@ -50,6 +50,55 @@ const ACTIVATION_VAD_PRE_ROLL_BYTES: usize = TARGET_SAMPLE_RATE as usize * 2 / 5 const ACTIVATION_VAD_HANGOVER_CHUNKS: usize = 8; const VOICE_CAPTURE_KEY_PREFIX: u64 = 1 << 63; +// [DEBUG-VOICE-1003] Temporary P1-P6 diagnostics; remove only after user confirmation. +// Audio/model work never waits for disk I/O. Only the dedicated NDJSON writer opens a file. +pub(crate) fn voice_diagnostic( + probe_id: &'static str, + hypothesis_ids: &'static [&'static str], + event: &'static str, + outcome: &'static str, + safe: serde_json::Value, +) { + #[cfg(all(target_os = "windows", not(test)))] + { + use std::io::Write; + static WRITER: OnceLock>> = OnceLock::new(); + static SEQUENCE: AtomicU64 = AtomicU64::new(0); + static DROPPED: AtomicU64 = AtomicU64::new(0); + static RUN: OnceLock = OnceLock::new(); + let run = RUN.get_or_init(|| format!("r{:x}", std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH).unwrap_or_default().as_nanos())); + let writer = WRITER.get_or_init(|| { + let (tx, rx) = sync_channel::(4096); + std::thread::Builder::new().name("voice-1003-diagnostics".into()).spawn(move || { + let path = std::env::temp_dir().join("openless-1003-3c02ffc1.ndjson"); + let Ok(mut file) = std::fs::OpenOptions::new().create(true).append(true).open(path) else { + return; + }; + for record in rx { + let Ok(mut line) = serde_json::to_vec(&record) else { continue; }; + line.push(b'\n'); + if file.write_all(&line).is_err() { break; } + } + }).ok().map(|_| tx) + }); + if let Some(writer) = writer { + let record = serde_json::json!({ + "schema": "diagnosing-bugs.v1", "session_id": "voice-1003-3c02ffc1", + "run_id": run, "probe_id": probe_id, "hypothesis_ids": hypothesis_ids, + "event": event, "outcome": outcome, "safe": safe, + "seq": SEQUENCE.fetch_add(1, Ordering::Relaxed), + "dropped": DROPPED.load(Ordering::Relaxed), + "ts": std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default().as_millis().to_string(), + }); + if writer.try_send(record).is_err() { DROPPED.fetch_add(1, Ordering::Relaxed); } + } + } + #[cfg(any(not(target_os = "windows"), test))] + let _ = (probe_id, hypothesis_ids, event, outcome, safe); +} + fn voice_capture_key(generation: u64) -> u64 { generation | VOICE_CAPTURE_KEY_PREFIX } @@ -479,7 +528,22 @@ impl WarmCaptureSink { } fn deliver_activation(&self, consumer: &Arc, pcm: &[u8]) { - let gated = self.activation_vad.lock().gate(pcm); + let (gated, was_active, active, hangover) = { + let mut vad = self.activation_vad.lock(); + let was_active = vad.speech_active; + let gated = vad.gate(pcm); + (gated, was_active, vad.speech_active, vad.hangover_chunks) + }; + // [DEBUG-VOICE-1003] P1 / H1: transitions plus bounded coverage of sustained noise/silence. + static BLOCKS: AtomicU64 = AtomicU64::new(0); + let blocks = BLOCKS.fetch_add(1, Ordering::Relaxed); + if was_active != active || gated.end_of_speech || blocks % 64 == 0 { + voice_diagnostic("P1", &["H1"], "vad_gate", "observed", serde_json::json!({ + "bytes": pcm.len(), "active": active, "hangover": hangover, + "forwarded": gated.pcm.is_some(), "boundary": gated.end_of_speech, + "count": blocks, + })); + } if let Some(pcm) = gated.pcm { consumer.consume_pcm_chunk(&pcm); } From 73e39abcd8b1d54377bd3e0b508c9d16a42cb65c Mon Sep 17 00:00:00 2001 From: HKLHaoBin Date: Mon, 19 Oct 2026 13:41:26 +0800 Subject: [PATCH 05/12] fix: include JSON dependency in recorder test fixture Co-Authored-By: Claude Fable 5 --- openless-all/app/scripts/test-recorder-module.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openless-all/app/scripts/test-recorder-module.py b/openless-all/app/scripts/test-recorder-module.py index 8e1e828de..2fac17a0e 100644 --- a/openless-all/app/scripts/test-recorder-module.py +++ b/openless-all/app/scripts/test-recorder-module.py @@ -22,6 +22,7 @@ cpal = "=0.15.3" parking_lot = "0.12" serde = { version = "1", features = ["derive"] } +serde_json = "1" thiserror = "1" log = "0.4" """) From d2a7fbe92ea522d03ab6efedf531574e6ad9d0f8 Mon Sep 17 00:00:00 2001 From: HKLHaoBin Date: Mon, 19 Oct 2026 16:17:23 +0800 Subject: [PATCH 06/12] fix: supply tail context before activation segment EOF Experiment A1-tail-context / C1-nonterminal-padding: add bounded zero-sample context before nonterminal input_finished, retaining diagnostics for verification. Co-Authored-By: Claude Fable 5 --- .../src-tauri/src/asr/local/sherpa_runtime.rs | 54 +++++++++++++++++-- openless-all/app/src-tauri/src/recorder.rs | 1 + 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs b/openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs index 6b65ff95a..f2189b43f 100644 --- a/openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs +++ b/openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs @@ -794,10 +794,16 @@ impl SherpaOnlineSession { where F: Fn(&SherpaTokenEmission), { - { - let _probe = VoiceNativeProbe::enter(&self.diagnostic_active, "input_finished"); - self.stream.input_finished(); - } + finish_online_segment_input( + |tail| { + let _probe = VoiceNativeProbe::enter(&self.diagnostic_active, "tail_padding"); + self.stream.accept_waveform(16_000, tail); + }, + || { + let _probe = VoiceNativeProbe::enter(&self.diagnostic_active, "input_finished"); + self.stream.input_finished(); + }, + ); self.drain_ready_without_endpoint_reset(on_token); self.capture_result(on_token, false); if !self.last_partial_text.is_empty() { @@ -919,6 +925,18 @@ impl SherpaOnlineSession { } } +// Sherpa 1.13.2's online-decode-files.py supplies 0.66 s of zero samples before +// input_finished(): marking EOF alone does not provide the model's trailing context. +// This is bounded model input, not a wall-clock wait or continuous idle inference. +fn finish_online_segment_input( + accept_tail: impl FnOnce(&[f32]), + input_finished: impl FnOnce(), +) { + const TAIL: [f32; 16_000 * 66 / 100] = [0.0; 16_000 * 66 / 100]; + accept_tail(&TAIL); + input_finished(); +} + fn append_segment(text: &mut String, segment: &str) { let segment = segment.trim(); if segment.is_empty() { @@ -1173,6 +1191,34 @@ mod tests { assert!(pcm_s16le_to_f32(&[0]).is_err()); } + #[test] + fn segment_input_supplies_bounded_silent_context_before_eof() { + let events = std::cell::RefCell::new(Vec::new()); + finish_online_segment_input( + |tail| { + assert_eq!(tail.len(), 10_560); + assert!(tail.iter().all(|sample| *sample == 0.0)); + events.borrow_mut().push(("tail", tail.len())); + }, + || events.borrow_mut().push(("finished", 0)), + ); + assert_eq!(*events.borrow(), vec![("tail", 10_560), ("finished", 0)]); + } + + #[test] + fn repeated_segment_boundaries_do_not_accumulate_tail_context() { + let samples = std::cell::Cell::new(0); + let boundaries = std::cell::Cell::new(0); + for _ in 0..2 { + finish_online_segment_input( + |tail| samples.set(samples.get() + tail.len()), + || boundaries.set(boundaries.get() + 1), + ); + } + assert_eq!(samples.get(), 21_120); + assert_eq!(boundaries.get(), 2); + } + #[test] fn append_segment_inserts_space_between_segments() { let mut text = String::new(); diff --git a/openless-all/app/src-tauri/src/recorder.rs b/openless-all/app/src-tauri/src/recorder.rs index b3f2bbc56..94df9f5ff 100644 --- a/openless-all/app/src-tauri/src/recorder.rs +++ b/openless-all/app/src-tauri/src/recorder.rs @@ -85,6 +85,7 @@ pub(crate) fn voice_diagnostic( if let Some(writer) = writer { let record = serde_json::json!({ "schema": "diagnosing-bugs.v1", "session_id": "voice-1003-3c02ffc1", + "attempt_id": "A1-tail-context", "change_id": "C1-nonterminal-padding", "run_id": run, "probe_id": probe_id, "hypothesis_ids": hypothesis_ids, "event": event, "outcome": outcome, "safe": safe, "seq": SEQUENCE.fetch_add(1, Ordering::Relaxed), From e776f79824ada78baa8429ac2548c57a2167f489 Mon Sep 17 00:00:00 2001 From: HKLHaoBin Date: Mon, 19 Oct 2026 21:48:32 +0800 Subject: [PATCH 07/12] fix: supply tail context before terminal streaming EOF Reuse bounded Sherpa tail input when formal transcription finishes so final tokens can decode before transcript cleanup. Retain diagnostic probes and identify the A2 terminal-padding experiment. Co-Authored-By: Claude Fable 5 --- .../src-tauri/src/asr/local/sherpa_runtime.rs | 22 ++++++++++++------- openless-all/app/src-tauri/src/recorder.rs | 2 +- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs b/openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs index f2189b43f..dc5d31346 100644 --- a/openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs +++ b/openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs @@ -677,7 +677,7 @@ pub struct SherpaOnlineSession { pending_new_segment: bool, } -// [DEBUG-VOICE-1003] P3 / H2,H4: native entry/return and per-recognizer overlap. +// [DEBUG-VOICE-1003] P3 / H2,H4,H6: native entry/return and per-recognizer overlap. #[cfg(target_os = "windows")] struct VoiceNativeProbe<'a> { active: &'a std::sync::atomic::AtomicUsize, @@ -688,7 +688,7 @@ struct VoiceNativeProbe<'a> { impl<'a> VoiceNativeProbe<'a> { fn enter(active: &'a std::sync::atomic::AtomicUsize, stage: &'static str) -> Self { let count = active.fetch_add(1, Ordering::Relaxed) + 1; - crate::recorder::voice_diagnostic("P3", &["H2", "H4"], "native", "entered", + crate::recorder::voice_diagnostic("P3", &["H2", "H4", "H6"], "native", "entered", serde_json::json!({"stage": stage, "count": count})); Self { active, stage } } @@ -698,7 +698,7 @@ impl<'a> VoiceNativeProbe<'a> { impl Drop for VoiceNativeProbe<'_> { fn drop(&mut self) { let count = self.active.fetch_sub(1, Ordering::Relaxed) - 1; - crate::recorder::voice_diagnostic("P3", &["H2", "H4"], "native", "returned", + crate::recorder::voice_diagnostic("P3", &["H2", "H4", "H6"], "native", "returned", serde_json::json!({"stage": self.stage, "count": count})); } } @@ -772,10 +772,16 @@ impl SherpaOnlineSession { where F: Fn(&SherpaTokenEmission), { - { - let _probe = VoiceNativeProbe::enter(&self.diagnostic_active, "input_finished"); - self.stream.input_finished(); - } + finish_online_segment_input( + |tail| { + let _probe = VoiceNativeProbe::enter(&self.diagnostic_active, "tail_padding"); + self.stream.accept_waveform(16_000, tail); + }, + || { + let _probe = VoiceNativeProbe::enter(&self.diagnostic_active, "input_finished"); + self.stream.input_finished(); + }, + ); self.drain_ready(on_token); self.capture_result(on_token, false); Ok(self.final_text()) @@ -864,7 +870,7 @@ impl SherpaOnlineSession { let _probe = VoiceNativeProbe::enter(&self.diagnostic_active, "result"); self.recognizer.get_result(&self.stream) }; - crate::recorder::voice_diagnostic("P3", &["H2", "H4"], "native_result", "observed", + crate::recorder::voice_diagnostic("P3", &["H2", "H4", "H6"], "native_result", "observed", serde_json::json!({"present": result.is_some(), "count": result.as_ref().map(|result| result.text.chars().count()).unwrap_or(0), "stage": if allow_endpoint_reset { "streaming" } else { "flush" }})); diff --git a/openless-all/app/src-tauri/src/recorder.rs b/openless-all/app/src-tauri/src/recorder.rs index 94df9f5ff..bf6d3991a 100644 --- a/openless-all/app/src-tauri/src/recorder.rs +++ b/openless-all/app/src-tauri/src/recorder.rs @@ -85,7 +85,7 @@ pub(crate) fn voice_diagnostic( if let Some(writer) = writer { let record = serde_json::json!({ "schema": "diagnosing-bugs.v1", "session_id": "voice-1003-3c02ffc1", - "attempt_id": "A1-tail-context", "change_id": "C1-nonterminal-padding", + "attempt_id": "A2-terminal-tail", "change_id": "C2-terminal-padding", "run_id": run, "probe_id": probe_id, "hypothesis_ids": hypothesis_ids, "event": event, "outcome": outcome, "safe": safe, "seq": SEQUENCE.fetch_add(1, Ordering::Relaxed), From 9c75062ce835dbf29dccd84053f44e4af4c721af Mon Sep 17 00:00:00 2001 From: HKLHaoBin Date: Wed, 21 Oct 2026 02:00:59 +0800 Subject: [PATCH 08/12] fix(core): complete imported final insertion test dependencies Set the optional voice activation runtime to None in the upstream polished-final-insertion fixture so the merged Core tests compile on all platforms. Co-Authored-By: Claude Opus 5 --- openless-all/app/crates/openless-core/src/api.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/openless-all/app/crates/openless-core/src/api.rs b/openless-all/app/crates/openless-core/src/api.rs index 70ffdf429..4f4c7354e 100644 --- a/openless-all/app/crates/openless-core/src/api.rs +++ b/openless-all/app/crates/openless-core/src/api.rs @@ -15735,6 +15735,7 @@ mod tests { credential_store: Arc::new(crate::credentials::InMemoryCredentialStore::default()), services: crate::domains::BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, marketplace_config: None, selection_runtime: None, selection_polisher: None, From 5372769af3fff3cdc4aa9959129616a242a139e6 Mon Sep 17 00:00:00 2001 From: HKLHaoBin Date: Wed, 21 Oct 2026 02:19:31 +0800 Subject: [PATCH 09/12] fix(ci): fetch pinned installer smoke tooling without submodules Read the smoke script from the workflow commit through the authenticated GitHub contents API. Preserve byte contents and installer checks while avoiding checkout auth cleanup traversing optional vendor gitlinks. Co-Authored-By: Claude Opus 5 --- .github/workflows/release-tauri.yml | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release-tauri.yml b/.github/workflows/release-tauri.yml index b4a65a75f..feecb025a 100644 --- a/.github/workflows/release-tauri.yml +++ b/.github/workflows/release-tauri.yml @@ -452,15 +452,23 @@ jobs: # Tests are tooling, not application payload. A targeted repair checks out # immutable app source above, but uses the reviewed workflow commit's smoke # script so fixes to test semantics do not require moving a release tag. - - name: Checkout installer smoke tooling + - name: Fetch installer smoke tooling if: matrix.platform == 'windows-latest' - uses: actions/checkout@v4 - with: - ref: ${{ github.workflow_sha }} - path: release-tooling - sparse-checkout: openless-all/app/scripts/windows-ime-install-smoke.ps1 - sparse-checkout-cone-mode: false - persist-credentials: false + shell: pwsh + env: + GH_TOKEN: ${{ github.token }} + TOOLING_SHA: ${{ github.workflow_sha }} + run: | + # Fetch only the pinned script: a sparse checkout still visits unrelated + # gitlinks during auth cleanup, including optional vendor runtimes. + $scriptPath = 'openless-all/app/scripts/windows-ime-install-smoke.ps1' + $encoded = gh api "repos/$env:GITHUB_REPOSITORY/contents/$scriptPath" --method GET -f "ref=$env:TOOLING_SHA" --jq '.content' + if ($LASTEXITCODE -ne 0) { throw 'Failed to fetch pinned installer smoke tooling.' } + $bytes = [Convert]::FromBase64String(($encoded -join '')) + if ($bytes.Length -eq 0) { throw 'Pinned installer smoke tooling is empty.' } + $destination = Join-Path $env:GITHUB_WORKSPACE "release-tooling/$scriptPath" + New-Item -ItemType Directory -Force -Path (Split-Path $destination -Parent) | Out-Null + [System.IO.File]::WriteAllBytes($destination, $bytes) - name: Verify Windows installers register TSF IME if: matrix.platform == 'windows-latest' From 433818859bba5d814b133eb82a7f771b4063b730 Mon Sep 17 00:00:00 2001 From: HKLHaoBin Date: Wed, 21 Oct 2026 02:39:15 +0800 Subject: [PATCH 10/12] test(ci): cover pinned installer smoke tooling fetch Keep immutable workflow revision, authentication, failure propagation, and byte-preserving script retrieval assertions aligned with the workflow. Retain all installer registration checks. Co-Authored-By: Claude Opus 5 --- .../windows-msi-release-contract.test.mjs | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/openless-all/app/scripts/windows-msi-release-contract.test.mjs b/openless-all/app/scripts/windows-msi-release-contract.test.mjs index d3fec672e..b926339d8 100644 --- a/openless-all/app/scripts/windows-msi-release-contract.test.mjs +++ b/openless-all/app/scripts/windows-msi-release-contract.test.mjs @@ -85,10 +85,22 @@ const publisher = steps.find((s) => s.uses?.startsWith('softprops/action-gh-rele assert.equal(publisher.with.tag_name, '${{ env.OPENLESS_RELEASE_TAG }}'); assert.equal(publisher.with.draft, true); assert.ok(steps.find((s) => s.name === 'Confirm repair target remains draft')); -const smokeTooling = steps.find((s) => s.name === 'Checkout installer smoke tooling'); -assert.equal(smokeTooling.with.ref, '${{ github.workflow_sha }}'); -assert.equal(smokeTooling.with['sparse-checkout'], 'openless-all/app/scripts/windows-ime-install-smoke.ps1'); -assert.equal(smokeTooling.with['persist-credentials'], false); +const smokeTooling = steps.find((s) => s.name === 'Fetch installer smoke tooling'); +assert.ok(smokeTooling, 'Windows smoke tooling must be fetched without checking out unrelated gitlinks'); +assert.equal(smokeTooling.if, "matrix.platform == 'windows-latest'"); +assert.equal(smokeTooling.shell, 'pwsh'); +assert.equal(smokeTooling.env.TOOLING_SHA, '${{ github.workflow_sha }}'); +assert.equal(smokeTooling.env.GH_TOKEN, '${{ github.token }}'); +assert.equal(smokeTooling.uses, undefined); +assert.match(smokeTooling.run, /\$scriptPath = 'openless-all\/app\/scripts\/windows-ime-install-smoke\.ps1'/); +assert.match(smokeTooling.run, /gh api "repos\/\$env:GITHUB_REPOSITORY\/contents\/\$scriptPath"/); +assert.match(smokeTooling.run, /--method GET -f "ref=\$env:TOOLING_SHA" --jq '\.content'/); +assert.match(smokeTooling.run, /if \(\$LASTEXITCODE -ne 0\) \{ throw/); +assert.match(smokeTooling.run, /FromBase64String\(\(\$encoded -join ''\)\)/); +assert.match(smokeTooling.run, /if \(\$bytes\.Length -eq 0\) \{ throw/); +assert.match(smokeTooling.run, /Join-Path \$env:GITHUB_WORKSPACE "release-tooling\/\$scriptPath"/); +assert.match(smokeTooling.run, /WriteAllBytes\(\$destination, \$bytes\)/); +assert.doesNotMatch(smokeTooling.run, /git checkout|git submodule|continue-on-error/); const smoke = steps.find((s) => s.name === 'Verify Windows installers register TSF IME'); assert.match(smoke.run, /-File \$smokeScript/); assert.match( From 4193770a062cfa87c8ad172eea4d5f9a176c9f85 Mon Sep 17 00:00:00 2001 From: Chris233 Date: Wed, 21 Oct 2026 11:14:50 +0800 Subject: [PATCH 11/12] fix(voice-activation): preserve sessions and command boundaries --- .../app/crates/openless-core/src/api.rs | 551 +++++++++++++----- .../openless-core/src/dictation_context.rs | 3 + .../openless-core/src/dictation_engine.rs | 195 ++++++- .../app/crates/openless-core/src/lib.rs | 7 +- .../openless-core/src/silence_auto_stop.rs | 38 ++ .../openless-core/src/voice_activation.rs | 60 +- .../app/e2e/settings-regressions.spec.ts | 58 ++ .../src-tauri/src/asr/local/sherpa_runtime.rs | 86 ++- openless-all/app/src-tauri/src/recorder.rs | 73 ++- openless-all/app/src/i18n/de.ts | 2 + openless-all/app/src/i18n/en.ts | 2 + openless-all/app/src/i18n/es.ts | 2 + openless-all/app/src/i18n/fr.ts | 4 +- openless-all/app/src/i18n/ja.ts | 2 + openless-all/app/src/i18n/ko.ts | 2 + openless-all/app/src/i18n/zh-CN.ts | 4 +- openless-all/app/src/i18n/zh-TW.ts | 1 + .../pages/settings/RecordingInputSection.tsx | 2 +- .../app/src/pages/settings/navigation.test.ts | 32 +- .../app/src/pages/settings/navigation.ts | 13 +- openless-all/app/src/pages/settings/tabs.tsx | 43 +- 21 files changed, 937 insertions(+), 243 deletions(-) diff --git a/openless-all/app/crates/openless-core/src/api.rs b/openless-all/app/crates/openless-core/src/api.rs index 47dddecdd..670275f71 100644 --- a/openless-all/app/crates/openless-core/src/api.rs +++ b/openless-all/app/crates/openless-core/src/api.rs @@ -2314,7 +2314,6 @@ struct BackendEngineProgress { state: Arc>, phase_changed: Arc, text_insertions: Arc>>, - voice_activation_session: Arc>>, raw_transcripts: Mutex>, } @@ -2440,20 +2439,6 @@ impl EngineProgressSink for BackendEngineProgress { } } EngineProgress::TranscriptDelta(mut delta) => { - let activation_phrases = self - .voice_activation_session - .lock() - .expect("voice activation session lock poisoned") - .as_ref() - .filter(|session| session.session_id == session_id) - .map(|session| { - ( - session.wake_phrase.clone(), - session - .finish_enabled - .then(|| session.finish_phrase.clone()), - ) - }); let mut state = self.state.write().expect("backend state lock poisoned"); ensure_active_session(&state, session_id)?; if !matches!( @@ -2467,7 +2452,17 @@ impl EngineProgressSink for BackendEngineProgress { "transcript delta arrived after transcription completed", )); } - if let Some((wake_phrase, finish_phrase)) = activation_phrases { + // Traditional final text was cleaned before correction and polishing. + let activation_phrases = state + .dictation_context + .as_ref() + .filter(|context| { + !delta.is_final + || context.pipeline_mode + == crate::shared_types::PipelineMode::Multimodal + }) + .and_then(|context| context.voice_activation.clone()); + if let Some(commands) = activation_phrases { let mut raw_transcripts = self .raw_transcripts .lock() @@ -2485,19 +2480,7 @@ impl EngineProgressSink for BackendEngineProgress { transcript }); raw_transcript.apply(&delta)?; - let mut cleaned = raw_transcript.text().to_string(); - if let Some(stripped) = - crate::voice_activation::strip_phrase_prefix(&cleaned, &wake_phrase) - { - cleaned = stripped; - } - if let Some(finish_phrase) = finish_phrase { - if let Some(stripped) = - crate::voice_activation::strip_phrase_suffix(&cleaned, &finish_phrase) - { - cleaned = stripped; - } - } + let cleaned = commands.clean(raw_transcript.text()); delta = crate::types::TranscriptDelta { text: cleaned, offset: 0, @@ -2894,9 +2877,7 @@ fn voice_activation_failure_phase( code: BackendErrorCode, ) -> crate::voice_activation::VoiceActivationPhase { match code { - BackendErrorCode::Unsupported => { - crate::voice_activation::VoiceActivationPhase::Unavailable - } + BackendErrorCode::Unsupported => crate::voice_activation::VoiceActivationPhase::Unavailable, BackendErrorCode::PermissionDenied => { crate::voice_activation::VoiceActivationPhase::PermissionDenied } @@ -3600,9 +3581,7 @@ impl OpenLessBackend { || self .events .latest_voice_activation_state() - .map(|state| { - state.phase != crate::voice_activation::VoiceActivationPhase::Disabled - }) + .map(|state| state.phase != crate::voice_activation::VoiceActivationPhase::Disabled) .unwrap_or(false); if let Some(handle) = handle { handle @@ -3654,9 +3633,7 @@ impl OpenLessBackend { || self .events .latest_voice_activation_state() - .map(|state| { - state.phase != crate::voice_activation::VoiceActivationPhase::Disabled - }) + .map(|state| state.phase != crate::voice_activation::VoiceActivationPhase::Disabled) .unwrap_or(false); if let Some(handle) = handle { handle @@ -3798,26 +3775,6 @@ impl OpenLessBackend { let _ = backend.cancel_dictation(Some(session_id)).await; return; } - let preferences = backend.get_preferences(); - let timeout = crate::voice_activation::VoiceActivationConfig { - timeout_enabled: preferences.voice_activation_timeout_enabled, - timeout_seconds: preferences.voice_activation_timeout_seconds, - ..crate::voice_activation::VoiceActivationConfig::default() - } - .normalized() - .ok() - .and_then(|config| config.timeout()); - let voice_session = crate::voice_activation::VoiceActivationSession { - generation: crate::voice_activation::VoiceActivationGeneration( - signal_generation, - ), - session_id, - started_at: std::time::Instant::now(), - timeout, - wake_phrase: preferences.voice_activation_wake_phrase.clone(), - finish_enabled: preferences.voice_activation_finish_enabled, - finish_phrase: preferences.voice_activation_finish_phrase, - }; let committed = { let _configure_gate = backend.voice_activation_configure_gate.lock().await; @@ -3834,11 +3791,6 @@ impl OpenLessBackend { { false } else { - *backend - .voice_activation_session - .lock() - .expect("voice activation session lock poisoned") = - Some(voice_session); handle.bind_dictation_session(session_id); true } @@ -3858,19 +3810,6 @@ impl OpenLessBackend { }, ), ); - if let Some(timeout) = timeout { - let timeout_backend = Arc::clone(&backend); - backend.deps.task_spawner.spawn(Box::pin(async move { - tokio::time::sleep(timeout).await; - timeout_backend - .finish_voice_activation( - signal_generation, - session_id, - crate::voice_activation::VoiceActivationStopReason::Timeout, - ) - .await; - })); - } } Err(error) => { log::warn!("voice dictation start failed: {error}"); @@ -3947,11 +3886,31 @@ impl OpenLessBackend { "voice activation generation is stale", )); } + let preferences = self.get_preferences(); + let activation_config = match (crate::voice_activation::VoiceActivationConfig { + wake_phrase: preferences.voice_activation_wake_phrase.clone(), + finish_enabled: preferences.voice_activation_finish_enabled, + finish_phrase: preferences.voice_activation_finish_phrase.clone(), + timeout_enabled: preferences.voice_activation_timeout_enabled, + timeout_seconds: preferences.voice_activation_timeout_seconds, + ..crate::voice_activation::VoiceActivationConfig::default() + }) + .normalized() + { + Ok(config) => config, + Err(error) => { + let _ = capture.discard().await; + return Err(BackendError::new( + BackendErrorCode::InvalidArgument, + error.to_string(), + )); + } + }; self.pending_audio_captures .lock() .expect("pending audio capture lock poisoned") .insert(PendingAudioCaptureKey::VoiceActivation(generation), capture); - let reservation = match self.reserve_dictation_session( + let mut reservation = match self.reserve_dictation_session( true, DictationOutputTarget::ForegroundApp, Some(PendingAudioCaptureKey::VoiceActivation(generation)), @@ -3966,25 +3925,15 @@ impl OpenLessBackend { } }; let session_id = reservation.session_id; - let preferences = self.get_preferences(); - let activation_config = crate::voice_activation::VoiceActivationConfig { - wake_phrase: preferences.voice_activation_wake_phrase.clone(), - finish_enabled: preferences.voice_activation_finish_enabled, - finish_phrase: preferences.voice_activation_finish_phrase.clone(), - timeout_enabled: preferences.voice_activation_timeout_enabled, - timeout_seconds: preferences.voice_activation_timeout_seconds, - ..crate::voice_activation::VoiceActivationConfig::default() - } - .normalized() - .map_err(|error| BackendError::new(BackendErrorCode::InvalidArgument, error.to_string()))?; { let _configure_gate = self.voice_activation_configure_gate.lock().await; if self.voice_activation_generation.load(Ordering::Acquire) != generation { - self.voice_sessions.release(session_id); - self.discard_pending_audio_capture(PendingAudioCaptureKey::VoiceActivation( - generation, - )) - .await?; + if let Some(capture) = reservation.pending_capture.take() { + if let Err(error) = capture.discard().await { + log::warn!("discard stale voice capture failed: {error}"); + } + } + self.reset_dictation_session(session_id); return Err(BackendError::new( BackendErrorCode::Cancelled, "voice activation generation became stale while starting dictation", @@ -3997,13 +3946,14 @@ impl OpenLessBackend { crate::voice_activation::VoiceActivationSession { generation: crate::voice_activation::VoiceActivationGeneration(generation), session_id, - started_at: std::time::Instant::now(), timeout: activation_config.timeout(), - wake_phrase: activation_config.wake_phrase.clone(), - finish_enabled: activation_config.finish_enabled, - finish_phrase: activation_config.finish_phrase.clone(), - }, - ); + commands: crate::voice_activation::VoiceActivationTextPolicy { + wake_phrase: activation_config.wake_phrase.clone(), + finish_phrase: activation_config + .finish_enabled + .then(|| activation_config.finish_phrase.clone()), + }, + }); } match self .start_reserved_dictation(reservation, DictationStartOptions::default()) @@ -4985,7 +4935,6 @@ impl OpenLessBackend { state: Arc::clone(&self.state), phase_changed: Arc::clone(&self.phase_changed), text_insertions: Arc::clone(&self.text_insertions), - voice_activation_session: Arc::clone(&self.voice_activation_session), raw_transcripts: Mutex::new(HashMap::new()), }) } @@ -7543,6 +7492,29 @@ impl OpenLessBackend { return Err(error); } }; + let activation = self + .voice_activation_session + .lock() + .expect("voice activation session lock poisoned") + .as_ref() + .filter(|session| session.session_id == session_id) + .cloned(); + let context = if let Some(activation) = activation { + let mut context = (*context).clone(); + context.voice_activation = Some(activation.commands); + if let Some(timeout) = activation.timeout { + let timeout_ms = timeout.as_millis() as u64; + context.recording.silence_after_ms = Some( + context + .recording + .silence_after_ms + .map_or(timeout_ms, |general| general.min(timeout_ms)), + ); + } + Arc::new(context) + } else { + context + }; let context = { let mut state = self.state.write().expect("backend state lock poisoned"); if let Err(error) = ensure_running(&state) { @@ -7569,10 +7541,17 @@ impl OpenLessBackend { SilenceMonitor { session_id, started_at, - detector: crate::silence_auto_stop::SilenceAutoStop::new( - std::time::Duration::from_millis(silence_ms), - started_at, - ), + detector: if context.voice_activation.is_some() { + crate::silence_auto_stop::SilenceAutoStop::after_speech( + std::time::Duration::from_millis(silence_ms), + started_at, + ) + } else { + crate::silence_auto_stop::SilenceAutoStop::new( + std::time::Duration::from_millis(silence_ms), + started_at, + ) + }, } }); let context = match state.dictation_start_output_target { @@ -8015,50 +7994,12 @@ impl OpenLessBackend { } }; - let activation_phrases = self - .voice_activation_session - .lock() - .expect("voice activation session lock poisoned") - .as_ref() - .filter(|session| session.session_id == session_id) - .map(|session| { - ( - session.wake_phrase.clone(), - session - .finish_enabled - .then(|| session.finish_phrase.clone()), - ) - }); - if let Some((wake_phrase, finish_phrase)) = activation_phrases { - for text in [ - &mut engine_result.raw_text, - &mut engine_result.polished_text, - ] { - if let Some(stripped) = - crate::voice_activation::strip_phrase_prefix(text, &wake_phrase) - { - *text = stripped; - } - if let Some(finish_phrase) = finish_phrase.as_ref() { - if let Some(stripped) = - crate::voice_activation::strip_phrase_suffix(text, finish_phrase) - { - *text = stripped; - } - } - } - if let Some(asr_transcript) = engine_result.asr_transcript.as_mut() { - if let Some(stripped) = - crate::voice_activation::strip_phrase_prefix(asr_transcript, &wake_phrase) - { - *asr_transcript = stripped; - } - if let Some(finish_phrase) = finish_phrase.as_ref() { - if let Some(stripped) = - crate::voice_activation::strip_phrase_suffix(asr_transcript, finish_phrase) - { - *asr_transcript = stripped; - } + if context.pipeline_mode == crate::shared_types::PipelineMode::Multimodal { + if let Some(commands) = context.voice_activation.as_ref() { + engine_result.raw_text = commands.clean(&engine_result.raw_text); + engine_result.polished_text = commands.clean(&engine_result.polished_text); + if let Some(text) = engine_result.asr_transcript.as_mut() { + *text = commands.clean(text); } } } @@ -9183,6 +9124,60 @@ mod tests { fn assert_send_sync() {} + #[tokio::test] + async fn voice_activation_cancelled_handoff_releases_reserved_session() { + struct Capture(Arc, bool); + impl PendingAudioCapture for Capture { + fn commit( + self: Box, + _session_id: SessionId, + _context: Arc, + _consumer: Arc, + _progress: Arc, + ) -> BoxFuture<'static, Result, BackendError>> + { + panic!("stale handoff must not commit"); + } + fn discard(self: Box) -> BoxFuture<'static, Result<(), BackendError>> { + Box::pin(async move { + self.0.fetch_add(1, Ordering::SeqCst); + if self.1 { + Err(BackendError::new( + BackendErrorCode::Platform, + "discard failed", + )) + } else { + Ok(()) + } + }) + } + } + for discard_fails in [false, true] { + let (backend, _) = backend(); + backend.start().await.unwrap(); + let generation = backend.voice_activation_generation.load(Ordering::Acquire); + let discarded = Arc::new(AtomicU64::new(0)); + let gate = backend.voice_activation_configure_gate.lock().await; + let mut start = Box::pin(backend.start_voice_dictation_with_capture( + Box::new(Capture(Arc::clone(&discarded), discard_fails)), + generation, + )); + assert!(futures_util::poll!(&mut start).is_pending()); + assert_eq!(backend.snapshot().dictation.phase, DictationPhase::Starting); + backend + .voice_activation_generation + .fetch_add(1, Ordering::AcqRel); + drop(gate); + assert_eq!(start.await.unwrap_err().code, BackendErrorCode::Cancelled); + assert!(backend.snapshot().dictation.session_id.is_none()); + assert_eq!(discarded.load(Ordering::SeqCst), 1); + let next = backend + .reserve_dictation_session(false, DictationOutputTarget::ForegroundApp, None) + .unwrap(); + backend.reset_dictation_session(next.session_id); + } + } + #[test] fn vocabulary_learning_settings_migrate_normalize_and_roundtrip() { let old: UserPreferences = serde_json::from_str("{}").unwrap(); @@ -13845,6 +13840,265 @@ mod tests { backend.shutdown().await.unwrap(); } + async fn start_voice_test_session( + backend: &OpenLessBackend, + timeout: Option, + insert_text: bool, + ) -> SessionId { + let reservation = backend + .reserve_dictation_session(insert_text, DictationOutputTarget::ForegroundApp, None) + .unwrap(); + let id = reservation.session_id; + *backend.voice_activation_session.lock().unwrap() = + Some(crate::voice_activation::VoiceActivationSession { + generation: crate::voice_activation::VoiceActivationGeneration( + backend.voice_activation_generation.load(Ordering::Acquire), + ), + session_id: id, + timeout, + commands: crate::voice_activation::VoiceActivationTextPolicy { + wake_phrase: "开始语音录入".into(), + finish_phrase: Some("完成语音录入".into()), + }, + }); + backend + .start_reserved_dictation( + reservation, + DictationStartOptions { + insert_text, + ..Default::default() + }, + ) + .await + .unwrap() + } + + #[tokio::test] + async fn voice_activation_silence_uses_earliest_enabled_timeout_and_rejects_old_session() { + for (voice_timeout, general_timeout, expected) in [ + (Some(15), None, Some(15)), + (None, None, None), + (Some(15), Some(3), Some(3)), + (None, Some(3), Some(3)), + ] { + let (backend, _) = backend(); + let mut preferences = backend.get_preferences(); + preferences.silence_auto_stop_enabled = general_timeout.is_some(); + preferences.silence_auto_stop_seconds = general_timeout.unwrap_or(3) as f32; + preferences.hotkey.mode = crate::shared_types::HotkeyMode::Toggle; + backend.set_preferences(preferences).unwrap(); + backend.start().await.unwrap(); + let id = start_voice_test_session( + &backend, + voice_timeout.map(std::time::Duration::from_secs), + false, + ) + .await; + let context = backend + .state + .read() + .unwrap() + .dictation_context + .clone() + .unwrap(); + assert_eq!( + context.recording.silence_after_ms, + expected.map(|seconds| seconds * 1000) + ); + let commands = context.voice_activation.clone(); + assert!(commands.is_some()); + assert_eq!( + context.with_translation_requested(true).voice_activation, + commands + ); + assert_eq!( + context + .with_output_target(DictationOutputTarget::QuickNote) + .voice_activation, + commands + ); + let progress = backend.engine_progress_sink(); + for elapsed_ms in (0..=30_000).step_by(100) { + progress + .publish( + id, + EngineProgress::RecordingLevel { + elapsed_ms, + level: 0.1, + }, + ) + .unwrap(); + } + let count_stops = || { + backend.replay_events_after(0).events.iter().filter(|event| + matches!(&event.kind, BackendEventKind::RecordingControlRequested(request) if request.session_id == id) + ).count() + }; + assert_eq!(count_stops(), 0); + let deadline = 30_000 + expected.unwrap_or(300) * 1000; + progress + .publish( + id, + EngineProgress::RecordingLevel { + elapsed_ms: deadline - 1, + level: 0.0, + }, + ) + .unwrap(); + assert_eq!(count_stops(), 0); + progress + .publish( + id, + EngineProgress::RecordingLevel { + elapsed_ms: deadline, + level: 0.0, + }, + ) + .unwrap(); + assert_eq!(count_stops(), usize::from(expected.is_some())); + progress + .publish( + id, + EngineProgress::RecordingLevel { + elapsed_ms: deadline + 100, + level: 0.0, + }, + ) + .unwrap(); + assert_eq!(count_stops(), usize::from(expected.is_some())); + let generation = backend.voice_activation_generation.load(Ordering::Acquire); + backend.cancel_dictation(Some(id)).await.unwrap(); + let next = backend.start_dictation().await.unwrap(); + assert!(progress + .publish( + id, + EngineProgress::RecordingLevel { + elapsed_ms: deadline + 200, + level: 0.0 + } + ) + .is_err()); + backend + .finish_voice_activation( + generation, + id, + crate::voice_activation::VoiceActivationStopReason::Timeout, + ) + .await; + assert_eq!(backend.snapshot().dictation.session_id, Some(next)); + backend.cancel_dictation(Some(next)).await.unwrap(); + } + } + + #[tokio::test] + async fn voice_activation_persists_clean_body_and_never_inserts_command_only_audio() { + use crate::testing::{ + FixtureAudioRecorder, FixtureInsertionAction, FixtureTextInserter, FixtureTextPolisher, + FixtureTranscriptionEngine, + }; + for body in ["正文", "开始语音录入正文", ""] { + let dir = TestDataDir::new("voice-command-history"); + let polisher = Arc::new(FixtureTextPolisher::successful("完整正文")); + let inserter = Arc::new(FixtureTextInserter::with_outcome(InsertOutcome::Inserted)); + let engine = crate::PipelineDictationEngine::new( + Arc::new(FixtureAudioRecorder::new(vec![vec![1, 0, 2, 0]], vec![])), + Arc::new(FixtureTranscriptionEngine::successful( + format!("开始语音录入 {body} 完成语音录入"), + 1000, + )), + polisher.clone(), + ); + let backend = OpenLessBackend::new( + BackendConfig { + data_dir: dir.path().to_path_buf(), + ..Default::default() + }, + BackendDependencies { + host_actions: Arc::new(FakeHost::default()), + text_inserter: inserter.clone(), + dictation_engine: Arc::new(engine), + credential_store: llm_configured_credentials(), + ..BackendDependencies::unsupported() + }, + ) + .unwrap(); + let mut prefs = backend.get_preferences(); + prefs.streaming_insert = false; + backend.set_preferences(prefs).unwrap(); + backend.start().await.unwrap(); + let id = start_voice_test_session(&backend, None, true).await; + let result = backend.stop_dictation_session(id).await; + let history = backend.list_history().unwrap(); + assert_eq!(history.len(), 1); + assert_eq!(history[0].raw_transcript, body); + let final_transcript = + backend + .replay_events_after(0) + .events + .into_iter() + .find_map(|event| match event.kind { + BackendEventKind::TranscriptDelta(delta) if delta.is_final => { + Some(delta.text) + } + _ => None, + }); + assert_eq!(final_transcript.as_deref(), Some(body)); + if body.is_empty() { + assert!(result.is_err()); + assert!(history[0].final_text.is_empty()); + assert!(polisher.inputs().is_empty()); + assert!(!inserter.actions().iter().any(|action| matches!( + action, + FixtureInsertionAction::Insert { .. } | FixtureInsertionAction::Write { .. } + ))); + } else { + assert_eq!(result.unwrap().polished_text, "完整正文"); + assert_eq!(history[0].final_text, "完整正文"); + assert_eq!(polisher.inputs(), vec![body]); + assert!(inserter.actions().iter().any(|action| matches!(action, + FixtureInsertionAction::Insert { text, .. } if text == "完整正文"))); + } + assert!(backend.snapshot().dictation.session_id.is_none()); + } + } + + #[tokio::test] + async fn voice_activation_transcript_revisions_keep_raw_offsets() { + let (backend, _) = backend(); + backend.start().await.unwrap(); + let id = start_voice_test_session(&backend, None, false).await; + let progress = backend.engine_progress_sink(); + for (text, offset, expected) in [ + ("开始语音录入你好", 0, "你好"), + ("世界完成语音录入", 8, "你好世界"), + ("你好，世界完成语音录入", 6, "你好，世界"), + ] { + progress + .publish( + id, + EngineProgress::TranscriptDelta(crate::TranscriptDelta { + text: text.into(), + offset, + is_final: false, + }), + ) + .unwrap(); + assert_eq!( + backend.state.read().unwrap().transcripts[&id].text(), + expected + ); + if offset == 0 { + backend + .stop_voice_activation( + crate::voice_activation::VoiceActivationStopReason::Disabled, + ) + .await + .unwrap(); + } + } + backend.cancel_dictation(Some(id)).await.unwrap(); + } + #[tokio::test] async fn recording_readiness_waits_for_the_first_pcm_even_when_its_meter_is_zero() { let (backend, _) = backend(); @@ -17444,7 +17698,6 @@ mod tests { state: Arc::clone(&backend.state), phase_changed: Arc::clone(&backend.phase_changed), text_insertions: Arc::clone(&backend.text_insertions), - voice_activation_session: Arc::clone(&backend.voice_activation_session), raw_transcripts: Mutex::new(HashMap::new()), }; backend.cancel_dictation(Some(session)).await.unwrap(); diff --git a/openless-all/app/crates/openless-core/src/dictation_context.rs b/openless-all/app/crates/openless-core/src/dictation_context.rs index c06c3ef74..9a9cb977a 100644 --- a/openless-all/app/crates/openless-core/src/dictation_context.rs +++ b/openless-all/app/crates/openless-core/src/dictation_context.rs @@ -198,6 +198,8 @@ pub struct DictationContext { pub recording: RecordingPlan, pub pipeline_mode: PipelineMode, pub correction_rules: Vec, + /// Runtime-only commands, captured only for voice-started dictation. + pub voice_activation: Option, pub asr: ProviderInvocation, /// Local model a failed cloud ASR call is retried with, frozen at session /// start (see local_asr_fallback()); None when the fallback doesn't apply. @@ -340,6 +342,7 @@ impl DictationContext { preferences.pipeline_mode, ), correction_rules: Vec::new(), + voice_activation: None, asr, asr_fallback, llm, diff --git a/openless-all/app/crates/openless-core/src/dictation_engine.rs b/openless-all/app/crates/openless-core/src/dictation_engine.rs index 9e140803f..00c337f8f 100644 --- a/openless-all/app/crates/openless-core/src/dictation_engine.rs +++ b/openless-all/app/crates/openless-core/src/dictation_engine.rs @@ -644,7 +644,9 @@ impl DictationEngine for PipelineDictationEngine { ) }); let race = match standby { - Some(standby) => race_cloud_with_local_standby(transcription.finish(), standby).await, + Some(standby) => { + race_cloud_with_local_standby(transcription.finish(), standby).await + } None => AsrRace::Cloud { result: transcription.finish().await, standby: None, @@ -825,6 +827,9 @@ impl DictationEngine for PipelineDictationEngine { ) .into()); } + if let Some(commands) = context.voice_activation.as_ref() { + transcript.text = commands.clean(&transcript.text); + } let original_asr_text = transcript.text.clone(); transcript.text = crate::correction::apply_correction_rules( &transcript.text, @@ -869,6 +874,7 @@ impl DictationEngine for PipelineDictationEngine { // Optional polish may be skipped after fallback; an explicit translation is still required. let translation_active = context.polish.translation_active; let uses_polisher = context.uses_llm_polisher() + && !(context.voice_activation.is_some() && transcript.text.trim().is_empty()) && !skips_polish_for_short_text && (!local_fallback_used || translation_active); log::info!( @@ -1230,7 +1236,10 @@ struct ReplayPlan { backoff: bool, } -const CLOUD_RETRY: ReplayPlan = ReplayPlan { attempts: 2, backoff: true }; +const CLOUD_RETRY: ReplayPlan = ReplayPlan { + attempts: 2, + backoff: true, +}; type AsrAttempt = ( crate::ports::TranscriptOutput, @@ -3464,6 +3473,126 @@ mod tests { assert_eq!(result.polished_text, "raw text"); } + #[tokio::test] + async fn voice_activation_commands_are_cleaned_before_correction_and_polishing() { + use crate::testing::{ + FixtureAudioRecorder, FixtureTextPolisher, FixtureTranscriptionEngine, + }; + for (input, voice, finish, expected, original) in [ + ( + "开始，文字录入。 错字 完成，文字录入。", + true, + true, + "正文", + Some("错字"), + ), + ( + "开始文字录入 正文完成文字录入之前 完成文字录入", + true, + true, + "正文完成文字录入之前", + None, + ), + ( + "开始文字录入 正文完成文字录入", + true, + false, + "正文完成文字录入", + None, + ), + ("开始文字录入 完成文字录入", true, true, "", None), + ( + "开始文字录入 正文 完成文字录入", + false, + true, + "开始文字录入 正文 完成文字录入", + None, + ), + ] { + let polisher = Arc::new(FixtureTextPolisher::successful("润色结果")); + let engine = PipelineDictationEngine::new( + Arc::new(FixtureAudioRecorder::new(vec![vec![1, 0, 2, 0]], vec![])), + Arc::new(FixtureTranscriptionEngine::successful(input, 1_000)), + polisher.clone(), + ); + let mut context = DictationContext::default(); + context.polish.polish_skip_max_chars = 0; + context.voice_activation = + voice.then(|| crate::voice_activation::VoiceActivationTextPolicy { + wake_phrase: "开始文字录入".into(), + finish_phrase: finish.then(|| "完成文字录入".into()), + }); + context.correction_rules.push(crate::types::CorrectionRule { + id: "voice-cleanup".into(), + pattern: "错字".into(), + replacement: "正文".into(), + enabled: true, + created_at: String::new(), + source: Default::default(), + }); + let session_id = SessionId::new(); + let progress = Arc::new(RecordingProgress::default()); + engine + .start(session_id, Arc::new(context), progress.clone()) + .await + .unwrap(); + let result = engine.finish(session_id, progress.clone()).await.unwrap(); + assert_eq!(result.raw_text, expected); + assert_eq!(result.asr_transcript.as_deref(), original); + if expected.is_empty() { + assert!(polisher.inputs().is_empty()); + assert!(result.polished_text.is_empty()); + } else { + assert_eq!(polisher.inputs(), vec![expected]); + assert_eq!(result.polished_text, "润色结果"); + } + assert!(progress.events.lock().unwrap().iter().any(|event| matches!(event, + EngineProgress::TranscriptDelta(delta) if delta.is_final && delta.text == expected))); + } + } + + #[tokio::test] + async fn voice_activation_polish_failure_retains_only_spoken_body() { + use crate::testing::{ + FixtureAudioRecorder, FixtureTextPolisher, FixtureTranscriptionEngine, + }; + for policy in [PolishFailurePolicy::Fail, PolishFailurePolicy::UseRawText] { + let polisher = Arc::new(FixtureTextPolisher::failing(BackendError::new( + BackendErrorCode::Provider, + "fixture", + ))); + let engine = PipelineDictationEngine::new( + Arc::new(FixtureAudioRecorder::new(vec![vec![1, 0, 2, 0]], vec![])), + Arc::new(FixtureTranscriptionEngine::successful( + "开始语音录入 正文 完成语音录入", + 1000, + )), + polisher.clone(), + ) + .with_polish_failure_policy(policy); + let mut context = DictationContext::default(); + context.voice_activation = Some(crate::voice_activation::VoiceActivationTextPolicy { + wake_phrase: "开始语音录入".into(), + finish_phrase: Some("完成语音录入".into()), + }); + let id = SessionId::new(); + let progress = Arc::new(RecordingProgress::default()); + engine + .start(id, Arc::new(context), progress.clone()) + .await + .unwrap(); + let result = engine.finish(id, progress).await; + assert_eq!(polisher.inputs(), vec!["正文"]); + match result { + Ok(output) => { + assert_eq!(output.raw_text, "正文"); + assert_eq!(output.polished_text, "正文"); + } + Err(failure) => assert_eq!(failure.raw_text.as_deref(), Some("正文")), + } + } + } + #[tokio::test] async fn short_transcript_under_threshold_skips_polish() { // Issue #1032: "raw text" has 7 non-whitespace characters, within a limit of 8. @@ -3755,7 +3884,9 @@ mod tests { /// started with, and records that provider — so a test can tell the cloud /// call, its retries and the local fallback apart. struct RoutedTranscriber { - outputs: Mutex>>>, + outputs: Mutex< + std::collections::HashMap>>, + >, started: Arc>>, /// How long each provider's finish takes; providers not listed answer at once. delays: std::collections::HashMap, @@ -3828,7 +3959,11 @@ mod tests { fn fallback_test_engine( cloud: Vec>, local: Vec>, - ) -> (PipelineDictationEngine, Arc, Arc>>) { + ) -> ( + PipelineDictationEngine, + Arc, + Arc>>, + ) { let (engine, progress, started, _) = fallback_polish_test_engine( cloud, local, @@ -3896,14 +4031,21 @@ mod tests { contexts: Arc::new(Mutex::new(Vec::new())), }), ); - (engine, Arc::new(RecordingProgress::default()), started, polish_calls) + ( + engine, + Arc::new(RecordingProgress::default()), + started, + polish_calls, + ) } /// A polishing session (not raw mode) on CLOUD with the LOCAL fallback. fn polishing_fallback_context() -> Arc { let mut context = DictationContext { asr: crate::dictation_context::ProviderInvocation::new(CLOUD, CLOUD), - asr_fallback: Some(crate::dictation_context::ProviderInvocation::new(LOCAL, LOCAL)), + asr_fallback: Some(crate::dictation_context::ProviderInvocation::new( + LOCAL, LOCAL, + )), ..DictationContext::default() }; assert!(context.uses_llm_polisher(), "fixture must exercise the polisher"); @@ -4008,7 +4150,11 @@ mod tests { #[tokio::test] async fn a_local_fallback_skips_polish_and_reports_no_failure() { let (engine, progress, started, polish_calls) = fallback_polish_test_engine( - vec![Err(BackendError::new(BackendErrorCode::Provider, "offline").retryable(true))], + vec![Err(BackendError::new( + BackendErrorCode::Provider, + "offline", + ) + .retryable(true))], vec![transcript("本地识别")], Err(BackendError::new(BackendErrorCode::Provider, "offline")), ); @@ -4037,7 +4183,10 @@ mod tests { let (engine, progress, _, polish_calls) = fallback_polish_test_engine( vec![], vec![transcript("本地识别")], - Err(BackendError::new(BackendErrorCode::Provider, "invalid api key")), + Err(BackendError::new( + BackendErrorCode::Provider, + "invalid api key", + )), ); let session_id = SessionId::new(); engine.start(session_id, local_polishing_context(), progress.clone()).await.unwrap(); @@ -4090,18 +4239,27 @@ mod tests { fn fallback_context() -> Arc { let mut context = (*raw_dictation_context()).clone(); context.asr = crate::dictation_context::ProviderInvocation::new(CLOUD, CLOUD); - context.asr_fallback = Some(crate::dictation_context::ProviderInvocation::new(LOCAL, LOCAL)); + context.asr_fallback = Some(crate::dictation_context::ProviderInvocation::new( + LOCAL, LOCAL, + )); Arc::new(context) } fn transcript(text: &str) -> Result { - Ok(TranscriptOutput { text: text.into(), duration_ms: 25 }) + Ok(TranscriptOutput { + text: text.into(), + duration_ms: 25, + }) } #[tokio::test] async fn failed_cloud_asr_goes_straight_to_the_local_fallback() { let (engine, progress, started) = fallback_test_engine( - vec![Err(BackendError::new(BackendErrorCode::Provider, "offline").retryable(true))], + vec![Err(BackendError::new( + BackendErrorCode::Provider, + "offline", + ) + .retryable(true))], vec![transcript("本地识别")], ); let session_id = SessionId::new(); @@ -4117,7 +4275,10 @@ mod tests { #[tokio::test] async fn a_non_retryable_cloud_error_also_falls_back() { let (engine, progress, started) = fallback_test_engine( - vec![Err(BackendError::new(BackendErrorCode::Provider, "quota exhausted"))], + vec![Err(BackendError::new( + BackendErrorCode::Provider, + "quota exhausted", + ))], vec![transcript("本地识别")], ); let session_id = SessionId::new(); @@ -4135,7 +4296,10 @@ mod tests { || Err(BackendError::new(BackendErrorCode::Provider, "offline").retryable(true)); let (engine, progress, started) = fallback_test_engine( vec![temporary(), temporary(), temporary()], - vec![Err(BackendError::new(BackendErrorCode::Unsupported, "model not downloaded"))], + vec![Err(BackendError::new( + BackendErrorCode::Unsupported, + "model not downloaded", + ))], ); let session_id = SessionId::new(); engine.start(session_id, fallback_context(), progress.clone()).await.unwrap(); @@ -4206,7 +4370,10 @@ mod tests { let (engine, progress, _, _) = racing_test_engine( &[(CLOUD, 200)], vec![transcript("云端识别")], - vec![Err(BackendError::new(BackendErrorCode::Unsupported, "model not downloaded"))], + vec![Err(BackendError::new( + BackendErrorCode::Unsupported, + "model not downloaded", + ))], Ok(crate::ports::PolishOutput::text("unused")), ); let session_id = SessionId::new(); diff --git a/openless-all/app/crates/openless-core/src/lib.rs b/openless-all/app/crates/openless-core/src/lib.rs index 2404bd4d4..92964d687 100644 --- a/openless-all/app/crates/openless-core/src/lib.rs +++ b/openless-all/app/crates/openless-core/src/lib.rs @@ -384,9 +384,10 @@ pub use voice_activation::{ normalize_phrase, validate_phrase, PhraseValidationError, VoiceActivationCommand, VoiceActivationConfig, VoiceActivationDecision, VoiceActivationGeneration, VoiceActivationPhase, VoiceActivationSession, VoiceActivationSignal, - VoiceActivationStateSnapshot, VoiceActivationStopReason, VoiceCommandMatch, - VoiceCommandRecognizer, VoicePhraseMatcher, DEFAULT_FINISH_PHRASE, DEFAULT_TIMEOUT_SECONDS, - DEFAULT_WAKE_PHRASE, MAX_PHRASE_CHARS, MAX_TIMEOUT_SECONDS, MIN_TIMEOUT_SECONDS, + VoiceActivationStateSnapshot, VoiceActivationStopReason, VoiceActivationTextPolicy, + VoiceCommandMatch, VoiceCommandRecognizer, VoicePhraseMatcher, DEFAULT_FINISH_PHRASE, + DEFAULT_TIMEOUT_SECONDS, DEFAULT_WAKE_PHRASE, MAX_PHRASE_CHARS, MAX_TIMEOUT_SECONDS, + MIN_TIMEOUT_SECONDS, }; pub use voice_edit_session::{ TextSelection, VoiceEditCommit, VoiceEditContext, VoiceEditError, VoiceEditPhase, diff --git a/openless-all/app/crates/openless-core/src/silence_auto_stop.rs b/openless-all/app/crates/openless-core/src/silence_auto_stop.rs index ebd19ae72..21f27ad0b 100644 --- a/openless-all/app/crates/openless-core/src/silence_auto_stop.rs +++ b/openless-all/app/crates/openless-core/src/silence_auto_stop.rs @@ -31,6 +31,15 @@ impl SilenceAutoStop { } } + /// A wake command already established speech before formal capture began. + pub fn after_speech(silence_after_speech: Duration, started_at: Instant) -> Self { + Self { + speech_detected: true, + last_speech_at: Some(started_at), + ..Self::new(silence_after_speech, started_at) + } + } + pub fn on_level(&mut self, level: f32, now: Instant) -> Option { if self.decided { return None; @@ -72,6 +81,35 @@ mod tests { .find_map(|(level, offset)| detector.on_level(*level, base + *offset)) } + #[test] + fn voice_activation_timeout_tracks_speech_instead_of_total_duration() { + let base = Instant::now(); + let mut detector = SilenceAutoStop::after_speech(Duration::from_secs(15), base); + for seconds in 0..=30 { + assert_eq!( + detector.on_level(0.1, base + Duration::from_secs(seconds)), + None + ); + } + assert_eq!(detector.on_level(0.0, base + Duration::from_secs(44)), None); + assert_eq!( + detector.on_level(0.0, base + Duration::from_secs(45)), + Some(SilenceDecision::Stop) + ); + assert_eq!(detector.on_level(0.0, base + Duration::from_secs(60)), None); + } + + #[test] + fn voice_activation_without_body_uses_configured_silence_timeout() { + let base = Instant::now(); + let mut detector = SilenceAutoStop::after_speech(Duration::from_secs(15), base); + assert_eq!(detector.on_level(0.0, base + Duration::from_secs(10)), None); + assert_eq!( + detector.on_level(0.0, base + Duration::from_secs(15)), + Some(SilenceDecision::Stop) + ); + } + #[test] fn speech_then_silence_stops_once() { let base = Instant::now(); diff --git a/openless-all/app/crates/openless-core/src/voice_activation.rs b/openless-all/app/crates/openless-core/src/voice_activation.rs index 8f2393fc1..1faf28ded 100644 --- a/openless-all/app/crates/openless-core/src/voice_activation.rs +++ b/openless-all/app/crates/openless-core/src/voice_activation.rs @@ -4,7 +4,7 @@ //! only consumes revised recognition text and produces command decisions, so a //! standby listener can never accidentally publish normal transcript events. -use std::time::{Duration, Instant}; +use std::time::Duration; use unicode_normalization::UnicodeNormalization; @@ -386,23 +386,32 @@ impl VoiceActivationGeneration { } } +/// Commands frozen for a voice-started session; ordinary dictation has no policy. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VoiceActivationTextPolicy { + pub wake_phrase: String, + pub finish_phrase: Option, +} + +impl VoiceActivationTextPolicy { + pub fn clean(&self, text: &str) -> String { + let body = strip_phrase_prefix(text, &self.wake_phrase).unwrap_or_else(|| text.to_owned()); + self.finish_phrase + .as_ref() + .and_then(|phrase| strip_phrase_suffix(&body, phrase)) + .unwrap_or(body) + } +} + #[derive(Debug, Clone)] pub struct VoiceActivationSession { pub generation: VoiceActivationGeneration, pub session_id: SessionId, - pub started_at: Instant, pub timeout: Option, - pub wake_phrase: String, - pub finish_enabled: bool, - pub finish_phrase: String, + pub commands: VoiceActivationTextPolicy, } impl VoiceActivationSession { - pub fn timed_out_at(&self, now: Instant) -> bool { - self.timeout - .is_some_and(|timeout| now.saturating_duration_since(self.started_at) >= timeout) - } - pub fn accepts(&self, generation: VoiceActivationGeneration, session_id: SessionId) -> bool { self.generation == generation && self.session_id == session_id } @@ -519,11 +528,11 @@ mod tests { let session = VoiceActivationSession { generation: VoiceActivationGeneration(7), session_id, - started_at: Instant::now(), timeout: Some(Duration::from_secs(1)), - wake_phrase: DEFAULT_WAKE_PHRASE.into(), - finish_enabled: true, - finish_phrase: DEFAULT_FINISH_PHRASE.into(), + commands: VoiceActivationTextPolicy { + wake_phrase: DEFAULT_WAKE_PHRASE.into(), + finish_phrase: Some(DEFAULT_FINISH_PHRASE.into()), + }, }; assert!(session.accepts(VoiceActivationGeneration(7), session_id)); assert!(!session.accepts(VoiceActivationGeneration(6), session_id)); @@ -540,16 +549,17 @@ mod tests { config.normalized().unwrap().timeout_seconds, MIN_TIMEOUT_SECONDS ); - let session = VoiceActivationSession { - generation: VoiceActivationGeneration(1), - session_id: SessionId::new(), - started_at: Instant::now(), - timeout: Some(Duration::from_secs(1)), - wake_phrase: DEFAULT_WAKE_PHRASE.into(), - finish_enabled: true, - finish_phrase: DEFAULT_FINISH_PHRASE.into(), - }; - assert!(!session.timed_out_at(Instant::now())); - assert!(session.timed_out_at(session.started_at + Duration::from_secs(1))); + assert_eq!( + config.normalized().unwrap().timeout(), + Some(Duration::from_secs(5)) + ); + assert_eq!( + VoiceActivationConfig { + timeout_enabled: false, + ..config + } + .timeout(), + None + ); } } diff --git a/openless-all/app/e2e/settings-regressions.spec.ts b/openless-all/app/e2e/settings-regressions.spec.ts index 4503c2993..1664d4229 100644 --- a/openless-all/app/e2e/settings-regressions.spec.ts +++ b/openless-all/app/e2e/settings-regressions.spec.ts @@ -227,3 +227,61 @@ test('conditional notices animate actual layout downward and retain content duri await expect(page.getByTestId('notice')).toHaveText('Working status'); await expect(page.getByTestId('notice')).toBeHidden(); }); + +test('voice activation opens model setup without leaving the multimodal pipeline', async ({ + page, +}, testInfo) => { + await page.addInitScript(() => { + Object.defineProperty(navigator, 'platform', { get: () => 'Win32' }); + Object.defineProperty(navigator, 'userAgent', { get: () => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)' }); + }); + await page.goto('http://127.0.0.1:5326/?os=win'); + await page.getByRole('button', { name: '设置', exact: true }).click(); + const dialog = page.getByRole('dialog', { name: '设置', exact: true }); + const categories = dialog.getByRole('navigation', { name: '设置分类' }); + await categories.getByRole('button', { name: 'AI 服务与模型', exact: true }).click(); + await dialog.getByRole('button', { name: '多模态模式', exact: true }).click(); + const models = dialog + .locator('.ol-service-views') + .getByRole('button', { name: /^本地模型/ }); + await expect(models).toBeDisabled(); + await categories.getByRole('button', { name: '录音与输入', exact: true }).click(); + await dialog.getByRole('switch', { name: '语音唤醒听写', exact: true }).click(); + const download = dialog.getByRole('button', { + name: '前往 AI 服务与模型下载 Zipformer', + exact: true, + }); + await expect(download).toBeVisible(); + const providerState = () => + page.evaluate(async () => { + const { mockSettings } = await import('/src/lib/ipc/mock-data.ts'); + return [ + mockSettings.pipelineMode, + mockSettings.activeAsrProvider, + mockSettings.activeOmniProvider, + ]; + }); + const before = await providerState(); + await download.focus(); + await page.keyboard.press('Enter'); + await expect(models).toHaveAttribute('aria-pressed', 'true'); + await expect(dialog.getByRole('button', { name: '多模态模式', exact: true })).toHaveAttribute( + 'aria-pressed', + 'true', + ); + await expect( + dialog.getByText('此处准备语音唤醒模型，听写仍使用当前多模态服务。', { exact: true }), + ).toBeVisible(); + await expect( + dialog.locator('.ol-service-views').getByRole('button', { name: /^语言模型/ }), + ).toBeDisabled(); + await expect( + dialog.locator('.ol-service-views').getByRole('button', { name: /^语音识别/ }), + ).toBeDisabled(); + expect(await providerState()).toEqual(before); + await page.screenshot({ path: testInfo.outputPath('voice-activation-model-setup.png') }); + await categories.getByRole('button', { name: '录音与输入', exact: true }).click(); + await dialog.getByRole('switch', { name: '语音唤醒听写', exact: true }).click(); + await categories.getByRole('button', { name: 'AI 服务与模型', exact: true }).click(); + await expect(models).toBeDisabled(); +}); diff --git a/openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs b/openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs index da37d1129..88d2e5944 100644 --- a/openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs +++ b/openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs @@ -340,7 +340,8 @@ impl SherpaOnnxRuntime { #[cfg(local_asr_sherpa)] crate::recorder::voice_diagnostic("P5", &["H4"], "cache_owner", "releasing", serde_json::json!({"count": Arc::strong_count(&loaded.recognizer), - "active": loaded.diagnostic_active.load(Ordering::Relaxed)})); + "active": loaded.diagnostic_active.load(Ordering::Relaxed)}), + ); } state.online_loaded = None; Ok(()) @@ -364,7 +365,8 @@ impl SherpaOnnxRuntime { #[cfg(local_asr_sherpa)] crate::recorder::voice_diagnostic("P5", &["H4"], "cache_owner", "releasing", serde_json::json!({"count": Arc::strong_count(&loaded.recognizer), - "active": loaded.diagnostic_active.load(Ordering::Relaxed)})); + "active": loaded.diagnostic_active.load(Ordering::Relaxed)}), + ); } state.online_loaded = None; Ok(true) @@ -688,8 +690,13 @@ struct VoiceNativeProbe<'a> { impl<'a> VoiceNativeProbe<'a> { fn enter(active: &'a std::sync::atomic::AtomicUsize, stage: &'static str) -> Self { let count = active.fetch_add(1, Ordering::Relaxed) + 1; - crate::recorder::voice_diagnostic("P3", &["H2", "H4", "H6"], "native", "entered", - serde_json::json!({"stage": stage, "count": count})); + crate::recorder::voice_diagnostic( + "P3", + &["H2", "H4", "H6"], + "native", + "entered", + serde_json::json!({"stage": stage, "count": count}), + ); Self { active, stage } } } @@ -698,8 +705,13 @@ impl<'a> VoiceNativeProbe<'a> { impl Drop for VoiceNativeProbe<'_> { fn drop(&mut self) { let count = self.active.fetch_sub(1, Ordering::Relaxed) - 1; - crate::recorder::voice_diagnostic("P3", &["H2", "H4", "H6"], "native", "returned", - serde_json::json!({"stage": self.stage, "count": count})); + crate::recorder::voice_diagnostic( + "P3", + &["H2", "H4", "H6"], + "native", + "returned", + serde_json::json!({"stage": self.stage, "count": count}), + ); } } @@ -710,7 +722,8 @@ impl Drop for SherpaOnlineSession { fn drop(&mut self) { crate::recorder::voice_diagnostic("P5", &["H4"], "session_owner", "dropping", serde_json::json!({"count": Arc::strong_count(&self.recognizer), - "active": self.diagnostic_active.load(Ordering::Relaxed)})); + "active": self.diagnostic_active.load(Ordering::Relaxed)}), + ); } } @@ -819,8 +832,9 @@ impl SherpaOnlineSession { self.last_emitted_text.clear(); self.pending_new_segment = true; { - let _probe = VoiceNativeProbe::enter(&self.diagnostic_active, "reset"); - self.recognizer.reset(&self.stream); + // EOF also closes feature extraction; recognizer reset only resets decoding. + let _probe = VoiceNativeProbe::enter(&self.diagnostic_active, "new_stream"); + self.stream = self.recognizer.create_stream(); } Ok(()) } @@ -873,7 +887,8 @@ impl SherpaOnlineSession { crate::recorder::voice_diagnostic("P3", &["H2", "H4", "H6"], "native_result", "observed", serde_json::json!({"present": result.is_some(), "count": result.as_ref().map(|result| result.text.chars().count()).unwrap_or(0), - "stage": if allow_endpoint_reset { "streaming" } else { "flush" }})); + "stage": if allow_endpoint_reset { "streaming" } else { "flush" }}), + ); let Some(result) = result else { return; }; @@ -934,10 +949,7 @@ impl SherpaOnlineSession { // Sherpa 1.13.2's online-decode-files.py supplies 0.66 s of zero samples before // input_finished(): marking EOF alone does not provide the model's trailing context. // This is bounded model input, not a wall-clock wait or continuous idle inference. -fn finish_online_segment_input( - accept_tail: impl FnOnce(&[f32]), - input_finished: impl FnOnce(), -) { +fn finish_online_segment_input(accept_tail: impl FnOnce(&[f32]), input_finished: impl FnOnce()) { const TAIL: [f32; 16_000 * 66 / 100] = [0.0; 16_000 * 66 / 100]; accept_tail(&TAIL); input_finished(); @@ -957,8 +969,13 @@ fn append_segment(text: &mut String, segment: &str) { #[cfg(local_asr_sherpa)] fn create_online_session_from_loaded(loaded: LoadedOnlineModel) -> Result { let stream = loaded.recognizer.create_stream(); - crate::recorder::voice_diagnostic("P5", &["H4"], "session_owner", "created", - serde_json::json!({"count": Arc::strong_count(&loaded.recognizer)})); + crate::recorder::voice_diagnostic( + "P5", + &["H4"], + "session_owner", + "created", + serde_json::json!({"count": Arc::strong_count(&loaded.recognizer)}), + ); Ok(SherpaOnlineSession { diagnostic_active: loaded.diagnostic_active, alias: loaded.alias, @@ -1225,6 +1242,43 @@ mod tests { assert_eq!(boundaries.get(), 2); } + #[cfg(local_asr_sherpa)] + #[tokio::test] + #[ignore = "requires explicit OPENLESS_SHERPA_TEST_MODEL_DIR and OPENLESS_SHERPA_TEST_WAV; never downloads"] + async fn voice_activation_decodes_a_second_segment_after_eof() { + let model_dir = std::env::var_os("OPENLESS_SHERPA_TEST_MODEL_DIR") + .expect("explicit Zipformer test model directory"); + let wav_path = std::env::var_os("OPENLESS_SHERPA_TEST_WAV") + .expect("explicit 16 kHz mono speech fixture"); + let wav = std::fs::read(wav_path).unwrap(); + let pcm = openless_core::audio::decode_dictation_wav(&wav).unwrap(); + assert!(!pcm.is_empty()); + let runtime = SherpaOnnxRuntime::new(); + let mut session = runtime + .create_online_session("zipformer-bilingual-zh-en-streaming", Path::new(&model_dir)) + .await + .unwrap(); + session.stream.set_option("openless-test-segment", "first"); + for chunk in pcm.chunks(640) { + session.accept_pcm_chunk(chunk, &|_| {}).unwrap(); + } + session.flush_segment(&|_| {}).unwrap(); + let first = session.final_text(); + assert!( + !first.is_empty(), + "the fixture must contain recognizable speech" + ); + assert!( + !session.stream.has_option("openless-test-segment"), + "EOF must replace the feature stream" + ); + for chunk in pcm.chunks(640) { + session.accept_pcm_chunk(chunk, &|_| {}).unwrap(); + } + let result = session.finish(&|_| {}).unwrap(); + assert_eq!(result, format!("{first} {first}")); + } + #[test] fn append_segment_inserts_space_between_segments() { let mut text = String::new(); diff --git a/openless-all/app/src-tauri/src/recorder.rs b/openless-all/app/src-tauri/src/recorder.rs index bf6d3991a..f67f52952 100644 --- a/openless-all/app/src-tauri/src/recorder.rs +++ b/openless-all/app/src-tauri/src/recorder.rs @@ -66,8 +66,15 @@ pub(crate) fn voice_diagnostic( static SEQUENCE: AtomicU64 = AtomicU64::new(0); static DROPPED: AtomicU64 = AtomicU64::new(0); static RUN: OnceLock = OnceLock::new(); - let run = RUN.get_or_init(|| format!("r{:x}", std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH).unwrap_or_default().as_nanos())); + let run = RUN.get_or_init(|| { + format!( + "r{:x}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() + ) + }); let writer = WRITER.get_or_init(|| { let (tx, rx) = sync_channel::(4096); std::thread::Builder::new().name("voice-1003-diagnostics".into()).spawn(move || { @@ -390,9 +397,7 @@ struct WarmCaptureSink { activation_consumer: Mutex>>, activation_consumer_during_capture: AtomicBool, activation_generation: Mutex>, - activation_fault_handler: Mutex< - Option<(u64, Arc)>, - >, + activation_fault_handler: Mutex)>>, activation_vad: Mutex, max_bytes: usize, } @@ -434,11 +439,6 @@ impl ActivationVad { end_of_speech: false, }; } - WarmCaptureSink::append_bounded( - &mut self.pre_roll, - pcm, - ACTIVATION_VAD_PRE_ROLL_BYTES, - ); let speech = pcm_rms_level(pcm) >= ACTIVATION_VAD_RMS_THRESHOLD; if speech { let mut output = if self.speech_active { @@ -446,9 +446,9 @@ impl ActivationVad { } else { self.pre_roll.iter().copied().collect() }; - if output.is_empty() { - output.extend_from_slice(pcm); - } + // The current block can be a complete handoff buffer, not just a device chunk. + output.extend_from_slice(pcm); + self.pre_roll.clear(); self.speech_active = true; self.hangover_chunks = ACTIVATION_VAD_HANGOVER_CHUNKS; return ActivationVadOutput { @@ -463,6 +463,7 @@ impl ActivationVad { end_of_speech: false, }; } + WarmCaptureSink::append_bounded(&mut self.pre_roll, pcm, ACTIVATION_VAD_PRE_ROLL_BYTES); let end_of_speech = self.speech_active; self.speech_active = false; ActivationVadOutput { @@ -539,11 +540,17 @@ impl WarmCaptureSink { static BLOCKS: AtomicU64 = AtomicU64::new(0); let blocks = BLOCKS.fetch_add(1, Ordering::Relaxed); if was_active != active || gated.end_of_speech || blocks % 64 == 0 { - voice_diagnostic("P1", &["H1"], "vad_gate", "observed", serde_json::json!({ - "bytes": pcm.len(), "active": active, "hangover": hangover, - "forwarded": gated.pcm.is_some(), "boundary": gated.end_of_speech, - "count": blocks, - })); + voice_diagnostic( + "P1", + &["H1"], + "vad_gate", + "observed", + serde_json::json!({ + "bytes": pcm.len(), "active": active, "hangover": hangover, + "forwarded": gated.pcm.is_some(), "boundary": gated.end_of_speech, + "count": blocks, + }), + ); } if let Some(pcm) = gated.pcm { consumer.consume_pcm_chunk(&pcm); @@ -2774,11 +2781,12 @@ mod tests { let sink = WarmCaptureSink::new(STAGED_CAPTURE_MAX_BYTES); let formal = Arc::new(RecordingConsumer::default()); let finish = Arc::new(RecordingConsumer::default()); - sink.arm(7, false).unwrap(); + sink.arm_with_limit(7, false, VOICE_STAGED_CAPTURE_MAX_BYTES) + .unwrap(); sink.set_activation_consumer_during_capture(Some( - Arc::clone(&finish) as Arc, + Arc::clone(&finish) as Arc )); - let speech = [0x00, 0x40, 0x00, 0x40]; + let speech = [0x00, 0x40].repeat(16_000); sink.consume_pcm_chunk(&speech); sink.promote( @@ -2833,6 +2841,29 @@ mod tests { assert!(chunks[0].ends_with(&speech)); } + #[test] + fn activation_vad_preserves_large_blocks_and_does_not_replay_forwarded_audio() { + let mut vad = ActivationVad::default(); + let silence = vec![0; ACTIVATION_VAD_PRE_ROLL_BYTES + 320]; + assert!(vad.gate(&silence).pcm.is_none()); + let speech = [0x00, 0x40].repeat(16_000); + let first = vad.gate(&speech).pcm.unwrap(); + assert_eq!(first.len(), ACTIVATION_VAD_PRE_ROLL_BYTES + speech.len()); + assert_eq!( + &first[..ACTIVATION_VAD_PRE_ROLL_BYTES], + &silence[..ACTIVATION_VAD_PRE_ROLL_BYTES] + ); + assert_eq!(&first[ACTIVATION_VAD_PRE_ROLL_BYTES..], &speech); + assert_eq!(vad.gate(&speech[..320]).pcm.unwrap(), speech[..320]); + for _ in 0..ACTIVATION_VAD_HANGOVER_CHUNKS { + assert_eq!(vad.gate(&[0; 320]).pcm.unwrap(), [0; 320]); + } + assert!(vad.gate(&[0; 320]).end_of_speech); + let next = vad.gate(&speech[..320]).pcm.unwrap(); + assert_eq!(next.len(), 640); + assert_eq!(&next[320..], &speech[..320]); + } + #[test] fn activation_vad_hangover_is_bounded() { let mut vad = ActivationVad::default(); diff --git a/openless-all/app/src/i18n/de.ts b/openless-all/app/src/i18n/de.ts index a1431c4cc..dcbb77aee 100644 --- a/openless-all/app/src/i18n/de.ts +++ b/openless-all/app/src/i18n/de.ts @@ -1639,6 +1639,8 @@ export const de: typeof zhCN = { voiceActivationDesc: 'Lokal auf eine Aktivierungsphrase warten, um das Diktat zu starten.', voiceActivationLocalOnly: 'Die Erkennung bleibt auf diesem Gerät; Bereitschaftsaudio wird weder hochgeladen noch archiviert.', + voiceActivationMultimodalModels: + 'Hier das Sprachaktivierungsmodell vorbereiten. Das Diktat nutzt weiterhin den aktuellen multimodalen Dienst.', voiceActivationModelMissing: 'Das lokale Zipformer-Modell ist nicht installiert.', voiceActivationDownloadZipformer: 'Zipformer herunterladen', voiceActivationWakeEnabled: 'Aktivierungsphrase', diff --git a/openless-all/app/src/i18n/en.ts b/openless-all/app/src/i18n/en.ts index bd9786053..949e5e670 100644 --- a/openless-all/app/src/i18n/en.ts +++ b/openless-all/app/src/i18n/en.ts @@ -1607,6 +1607,8 @@ export const en: typeof zhCN = { voiceActivationDesc: 'Listen locally for a wake phrase to start dictation.', voiceActivationLocalOnly: 'Recognition stays on this device; standby audio is not uploaded or archived.', + voiceActivationMultimodalModels: + 'Prepare the voice activation model here. Dictation still uses your current multimodal service.', voiceActivationModelMissing: 'The local Zipformer model is not installed.', voiceActivationDownloadZipformer: 'Download Zipformer', voiceActivationWakeEnabled: 'Wake phrase', diff --git a/openless-all/app/src/i18n/es.ts b/openless-all/app/src/i18n/es.ts index 399cea2d1..0a0066fce 100644 --- a/openless-all/app/src/i18n/es.ts +++ b/openless-all/app/src/i18n/es.ts @@ -1631,6 +1631,8 @@ export const es: typeof zhCN = { voiceActivationDesc: 'Escucha localmente una frase de activación para iniciar el dictado.', voiceActivationLocalOnly: 'El reconocimiento permanece en este dispositivo; el audio de espera no se sube ni se archiva.', + voiceActivationMultimodalModels: + 'Prepara aquí el modelo de activación por voz. El dictado sigue usando el servicio multimodal actual.', voiceActivationModelMissing: 'El modelo local Zipformer no está instalado.', voiceActivationDownloadZipformer: 'Descargar Zipformer', voiceActivationWakeEnabled: 'Frase de activación', diff --git a/openless-all/app/src/i18n/fr.ts b/openless-all/app/src/i18n/fr.ts index 8b8319702..11e5bce16 100644 --- a/openless-all/app/src/i18n/fr.ts +++ b/openless-all/app/src/i18n/fr.ts @@ -1652,6 +1652,8 @@ export const fr: typeof zhCN = { voiceActivationDesc: 'Écouter localement une phrase d’activation pour démarrer la dictée.', voiceActivationLocalOnly: 'La reconnaissance reste sur cet appareil ; l’audio en veille n’est ni envoyé ni archivé.', + voiceActivationMultimodalModels: + 'Préparez ici le modèle d’activation vocale. La dictée utilise toujours votre service multimodal actuel.', voiceActivationModelMissing: 'Le modèle local Zipformer n’est pas installé.', voiceActivationDownloadZipformer: 'Télécharger Zipformer', voiceActivationWakeEnabled: 'Phrase d’activation', @@ -2302,7 +2304,7 @@ export const fr: typeof zhCN = { 'L’inférence locale est plus lente ; un appareil insuffisamment puissant peut omettre des mots.', localAsrFallbackLabel: 'Utiliser le modèle local si le cloud échoue', localAsrFallbackDesc: - 'En cas de connexion faible ou absente, transcrit l\'enregistrement avec le modèle local téléchargé au lieu d\'attendre les nouvelles tentatives du cloud. Nécessite un modèle local téléchargé.', + "En cas de connexion faible ou absente, transcrit l'enregistrement avec le modèle local téléchargé au lieu d'attendre les nouvelles tentatives du cloud. Nécessite un modèle local téléchargé.", qwen3Desc: 'Une fois activé, il remplace le fournisseur ASR.', sherpaDesc: 'Une fois activé, il remplace le fournisseur ASR.', foundryDesc: 'Une fois activé, il remplace le fournisseur ASR.', diff --git a/openless-all/app/src/i18n/ja.ts b/openless-all/app/src/i18n/ja.ts index 86096642d..ee0e5251d 100644 --- a/openless-all/app/src/i18n/ja.ts +++ b/openless-all/app/src/i18n/ja.ts @@ -1594,6 +1594,8 @@ export const ja: typeof zhCN = { voiceActivationDesc: '起動フレーズをローカルで待ち受けて音声入力を開始します。', voiceActivationLocalOnly: '認識はこの端末内で行われます。待機中の音声は送信・保存されません。', + voiceActivationMultimodalModels: + 'ここで音声起動用モデルを準備します。音声入力には引き続き現在のマルチモーダルサービスを使用します。', voiceActivationModelMissing: 'ローカル Zipformer モデルがインストールされていません。', voiceActivationDownloadZipformer: 'Zipformer をダウンロード', voiceActivationWakeEnabled: '起動フレーズ', diff --git a/openless-all/app/src/i18n/ko.ts b/openless-all/app/src/i18n/ko.ts index 65a58f4eb..b4a5b5b8f 100644 --- a/openless-all/app/src/i18n/ko.ts +++ b/openless-all/app/src/i18n/ko.ts @@ -1582,6 +1582,8 @@ export const ko: typeof zhCN = { voiceActivationDesc: '받아쓰기 시작 구문을 로컬에서 감지합니다.', voiceActivationLocalOnly: '인식은 이 기기에서만 수행되며 대기 오디오는 업로드하거나 보관하지 않습니다.', + voiceActivationMultimodalModels: + '여기에서 음성 활성화 모델을 준비합니다. 받아쓰기는 현재 멀티모달 서비스를 계속 사용합니다.', voiceActivationModelMissing: '로컬 Zipformer 모델이 설치되지 않았습니다.', voiceActivationDownloadZipformer: 'Zipformer 다운로드', voiceActivationWakeEnabled: '시작 구문', diff --git a/openless-all/app/src/i18n/zh-CN.ts b/openless-all/app/src/i18n/zh-CN.ts index 49eefb4e7..4f1c91d69 100644 --- a/openless-all/app/src/i18n/zh-CN.ts +++ b/openless-all/app/src/i18n/zh-CN.ts @@ -1455,6 +1455,7 @@ export const zhCN = { '仅在 Windows 桌面端启用本地语音命令唤醒听写。待机音频只在内存中处理，不会上传或保存；会增加功耗并保持麦克风指示灯亮起。', voiceActivationLocalOnly: '本地识别：说“开始语音录入”开始，说“完成语音录入”结束。需要已安装本地 Zipformer 模型。', + voiceActivationMultimodalModels: '此处准备语音唤醒模型，听写仍使用当前多模态服务。', voiceActivationModelMissing: '请先在“AI 服务与模型”的本地模型页准备 Zipformer。', voiceActivationDownloadZipformer: '前往 AI 服务与模型下载 Zipformer', voiceActivationWakeEnabled: '启用开始命令', @@ -1462,7 +1463,8 @@ export const zhCN = { voiceActivationFinishEnabled: '启用完成命令', voiceActivationFinishPhrase: '完成命令', voiceActivationTimeoutEnabled: '超时自动结束', - voiceActivationTimeoutSeconds: '超时秒数', muteDuringRecordingLabel: '录音时静音', + voiceActivationTimeoutSeconds: '超时秒数', + muteDuringRecordingLabel: '录音时静音', muteDuringRecordingDesc: '录音期间临时静音系统输出，避免扬声器回音。', audioCueLabel: '录音提示音', audioCueDesc: '按下热键开始录音时播放一段合成提示音，提醒已开始录音。胶囊隐藏时也会响。', diff --git a/openless-all/app/src/i18n/zh-TW.ts b/openless-all/app/src/i18n/zh-TW.ts index 8932972c9..276f9e2de 100644 --- a/openless-all/app/src/i18n/zh-TW.ts +++ b/openless-all/app/src/i18n/zh-TW.ts @@ -1526,6 +1526,7 @@ export const zhTW: typeof zhCN = { voiceActivationLabel: '語音啟用', voiceActivationDesc: '在本機等待喚醒詞以開始聽寫。', voiceActivationLocalOnly: '辨識只在此裝置上進行；待機音訊不會上傳或封存。', + voiceActivationMultimodalModels: '在此準備語音喚醒模型，聽寫仍使用目前的多模態服務。', voiceActivationModelMissing: '請先在「AI 服務與模型」的本機模型頁準備 Zipformer。', voiceActivationDownloadZipformer: '前往 AI 服務與模型下載 Zipformer', voiceActivationWakeEnabled: '喚醒詞', diff --git a/openless-all/app/src/pages/settings/RecordingInputSection.tsx b/openless-all/app/src/pages/settings/RecordingInputSection.tsx index a59461dd0..aadbf1ea5 100644 --- a/openless-all/app/src/pages/settings/RecordingInputSection.tsx +++ b/openless-all/app/src/pages/settings/RecordingInputSection.tsx @@ -174,7 +174,7 @@ export function RecordingInputSection({ }, [os]); useEffect(() => { - if (!isTauri || os !== 'win' || !prefs?.voiceActivationEnabled) { + if (os !== 'win' || !prefs?.voiceActivationEnabled) { voiceActivationStateObservedRef.current = false; setVoiceActivationModelMissing(false); return; diff --git a/openless-all/app/src/pages/settings/navigation.test.ts b/openless-all/app/src/pages/settings/navigation.test.ts index e8a5af0ec..d28c13e6e 100644 --- a/openless-all/app/src/pages/settings/navigation.test.ts +++ b/openless-all/app/src/pages/settings/navigation.test.ts @@ -207,8 +207,16 @@ assert.deepEqual( ); console.log('settings service view tests passed'); -assert.equal(visibleSettingsSections(false, 'harmony').some((item) => item.id === 'inputMethod'), false, 'Harmony wiring does not add an Android input-method navigation entry'); -assert.deepEqual(visibleAdvancedPages('harmony', 'harmony').map((item) => item.id), ['testingFeatures', 'vocabularyLearning', 'customSyncServer', 'debug'], 'Harmony retains shared utility settings without desktop coding-agent controls'); +assert.equal( + visibleSettingsSections(false, 'harmony').some((item) => item.id === 'inputMethod'), + false, + 'Harmony wiring does not add an Android input-method navigation entry', +); +assert.deepEqual( + visibleAdvancedPages('harmony', 'harmony').map((item) => item.id), + ['testingFeatures', 'vocabularyLearning', 'customSyncServer', 'debug'], + 'Harmony retains shared utility settings without desktop coding-agent controls', +); for (const os of ['mac', 'win', 'android'] as const) { assert.equal( visibleAdvancedPages('desktop', os).some((item) => @@ -230,3 +238,23 @@ assert.deepEqual( ['lessComputer', 'claudeConsole'], 'macOS exposes both testing features after opt-in', ); + +assert.equal( + resolveServiceView('models', serviceViews, true, 'win', true), + 'models', + 'voice activation can prepare models without changing the multimodal pipeline', +); +assert.equal( + isServiceViewInactive('models', true, 'win', true), + false, + 'the model button and resolved view agree for voice activation', +); +for (const os of ['mac', 'harmony', 'android'] as const) { + assert.equal( + isServiceViewInactive('models', true, os, true), + true, + 'voice activation does not unlock models on other platforms', + ); +} +assert.equal(isServiceViewInactive('llm', true, 'win', true), true, 'LLM settings remain inactive'); +assert.equal(isServiceViewInactive('asr', true, 'win', true), true, 'ASR settings remain inactive'); diff --git a/openless-all/app/src/pages/settings/navigation.ts b/openless-all/app/src/pages/settings/navigation.ts index 2e0794501..6be04caf6 100644 --- a/openless-all/app/src/pages/settings/navigation.ts +++ b/openless-all/app/src/pages/settings/navigation.ts @@ -107,7 +107,13 @@ export function availableServiceViews(localModels: boolean, os?: OS): ServiceVie ]; } -export function isServiceViewInactive(view: ServiceViewId, multimodal: boolean, os?: OS): boolean { +export function isServiceViewInactive( + view: ServiceViewId, + multimodal: boolean, + os?: OS, + voiceActivationModels = false, +): boolean { + if (view === 'models' && os === 'win' && voiceActivationModels) return false; if (view === 'models' && os === 'mac') return true; return multimodal ? view !== 'omni' && view !== 'connections' : view === 'omni'; } @@ -117,7 +123,10 @@ export function resolveServiceView( available: ServiceViewId[], multimodal: boolean, os?: OS, + voiceActivationModels = false, ): ServiceViewId { - const enabled = available.filter((view) => !isServiceViewInactive(view, multimodal, os)); + const enabled = available.filter( + (view) => !isServiceViewInactive(view, multimodal, os, voiceActivationModels), + ); return enabled.includes(requested) ? requested : enabled[0]; } diff --git a/openless-all/app/src/pages/settings/tabs.tsx b/openless-all/app/src/pages/settings/tabs.tsx index c9bf16fd5..77ae3ef56 100644 --- a/openless-all/app/src/pages/settings/tabs.tsx +++ b/openless-all/app/src/pages/settings/tabs.tsx @@ -64,7 +64,10 @@ function usePlatformCaps(): PlatformCapabilities | null { } // Recording & input: from recording to text insertion, plus phone input. -export function GeneralTab({ os, onOpenLocalModels }: { os?: OS; onOpenLocalModels?: () => void } = {}) { +export function GeneralTab({ + os, + onOpenLocalModels, +}: { os?: OS; onOpenLocalModels?: () => void } = {}) { const platformCaps = usePlatformCaps(); const showRemoteInput = platformCaps?.platform === 'desktop'; @@ -128,15 +131,21 @@ export function ServicesTab({ const showLocalModel = detectOS() !== 'mac' && platformCaps?.supportsLocalAsr === true; const multimodal = prefs?.multimodalPipelineEnabled === true && prefs.pipelineMode === 'multimodal'; - const [view, setView] = useState( - initialView ?? (asrNavigation ? 'asr' : 'llm'), - ); + const [view, setView] = useState(initialView ?? (asrNavigation ? 'asr' : 'llm')); useEffect(() => { if (initialView) setView(initialView); else if (asrNavigation) setView('asr'); }, [initialView, asrNavigation]); + const voiceActivationModels = + showLocalModel && detectOS() === 'win' && prefs?.voiceActivationEnabled === true; const views = availableServiceViews(showLocalModel, detectOS()); - const requestedView = resolveServiceView(view, views, multimodal, detectOS()); + const requestedView = resolveServiceView( + view, + views, + multimodal, + detectOS(), + voiceActivationModels, + ); const contentRef = useRef(null); const selectedView = usePageTransition(requestedView, contentRef, false); const pipelineRef = useRef(null); @@ -267,7 +276,12 @@ export function ServicesTab({ ); } - const inactive = isServiceViewInactive(id, multimodal, detectOS()); + const inactive = isServiceViewInactive( + id, + multimodal, + detectOS(), + voiceActivationModels, + ); const required = !inactive && (id === 'llm' || id === 'asr'); const configured = id === 'llm' ? requiredConfigured.llm : id === 'asr' ? requiredConfigured.asr : false; @@ -321,12 +335,25 @@ export function ServicesTab({
{/* The LocalModelPicker inside the channel editor jumps to this view through this context. */} setView('models') : null} + value={ + showLocalModel && (!multimodal || voiceActivationModels) + ? () => setView('models') + : null + } > {selectedView === 'llm' && } {selectedView === 'asr' && } {selectedView === 'omni' && } - {showLocalModel && selectedView === 'models' && } + {showLocalModel && selectedView === 'models' && ( + <> + {multimodal && voiceActivationModels && ( +

+ {t('settings.recording.voiceActivationMultimodalModels')} +

+ )} + + + )} {selectedView === 'connections' && ( <> From ab4d1fc363c0c4e51fd32f86ecbc914c48f2a070 Mon Sep 17 00:00:00 2001 From: Chris233 Date: Wed, 21 Oct 2026 11:25:52 +0800 Subject: [PATCH 12/12] test(voice-activation): initialize context for strict clippy --- .../app/crates/openless-core/src/dictation_engine.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/openless-all/app/crates/openless-core/src/dictation_engine.rs b/openless-all/app/crates/openless-core/src/dictation_engine.rs index 00c337f8f..01c777f60 100644 --- a/openless-all/app/crates/openless-core/src/dictation_engine.rs +++ b/openless-all/app/crates/openless-core/src/dictation_engine.rs @@ -3570,11 +3570,13 @@ mod tests { polisher.clone(), ) .with_polish_failure_policy(policy); - let mut context = DictationContext::default(); - context.voice_activation = Some(crate::voice_activation::VoiceActivationTextPolicy { - wake_phrase: "开始语音录入".into(), - finish_phrase: Some("完成语音录入".into()), - }); + let context = DictationContext { + voice_activation: Some(crate::voice_activation::VoiceActivationTextPolicy { + wake_phrase: "开始语音录入".into(), + finish_phrase: Some("完成语音录入".into()), + }), + ..Default::default() + }; let id = SessionId::new(); let progress = Arc::new(RecordingProgress::default()); engine