diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 65ab70c5a..4256269ac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,6 +19,11 @@ on: type: choice options: [all, macos] default: all + macos_runner: + description: macOS runner (Intel fallback when hosted ARM capacity is unavailable) + type: choice + options: [macos-latest, macos-15-intel] + default: macos-latest # Cancel old runs when the same PR is pushed repeatedly; workflow_dispatch is isolated by run_id. concurrency: @@ -399,7 +404,8 @@ jobs: fail-fast: false matrix: include: ${{ fromJSON(github.event_name == 'workflow_dispatch' && inputs.platform == 'macos' && '[{"os":"macos-latest","label":"macOS","preflight":false}]' || '[{"os":"macos-latest","label":"macOS","preflight":false},{"os":"windows-latest","label":"Windows","preflight":true}]') }} - runs-on: ${{ matrix.os }} + # Dispatch-only fallback; push/PR runs keep the default matrix and all test gates. + runs-on: ${{ matrix.label == 'macOS' && github.event_name == 'workflow_dispatch' && inputs.macos_runner || matrix.os }} timeout-minutes: 60 env: # The shared Core and full Tauri test binary compile together; limit concurrency @@ -601,7 +607,7 @@ jobs: # with stable, which is a superset of the MSRV promise). if: needs.changes.outputs.msrv != 'false' && needs.changes.outputs.msrv_needed == 'true' name: macOS Rust 1.99 MSRV - runs-on: macos-latest + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.macos_runner || 'macos-latest' }} timeout-minutes: 60 env: CARGO_BUILD_JOBS: 2 diff --git a/docs/issue-1223-aec-feasibility.md b/docs/issue-1223-aec-feasibility.md new file mode 100644 index 000000000..7d6b178a4 --- /dev/null +++ b/docs/issue-1223-aec-feasibility.md @@ -0,0 +1,159 @@ +# Issue #1223: playback-preserving acoustic echo cancellation + +**Status:** implementation and feasibility note; the baseline research below is supplemented by the feature-branch implementation. +**Baseline revision examined:** `de072a88373a222280a8c22948092a260880196c` (`upstream/beta`, 2.0.2 Beta.1). +**Issue:** [Open-Less/openless#1223](https://github.com/Open-Less/openless/issues/1223) + +## Executive conclusion + +Issue #1223 is technically feasible, but not as a platform-neutral switch around the existing microphone normalizer. Acoustic echo cancellation (AEC) needs two time-related signals: the microphone/capture signal and a render/playback reference, plus a usable estimate of the delay between them. The official WebRTC API exposes this explicitly through `ProcessReverseStream()` for render audio, `ProcessStream()` for capture audio, and `set_stream_delay_ms()` for their timing relationship ([WebRTC Audio Processing API](https://webrtc.googlesource.com/src/+/refs/heads/main/api/audio/audio_processing.h)). A microphone-only RMS gate or gain stage can reduce quiet noise, but it cannot reliably identify and subtract speech from a video. + +The practical recommendation is an **optional, capability-reported, best-effort feature** that leaves playback and volume untouched. Keep “Mute while recording” as the explicit fallback. Implement and validate one platform at a time rather than promising universal AEC: + +1. Add a capability probe, diagnostics, fallback behavior, and an offline/fixture test harness. +2. Prototype Windows with WASAPI loopback plus a proven AEC implementation such as WebRTC APM/AEC3. +3. Add a PipeWire-specific Linux path where the echo-cancel virtual sink/source can own the playback reference. +4. Investigate macOS VoiceProcessingIO or a supported Core Audio render tap; do not claim that VoiceProcessingIO cancels arbitrary applications whose playback is outside that unit. +5. Add Android’s native `AudioRecord`/`AcousticEchoCanceler` path only where the device reports the effect as available. + +## 1. Issue and GitHub history + +### User request + +The issue asks for an optional playback-suppression/AEC setting so a video can keep playing while OpenLess recognizes the user’s speech. It explicitly accepts device- and environment-dependent results and asks that supported platforms and limitations be documented ([issue body](https://github.com/Open-Less/openless/issues/1223)). + +The two issue comments are: + +- On 2026-10-17, `HKLHaoBin` explained that AEC was omitted from the earlier audio-enhancement work because of CPU cost, additional dependencies, and implementation/maintenance complexity, and offered to try an implementation ([comment 6237559767](https://github.com/Open-Less/openless/issues/1223#issuecomment-6237559767)). +- On 2026-10-17, the reporter said a closed-source competitor appears to preserve video playback while preventing the video voice from entering recognition ([comment 6237637338](https://github.com/Open-Less/openless/issues/1223#issuecomment-6237637338)). This is a user observation, not an independently verified technical claim. + +`gh api --paginate repos/Open-Less/openless/issues/1223/events` returned an empty list. The issue timeline contained only the two `commented` entries above; it contained no cross-reference, linked-commit, or pull-request event. `gh search prs 1223 --repo Open-Less/openless` and `gh search commits 1223 --repo Open-Less/openless` returned no results at investigation time. The issue body does explicitly reference [#269](https://github.com/Open-Less/openless/issues/269), so that issue and its implementation PR are relevant context, but they are not direct links from the #1223 timeline. + +### Related repository history + +- [#269](https://github.com/Open-Less/openless/issues/269) requested pausing and restoring background media. It is closed. +- [PR #274](https://github.com/Open-Less/openless/pull/274) implemented the current “Mute while recording” behavior. Its description says the implementation mutes the system output instead of pausing each media application. Its automated review recorded the important limitation: media continues advancing while muted, and macOS/Linux needed real-device verification. The resulting commits were `9511eb6`, `bb0fefc`, and `78a3846`. +- [PR #1194](https://github.com/Open-Less/openless/pull/1194), merged for #1026, added opt-in RMS/downmix microphone enhancement and explicitly documented that it does **not** include echo cancellation. Its merge commit is `44972015349cac3a52c4a28190675554a8986fcb`. + +## 2. Current audio architecture + +The following bullets describe the committed code at the baseline revision above; they explain why the feature branch adds both a shared DSP seam and host-specific reference capture: + +- `openless-all/app/crates/openless-core/src/audio.rs` owns the canonical 16 kHz, mono, signed-16-bit PCM normalization contract. The baseline `AudioProcessingConfig`/`AudioProcessor` path implemented optional microphone enhancement/noise gating but had no render-reference input or AEC state ([file at baseline revision](https://github.com/Open-Less/openless/blob/de072a88373a222280a8c22948092a260880196c/openless-all/app/crates/openless-core/src/audio.rs)). +- `openless-all/app/src-tauri/src/recorder.rs` opens a `cpal` input stream, converts the selected microphone format to `f32`, then sends microphone callback data through the processor/normalizer. The baseline had no corresponding render/loopback stream in this adapter ([file](https://github.com/Open-Less/openless/blob/de072a88373a222280a8c22948092a260880196c/openless-all/app/src-tauri/src/recorder.rs)). +- `openless-all/app/linux-egui/src/audio.rs` likewise owned a `cpal` microphone input stream and forwarded normalized chunks. The baseline had no playback-monitor/reference capture path ([file](https://github.com/Open-Less/openless/blob/de072a88373a222280a8c22948092a260880196c/openless-all/app/linux-egui/src/audio.rs)). +- `openless-all/app/src-tauri/src/audio_mute.rs` is an output-mute guard: Windows uses `IAudioEndpointVolume`, macOS invokes `osascript` to set output mute, and Linux tries `wpctl` then `pactl`. It records and restores mute state, but it does not observe or process the render signal ([file](https://github.com/Open-Less/openless/blob/de072a88373a222280a8c22948092a260880196c/openless-all/app/src-tauri/src/audio_mute.rs)). +- `openless-all/app/crates/openless-core/src/ports.rs` states that each host owns device selection, permissions, resampling, and native stream implementation. This is the seam used by the feature branch for host-specific render-reference capture while keeping cancellation inside the existing PCM pipeline ([file](https://github.com/Open-Less/openless/blob/de072a88373a222280a8c22948092a260880196c/openless-all/app/crates/openless-core/src/ports.rs)). + +### Feature-branch implementation boundary + +The feature branch adds a session-frozen, opt-in `echo_cancellation_enabled` preference, excluded from cloud sync because it is host/device dependent. `ReferenceAudioBuffer` keeps at most two seconds of downmixed render samples and bypasses cancellation on startup starvation or underrun. `AudioProcessor::process_with_reference` resamples microphone and reference data to the canonical rate, applies a bounded adaptive canceller before the existing microphone enhancement, and preserves the old microphone-only path when the switch is off or a reference is unavailable. + +The Tauri adapter uses cpal 0.18’s render-endpoint input path on desktop targets where the backend supplies loopback (WASAPI and cpal’s macOS CoreAudio loopback support). The Linux egui adapter attempts PipeWire sink capture and PulseAudio monitor sources; ALSA remains microphone-only. All reference samples remain transient and are not archived or transmitted. Any unavailable, invalidated, or runtime-failed reference leaves ordinary recording running without changing system volume or media playback state. The independent `mute_during_recording` option remains unchanged and is still the explicit output-mute fallback. + +## 3. What AEC requires + +### Render reference and timing are mandatory inputs + +The official WebRTC Audio Processing API describes approximately 10 ms linear PCM frames, sends the reverse/render stream to `ProcessReverseStream()`, sends microphone/capture audio to `ProcessStream()`, and exposes a stream-delay setting for the timing relationship. Its AEC configuration is therefore fundamentally different from a capture-only gain or noise gate ([WebRTC API header](https://webrtc.googlesource.com/src/+/refs/heads/main/api/audio/audio_processing.h)). WebRTC’s AEC3 interface similarly describes 10 ms band-split frames, a render-analysis path that stores the render signal, and a capture-processing path that removes estimated echo ([AEC3 header](https://webrtc.googlesource.com/src/+/refs/heads/main/modules/audio_processing/aec3/echo_canceller3.h)). + +A FIFO of samples is not automatically a valid reference. The render and capture clocks can differ, the acoustic path has an unknown delay and impulse response, and output routing can change. The implementation must either use a platform voice-processing unit or maintain timestamps/delay and bounded drift handling in a software AEC path. + +### Why the existing enhancement is insufficient + +PR #1194’s enhancement is a stateful RMS/noise-floor gate and gain/soft-clip stage. It has no render input, delay estimate, adaptive echo filter, or double-talk reference. It should remain separately configurable; enabling it must not be presented as enabling AEC ([PR #1194](https://github.com/Open-Less/openless/pull/1194)). + +## 4. Platform feasibility and constraints + +| Platform | Official capability | Feasibility for OpenLess | Main limitation / required adapter | +| --- | --- | --- | --- | +| Windows | WASAPI loopback captures the stream played by a render endpoint; Microsoft says loopback exists primarily to support AEC. It is shared-mode only, and endpoint/hardware/DRM behavior matters ([Loopback Recording](https://learn.microsoft.com/en-us/windows/win32/coreaudio/loopback-recording)). Windows also exposes Communications/Speech processing modes and effect discovery, but modes are driver-defined and optional ([Audio Signal Processing Modes](https://learn.microsoft.com/en-us/windows-hardware/drivers/audio/audio-signal-processing-modes); [APO architecture](https://learn.microsoft.com/en-us/windows-hardware/drivers/audio/audio-processing-object-architecture)). | **Best first desktop prototype.** Passive WASAPI loopback can preserve audible playback while supplying a reference to WebRTC APM/AEC3. | The current `cpal` input path does not expose this render-reference lifecycle. Use a Windows-specific WASAPI adapter, select/track the actual render endpoint, handle shared-mode and protected-content gaps, and avoid assuming a driver APO is present. | +| macOS | Apple’s VoiceProcessingIO unit provides voice processing such as echo cancellation and AGC ([VoiceProcessingIO](https://developer.apple.com/documentation/audiotoolbox/kaudiounitsubtype_voiceprocessingio)). | **Feasible for a controlled voice-I/O route; uncertain for arbitrary system playback.** | VoiceProcessingIO expects application playback through its output path. OpenLess currently mutes the system endpoint and does not own every application’s playback graph. A separate supported Core Audio render tap/monitor path and permissions would need investigation; otherwise report unsupported and retain fallback. Voice processing can change speech-oriented audio characteristics, so compare near-end fidelity. | +| Linux (PipeWire) | PipeWire’s official echo-cancel module creates virtual echo-cancel capture/playback nodes and streams. Playback sent to the echo-cancel sink supplies the reference; `monitor.mode` can capture a default sink monitor. The documented backend is `aec/libspa-aec-webrtc` ([PipeWire Echo Cancel](https://docs.pipewire.org/page_module_echo_cancel.html)). | **Feasible when PipeWire can own or monitor the playback route.** | The current `linux-egui` recorder is a generic `cpal` input path. Add a PipeWire-specific node/monitor adapter or use the virtual source/sink; do not assume an ALSA/Pulse/default-device capture has a synchronized monitor. Device/route changes and non-PipeWire systems require capability fallback. | +| Android | `AcousticEchoCanceler` was added in API 16, is device-dependent, must be checked with `isAvailable()`, and attaches to the `AudioRecord` audio session; `create()` may return `null` ([Android API reference](https://developer.android.com/reference/android/media/audiofx/AcousticEchoCanceler)). | **Feasible only through a native Android capture path on supported devices.** | The current shared recorder uses `cpal`; the repository’s Kotlin code handles microphone permission but does not expose an `AudioRecord` session/effect adapter. Add native session ownership, route-specific tests, and a clear unavailable state. | +| Portable software fallback | WebRTC APM/AEC3 is a cross-platform algorithmic option, but its official API still requires render and capture frames plus timing ([APM header](https://webrtc.googlesource.com/src/+/refs/heads/main/api/audio/audio_processing.h)). | **Feasible only where each host can provide a reliable render/monitor reference.** | A portable library does not create a system loopback source. Adding it without a platform reference would produce an apparent setting with weak or misleading results. It also adds binary size, CPU, latency, and real-time integration work; these are consistent with the complexity concerns recorded in #1223’s first comment. | + +The W3C `echoCancellation` constraint is a useful model for product wording, not a guarantee that every source supports AEC. The specification says `true` must attempt at least remote-only cancellation, while `all` must attempt all system-played sound; it also requires capabilities to report only `false` when cancellation is unavailable ([Media Capture and Streams, `echoCancellation`](https://w3c.github.io/mediacapture-main/#def-constraint-echoCancellation)). OpenLess should expose an equivalent capability/unsupported result instead of claiming universal support. + +## 5. Risks and mitigations + +1. **Wrong or missing reference.** A default render monitor may not correspond to the selected output, a Bluetooth route may add variable delay, and applications may use another sink. Mitigate with endpoint identity, timestamps, route-change invalidation, reference starvation bypass, and a visible capability/diagnostic state. +2. **Double-talk damage.** The user and video speaker can talk simultaneously. AEC must protect near-end speech; use a proven double-talk strategy and measure near-end word recall, not only echo reduction. +3. **Clock drift and buffering.** Separate render/capture clocks eventually move out of alignment. Bound queues, monitor underflow/overflow, and resample or adapt delay rather than draining an un-timestamped FIFO indefinitely. +4. **Real-time callback safety.** Audio callbacks must not block on a long DSP operation, allocate without bounds, or wait on a worker. Keep callback work bounded and use a lock-free or carefully bounded handoff where the platform requires it. +5. **Native processing conflicts.** Windows driver APOs and Android/macOS voice processing may already apply AEC/AGC/NS. Discover/record active effects where the platform permits and avoid unintentionally stacking processors ([Windows effect discovery guidance](https://learn.microsoft.com/en-us/windows-hardware/drivers/audio/audio-signal-processing-modes)). +6. **Protected or inaccessible playback.** Microsoft documents that loopback behavior depends on endpoint capabilities and that protected content may not be capturable ([WASAPI loopback](https://learn.microsoft.com/en-us/windows/win32/coreaudio/loopback-recording)). Treat reference loss as unsupported/bypass, never as silence that could train the canceller incorrectly. +7. **Privacy.** A render reference can contain all system audio, not only the video selected by the user. Keep it in-process, do not archive or transmit it, and document that it is used only transiently to process microphone PCM. +8. **Failure cleanup.** Existing mute behavior is best-effort and uses a guard restored on drop ([`audio_mute.rs`](https://github.com/Open-Less/openless/blob/de072a88373a222280a8c22948092a260880196c/openless-all/app/src-tauri/src/audio_mute.rs)). AEC startup, route changes, cancellation, recorder errors, and shutdown must preserve the same cleanup guarantees without muting playback when AEC is selected. + +## 6. Phased implementation recommendation + +### Phase 0: contract and observability + +- Add a session-frozen AEC mode separate from `mute_during_recording` and microphone enhancement: `off`, `native`, or `software` is preferable to a boolean if the UI needs to explain fallback. +- Define a host capability response containing platform, reference source, endpoint identity, availability, and reason for unavailability. +- Preserve current mute as an explicit fallback, never silently mute when the user selected playback-preserving AEC. +- Add deterministic tests for PCM framing, reference underflow/overflow, reset, cancellation, and “no reference means bypass.” +- Add an offline fixture containing far-end audio, a known delayed/filtered echo, near-end speech, and double-talk. Test both echo reduction and preservation of near-end samples. + +### Phase 1: Windows proof of value + +- Implement a Windows WASAPI shared-mode loopback capture on the render endpoint used for playback. +- Feed 10 ms render and microphone frames into WebRTC APM/AEC3, preserving stream delay and route identity. +- Measure speaker, wired, Bluetooth, default-device change, device unplug, Remote Desktop, and protected-content cases before exposing the setting broadly. +- Keep AEC off by default until real-device results meet the acceptance gates below. + +### Phase 2: Linux PipeWire + +- Detect PipeWire and create/use the documented echo-cancel source/sink or monitor path. Keep the generic `cpal` path as fallback. +- Test default sink monitor, per-application routing, route changes, PulseAudio compatibility mode, and systems without PipeWire. +- Do not advertise “Linux support” as a single guarantee; report the backend and route capability. + +### Phase 3: macOS and Android + +- macOS: prototype VoiceProcessingIO only for a controlled voice-processing route; separately investigate a supported system render tap and permission requirements. If arbitrary app playback cannot be referenced reliably, mark the feature unavailable rather than muting or claiming cancellation. +- Android: add a Kotlin/native `AudioRecord` adapter, attach `AcousticEchoCanceler` to the correct session, check both availability and `create()` result, and test speaker/headset/Bluetooth/device-vendor combinations. + +### Phase 4: product hardening + +- Add per-platform support text, diagnostic logging without raw audio, route-change handling, power/CPU telemetry, and a one-click AEC sample test. +- Keep the existing mute option and clarify that it is the deterministic fallback; keep microphone enhancement separately described as noise suppression, not AEC. + +## 7. Proposed acceptance and test matrix + +The following are proposed product gates, not guarantees stated by platform vendors. Record the exact device, OS, audio route, sample rates, output endpoint, and AEC mode for every run. + +| Test area | Required cases | Evidence / gate | +| --- | --- | --- | +| Baseline and toggle semantics | AEC off; enhancement off/on; mute off/on; AEC setting changed between sessions | Existing behavior is unchanged when AEC is off. AEC does not call endpoint mute or change system volume. Mute restores the user’s original state on normal stop, cancel, startup failure, runtime fault, and shutdown. | +| Far-end-only | Video speech, music, tones, silence at microphone; several levels and delays | Compare processed and unprocessed microphone recordings. False transcription rate decreases materially; report ERLE/SNR or another reproducible echo metric. | +| Near-end-only | User speech with no playback; quiet/normal/loud speech; short words and sustained speech | Near-end word recall/WER remains within an agreed regression budget; no audible clipping or gate-induced loss in the archived processed PCM. | +| Double talk | Video speech overlaps user speech at beginning, middle, and end of utterances | Near-end speech remains intelligible and transcribes; cancellation does not treat the user as echo. | +| Route/device matrix | Built-in speaker + mic, USB mic, wired headset, Bluetooth headset/speaker, output and input changes during capture | Each combination is either validated or reports unsupported/bypassed. No stale reference after endpoint change. | +| Platform matrix | Windows WASAPI loopback; PipeWire source/sink and monitor mode; macOS VoiceProcessingIO/tap candidate; Android supported and unsupported devices | Capability result identifies backend and reason. Unsupported platforms keep working without a crash and without unexpected mute. | +| Faults and permissions | Missing microphone permission, missing render permission/device, protected content, route unplug, stream underflow/overflow | Recording remains usable through documented bypass/fallback; no raw render audio is persisted or transmitted; cleanup is verified. | +| Timing and performance | Start/stop latency, reference starvation, clock drift, 10-minute capture, repeated sessions | Bounded queue memory, no callback watchdog failures, agreed p95 CPU/latency/power budget, and no unbounded growth. | +| Regression | History playback/retranscription, streaming ASR, QA/selection voice, archive retention, concurrent lifecycle races | Existing recorder/core contract tests pass; processed archive format remains canonical 16 kHz mono PCM WAV where enabled. | + +## Current implementation decision + +The feature branch follows the best-effort design above: AEC is opt-in, session-frozen, capability-by-fallback, and independent from output muting. The shared software canceller is covered by deterministic signal tests, while host-specific loopback and monitor paths remain subject to platform/device validation. The setting must not be presented as a guarantee of perfect cancellation. Windows WASAPI and macOS CoreAudio loopback are the primary desktop paths; PipeWire/Pulse monitor capture is attempted on Linux; unsupported routes, ALSA-only capture, mobile builds, protected content, and unavailable monitors continue with ordinary microphone processing. A route changed during an active session is not dynamically rebound by this phase; users must stop and start a new dictation session for the new output route to be selected, and the route-change acceptance cases remain hardware-validation work. + +The remaining acceptance work is real-device validation of endpoint identity, route changes, Bluetooth latency, double-talk speech preservation, and long-running clock drift. Those hardware-only results should be recorded per platform rather than inferred from the offline tests. + +## Primary sources + +- [Issue #1223 and comments](https://github.com/Open-Less/openless/issues/1223) +- [PR #274: mute system output](https://github.com/Open-Less/openless/pull/274) +- [PR #1194: lightweight microphone enhancement](https://github.com/Open-Less/openless/pull/1194) +- [WebRTC Audio Processing API](https://webrtc.googlesource.com/src/+/refs/heads/main/api/audio/audio_processing.h) +- [WebRTC EchoCanceller3](https://webrtc.googlesource.com/src/+/refs/heads/main/modules/audio_processing/aec3/echo_canceller3.h) +- [Microsoft WASAPI Loopback Recording](https://learn.microsoft.com/en-us/windows/win32/coreaudio/loopback-recording) +- [Microsoft Audio Signal Processing Modes](https://learn.microsoft.com/en-us/windows-hardware/drivers/audio/audio-signal-processing-modes) +- [Microsoft Audio Processing Object Architecture](https://learn.microsoft.com/en-us/windows-hardware/drivers/audio/audio-processing-object-architecture) +- [Apple VoiceProcessingIO](https://developer.apple.com/documentation/audiotoolbox/kaudiounitsubtype_voiceprocessingio) +- [PipeWire Echo Cancel module](https://docs.pipewire.org/page_module_echo_cancel.html) +- [Android AcousticEchoCanceler](https://developer.android.com/reference/android/media/audiofx/AcousticEchoCanceler) +- [W3C `echoCancellation` constraint](https://w3c.github.io/mediacapture-main/#def-constraint-echoCancellation) + +*The baseline history and primary-source citations were collected during the read-only investigation; the feature-branch implementation notes above describe the changes now under test.* diff --git a/openless-all/app/crates/openless-core/src/api.rs b/openless-all/app/crates/openless-core/src/api.rs index 670275f71..6acb2b752 100644 --- a/openless-all/app/crates/openless-core/src/api.rs +++ b/openless-all/app/crates/openless-core/src/api.rs @@ -5288,6 +5288,7 @@ impl OpenLessBackend { pre_roll_ms: 500, audio_processing: crate::audio::AudioProcessingConfig { enhancement_enabled: preferences.audio_enhancement_enabled, + echo_cancellation_enabled: preferences.echo_cancellation_enabled, }, }; match self.deps.dictation_engine.arm_audio_capture(request) { diff --git a/openless-all/app/crates/openless-core/src/audio.rs b/openless-all/app/crates/openless-core/src/audio.rs index acfc652e4..52f6b4781 100644 --- a/openless-all/app/crates/openless-core/src/audio.rs +++ b/openless-all/app/crates/openless-core/src/audio.rs @@ -1,7 +1,11 @@ //! Framework-independent audio normalization used by native recorder adapters. +use std::collections::VecDeque; +use std::sync::Mutex; + pub const DICTATION_SAMPLE_RATE: u32 = 16_000; const LEVEL_RMS_GAIN: f32 = 4.0; +const REFERENCE_BUFFER_MAX_SECONDS: usize = 2; #[derive(Debug, Clone, PartialEq)] pub struct NormalizedPcmChunk { @@ -13,6 +17,84 @@ pub struct NormalizedPcmChunk { #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct AudioProcessingConfig { pub enhancement_enabled: bool, + /// Use a native render/monitor reference when the host can provide one. + /// This is deliberately independent from output muting and is best effort: + /// a recorder remains usable when the platform has no monitor stream. + pub echo_cancellation_enabled: bool, +} + +/// Bounded render samples shared by a native render callback and the processing worker. +/// +/// The callback only performs a short mutex-protected append and never waits for the +/// microphone worker. When the buffer is empty, the worker bypasses cancellation for +/// that microphone chunk instead of treating missing reference samples as silence. +#[derive(Debug)] +pub struct ReferenceAudioBuffer { + sample_rate: u32, + samples: Mutex>, +} + +impl ReferenceAudioBuffer { + pub fn new(sample_rate: u32) -> Self { + Self { + sample_rate, + samples: Mutex::new(VecDeque::with_capacity( + sample_rate as usize * REFERENCE_BUFFER_MAX_SECONDS, + )), + } + } + + pub fn sample_rate(&self) -> u32 { + self.sample_rate + } + + pub fn push_interleaved(&self, samples: &[f32], channels: usize) { + if channels == 0 || self.sample_rate == 0 || samples.is_empty() { + return; + } + let Ok(mut queue) = self.samples.try_lock() else { + return; + }; + let capacity = self.sample_rate as usize * REFERENCE_BUFFER_MAX_SECONDS; + for frame in samples.chunks_exact(channels) { + let mono = frame + .iter() + .map(|sample| if sample.is_finite() { *sample } else { 0.0 }) + .sum::() + / channels as f32; + if queue.len() == capacity { + queue.pop_front(); + } + queue.push_back(mono); + } + } + + /// Take the render duration corresponding to a microphone callback. + /// Returns `None` when the render stream has not produced enough samples yet. + pub fn take_for_input(&self, input_frames: usize, input_sample_rate: u32) -> Option> { + if input_frames == 0 || input_sample_rate == 0 || self.sample_rate == 0 { + return None; + } + let needed = ((input_frames as u64 * self.sample_rate as u64) + .saturating_add(input_sample_rate as u64 - 1) + / input_sample_rate as u64) as usize; + let mut queue = self.samples.lock().expect("reference buffer lock poisoned"); + if queue.len() < needed { + // A partial queue represents an underrun, not silence. Discard it so the next + // successful read cannot feed stale pre-gap render samples into the canceller. + queue.clear(); + return None; + } + Some(queue.drain(..needed).collect()) + } + + #[cfg(test)] + fn len(&self) -> usize { + self.samples + .lock() + .expect("reference buffer lock poisoned") + .len() + } } /// Small, deterministic microphone enhancement that does not require a model @@ -210,24 +292,119 @@ impl AudioEnhancer { } } +const ECHO_FILTER_TAPS: usize = 1_024; +const ECHO_ADAPTATION_RATE: f32 = 0.12; +const ECHO_DOUBLE_TALK_RATIO: f32 = 2.5; +const ECHO_MIN_REFERENCE_POWER: f32 = 0.000_001; + +/// Conservative normalized-LMS echo canceller for a mono render reference. +/// +/// The filter is intentionally bounded and stateful. It learns the acoustic path while +/// the render signal is present, but pauses adaptation when near-end energy is clearly +/// stronger than the estimated echo (double talk). It is not a replacement for a +/// platform voice-processing stack; hosts therefore treat it as an optional best-effort +/// processor and bypass it when the render reference is unavailable. +#[derive(Debug, Clone)] +struct EchoCanceller { + coefficients: Vec, + history: Vec, + history_cursor: usize, +} + +impl Default for EchoCanceller { + fn default() -> Self { + Self { + coefficients: vec![0.0; ECHO_FILTER_TAPS], + history: vec![0.0; ECHO_FILTER_TAPS], + history_cursor: 0, + } + } +} + +impl EchoCanceller { + fn reset(&mut self) { + self.coefficients.fill(0.0); + self.history.fill(0.0); + self.history_cursor = 0; + } + + fn process(&mut self, microphone: &[f32], reference: &[f32]) -> Vec { + if microphone.is_empty() || reference.is_empty() { + return microphone.to_vec(); + } + let mut output = Vec::with_capacity(microphone.len()); + for (index, &sample) in microphone.iter().enumerate() { + let Some(&reference_sample) = reference.get(index) else { + // A resampler may produce one fewer sample at a chunk boundary. Do not + // repeat the last render value across the missing tail; preserve the + // microphone sample and let the next reference chunk re-establish alignment. + output.push(sample); + continue; + }; + self.history[self.history_cursor] = reference_sample; + let mut estimate = 0.0; + let mut reference_power = 0.0; + for tap in 0..ECHO_FILTER_TAPS { + let history_index = + (self.history_cursor + ECHO_FILTER_TAPS - tap) % ECHO_FILTER_TAPS; + let value = self.history[history_index]; + estimate += self.coefficients[tap] * value; + reference_power += value * value; + } + let error = sample - estimate; + let near_power = sample * sample; + let should_adapt = reference_power > ECHO_MIN_REFERENCE_POWER + && (estimate.abs() < 0.01 + || estimate * estimate * ECHO_DOUBLE_TALK_RATIO >= near_power + || near_power < ECHO_MIN_REFERENCE_POWER); + if should_adapt { + let normalization = reference_power + 0.000_01; + for tap in 0..ECHO_FILTER_TAPS { + let history_index = + (self.history_cursor + ECHO_FILTER_TAPS - tap) % ECHO_FILTER_TAPS; + self.coefficients[tap] += + ECHO_ADAPTATION_RATE * error * self.history[history_index] / normalization; + self.coefficients[tap] = self.coefficients[tap].clamp(-2.0, 2.0); + } + } + output.push(if error.is_finite() { error } else { 0.0 }); + self.history_cursor = (self.history_cursor + 1) % ECHO_FILTER_TAPS; + } + output + } +} + /// Shared stateful processor used by native microphone hosts. #[derive(Debug)] pub struct AudioProcessor { normalizer: PcmNormalizer, + microphone_resampler: PcmNormalizer, + reference_resampler: PcmNormalizer, enhancer: AudioEnhancer, + echo_canceller: Option, } impl AudioProcessor { pub fn new(config: AudioProcessingConfig) -> Self { Self { normalizer: PcmNormalizer::default(), + microphone_resampler: PcmNormalizer::default(), + reference_resampler: PcmNormalizer::default(), enhancer: AudioEnhancer::new(config), + echo_canceller: config + .echo_cancellation_enabled + .then(EchoCanceller::default), } } pub fn reset(&mut self) { self.normalizer = PcmNormalizer::default(); + self.microphone_resampler = PcmNormalizer::default(); + self.reference_resampler = PcmNormalizer::default(); self.enhancer.reset(); + if let Some(canceller) = &mut self.echo_canceller { + canceller.reset(); + } } pub fn process( @@ -247,6 +424,59 @@ impl AudioProcessor { self.normalizer .process(&enhanced, channels, input_sample_rate) } + + /// Process microphone audio with an optional native render reference. + /// + /// A missing reference is a normal condition during startup or on platforms without + /// monitor capture; in that case the microphone follows the existing processing path. + pub fn process_with_reference( + &mut self, + interleaved: &[f32], + channels: usize, + input_sample_rate: u32, + reference: Option<&[f32]>, + reference_sample_rate: u32, + ) -> Option { + let Some(canceller) = self.echo_canceller.as_mut() else { + return self.process(interleaved, channels, input_sample_rate); + }; + let Some(reference) = reference.filter(|samples| !samples.is_empty()) else { + // A reference gap invalidates the learned delay/history relationship. Bypass this + // microphone chunk and relearn from fresh render samples instead of carrying stale + // pre-gap state into the next cancellation attempt. + canceller.reset(); + self.microphone_resampler = PcmNormalizer::default(); + self.reference_resampler = PcmNormalizer::default(); + return self.process(interleaved, channels, input_sample_rate); + }; + let microphone = + self.microphone_resampler + .process_f32(interleaved, channels, input_sample_rate); + let render = self + .reference_resampler + .process_f32(reference, 1, reference_sample_rate); + if microphone.is_empty() || render.is_empty() { + // The resampler can also reject a malformed/zero-rate reference. Reset the + // canceller before the microphone-only fallback for the same gap semantics. + canceller.reset(); + self.microphone_resampler = PcmNormalizer::default(); + self.reference_resampler = PcmNormalizer::default(); + return self.process(interleaved, channels, input_sample_rate); + } + if render.len() < microphone.len() { + // A short resampled reference is an alignment gap, not permission to repeat + // the final render sample. Bypass this chunk and relearn from a clean history. + canceller.reset(); + self.microphone_resampler = PcmNormalizer::default(); + self.reference_resampler = PcmNormalizer::default(); + return self.process(interleaved, channels, input_sample_rate); + } + let mut cancelled = canceller.process(µphone, &render); + self.enhancer + .process_in_place_with_format(&mut cancelled, 1, DICTATION_SAMPLE_RATE); + self.normalizer + .process(&cancelled, 1, DICTATION_SAMPLE_RATE) + } } /// Stateful interleaved-audio normalizer. @@ -357,11 +587,7 @@ impl PcmNormalizer { channels: usize, input_sample_rate: u32, ) -> Option { - if interleaved.is_empty() || channels == 0 || input_sample_rate == 0 { - return None; - } - let mono = downmix_to_mono(interleaved, channels); - let resampled = self.resample(&mono, input_sample_rate, DICTATION_SAMPLE_RATE); + let resampled = self.process_f32(interleaved, channels, input_sample_rate); if resampled.is_empty() { return None; } @@ -372,6 +598,19 @@ impl PcmNormalizer { }) } + fn process_f32( + &mut self, + interleaved: &[f32], + channels: usize, + input_sample_rate: u32, + ) -> Vec { + if interleaved.is_empty() || channels == 0 || input_sample_rate == 0 { + return Vec::new(); + } + let mono = downmix_to_mono(interleaved, channels); + self.resample(&mono, input_sample_rate, DICTATION_SAMPLE_RATE) + } + fn resample(&mut self, samples: &[f32], source_rate: u32, target_rate: u32) -> Vec { if samples.is_empty() { return Vec::new(); @@ -477,6 +716,7 @@ mod tests { fn enhancement_boosts_quiet_samples_and_soft_clips_loud_samples() { let mut enhancer = AudioEnhancer::new(AudioProcessingConfig { enhancement_enabled: true, + ..Default::default() }); let mut samples = [0.5, -0.5, 1.0, -1.0]; enhancer.process_in_place(&mut samples); @@ -490,6 +730,7 @@ mod tests { fn enhancement_replaces_non_finite_samples_and_stays_in_range() { let mut enhancer = AudioEnhancer::new(AudioProcessingConfig { enhancement_enabled: true, + ..Default::default() }); let mut samples = [f32::NAN, f32::INFINITY, f32::NEG_INFINITY, 2.0, -2.0]; enhancer.process_in_place(&mut samples); @@ -502,6 +743,7 @@ mod tests { fn noise_suppression_attenuates_quiet_frames_and_recovers_for_speech() { let mut enhancer = AudioEnhancer::new(AudioProcessingConfig { enhancement_enabled: true, + ..Default::default() }); let quiet = [0.004_f32; 160]; let mut processed_quiet = quiet; @@ -520,6 +762,7 @@ mod tests { fn noise_suppression_reset_restores_initial_gain_state() { let mut enhancer = AudioEnhancer::new(AudioProcessingConfig { enhancement_enabled: true, + ..Default::default() }); let quiet = [0.004_f32; 160]; let mut settled = quiet; @@ -536,6 +779,7 @@ mod tests { fn noise_suppression_keeps_stereo_frame_processing_consistent() { let mut enhancer = AudioEnhancer::new(AudioProcessingConfig { enhancement_enabled: true, + ..Default::default() }); let mut samples = [0.004_f32; 320]; enhancer.process_in_place_with_format(&mut samples, 2, DICTATION_SAMPLE_RATE); @@ -547,6 +791,7 @@ mod tests { fn noise_suppression_does_not_learn_sustained_speech_as_noise() { let mut enhancer = AudioEnhancer::new(AudioProcessingConfig { enhancement_enabled: true, + ..Default::default() }); for _ in 0..1_000 { let mut speech = [0.2_f32; 160]; @@ -559,6 +804,7 @@ mod tests { fn noise_suppression_uses_downmixed_rms_for_stereo_input() { let mut enhancer = AudioEnhancer::new(AudioProcessingConfig { enhancement_enabled: true, + ..Default::default() }); let mut out_of_phase = [0.2_f32; 320]; for frame in out_of_phase.as_chunks_mut::<2>().0 { @@ -587,6 +833,7 @@ mod tests { let quiet = [0.004_f32; 160]; let mut processor = AudioProcessor::new(AudioProcessingConfig { enhancement_enabled: true, + ..Default::default() }); let mut output = None; for _ in 0..24 { @@ -614,6 +861,7 @@ mod tests { fn processor_enhancement_preserves_canonical_output_contract() { let mut processor = AudioProcessor::new(AudioProcessingConfig { enhancement_enabled: true, + ..Default::default() }); let output = processor .process(&[0.25, -0.25, 0.5, -0.5], 1, DICTATION_SAMPLE_RATE) @@ -636,6 +884,134 @@ mod tests { assert_eq!(decode(&output.pcm_i16_le), vec![0, 16383, 32767]); } + #[test] + fn reference_buffer_bounds_and_matches_input_duration() { + let buffer = ReferenceAudioBuffer::new(8_000); + buffer.push_interleaved(&[0.1; 8_000 * 3], 1); + assert_eq!(buffer.len(), 8_000 * REFERENCE_BUFFER_MAX_SECONDS); + let chunk = buffer.take_for_input(160, 16_000).unwrap(); + assert_eq!(chunk.len(), 80); + assert_eq!(buffer.take_for_input(32_000, 16_000), None); + } + + #[test] + fn reference_buffer_discards_partial_underrun() { + let buffer = ReferenceAudioBuffer::new(DICTATION_SAMPLE_RATE); + buffer.push_interleaved(&[0.25; 80], 1); + assert_eq!(buffer.take_for_input(160, DICTATION_SAMPLE_RATE), None); + + buffer.push_interleaved(&[0.75; 160], 1); + let chunk = buffer + .take_for_input(160, DICTATION_SAMPLE_RATE) + .expect("fresh render samples should satisfy the next input frame"); + assert!(chunk + .iter() + .all(|sample| (*sample - 0.75).abs() < f32::EPSILON)); + } + + #[test] + fn echo_canceller_reduces_delayed_render_signal() { + let mut canceller = EchoCanceller::default(); + let mut reference = vec![0.0_f32; 16_000]; + for (index, sample) in reference.iter_mut().enumerate() { + *sample = (index as f32 * 0.071).sin() * 0.25; + } + let delay = 96; + let microphone = reference + .iter() + .enumerate() + .map(|(index, _)| { + reference + .get(index.saturating_sub(delay)) + .copied() + .unwrap_or_default() + * 0.7 + }) + .collect::>(); + let mut cancelled = Vec::new(); + for (mic, far) in microphone.chunks(160).zip(reference.chunks(160)) { + cancelled.extend(canceller.process(mic, far)); + } + let input_rms = microphone[2_000..] + .iter() + .map(|sample| sample * sample) + .sum::(); + let output_rms = cancelled[2_000..] + .iter() + .map(|sample| sample * sample) + .sum::(); + assert!(output_rms < input_rms * 0.35, "{output_rms} >= {input_rms}"); + } + + #[test] + fn echo_canceller_preserves_near_end_during_double_talk() { + let mut canceller = EchoCanceller::default(); + let mut reference = vec![0.0_f32; 12_000]; + for (index, sample) in reference.iter_mut().enumerate() { + // Deterministic broadband-ish reference gives the adaptive filter enough + // information to learn the delayed acoustic path. + *sample = (((index as u32) + .wrapping_mul(1_103_515_245) + .wrapping_add(12_345) + >> 16) as f32 + / 32_768.0 + - 1.0) + * 0.2; + } + let delay = 96; + let echo = |index: usize| { + reference + .get(index.saturating_sub(delay)) + .copied() + .unwrap_or_default() + * 0.7 + }; + for (index, far) in reference.chunks(160).enumerate() { + let start = index * 160; + let microphone = (start..start + far.len()).map(echo).collect::>(); + let _ = canceller.process(µphone, far); + } + + let mut output = Vec::new(); + let mut near_end = Vec::new(); + for (index, far) in reference[6_000..].chunks(160).enumerate() { + let start = 6_000 + index * 160; + let near = (start..start + far.len()) + .map(|sample| (sample as f32 * 0.017).sin() * 0.22) + .collect::>(); + let microphone = far + .iter() + .enumerate() + .map(|(offset, _sample)| echo(start + offset) + near[offset]) + .collect::>(); + output.extend(canceller.process(µphone, far)); + near_end.extend(near); + } + let dot = output + .iter() + .zip(&near_end) + .map(|(processed, near)| processed * near) + .sum::(); + let near_power = near_end.iter().map(|sample| sample * sample).sum::(); + assert!( + dot > near_power * 0.35, + "near-end correlation was suppressed" + ); + } + #[test] + fn processor_without_render_reference_falls_back_to_microphone_path() { + let input = [0.1_f32; 160]; + let mut with_aec = AudioProcessor::new(AudioProcessingConfig { + echo_cancellation_enabled: true, + ..Default::default() + }); + let mut ordinary = AudioProcessor::new(AudioProcessingConfig::default()); + assert_eq!( + with_aec.process_with_reference(&input, 1, DICTATION_SAMPLE_RATE, None, 16_000), + ordinary.process(&input, 1, DICTATION_SAMPLE_RATE) + ); + } + #[test] fn wav_encoder_preserves_canonical_pcm_and_rejects_partial_samples() { let pcm = [1, 0, 255, 127]; diff --git a/openless-all/app/crates/openless-core/src/cloud_sync_e2ee_documents/registry.rs b/openless-all/app/crates/openless-core/src/cloud_sync_e2ee_documents/registry.rs index 67cc380e5..49330c866 100644 --- a/openless-all/app/crates/openless-core/src/cloud_sync_e2ee_documents/registry.rs +++ b/openless-all/app/crates/openless-core/src/cloud_sync_e2ee_documents/registry.rs @@ -193,6 +193,13 @@ pub const PREFERENCE_FIELDS: &[PreferenceField] = &[ shape: PreferenceShape::Boolean, reason: "local_only_microphone_capture", }, + PreferenceField { + rust_name: "echo_cancellation_enabled", + key: "echoCancellationEnabled", + class: PreferenceClass::Excluded, + shape: PreferenceShape::Boolean, + reason: "local_only_microphone_capture", + }, PreferenceField { rust_name: "voice_activation_enabled", key: "voiceActivationEnabled", diff --git a/openless-all/app/crates/openless-core/src/dictation_context.rs b/openless-all/app/crates/openless-core/src/dictation_context.rs index 9a9cb977a..cd4986f97 100644 --- a/openless-all/app/crates/openless-core/src/dictation_context.rs +++ b/openless-all/app/crates/openless-core/src/dictation_context.rs @@ -335,6 +335,7 @@ impl DictationContext { .then(|| (preferences.silence_auto_stop_seconds * 1_000.0).round() as u64), audio_processing: AudioProcessingConfig { enhancement_enabled: preferences.audio_enhancement_enabled, + echo_cancellation_enabled: preferences.echo_cancellation_enabled, }, }, pipeline_mode: crate::shared_types::effective_pipeline_mode( diff --git a/openless-all/app/crates/openless-core/src/experimental_asr/tests.rs b/openless-all/app/crates/openless-core/src/experimental_asr/tests.rs index 7d8fef64d..a4725742a 100644 --- a/openless-all/app/crates/openless-core/src/experimental_asr/tests.rs +++ b/openless-all/app/crates/openless-core/src/experimental_asr/tests.rs @@ -593,7 +593,12 @@ async fn optional_runtime_files_are_not_download_progress() { let directory = f.store.model_dir(&Model::TextStream.target()).unwrap(); std::fs::create_dir_all(directory.join(".runtime/cache")).unwrap(); let unrelated = std::fs::File::create(directory.join(".runtime/cache/wheels")).unwrap(); - unrelated.set_len(5 * 1024 * 1024 * 1024).unwrap(); + // A modest non-model file is enough to catch accidental runtime-directory + // accounting without relying on sparse-file support from the host filesystem. + unrelated.set_len(5 * 1024 * 1024).unwrap(); + // Release the handle before the download commits a replacement model directory; + // Windows otherwise reports access denied while removing the fixture tree. + drop(unrelated); let status = f.service.status().unwrap(); assert_eq!(status[Model::TextStream.index()].downloaded_bytes, 0); f.service.set_enabled(Model::TextStream, true).unwrap(); diff --git a/openless-all/app/crates/openless-core/src/lib.rs b/openless-all/app/crates/openless-core/src/lib.rs index 92964d687..edfc51618 100644 --- a/openless-all/app/crates/openless-core/src/lib.rs +++ b/openless-all/app/crates/openless-core/src/lib.rs @@ -224,7 +224,7 @@ pub use api::{ }; pub use audio::{ encode_dictation_wav, AudioEnhancer, AudioProcessingConfig, AudioProcessor, NormalizedPcmChunk, - PcmNormalizer, DICTATION_SAMPLE_RATE, + PcmNormalizer, ReferenceAudioBuffer, DICTATION_SAMPLE_RATE, }; pub use auxiliary::{ AsrCallLabel, AuxiliaryApi, RepolishRequest, RetranscriptionFailure, RetranscriptionResult, diff --git a/openless-all/app/crates/openless-core/src/preferences.rs b/openless-all/app/crates/openless-core/src/preferences.rs index 3df4a7aa1..1f5c00649 100644 --- a/openless-all/app/crates/openless-core/src/preferences.rs +++ b/openless-all/app/crates/openless-core/src/preferences.rs @@ -124,6 +124,7 @@ fn preferences_json_preserving_unknown( fn strip_local_only_preferences(value: &mut serde_json::Value) { if let serde_json::Value::Object(object) = value { object.remove("lowLatencyDictationEnabled"); + object.remove("echoCancellationEnabled"); object.remove("lowMemoryBackgroundEnabled"); } } @@ -311,6 +312,10 @@ impl PreferencesStore { "lowLatencyDictationEnabled".to_string(), serde_json::Value::Bool(state.low_latency_dictation_enabled), ); + object.insert( + "echoCancellationEnabled".to_string(), + serde_json::Value::Bool(state.echo_cancellation_enabled), + ); } let parsed: UserPreferences = serde_json::from_value(value.clone()) .map_err(|_| persistence_error("validate restored preferences"))?; @@ -480,9 +485,11 @@ mod tests { let mut value = serde_json::json!({ "lowLatencyDictationEnabled": true, "microphoneDeviceName": "Built-in Mic", + "echoCancellationEnabled": true, }); strip_local_only_preferences(&mut value); assert!(value.get("lowLatencyDictationEnabled").is_none()); + assert!(value.get("echoCancellationEnabled").is_none()); assert_eq!(value["microphoneDeviceName"], "Built-in Mic"); } @@ -490,6 +497,7 @@ mod tests { fn missing_low_latency_preference_defaults_off() { let preferences: UserPreferences = serde_json::from_value(serde_json::json!({})).unwrap(); assert!(!preferences.low_latency_dictation_enabled); + assert!(!preferences.echo_cancellation_enabled); } #[test] diff --git a/openless-all/app/crates/openless-core/src/shared_types.rs b/openless-all/app/crates/openless-core/src/shared_types.rs index a133a6ff4..3f71bf602 100644 --- a/openless-all/app/crates/openless-core/src/shared_types.rs +++ b/openless-all/app/crates/openless-core/src/shared_types.rs @@ -427,6 +427,10 @@ pub struct UserPreferences { /// Disabled by default until an audio-quality baseline supports changing it. #[serde(default)] pub audio_enhancement_enabled: bool, + /// Use a desktop render/monitor reference for best-effort echo cancellation. + /// Disabled by default and local to the host audio device. + #[serde(default)] + pub echo_cancellation_enabled: bool, /// Reconnect the current ASR after recording ends and submit the whole PCM. Default off. #[serde(default)] pub stable_transcription_enabled: bool, @@ -1023,6 +1027,8 @@ struct UserPreferencesWire { #[serde(default)] audio_enhancement_enabled: bool, #[serde(default)] + echo_cancellation_enabled: bool, + #[serde(default)] stable_transcription_enabled: bool, #[serde(default)] low_latency_dictation_enabled: bool, @@ -1305,6 +1311,7 @@ impl Default for UserPreferencesWire { capsule_transcript_font_size: prefs.capsule_transcript_font_size, mute_during_recording: prefs.mute_during_recording, audio_enhancement_enabled: prefs.audio_enhancement_enabled, + echo_cancellation_enabled: prefs.echo_cancellation_enabled, stable_transcription_enabled: prefs.stable_transcription_enabled, low_latency_dictation_enabled: prefs.low_latency_dictation_enabled, low_memory_background_enabled: prefs.low_memory_background_enabled, @@ -1524,6 +1531,7 @@ impl<'de> Deserialize<'de> for UserPreferences { capsule_transcript_font_size: wire.capsule_transcript_font_size.clamp(12, 20), mute_during_recording: wire.mute_during_recording, audio_enhancement_enabled: wire.audio_enhancement_enabled, + echo_cancellation_enabled: wire.echo_cancellation_enabled, stable_transcription_enabled: wire.stable_transcription_enabled, low_latency_dictation_enabled: wire.low_latency_dictation_enabled, low_memory_background_enabled: wire.low_memory_background_enabled, @@ -1919,6 +1927,7 @@ impl Default for UserPreferences { capsule_transcript_font_size: default_capsule_transcript_font_size(), mute_during_recording: false, audio_enhancement_enabled: false, + echo_cancellation_enabled: false, stable_transcription_enabled: false, low_latency_dictation_enabled: false, low_memory_background_enabled: false, diff --git a/openless-all/app/linux-egui/src/audio.rs b/openless-all/app/linux-egui/src/audio.rs index 3c51ae00d..c712e3bf1 100644 --- a/openless-all/app/linux-egui/src/audio.rs +++ b/openless-all/app/linux-egui/src/audio.rs @@ -76,6 +76,7 @@ impl AudioProcessingWorker { consumer: Arc, progress: Arc, writer: Option>>, + reference_slot: Arc>>>, ) -> Result<(Self, SyncSender), BackendError> { let (sender, receiver) = sync_channel::(AUDIO_PROCESSING_QUEUE_CAPACITY); let started = std::time::Instant::now(); @@ -84,9 +85,24 @@ impl AudioProcessingWorker { .spawn(move || { let mut processor = openless_core::AudioProcessor::new(processing); while let Ok(frame) = receiver.recv() { - if let Some(chunk) = - processor.process(&frame.samples, frame.channels, frame.input_sample_rate) - { + let reference = reference_slot + .lock() + .expect("Linux echo reference lock poisoned") + .clone(); + let reference_rate = reference.as_ref().map(|buffer| buffer.sample_rate()); + let reference_samples = reference.as_ref().and_then(|buffer| { + buffer.take_for_input( + frame.samples.len() / frame.channels.max(1), + frame.input_sample_rate, + ) + }); + if let Some(chunk) = processor.process_with_reference( + &frame.samples, + frame.channels, + frame.input_sample_rate, + reference_samples.as_deref(), + reference_rate.unwrap_or(0), + ) { consumer.consume_pcm_chunk(&chunk.pcm_i16_le); if let Some(writer) = &writer { if let Err(error) = writer @@ -412,14 +428,20 @@ fn run_audio_thread(args: LinuxAudioThreadArgs) { runtime_error, startup, } = args; - let (processor, frame_tx) = - match AudioProcessingWorker::start(audio_processing, consumer, progress, writer) { - Ok(worker) => worker, - Err(error) => { - let _ = startup.send(Err(error)); - return; - } - }; + let reference_slot = Arc::new(std::sync::Mutex::new(None)); + let (processor, frame_tx) = match AudioProcessingWorker::start( + audio_processing, + consumer, + progress, + writer, + Arc::clone(&reference_slot), + ) { + Ok(worker) => worker, + Err(error) => { + let _ = startup.send(Err(error)); + return; + } + }; let mut result = Err(BackendError::new( BackendErrorCode::Platform, "no Linux audio backend is available", @@ -449,6 +471,8 @@ fn run_audio_thread(args: LinuxAudioThreadArgs) { &frame_tx, &stop, &runtime_error, + &reference_slot, + audio_processing.echo_cancellation_enabled, ) { Ok(stream) => { result = Ok(stream); @@ -465,10 +489,10 @@ fn run_audio_thread(args: LinuxAudioThreadArgs) { } } - let stream = match result { - Ok(stream) => { + let streams = match result { + Ok(streams) => { let _ = startup.send(Ok(())); - stream + streams } Err(error) => { drop(frame_tx); @@ -480,7 +504,12 @@ fn run_audio_thread(args: LinuxAudioThreadArgs) { while !stop.load(std::sync::atomic::Ordering::Acquire) { std::thread::park_timeout(std::time::Duration::from_millis(25)); } - drop(stream); + drop(streams.microphone); + reference_slot + .lock() + .expect("Linux echo reference lock poisoned") + .take(); + drop(streams.reference); drop(frame_tx); let _ = processor.finish(); } @@ -492,6 +521,12 @@ fn audio_backend_order() -> [&'static str; 3] { ["pipewire", "pulseaudio", "alsa"] } +#[cfg(target_os = "linux")] +struct LinuxAudioStreams { + microphone: cpal::Stream, + reference: Option, +} + #[cfg(target_os = "linux")] #[allow(clippy::too_many_arguments)] fn try_start_audio_stream( @@ -501,18 +536,176 @@ fn try_start_audio_stream( frame_tx: &SyncSender, stop: &Arc, runtime_error: &Arc>>, + reference_slot: &Arc>>>, + echo_cancellation_enabled: bool, +) -> Result { + let device = select_input_device(host, preferred_device_name).map_err(|error| { + BackendError::new(error.code, format!("{backend} backend: {}", error.message)) + })?; + let stream = try_start_microphone_stream(&device, frame_tx, stop, runtime_error)?; + let reference_stream = if echo_cancellation_enabled && backend != "alsa" { + match start_reference_stream(host, backend, reference_slot, stop) { + Ok(stream) => Some(stream), + Err(error) => { + log::warn!( + "{backend} playback monitor unavailable; recording without AEC: {error}" + ); + if backend == "pipewire" { + start_pulse_reference_fallback(reference_slot, stop) + } else { + None + } + } + } + } else { + None + }; + Ok(LinuxAudioStreams { + microphone: stream, + reference: reference_stream, + }) +} + +#[cfg(target_os = "linux")] +fn select_playback_monitor_device( + host: &cpal::Host, + backend: &str, +) -> Result { + use cpal::traits::{DeviceTrait, HostTrait}; + + let default_output_name = host + .default_output_device() + .and_then(|device| device.description().ok()) + .map(|description| description.name().to_ascii_lowercase()); + let devices = host + .devices() + .map_err(|error| classify_audio_error("enumerate playback monitors", error.to_string()))?; + let mut candidates = Vec::new(); + for device in devices { + if !device.supports_input() { + continue; + } + let name = device + .description() + .map(|description| description.name().to_ascii_lowercase()) + .unwrap_or_default(); + let matches_default = !name.is_empty() + && default_output_name.as_deref().is_some_and(|default_name| { + name == default_name + || name.contains(default_name) + || default_name.contains(name.as_str()) + }); + match backend { + // PulseAudio exposes sink monitors as input sources whose description contains + // "monitor". Prefer the monitor belonging to the current default output route. + "pulseaudio" if name.contains("monitor") => { + if matches_default { + return Ok(device); + } + candidates.push(device); + } + // cpal's PipeWire backend marks sink nodes as duplex and sets + // STREAM_CAPTURE_SINK for input streams. Prefer the synthetic default sink, then + // a sink matching the current default output route. + "pipewire" if name == "default_sink" || name.contains("default sink") => { + return Ok(device); + } + "pipewire" if name.contains("sink") => { + if matches_default { + return Ok(device); + } + candidates.push(device); + } + _ => {} + } + } + if candidates.len() == 1 { + return Ok(candidates + .pop() + .expect("one playback monitor candidate must be present")); + } + Err(BackendError::new( + BackendErrorCode::Unsupported, + format!("{backend} playback monitor source for the default output was not found"), + )) +} + +#[cfg(target_os = "linux")] +fn start_pulse_reference_fallback( + reference_slot: &Arc>>>, + stop: &Arc, +) -> Option { + let host_id = cpal::available_hosts() + .into_iter() + .find(|id| id.name().eq_ignore_ascii_case("pulseaudio"))?; + let host = match cpal::host_from_id(host_id) { + Ok(host) => host, + Err(error) => { + log::debug!("PulseAudio reference fallback host unavailable: {error}"); + return None; + } + }; + match start_reference_stream(&host, "pulseaudio", reference_slot, stop) { + Ok(stream) => { + log::info!("using PulseAudio playback monitor as PipeWire AEC fallback"); + Some(stream) + } + Err(error) => { + log::debug!("PulseAudio playback monitor fallback unavailable: {error}"); + None + } + } +} + +#[cfg(target_os = "linux")] +fn start_reference_stream( + host: &cpal::Host, + backend: &str, + reference_slot: &Arc>>>, + stop: &Arc, ) -> Result { use cpal::traits::{DeviceTrait, StreamTrait}; - let device = select_input_device(host, preferred_device_name).map_err(|error| { - BackendError::new(error.code, format!("{backend} backend: {}", error.message)) + let device = select_playback_monitor_device(host, backend)?; + let supported = device.default_input_config().map_err(|error| { + classify_audio_error("default playback monitor config", error.to_string()) })?; + let config: cpal::StreamConfig = supported.config(); + let sample_rate = config.sample_rate; + let channels = usize::from(config.channels); + let reference = Arc::new(openless_core::ReferenceAudioBuffer::new(sample_rate)); + let stream = build_linux_reference_stream( + &device, + &config, + supported.sample_format(), + Arc::clone(&reference), + channels, + Arc::clone(stop), + )?; + stream + .play() + .map_err(|error| classify_audio_error("start playback monitor", error.to_string()))?; + *reference_slot + .lock() + .expect("Linux echo reference lock poisoned") = Some(reference); + Ok(stream) +} + +#[cfg(target_os = "linux")] +fn try_start_microphone_stream( + device: &cpal::Device, + frame_tx: &SyncSender, + stop: &Arc, + runtime_error: &Arc>>, +) -> Result { + use cpal::traits::{DeviceTrait, StreamTrait}; + let mut native = None; let mut start = |supported: cpal::SupportedStreamConfig| -> Result { let accepting = Arc::new(std::sync::atomic::AtomicBool::new(false)); let config = supported.config(); let stream = build_input_stream( - &device, + device, &config, supported.sample_format(), supported.sample_rate(), @@ -595,6 +788,57 @@ fn native_input_alternatives( candidates } +#[cfg(target_os = "linux")] +fn build_linux_reference_stream( + device: &cpal::Device, + config: &cpal::StreamConfig, + sample_format: cpal::SampleFormat, + reference: Arc, + channels: usize, + stop: Arc, +) -> Result { + use cpal::traits::DeviceTrait; + macro_rules! make_stream { + ($t:ty, $to_f32:expr) => {{ + let reference = Arc::clone(&reference); + let stop_for_callback = Arc::clone(&stop); + device + .build_input_stream( + *config, + move |data: &[$t], _| { + if stop_for_callback.load(std::sync::atomic::Ordering::Acquire) { + return; + } + let samples = data.iter().copied().map($to_f32).collect::>(); + reference.push_interleaved(&samples, channels); + }, + move |error| { + log::warn!("[linux-audio] playback monitor stream error: {error}"); + }, + None, + ) + .map_err(|error| classify_audio_error("build playback monitor", error.to_string())) + }}; + } + match sample_format { + cpal::SampleFormat::F32 => make_stream!(f32, |s: f32| s), + cpal::SampleFormat::F64 => make_stream!(f64, |s: f64| s as f32), + cpal::SampleFormat::I64 => make_stream!(i64, |s: i64| (s as f64 / i64::MAX as f64) as f32), + cpal::SampleFormat::U64 => make_stream!(u64, |s: u64| ((s as f64 / u64::MAX as f64) * 2.0 + - 1.0) as f32), + cpal::SampleFormat::U32 => make_stream!(u32, |s: u32| (s as f64 / u32::MAX as f64 * 2.0 + - 1.0) as f32), + cpal::SampleFormat::I16 => make_stream!(i16, |s: i16| s as f32 / i16::MAX as f32), + cpal::SampleFormat::U16 => make_stream!(u16, |s: u16| (s as f32 - 32768.0) / 32768.0), + cpal::SampleFormat::I32 => make_stream!(i32, |s: i32| s as f32 / i32::MAX as f32), + cpal::SampleFormat::I8 => make_stream!(i8, |s: i8| s as f32 / i8::MAX as f32), + cpal::SampleFormat::U8 => make_stream!(u8, |s: u8| (s as f32 - 128.0) / 128.0), + other => Err(BackendError::new( + BackendErrorCode::Unsupported, + format!("unsupported playback monitor sample format: {other:?}"), + )), + } +} #[cfg(target_os = "linux")] fn select_input_device( host: &cpal::Host, diff --git a/openless-all/app/linux-egui/src/i18n.rs b/openless-all/app/linux-egui/src/i18n.rs index cb7d5b198..7fc883a4d 100644 --- a/openless-all/app/linux-egui/src/i18n.rs +++ b/openless-all/app/linux-egui/src/i18n.rs @@ -2548,6 +2548,16 @@ pub const CATALOG: &[Msg] = &[ "가벼운 마이크 향상", ), }, + Msg { + key: "settings.recording.echo_cancellation_label", + text: row( + "播放回声抑制", + "播放回聲抑制", + "Playback echo cancellation", + "再生エコーキャンセル", + "재생 에코 제거", + ), + }, Msg { key: "settings.recording.mute_during_recording_label", text: row( @@ -2558,7 +2568,7 @@ pub const CATALOG: &[Msg] = &[ "녹음 중 음소거", ), }, - Msg { + Msg { key: "settings.recording.paste_shortcut_label", text: row( "模拟粘贴快捷键", @@ -4660,6 +4670,16 @@ pub const CATALOG: &[Msg] = &[ ), }, Msg { + key: "settings.recording.echo_cancellation_desc", + text: row( + "保持系统播放声音，同时使用桌面音频输出作为参考抑制回声。部分设备、蓝牙路径或 Linux ALSA 环境可能不可用。", + "保持系統播放聲音，同時使用桌面音訊輸出作為參考抑制回聲。部分裝置、藍牙路徑或 Linux ALSA 環境可能不可用。", + "Use desktop output as a reference while keeping playback audible; some devices, Bluetooth routes, or Linux ALSA setups may be unavailable.", + "再生音を聞きながらデスクトップ出力を参照してエコーを抑制します。デバイス、Bluetooth、Linux ALSA では利用できない場合があります。", + "재생을 계속 들으면서 데스크톱 출력을 참조해 에코를 줄입니다. 일부 장치, Bluetooth 경로 또는 Linux ALSA에서는 사용할 수 없을 수 있습니다.", + ), + }, + Msg { key: "settings.recording.paste_shortcut_desc", text: row( "插入时模拟按下的粘贴键，部分终端类应用需要 Ctrl+Shift+V（仅 Windows / Linux）。", diff --git a/openless-all/app/linux-egui/src/linux_app/settings_save.rs b/openless-all/app/linux-egui/src/linux_app/settings_save.rs index d7e83d1b4..ae41fdacb 100644 --- a/openless-all/app/linux-egui/src/linux_app/settings_save.rs +++ b/openless-all/app/linux-egui/src/linux_app/settings_save.rs @@ -227,6 +227,9 @@ impl OpenLessEguiApp { frontend::view_model::SettingsField::AudioEnhancement => { preferences.audio_enhancement_enabled = !preferences.audio_enhancement_enabled; } + frontend::view_model::SettingsField::EchoCancellation => { + preferences.echo_cancellation_enabled = !preferences.echo_cancellation_enabled; + } frontend::view_model::SettingsField::RecordAudioForDebug => { preferences.record_audio_for_debug = !preferences.record_audio_for_debug; } diff --git a/openless-all/app/linux-egui/src/main.rs b/openless-all/app/linux-egui/src/main.rs index bb6022bc0..737e4e2e2 100644 --- a/openless-all/app/linux-egui/src/main.rs +++ b/openless-all/app/linux-egui/src/main.rs @@ -5653,6 +5653,7 @@ mod linux_app { }; s.mute_while_recording = prefs.mute_during_recording; s.audio_enhancement = prefs.audio_enhancement_enabled; + s.echo_cancellation = prefs.echo_cancellation_enabled; s.audio_cue = prefs.audio_cue_on_record; s.launch_at_login = prefs.launch_at_login; s.streaming_save_clipboard = prefs.streaming_insert_save_clipboard; diff --git a/openless-all/app/linux-egui/src/ui/frontend/settings.rs b/openless-all/app/linux-egui/src/ui/frontend/settings.rs index e3266df04..a467da07f 100644 --- a/openless-all/app/linux-egui/src/ui/frontend/settings.rs +++ b/openless-all/app/linux-egui/src/ui/frontend/settings.rs @@ -737,6 +737,17 @@ fn general(ui: &mut egui::Ui, vm: &mut FrontendViewModel, actions: &mut Vec, pub mute_while_recording: bool, pub audio_enhancement: bool, + pub echo_cancellation: bool, pub audio_cue: bool, /// 录音胶囊开关与样式（Tauri `capsuleLabel` / `capsuleStyleLabel`）。 pub show_capsule: bool, @@ -1244,6 +1247,7 @@ impl Default for SettingsFields { microphone_error: None, mute_while_recording: false, audio_enhancement: false, + echo_cancellation: false, audio_cue: false, show_capsule: true, capsule_style: 0, diff --git a/openless-all/app/scripts/ci-macos-runner-contract.test.mjs b/openless-all/app/scripts/ci-macos-runner-contract.test.mjs new file mode 100644 index 000000000..0ee224be3 --- /dev/null +++ b/openless-all/app/scripts/ci-macos-runner-contract.test.mjs @@ -0,0 +1,36 @@ +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import yaml from 'js-yaml'; + +const workflow = yaml.load( + await readFile(new URL('../../../.github/workflows/ci.yml', import.meta.url), 'utf8'), +); + +const runnerInput = workflow.on.workflow_dispatch.inputs.macos_runner; +assert.equal(runnerInput.type, 'choice'); +assert.equal(runnerInput.default, 'macos-latest'); +assert.deepEqual(runnerInput.options, ['macos-latest', 'macos-15-intel']); + +const crossPlatform = workflow.jobs['cross-platform']; +assert.equal( + crossPlatform['runs-on'], + "${{ matrix.label == 'macOS' && github.event_name == 'workflow_dispatch' && inputs.macos_runner || matrix.os }}", + 'Only manual macOS checks may override the default matrix runner', +); +assert.equal( + workflow.jobs['macos-msrv']['runs-on'], + "${{ github.event_name == 'workflow_dispatch' && inputs.macos_runner || 'macos-latest' }}", + 'MSRV must share the manual fallback and keep its default for push/PR events', +); +assert.ok(crossPlatform.strategy.matrix.include.includes('"os":"windows-latest"')); +assert.equal(crossPlatform.strategy['fail-fast'], false); +for (const gate of [ + 'Build frontend and run frontend/contract tests', + 'Run Codex writable-roots sandbox contract', + 'Run Rust backend unit tests', + 'Run native recorder module tests (Windows)', +]) { + assert.ok(crossPlatform.steps.some((step) => step.name === gate), `${gate} must remain enabled`); +} + +console.log('ci-macos-runner-contract.test.mjs passed'); diff --git a/openless-all/app/scripts/test-recorder-module.py b/openless-all/app/scripts/test-recorder-module.py index 2fac17a0e..d83989c1c 100644 --- a/openless-all/app/scripts/test-recorder-module.py +++ b/openless-all/app/scripts/test-recorder-module.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 -"""Run the native recorder tests without building Tauri/MLX or fetching ASR submodules. +"""Run native recorder and audio cue tests without Tauri/MLX or ASR submodules. The bridge trait below matches the only core interface used by recorder.rs. -This checks the production recorder module; it does not replace a desktop cargo check. +This checks both production audio modules; it does not replace a desktop cargo check. """ import json from pathlib import Path @@ -19,7 +19,7 @@ version = "0.1.0" edition = "2021" [dependencies] -cpal = "=0.15.3" +cpal = "=0.18.2" parking_lot = "0.12" serde = { version = "1", features = ["derive"] } serde_json = "1" @@ -32,8 +32,9 @@ "#[derive(Debug, Clone)] pub struct BackendError { pub code: BackendErrorCode, pub message: String }\n" "impl BackendError { pub fn new(code: BackendErrorCode, message: impl Into) -> Self { Self { code, message: message.into() } } }\n" + "#[path = " + json.dumps(str(recorder.parents[2] / "crates/openless-core/src/audio.rs"), ensure_ascii=False) + "]\npub mod audio;\n" - "pub use audio::{AudioProcessingConfig, AudioProcessor};\n" + "pub use audio::{AudioProcessingConfig, AudioProcessor, ReferenceAudioBuffer};\n" "pub trait AudioConsumer: Send + Sync { fn consume_pcm_chunk(&self, pcm: &[u8]); }\n" + "#[path = " + json.dumps(str(recorder), ensure_ascii=False) + "]\npub mod recorder;\n" + + "#[path = " + json.dumps(str(recorder.with_name("native_audio_cue.rs")), ensure_ascii=False) + "]\npub mod native_audio_cue;\n" ) subprocess.run(["cargo", "test", "--manifest-path", str(root / "Cargo.toml"), *sys.argv[1:]], check=True) diff --git a/openless-all/app/src-tauri/Cargo.lock b/openless-all/app/src-tauri/Cargo.lock index 5e1709cdb..75bee31dc 100644 --- a/openless-all/app/src-tauri/Cargo.lock +++ b/openless-all/app/src-tauri/Cargo.lock @@ -66,9 +66,9 @@ dependencies = [ [[package]] name = "alsa" -version = "0.9.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed7572b7ba83a31e20d1b48970ee402d2e3e0537dcfe0a3ff4d6eb7508617d43" +checksum = "812947049edcd670a82cd5c73c3661d2e58468577ba8489de58e1a73c04cbd5d" dependencies = [ "alsa-sys", "bitflags 2.13.2", @@ -78,9 +78,9 @@ dependencies = [ [[package]] name = "alsa-sys" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db8fee663d06c4e303404ef5f40488a53e062f89ba8bfed81f42325aafad1527" +checksum = "ad7569085a265dd3f607ebecce7458eaab2132a84393534c95b18dcbc3f31e04" dependencies = [ "libc", "pkg-config", @@ -1141,13 +1141,16 @@ checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" [[package]] name = "coreaudio-rs" -version = "0.11.3" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "321077172d79c662f64f5071a03120748d5bb652f5231570141be24cfcd2bace" +checksum = "7d5d7dca3ebcf65a035582c9ad4385371a9d9ee6537474d2a278f4e1e475bb58" dependencies = [ - "bitflags 1.3.2", - "core-foundation-sys", - "coreaudio-sys", + "bitflags 2.13.2", + "libc", + "objc2-audio-toolbox", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", ] [[package]] @@ -1161,25 +1164,32 @@ dependencies = [ [[package]] name = "cpal" -version = "0.15.3" +version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "873dab07c8f743075e57f524c583985fbaf745602acbe916a01539364369a779" +checksum = "6f02e8d0327b42d3e2e4ab2119af397344eb9fc54a34bf0ddeaa1277af8681f1" dependencies = [ "alsa", - "core-foundation-sys", + "block2 0.6.2", "coreaudio-rs", "dasp_sample", - "jni 0.21.1", + "jni 0.22.4", "js-sys", "libc", "mach2", - "ndk 0.8.0", + "ndk", "ndk-context", - "oboe", - "wasm-bindgen", - "wasm-bindgen-futures", + "num-derive", + "num-traits", + "objc2 0.6.5", + "objc2-audio-toolbox", + "objc2-avf-audio", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation 0.3.2", "web-sys", - "windows 0.54.0", + "windows 0.62.2", + "windows-core 0.62.2", ] [[package]] @@ -3307,12 +3317,9 @@ dependencies = [ [[package]] name = "mach2" -version = "0.4.3" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d640282b302c0bb0a2a8e0233ead9035e3bed871f0b7e81fe4a1ec829765db44" -dependencies = [ - "libc", -] +checksum = "dae608c151f68243f2b000364e1f7b186d9c29845f7d2d85bd31b9ad77ad552b" [[package]] name = "malloc_buf" @@ -3523,20 +3530,6 @@ dependencies = [ "tempfile", ] -[[package]] -name = "ndk" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2076a31b7010b17a38c01907c45b945e8f11495ee4dd588309718901b1f7a5b7" -dependencies = [ - "bitflags 2.13.2", - "jni-sys 0.3.1", - "log", - "ndk-sys 0.5.0+25.2.9519653", - "num_enum", - "thiserror 1.0.69", -] - [[package]] name = "ndk" version = "0.9.0" @@ -3546,7 +3539,7 @@ dependencies = [ "bitflags 2.13.2", "jni-sys 0.3.1", "log", - "ndk-sys 0.6.0+11769913", + "ndk-sys", "num_enum", "raw-window-handle", "thiserror 1.0.69", @@ -3558,15 +3551,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" -[[package]] -name = "ndk-sys" -version = "0.5.0+25.2.9519653" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c196769dd60fd4f363e11d948139556a344e79d451aeb2fa2fd040738ef7691" -dependencies = [ - "jni-sys 0.3.1", -] - [[package]] name = "ndk-sys" version = "0.6.0+11769913" @@ -3789,6 +3773,32 @@ dependencies = [ "objc2-quartz-core 0.3.2", ] +[[package]] +name = "objc2-audio-toolbox" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6948501a91121d6399b79abaa33a8aa4ea7857fe019f341b8c23ad6e81b79b08" +dependencies = [ + "bitflags 2.13.2", + "libc", + "objc2 0.6.5", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation 0.3.2", +] + +[[package]] +name = "objc2-avf-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13a380031deed8e99db00065c45937da434ca987c034e13b87e4441f9e4090be" +dependencies = [ + "bitflags 2.13.2", + "objc2 0.6.5", + "objc2-foundation 0.3.2", +] + [[package]] name = "objc2-cloud-kit" version = "0.3.2" @@ -3800,6 +3810,29 @@ dependencies = [ "objc2-foundation 0.3.2", ] +[[package]] +name = "objc2-core-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1eebcea8b0dbff5f7c8504f3107c68fc061a3eb44932051c8cf8a68d969c3b2" +dependencies = [ + "dispatch2", + "objc2 0.6.5", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation 0.3.2", +] + +[[package]] +name = "objc2-core-audio-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a89f2ec274a0cf4a32642b2991e8b351a404d290da87bb6a9a9d8632490bd1c" +dependencies = [ + "bitflags 2.13.2", + "objc2 0.6.5", +] + [[package]] name = "objc2-core-data" version = "0.2.2" @@ -3829,7 +3862,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ "bitflags 2.13.2", + "block2 0.6.2", "dispatch2", + "libc", "objc2 0.6.5", ] @@ -4068,29 +4103,6 @@ dependencies = [ "objc", ] -[[package]] -name = "oboe" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8b61bebd49e5d43f5f8cc7ee2891c16e0f41ec7954d36bcb6c14c5e0de867fb" -dependencies = [ - "jni 0.21.1", - "ndk 0.8.0", - "ndk-context", - "num-derive", - "num-traits", - "oboe-sys", -] - -[[package]] -name = "oboe-sys" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8bb09a4a2b1d668170cfe0a7d5bc103f8999fb316c98099b6a9939c9f2e79d" -dependencies = [ - "cc", -] - [[package]] name = "oid-registry" version = "0.7.1" @@ -5775,7 +5787,7 @@ checksum = "aac18da81ebbf05109ab275b157c22a653bb3c12cf884450179942f81bcbf6c3" dependencies = [ "bytemuck", "js-sys", - "ndk 0.9.0", + "ndk", "objc2 0.6.5", "objc2-core-foundation", "objc2-core-graphics", @@ -5992,8 +6004,8 @@ dependencies = [ "jni 0.21.1", "libc", "log", - "ndk 0.9.0", - "ndk-sys 0.6.0+11769913", + "ndk", + "ndk-sys", "objc2 0.6.5", "objc2-app-kit 0.3.2", "objc2-foundation 0.3.2", @@ -6032,9 +6044,9 @@ dependencies = [ "jni 0.21.1", "libc", "log", - "ndk 0.9.0", + "ndk", "ndk-context", - "ndk-sys 0.6.0+11769913", + "ndk-sys", "objc2 0.6.5", "objc2-app-kit 0.3.2", "objc2-foundation 0.3.2", @@ -7475,16 +7487,6 @@ dependencies = [ "windows-version", ] -[[package]] -name = "windows" -version = "0.54.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9252e5725dbed82865af151df558e754e4a3c2c30818359eb17465f1346a1b49" -dependencies = [ - "windows-core 0.54.0", - "windows-targets 0.52.6", -] - [[package]] name = "windows" version = "0.58.0" @@ -7538,16 +7540,6 @@ dependencies = [ "windows-core 0.62.2", ] -[[package]] -name = "windows-core" -version = "0.54.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12661b9c89351d684a50a8a643ce5f608e20243b9fb84687800163429f161d65" -dependencies = [ - "windows-result 0.1.2", - "windows-targets 0.52.6", -] - [[package]] name = "windows-core" version = "0.58.0" @@ -7696,15 +7688,6 @@ dependencies = [ "windows-strings 0.5.1", ] -[[package]] -name = "windows-result" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" -dependencies = [ - "windows-targets 0.52.6", -] - [[package]] name = "windows-result" version = "0.2.0" @@ -8154,7 +8137,7 @@ dependencies = [ "javascriptcore-rs", "jni 0.21.1", "libc", - "ndk 0.9.0", + "ndk", "objc2 0.6.5", "objc2-app-kit 0.3.2", "objc2-core-foundation", diff --git a/openless-all/app/src-tauri/Cargo.toml b/openless-all/app/src-tauri/Cargo.toml index b631ed04c..8a32ebe5c 100644 --- a/openless-all/app/src-tauri/Cargo.toml +++ b/openless-all/app/src-tauri/Cargo.toml @@ -75,8 +75,9 @@ regex = "1" bytes = "1" url = "2" raw-window-handle = "0.6" -# Audio capture — shared across desktop and mobile. -cpal = "0.15" +# Audio capture — cpal 0.18 provides the desktop render-loopback path and remains +# available to mobile builds through the same cross-platform API. +cpal = "0.18.2" [target.'cfg(any(target_os = "android", target_os = "ios"))'.dependencies] tauri-plugin-fs = "2.7.1" diff --git a/openless-all/app/src-tauri/src/commands/settings.rs b/openless-all/app/src-tauri/src/commands/settings.rs index e3c989a03..eb8bf4fc5 100644 --- a/openless-all/app/src-tauri/src/commands/settings.rs +++ b/openless-all/app/src-tauri/src/commands/settings.rs @@ -231,6 +231,7 @@ pub(crate) fn sync_low_latency_capture_preferences(prefs: &UserPreferences) { .then(|| prefs.microphone_device_name.clone()), openless_core::AudioProcessingConfig { enhancement_enabled: prefs.audio_enhancement_enabled, + echo_cancellation_enabled: prefs.echo_cancellation_enabled, }, ); } diff --git a/openless-all/app/src-tauri/src/core_adapters.rs b/openless-all/app/src-tauri/src/core_adapters.rs index 98af79c56..2a8e144dc 100644 --- a/openless-all/app/src-tauri/src/core_adapters.rs +++ b/openless-all/app/src-tauri/src/core_adapters.rs @@ -150,6 +150,7 @@ pub(crate) fn backend_dependencies( .then(|| prefs.microphone_device_name.clone()), openless_core::AudioProcessingConfig { enhancement_enabled: prefs.audio_enhancement_enabled, + echo_cancellation_enabled: prefs.echo_cancellation_enabled, }, ); } @@ -2000,6 +2001,7 @@ impl openless_core::VoiceActivationRuntime for TauriVoiceActivationRuntime { ); let audio_processing = openless_core::AudioProcessingConfig { enhancement_enabled: preferences_snapshot.audio_enhancement_enabled, + echo_cancellation_enabled: preferences_snapshot.echo_cancellation_enabled, }; if let Some(wake_asr) = wake_asr.as_ref() { warm_hub.set_activation_generation(config.activation_generation); @@ -2264,6 +2266,7 @@ impl openless_core::VoiceActivationHandle for TauriVoiceActivationHandle { .then(|| prefs.microphone_device_name.clone()), openless_core::AudioProcessingConfig { enhancement_enabled: prefs.audio_enhancement_enabled, + echo_cancellation_enabled: prefs.echo_cancellation_enabled, }, ); } diff --git a/openless-all/app/src-tauri/src/native_audio_cue.rs b/openless-all/app/src-tauri/src/native_audio_cue.rs index c8af87833..e7bfbde53 100644 --- a/openless-all/app/src-tauri/src/native_audio_cue.rs +++ b/openless-all/app/src-tauri/src/native_audio_cue.rs @@ -44,12 +44,12 @@ fn play(generation: u64) -> Result<(), String> { let format = supported.sample_format(); let config: cpal::StreamConfig = supported.into(); let channels = config.channels as usize; - let sample_rate = config.sample_rate.0 as f32; + let sample_rate = config.sample_rate as f32; macro_rules! stream { ($type:ty, $convert:expr) => {{ let mut position = 0usize; device.build_output_stream( - &config, + config, move |data: &mut [$type], _: &cpal::OutputCallbackInfo| { for frame in data.chunks_mut(channels) { let value = if GENERATION.load(Ordering::Relaxed) == generation { diff --git a/openless-all/app/src-tauri/src/permissions.rs b/openless-all/app/src-tauri/src/permissions.rs index 9a4d58dcb..f3d0aed20 100644 --- a/openless-all/app/src-tauri/src/permissions.rs +++ b/openless-all/app/src-tauri/src/permissions.rs @@ -412,7 +412,7 @@ mod platform { ($t:ty) => { device .build_input_stream::<$t, _, _>( - config, + *config, move |_data: &[$t], _info| {}, err_cb, None, diff --git a/openless-all/app/src-tauri/src/recorder.rs b/openless-all/app/src-tauri/src/recorder.rs index d5acb8839..f27fd0d43 100644 --- a/openless-all/app/src-tauri/src/recorder.rs +++ b/openless-all/app/src-tauri/src/recorder.rs @@ -120,6 +120,118 @@ fn voice_capture_key(generation: u64) -> u64 { generation | VOICE_CAPTURE_KEY_PREFIX } +/// [DEBUG-ISSUE1223] Per-capture state shared by the two native stream roles. +/// Only the first callback and callbacks immediately following errors are logged. +struct CaptureDiagnostics { + run_id: u64, + microphone: StreamDiagnostics, + reference: StreamDiagnostics, +} + +#[derive(Default)] +struct StreamDiagnostics { + callbacks: AtomicUsize, + errors: AtomicUsize, + error_pending: AtomicBool, +} + +impl StreamDiagnostics { + fn record_error(&self) -> usize { + let count = self.errors.fetch_add(1, Ordering::Relaxed) + 1; + self.error_pending.store(true, Ordering::Release); + count + } + + fn record_callback(&self) -> (usize, bool) { + let count = self.callbacks.fetch_add(1, Ordering::Relaxed) + 1; + let after_error = self.error_pending.swap(false, Ordering::AcqRel); + (count, after_error) + } +} + +impl CaptureDiagnostics { + fn new() -> Self { + static NEXT_RUN: AtomicU64 = AtomicU64::new(1); + Self { + run_id: NEXT_RUN.fetch_add(1, Ordering::Relaxed), + microphone: StreamDiagnostics::default(), + reference: StreamDiagnostics::default(), + } + } + + /// Only code-controlled enums, booleans and numeric facts may enter the payload. + /// Use the existing logger; do not open files on native callback threads. + fn event( + &self, + probe_id: &str, + hypothesis_ids: &str, + event: &str, + outcome: &str, + safe_json: &str, + ) { + let timestamp_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis(); + log::info!( + "[DEBUG-ISSUE1223] {{\"schema\":\"diagnosing-bugs.v1\",\"session_id\":\"s1223-20261019\",\"run_id\":\"pid-{}-capture-{}\",\"probe_id\":\"{}\",\"hypothesis_ids\":{},\"event\":\"{}\",\"outcome\":\"{}\",\"safe\":{},\"ts\":{}}}", + std::process::id(), + self.run_id, + probe_id, + hypothesis_ids, + event, + outcome, + safe_json, + timestamp_ms, + ); + } + + fn data_callback(&self, reference: bool, accepting_audio: bool, frames: usize) { + let stream = if reference { + &self.reference + } else { + &self.microphone + }; + let (count, after_error) = stream.record_callback(); + if count == 1 || after_error { + self.event( + "P5", + "[\"H1\",\"H2\",\"H3\"]", + "data_callback", + if after_error { "after_error" } else { "first_callback" }, + &format!( + "{{\"stream_role\":\"{}\",\"accepting_audio\":{},\"callback_count\":{},\"error_count\":{},\"frames\":{}}}", + if reference { "reference" } else { "microphone" }, + accepting_audio, + count, + stream.errors.load(Ordering::Relaxed), + frames, + ), + ); + } + } +} + +fn cpal_error_kind_code(error: &cpal::Error) -> &'static str { + match error.kind() { + cpal::ErrorKind::DeviceBusy => "device_busy", + cpal::ErrorKind::DeviceChanged => "device_changed", + cpal::ErrorKind::DeviceNotAvailable => "device_not_available", + cpal::ErrorKind::HostUnavailable => "host_unavailable", + cpal::ErrorKind::InvalidInput => "invalid_input", + cpal::ErrorKind::PermissionDenied => "permission_denied", + cpal::ErrorKind::RealtimeDenied => "realtime_denied", + cpal::ErrorKind::ResourceExhausted => "resource_exhausted", + cpal::ErrorKind::StreamInvalidated => "stream_invalidated", + cpal::ErrorKind::UnsupportedConfig => "unsupported_config", + cpal::ErrorKind::UnsupportedOperation => "unsupported_operation", + cpal::ErrorKind::Xrun => "xrun", + cpal::ErrorKind::BackendError => "backend_error", + cpal::ErrorKind::Other => "other", + _ => "unknown", + } +} + /// Downstream that receives resampled Int16 PCM bytes (little-endian). pub trait AudioConsumer: Send + Sync { /// Each delivery is a little-endian byte sequence of Int16 samples; @@ -1774,22 +1886,29 @@ impl Drop for StagedRecorder { } } +fn device_name(device: &cpal::Device) -> Option { + device + .description() + .ok() + .map(|description| description.name().to_owned()) +} + /// Enumerate input devices, marking the system default device. pub fn list_input_devices() -> Result, RecorderError> { let host = cpal::default_host(); let default_name = host .default_input_device() - .and_then(|device| device.name().ok()); + .and_then(|device| device_name(&device)); let devices = host .input_devices() .map_err(|e| RecorderError::EngineFailed(format!("input_devices: {e}")))?; let mut result = Vec::new(); for device in devices { - let name = match device.name() { - Ok(name) => name, - Err(err) => { - log::warn!("[recorder] failed to read input device name: {err}"); + let name = match device_name(&device) { + Some(name) => name, + None => { + log::warn!("[recorder] failed to read input device name"); continue; } }; @@ -1815,12 +1934,39 @@ fn run_audio_thread( startup_tx: Sender>, runtime_error_tx: Sender, ) { + let (reference_stream, reference_buffer) = if processing.echo_cancellation_enabled { + match start_reference_stream(&stop_flag, &state.diagnostics) { + Ok((stream, buffer)) => (Some(stream), Some(buffer)), + Err(error) => { + log::warn!("[recorder] playback echo reference unavailable; recording without AEC: {error}"); + (None, None) + } + } + } else { + (None, None) + }; + state.diagnostics.event( + "P1", + "[\"H1\",\"H2\",\"H3\"]", + "reference_start", + if reference_buffer.is_some() { + "available" + } else { + "fallback" + }, + &format!( + "{{\"aec_requested\":{},\"reference_available\":{}}}", + processing.echo_cancellation_enabled, + reference_buffer.is_some(), + ), + ); let (processor, frame_tx) = match AudioProcessingWorker::start( processing, consumer, level_handler, archiver, Arc::clone(&state), + reference_buffer, ) { Ok(worker) => worker, Err(error) => { @@ -1922,6 +2068,17 @@ fn run_audio_thread( "[recorder] watchdog: 录音回调已停止 {} 秒，触发错误恢复", elapsed.as_secs() ); + state_for_watchdog.diagnostics.event( + "P4", + "[\"H4\"]", + "watchdog_fault", + "callback_silent", + &format!( + "{{\"callback_count\":{},\"elapsed_secs\":{}}}", + state_for_watchdog.diagnostics.microphone.callbacks.load(Ordering::Relaxed), + elapsed.as_secs(), + ), + ); let _ = runtime_error_tx_for_watchdog.send(RecorderError::EngineFailed( format!("录音回调静默停止 {} 秒", elapsed.as_secs()), @@ -1937,6 +2094,16 @@ fn run_audio_thread( "[recorder] watchdog: {} 秒内未收到首次回调，触发错误恢复", elapsed.as_secs() ); + state_for_watchdog.diagnostics.event( + "P4", + "[\"H4\"]", + "watchdog_fault", + "first_callback_timeout", + &format!( + "{{\"callback_count\":0,\"elapsed_secs\":{}}}", + elapsed.as_secs(), + ), + ); let _ = runtime_error_tx_for_watchdog.send(RecorderError::EngineFailed( format!("录音启动后 {} 秒内未收到回调", elapsed.as_secs()), @@ -1958,7 +2125,7 @@ fn run_audio_thread( // callback racing with shutdown from adding work after the worker drain begins. state.accepting_audio.store(false, Ordering::Release); // Pause explicitly before drop. - // On macOS coreaudio, cpal 0.15's plain drop(Stream) does not synchronously call + // On macOS coreaudio, cpal 0.18's plain drop(Stream) does not synchronously call // AudioOutputUnitStop, so the AudioUnit render callback keeps firing and process_callback // still logs cb# lines at ~5ms/frame, making macOS think the mic is still in use (the // orange dot never turns off). pause() goes through StreamTrait::pause — on the coreaudio @@ -1969,6 +2136,13 @@ fn run_audio_thread( log::warn!("[recorder] cpal Stream pause before drop failed: {err}"); } drop(stream); + if let Some(reference_stream) = reference_stream { + if let Err(err) = reference_stream.pause() { + log::warn!("[recorder] echo reference pause before drop failed: {err}"); + } + drop(reference_stream); + log::info!("[recorder] echo reference stream dropped"); + } log::info!("[recorder] cpal Stream dropped (mic released)"); *state.processor_sender.lock() = None; drop(frame_tx); @@ -1991,7 +2165,7 @@ fn build_input_stream( ) -> Result<(cpal::Stream, Arc), RecorderError> { let host = cpal::default_host(); let selected = select_input_device(&host, microphone_device_name.as_deref())?; - let selected_name = selected.name().ok(); + let selected_name = device_name(&selected); let start = |device: &cpal::Device| { start_device_stream(device, &frame_tx, &state, &stop_flag, &runtime_error_tx) }; @@ -2002,7 +2176,7 @@ fn build_input_stream( }; log::warn!("[recorder] selected microphone failed: {initial_error}; trying other inputs"); let default = host.default_input_device(); - let default_name = default.as_ref().and_then(|device| device.name().ok()); + let default_name = default.as_ref().and_then(device_name); let mut candidates = Vec::new(); if let Some(default) = default { if default_name != selected_name { @@ -2011,7 +2185,7 @@ fn build_input_stream( } if let Ok(devices) = host.input_devices() { candidates.extend(devices.filter(|device| { - let name = device.name().ok(); + let name = device_name(device); name.is_none() || (name != selected_name && name != default_name) })); } @@ -2073,13 +2247,13 @@ fn native_input_alternatives( default: Option<&cpal::SupportedStreamConfig>, ranges: impl IntoIterator, ) -> Vec { - let native_rate = default.map(|config| config.sample_rate().0); + let native_rate = default.map(|config| config.sample_rate()); let mut candidates = Vec::new(); for range in ranges { - let min = range.min_sample_rate().0; - let max = range.max_sample_rate().0; + let min = range.min_sample_rate(); + let max = range.max_sample_rate(); for rate in [native_rate.unwrap_or(max).clamp(min, max), min, max] { - let candidate = range.with_sample_rate(cpal::SampleRate(rate)); + let candidate = range.with_sample_rate(rate); let same = |other: &cpal::SupportedStreamConfig| { other.config() == candidate.config() && other.sample_format() == candidate.sample_format() @@ -2109,7 +2283,7 @@ fn start_native_config( frame_tx.clone(), Arc::clone(state), Arc::clone(stop_flag), - config.sample_rate.0, + config.sample_rate, config.channels as usize, runtime_error_tx.clone(), )?; @@ -2119,8 +2293,8 @@ fn start_native_config( state.accepting_audio.store(true, Ordering::Release); log::info!( "[recorder] inputDevice={} inputFormat sampleRate={} channels={} fmt={:?}", - device.name().unwrap_or_else(|_| "".into()), - config.sample_rate.0, + device_name(device).unwrap_or_else(|| "".into()), + config.sample_rate, config.channels, supported.sample_format() ); @@ -2139,7 +2313,7 @@ fn select_input_device( .input_devices() .map_err(|e| RecorderError::EngineFailed(format!("input_devices: {e}")))?; for device in devices { - if device.name().ok().as_deref() == Some(preferred) { + if device_name(&device).as_deref() == Some(preferred) { return Ok(device); } } @@ -2173,7 +2347,7 @@ fn classify_default_config_err(msg: String) -> RecorderError { } /// Startup-time build_stream failure: same classification as above; may be a permission issue. -fn classify_build_stream_err(err: cpal::BuildStreamError) -> RecorderError { +fn classify_build_stream_err(err: cpal::Error) -> RecorderError { let msg = err.to_string(); let lower = msg.to_lowercase(); if is_no_device_error(&lower) { @@ -2205,8 +2379,58 @@ fn is_no_device_error(lower: &str) -> bool { .any(|pattern| lower.contains(pattern)) } -/// `SupportedStreamConfig` -> the concrete build call for that SampleFormat. -/// Only common cpal float and integer formats are supported; others fall back to an error. +fn build_input_stream_compat( + device: &cpal::Device, + config: StreamConfig, + data_callback: D, + error_callback: E, +) -> Result +where + T: cpal::SizedSample, + D: FnMut(&[T], &cpal::InputCallbackInfo) + Send + 'static, + E: FnMut(cpal::Error) + Send + 'static, +{ + device.build_input_stream(config, data_callback, error_callback, None) +} + +fn handle_microphone_stream_error( + state: &StreamState, + error: cpal::Error, + runtime_error_tx: &Sender, +) { + if !state.accepting_audio.load(Ordering::Acquire) { + return; + } + let error_kind = cpal_error_kind_code(&error); + let recoverable_xrun = error.kind() == cpal::ErrorKind::Xrun; + let error_count = state.diagnostics.microphone.record_error(); + state.diagnostics.event( + "P2", + "[\"H1\",\"H2\",\"H3\"]", + "microphone_stream_error", + if recoverable_xrun { + "xrun_recoverable" + } else { + "fatal_candidate" + }, + &format!( + "{{\"stream_role\":\"microphone\",\"error_kind\":\"{}\",\"accepting_audio\":true,\"error_count\":{},\"callback_count\":{}}}", + error_kind, + error_count, + state.diagnostics.microphone.callbacks.load(Ordering::Relaxed), + ), + ); + if recoverable_xrun { + // CPAL can report a discontinuity and deliver the next capture packet immediately. + // Do not cancel usable capture or refresh liveness here: the existing watchdog must + // still detect a stream that stops delivering data after this notification. + log::warn!("[recorder] microphone xrun; continuing capture (error_count={error_count})"); + return; + } + log::error!("[recorder] stream error: {error}"); + let _ = runtime_error_tx.send(RecorderError::EngineFailed(format!("stream: {error}"))); +} + #[allow(clippy::too_many_arguments)] fn build_stream_for_format( device: &cpal::Device, @@ -2228,18 +2452,15 @@ fn build_stream_for_format( let runtime_error_for_error = runtime_error_tx.clone(); let error_state = Arc::clone(&state); let err_cb = move |err| { - if !error_state.accepting_audio.load(Ordering::Acquire) { - return; - } - log::error!("[recorder] stream error: {err}"); - let _ = runtime_error_for_error - .send(RecorderError::EngineFailed(format!("stream: {err}"))); + handle_microphone_stream_error(&error_state, err, &runtime_error_for_error); }; - device - .build_input_stream::<$t, _, _>( - config, + build_input_stream_compat::<$t, _, _>( + device, + *config, move |data: &[$t], _info| { - if !state.accepting_audio.load(Ordering::Acquire) { + let accepting = state.accepting_audio.load(Ordering::Acquire); + state.diagnostics.data_callback(false, accepting, data.len() / channels.max(1)); + if !accepting { return; } let mut floats = Vec::with_capacity(data.len()); @@ -2257,6 +2478,17 @@ fn build_stream_for_format( stop_flag.store(true, Ordering::Release); } Err(TrySendError::Full(_)) => { + state.diagnostics.event( + "P4", + "[\"H4\"]", + "processing_backpressure", + "queue_full", + &format!( + "{{\"queue_capacity\":{},\"callback_count\":{}}}", + AUDIO_PROCESSING_QUEUE_CAPACITY, + state.diagnostics.microphone.callbacks.load(Ordering::Relaxed), + ), + ); if state.accepting_audio.swap(false, Ordering::AcqRel) { let _ = runtime_error_for_callback.send( RecorderError::EngineFailed( @@ -2269,7 +2501,6 @@ fn build_stream_for_format( } }, err_cb, - None, ) .map_err(classify_build_stream_err) }}; @@ -2300,6 +2531,127 @@ fn build_stream_for_format( } } +#[cfg(not(any(target_os = "android", target_os = "ios")))] +fn start_reference_stream( + stop_flag: &Arc, + diagnostics: &Arc, +) -> Result<(cpal::Stream, Arc), RecorderError> { + let host = cpal::default_host(); + let device = host + .default_output_device() + .ok_or_else(|| RecorderError::EngineFailed("no default output device for echo reference".into()))?; + let supported = device + .default_output_config() + .map_err(|error| RecorderError::EngineFailed(format!("default output config: {error}")))?; + let config = supported.config(); + let sample_rate = config.sample_rate; + let channels = config.channels as usize; + let reference = Arc::new(openless_core::ReferenceAudioBuffer::new(sample_rate)); + let reference_for_callback = Arc::clone(&reference); + let stop_for_callback = Arc::clone(stop_flag); + let device_name = device_name(&device).unwrap_or_else(|| "".into()); + let stream = build_reference_stream_for_format( + &device, + &config, + supported.sample_format(), + reference_for_callback, + channels, + stop_for_callback, + Arc::clone(diagnostics), + )?; + stream + .play() + .map_err(|error| RecorderError::EngineFailed(format!("play echo reference: {error}")))?; + log::info!( + "[recorder] echo reference outputDevice={} sampleRate={} channels={} fmt={:?}", + device_name, + sample_rate, + channels, + supported.sample_format() + ); + Ok((stream, reference)) +} + +#[cfg(any(target_os = "android", target_os = "ios"))] +fn start_reference_stream( + _stop_flag: &Arc, + _diagnostics: &Arc, +) -> Result<(cpal::Stream, Arc), RecorderError> { + Err(RecorderError::EngineFailed( + "native echo reference is unavailable on mobile".into(), + )) +} + +#[cfg(not(any(target_os = "android", target_os = "ios")))] +fn build_reference_stream_for_format( + device: &cpal::Device, + config: &StreamConfig, + sample_format: SampleFormat, + reference: Arc, + channels: usize, + stop_flag: Arc, + diagnostics: Arc, +) -> Result { + macro_rules! make_reference_stream { + ($t:ty, $to_f32:expr) => {{ + let reference = Arc::clone(&reference); + let stop_for_callback = Arc::clone(&stop_flag); + let callback_diagnostics = Arc::clone(&diagnostics); + let error_diagnostics = Arc::clone(&diagnostics); + build_input_stream_compat::<$t, _, _>( + device, + *config, + move |data: &[$t], _info| { + let accepting = !stop_for_callback.load(Ordering::Acquire); + callback_diagnostics.data_callback(true, accepting, data.len() / channels.max(1)); + if !accepting { + return; + } + let samples = data.iter().copied().map($to_f32).collect::>(); + reference.push_interleaved(&samples, channels); + }, + move |error| { + let error_kind = cpal_error_kind_code(&error); + let error_count = error_diagnostics.reference.record_error(); + error_diagnostics.event( + "P3", + "[\"H1\",\"H2\"]", + "reference_stream_error", + if error_kind == "xrun" { + "reference_xrun" + } else { + "reference_fault" + }, + &format!( + "{{\"stream_role\":\"reference\",\"error_kind\":\"{}\",\"error_count\":{},\"callback_count\":{}}}", + error_kind, + error_count, + error_diagnostics.reference.callbacks.load(Ordering::Relaxed), + ), + ); + log::warn!("[recorder] echo reference stream stopped: {error}"); + }, + ) + .map_err(classify_build_stream_err) + }}; + } + match sample_format { + SampleFormat::F32 => make_reference_stream!(f32, |s: f32| s), + SampleFormat::F64 => make_reference_stream!(f64, |s: f64| s as f32), + SampleFormat::I64 => make_reference_stream!(i64, |s: i64| (s as f64 / i64::MAX as f64) as f32), + SampleFormat::U64 => make_reference_stream!(u64, |s: u64| ((s as f64 / u64::MAX as f64) * 2.0 - 1.0) as f32), + SampleFormat::U32 => make_reference_stream!(u32, |s: u32| (s as f64 / u32::MAX as f64 * 2.0 - 1.0) as f32), + SampleFormat::I16 => make_reference_stream!(i16, |s: i16| s as f32 / i16::MAX as f32), + SampleFormat::U16 => make_reference_stream!(u16, |s: u16| (s as f32 - 32768.0) / 32768.0), + SampleFormat::I32 => make_reference_stream!(i32, |s: i32| s as f32 / i32::MAX as f32), + SampleFormat::I8 => make_reference_stream!(i8, |s: i8| s as f32 / i8::MAX as f32), + SampleFormat::U8 => make_reference_stream!(u8, |s: u8| (s as f32 - 128.0) / 128.0), + other => Err(RecorderError::EngineFailed(format!( + "unsupported echo reference sample format: {other:?}" + ))), + } +} + struct RawAudioFrame { samples: Vec, channels: usize, @@ -2324,9 +2676,11 @@ impl AudioProcessingWorker { level_handler: Arc, archiver: Option>, state: Arc, + reference: Option>, ) -> Result<(Self, SyncSender), RecorderError> { let (sender, receiver) = sync_channel::(AUDIO_PROCESSING_QUEUE_CAPACITY); + let reference_rate = reference.as_ref().map(|buffer| buffer.sample_rate()); let join_handle = thread::Builder::new() .name("openless-audio-processor".into()) .spawn(move || { @@ -2334,11 +2688,21 @@ impl AudioProcessingWorker { while let Ok(message) = receiver.recv() { match message { AudioProcessingMessage::Frame(frame) => { - if let Some(chunk) = processor.process( + let reference_samples = reference.as_ref().and_then(|buffer| { + buffer.take_for_input( + frame.samples.len() / frame.channels.max(1), + frame.input_sample_rate, + ) + }); + let reference_slice = reference_samples.as_deref(); + let chunk = processor.process_with_reference( &frame.samples, frame.channels, frame.input_sample_rate, - ) { + reference_slice, + reference_rate.unwrap_or(0), + ); + if let Some(chunk) = chunk { consumer.consume_pcm_chunk(&chunk.pcm_i16_le); if let Some(archiver) = &archiver { archiver.append(&chunk.pcm_i16_le); @@ -2382,6 +2746,7 @@ impl AudioProcessingWorker { /// State carried across callbacks: resample leftovers, diagnostic counters and peak. struct StreamState { + diagnostics: Arc, processor_sender: Mutex>>, /// Fractional position left unconsumed by the previous callback; linear-interpolation /// resampling spans buffers. @@ -2400,6 +2765,7 @@ struct StreamState { impl StreamState { fn new() -> Self { Self { + diagnostics: Arc::new(CaptureDiagnostics::new()), processor_sender: Mutex::new(None), resample_phase: Mutex::new(0.0), last_sample: Mutex::new(0.0), @@ -2692,6 +3058,135 @@ mod tests { use super::*; use std::sync::{Arc, Mutex as StdMutex}; + #[test] + fn microphone_xrun_keeps_capture_and_error_channel_open() { + let state = StreamState::new(); + let (sender, receiver) = channel(); + handle_microphone_stream_error( + &state, + cpal::Error::new(cpal::ErrorKind::Xrun), + &sender, + ); + assert!(matches!( + receiver.try_recv(), + Err(std::sync::mpsc::TryRecvError::Empty) + )); + assert!(state.accepting_audio.load(Ordering::Acquire)); + assert!(state.last_callback_time.lock().is_none()); + assert_eq!(state.diagnostics.microphone.record_callback(), (1, true)); + assert_eq!( + state.diagnostics.microphone.errors.load(Ordering::Relaxed), + 1 + ); + } + + #[test] + fn repeated_xruns_do_not_refresh_the_watchdog_or_hide_device_loss() { + let state = StreamState::new(); + let last_callback = std::time::Instant::now() - std::time::Duration::from_secs(10); + *state.last_callback_time.lock() = Some(last_callback); + let (sender, receiver) = channel(); + for _ in 0..8 { + handle_microphone_stream_error( + &state, + cpal::Error::new(cpal::ErrorKind::Xrun), + &sender, + ); + } + assert!(matches!( + receiver.try_recv(), + Err(std::sync::mpsc::TryRecvError::Empty) + )); + assert_eq!(*state.last_callback_time.lock(), Some(last_callback)); + assert!(state.accepting_audio.load(Ordering::Acquire)); + handle_microphone_stream_error( + &state, + cpal::Error::new(cpal::ErrorKind::DeviceNotAvailable), + &sender, + ); + assert!(matches!( + receiver.try_recv(), + Ok(RecorderError::EngineFailed(_)) + )); + } + + #[test] + fn microphone_non_xrun_errors_remain_fatal() { + for kind in [ + cpal::ErrorKind::DeviceNotAvailable, + cpal::ErrorKind::StreamInvalidated, + cpal::ErrorKind::PermissionDenied, + cpal::ErrorKind::BackendError, + cpal::ErrorKind::Other, + ] { + let state = StreamState::new(); + let (sender, receiver) = channel(); + handle_microphone_stream_error(&state, cpal::Error::new(kind), &sender); + assert!( + matches!(receiver.try_recv(), Ok(RecorderError::EngineFailed(_))), + "{kind:?}" + ); + } + } + + #[test] + fn microphone_stream_errors_after_stop_are_ignored() { + let state = StreamState::new(); + state.accepting_audio.store(false, Ordering::Release); + let (sender, receiver) = channel(); + for kind in [cpal::ErrorKind::Xrun, cpal::ErrorKind::DeviceNotAvailable] { + handle_microphone_stream_error(&state, cpal::Error::new(kind), &sender); + } + assert!(matches!( + receiver.try_recv(), + Err(std::sync::mpsc::TryRecvError::Empty) + )); + assert_eq!( + state.diagnostics.microphone.errors.load(Ordering::Relaxed), + 0 + ); + } + + #[test] + fn diagnostic_callback_marks_only_the_next_callback_after_errors() { + let stream = StreamDiagnostics::default(); + assert_eq!(stream.record_callback(), (1, false)); + assert_eq!(stream.record_error(), 1); + assert_eq!(stream.record_callback(), (2, true)); + assert_eq!(stream.record_callback(), (3, false)); + assert_eq!(stream.record_error(), 2); + assert_eq!(stream.record_error(), 3); + assert_eq!(stream.record_callback(), (4, true)); + assert_eq!(stream.record_callback(), (5, false)); + assert_eq!(stream.errors.load(Ordering::Relaxed), 3); + } + + #[test] + fn diagnostic_stream_roles_and_capture_runs_are_independent() { + let first = CaptureDiagnostics::new(); + let second = CaptureDiagnostics::new(); + assert_ne!(first.run_id, second.run_id); + first.microphone.record_error(); + assert_eq!(first.reference.record_callback(), (1, false)); + assert_eq!(first.microphone.record_callback(), (1, true)); + assert_eq!(second.microphone.record_callback(), (1, false)); + assert_eq!(first.reference.errors.load(Ordering::Relaxed), 0); + } + + #[test] + fn diagnostic_error_code_uses_only_structured_kind() { + let error = cpal::Error::with_message(cpal::ErrorKind::Xrun, "private backend detail"); + assert_eq!(cpal_error_kind_code(&error), "xrun"); + assert_eq!( + cpal_error_kind_code(&cpal::Error::new(cpal::ErrorKind::DeviceNotAvailable)), + "device_not_available", + ); + assert_eq!( + cpal_error_kind_code(&cpal::Error::new(cpal::ErrorKind::StreamInvalidated)), + "stream_invalidated", + ); + } + #[derive(Default)] struct RecordingConsumer { chunks: StdMutex>>, @@ -2715,11 +3210,13 @@ mod tests { let (worker, sender) = AudioProcessingWorker::start( openless_core::AudioProcessingConfig { enhancement_enabled: true, + ..Default::default() }, Arc::clone(&consumer) as Arc, Arc::new(|_| {}), None, state, + None, ) .unwrap(); sender @@ -2743,11 +3240,13 @@ mod tests { let (worker, sender) = AudioProcessingWorker::start( openless_core::AudioProcessingConfig { enhancement_enabled: false, + ..Default::default() }, Arc::clone(&consumer) as Arc, Arc::new(|_| {}), None, Arc::new(StreamState::new()), + None, ) .unwrap(); for _ in 0..8 { @@ -2943,17 +3442,17 @@ mod tests { fn native_alternatives_preserve_channels_format_and_advertised_rates() { let range = cpal::SupportedStreamConfigRange::new( 2, - cpal::SampleRate(44_100), - cpal::SampleRate(48_000), + 44_100, + 48_000, cpal::SupportedBufferSize::Unknown, SampleFormat::F32, ); - let native = range.with_sample_rate(cpal::SampleRate(48_000)); + let native = range.with_sample_rate(48_000); let alternatives = native_input_alternatives(Some(&native), [range, range]); assert_eq!(alternatives.len(), 1); assert_eq!(alternatives[0].channels(), 2); assert_eq!(alternatives[0].sample_format(), SampleFormat::F32); - assert_eq!(alternatives[0].sample_rate().0, 44_100); + assert_eq!(alternatives[0].sample_rate(), 44_100); } #[test] diff --git a/openless-all/app/src/i18n/de.ts b/openless-all/app/src/i18n/de.ts index ac3e6e9ac..596671874 100644 --- a/openless-all/app/src/i18n/de.ts +++ b/openless-all/app/src/i18n/de.ts @@ -1554,6 +1554,9 @@ export const de: typeof zhCN = { audioEnhancementLabel: 'Leichte Mikrofonverbesserung', audioEnhancementDesc: 'Wendet eine leichte Verstärkung, weiche Begrenzung und konservative Rauschunterdrückung auf das lokale Mikrofon an. Keine Echounterdrückung. Standardmäßig aus.', + echoCancellationLabel: 'Wiedergabe-Echounterdrückung', + echoCancellationDesc: + 'Verwendet die Desktop-Audioausgabe als Referenz, während die Wiedergabe hörbar bleibt. Die Funktion arbeitet best effort und kann bei bestimmten Geräten, Bluetooth-Routen oder Linux-ALSA nicht verfügbar sein.', muteDuringRecordingLabel: 'Während der Aufnahme stummschalten', muteDuringRecordingDesc: 'Schaltet die Systemausgabe während der Spracheingabe vorübergehend stumm, um Lautsprecherechos zu vermeiden.', diff --git a/openless-all/app/src/i18n/en.ts b/openless-all/app/src/i18n/en.ts index 304d39dec..c0f7a4601 100644 --- a/openless-all/app/src/i18n/en.ts +++ b/openless-all/app/src/i18n/en.ts @@ -1524,6 +1524,9 @@ export const en: typeof zhCN = { audioEnhancementLabel: 'Lightweight microphone enhancement', audioEnhancementDesc: 'Applies light gain, soft limiting, and conservative noise suppression to the local microphone. It does not include echo cancellation. Off by default.', + echoCancellationLabel: 'Playback echo cancellation', + echoCancellationDesc: + 'Use the desktop audio output as a reference while keeping playback audible. This is best effort and may be unavailable for some devices, Bluetooth routes, or Linux ALSA setups.', muteDuringRecordingLabel: 'Mute while recording', muteDuringRecordingDesc: 'Temporarily mute system output during voice input to avoid speaker echo.', diff --git a/openless-all/app/src/i18n/es.ts b/openless-all/app/src/i18n/es.ts index 831db88b6..e5a53968a 100644 --- a/openless-all/app/src/i18n/es.ts +++ b/openless-all/app/src/i18n/es.ts @@ -1545,6 +1545,9 @@ export const es: typeof zhCN = { audioEnhancementLabel: 'Mejora ligera del micrófono', audioEnhancementDesc: 'Aplica una ganancia ligera, limitación suave y reducción de ruido conservadora al micrófono local. No incluye cancelación de eco. Desactivada de forma predeterminada.', + echoCancellationLabel: 'Cancelación del eco de reproducción', + echoCancellationDesc: + 'Usa la salida de audio del escritorio como referencia mientras mantiene el audio audible. Es una función de mejor esfuerzo y puede no estar disponible con ciertos dispositivos, rutas Bluetooth o ALSA en Linux.', muteDuringRecordingLabel: 'Silenciar durante la grabación', muteDuringRecordingDesc: 'Silencia temporalmente el sonido del sistema durante la entrada de voz para evitar el eco de los altavoces.', diff --git a/openless-all/app/src/i18n/fr.ts b/openless-all/app/src/i18n/fr.ts index 5a49334dc..511d4013a 100644 --- a/openless-all/app/src/i18n/fr.ts +++ b/openless-all/app/src/i18n/fr.ts @@ -1565,6 +1565,9 @@ export const fr: typeof zhCN = { audioEnhancementLabel: 'Amélioration légère du microphone', audioEnhancementDesc: 'Applique un gain léger, une limitation douce et une réduction prudente du bruit au microphone local. N’inclut pas l’annulation d’écho. Désactivée par défaut.', + echoCancellationLabel: 'Annulation de l’écho de lecture', + echoCancellationDesc: + 'Utilise la sortie audio du bureau comme référence tout en laissant la lecture audible. Cette fonction est expérimentale et peut être indisponible avec certains appareils, Bluetooth ou ALSA sous Linux.', muteDuringRecordingLabel: 'Couper le son pendant l’enregistrement', muteDuringRecordingDesc: 'Coupe temporairement le son système pendant la saisie vocale pour éviter l’écho des haut-parleurs.', diff --git a/openless-all/app/src/i18n/ja.ts b/openless-all/app/src/i18n/ja.ts index c894554ed..48b23cdea 100644 --- a/openless-all/app/src/i18n/ja.ts +++ b/openless-all/app/src/i18n/ja.ts @@ -1512,6 +1512,9 @@ export const ja: typeof zhCN = { audioEnhancementLabel: '軽量マイク拡張', audioEnhancementDesc: 'ローカルマイクに軽いノイズ抑制、ゲイン、ソフトリミッターを適用します。エコーキャンセルは含みません。既定ではオフです。', + echoCancellationLabel: '再生エコーキャンセル', + echoCancellationDesc: + '再生音を聞きながら、デスクトップの出力を参照してエコーを抑制します。ベストエフォートの機能で、デバイスや Bluetooth、Linux ALSA では利用できない場合があります。', muteDuringRecordingLabel: '録音中はミュート', muteDuringRecordingDesc: '録音中にシステム出力を一時的にミュートし、スピーカーのエコーを防ぎます。', diff --git a/openless-all/app/src/i18n/ko.ts b/openless-all/app/src/i18n/ko.ts index 59897bc5a..e4e1b13f7 100644 --- a/openless-all/app/src/i18n/ko.ts +++ b/openless-all/app/src/i18n/ko.ts @@ -1500,6 +1500,9 @@ export const ko: typeof zhCN = { audioEnhancementLabel: '가벼운 마이크 향상', audioEnhancementDesc: '로컬 마이크에 가벼운 소음 억제, 게인과 소프트 리미터를 적용합니다. 에코 제거는 포함하지 않으며 기본값은 꺼짐입니다.', + echoCancellationLabel: '재생 에코 제거', + echoCancellationDesc: + '재생을 계속 들으면서 데스크톱 오디오 출력을 참조해 에코를 줄입니다. 최선의 노력으로 동작하며 일부 장치, Bluetooth 경로 또는 Linux ALSA에서는 사용할 수 없을 수 있습니다.', muteDuringRecordingLabel: '녹음 중 음소거', muteDuringRecordingDesc: '녹음 중 시스템 출력을 일시적으로 음소거하여 스피커 에코를 방지합니다.', diff --git a/openless-all/app/src/i18n/zh-CN.ts b/openless-all/app/src/i18n/zh-CN.ts index 021c8516d..228594af7 100644 --- a/openless-all/app/src/i18n/zh-CN.ts +++ b/openless-all/app/src/i18n/zh-CN.ts @@ -1450,6 +1450,9 @@ export const zhCN = { '保持麦克风流就绪，并在内存中保留最多 500 毫秒音频，减少按下热键后第一句话被漏识别的情况。确认录音前不会保存或上传音频；此模式会增加功耗并保持系统麦克风指示灯亮起。', audioEnhancementLabel: '轻量麦克风增强', audioEnhancementDesc: '对本机麦克风应用轻量降噪、增益与软限幅，不包含回声消除。默认关闭。', + echoCancellationLabel: '播放回声抑制', + echoCancellationDesc: + '在保持系统播放声音的同时，使用桌面音频输出作为参考抑制回声。此功能为尽力而为，部分设备、蓝牙路径或 Linux ALSA 环境可能不可用。', voiceActivationLabel: '语音唤醒听写', voiceActivationDesc: '仅在 Windows 桌面端启用本地语音命令唤醒听写。待机音频只在内存中处理，不会上传或保存；会增加功耗并保持麦克风指示灯亮起。', diff --git a/openless-all/app/src/i18n/zh-TW.ts b/openless-all/app/src/i18n/zh-TW.ts index 9498ddf6c..43db2ee90 100644 --- a/openless-all/app/src/i18n/zh-TW.ts +++ b/openless-all/app/src/i18n/zh-TW.ts @@ -1452,6 +1452,9 @@ export const zhTW: typeof zhCN = { '保持麥克風串流就緒，並在記憶體中保留最多 500 毫秒音訊，減少按下熱鍵後第一句話被漏辨識的情況。確認錄音前不會儲存或上傳音訊；此模式會增加耗電並保持系統麥克風指示燈亮起。', audioEnhancementLabel: '輕量麥克風增強', audioEnhancementDesc: '對本機麥克風套用輕量降噪、增益與軟限幅，不包含回聲消除。預設關閉。', + echoCancellationLabel: '播放回聲抑制', + echoCancellationDesc: + '在保持系統播放聲音的同時，使用桌面音訊輸出作為參考抑制回聲。此功能為盡力而為，部分裝置、藍牙路徑或 Linux ALSA 環境可能不可用。', muteDuringRecordingLabel: '錄音時靜音', muteDuringRecordingDesc: '錄音期間臨時靜音系統輸出，避免揚聲器迴音。', audioCueLabel: '錄音提示音', diff --git a/openless-all/app/src/lib/ipc/mock-data.ts b/openless-all/app/src/lib/ipc/mock-data.ts index baa9046dd..42e40be0b 100644 --- a/openless-all/app/src/lib/ipc/mock-data.ts +++ b/openless-all/app/src/lib/ipc/mock-data.ts @@ -55,6 +55,7 @@ export let mockSettings: UserPreferences = { lowLatencyDictationEnabled: false, lowMemoryBackgroundEnabled: false, audioEnhancementEnabled: false, + echoCancellationEnabled: false, voiceActivationEnabled: false, voiceActivationWakeEnabled: true, voiceActivationFinishEnabled: true, diff --git a/openless-all/app/src/lib/types.ts b/openless-all/app/src/lib/types.ts index e45098ce0..61014da76 100644 --- a/openless-all/app/src/lib/types.ts +++ b/openless-all/app/src/lib/types.ts @@ -375,6 +375,8 @@ export interface UserPreferences { lowMemoryBackgroundEnabled: boolean; /** Apply conservative noise suppression, lightweight gain, and soft limiting to local microphone input. Disabled by default. */ audioEnhancementEnabled: boolean; + /** Use a desktop render/monitor reference for best-effort echo cancellation. Disabled by default. */ + echoCancellationEnabled: boolean; /** Enable local-only voice activation while idle (Windows desktop only). */ voiceActivationEnabled: boolean; voiceActivationWakeEnabled: boolean; diff --git a/openless-all/app/src/pages/settings/RecordingInputSection.tsx b/openless-all/app/src/pages/settings/RecordingInputSection.tsx index 81fb37227..9b5c76a99 100644 --- a/openless-all/app/src/pages/settings/RecordingInputSection.tsx +++ b/openless-all/app/src/pages/settings/RecordingInputSection.tsx @@ -78,6 +78,7 @@ export function RecordingInputSection({ const [platformCaps, setPlatformCaps] = useState( getCachedPlatformCapabilities, ); + const supportsDesktopEchoCancellation = platformCaps?.platform === 'desktop'; const [microphoneDevices, setMicrophoneDevices] = useState([]); const [microphoneDevicesLoaded, setMicrophoneDevicesLoaded] = useState(false); const [microphoneDevicesError, setMicrophoneDevicesError] = useState(null); @@ -292,6 +293,8 @@ export function RecordingInputSection({ savePrefs({ ...prefs, lowLatencyDictationEnabled }); const onAudioEnhancementChange = (audioEnhancementEnabled: boolean) => savePrefs({ ...prefs, audioEnhancementEnabled }); + const onEchoCancellationChange = (echoCancellationEnabled: boolean) => + savePrefs({ ...prefs, echoCancellationEnabled }); const onMicrophoneDeviceChange = (microphoneDeviceName: string) => savePrefs({ ...prefs, microphoneDeviceName }); const onRestoreClipboardChange = (restoreClipboardAfterPaste: boolean) => @@ -552,6 +555,17 @@ export function RecordingInputSection({ )} + {supportsDesktopEchoCancellation && ( + + + + )} {!isAndroid && (