diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d5b953220..8537819cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -169,6 +169,12 @@ jobs: - name: Check Android updater pubkey matches tauri.conf.json run: npm run check:android-updater-pubkey + # arm64 Android links sherpa-onnx's shared libraries (local ASR). Its + # build script finds them through SHERPA_ONNX_LIB_DIR and has no Android + # prebuilt download of its own. + - name: Prepare sherpa-onnx Android libraries + run: echo "SHERPA_ONNX_LIB_DIR=$(node scripts/sherpa-onnx-android.mjs)" >> "$GITHUB_ENV" + - name: Check Tauri backend (Android target) run: cargo check --locked --manifest-path src-tauri/Cargo.toml --target aarch64-linux-android diff --git a/openless-all/app/android/kotlin/OpenLessImeService.kt b/openless-all/app/android/kotlin/OpenLessImeService.kt index 98a3381b5..c60c5f5aa 100644 --- a/openless-all/app/android/kotlin/OpenLessImeService.kt +++ b/openless-all/app/android/kotlin/OpenLessImeService.kt @@ -328,6 +328,14 @@ class OpenLessImeService : InputMethodService(), OpenLessOverlayBridge.OverlaySt // voiceButton/status below. private var backendLinkIndicator: View? = null private var backendLinkPulseAnimator: android.animation.ObjectAnimator? = null + // Which recognizer the next dictation uses, as nativeAsrRoute() reports + // it ("cloud", "local", "localRaw", "unavailable"). The idle dot shows + // local in dark green and pure-local raw output (no usable polish + // channel, or no network to reach it) in light white. Refreshed off the + // main thread by refreshAsrRoute(). + private var asrRoute = "cloud" + private var networkAvailable = true + private val asrRouteRefreshing = java.util.concurrent.atomic.AtomicBoolean(false) // Updated only by the heartbeat check (see startBackendHeartbeat()), // not by the normal recording/processing/done states — those already // take priority in updateBackendLinkIndicator(). False means the most @@ -3416,6 +3424,8 @@ class OpenLessImeService : InputMethodService(), OpenLessOverlayBridge.OverlaySt // show. Keyboard settings write height/raise prefs while that cached // view is still the old footprint — rebuild when they diverge so the // next focus actually shows what the settings preview promised. + // A channel may have been turned off in settings since the last show. + if (isBackendReady()) refreshAsrRoute() val key = keyboardFootprintKey() if (appliedKeyboardFootprintKey != key) { appliedKeyboardFootprintKey = key @@ -3613,6 +3623,8 @@ class OpenLessImeService : InputMethodService(), OpenLessOverlayBridge.OverlaySt if (!ready) { OpenLessProcessRestartStats(this, "heartbeat").recordStart() ensureBackendReady() + } else { + refreshAsrRoute() } } @@ -3637,13 +3649,51 @@ class OpenLessImeService : InputMethodService(), OpenLessOverlayBridge.OverlaySt recording -> LINK_COLOR_RECORDING processing -> LINK_COLOR_PROCESSING !backendLinkHealthy -> LINK_COLOR_ISSUE + asrRoute == "localRaw" || (asrRoute == "local" && !networkAvailable) -> LINK_COLOR_READY_LOCAL_RAW + asrRoute == "local" -> LINK_COLOR_READY_LOCAL + asrRoute == "unavailable" -> LINK_COLOR_ISSUE else -> LINK_COLOR_READY } - (backendLinkIndicator?.background as? GradientDrawable)?.setColor(color) + (backendLinkIndicator?.background as? GradientDrawable)?.apply { + setColor(color) + // The light-white dot would vanish on the light keyboard without an edge. + if (color == LINK_COLOR_READY_LOCAL_RAW) setStroke(dp(1), Color.rgb(148, 163, 184)) else setStroke(0, Color.TRANSPARENT) + } + val ready = color == LINK_COLOR_READY || color == LINK_COLOR_READY_LOCAL || color == LINK_COLOR_READY_LOCAL_RAW backendLinkPulseAnimator?.duration = - if (color == LINK_COLOR_READY) BACKEND_LINK_READY_PULSE_DURATION_MS else BACKEND_LINK_PULSE_DURATION_MS + if (ready) BACKEND_LINK_READY_PULSE_DURATION_MS else BACKEND_LINK_PULSE_DURATION_MS + } + + /** + * Asks the backend whether the next dictation runs locally and repaints + * the dot. The call reads the credential store, so it runs on its own + * thread; overlapping requests (heartbeat plus a keyboard show) collapse + * into one. + */ + private fun refreshAsrRoute() { + if (!asrRouteRefreshing.compareAndSet(false, true)) return + Thread { + val route = runCatching { OpenLessNative.nativeAsrRoute() }.getOrNull() ?: "unavailable" + val online = hasValidatedNetwork() + backendHeartbeatHandler.post { + asrRouteRefreshing.set(false) + if (route != asrRoute || online != networkAvailable) { + asrRoute = route + networkAvailable = online + updateBackendLinkIndicator() + } + } + }.apply { name = "openless-asr-route" }.start() } + /** Whether the active network reaches the internet — polish needs it; local recognition doesn't. */ + private fun hasValidatedNetwork(): Boolean = runCatching { + val connectivity = getSystemService(android.net.ConnectivityManager::class.java) ?: return@runCatching true + val capabilities = connectivity.getNetworkCapabilities(connectivity.activeNetwork) ?: return@runCatching false + capabilities.hasCapability(android.net.NetworkCapabilities.NET_CAPABILITY_INTERNET) && + capabilities.hasCapability(android.net.NetworkCapabilities.NET_CAPABILITY_VALIDATED) + }.getOrDefault(true) + // Guards scheduleBackendReadyRecheck() so a second mic tap while a // recheck loop is already running doesn't stack a duplicate one. private var awaitingBackendReadyRecheck = false @@ -5915,6 +5965,10 @@ class OpenLessImeService : InputMethodService(), OpenLessOverlayBridge.OverlaySt // (recording/processing), plus a light-green "ready" and an amber // "link issue" that overlay doesn't have. private val LINK_COLOR_READY = Color.rgb(134, 239, 172) + // Ready, and the next dictation runs on the local model (asrRoute). + private val LINK_COLOR_READY_LOCAL = Color.rgb(21, 128, 61) + // Ready, local model and raw output: no usable polish channel or no network. + private val LINK_COLOR_READY_LOCAL_RAW = Color.rgb(241, 245, 249) private val LINK_COLOR_RECORDING = Color.rgb(244, 63, 94) // Recording with rawModeArmed set (mic swipe-up: skip LLM polish on // stop) — distinct orange so glancing at the dot tells the two diff --git a/openless-all/app/android/kotlin/OpenLessKeyboardSettingsActivity.kt b/openless-all/app/android/kotlin/OpenLessKeyboardSettingsActivity.kt index 8a806a7ad..d866f8692 100644 --- a/openless-all/app/android/kotlin/OpenLessKeyboardSettingsActivity.kt +++ b/openless-all/app/android/kotlin/OpenLessKeyboardSettingsActivity.kt @@ -578,6 +578,21 @@ class OpenLessKeyboardSettingsActivity : Activity() { }, ) + // "语音识别来源": how many dictations in history the cloud and the local + // model each recognized (with the local standby racing the cloud + // either can win), the latest one's kind highlighted. + content.addView(sectionLabel(ui("语音识别来源", "Speech recognition source"))) + val asrUsageText = TextView(this).apply { + text = ui("读取中…", "Loading…") + textSize = 14f + setTextColor(tone(Color.rgb(220, 220, 220), Color.rgb(50, 50, 55))) + } + content.addView(asrUsageText) + Thread { + val usage = runCatching { OpenLessNative.nativeAsrUsage() }.getOrNull() + runOnUiThread { asrUsageText.text = asrUsageSummary(usage) } + }.apply { name = "openless-asr-usage" }.start() + // build_first_seen_wall_time is written by OpenLessApplication's // resetRestartStatsOnVersionBump() at the exact moment it last // zeroed the restart-cause counters below — i.e. "counting since @@ -744,6 +759,25 @@ class OpenLessKeyboardSettingsActivity : Activity() { } } + /** "云 N · 本 M" from nativeAsrUsage()'s ",,", the latest kind bold and colored. */ + private fun asrUsageSummary(usage: String?): CharSequence { + val parts = usage?.split(',') ?: return ui("后台服务未就绪", "Backend not ready") + val local = parts.getOrNull(0)?.toIntOrNull() ?: 0 + val cloud = parts.getOrNull(1)?.toIntOrNull() ?: 0 + if (local + cloud == 0) return ui("暂无听写记录", "No dictations yet") + val cloudText = ui("云 $cloud", "Cloud $cloud") + val localText = ui("本 $local", "Local $local") + val text = android.text.SpannableString("$cloudText · $localText") + val (start, end, color) = when (parts.getOrNull(2)) { + "cloud" -> Triple(0, cloudText.length, tone(Color.rgb(96, 165, 250), Color.rgb(37, 99, 235))) + "local" -> Triple(text.length - localText.length, text.length, tone(Color.rgb(74, 222, 128), Color.rgb(21, 128, 61))) + else -> return text + } + text.setSpan(android.text.style.ForegroundColorSpan(color), start, end, android.text.Spanned.SPAN_EXCLUSIVE_EXCLUSIVE) + text.setSpan(android.text.style.StyleSpan(android.graphics.Typeface.BOLD), start, end, android.text.Spanned.SPAN_EXCLUSIVE_EXCLUSIVE) + return text + } + private fun sectionLabel(text: String): View = TextView(this).apply { this.text = text textSize = 12f diff --git a/openless-all/app/android/kotlin/OpenLessNative.kt b/openless-all/app/android/kotlin/OpenLessNative.kt index 602595940..c3f41c6b9 100644 --- a/openless-all/app/android/kotlin/OpenLessNative.kt +++ b/openless-all/app/android/kotlin/OpenLessNative.kt @@ -95,6 +95,11 @@ object OpenLessNative { @JvmStatic external fun nativeHasRegisteredActivityContext(): Boolean @JvmStatic external fun nativeBackendSnapshot(): String + + /** "local", "cloud" or "unavailable" — which recognizer the next dictation uses. Blocks on a credential read: call off the main thread. */ + @JvmStatic external fun nativeAsrRoute(): String? + /** ",," dictation counts from history; latest is "local", "cloud" or empty. Reads history: call off the main thread. */ + @JvmStatic external fun nativeAsrUsage(): String? @JvmStatic external fun nativeImeCommand(json: String): String @JvmStatic external fun nativeCloudPinyin(pinyin: String): String @JvmStatic external fun nativePublicDictionary(base: String, action: String): String diff --git a/openless-all/app/crates/openless-core/src/api.rs b/openless-all/app/crates/openless-core/src/api.rs index ee9124023..0ead27d68 100644 --- a/openless-all/app/crates/openless-core/src/api.rs +++ b/openless-all/app/crates/openless-core/src/api.rs @@ -2305,6 +2305,29 @@ impl EngineProgressSink for BackendEngineProgress { } } +/// Which kind of recognizer the next dictation uses — see OpenLessBackend::asr_route(). +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub enum AsrRoute { + Cloud, + Local, + /// Local recognition with no usable polish channel: the raw transcript + /// is the output (see capture_dictation_context_with_host). + LocalRaw, + /// No usable channel: the next dictation would fail to start. + Unavailable, +} + +/// Which recognizer produced the dictations in history — see +/// OpenLessBackend::asr_usage(). +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct AsrUsage { + pub local: u64, + pub cloud: u64, + /// Whether the latest dictation was recognized locally; None without history. + pub last_local: Option, +} + pub struct OpenLessBackend { config: BackendConfig, startup_error: Option, @@ -4949,6 +4972,242 @@ impl OpenLessBackend { self.release_local_asr_provider(&previous.1).await } + /// Whether the selected channel is one of the shared cloud ASR providers. + /// Disabled channel errors also occur for local channels, which must not + /// silently switch to another local model. + async fn selected_asr_channel_is_shared_cloud(&self, preference_fallback: &str) -> bool { + let selected_id = match self + .deps + .credential_store + .active_provider(ProviderSlot::Asr) + .await + { + Ok(provider) if !provider.trim().is_empty() => provider, + _ => preference_fallback.to_string(), + }; + let provider_type = match self + .deps + .credential_store + .list_channels(ChannelKind::Asr) + .await + { + Ok(channels) => channels + .into_iter() + .find(|channel| channel.id == selected_id) + .map(|channel| channel.provider_type) + .unwrap_or(selected_id), + Err(_) => selected_id, + }; + crate::cloud_providers::SHARED_CLOUD_ASR_PROVIDER_TYPES.contains(&provider_type.as_str()) + } + + /// The local model dictation switches to when the selected cloud ASR + /// channel is turned off or unconfigured: only where local ASR runs, with a sherpa-onnx + /// model selected and its files installed — otherwise the disabled-channel + /// error stands. Model, language and keep-loaded time are filled in from + /// the same preferences a manual switch would use (DictationContext::capture). + fn local_asr_switch_target( + &self, + preferences: &crate::UserPreferences, + ) -> Option { + if !crate::shared_types::PlatformCapabilities::current().supports_local_asr { + return None; + } + let target = crate::LocalAsrTarget::parse( + crate::LocalAsrRuntime::SherpaOnnx, + preferences.sherpa_onnx_model.trim(), + ) + .ok()?; + let installed = self + .deps + .services + .model_store + .as_ref()? + .is_installed(&target) + .unwrap_or(false); + installed.then(|| { + crate::dictation_context::ProviderInvocation::for_provider( + crate::LocalAsrRuntime::SherpaOnnx.provider_id(), + ) + }) + } + + async fn resolve_dictation_asr_provider( + &self, + preferences: &crate::UserPreferences, + ) -> Result { + let provider = match self + .resolve_session_provider(ProviderSlot::Asr, &preferences.active_asr_provider) + .await + { + Ok(provider) => provider, + Err(error) + if error.code == BackendErrorCode::InvalidState + && self + .selected_asr_channel_is_shared_cloud(&preferences.active_asr_provider) + .await => + { + return self.local_asr_switch_target(preferences).ok_or(error); + } + Err(error) => return Err(error), + }; + if crate::cloud_providers::SHARED_CLOUD_ASR_PROVIDER_TYPES + .contains(&provider.provider_type.as_str()) + { + if let Some(local) = self.local_asr_switch_target(preferences) { + if !crate::cloud_providers::asr_channel_configured( + self.deps.credential_store.as_ref(), + &provider, + ) + .await? + { + return Ok(local); + } + } + } + Ok(provider) + } + + async fn selected_local_asr_model_is_installed( + &self, + runtime: crate::LocalAsrRuntime, + preferences: &crate::UserPreferences, + ) -> bool { + let model = match runtime { + crate::LocalAsrRuntime::Foundry => &preferences.foundry_local_asr_model, + crate::LocalAsrRuntime::SherpaOnnx => &preferences.sherpa_onnx_model, + crate::LocalAsrRuntime::Generic => return false, + }; + let Ok(target) = crate::LocalAsrTarget::parse(runtime, model.trim()) else { + return false; + }; + self.deps + .services + .local_asr + .list_models(runtime) + .await + .ok() + .is_some_and(|models| { + models + .into_iter() + .any(|entry| entry.target == target && entry.installed) + }) + } + + /// Which recognizer the next dictation will use, for hosts that show it + /// (the Android keyboard's status dot): the selected channel's own kind, + /// or Local when that channel is disabled or unconfigured and an installed + /// local model steps in. + pub async fn asr_route(&self) -> AsrRoute { + let preferences = self.get_preferences(); + let local = match self.resolve_dictation_asr_provider(&preferences).await { + Ok(provider) => { + let Some(runtime) = Self::local_asr_runtime(&provider.provider_type) else { + return AsrRoute::Cloud; + }; + if !self.deps.services.local_asr.engine_available(runtime) { + return AsrRoute::Unavailable; + } + if runtime == crate::LocalAsrRuntime::Foundry { + // Foundry's engine is compiled in even when the selected + // native model receipt is missing or a download/storage + // operation requires a process restart. Do not advertise a + // route that the next prepare call will reject. + let status = match self + .deps + .services + .local_asr + .runtime_status(runtime) + .await + { + Ok(status) => status, + Err(_) => return AsrRoute::Unavailable, + }; + if status.restart_required + || !self + .selected_local_asr_model_is_installed(runtime, &preferences) + .await + { + return AsrRoute::Unavailable; + } + } + // Sherpa is a real local route only when the selected model is + // installed on this ABI. The native engine otherwise rejects + // the dictation after the route has already been displayed. + if runtime == crate::LocalAsrRuntime::SherpaOnnx + && self.local_asr_switch_target(&preferences).is_none() + { + return AsrRoute::Unavailable; + } + true + } + Err(_) => return AsrRoute::Unavailable, + }; + if !local { + return AsrRoute::Cloud; + } + // Same test the dictation context applies: a polish channel that is + // turned off or not configured means raw output. + let polish_usable = match self + .resolve_session_provider(ProviderSlot::Llm, &preferences.active_llm_provider) + .await + { + Ok(llm) => { + crate::cloud_providers::llm_channel_configured( + self.deps.credential_store.as_ref(), + &llm.provider_id, + &llm.provider_type, + llm.model.as_deref(), + ) + .await + } + Err(_) => false, + }; + if polish_usable { + AsrRoute::Local + } else { + AsrRoute::LocalRaw + } + } + + /// How many dictations in history the local model and the cloud each + /// recognized, and which one the latest used. Counted from history, so it + /// survives restarts and follows the user's retention and deletions. + pub fn asr_usage(&self) -> Result { + let mut usage = AsrUsage::default(); + let mut latest: Option<(String, bool)> = None; + for entry in self.history.list()? { + // Recording drafts and terminal ASR failures retain the provider + // selected at start-up, but they never produced a recognition. + if matches!( + entry.error_code.as_deref(), + Some( + "recording" + | "cancelled" + | "recordingFailed" + | "transcribeFailed" + | "emptyTranscript" + ) + ) { + continue; + } + let Some(provider) = entry.asr_provider.as_deref() else { + continue; + }; + let local = crate::dictation_context::is_local_asr_provider(provider); + if local { + usage.local += 1; + } else { + usage.cloud += 1; + } + if latest.as_ref().is_none_or(|(created_at, _)| entry.created_at > *created_at) { + latest = Some((entry.created_at, local)); + } + } + usage.last_local = latest.map(|(_, local)| local); + Ok(usage) + } + fn local_asr_runtime(provider: &str) -> Option { match provider { "local-qwen3" | "local-qwen3-mlx" | "local-qwen3-c" | "local-whisper" @@ -7393,8 +7652,7 @@ impl OpenLessBackend { ); let traditional = pipeline_mode == crate::shared_types::PipelineMode::Traditional; let active_asr_provider = if traditional && purpose != DictationContextPurpose::QaText { - self.resolve_session_provider(ProviderSlot::Asr, &preferences.active_asr_provider) - .await? + self.resolve_dictation_asr_provider(&preferences).await? } else { crate::dictation_context::ProviderInvocation::for_provider( &preferences.active_asr_provider, @@ -7477,6 +7735,17 @@ impl OpenLessBackend { { context = context.with_unconfigured_llm_raw(); } + // Pure local: recognition runs on this device and the polish channel + // is turned off too — output the raw transcript instead of refusing to + // start. The error is kept below for a later translation request. + if purpose == DictationContextPurpose::Dictation + && traditional + && deferred_llm_error.is_some() + && context.uses_llm_polisher() + && crate::dictation_context::is_local_asr_provider(&context.asr.provider_type) + { + context = context.with_unconfigured_llm_raw(); + } if let Some(error) = deferred_llm_error { if purpose != DictationContextPurpose::Dictation || context.uses_llm_polisher() { return Err(error); @@ -9898,6 +10167,109 @@ mod tests { } } + #[test] + fn asr_usage_counts_local_and_cloud_dictations_and_the_latest_kind() { + let data_dir = std::env::temp_dir().join(format!( + "openless-core-backend-asr-usage-{}", + uuid::Uuid::new_v4().simple() + )); + let backend = backend_with_dictation_engine(data_dir.clone(), Arc::new(FakeEngine)); + assert_eq!(backend.asr_usage().unwrap(), AsrUsage::default()); + + let entries = [ + ("a", "2026-10-18T10:00:00+00:00", Some("volcengine")), + ("b", "2026-10-18T10:01:00+00:00", Some("sherpa-onnx-local")), + ("c", "2026-10-18T10:02:00+00:00", None), + ("d", "2026-10-18T10:03:00+00:00", Some("volcengine")), + ("e", "2026-10-18T10:04:00+00:00", Some("sherpa-onnx-local")), + ]; + for (id, created_at, provider) in entries { + let mut entry = history_session(id); + entry.created_at = created_at.into(); + entry.asr_provider = provider.map(Into::into); + backend.append_history(entry, 30, None).unwrap(); + } + + assert_eq!( + backend.asr_usage().unwrap(), + AsrUsage { local: 2, cloud: 2, last_local: Some(true) } + ); + let _ = std::fs::remove_dir_all(data_dir); + } + + #[test] + fn asr_usage_ignores_entries_without_a_successful_recognition() { + let data_dir = std::env::temp_dir().join(format!( + "openless-core-backend-asr-usage-failures-{}", + uuid::Uuid::new_v4().simple() + )); + let backend = backend_with_dictation_engine(data_dir.clone(), Arc::new(FakeEngine)); + + let entries = [ + ( + "completed-local", + "2026-10-18T10:00:00+00:00", + "sherpa-onnx-local", + None, + ), + ( + "recording-draft", + "2026-10-18T10:01:00+00:00", + "volcengine", + Some("recording"), + ), + ( + "cancelled", + "2026-10-18T10:02:00+00:00", + "sherpa-onnx-local", + Some("cancelled"), + ), + ( + "recording-failed", + "2026-10-18T10:03:00+00:00", + "volcengine", + Some("recordingFailed"), + ), + ( + "transcribe-failed", + "2026-10-18T10:04:00+00:00", + "sherpa-onnx-local", + Some("transcribeFailed"), + ), + ( + "empty-transcript", + "2026-10-18T10:05:00+00:00", + "volcengine", + Some("emptyTranscript"), + ), + ( + "insert-failed-after-asr", + "2026-10-18T10:06:00+00:00", + "volcengine", + Some("insertFailed"), + ), + ( + "polish-failed-after-asr", + "2026-10-18T10:07:00+00:00", + "sherpa-onnx-local", + Some("polishFailed"), + ), + ]; + for (id, created_at, provider, error_code) in entries { + let mut entry = history_session(id); + entry.created_at = created_at.into(); + entry.asr_provider = Some(provider.into()); + entry.error_code = error_code.map(Into::into); + backend.append_history(entry, 30, None).unwrap(); + } + + assert_eq!( + backend.asr_usage().unwrap(), + AsrUsage { local: 2, cloud: 1, last_local: Some(true) } + ); + let _ = std::fs::remove_dir_all(data_dir); + } + #[test] fn vocabulary_facade_persists_shared_types_and_publishes_revisions() { let data_dir = std::env::temp_dir().join(format!( @@ -13419,6 +13791,126 @@ mod tests { backend.shutdown().await.unwrap(); } + #[cfg(target_os = "windows")] + #[tokio::test] + async fn unconfigured_cloud_capture_switches_only_with_an_installed_local_model() { + let data_dir = TestDataDir::new("unconfigured-cloud-local-switch"); + let mut backend = + backend_with_dictation_engine(data_dir.path().to_path_buf(), Arc::new(FakeEngine)); + backend.deps.services.model_store = Some(Arc::new( + crate::ModelStore::new( + crate::ModelStoreConfig::new(data_dir.path().join("models")).unwrap(), + ) + .unwrap(), + )); + let channel = backend + .create_channel(ChannelKind::Asr, "whisper".into(), "Cloud ASR".into()) + .await + .unwrap(); + backend + .set_active_provider(ProviderSlot::Asr, channel.clone()) + .await + .unwrap(); + let options = DictationStartOptions::default(); + let capture = + || backend.capture_dictation_context(&options, DictationContextPurpose::Dictation); + assert_eq!(capture().await.unwrap().asr.provider_type, "whisper"); + let target = crate::LocalAsrTarget::parse( + crate::LocalAsrRuntime::SherpaOnnx, + "sense-voice-small-zh", + ) + .unwrap(); + let dir = backend + .deps + .services + .model_store + .as_ref() + .unwrap() + .model_dir(&target) + .unwrap(); + std::fs::create_dir_all(&dir).unwrap(); + for (file, bytes) in [ + (crate::MODEL_READY_SENTINEL, "ready"), + ("model.int8.onnx", "model"), + ("tokens.txt", "tokens"), + ] { + std::fs::write(dir.join(file), bytes).unwrap(); + } + let local = capture().await.unwrap(); + assert_eq!(local.asr.provider_type, "sherpa-onnx-local"); + assert_eq!(local.asr.model.as_deref(), Some("sense-voice-small-zh")); + backend + .deps + .credential_store + .write( + crate::CredentialKey::new( + crate::CredentialNamespace::Asr, + Some(channel), + "asr.api_key", + ) + .unwrap(), + crate::SecretValue::new("fixture-key"), + ) + .await + .unwrap(); + assert_eq!(capture().await.unwrap().asr.provider_type, "whisper"); + } + + #[tokio::test] + async fn local_recognition_with_a_disabled_llm_dictates_raw_text() { + let data_dir = TestDataDir::new("local-disabled-llm"); + let polisher = Arc::new(crate::testing::FixtureTextPolisher::successful( + "must not run", + )); + let backend = backend_with_dictation_engine( + data_dir.path().to_path_buf(), + Arc::new(crate::PipelineDictationEngine::new( + Arc::new(crate::testing::FixtureAudioRecorder::new( + vec![vec![1, 0]], + Vec::new(), + )), + Arc::new(crate::testing::FixtureTranscriptionEngine::successful( + "raw words", + 80, + )), + polisher.clone(), + )), + ); + let llm = backend + .create_channel( + ChannelKind::Llm, + "ark".to_string(), + "Unused LLM".to_string(), + ) + .await + .unwrap(); + let mut preferences = backend.get_preferences(); + preferences.active_llm_provider = llm.clone(); + preferences.active_asr_provider = "sherpa-onnx-local".to_string(); + backend.set_preferences(preferences).unwrap(); + backend + .set_active_provider(ProviderSlot::Llm, llm.clone()) + .await + .unwrap(); + backend + .set_channel_enabled(ChannelKind::Llm, llm, false) + .await + .unwrap(); + // A polishing style: with cloud ASR this start would be refused. + backend.activate_style_pack("builtin.light").unwrap(); + backend.start().await.unwrap(); + + backend + .start_dictation() + .await + .expect("local recognition dictates raw text without a polish channel"); + let result = backend.stop_dictation().await.unwrap(); + + assert_eq!(result.polished_text, "raw words"); + assert!(polisher.inputs().is_empty()); + backend.shutdown().await.unwrap(); + } + #[tokio::test] async fn raw_dictation_remains_usable_when_the_last_llm_channel_is_disabled() { let data_dir = TestDataDir::new("raw-disabled-llm"); @@ -13449,6 +13941,9 @@ mod tests { .unwrap(); let mut preferences = backend.get_preferences(); preferences.active_llm_provider = llm.clone(); + // Cloud ASR: with local recognition a disabled LLM means raw output + // instead (local_recognition_with_a_disabled_llm_dictates_raw_text). + preferences.active_asr_provider = "volcengine".to_string(); preferences.working_languages = vec!["简体中文".to_string()]; preferences.translation_target_language = "English".to_string(); backend.set_preferences(preferences).unwrap(); diff --git a/openless-all/app/crates/openless-core/src/cloud_providers.rs b/openless-all/app/crates/openless-core/src/cloud_providers.rs index 9372fc885..a542bb897 100644 --- a/openless-all/app/crates/openless-core/src/cloud_providers.rs +++ b/openless-all/app/crates/openless-core/src/cloud_providers.rs @@ -411,6 +411,61 @@ where } } +async fn read_asr_channel_credentials( + credentials: &dyn CredentialStore, + channel_id: &str, +) -> Result>, BackendError> { + let mut values = HashMap::new(); + for account in ASR_SNAPSHOT_ACCOUNTS { + values.insert( + *account, + read_channel_credential(credentials, CredentialNamespace::Asr, channel_id, account) + .await?, + ); + } + Ok(values) +} + +pub(crate) async fn asr_channel_configured( + credentials: &dyn CredentialStore, + asr: &crate::dictation_context::ProviderInvocation, +) -> Result { + let values = read_asr_channel_credentials(credentials, &asr.provider_id).await?; + let value = |account: &str| { + values + .get(account) + .and_then(|value| value.as_deref()) + .map(str::trim) + .filter(|value| !value.is_empty()) + }; + let configuration = crate::provider_rules::CredentialConfiguration { + asr_api_key: value(ASR_API_KEY_ACCOUNT).is_some(), + asr_endpoint: value(ASR_ENDPOINT_ACCOUNT).is_some(), + asr_model: asr + .model + .as_deref() + .is_some_and(|model| !model.trim().is_empty()) + || value(ASR_MODEL_ACCOUNT).is_some(), + volcengine_service: value(crate::credentials::VOLCENGINE_SERVICE_ACCOUNT) + .map(str::to_string), + volcengine_auth_mode: value(VOLCENGINE_AUTH_MODE_ACCOUNT).map(str::to_string), + volcengine_app_key: value(VOLCENGINE_APP_KEY_ACCOUNT).is_some(), + volcengine_access_key: value(VOLCENGINE_ACCESS_KEY_ACCOUNT).is_some(), + volcengine_api_key: value(VOLCENGINE_API_KEY_ACCOUNT).is_some(), + xfyun_app_id: value(XFYUN_APP_ID_ACCOUNT).is_some(), + xfyun_api_key: value(XFYUN_API_KEY_ACCOUNT).is_some(), + tencent_cloud_app_id: value(TENCENT_CLOUD_APP_ID_ACCOUNT).is_some(), + tencent_cloud_secret_id: value(TENCENT_CLOUD_SECRET_ID_ACCOUNT).is_some(), + tencent_cloud_secret_key: value(TENCENT_CLOUD_SECRET_KEY_ACCOUNT).is_some(), + ..Default::default() + }; + Ok(crate::provider_rules::asr_configured( + &asr.provider_type, + &configuration, + None, + )) +} + async fn prepare_cloud_transcription( credentials: &dyn CredentialStore, context: Arc, @@ -431,14 +486,7 @@ async fn prepare_cloud_transcription( "shared cloud ASR provider is not supported", )); } - let mut values = HashMap::new(); - for account in ASR_SNAPSHOT_ACCOUNTS { - values.insert( - *account, - read_channel_credential(credentials, CredentialNamespace::Asr, &channel_id, account) - .await?, - ); - } + let values = read_asr_channel_credentials(credentials, &channel_id).await?; let value = |account: &str| values.get(account).cloned().flatten(); let model = context .asr @@ -1241,7 +1289,7 @@ async fn run_cloud_polish( if let Some(error) = stream.take_error() { return Err(error); } - let polished = result.map_err(map_llm_error)?; + let polished = result.map_err(map_polish_error)?; if cancellation.is_cancelled() { return Err(cancelled_provider_error()); } @@ -1297,7 +1345,7 @@ async fn run_cloud_polish( if let Some(error) = stream.take_error() { return Err(error); } - let text = result.map_err(map_llm_error)?; + let text = result.map_err(map_polish_error)?; PolishOutput::text(text.trim().to_owned()) } } @@ -1522,6 +1570,18 @@ fn map_llm_error(error: impl std::fmt::Display) -> BackendError { ) } +/// A polish failure, marked retryable when the network never answered +/// (connection failure or timeout) — unlike a rejected key, exhausted quota or +/// malformed reply. The dictation pipeline uses it to tell "offline" apart +/// when the transcript came from a local model. +fn map_polish_error(error: crate::polish::LLMError) -> BackendError { + let transient = matches!( + error, + crate::polish::LLMError::Network(_) | crate::polish::LLMError::Timeout + ); + map_llm_error(error).retryable(transient) +} + pub struct SharedAuxiliaryTextPolisher { credentials: Arc, traditional: Arc, @@ -3048,6 +3108,66 @@ mod tests { } } + #[tokio::test] + async fn asr_channel_configured_matches_provider_credential_requirements() { + for (provider, accounts) in [ + ("whisper", vec![ASR_API_KEY_ACCOUNT]), + ("bailian", vec![ASR_API_KEY_ACCOUNT]), + ( + "volcengine", + vec![VOLCENGINE_APP_KEY_ACCOUNT, VOLCENGINE_ACCESS_KEY_ACCOUNT], + ), + ("iflytek", vec![XFYUN_APP_ID_ACCOUNT, XFYUN_API_KEY_ACCOUNT]), + ( + "tencent-cloud", + vec![ + TENCENT_CLOUD_APP_ID_ACCOUNT, + TENCENT_CLOUD_SECRET_ID_ACCOUNT, + TENCENT_CLOUD_SECRET_KEY_ACCOUNT, + ], + ), + ] { + let store = InMemoryCredentialStore::default(); + let invocation = ProviderInvocation::new("cloud-channel", provider); + assert!( + !asr_channel_configured(&store, &invocation).await.unwrap(), + "{provider}" + ); + for account in accounts { + assert!(!asr_channel_configured(&store, &invocation).await.unwrap()); + write_channel_secret( + &store, + CredentialNamespace::Asr, + "cloud-channel", + account, + "fixture-key", + ) + .await; + } + assert!( + asr_channel_configured(&store, &invocation).await.unwrap(), + "{provider}" + ); + let context = Arc::new(DictationContext { + asr: invocation, + ..Default::default() + }); + assert!( + prepare_cloud_transcription(&store, context).await.is_ok(), + "{provider}" + ); + } + } + + #[tokio::test] + async fn asr_configuration_does_not_hide_credential_read_errors() { + let invocation = ProviderInvocation::new("channel", "whisper"); + let error = asr_channel_configured(&crate::UnsupportedCredentialStore, &invocation) + .await + .unwrap_err(); + assert_eq!(error.code, BackendErrorCode::Unsupported); + } + #[tokio::test] async fn llm_channel_configured_matches_polisher_requirements() { let store = InMemoryCredentialStore::default(); diff --git a/openless-all/app/crates/openless-core/src/cloud_sync_e2ee_documents/registry.rs b/openless-all/app/crates/openless-core/src/cloud_sync_e2ee_documents/registry.rs index 4ac190fef..1b5ee5a22 100644 --- a/openless-all/app/crates/openless-core/src/cloud_sync_e2ee_documents/registry.rs +++ b/openless-all/app/crates/openless-core/src/cloud_sync_e2ee_documents/registry.rs @@ -627,6 +627,13 @@ pub const PREFERENCE_FIELDS: &[PreferenceField] = &[ shape: PreferenceShape::Unsigned32, reason: "device_bound", }, + PreferenceField { + rust_name: "local_asr_fallback_enabled", + key: "localAsrFallbackEnabled", + class: PreferenceClass::DeviceProfile, + shape: PreferenceShape::Boolean, + reason: "device_bound", + }, PreferenceField { rust_name: "update_channel", key: "updateChannel", diff --git a/openless-all/app/crates/openless-core/src/dictation_context.rs b/openless-all/app/crates/openless-core/src/dictation_context.rs index 93a6cd75b..d12bb71c0 100644 --- a/openless-all/app/crates/openless-core/src/dictation_context.rs +++ b/openless-all/app/crates/openless-core/src/dictation_context.rs @@ -197,6 +197,9 @@ pub struct DictationContext { pub pipeline_mode: PipelineMode, pub correction_rules: Vec, pub asr: ProviderInvocation, + /// Local model a failed cloud ASR call is retried with, frozen at session + /// start (see local_asr_fallback()); None when the fallback doesn't apply. + pub asr_fallback: Option, pub llm: ProviderInvocation, /// Raw may start without an LLM. If translation is requested at stop, /// preserve the start-time resolution failure instead of using a disabled @@ -293,6 +296,11 @@ impl DictationContext { } let prior_turns = eligible_polish_context_turns(recent_history, &style_pack.id, translation_active); + let asr_fallback = local_asr_fallback( + preferences, + &asr, + crate::shared_types::PlatformCapabilities::current().supports_local_asr, + ); Self { audio_source: options.audio_source, output_target: options.output_target, @@ -331,6 +339,7 @@ impl DictationContext { ), correction_rules: Vec::new(), asr, + asr_fallback, llm, deferred_llm_error: None, unconfigured_llm_raw: None, @@ -652,6 +661,51 @@ pub fn eligible_polish_context_turns( .collect() } +/// Providers that recognize on this device: no network is involved, so a +/// session on one of them treats polish as optional (see dictation_engine). +pub(crate) fn is_local_asr_provider(provider: &str) -> bool { + matches!( + provider, + "apple-speech" + | "local-qwen3" + | "local-qwen3-mlx" + | "local-qwen3-c" + | "local-whisper" + | "apple-whisper" + | "foundry-local" + | "foundry-whisper" + | "foundry-local-whisper" + | "sherpa-onnx" + | "sherpa-onnx-local" + ) || crate::experimental_asr::ExperimentalAsrModel::from_provider(provider).is_some() +} + +/// The sherpa-onnx model a failed cloud ASR call falls back to: only when the +/// user enabled it, this platform has local ASR, the session uses one of the +/// shared cloud providers (a local provider has nothing to fall back to) and a +/// model is selected. Built from the same preferences a manual switch to the +/// local model would use; whether the model is actually installed is left to +/// the native engine, which refuses to start without it. +pub(crate) fn local_asr_fallback( + preferences: &UserPreferences, + asr: &ProviderInvocation, + local_asr_supported: bool, +) -> Option { + if !preferences.local_asr_fallback_enabled + || !local_asr_supported + || !crate::cloud_providers::SHARED_CLOUD_ASR_PROVIDER_TYPES.contains(&asr.provider_type.as_str()) + { + return None; + } + let provider = crate::local_asr_catalog::LocalAsrRuntime::SherpaOnnx.provider_id(); + let mut invocation = ProviderInvocation::new(provider, provider); + invocation.model = Some(non_blank(&preferences.sherpa_onnx_model)?); + invocation.language = non_blank(&preferences.sherpa_onnx_language_hint); + invocation.keep_loaded_secs = Some(preferences.sherpa_onnx_keep_loaded_secs); + invocation.prompt = asr.prompt.clone(); + Some(invocation) +} + fn selected_asr_details( preferences: &UserPreferences, provider: &str, @@ -700,6 +754,53 @@ mod tests { use super::*; use crate::style_packs::builtin_style_pack_for_mode; + fn fallback_preferences() -> UserPreferences { + UserPreferences { + local_asr_fallback_enabled: true, + sherpa_onnx_model: "sense-voice-small-zh".into(), + sherpa_onnx_language_hint: "zh".into(), + sherpa_onnx_keep_loaded_secs: 300, + ..UserPreferences::default() + } + } + + fn cloud_asr() -> ProviderInvocation { + let cloud = crate::cloud_providers::SHARED_CLOUD_ASR_PROVIDER_TYPES[0]; + let mut asr = ProviderInvocation::new("my-cloud-channel", cloud); + asr.prompt = Some("OpenLess.".into()); + asr + } + + #[test] + fn local_fallback_uses_the_selected_sherpa_model_and_its_settings() { + let fallback = local_asr_fallback(&fallback_preferences(), &cloud_asr(), true).unwrap(); + assert_eq!(fallback.provider_type, "sherpa-onnx-local"); + assert_eq!(fallback.provider_id, "sherpa-onnx-local"); + assert_eq!(fallback.model.as_deref(), Some("sense-voice-small-zh")); + assert_eq!(fallback.language.as_deref(), Some("zh")); + assert_eq!(fallback.keep_loaded_secs, Some(300)); + // Hotwords carry over from the session's ASR prompt. + assert_eq!(fallback.prompt.as_deref(), Some("OpenLess.")); + } + + #[test] + fn local_fallback_needs_the_switch_local_asr_a_cloud_provider_and_a_model() { + let preferences = fallback_preferences(); + let mut off = preferences.clone(); + off.local_asr_fallback_enabled = false; + assert_eq!(local_asr_fallback(&off, &cloud_asr(), true), None); + // Platforms without local ASR. + assert_eq!(local_asr_fallback(&preferences, &cloud_asr(), false), None); + // A local (or unknown) provider has nothing to fall back to. + let local = ProviderInvocation::for_provider("sherpa-onnx-local"); + assert_eq!(local_asr_fallback(&preferences, &local, true), None); + let unknown = ProviderInvocation::for_provider("experimental-textstream"); + assert_eq!(local_asr_fallback(&preferences, &unknown, true), None); + let mut no_model = preferences.clone(); + no_model.sherpa_onnx_model = " ".into(); + assert_eq!(local_asr_fallback(&no_model, &cloud_asr(), true), None); + } + #[test] fn selection_polish_uses_the_selected_text_envelope() { let mut context = DictationContext::default(); @@ -894,6 +995,19 @@ mod tests { assert!(!ordinary.uses_llm_polisher()); } + #[test] + fn native_and_experimental_asr_providers_are_local() { + for provider in [ + "apple-speech", + "experimental-sensevoice", + "experimental-embedding-gemma", + "experimental-textstream", + ] { + assert!(is_local_asr_provider(provider), "{provider}"); + } + assert!(!is_local_asr_provider("volcengine")); + } + #[test] fn short_text_polish_skip_respects_threshold_and_translation() { let mut context = DictationContext::default(); diff --git a/openless-all/app/crates/openless-core/src/dictation_engine.rs b/openless-all/app/crates/openless-core/src/dictation_engine.rs index 376ab6b23..121b7fa10 100644 --- a/openless-all/app/crates/openless-core/src/dictation_engine.rs +++ b/openless-all/app/crates/openless-core/src/dictation_engine.rs @@ -232,6 +232,7 @@ impl DictationEngine for PipelineDictationEngine { )); } let stable = context.recording.transcribe_after_stop; + let has_local_fallback = context.asr_fallback.is_some(); let transcript_partials: Arc = if stable { Arc::new(DiscardTextStream) } else { @@ -353,18 +354,25 @@ impl DictationEngine for PipelineDictationEngine { })); } else if !stable { if let Err(error) = buffered.attach().await { - let recording = session - .resources - .lock() - .expect("pipeline resource lock poisoned") - .recording - .take(); - if let Some(recording) = recording { - let _ = recording.stop().await; + // Keep capturing for the local replay; the buffer retains the cloud error. + if has_local_fallback && !session.cancelled.load(Ordering::Acquire) { + log::warn!( + "[dictation] cloud ASR startup failed; keeping local fallback: {error}" + ); + } else { + let recording = session + .resources + .lock() + .expect("pipeline resource lock poisoned") + .recording + .take(); + if let Some(recording) = recording { + let _ = recording.stop().await; + } + let _ = cancel_transcription_once(&session, Arc::clone(&buffered)).await; + remove_session(&sessions, session_id, &session); + return Err(error); } - let _ = cancel_transcription_once(&session, Arc::clone(&buffered)).await; - remove_session(&sessions, session_id, &session); - return Err(error); } } if session.cancelled.load(Ordering::Acquire) { @@ -529,6 +537,7 @@ impl DictationEngine for PipelineDictationEngine { ) -> BoxFuture<'static, Result> { let sessions = Arc::clone(&self.sessions); let polisher = Arc::clone(&self.polisher); + let transcription_engine = Arc::clone(&self.transcription); let policy = self.polish_failure_policy; Box::pin(async move { let session = find_session(&sessions, session_id)?; @@ -618,7 +627,29 @@ impl DictationEngine for PipelineDictationEngine { )?; let asr_started = std::time::Instant::now(); let mut asr_call_label = transcription.asr_call_label(); - let transcription_result = transcription.finish().await; + // With a local fallback configured, the local model stands by on + // the same recording in case the cloud is slow or fails. + let standby = context + .asr_fallback + .is_some() + .then(|| archive.clone().filter(|archive| archive.is_available())) + .flatten() + .map(|archive| { + LocalStandby::new( + Arc::clone(&transcription_engine), + Arc::clone(&context), + Arc::clone(&session), + session_id, + archive, + ) + }); + let race = match standby { + Some(standby) => race_cloud_with_local_standby(transcription.finish(), standby).await, + None => AsrRace::Cloud { + result: transcription.finish().await, + standby: None, + }, + }; asr_call_label = transcription.asr_call_label().or(asr_call_label); for notification in transcription.take_progress_notifications() { publish_progress( @@ -628,14 +659,58 @@ impl DictationEngine for PipelineDictationEngine { EngineProgress::Notification(notification), )?; } - let mut transcript = match transcription_result { - Ok(transcript) => { + // Set when the cloud ASR failed or was too slow and the local + // fallback produced the transcript: the network is most likely down + // or weak, so polish is skipped. + let mut local_fallback_used = false; + // Set when the local standby simply answered first: the cloud may be + // healthy, so polish still runs. + let mut local_won_race = false; + let mut transcript = match race { + AsrRace::Local((transcript, label)) => { + log::info!( + "[dictation] local {} answered before cloud ASR {}; using the local transcript", + context.asr_fallback.as_ref().map_or("", |fallback| fallback.provider_type.as_str()), + context.asr.provider_type, + ); + let _ = cancel_transcription_once(&session, Arc::clone(&transcription)).await; + asr_call_label = label; + local_won_race = true; + transcript + } + AsrRace::Cloud { + result: Ok(transcript), + standby, + } => { session .transcription_finished .store(true, Ordering::Release); - transcript + if transcript.text.trim().is_empty() { + if let Some(standby) = standby { + if let Some((local, label)) = standby.result().await { + log::warn!( + "[dictation] cloud ASR returned an empty transcript; using the local fallback" + ); + asr_call_label = label; + local_fallback_used = true; + local + } else { + transcript + } + } else { + transcript + } + } else { + if let Some(standby) = standby { + standby.abandon().await; + } + transcript + } } - Err(first_error) => { + AsrRace::Cloud { + result: Err(first_error), + standby, + } => { let cancelled = session.cancelled.load(Ordering::Acquire); let prepared = prepared.ok_or_else(|| { BackendError::new( @@ -643,6 +718,8 @@ impl DictationEngine for PipelineDictationEngine { "transcription provider preparation is unavailable", ) })?; + // The recording is replayed for a cloud retry (retryable + // errors only). let retry_pcm = if !cancelled && first_error.retryable { match archive.as_ref().filter(|archive| archive.is_available()) { Some(archive) => { @@ -654,22 +731,64 @@ impl DictationEngine for PipelineDictationEngine { None }; let _ = cancel_transcription_once(&session, Arc::clone(&transcription)).await; - match retry_pcm { - Some(pcm) => match retry_transcription( - prepared, - Arc::clone(&session), - session_id, - Arc::clone(&progress), - pcm, - ) - .await - { - Ok((transcript, label)) => { - asr_call_label = label; - transcript - } - Err((error, label)) => { - asr_call_label = label.or(asr_call_label); + // Local first, on any cloud error — an expired key or + // exhausted quota is as much a reason to go local as a dead + // network, and on a weak or absent network the cloud retry + // would only add its backoff and timeouts before failing. + let local = match standby { + Some(standby) if !cancelled => { + log::warn!( + "[dictation] cloud ASR {} failed ({:?}: {}); using the local fallback", + context.asr.provider_type, + first_error.code, + first_error.message, + ); + standby.result().await + } + Some(standby) => { + standby.abandon().await; + None + } + None => None, + }; + if let Some((transcript, label)) = local { + asr_call_label = label; + local_fallback_used = true; + transcript + } else { + match retry_pcm { + Some(pcm) => match retry_transcription( + prepared, + Arc::clone(&session), + session_id, + Arc::clone(&progress), + pcm, + CLOUD_RETRY, + ) + .await + { + Ok((transcript, label)) => { + asr_call_label = label; + transcript + } + Err((error, label)) => { + asr_call_label = label.or(asr_call_label); + demote_failed_archive( + archive.as_ref(), + context.output_target, + &mut has_audio_recording, + ) + .await; + remove_session(&sessions, session_id, &session); + let mut failure = + EngineFailure::new(error, EngineFailureStage::Transcribing); + failure.asr_ms = Some(asr_started.elapsed().as_millis() as u64); + failure.has_audio_recording = has_audio_recording; + failure.asr_call_label = asr_call_label; + return Err(failure); + } + }, + None => { demote_failed_archive( archive.as_ref(), context.output_target, @@ -677,6 +796,13 @@ impl DictationEngine for PipelineDictationEngine { ) .await; remove_session(&sessions, session_id, &session); + let error = if cancelled { + cancelled_error( + "dictation was cancelled while transcription was finishing", + ) + } else { + first_error + }; let mut failure = EngineFailure::new(error, EngineFailureStage::Transcribing); failure.asr_ms = Some(asr_started.elapsed().as_millis() as u64); @@ -684,28 +810,6 @@ impl DictationEngine for PipelineDictationEngine { failure.asr_call_label = asr_call_label; return Err(failure); } - }, - None => { - demote_failed_archive( - archive.as_ref(), - context.output_target, - &mut has_audio_recording, - ) - .await; - remove_session(&sessions, session_id, &session); - let error = if cancelled { - cancelled_error( - "dictation was cancelled while transcription was finishing", - ) - } else { - first_error - }; - let mut failure = - EngineFailure::new(error, EngineFailureStage::Transcribing); - failure.asr_ms = Some(asr_started.elapsed().as_millis() as u64); - failure.has_audio_recording = has_audio_recording; - failure.asr_call_label = asr_call_label; - return Err(failure); } } } @@ -761,8 +865,16 @@ impl DictationEngine for PipelineDictationEngine { }), )?; // Issue #1032: short transcripts skip polish (user-configured threshold). + // Optional polish may be skipped after fallback; an explicit translation is still required. + let translation_active = context.polish.translation_active; let uses_polisher = context.uses_llm_polisher() - && !context.skips_polish_for_short_text(&transcript.text); + && !context.skips_polish_for_short_text(&transcript.text) + && (!local_fallback_used || translation_active); + // Local recognition makes polish optional: when it fails because + // the network never answered, the raw transcript is the result. + let local_asr_used = local_fallback_used + || local_won_race + || crate::dictation_context::is_local_asr_provider(&context.asr.provider_type); let (polish_output, polish_failed, polish_ms) = if uses_polisher { publish_progress( &session, @@ -799,9 +911,11 @@ impl DictationEngine for PipelineDictationEngine { }; let result = match output { Ok(text) => (text, false), + // Offline after local recognition: the raw transcript is the + // expected output. A rejected key or quota is still reported. Err(error) if can_fallback_to_raw(policy, &error) => ( crate::ports::PolishOutput::text(transcript.text.clone()), - true, + !(local_asr_used && error.retryable && !translation_active), ), Err(error) => { let polish_ms = Some(polish_started.elapsed().as_millis() as u64); @@ -1098,12 +1212,167 @@ fn remove_session( } } +/// How a recorded-audio replay is attempted: the cloud retry backs off and +/// tries twice. +#[derive(Clone, Copy)] +struct ReplayPlan { + attempts: u64, + backoff: bool, +} + +const CLOUD_RETRY: ReplayPlan = ReplayPlan { attempts: 2, backoff: true }; + +type AsrAttempt = ( + crate::ports::TranscriptOutput, + Option, +); + +/// The outcome of racing the cloud ASR against the local standby. +enum AsrRace { + /// The cloud answered first (or the local model had nothing usable). + /// `standby` is the local transcription still pending, if any, for the + /// caller to use after a cloud error or to abandon. + Cloud { + result: Result, + standby: Option, + }, + /// The local model finished first with a usable transcript. + Local(AsrAttempt), +} + +/// Runs the cloud ASR and the local standby side by side once recording has +/// stopped; whichever returns a transcript first is used. The cloud wins a tie, +/// and a failed local run leaves the cloud to finish on its own. +async fn race_cloud_with_local_standby( + mut cloud: BoxFuture<'static, Result>, + mut standby: LocalStandby, +) -> AsrRace { + tokio::select! { + biased; + result = &mut cloud => AsrRace::Cloud { + result, + standby: Some(standby), + }, + local = &mut standby.run => match local { + Some(local) => AsrRace::Local(local), + None => AsrRace::Cloud { + result: cloud.await, + standby: None, + }, + }, + } +} + +/// The session's local fallback model transcribing the frozen recording, +/// outside the pipeline's registered resources so that it can run alongside +/// the cloud call. Yields None when there is no usable result — the caller then +/// carries on with the cloud's outcome, so a missing or broken local model never +/// replaces the error the user's provider actually gave. +struct LocalStandby { + run: BoxFuture<'static, Option>, + /// The started local session until it finishes, so that an abandoned + /// standby can still release its model. + started: Arc>>>, +} + +impl LocalStandby { + fn new( + engine: Arc, + context: Arc, + session: Arc, + session_id: SessionId, + archive: Arc, + ) -> Self { + let started: Arc>>> = Arc::new(Mutex::new(None)); + let slot = Arc::clone(&started); + let run = Box::pin(async move { + let transcribe = async { + let fallback = context.asr_fallback.clone()?; + let pcm = archive.read_pcm().await.ok().filter(|pcm| !pcm.is_empty())?; + let mut local = (*context).clone(); + local.asr = fallback; + local.asr_fallback = None; + let prepared = match engine.prepare(session_id, Arc::new(local)).await { + Ok(prepared) => prepared, + Err(error) => { + log::warn!("[dictation] local ASR standby unavailable: {}", error.message); + return None; + } + }; + // Partials stay private: the cloud may still win. + let transcription = match prepared.start(Arc::new(DiscardedTextStream)).await { + Ok(transcription) => transcription, + Err(error) => { + log::warn!("[dictation] local ASR standby failed to start: {}", error.message); + return None; + } + }; + *slot.lock().expect("local standby lock poisoned") = Some(Arc::clone(&transcription)); + let label = transcription.asr_call_label(); + transcription.consume_pcm_chunk(&pcm); + let result = transcription.finish().await; + slot.lock().expect("local standby lock poisoned").take(); + match result { + Ok(output) if output.text.trim().is_empty() => { + log::warn!("[dictation] local ASR standby returned an empty transcript"); + None + } + Ok(output) => Some((output, label)), + Err(error) => { + log::warn!("[dictation] local ASR standby failed: {}", error.message); + None + } + } + }; + tokio::select! { + result = transcribe => result, + _ = wait_until_cancelled(&session) => { + let started = slot.lock().expect("local standby lock poisoned").take(); + if let Some(transcription) = started { + let _ = transcription.cancel().await; + } + None + } + } + }); + Self { run, started } + } + + async fn result(self) -> Option { + self.run.await + } + + /// Stops a standby whose result is no longer needed and releases its model. + async fn abandon(self) { + drop(self.run); + let started = self.started.lock().expect("local standby lock poisoned").take(); + if let Some(transcription) = started { + let _ = transcription.cancel().await; + } + } +} + +async fn wait_until_cancelled(session: &PipelineSession) { + while !session.cancelled.load(Ordering::Acquire) { + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } +} + +struct DiscardedTextStream; + +impl TextStreamSink for DiscardedTextStream { + fn publish(&self, _chunk: TextStreamChunk) -> Result<(), BackendError> { + Ok(()) + } +} + async fn retry_transcription( prepared: Arc, session: Arc, session_id: SessionId, progress: Arc, pcm: Vec, + plan: ReplayPlan, ) -> Result< ( crate::ports::TranscriptOutput, @@ -1112,11 +1381,13 @@ async fn retry_transcription( (BackendError, Option), > { let mut last_label = None; - for attempt in 1..=2_u64 { - if let Err(error) = - cancellable_backoff(&session, std::time::Duration::from_millis(500 * attempt)).await - { - return Err((error, last_label)); + for attempt in 1..=plan.attempts { + if plan.backoff { + if let Err(error) = + cancellable_backoff(&session, std::time::Duration::from_millis(500 * attempt)).await + { + return Err((error, last_label)); + } } let partials: Arc = Arc::new(TranscriptProgressForwarder { session_id, @@ -1130,7 +1401,7 @@ async fn retry_transcription( last_label, )); } - Err(error) if error.retryable && attempt < 2 => continue, + Err(error) if error.retryable && attempt < plan.attempts => continue, Err(error) => return Err((error, last_label)), }; last_label = transcription.asr_call_label(); @@ -1194,7 +1465,7 @@ async fn retry_transcription( } Err(error) => { let _ = cancel_transcription_once(&session, transcription).await; - if !error.retryable || attempt == 2 { + if !error.retryable || attempt == plan.attempts { return Err((error, last_label)); } } @@ -3377,6 +3648,513 @@ mod tests { Arc::new(context) } + /// Answers each start from the queue of the provider the session was + /// started with, and records that provider — so a test can tell the cloud + /// call, its retries and the local fallback apart. + struct RoutedTranscriber { + outputs: Mutex>>>, + started: Arc>>, + /// How long each provider's finish takes; providers not listed answer at once. + delays: std::collections::HashMap, + } + + struct DelayedTranscriptionSession { + output: Result, + delay: std::time::Duration, + cancels: Arc, + } + + impl AudioConsumer for DelayedTranscriptionSession { + fn consume_pcm_chunk(&self, _pcm: &[u8]) {} + } + + impl TranscriptionSession for DelayedTranscriptionSession { + fn finish(&self) -> BoxFuture<'static, Result> { + let output = self.output.clone(); + let delay = self.delay; + Box::pin(async move { + tokio::time::sleep(delay).await; + output + }) + } + + fn cancel(&self) -> BoxFuture<'static, Result<(), BackendError>> { + self.cancels.fetch_add(1, Ordering::SeqCst); + Box::pin(async { Ok(()) }) + } + } + + impl TranscriptionEngine for RoutedTranscriber { + fn start( + &self, + _session_id: SessionId, + context: Arc, + _partials: Arc, + ) -> BoxFuture<'static, Result, BackendError>> { + let provider = context.asr.provider_type.clone(); + self.started.lock().unwrap().push(provider.clone()); + let output = self + .outputs + .lock() + .unwrap() + .get_mut(&provider) + .and_then(VecDeque::pop_front) + .unwrap_or_else(|| panic!("no fixture output left for {provider}")); + if let Some(delay) = self.delays.get(&provider).copied() { + return Box::pin(async move { + Ok(Arc::new(DelayedTranscriptionSession { + output, + delay, + cancels: Arc::new(AtomicUsize::new(0)), + }) as Arc) + }); + } + Box::pin(async move { + Ok(Arc::new(RetryTranscriptionSession { + output, + pcm: Arc::new(Mutex::new(Vec::new())), + }) as Arc) + }) + } + } + + const CLOUD: &str = "volcengine"; + const LOCAL: &str = "sherpa-onnx-local"; + + /// A raw-mode session on CLOUD whose fallback (when set) is LOCAL. + fn fallback_test_engine( + cloud: Vec>, + local: Vec>, + ) -> (PipelineDictationEngine, Arc, Arc>>) { + let (engine, progress, started, _) = fallback_polish_test_engine( + cloud, + local, + Ok(crate::ports::PolishOutput::text("unused")), + ); + (engine, progress, started) + } + + /// The engine, its progress sink, the providers started in order and the + /// polisher's call count. + type FallbackTestEngine = ( + PipelineDictationEngine, + Arc, + Arc>>, + Arc, + ); + + /// fallback_test_engine() with the polisher's result chosen, also + /// returning how many times it was called. + fn fallback_polish_test_engine( + cloud: Vec>, + local: Vec>, + polish: Result, + ) -> FallbackTestEngine { + racing_test_engine(&[], cloud, local, polish) + } + + /// fallback_polish_test_engine() with each listed provider taking the + /// given time to finish. + fn racing_test_engine( + delays: &[(&str, u64)], + cloud: Vec>, + local: Vec>, + polish: Result, + ) -> FallbackTestEngine { + let started = Arc::new(Mutex::new(Vec::new())); + let polish_calls = Arc::new(AtomicUsize::new(0)); + let engine = PipelineDictationEngine::new( + Arc::new(FixtureRecorder { + stops: Arc::new(AtomicUsize::new(0)), + archive: Arc::new(FixtureArchive { + available: AtomicBool::new(true), + discards: Arc::new(AtomicUsize::new(0)), + pcm: vec![1, 0, 2, 0], + }), + fail: false, + }), + Arc::new(RoutedTranscriber { + outputs: Mutex::new(std::collections::HashMap::from([ + (CLOUD.to_string(), cloud.into()), + (LOCAL.to_string(), local.into()), + ])), + started: Arc::clone(&started), + delays: delays + .iter() + .map(|(provider, ms)| { + (provider.to_string(), std::time::Duration::from_millis(*ms)) + }) + .collect(), + }), + Arc::new(FixturePolisher { + result: polish, + calls: Arc::clone(&polish_calls), + cancels: Arc::new(AtomicUsize::new(0)), + contexts: Arc::new(Mutex::new(Vec::new())), + }), + ); + (engine, Arc::new(RecordingProgress::default()), started, polish_calls) + } + + /// A polishing session (not raw mode) on CLOUD with the LOCAL fallback. + fn polishing_fallback_context() -> Arc { + let mut context = DictationContext { + asr: crate::dictation_context::ProviderInvocation::new(CLOUD, CLOUD), + asr_fallback: Some(crate::dictation_context::ProviderInvocation::new(LOCAL, LOCAL)), + ..DictationContext::default() + }; + assert!(context.uses_llm_polisher(), "fixture must exercise the polisher"); + context.polish.polish_skip_max_chars = 0; + Arc::new(context) + } + + #[tokio::test] + async fn realtime_cloud_start_failure_preserves_local_fallback_and_original_error() { + struct FailingCloudStart(Arc); + impl TranscriptionEngine for FailingCloudStart { + fn start( + &self, + id: SessionId, + context: Arc, + partials: Arc, + ) -> BoxFuture<'static, Result, BackendError>> + { + if context.asr.provider_type == CLOUD { + Box::pin(async { + Err(BackendError::new( + BackendErrorCode::Provider, + "cloud connection failed", + )) + }) + } else { + self.0.start(id, context, partials) + } + } + } + for local_succeeds in [true, false] { + let local = if local_succeeds { + transcript("local transcript") + } else { + Err(BackendError::new( + BackendErrorCode::InvalidState, + "model missing", + )) + }; + let (mut engine, progress, _) = fallback_test_engine(vec![], vec![local]); + engine.transcription = Arc::new(FailingCloudStart(Arc::clone(&engine.transcription))); + let id = SessionId::new(); + engine + .start(id, fallback_context(), progress.clone()) + .await + .unwrap(); + let result = engine.finish(id, progress).await; + if local_succeeds { + assert_eq!(result.unwrap().raw_text, "local transcript"); + } else { + assert_eq!(result.unwrap_err().error.message, "cloud connection failed"); + } + assert!(engine.sessions.lock().unwrap().is_empty()); + } + } + + #[tokio::test] + async fn local_recognition_preserves_translation_and_reports_translation_failure() { + for fallback in [false, true] { + for polish_succeeds in [false, true] { + let polish = if polish_succeeds { + Ok(crate::ports::PolishOutput::text("Hello")) + } else { + Err(BackendError::new(BackendErrorCode::Provider, "LLM offline") + .retryable(true)) + }; + let (engine, progress, _, calls) = fallback_polish_test_engine( + if fallback { + vec![Err(BackendError::new( + BackendErrorCode::Provider, + "ASR quota exhausted", + ))] + } else { + vec![] + }, + vec![transcript("你好")], + polish, + ); + let mut context = if fallback { + (*polishing_fallback_context()).clone() + } else { + (*local_polishing_context()).clone() + }; + context.polish.translation_active = true; + context.polish.translation_target_language = "English".into(); + let id = SessionId::new(); + engine + .start(id, Arc::new(context), progress.clone()) + .await + .unwrap(); + let result = engine.finish(id, progress).await.unwrap(); + assert_eq!(calls.load(Ordering::SeqCst), 1); + assert_eq!( + result.polished_text, + if polish_succeeds { "Hello" } else { "你好" } + ); + assert_eq!(result.polish_failed, !polish_succeeds); + } + } + } + + #[tokio::test] + async fn a_local_fallback_skips_polish_and_reports_no_failure() { + let (engine, progress, started, polish_calls) = fallback_polish_test_engine( + vec![Err(BackendError::new(BackendErrorCode::Provider, "offline").retryable(true))], + vec![transcript("本地识别")], + Err(BackendError::new(BackendErrorCode::Provider, "offline")), + ); + let session_id = SessionId::new(); + engine.start(session_id, polishing_fallback_context(), progress.clone()).await.unwrap(); + + let result = engine.finish(session_id, progress).await.unwrap(); + + assert_eq!(result.raw_text, "本地识别"); + assert_eq!(result.polished_text, "本地识别"); + assert!(!result.polish_failed); + // The cloud was unreachable a moment ago: no polish request at all. + assert_eq!(polish_calls.load(Ordering::SeqCst), 0); + assert_eq!(started.lock().unwrap().as_slice(), &[CLOUD, LOCAL]); + } + + fn local_polishing_context() -> Arc { + let mut context = (*polishing_fallback_context()).clone(); + context.asr = crate::dictation_context::ProviderInvocation::new(LOCAL, LOCAL); + context.asr_fallback = None; + Arc::new(context) + } + + #[tokio::test] + async fn a_rejected_polish_after_local_recognition_is_still_reported() { + let (engine, progress, _, polish_calls) = fallback_polish_test_engine( + vec![], + vec![transcript("本地识别")], + Err(BackendError::new(BackendErrorCode::Provider, "invalid api key")), + ); + let session_id = SessionId::new(); + engine.start(session_id, local_polishing_context(), progress.clone()).await.unwrap(); + + let result = engine.finish(session_id, progress).await.unwrap(); + + assert_eq!(result.polished_text, "本地识别"); + assert!(result.polish_failed, "a rejected key must not be hidden as offline"); + assert_eq!(polish_calls.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn an_offline_polish_after_local_recognition_is_raw_output_not_a_failure() { + let (engine, progress, started, polish_calls) = fallback_polish_test_engine( + vec![], + vec![transcript("本地识别")], + Err(BackendError::new(BackendErrorCode::Provider, "offline").retryable(true)), + ); + let mut context = (*polishing_fallback_context()).clone(); + context.asr = crate::dictation_context::ProviderInvocation::new(LOCAL, LOCAL); + context.asr_fallback = None; + let session_id = SessionId::new(); + engine.start(session_id, Arc::new(context), progress.clone()).await.unwrap(); + + let result = engine.finish(session_id, progress).await.unwrap(); + + assert_eq!(result.polished_text, "本地识别"); + assert!(!result.polish_failed); + assert_eq!(polish_calls.load(Ordering::SeqCst), 1); + assert_eq!(started.lock().unwrap().as_slice(), &[LOCAL]); + } + + #[tokio::test] + async fn a_failed_polish_after_cloud_recognition_is_still_reported() { + let (engine, progress, _, polish_calls) = fallback_polish_test_engine( + vec![transcript("云端识别")], + vec![], + Err(BackendError::new(BackendErrorCode::Provider, "offline")), + ); + let session_id = SessionId::new(); + engine.start(session_id, polishing_fallback_context(), progress.clone()).await.unwrap(); + + let result = engine.finish(session_id, progress).await.unwrap(); + + assert_eq!(result.polished_text, "云端识别"); + assert!(result.polish_failed); + assert_eq!(polish_calls.load(Ordering::SeqCst), 1); + } + + fn fallback_context() -> Arc { + let mut context = (*raw_dictation_context()).clone(); + context.asr = crate::dictation_context::ProviderInvocation::new(CLOUD, CLOUD); + context.asr_fallback = Some(crate::dictation_context::ProviderInvocation::new(LOCAL, LOCAL)); + Arc::new(context) + } + + fn transcript(text: &str) -> Result { + Ok(TranscriptOutput { text: text.into(), duration_ms: 25 }) + } + + #[tokio::test] + async fn failed_cloud_asr_goes_straight_to_the_local_fallback() { + let (engine, progress, started) = fallback_test_engine( + vec![Err(BackendError::new(BackendErrorCode::Provider, "offline").retryable(true))], + vec![transcript("本地识别")], + ); + let session_id = SessionId::new(); + engine.start(session_id, fallback_context(), progress.clone()).await.unwrap(); + + let result = engine.finish(session_id, progress).await.unwrap(); + + assert_eq!(result.raw_text, "本地识别"); + // No cloud retry (and no backoff) before the local model runs. + assert_eq!(started.lock().unwrap().as_slice(), &[CLOUD, LOCAL]); + } + + #[tokio::test] + async fn a_non_retryable_cloud_error_also_falls_back() { + let (engine, progress, started) = fallback_test_engine( + vec![Err(BackendError::new(BackendErrorCode::Provider, "quota exhausted"))], + vec![transcript("本地识别")], + ); + let session_id = SessionId::new(); + engine.start(session_id, fallback_context(), progress.clone()).await.unwrap(); + + let result = engine.finish(session_id, progress).await.unwrap(); + + assert_eq!(result.raw_text, "本地识别"); + assert_eq!(started.lock().unwrap().as_slice(), &[CLOUD, LOCAL]); + } + + #[tokio::test] + async fn a_failed_local_fallback_keeps_the_cloud_retry_and_the_cloud_error() { + let temporary = + || Err(BackendError::new(BackendErrorCode::Provider, "offline").retryable(true)); + let (engine, progress, started) = fallback_test_engine( + vec![temporary(), temporary(), temporary()], + vec![Err(BackendError::new(BackendErrorCode::Unsupported, "model not downloaded"))], + ); + let session_id = SessionId::new(); + engine.start(session_id, fallback_context(), progress.clone()).await.unwrap(); + + let failure = engine.finish(session_id, progress).await.unwrap_err(); + + // The cloud's own error, not the local model's. + assert_eq!(failure.error.code, BackendErrorCode::Provider); + assert_eq!(failure.error.message, "offline"); + assert_eq!(started.lock().unwrap().as_slice(), &[CLOUD, LOCAL, CLOUD, CLOUD]); + } + + #[tokio::test] + async fn without_a_fallback_the_cloud_retry_is_unchanged() { + let temporary = + || Err(BackendError::new(BackendErrorCode::Provider, "offline").retryable(true)); + let (engine, progress, started) = + fallback_test_engine(vec![temporary(), transcript("cloud retry")], vec![]); + let mut context = (*fallback_context()).clone(); + context.asr_fallback = None; + let session_id = SessionId::new(); + engine.start(session_id, Arc::new(context), progress.clone()).await.unwrap(); + + let result = engine.finish(session_id, progress).await.unwrap(); + + assert_eq!(result.raw_text, "cloud retry"); + assert_eq!(started.lock().unwrap().as_slice(), &[CLOUD, CLOUD]); + } + + #[tokio::test] + async fn the_local_standby_wins_when_it_answers_before_a_slow_cloud() { + let (engine, progress, started, polish_calls) = racing_test_engine( + &[(CLOUD, 2_000), (LOCAL, 20)], + vec![transcript("云端识别")], + vec![transcript("本地识别")], + Ok(crate::ports::PolishOutput::text("润色后")), + ); + let session_id = SessionId::new(); + engine.start(session_id, polishing_fallback_context(), progress.clone()).await.unwrap(); + + let result = engine.finish(session_id, progress).await.unwrap(); + + assert_eq!(result.raw_text, "本地识别"); + // Only slower, not unreachable: the transcript is still polished. + assert_eq!(result.polished_text, "润色后"); + assert_eq!(polish_calls.load(Ordering::SeqCst), 1); + assert_eq!(started.lock().unwrap().as_slice(), &[CLOUD, LOCAL]); + } + + #[tokio::test] + async fn the_cloud_wins_when_it_answers_before_the_local_standby() { + let (engine, progress, _, _) = racing_test_engine( + &[(CLOUD, 20), (LOCAL, 2_000)], + vec![transcript("云端识别")], + vec![transcript("本地识别")], + Ok(crate::ports::PolishOutput::text("unused")), + ); + let session_id = SessionId::new(); + engine.start(session_id, fallback_context(), progress.clone()).await.unwrap(); + + let result = engine.finish(session_id, progress).await.unwrap(); + + assert_eq!(result.raw_text, "云端识别"); + } + + #[tokio::test] + async fn a_failed_local_standby_leaves_a_slow_cloud_to_finish() { + let (engine, progress, _, _) = racing_test_engine( + &[(CLOUD, 200)], + vec![transcript("云端识别")], + vec![Err(BackendError::new(BackendErrorCode::Unsupported, "model not downloaded"))], + Ok(crate::ports::PolishOutput::text("unused")), + ); + let session_id = SessionId::new(); + engine.start(session_id, fallback_context(), progress.clone()).await.unwrap(); + + let result = engine.finish(session_id, progress).await.unwrap(); + + assert_eq!(result.raw_text, "云端识别"); + } + + #[tokio::test] + async fn an_empty_cloud_result_uses_a_valid_local_standby() { + let (engine, progress, started, polish_calls) = racing_test_engine( + &[(CLOUD, 20), (LOCAL, 200)], + vec![transcript(" \n\t")], + vec![transcript("本地识别")], + Ok(crate::ports::PolishOutput::text("unused")), + ); + let session_id = SessionId::new(); + engine + .start(session_id, polishing_fallback_context(), progress.clone()) + .await + .unwrap(); + + let result = engine.finish(session_id, progress).await.unwrap(); + + assert_eq!(result.raw_text, "本地识别"); + assert_eq!(polish_calls.load(Ordering::SeqCst), 0); + assert_eq!(started.lock().unwrap().as_slice(), &[CLOUD, LOCAL]); + } + + #[tokio::test] + async fn an_empty_local_standby_leaves_a_slow_cloud_to_finish() { + let (engine, progress, started, _) = racing_test_engine( + &[(CLOUD, 200), (LOCAL, 20)], + vec![transcript("云端识别")], + vec![transcript(" \n\t")], + Ok(crate::ports::PolishOutput::text("unused")), + ); + let session_id = SessionId::new(); + engine + .start(session_id, fallback_context(), progress.clone()) + .await + .unwrap(); + + let result = engine.finish(session_id, progress).await.unwrap(); + + assert_eq!(result.raw_text, "云端识别"); + assert_eq!(started.lock().unwrap().as_slice(), &[CLOUD, LOCAL]); + } + #[tokio::test] async fn retryable_asr_failure_stops_after_two_retries() { let temporary = diff --git a/openless-all/app/crates/openless-core/src/domains.rs b/openless-all/app/crates/openless-core/src/domains.rs index 40af1fb12..1c87cab17 100644 --- a/openless-all/app/crates/openless-core/src/domains.rs +++ b/openless-all/app/crates/openless-core/src/domains.rs @@ -302,6 +302,10 @@ pub trait LocalAsrApi: Send + Sync { &self, target: LocalAsrTarget, ) -> BoxFuture<'static, Result>; + /// Whether the host has a native engine for this runtime. + fn engine_available(&self, _runtime: LocalAsrRuntime) -> bool { + false + } /// Run the native smoke test for a specific ASR channel without changing /// the globally active channel. fn test_channel( diff --git a/openless-all/app/crates/openless-core/src/lib.rs b/openless-all/app/crates/openless-core/src/lib.rs index 154f693ef..b04cd838f 100644 --- a/openless-all/app/crates/openless-core/src/lib.rs +++ b/openless-all/app/crates/openless-core/src/lib.rs @@ -214,7 +214,7 @@ pub mod contract { pub use activity::{ActivityDay, ActivityStore, DayStats}; pub use api::{ - BackendRepositories, BackendSnapshot, CliDispatchOutcome, DictationHotkeyDispatchOptions, + AsrRoute, AsrUsage, BackendRepositories, BackendSnapshot, CliDispatchOutcome, DictationHotkeyDispatchOptions, DictationHotkeyEdge, LessComputerHotkeyAction, LessComputerVoiceFinish, LessComputerVoiceOptions, LessComputerVoiceSession, OpenLessBackend, QaVoiceCaptureResult, QaVoiceCaptureSession, StartupSnapshot, VoiceTranscriptionSession, diff --git a/openless-all/app/crates/openless-core/src/local_asr_service.rs b/openless-all/app/crates/openless-core/src/local_asr_service.rs index 06251efaf..dcb742525 100644 --- a/openless-all/app/crates/openless-core/src/local_asr_service.rs +++ b/openless-all/app/crates/openless-core/src/local_asr_service.rs @@ -563,6 +563,10 @@ fn restore_activation_preferences( } impl LocalAsrApi for LocalAsrService { + fn engine_available(&self, runtime: LocalAsrRuntime) -> bool { + self.runtime.engine_available(runtime) + } + fn activate( &self, request: LocalAsrActivationRequest, @@ -1239,15 +1243,25 @@ impl LocalAsrApi for LocalAsrService { ) })?; let provider_type = channel.provider_type; - let model_id = { + let (target_runtime, model_id) = { let preferences = preferences.get(); match provider_type.as_str() { - "local-whisper" | "apple-whisper" => { - preferences.local_whisper_active_model.clone() - } - "local-qwen3" | "local-qwen3-mlx" | "local-qwen3-c" => { - preferences.local_asr_active_model.clone() - } + "local-whisper" | "apple-whisper" => ( + LocalAsrRuntime::Generic, + preferences.local_whisper_active_model.clone(), + ), + "local-qwen3" | "local-qwen3-mlx" | "local-qwen3-c" => ( + LocalAsrRuntime::Generic, + preferences.local_asr_active_model.clone(), + ), + "foundry-local" | "foundry-whisper" | "foundry-local-whisper" => ( + LocalAsrRuntime::Foundry, + preferences.foundry_local_asr_model.clone(), + ), + "sherpa-onnx" | "sherpa-onnx-local" => ( + LocalAsrRuntime::SherpaOnnx, + preferences.sherpa_onnx_model.clone(), + ), _ => { return Err(BackendError::new( BackendErrorCode::Unsupported, @@ -1262,7 +1276,7 @@ impl LocalAsrApi for LocalAsrService { "local ASR model is not configured", )); } - let target = LocalAsrTarget::parse(LocalAsrRuntime::Generic, model_id)?; + let target = LocalAsrTarget::parse(target_runtime, model_id)?; if !runtime.supports_model(&target) { return Err(BackendError::new(BackendErrorCode::Unsupported, "This local ASR engine is retired or unavailable")); } diff --git a/openless-all/app/crates/openless-core/src/model_store.rs b/openless-all/app/crates/openless-core/src/model_store.rs index 8160a5961..aa72a1999 100644 --- a/openless-all/app/crates/openless-core/src/model_store.rs +++ b/openless-all/app/crates/openless-core/src/model_store.rs @@ -2485,6 +2485,9 @@ fn write_relocation_journal(root: &Path, source: &Path) -> Result<(), BackendErr } fn commit_staging(staging: &Path, destination: &Path) -> Result<(), BackendError> { + if let Some(parent) = destination.parent() { + std::fs::create_dir_all(parent).map_err(platform_error)?; + } let backup = destination.with_file_name(format!( ".{}.previous-{}", destination @@ -3463,6 +3466,42 @@ mod tests { let _ = std::fs::remove_dir_all(root); } + #[tokio::test] + async fn sherpa_download_creates_runtime_parent_before_commit() { + let root = + std::env::temp_dir().join(format!("openless-sherpa-model-store-{}", uuid::Uuid::new_v4())); + let body = b"0123456789".to_vec(); + let transport = Arc::new(FakeTransport { + calls: Arc::new(AtomicUsize::new(0)), + body: body.clone(), + ignore_range: false, + }); + let mut config = ModelStoreConfig::new(root.clone()).unwrap(); + config.chunk_size_bytes = 4; + let store = ModelStore::with_transport(config, transport); + let target = + LocalAsrTarget::parse(LocalAsrRuntime::SherpaOnnx, "sense-voice-small-zh").unwrap(); + let manifest = ModelManifest::new( + target, + "org/demo", + vec![ModelFile { + path: "weights.bin".into(), + url: "https://example.test/weights.bin".into(), + size_bytes: body.len() as u64, + sha256: Some(format!("{:x}", Sha256::digest(&body))), + }], + ) + .unwrap(); + + let result = store.download(manifest).await; + + assert!(result.is_ok(), "Sherpa model download failed: {result:?}"); + assert!(root + .join("sherpa-onnx/sense-voice-small-zh/weights.bin") + .is_file()); + let _ = std::fs::remove_dir_all(root); + } + #[test] fn model_list_restores_trusted_partial_bytes_and_cleans_bad_indexes() { let root = std::env::temp_dir().join(format!( diff --git a/openless-all/app/crates/openless-core/src/shared_types.rs b/openless-all/app/crates/openless-core/src/shared_types.rs index 9ff98cb1f..0b5c3ed7c 100644 --- a/openless-all/app/crates/openless-core/src/shared_types.rs +++ b/openless-all/app/crates/openless-core/src/shared_types.rs @@ -696,6 +696,11 @@ pub struct UserPreferences { /// (seconds); same semantics as foundry/qwen3. #[serde(default = "default_local_asr_keep_loaded_secs")] pub sherpa_onnx_keep_loaded_secs: u32, + /// When a cloud ASR provider fails, transcribe the recorded audio with the + /// selected sherpa-onnx model instead (weak or no network). Off by default; + /// only takes effect where local ASR is supported and a model is installed. + #[serde(default)] + pub local_asr_fallback_enabled: bool, /// Auto-update channel. stable = background auto-update checks the /// stable manifest; beta = the beta manifest. Manual check buttons pass /// the channel explicitly, decoupled from this pref. @@ -1115,6 +1120,8 @@ struct UserPreferencesWire { #[serde(default = "default_local_asr_keep_loaded_secs")] sherpa_onnx_keep_loaded_secs: u32, #[serde(default)] + local_asr_fallback_enabled: bool, + #[serde(default)] update_channel: UpdateChannel, #[serde(default)] update_channel_explicit: Option, @@ -1311,6 +1318,7 @@ impl Default for UserPreferencesWire { sherpa_onnx_model: prefs.sherpa_onnx_model, sherpa_onnx_language_hint: prefs.sherpa_onnx_language_hint, sherpa_onnx_keep_loaded_secs: prefs.sherpa_onnx_keep_loaded_secs, + local_asr_fallback_enabled: prefs.local_asr_fallback_enabled, update_channel: prefs.update_channel, // None preserves the fact that the old config lacked the marker; deserialization treats only historical Beta as explicit. update_channel_explicit: None, @@ -1529,6 +1537,7 @@ impl<'de> Deserialize<'de> for UserPreferences { sherpa_onnx_model: wire.sherpa_onnx_model, sherpa_onnx_language_hint: wire.sherpa_onnx_language_hint, sherpa_onnx_keep_loaded_secs: wire.sherpa_onnx_keep_loaded_secs, + local_asr_fallback_enabled: wire.local_asr_fallback_enabled, update_channel: wire.update_channel, update_channel_explicit, history_retention_days: wire.history_retention_days, @@ -1894,6 +1903,7 @@ impl Default for UserPreferences { sherpa_onnx_model: default_sherpa_onnx_model(), sherpa_onnx_language_hint: String::new(), sherpa_onnx_keep_loaded_secs: default_local_asr_keep_loaded_secs(), + local_asr_fallback_enabled: false, update_channel: UpdateChannel::default(), update_channel_explicit: false, history_retention_days: default_history_retention_days(), @@ -2502,7 +2512,8 @@ impl PlatformCapabilities { supports_overlay: true, supports_desktop_hotkey: false, supports_tray: false, - supports_local_asr: false, + // The app links sherpa-onnx on arm64 only; other ABIs stay cloud-only. + supports_local_asr: cfg!(target_arch = "aarch64"), supports_local_qwen3_mlx: false, supports_in_app_dictation: true, supports_auto_update: true, @@ -2537,8 +2548,9 @@ impl PlatformCapabilities { supports_overlay: true, supports_desktop_hotkey: true, supports_tray: true, - // Legacy manager is retired on macOS; experiments use their own entry. - supports_local_asr: cfg!(any(target_os = "linux", target_os = "windows")), + // The Tauri native local engine is available on Windows; Linux keeps the + // retired generic provider metadata but has no inference runtime. + supports_local_asr: cfg!(target_os = "windows"), supports_local_qwen3_mlx: false, supports_in_app_dictation: false, supports_auto_update: true, diff --git a/openless-all/app/crates/openless-core/tests/local_asr_contract.rs b/openless-all/app/crates/openless-core/tests/local_asr_contract.rs index 94c02c4c6..5865d3aad 100644 --- a/openless-all/app/crates/openless-core/tests/local_asr_contract.rs +++ b/openless-all/app/crates/openless-core/tests/local_asr_contract.rs @@ -1,6 +1,6 @@ use futures_util::future::BoxFuture; use openless_core::{ - normalize_foundry_language_hint, normalize_sherpa_language_hint, BackendConfig, + normalize_foundry_language_hint, normalize_sherpa_language_hint, AsrRoute, BackendConfig, BackendDependencies, BackendError, BackendErrorCode, BackendEventKind, ChannelKind, ChannelMutation, ChannelMutationResult, ChannelSummary, CredentialKey, CredentialStore, CredentialsStatus, FoundryRuntimeSource, InMemoryCredentialStore, LocalAsrActivationRequest, @@ -581,6 +581,199 @@ async fn channel_test_requires_a_non_blank_transcript() { let _ = std::fs::remove_dir_all(data_dir); } +#[tokio::test] +async fn foundry_channel_test_uses_the_foundry_runtime() { + let (data_dir, runtime, backend) = local_asr_backend(); + let target = LocalAsrTarget::parse(LocalAsrRuntime::Foundry, "whisper-small").unwrap(); + backend + .services() + .local_asr + .set_active_model(target.clone()) + .await + .unwrap(); + let channel_id = backend + .create_channel( + ChannelKind::Asr, + "foundry-local-whisper".into(), + "Foundry Local".into(), + ) + .await + .unwrap(); + + *runtime.test_transcript.lock().unwrap() = "Hello from Foundry".into(); + let result = backend + .services() + .local_asr + .test_channel(channel_id) + .await + .unwrap(); + + assert_eq!(result.target, target); + assert_eq!(result.backend, "foundry-local-whisper"); + assert_eq!(result.transcribed_text, "Hello from Foundry"); + assert_eq!( + runtime.tested_providers.lock().unwrap().as_slice(), + ["foundry-local-whisper"] + ); + let _ = std::fs::remove_dir_all(data_dir); +} + +#[tokio::test] +async fn sherpa_channel_test_uses_the_sherpa_runtime() { + let (data_dir, runtime, backend) = local_asr_backend(); + let target = + LocalAsrTarget::parse(LocalAsrRuntime::SherpaOnnx, "sense-voice-small-zh").unwrap(); + let model_store = + ModelStore::new(ModelStoreConfig::new(data_dir.join("models")).unwrap()).unwrap(); + let model_dir = model_store.model_dir(&target).unwrap(); + std::fs::create_dir_all(&model_dir).unwrap(); + std::fs::write( + model_dir.join(openless_core::MODEL_READY_SENTINEL), + b"ready", + ) + .unwrap(); + std::fs::write(model_dir.join("model.int8.onnx"), b"model").unwrap(); + std::fs::write(model_dir.join("tokens.txt"), b"tokens").unwrap(); + + backend + .services() + .local_asr + .set_active_model(target.clone()) + .await + .unwrap(); + let channel_id = backend + .create_channel( + ChannelKind::Asr, + "sherpa-onnx-local".into(), + "Local Sherpa".into(), + ) + .await + .unwrap(); + + *runtime.test_transcript.lock().unwrap() = "Hello from Sherpa".into(); + let result = backend + .services() + .local_asr + .test_channel(channel_id) + .await + .unwrap(); + + assert_eq!(result.target, target); + assert_eq!(result.backend, "sherpa-onnx-local"); + assert_eq!(result.transcribed_text, "Hello from Sherpa"); + assert_eq!( + runtime.tested_providers.lock().unwrap().as_slice(), + ["sherpa-onnx-local"] + ); + let _ = std::fs::remove_dir_all(data_dir); +} + +#[cfg(target_os = "windows")] +#[tokio::test] +async fn cloud_route_switches_to_local_when_unconfigured_or_disabled() { + let (data_dir, _, backend) = local_asr_backend(); + let target = + LocalAsrTarget::parse(LocalAsrRuntime::SherpaOnnx, "sense-voice-small-zh").unwrap(); + install_fixture(&data_dir, &target); + backend + .services() + .local_asr + .set_active_model(target) + .await + .unwrap(); + let channel = backend + .create_channel(ChannelKind::Asr, "whisper".into(), "Cloud ASR".into()) + .await + .unwrap(); + backend + .set_active_provider(ProviderSlot::Asr, channel.clone()) + .await + .unwrap(); + assert_eq!(backend.asr_route().await, AsrRoute::LocalRaw); + backend + .set_channel_enabled(ChannelKind::Asr, channel, false) + .await + .unwrap(); + assert_eq!(backend.asr_route().await, AsrRoute::LocalRaw); + let _ = std::fs::remove_dir_all(data_dir); +} + +#[tokio::test] +async fn disabled_local_channel_does_not_switch_to_a_different_local_model() { + let (data_dir, _runtime, backend) = local_asr_backend_with_credentials( + Arc::new(InMemoryCredentialStore::default()), + Some("sherpa-onnx-local"), + ); + let target = + LocalAsrTarget::parse(LocalAsrRuntime::SherpaOnnx, "sense-voice-small-zh").unwrap(); + install_fixture(&data_dir, &target); + backend + .services() + .local_asr + .set_active_model(target) + .await + .unwrap(); + let local = backend + .create_channel( + ChannelKind::Asr, + "sherpa-onnx-local".into(), + "Local Sherpa".into(), + ) + .await + .unwrap(); + backend + .set_active_provider(ProviderSlot::Asr, local.clone()) + .await + .unwrap(); + backend + .set_channel_enabled(ChannelKind::Asr, local, false) + .await + .unwrap(); + + assert_eq!(backend.asr_route().await, AsrRoute::Unavailable); + backend.start().await.unwrap(); + assert_eq!( + backend.start_dictation().await.unwrap_err().code, + BackendErrorCode::InvalidState + ); + backend.shutdown().await.unwrap(); + let _ = std::fs::remove_dir_all(data_dir); +} + +#[tokio::test] +async fn foundry_route_is_unavailable_when_restart_is_required() { + let (data_dir, runtime, backend) = local_asr_backend_with_credentials( + Arc::new(InMemoryCredentialStore::default()), + Some("foundry-local-whisper"), + ); + backend + .services() + .local_asr + .set_active_model( + LocalAsrTarget::parse(LocalAsrRuntime::Foundry, "whisper-small").unwrap(), + ) + .await + .unwrap(); + let foundry = backend + .create_channel( + ChannelKind::Asr, + "foundry-local-whisper".into(), + "Foundry Local".into(), + ) + .await + .unwrap(); + backend + .set_active_provider(ProviderSlot::Asr, foundry) + .await + .unwrap(); + runtime + .restart_needed + .store(true, std::sync::atomic::Ordering::SeqCst); + + assert_eq!(backend.asr_route().await, AsrRoute::Unavailable); + let _ = std::fs::remove_dir_all(data_dir); +} + #[tokio::test] async fn local_asr_activation_owns_channel_creation_and_enabling() { for (runtime_kind, model_id, provider_id) in [ diff --git a/openless-all/app/package.json b/openless-all/app/package.json index 8d97c795e..5f0f916e6 100644 --- a/openless-all/app/package.json +++ b/openless-all/app/package.json @@ -12,7 +12,7 @@ "preview": "vite preview", "tauri": "tauri", "tauri:android:init": "tauri android init", - "tauri:android:dev": "tauri android dev", + "tauri:android:dev": "node scripts/run-android-tauri-dev.mjs", "tauri:android:build": "node scripts/run-android-tauri-build.mjs --debug", "tauri:android:build:debug": "node scripts/run-android-tauri-build.mjs --debug", "tauri:android:build:release": "node scripts/run-android-tauri-build.mjs --release", diff --git a/openless-all/app/scripts/build-mobile-rime.py b/openless-all/app/scripts/build-mobile-rime.py index dfb724ed7..1005c91c9 100644 --- a/openless-all/app/scripts/build-mobile-rime.py +++ b/openless-all/app/scripts/build-mobile-rime.py @@ -7,6 +7,7 @@ from pathlib import Path import shutil import subprocess +import sys import tarfile import urllib.request @@ -38,7 +39,19 @@ def fetch(name, url): unpack = CACHE / 'sources' / (name + '-unpack') unpack.mkdir(parents=True, exist_ok=True) with tarfile.open(archive) as tar: - tar.extractall(unpack, filter='data') + if sys.version_info >= (3, 12): + tar.extractall(unpack, filter='data') + else: + # Python 3.11 and older do not support TarFile's data filter. Reject + # traversal and link entries before extracting with the legacy API. + root = os.path.realpath(unpack) + for member in tar.getmembers(): + target = os.path.realpath(os.path.join(root, member.name)) + if os.path.commonpath((root, target)) != root: + raise RuntimeError('Unsafe archive path: ' + member.name) + if member.issym() or member.islnk(): + raise RuntimeError('Unsafe archive link: ' + member.name) + tar.extractall(unpack) roots = list(unpack.iterdir()) if len(roots) != 1: raise RuntimeError('Unexpected archive structure: ' + name) diff --git a/openless-all/app/scripts/macos-local-asr-retirement-contract.test.mjs b/openless-all/app/scripts/macos-local-asr-retirement-contract.test.mjs index 4dd42fd6a..ddb5e7c89 100644 --- a/openless-all/app/scripts/macos-local-asr-retirement-contract.test.mjs +++ b/openless-all/app/scripts/macos-local-asr-retirement-contract.test.mjs @@ -19,7 +19,16 @@ assert.match(router, /apple_speech::AppleSpeechTranscriptionEngine/); const apple = read('src-tauri/src/asr/apple_speech.rs'); assert.doesNotMatch(apple, /ModelStore|ExperimentalNativeRuntime|download_model/); assert.match(read('src-tauri/Info.plist'), /NSSpeechRecognitionUsageDescription/); -assert.match(adapters, /cfg\(target_os = "windows"\)\]\s+struct TauriNativeTranscriptionEngine/); +// The native Sherpa/Foundry engine compiles only where build.rs sets +// `local_asr_sherpa` (Windows, arm64 Android) — never on macOS. +assert.match(adapters, /cfg\(local_asr_sherpa\)\]\s+struct TauriNativeTranscriptionEngine/); +const buildScript = read('src-tauri/build.rs'); +const sherpaCfg = buildScript.slice( + buildScript.indexOf('rustc-check-cfg=cfg(local_asr_sherpa)'), + buildScript.indexOf('cargo:rustc-cfg=local_asr_sherpa'), +); +assert.match(sherpaCfg, /cargo_target_os == "windows" \|\| \(target_os == "android" && target_arch == "aarch64"\)/); +assert.doesNotMatch(sherpaCfg, /macos/); const availability = adapters.slice( adapters.indexOf('impl openless_core::ModelRuntimeAdapter'), adapters.indexOf('fn inspect_native_models'), diff --git a/openless-all/app/scripts/merge-android-overlay-manifest.mjs b/openless-all/app/scripts/merge-android-overlay-manifest.mjs index f40d45162..bf4ba69e7 100644 --- a/openless-all/app/scripts/merge-android-overlay-manifest.mjs +++ b/openless-all/app/scripts/merge-android-overlay-manifest.mjs @@ -14,6 +14,8 @@ const PERMISSIONS = [ 'android.permission.FOREGROUND_SERVICE_SPECIAL_USE', 'android.permission.REQUEST_IGNORE_BATTERY_OPTIMIZATIONS', 'android.permission.POST_NOTIFICATIONS', + // The keyboard's status dot shows pure-local raw output while offline. + 'android.permission.ACCESS_NETWORK_STATE', ]; const APPLICATION_SNIPPET = ` diff --git a/openless-all/app/scripts/run-android-tauri-build.mjs b/openless-all/app/scripts/run-android-tauri-build.mjs index f2a1b8a9f..3c2b0aaf9 100644 --- a/openless-all/app/scripts/run-android-tauri-build.mjs +++ b/openless-all/app/scripts/run-android-tauri-build.mjs @@ -7,11 +7,50 @@ * node scripts/run-android-tauri-build.mjs --release */ import { spawnSync } from 'node:child_process'; +import { copyFileSync, existsSync, mkdirSync } from 'node:fs'; +import { join, resolve as resolvePath } from 'node:path'; import process from 'node:process'; -import { parseAndroidAbis, ANDROID_ABI_MATRIX, entryForAbi } from './android-abi-matrix.mjs'; import { fileURLToPath } from 'node:url'; +import { parseAndroidAbis, ANDROID_ABI_MATRIX, entryForAbi } from './android-abi-matrix.mjs'; +import { + SHERPA_ONNX_ANDROID_ABI, + SHERPA_ONNX_ANDROID_LIBS, + ensureSherpaOnnxAndroidLibs, +} from './sherpa-onnx-android.mjs'; -function resolveTargets() { +const appRoot = fileURLToPath(new URL('..', import.meta.url)); + +/** + * Local ASR links sherpa-onnx on arm64 only (see sherpa-onnx-android.mjs): + * point the crate at the verified libraries and put them in jniLibs so the + * arm64 APK ships what libopenless_lib.so needs at load time. + */ +export async function prepareLocalAsrLibs(targets, env) { + if (!targets.includes('aarch64')) return; + const libDir = await ensureSherpaOnnxAndroidLibs(); + env.SHERPA_ONNX_LIB_DIR = libDir; + const jniLibs = join( + appRoot, + 'src-tauri', + 'gen', + 'android', + 'app', + 'src', + 'main', + 'jniLibs', + SHERPA_ONNX_ANDROID_ABI, + ); + if (!existsSync(join(appRoot, 'src-tauri', 'gen', 'android'))) { + throw new Error( + 'Generated Android project not found under src-tauri/gen/android; run "npm run tauri -- android init --ci" first.', + ); + } + mkdirSync(jniLibs, { recursive: true }); + for (const lib of SHERPA_ONNX_ANDROID_LIBS) copyFileSync(join(libDir, lib), join(jniLibs, lib)); + console.log(`[android-build] local ASR libraries: ${libDir}`); +} + +export function resolveTargets() { const raw = process.env.OPENLESS_ANDROID_TARGETS ?? ''; if (!raw.trim()) { return ANDROID_ABI_MATRIX.map((e) => e.abi); @@ -21,21 +60,16 @@ function resolveTargets() { }).map((e) => e.abi); } -function main() { +async function main() { const mode = process.argv[2]; if (mode !== '--debug' && mode !== '--release') { console.error('Usage: node scripts/run-android-tauri-build.mjs --debug|--release'); process.exit(1); } const targets = resolveTargets(); - const rimeBuilder = fileURLToPath(new URL('./build-mobile-rime.py', import.meta.url)); - for (const target of targets) { - const rime = spawnSync(process.env.PYTHON ?? 'python3', [rimeBuilder, 'android', '--stage', '--abi', entryForAbi(target).gradleAbi], { - stdio: 'inherit', env: process.env, - }); - if (rime.error) throw rime.error; - if (rime.status !== 0) process.exit(rime.status ?? 1); - } + const env = { ...process.env }; + await prepareLocalAsrLibs(targets, env); + stageMobileRime(targets, env); const args = ['tauri', 'android', 'build', '--apk', '--split-per-abi', '--target', ...targets]; if (mode === '--debug') { args.push('--debug'); @@ -53,7 +87,7 @@ function main() { const bin = process.platform === 'win32' ? 'npx.cmd' : 'npx'; const result = spawnSync(bin, args, { stdio: 'inherit', - env: process.env, + env, shell: process.platform === 'win32', }); if (result.error) { @@ -62,4 +96,45 @@ function main() { process.exit(result.status ?? 1); } -main(); +function resolvePython(env) { + if (env.PYTHON?.trim()) { + return { command: env.PYTHON, prefix: [] }; + } + if (process.platform !== 'win32') { + return { command: 'python3', prefix: [] }; + } + for (const command of ['python', 'py']) { + const probe = spawnSync(command, ['--version'], { + stdio: 'ignore', + env, + shell: true, + }); + if (!probe.error && probe.status === 0) { + return { command, prefix: command === 'py' ? ['-3'] : [] }; + } + } + return { command: 'python', prefix: [] }; +} + +export function stageMobileRime(targets, env) { + const rimeBuilder = fileURLToPath(new URL('./build-mobile-rime.py', import.meta.url)); + const python = resolvePython(env); + for (const target of targets) { + const rime = spawnSync( + python.command, + [...python.prefix, rimeBuilder, 'android', '--stage', '--abi', entryForAbi(target).gradleAbi], + { stdio: 'inherit', env, shell: process.platform === 'win32' }, + ); + if (rime.error) throw rime.error; + if (rime.status !== 0) { + throw new Error(`staging Rime for ${target} failed (exit ${rime.status})`); + } + } +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === resolvePath(process.argv[1])) { + main().catch((error) => { + console.error(`[android-build] ${error.message ?? error}`); + process.exit(1); + }); +} diff --git a/openless-all/app/scripts/run-android-tauri-dev.mjs b/openless-all/app/scripts/run-android-tauri-dev.mjs new file mode 100644 index 000000000..10f953fee --- /dev/null +++ b/openless-all/app/scripts/run-android-tauri-dev.mjs @@ -0,0 +1,79 @@ +/** + * Prepare native Android dependencies and run `tauri android dev`. + * + * The direct Tauri dev command does not know about the verified Sherpa + * libraries or the mobile Rime staging step used by the release wrapper. + */ +import { spawnSync } from 'node:child_process'; +import { existsSync } from 'node:fs'; +import { join } from 'node:path'; +import process from 'node:process'; +import { fileURLToPath } from 'node:url'; +import { + prepareLocalAsrLibs, + resolveTargets, + stageMobileRime, +} from './run-android-tauri-build.mjs'; + +const appRoot = fileURLToPath(new URL('..', import.meta.url)); +const generatedAndroid = join(appRoot, 'src-tauri', 'gen', 'android'); + +function npxBin() { + return process.platform === 'win32' ? 'npx.cmd' : 'npx'; +} + +function run(args, env) { + const result = spawnSync(npxBin(), args, { + stdio: 'inherit', + env, + shell: process.platform === 'win32', + }); + if (result.error) throw result.error; + if (result.status !== 0) { + throw new Error(`${args.join(' ')} failed (exit ${result.status})`); + } +} + +function runNode(script, args, env) { + const command = [script, ...args]; + const result = spawnSync(process.execPath, command, { + stdio: 'inherit', + env, + shell: false, + }); + if (result.error) throw result.error; + if (result.status !== 0) { + throw new Error(`${command.join(' ')} failed (exit ${result.status})`); + } +} + +async function main() { + const env = { ...process.env }; + const targets = resolveTargets(); + if (!existsSync(generatedAndroid)) { + console.log('[android-dev] generated Android project missing; initializing it'); + run(['tauri', 'android', 'init', '--ci'], env); + } + // Keep direct development builds identical to the release wrapper: Tauri's + // generated project is only a base, so copy the keyboard sources and apply + // the manifest/Gradle integrations before starting Gradle. + for (const script of [ + 'copy-android-scaffolding.mjs', + 'merge-android-v1-manifest.mjs', + 'merge-android-overlay-manifest.mjs', + 'merge-android-shizuku-manifest.mjs', + 'patch-android-shizuku-deps.mjs', + 'merge-android-updater-manifest.mjs', + ]) { + runNode(join(appRoot, 'scripts', script), [], env); + } + await prepareLocalAsrLibs(targets, env); + stageMobileRime(targets, env); + console.log('[android-dev] starting Tauri Android development mode'); + run(['tauri', 'android', 'dev'], env); +} + +main().catch((error) => { + console.error(`[android-dev] ${error.message ?? error}`); + process.exit(1); +}); diff --git a/openless-all/app/scripts/sherpa-onnx-android.mjs b/openless-all/app/scripts/sherpa-onnx-android.mjs new file mode 100644 index 000000000..2ee65ef72 --- /dev/null +++ b/openless-all/app/scripts/sherpa-onnx-android.mjs @@ -0,0 +1,98 @@ +/** + * Fetches the official sherpa-onnx Android libraries that local ASR links on + * arm64 Android builds. + * + * The sherpa-onnx crate downloads prebuilt libraries for desktop targets only; + * for Android it reads SHERPA_ONNX_LIB_DIR. This pins the release, checks the + * archive's size and SHA-256, and extracts the arm64-v8a + * libsherpa-onnx-c-api.so and libonnxruntime.so into a cache under + * src-tauri/target/, so a build never links an unverified download. + * + * Usage (also called by run-android-tauri-build.mjs): + * node scripts/sherpa-onnx-android.mjs # prints the library directory + */ +import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { createReadStream, existsSync, mkdirSync, renameSync, rmSync, statSync } from 'node:fs'; +import { writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import process from 'node:process'; +import { fileURLToPath } from 'node:url'; + +// Must match the sherpa-onnx crate version in src-tauri/Cargo.toml. +export const SHERPA_ONNX_VERSION = '1.13.2'; +export const SHERPA_ONNX_ANDROID_ARCHIVE = { + name: `sherpa-onnx-v${SHERPA_ONNX_VERSION}-android.tar.bz2`, + url: `https://github.com/k2-fsa/sherpa-onnx/releases/download/v${SHERPA_ONNX_VERSION}/sherpa-onnx-v${SHERPA_ONNX_VERSION}-android.tar.bz2`, + sizeBytes: 52_236_308, + sha256: 'fc4d17941152941a883b0cfabfc9acac118682324e9f97df6c1ae1360bc7bc8e', +}; +/** The only ABI local ASR is built for; the others stay cloud-only. */ +export const SHERPA_ONNX_ANDROID_ABI = 'arm64-v8a'; +/** What the app links and ships: libopenless_lib.so needs the C API, which needs onnxruntime. */ +export const SHERPA_ONNX_ANDROID_LIBS = ['libsherpa-onnx-c-api.so', 'libonnxruntime.so']; + +const appRoot = fileURLToPath(new URL('..', import.meta.url)); + +export function sherpaOnnxAndroidCacheDir(root = appRoot) { + return join(root, 'src-tauri', 'target', 'sherpa-onnx-android', `v${SHERPA_ONNX_VERSION}`); +} + +export function sherpaOnnxAndroidLibDir(root = appRoot) { + return join(sherpaOnnxAndroidCacheDir(root), 'jniLibs', SHERPA_ONNX_ANDROID_ABI); +} + +async function sha256Of(path) { + const hash = createHash('sha256'); + for await (const chunk of createReadStream(path)) hash.update(chunk); + return hash.digest('hex'); +} + +/** Returns the directory holding the verified arm64 libraries, downloading them first if needed. */ +export async function ensureSherpaOnnxAndroidLibs({ root = appRoot, log = console.log } = {}) { + const libDir = sherpaOnnxAndroidLibDir(root); + if (SHERPA_ONNX_ANDROID_LIBS.every((lib) => existsSync(join(libDir, lib)))) return libDir; + + const cacheDir = sherpaOnnxAndroidCacheDir(root); + mkdirSync(cacheDir, { recursive: true }); + const archive = join(cacheDir, SHERPA_ONNX_ANDROID_ARCHIVE.name); + const valid = async () => + existsSync(archive) && + statSync(archive).size === SHERPA_ONNX_ANDROID_ARCHIVE.sizeBytes && + (await sha256Of(archive)) === SHERPA_ONNX_ANDROID_ARCHIVE.sha256; + + if (!(await valid())) { + log(`[sherpa-onnx] downloading ${SHERPA_ONNX_ANDROID_ARCHIVE.url}`); + const response = await fetch(SHERPA_ONNX_ANDROID_ARCHIVE.url); + if (!response.ok) throw new Error(`sherpa-onnx download failed: HTTP ${response.status}`); + const partial = `${archive}.part`; + await writeFile(partial, Buffer.from(await response.arrayBuffer())); + renameSync(partial, archive); + if (!(await valid())) { + rmSync(archive, { force: true }); + throw new Error(`sherpa-onnx archive failed size/SHA-256 verification: ${SHERPA_ONNX_ANDROID_ARCHIVE.name}`); + } + } + + const members = SHERPA_ONNX_ANDROID_LIBS.map((lib) => `./jniLibs/${SHERPA_ONNX_ANDROID_ABI}/${lib}`); + const result = spawnSync('tar', ['-xjf', SHERPA_ONNX_ANDROID_ARCHIVE.name, ...members], { + cwd: cacheDir, + stdio: 'inherit', + }); + if (result.error) throw result.error; + if (result.status !== 0) throw new Error(`extracting ${SHERPA_ONNX_ANDROID_ARCHIVE.name} failed (tar exit ${result.status})`); + for (const lib of SHERPA_ONNX_ANDROID_LIBS) { + if (!existsSync(join(libDir, lib))) throw new Error(`${lib} missing from ${SHERPA_ONNX_ANDROID_ARCHIVE.name}`); + } + log(`[sherpa-onnx] libraries ready in ${libDir}`); + return libDir; +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { + ensureSherpaOnnxAndroidLibs({ log: (line) => console.error(line) }) + .then((dir) => console.log(dir)) + .catch((error) => { + console.error(error.message ?? error); + process.exit(1); + }); +} diff --git a/openless-all/app/src-tauri/Cargo.toml b/openless-all/app/src-tauri/Cargo.toml index 5794b4863..36b7f677d 100644 --- a/openless-all/app/src-tauri/Cargo.toml +++ b/openless-all/app/src-tauri/Cargo.toml @@ -122,6 +122,13 @@ jni = "0.21" ndk-context = "0.1" tao = "0.35" +# Local ASR on arm64 Android: the crate has no Android prebuilt download, so +# scripts/run-android-tauri-build.mjs points SHERPA_ONNX_LIB_DIR at the +# verified official libraries (scripts/sherpa-onnx-android.mjs) and ships them +# in jniLibs. Other Android ABIs stay cloud-only. +[target.'cfg(all(target_os = "android", target_arch = "aarch64"))'.dependencies] +sherpa-onnx = { version = "=1.13.2", default-features = false, features = ["shared"] } + [target.'cfg(target_os = "macos")'.dependencies] sherpa-onnx = { version = "=1.13.2", default-features = false, features = ["static"] } security-framework = "=3.7.0" diff --git a/openless-all/app/src-tauri/assets/local-asr-test.wav b/openless-all/app/src-tauri/assets/local-asr-test.wav new file mode 100644 index 000000000..4f213a3dc Binary files /dev/null and b/openless-all/app/src-tauri/assets/local-asr-test.wav differ diff --git a/openless-all/app/src-tauri/build.rs b/openless-all/app/src-tauri/build.rs index fd3b2a50f..914dac69e 100644 --- a/openless-all/app/src-tauri/build.rs +++ b/openless-all/app/src-tauri/build.rs @@ -29,6 +29,17 @@ fn main() { link_android_cpp_runtime(); } + // `local_asr_sherpa`: targets that link sherpa-onnx for local ASR — + // Windows, and arm64 Android (see the Cargo.toml dependency comment). + // classify_target_os() only names android/macos/linux, so Windows is + // read from Cargo's own variable. + println!("cargo::rustc-check-cfg=cfg(local_asr_sherpa)"); + let cargo_target_os = std::env::var("CARGO_CFG_TARGET_OS").unwrap_or_default(); + let target_arch = std::env::var("CARGO_CFG_TARGET_ARCH").unwrap_or_default(); + if cargo_target_os == "windows" || (target_os == "android" && target_arch == "aarch64") { + println!("cargo:rustc-cfg=local_asr_sherpa"); + } + tauri_build::build(); } diff --git a/openless-all/app/src-tauri/src/android/native_bridge.rs b/openless-all/app/src-tauri/src/android/native_bridge.rs index f1510392c..d66d1b747 100644 --- a/openless-all/app/src-tauri/src/android/native_bridge.rs +++ b/openless-all/app/src-tauri/src/android/native_bridge.rs @@ -1330,6 +1330,51 @@ mod jni_exports { spawn_remove_correction_rule(pattern_str); } + /// "local", "cloud" or "unavailable": which recognizer the next dictation + /// uses (OpenLessBackend::asr_route()), for the keyboard's status dot. + /// Blocks on a credential read — call it off the main thread. + #[no_mangle] + pub unsafe extern "system" fn Java_com_openless_app_OpenLessNative_nativeAsrRoute( + env: *mut JNIEnv, + _class: JClass, + ) -> jstring { + let route = match CORE_BACKEND.get() { + Some(backend) => match tauri::async_runtime::block_on(backend.asr_route()) { + openless_core::AsrRoute::Local => "local", + openless_core::AsrRoute::LocalRaw => "localRaw", + openless_core::AsrRoute::Cloud => "cloud", + openless_core::AsrRoute::Unavailable => "unavailable", + }, + None => "unavailable", + }; + match JniEnv::from_raw(env) { + Ok(mut env) => crate::android::jni::android::export_jstring(&mut env, route), + Err(_) => std::ptr::null_mut(), + } + } + + /// ",," from history, latest being "local", "cloud" + /// or empty; null when the backend is not up. + #[no_mangle] + pub unsafe extern "system" fn Java_com_openless_app_OpenLessNative_nativeAsrUsage( + env: *mut JNIEnv, + _class: JClass, + ) -> jstring { + let Some(usage) = CORE_BACKEND.get().and_then(|backend| backend.asr_usage().ok()) else { + return std::ptr::null_mut(); + }; + let latest = match usage.last_local { + Some(true) => "local", + Some(false) => "cloud", + None => "", + }; + let encoded = format!("{},{},{}", usage.local, usage.cloud, latest); + match JniEnv::from_raw(env) { + Ok(mut env) => crate::android::jni::android::export_jstring(&mut env, &encoded), + Err(_) => std::ptr::null_mut(), + } + } + #[no_mangle] pub unsafe extern "system" fn Java_com_openless_app_OpenLessNative_nativeBackendSnapshot( env: *mut JNIEnv, diff --git a/openless-all/app/src-tauri/src/asr/local/mod.rs b/openless-all/app/src-tauri/src/asr/local/mod.rs index 71aa28c36..2da195d03 100644 --- a/openless-all/app/src-tauri/src/asr/local/mod.rs +++ b/openless-all/app/src-tauri/src/asr/local/mod.rs @@ -1,6 +1,6 @@ //! Native local speech runtimes. macOS uses opt-in experimental modules only; //! old IDs remain metadata-only and cannot link their former inference libraries. -#[cfg(any(target_os = "windows", test))] +#[cfg(any(local_asr_sherpa, test))] mod blocking_decode; pub mod experimental; pub mod foundry; diff --git a/openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs b/openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs index 7de3b00f7..58f4eba0f 100644 --- a/openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs +++ b/openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs @@ -18,7 +18,7 @@ use anyhow::{Context, Result}; use parking_lot::Mutex; use tokio::sync::Mutex as AsyncMutex; -#[cfg(target_os = "windows")] +#[cfg(local_asr_sherpa)] use super::blocking_decode::run_serialized_decode; use crate::asr::local::sherpa::{ @@ -28,7 +28,7 @@ use crate::asr::local::sherpa::{ const QWEN3_ASR_CHUNK_LIMIT_MS: u64 = 30_000; const QWEN3_ASR_MAX_NEW_TOKENS: i32 = 256; -#[cfg(target_os = "windows")] +#[cfg(local_asr_sherpa)] use sherpa_onnx::{ OfflineParaformerModelConfig, OfflineQwen3ASRModelConfig, OfflineRecognizer, OfflineRecognizerConfig, OfflineSenseVoiceModelConfig, OfflineWhisperModelConfig, @@ -40,9 +40,9 @@ use sherpa_onnx::{ #[derive(Clone)] struct LoadedOfflineModel { alias: String, - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] recognizer: Arc, - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] decode_gate: Arc, } @@ -50,7 +50,7 @@ struct LoadedOfflineModel { #[derive(Clone)] struct LoadedOnlineModel { alias: String, - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] recognizer: Arc, } @@ -92,11 +92,11 @@ impl SherpaOnnxRuntime { } } - /// Whether the runtime actually has inference capability today. Only Windows currently - /// wires in the `sherpa-onnx` offline recognizer. + /// Whether the runtime actually has inference capability today: the targets that link + /// `sherpa-onnx` (Windows, arm64 Android — the `local_asr_sherpa` cfg from build.rs). #[allow(dead_code)] pub fn is_available(&self) -> bool { - cfg!(target_os = "windows") + cfg!(local_asr_sherpa) } pub async fn status_snapshot(&self, active_model: &str) -> SherpaRuntimeStatus { @@ -457,7 +457,7 @@ enum LoadedModel { Online(LoadedOnlineModel), } -#[cfg(target_os = "windows")] +#[cfg(local_asr_sherpa)] async fn load_model(alias: &str, dir: &Path) -> Result { let alias = alias.to_string(); let dir = dir.to_path_buf(); @@ -485,7 +485,7 @@ async fn load_model(alias: &str, dir: &Path) -> Result { .map_err(|e| anyhow::anyhow!("sherpa-onnx load join failed: {e:#}"))? } -#[cfg(not(target_os = "windows"))] +#[cfg(not(local_asr_sherpa))] async fn load_model(alias: &str, _dir: &Path) -> Result { match sherpa_target(alias)?.sherpa_execution_mode() { Some(openless_core::LocalAsrExecutionMode::Offline) => { @@ -502,7 +502,7 @@ async fn load_model(alias: &str, _dir: &Path) -> Result { } } -#[cfg(target_os = "windows")] +#[cfg(local_asr_sherpa)] fn create_offline_recognizer(alias: &str, dir: &Path) -> Result { let mut config = OfflineRecognizerConfig::default(); config.model_config.num_threads = std::thread::available_parallelism() @@ -556,7 +556,7 @@ fn create_offline_recognizer(alias: &str, dir: &Path) -> Result Result { let mut config = OnlineRecognizerConfig::default(); config.model_config.num_threads = std::thread::available_parallelism() @@ -586,7 +586,7 @@ fn create_online_recognizer(alias: &str, dir: &Path) -> Result .ok_or_else(|| anyhow::anyhow!("create sherpa-onnx online recognizer failed")) } -#[cfg(target_os = "windows")] +#[cfg(local_asr_sherpa)] fn path_to_string(path: &Path) -> Result { Ok(path .to_str() @@ -594,7 +594,7 @@ fn path_to_string(path: &Path) -> Result { .to_string()) } -#[cfg(target_os = "windows")] +#[cfg(local_asr_sherpa)] async fn transcribe_loaded_model( loaded: LoadedOfflineModel, pcm_chunks: Vec>, @@ -630,7 +630,7 @@ async fn transcribe_loaded_model( .await } -#[cfg(not(target_os = "windows"))] +#[cfg(not(local_asr_sherpa))] async fn transcribe_loaded_model( _loaded: LoadedOfflineModel, _pcm_chunks: Vec>, @@ -642,9 +642,9 @@ async fn transcribe_loaded_model( pub struct SherpaOnlineSession { alias: String, - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] recognizer: Arc, - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] stream: sherpa_onnx::OnlineStream, committed_text: String, last_partial_text: String, @@ -674,7 +674,7 @@ impl SherpaOnlineSession { self.finish_inner(on_token) } - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] fn accept_samples(&mut self, samples: &[f32], on_token: &F) -> Result<()> where F: Fn(&str), @@ -684,7 +684,7 @@ impl SherpaOnlineSession { Ok(()) } - #[cfg(not(target_os = "windows"))] + #[cfg(not(local_asr_sherpa))] fn accept_samples(&mut self, _samples: &[f32], _on_token: &F) -> Result<()> where F: Fn(&str), @@ -692,7 +692,7 @@ impl SherpaOnlineSession { Ok(()) } - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] fn finish_inner(&mut self, on_token: &F) -> Result where F: Fn(&str), @@ -703,7 +703,7 @@ impl SherpaOnlineSession { Ok(self.final_text()) } - #[cfg(not(target_os = "windows"))] + #[cfg(not(local_asr_sherpa))] fn finish_inner(&mut self, _on_token: &F) -> Result where F: Fn(&str), @@ -711,7 +711,7 @@ impl SherpaOnlineSession { Ok(self.final_text()) } - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] fn drain_ready(&mut self, on_token: &F) where F: Fn(&str), @@ -722,7 +722,7 @@ impl SherpaOnlineSession { } } - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] fn capture_result(&mut self, on_token: &F, allow_endpoint_reset: bool) where F: Fn(&str), @@ -781,7 +781,7 @@ fn append_segment(text: &mut String, segment: &str) { text.push_str(segment); } -#[cfg(target_os = "windows")] +#[cfg(local_asr_sherpa)] fn create_online_session_from_loaded(loaded: LoadedOnlineModel) -> Result { let stream = loaded.recognizer.create_stream(); Ok(SherpaOnlineSession { @@ -794,7 +794,7 @@ fn create_online_session_from_loaded(loaded: LoadedOnlineModel) -> Result Result { Ok(SherpaOnlineSession { alias: loaded.alias, @@ -845,7 +845,7 @@ mod tests { let status = runtime.status_snapshot("sense-voice-small-zh").await; assert_eq!(status.provider_id, PROVIDER_ID); - assert_eq!(status.available, cfg!(target_os = "windows")); + assert_eq!(status.available, cfg!(local_asr_sherpa)); assert!(!status.runtime_ready); assert_eq!(status.active_model, "sense-voice-small-zh"); assert_eq!(status.loaded_model_id, None); @@ -856,7 +856,7 @@ mod tests { assert_eq!(status.last_error, None); } - #[cfg(not(target_os = "windows"))] + #[cfg(not(local_asr_sherpa))] #[tokio::test] async fn status_snapshot_only_marks_active_model_ready() { let runtime = SherpaOnnxRuntime::new(); @@ -919,7 +919,7 @@ mod tests { #[tokio::test] async fn release_now_clears_loaded_model() { let runtime = SherpaOnnxRuntime::new(); - #[cfg(not(target_os = "windows"))] + #[cfg(not(local_asr_sherpa))] { runtime.state.lock().offline_loaded = Some(LoadedOfflineModel { alias: "sense-voice-small-zh".into(), @@ -1027,7 +1027,7 @@ mod tests { assert_eq!(text, "你好 world"); } - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] #[test] fn sherpa_onnx_crate_exposes_online_recognizer_api_for_streaming_phase() { let mut config = sherpa_onnx::OnlineRecognizerConfig::default(); diff --git a/openless-all/app/src-tauri/src/asr/local/test_run.rs b/openless-all/app/src-tauri/src/asr/local/test_run.rs index 2d8e96e56..6f1593089 100644 --- a/openless-all/app/src-tauri/src/asr/local/test_run.rs +++ b/openless-all/app/src-tauri/src/asr/local/test_run.rs @@ -3,7 +3,7 @@ use anyhow::Result; /// Walk the RIFF chunk chain strictly to find "data" — both jfk.wav and test_speech.wav carry /// LIST/INFO metadata after the fmt chunk, so a hardcoded +44 offset reads garbage. -pub(super) fn decode_wav_16k_mono(bytes: &[u8]) -> Result> { +pub(crate) fn decode_wav_16k_mono(bytes: &[u8]) -> Result> { if bytes.len() < 44 || &bytes[0..4] != b"RIFF" || &bytes[8..12] != b"WAVE" { anyhow::bail!("不是有效的 RIFF/WAVE 文件"); } diff --git a/openless-all/app/src-tauri/src/commands/credentials.rs b/openless-all/app/src-tauri/src/commands/credentials.rs index 9864d8369..d34495686 100644 --- a/openless-all/app/src-tauri/src/commands/credentials.rs +++ b/openless-all/app/src-tauri/src/commands/credentials.rs @@ -85,15 +85,7 @@ pub(crate) fn active_foundry_asr_is_supported(provider: &str) -> bool { } pub(crate) fn active_sherpa_asr_is_supported(provider: &str) -> bool { - #[cfg(all(not(mobile), target_os = "windows"))] - { - provider == crate::asr::local::sherpa::PROVIDER_ID - } - #[cfg(not(all(not(mobile), target_os = "windows")))] - { - let _ = provider; - false - } + cfg!(local_asr_sherpa) && provider == crate::asr::local::sherpa::PROVIDER_ID } impl openless_core::CredentialStore for SystemCredentialStore { @@ -326,7 +318,11 @@ fn credentials_status( let asr_configured = openless_core::provider_rules::asr_configured( &active_asr_provider, &configuration, - local_asr_configured(&active_asr_provider, model_store), + local_asr_configured( + &active_asr_provider, + model_store, + Some(&preferences.sherpa_onnx_model), + ), ); let llm_configured = openless_core::provider_rules::llm_configured(&active_llm_provider, &configuration); @@ -590,13 +586,14 @@ fn asr_configured_for_provider_with_model_store( openless_core::provider_rules::asr_configured( provider, &credential_configuration(snap, "", false), - local_asr_configured(provider, model_store), + local_asr_configured(provider, model_store, None), ) } fn local_asr_configured( provider: &str, model_store: Option<&openless_core::ModelStore>, + sherpa_model: Option<&str>, ) -> Option { if provider == "apple-speech" { return Some(cfg!(target_os = "macos")); @@ -616,7 +613,14 @@ fn local_asr_configured( return Some(active_foundry_asr_is_supported(provider)); } if provider == crate::asr::local::sherpa::PROVIDER_ID { - return Some(active_sherpa_asr_is_supported(provider)); + let model_installed = sherpa_model.is_none_or(|model| { + openless_core::LocalAsrTarget::parse(openless_core::LocalAsrRuntime::SherpaOnnx, model) + .ok() + .is_some_and(|target| { + model_store.is_some_and(|store| store.is_installed(&target).unwrap_or(false)) + }) + }); + return Some(active_sherpa_asr_is_supported(provider) && model_installed); } None } @@ -1036,6 +1040,45 @@ mod tests { } } + #[test] + fn sherpa_configuration_requires_the_selected_model_receipt() { + let data_dir = std::env::temp_dir().join(format!( + "openless-credentials-sherpa-config-{}", + uuid::Uuid::new_v4() + )); + let store = openless_core::ModelStore::new( + openless_core::ModelStoreConfig::new(data_dir.join("models")).unwrap(), + ) + .unwrap(); + let provider = crate::asr::local::sherpa::PROVIDER_ID; + let model = "sense-voice-small-zh"; + + assert_eq!( + local_asr_configured(provider, Some(&store), Some(model)), + Some(false) + ); + let target = openless_core::LocalAsrTarget::parse( + openless_core::LocalAsrRuntime::SherpaOnnx, + model, + ) + .unwrap(); + let model_dir = store.model_dir(&target).unwrap(); + std::fs::create_dir_all(&model_dir).unwrap(); + std::fs::write( + model_dir.join(openless_core::MODEL_READY_SENTINEL), + b"ready", + ) + .unwrap(); + std::fs::write(model_dir.join("model.int8.onnx"), b"model").unwrap(); + std::fs::write(model_dir.join("tokens.txt"), b"tokens").unwrap(); + + assert_eq!( + local_asr_configured(provider, Some(&store), Some(model)), + Some(active_sherpa_asr_is_supported(provider)) + ); + let _ = std::fs::remove_dir_all(data_dir); + } + #[test] fn core_llm_accounts_are_supported_by_the_tauri_vault_adapter() { for account in openless_core::llm_protocol::CONFIG_ACCOUNTS { diff --git a/openless-all/app/src-tauri/src/commands/mod.rs b/openless-all/app/src-tauri/src/commands/mod.rs index 00675353f..c73831c8c 100644 --- a/openless-all/app/src-tauri/src/commands/mod.rs +++ b/openless-all/app/src-tauri/src/commands/mod.rs @@ -61,7 +61,9 @@ mod foundry_asr; mod github_oauth; mod history; mod hotkeys; -#[cfg(not(mobile))] +// Local ASR commands also exist on arm64 Android, which links sherpa-onnx +// (build.rs `local_asr_sherpa`). +#[cfg(any(not(mobile), local_asr_sherpa))] mod local_asr; mod marketplace; mod misc; @@ -77,7 +79,7 @@ mod selection_polish_preview; #[cfg(all(not(mobile), target_os = "windows"))] mod selection_voice; mod settings; -#[cfg(not(mobile))] +#[cfg(any(not(mobile), local_asr_sherpa))] mod sherpa_asr; mod style_packs; mod voice_edit; @@ -95,7 +97,7 @@ pub use foundry_asr::*; pub use github_oauth::*; pub use history::*; pub use hotkeys::*; -#[cfg(not(mobile))] +#[cfg(any(not(mobile), local_asr_sherpa))] pub use local_asr::*; pub use marketplace::*; pub use misc::*; @@ -111,9 +113,9 @@ pub use selection_polish_preview::*; #[cfg(all(not(mobile), target_os = "windows"))] pub use selection_voice::*; pub use settings::*; -#[cfg(not(mobile))] -// Sherpa commands are registered on Windows only; other desktop targets keep -// an empty re-export. +#[cfg(any(not(mobile), local_asr_sherpa))] +// Sherpa commands are registered on Windows and arm64 Android; other desktop +// targets keep an empty re-export. #[allow(unused_imports)] pub use sherpa_asr::*; pub use style_packs::*; @@ -247,6 +249,13 @@ pub(crate) fn open_path_in_file_manager(path: &std::path::Path) -> Result<(), St .map_err(|e| e.to_string()) } +/// Model folders live in app-private storage on Android, which no file +/// manager can open; the local ASR "reveal" commands report that instead. +#[cfg(all(target_os = "android", local_asr_sherpa))] +pub(crate) fn open_path_in_file_manager(_path: &std::path::Path) -> Result<(), String> { + Err("Opening the model folder is not supported on Android".to_string()) +} + #[cfg(test)] mod tests { use super::{is_valid_local_pack_id, is_valid_session_id}; diff --git a/openless-all/app/src-tauri/src/coordinator.rs b/openless-all/app/src-tauri/src/coordinator.rs index 41eeb689f..2ace33447 100644 --- a/openless-all/app/src-tauri/src/coordinator.rs +++ b/openless-all/app/src-tauri/src/coordinator.rs @@ -684,6 +684,14 @@ impl Coordinator { ); let app = crate::core_adapters::app_handle_slot(); + // arm64 Android links sherpa-onnx (build.rs `local_asr_sherpa`) and + // gets the same runtime pair Windows builds in new_with_local_runtimes(). + #[cfg(local_asr_sherpa)] + let native_asr = crate::core_adapters::TauriNativeAsrDependencies::new( + Arc::new(crate::asr::local::FoundryLocalRuntime::new()), + Arc::new(crate::asr::local::SherpaOnnxRuntime::new()), + ); + #[cfg(not(local_asr_sherpa))] let native_asr = crate::core_adapters::TauriNativeAsrDependencies::new(); let hotkey_status = Arc::new(Mutex::new(HotkeyStatus::default())); let qa_context = Arc::new(TauriQaHostContext::default()); diff --git a/openless-all/app/src-tauri/src/core_adapters.rs b/openless-all/app/src-tauri/src/core_adapters.rs index 2b83bfae2..f583e18ec 100644 --- a/openless-all/app/src-tauri/src/core_adapters.rs +++ b/openless-all/app/src-tauri/src/core_adapters.rs @@ -56,14 +56,14 @@ impl openless_core::TaskSpawner for TauriTaskSpawner { pub(crate) struct TauriNativeAsrDependencies { foundry: Arc, sherpa: Arc, - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] foundry_generation: Arc, - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] sherpa_generation: Arc, } impl TauriNativeAsrDependencies { - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] pub(crate) fn new( foundry: Arc, sherpa: Arc, @@ -76,7 +76,7 @@ impl TauriNativeAsrDependencies { } } - #[cfg(not(target_os = "windows"))] + #[cfg(not(local_asr_sherpa))] pub(crate) fn new() -> Self { Self { foundry: Arc::new(crate::asr::local::FoundryLocalRuntime::new()), @@ -195,26 +195,32 @@ pub(crate) fn backend_dependencies( .register(*provider_type, Arc::clone(&production_asr)) .expect("built-in ASR provider ids are non-empty"); } - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] let native_asr: Arc = Arc::new(TauriNativeTranscriptionEngine::new( native_asr_dependencies, model_store.clone(), Arc::clone(&backend), )); - #[cfg(not(target_os = "windows"))] + #[cfg(not(local_asr_sherpa))] let _ = native_asr_dependencies; + // Foundry is Windows-only; Sherpa runs wherever sherpa-onnx is linked + // (Windows and arm64 Android — see build.rs). #[cfg(target_os = "windows")] for provider_type in [ crate::asr::local::foundry::PROVIDER_ID, "foundry-local", "foundry-whisper", - crate::asr::local::sherpa::PROVIDER_ID, - "sherpa-onnx", ] { transcription .register(provider_type, Arc::clone(&native_asr)) .expect("native ASR provider ids are non-empty"); } + #[cfg(local_asr_sherpa)] + for provider_type in [crate::asr::local::sherpa::PROVIDER_ID, "sherpa-onnx"] { + transcription + .register(provider_type, Arc::clone(&native_asr)) + .expect("native ASR provider ids are non-empty"); + } #[cfg(target_os = "macos")] transcription .register( @@ -350,17 +356,13 @@ impl openless_core::ModelRuntimeAdapter for TauriLocalAsrRuntimeAdapter { fn engine_available(&self, runtime: openless_core::LocalAsrRuntime) -> bool { match runtime { openless_core::LocalAsrRuntime::Generic => false, - openless_core::LocalAsrRuntime::Foundry - | openless_core::LocalAsrRuntime::SherpaOnnx => cfg!(target_os = "windows"), + openless_core::LocalAsrRuntime::Foundry => cfg!(target_os = "windows"), + openless_core::LocalAsrRuntime::SherpaOnnx => cfg!(local_asr_sherpa), } } fn supports_model(&self, target: &openless_core::LocalAsrTarget) -> bool { - match target.runtime { - openless_core::LocalAsrRuntime::Generic => false, - openless_core::LocalAsrRuntime::Foundry - | openless_core::LocalAsrRuntime::SherpaOnnx => cfg!(target_os = "windows"), - } + self.engine_available(target.runtime) } fn inspect_native_models( @@ -624,7 +626,7 @@ impl openless_core::ModelRuntimeAdapter for TauriLocalAsrRuntimeAdapter { model_dir: PathBuf, progress: openless_core::ModelPrepareProgressSink, ) -> BoxFuture<'static, Result> { - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] self.invalidate_scheduled_release(target.runtime); let foundry = Arc::clone(&self.native.foundry); let sherpa = Arc::clone(&self.native.sherpa); @@ -755,7 +757,7 @@ impl openless_core::ModelRuntimeAdapter for TauriLocalAsrRuntimeAdapter { &self, runtime: openless_core::LocalAsrRuntime, ) -> BoxFuture<'static, Result<(), BackendError>> { - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] self.invalidate_scheduled_release(runtime); let foundry = Arc::clone(&self.native.foundry); let sherpa = Arc::clone(&self.native.sherpa); @@ -799,13 +801,66 @@ impl openless_core::ModelRuntimeAdapter for TauriLocalAsrRuntimeAdapter { ) } + fn test_model_for_provider( + &self, + target: openless_core::LocalAsrTarget, + model_dir: PathBuf, + provider_type: String, + ) -> BoxFuture<'static, Result> { + #[cfg(local_asr_sherpa)] + { + match target.runtime { + openless_core::LocalAsrRuntime::SherpaOnnx => { + let sherpa = Arc::clone(&self.native.sherpa); + Box::pin(test_sherpa_model(target, model_dir, provider_type, sherpa)) + } + openless_core::LocalAsrRuntime::Foundry => { + #[cfg(target_os = "windows")] + { + let foundry = Arc::clone(&self.native.foundry); + Box::pin(test_foundry_model(target, provider_type, foundry)) + } + #[cfg(not(target_os = "windows"))] + { + let _ = (target, model_dir, provider_type); + Box::pin(async { + Err(BackendError::new( + BackendErrorCode::Unsupported, + "Foundry Local Whisper is not available on this platform", + )) + }) + } + } + openless_core::LocalAsrRuntime::Generic => { + let _ = (target, model_dir, provider_type); + Box::pin(async { + Err(BackendError::new( + BackendErrorCode::Unsupported, + "legacy generic local ASR verification is not supported", + )) + }) + } + } + } + #[cfg(not(local_asr_sherpa))] + { + let _ = (target, model_dir, provider_type); + Box::pin(async { + Err(BackendError::new( + BackendErrorCode::Unsupported, + "native local ASR is not available on this platform", + )) + }) + } + } + fn invalidate_route(&self, runtime: openless_core::LocalAsrRuntime) { if runtime == openless_core::LocalAsrRuntime::Foundry { self.native.foundry.invalidate_route(); } } - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] fn invalidate_scheduled_release(&self, runtime: openless_core::LocalAsrRuntime) { match runtime { openless_core::LocalAsrRuntime::Foundry => &self.native.foundry_generation, @@ -816,6 +871,143 @@ impl openless_core::ModelRuntimeAdapter for TauriLocalAsrRuntimeAdapter { } } +#[cfg(local_asr_sherpa)] +const LOCAL_ASR_TEST_WAV: &[u8] = include_bytes!("../assets/local-asr-test.wav"); + +#[cfg(all(local_asr_sherpa, target_os = "windows"))] +async fn test_foundry_model( + target: openless_core::LocalAsrTarget, + provider_type: String, + foundry: Arc, +) -> Result { + if target.runtime != openless_core::LocalAsrRuntime::Foundry + || !matches!( + provider_type.as_str(), + "foundry-local" | "foundry-whisper" | "foundry-local-whisper" + ) + { + return Err(BackendError::new( + BackendErrorCode::Unsupported, + "native Foundry channel verification requires a Foundry provider", + )); + } + let samples = crate::asr::local::test_run::decode_wav_16k_mono(LOCAL_ASR_TEST_WAV) + .map_err(|error| local_asr_backend_error(BackendErrorCode::Platform, error))?; + let audio_ms = samples.len() as u64 * 1_000 / 16_000; + let pcm = samples + .iter() + .flat_map(|sample| { + ((*sample * 32_768.0).clamp(-32_768.0, 32_767.0) as i16).to_le_bytes() + }) + .collect::>(); + let model_id = target.model_id().to_string(); + + let load_started = Instant::now(); + foundry + .ensure_loaded(&model_id, "auto") + .await + .map_err(map_native_asr_error)?; + let load_ms = load_started.elapsed().as_millis() as u64; + + let provider = crate::asr::local::FoundryLocalWhisperAsr::new( + Arc::clone(&foundry), + model_id, + "auto".into(), + None, + ); + LegacyAudioConsumer::consume_pcm_chunk(&provider, &pcm); + let transcribe_started = Instant::now(); + let result = provider + .transcribe_with_fallback_notice(std::time::Duration::from_secs(30), Arc::new(|_| {})) + .await + .map_err(map_native_asr_error); + if let Err(error) = foundry.release_now().await { + log::warn!("[local-asr-test] failed to release Foundry model: {error:#}"); + } + let output = result?; + let transcribe_ms = transcribe_started.elapsed().as_millis() as u64; + + Ok(openless_core::LocalAsrTestResult { + target, + backend: provider_type, + expected_text: "Hello. This is a test of the Voxtrail speech-to-text system.".into(), + transcribed_text: output.raw.text, + audio_ms, + load_ms, + transcribe_ms, + }) +} + +#[cfg(local_asr_sherpa)] +async fn test_sherpa_model( + target: openless_core::LocalAsrTarget, + model_dir: PathBuf, + provider_type: String, + sherpa: Arc, +) -> Result { + if target.runtime != openless_core::LocalAsrRuntime::SherpaOnnx + || !matches!(provider_type.as_str(), "sherpa-onnx" | "sherpa-onnx-local") + { + return Err(BackendError::new( + BackendErrorCode::Unsupported, + "native Sherpa channel verification requires a Sherpa provider", + )); + } + let samples = crate::asr::local::test_run::decode_wav_16k_mono(LOCAL_ASR_TEST_WAV) + .map_err(|error| local_asr_backend_error(BackendErrorCode::Platform, error))?; + let audio_ms = samples.len() as u64 * 1_000 / 16_000; + let pcm = samples + .iter() + .flat_map(|sample| { + ((*sample * 32_768.0).clamp(-32_768.0, 32_767.0) as i16).to_le_bytes() + }) + .collect::>(); + let model_id = target.model_id().to_string(); + + let load_started = Instant::now(); + if let Err(error) = sherpa.ensure_loaded(&model_id, &model_dir).await { + if let Err(release_error) = sherpa.release_now().await { + log::warn!("[local-asr-test] failed to release Sherpa model: {release_error:#}"); + } + return Err(map_native_asr_error(error)); + } + let load_ms = load_started.elapsed().as_millis() as u64; + + let transcribe_started = Instant::now(); + let result = async { + let provider = crate::asr::local::SherpaOnnxAsr::new_for_model( + Arc::clone(&sherpa), + model_id, + model_dir, + None, + None, + ) + .await + .map_err(map_native_asr_error)?; + LegacyAudioConsumer::consume_pcm_chunk(&provider, &pcm); + provider + .transcribe(std::time::Duration::from_secs(30)) + .await + .map_err(map_native_asr_error) + } + .await; + if let Err(error) = sherpa.release_now().await { + log::warn!("[local-asr-test] failed to release Sherpa model: {error:#}"); + } + let output = result?; + let transcribe_ms = transcribe_started.elapsed().as_millis() as u64; + + Ok(openless_core::LocalAsrTestResult { + target, + backend: provider_type, + expected_text: "Hello. This is a test of the Voxtrail speech-to-text system.".into(), + transcribed_text: output.text, + audio_ms, + load_ms, + transcribe_ms, + }) +} + impl TauriLocalAsrRuntimeAdapter { fn preload_for_lease( &self, @@ -824,9 +1016,9 @@ impl TauriLocalAsrRuntimeAdapter { provider_type: String, _activation_generation: Option, ) -> BoxFuture<'static, Result<(), BackendError>> { - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] let foundry = Arc::clone(&self.native.foundry); - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] let sherpa = Arc::clone(&self.native.sherpa); Box::pin(async move { if target.runtime != openless_core::LocalAsrRuntime::Generic { @@ -834,12 +1026,19 @@ impl TauriLocalAsrRuntimeAdapter { // verifies the receipt. preload can also be called standalone, so the really // loaded alias must be checked — never fake success, and never overturn a // successful prepare with Unsupported like the old implementation did. - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] { let ready = match target.runtime { + #[cfg(target_os = "windows")] openless_core::LocalAsrRuntime::Foundry => { foundry.is_loaded_for(target.model_id()) } + // Foundry is Windows-only; nothing is ever loaded for it elsewhere. + #[cfg(not(target_os = "windows"))] + openless_core::LocalAsrRuntime::Foundry => { + let _ = &foundry; + false + } openless_core::LocalAsrRuntime::SherpaOnnx => { let status = sherpa.status_snapshot(target.model_id()).await; status.runtime_ready @@ -854,11 +1053,11 @@ impl TauriLocalAsrRuntimeAdapter { ) }); } - #[cfg(not(target_os = "windows"))] + #[cfg(not(local_asr_sherpa))] { return Err(BackendError::new( BackendErrorCode::Unsupported, - "Foundry and Sherpa are only available on Windows", + "Foundry and Sherpa are not available on this platform", )); } } @@ -1474,15 +1673,15 @@ impl openless_core::PlatformApi for TauriPlatformApi { } } -#[cfg(target_os = "windows")] +#[cfg(local_asr_sherpa)] struct TauriNativeTranscriptionEngine { dependencies: TauriNativeAsrDependencies, model_store: Option>, - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] backend: BackendSlot, } -#[cfg(target_os = "windows")] +#[cfg(local_asr_sherpa)] impl TauriNativeTranscriptionEngine { fn new( dependencies: TauriNativeAsrDependencies, @@ -1492,45 +1691,106 @@ impl TauriNativeTranscriptionEngine { Self { dependencies, model_store, - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] backend: _backend, } } } -#[cfg(target_os = "windows")] +#[cfg(local_asr_sherpa)] #[derive(Clone)] enum TauriNativeTranscriptionSessionKind { - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] Foundry { provider: Arc, runtime: Arc, }, - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] Sherpa { provider: Arc, runtime: Arc, }, } -#[cfg(target_os = "windows")] +#[cfg(local_asr_sherpa)] #[derive(Clone)] struct TauriNativeTranscriptionSession { kind: TauriNativeTranscriptionSessionKind, asr_call_label: openless_core::AsrCallLabel, - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] backend: BackendSlot, - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] session_id: SessionId, - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] generation: u64, - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] current_generation: Arc, keep_loaded_secs: u32, released: Arc, } -#[cfg(target_os = "windows")] +#[cfg(local_asr_sherpa)] +enum NativeStartReleaseRuntime { + Foundry(Arc), + Sherpa(Arc), +} + +#[cfg(local_asr_sherpa)] +struct NativeStartReleaseGuard { + runtime: NativeStartReleaseRuntime, + keep_loaded_secs: u32, + generation: u64, + current_generation: Arc, + armed: bool, +} + +#[cfg(local_asr_sherpa)] +impl NativeStartReleaseGuard { + fn new( + runtime: NativeStartReleaseRuntime, + keep_loaded_secs: u32, + generation: u64, + current_generation: Arc, + ) -> Self { + Self { + runtime, + keep_loaded_secs, + generation, + current_generation, + armed: true, + } + } + + fn disarm(&mut self) { + self.armed = false; + } +} + +#[cfg(local_asr_sherpa)] +impl Drop for NativeStartReleaseGuard { + fn drop(&mut self) { + if !self.armed { + return; + } + match &self.runtime { + NativeStartReleaseRuntime::Foundry(runtime) => schedule_foundry_release( + Arc::clone(runtime), + None, + self.keep_loaded_secs, + self.generation, + Arc::clone(&self.current_generation), + ), + NativeStartReleaseRuntime::Sherpa(runtime) => schedule_sherpa_release( + Arc::clone(runtime), + self.keep_loaded_secs, + self.generation, + Arc::clone(&self.current_generation), + ), + } + } +} + +#[cfg(local_asr_sherpa)] impl TranscriptionEngine for TauriNativeTranscriptionEngine { fn start( &self, @@ -1541,7 +1801,7 @@ impl TranscriptionEngine for TauriNativeTranscriptionEngine { // The two Windows runtimes count generations independently; switching to Sherpa must not // permanently invalidate Foundry's release. Settings-page prepare/release also shares // these generations: an old session must not unload a newly enabled model. - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] let current_generation = Arc::clone( if context.asr.provider_type == openless_core::LocalAsrRuntime::Foundry.provider_id() { &self.dependencies.foundry_generation @@ -1549,21 +1809,33 @@ impl TranscriptionEngine for TauriNativeTranscriptionEngine { &self.dependencies.sherpa_generation }, ); - #[cfg(target_os = "windows")] - let generation = current_generation.fetch_add(1, Ordering::AcqRel) + 1; let keep_loaded_secs = context.asr.keep_loaded_secs.unwrap_or(0); - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] let foundry = Arc::clone(&self.dependencies.foundry); - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] let backend = Arc::clone(&self.backend); - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] let sherpa = Arc::clone(&self.dependencies.sherpa); let model_store = self.model_store.clone(); Box::pin(async move { let provider_type = context.asr.provider_type.as_str(); - #[cfg(target_os = "windows")] + let generation = current_generation.fetch_add(1, Ordering::AcqRel) + 1; + let release_runtime = if provider_type + == openless_core::LocalAsrRuntime::Foundry.provider_id() + { + NativeStartReleaseRuntime::Foundry(Arc::clone(&foundry)) + } else { + NativeStartReleaseRuntime::Sherpa(Arc::clone(&sherpa)) + }; + let mut start_release_guard = NativeStartReleaseGuard::new( + release_runtime, + keep_loaded_secs, + generation, + Arc::clone(¤t_generation), + ); + #[cfg(local_asr_sherpa)] let (kind, label_model) = if provider_type == openless_core::LocalAsrRuntime::Foundry.provider_id() { @@ -1585,6 +1857,9 @@ impl TranscriptionEngine for TauriNativeTranscriptionEngine { }); ( TauriNativeTranscriptionSessionKind::Foundry { + // Foundry is never registered off Windows (see backend_dependencies); + // the non-Windows constructor only keeps this branch compiling there. + #[cfg(target_os = "windows")] provider: Arc::new(crate::asr::local::FoundryLocalWhisperAsr::new( Arc::clone(&foundry), model.clone(), @@ -1595,11 +1870,16 @@ impl TranscriptionEngine for TauriNativeTranscriptionEngine { .unwrap_or_else(|| "auto".to_string()), context.asr.language.clone(), )), + #[cfg(not(target_os = "windows"))] + provider: Arc::new(crate::asr::local::FoundryLocalWhisperAsr::new( + model.clone(), + context.asr.language.clone(), + )), runtime: foundry, }, Some(model), ) - } else if provider_type == openless_core::LocalAsrRuntime::SherpaOnnx.provider_id() { + } else if matches!(provider_type, "sherpa-onnx-local" | "sherpa-onnx") { let model = context .asr .model @@ -1669,39 +1949,43 @@ impl TranscriptionEngine for TauriNativeTranscriptionEngine { )); }; - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] let _ = partials; let asr_call_label = openless_core::AsrCallLabel::new(context.asr.provider_type.clone(), label_model); - Ok(Arc::new(TauriNativeTranscriptionSession { + let session = Arc::new(TauriNativeTranscriptionSession { kind, asr_call_label, - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] backend, - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] session_id: _session_id, - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] generation, - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] current_generation, keep_loaded_secs, released: Arc::new(AtomicBool::new(false)), - }) as Arc) + }); + // A dropped native start future is cleaned by NativeStartReleaseGuard. Once the + // session exists, its finish/cancel path owns the same generation and release. + start_release_guard.disarm(); + Ok(session as Arc) }) } } -#[cfg(target_os = "windows")] +#[cfg(local_asr_sherpa)] impl CoreAudioConsumer for TauriNativeTranscriptionSession { fn consume_pcm_chunk(&self, pcm: &[u8]) { match &self.kind { - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] TauriNativeTranscriptionSessionKind::Foundry { provider, .. } => { LegacyAudioConsumer::consume_pcm_chunk(provider.as_ref(), pcm); } - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] TauriNativeTranscriptionSessionKind::Sherpa { provider, .. } => { LegacyAudioConsumer::consume_pcm_chunk(provider.as_ref(), pcm); } @@ -1709,27 +1993,27 @@ impl CoreAudioConsumer for TauriNativeTranscriptionSession { } } -#[cfg(target_os = "windows")] +#[cfg(local_asr_sherpa)] impl TranscriptionSession for TauriNativeTranscriptionSession { fn asr_call_label(&self) -> Option { Some(self.asr_call_label.clone()) } fn finish(&self) -> BoxFuture<'static, Result> { - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] use crate::asr::local::foundry_runtime as foundry; let session = self.clone(); let kind = self.kind.clone(); Box::pin(async move { - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] let mut recovery = None; // Don't arrange the release before each branch's `?`: all results flow through this // one point. cancel and finish may run concurrently; `released` guarantees resource // teardown is arranged only once, and an old generation can't clear a new session. let result = async { let output = match kind { - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] TauriNativeTranscriptionSessionKind::Foundry { provider, .. } => { let timeout = openless_core::provider_rules::native_transcribe_timeout( "foundry-local-whisper", provider.buffer_duration_ms()); @@ -1753,9 +2037,13 @@ impl TranscriptionSession for TauriNativeTranscriptionSession { log::error!( "[core-adapter] Foundry retranscription reached terminal fallback error: {error:#}" ); + #[cfg(target_os = "windows")] + let message = foundry::FOUNDRY_FALLBACK_TERMINAL_USER_MESSAGE; + #[cfg(not(target_os = "windows"))] + let message = "Foundry is only available on Windows"; let mut backend_error = BackendError::new( BackendErrorCode::Provider, - foundry::FOUNDRY_FALLBACK_TERMINAL_USER_MESSAGE, + message, ); backend_error.details = Some(serde_json::json!({ "terminal": "foundry_fallback" @@ -1767,7 +2055,7 @@ impl TranscriptionSession for TauriNativeTranscriptionSession { recovery = result.primary_recovery; result.raw } - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] TauriNativeTranscriptionSessionKind::Sherpa { provider, .. } => { let timeout = openless_core::provider_rules::native_transcribe_timeout( "sherpa-onnx-local", provider.buffer_duration_ms()); @@ -1786,7 +2074,7 @@ impl TranscriptionSession for TauriNativeTranscriptionSession { if result.is_err() { session.cancel_native(); } - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] session.release_native(result.is_err(), recovery); result }) @@ -1796,14 +2084,14 @@ impl TranscriptionSession for TauriNativeTranscriptionSession { let session = self.clone(); Box::pin(async move { session.cancel_native(); - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] session.release_native(true, None); Ok(()) }) } } -#[cfg(target_os = "windows")] +#[cfg(local_asr_sherpa)] fn foundry_transcription_notices( backend: BackendSlot, session_id: SessionId, @@ -1829,13 +2117,13 @@ fn foundry_transcription_notices( }) } -#[cfg(target_os = "windows")] +#[cfg(local_asr_sherpa)] impl TauriNativeTranscriptionSession { fn cancel_native(&self) { match &self.kind { - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] TauriNativeTranscriptionSessionKind::Foundry { provider, .. } => provider.cancel(), - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] TauriNativeTranscriptionSessionKind::Sherpa { provider, .. } => provider.cancel(), } } @@ -1843,7 +2131,7 @@ impl TauriNativeTranscriptionSession { fn release_native( &self, discard: bool, - #[cfg(target_os = "windows")] recovery: Option< + #[cfg(local_asr_sherpa)] recovery: Option< crate::asr::local::foundry_runtime::FoundryPrimaryRecoveryToken, >, ) { @@ -1851,7 +2139,7 @@ impl TauriNativeTranscriptionSession { return; } match &self.kind { - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] TauriNativeTranscriptionSessionKind::Foundry { runtime, .. } => { schedule_foundry_release( Arc::clone(runtime), @@ -1861,7 +2149,7 @@ impl TauriNativeTranscriptionSession { Arc::clone(&self.current_generation), ); } - #[cfg(target_os = "windows")] + #[cfg(local_asr_sherpa)] TauriNativeTranscriptionSessionKind::Sherpa { runtime, .. } => { schedule_sherpa_release( Arc::clone(runtime), @@ -1875,13 +2163,13 @@ impl TauriNativeTranscriptionSession { } } -#[cfg(any(target_os = "windows", target_os = "macos", test))] +#[cfg(any(target_os = "windows", target_os = "macos", local_asr_sherpa, test))] fn local_asr_release_delay(keep_loaded_secs: u32) -> Option { (keep_loaded_secs != openless_core::LOCAL_ASR_KEEP_LOADED_FOREVER_SECS) .then(|| std::time::Duration::from_secs(keep_loaded_secs as u64)) } -#[cfg(target_os = "windows")] +#[cfg(local_asr_sherpa)] fn schedule_foundry_release( runtime: Arc, recovery: Option, @@ -1889,6 +2177,10 @@ fn schedule_foundry_release( generation: u64, current_generation: Arc, ) { + // Foundry only runs on Windows; elsewhere there is never a loaded Foundry model to release. + #[cfg(not(target_os = "windows"))] + let _ = (runtime, recovery, keep_loaded_secs, generation, current_generation); + #[cfg(target_os = "windows")] tauri::async_runtime::spawn(async move { if current_generation.load(Ordering::Acquire) != generation { return; @@ -1929,7 +2221,7 @@ fn schedule_foundry_release( }); } -#[cfg(target_os = "windows")] +#[cfg(local_asr_sherpa)] fn schedule_sherpa_release( runtime: Arc, keep_loaded_secs: u32, @@ -1954,7 +2246,7 @@ fn schedule_sherpa_release( }); } -#[cfg(any(target_os = "windows", target_os = "macos"))] +#[cfg(any(target_os = "windows", target_os = "macos", local_asr_sherpa))] fn map_native_asr_error(error: impl std::fmt::Display) -> BackendError { BackendError::new( BackendErrorCode::Provider, diff --git a/openless-all/app/src-tauri/src/lib.rs b/openless-all/app/src-tauri/src/lib.rs index 37239aff6..9013f964d 100644 --- a/openless-all/app/src-tauri/src/lib.rs +++ b/openless-all/app/src-tauri/src/lib.rs @@ -523,7 +523,8 @@ macro_rules! app_invoke_handler_desktop { }; } -/// Android/iOS: only commands usable without desktop hotkeys, tray, updater, or local ASR. +/// Android/iOS: only commands usable without desktop hotkeys, tray or updater; +/// local ASR commands only where sherpa-onnx is linked (arm64 Android). #[macro_export] macro_rules! app_invoke_handler_mobile { () => { @@ -681,6 +682,77 @@ macro_rules! app_invoke_handler_mobile { $crate::restart_app, $crate::reset_accessibility_permission_and_restart_app, $crate::log_client_error, + // Local ASR: arm64 Android links sherpa-onnx (build.rs `local_asr_sherpa`). + #[cfg(local_asr_sherpa)] + $crate::commands::local_asr_get_settings, + #[cfg(local_asr_sherpa)] + $crate::commands::local_asr_storage_settings, + #[cfg(local_asr_sherpa)] + $crate::commands::local_asr_set_models_base_dir, + #[cfg(local_asr_sherpa)] + $crate::commands::local_asr_activate, + #[cfg(local_asr_sherpa)] + $crate::commands::local_asr_set_active_model, + #[cfg(local_asr_sherpa)] + $crate::commands::local_asr_set_mirror, + #[cfg(local_asr_sherpa)] + $crate::commands::local_asr_list_models, + #[cfg(local_asr_sherpa)] + $crate::commands::local_asr_fetch_remote_info, + #[cfg(local_asr_sherpa)] + $crate::commands::local_asr_fetch_hf_card, + #[cfg(local_asr_sherpa)] + $crate::commands::local_asr_download_model, + #[cfg(local_asr_sherpa)] + $crate::commands::local_asr_cancel_download, + #[cfg(local_asr_sherpa)] + $crate::commands::local_asr_delete_model, + #[cfg(local_asr_sherpa)] + $crate::commands::local_asr_cleanup_incomplete, + #[cfg(local_asr_sherpa)] + $crate::commands::local_asr_model_dir, + #[cfg(local_asr_sherpa)] + $crate::commands::local_asr_reveal_model_dir, + #[cfg(local_asr_sherpa)] + $crate::commands::local_asr_reveal_models_root, + #[cfg(local_asr_sherpa)] + $crate::commands::local_asr_test_model, + #[cfg(local_asr_sherpa)] + $crate::commands::local_asr_test_channel, + #[cfg(local_asr_sherpa)] + $crate::commands::local_asr_engine_status, + #[cfg(local_asr_sherpa)] + $crate::commands::local_asr_release_engine, + #[cfg(local_asr_sherpa)] + $crate::commands::local_asr_preload, + #[cfg(local_asr_sherpa)] + $crate::commands::local_asr_set_keep_loaded_secs, + #[cfg(local_asr_sherpa)] + $crate::commands::sherpa_onnx_asr_status, + #[cfg(local_asr_sherpa)] + $crate::commands::sherpa_onnx_asr_catalog, + #[cfg(local_asr_sherpa)] + $crate::commands::sherpa_onnx_asr_fetch_remote_info, + #[cfg(local_asr_sherpa)] + $crate::commands::sherpa_onnx_asr_download_model, + #[cfg(local_asr_sherpa)] + $crate::commands::sherpa_onnx_asr_cancel_download, + #[cfg(local_asr_sherpa)] + $crate::commands::sherpa_onnx_asr_set_model, + #[cfg(local_asr_sherpa)] + $crate::commands::sherpa_onnx_asr_set_language_hint, + #[cfg(local_asr_sherpa)] + $crate::commands::sherpa_onnx_asr_prepare, + #[cfg(local_asr_sherpa)] + $crate::commands::sherpa_onnx_asr_cancel_prepare, + #[cfg(local_asr_sherpa)] + $crate::commands::sherpa_onnx_asr_release, + #[cfg(local_asr_sherpa)] + $crate::commands::sherpa_onnx_asr_model_dir, + #[cfg(local_asr_sherpa)] + $crate::commands::sherpa_onnx_asr_delete_model, + #[cfg(local_asr_sherpa)] + $crate::commands::sherpa_onnx_asr_reveal_model_dir, ] }; } diff --git a/openless-all/app/src/i18n/de.ts b/openless-all/app/src/i18n/de.ts index 2d05a04f5..4d22fe3a7 100644 --- a/openless-all/app/src/i18n/de.ts +++ b/openless-all/app/src/i18n/de.ts @@ -2254,6 +2254,9 @@ export const de: typeof zhCN = { 'Verlagert die Transkription von Cloud-ASR auf das Gerät. Für Offline-Nutzung oder sensible Daten.', localAsrWarningShort: 'Lokale Inferenz ist langsamer. Zu schwache Hardware kann Wörter verlieren.', + localAsrFallbackLabel: 'Bei Cloud-Fehler lokales Modell verwenden', + localAsrFallbackDesc: + 'Bei schwacher oder fehlender Verbindung wird die Aufnahme mit dem heruntergeladenen lokalen Modell erkannt, statt auf Cloud-Wiederholungen zu warten. Erfordert ein heruntergeladenes lokales Modell.', qwen3Desc: 'Nach der Aktivierung übernimmt dieses Modell die Spracherkennung.', sherpaDesc: 'Nach der Aktivierung übernimmt dieses Modell die Spracherkennung.', foundryDesc: 'Nach der Aktivierung übernimmt dieses Modell die Spracherkennung.', diff --git a/openless-all/app/src/i18n/en.ts b/openless-all/app/src/i18n/en.ts index 353b3b7ba..99fc48b17 100644 --- a/openless-all/app/src/i18n/en.ts +++ b/openless-all/app/src/i18n/en.ts @@ -2193,6 +2193,9 @@ export const en: typeof zhCN = { localAsrDesc: 'Move transcription from cloud ASR to on-device inference. Offline / privacy-sensitive use only.', localAsrWarningShort: 'Local inference is slower; under-spec hardware may drop words.', + localAsrFallbackLabel: 'Use the local model when cloud ASR fails', + localAsrFallbackDesc: + 'On a weak or missing connection, transcribe the recording with the downloaded local model instead of waiting for cloud retries. Requires a downloaded local model.', qwen3Desc: 'Once enabled, the ASR provider will be taken over.', sherpaDesc: 'Once enabled, the ASR provider will be taken over.', foundryDesc: 'Once enabled, the ASR provider will be taken over.', diff --git a/openless-all/app/src/i18n/es.ts b/openless-all/app/src/i18n/es.ts index 2811b88d3..926ee0ec9 100644 --- a/openless-all/app/src/i18n/es.ts +++ b/openless-all/app/src/i18n/es.ts @@ -2235,6 +2235,9 @@ export const es: typeof zhCN = { 'Sustituye el ASR en la nube por inferencia en el dispositivo. Para uso sin conexión o con datos sensibles.', localAsrWarningShort: 'La inferencia local es más lenta; un equipo poco potente puede omitir palabras.', + localAsrFallbackLabel: 'Usar el modelo local si falla la nube', + localAsrFallbackDesc: + 'Con conexión débil o sin conexión, transcribe la grabación con el modelo local descargado en lugar de esperar los reintentos en la nube. Requiere un modelo local descargado.', qwen3Desc: 'Al activarlo, sustituirá al proveedor ASR.', sherpaDesc: 'Al activarlo, sustituirá al proveedor ASR.', foundryDesc: 'Al activarlo, sustituirá al proveedor ASR.', diff --git a/openless-all/app/src/i18n/fr.ts b/openless-all/app/src/i18n/fr.ts index 84cbb76a6..a5330392a 100644 --- a/openless-all/app/src/i18n/fr.ts +++ b/openless-all/app/src/i18n/fr.ts @@ -2264,6 +2264,9 @@ export const fr: typeof zhCN = { 'Remplace l’ASR dans le cloud par l’inférence sur l’appareil. Pour l’usage hors ligne ou les données sensibles.', localAsrWarningShort: 'L’inférence locale est plus lente ; un appareil insuffisamment puissant peut omettre des mots.', + localAsrFallbackLabel: 'Utiliser le modèle local si le cloud échoue', + localAsrFallbackDesc: + 'En cas de connexion faible ou absente, transcrit l\'enregistrement avec le modèle local téléchargé au lieu d\'attendre les nouvelles tentatives du cloud. Nécessite un modèle local téléchargé.', qwen3Desc: 'Une fois activé, il remplace le fournisseur ASR.', sherpaDesc: 'Une fois activé, il remplace le fournisseur ASR.', foundryDesc: 'Une fois activé, il remplace le fournisseur ASR.', diff --git a/openless-all/app/src/i18n/ja.ts b/openless-all/app/src/i18n/ja.ts index 4743df55f..712e5b009 100644 --- a/openless-all/app/src/i18n/ja.ts +++ b/openless-all/app/src/i18n/ja.ts @@ -2163,6 +2163,9 @@ export const ja: typeof zhCN = { localAsrTitle: 'ローカル ASR モデル', localAsrDesc: '転写をクラウドから本機推論に切り替えます。オフライン／プライバシー重視向け。', localAsrWarningShort: 'ローカル推論は遅く、スペック不足では欠字の可能性があります。', + localAsrFallbackLabel: 'クラウド認識に失敗したらローカルモデルを使う', + localAsrFallbackDesc: + '通信が不安定、またはオフラインのとき、クラウドの再試行を待たずにダウンロード済みのローカルモデルで録音を認識します。ローカルモデルのダウンロードが必要です。', qwen3Desc: '有効化すると ASR プロバイダーが引き継がれます。', sherpaDesc: '有効化すると ASR プロバイダーが引き継がれます。', foundryDesc: '有効化すると ASR プロバイダーが引き継がれます。', diff --git a/openless-all/app/src/i18n/ko.ts b/openless-all/app/src/i18n/ko.ts index 1ae922941..c5ccc0c75 100644 --- a/openless-all/app/src/i18n/ko.ts +++ b/openless-all/app/src/i18n/ko.ts @@ -2147,6 +2147,9 @@ export const ko: typeof zhCN = { localAsrDesc: '전사를 클라우드에서 로컬 추론으로 전환합니다. 오프라인 / 프라이버시용에만 권장됩니다.', localAsrWarningShort: '로컬 추론은 느리며, 사양 부족 시 글자 누락이 발생할 수 있습니다.', + localAsrFallbackLabel: '클라우드 인식이 실패하면 로컬 모델 사용', + localAsrFallbackDesc: + '네트워크가 불안정하거나 오프라인일 때 클라우드 재시도를 기다리지 않고 다운로드한 로컬 모델로 녹음을 인식합니다. 로컬 모델을 먼저 다운로드해야 합니다.', qwen3Desc: '활성화하면 ASR 프로바이더가 인수됩니다.', sherpaDesc: '활성화하면 ASR 프로바이더가 인수됩니다.', foundryDesc: '활성화하면 ASR 프로바이더가 인수됩니다.', diff --git a/openless-all/app/src/i18n/zh-CN.ts b/openless-all/app/src/i18n/zh-CN.ts index 6f6146d86..e61a1f8c0 100644 --- a/openless-all/app/src/i18n/zh-CN.ts +++ b/openless-all/app/src/i18n/zh-CN.ts @@ -2065,6 +2065,9 @@ export const zhCN = { localAsrTitle: '本地 ASR 模型', localAsrDesc: '把转写从云端切到本机推理。仅推荐离线 / 隐私敏感场景。', localAsrWarningShort: '本地推理较慢，配置不足时可能吞字。', + localAsrFallbackLabel: '云端识别失败时用本地模型', + localAsrFallbackDesc: + '网络不好或没网时，用已下载的本地模型识别录音，不再等云端重试。需要先下载本地模型。', qwen3Desc: '启动之后，ASR 提供商将被接管。', sherpaDesc: '启动之后，ASR 提供商将被接管。', foundryDesc: '启动之后，ASR 提供商将被接管。', diff --git a/openless-all/app/src/i18n/zh-TW.ts b/openless-all/app/src/i18n/zh-TW.ts index caf51beaf..d2327e65b 100644 --- a/openless-all/app/src/i18n/zh-TW.ts +++ b/openless-all/app/src/i18n/zh-TW.ts @@ -2066,6 +2066,9 @@ export const zhTW: typeof zhCN = { localAsrTitle: '本地 ASR 模型', localAsrDesc: '把轉寫從雲端切到本機推理。僅推薦離線 / 隱私敏感場景。', localAsrWarningShort: '本地推理較慢，配置不足時可能吞字。', + localAsrFallbackLabel: '雲端辨識失敗時改用本機模型', + localAsrFallbackDesc: + '網路不佳或沒有網路時，用已下載的本機模型辨識錄音，不再等待雲端重試。需要先下載本機模型。', qwen3Desc: '啟動之後，ASR 提供商將被接管。', sherpaDesc: '啟用後，ASR 提供商將被接管。', foundryDesc: '啟動之後，ASR 提供商將被接管。', diff --git a/openless-all/app/src/lib/ipc/mock-data.ts b/openless-all/app/src/lib/ipc/mock-data.ts index b27080e99..3ca19f0b4 100644 --- a/openless-all/app/src/lib/ipc/mock-data.ts +++ b/openless-all/app/src/lib/ipc/mock-data.ts @@ -119,6 +119,7 @@ export let mockSettings: UserPreferences = { sherpaOnnxModel: 'sense-voice-small-zh', sherpaOnnxLanguageHint: '', sherpaOnnxKeepLoadedSecs: 300, + localAsrFallbackEnabled: false, historyRetentionDays: 7, polishContextWindowMinutes: 5, polishSkipMaxChars: 0, diff --git a/openless-all/app/src/lib/types.ts b/openless-all/app/src/lib/types.ts index cfccc2304..42f6c98a1 100644 --- a/openless-all/app/src/lib/types.ts +++ b/openless-all/app/src/lib/types.ts @@ -493,6 +493,8 @@ export interface UserPreferences { sherpaOnnxLanguageHint: string; /** Seconds the Windows sherpa-onnx model stays loaded in the runtime. */ sherpaOnnxKeepLoadedSecs: number; + /** When a cloud ASR provider fails, transcribe the recording with the selected sherpa-onnx model. */ + localAsrFallbackEnabled: boolean; /** History retention days. 0 = no time-based cleanup (the 200-entry cap still applies). Default 7. */ historyRetentionDays: number; /** Conversation-aware polish context window (minutes). 0 = off. Default 5. See PR-A. */ diff --git a/openless-all/app/src/pages/LocalAsr/components.tsx b/openless-all/app/src/pages/LocalAsr/components.tsx index bd5d0d0ec..070cabd1b 100644 --- a/openless-all/app/src/pages/LocalAsr/components.tsx +++ b/openless-all/app/src/pages/LocalAsr/components.tsx @@ -668,6 +668,7 @@ export function ModelDetailPanel({ onCancel, onDelete, onReveal, + showReveal = true, onTest, onCleanup, showTest, @@ -683,6 +684,7 @@ export function ModelDetailPanel({ onCancel: () => void; onDelete: () => void; onReveal: () => void; + showReveal?: boolean; onTest: () => void; /** Clean up the staging directory of an interrupted download (shown only when leftovers exist and no download is running). */ onCleanup?: () => void; @@ -733,13 +735,15 @@ export function ModelDetailPanel({ )} {entry.isDownloaded && ( <> - - {t('localAsr.revealDir')} - + {showReveal && ( + + {t('localAsr.revealDir')} + + )} void; onStart: () => void; onClose: () => void; + /** Download source picker shown in the header, so the source is chosen where the download starts. */ + mirror?: { + label: string; + value: string; + options: { value: string; label: string }[]; + onChange: (value: string) => void; + }; }) { const { t } = useTranslation(); const stackLayout = useLayoutStack(760); @@ -882,6 +894,24 @@ export function DownloadDialog({

{t('localAsr.downloadDialogTitle')}

{t('localAsr.downloadDialogDesc')}

+ {mirror && ( + // A native select: on Android it opens the system picker, which no + // dialog layer or clipping can hide. + + )}