diff --git a/openless-all/app/crates/openless-core/src/api.rs b/openless-all/app/crates/openless-core/src/api.rs index 0d13554a4..3450433dc 100644 --- a/openless-all/app/crates/openless-core/src/api.rs +++ b/openless-all/app/crates/openless-core/src/api.rs @@ -2595,26 +2595,34 @@ impl EditObservationSink for CoreEditObservationSink { if self.generation.load(Ordering::Acquire) != self.expected_generation { return false; } - if !crate::host_document::edit_is_within_typed_text(&edit, &self.typed_text) { - return false; - } - let Some(rule) = crate::host_document::learned_rule_with_max_chars( + let rules = crate::host_document::learned_rules_with_max_chars( &edit, self.settings.max_phrase_chars as usize, - ) else { - return false; - }; - if let Err(error) = queue_pending_correction_state( - &self.pending, - &self.events, - rule.pattern, - rule.replacement, - Some((&self.generation, self.expected_generation)), - self.settings.suggestion_seconds, - ) { - log::warn!("failed to queue observed correction: {error}"); + ); + let mut published = false; + for rule in rules { + // Validate the recovered word, not a lone changed character (or + // an empty spelling insertion). Reject words absent from this insertion. + if !self.typed_text.contains(&rule.pattern) { + log::debug!("observed vocabulary candidate outside inserted text"); + continue; + } + match queue_pending_correction_state( + &self.pending, + &self.events, + rule.pattern, + rule.replacement, + Some((&self.generation, self.expected_generation)), + self.settings.suggestion_seconds, + ) { + Ok(_) => published = true, + Err(error) => log::warn!("failed to queue observed correction: {error}"), + } } - true + if !published { + log::debug!("observed edit produced no eligible vocabulary candidate"); + } + published } } @@ -8272,6 +8280,41 @@ mod tests { assert!(backend.get_preferences().vocabulary_learning_enabled); } + #[test] + fn observed_spelling_correction_saves_only_the_complete_confirmed_word() { + let (backend, _) = backend(); + let sink = CoreEditObservationSink { + settings: backend.get_preferences().vocabulary_learning_settings, + expected_generation: backend.edit_observation_generation.load(Ordering::Acquire), + generation: Arc::clone(&backend.edit_observation_generation), + typed_text: "你好,ZIP和codx".into(), + pending: Arc::clone(&backend.pending_corrections), + events: Arc::clone(&backend.events), + }; + assert!(sink.publish( + crate::host_document::minimal_edit("你好,ZIP和codx", "你好,VIP和codex。").unwrap() + )); + let suggestions = backend.pending_corrections(); + assert_eq!(suggestions.len(), 2); + assert_eq!(suggestions[0].pattern, "ZIP"); + assert_eq!(suggestions[0].replacement, "VIP"); + assert_eq!(suggestions[1].pattern, "codx"); + assert_eq!(suggestions[1].replacement, "codex"); + assert!(backend.list_vocabulary().unwrap().is_empty()); + backend + .accept_pending_correction(&suggestions[0].id) + .unwrap(); + let vocabulary = backend.list_vocabulary().unwrap(); + assert_eq!(vocabulary.len(), 1); + assert_eq!(vocabulary[0].phrase, "VIP"); + assert!(backend.list_correction_rules().unwrap().is_empty()); + + // A matching changed letter is insufficient when the recovered word + // came from outside this insertion. + assert!(!sink.publish(crate::host_document::minimal_edit("Zebra", "Vebra").unwrap())); + assert_eq!(backend.pending_corrections().len(), 1); + } + #[test] fn vocabulary_learning_expired_suggestions_cannot_be_accepted() { let (backend, _) = backend(); diff --git a/openless-all/app/crates/openless-core/src/host_document/diff.rs b/openless-all/app/crates/openless-core/src/host_document/diff.rs index 63c0003ed..20d0f5fcf 100644 --- a/openless-all/app/crates/openless-core/src/host_document/diff.rs +++ b/openless-all/app/crates/openless-core/src/host_document/diff.rs @@ -1,3 +1,5 @@ +use unicode_normalization::char::is_combining_mark; + const MAX_EDIT_CHARS: usize = 64; const CONTEXT_CHARS: usize = 256; const MIN_PATTERN_CHARS: usize = 2; @@ -26,9 +28,7 @@ pub fn minimal_edit(before_text: &str, after_text: &str) -> Option { .count(); let source: String = old[prefix..old.len() - suffix].iter().collect(); let target: String = new[prefix..new.len() - suffix].iter().collect(); - if source.is_empty() - || source.chars().count().max(target.chars().count()) > MAX_EDIT_CHARS - || source.trim().is_empty() + if source.chars().count().max(target.chars().count()) > MAX_EDIT_CHARS || strip_whitespace(&source) == strip_whitespace(&target) { return None; @@ -75,47 +75,274 @@ pub fn learned_rule(edit: &EditPair) -> Option { } pub fn learned_rule_with_max_chars(edit: &EditPair, max_chars: usize) -> Option { - if !is_vocab_worthy_with_max_chars(edit, max_chars) { - return None; + let mut rules = learned_rules_with_max_chars(edit, max_chars); + // The singular API must not silently discard a second correction. + (rules.len() == 1).then(|| rules.remove(0)) +} + +/// Split bounded character changes before recovering vocabulary boundaries. +/// Formatting changes and continued typing must not swallow a word correction. +/// This is deliberately local: CJK context remains a conservative two-character +/// heuristic, not a claim to semantic word segmentation. +pub fn learned_rules_with_max_chars(edit: &EditPair, max_chars: usize) -> Vec { + let source: Vec = edit.source.chars().collect(); + let target: Vec = edit.target.chars().collect(); + if source.len().max(target.len()) > MAX_EDIT_CHARS { + return Vec::new(); } let before: Vec = edit.before.chars().collect(); let after: Vec = edit.after.chars().collect(); - let mut left = 0; - let mut right = 0; - while edit.source.trim().chars().count() + left + right < MIN_PATTERN_CHARS { - if before.len() > left && !before[before.len() - left - 1].is_whitespace() { - left += 1; - } else if after.len() > right && !after[right].is_whitespace() { - right += 1; - } else { + let old: Vec = before + .iter() + .chain(&source) + .chain(&after) + .copied() + .collect(); + let new: Vec = before + .iter() + .chain(&target) + .chain(&after) + .copied() + .collect(); + let mut rules = Vec::new(); + for (old_start, old_end, new_start, new_end) in changed_ranges(&source, &target) { + let (old_start, old_end) = + trim_formatting(&old, old_start + before.len(), old_end + before.len()); + let (new_start, new_end) = + trim_formatting(&new, new_start + before.len(), new_end + before.len()); + let old_word = identifier_span(&old, old_start, old_end); + let new_word = identifier_span(&new, new_start, new_end); + let (pattern, replacement) = match (old_word, new_word) { + (Some((a, b)), Some((c, d))) => ( + old[a..b].iter().collect::(), + new[c..d].iter().collect::(), + ), + (Some((a, b)), None) => { + let Some((c, d)) = leading_word_span(&new, new_start, new_end) else { + // Do not join a partial identifier to neighboring punctuation. + continue; + }; + (old[a..b].iter().collect(), new[c..d].iter().collect()) + } + (None, Some((c, d))) => { + let Some((a, b)) = leading_word_span(&old, old_start, old_end) else { + // Do not join a partial identifier to neighboring punctuation. + continue; + }; + (old[a..b].iter().collect(), new[c..d].iter().collect()) + } + (None, None) => { + if old_start == old_end + || new_start == new_end + || !old[old_start..old_end].iter().all(|&c| is_cjk(c)) + || !new[new_start..new_end].iter().all(|&c| is_cjk(c)) + { + continue; + } + let mut a = old_start; + let mut b = old_end; + let mut c = new_start; + let mut d = new_end; + while b - a < MIN_PATTERN_CHARS { + if a > 0 && c > 0 && old[a - 1] == new[c - 1] && is_cjk(old[a - 1]) { + a -= 1; + c -= 1; + } else if b < old.len() && d < new.len() && old[b] == new[d] && is_cjk(old[b]) { + b += 1; + d += 1; + } else { + break; + } + } + (old[a..b].iter().collect(), new[c..d].iter().collect()) + } + }; + let candidate = EditPair { + source: pattern.clone(), + target: replacement.clone(), + before: String::new(), + after: String::new(), + }; + if pattern != replacement + && pattern.chars().count() >= MIN_PATTERN_CHARS + && is_vocab_worthy_with_max_chars(&candidate, max_chars) + && pattern.chars().any(char::is_alphabetic) + && replacement.chars().any(char::is_alphabetic) + { + let rule = LearnedRule { + pattern, + replacement, + }; + if !rules.contains(&rule) { + rules.push(rule); + } + } + } + rules +} + +/// LCS is quadratic only in the changed region, capped at 64 scalars per side. +/// Equal runs separate independent edits; adjacent delete/insert steps form +/// one replacement so a spelling correction is never learned as two fragments. +fn changed_ranges(old: &[char], new: &[char]) -> Vec<(usize, usize, usize, usize)> { + let mut lcs = vec![vec![0; new.len() + 1]; old.len() + 1]; + for i in (0..old.len()).rev() { + for j in (0..new.len()).rev() { + lcs[i][j] = if old[i] == new[j] { + 1 + lcs[i + 1][j + 1] + } else { + lcs[i + 1][j].max(lcs[i][j + 1]) + }; + } + } + let (mut i, mut j) = (0, 0); + let mut ranges = Vec::new(); + while i < old.len() || j < new.len() { + if i < old.len() && j < new.len() && old[i] == new[j] { + i += 1; + j += 1; + continue; + } + let (a, b) = (i, j); + while i < old.len() || j < new.len() { + if i < old.len() && j < new.len() && old[i] == new[j] { + break; + } + if j == new.len() || (i < old.len() && lcs[i + 1][j] >= lcs[i][j + 1]) { + i += 1; + } else { + j += 1; + } + } + ranges.push((a, i, b, j)); + } + ranges +} + +fn identifier_char(text: &[char], index: usize) -> bool { + let c = text[index]; + c.is_ascii_alphanumeric() + || (matches!(c, '_' | '-' | '.' | '\'') + && index > 0 + && index + 1 < text.len() + && text[index - 1].is_ascii_alphanumeric() + && text[index + 1].is_ascii_alphanumeric()) +} + +fn identifier_span(text: &[char], start: usize, end: usize) -> Option<(usize, usize)> { + // An empty diff inside a word is a spelling insertion/deletion. At a word + // edge it could just be new typing, so require characters on both sides. + if start == end { + if start == 0 + || start == text.len() + || !identifier_char(text, start - 1) + || !identifier_char(text, start) + { return None; } + } else if !(start..end).all(|i| identifier_char(text, i)) { + return None; + } + let (mut a, mut b) = (start, end); + while a > 0 && identifier_char(text, a - 1) { + a -= 1; + } + while b < text.len() && identifier_char(text, b) { + b += 1; + } + // ASCII recovery must not truncate an accented/combining-script word. + let non_ascii_word = + |c: char| (!c.is_ascii() && c.is_alphanumeric() && !is_cjk(c)) || is_combining_mark(c); + if (a > 0 && non_ascii_word(text[a - 1])) || (b < text.len() && non_ascii_word(text[b])) { + return None; + } + // A component of a URL, filesystem path or email is not a vocabulary word. + if (a > 0 && matches!(text[a - 1], '/' | '\\' | '@' | ':')) + || (b < text.len() && matches!(text[b], '/' | '\\' | '@' | ':')) + || looks_like_host(&text[a..b]) + { + return None; + } + Some((a, b)) +} + +fn leading_word_span(text: &[char], start: usize, end: usize) -> Option<(usize, usize)> { + let mut word_end = start; + while word_end < end && identifier_char(text, word_end) { + word_end += 1; } - let prefix: String = before[before.len() - left..].iter().collect(); - let suffix: String = after[..right].iter().collect(); - let pattern = format!("{prefix}{}{suffix}", edit.source) - .trim() - .to_string(); - let replacement = format!("{prefix}{}{suffix}", edit.target) - .trim() - .to_string(); - (!pattern.is_empty() - && !replacement.is_empty() - && pattern.chars().count() <= max_chars - && replacement.chars().count() <= max_chars) - .then_some(LearnedRule { - pattern, - replacement, + if word_end > start { + return identifier_span(text, start, word_end); + } + let mut cjk_end = start; + while cjk_end < end && is_cjk(text[cjk_end]) { + cjk_end += 1; + } + (cjk_end > start).then_some((start, cjk_end)) +} + +fn looks_like_host(text: &[char]) -> bool { + let value: String = text.iter().collect(); + let labels: Vec<&str> = value.split('.').collect(); + labels.len() >= 2 + && labels.iter().all(|label| { + !label.is_empty() && label.chars().all(|c| c.is_ascii_alphanumeric() || c == '-') }) + && labels[0].chars().any(|c| c.is_ascii_alphabetic()) + && labels + .last() + .is_some_and(|label| label.chars().any(|c| c.is_ascii_alphabetic())) +} + +fn trim_formatting(text: &[char], mut start: usize, mut end: usize) -> (usize, usize) { + // Strip only outer formatting. Internal dots/hyphens remain part of an + // identifier; internal sentence boundaries still reject a phrase rewrite. + let formatting = |c: char| { + c.is_whitespace() + || matches!( + c, + '。' | ',' + | '、' + | '?' + | '!' + | ';' + | ':' + | '.' + | ',' + | '?' + | '!' + | ';' + | ':' + | '(' + | ')' + | '(' + | ')' + | '"' + | '“' + | '”' + | '‘' + | '’' + ) + }; + while start < end && formatting(text[start]) && !identifier_char(text, start) { + start += 1; + } + while start < end && formatting(text[end - 1]) && !identifier_char(text, end - 1) { + end -= 1; + } + (start, end) +} + +fn is_cjk(c: char) -> bool { + matches!(c, '\u{3400}'..='\u{4DBF}' | '\u{4E00}'..='\u{9FFF}' | '\u{20000}'..='\u{323AF}') } fn crosses_boundary(value: &str) -> bool { - value.chars().any(|c| { - matches!( - c, - '\n' | '\r' | '。' | '?' | '!' | ';' | ',' | '、' | ':' | '?' | '!' | ';' - ) - }) + let chars: Vec = value.chars().collect(); + chars + .iter() + .enumerate() + .any(|(i, &c)| !c.is_alphabetic() && !identifier_char(&chars, i)) } pub fn edit_is_within_typed_text(edit: &EditPair, typed_text: &str) -> bool { @@ -158,4 +385,116 @@ mod tests { ("大禹", "大鱼") ); } + fn assert_rule(before: &str, after: &str, pattern: &str, replacement: &str) { + let edit = minimal_edit(before, after).unwrap(); + assert_eq!( + learned_rule(&edit), + Some(LearnedRule { + pattern: pattern.into(), + replacement: replacement.into(), + }), + "{before:?} -> {after:?}" + ); + } + + #[test] + fn restores_complete_identifiers_at_every_spelling_position() { + for (before, after, pattern, replacement) in [ + ("你好,ZIP", "你好,VIP", "ZIP", "VIP"), + ("你好,ZIP。", "你好,VIP。", "ZIP", "VIP"), + ("你好, ZIP", "你好, VIP", "ZIP", "VIP"), + ("我用codez", "我用codex", "codez", "codex"), + ("我用codx", "我用codex", "codx", "codex"), + ("我用codeex", "我用codex", "codeex", "codex"), + ("使用Qwen2", "使用Qwen3", "Qwen2", "Qwen3"), + ("使用GPT-6.0。", "使用GPT-6.1。", "GPT-6.0", "GPT-6.1"), + ("用gpt_oss", "用gpt_osx", "gpt_oss", "gpt_osx"), + ("(ZIP)", "(VIP)", "ZIP", "VIP"), + ("今天讲大禹", "今天讲Codex", "大禹", "Codex"), + ] { + assert_rule(before, after, pattern, replacement); + } + } + + #[test] + fn separates_word_corrections_from_formatting_and_continued_typing() { + for (before, after, pattern, replacement) in [ + ("你好,ZIP", "你好,VIP。", "ZIP", "VIP"), + ("你好,ZIP.", "你好,VIP!", "ZIP", "VIP"), + ("你好,ZIP", "你好,Codex。", "ZIP", "Codex"), + ("你好,ZIP", "你好,VIP。继续写下一句", "ZIP", "VIP"), + ("你好,foo", "你好,bar。继续写", "foo", "bar"), + ("你好,foo", "你好,bar next", "foo", "bar"), + ] { + assert_rule(before, after, pattern, replacement); + } + } + + #[test] + fn handles_multiple_word_corrections_without_fragment_duplicates() { + let edit = minimal_edit("ZIP和codx", "VIP和codex。").unwrap(); + assert_eq!( + learned_rules_with_max_chars(&edit, 12), + vec![ + LearnedRule { + pattern: "ZIP".into(), + replacement: "VIP".into() + }, + LearnedRule { + pattern: "codx".into(), + replacement: "codex".into() + }, + ] + ); + assert!(learned_rule(&edit).is_none()); + assert_rule("gpt_oss", "GPT_OSS", "gpt_oss", "GPT_OSS"); + } + + #[test] + fn rejects_formatting_deletion_new_words_and_non_vocabulary_values() { + for (before, after) in [ + ("你好,ZIP", "你好,ZIP。"), + ("你好,ZIP。", "你好,ZIP!"), + ("你好 ZIP", "你好 ZIP"), + ("ZIP", ""), + ("你好", "你好 VIP"), + ("123", "124"), + ("https://codx.com", "https://codex.com"), + ("/tmp/codx", "/tmp/codex"), + ("codx@example.com", "codex@example.com"), + ("https://foo.com", "https://bar.com"), + ("example.com", "example.net"), + ("www.example.com", "www.example.org"), + ("/tmp/foo", "/tmp/bar"), + ("foo@example.com", "bar@example.com"), + (",甲", ",乙"), + ("😀甲", "😀乙"), + ("café", "cazé"), + ("cafe\u{301}", "cake\u{301}"), + ] { + if let Some(edit) = minimal_edit(before, after) { + assert!( + learned_rules_with_max_chars(&edit, 32).is_empty(), + "{before:?} -> {after:?}" + ); + } + } + } + + #[test] + fn validates_expanded_word_length_without_truncating_it() { + let edit = minimal_edit("abcdefZ", "abcdefV").unwrap(); + assert!(learned_rules_with_max_chars(&edit, 6).is_empty()); + assert_eq!( + learned_rule_with_max_chars(&edit, 7).unwrap().replacement, + "abcdefV" + ); + let edit = EditPair { + source: "a".repeat(65), + target: "b".into(), + before: String::new(), + after: String::new(), + }; + assert!(learned_rules_with_max_chars(&edit, 100).is_empty()); + } } diff --git a/openless-all/app/crates/openless-core/src/host_document/mod.rs b/openless-all/app/crates/openless-core/src/host_document/mod.rs index 1f0825b3e..e5f365351 100644 --- a/openless-all/app/crates/openless-core/src/host_document/mod.rs +++ b/openless-all/app/crates/openless-core/src/host_document/mod.rs @@ -9,7 +9,7 @@ mod window; pub use diff::{ edit_is_within_typed_text, is_vocab_worthy, learned_rule, learned_rule_with_max_chars, - minimal_edit, EditPair, LearnedRule, + learned_rules_with_max_chars, minimal_edit, EditPair, LearnedRule, }; pub use observation::ObservedInsertion; pub use window::{