diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2821c2b6c..139c1168d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -417,6 +417,8 @@ jobs: # Rime is staged for this emulator ABI; do not launch the Tauri Rust build. src-tauri/gen/android/gradlew --project-dir src-tauri/gen/android \ :app:connectedX86_64DebugAndroidTest -x :app:rustBuildX86_64Debug --no-daemon + # Regression evidence written by ImeAutoTextIntegrationTest (#1230). + adb -s emulator-5554 logcat -d -s Ime1230Evidence:I '*:S' - name: Upload Android emulator failure log if: failure() diff --git a/openless-all/app/android/kotlin/EnglishAutoText.kt b/openless-all/app/android/kotlin/EnglishAutoText.kt new file mode 100644 index 000000000..0c8af6f8e --- /dev/null +++ b/openless-all/app/android/kotlin/EnglishAutoText.kt @@ -0,0 +1,58 @@ +package com.openless.app + +import android.text.InputType + +/** + * The English keyboard's small typing conveniences — sentence-start + * capitals, double-space full stop, punctuation pulling in the space a + * candidate pick added — as pure decisions on the text around the caret. + * [OpenLessImeService] reads the field and carries them out. + */ +internal object EnglishAutoText { + /** Marks that take the place of a candidate's trailing space: "word " + "," becomes "word, ". */ + private val SPACE_SWALLOWING_PUNCTUATION = setOf(".", ",", "!", "?", ";", ":") + private val SENTENCE_ENDS = setOf('.', '!', '?') + + /** How much text before the caret [isSentenceStart] needs to see. */ + const val SENTENCE_LOOKBEHIND = 64 + + /** + * True at the start of the field, of a line, or after a sentence-ending + * mark followed by whitespace. [before] is up to [SENTENCE_LOOKBEHIND] + * characters before the caret. + */ + fun isSentenceStart(before: CharSequence): Boolean { + if (before.isBlank()) return true + val last = before.last() + if (last == '\n') return true + if (!last.isWhitespace()) return false + val mark = before.trimEnd().lastOrNull() ?: return true + return mark in SENTENCE_ENDS + } + + /** Fields where a capital nobody asked for would be wrong: addresses, URLs, passwords, numbers. */ + fun allowsAutoCapitals(inputType: Int): Boolean { + if (inputType and InputType.TYPE_MASK_CLASS != InputType.TYPE_CLASS_TEXT) return false + return when (inputType and InputType.TYPE_MASK_VARIATION) { + InputType.TYPE_TEXT_VARIATION_EMAIL_ADDRESS, + InputType.TYPE_TEXT_VARIATION_WEB_EMAIL_ADDRESS, + InputType.TYPE_TEXT_VARIATION_URI, + InputType.TYPE_TEXT_VARIATION_PASSWORD, + InputType.TYPE_TEXT_VARIATION_VISIBLE_PASSWORD, + InputType.TYPE_TEXT_VARIATION_WEB_PASSWORD -> false + else -> true + } + } + + /** + * A second quick space turns "word " into "word. ". Only right after a + * letter or digit and its single space — never after punctuation, a + * second space or a line break. + */ + fun doubleSpaceMakesFullStop(before: CharSequence): Boolean { + if (before.length < 2 || before.last() != ' ') return false + return before[before.length - 2].isLetterOrDigit() + } + + fun swallowsAutoSpace(char: String): Boolean = char in SPACE_SWALLOWING_PUNCTUATION +} diff --git a/openless-all/app/android/kotlin/EnglishUserFrequency.kt b/openless-all/app/android/kotlin/EnglishUserFrequency.kt index ce9b85b5f..4f8e967e1 100644 --- a/openless-all/app/android/kotlin/EnglishUserFrequency.kt +++ b/openless-all/app/android/kotlin/EnglishUserFrequency.kt @@ -22,6 +22,7 @@ import kotlin.math.ln internal class EnglishUserFrequency(context: Context) { private val preferences = context.applicationContext.getSharedPreferences(STORE, Context.MODE_PRIVATE) private val customWordsPrefs = context.applicationContext.getSharedPreferences(CUSTOM_STORE, Context.MODE_PRIVATE) + private val cache = FrequencyStampCache.of(context, STORE) // Tracks entry count without paying preferences.all's full-map-copy cost // on every record() — see trimIfNeeded(). -1 means "not yet known", @@ -30,22 +31,19 @@ internal class EnglishUserFrequency(context: Context) { private var approxSize = -1 fun score(word: String, now: Long = System.currentTimeMillis()): Double { - val value = preferences.getString(key(word), null) ?: return 0.0 - val parts = value.split(',') - val count = parts.getOrNull(0)?.toDoubleOrNull() ?: return 0.0 - val last = parts.getOrNull(1)?.toLongOrNull() ?: return 0.0 - val ageDays = ((now - last).coerceAtLeast(0L)).toDouble() / MILLIS_PER_DAY - return ln(1.0 + count) * COUNT_WEIGHT + exp(-ageDays / RECENCY_HALF_LIFE_DAYS) * RECENCY_WEIGHT + val stamp = cache.get(key(word)) ?: return 0.0 + val ageDays = ((now - stamp.lastUsedMs).coerceAtLeast(0L)).toDouble() / MILLIS_PER_DAY + return ln(1.0 + stamp.count) * COUNT_WEIGHT + exp(-ageDays / RECENCY_HALF_LIFE_DAYS) * RECENCY_WEIGHT } fun record(word: String) { if (word.isBlank()) return val entryKey = key(word) val isNewEntry = !preferences.contains(entryKey) - val old = preferences.getString(entryKey, null)?.split(',') - val count = (old?.getOrNull(0)?.toIntOrNull() ?: 0) + 1 + val count = (cache.get(entryKey)?.count ?: 0) + 1 val now = System.currentTimeMillis() preferences.edit().putString(entryKey, "$count,$now").apply() + cache.put(entryKey, FrequencyStampCache.Stamp(count, now)) // Re-recording an existing word (by far the common case) never // touches preferences.all at all now — only a genuinely new word // can push the count past MAX_ENTRIES, so that's the only case @@ -59,6 +57,7 @@ internal class EnglishUserFrequency(context: Context) { fun clear() { preferences.edit().clear().apply() + cache.invalidate() approxSize = 0 } diff --git a/openless-all/app/android/kotlin/FrequencyStampCache.kt b/openless-all/app/android/kotlin/FrequencyStampCache.kt new file mode 100644 index 000000000..2f28a1fe1 --- /dev/null +++ b/openless-all/app/android/kotlin/FrequencyStampCache.kt @@ -0,0 +1,110 @@ +package com.openless.app + +import android.content.Context +import android.content.SharedPreferences +import java.util.concurrent.ConcurrentHashMap + +/** + * Parsed, in-memory copy of one "count,lastUsedMs" usage store, shared by + * every reader of that store in the process. + * + * Ranking looks a score up for every candidate it considers — reading and + * splitting the stored string each time made that the slow part of a + * stroke query. Every StrokeUserFrequency/EnglishUserFrequency instance + * for a store gets this one cache, kept + * current by a preference listener, so a write from any instance (the + * keyboard, the settings page clearing it, an import) is seen by all. + */ +internal class FrequencyStampCache private constructor( + private val preferences: SharedPreferences, + private val groupOf: ((String) -> String?)?, +) { + data class Stamp(val count: Int, val lastUsedMs: Long) + + private val stamps = ConcurrentHashMap() + // groupOf(key) -> keys; lets the stroke store list every character + // picked for one code without scanning the whole store. + private val groups = ConcurrentHashMap>() + @Volatile + private var loaded = false + + // Held as a field: SharedPreferences only keeps a weak reference to it. + private val listener = SharedPreferences.OnSharedPreferenceChangeListener { _, key -> + if (key == null) invalidate() else refresh(key) + } + + init { + preferences.registerOnSharedPreferenceChangeListener(listener) + } + + fun get(key: String): Stamp? { + ensureLoaded() + return stamps[key] + } + + fun keysInGroup(group: String): List { + ensureLoaded() + return groups[group]?.toList().orEmpty() + } + + /** Call alongside the preference write, so a query right after sees it before the listener runs. */ + fun put(key: String, stamp: Stamp) { + ensureLoaded() + store(key, stamp) + } + + /** After a wholesale change (clear, import) — clear() alone doesn't notify listeners before API 30. */ + fun invalidate() { + synchronized(this) { + loaded = false + stamps.clear() + groups.clear() + } + } + + private fun ensureLoaded() { + if (loaded) return + synchronized(this) { + if (loaded) return + preferences.all.forEach { (key, value) -> parseStamp(value as? String)?.let { store(key, it) } } + loaded = true + } + } + + private fun refresh(key: String) { + if (!loaded) return + val stamp = parseStamp(preferences.getString(key, null)) + if (stamp == null) { + stamps.remove(key) + groupOf?.invoke(key)?.let { group -> groups[group]?.remove(key) } + } else { + store(key, stamp) + } + } + + private fun store(key: String, stamp: Stamp) { + stamps[key] = stamp + groupOf?.invoke(key)?.let { group -> groups.getOrPut(group) { ConcurrentHashMap.newKeySet() }.add(key) } + } + + companion object { + private val registry = HashMap() + + /** The process-wide cache for [store]; [groupOf] only takes effect for whoever opens it first. */ + fun of(context: Context, store: String, groupOf: ((String) -> String?)? = null): FrequencyStampCache = + synchronized(registry) { + registry.getOrPut(store) { + FrequencyStampCache(context.applicationContext.getSharedPreferences(store, Context.MODE_PRIVATE), groupOf) + } + } + + /** "count,lastUsedMs" -> Stamp, or null for anything else. */ + fun parseStamp(value: String?): Stamp? { + val comma = value?.indexOf(',') ?: return null + if (comma <= 0) return null + val count = value.substring(0, comma).toIntOrNull() ?: return null + val last = value.substring(comma + 1).toLongOrNull() ?: return null + return Stamp(count, last) + } + } +} diff --git a/openless-all/app/android/kotlin/ImeLearningPolicy.kt b/openless-all/app/android/kotlin/ImeLearningPolicy.kt index 18f84f83d..88f3f2446 100644 --- a/openless-all/app/android/kotlin/ImeLearningPolicy.kt +++ b/openless-all/app/android/kotlin/ImeLearningPolicy.kt @@ -6,8 +6,8 @@ import android.view.inputmethod.EditorInfo internal object ImeLearningPolicy { const val PRIVACY_VERSION = 1 - private val stores = listOf("openless_english_frequency", "openless_english_custom_words", - "openless_pinyin_frequency", "openless_pinyin_learned_phrases") + // LitePinyin's two stores are deleted outright by OpenLessRime.initialize(). + private val stores = listOf("openless_english_frequency", "openless_english_custom_words") fun isPassword(inputType: Int): Boolean { val kind = inputType and InputType.TYPE_MASK_CLASS diff --git a/openless-all/app/android/kotlin/KeyboardDataFile.kt b/openless-all/app/android/kotlin/KeyboardDataFile.kt new file mode 100644 index 000000000..1be082903 --- /dev/null +++ b/openless-all/app/android/kotlin/KeyboardDataFile.kt @@ -0,0 +1,225 @@ +package com.openless.app + +import android.content.Context +import android.content.SharedPreferences +import org.json.JSONArray +import org.json.JSONObject +import java.io.File + +/** + * The keyboard settings page's own export/import, in one JSON file: that + * page's preferences (keyboard size, haptics and the other switches), the + * Rime pinyin user dictionary, the stroke and association picks, and the + * cloud notes webhook (its token in plaintext). App + * settings are not part of it; they travel through cloud sync and the + * migration bundle. Importing merges the learning data into what this + * device already learned and never deletes it. + */ +internal object KeyboardDataFile { + const val VERSION = 1 + // The pinyin snapshot alone may reach RimeUserDictionaryFile.MAX_BYTES. + const val MAX_BYTES = 48 * 1024 * 1024 + private const val MAX_PICKS = 50_000 + private const val PREFS_STORE = "openless_ime_ui" + + enum class Section(val key: String, val labelZh: String, val labelEn: String) { + SETTINGS("keyboardSettings", "键盘设置(高度、震动、开关)", "Keyboard settings (size, haptics, switches)"), + PINYIN("pinyinUserDictionary", "拼音用户词典", "Pinyin user dictionary"), + STROKE("strokeCodes", "笔画调频", "Stroke ranking"), + ASSOCIATION("associations", "联想学习", "Association learning"), + CLOUD_NOTES("cloudNotes", "云笔记地址 / Token(明文)", "Cloud notes URL / token (plaintext)"), + } + + /** An export starts with everything checked, the plaintext token included (the checklist marks it). */ + val defaultExport: Set
= Section.entries.toSet() + + /** An import starts without the token, so another file never replaces this device's by accident. */ + val defaultImport: Set
= Section.entries.toSet() - Section.CLOUD_NOTES + + /** One preference of the keyboard settings page: its file key, preference key and accepted values. */ + private class Setting(val key: String, val pref: String, val accepts: (Any) -> Boolean, val read: (SharedPreferences) -> Any) + + private fun bool(key: String, pref: String, default: Boolean) = + Setting(key, pref, { it is Boolean }) { it.getBoolean(pref, default) } + + private fun int(key: String, pref: String, range: IntRange, default: Int) = + Setting(key, pref, { it is Int && it in range }) { it.getInt(pref, default).coerceIn(range) } + + private val settings = listOf( + int("keyboardHeightDp", OpenLessImeService.PREF_KEYBOARD_HEIGHT_DP, + OpenLessImeService.MIN_KEYBOARD_HEIGHT_DP..OpenLessImeService.MAX_KEYBOARD_HEIGHT_DP, OpenLessImeService.DEFAULT_KEYBOARD_HEIGHT_DP), + int("keyboardRaiseDp", OpenLessImeService.PREF_KEYBOARD_RAISE_DP, + OpenLessImeService.MIN_KEYBOARD_RAISE_DP..OpenLessImeService.MAX_KEYBOARD_RAISE_DP, OpenLessImeService.DEFAULT_KEYBOARD_RAISE_DP), + bool("hapticEnabled", "key_haptic_enabled", true), + int("hapticAmplitude", "key_haptic_amplitude", 1..255, 55), + // Stored as a Long; the page's slider keeps it at 1..125. + Setting("hapticDurationMs", "key_haptic_duration_ms", { it is Int && it in 1..1000 }) { + it.getLong("key_haptic_duration_ms", 12L).coerceIn(1L, 1000L).toInt() + }, + bool("debugFeaturesEnabled", "key_debug_features_enabled", false), + bool("cloudPinyinEnabled", "cloud_pinyin_enabled", false), + bool("publicDictionaryUpdates", "public_dictionary_updates", true), + bool("englishSuggestionsEnabled", "english_suggestions_enabled", true), + Setting("fuzzyPinyin", "pinyin_fuzzy_groups", { value -> + value is List<*> && value.all { id -> RimeFuzzyPinyin.groups.any { it.id == id } } + }) { RimeFuzzyPinyin.selected(it).toList() }, + ) + + /** Whether [value] (a JSON value with arrays as lists) is acceptable for the keyboard setting [key]. */ + fun acceptsSetting(key: String, value: Any): Boolean = settings.firstOrNull { it.key == key }?.accepts?.invoke(value) == true + + /** Whether a cloud notes webhook address may be imported: blank, or http(s). */ + fun acceptsWebhookUrl(url: String): Boolean = + url.isBlank() || runCatching { java.net.URI(url).scheme in listOf("http", "https") }.getOrDefault(false) + + class Contents( + val settings: Map, + val pinyinSnapshot: String?, + val strokeCodes: Map, + val associations: Map, + val cloudNotes: Pair?, + ) { + private val pinyinCount = pinyinSnapshot?.let { RimeUserDictionaryFile.entryCount(it) } ?: 0 + + fun count(section: Section): Int = when (section) { + Section.SETTINGS -> settings.size + Section.PINYIN -> pinyinCount + Section.STROKE -> strokeCodes.size + Section.ASSOCIATION -> associations.size + Section.CLOUD_NOTES -> if (cloudNotes == null) 0 else 1 + } + + /** Sections that hold something to import. */ + fun present(): List
= Section.entries.filter { count(it) > 0 } + } + + /** The chosen [sections] as a file. Off the main thread: reading the pinyin dictionary closes the open Rime sessions. */ + fun export(context: Context, sections: Set
): String { + val app = context.applicationContext + val prefs = app.getSharedPreferences(PREFS_STORE, Context.MODE_PRIVATE) + val pinyin = if (Section.PINYIN !in sections) null else { + val scratch = File(app.cacheDir, "export-pinyin_simp.userdb.txt") + try { + if (OpenLessRime.exportUserDictionary(app, scratch)) scratch.readText() else null + } finally { scratch.delete() } + } + return JSONObject().apply { + put("openlessKeyboardDataVersion", VERSION) + put( + "exportedAt", + java.text.SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX", java.util.Locale.US).format(java.util.Date()), + ) + if (Section.SETTINGS in sections) { + put(Section.SETTINGS.key, JSONObject().apply { + for (setting in settings) { + val value = setting.read(prefs) + put(setting.key, if (value is List<*>) JSONArray(value) else value) + } + }) + } + pinyin?.let { put(Section.PINYIN.key, it) } + if (Section.STROKE in sections) put(Section.STROKE.key, JSONObject(StrokeUserFrequency.strokeCodes(app).exportAll())) + if (Section.ASSOCIATION in sections) put(Section.ASSOCIATION.key, JSONObject(StrokeUserFrequency.associations(app).exportAll())) + if (Section.CLOUD_NOTES in sections) { + put(Section.CLOUD_NOTES.key, JSONObject().apply { + put("webhookUrl", prefs.getString("key_cloud_note_webhook_url", "") ?: "") + put("webhookToken", prefs.getString("key_cloud_note_webhook_token", "") ?: "") + }) + } + }.toString() + } + + /** Parses and checks every section; throws IllegalArgumentException for anything that is not a valid keyboard data file. */ + fun read(json: String): Contents { + val root = runCatching { JSONObject(json) }.getOrElse { throw IllegalArgumentException("Not JSON") } + require(root.optInt("openlessKeyboardDataVersion", 0) == VERSION) { "Unsupported keyboard data version" } + val pinyin = root.opt(Section.PINYIN.key)?.let { value -> + require(value is String) { "Invalid pinyin dictionary" } + value.also { RimeUserDictionaryFile.entryCount(it) } + } + val cloudNotes = root.opt(Section.CLOUD_NOTES.key)?.let { value -> + require(value is JSONObject) { "Invalid cloud notes" } + val url = value.optString("webhookUrl", "") + require(acceptsWebhookUrl(url)) { "Invalid webhook URL" } + url to value.optString("webhookToken", "") + } + return Contents( + readSettings(root), + pinyin, + picks(root, Section.STROKE, strokeKeys = true), + picks(root, Section.ASSOCIATION, strokeKeys = false), + cloudNotes, + ) + } + + private fun readSettings(root: JSONObject): Map { + val value = root.opt(Section.SETTINGS.key) ?: return emptyMap() + require(value is JSONObject) { "Invalid keyboard settings" } + val result = mutableMapOf() + for (key in value.keys()) { + val raw = value.get(key) + val item: Any = if (raw is JSONArray) (0 until raw.length()).map { raw.get(it) } else raw + require(acceptsSetting(key, item)) { "Invalid keyboard setting $key" } + result[key] = item + } + return result + } + + private fun picks(root: JSONObject, section: Section, strokeKeys: Boolean): Map { + val value = root.opt(section.key) ?: return emptyMap() + require(value is JSONObject && value.length() <= MAX_PICKS) { "Invalid ${section.key}" } + val entries = mutableMapOf() + for (key in value.keys()) { + val stamp = value.get(key) + require( + stamp is String && StrokeUserFrequency.isValidEntry(key, stamp) && + StrokeUserFrequency.isStrokeCodeKey(key) == strokeKeys, + ) { "Invalid ${section.key} entry" } + entries[key] = stamp + } + return entries + } + + /** Applies the chosen [sections] of [contents]: settings and cloud notes are set, learning data is merged. Off the main thread. */ + fun import(context: Context, contents: Contents, sections: Set
) { + val app = context.applicationContext + val prefs = app.getSharedPreferences(PREFS_STORE, Context.MODE_PRIVATE) + if (Section.SETTINGS in sections && contents.settings.isNotEmpty()) { + val fuzzyBefore = RimeFuzzyPinyin.selected(prefs) + // Haptics, English suggestions and debug features are also in the + // migration bundle, so the write goes through its fence. + check(OpenLessMigrationState.settings { + val editor = prefs.edit() + for ((key, value) in contents.settings) { + val setting = settings.first { it.key == key } + when { + key == "fuzzyPinyin" -> RimeFuzzyPinyin.save(prefs, (value as List<*>).map { it as String }.toSet()) + key == "hapticDurationMs" -> editor.putLong(setting.pref, (value as Int).toLong()) + value is Boolean -> editor.putBoolean(setting.pref, value) + value is Int -> editor.putInt(setting.pref, value) + } + } + check(editor.commit()) { "Keyboard settings save failed" } + }) { "migration_android_busy" } + val fuzzyAfter = RimeFuzzyPinyin.selected(prefs) + if (fuzzyAfter != fuzzyBefore) OpenLessRime.applyFuzzyPinyin(app, fuzzyAfter) + } + if (Section.PINYIN in sections && contents.pinyinSnapshot != null) { + // Rime parses a snapshot as text only when its name ends in .userdb.txt. + val scratch = File(app.cacheDir, "import-pinyin_simp.userdb.txt") + try { + scratch.writeText(contents.pinyinSnapshot) + OpenLessRime.importUserDictionary(app, scratch) + } finally { scratch.delete() } + } + if (Section.STROKE in sections) StrokeUserFrequency.strokeCodes(app).mergeAll(contents.strokeCodes) + if (Section.ASSOCIATION in sections) StrokeUserFrequency.associations(app).mergeAll(contents.associations) + if (Section.CLOUD_NOTES in sections) { + contents.cloudNotes?.let { (url, token) -> + check(prefs.edit().putString("key_cloud_note_webhook_url", url).putString("key_cloud_note_webhook_token", token).commit()) { + "Cloud notes save failed" + } + } + } + } +} diff --git a/openless-all/app/android/kotlin/LegacyPinyinBackup.kt b/openless-all/app/android/kotlin/LegacyPinyinBackup.kt deleted file mode 100644 index 7dab7cc09..000000000 --- a/openless-all/app/android/kotlin/LegacyPinyinBackup.kt +++ /dev/null @@ -1,24 +0,0 @@ -package com.openless.app - -import android.content.Context - -/** Old backup-format compatibility only; never queried by the Rime engine. */ -internal class LegacyPinyinBackup(context: Context) { - private val preferences = context.applicationContext.getSharedPreferences( - "openless_pinyin_learned_phrases", Context.MODE_PRIVATE, - ) - - fun entryCount(): Int = exportAll().size - - fun exportAll(): Map = preferences.all.mapNotNull { (key, value) -> - (value as? String)?.let { key to it } - }.toMap() - - fun importAll(entries: Map) { - check(OpenLessMigrationState.settings { - check(preferences.edit().clear().apply { - entries.forEach { (key, value) -> putString(key, value) } - }.commit()) { "Legacy pinyin backup save failed" } - }) { "migration_android_busy" } - } -} diff --git a/openless-all/app/android/kotlin/OpenLessImeService.kt b/openless-all/app/android/kotlin/OpenLessImeService.kt index 8787b73a6..093ffe920 100644 --- a/openless-all/app/android/kotlin/OpenLessImeService.kt +++ b/openless-all/app/android/kotlin/OpenLessImeService.kt @@ -292,6 +292,16 @@ class OpenLessImeService : InputMethodService(), OpenLessOverlayBridge.OverlaySt private var clipboardHistoryMode = false private var clipboardHistoryCategory = OpenLessClipboardHistory.Category.ALL private var shiftState = ShiftState.OFF + // SHIFT_ONCE that updateAutoShift() turned on for a sentence start, as + // opposed to the user's own Shift tap — only this kind is turned back off + // when the caret leaves the sentence start. + private var autoShift = false + // The space selectEnglishCandidate() appended, still right before the + // caret: sentence punctuation typed next takes its place ("word, "). + private data class EnglishEditAnchor(val connection: android.view.inputmethod.InputConnection, val offset: Int) + private var englishAutoSpace: EnglishEditAnchor? = null + private var lastEnglishSpace: EnglishEditAnchor? = null + private var lastEnglishSpaceAt = 0L private var state = "idle" private var currentMessage = "点击开始说话" private var status: TextView? = null @@ -318,6 +328,15 @@ class OpenLessImeService : InputMethodService(), OpenLessOverlayBridge.OverlaySt // view tree away via refreshInputView() and would otherwise leave // stale View keys behind. private val englishSwipeSymbols = HashMap() + // The letter keys and Shift key of the English panel on screen, so a + // Shift change can relabel them in place (applyShiftState()) instead of + // rebuilding the whole keyboard. Entries from an older panel are simply + // no longer attached to the window and get skipped. + private val englishLetterKeys = ArrayList>() + private var englishShiftKey: ShiftKeyView? = null + // Binding behind each reusable candidate view in englishCandidateRow — + // see populateLatinCandidateRow(). + private val latinCandidateSlots = java.util.WeakHashMap() private var voiceButton: VoiceButton? = null // Silence-detection for the main voice panel: if the mic capture never // reports a meaningful level for a while after recording starts, the @@ -556,8 +575,10 @@ class OpenLessImeService : InputMethodService(), OpenLessOverlayBridge.OverlaySt rimePinyinController.clear() latinInputMode = latinInputMode.toggled() saveLatinInputMode(latinInputMode) + clearEnglishAutoText() refreshLatinCandidateBar() performKeyHaptic() + updateAutoShift(refresh = false) refreshInputView() } @@ -1266,6 +1287,9 @@ class OpenLessImeService : InputMethodService(), OpenLessOverlayBridge.OverlaySt clipboardHistoryMode = false clipboardHistoryCategory = OpenLessClipboardHistory.Category.ALL shiftState = ShiftState.OFF + autoShift = false + clearEnglishAutoText() + updateAutoShift(refresh = false) if (slideDirection != null) refreshInputView(slideDirection) else refreshInputView() } @@ -1421,7 +1445,7 @@ class OpenLessImeService : InputMethodService(), OpenLessOverlayBridge.OverlaySt } finalizeEnglishComposingWord() rimePinyinController.clear() - currentInputConnection?.commitText(" ", 1) + commitEnglishSpace() }, swipeUpAction = { commitEnglishChar(".") }, swipePreview = ".", longPressAction = { toggleLatinInputMode() }).apply { // Same size as Return (17sp — keyboardKey()'s own special case // for that label), per real-device feedback. @@ -1573,7 +1597,10 @@ class OpenLessImeService : InputMethodService(), OpenLessOverlayBridge.OverlaySt * add straight into the same now-fresh map without re-clearing it. */ private fun addEnglishCharRow(parent: LinearLayout, keys: List, swipeSymbols: List? = null) { - if (swipeSymbols != null) englishSwipeSymbols.clear() + if (swipeSymbols != null) { + englishSwipeSymbols.clear() + englishLetterKeys.clear() + } val row = LinearLayout(this).apply { gravity = android.view.Gravity.CENTER } keys.forEachIndexed { index, key -> row.addView(buildEnglishCharKey(key, 1f, swipeSymbol = swipeSymbols?.getOrNull(index))) @@ -1604,7 +1631,9 @@ class OpenLessImeService : InputMethodService(), OpenLessOverlayBridge.OverlaySt private fun handleEnglishBottomModeToggle() { englishLayer = if (englishLayer == EnglishLayer.LETTERS) EnglishLayer.NUMBERS else EnglishLayer.LETTERS shiftState = ShiftState.OFF + autoShift = false finalizeEnglishComposingWord() + updateAutoShift(refresh = false) refreshInputView() } @@ -1615,21 +1644,102 @@ class OpenLessImeService : InputMethodService(), OpenLessOverlayBridge.OverlaySt } private fun cycleEnglishShift() { - // Cycles lowercase -> capitalize-next -> caps-lock -> lowercase. - shiftState = when (shiftState) { - ShiftState.OFF -> ShiftState.SHIFT_ONCE - ShiftState.SHIFT_ONCE -> ShiftState.CAPS_LOCK - ShiftState.CAPS_LOCK -> ShiftState.OFF + // Cycles lowercase -> capitalize-next -> caps-lock -> lowercase. A + // capital turned on for a sentence start goes straight back off: the + // user tapping Shift there wants lowercase, not caps lock. + shiftState = when { + autoShift && shiftState == ShiftState.SHIFT_ONCE -> ShiftState.OFF + shiftState == ShiftState.OFF -> ShiftState.SHIFT_ONCE + shiftState == ShiftState.SHIFT_ONCE -> ShiftState.CAPS_LOCK + else -> ShiftState.OFF + } + autoShift = false + applyShiftState() + } + + /** + * Turns a one-shot capital on at a sentence start (see + * EnglishAutoText.isSentenceStart) and back off once the caret leaves + * it — but only a capital it turned on itself; the user's own Shift or + * caps lock is never overridden. Pass refresh = false when the caller + * rebuilds the input view anyway. + */ + private fun updateAutoShift(refresh: Boolean = true) { + if (shiftState == ShiftState.CAPS_LOCK || (shiftState == ShiftState.SHIFT_ONCE && !autoShift)) return + val applies = inputMode == InputMode.ENGLISH && + latinInputMode == LatinInputMode.ENGLISH && + englishLayer == EnglishLayer.LETTERS && + EnglishAutoText.allowsAutoCapitals(currentInputEditorInfo?.inputType ?: 0) + val wanted = applies && currentInputConnection + ?.getTextBeforeCursor(EnglishAutoText.SENTENCE_LOOKBEHIND, 0) + ?.let { EnglishAutoText.isSentenceStart(it) } == true + val next = when { + wanted && shiftState == ShiftState.OFF -> ShiftState.SHIFT_ONCE + !wanted && autoShift -> ShiftState.OFF + else -> return + } + shiftState = next + autoShift = next == ShiftState.SHIFT_ONCE + if (refresh) applyShiftState() + } + + private fun englishEditAnchor(connection: android.view.inputmethod.InputConnection): EnglishEditAnchor? { + val extracted = connection.getExtractedText(android.view.inputmethod.ExtractedTextRequest(), 0) ?: return null + if (extracted.selectionStart < 0 || extracted.selectionStart != extracted.selectionEnd) return null + return EnglishEditAnchor(connection, extracted.startOffset + extracted.selectionStart) + } + + private fun clearEnglishAutoText() { + englishAutoSpace = null + lastEnglishSpace = null + lastEnglishSpaceAt = 0L + } + + private fun invalidateEnglishAutoTextSelection(start: Int, end: Int) { + if (start != end || start < 0) { + clearEnglishAutoText() + return + } + if (englishAutoSpace?.offset != start) englishAutoSpace = null + if (lastEnglishSpace?.offset != start) { + lastEnglishSpace = null + lastEnglishSpaceAt = 0L + } + } + + /** Space on the English/pinyin keyboard; in English mode a second quick tap after a word turns "word " into "word. ". */ + private fun commitEnglishSpace() { + val connection = currentInputConnection ?: return + val now = android.os.SystemClock.uptimeMillis() + val position = englishEditAnchor(connection) + val elapsed = now - lastEnglishSpaceAt + val quickSecondTap = lastEnglishSpace != null && lastEnglishSpace == position && + elapsed >= 0L && elapsed < DOUBLE_SPACE_WINDOW_MS + clearEnglishAutoText() + if (quickSecondTap && + latinInputMode == LatinInputMode.ENGLISH && + !ImeLearningPolicy.isPassword(currentInputEditorInfo?.inputType ?: 0) && + EnglishAutoText.doubleSpaceMakesFullStop(connection.getTextBeforeCursor(2, 0) ?: "") + ) { + if (connection.deleteSurroundingText(1, 0)) connection.commitText(". ", 1) + return + } + if (connection.commitText(" ", 1)) { + lastEnglishSpace = englishEditAnchor(connection) + lastEnglishSpaceAt = now } - refreshInputView() } /** Backspace on the English keyboard: same field edit as everywhere else, plus keeping englishComposingWord (and therefore the candidate bar) in sync. In Pinyin mode, deletes from the (never-committed) encoding instead — falls back to the normal field edit only once the encoding is already empty (plan 3.3: "编码为空时 Backspace 恢复现有普通删除"). */ private fun englishDeleteBackward() { if (latinInputMode == LatinInputMode.PINYIN) { - rimePinyinController.backspace({ renderPinyinCandidates(it) }, { deleteBackward() }) + rimePinyinController.backspace({ renderPinyinCandidates(it) }, { + rimePinyinController.resetAssociationContext() + deleteBackward() + }) return } + clearEnglishAutoText() deleteBackward() if (englishComposingWord.isNotEmpty()) { englishComposingWord.deleteCharAt(englishComposingWord.length - 1) @@ -1692,6 +1802,84 @@ class OpenLessImeService : InputMethodService(), OpenLessOverlayBridge.OverlaySt } } + /** What a reusable candidate view in englishCandidateRow does when tapped, swapped on every reuse. */ + private class LatinCandidateSlot(val hasLongPress: Boolean) { + var action: () -> Unit = {} + var longPress: () -> Unit = {} + } + + private class LatinCandidate(val label: String, val longPress: (() -> Unit)?, val action: () -> Unit) + + private var pinyinEncodingLabel: TextView? = null + + /** + * Fills englishCandidateRow with [candidates] (after an optional + * non-clickable [leadingLabel], pinyin's in-progress encoding), reusing + * the candidate views already there — same idea as the stroke row's + * populateCandidateRow(): this runs on every keystroke, and creating a + * dozen keyboardKey() views each time was the costly part. A view's tap + * and long-press go through its LatinCandidateSlot, so reusing it only + * swaps the slot's actions; keyboardKey()'s own long-press handling + * (including swallowing the tap that ends a long press) stays intact. + * Only a view of the same kind is reused — whether it has a long press + * is fixed when keyboardKey() builds it. + */ + private fun populateLatinCandidateRow(candidates: List, leadingLabel: String? = null) { + val row = englishCandidateRow ?: return + val firstIsLabel = row.childCount > 0 && row.getChildAt(0) === pinyinEncodingLabel + var offset = 0 + if (leadingLabel != null) { + val label = if (firstIsLabel) pinyinEncodingLabel!! else TextView(this).apply { + textSize = 16f + setSingleLine(true) + gravity = android.view.Gravity.CENTER + setTextColor(tone(Color.rgb(180, 180, 180), Color.rgb(120, 120, 125))) + setPadding(dp(9), 0, dp(9), 0) + isClickable = false + }.also { + pinyinEncodingLabel = it + row.addView(it, 0, LinearLayout.LayoutParams(ViewGroup.LayoutParams.WRAP_CONTENT, ViewGroup.LayoutParams.MATCH_PARENT)) + } + label.text = leadingLabel + offset = 1 + } else if (firstIsLabel) { + row.removeViewAt(0) + } + candidates.forEachIndexed { index, candidate -> + val position = offset + index + val existing = row.getChildAt(position) as? TextView + val existingSlot = existing?.let { latinCandidateSlots[it] } + val view: TextView + val slot: LatinCandidateSlot + if (existing != null && existingSlot != null && existingSlot.hasLongPress == (candidate.longPress != null)) { + view = existing + slot = existingSlot + } else { + val fresh = LatinCandidateSlot(candidate.longPress != null) + slot = fresh + view = candidateItemView( + candidate.label, + isFirst = index == 0, + onLongPress = if (fresh.hasLongPress) ({ fresh.longPress() }) else null, + action = { fresh.action() }, + ) + latinCandidateSlots[view] = fresh + if (existing != null || position < row.childCount) row.removeViewAt(position) + row.addView(view, position, LinearLayout.LayoutParams(ViewGroup.LayoutParams.WRAP_CONTENT, ViewGroup.LayoutParams.MATCH_PARENT)) + } + slot.action = candidate.action + candidate.longPress?.let { slot.longPress = it } + if (view.text.toString() != candidate.label) view.text = candidate.label + view.contentDescription = candidate.label + styleCandidateFirstState(view, index == 0) + } + while (row.childCount > offset + candidates.size) row.removeViewAt(row.childCount - 1) + // Shared with the stroke panel's own candidate row (only one of the + // two is ever visible at a time) — see showCandidateOverlay(). + candidateOverlayEntries = candidates.map { it.label to it.action } + if (candidates.isEmpty()) englishExpandCandidatesButton?.visibility = View.GONE + } + /** * "Show more candidates" overlay — a PopupWindow anchored below the * candidate row, wrapping the current full candidate/association list @@ -2420,7 +2608,7 @@ class OpenLessImeService : InputMethodService(), OpenLessOverlayBridge.OverlaySt StrokeActionView(this, graphicActionCode, graphicRotation) } label == "←" || label == "↵" -> ActionSymbolView(this, label, isDarkTheme) - label == "⇧" -> ShiftKeyView(this, shiftState, isDarkTheme) + label == "⇧" -> ShiftKeyView(this, shiftState, isDarkTheme).also { englishShiftKey = it } midDivider -> MidDividerTextView(this, tone(Color.rgb(100, 100, 100), Color.rgb(205, 205, 209))) else -> TextView(this) } @@ -2732,11 +2920,36 @@ class OpenLessImeService : InputMethodService(), OpenLessOverlayBridge.OverlaySt * This is a dedicated builder, not a keyboardKey() variant, precisely * so none of this touch handling can affect any other panel's keys. */ - private fun buildEnglishCharKey(baseChar: String, weight: Float, swipeSymbol: String? = null): View { - fun charFor(view: View): String { - val base = view.tag as? String ?: return "" - return if (shiftState != ShiftState.OFF && base.length == 1 && base[0].isLetter()) base.uppercase() else base + /** What a char key types right now: its base character, uppercased for a letter while Shift is on. */ + private fun englishCharFor(view: View): String { + val base = view.tag as? String ?: return "" + return if (shiftState != ShiftState.OFF && base.length == 1 && base[0].isLetter()) base.uppercase() else base + } + + /** + * Shows the current shiftState on the English panel: relabels the letter + * keys and redraws Shift in place. Typing already reads shiftState at the + * moment of the tap (englishCharFor), so only the labels were stale — + * rebuilding the whole keyboard for that ran on every sentence start and + * every capital. Falls back to a rebuild when that panel isn't on screen. + */ + private fun applyShiftState() { + val keys = englishLetterKeys.filter { it.first.isAttachedToWindow } + val shiftKey = englishShiftKey?.takeIf { it.isAttachedToWindow } + if (keys.isEmpty() || shiftKey == null) { + refreshInputView() + return } + keys.forEach { (key, letter) -> + val char = englishCharFor(key) + letter.text = char + key.contentDescription = char + } + shiftKey.state = shiftState + } + + private fun buildEnglishCharKey(baseChar: String, weight: Float, swipeSymbol: String? = null): View { + fun charFor(view: View): String = englishCharFor(view) val letterView = TextView(this).apply { textSize = 22f typeface = Typeface.create("sans-serif-medium", Typeface.NORMAL) @@ -2753,6 +2966,7 @@ class OpenLessImeService : InputMethodService(), OpenLessOverlayBridge.OverlaySt tag = baseChar if (swipeSymbol != null) englishSwipeSymbols[this] = swipeSymbol letterView.text = charFor(this) + if (baseChar.length == 1 && baseChar[0].isLetter()) englishLetterKeys += this to letterView addView(letterView, FrameLayout.LayoutParams(FrameLayout.LayoutParams.MATCH_PARENT, FrameLayout.LayoutParams.MATCH_PARENT)) if (swipeSymbol != null) { addView( @@ -2944,16 +3158,24 @@ class OpenLessImeService : InputMethodService(), OpenLessOverlayBridge.OverlaySt // encoding buffer in Pinyin mode; see // RimePinyinController's own doc comment. rimePinyinController.appendLetter(char[0]) { renderPinyinCandidates(it) } - if (shiftState == ShiftState.SHIFT_ONCE) { - shiftState = ShiftState.OFF - refreshInputView() - } + consumeShiftOnce() return } rimePinyinController.insertLiteral(char) { renderPinyinCandidates(it) } return } - if (currentInputConnection?.commitText(char, 1) != true) return + val connection = currentInputConnection ?: return + val replaceAutoSpace = englishAutoSpace != null && englishAutoSpace == englishEditAnchor(connection) && + EnglishAutoText.swallowsAutoSpace(char) && connection.getTextBeforeCursor(1, 0)?.toString() == " " + clearEnglishAutoText() + if (replaceAutoSpace) { + if (!connection.deleteSurroundingText(1, 0) || !connection.commitText("$char ", 1)) return + englishAutoSpace = englishEditAnchor(connection) + finalizeEnglishComposingWord() + consumeShiftOnce() + return + } + if (!connection.commitText(char, 1)) return if (personalizedLearningAllowed() && char.length == 1 && char[0].isLetter()) { englishComposingWord.append(char.lowercase()) updateEnglishCandidates() @@ -2964,10 +3186,15 @@ class OpenLessImeService : InputMethodService(), OpenLessOverlayBridge.OverlaySt // benefit for that particular boundary. finalizeEnglishComposingWord() } - if (shiftState == ShiftState.SHIFT_ONCE) { - shiftState = ShiftState.OFF - refreshInputView() - } + consumeShiftOnce() + } + + /** A one-shot capital (the user's or updateAutoShift()'s) is spent by the character just typed. */ + private fun consumeShiftOnce() { + if (shiftState != ShiftState.SHIFT_ONCE) return + shiftState = ShiftState.OFF + autoShift = false + applyShiftState() } /** Ends the current word: records it for user-frequency learning (if long enough to be a real word) and clears the tracking buffer. */ @@ -3080,25 +3307,9 @@ class OpenLessImeService : InputMethodService(), OpenLessOverlayBridge.OverlaySt } private fun renderEnglishCandidates(words: List) { - val row = englishCandidateRow ?: return - row.removeAllViews() - val overlayEntries = mutableListOf Unit>>() - words.forEachIndexed { index, word -> - row.addView( - candidateItemView( - word, - isFirst = index == 0, - action = { selectEnglishCandidate(word) }, - onLongPress = { forgetEnglishCandidate(word) }, - ), - LinearLayout.LayoutParams(ViewGroup.LayoutParams.WRAP_CONTENT, ViewGroup.LayoutParams.MATCH_PARENT), - ) - overlayEntries.add(word to { selectEnglishCandidate(word) }) - } - // Shared with the stroke panel's own candidate row (only one of the - // two is ever visible at a time) — see showCandidateOverlay(). - candidateOverlayEntries = overlayEntries - if (words.isEmpty()) englishExpandCandidatesButton?.visibility = View.GONE + populateLatinCandidateRow( + words.map { word -> LatinCandidate(word, longPress = { forgetEnglishCandidate(word) }) { selectEnglishCandidate(word) } }, + ) } /** @@ -3121,7 +3332,9 @@ class OpenLessImeService : InputMethodService(), OpenLessOverlayBridge.OverlaySt break } } + clearEnglishAutoText() if (!connection.commitText("$word ", 1)) return + englishAutoSpace = englishEditAnchor(connection) englishCandidateProvider.recordCommit(word) englishComposingWord.clear() updateEnglishCandidates() @@ -3153,37 +3366,10 @@ class OpenLessImeService : InputMethodService(), OpenLessOverlayBridge.OverlaySt * or container so the fixed 300dp panel height is never touched. */ private fun renderPinyinCandidates(candidates: List) { - val row = englishCandidateRow ?: return - row.removeAllViews() - val encoding = rimePinyinController.currentEncoding() - if (encoding.isNotEmpty()) { - val label = TextView(this).apply { - text = encoding - textSize = 16f - setSingleLine(true) - gravity = android.view.Gravity.CENTER - setTextColor(tone(Color.rgb(180, 180, 180), Color.rgb(120, 120, 125))) - setPadding(dp(9), 0, dp(9), 0) - isClickable = false - } - row.addView(label, LinearLayout.LayoutParams(ViewGroup.LayoutParams.WRAP_CONTENT, ViewGroup.LayoutParams.MATCH_PARENT)) - } - val overlayEntries = mutableListOf Unit>>() - candidates.forEachIndexed { index, char -> - row.addView( - candidateItemView( - char, - isFirst = index == 0, - action = { selectPinyinCandidate(char) }, - ), - LinearLayout.LayoutParams(ViewGroup.LayoutParams.WRAP_CONTENT, ViewGroup.LayoutParams.MATCH_PARENT), - ) - overlayEntries.add(char to { selectPinyinCandidate(char) }) - } - // Shared with the English/stroke candidate rows (only one of the - // three is ever visible at a time) — see showCandidateOverlay(). - candidateOverlayEntries = overlayEntries - if (candidates.isEmpty()) englishExpandCandidatesButton?.visibility = View.GONE + populateLatinCandidateRow( + candidates.map { char -> LatinCandidate(char, longPress = null) { selectPinyinCandidate(char) } }, + leadingLabel = rimePinyinController.currentEncoding().takeIf { it.isNotEmpty() }, + ) } /** Candidate tap is the ONLY way a pinyin candidate reaches the field — see toggleLatinInputMode()'s doc comment on why Space deliberately never does this. */ @@ -3218,23 +3404,9 @@ class OpenLessImeService : InputMethodService(), OpenLessOverlayBridge.OverlaySt /** Same row/candidateItemView() machinery as renderPinyinCandidates(), but no encoding label (there's no in-progress encoding here) and suffix-based commit (see selectPinyinAssociation()) instead of committing the whole label — mirrors StrokeInputController.refreshAssociations() exactly. */ private fun renderPinyinAssociations(candidates: List) { - val row = englishCandidateRow ?: return - row.removeAllViews() - val overlayEntries = mutableListOf Unit>>() - candidates.forEachIndexed { index, candidate -> - val displayText = outputScript(candidate.text) - row.addView( - candidateItemView( - displayText, - isFirst = index == 0, - action = { selectPinyinAssociation(candidate) }, - ), - LinearLayout.LayoutParams(ViewGroup.LayoutParams.WRAP_CONTENT, ViewGroup.LayoutParams.MATCH_PARENT), - ) - overlayEntries.add(displayText to { selectPinyinAssociation(candidate) }) - } - candidateOverlayEntries = overlayEntries - if (candidates.isEmpty()) englishExpandCandidatesButton?.visibility = View.GONE + populateLatinCandidateRow( + candidates.map { candidate -> LatinCandidate(outputScript(candidate.text), longPress = null) { selectPinyinAssociation(candidate) } }, + ) } /** @@ -3276,6 +3448,7 @@ class OpenLessImeService : InputMethodService(), OpenLessOverlayBridge.OverlaySt } internal fun sendEnterKey() { + clearEnglishAutoText() if (inputMode == InputMode.ENGLISH && latinInputMode == LatinInputMode.PINYIN && !ImeLearningPolicy.isPassword(currentInputEditorInfo?.inputType ?: 0)) { rimePinyinController.confirmFirst({ renderPinyinCandidates(it) }, { sendEditorEnterKey() }) @@ -3305,6 +3478,11 @@ class OpenLessImeService : InputMethodService(), OpenLessOverlayBridge.OverlaySt refreshLanguage() strokeController.resetForNewInputSession() englishLayer = EnglishLayer.LETTERS + // An automatic capital belonged to the old field's sentence start; + // onStartInputView() works out the new field's. + if (autoShift) shiftState = ShiftState.OFF + autoShift = false + clearEnglishAutoText() englishComposingWord.clear() englishCandidateQueryEpoch++ rimePinyinController.clear() @@ -3313,6 +3491,14 @@ class OpenLessImeService : InputMethodService(), OpenLessOverlayBridge.OverlaySt rimePinyinController.bindCommit { text -> if (ImeLearningPolicy.isPassword(currentInputEditorInfo?.inputType ?: 0)) false else if (currentInputConnection?.commitText(text, 1) == true) { + // Typing an offered association out by hand counts as picking it. + if (personalizedLearningAllowed() && OpenLessAndroidPreferences.strokeUsageEnabled(this)) { + strokeController.phraseRepository.recordTypedContinuation( + rimePinyinController.associationLearning.advance(rimePinyinController.associationContext(), text), + ) + } else { + rimePinyinController.associationLearning.reset() + } rimePinyinController.recordCommittedText(text) performKeyHaptic() true @@ -3348,7 +3534,9 @@ class OpenLessImeService : InputMethodService(), OpenLessOverlayBridge.OverlaySt // A channel may have been turned off in settings since the last show. if (isBackendReady()) refreshAsrRoute() val key = keyboardFootprintKey() - if (appliedKeyboardFootprintKey != key) { + val rebuild = appliedKeyboardFootprintKey != key + updateAutoShift(refresh = !rebuild) + if (rebuild) { appliedKeyboardFootprintKey = key refreshInputView() } @@ -3358,6 +3546,7 @@ class OpenLessImeService : InputMethodService(), OpenLessOverlayBridge.OverlaySt "${keyboardPanelHeightDp()}:${raiseHeightDp(this)}:$inputMode:$latinInputMode:${personalizedLearningAllowed()}" override fun onFinishInputView(finishingInput: Boolean) { + clearEnglishAutoText() if (draftEdit != null && (recording || processing)) cancelDictation() closeDraft() super.onFinishInputView(finishingInput) @@ -3380,6 +3569,7 @@ class OpenLessImeService : InputMethodService(), OpenLessOverlayBridge.OverlaySt } override fun onFinishInput() { + clearEnglishAutoText() if (draftEdit != null && (recording || processing)) cancelDictation() closeDraft() cancelImeSession() @@ -3425,9 +3615,38 @@ class OpenLessImeService : InputMethodService(), OpenLessOverlayBridge.OverlaySt candidatesEnd: Int, ) { super.onUpdateSelection(oldSelStart, oldSelEnd, newSelStart, newSelEnd, candidatesStart, candidatesEnd) + if (draftEdit == null) invalidateEnglishAutoTextSelection(newSelStart, newSelEnd) // Selecting another span must not discard the user's pending undo/redo. if (newSelStart < 0 || newSelStart != newSelEnd) return invalidateDictationResultIfTextChanged() + if (newSelStart == 0) clearCandidatesIfFieldEmptied() + updateAutoShift() + } + + /** + * The host cleared the whole field without restarting input — a chat + * app's send button does exactly this. Associations (and an English word + * being typed) refer to text that is gone, so they must not linger in the + * candidate row; the stroke panel also drops a half-typed code and its + * code row. A pinyin encoding still being composed is left alone. + */ + private fun clearCandidatesIfFieldEmptied() { + // Typing goes to the draft box while it is open; the host field being + // empty says nothing about what the candidate row refers to. + if (draftEdit != null) return + val connection = currentInputConnection ?: return + val before = connection.getTextBeforeCursor(1, 0) ?: return + val after = connection.getTextAfterCursor(1, 0) ?: return + if (before.isNotEmpty() || after.isNotEmpty()) return + strokeController.clearAfterFieldEmptied() + rimePinyinController.resetAssociationContext() + if (inputMode != InputMode.ENGLISH) return + if (latinInputMode == LatinInputMode.PINYIN) { + if (rimePinyinController.isEmpty()) renderPinyinAssociations(emptyList()) + } else if (englishComposingWord.isNotEmpty()) { + englishComposingWord.clear() + updateEnglishCandidates() + } } internal fun toggleDictation(initialCloud: Boolean = false) { @@ -3984,6 +4203,7 @@ class OpenLessImeService : InputMethodService(), OpenLessOverlayBridge.OverlaySt /** Composition and dictation-result state belongs to one text target; drop it when the target flips between the host field and the draft. */ private fun resetEditingStateForTargetChange() { + clearEnglishAutoText() englishComposingWord.clear() englishCandidateQueryEpoch++ rimePinyinController.clear() @@ -4005,7 +4225,12 @@ class OpenLessImeService : InputMethodService(), OpenLessOverlayBridge.OverlaySt draftMessage = null draftMessageIsError = false draftConnection = null - draftEdit = android.widget.EditText(this).apply { + draftEdit = object : android.widget.EditText(this) { + override fun onSelectionChanged(selStart: Int, selEnd: Int) { + super.onSelectionChanged(selStart, selEnd) + invalidateEnglishAutoTextSelection(selStart, selEnd) + } + }.apply { inputType = InputType.TYPE_CLASS_TEXT or InputType.TYPE_TEXT_FLAG_MULTI_LINE or InputType.TYPE_TEXT_FLAG_NO_SUGGESTIONS imeOptions = EditorInfo.IME_FLAG_NO_EXTRACT_UI @@ -5757,9 +5982,15 @@ class OpenLessImeService : InputMethodService(), OpenLessOverlayBridge.OverlaySt */ private class ShiftKeyView( context: android.content.Context, - private val state: ShiftState, + initialState: ShiftState, private val darkTheme: Boolean, ) : TextView(context) { + var state: ShiftState = initialState + set(value) { + if (field == value) return + field = value + invalidate() + } private val mutedColor = if (darkTheme) Color.rgb(190, 190, 190) else Color.rgb(120, 120, 125) private val activeColor = if (darkTheme) Color.rgb(245, 245, 245) else Color.rgb(30, 30, 34) private val paint = Paint(Paint.ANTI_ALIAS_FLAG).apply { @@ -6225,6 +6456,7 @@ class OpenLessImeService : InputMethodService(), OpenLessOverlayBridge.OverlaySt // keystroke; comfortably more than can fit on screen at once so // scrolling actually reveals more real options. private const val ENGLISH_CANDIDATE_QUERY_LIMIT = 10 + private const val DOUBLE_SPACE_WINDOW_MS = 500L // buildClipboardView()'s "recent clips" strip. Originally 92dp for // 4 equal-weight rows + 5 divider lines (dp(1) each), raised ~1.2x // to 109dp per user request: (92-5)/4=21.75dp/row * 1.2 = 26.1, diff --git a/openless-all/app/android/kotlin/OpenLessKeyboardSettingsActivity.kt b/openless-all/app/android/kotlin/OpenLessKeyboardSettingsActivity.kt index f7282319c..cac93168d 100644 --- a/openless-all/app/android/kotlin/OpenLessKeyboardSettingsActivity.kt +++ b/openless-all/app/android/kotlin/OpenLessKeyboardSettingsActivity.kt @@ -260,6 +260,25 @@ class OpenLessKeyboardSettingsActivity : Activity() { isChecked = prefs.getBoolean("public_dictionary_updates", true) setOnCheckedChangeListener { _, checked -> prefs.edit().putBoolean("public_dictionary_updates", checked).apply() } }) + content.addView( + sectionLabel(ui("模糊音", "Fuzzy pinyin")), + LinearLayout.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.WRAP_CONTENT).apply { + topMargin = dp(14) + }, + ) + val fuzzy = RimeFuzzyPinyin.selected(prefs).toMutableSet() + for (group in RimeFuzzyPinyin.groups) { + content.addView(Switch(this).apply { + text = ui(group.labelZh, group.labelEn) + isChecked = group.id in fuzzy + setOnCheckedChangeListener { _, checked -> + if (checked) fuzzy += group.id else fuzzy -= group.id + RimeFuzzyPinyin.save(prefs, fuzzy) + scheduleFuzzyRedeploy() + } + }) + } + content.addView(View(this), LinearLayout.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, dp(14))) content.addView(sectionLabel(ui("英文键盘", "English keyboard"))) val englishSuggestionsRow = LinearLayout(this).apply { gravity = Gravity.CENTER_VERTICAL } englishSuggestionsRow.addView( @@ -464,28 +483,47 @@ class OpenLessKeyboardSettingsActivity : Activity() { // live in the app's own WebView settings, not this native page. content.addView(sectionLabel(ui("个人偏好数据", "Personal preference data"))) // 笔画输入的个人调频数据:记录"这个笔画码你选过哪个字",越用越靠前 - // 排序,不影响词库本身,也不会同步到云端。 - val personalFrequency = StrokeUserFrequency(this) - content.addView( - TextView(this).apply { - text = ui( - "笔画调频:已记录 ${personalFrequency.size()} / ${personalFrequency.capacity()} 条", - "Stroke ranking: ${personalFrequency.size()} / ${personalFrequency.capacity()} entries recorded", - ) - textSize = 14f - setTextColor(tone(Color.rgb(200, 200, 200), Color.rgb(70, 70, 75))) - }, - ) - // Retain old backups without presenting them as active Rime learning. - val learnedPhrases = LegacyPinyinBackup(this) + // 排序,不影响词库本身,也不会同步到云端。联想学习单独一个存储、单独 + // 计上限,两者不再互相挤占。 + val strokePicks = StrokeUserFrequency.strokeCodes(this) + val associationPicks = StrokeUserFrequency.associations(this) + for ((labelZh, labelEn, store) in listOf( + Triple("笔画调频", "Stroke ranking", strokePicks), + Triple("联想学习", "Association learning", associationPicks), + )) { + content.addView( + TextView(this).apply { + text = ui( + "$labelZh:已记录 ${store.size()} / ${store.capacity()} 条", + "$labelEn: ${store.size()} / ${store.capacity()} entries recorded", + ) + textSize = 14f + setTextColor(tone(Color.rgb(200, 200, 200), Color.rgb(70, 70, 75))) + }, + ) + } + // 拼音的选词调频和造词由 Rime 的用户词典完成;这里只读条数。 + pinyinDictionaryText = TextView(this).apply { + text = ui("拼音用户词典:读取中…", "Pinyin user dictionary: loading…") + textSize = 14f + setTextColor(tone(Color.rgb(200, 200, 200), Color.rgb(70, 70, 75))) + } + content.addView(pinyinDictionaryText) + refreshPinyinDictionaryCount() + // 键盘自己的学习数据(上面三项)单独导出 / 导入,和 App 设置无关: + // 设置走云同步,这里不进迁移文件,也不参与加密同步。 + val dataActions = LinearLayout(this).apply { gravity = Gravity.CENTER_VERTICAL } + dataActions.addView(dictionaryAction(ui("导出键盘数据 →", "Export keyboard data →")) { showKeyboardDataExportChoices() }) + dataActions.addView(dictionaryAction(ui("导入键盘数据 →", "Import keyboard data →")) { pickKeyboardDataImportSource() }) + content.addView(dataActions) content.addView( TextView(this).apply { text = ui( - "旧版拼音备份:${learnedPhrases.entryCount()} 条(不参与候选)", - "Legacy pinyin backup: ${learnedPhrases.entryCount()} entries (not used for candidates)", + "可选择导出本页的键盘设置(高度、震动等)、拼音用户词典、笔画调频、联想学习,以及云笔记地址 / Token(明文)。不含 App 设置(设置通过云同步)。文件里有你学过的词,请妥善保管。导入时可选择部分:设置会被覆盖,学习数据与本机合并(同一条取较高的次数),不会删除现有内容。", + "Choose what to export: this page's keyboard settings (size, haptics…), the pinyin user dictionary, stroke ranking, association learning, and the cloud notes URL / token (plaintext). App settings are not included; cloud sync carries them. The file holds words you have learned; keep it private. On import, pick the parts: settings are replaced, learning data merges into this device's (the higher count wins) and nothing is removed.", ) - textSize = 14f - setTextColor(tone(Color.rgb(200, 200, 200), Color.rgb(70, 70, 75))) + textSize = 12f + setTextColor(tone(Color.rgb(150, 150, 150), Color.rgb(110, 110, 115))) setPadding(0, 0, 0, dp(14)) }, ) @@ -567,6 +605,174 @@ class OpenLessKeyboardSettingsActivity : Activity() { return root } + private var pinyinDictionaryText: TextView? = null + + private fun dictionaryAction(text: String, onClick: () -> Unit): View = TextView(this).apply { + this.text = text + textSize = 15f + setTypeface(typeface, android.graphics.Typeface.BOLD) + setTextColor(tone(Color.rgb(94, 234, 212), Color.rgb(15, 118, 110))) + isClickable = true + setPadding(0, dp(8), dp(20), dp(8)) + setOnClickListener { onClick() } + } + + private fun toast(zh: String, en: String) { + Toast.makeText(applicationContext, ui(zh, en), Toast.LENGTH_LONG).show() + } + + /** Applies the saved fuzzy-pinyin groups; toggles made while one redeploy runs are folded into the next. */ + private fun scheduleFuzzyRedeploy() { + if (!fuzzyRedeployPending.compareAndSet(false, true)) return + toast("正在更新拼音方案…", "Updating the pinyin schema…") + val app = applicationContext + rimeWorker.execute { + fuzzyRedeployPending.set(false) + val result = runCatching { OpenLessRime.applyFuzzyPinyin(app, RimeFuzzyPinyin.selected(prefs)) } + runOnUiThread { + if (result.isSuccess) toast("模糊音已更新", "Fuzzy pinyin updated") + else toast("模糊音更新失败:${result.exceptionOrNull()?.message}", "Fuzzy pinyin update failed: ${result.exceptionOrNull()?.message}") + } + } + } + + /** Exports to a scratch file and counts it — the only way to read the dictionary while Rime owns it. */ + private fun refreshPinyinDictionaryCount() { + val app = applicationContext + rimeWorker.execute { + val count = runCatching { + val scratch = java.io.File(app.cacheDir, "count-pinyin_simp.userdb.txt") + try { + if (OpenLessRime.exportUserDictionary(app, scratch)) RimeUserDictionaryFile.entryCount(scratch.readText()) else 0 + } finally { scratch.delete() } + }.getOrNull() + runOnUiThread { + pinyinDictionaryText?.text = if (count == null) ui("拼音用户词典:读取失败", "Pinyin user dictionary: unavailable") + else ui("拼音用户词典:$count 条", "Pinyin user dictionary: $count entries") + } + } + } + + private var pendingExport: Set = KeyboardDataFile.defaultExport + + /** A checklist of [sections], [preset] checked; [onConfirm] gets the checked ones (never called with none). */ + private fun chooseSections( + title: String, + sections: List, + preset: Set, + label: (KeyboardDataFile.Section) -> String, + action: String, + onConfirm: (Set) -> Unit, + ) { + val checked = BooleanArray(sections.size) { sections[it] in preset } + android.app.AlertDialog.Builder(this) + .setTitle(title) + .setMultiChoiceItems(sections.map(label).toTypedArray(), checked) { _, index, isChecked -> checked[index] = isChecked } + .setPositiveButton(action) { _, _ -> + val chosen = sections.filterIndexed { index, _ -> checked[index] }.toSet() + if (chosen.isNotEmpty()) onConfirm(chosen) + } + .setNegativeButton(ui("取消", "Cancel"), null) + .show() + } + + private fun showKeyboardDataExportChoices() { + chooseSections( + ui("选择要导出的内容", "Choose what to export"), + KeyboardDataFile.Section.entries, + KeyboardDataFile.defaultExport, + { ui(it.labelZh, it.labelEn) }, + ui("下一步", "Next"), + ) { pendingExport = it; pickKeyboardDataExportTarget() } + } + + private fun pickKeyboardDataExportTarget() { + val timestamp = android.text.format.DateFormat.format("yyyyMMdd-HHmm", System.currentTimeMillis()) + val intent = android.content.Intent(android.content.Intent.ACTION_CREATE_DOCUMENT).apply { + addCategory(android.content.Intent.CATEGORY_OPENABLE) + type = "application/json" + putExtra(android.content.Intent.EXTRA_TITLE, "openless-keyboard-data-$timestamp.json") + } + runCatching { startActivityForResult(intent, REQUEST_KEYBOARD_DATA_EXPORT) } + .onFailure { toast("找不到文件管理器", "No file manager available") } + } + + private fun pickKeyboardDataImportSource() { + val intent = android.content.Intent(android.content.Intent.ACTION_OPEN_DOCUMENT).apply { + addCategory(android.content.Intent.CATEGORY_OPENABLE) + // Shared files often lose their JSON MIME type on the way. + type = "*/*" + } + runCatching { startActivityForResult(intent, REQUEST_KEYBOARD_DATA_IMPORT) } + .onFailure { toast("找不到文件管理器", "No file manager available") } + } + + /** Lists the parts [contents] actually holds, with counts, all but the token checked. */ + private fun showKeyboardDataImportChoices(contents: KeyboardDataFile.Contents) { + val sections = contents.present() + if (sections.isEmpty()) { + toast("文件里没有可导入的数据", "The file holds nothing to import") + return + } + chooseSections( + ui("选择要导入的内容", "Choose what to import"), + sections, + KeyboardDataFile.defaultImport, + { ui("${it.labelZh}(${contents.count(it)} 项)", "${it.labelEn} (${contents.count(it)})") }, + ui("导入", "Import"), + ) { chosen -> + val app = applicationContext + rimeWorker.execute { + val result = runCatching { KeyboardDataFile.import(app, contents, chosen) } + runOnUiThread { + result.onSuccess { toast("已导入:${chosen.joinToString("、") { it.labelZh }}", "Imported: ${chosen.joinToString(", ") { it.labelEn }}") } + .onFailure { toast("导入失败:${it.message}", "Import failed: ${it.message}") } + if (!isFinishing && !isDestroyed) setContentView(buildContent()) + } + } + } + } + + override fun onActivityResult(requestCode: Int, resultCode: Int, data: android.content.Intent?) { + super.onActivityResult(requestCode, resultCode, data) + val uri = data?.data?.toString() + if (resultCode != RESULT_OK || uri == null) return + val app = applicationContext + when (requestCode) { + REQUEST_KEYBOARD_DATA_EXPORT -> rimeWorker.execute { + val result = runCatching { + val json = KeyboardDataFile.export(app, pendingExport) + check(OpenLessContentWriter.writeBytes(app, uri, json.toByteArray(Charsets.UTF_8))) { "write failed" } + } + runOnUiThread { + result.onSuccess { toast("键盘数据已导出", "Keyboard data exported") } + .onFailure { toast("导出失败:${it.message}", "Export failed: ${it.message}") } + refreshPinyinDictionaryCount() + } + } + REQUEST_KEYBOARD_DATA_IMPORT -> rimeWorker.execute { + val result = runCatching { + val bytes = OpenLessContentReader.readBytes(app, uri, KeyboardDataFile.MAX_BYTES) + ?: error(ui("读取失败或文件超过 48 MiB", "Read failed or file exceeds 48 MiB")) + runCatching { KeyboardDataFile.read(String(bytes, Charsets.UTF_8)) } + .getOrElse { error(ui("不是 OpenLess 键盘数据文件", "Not an OpenLess keyboard data file")) } + } + runOnUiThread { + result.onSuccess { if (!isFinishing && !isDestroyed) showKeyboardDataImportChoices(it) } + .onFailure { toast("导入失败:${it.message}", "Import failed: ${it.message}") } + } + } + } + } + + private companion object { + const val REQUEST_KEYBOARD_DATA_EXPORT = 7101 + const val REQUEST_KEYBOARD_DATA_IMPORT = 7102 + // Outlives the Activity so a redeploy or import finishes even if the page closes; also serializes them. + val rimeWorker: java.util.concurrent.ExecutorService = java.util.concurrent.Executors.newSingleThreadExecutor() + val fuzzyRedeployPending = java.util.concurrent.atomic.AtomicBoolean(false) + } + /** Fires a one-shot vibration with the sliders' current (already-saved) values, so a change is felt immediately. */ private fun fireTestVibration(amplitude: Int, durationMs: Int) { runCatching { diff --git a/openless-all/app/android/kotlin/OpenLessRime.kt b/openless-all/app/android/kotlin/OpenLessRime.kt index 34d516168..3a65e63f9 100644 --- a/openless-all/app/android/kotlin/OpenLessRime.kt +++ b/openless-all/app/android/kotlin/OpenLessRime.kt @@ -9,9 +9,14 @@ internal object OpenLessRime { private var initialized = false private val dictionaryWorker = java.util.concurrent.Executors.newSingleThreadExecutor() private var nextDictionaryCheck = 0L + /** Bumped whenever the native side closed every session; a controller holding an older one starts a new one. */ + @Volatile var sessionGeneration = 0L + private set external fun nativeRequest(operation: String, session: String, argument: String): String @Synchronized fun initialize(context: Context) { if (initialized) { refreshPublicDictionary(context); return } + // Rime replaced LitePinyin; its stores are no longer read or backed up. + for (name in listOf("openless_pinyin_frequency", "openless_pinyin_learned_phrases")) context.deleteSharedPreferences(name) val shared = File(context.filesDir, "rime-shared-1.17.0-ice-da1fbe60-v1").apply { mkdirs() } val files = context.assets.list("rime") ?: error("Rime assets unavailable") require(files.isNotEmpty()) @@ -60,4 +65,30 @@ internal object OpenLessRime { check(response.optBoolean("ok")) { response.optString("error", "Rime request failed") } return response } + + private fun closingSessions(operation: String, argument: String): JSONObject = + try { request(operation, "", argument) } finally { sessionGeneration++ } + + /** Writes the pinyin user dictionary snapshot to [target]; false when nothing has been learned yet. Off the main thread. */ + fun exportUserDictionary(context: Context, target: File): Boolean { + initialize(context) + return closingSessions("user_dict_export", target.absolutePath).optBoolean("exported") + } + + /** Merges a snapshot already checked by [RimeUserDictionaryFile.entryCount] into the user dictionary. Off the main thread. */ + fun importUserDictionary(context: Context, source: File) { + initialize(context) + closingSessions("user_dict_import", source.absolutePath) + } + + /** Rewrites both pinyin schemas' patches for [groups] and redeploys them. Off the main thread. */ + fun applyFuzzyPinyin(context: Context, groups: Set) { + initialize(context) + val patch = RimeFuzzyPinyin.customYaml(groups) + for (schema in listOf("openless_pinyin_learning", "openless_pinyin_private")) { + val file = File(context.filesDir, "rime-user/$schema.custom.yaml") + if (patch == null) file.delete() else file.writeText(patch) + } + closingSessions("redeploy", "") + } } diff --git a/openless-all/app/android/kotlin/OpenLessSettingsExport.kt b/openless-all/app/android/kotlin/OpenLessSettingsExport.kt index aa32c3733..0324ff9ca 100644 --- a/openless-all/app/android/kotlin/OpenLessSettingsExport.kt +++ b/openless-all/app/android/kotlin/OpenLessSettingsExport.kt @@ -5,7 +5,7 @@ import org.json.JSONObject /** * Coordinates the native local migration page's legacy JSON compatibility flow — collects - * whichever of the seven categories the user selected into one JSON object + * whichever of the five categories the user selected into one JSON object * for export, and applies whichever categories are both present in an * imported file and still checked. Deliberately its own file rather than * folded into an Activity, keeping this self-contained @@ -26,7 +26,6 @@ internal object OpenLessSettingsExport { CLOUD_NOTES("cloudNotes", "云笔记地址 / Token", "Cloud notes URL / token"), HAPTIC("haptic", "震动反馈", "Haptic feedback"), STROKE_FREQUENCY("strokeFrequency", "笔画调频", "Stroke ranking"), - PINYIN_LEARNED_PHRASES("pinyinLearnedPhrases", "旧版拼音备份", "Legacy pinyin backup"), PROVIDER_SELECTION("providerSelection", "ASR / LLM / 风格包 选择", "ASR / LLM / style pack selection"), CREDENTIALS("credentials", "ASR / LLM API Key(明文,敏感)", "ASR / LLM API keys (plaintext, sensitive)"), } @@ -63,10 +62,7 @@ internal object OpenLessSettingsExport { ) } if (Category.STROKE_FREQUENCY in selected) { - root.put(Category.STROKE_FREQUENCY.key, JSONObject(StrokeUserFrequency(app).exportAll())) - } - if (Category.PINYIN_LEARNED_PHRASES in selected) { - root.put(Category.PINYIN_LEARNED_PHRASES.key, JSONObject(LegacyPinyinBackup(app).exportAll())) + root.put(Category.STROKE_FREQUENCY.key, JSONObject(StrokeUserFrequency.exportCombined(app))) } if (Category.PROVIDER_SELECTION in selected || Category.CREDENTIALS in selected) { val response = JSONObject(OpenLessNative.nativeExportProviderSettings(Category.CREDENTIALS in selected)) @@ -94,8 +90,10 @@ internal object OpenLessSettingsExport { put("englishSuggestionsEnabled", prefs.getBoolean("english_suggestions_enabled", true)) put("debugFeaturesEnabled", prefs.getBoolean("key_debug_features_enabled", false)) }) - put("strokeFrequency", JSONObject(StrokeUserFrequency(app).exportAll())) - put("pinyinLearnedPhrases", JSONObject(LegacyPinyinBackup(app).exportAll())) + put("strokeFrequency", JSONObject(StrokeUserFrequency.exportCombined(app))) + // Bundle schema 1 requires this field. It held LitePinyin's learning, + // which Rime replaced; the Rime user dictionary has its own export. + put("pinyinLearnedPhrases", JSONObject()) }.toString() } @@ -118,9 +116,8 @@ internal object OpenLessSettingsExport { beforeWrite("preferences") check(editor.commit()) { "Portable settings save failed" } beforeWrite("strokeFrequency") - StrokeUserFrequency(app).importAll(root.getJSONObject("strokeFrequency").toStringMap()) - beforeWrite("pinyinLearnedPhrases") - LegacyPinyinBackup(app).importAll(root.getJSONObject("pinyinLearnedPhrases").toStringMap()) + StrokeUserFrequency.importCombined(app, root.getJSONObject("strokeFrequency").toStringMap()) + // "pinyinLearnedPhrases" is validated but no longer restored anywhere (see exportPortable()). } private fun parsePortable(json: String): JSONObject { @@ -186,12 +183,7 @@ internal object OpenLessSettingsExport { require(amplitude in 1..255 && duration in 1..1000) { "Invalid haptic settings" } check(OpenLessMigrationState.settings { check(prefs.edit().putBoolean("key_haptic_enabled", enabled).putInt("key_haptic_amplitude", amplitude).putLong("key_haptic_duration_ms", duration).commit()) }) { "migration_android_busy" } } - Category.STROKE_FREQUENCY -> StrokeUserFrequency(app).importAll(value.toStringMap()) - Category.PINYIN_LEARNED_PHRASES -> { - require(root.optInt("learningPrivacyVersion", 0) >= ImeLearningPolicy.PRIVACY_VERSION) { "Legacy learned phrases cannot be safely restored" } - check(ImeLearningPolicy.migrate(app)) { "Learning data cleanup failed" } - LegacyPinyinBackup(app).importAll(value.toStringMap()) - } + Category.STROKE_FREQUENCY -> StrokeUserFrequency.importCombined(app, value.toStringMap()) else -> Unit } applied.add(category) diff --git a/openless-all/app/android/kotlin/RimeFuzzyPinyin.kt b/openless-all/app/android/kotlin/RimeFuzzyPinyin.kt new file mode 100644 index 000000000..3c56f52e8 --- /dev/null +++ b/openless-all/app/android/kotlin/RimeFuzzyPinyin.kt @@ -0,0 +1,48 @@ +package com.openless.app + +import android.content.SharedPreferences + +/** + * Optional fuzzy-pinyin pairs for the Rime pinyin schemas. Each group is a + * pair of rime-ice's own (commented-out) spelling rules, appended to + * `speller/algebra` through a `.custom.yaml` patch in the user data dir — + * both pinyin schemas get the same patch, since they share one compiled + * spelling index. + */ +internal object RimeFuzzyPinyin { + data class Group(val id: String, val labelZh: String, val labelEn: String, val rules: List) + + val groups = listOf( + Group("zcs", "z c s = zh ch sh", "z c s = zh ch sh", listOf("derive/^([zcs])h/\$1/", "derive/^([zcs])([^h])/\$1h\$2/")), + Group("nl", "n = l", "n = l", listOf("derive/^n/l/", "derive/^l/n/")), + Group("fh", "f = h", "f = h", listOf("derive/^f/h/", "derive/^h/f/")), + Group("rl", "r = l", "r = l", listOf("derive/^r/l/", "derive/^l/r/")), + // an$ also covers ian/uan, so this pair includes ian = iang and uan = uang. + Group("ang", "an = ang(含 ian、uan)", "an = ang (incl. ian, uan)", listOf("derive/ang\$/an/", "derive/an\$/ang/")), + Group("eng", "en = eng", "en = eng", listOf("derive/eng\$/en/", "derive/en\$/eng/")), + Group("ing", "in = ing", "in = ing", listOf("derive/ing\$/in/", "derive/in\$/ing/")), + ) + + private const val KEY = "pinyin_fuzzy_groups" + + fun selected(prefs: SharedPreferences): Set { + val known = groups.map { it.id }.toSet() + return prefs.getString(KEY, "").orEmpty().split(',').filter { it in known }.toSet() + } + + fun save(prefs: SharedPreferences, ids: Set) { + prefs.edit().putString(KEY, groups.map { it.id }.filter { it in ids }.joinToString(",")).apply() + } + + /** The schema patch for [ids], or null when no group is on (the patch file is then removed). */ + fun customYaml(ids: Set): String? { + val rules = groups.filter { it.id in ids }.flatMap { it.rules } + if (rules.isEmpty()) return null + return buildString { + append("# Written by OpenLess keyboard settings (fuzzy pinyin); changes here are overwritten.\n") + append("patch:\n") + append(" \"speller/algebra/+\":\n") + rules.forEach { append(" - ").append(it).append('\n') } + } + } +} diff --git a/openless-all/app/android/kotlin/RimePinyinController.kt b/openless-all/app/android/kotlin/RimePinyinController.kt index 188831dc1..3a24085e3 100644 --- a/openless-all/app/android/kotlin/RimePinyinController.kt +++ b/openless-all/app/android/kotlin/RimePinyinController.kt @@ -23,10 +23,12 @@ internal class RimePinyinController(context: Context) { @Volatile private var cloudChoices: Set = emptySet() @Volatile private var candidateRevision = 0L private var session = "" + private var sessionGeneration = 0L private var pending = "" private var pendingLiteral = "" private var commit: (String) -> Boolean = { false } private val confirmedText = StringBuilder() + val associationLearning = TypedContinuationHistory() private var associationEpoch = 0L fun bindCommit(handler: (String) -> Boolean) { commit = handler } fun currentEncoding(): String = encoding @@ -57,7 +59,14 @@ internal class RimePinyinController(context: Context) { try { if (epoch != generation) return@execute OpenLessRime.initialize(context) - if (session.isEmpty()) session = OpenLessRime.request("create", "", if (learning) "learning" else "private").getString("session") + // A dictionary import/export or a fuzzy-pinyin redeploy closed every native session. + if (session.isNotEmpty() && sessionGeneration != OpenLessRime.sessionGeneration) { + session = ""; pending = ""; pendingLiteral = "" + } + if (session.isEmpty()) { + sessionGeneration = OpenLessRime.sessionGeneration + session = OpenLessRime.request("create", "", if (learning) "learning" else "private").getString("session") + } if (pending.isNotEmpty()) { check(deliver(pending, epoch)) { "Editor write failed" } OpenLessRime.request("ack", session); pending = "" @@ -180,7 +189,9 @@ internal class RimePinyinController(context: Context) { if (confirmedText.length > StrokeInputController.MAX_ASSOCIATION_CONTEXT) confirmedText.delete(0, confirmedText.length - StrokeInputController.MAX_ASSOCIATION_CONTEXT) } - fun resetAssociationContext() { confirmedText.clear(); associationEpoch++ } + /** What was committed just before the next commit, for StrokePhraseRepository.recordTypedContinuation(). */ + fun associationContext(): String = confirmedText.toString() + fun resetAssociationContext() { confirmedText.clear(); associationLearning.reset(); associationEpoch++ } fun queryAssociations(repository: StrokePhraseRepository, update: (List) -> Unit) { val text = confirmedText.toString() if (text.isEmpty()) { update(emptyList()); return } diff --git a/openless-all/app/android/kotlin/RimeUserDictionaryFile.kt b/openless-all/app/android/kotlin/RimeUserDictionaryFile.kt new file mode 100644 index 000000000..d46db7f26 --- /dev/null +++ b/openless-all/app/android/kotlin/RimeUserDictionaryFile.kt @@ -0,0 +1,40 @@ +package com.openless.app + +/** + * Checks a Rime user dictionary snapshot (`pinyin_simp.userdb.txt`, the text + * format librime's backup writes) before it is merged. Rime merges a + * snapshot into whichever dictionary its header names, so anything that is + * not this keyboard's pinyin dictionary is refused here. + */ +internal object RimeUserDictionaryFile { + const val MAX_BYTES = 32 * 1024 * 1024 + + /** + * Entries in [text] that still count as learned (commit count above zero; + * Rime keeps deleted words as negative counts). Throws + * IllegalArgumentException when [text] is not a pinyin user dictionary. + */ + fun entryCount(text: String): Int { + val lines = text.lineSequence().map { it.trimEnd('\r') } + var dbName: String? = null + var dbType: String? = null + var entries = 0 + for (line in lines) { + when { + line.startsWith("#@/db_name\t") -> dbName = line.substringAfter('\t') + line.startsWith("#@/db_type\t") -> dbType = line.substringAfter('\t') + line.isBlank() || line.startsWith("#") -> Unit + else -> { + val columns = line.split('\t') + require(columns.size >= 2 && columns[0].isNotBlank() && columns[1].isNotEmpty()) { "Not a user dictionary row" } + if (commits(columns.getOrNull(2)) > 0) entries++ + } + } + } + require(dbName == "pinyin_simp" && dbType == "userdb") { "Not a pinyin user dictionary" } + return entries + } + + private fun commits(value: String?): Int = + value?.split(' ')?.firstOrNull { it.startsWith("c=") }?.substring(2)?.toIntOrNull() ?: 1 +} diff --git a/openless-all/app/android/kotlin/StrokeInput.kt b/openless-all/app/android/kotlin/StrokeInput.kt index 0946b16e3..070b38de1 100644 --- a/openless-all/app/android/kotlin/StrokeInput.kt +++ b/openless-all/app/android/kotlin/StrokeInput.kt @@ -27,16 +27,18 @@ internal class StrokeInputRepository(context: Context) { "六" to "n", "七" to "h", "八" to "p", "九" to "p", "零" to "n", ) + // Every list in here is in corpus-frequency order (see ensureLoaded()), + // so a query only walks it until it has enough matches. private val index = AtomicReference>>>(emptyMap()) private val frequency = AtomicReference>(emptyMap()) + private val codesByChar = AtomicReference>>(emptyMap()) private val loading = Any() // Personal per-code preference, learned from picking a non-top candidate. // Kept global (not per-target-app, unlike phrase association) since which - // character you mean for a given stroke code is a habit, not context — - // sharing StrokeUserFrequency's store/cap with the phrase repository is - // intentional, both are "how you personally type" data. - private val userFrequency = StrokeUserFrequency(context) + // character you mean for a given stroke code is a habit, not context. + // Its own store and cap, separate from the association picks. + private val userFrequency = StrokeUserFrequency.strokeCodes(context) /** Warm the offline index before the first stroke key is pressed. */ fun preloadAsync() { @@ -74,33 +76,44 @@ internal class StrokeInputRepository(context: Context) { fun searchAsync(pattern: String, callback: (List) -> Unit) { executor.execute { ensureLoaded() - val lookupKey = pattern.takeWhile { it != '*' }.take(PREFIX_INDEX_LENGTH) - val freq = frequency.get() - val result = if (pattern.isEmpty()) emptyList() else index.get() - .getOrDefault(lookupKey, emptyList()) - .asSequence() - .filter { (_, code) -> matches(pattern, code) } - .map { it.first } - .distinct() - .toList() - // Personal pick for this exact code always outranks the - // static corpus frequency — corpus counts span 1 to several - // million, so adding a personal score would never move the - // needle for common characters. Candidates with no personal - // score (0.0, the common case) fall through to the corpus - // order unchanged. - .sortedWith( - compareByDescending { userFrequency.score(pattern, it) } - .thenByDescending { freq[it] ?: 0L }, - ) - .take(MAX_CANDIDATES) + val startedAt = android.os.SystemClock.elapsedRealtime() + val result = if (pattern.isEmpty()) emptyList() else { + val freq = frequency.get() + val codes = codesByChar.get() + // Only characters still spelled by this code: the store + // outlives dictionary updates. + val picks = userFrequency.picksFor(pattern).filter { (char, _) -> + codes[char]?.any { strokeCodeMatches(pattern, it) } == true + } + rankStrokeCandidates( + index.get().getOrDefault(pattern.takeWhile { it != '*' }.take(PREFIX_INDEX_LENGTH), emptyList()), + pattern, + picks, + MAX_CANDIDATES, + ) { freq[it] ?: 0L } + } + android.util.Log.d( + "OpenLessStroke", + "query pattern=$pattern results=${result.size} elapsed=${android.os.SystemClock.elapsedRealtime() - startedAt}ms", + ) android.os.Handler(android.os.Looper.getMainLooper()).post { callback(result) } } } - /** Records that `candidate` was hand-picked over the top result for `pattern`, off the caller's thread. */ - fun recordPersonalPick(pattern: String, candidate: String) { - executor.execute { userFrequency.record(pattern, candidate) } + /** + * Records a pick of `candidate` for `pattern`, off the caller's thread — + * when it was hand-picked over the top result ([wasTop] false), or when + * it is on top only because of earlier picks. The latter keeps a learned + * character's score fresh: recording corrections alone let it decay and + * lose its place to the next one-off pick. Picking a corpus-top + * character that was never learned records nothing. + */ + fun recordPick(pattern: String, candidate: String, wasTop: Boolean) { + executor.execute { + if (shouldRecordStrokePick(wasTop, userFrequency.score(pattern, candidate) > 0.0)) { + userFrequency.record(pattern, candidate) + } + } } fun shutdown() = executor.shutdownNow() @@ -110,16 +123,23 @@ internal class StrokeInputRepository(context: Context) { synchronized(loading) { if (index.get().isNotEmpty()) return val startedAt = android.os.SystemClock.elapsedRealtime() - val entries = loadEntries(appContext) + val freq = loadFrequency(appContext) + // Sorted once here (stable, so equal counts keep file order) and + // bucketed in that order: every bucket comes out frequency-sorted + // without sorting each one, and no query ever sorts again. + val entries = loadEntries(appContext).sortedByDescending { freq[it.first] ?: 0L } val buckets = HashMap>>() + val codes = HashMap>() buckets[""] = entries.toMutableList() entries.forEach { entry -> val code = entry.second (1..minOf(PREFIX_INDEX_LENGTH, code.length)).forEach { length -> buckets.getOrPut(code.substring(0, length)) { ArrayList() }.add(entry) } + codes.getOrPut(entry.first) { ArrayList(1) }.add(code) } - frequency.set(loadFrequency(appContext)) + frequency.set(freq) + codesByChar.set(codes) index.set(buckets) android.util.Log.i( "OpenLessStroke", @@ -129,13 +149,47 @@ internal class StrokeInputRepository(context: Context) { } } - private fun matches(pattern: String, code: String): Boolean { - if (pattern.length > code.length) return false - return pattern.indices.all { index -> pattern[index] == '*' || pattern[index] == code[index] } - } - private companion object { const val PREFIX_INDEX_LENGTH = 4 const val MAX_CANDIDATES = 36 } } + +/** A stroke code matches a typed pattern when it starts with it, '*' standing for any one stroke. */ +internal fun strokeCodeMatches(pattern: String, code: String): Boolean { + if (pattern.length > code.length) return false + return pattern.indices.all { index -> pattern[index] == '*' || pattern[index] == code[index] } +} + +/** + * Candidates for [pattern]: the user's own picks for this exact code first + * (highest score first, corpus frequency breaking ties), then the remaining + * matches from [entries] — which must already be in corpus-frequency order — + * each character once, until [limit]. A personal pick always outranks the + * corpus: counts there span 1 to several million, so adding a personal score + * would never move a common character. + * + * Same order as sorting every match by (score, frequency), but it stops as + * soon as the row is full — the first stroke alone matches up to ~48k + * entries, and scoring and sorting all of them made that keystroke slow. + */ +internal fun rankStrokeCandidates( + entries: List>, + pattern: String, + picks: List>, + limit: Int, + frequencyOf: (String) -> Long, +): List { + val result = LinkedHashSet() + picks.filter { it.second > 0.0 } + .sortedWith(compareByDescending> { it.second }.thenByDescending { frequencyOf(it.first) }) + .forEach { if (result.size < limit) result += it.first } + for ((char, code) in entries) { + if (result.size >= limit) break + if (char !in result && strokeCodeMatches(pattern, code)) result += char + } + return result.toList() +} + +/** See StrokeInputRepository.recordPick(): a correction always counts, a top pick only once it was learned. */ +internal fun shouldRecordStrokePick(wasTop: Boolean, alreadyLearned: Boolean): Boolean = !wasTop || alreadyLearned diff --git a/openless-all/app/android/kotlin/StrokeInputController.kt b/openless-all/app/android/kotlin/StrokeInputController.kt index 25f7fb564..c64955068 100644 --- a/openless-all/app/android/kotlin/StrokeInputController.kt +++ b/openless-all/app/android/kotlin/StrokeInputController.kt @@ -41,6 +41,7 @@ internal class StrokeInputController(private val service: OpenLessImeService) { private val wordSegments = mutableListOf() private var lastStrokeCandidates: List = emptyList() private var confirmedText = "" + private val associationLearning = TypedContinuationHistory() private var phraseQueryEpoch = 0L private var strokePreview: TextView? = null private var clearStrokeButton: TextView? = null @@ -79,6 +80,7 @@ internal class StrokeInputController(private val service: OpenLessImeService) { /** Matches selectInputMode()'s own old inline reset exactly — mode-switch also resets the number/symbol sub-panel and the punctuation rail's page, which resetForNewInputSession() below does not. */ fun resetForModeSwitch() { + associationLearning.reset() strokeNumberMode = false numberSymbolMode = false symbolPageIndex = 0 @@ -91,17 +93,39 @@ internal class StrokeInputController(private val service: OpenLessImeService) { /** Matches onStartInput()'s own old inline reset exactly — a fresh input session also drops any in-progress word segments/candidates, but leaves the punctuation rail's page alone. */ fun resetForNewInputSession() { + associationLearning.reset() strokeNumberMode = false numberSymbolMode = false symbolPageIndex = 0 confirmedText = "" phraseQueryEpoch++ strokeCode = "" + strokeQueryEpoch++ wordSegments.clear() lastStrokeCandidates = emptyList() + // The panel isn't rebuilt for a new session, so the views still show the old field's code and candidates. + strokePreview?.text = "" + strokeCandidates?.removeAllViews() updateClearStrokeButtonVisibility() } + /** + * The host emptied the field (e.g. a chat app's send button): start over + * — no association context, no half-typed code or 分词 segments, and an + * empty code row and candidate row. The candidate overlay entries are + * shared with the English panel, so they're only touched while this + * panel is the one showing. + */ + fun clearAfterFieldEmptied() { + associationLearning.reset() + if (service.inputMode == OpenLessImeService.InputMode.STROKE) { + abandonComposition() + } else { + confirmedText = "" + phraseQueryEpoch++ + } + } + fun buildStrokeView(): View { // Matches the punctuation rail's own 0.16f width share below (body's // "0.16f/0.65f/0.19f" split) so a downward drag anywhere on the rail @@ -234,7 +258,7 @@ internal class StrokeInputController(private val service: OpenLessImeService) { background = service.roundedButton(service.tone(Color.rgb(45, 45, 45), Color.rgb(230, 230, 234)), service.dp(4)) } punctuationGroups[punctuationGroupIndex].forEachIndexed { index, mark -> - punctuation.addView(service.keyboardKey(mark, 1f, action = { service.currentInputConnection?.commitText(mark, 1) }).apply { + punctuation.addView(service.keyboardKey(mark, 1f, action = { commitDirect(mark) }).apply { textSize = 18f // The rail is one connected key surface; separators provide the only visual split. background = GradientDrawable().apply { setColor(Color.TRANSPARENT) } @@ -266,9 +290,9 @@ internal class StrokeInputController(private val service: OpenLessImeService) { background = service.roundedButton(service.tone(Color.rgb(112, 78, 92), Color.rgb(232, 205, 213)), service.dp(5)) } } else if (code == "voice") service.keyboardKey("0", 1f, action = { - service.currentInputConnection?.commitText(" ", 1) + commitDirect(" ") }, swipeUpAction = { - service.currentInputConnection?.commitText("0", 1) + commitDirect("0") }, swipePreview = "0", microphoneIcon = true, longPressDelayMs = 900L, longPressAction = { service.inputMode = OpenLessImeService.InputMode.VOICE service.saveInputMode(service.inputMode) @@ -286,10 +310,10 @@ internal class StrokeInputController(private val service: OpenLessImeService) { service.refreshInputView() } " " -> segmentStroke() - else -> if (code in listOf("h", "s", "p", "n", "z", "*")) appendStroke(code) else service.currentInputConnection?.commitText(code, 1) + else -> if (code in listOf("h", "s", "p", "n", "z", "*")) appendStroke(code) else commitDirect(code) } }, swipeUpAction = swipeDigit?.let { digit -> - { service.currentInputConnection?.commitText(digit, 1) } + { commitDirect(digit) } }, swipePreview = swipeDigit, strokeIconCode = code.takeIf { it in listOf("h", "s", "p", "n", "z") }, graphicCode = code.takeIf { @@ -313,7 +337,7 @@ internal class StrokeInputController(private val service: OpenLessImeService) { body.addView(grid, LinearLayout.LayoutParams(0, ViewGroup.LayoutParams.MATCH_PARENT, 0.65f)) val actions = LinearLayout(service).apply { orientation = LinearLayout.VERTICAL; gravity = Gravity.CENTER } - listOf("←" to { deleteStroke() }, "↵" to { service.sendEnterKey() }, "清除" to { clearStrokes() }, "123" to { + listOf("←" to { deleteStroke() }, "↵" to { abandonComposition(); service.sendEnterKey() }, "清除" to { clearStrokes() }, "123" to { strokeNumberMode = true numberSymbolMode = false symbolPageIndex = 0 @@ -422,7 +446,7 @@ internal class StrokeInputController(private val service: OpenLessImeService) { symbolPageIndex = 0 service.refreshInputView() }) - else -> ({ service.currentInputConnection?.commitText(label, 1) }) + else -> ({ commitDirect(label) }) } row.addView(service.keyboardKey(label, 1f, action, repeatOnLongPress = rowIndex == 0 && isAction, repeatAction = action).apply { textSize = if (isAction) 15f else 20f @@ -563,10 +587,10 @@ internal class StrokeInputController(private val service: OpenLessImeService) { /** Commits the current character together with any segments already marked via 分词. */ private fun commitStrokeCandidate(candidate: String) { val code = strokeCode - val correction = candidate != lastStrokeCandidates.firstOrNull() - if (commitWord((wordSegments + candidate).joinToString("")) && correction && + val wasTop = candidate == lastStrokeCandidates.firstOrNull() + if (commitWord((wordSegments + candidate).joinToString("")) && service.personalizedLearningAllowed() && OpenLessAndroidPreferences.strokeUsageEnabled(service) && code.isNotEmpty()) { - strokeRepository.recordPersonalPick(code, candidate) + strokeRepository.recordPick(code, candidate, wasTop) } } @@ -576,7 +600,9 @@ internal class StrokeInputController(private val service: OpenLessImeService) { val contextBeforeCommit = confirmedText.takeLast(MAX_ASSOCIATION_CONTEXT) if (!connection.commitText(service.outputScript(word), 1)) return false if (service.personalizedLearningAllowed() && OpenLessAndroidPreferences.strokeUsageEnabled(service)) { - phraseRepository.recordUsage(contextBeforeCommit, word) + phraseRepository.recordTypedContinuation(associationLearning.advance(contextBeforeCommit, word)) + } else { + associationLearning.reset() } if (service.personalizedLearningAllowed()) confirmedText = (confirmedText + word).takeLast(MAX_ASSOCIATION_CONTEXT) clearStrokes() @@ -614,12 +640,14 @@ internal class StrokeInputController(private val service: OpenLessImeService) { if (service.personalizedLearningAllowed() && OpenLessAndroidPreferences.strokeUsageEnabled(service)) { phraseRepository.recordUsage(matchedContext, displayText) } + associationLearning.reset() confirmedText = (confirmedText + suffix).takeLast(MAX_ASSOCIATION_CONTEXT) clearStrokes() refreshAssociations() } private fun deleteStroke() { + associationLearning.reset() if (strokeCode.isNotEmpty()) { strokeCode = strokeCode.dropLast(1) strokeQueryEpoch++ @@ -656,6 +684,26 @@ internal class StrokeInputController(private val service: OpenLessImeService) { updateClearStrokeButtonVisibility() } + /** + * Something other than a candidate is about to reach the field (a mark, + * digit, space, Return): drop the half-typed code, its segments and the + * association context with them — what's on screen next no longer + * follows the last committed word, and a stale code row would sit there + * meaning nothing. Same as pinyin abandoning its encoding on punctuation. + */ + private fun abandonComposition() { + associationLearning.reset() + clearStrokes() + confirmedText = "" + phraseQueryEpoch++ + service.candidateOverlayEntries = emptyList() + } + + private fun commitDirect(text: String) { + abandonComposition() + service.currentInputConnection?.commitText(text, 1) + } + // internal, not private: RimePinyinController's own association context // buffer reuses the same cap for consistency (see phraseRepository's own // doc comment on why the two share one StrokePhraseRepository instance). diff --git a/openless-all/app/android/kotlin/StrokePhraseRepository.kt b/openless-all/app/android/kotlin/StrokePhraseRepository.kt index 7a90611f9..0a8bf3c32 100644 --- a/openless-all/app/android/kotlin/StrokePhraseRepository.kt +++ b/openless-all/app/android/kotlin/StrokePhraseRepository.kt @@ -47,9 +47,9 @@ internal class StrokePhraseRepository(context: Context) { executor.execute { val startedAt = android.os.SystemClock.elapsedRealtime() ensureLoaded() - val result = findLongestSuffix(prefix).sortedWith(compareByDescending { - it.baseWeight + userFrequency.score(it.matchedPrefix.ifEmpty { prefix }, it.text).toInt() - }) + val result = rankAssociations(findLongestSuffix(prefix)) { + userFrequency.score(it.matchedPrefix.ifEmpty { prefix }, it.text) + } // Temporary-diagnostic-grade, not removed after: cheap (one // log line per commit) and directly answers "is this slow // because of the cold trie build, the query itself, or is it @@ -70,6 +70,17 @@ internal class StrokePhraseRepository(context: Context) { executor.execute { userFrequency.record(context, candidate) } } + /** Records completed associations across one or more successful editor commits. */ + fun recordTypedContinuation(continuations: List) { + if (continuations.isEmpty()) return + executor.execute { + ensureLoaded() + continuations.mapNotNull { typedContinuation(it.contextBefore, it.word, ::find) } + .distinctBy { it.matchedPrefix to it.text } + .forEach { userFrequency.record(it.matchedPrefix, it.text) } + } + } + fun shutdown() = executor.shutdownNow() private fun ensureLoaded() { @@ -151,13 +162,7 @@ internal class StrokePhraseRepository(context: Context) { return result } - private fun findLongestSuffix(prefix: String): List { - for (length in prefix.length downTo 1) { - val result = find(prefix.takeLast(length)) - if (result.isNotEmpty()) return result - } - return emptyList() - } + private fun findLongestSuffix(prefix: String): List = longestSuffixExtensions(prefix, ::find) private companion object { const val NODE_TOP_N = 12 @@ -168,5 +173,78 @@ internal class StrokePhraseRepository(context: Context) { const val CACHE_SIZE = 256 } - private val userFrequency = StrokeUserFrequency(context) + private val userFrequency = StrokeUserFrequency.associations(context) +} + +/** + * Association candidates for [context]: the phrases under its longest + * trailing substring that [lookup] knows, minus the phrase that substring + * already completes. Every phrase sits in the trie node of its own last + * character too, so once a whole phrase is on screen the lookup returns it + * again — a candidate with nothing left to commit, which tapping can't + * clear. When that phrase is all there is, the result is empty rather than + * a retry with a shorter substring: the row should clear, not jump to + * phrases that only share the last character or two. + */ +internal fun longestSuffixExtensions( + context: String, + lookup: (String) -> List, +): List { + for (length in context.length downTo 1) { + val result = lookup(context.takeLast(length)) + if (result.isNotEmpty()) return result.filter { it.text != it.matchedPrefix } + } + return emptyList() +} + +/** + * Order for the association row: phrases the user has picked before (a + * positive [userScore]) first, by that score, then the rest by corpus + * weight. Corpus weights run from 1 to several hundred thousand while a + * user score stays in the hundreds, so adding the two — the old order — + * let a habit move only rare phrases; same reasoning as the stroke + * candidate order in StrokeInputRepository.searchAsync(). + */ +internal fun rankAssociations( + candidates: List, + userScore: (StrokePhraseRepository.Candidate) -> Double, +): List = candidates + .map { it to userScore(it) } + .sortedWith( + compareByDescending> { it.second } + .thenByDescending { it.first.baseWeight }, + ) + .map { it.first } + +/** The association offered for [contextBefore] that [word], typed right after it, completes exactly — or null. */ +internal fun typedContinuation( + contextBefore: String, + word: String, + lookup: (String) -> List, +): StrokePhraseRepository.Candidate? = + longestSuffixExtensions(contextBefore, lookup).firstOrNull { it.text == it.matchedPrefix + word } + +internal data class TypedContinuation(val contextBefore: String, val word: String) + +/** Main-thread commit boundaries; background lookups receive immutable snapshots. */ +internal class TypedContinuationHistory { + private var expectedContext: String? = null + private var continuations = emptyList() + + fun reset() { + expectedContext = null + continuations = emptyList() + } + + fun advance(contextBefore: String, word: String): List { + if (word.isEmpty()) return emptyList() + if (contextBefore != expectedContext) reset() + val starts = continuations + if (contextBefore.isNotEmpty()) { + listOf(TypedContinuation(contextBefore, "")) + } else emptyList() + continuations = starts.map { it.copy(word = it.word + word) } + .filter { it.word.length < StrokeInputController.MAX_ASSOCIATION_CONTEXT } + expectedContext = (contextBefore + word).takeLast(StrokeInputController.MAX_ASSOCIATION_CONTEXT) + return continuations + } } diff --git a/openless-all/app/android/kotlin/StrokeUserFrequency.kt b/openless-all/app/android/kotlin/StrokeUserFrequency.kt index 1cd3bef38..c5e257a72 100644 --- a/openless-all/app/android/kotlin/StrokeUserFrequency.kt +++ b/openless-all/app/android/kotlin/StrokeUserFrequency.kt @@ -7,10 +7,16 @@ import kotlin.math.ln /** * Local-only candidate preference store, shared globally across target apps * — which characters/phrases you mean for a given context or stroke code is - * a personal typing habit, not something that differs by app. + * a personal typing habit, not something that differs by app. Two separate + * stores, each with its own cap: [strokeCodes] (stroke code -> picked + * character) and [associations] (text before -> picked association), so + * heavy use of one never evicts the other. */ -internal class StrokeUserFrequency(context: Context) { - private val preferences = context.applicationContext.getSharedPreferences(STORE, Context.MODE_PRIVATE) +internal class StrokeUserFrequency private constructor(context: Context, store: String, private val maxEntries: Int) { + private val preferences = context.applicationContext.getSharedPreferences(store, Context.MODE_PRIVATE) + private val cache = FrequencyStampCache.of(context, store) { key -> + key.indexOf(SEPARATOR).takeIf { it >= 0 }?.let { key.substring(0, it) } + } // Tracks entry count without paying preferences.all's full-map-copy cost // on every record() — see trimIfNeeded(). -1 means "not yet known", @@ -19,31 +25,35 @@ internal class StrokeUserFrequency(context: Context) { private var approxSize = -1 fun score(context: String, candidate: String, now: Long = System.currentTimeMillis()): Double { - val value = preferences.getString(key(context, candidate), null) ?: return 0.0 - val parts = value.split(',') - val count = parts.getOrNull(0)?.toDoubleOrNull() ?: return 0.0 - val last = parts.getOrNull(1)?.toLongOrNull() ?: return 0.0 - val ageDays = ((now - last).coerceAtLeast(0L)).toDouble() / MILLIS_PER_DAY - return ln(1.0 + count) * COUNT_WEIGHT + exp(-ageDays / RECENCY_HALF_LIFE_DAYS) * RECENCY_WEIGHT + val stamp = cache.get(key(context, candidate)) ?: return 0.0 + val ageDays = ((now - stamp.lastUsedMs).coerceAtLeast(0L)).toDouble() / MILLIS_PER_DAY + return ln(1.0 + stamp.count) * COUNT_WEIGHT + exp(-ageDays / RECENCY_HALF_LIFE_DAYS) * RECENCY_WEIGHT } + /** Every candidate recorded for exactly [context], with its score — read from memory, no store scan. */ + fun picksFor(context: String, now: Long = System.currentTimeMillis()): List> = + cache.keysInGroup(context).map { key -> + val candidate = key.substring(context.length + SEPARATOR.length) + candidate to score(context, candidate, now) + } + fun record(context: String, candidate: String) { if (candidate.isBlank()) return OpenLessMigrationState.learn { val entryKey = key(context, candidate) val isNewEntry = !preferences.contains(entryKey) - val old = preferences.getString(entryKey, null)?.split(',') - val count = (old?.getOrNull(0)?.toIntOrNull() ?: 0) + 1 + val count = (cache.get(entryKey)?.count ?: 0) + 1 val now = System.currentTimeMillis() preferences.edit().putString(entryKey, "$count,$now").apply() + cache.put(entryKey, FrequencyStampCache.Stamp(count, now)) // Re-recording an existing entry (by far the common case — the same // handful of characters/phrases typed over and over) never touches // preferences.all at all now. Only a genuinely new key can push the - // count past MAX_ENTRIES, so that's the only case worth checking. + // count past the cap, so that's the only case worth checking. if (isNewEntry) { val size = resolvedSize() + 1 approxSize = size - if (size > MAX_ENTRIES) trimIfNeeded() + if (size > maxEntries) trimIfNeeded() } } } @@ -51,14 +61,15 @@ internal class StrokeUserFrequency(context: Context) { fun clear() { check(OpenLessMigrationState.settings { preferences.edit().clear().apply() + cache.invalidate() approxSize = 0 }) { "migration_android_busy" } } - /** Current entry count, for showing usage against [MAX_ENTRIES] in settings. */ + /** Current entry count, for showing usage against [capacity] in settings. */ fun size(): Int = resolvedSize() - fun capacity(): Int = MAX_ENTRIES + fun capacity(): Int = maxEntries /** Every stored entry as a plain "contextcandidate"->"count,lastUsedMs" map, for the settings page's export/import feature — see OpenLessSettingsExport. */ fun exportAll(): Map { @@ -70,10 +81,23 @@ internal class StrokeUserFrequency(context: Context) { fun importAll(entries: Map) { check(OpenLessMigrationState.settings { check(preferences.edit().clear().apply { entries.forEach { (key, value) -> putString(key, value) } }.commit()) { "Stroke learning save failed" } + cache.invalidate() approxSize = entries.size }) { "migration_android_busy" } } + /** Merges [entries] (already checked with [isValidEntry]) into this store: per entry the higher count and the later use win; past the cap the least recently used go. */ + fun mergeAll(entries: Map) { + check(OpenLessMigrationState.settings { + val current = exportAll() + check(preferences.edit().apply { + entries.forEach { (key, value) -> putString(key, current[key]?.let { mergeStamps(it, value) } ?: value) } + }.commit()) { "Stroke learning save failed" } + cache.invalidate() + trimIfNeeded() + }) { "migration_android_busy" } + } + private fun key(context: String, candidate: String) = "$context$SEPARATOR$candidate" private fun resolvedSize(): Int { @@ -84,11 +108,14 @@ internal class StrokeUserFrequency(context: Context) { return size } + // Past the cap, drops the least recently used down to TRIM_TO of it in one + // pass, so the full copy and sort run once per batch rather than for every + // new entry once the store is full. private fun trimIfNeeded() { val all = preferences.all approxSize = all.size - if (all.size <= MAX_ENTRIES) return - val removeCount = all.size - MAX_ENTRIES + if (all.size <= maxEntries) return + val removeCount = all.size - trimTarget(maxEntries) val oldest = all.entries .sortedBy { it.value.toString().substringAfterLast(',').toLongOrNull() ?: 0L } .take(removeCount) @@ -98,16 +125,81 @@ internal class StrokeUserFrequency(context: Context) { approxSize = all.size - oldest.size } - private companion object { - const val STORE = "openless_stroke_frequency" - const val SEPARATOR = "" - // Shared by both phrase-association contexts and stroke-code - // contexts; each entry is well under 100 bytes so even the full cap - // stays a few hundred KB, cheap to parse into memory on first use. - const val MAX_ENTRIES = 3000 - const val MILLIS_PER_DAY = 86_400_000L - const val RECENCY_HALF_LIFE_DAYS = 30.0 - const val COUNT_WEIGHT = 80.0 - const val RECENCY_WEIGHT = 24.0 + companion object { + private const val STROKE_STORE = "openless_stroke_frequency" + private const val ASSOCIATION_STORE = "openless_association_frequency" + private const val SEPARATOR = "" + // Each entry is well under 100 bytes, so even the larger cap is under a + // megabyte to parse on first use. Together they stay within the + // migration bundle's 20,000-entry limit for "strokeFrequency". + private const val MAX_STROKE_ENTRIES = 5000 + private const val MAX_ASSOCIATION_ENTRIES = 10_000 + private const val TRIM_TO = 0.9 + private const val MILLIS_PER_DAY = 86_400_000L + private const val RECENCY_HALF_LIFE_DAYS = 30.0 + private const val COUNT_WEIGHT = 80.0 + private const val RECENCY_WEIGHT = 24.0 + @Volatile private var split = false + + fun strokeCodes(context: Context) = StrokeUserFrequency(context, STROKE_STORE, MAX_STROKE_ENTRIES).also { splitSharedStore(context) } + + fun associations(context: Context) = StrokeUserFrequency(context, ASSOCIATION_STORE, MAX_ASSOCIATION_ENTRIES).also { splitSharedStore(context) } + + /** How many entries a store over [cap] keeps after trimming. */ + fun trimTarget(cap: Int): Int = (cap * TRIM_TO).toInt() + + /** A "contextcandidate" -> "count,lastUsedMs" entry as record() writes it. */ + fun isValidEntry(key: String, value: String): Boolean { + val separator = key.indexOf(SEPARATOR) + return separator > 0 && separator < key.length - 1 && key.toByteArray(Charsets.UTF_8).size <= 256 && + FrequencyStampCache.parseStamp(value)?.let { it.count > 0 && it.lastUsedMs >= 0 } == true + } + + /** Two records of one entry as one: the higher count and the later use. Both must be valid stamps. */ + fun mergeStamps(a: String, b: String): String { + val first = checkNotNull(FrequencyStampCache.parseStamp(a)) + val second = checkNotNull(FrequencyStampCache.parseStamp(b)) + return "${maxOf(first.count, second.count)},${maxOf(first.lastUsedMs, second.lastUsedMs)}" + } + + /** Stroke-code contexts are spelled with stroke keys only; association contexts are committed text. */ + fun isStrokeCodeKey(key: String): Boolean { + val separator = key.indexOf(SEPARATOR) + return separator > 0 && key.substring(0, separator).all { it in "hspnz*" } + } + + /** Both stores as one map — the shape backups and the migration bundle's "strokeFrequency" have always had. */ + fun exportCombined(context: Context): Map = + strokeCodes(context).exportAll() + associations(context).exportAll() + + /** Replaces both stores from a combined map, each entry going to the store its key belongs to. */ + fun importCombined(context: Context, entries: Map) { + val (codes, phrases) = entries.entries.partition { isStrokeCodeKey(it.key) } + strokeCodes(context).importAll(codes.associate { it.toPair() }) + associations(context).importAll(phrases.associate { it.toPair() }) + } + + // Until both kinds got their own store they shared the stroke one; + // move the association entries out once. + private fun splitSharedStore(context: Context) { + if (split) return + synchronized(this) { + if (split) return + split = true + val app = context.applicationContext + val marker = app.getSharedPreferences("openless_learning_layout", Context.MODE_PRIVATE) + if (marker.getBoolean("associations_split", false)) return + OpenLessMigrationState.learn { + val stroke = app.getSharedPreferences(STROKE_STORE, Context.MODE_PRIVATE) + val moved = stroke.all.filter { (key, value) -> value is String && !isStrokeCodeKey(key) } + if (moved.isNotEmpty()) { + val target = app.getSharedPreferences(ASSOCIATION_STORE, Context.MODE_PRIVATE) + check(target.edit().apply { moved.forEach { (key, value) -> putString(key, value as String) } }.commit()) + check(stroke.edit().apply { moved.keys.forEach { remove(it) } }.commit()) + } + marker.edit().putBoolean("associations_split", true).apply() + } + } + } } } diff --git a/openless-all/app/android/kotlin/androidTest/ImeAutoTextIntegrationTest.kt b/openless-all/app/android/kotlin/androidTest/ImeAutoTextIntegrationTest.kt new file mode 100644 index 000000000..e4d57a2d7 --- /dev/null +++ b/openless-all/app/android/kotlin/androidTest/ImeAutoTextIntegrationTest.kt @@ -0,0 +1,217 @@ +package com.openless.app + +import android.content.ContextWrapper +import android.text.Editable +import android.view.inputmethod.BaseInputConnection +import android.view.inputmethod.ExtractedText +import android.view.inputmethod.ExtractedTextRequest +import android.view.inputmethod.InputConnection +import android.view.inputmethod.InputConnectionWrapper +import android.widget.EditText +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import org.json.JSONObject +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.RunWith +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit + +@RunWith(AndroidJUnit4::class) +class ImeAutoTextIntegrationTest { + private val instrumentation = InstrumentationRegistry.getInstrumentation() + + @Test fun candidateSpaceDoesNotFollowCaretMoves() = instrumentation.runOnMainSync { + val (service, edit) = editor("old new ") + armCandidateSpace(service) + edit.setSelection(4) + service.onUpdateSelection(8, 8, 4, 4, -1, -1) + call(service, "commitEnglishChar", ",") + val changedUnrelatedSpace = edit.text.toString() == "old, new " + probe("P1", "H1", changedUnrelatedSpace) + assertFalse(changedUnrelatedSpace) + assertEquals("old ,new ", edit.text.toString()) + } + + @Test fun doubleSpaceDoesNotFollowCaretMoves() = instrumentation.runOnMainSync { + val (service, edit) = editor("one two ") + edit.setSelection(3) + call(service, "commitEnglishSpace") + edit.setSelection(edit.length()) + service.onUpdateSelection(4, 4, 9, 9, -1, -1) + call(service, "commitEnglishSpace") + val changedUnrelatedSpace = edit.text.toString().endsWith("two. ") + probe("P2", "H2", changedUnrelatedSpace) + assertFalse(changedUnrelatedSpace) + assertEquals("one two ", edit.text.toString()) + } + + @Test fun splitAssociationLearningRetainsTheOriginalPrefix() { + val lookup: (String) -> List = { prefix -> + listOf("天气", "天气预报").filter { it.startsWith(prefix) } + .map { StrokePhraseRepository.Candidate(it, 1, prefix) } + } + val history = TypedContinuationHistory() + assertEquals(emptyList(), history.advance("天气", "预").mapNotNull { + typedContinuation(it.contextBefore, it.word, lookup) + }) + val completed = history.advance("天气预", "报").mapNotNull { + typedContinuation(it.contextBefore, it.word, lookup) + } + val missesOriginalPrefix = completed.none { it.matchedPrefix == "天气" } + probe("P3", "H3", missesOriginalPrefix) + assertFalse(missesOriginalPrefix) + assertEquals(listOf("天气", "天气预"), completed.map { it.matchedPrefix }) + } + + @Test fun candidatePunctuationAndOwnSelectionUpdatesStillWork() = instrumentation.runOnMainSync { + val (service, edit) = editor("word ") + armCandidateSpace(service) + service.onUpdateSelection(4, 4, 5, 5, -1, -1) + call(service, "commitEnglishChar", "?") + service.onUpdateSelection(5, 5, 6, 6, -1, -1) + call(service, "commitEnglishChar", "!") + assertEquals("word?! ", edit.text.toString()) + } + + @Test fun consecutiveSpacesStillMakeOneFullStop() = instrumentation.runOnMainSync { + val (service, edit) = editor("word") + call(service, "commitEnglishSpace") + service.onUpdateSelection(4, 4, 5, 5, -1, -1) + call(service, "commitEnglishSpace") + assertEquals("word. ", edit.text.toString()) + call(service, "commitEnglishSpace") + assertEquals("word. ", edit.text.toString()) + } + + @Test fun selectedTextAndUnknownPositionsDoNotDeleteNeighboringSpaces() = instrumentation.runOnMainSync { + val (service, edit) = editor("a word ") + armCandidateSpace(service) + edit.setSelection(2, 6) + call(service, "commitEnglishChar", ",") + assertEquals("a , ", edit.text.toString()) + val (unknownService, unknownEdit) = editor("word ", reportsPosition = false) + armCandidateSpace(unknownService) + call(unknownService, "commitEnglishChar", ",") + assertEquals("word ,", unknownEdit.text.toString()) + call(unknownService, "commitEnglishSpace") + call(unknownService, "commitEnglishSpace") + assertEquals("word , ", unknownEdit.text.toString()) + } + + @Test fun replacingTheInputConnectionDropsTheOldCandidateSpace() = instrumentation.runOnMainSync { + val (service, edit) = editor("word ") + armCandidateSpace(service) + setField(service, "draftConnection", connection(edit, true)) + call(service, "commitEnglishChar", ",") + assertEquals("word ,", edit.text.toString()) + } + + @Test fun pinyinEditorBackspaceDropsDeletedAssociationContext() { + lateinit var controller: RimePinyinController + var controllerCreated = false + lateinit var edit: EditText + val deleted = CountDownLatch(1) + instrumentation.targetContext.getSharedPreferences("openless_ime_ui", android.content.Context.MODE_PRIVATE) + .edit().putBoolean("public_dictionary_updates", false).commit() + OpenLessRime.initialize(instrumentation.targetContext) + try { + instrumentation.runOnMainSync { + val (service, field) = editor("天气预") + edit = field + setField(service, "latinInputMode", OpenLessImeService.LatinInputMode.PINYIN) + controller = OpenLessImeService::class.java.getDeclaredMethod("getRimePinyinController") + .apply { isAccessible = true }.invoke(service) as RimePinyinController + controllerCreated = true + controller.setLearningAllowed(true) + controller.recordCommittedText("天气") + controller.associationLearning.advance("天气", "预") + controller.recordCommittedText("预") + setField(service, "draftConnection", object : InputConnectionWrapper(connection(edit, true), false) { + override fun deleteSurroundingText(beforeLength: Int, afterLength: Int): Boolean { + val result = super.deleteSurroundingText(beforeLength, afterLength) + deleted.countDown() + return result + } + }) + call(service, "englishDeleteBackward") + } + assertTrue("Empty-encoding Backspace must reach the editor", deleted.await(20, TimeUnit.SECONDS)) + instrumentation.runOnMainSync { + assertEquals("天气", edit.text.toString()) + assertEquals("", controller.associationContext()) + val lookup: (String) -> List = { prefix -> + listOf("天气预报").filter { it.startsWith(prefix) } + .map { StrokePhraseRepository.Candidate(it, 1, prefix) } + } + val learned = controller.associationLearning.advance(controller.associationContext(), "报") + .mapNotNull { typedContinuation(it.contextBefore, it.word, lookup) } + assertEquals(emptyList(), learned) + } + } finally { + instrumentation.runOnMainSync { if (controllerCreated) controller.shutdown() } + } + } + + private fun editor(text: String, reportsPosition: Boolean = true): Pair { + val service = OpenLessImeService() + ContextWrapper::class.java.getDeclaredMethod("attachBaseContext", android.content.Context::class.java) + .apply { isAccessible = true }.invoke(service, instrumentation.targetContext) + service.inputMode = OpenLessImeService.InputMode.ENGLISH + val edit = EditText(instrumentation.targetContext).apply { + setText(text) + setSelection(length()) + } + setField(service, "draftEdit", edit) + setField(service, "draftConnection", connection(edit, reportsPosition)) + return service to edit + } + + private fun connection(edit: EditText, reportsPosition: Boolean): InputConnection = object : BaseInputConnection(edit, true) { + override fun getEditable(): Editable = edit.text + override fun getExtractedText(request: ExtractedTextRequest?, flags: Int): ExtractedText? { + if (!reportsPosition) return null + return ExtractedText().apply { + text = edit.text.toString() + startOffset = 0 + selectionStart = edit.selectionStart + selectionEnd = edit.selectionEnd + } + } + } + + private fun armCandidateSpace(service: OpenLessImeService) { + val anchor = OpenLessImeService::class.java.getDeclaredMethod("englishEditAnchor", InputConnection::class.java) + .apply { isAccessible = true }.invoke(service, service.currentInputConnection) + setField(service, "englishAutoSpace", anchor) + } + + private fun setField(service: OpenLessImeService, name: String, value: Any?) { + OpenLessImeService::class.java.getDeclaredField(name).apply { isAccessible = true }.set(service, value) + } + + private fun call(service: OpenLessImeService, name: String, vararg args: String) { + OpenLessImeService::class.java.getDeclaredMethod(name, *args.map { String::class.java }.toTypedArray()) + .apply { isAccessible = true }.invoke(service, *args) + } + + private fun probe(id: String, hypothesis: String, observed: Boolean) { + val record = JSONObject().apply { + put("schema", "diagnosing-bugs.v1") + put("session_id", "a5809584") + put("run_id", "ime1230-fixed") + put("probe_id", id) + put("hypothesis_ids", org.json.JSONArray(listOf(hypothesis))) + put("event", "regression_boundary") + put("outcome", if (observed) "bug_observed" else "bug_absent") + put("safe", JSONObject().put("matched", observed)) + } + android.util.Log.i("Ime1230Evidence", record.toString()) + synchronized(ImeAutoTextIntegrationTest::class.java) { + java.io.File(instrumentation.targetContext.filesDir, "ime1230-a5809584.ndjson") + .appendText("$record\n") + } + } +} diff --git a/openless-all/app/android/kotlin/androidTest/ImeLearningMigrationTest.kt b/openless-all/app/android/kotlin/androidTest/ImeLearningMigrationTest.kt index f0223d974..c289f3a95 100644 --- a/openless-all/app/android/kotlin/androidTest/ImeLearningMigrationTest.kt +++ b/openless-all/app/android/kotlin/androidTest/ImeLearningMigrationTest.kt @@ -11,7 +11,7 @@ import org.junit.runner.RunWith class ImeLearningMigrationTest { @Test fun clearsOnlyLegacyAutomaticLearningAndRunsOnce() { val context = InstrumentationRegistry.getInstrumentation().targetContext - val stores = listOf("openless_english_frequency", "openless_english_custom_words", "openless_pinyin_frequency", "openless_pinyin_learned_phrases") + val stores = listOf("openless_english_frequency", "openless_english_custom_words") val marker = context.getSharedPreferences("openless_learning_privacy", Context.MODE_PRIVATE) assertTrue(marker.edit().clear().commit()) val unrelated = context.getSharedPreferences("openless_clipboard_history", Context.MODE_PRIVATE) @@ -35,11 +35,9 @@ class ImeLearningMigrationTest { assertTrue(result.applied.isEmpty()) assertTrue(result.errors.containsKey(OpenLessSettingsExport.Category.HAPTIC)) assertEquals(80, prefs.getInt("key_haptic_amplitude", 0)) - val legacy = OpenLessSettingsExport.import(context, - """{"openlessSettingsExportVersion":1,"pinyinLearnedPhrases":{"mm":"legacy,3"}}""", - setOf(OpenLessSettingsExport.Category.PINYIN_LEARNED_PHRASES)) - assertTrue(legacy.applied.isEmpty()) - assertTrue(legacy.errors.isNotEmpty()) + // LitePinyin backups are no longer offered for import. + assertTrue(OpenLessSettingsExport.categoriesPresent( + """{"openlessSettingsExportVersion":1,"pinyinLearnedPhrases":{"mm":"legacy,3"}}""").isEmpty()) } } diff --git a/openless-all/app/android/kotlin/androidTest/OpenLessMigrationBundleInstrumentedTest.kt b/openless-all/app/android/kotlin/androidTest/OpenLessMigrationBundleInstrumentedTest.kt index 5cf6cfa26..85c6e71c3 100644 --- a/openless-all/app/android/kotlin/androidTest/OpenLessMigrationBundleInstrumentedTest.kt +++ b/openless-all/app/android/kotlin/androidTest/OpenLessMigrationBundleInstrumentedTest.kt @@ -33,9 +33,8 @@ class OpenLessMigrationBundleInstrumentedTest { val next = JSONObject(before).apply { getJSONObject("preferences").put("locale", "zh-CN").put("themeMode", "dark") getJSONObject("strokeFrequency").put("new-word", "1,2") - getJSONObject("pinyinLearnedPhrases").put("xx", "new-word,3") }.toString() - for (stage in listOf("preferences", "strokeFrequency", "pinyinLearnedPhrases")) { + for (stage in listOf("preferences", "strokeFrequency")) { val lease = OpenLessMigrationState.acquire(JSONObject(OpenLessMigrationState.capture(context)).getString("revision")) try { val result = runCatching { @@ -61,7 +60,7 @@ class OpenLessMigrationBundleInstrumentedTest { val lease = OpenLessMigrationState.acquire(captured.getString("revision")) try { assertFalse(OpenLessMigrationState.settings { error("must not run") }) - StrokeUserFrequency(context).record("test", "word") + StrokeUserFrequency.associations(context).record("test", "word") assertEquals(0, JSONObject(OpenLessMigrationState.captureLocked(context, lease)).getJSONObject("strokeFrequency").length()) OpenLessMigrationState.replace(context, lease, captured.getJSONObject("value").toString()) } finally { OpenLessMigrationState.finish(lease, false) } @@ -100,26 +99,18 @@ class OpenLessMigrationBundleInstrumentedTest { } @Test - fun legacyPinyinBackupRoundTripsUnderTheMigrationFence() { + fun legacyPinyinEntriesAreAcceptedButNotRestored() { val context = isolatedContext() OpenLessMigrationState.initialize(context, false) - // Complete the privacy cleanup before seeding a compatible legacy backup. - OpenLessMigrationState.capture(context) - val backup = LegacyPinyinBackup(context) - backup.importAll(mapOf("before" to "旧词,2")) val before = JSONObject(OpenLessMigrationState.capture(context)) - assertEquals("旧词,2", before.getJSONObject("value").getJSONObject("pinyinLearnedPhrases").getString("before")) - val next = JSONObject(before.getJSONObject("value").toString()).apply { - put("pinyinLearnedPhrases", JSONObject().put("after", "新词,3")) + assertEquals(0, before.getJSONObject("value").getJSONObject("pinyinLearnedPhrases").length()) + val legacy = JSONObject(before.getJSONObject("value").toString()).apply { + put("pinyinLearnedPhrases", JSONObject().put("before", "旧词,2")) } val lease = OpenLessMigrationState.acquire(before.getString("revision")) try { - assertTrue(runCatching { backup.importAll(mapOf("unexpected" to "blocked,1")) }.isFailure) - assertEquals(mapOf("before" to "旧词,2"), backup.exportAll()) - OpenLessMigrationState.replace(context, lease, next.toString()) - assertEquals(mapOf("after" to "新词,3"), backup.exportAll()) - OpenLessMigrationState.replace(context, lease, before.getJSONObject("value").toString()) - assertEquals(mapOf("before" to "旧词,2"), backup.exportAll()) + OpenLessMigrationState.replace(context, lease, legacy.toString()) + assertEquals(0, JSONObject(OpenLessMigrationState.captureLocked(context, lease)).getJSONObject("pinyinLearnedPhrases").length()) } finally { OpenLessMigrationState.finish(lease, false) } } diff --git a/openless-all/app/android/kotlin/androidTest/RimeEngineIntegrationTest.kt b/openless-all/app/android/kotlin/androidTest/RimeEngineIntegrationTest.kt index c5a4999d5..c2d6383be 100644 --- a/openless-all/app/android/kotlin/androidTest/RimeEngineIntegrationTest.kt +++ b/openless-all/app/android/kotlin/androidTest/RimeEngineIntegrationTest.kt @@ -197,6 +197,97 @@ class RimeEngineIntegrationTest { } finally { OpenLessRime.request("destroy", privateSession) } } + @Test fun userDictionaryExportsAndMergesSnapshots() { + val context = InstrumentationRegistry.getInstrumentation().targetContext + OpenLessRime.initialize(context) + repeat(3) { + val session = OpenLessRime.request("create", "", "learning").getString("session") + try { + var state = typePinyin(session, "nihao") + state = choose(session, state, "拟") + state = choose(session, OpenLessRime.request("candidates", session), "蚝") + assertEquals("拟蚝", state.getString("commit")) + OpenLessRime.request("ack", session) + } finally { OpenLessRime.request("destroy", session) } + } + val open = OpenLessRime.request("create", "", "learning").getString("session") + val exported = java.io.File(context.cacheDir, "test-export-pinyin_simp.userdb.txt") + val incoming = java.io.File(context.cacheDir, "test-import-pinyin_simp.userdb.txt") + try { + assertTrue(OpenLessRime.exportUserDictionary(context, exported)) + // Exporting closes every session so the dictionary can be read. + try { + OpenLessRime.request("state", open) + fail("Sessions must be closed by an export") + } catch (_: IllegalStateException) {} + val text = exported.readText() + assertTrue(RimeUserDictionaryFile.entryCount(text) > 0) + assertTrue(text.lines().any { it.split('\t').getOrNull(1) == "拟蚝" }) + // Another device's snapshot holding a word this one never learned — + // on a spelling no other test types, since the dictionary is shared, + // and made of rare characters that sentence composition (which + // builds 剖剖 from the common 剖) never puts together on its own. + val header = text.lines().filter { it.startsWith("#") }.joinToString("\n") + fun importRow(row: String) { + incoming.writeText("$header\n$row\n") + OpenLessRime.importUserDictionary(context, incoming) + } + fun candidatesFor(input: String): List { + val session = OpenLessRime.request("create", "", "learning").getString("session") + try { + val rows = typePinyin(session, input).getJSONArray("candidates") + return (0 until rows.length()).map { rows.getString(it) } + } finally { OpenLessRime.request("destroy", session) } + } + assertFalse("No such word before the import", "抔掊" in candidatesFor("poupou")) + try { + importRow("pou pou \t抔掊\tc=9 d=9 t=1") + assertTrue("Imported words become candidates", "抔掊" in candidatesFor("poupou")) + assertTrue("Importing merges instead of replacing", "拟蚝" in candidatesFor("nihao")) + } finally { + // A larger negative count wins the merge and marks the word deleted. + importRow("pou pou \t抔掊\tc=-10 d=0 t=1") + } + assertFalse("The cleanup import deletes the word again", "抔掊" in candidatesFor("poupou")) + } finally { + exported.delete() + incoming.delete() + } + } + + @Test fun fuzzyPinyinPatchIsDeployedAndRemoved() { + val context = InstrumentationRegistry.getInstrumentation().targetContext + OpenLessRime.initialize(context) + fun candidatesFor(input: String): List { + val session = OpenLessRime.request("create", "", "private").getString("session") + try { + val rows = typePinyin(session, input).getJSONArray("candidates") + return (0 until rows.length()).map { rows.getString(it) } + } finally { OpenLessRime.request("destroy", session) } + } + assertFalse("知" in candidatesFor("zi")) + try { + OpenLessRime.applyFuzzyPinyin(context, setOf("zcs")) + assertTrue("z = zh after the patch", "知" in candidatesFor("zi")) + } finally { + OpenLessRime.applyFuzzyPinyin(context, emptySet()) + } + assertFalse(java.io.File(context.filesDir, "rime-user/openless_pinyin_private.custom.yaml").exists()) + assertFalse("知" in candidatesFor("zi")) + } + + @Test fun controllerStartsANewSessionAfterSessionsWereClosed() = withController { controller, committed -> + drain(controller) { for (letter in "ni") controller.appendLetter(letter) {} } + val context = InstrumentationRegistry.getInstrumentation().targetContext + OpenLessRime.exportUserDictionary(context, java.io.File(context.cacheDir, "test-reset-pinyin_simp.userdb.txt").apply { deleteOnExit() }) + // The half-typed "ni" went with the closed session; typing continues in a new one. + drain(controller) { + for (letter in "nihao") controller.appendLetter(letter) {} + controller.space {} + } + assertEquals("你好", committed.toString()) + } + private fun withController(test: (RimePinyinController, StringBuilder) -> Unit) { val instrumentation = InstrumentationRegistry.getInstrumentation() // Cold dictionary compilation is a one-time deployment, not input queue work. diff --git a/openless-all/app/android/kotlin/test/EnglishAutoTextTest.kt b/openless-all/app/android/kotlin/test/EnglishAutoTextTest.kt new file mode 100644 index 000000000..00fb1c283 --- /dev/null +++ b/openless-all/app/android/kotlin/test/EnglishAutoTextTest.kt @@ -0,0 +1,52 @@ +package com.openless.app + +import android.text.InputType +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +class EnglishAutoTextTest { + @Test fun sentenceStartsAtTheFieldLineOrAfterAFinishedSentence() { + assertTrue(EnglishAutoText.isSentenceStart("")) + assertTrue(EnglishAutoText.isSentenceStart(" ")) + assertTrue(EnglishAutoText.isSentenceStart("Hello there\n")) + assertTrue(EnglishAutoText.isSentenceStart("Done. ")) + assertTrue(EnglishAutoText.isSentenceStart("Really? ")) + assertTrue(EnglishAutoText.isSentenceStart("Wow! ")) + } + + @Test fun midSentenceIsNotASentenceStart() { + assertFalse(EnglishAutoText.isSentenceStart("Hello")) + assertFalse(EnglishAutoText.isSentenceStart("Hello ")) + assertFalse(EnglishAutoText.isSentenceStart("one, ")) + // The mark must be followed by whitespace: "3." or "e.g." mid-word isn't an end yet. + assertFalse(EnglishAutoText.isSentenceStart("Version 3.")) + } + + @Test fun capitalsOnlyInOrdinaryTextFields() { + val text = InputType.TYPE_CLASS_TEXT + assertTrue(EnglishAutoText.allowsAutoCapitals(text)) + assertTrue(EnglishAutoText.allowsAutoCapitals(text or InputType.TYPE_TEXT_FLAG_MULTI_LINE)) + assertTrue(EnglishAutoText.allowsAutoCapitals(text or InputType.TYPE_TEXT_VARIATION_SHORT_MESSAGE)) + assertFalse(EnglishAutoText.allowsAutoCapitals(text or InputType.TYPE_TEXT_VARIATION_EMAIL_ADDRESS)) + assertFalse(EnglishAutoText.allowsAutoCapitals(text or InputType.TYPE_TEXT_VARIATION_URI)) + assertFalse(EnglishAutoText.allowsAutoCapitals(text or InputType.TYPE_TEXT_VARIATION_PASSWORD)) + assertFalse(EnglishAutoText.allowsAutoCapitals(text or InputType.TYPE_TEXT_VARIATION_VISIBLE_PASSWORD)) + assertFalse(EnglishAutoText.allowsAutoCapitals(InputType.TYPE_CLASS_NUMBER)) + } + + @Test fun doubleSpaceFullStopOnlyRightAfterAWord() { + assertTrue(EnglishAutoText.doubleSpaceMakesFullStop("hello ")) + assertTrue(EnglishAutoText.doubleSpaceMakesFullStop("at 5 ")) + assertFalse(EnglishAutoText.doubleSpaceMakesFullStop(" ")) + assertFalse(EnglishAutoText.doubleSpaceMakesFullStop("hello")) + assertFalse(EnglishAutoText.doubleSpaceMakesFullStop("hello. ")) + assertFalse(EnglishAutoText.doubleSpaceMakesFullStop("hello ")) + assertFalse(EnglishAutoText.doubleSpaceMakesFullStop("hello\n ")) + } + + @Test fun onlySentencePunctuationSwallowsTheAutoSpace() { + listOf(".", ",", "!", "?", ";", ":").forEach { assertTrue(it, EnglishAutoText.swallowsAutoSpace(it)) } + listOf("(", "'", "-", "@", "a", "1").forEach { assertFalse(it, EnglishAutoText.swallowsAutoSpace(it)) } + } +} diff --git a/openless-all/app/android/kotlin/test/InputLearningRulesTest.kt b/openless-all/app/android/kotlin/test/InputLearningRulesTest.kt new file mode 100644 index 000000000..07cf276e4 --- /dev/null +++ b/openless-all/app/android/kotlin/test/InputLearningRulesTest.kt @@ -0,0 +1,70 @@ +package com.openless.app + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class InputLearningRulesTest { + /** The old query: every match, scored and fully sorted. */ + private fun naiveStrokeRank( + fileOrder: List>, + pattern: String, + scores: Map, + freq: Map, + limit: Int, + ): List = fileOrder + .filter { (_, code) -> strokeCodeMatches(pattern, code) } + .map { it.first } + .distinct() + .sortedWith(compareByDescending { scores[it] ?: 0.0 }.thenByDescending { freq[it] ?: 0L }) + .take(limit) + + @Test fun strokeRankingMatchesTheOldFullSortOnRandomData() { + val random = java.util.Random(42) + val chars = (0 until 60).map { "字$it" } + val freq = chars.associateWith { (random.nextInt(5) * 100).toLong() } // plenty of ties + val fileOrder = (0 until 400).map { + chars[random.nextInt(chars.size)] to (1..(1 + random.nextInt(5))).map { "hspnz"[random.nextInt(5)] }.joinToString("") + } + val sorted = fileOrder.sortedByDescending { freq[it.first] ?: 0L } + listOf("h", "s", "hs", "p*", "*z", "hsp", "nn", "zzzzzz").forEach { pattern -> + val matching = fileOrder.filter { strokeCodeMatches(pattern, it.second) }.map { it.first }.distinct() + // Distinct scores: real ones carry a millisecond timestamp, so two picks never tie. + val scores = matching.shuffled(random).take(3).withIndex().associate { (i, char) -> char to i * 50.0 + 1.0 } + listOf(1, 5, 36).forEach { limit -> + assertEquals( + "pattern=$pattern limit=$limit", + naiveStrokeRank(fileOrder, pattern, scores, freq, limit), + rankStrokeCandidates(sorted, pattern, scores.toList(), limit) { freq[it] ?: 0L }, + ) + } + } + } + + @Test fun aPickedCharacterLeadsEvenWithTheLowestFrequency() { + val entries = listOf("的" to "p", "了" to "z", "人" to "p", "仁" to "pp") + val freq = mapOf("的" to 1000L, "人" to 500L, "仁" to 1L) + assertEquals(listOf("仁", "的", "人"), rankStrokeCandidates(entries, "p", listOf("仁" to 79.0), 36) { freq[it] ?: 0L }) + assertEquals(listOf("仁"), rankStrokeCandidates(entries, "p", listOf("仁" to 79.0), 1) { freq[it] ?: 0L }) + assertEquals(listOf("的", "人", "仁"), rankStrokeCandidates(entries, "p", emptyList(), 36) { freq[it] ?: 0L }) + } + + @Test fun storedUsageParsesOnlyWellFormedValues() { + assertEquals(FrequencyStampCache.Stamp(3, 1700000000000L), FrequencyStampCache.parseStamp("3,1700000000000")) + assertNull(FrequencyStampCache.parseStamp(null)) + assertNull(FrequencyStampCache.parseStamp("")) + assertNull(FrequencyStampCache.parseStamp("3")) + assertNull(FrequencyStampCache.parseStamp(",17")) + assertNull(FrequencyStampCache.parseStamp("x,17")) + assertNull(FrequencyStampCache.parseStamp("3,abc")) + } + + @Test fun strokePickIsRecordedWhenCorrectedOrAlreadyLearned() { + assertTrue(shouldRecordStrokePick(wasTop = false, alreadyLearned = false)) + assertTrue(shouldRecordStrokePick(wasTop = false, alreadyLearned = true)) + assertTrue(shouldRecordStrokePick(wasTop = true, alreadyLearned = true)) + assertFalse(shouldRecordStrokePick(wasTop = true, alreadyLearned = false)) + } +} diff --git a/openless-all/app/android/kotlin/test/RimePinyinDataTest.kt b/openless-all/app/android/kotlin/test/RimePinyinDataTest.kt new file mode 100644 index 000000000..3cb06e645 --- /dev/null +++ b/openless-all/app/android/kotlin/test/RimePinyinDataTest.kt @@ -0,0 +1,129 @@ +package com.openless.app + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertThrows +import org.junit.Assert.assertTrue +import org.junit.Test + +class RimePinyinDataTest { + private val sep = "\u0001" + + @Test + fun fuzzyPatchAppendsTheSelectedRulesInGroupOrder() { + assertNull(RimeFuzzyPinyin.customYaml(emptySet())) + assertNull(RimeFuzzyPinyin.customYaml(setOf("unknown"))) + val yaml = RimeFuzzyPinyin.customYaml(setOf("ing", "zcs"))!! + assertEquals( + listOf( + "patch:", + " \"speller/algebra/+\":", + " - derive/^([zcs])h/\$1/", + " - derive/^([zcs])([^h])/\$1h\$2/", + " - derive/ing\$/in/", + " - derive/in\$/ing/", + ), + yaml.lines().filter { it.isNotEmpty() && !it.startsWith("#") }, + ) + } + + @Test + fun everyFuzzyGroupIsAPairOfDeriveRules() { + assertEquals(RimeFuzzyPinyin.groups.size, RimeFuzzyPinyin.groups.map { it.id }.toSet().size) + for (group in RimeFuzzyPinyin.groups) { + assertEquals(2, group.rules.size) + assertTrue(group.rules.all { it.startsWith("derive/") && it.count { c -> c == '/' } == 3 }) + } + } + + private fun snapshot(name: String = "pinyin_simp", type: String = "userdb", vararg rows: String) = buildString { + append("# Rime user dictionary\n") + append("#@/db_name\t$name\n") + append("#@/db_type\t$type\n") + append("#@/tick\t42\n") + rows.forEach { append(it).append('\n') } + } + + @Test + fun userDictionaryCountsLearnedRowsOnly() { + val text = snapshot( + rows = arrayOf("ni hao \t你好\tc=3 d=1.2 t=40", "shan chu \t删除\tc=-1 d=0.5 t=30", "zao ci \t造词\tc=1 d=0.9 t=41"), + ) + assertEquals(2, RimeUserDictionaryFile.entryCount(text)) + assertEquals(1, RimeUserDictionaryFile.entryCount(text.replace("\n", "\r\n").replace("c=1 ", "c=0 "))) + } + + @Test + fun userDictionaryRejectsOtherDictionariesAndMalformedRows() { + assertEquals(0, RimeUserDictionaryFile.entryCount(snapshot())) + assertThrows(IllegalArgumentException::class.java) { RimeUserDictionaryFile.entryCount(snapshot(name = "luna_pinyin")) } + assertThrows(IllegalArgumentException::class.java) { RimeUserDictionaryFile.entryCount(snapshot(type = "tabledb")) } + assertThrows(IllegalArgumentException::class.java) { RimeUserDictionaryFile.entryCount("ni hao \t你好\tc=1\n") } + assertThrows(IllegalArgumentException::class.java) { RimeUserDictionaryFile.entryCount(snapshot(rows = arrayOf("just text"))) } + assertThrows(IllegalArgumentException::class.java) { RimeUserDictionaryFile.entryCount("{\"openlessSettingsExportVersion\":2}") } + } + + @Test + fun learningEntriesAreCheckedAndMergedByHigherCountAndLaterUse() { + assertTrue(StrokeUserFrequency.isValidEntry("hsp${sep}王", "3,1700000000000")) + assertFalse(StrokeUserFrequency.isValidEntry("hsp${sep}王", "0,1700000000000")) + assertFalse(StrokeUserFrequency.isValidEntry("hsp${sep}王", "3")) + assertFalse(StrokeUserFrequency.isValidEntry("hsp${sep}", "3,1")) + assertFalse(StrokeUserFrequency.isValidEntry("hsp王", "3,1")) + assertFalse(StrokeUserFrequency.isValidEntry("天".repeat(90) + "${sep}气", "3,1")) + assertEquals("5,200", StrokeUserFrequency.mergeStamps("5,100", "2,200")) + assertEquals("7,300", StrokeUserFrequency.mergeStamps("1,300", "7,50")) + } + + @Test + fun aFullStoreIsTrimmedInOneBatch() { + assertEquals(4500, StrokeUserFrequency.trimTarget(5000)) + assertEquals(9000, StrokeUserFrequency.trimTarget(10_000)) + } + + @Test + fun keyboardSettingsAcceptOnlyKnownKeysWithValidValues() { + assertTrue(KeyboardDataFile.acceptsSetting("keyboardHeightDp", 300)) + assertFalse(KeyboardDataFile.acceptsSetting("keyboardHeightDp", 900)) + assertFalse(KeyboardDataFile.acceptsSetting("keyboardHeightDp", "300")) + assertTrue(KeyboardDataFile.acceptsSetting("keyboardRaiseDp", 0)) + assertTrue(KeyboardDataFile.acceptsSetting("hapticAmplitude", 255)) + assertFalse(KeyboardDataFile.acceptsSetting("hapticAmplitude", 0)) + assertTrue(KeyboardDataFile.acceptsSetting("hapticDurationMs", 12)) + assertFalse(KeyboardDataFile.acceptsSetting("hapticDurationMs", 5000)) + assertTrue(KeyboardDataFile.acceptsSetting("cloudPinyinEnabled", false)) + assertFalse(KeyboardDataFile.acceptsSetting("cloudPinyinEnabled", 1)) + assertTrue(KeyboardDataFile.acceptsSetting("fuzzyPinyin", listOf("zcs", "ing"))) + assertTrue(KeyboardDataFile.acceptsSetting("fuzzyPinyin", emptyList())) + assertFalse(KeyboardDataFile.acceptsSetting("fuzzyPinyin", listOf("zcs", "bogus"))) + assertFalse(KeyboardDataFile.acceptsSetting("locale", "zh-CN")) + assertFalse(KeyboardDataFile.acceptsSetting("key_cloud_note_webhook_token", "secret")) + } + + @Test + fun webhookAddressesMustBeBlankOrHttp() { + assertTrue(KeyboardDataFile.acceptsWebhookUrl("")) + assertTrue(KeyboardDataFile.acceptsWebhookUrl("https://example.com/capture_ingest.php")) + assertFalse(KeyboardDataFile.acceptsWebhookUrl("file:///sdcard/notes")) + assertFalse(KeyboardDataFile.acceptsWebhookUrl("javascript:alert(1)")) + assertFalse(KeyboardDataFile.acceptsWebhookUrl("not a url")) + } + + @Test + fun everythingIsExportedButTheTokenIsNotImportedUnlessChosen() { + assertEquals(KeyboardDataFile.Section.entries.toSet(), KeyboardDataFile.defaultExport) + assertFalse(KeyboardDataFile.Section.CLOUD_NOTES in KeyboardDataFile.defaultImport) + assertEquals(KeyboardDataFile.Section.entries.size - 1, KeyboardDataFile.defaultImport.size) + } + + @Test + fun strokeCodeKeysAreToldApartFromAssociationKeys() { + assertTrue(StrokeUserFrequency.isStrokeCodeKey("hsp${sep}王")) + assertTrue(StrokeUserFrequency.isStrokeCodeKey("h*z${sep}过")) + assertFalse(StrokeUserFrequency.isStrokeCodeKey("天气${sep}天气预报")) + assertFalse(StrokeUserFrequency.isStrokeCodeKey("test${sep}word")) + assertFalse(StrokeUserFrequency.isStrokeCodeKey("${sep}空上下文")) + assertFalse(StrokeUserFrequency.isStrokeCodeKey("hsp")) + } +} diff --git a/openless-all/app/android/kotlin/test/StrokePhraseSuffixTest.kt b/openless-all/app/android/kotlin/test/StrokePhraseSuffixTest.kt new file mode 100644 index 000000000..8fc5e9dcd --- /dev/null +++ b/openless-all/app/android/kotlin/test/StrokePhraseSuffixTest.kt @@ -0,0 +1,121 @@ +package com.openless.app + +import com.openless.app.StrokePhraseRepository.Candidate +import org.junit.Assert.assertEquals +import org.junit.Test + +class StrokePhraseSuffixTest { + /** A trie stand-in: every phrase is listed under each of its prefixes, itself included. */ + private fun lookupOf(vararg phrases: String): (String) -> List = { prefix -> + phrases.filter { it.startsWith(prefix) }.map { Candidate(it, 1, prefix) } + } + + private fun texts(context: String, lookup: (String) -> List) = + longestSuffixExtensions(context, lookup).map { it.text } + + @Test fun phrasesContinuingTheLongestSuffixAreOffered() { + assertEquals(listOf("天气好", "天气预报"), texts("今天天气", lookupOf("天气好", "天气预报", "气球"))) + } + + @Test fun thePhraseTheSuffixAlreadyCompletesIsDropped() { + assertEquals(listOf("天气预报"), texts("今天天气", lookupOf("天气", "天气预报"))) + } + + /** The reported bug: after the last association was picked, it stayed in the row. */ + @Test fun nothingIsOfferedOnceTheLastPhraseIsOnScreen() { + assertEquals(emptyList(), texts("今天天气预报", lookupOf("天气预报", "预报员", "报告"))) + } + + @Test fun aShorterSuffixIsTriedOnlyWhenTheLongerOneMatchesNothing() { + assertEquals(listOf("好的"), texts("你好", lookupOf("好的"))) + } + + @Test fun emptyContextOrNoMatchGivesNothing() { + assertEquals(emptyList(), texts("", lookupOf("天气"))) + assertEquals(emptyList(), texts("你好", lookupOf("天气"))) + } + + private fun ranked(weights: Map, scores: Map) = + rankAssociations(weights.map { (text, weight) -> Candidate(text, weight, "天气") }) { scores[it.text] ?: 0.0 } + .map { it.text } + + /** The old additive order could never lift 天气图 (weight 7) over 天气预报 (2054), however often it was picked. */ + @Test fun aPickedPhraseOutranksAnUnpickedOneWhateverTheCorpusWeight() { + val weights = mapOf("天气预报" to 2054, "天气形势" to 12, "天气图" to 7) + assertEquals(listOf("天气图", "天气预报", "天气形势"), ranked(weights, mapOf("天气图" to 79.0))) + } + + @Test fun pickedPhrasesFollowTheirScoreAndTheRestTheirWeight() { + val weights = mapOf("天气预报" to 2054, "天气形势" to 12, "天气图" to 7, "天气变化" to 1) + assertEquals( + listOf("天气变化", "天气图", "天气预报", "天气形势"), + ranked(weights, mapOf("天气图" to 79.0, "天气变化" to 216.0)), + ) + assertEquals(listOf("天气预报", "天气形势", "天气图", "天气变化"), ranked(weights, emptyMap())) + } + + @Test fun typingOutAnOfferedAssociationCountsAsPickingIt() { + val lookup = lookupOf("天气", "天气预报", "天气好") + val hit = typedContinuation("今天天气", "预报", lookup) + assertEquals("天气预报", hit?.text) + assertEquals("天气", hit?.matchedPrefix) + } + + @Test fun splitCommitsLearnTheOriginalAssociationPrefix() { + val lookup = lookupOf("天气", "天气预报") + val history = TypedContinuationHistory() + assertEquals(emptyList(), history.advance("天气", "预").mapNotNull { + typedContinuation(it.contextBefore, it.word, lookup) + }) + val completed = history.advance("天气预", "报").mapNotNull { + typedContinuation(it.contextBefore, it.word, lookup) + } + assertEquals(listOf("天气", "天气预"), completed.map { it.matchedPrefix }) + assertEquals(listOf("天气预报", "天气预报"), completed.map { it.text }) + } + + @Test fun batchCommitsStillLearnAndUnrelatedTextDoesNot() { + val lookup = lookupOf("天气", "天气预报") + val history = TypedContinuationHistory() + val batch = history.advance("天气", "预报").mapNotNull { + typedContinuation(it.contextBefore, it.word, lookup) + } + assertEquals(listOf("天气"), batch.map { it.matchedPrefix }) + assertEquals(emptyList(), history.advance("天气预报", "雨").mapNotNull { + typedContinuation(it.contextBefore, it.word, lookup) + }) + } + + @Test fun resetAndContextChangesDiscardPendingContinuations() { + val history = TypedContinuationHistory() + history.advance("天气", "预") + history.reset() + assertEquals(listOf(TypedContinuation("天气预", "报")), history.advance("天气预", "报")) + history.advance("天气", "预") + assertEquals(listOf(TypedContinuation("别处", "报")), history.advance("别处", "报")) + } + + @Test fun historyIsBoundedAndDoesNotInventCommitBoundaries() { + val history = TypedContinuationHistory() + assertEquals(emptyList(), history.advance("", "天气")) + assertEquals(listOf(TypedContinuation("天气", "预报")), history.advance("天气", "预报")) + var context = "天气预报" + repeat(20) { + val continuations = history.advance(context, "字") + assertEquals(true, continuations.size <= 7) + assertEquals(true, continuations.all { it.word.length < 8 }) + context = (context + "字").takeLast(8) + } + } + + @Test fun typingSomethingElseCountsAsNothing() { + val lookup = lookupOf("天气", "天气预报") + // Only part of the association, more than it, or unrelated text. + assertEquals(null, typedContinuation("今天天气", "预", lookup)) + assertEquals(null, typedContinuation("今天天气", "预报员", lookup)) + assertEquals(null, typedContinuation("今天天气", "很好", lookup)) + // The phrase that is already complete is never offered, so it can't be "typed out". + assertEquals(null, typedContinuation("今天", "天气", lookupOf("今天"))) + assertEquals(null, typedContinuation("", "天气", lookup)) + } +} diff --git a/openless-all/app/mobile-ime/README.md b/openless-all/app/mobile-ime/README.md index ce45fc01c..1a7728b3e 100644 --- a/openless-all/app/mobile-ime/README.md +++ b/openless-all/app/mobile-ime/README.md @@ -29,7 +29,9 @@ Android 的空格、删除、回车与字母在同一个串行队列中执行, Android 继续遵守 `IME_FLAG_NO_PERSONALIZED_LEARNING` 和密码字段策略;不学习的会话使用独立 `openless_pinyin_private` 方案,既不训练也不读取个人词典。鸿蒙密码框不创建中文会话;普通字段的本地学习在获取系统输入框属性后才启用,自检始终使用私密会话。共享资源目录升级为 `rime-shared-1.17.0-ice-da1fbe60-v1`,避免已安装应用因旧资源缓存而漏装学习方案;个人词典目录保持原位。 -旧 Android LitePinyin 算法及内置 TSV 已删除;旧备份格式只保留一个不参与输入的兼容适配器。 +旧 Android LitePinyin 算法、内置 TSV 及其备份适配器均已删除;迁移文件的 `pinyinLearnedPhrases` 字段为保持格式兼容仍保留,但始终为空,导入时忽略。 + +用户词典与模糊音:`user_dict_export` / `user_dict_import` 通过 librime levers 的 `backup_user_dict` / `restore_user_dict` 读写 `pinyin_simp` 的文本快照(`.userdb.txt`,导入为合并:同一词条取较大的使用次数与权重);`redeploy` 在 `rime-user/<方案>.custom.yaml` 改动后重新部署两个拼音方案。三者都会先销毁全部会话(提交用户词典事务并释放数据库锁),调用方需重建会话。Android 在键盘设置中提供导出 / 导入与模糊音开关,不进入迁移文件或加密同步。`rime_levers_api.h` 原样复制自 librime 1.17.0,因其安装步骤只导出 `rime_api.h`。 可评估的上游扩展(当前未启用):[librime-predict](https://github.com/rime/librime-predict) 提供下一个词预测,需额外预测数据;[librime-lua](https://github.com/hchunhui/librime-lua) 用于复用成熟方案脚本;[librime-octagram](https://github.com/lotem/librime-octagram) 及其模型可用于语法加权。扩展与数据需分别验证 OHOS / Android 构建、版本兼容及安装体积,不自行实现平行算法。当前个人学习直接使用 librime 原生用户词典。 diff --git a/openless-all/app/mobile-ime/rime_engine.cpp b/openless-all/app/mobile-ime/rime_engine.cpp index 91ad0b013..5f4ff6b7d 100644 --- a/openless-all/app/mobile-ime/rime_engine.cpp +++ b/openless-all/app/mobile-ime/rime_engine.cpp @@ -1,5 +1,6 @@ #include "rime_engine.h" #include +#include "rime_levers_api.h" #include #include #include @@ -9,7 +10,13 @@ #include #include +// librime is linked statically: referencing the module keeps its registration +// (and the user dictionary manager behind it) from being dropped by the linker. +void rime_require_module_levers(); // C++ linkage, as RIME_REGISTER_MODULE defines it + namespace { +const char* const kPinyinSchemas[] = {"openless_pinyin_private", "openless_pinyin_learning"}; +const char* const kUserDict = "pinyin_simp"; std::mutex mutex; RimeApi* api = nullptr; std::string shared_dir, user_dir; @@ -68,6 +75,24 @@ std::string snapshot(RimeSessionId id, bool expanded = false, int offset = 0, in api->free_context(&context); return out; } +void deploy_pinyin_schemas(RimeApi* target) { + for (const auto* name : kPinyinSchemas) { + const auto schema = shared_dir + "/" + name + ".schema.yaml"; + if (!target->deploy_schema(schema.c_str())) throw std::runtime_error("Rime schema deployment failed"); + } +} +// Destroying a session commits its user dictionary transaction and releases +// the database lock, which the user dictionary manager needs to open it. +void destroy_all_sessions() { + for (const auto& entry : sessions) api->destroy_session(entry.first); + sessions.clear(); +} +RimeLeversApi* levers() { + auto* module = api->find_module("levers"); + auto* levers = module && module->get_api ? reinterpret_cast(module->get_api()) : nullptr; + if (!levers) throw std::runtime_error("Rime user dictionary tools unavailable"); + return levers; +} std::string request(const std::string& op, const std::string& handle, const std::string& arg) { if (op == "initialize") { if (api) { @@ -85,17 +110,49 @@ std::string request(const std::string& op, const std::string& handle, const std: candidate->setup(&traits); candidate->initialize(&traits); if (candidate->start_maintenance(True)) candidate->join_maintenance_thread(); - for (const auto* name : {"openless_pinyin_private", "openless_pinyin_learning"}) { - const auto schema = shared_dir + "/" + name + ".schema.yaml"; - if (!candidate->deploy_schema(schema.c_str())) { - candidate->finalize(); - throw std::runtime_error("Rime schema deployment failed"); - } - } + try { deploy_pinyin_schemas(candidate); } + catch (...) { candidate->finalize(); throw; } + // On a fresh profile the first installation_update only writes + // installation.yaml and leaves the sync dir as the relative "sync" + // (the read-only process cwd on Android). A second run reads that file + // and points it into the user data dir, where dictionary backups go. + candidate->run_task("installation_update"); api = candidate; + rime_require_module_levers(); return "{\"ok\":true}"; } if (!api) throw std::runtime_error("Rime is not initialized"); + if (op == "redeploy") { + // Rebuild after a schema .custom.yaml changed; deleting the compiled + // schema keeps a same-second rewrite from looking up to date. + destroy_all_sessions(); + for (const auto* name : kPinyinSchemas) { + std::error_code ignored; + std::filesystem::remove(user_dir + "/build/" + name + ".schema.yaml", ignored); + } + deploy_pinyin_schemas(api); + return "{\"ok\":true}"; + } + if (op == "user_dict_export") { + if (arg.empty()) throw std::runtime_error("Export path required"); + destroy_all_sessions(); + if (!std::filesystem::exists(user_dir + "/" + kUserDict + ".userdb")) + return "{\"ok\":true,\"exported\":false}"; + char sync_dir[1024] = {0}; + api->get_user_data_sync_dir(sync_dir, sizeof(sync_dir)); + if (!std::filesystem::path(sync_dir).is_absolute()) throw std::runtime_error("Rime sync directory unavailable"); + if (!levers()->backup_user_dict(kUserDict)) throw std::runtime_error("Rime user dictionary backup failed"); + std::filesystem::copy_file(std::string(sync_dir) + "/" + kUserDict + ".userdb.txt", arg, + std::filesystem::copy_options::overwrite_existing); + return "{\"ok\":true,\"exported\":true}"; + } + if (op == "user_dict_import") { + if (arg.empty()) throw std::runtime_error("Import path required"); + destroy_all_sessions(); + // Merges like Rime's own sync: the larger commit count and weight win. + if (!levers()->restore_user_dict(arg.c_str())) throw std::runtime_error("Rime user dictionary import failed"); + return "{\"ok\":true}"; + } if (op == "create") { auto id = api->create_session(); if (!id) throw std::runtime_error("Rime session creation failed"); diff --git a/openless-all/app/mobile-ime/rime_levers_api.h b/openless-all/app/mobile-ime/rime_levers_api.h new file mode 100644 index 000000000..eef264e6e --- /dev/null +++ b/openless-all/app/mobile-ime/rime_levers_api.h @@ -0,0 +1,96 @@ +// Copied unchanged from librime 1.17.0 (src/rime_levers_api.h): its install step +// only ships rime_api.h, and the user dictionary tools are declared here. +/* + * Copyright RIME Developers + * Distributed under the BSD License + * + * 2013-10-21 GONG Chen + */ +#ifndef RIME_LEVERS_API_H_ +#define RIME_LEVERS_API_H_ + +#include "rime_api.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + char placeholder; +} RimeCustomSettings; + +typedef struct { + char placeholder; +} RimeSwitcherSettings; + +typedef struct { + char placeholder; +} RimeSchemaInfo; + +typedef struct { + void* ptr; + size_t i; +} RimeUserDictIterator; + +typedef struct RIME_FLAVORED(rime_levers_api_t) { + int data_size; + + RimeCustomSettings* (*custom_settings_init)(const char* config_id, + const char* generator_id); + void (*custom_settings_destroy)(RimeCustomSettings* settings); + Bool (*load_settings)(RimeCustomSettings* settings); + Bool (*save_settings)(RimeCustomSettings* settings); + Bool (*customize_bool)(RimeCustomSettings* settings, + const char* key, + Bool value); + Bool (*customize_int)(RimeCustomSettings* settings, + const char* key, + int value); + Bool (*customize_double)(RimeCustomSettings* settings, + const char* key, + double value); + Bool (*customize_string)(RimeCustomSettings* settings, + const char* key, + const char* value); + Bool (*is_first_run)(RimeCustomSettings* settings); + Bool (*settings_is_modified)(RimeCustomSettings* settings); + Bool (*settings_get_config)(RimeCustomSettings* settings, RimeConfig* config); + + RimeSwitcherSettings* (*switcher_settings_init)(); + Bool (*get_available_schema_list)(RimeSwitcherSettings* settings, + RimeSchemaList* list); + Bool (*get_selected_schema_list)(RimeSwitcherSettings* settings, + RimeSchemaList* list); + void (*schema_list_destroy)(RimeSchemaList* list); + const char* (*get_schema_id)(RimeSchemaInfo* info); + const char* (*get_schema_name)(RimeSchemaInfo* info); + const char* (*get_schema_version)(RimeSchemaInfo* info); + const char* (*get_schema_author)(RimeSchemaInfo* info); + const char* (*get_schema_description)(RimeSchemaInfo* info); + const char* (*get_schema_file_path)(RimeSchemaInfo* info); + Bool (*select_schemas)(RimeSwitcherSettings* settings, + const char* schema_id_list[], + int count); + const char* (*get_hotkeys)(RimeSwitcherSettings* settings); + Bool (*set_hotkeys)(RimeSwitcherSettings* settings, const char* hotkeys); + + Bool (*user_dict_iterator_init)(RimeUserDictIterator* iter); + void (*user_dict_iterator_destroy)(RimeUserDictIterator* iter); + const char* (*next_user_dict)(RimeUserDictIterator* iter); + Bool (*backup_user_dict)(const char* dict_name); + Bool (*restore_user_dict)(const char* snapshot_file); + int (*export_user_dict)(const char* dict_name, const char* text_file); + int (*import_user_dict)(const char* dict_name, const char* text_file); + + // patch a list or a map + Bool (*customize_item)(RimeCustomSettings* settings, + const char* key, + RimeConfig* value); + +} RIME_FLAVORED(RimeLeversApi); + +#ifdef __cplusplus +} +#endif + +#endif // RIME_LEVERS_API_H_ diff --git a/openless-all/app/scripts/copy-android-scaffolding.mjs b/openless-all/app/scripts/copy-android-scaffolding.mjs index bfe5677c8..ad8a488bf 100644 --- a/openless-all/app/scripts/copy-android-scaffolding.mjs +++ b/openless-all/app/scripts/copy-android-scaffolding.mjs @@ -71,12 +71,16 @@ const KOTLIN_FILES = [ 'StrokeInput.kt', 'OpenLessRime.kt', 'RimePinyinController.kt', - 'LegacyPinyinBackup.kt', + 'RimeFuzzyPinyin.kt', + 'RimeUserDictionaryFile.kt', + 'KeyboardDataFile.kt', 'OpenLessSettingsExport.kt', 'StrokePhraseRepository.kt', 'StrokeUserFrequency.kt', + 'FrequencyStampCache.kt', 'EnglishCandidateProvider.kt', 'EnglishUserFrequency.kt', + 'EnglishAutoText.kt', 'OpenLessRuntimeService.kt', 'OpenLessBackendWarmupActivity.kt', 'OpenLessClipboardHistory.kt', @@ -93,6 +97,10 @@ const KOTLIN_TEST_FILES = [ 'DictationSpanTest.kt', 'ImePrivacyPolicyTest.kt', 'ImeDraftReplacementTest.kt', + 'StrokePhraseSuffixTest.kt', + 'EnglishAutoTextTest.kt', + 'InputLearningRulesTest.kt', + 'RimePinyinDataTest.kt', 'LatinInputModeTest.kt', 'OpenLessContentReaderTest.kt', 'OpenLessMigrationBundleContractTest.kt', @@ -111,6 +119,7 @@ const KOTLIN_ANDROID_TEST_FILES = [ 'ImeLearningMigrationTest.kt', 'OpenLessMigrationBundleInstrumentedTest.kt', 'RimeEngineIntegrationTest.kt', + 'ImeAutoTextIntegrationTest.kt', ]; const XML_FILES = [ @@ -415,6 +424,7 @@ function main() { 'LitePinyinRepository.kt', 'LitePinyinUserFrequency.kt', 'LitePinyinLearnedPhrases.kt', + 'LegacyPinyinBackup.kt', ]) { const path = join(kotlinDest, name); if (dryRun) console.log(`[dry-run] Would remove retired source ${path}`); diff --git a/openless-all/app/scripts/local-bundle-contract.test.mjs b/openless-all/app/scripts/local-bundle-contract.test.mjs index 212b4040d..319f8888b 100644 --- a/openless-all/app/scripts/local-bundle-contract.test.mjs +++ b/openless-all/app/scripts/local-bundle-contract.test.mjs @@ -20,7 +20,6 @@ const permissionsCommands = read('src-tauri/src/commands/permissions_cmds.rs'); const mobileHandler = read('src-tauri/src/lib.rs'); const application = read('android/kotlin/OpenLessApplication.kt'); const settingsExport = read('android/kotlin/OpenLessSettingsExport.kt'); -const legacyPinyin = read('android/kotlin/LegacyPinyinBackup.kt'); assert.match(policy, /openless\.android\.migration-bundle/); assert.match(migrationActivity, /nativeExportLocalMigrationBundle/); @@ -55,10 +54,18 @@ assert.doesNotMatch( ); assert.doesNotMatch(keyboardActivity, /showExportCategoryDialog|launchImportPicker/); assert.match(keyboardActivity, /nativeAsrUsage/); -assert.match(keyboardActivity, /LegacyPinyinBackup/); +assert.doesNotMatch(keyboardActivity, /LegacyPinyinBackup/); +// The keyboard page exports its own learning data, separately from the migration bundle. +const keyboardData = read('android/kotlin/KeyboardDataFile.kt'); +assert.match(keyboardActivity, /KeyboardDataFile\.export\(/); +assert.match(keyboardActivity, /KeyboardDataFile\.import\(/); +assert.match(keyboardData, /OpenLessRime\.exportUserDictionary/); +assert.match(keyboardData, /OpenLessRime\.importUserDictionary/); +assert.doesNotMatch(keyboardData, /nativeExportLocalMigrationBundle|OpenLessSettingsExport/); assert.doesNotMatch(settingsExport, /LitePinyinLearnedPhrases/); -assert.match(settingsExport, /pinyinLearnedPhrases.*LegacyPinyinBackup/); -assert.match(legacyPinyin, /OpenLessMigrationState\.settings/); +// Bundle schema 1 keeps the field, always empty now that Rime replaced LitePinyin. +assert.match(settingsExport, /put\("pinyinLearnedPhrases", JSONObject\(\)\)/); +assert.equal(existsSync(join(appRoot, 'android/kotlin/LegacyPinyinBackup.kt')), false); assert.equal(existsSync(join(appRoot, 'android/kotlin/LitePinyinLearnedPhrases.kt')), false); assert.match(scaffolding, /OpenLessLocalMigrationActivity\.kt/); assert.match(native, /nativeExportLocalMigrationBundle/);