Sort completion candidates by frecency from history

changelog.md

@@ -1,6 +1,11 @@
 Upcoming (TBD)
 ==============
 
+Features
+--------
+* Sort completion candidates by frecency from history.
+
+
 Internal
 ---------
 * Upgrade `pygments` to v2.21.0, removing hacks for `set*` identifiers.

mycli/client_query.py

@@ -59,6 +59,7 @@ def refresh_completions(self, reset: bool = False) -> list[SQLResult]:
                 "keyword_casing": self.completer.keyword_casing,
                 "indexed_column_suffix": self.completer.indexed_column_suffix,
                 "config_property_names": self.completer.config_property_names,
+                'frecency_provider': self.completer.frecency_provider,
             },
         )
 

mycli/main_modes/repl.py

@@ -74,7 +74,7 @@
     prepare_polars_transform,
     run_polars_transform,
 )
-from mycli.packages.ptoolkit.history import FileHistoryWithTimestamp
+from mycli.packages.ptoolkit.history import FRECENCY_HISTORY_ENTRIES, FRECENCY_REFRESH_INTERVAL, FileHistoryWithTimestamp
 from mycli.packages.special.utils import format_uptime, get_ssl_version, get_uptime, get_warning_count
 from mycli.packages.sql_utils import (
     extract_new_password,
@@ -146,7 +146,14 @@ def complete_while_typing_filter() -> bool:
 def _create_history(mycli: 'MyCli') -> FileHistoryWithTimestamp | None:
     history_file = os.path.expanduser(os.environ.get('MYCLI_HISTFILE', mycli.config['main'].get('history_file', '~/.mycli-history')))
     if dir_path_exists(history_file):
-        return FileHistoryWithTimestamp(history_file)
+        configured_history_entries = mycli.config['main'].get('frecency_history_entries', FRECENCY_HISTORY_ENTRIES)
+        frecency_history_entries = int(configured_history_entries) if str(configured_history_entries).strip() else 0
+        frecency_refresh_interval = int(mycli.config['main'].get('frecency_refresh_interval', FRECENCY_REFRESH_INTERVAL))
+        return FileHistoryWithTimestamp(
+            history_file,
+            frecency_history_entries=frecency_history_entries,
+            frecency_refresh_interval=frecency_refresh_interval,
+        )
 
     mycli.echo(
         f'Error: Unable to open the history file "{history_file}". Your query history will not be saved.',
@@ -1113,10 +1120,12 @@ def main_repl(mycli: 'MyCli') -> None:
 
     mycli.configure_pager()
     _configure_editor(mycli)
+    history = _create_history(mycli)
+    if history is not None:
+        mycli.completer.frecency_provider = lambda: history.frecency
     if mycli.smart_completion and not mycli.sandbox_mode:
         mycli.refresh_completions()
 
-    history = _create_history(mycli)
     key_bindings = mycli_bindings(mycli)
     _show_startup_banner(mycli, sqlexecute)
     _build_prompt_session(mycli, state, history, key_bindings)

mycli/myclirc

@@ -54,6 +54,14 @@ destructive_keywords = DROP SHUTDOWN DELETE TRUNCATE ALTER UPDATE
 # interactive query history location.
 history_file = ~/.mycli-history
 
+# Number of recent history entries used to calculate frecency. Set to 0 or
+# leave empty to disable frecency calculation entirely.
+frecency_history_entries = 1000
+
+# Number of REPL entries between frecency recalculation. Set to 0 to disable
+# periodic recalculation.
+frecency_refresh_interval = 50
+
 # log_file location.
 log_file = ~/.mycli.log
 

mycli/packages/ptoolkit/history.py

@@ -1,28 +1,152 @@
+from collections import defaultdict
+from collections.abc import Iterable, Mapping
+from itertools import islice
+import logging
 import os
-from typing import Union
+import re
+import threading
 
 from prompt_toolkit.history import FileHistory
+from sqlglot import Token, TokenType, tokenize
+from sqlglot.errors import TokenError
 
 from mycli.packages.sql_utils import is_password_change
 
-_StrOrBytesPath = Union[str, bytes, os.PathLike]
+logger = logging.getLogger(__name__)
+
+_StrOrBytesPath = str | bytes | os.PathLike[str] | os.PathLike[bytes]
+FRECENCY_HISTORY_ENTRIES = 1000
+FRECENCY_REFRESH_INTERVAL = 50
+_FRECENCY_LITERAL_TYPES = frozenset({
+    TokenType.STRING,
+    TokenType.NUMBER,
+    TokenType.BIT_STRING,
+    TokenType.HEX_STRING,
+    TokenType.BYTE_STRING,
+    TokenType.NATIONAL_STRING,
+    TokenType.RAW_STRING,
+    TokenType.HEREDOC_STRING,
+    TokenType.UNICODE_STRING,
+})
+_FRECENCY_WORD_PATTERN = re.compile(r'^[^\W\d][\w$]*(?:\s+[^\W\d][\w$]*)*$')
+
+
+def _normalize_frecency_token(token: Token) -> str | None:
+    if token.token_type in _FRECENCY_LITERAL_TYPES:
+        return None
+    if token.token_type != TokenType.IDENTIFIER and not _FRECENCY_WORD_PATTERN.fullmatch(token.text):
+        return None
+    return token.text.casefold() or None
+
+
+def _calculate_frecency(
+    entries: Iterable[str],
+    history_entries: int = FRECENCY_HISTORY_ENTRIES,
+) -> dict[str, float]:
+    frecency: defaultdict[str, float] = defaultdict(float)
+    for position, entry in enumerate(islice(entries, max(0, history_entries))):
+        weight = 1 / (position + 1)
+        try:
+            tokens = tokenize(entry, dialect='mysql')
+        except TokenError:
+            continue
+        for token in tokens:
+            if normalized := _normalize_frecency_token(token):
+                frecency[normalized] += weight
+    return dict(frecency)
+
+
+def frecency_score(text: str, frecency: Mapping[str, float]) -> float:
+    try:
+        tokens = tokenize(text, dialect='mysql')
+    except TokenError:
+        return 0.0
+    normalized_tokens = [normalized for token in tokens if (normalized := _normalize_frecency_token(token))]
+    if not normalized_tokens:
+        return 0.0
+    return sum(frecency.get(token, 0.0) for token in normalized_tokens) / len(normalized_tokens)
 
 
 class FileHistoryWithTimestamp(FileHistory):
     """
     :class:`.FileHistory` class that stores all strings in a file with timestamp.
     """
 
-    def __init__(self, filename: _StrOrBytesPath) -> None:
+    def __init__(
+        self,
+        filename: _StrOrBytesPath,
+        frecency_history_entries: int = FRECENCY_HISTORY_ENTRIES,
+        frecency_refresh_interval: int = FRECENCY_REFRESH_INTERVAL,
+    ) -> None:
         self.filename = filename
         super().__init__(filename)
+        self.frecency_history_entries = max(0, frecency_history_entries)
+        self.frecency_refresh_interval = max(0, frecency_refresh_interval)
+        self._frecency: dict[str, float] = {}
+        self._frecency_lock = threading.Lock()
+        self._frecency_generation = 0
+        self._frecency_thread: threading.Thread | None = None
+        self._frecency_entries_since_refresh = 0
+        if self.frecency_history_entries:
+            self._request_frecency_refresh()
+
+    @property
+    def frecency(self) -> dict[str, float]:
+        with self._frecency_lock:
+            return self._frecency
+
+    def _request_frecency_refresh(self) -> None:
+        with self._frecency_lock:
+            self._frecency_generation += 1
+            if self._frecency_thread is not None:
+                return
+            thread = threading.Thread(target=self._refresh_frecency, name='frecency_refresh', daemon=True)
+            self._frecency_thread = thread
+
+        try:
+            thread.start()
+        except Exception:
+            with self._frecency_lock:
+                if self._frecency_thread is thread:
+                    self._frecency_thread = None
+            logger.exception('Failed to start history frecency calculation.')
+
+    def _refresh_frecency(self) -> None:
+        while True:
+            with self._frecency_lock:
+                generation = self._frecency_generation
+
+            try:
+                frecency = _calculate_frecency(self.load_history_strings(), self.frecency_history_entries)
+            except Exception:
+                logger.exception('Failed to calculate history frecency.')
+                with self._frecency_lock:
+                    if self._frecency_generation != generation:
+                        continue
+                    self._frecency_thread = None
+                return
+
+            with self._frecency_lock:
+                self._frecency = frecency
+                if self._frecency_generation == generation:
+                    self._frecency_thread = None
+                    return
 
     def append_string(self, string: str) -> None:
         "Add string to the history."
         self._loaded_strings.insert(0, string)
         if is_password_change(string):
             return
         self.store_string(string)
+        if not self.frecency_history_entries or not self.frecency_refresh_interval:
+            return
+        with self._frecency_lock:
+            self._frecency_entries_since_refresh += 1
+            refresh = self._frecency_entries_since_refresh >= self.frecency_refresh_interval
+            if refresh:
+                self._frecency_entries_since_refresh = 0
+        if refresh:
+            self._request_frecency_refresh()
 
     def load_history_with_timestamp(self) -> list[tuple[str, str]]:
         """

mycli/sqlcompleter.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from collections import Counter
+from collections.abc import Callable, Mapping
 from enum import IntEnum
 import logging
 import os
@@ -18,6 +19,7 @@
 from mycli.compat import WIN
 from mycli.packages.completion_engine import is_inside_quotes, suggest_type
 from mycli.packages.filepaths import complete_path, parse_path, suggest_path
+from mycli.packages.ptoolkit.history import frecency_score
 from mycli.packages.special import llm
 from mycli.packages.special.dsn_aliases import DsnAliases
 from mycli.packages.special.favoritequeries import (
@@ -950,11 +952,13 @@ def __init__(
         keyword_casing: str = "auto",
         indexed_column_suffix: str = '*',
         config_property_names: Collection[str] = (),
+        frecency_provider: Callable[[], Mapping[str, float]] | None = None,
     ) -> None:
         super(self.__class__, self).__init__()
         self.smart_completion = smart_completion
         self.indexed_column_suffix = indexed_column_suffix
         self.config_property_names = tuple(sorted(config_property_names))
+        self.frecency_provider = frecency_provider
         self.reserved_words = set()
         for x in self.keywords:
             self.reserved_words.update(x.split())
@@ -1437,20 +1441,23 @@ def get_completions(
         last_for_len = last_word(word_before_cursor, include="most_punctuations")
         text_for_len = last_for_len.lower()
         last_for_len_paths = last_word(word_before_cursor, include='alphanum_underscore')
+        frecency = self.frecency_provider() if self.frecency_provider is not None else {}
 
         if smart_completion is None:
             smart_completion = self.smart_completion
 
         # If smart_completion is off then match any word that starts with
         # 'word_before_cursor'.
         if not smart_completion:
-            matches = self.find_matches(
+            matches: Iterable[tuple[str, int]] = self.find_matches(
                 word_before_cursor,
                 self.all_completions,
                 start_only=True,
                 fuzzy=False,
                 text_before_cursor=document.text_before_cursor,
             )
+            if frecency:
+                matches = sorted(matches, key=lambda item: -frecency_score(item[0], frecency))
             return (Completion(x[0], -len(text_for_len)) for x in matches)
 
         completions: list[tuple[str, int, int]] = []
@@ -1811,15 +1818,16 @@ def get_completions(
 
         def completion_sort_key(item: tuple[str, int, int], text_for_len: str):
             candidate, fuzziness, rank = item
+            candidate_frecency = frecency_score(candidate, frecency) if frecency else 0.0
             if not text_for_len:
-                # sort only by the rank (the order of the completion type)
-                return (0, rank, 0)
+                # Sort by the rank (the order of the completion type), then frecency.
+                return (0, rank, -candidate_frecency, 0)
             elif candidate.lower().startswith(text_for_len):
-                # sort only by the length of the candidate
-                return (0, 0, -1000 + len(candidate))
-            # sort by fuzziness and rank
+                # Direct prefix matches are equally relevant; prefer frecency before length.
+                return (0, 0, -candidate_frecency, -1000 + len(candidate))
+            # Sort by fuzziness, rank, and frecency.
             # todo add alpha here, or original order?
-            return (fuzziness, rank, 0)
+            return (fuzziness, rank, -candidate_frecency, 0)
 
         if rigid_sort:
             uniq_completions_str = dict.fromkeys(x[0] for x in completions)

test/myclirc

@@ -54,6 +54,14 @@ destructive_keywords = DROP SHUTDOWN DELETE TRUNCATE ALTER UPDATE
 # interactive query history location.
 history_file = ~/.mycli-history
 
+# Number of recent history entries used to calculate frecency. Set to 0 or
+# leave empty to disable frecency calculation entirely.
+frecency_history_entries = 1000
+
+# Number of REPL entries between frecency recalculation. Set to 0 to disable
+# periodic recalculation.
+frecency_refresh_interval = 50
+
 # log_file location.
 log_file = ~/.mycli.test.log
 

test/pytests/test_client_query.py

@@ -27,6 +27,7 @@ def make_refresh_cli() -> tuple[Any, dict[str, Any]]:
     cli._on_completions_refreshed = callback
     cli.completer = SimpleNamespace(
         config_property_names=('main.show_warnings',),
+        frecency_provider=lambda: {'select': 1.0},
         keyword_casing='upper',
         indexed_column_suffix=' [indexed]',
         set_dbname=lambda dbname: state['set_dbname_calls'].append(dbname),
@@ -72,6 +73,7 @@ def test_refresh_completions_passes_options_to_refresher() -> None:
                 'keyword_casing': 'upper',
                 'indexed_column_suffix': ' [indexed]',
                 'config_property_names': ('main.show_warnings',),
+                'frecency_provider': cli.completer.frecency_provider,
             },
         )
     ]
@@ -101,6 +103,7 @@ def test_refresh_completions_updates_dbname_when_reset() -> None:
     cli.sqlexecute = SimpleNamespace(dbname='next_db')
     cli.completer = SimpleNamespace(
         config_property_names=(),
+        frecency_provider=None,
         keyword_casing='lower',
         indexed_column_suffix='*',
         set_dbname=lambda dbname: set_dbname_calls.append(dbname),
@@ -121,6 +124,7 @@ def test_refresh_completions_uses_lock_when_reset() -> None:
     cli._completer_lock = cast(Any, ReusableLock(lambda: entered_lock.__setitem__('count', entered_lock['count'] + 1)))
     cli.completer = SimpleNamespace(
         config_property_names=(),
+        frecency_provider=None,
         keyword_casing='lower',
         indexed_column_suffix='*',
         set_dbname=lambda dbname: None,