diff --git a/changelog.md b/changelog.md index 5e080b8c6..4fe08345c 100644 --- a/changelog.md +++ b/changelog.md @@ -1,6 +1,11 @@ Upcoming (TBD) ============== +Features +-------- +* Sort completion candidates by frecency from history. + + Internal --------- * Upgrade `pygments` to v2.21.0, removing hacks for `set*` identifiers. diff --git a/mycli/client_query.py b/mycli/client_query.py index 5ae94b436..7970f80e6 100644 --- a/mycli/client_query.py +++ b/mycli/client_query.py @@ -59,6 +59,7 @@ def refresh_completions(self, reset: bool = False) -> list[SQLResult]: "keyword_casing": self.completer.keyword_casing, "indexed_column_suffix": self.completer.indexed_column_suffix, "config_property_names": self.completer.config_property_names, + 'frecency_provider': self.completer.frecency_provider, }, ) diff --git a/mycli/main_modes/repl.py b/mycli/main_modes/repl.py index 917cb07bb..c5d97d914 100644 --- a/mycli/main_modes/repl.py +++ b/mycli/main_modes/repl.py @@ -74,7 +74,7 @@ prepare_polars_transform, run_polars_transform, ) -from mycli.packages.ptoolkit.history import FileHistoryWithTimestamp +from mycli.packages.ptoolkit.history import FRECENCY_HISTORY_ENTRIES, FRECENCY_REFRESH_INTERVAL, FileHistoryWithTimestamp from mycli.packages.special.utils import format_uptime, get_ssl_version, get_uptime, get_warning_count from mycli.packages.sql_utils import ( extract_new_password, @@ -146,7 +146,13 @@ def complete_while_typing_filter() -> bool: def _create_history(mycli: 'MyCli') -> FileHistoryWithTimestamp | None: history_file = os.path.expanduser(os.environ.get('MYCLI_HISTFILE', mycli.config['main'].get('history_file', '~/.mycli-history'))) if dir_path_exists(history_file): - return FileHistoryWithTimestamp(history_file) + frecency_history_entries = int(mycli.config['main'].get('frecency_history_entries', FRECENCY_HISTORY_ENTRIES) or 0) + frecency_refresh_interval = int(mycli.config['main'].get('frecency_refresh_interval', FRECENCY_REFRESH_INTERVAL) or 0) + return FileHistoryWithTimestamp( + history_file, + frecency_history_entries=frecency_history_entries, + frecency_refresh_interval=frecency_refresh_interval, + ) mycli.echo( f'Error: Unable to open the history file "{history_file}". Your query history will not be saved.', @@ -1113,10 +1119,12 @@ def main_repl(mycli: 'MyCli') -> None: mycli.configure_pager() _configure_editor(mycli) + history = _create_history(mycli) + if history is not None: + mycli.completer.frecency_provider = lambda: history.frecency if mycli.smart_completion and not mycli.sandbox_mode: mycli.refresh_completions() - history = _create_history(mycli) key_bindings = mycli_bindings(mycli) _show_startup_banner(mycli, sqlexecute) _build_prompt_session(mycli, state, history, key_bindings) diff --git a/mycli/myclirc b/mycli/myclirc index 4c750e794..f2fd16e6a 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -54,6 +54,14 @@ destructive_keywords = DROP SHUTDOWN DELETE TRUNCATE ALTER UPDATE # interactive query history location. history_file = ~/.mycli-history +# Number of recent history entries used to calculate frecency. Set to 0 or +# leave empty to disable frecency calculation entirely. +frecency_history_entries = 1000 + +# Number of REPL entries between frecency recalculation. Set to 0 to disable +# periodic recalculation. +frecency_refresh_interval = 50 + # log_file location. log_file = ~/.mycli.log diff --git a/mycli/packages/ptoolkit/history.py b/mycli/packages/ptoolkit/history.py index 982bc7748..78b2ba083 100644 --- a/mycli/packages/ptoolkit/history.py +++ b/mycli/packages/ptoolkit/history.py @@ -1,11 +1,77 @@ +from collections import defaultdict +from collections.abc import Iterable, Mapping +from functools import lru_cache +from itertools import islice +import logging import os -from typing import Union +import re +import threading from prompt_toolkit.history import FileHistory +from sqlglot import Token, TokenType, tokenize +from sqlglot.errors import TokenError from mycli.packages.sql_utils import is_password_change -_StrOrBytesPath = Union[str, bytes, os.PathLike] +logger = logging.getLogger(__name__) + +_StrOrBytesPath = str | bytes | os.PathLike[str] | os.PathLike[bytes] +FRECENCY_HISTORY_ENTRIES = 1000 +FRECENCY_REFRESH_INTERVAL = 50 +_FRECENCY_LITERAL_TYPES = frozenset({ + TokenType.STRING, + TokenType.NUMBER, + TokenType.BIT_STRING, + TokenType.HEX_STRING, + TokenType.BYTE_STRING, + TokenType.NATIONAL_STRING, + TokenType.RAW_STRING, + TokenType.HEREDOC_STRING, + TokenType.UNICODE_STRING, +}) +_FRECENCY_WORD_PATTERN = re.compile(r'^[^\W\d][\w$]*(?:\s+[^\W\d][\w$]*)*$') + + +def _normalize_frecency_token(token: Token) -> str | None: + if token.token_type in _FRECENCY_LITERAL_TYPES: + return None + if token.token_type != TokenType.IDENTIFIER and not _FRECENCY_WORD_PATTERN.fullmatch(token.text): + return None + return token.text.casefold() or None + + +def _calculate_frecency( + entries: Iterable[str], + history_entries: int = FRECENCY_HISTORY_ENTRIES, +) -> dict[str, float]: + frecency: defaultdict[str, float] = defaultdict(float) + for position, entry in enumerate(islice(entries, max(0, history_entries))): + weight = 1 / (position + 1) + try: + tokens = tokenize(entry, dialect='mysql') + except TokenError: + continue + for token in tokens: + if normalized := _normalize_frecency_token(token): + frecency[normalized] += weight + return dict(frecency) + + +@lru_cache(maxsize=8192) +def _frecency_tokens(text: str) -> tuple[str, ...]: + """Tokenize a completion candidate. Cached: candidates repeat on every keystroke.""" + try: + tokens = tokenize(text, dialect='mysql') + except TokenError: + return () + return tuple(normalized for token in tokens if (normalized := _normalize_frecency_token(token))) + + +def frecency_score(text: str, frecency: Mapping[str, float]) -> float: + normalized_tokens = _frecency_tokens(text) + if not normalized_tokens: + return 0.0 + return sum(frecency.get(token, 0.0) for token in normalized_tokens) / len(normalized_tokens) class FileHistoryWithTimestamp(FileHistory): @@ -13,9 +79,65 @@ class FileHistoryWithTimestamp(FileHistory): :class:`.FileHistory` class that stores all strings in a file with timestamp. """ - def __init__(self, filename: _StrOrBytesPath) -> None: + def __init__( + self, + filename: _StrOrBytesPath, + frecency_history_entries: int = FRECENCY_HISTORY_ENTRIES, + frecency_refresh_interval: int = FRECENCY_REFRESH_INTERVAL, + ) -> None: self.filename = filename super().__init__(filename) + self.frecency_history_entries = max(0, frecency_history_entries) + self.frecency_refresh_interval = max(0, frecency_refresh_interval) + self._frecency: dict[str, float] = {} + self._frecency_lock = threading.Lock() + self._frecency_generation = 0 + self._frecency_thread: threading.Thread | None = None + self._frecency_entries_since_refresh = 0 + if self.frecency_history_entries: + self._request_frecency_refresh() + + @property + def frecency(self) -> dict[str, float]: + with self._frecency_lock: + return self._frecency + + def _request_frecency_refresh(self) -> None: + with self._frecency_lock: + self._frecency_generation += 1 + if self._frecency_thread is not None: + return + thread = threading.Thread(target=self._refresh_frecency, name='frecency_refresh', daemon=True) + self._frecency_thread = thread + + try: + thread.start() + except Exception: + with self._frecency_lock: + if self._frecency_thread is thread: + self._frecency_thread = None + logger.exception('Failed to start history frecency calculation.') + + def _refresh_frecency(self) -> None: + while True: + with self._frecency_lock: + generation = self._frecency_generation + + try: + frecency = _calculate_frecency(self.load_history_strings(), self.frecency_history_entries) + except Exception: + logger.exception('Failed to calculate history frecency.') + with self._frecency_lock: + if self._frecency_generation != generation: + continue + self._frecency_thread = None + return + + with self._frecency_lock: + self._frecency = frecency + if self._frecency_generation == generation: + self._frecency_thread = None + return def append_string(self, string: str) -> None: "Add string to the history." @@ -23,6 +145,15 @@ def append_string(self, string: str) -> None: if is_password_change(string): return self.store_string(string) + if not self.frecency_history_entries or not self.frecency_refresh_interval: + return + with self._frecency_lock: + self._frecency_entries_since_refresh += 1 + refresh = self._frecency_entries_since_refresh >= self.frecency_refresh_interval + if refresh: + self._frecency_entries_since_refresh = 0 + if refresh: + self._request_frecency_refresh() def load_history_with_timestamp(self) -> list[tuple[str, str]]: """ diff --git a/mycli/sqlcompleter.py b/mycli/sqlcompleter.py index 68b3d2598..3ff0f2075 100644 --- a/mycli/sqlcompleter.py +++ b/mycli/sqlcompleter.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections import Counter +from collections.abc import Callable, Mapping from enum import IntEnum import logging import os @@ -18,6 +19,7 @@ from mycli.compat import WIN from mycli.packages.completion_engine import is_inside_quotes, suggest_type from mycli.packages.filepaths import complete_path, parse_path, suggest_path +from mycli.packages.ptoolkit.history import frecency_score from mycli.packages.special import llm from mycli.packages.special.dsn_aliases import DsnAliases from mycli.packages.special.favoritequeries import ( @@ -950,11 +952,13 @@ def __init__( keyword_casing: str = "auto", indexed_column_suffix: str = '*', config_property_names: Collection[str] = (), + frecency_provider: Callable[[], Mapping[str, float]] | None = None, ) -> None: super(self.__class__, self).__init__() self.smart_completion = smart_completion self.indexed_column_suffix = indexed_column_suffix self.config_property_names = tuple(sorted(config_property_names)) + self.frecency_provider = frecency_provider self.reserved_words = set() for x in self.keywords: self.reserved_words.update(x.split()) @@ -1437,6 +1441,7 @@ def get_completions( last_for_len = last_word(word_before_cursor, include="most_punctuations") text_for_len = last_for_len.lower() last_for_len_paths = last_word(word_before_cursor, include='alphanum_underscore') + frecency = self.frecency_provider() if self.frecency_provider is not None else {} if smart_completion is None: smart_completion = self.smart_completion @@ -1444,13 +1449,15 @@ def get_completions( # If smart_completion is off then match any word that starts with # 'word_before_cursor'. if not smart_completion: - matches = self.find_matches( + matches: Iterable[tuple[str, int]] = self.find_matches( word_before_cursor, self.all_completions, start_only=True, fuzzy=False, text_before_cursor=document.text_before_cursor, ) + if frecency: + matches = sorted(matches, key=lambda item: -frecency_score(item[0], frecency)) return (Completion(x[0], -len(text_for_len)) for x in matches) completions: list[tuple[str, int, int]] = [] @@ -1811,15 +1818,16 @@ def get_completions( def completion_sort_key(item: tuple[str, int, int], text_for_len: str): candidate, fuzziness, rank = item + candidate_frecency = frecency_score(candidate, frecency) if frecency else 0.0 if not text_for_len: - # sort only by the rank (the order of the completion type) - return (0, rank, 0) + # Sort by the rank (the order of the completion type), then frecency. + return (0, rank, -candidate_frecency, 0) elif candidate.lower().startswith(text_for_len): - # sort only by the length of the candidate - return (0, 0, -1000 + len(candidate)) - # sort by fuzziness and rank + # Direct prefix matches are equally relevant; prefer frecency before length. + return (0, 0, -candidate_frecency, -1000 + len(candidate)) + # Sort by fuzziness, rank, and frecency. # todo add alpha here, or original order? - return (fuzziness, rank, 0) + return (fuzziness, rank, -candidate_frecency, 0) if rigid_sort: uniq_completions_str = dict.fromkeys(x[0] for x in completions) diff --git a/test/myclirc b/test/myclirc index a4bcd270d..a422fddc6 100644 --- a/test/myclirc +++ b/test/myclirc @@ -54,6 +54,14 @@ destructive_keywords = DROP SHUTDOWN DELETE TRUNCATE ALTER UPDATE # interactive query history location. history_file = ~/.mycli-history +# Number of recent history entries used to calculate frecency. Set to 0 or +# leave empty to disable frecency calculation entirely. +frecency_history_entries = 1000 + +# Number of REPL entries between frecency recalculation. Set to 0 to disable +# periodic recalculation. +frecency_refresh_interval = 50 + # log_file location. log_file = ~/.mycli.test.log diff --git a/test/pytests/test_client_query.py b/test/pytests/test_client_query.py index 3e2108cfb..080a12d0f 100644 --- a/test/pytests/test_client_query.py +++ b/test/pytests/test_client_query.py @@ -27,6 +27,7 @@ def make_refresh_cli() -> tuple[Any, dict[str, Any]]: cli._on_completions_refreshed = callback cli.completer = SimpleNamespace( config_property_names=('main.show_warnings',), + frecency_provider=lambda: {'select': 1.0}, keyword_casing='upper', indexed_column_suffix=' [indexed]', set_dbname=lambda dbname: state['set_dbname_calls'].append(dbname), @@ -72,6 +73,7 @@ def test_refresh_completions_passes_options_to_refresher() -> None: 'keyword_casing': 'upper', 'indexed_column_suffix': ' [indexed]', 'config_property_names': ('main.show_warnings',), + 'frecency_provider': cli.completer.frecency_provider, }, ) ] @@ -101,6 +103,7 @@ def test_refresh_completions_updates_dbname_when_reset() -> None: cli.sqlexecute = SimpleNamespace(dbname='next_db') cli.completer = SimpleNamespace( config_property_names=(), + frecency_provider=None, keyword_casing='lower', indexed_column_suffix='*', set_dbname=lambda dbname: set_dbname_calls.append(dbname), @@ -121,6 +124,7 @@ def test_refresh_completions_uses_lock_when_reset() -> None: cli._completer_lock = cast(Any, ReusableLock(lambda: entered_lock.__setitem__('count', entered_lock['count'] + 1))) cli.completer = SimpleNamespace( config_property_names=(), + frecency_provider=None, keyword_casing='lower', indexed_column_suffix='*', set_dbname=lambda dbname: None, diff --git a/test/pytests/test_main_modes_repl.py b/test/pytests/test_main_modes_repl.py index b39cbf937..e6141a3fb 100644 --- a/test/pytests/test_main_modes_repl.py +++ b/test/pytests/test_main_modes_repl.py @@ -351,11 +351,19 @@ def test_complete_while_typing_filter_does_not_treat_block_comments_as_slash_com def test_repl_create_history(monkeypatch: pytest.MonkeyPatch) -> None: cli = make_repl_cli() + cli.config['main']['frecency_history_entries'] = '25' + cli.config['main']['frecency_refresh_interval'] = '10' monkeypatch.setenv('MYCLI_HISTFILE', '~/override-history') monkeypatch.setattr(repl_mode, 'dir_path_exists', lambda path: True) - monkeypatch.setattr(repl_mode, 'FileHistoryWithTimestamp', lambda path: f'history:{path}') + monkeypatch.setattr( + repl_mode, + 'FileHistoryWithTimestamp', + lambda path, frecency_history_entries, frecency_refresh_interval: ( + f'history:{path}:{frecency_history_entries}:{frecency_refresh_interval}' + ), + ) history = cast(Any, repl_mode._create_history(cli)) - assert history == f'history:{os.path.expanduser("~/override-history")}' + assert history == f'history:{os.path.expanduser("~/override-history")}:25:10' monkeypatch.delenv('MYCLI_HISTFILE') monkeypatch.setattr(repl_mode, 'dir_path_exists', lambda path: False) @@ -363,6 +371,31 @@ def test_repl_create_history(monkeypatch: pytest.MonkeyPatch) -> None: assert 'Unable to open the history file' in cli.echo_calls[-1] +@pytest.mark.parametrize('configured_history_entries', ['', '0']) +def test_repl_create_history_can_disable_frecency( + monkeypatch: pytest.MonkeyPatch, + configured_history_entries: str, +) -> None: + cli = make_repl_cli() + cli.config['main']['frecency_history_entries'] = configured_history_entries + monkeypatch.setattr(repl_mode, 'dir_path_exists', lambda path: True) + history_arguments: dict[str, int] = {} + + def create_history( + path: str, + frecency_history_entries: int, + frecency_refresh_interval: int, + ) -> str: + history_arguments['frecency_history_entries'] = frecency_history_entries + return path + + monkeypatch.setattr(repl_mode, 'FileHistoryWithTimestamp', create_history) + + repl_mode._create_history(cli) + + assert history_arguments['frecency_history_entries'] == 0 + + def test_repl_picker_helpers_cover_present_and_missing_resources(monkeypatch: pytest.MonkeyPatch) -> None: files = { 'AUTHORS': '* Alice\n* Bob\n', @@ -2140,7 +2173,9 @@ def test_main_repl_covers_setup_loop_and_goodbye(monkeypatch: pytest.MonkeyPatch cli.verbosity = 0 cli.smart_completion = True loop_iterations: list[int] = [] - monkeypatch.setattr(repl_mode, '_create_history', lambda mycli: 'history') + history = SimpleNamespace(frecency={'select': 1.0}) + cli.completer = SimpleNamespace(frecency_provider=None) + monkeypatch.setattr(repl_mode, '_create_history', lambda mycli: history) monkeypatch.setattr(repl_mode, 'mycli_bindings', lambda mycli: 'bindings') monkeypatch.setattr(repl_mode, '_show_startup_banner', lambda mycli, sqlexecute: None) monkeypatch.setattr( @@ -2162,6 +2197,7 @@ def fake_one_iteration(mycli: Any, state: repl_mode.ReplState) -> None: repl_mode.main_repl(cli) assert cli.pager_configured == 1 + assert cli.completer.frecency_provider() == {'select': 1.0} assert cli.refresh_calls == [False] assert cli.title_calls == 1 assert loop_iterations == [0, 1] @@ -2173,7 +2209,9 @@ def test_main_repl_covers_no_refresh_and_quiet_exit(monkeypatch: pytest.MonkeyPa cli = make_repl_cli(SimpleNamespace()) cli.verbosity = -1 cli.smart_completion = False - monkeypatch.setattr(repl_mode, '_create_history', lambda mycli: 'history') + history = SimpleNamespace(frecency={}) + cli.completer = SimpleNamespace(frecency_provider=None) + monkeypatch.setattr(repl_mode, '_create_history', lambda mycli: history) monkeypatch.setattr(repl_mode, 'mycli_bindings', lambda mycli: 'bindings') monkeypatch.setattr(repl_mode, '_show_startup_banner', lambda mycli, sqlexecute: None) monkeypatch.setattr( diff --git a/test/pytests/test_ptoolkit_history.py b/test/pytests/test_ptoolkit_history.py index ce54b5907..a50fa99e3 100644 --- a/test/pytests/test_ptoolkit_history.py +++ b/test/pytests/test_ptoolkit_history.py @@ -1,9 +1,19 @@ # type: ignore from pathlib import Path +import threading + +import pytest from mycli.packages.ptoolkit import history as history_module -from mycli.packages.ptoolkit.history import FileHistoryWithTimestamp +from mycli.packages.ptoolkit.history import FileHistoryWithTimestamp, frecency_score + + +def wait_for_frecency_refresh(history: FileHistoryWithTimestamp) -> None: + while history._frecency_thread is not None: + thread = history._frecency_thread + thread.join(timeout=5) + assert not thread.is_alive() def test_file_history_with_timestamp_sets_filename(tmp_path: Path) -> None: @@ -12,6 +22,319 @@ def test_file_history_with_timestamp_sets_filename(tmp_path: Path) -> None: history = FileHistoryWithTimestamp(history_path) assert history.filename == history_path + assert history.frecency == {} + + +def test_history_frecency_weights_normalizes_and_filters_tokens() -> None: + entries = [ + 'SELECT foo, foo, 123, \'foo\', "bar", `Mixed``Name` /* ignored */; /status', + 'select Foo FROM `mixed``name`', + ] + + frecency = history_module._calculate_frecency(entries) + + assert frecency == { + 'select': 1.5, + 'foo': 2.5, + 'mixed`name': 1.5, + 'status': 1.0, + 'from': 0.5, + } + + +def test_history_frecency_ignores_invalid_entries() -> None: + entries = ['SELECT "unterminated', 'SELECT valid_name'] + + assert history_module._calculate_frecency(entries) == { + 'select': 0.5, + 'valid_name': 0.5, + } + + +@pytest.mark.parametrize( + ('candidate', 'expected'), + [ + ('`Mixed``Name`', 4.0), + ('/status', 2.0), + ('ORDER BY unseen', 1.5), + ('foo bar', 2.0), + ("'literal'", 0.0), + ('"unterminated', 0.0), + ], +) +def test_frecency_score_normalizes_and_averages_candidate_tokens(candidate: str, expected: float) -> None: + frecency = {'mixed`name': 4.0, 'status': 2.0, 'order by': 3.0, 'foo': 3.0, 'bar': 1.0} + + assert frecency_score(candidate, frecency) == expected + + +def test_history_frecency_uses_only_newest_thousand_entries(tmp_path: Path) -> None: + history_path = tmp_path / 'history.txt' + history_path.write_text( + ''.join(f'\n# 2026-01-01 00:00:{index:04d}\n+SELECT token_{index}\n' for index in range(1001)), + encoding='utf-8', + ) + + history = FileHistoryWithTimestamp(history_path) + wait_for_frecency_refresh(history) + + assert 'token_0' not in history.frecency + assert history.frecency['token_1000'] == 1.0 + assert history.frecency['token_1'] == pytest.approx(0.001) + + +def test_history_frecency_uses_configured_entry_count(tmp_path: Path) -> None: + history_path = tmp_path / 'history.txt' + history_path.write_text( + '\n# old\n+SELECT old_token\n\n# middle\n+SELECT middle_token\n\n# new\n+SELECT new_token\n', + encoding='utf-8', + ) + + history = FileHistoryWithTimestamp(history_path, frecency_history_entries=2) + wait_for_frecency_refresh(history) + + assert history.frecency_history_entries == 2 + assert history.frecency['new_token'] == 1.0 + assert history.frecency['middle_token'] == 0.5 + assert 'old_token' not in history.frecency + + +@pytest.mark.parametrize('history_entries', [0, -1]) +def test_nonpositive_history_entry_count_disables_frecency( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + history_entries: int, +) -> None: + history_path = tmp_path / 'history.txt' + history_path.write_text('\n# entry\n+SELECT ignored\n', encoding='utf-8') + calculate_frecency = pytest.fail + monkeypatch.setattr(history_module, '_calculate_frecency', calculate_frecency) + + history = FileHistoryWithTimestamp(history_path, frecency_history_entries=history_entries, frecency_refresh_interval=1) + history.append_string('SELECT new_token') + + assert history.frecency_history_entries == 0 + assert history.frecency == {} + assert history._frecency_thread is None + + +def test_initial_frecency_is_calculated_in_background(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + calculation_started = threading.Event() + release_calculation = threading.Event() + + def calculate_frecency(entries, history_entries): + calculation_started.set() + assert release_calculation.wait(timeout=5) + return {'calculated': 1.0} + + monkeypatch.setattr(history_module, '_calculate_frecency', calculate_frecency) + + history = FileHistoryWithTimestamp(tmp_path / 'history.txt') + + assert calculation_started.wait(timeout=5) + assert history.frecency == {} + release_calculation.set() + wait_for_frecency_refresh(history) + assert history.frecency == {'calculated': 1.0} + + +def test_history_frecency_is_not_updated_when_history_is_appended(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + history_path = tmp_path / 'history.txt' + history_path.write_text('\n# 2026-01-01 00:00:00\n+SELECT existing\n', encoding='utf-8') + history = FileHistoryWithTimestamp(history_path) + wait_for_frecency_refresh(history) + original_frecency = history.frecency.copy() + monkeypatch.setattr(history, 'store_string', lambda _string: None) + + history.append_string('SELECT new_token') + + assert history.frecency == original_frecency + assert 'new_token' not in history.frecency + + +def test_history_frecency_is_recomputed_every_fifty_stored_entries(tmp_path: Path) -> None: + history = FileHistoryWithTimestamp(tmp_path / 'history.txt') + wait_for_frecency_refresh(history) + + for index in range(49): + history.append_string(f'SELECT token_{index}') + assert history.frecency == {} + + history.append_string('SELECT token_49') + wait_for_frecency_refresh(history) + assert history.frecency['token_49'] == 1.0 + assert history.frecency['token_0'] == pytest.approx(1 / 50) + + for index in range(50, 99): + history.append_string(f'SELECT token_{index}') + assert 'token_98' not in history.frecency + + history.append_string('SELECT token_99') + wait_for_frecency_refresh(history) + assert history.frecency['token_99'] == 1.0 + assert history.frecency['token_49'] == pytest.approx(1 / 51) + + +def test_history_frecency_uses_configured_refresh_interval(tmp_path: Path) -> None: + history = FileHistoryWithTimestamp(tmp_path / 'history.txt', frecency_refresh_interval=2) + wait_for_frecency_refresh(history) + + history.append_string('SELECT first_token') + assert history.frecency == {} + + history.append_string('SELECT second_token') + wait_for_frecency_refresh(history) + assert history.frecency_refresh_interval == 2 + assert history.frecency['second_token'] == 1.0 + assert history.frecency['first_token'] == 0.5 + + +@pytest.mark.parametrize('refresh_interval', [0, -1]) +def test_nonpositive_refresh_interval_disables_periodic_refresh(tmp_path: Path, refresh_interval: int) -> None: + history = FileHistoryWithTimestamp(tmp_path / 'history.txt', frecency_refresh_interval=refresh_interval) + wait_for_frecency_refresh(history) + + for index in range(50): + history.append_string(f'SELECT token_{index}') + + assert history.frecency_refresh_interval == 0 + assert history.frecency == {} + + +def test_password_changes_do_not_advance_frecency_refresh(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + history = FileHistoryWithTimestamp(tmp_path / 'history.txt') + wait_for_frecency_refresh(history) + monkeypatch.setattr(history_module, 'is_password_change', lambda string: string == 'password change') + + for index in range(49): + history.append_string(f'SELECT token_{index}') + history.append_string('password change') + + assert history.frecency == {} + + history.append_string('SELECT token_49') + wait_for_frecency_refresh(history) + assert history.frecency['token_49'] == 1.0 + assert 'password' not in history.frecency + + +def test_frecency_refresh_requests_are_coalesced(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + history = FileHistoryWithTimestamp(tmp_path / 'history.txt', frecency_refresh_interval=1) + wait_for_frecency_refresh(history) + calculation_started = threading.Event() + release_calculation = threading.Event() + calls: list[int] = [] + worker_threads: set[int | None] = set() + + def calculate_frecency(entries, history_entries): + calls.append(history_entries) + worker_threads.add(threading.current_thread().ident) + if len(calls) == 1: + calculation_started.set() + assert release_calculation.wait(timeout=5) + return {f'generation_{len(calls)}': 1.0} + + monkeypatch.setattr(history_module, '_calculate_frecency', calculate_frecency) + + history.append_string('SELECT first_token') + assert calculation_started.wait(timeout=5) + first_thread = history._frecency_thread + history.append_string('SELECT second_token') + assert history._frecency_thread is first_thread + assert first_thread.name == 'frecency_refresh' + assert first_thread.daemon is True + release_calculation.set() + wait_for_frecency_refresh(history) + + assert len(calls) == 2 + assert len(worker_threads) == 1 + assert history.frecency == {'generation_2': 1.0} + + +def test_frecency_failure_retains_snapshot_and_later_refresh_retries( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + history = FileHistoryWithTimestamp(tmp_path / 'history.txt', frecency_refresh_interval=1) + wait_for_frecency_refresh(history) + calls = 0 + logged_errors: list[str] = [] + + def calculate_frecency(entries, history_entries): + nonlocal calls + calls += 1 + if calls == 1: + raise RuntimeError('calculation failed') + return {'recovered': 1.0} + + monkeypatch.setattr(history_module, '_calculate_frecency', calculate_frecency) + monkeypatch.setattr(history_module.logger, 'exception', logged_errors.append) + + history.append_string('SELECT first_token') + wait_for_frecency_refresh(history) + assert history.frecency == {} + assert logged_errors == ['Failed to calculate history frecency.'] + + history.append_string('SELECT second_token') + wait_for_frecency_refresh(history) + assert history.frecency == {'recovered': 1.0} + + +def test_pending_frecency_refresh_retries_after_failure(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + history = FileHistoryWithTimestamp(tmp_path / 'history.txt', frecency_refresh_interval=1) + wait_for_frecency_refresh(history) + calculation_started = threading.Event() + release_calculation = threading.Event() + calls = 0 + + def calculate_frecency(entries, history_entries): + nonlocal calls + calls += 1 + if calls == 1: + calculation_started.set() + assert release_calculation.wait(timeout=5) + raise RuntimeError('calculation failed') + return {'recovered': 1.0} + + monkeypatch.setattr(history_module, '_calculate_frecency', calculate_frecency) + + history.append_string('SELECT first_token') + assert calculation_started.wait(timeout=5) + history.append_string('SELECT second_token') + release_calculation.set() + wait_for_frecency_refresh(history) + + assert calls == 2 + assert history.frecency == {'recovered': 1.0} + + +def test_frecency_thread_start_failure_can_be_retried(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + thread_class = history_module.threading.Thread + threads = [] + logged_errors: list[str] = [] + + class FailingThread: + def __init__(self, target, **kwargs): + self.history = target.__self__ + threads.append(self) + + def start(self): + raise RuntimeError('thread failed') + + monkeypatch.setattr(history_module.threading, 'Thread', FailingThread) + monkeypatch.setattr(history_module.logger, 'exception', logged_errors.append) + + history = FileHistoryWithTimestamp(tmp_path / 'history.txt') + + assert history is threads[0].history + assert history._frecency_thread is None + assert logged_errors == ['Failed to start history frecency calculation.'] + + monkeypatch.setattr(history_module.threading, 'Thread', thread_class) + history._request_frecency_refresh() + wait_for_frecency_refresh(history) + + assert history.frecency == {} def test_append_string_caches_and_stores_non_password_statement(tmp_path: Path, monkeypatch) -> None: diff --git a/test/pytests/test_sqlcompleter.py b/test/pytests/test_sqlcompleter.py index a3be727fe..c1e409c2b 100644 --- a/test/pytests/test_sqlcompleter.py +++ b/test/pytests/test_sqlcompleter.py @@ -380,6 +380,65 @@ def test_init_configures_indexed_column_suffix() -> None: assert completer.indexed_column_suffix == ' [indexed]' +def test_init_configures_frecency_sorting() -> None: + def provider() -> dict[str, float]: + return {'orders': 1.0} + + completer = SQLCompleter(frecency_provider=provider) + + assert completer.frecency_provider is provider + + +def test_get_completions_uses_frecency_before_prefix_length(monkeypatch) -> None: + completer = make_completer(frecency_provider=lambda: {'alphabet': 10.0}) + monkeypatch.setattr(mycli.sqlcompleter, 'suggest_type', lambda text, before: [{'type': 'column', 'tables': []}]) + monkeypatch.setattr(completer, 'populate_scoped_cols', lambda tables: ['ant', 'alphabet']) + monkeypatch.setattr(completer, 'populate_scoped_indexed_columns', lambda tables: []) + + result = [completion.text for completion in completer.get_completions(Document(text='SELECT a'), None)] + + assert result == ['alphabet', 'ant'] + + +def test_get_completions_preserves_stronger_fuzzy_match(monkeypatch) -> None: + completer = make_completer(frecency_provider=lambda: {'far': 100.0}) + monkeypatch.setattr(mycli.sqlcompleter, 'suggest_type', lambda text, before: [{'type': 'column', 'tables': []}]) + monkeypatch.setattr(completer, 'populate_scoped_cols', lambda tables: ['foo', 'far']) + monkeypatch.setattr(completer, 'populate_scoped_indexed_columns', lambda tables: []) + monkeypatch.setattr( + completer, + 'find_matches', + lambda *args, **kwargs: iter([('far', Fuzziness.RAPIDFUZZ), ('foo', Fuzziness.PERFECT)]), + ) + + result = [completion.text for completion in completer.get_completions(Document(text='SELECT x'), None)] + + assert result == ['foo', 'far'] + + +def test_naive_completions_use_live_frecency_provider() -> None: + frecency = {'bravo': 2.0} + completer = make_completer(smart_completion=False, frecency_provider=lambda: frecency) + completer.all_completions = {'alpha', 'bravo'} + + first = [completion.text for completion in completer.get_completions(Document(text=''), None)] + frecency = {'alpha': 3.0} + second = [completion.text for completion in completer.get_completions(Document(text=''), None)] + + assert first == ['bravo', 'alpha'] + assert second == ['alpha', 'bravo'] + + +def test_file_completions_preserve_rigid_ordering(monkeypatch) -> None: + completer = make_completer(frecency_provider=lambda: {'alpha': 100.0}) + monkeypatch.setattr(mycli.sqlcompleter, 'suggest_type', lambda text, before: [{'type': 'file_name'}]) + monkeypatch.setattr(completer, 'find_files', lambda word: iter([('zeta', 0), ('alpha', 0)])) + + result = [completion.text for completion in completer.get_completions(Document(text='/source '), None)] + + assert result == ['zeta', 'alpha'] + + def test_extend_metadata_helpers_and_logging(caplog) -> None: completer = make_completer() completer.set_dbname('missing')