gh-159672: Generate keyword typo hints in the parser

Author: pablogsal

Lib/test/test_traceback.py

@@ -1847,6 +1847,21 @@ def test_keyword_suggestions_from_command_string(self):
                 stderr_text = stderr.decode('utf-8')
                 self.assertIn(f"Did you mean '{expected_kw}'", stderr_text)
 
+    @cpython_only
+    def test_keyword_suggestions_from_single_input(self):
+        for source, keyword in self.TYPO_CASES:
+            for ending in ('', '\n'):
+                with self.subTest(source=source, ending=ending):
+                    with self.assertRaises(SyntaxError) as caught:
+                        compile(source + ending, "<test>", "single")
+                    self.assertEqual(caught.exception._metadata[0], keyword)
+                    with (unittest.mock.patch("builtins.compile") as compile_mock,
+                          unittest.mock.patch("builtins.open") as open_mock):
+                        text = ''.join(traceback.format_exception_only(caught.exception))
+                    self.assertIn(f"Did you mean '{keyword}'", text)
+                    compile_mock.assert_not_called()
+                    open_mock.assert_not_called()
+
     def test_no_keyword_suggestion_for_comma_errors(self):
         # When the parser identifies a missing comma, don't suggest
         # bogus keyword replacements like 'print' -> 'not'
@@ -1864,6 +1879,75 @@ def test_no_keyword_suggestion_for_comma_errors(self):
         self.assertIn("Perhaps you forgot a comma", stderr_text)
         self.assertNotIn("Did you mean", stderr_text)
 
+    @cpython_only
+    def test_parser_suggestions_need_no_formatting_work(self):
+        cases = [
+            ("def f():\n    returm value", "return", 2, 5),
+            ("é = 1; impor math", "import", 1, 8),
+            ("a = 1\nif True:\n    pass\nelso:\n    pass", "else", 4, 1),
+            ("[123 fur x\nin range(3)\nif x]", "for", 1, 6),
+            ("mach x:\n    case 1: pass", "match", 1, 1),
+            ("match x:\n    cse 1: pass", "case", 2, 5),
+            ("typ Alias = int", "type", 1, 1),
+            ("lazi import math", "lazy", 1, 1),
+            ("long_name = 1\n" * 85 + "whille True:\n    pass", "while", 86, 1),
+        ]
+        for source, keyword, lineno, offset in cases:
+            with self.subTest(source=source):
+                with self.assertRaises(SyntaxError) as caught:
+                    compile(source, "<test>", "exec")
+                error = caught.exception
+                original = error.args
+                with (unittest.mock.patch("builtins.compile") as compile_mock,
+                      unittest.mock.patch("builtins.open") as open_mock):
+                    exc = traceback.TracebackException.from_exception(error)
+                    first = ''.join(exc.format_exception_only())
+                    self.assertEqual(first, ''.join(exc.format_exception_only()))
+                compile_mock.assert_not_called()
+                open_mock.assert_not_called()
+                self.assertIn(f"Did you mean '{keyword}'?", first)
+                self.assertEqual((exc.lineno, exc.offset), (str(lineno), offset))
+                self.assertEqual(error.args, original)
+
+    @cpython_only
+    def test_keyword_trials_do_not_repeat_warnings(self):
+        import warnings
+        for prefix in ("value = '\\q'\n", "from . lazy import x\n"):
+            with self.subTest(prefix=prefix), warnings.catch_warnings(record=True) as caught:
+                warnings.simplefilter("always")
+                with self.assertRaises(SyntaxError) as error:
+                    compile(prefix + "impor math", "<test>", "exec")
+                self.assertEqual(len(caught), 1)
+                self.assertEqual(error.exception._metadata[0], "import")
+
+    @cpython_only
+    def test_keyword_replay_requires_complete_valid_input(self):
+        for source in ("dell 1", "impor math\nother =", "whille True:\npass",
+                       "switch +:\n case:", "function f(=):\n pass", "(mat x)",
+                       "impor math\n" + "pass\n" * 3000):
+            with self.subTest(source=source):
+                with self.assertRaises(SyntaxError) as caught:
+                    compile(source, "<test>", "exec")
+                # Action errors and a cache which stopped before EOF must not
+                # count as successful repairs.
+                self.assertIsNone(caught.exception._metadata)
+
+    @cpython_only
+    def test_keyword_replay_respects_feature_version(self):
+        import ast
+        for source in ("mach x:\n    case 1: pass", "mach x:", "switch x:\n case:"):
+            with self.subTest(source=source), self.assertRaises(SyntaxError) as caught:
+                ast.parse(source, feature_version=(3, 9))
+            self.assertIsNone(caught.exception._metadata)
+
+    @cpython_only
+    def test_keyword_trials_do_not_call_normalizer(self):
+        with unittest.mock.patch("unicodedata.normalize") as normalize:
+            with self.assertRaises(SyntaxError) as caught:
+                compile("whille True:\n    café = 1", "<test>", "exec")
+        normalize.assert_not_called()
+        self.assertIsNone(caught.exception._metadata)
+
 @requires_debug_ranges()
 @force_not_colorized_test_class
 class PurePythonTracebackErrorCaretTests(

Lib/traceback.py

@@ -10,10 +10,6 @@
 import textwrap
 import types
 import warnings
-import codeop
-import keyword
-import tokenize
-import io
 import importlib.util
 import pathlib
 
@@ -1182,7 +1178,13 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None,
             self.end_offset = exc_value.end_offset
             self.msg = exc_value.msg
             self._is_syntax_error = True
-            self._exc_metadata = getattr(exc_value, "_metadata", None)
+            metadata = getattr(exc_value, "_metadata", None)
+            if isinstance(metadata, tuple) and len(metadata) == 6:
+                (suggestion, self.text, lineno, self.offset,
+                 end_lineno, self.end_offset) = metadata
+                self.lineno = str(lineno)
+                self.end_lineno = str(end_lineno)
+                self.msg = f"invalid syntax. Did you mean '{suggestion}'?"
         elif exc_type and issubclass(exc_type, ImportError) and \
                 getattr(exc_value, "name_from", None) is not None:
             wrong_name = getattr(exc_value, "name_from", None)
@@ -1413,122 +1415,6 @@ def format_exception_only(self, *, show_group=False, _depth=0, **kwargs):
             for ex in self.exceptions:
                 yield from ex.format_exception_only(show_group=show_group, _depth=_depth+1, colorize=colorize)
 
-    def _find_keyword_typos(self):
-        assert self._is_syntax_error
-        try:
-            import _suggestions
-        except ImportError:
-            _suggestions = None
-
-        # Only try to find keyword typos if there is no custom message
-        if self.msg != "invalid syntax" and "Perhaps you forgot a comma" not in self.msg:
-            return
-
-        if not self._exc_metadata:
-            return
-
-        line, offset, source = self._exc_metadata
-        end_line = int(self.lineno) if self.lineno is not None else 0
-        lines = None
-        from_filename = False
-
-        if source is None:
-            if self.filename:
-                try:
-                    with open(self.filename) as f:
-                        lines = f.read().splitlines()
-                except Exception:
-                    line, end_line, offset = 0,1,0
-                else:
-                    from_filename = True
-            lines = lines if lines is not None else self.text.splitlines()
-        else:
-            lines = source.splitlines()
-
-        error_code = lines[line -1 if line > 0 else 0:end_line]
-        error_code = textwrap.dedent('\n'.join(error_code))
-
-        # Do not continue if the source is too large
-        if len(error_code) > 1024:
-            return
-
-        # If the original code doesn't raise SyntaxError, we can't validate
-        # that a keyword replacement actually fixes anything
-        try:
-            codeop.compile_command(error_code, symbol="exec", flags=codeop.PyCF_ONLY_AST)
-        except SyntaxError:
-            pass  # Good - the original code has a syntax error we might fix
-        else:
-            return  # Original code compiles or is incomplete - can't validate fixes
-
-        error_lines = error_code.splitlines()
-        tokens = tokenize.generate_tokens(io.StringIO(error_code).readline)
-        tokens_left_to_process = 10
-        import difflib
-        for token in tokens:
-            start, end = token.start, token.end
-            if token.type != tokenize.NAME:
-                continue
-            # Only consider NAME tokens on the same line as the error
-            the_end = end_line if line == 0 else end_line + 1
-            if from_filename and token.start[0]+line != the_end:
-                continue
-            wrong_name = token.string
-            if wrong_name in keyword.kwlist:
-                continue
-
-            # Limit the number of valid tokens to consider to not spend
-            # to much time in this function
-            tokens_left_to_process -= 1
-            if tokens_left_to_process < 0:
-                break
-            # Limit the number of possible matches to try
-            max_matches = 3
-            matches = []
-
-            hint = _get_cross_language_keyword_hint(wrong_name)
-            if hint:
-                matches.append(hint)
-            if _suggestions is not None:
-                suggestion = _suggestions._generate_suggestions(keyword.kwlist + keyword.softkwlist, wrong_name)
-                if suggestion:
-                    matches.append(suggestion)
-            matches.extend(
-                difflib.get_close_matches(
-                    wrong_name,
-                    keyword.kwlist + keyword.softkwlist,
-                    n=max_matches,
-                    cutoff=0.5
-                )
-            )
-            matches = matches[:max_matches]
-            for suggestion in matches:
-                if not suggestion or suggestion == wrong_name:
-                    continue
-                # Try to replace the token with the keyword
-                the_lines = error_lines.copy()
-                the_line = the_lines[start[0] - 1][:]
-                chars = list(the_line)
-                chars[token.start[1]:token.end[1]] = suggestion
-                the_lines[start[0] - 1] = ''.join(chars)
-                code = '\n'.join(the_lines)
-
-                # Check if it works
-                try:
-                    codeop.compile_command(code, symbol="exec", flags=codeop.PyCF_ONLY_AST)
-                except SyntaxError:
-                    continue
-
-                # Keep token.line but handle offsets correctly
-                self.text = token.line
-                self.offset = token.start[1] + 1
-                self.end_offset = token.end[1] + 1
-                self.lineno = start[0]
-                self.end_lineno = end[0]
-                self.msg = f"invalid syntax. Did you mean '{suggestion}'?"
-                return
-
-
     def _format_syntax_error(self, stype, **kwargs):
         """Format SyntaxError exceptions (internal helper)."""
         # Show exactly where the problem was found.
@@ -1555,9 +1441,6 @@ def _format_syntax_error(self, stype, **kwargs):
             # text  = "   foo\n"
             # rtext = "   foo"
             # ltext =    "foo"
-            with suppress(Exception):
-                self._find_keyword_typos()
-            text = self.text
             rtext = text.rstrip('\n')
             ltext = rtext.lstrip(' \n\f')
             spaces = len(rtext) - len(ltext)
@@ -1798,17 +1681,6 @@ def print(self, *, file=None, chain=True, **kwargs):
 })
 
 
-# Cross-language keyword suggestions.
-_CROSS_LANGUAGE_KEYWORD_HINTS = frozendict({
-    # C/C++ equivalents
-    'switch': 'match',
-    'delete': 'del',
-    # function define equivalents
-    'function': 'def',
-    'func': 'def',
-    'void': 'def',
-})
-
 def _substitution_cost(ch_a, ch_b):
     if ch_a == ch_b:
         return 0
@@ -1888,12 +1760,6 @@ def _get_cross_language_hint(obj, wrong_name):
     return None
 
 
-def _get_cross_language_keyword_hint(wrong_name):
-    """Check if wrong_name is a common keyword from another language
-    """
-    return _CROSS_LANGUAGE_KEYWORD_HINTS.get(wrong_name)
-
-
 def _get_safe___dir__(obj):
     # Use obj.__dir__() to avoid a TypeError when calling dir(obj).
     # See gh-131001 and gh-139933.

Misc/NEWS.d/next/Core and Builtins/2026-10-18-13-00-00.gh-issue-159672.keyword-typos.rst

@@ -0,0 +1,3 @@
+The parser now generates keyword typo suggestions from cached tokens. The
+:mod:`traceback` formatter displays these suggestions without source
+recompilation.

Parser/action_helpers.c

@@ -2072,7 +2072,7 @@ _warn_relative_import_of_lazy(Parser *p, asdl_seq *dots, expr_ty module)
     // the module name is insignificant, so this is parsed exactly like
     // `from .lazy import x` (an import of the relative module "lazy"), but it
     // is most likely a transposition of `lazy from . import x` (PEP 810).
-    if (p->call_invalid_rules) {
+    if (p->call_invalid_rules || p->keyword_trial != NULL) {
         return 0;
     }
 
@@ -2134,7 +2134,9 @@ _PyPegen_checked_from_import(Parser *p, asdl_seq *dots, expr_ty module_name,
             alias_ty alias = asdl_seq_GET(names, i);
             if (PyUnicode_CompareWithASCIIString(alias->name, "barry_as_FLUFL") == 0) {
                 p->flags |= PyPARSE_BARRY_AS_BDFL;
-                p->tok->barry_as_bdfl = 1;
+                if (p->keyword_trial == NULL) {
+                    p->tok->barry_as_bdfl = 1;
+                }
             }
         }
     }