gh-159672: Constrain literal conversion during keyword replay

Author: pablogsal

Include/internal/pycore_unicodeobject.h

@@ -337,10 +337,11 @@ PyAPI_FUNC(PyObject*) _PyUnicode_DecodeUnicodeEscapeInternal2(
     int *first_invalid_escape_char, /* on return, if not -1, contain the first
                                        invalid escaped char (<= 0xff) or invalid
                                        octal escape (> 0xff) in string. */
-    const char **first_invalid_escape_ptr); /* on return, if not NULL, may
+    const char **first_invalid_escape_ptr, /* on return, if not NULL, may
                                         point to the first invalid escaped
                                         char in string.
                                         May be NULL if errors is not NULL. */
+    bool allow_callbacks); /* Allow named escapes and registered error handlers. */
 
 /* --- Raw-Unicode-Escape Codecs ---------------------------------------------- */
 

Lib/test/test_traceback.py

@@ -1943,6 +1943,75 @@ def test_keyword_replay_respects_feature_version(self):
                 ast.parse(source, feature_version=(3, 9))
             self.assertIsNone(caught.exception._metadata)
 
+    @cpython_only
+    def test_keyword_trials_reject_invalid_literals(self):
+        for literal in ("b'é'", r"'\xzz'", r"'\U00110000'"):
+            for mode in ("exec", "eval", "single"):
+                source = "npt " + literal
+                with self.subTest(source=source, mode=mode):
+                    with self.assertRaises(SyntaxError) as caught:
+                        compile(source, "<test>", mode)
+                    self.assertIsNone(caught.exception._metadata)
+
+    @cpython_only
+    @requires_subprocess()
+    def test_keyword_trials_do_not_call_literal_hooks(self):
+        assert_python_ok('-c', textwrap.dedent(r'''
+            import codecs, sys, _pylong
+            calls = []
+
+            def forbidden(*args):
+                calls.append(args)
+                raise AssertionError("literal callback")
+
+            class WatchImports:
+                def find_spec(self, fullname, path=None, target=None):
+                    if fullname == "unicodedata":
+                        forbidden(fullname)
+
+            sys.meta_path.insert(0, WatchImports())
+            codecs.register_error("strict", forbidden)
+            _pylong.int_from_string = forbidden
+            sys.set_int_max_str_digits(0)
+            literals = [(r"'\N{SNOWMAN}'", False), (r"'\xzz'", False),
+                        (r"'\U00110000'", False), (r"f'\xzz'", False),
+                        (r"t'\xzz'", False), ("1" * 6001, False),
+                        (r"'\x61\u0062\U00000063'", True),
+                        (r"r'\N{SNOWMAN}'", True), (r"'\\N{SNOWMAN}'", True),
+                        (r"f'\x61{1}'", True), (r"t'\x61{1}'", True)]
+            for literal, expected in literals:
+                try:
+                    compile("impor math\nx = " + literal, "<test>", "exec")
+                except SyntaxError as error:
+                    assert (error._metadata is not None) == expected, literal
+                else:
+                    raise AssertionError(literal)
+            assert calls == [], calls
+            # A cached Unicode name table must not change the trial policy.
+            sys.meta_path.pop(0)
+            compile(r"x = '\N{SNOWMAN}'", "<test>", "exec")
+            try:
+                compile("impor math\nx = " + literals[0][0], "<test>", "exec")
+            except SyntaxError as error:
+                assert error._metadata is None
+            # The normal decoder must still honor registered error handlers.
+            try:
+                br"\xzz".decode("unicode_escape")
+            except AssertionError:
+                pass
+            assert len(calls) == 1, calls
+        '''))
+
+    @cpython_only
+    def test_keyword_trials_numeric_limit(self):
+        for digits, expected in ((640, True), (641, False)):
+            with self.subTest(digits=digits):
+                source = "x = " + "1" * digits
+                compile(source, "<test>", "exec")
+                with self.assertRaises(SyntaxError) as caught:
+                    compile("impor math\n" + source, "<test>", "exec")
+                self.assertEqual(caught.exception._metadata is not None, expected)
+
     @cpython_only
     def test_keyword_trials_do_not_call_normalizer(self):
         with unittest.mock.patch("unicodedata.normalize") as normalize:

Objects/unicodeobject.c

@@ -6548,7 +6548,8 @@ _PyUnicode_DecodeUnicodeEscapeInternal2(const char *s,
                                const char *errors,
                                Py_ssize_t *consumed,
                                int *first_invalid_escape_char,
-                               const char **first_invalid_escape_ptr)
+                               const char **first_invalid_escape_ptr,
+                               bool allow_callbacks)
 {
     const char *starts = s;
     const char *initial_starts = starts;
@@ -6708,6 +6709,10 @@ _PyUnicode_DecodeUnicodeEscapeInternal2(const char *s,
 
             /* \N{name} */
         case 'N':
+            if (!allow_callbacks) {
+                message = "named Unicode escapes are disabled";
+                goto error;
+            }
             ucnhash_capi = _PyUnicode_GetNameCAPI();
             if (ucnhash_capi == NULL) {
                 PyErr_SetString(
@@ -6767,6 +6772,14 @@ _PyUnicode_DecodeUnicodeEscapeInternal2(const char *s,
         }
       error:;
         Py_ssize_t endinpos = s-starts;
+        if (!allow_callbacks) {
+            make_decode_exception(&exc, "unicodeescape", starts, end - starts,
+                                  startinpos, endinpos, message);
+            if (exc != NULL) {
+                PyErr_SetObject(PyExc_UnicodeDecodeError, exc);
+            }
+            goto onError;
+        }
         writer.min_length = end - s + writer.pos;
         if (unicode_decode_call_errorhandler_writer(
                 errors, &errorHandler,
@@ -6803,7 +6816,8 @@ _PyUnicode_DecodeUnicodeEscapeStateful(const char *s,
     PyObject *result = _PyUnicode_DecodeUnicodeEscapeInternal2(s, size, errors,
                                                       consumed,
                                                       &first_invalid_escape_char,
-                                                      &first_invalid_escape_ptr);
+                                                      &first_invalid_escape_ptr,
+                                                      true);
     if (result == NULL)
         return NULL;
     if (first_invalid_escape_char != -1) {

Parser/pegen.c

@@ -1,5 +1,6 @@
 #include <Python.h>
 #include "pycore_ast.h"           // _PyAST_Validate(),
+#include "pycore_long.h"          // _PY_LONG_MAX_STR_DIGITS_THRESHOLD
 #include "pycore_pystate.h"       // _PyThreadState_GET()
 #include "pycore_parser.h"        // _PYPEGEN_NSTATISTICS
 #include "pycore_pyerrors.h"      // PyExc_IncompleteInputError
@@ -1422,6 +1423,15 @@ keyword_typo_metadata(Parser *p, PySyntaxErrorObject *error)
     {
         return NULL;
     }
+    // Conservatively bound numeric conversion before any trial. This also
+    // excludes the Python helper without changing normal number parsing.
+    for (int i = 0; i < p->fill; i++) {
+        Token *token = p->tokens[i];
+        if (token->type == NUMBER &&
+            PyBytes_GET_SIZE(token->bytes) > _PY_LONG_MAX_STR_DIGITS_THRESHOLD) {
+            return NULL;
+        }
+    }
     KeywordTypo candidates[MAX_KEYWORD_TYPO_CANDIDATES];
     int count = collect_keyword_typos(p, error, candidates);
     if (count <= 0) {

Parser/pegen_errors.c

@@ -104,6 +104,11 @@ _Pypegen_tokenizer_error(Parser *p)
 int
 _Pypegen_raise_decode_error(Parser *p)
 {
+    if (p->keyword_trial != NULL) {
+        // A trial can reject a literal without creating a syntax exception.
+        p->error_indicator = 1;
+        return -1;
+    }
     assert(PyErr_Occurred());
     const char *errtype = NULL;
     if (PyErr_ExceptionMatches(PyExc_UnicodeError)) {

Parser/string_parser.c

@@ -195,7 +195,11 @@ decode_unicode_with_escapes(Parser *parser, const char *s, size_t len, Token *t)
     const char *first_invalid_escape_ptr;
     v = _PyUnicode_DecodeUnicodeEscapeInternal2(buf, (Py_ssize_t)len, NULL, NULL,
                                                 &first_invalid_escape_char,
-                                                &first_invalid_escape_ptr);
+                                                &first_invalid_escape_ptr,
+                                                parser->keyword_trial == NULL);
+    if (v == NULL && parser->keyword_trial != NULL) {
+        parser->error_indicator = 1;
+    }
 
     // HACK: later we can simply pass the line no, since we don't preserve the tokens
     // when we are decoding the string but we preserve the line numbers.