gh-159672: Reject invalid literals during keyword replay

A trial can reject a bytes literal without a pending exception. Stop the decode-error path before it requires that exception, and preserve the original syntax error.

Author: pablogsal

Lib/test/test_traceback.py

@@ -1943,6 +1943,16 @@ def test_keyword_replay_respects_feature_version(self):
                 ast.parse(source, feature_version=(3, 9))
             self.assertIsNone(caught.exception._metadata)
 
+    @cpython_only
+    def test_keyword_trials_reject_invalid_literals(self):
+        for literal in ("b'é'", r"'\xzz'", r"'\U00110000'"):
+            for mode in ("exec", "eval", "single"):
+                source = "npt " + literal
+                with self.subTest(source=source, mode=mode):
+                    with self.assertRaises(SyntaxError) as caught:
+                        compile(source, "<test>", mode)
+                    self.assertIsNone(caught.exception._metadata)
+
     @cpython_only
     def test_keyword_trials_do_not_call_normalizer(self):
         with unittest.mock.patch("unicodedata.normalize") as normalize:

Parser/pegen_errors.c

@@ -104,6 +104,11 @@ _Pypegen_tokenizer_error(Parser *p)
 int
 _Pypegen_raise_decode_error(Parser *p)
 {
+    if (p->keyword_trial != NULL) {
+        // A trial can reject a literal without creating a syntax exception.
+        p->error_indicator = 1;
+        return -1;
+    }
     assert(PyErr_Occurred());
     const char *errtype = NULL;
     if (PyErr_ExceptionMatches(PyExc_UnicodeError)) {