From b5a74bfc0b534c69318ca4a73c8cd81f54ccdcf3 Mon Sep 17 00:00:00 2001
From: ApparentlyTejas <tejas.malhotra.14@gmail.com>
Date: Thu, 5 Nov 2026 00:58:49 +0100
Subject: [PATCH] Fix token 'end' offset for quoted identifiers, raw strings,
 and literals

_consume_literal, _consume_quoted_identifier, and
_consume_raw_string_literal all set the token's 'end' to the token's
length instead of start + length, so 'end' was wrong (and less than
'start') whenever the token wasn't at the start of the expression.
Every other token type in the lexer already uses start + len(...).

Fixes the quoted-identifier case from the report; the same bug also
affects backtick literals and raw string literals, so those are
fixed and covered too. Search results aren't affected, only the
position information.
---
 jmespath/lexer.py   |  6 +++---
 tests/test_lexer.py | 49 +++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 52 insertions(+), 3 deletions(-)

diff --git a/jmespath/lexer.py b/jmespath/lexer.py
index 8db05e37..ea106fde 100644
--- a/jmespath/lexer.py
+++ b/jmespath/lexer.py
@@ -174,7 +174,7 @@ def _consume_literal(self):
                                  message="Bad token %s" % lexeme)
         token_len = self._position - start
         return {'type': 'literal', 'value': parsed_json,
-                'start': start, 'end': token_len}
+                'start': start, 'end': start + token_len}
 
     def _consume_quoted_identifier(self):
         start = self._position
@@ -182,7 +182,7 @@ def _consume_quoted_identifier(self):
         try:
             token_len = self._position - start
             return {'type': 'quoted_identifier', 'value': loads(lexeme),
-                    'start': start, 'end': token_len}
+                    'start': start, 'end': start + token_len}
         except ValueError as e:
             error_message = str(e).split(':')[0]
             raise LexerError(lexer_position=start,
@@ -194,7 +194,7 @@ def _consume_raw_string_literal(self):
         lexeme = self._consume_until("'").replace("\\'", "'")
         token_len = self._position - start
         return {'type': 'literal', 'value': lexeme,
-                'start': start, 'end': token_len}
+                'start': start, 'end': start + token_len}
 
     def _match_or_else(self, expected, match_type, else_type):
         start = self._position
diff --git a/tests/test_lexer.py b/tests/test_lexer.py
index fbae0608..ba98bf38 100644
--- a/tests/test_lexer.py
+++ b/tests/test_lexer.py
@@ -143,6 +143,55 @@ def test_adds_quotes_when_invalid_json(self):
              ]
         )
 
+    def test_position_of_quoted_identifier_not_at_start(self):
+        # Regression test: 'end' used to be set to the token's length
+        # instead of start + length, so it was wrong (and less than
+        # 'start') whenever the token wasn't at the start of the
+        # expression.
+        tokens = list(self.lexer.tokenize('foo."q"'))
+        self.assertEqual(
+            tokens,
+            [{'type': 'unquoted_identifier', 'value': 'foo',
+              'start': 0, 'end': 3},
+             {'type': 'dot', 'value': '.',
+              'start': 3, 'end': 4},
+             {'type': 'quoted_identifier', 'value': 'q',
+              'start': 4, 'end': 7},
+             {'type': 'eof', 'value': '',
+              'start': 7, 'end': 7},
+             ]
+        )
+
+    def test_position_of_raw_string_literal_not_at_start(self):
+        tokens = list(self.lexer.tokenize("foo.'bar'"))
+        self.assertEqual(
+            tokens,
+            [{'type': 'unquoted_identifier', 'value': 'foo',
+              'start': 0, 'end': 3},
+             {'type': 'dot', 'value': '.',
+              'start': 3, 'end': 4},
+             {'type': 'literal', 'value': 'bar',
+              'start': 4, 'end': 9},
+             {'type': 'eof', 'value': '',
+              'start': 9, 'end': 9},
+             ]
+        )
+
+    def test_position_of_backtick_literal_not_at_start(self):
+        tokens = list(self.lexer.tokenize('foo.`"bar"`'))
+        self.assertEqual(
+            tokens,
+            [{'type': 'unquoted_identifier', 'value': 'foo',
+              'start': 0, 'end': 3},
+             {'type': 'dot', 'value': '.',
+              'start': 3, 'end': 4},
+             {'type': 'literal', 'value': 'bar',
+              'start': 4, 'end': 11},
+             {'type': 'eof', 'value': '',
+              'start': 11, 'end': 11},
+             ]
+        )
+
     def test_unknown_character(self):
         with self.assertRaises(LexerError) as e:
             tokens = list(self.lexer.tokenize('foo[0^]'))