diff --git a/CHANGELOG.rst b/CHANGELOG.rst index dc673df4..216f3f3f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,7 +1,9 @@ Next Release (TBD) ================== -* No changes yet. +* Fix ``end`` position offset for quoted identifier, literal, and raw string + literal tokens + (`issue #380 `__) 1.1.0 diff --git a/jmespath/lexer.py b/jmespath/lexer.py index 8db05e37..ea106fde 100644 --- a/jmespath/lexer.py +++ b/jmespath/lexer.py @@ -174,7 +174,7 @@ def _consume_literal(self): message="Bad token %s" % lexeme) token_len = self._position - start return {'type': 'literal', 'value': parsed_json, - 'start': start, 'end': token_len} + 'start': start, 'end': start + token_len} def _consume_quoted_identifier(self): start = self._position @@ -182,7 +182,7 @@ def _consume_quoted_identifier(self): try: token_len = self._position - start return {'type': 'quoted_identifier', 'value': loads(lexeme), - 'start': start, 'end': token_len} + 'start': start, 'end': start + token_len} except ValueError as e: error_message = str(e).split(':')[0] raise LexerError(lexer_position=start, @@ -194,7 +194,7 @@ def _consume_raw_string_literal(self): lexeme = self._consume_until("'").replace("\\'", "'") token_len = self._position - start return {'type': 'literal', 'value': lexeme, - 'start': start, 'end': token_len} + 'start': start, 'end': start + token_len} def _match_or_else(self, expected, match_type, else_type): start = self._position diff --git a/tests/test_lexer.py b/tests/test_lexer.py index fbae0608..162ee685 100644 --- a/tests/test_lexer.py +++ b/tests/test_lexer.py @@ -132,6 +132,27 @@ def test_position_multiple_tokens(self): ] ) + def test_position_quoted_identifier(self): + tokens = list(self.lexer.tokenize('foo."q"')) + quoted = [t for t in tokens if t['type'] == 'quoted_identifier'][0] + self.assertEqual(quoted['start'], 4) + self.assertEqual(quoted['end'], 6) + self.assertGreaterEqual(quoted['end'], quoted['start']) + + def test_position_literal_not_at_start(self): + tokens = list(self.lexer.tokenize('foo.`123`')) + literal = [t for t in tokens if t['type'] == 'literal'][0] + self.assertEqual(literal['start'], 4) + self.assertEqual(literal['end'], 8) + self.assertGreaterEqual(literal['end'], literal['start']) + + def test_position_raw_string_not_at_start(self): + tokens = list(self.lexer.tokenize("foo.'bar'")) + literal = [t for t in tokens if t['type'] == 'literal'][0] + self.assertEqual(literal['start'], 4) + self.assertEqual(literal['end'], 8) + self.assertGreaterEqual(literal['end'], literal['start']) + def test_adds_quotes_when_invalid_json(self): tokens = list(self.lexer.tokenize('`{{}`')) self.assertEqual(