Skip to content

Fix token 'end' offset for quoted identifiers, raw strings, and literals #393

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ApparentlyTejas wants to merge 1 commit into
base: develop
Choose a base branch
from
Open

Fix token 'end' offset for quoted identifiers, raw strings, and literals #393

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions jmespath/lexer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -174,15 +174,15 @@ def _consume_literal(self):
message="Bad token %s" % lexeme)
token_len = self._position - start
return {'type': 'literal', 'value': parsed_json,
'start': start, 'end': token_len}
'start': start, 'end': start + token_len}

def _consume_quoted_identifier(self):
start = self._position
lexeme = '"' + self._consume_until('"') + '"'
try:
token_len = self._position - start
return {'type': 'quoted_identifier', 'value': loads(lexeme),
'start': start, 'end': token_len}
'start': start, 'end': start + token_len}
except ValueError as e:
error_message = str(e).split(':')[0]
raise LexerError(lexer_position=start,
Expand All @@ -194,7 +194,7 @@ def _consume_raw_string_literal(self):
lexeme = self._consume_until("'").replace("\\'", "'")
token_len = self._position - start
return {'type': 'literal', 'value': lexeme,
'start': start, 'end': token_len}
'start': start, 'end': start + token_len}

def _match_or_else(self, expected, match_type, else_type):
start = self._position
Expand Down
49 changes: 49 additions & 0 deletions tests/test_lexer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,55 @@ def test_adds_quotes_when_invalid_json(self):
]
)

def test_position_of_quoted_identifier_not_at_start(self):
# Regression test: 'end' used to be set to the token's length
# instead of start + length, so it was wrong (and less than
# 'start') whenever the token wasn't at the start of the
# expression.
tokens = list(self.lexer.tokenize('foo."q"'))
self.assertEqual(
tokens,
[{'type': 'unquoted_identifier', 'value': 'foo',
'start': 0, 'end': 3},
{'type': 'dot', 'value': '.',
'start': 3, 'end': 4},
{'type': 'quoted_identifier', 'value': 'q',
'start': 4, 'end': 7},
{'type': 'eof', 'value': '',
'start': 7, 'end': 7},
]
)

def test_position_of_raw_string_literal_not_at_start(self):
tokens = list(self.lexer.tokenize("foo.'bar'"))
self.assertEqual(
tokens,
[{'type': 'unquoted_identifier', 'value': 'foo',
'start': 0, 'end': 3},
{'type': 'dot', 'value': '.',
'start': 3, 'end': 4},
{'type': 'literal', 'value': 'bar',
'start': 4, 'end': 9},
{'type': 'eof', 'value': '',
'start': 9, 'end': 9},
]
)

def test_position_of_backtick_literal_not_at_start(self):
tokens = list(self.lexer.tokenize('foo.`"bar"`'))
self.assertEqual(
tokens,
[{'type': 'unquoted_identifier', 'value': 'foo',
'start': 0, 'end': 3},
{'type': 'dot', 'value': '.',
'start': 3, 'end': 4},
{'type': 'literal', 'value': 'bar',
'start': 4, 'end': 11},
{'type': 'eof', 'value': '',
'start': 11, 'end': 11},
]
)

def test_unknown_character(self):
with self.assertRaises(LexerError) as e:
tokens = list(self.lexer.tokenize('foo[0^]'))
Expand Down