From 60d6a7bb9f1d932573c957162a72d617df82b2df Mon Sep 17 00:00:00 2001 From: Ajit Pratap Singh Date: Tue, 20 Oct 2026 12:44:58 +0530 Subject: [PATCH] fix(tokenizer): point unterminated-literal errors at the opener Unterminated quoted identifiers (newline), dollar-quoted strings and SQL Server bracket identifiers built error locations from raw cursor fields, reporting column 0 or the end of input. All tokenizer and StringLiteralReader error locations now go through the same rule as token spans (1-based line, byte column, a tab counting as one column; locationAt shares lineLocation with toSQLPosition) and point at the opening delimiter (escape errors: the character after the backslash). Golden digests regenerated; only error locations changed (verified: 0 token, error-code or message differences across 153 changed (entry, dialect) results). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KTj1PEbxzVCETkeYrZKcrE --- CHANGELOG.md | 12 ++ pkg/sql/tokenizer/error_location_test.go | 165 ++++++++++++++++++ pkg/sql/tokenizer/string_literal.go | 10 +- .../testdata/tokenizer_literal_golden.json | 68 +++++--- pkg/sql/tokenizer/tokenizer.go | 66 ++++--- 5 files changed, 258 insertions(+), 63 deletions(-) create mode 100644 pkg/sql/tokenizer/error_location_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b02ff2c1..6aa66845b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -370,6 +370,18 @@ acceptance, CLI refusals), so read **Behavior changes** before upgrading. - **Readable-style output that did not re-parse**: inner `;` and `;;` removed (#602, #631); readable-style re-parse failures over the testdata corpus fell from 36 to 6. +- **Tokenizer error locations point at the offending construct**: an + unterminated dollar-quoted string (`$$`/`$tag$`) and an unterminated SQL + Server `[identifier]` now report their opening delimiter instead of the end of + input, and a double-quoted identifier broken by a newline now reports its + opening quote on the correct column. These errors, invalid escape sequences + (which point at the character after the backslash) and `StringLiteralReader` + errors now use the same rule as token spans: 1-based line, 1-based byte + column, a tab counting as one column. Before, they used internal rune counts, + so the column drifted after multibyte characters and could be 0, and a dollar + quote could report the wrong line. The two errors with no position + (`keywords cannot be nil`, input too large) report column 1 instead of 0. + Error codes and messages are unchanged. ### Performance Numbers are quoted from each PR (benchstat on an Apple M4 unless stated; the diff --git a/pkg/sql/tokenizer/error_location_test.go b/pkg/sql/tokenizer/error_location_test.go new file mode 100644 index 000000000..64b90c57c --- /dev/null +++ b/pkg/sql/tokenizer/error_location_test.go @@ -0,0 +1,165 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tokenizer + +import ( + "errors" + "strings" + "testing" + + gerrors "github.com/ajitpratap0/GoSQLX/pkg/errors" + "github.com/ajitpratap0/GoSQLX/pkg/models" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" +) + +// refLocation is an independent reference for the public location rule: +// 1-based line, 1-based byte column, every byte (a tab included) one column. +func refLocation(input string, idx int) models.Location { + line, lineStart := 1, 0 + for i := 0; i < idx; i++ { + if input[i] == '\n' { + line++ + lineStart = i + 1 + } + } + return models.Location{Line: line, Column: 1 + idx - lineStart} +} + +// TestTokenizer_ErrorLocations checks that tokenizer errors report the byte +// that starts the offending construct (the opening delimiter of an +// unterminated literal, the escape character of a bad escape) under the same +// byte-based rule as token spans. +func TestTokenizer_ErrorLocations(t *testing.T) { + cases := []struct { + name string + dialect keywords.SQLDialect + sql string + // at is the substring whose first byte is the reported location. + at string + want models.Location + }{ + // Double-quoted identifier broken by a newline. + {"dquote newline first column", keywords.DialectPostgreSQL, "\n\"\n", "\"", models.Location{Line: 2, Column: 1}}, + {"dquote newline after tabs", keywords.DialectPostgreSQL, "SELECT\t\t\"a\nb\"", "\"", models.Location{Line: 1, Column: 9}}, + {"dquote newline after multibyte", keywords.DialectPostgreSQL, "SELECT 'é€𝄞', \"x\ny\"", "\"", models.Location{Line: 1, Column: 21}}, + {"dquote newline multi-line", keywords.DialectPostgreSQL, "SELECT 1\nFROM t\nWHERE\t\"a\nb\"", "\"", models.Location{Line: 3, Column: 7}}, + + // Unterminated dollar-quoted string: the tag's leading '$'. + {"dollar after tab", keywords.DialectPostgreSQL, "SELECT\t$$abc", "$$", models.Location{Line: 1, Column: 8}}, + {"dollar tag multi-line", keywords.DialectPostgreSQL, "SELECT 'é€',\n\t$tag$body\nmore", "$tag$", models.Location{Line: 2, Column: 2}}, + {"dollar after multibyte", keywords.DialectPostgreSQL, "SELECT 'é€𝄞' || $$abc", "$$", models.Location{Line: 1, Column: 23}}, + + // Unterminated SQL Server bracket identifier: the '['. + {"bracket after tab", keywords.DialectSQLServer, "SELECT\t[col", "[", models.Location{Line: 1, Column: 8}}, + {"bracket multi-line multibyte", keywords.DialectSQLServer, "SELECT N'é€',\n\t\t[abc def", "[", models.Location{Line: 2, Column: 3}}, + + // Invalid escape: the character after the backslash. + {"bad escape after tab and multibyte", keywords.DialectMySQL, "SELECT\t'é\\q'", "q'", models.Location{Line: 1, Column: 12}}, + {"bad escape multi-line", keywords.DialectMySQL, "SELECT 'a\nb€',\n\t'x\\q'", "q'", models.Location{Line: 3, Column: 5}}, + + // Regression guards for paths that already used the opener. + {"single quote multi-line", keywords.DialectPostgreSQL, "SELECT\t'é',\n\t'abc", "'abc", models.Location{Line: 2, Column: 2}}, + {"block comment after multibyte", keywords.DialectPostgreSQL, "SELECT 'é€'\t/* never closed", "/*", models.Location{Line: 1, Column: 16}}, + {"backtick after tab", keywords.DialectMySQL, "SELECT\t`a", "`", models.Location{Line: 1, Column: 8}}, + {"triple quote after tab", keywords.DialectPostgreSQL, "SELECT\t'''abc", "'''", models.Location{Line: 1, Column: 8}}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + idx := strings.Index(tc.sql, tc.at) + if idx < 0 { + t.Fatalf("marker %q not in %q", tc.at, tc.sql) + } + if ref := refLocation(tc.sql, idx); ref != tc.want { + t.Fatalf("case is inconsistent: byte %d maps to %+v, want %+v", idx, ref, tc.want) + } + + tkz := GetTokenizer() + defer PutTokenizer(tkz) + tkz.SetDialect(tc.dialect) + + check := func(api string, err error) { + t.Helper() + if err == nil { + t.Fatalf("%s(%q): expected an error", api, tc.sql) + } + var se *gerrors.Error + if !errors.As(err, &se) { + t.Fatalf("%s: got %T, want *errors.Error", api, err) + } + if se.Location != tc.want { + t.Errorf("%s(%q): location %+v, want %+v", api, tc.sql, se.Location, tc.want) + } + } + _, err := tkz.Tokenize([]byte(tc.sql)) + check("Tokenize", err) + _, err = tkz.TokenizeString(tc.sql) + check("TokenizeString", err) + }) + } +} + +// TestStringLiteralReader_ErrorLocations covers the standalone reader, which +// has no line index and derives locations from the byte offset directly. +func TestStringLiteralReader_ErrorLocations(t *testing.T) { + cases := []struct { + name string + input string + start int // byte offset of the opening quote + want models.Location + }{ + {"unterminated after tab and multibyte", "é\t'abc", 3, models.Location{Line: 1, Column: 4}}, + {"newline on second line", "x\n\t'ab\ncd'", 3, models.Location{Line: 2, Column: 2}}, + {"bad escape", "\t'é\\q'", 1, models.Location{Line: 1, Column: 6}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + pos := NewPosition(1, tc.start) + _, err := NewStringLiteralReader([]byte(tc.input), &pos, '\'').ReadStringLiteral() + if err == nil { + t.Fatalf("expected an error for %q", tc.input) + } + var se *gerrors.Error + if !errors.As(err, &se) { + t.Fatalf("got %T, want *errors.Error", err) + } + if se.Location != tc.want { + t.Errorf("location %+v, want %+v", se.Location, tc.want) + } + }) + } +} + +// TestPosition_LocationMatchesTokenSpans checks that Position.Location applies +// the same rule as token spans. +func TestPosition_LocationMatchesTokenSpans(t *testing.T) { + tkz := GetTokenizer() + defer PutTokenizer(tkz) + + input := "SELECT 1\n\téü\tx" + tokens, err := tkz.TokenizeString(input) + if err != nil { + t.Fatalf("TokenizeString: %v", err) + } + idx := strings.Index(input, "x") + got := NewPosition(1, idx).Location(tkz) + if want := (models.Location{Line: 2, Column: 7}); got != want { + t.Fatalf("Location() = %+v, want %+v", got, want) + } + last := tokens[len(tokens)-2] // the final token before EOF + if last.Token.Value != "x" || last.Start != got { + t.Fatalf("token %q starts at %+v, Location() = %+v", last.Token.Value, last.Start, got) + } +} diff --git a/pkg/sql/tokenizer/string_literal.go b/pkg/sql/tokenizer/string_literal.go index 3f55c3597..0b5a722fa 100644 --- a/pkg/sql/tokenizer/string_literal.go +++ b/pkg/sql/tokenizer/string_literal.go @@ -58,11 +58,13 @@ func (r *StringLiteralReader) ReadStringLiteral() (models.Token, error) { ch = quotedRune(ch, strict) if ch == '\\' { - // Handle escape sequences + // Handle escape sequences. Errors point at the character after + // the backslash (or end of input), matching Tokenizer. + escChar := r.pos.Index + 1 if err := r.handleEscapeSequence(&buf); err != nil { return models.Token{}, errors.InvalidSyntaxError( fmt.Sprintf("invalid escape sequence: %v", err), - models.Location{Line: r.pos.Line, Column: r.pos.Column}, + locationAt(r.input, escChar), string(r.input), ) } @@ -71,7 +73,7 @@ func (r *StringLiteralReader) ReadStringLiteral() (models.Token, error) { if ch == '\n' { return models.Token{}, errors.UnterminatedStringError( - models.Location{Line: startPos.Line, Column: startPos.Column}, + locationAt(r.input, startPos.Index), string(r.input), ) } @@ -105,7 +107,7 @@ func (r *StringLiteralReader) ReadStringLiteral() (models.Token, error) { } return models.Token{}, errors.UnterminatedStringError( - models.Location{Line: startPos.Line, Column: startPos.Column}, + locationAt(r.input, startPos.Index), string(r.input), ) } diff --git a/pkg/sql/tokenizer/testdata/tokenizer_literal_golden.json b/pkg/sql/tokenizer/testdata/tokenizer_literal_golden.json index d8b78aa01..f3b3ffc55 100644 --- a/pkg/sql/tokenizer/testdata/tokenizer_literal_golden.json +++ b/pkg/sql/tokenizer/testdata/tokenizer_literal_golden.json @@ -107,7 +107,7 @@ {"id":"file:../../../testdata/oracle/16_decode.sql","content_sha":"1e26890fd48c31779505b7a003e47454f951802e0b996c6aa1f98a3bf68ce845","digests":["bbe7e5ab71996250","bbe7e5ab71996250","bbe7e5ab71996250","bbe7e5ab71996250","bbe7e5ab71996250","bbe7e5ab71996250","bbe7e5ab71996250","bbe7e5ab71996250","bbe7e5ab71996250"]}, {"id":"file:../../../testdata/oracle/17_nvl_nvl2.sql","content_sha":"06dee8764ace42cb91f2991623ee295541716a216946fddcdb9b8fdb2d845ab0","digests":["9459f8736f093ff4","9459f8736f093ff4","9459f8736f093ff4","9459f8736f093ff4","9459f8736f093ff4","9459f8736f093ff4","9459f8736f093ff4","9459f8736f093ff4","9459f8736f093ff4"]}, {"id":"file:../../../testdata/oracle/18_date_functions.sql","content_sha":"77c5b59ca2d3b19b0f62aac902f42763858e6ab749cbae5ebc95686984f0b874","digests":["9955eefbdda9be9e","9955eefbdda9be9e","9955eefbdda9be9e","9955eefbdda9be9e","9955eefbdda9be9e","9955eefbdda9be9e","9955eefbdda9be9e","9955eefbdda9be9e","9955eefbdda9be9e"]}, -{"id":"file:../../../testdata/oracle/19_regexp_functions.sql","content_sha":"41ad2d9c69564422b012ebbb2370468bb1e33cabb04ff8f34a85f982961ff97a","digests":["8a3c74eb15558f5a","8a3c74eb15558f5a","8a3c74eb15558f5a","8a3c74eb15558f5a","19212e490bd5cbb8","19212e490bd5cbb8","19212e490bd5cbb8","19212e490bd5cbb8","19212e490bd5cbb8"]}, +{"id":"file:../../../testdata/oracle/19_regexp_functions.sql","content_sha":"41ad2d9c69564422b012ebbb2370468bb1e33cabb04ff8f34a85f982961ff97a","digests":["143605b1d079850b","143605b1d079850b","143605b1d079850b","143605b1d079850b","19212e490bd5cbb8","19212e490bd5cbb8","19212e490bd5cbb8","19212e490bd5cbb8","19212e490bd5cbb8"]}, {"id":"file:../../../testdata/oracle/20_with_clause.sql","content_sha":"6f744ce8c5f3720d4ec5588ff0db713ab583fbc901174f0050a03298e2337b8e","digests":["6f06ddfcdbaba29b","6f06ddfcdbaba29b","6f06ddfcdbaba29b","6f06ddfcdbaba29b","6f06ddfcdbaba29b","6f06ddfcdbaba29b","6f06ddfcdbaba29b","6f06ddfcdbaba29b","6f06ddfcdbaba29b"]}, {"id":"file:../../../testdata/postgresql/01_simple_select.sql","content_sha":"71ff01ad43e53b8d6b3b07dff0a9f87a14997c6b12b2d026efd79b40a4a29be2","digests":["766637ee7ca838d0","766637ee7ca838d0","766637ee7ca838d0","766637ee7ca838d0","766637ee7ca838d0","766637ee7ca838d0","766637ee7ca838d0","766637ee7ca838d0","766637ee7ca838d0"]}, {"id":"file:../../../testdata/postgresql/02_simple_insert.sql","content_sha":"db936f494a0579cea5a52c2ab989501efd23a930172b4181390249ec1d3ecc31","digests":["8d14aaa53afcbe67","8d14aaa53afcbe67","8d14aaa53afcbe67","8d14aaa53afcbe67","8d14aaa53afcbe67","8d14aaa53afcbe67","8d14aaa53afcbe67","8d14aaa53afcbe67","8d14aaa53afcbe67"]}, @@ -362,44 +362,44 @@ {"id":"edge-ctx:91","input":"SELECT 'a\nb' \"c\nd\", z\nFROM t","digests":["67ab489127cc7cdf","67ab489127cc7cdf","67ab489127cc7cdf","67ab489127cc7cdf","67ab489127cc7cdf","67ab489127cc7cdf","67ab489127cc7cdf","67ab489127cc7cdf","67ab489127cc7cdf"]}, {"id":"edge:92","input":"`é\nx` 'y\\q'","digests":["ea6ff8ff99904935","ea6ff8ff99904935","ea6ff8ff99904935","ea6ff8ff99904935","11a1d6ab65beea65","11a1d6ab65beea65","11a1d6ab65beea65","11a1d6ab65beea65","11a1d6ab65beea65"]}, {"id":"edge-ctx:92","input":"SELECT `é\nx` 'y\\q', z\nFROM t","digests":["8c6196a16c9eb467","8c6196a16c9eb467","8c6196a16c9eb467","8c6196a16c9eb467","ad3da9d7a65a118c","ad3da9d7a65a118c","ad3da9d7a65a118c","ad3da9d7a65a118c","ad3da9d7a65a118c"]}, -{"id":"edge:93","input":"'é€𝄞' \"c\nd\"","digests":["32b77c86f5a11fa8","32b77c86f5a11fa8","32b77c86f5a11fa8","32b77c86f5a11fa8","32b77c86f5a11fa8","32b77c86f5a11fa8","32b77c86f5a11fa8","32b77c86f5a11fa8","32b77c86f5a11fa8"]}, -{"id":"edge-ctx:93","input":"SELECT 'é€𝄞' \"c\nd\", z\nFROM t","digests":["c3cc5a00636d1c91","c3cc5a00636d1c91","c3cc5a00636d1c91","c3cc5a00636d1c91","c3cc5a00636d1c91","c3cc5a00636d1c91","c3cc5a00636d1c91","c3cc5a00636d1c91","c3cc5a00636d1c91"]}, -{"id":"edge:94","input":"\"é\" 'a\\q'","digests":["61a6d353757488ee","61a6d353757488ee","61a6d353757488ee","61a6d353757488ee","cc44a44f19d91d8f","cc44a44f19d91d8f","cc44a44f19d91d8f","cc44a44f19d91d8f","cc44a44f19d91d8f"]}, -{"id":"edge-ctx:94","input":"SELECT \"é\" 'a\\q', z\nFROM t","digests":["52d694da96f7d5a6","52d694da96f7d5a6","52d694da96f7d5a6","52d694da96f7d5a6","46f7449a8294353c","46f7449a8294353c","46f7449a8294353c","46f7449a8294353c","46f7449a8294353c"]}, +{"id":"edge:93","input":"'é€𝄞' \"c\nd\"","digests":["3506b903294d11ab","3506b903294d11ab","3506b903294d11ab","3506b903294d11ab","3506b903294d11ab","3506b903294d11ab","3506b903294d11ab","3506b903294d11ab","3506b903294d11ab"]}, +{"id":"edge-ctx:93","input":"SELECT 'é€𝄞' \"c\nd\", z\nFROM t","digests":["9e60b4d5ec484bcb","9e60b4d5ec484bcb","9e60b4d5ec484bcb","9e60b4d5ec484bcb","9e60b4d5ec484bcb","9e60b4d5ec484bcb","9e60b4d5ec484bcb","9e60b4d5ec484bcb","9e60b4d5ec484bcb"]}, +{"id":"edge:94","input":"\"é\" 'a\\q'","digests":["284635c184bac677","284635c184bac677","284635c184bac677","284635c184bac677","cc44a44f19d91d8f","cc44a44f19d91d8f","cc44a44f19d91d8f","cc44a44f19d91d8f","cc44a44f19d91d8f"]}, +{"id":"edge-ctx:94","input":"SELECT \"é\" 'a\\q', z\nFROM t","digests":["64c9d44c0aabcd4e","64c9d44c0aabcd4e","64c9d44c0aabcd4e","64c9d44c0aabcd4e","46f7449a8294353c","46f7449a8294353c","46f7449a8294353c","46f7449a8294353c","46f7449a8294353c"]}, {"id":"edge:95","input":"'''a\nb''' \"c\nd\"","digests":["797b8c780ecbdd5b","797b8c780ecbdd5b","797b8c780ecbdd5b","797b8c780ecbdd5b","797b8c780ecbdd5b","797b8c780ecbdd5b","797b8c780ecbdd5b","797b8c780ecbdd5b","797b8c780ecbdd5b"]}, {"id":"edge-ctx:95","input":"SELECT '''a\nb''' \"c\nd\", z\nFROM t","digests":["dd7a2c13021131b1","dd7a2c13021131b1","dd7a2c13021131b1","dd7a2c13021131b1","dd7a2c13021131b1","dd7a2c13021131b1","dd7a2c13021131b1","dd7a2c13021131b1","dd7a2c13021131b1"]}, {"id":"edge:96","input":"'''a\nb''' 'c\\q'","digests":["037fb68d528a080a","037fb68d528a080a","037fb68d528a080a","037fb68d528a080a","051db694cf3b02a0","051db694cf3b02a0","051db694cf3b02a0","051db694cf3b02a0","051db694cf3b02a0"]}, {"id":"edge-ctx:96","input":"SELECT '''a\nb''' 'c\\q', z\nFROM t","digests":["c1410465031775a1","c1410465031775a1","c1410465031775a1","c1410465031775a1","6d0ccaafe07321f3","6d0ccaafe07321f3","6d0ccaafe07321f3","6d0ccaafe07321f3","6d0ccaafe07321f3"]}, -{"id":"edge:97","input":"'''é€𝄞''' 'c\\q'","digests":["6d7aae8738d709ff","6d7aae8738d709ff","6d7aae8738d709ff","6d7aae8738d709ff","431e56e83a63448b","431e56e83a63448b","431e56e83a63448b","431e56e83a63448b","431e56e83a63448b"]}, -{"id":"edge-ctx:97","input":"SELECT '''é€𝄞''' 'c\\q', z\nFROM t","digests":["6f79d79389e030df","6f79d79389e030df","6f79d79389e030df","6f79d79389e030df","b96ade0392530325","b96ade0392530325","b96ade0392530325","b96ade0392530325","b96ade0392530325"]}, +{"id":"edge:97","input":"'''é€𝄞''' 'c\\q'","digests":["66124adc9bb22902","66124adc9bb22902","66124adc9bb22902","66124adc9bb22902","431e56e83a63448b","431e56e83a63448b","431e56e83a63448b","431e56e83a63448b","431e56e83a63448b"]}, +{"id":"edge-ctx:97","input":"SELECT '''é€𝄞''' 'c\\q', z\nFROM t","digests":["00dc21bb449e671c","00dc21bb449e671c","00dc21bb449e671c","00dc21bb449e671c","b96ade0392530325","b96ade0392530325","b96ade0392530325","b96ade0392530325","b96ade0392530325"]}, {"id":"edge:98","input":"N'é\nx' 'c\\q'","digests":["66f5782ae2938984","66f5782ae2938984","66f5782ae2938984","66f5782ae2938984","252f9b34da482a34","c6f95736203ffd5d","c6f95736203ffd5d","c6f95736203ffd5d","c6f95736203ffd5d"]}, {"id":"edge-ctx:98","input":"SELECT N'é\nx' 'c\\q', z\nFROM t","digests":["0da153e74efe66e1","0da153e74efe66e1","0da153e74efe66e1","0da153e74efe66e1","1823c537ffb8b58a","b0fc4d8ad066dfa6","b0fc4d8ad066dfa6","b0fc4d8ad066dfa6","b0fc4d8ad066dfa6"]}, -{"id":"edge:99","input":"N'é€𝄞' \"c\nd\"","digests":["3ebdba6d81c115ac","3ebdba6d81c115ac","3ebdba6d81c115ac","3ebdba6d81c115ac","3ebdba6d81c115ac","3ebdba6d81c115ac","3ebdba6d81c115ac","3ebdba6d81c115ac","3ebdba6d81c115ac"]}, -{"id":"edge-ctx:99","input":"SELECT N'é€𝄞' \"c\nd\", z\nFROM t","digests":["82be99766413716c","82be99766413716c","82be99766413716c","82be99766413716c","82be99766413716c","82be99766413716c","82be99766413716c","82be99766413716c","82be99766413716c"]}, -{"id":"edge:100","input":"‘a\nb’ 'c\\q'","digests":["f61b8e5637b55263","f61b8e5637b55263","f61b8e5637b55263","f61b8e5637b55263","951a44eec552533e","951a44eec552533e","951a44eec552533e","951a44eec552533e","951a44eec552533e"]}, -{"id":"edge-ctx:100","input":"SELECT ‘a\nb’ 'c\\q', z\nFROM t","digests":["18fb3a3937f00a0a","18fb3a3937f00a0a","18fb3a3937f00a0a","18fb3a3937f00a0a","59381a8641425502","59381a8641425502","59381a8641425502","59381a8641425502","59381a8641425502"]}, -{"id":"edge:101","input":"‘é€𝄞’ \"c\nd\"","digests":["ceff9c9b9fcc20c0","ceff9c9b9fcc20c0","ceff9c9b9fcc20c0","ceff9c9b9fcc20c0","ceff9c9b9fcc20c0","ceff9c9b9fcc20c0","ceff9c9b9fcc20c0","ceff9c9b9fcc20c0","ceff9c9b9fcc20c0"]}, -{"id":"edge-ctx:101","input":"SELECT ‘é€𝄞’ \"c\nd\", z\nFROM t","digests":["70f7126e1b655067","70f7126e1b655067","70f7126e1b655067","70f7126e1b655067","70f7126e1b655067","70f7126e1b655067","70f7126e1b655067","70f7126e1b655067","70f7126e1b655067"]}, +{"id":"edge:99","input":"N'é€𝄞' \"c\nd\"","digests":["3195a29420d98e96","3195a29420d98e96","3195a29420d98e96","3195a29420d98e96","3195a29420d98e96","3195a29420d98e96","3195a29420d98e96","3195a29420d98e96","3195a29420d98e96"]}, +{"id":"edge-ctx:99","input":"SELECT N'é€𝄞' \"c\nd\", z\nFROM t","digests":["0add70dade15bb4a","0add70dade15bb4a","0add70dade15bb4a","0add70dade15bb4a","0add70dade15bb4a","0add70dade15bb4a","0add70dade15bb4a","0add70dade15bb4a","0add70dade15bb4a"]}, +{"id":"edge:100","input":"‘a\nb’ 'c\\q'","digests":["f6a45e53f848f78c","f6a45e53f848f78c","f6a45e53f848f78c","f6a45e53f848f78c","951a44eec552533e","951a44eec552533e","951a44eec552533e","951a44eec552533e","951a44eec552533e"]}, +{"id":"edge-ctx:100","input":"SELECT ‘a\nb’ 'c\\q', z\nFROM t","digests":["7172f47e527ff685","7172f47e527ff685","7172f47e527ff685","7172f47e527ff685","59381a8641425502","59381a8641425502","59381a8641425502","59381a8641425502","59381a8641425502"]}, +{"id":"edge:101","input":"‘é€𝄞’ \"c\nd\"","digests":["92ce5b3d5d4e289e","92ce5b3d5d4e289e","92ce5b3d5d4e289e","92ce5b3d5d4e289e","92ce5b3d5d4e289e","92ce5b3d5d4e289e","92ce5b3d5d4e289e","92ce5b3d5d4e289e","92ce5b3d5d4e289e"]}, +{"id":"edge-ctx:101","input":"SELECT ‘é€𝄞’ \"c\nd\", z\nFROM t","digests":["d22586331e8334c0","d22586331e8334c0","d22586331e8334c0","d22586331e8334c0","d22586331e8334c0","d22586331e8334c0","d22586331e8334c0","d22586331e8334c0","d22586331e8334c0"]}, {"id":"edge:102","input":"“é\nx” 'c\\q'","digests":["ec7f63079a1816b6","ec7f63079a1816b6","ec7f63079a1816b6","ec7f63079a1816b6","ec7f63079a1816b6","ec7f63079a1816b6","ec7f63079a1816b6","ec7f63079a1816b6","ec7f63079a1816b6"]}, {"id":"edge-ctx:102","input":"SELECT “é\nx” 'c\\q', z\nFROM t","digests":["21802c0c437590ac","21802c0c437590ac","21802c0c437590ac","21802c0c437590ac","21802c0c437590ac","21802c0c437590ac","21802c0c437590ac","21802c0c437590ac","21802c0c437590ac"]}, -{"id":"edge:103","input":"“途 \"c\nd\"","digests":["569a7d7fa57d1497","569a7d7fa57d1497","569a7d7fa57d1497","569a7d7fa57d1497","569a7d7fa57d1497","569a7d7fa57d1497","569a7d7fa57d1497","569a7d7fa57d1497","569a7d7fa57d1497"]}, -{"id":"edge-ctx:103","input":"SELECT “途 \"c\nd\", z\nFROM t","digests":["acef5f3620b6ebf0","acef5f3620b6ebf0","acef5f3620b6ebf0","acef5f3620b6ebf0","acef5f3620b6ebf0","acef5f3620b6ebf0","acef5f3620b6ebf0","acef5f3620b6ebf0","acef5f3620b6ebf0"]}, +{"id":"edge:103","input":"“途 \"c\nd\"","digests":["2892372859beab23","2892372859beab23","2892372859beab23","2892372859beab23","2892372859beab23","2892372859beab23","2892372859beab23","2892372859beab23","2892372859beab23"]}, +{"id":"edge-ctx:103","input":"SELECT “途 \"c\nd\", z\nFROM t","digests":["fba361a74401f9c6","fba361a74401f9c6","fba361a74401f9c6","fba361a74401f9c6","fba361a74401f9c6","fba361a74401f9c6","fba361a74401f9c6","fba361a74401f9c6","fba361a74401f9c6"]}, {"id":"edge:104","input":"'a''\nb''c' 'c\\q'","digests":["5a76ba7765ac96ea","5a76ba7765ac96ea","5a76ba7765ac96ea","5a76ba7765ac96ea","5d352ca3505f280e","5d352ca3505f280e","5d352ca3505f280e","5d352ca3505f280e","5d352ca3505f280e"]}, {"id":"edge-ctx:104","input":"SELECT 'a''\nb''c' 'c\\q', z\nFROM t","digests":["b6301c147d791d85","b6301c147d791d85","b6301c147d791d85","b6301c147d791d85","ce8906be8c1ba43e","ce8906be8c1ba43e","ce8906be8c1ba43e","ce8906be8c1ba43e","ce8906be8c1ba43e"]}, {"id":"edge:105","input":"'a\\\nb' 'c\\q'","digests":["dc4bf912b0dc1f78","dc4bf912b0dc1f78","dc4bf912b0dc1f78","dc4bf912b0dc1f78","9443964d461251fa","9443964d461251fa","9443964d461251fa","9443964d461251fa","9443964d461251fa"]}, {"id":"edge-ctx:105","input":"SELECT 'a\\\nb' 'c\\q', z\nFROM t","digests":["7c1b8d228cc55283","7c1b8d228cc55283","7c1b8d228cc55283","7c1b8d228cc55283","5e6b33dc0257a152","5e6b33dc0257a152","5e6b33dc0257a152","5e6b33dc0257a152","5e6b33dc0257a152"]}, {"id":"edge:106","input":"`a``\nb` \"c\nd\"","digests":["51635d8c7e2b1424","51635d8c7e2b1424","51635d8c7e2b1424","51635d8c7e2b1424","51635d8c7e2b1424","51635d8c7e2b1424","51635d8c7e2b1424","51635d8c7e2b1424","51635d8c7e2b1424"]}, {"id":"edge-ctx:106","input":"SELECT `a``\nb` \"c\nd\", z\nFROM t","digests":["9db126c88689e383","9db126c88689e383","9db126c88689e383","9db126c88689e383","9db126c88689e383","9db126c88689e383","9db126c88689e383","9db126c88689e383","9db126c88689e383"]}, -{"id":"edge:107","input":"\"a\"\"é\" 'c\\q'","digests":["9fe5ddeacbd067b6","9fe5ddeacbd067b6","9fe5ddeacbd067b6","9fe5ddeacbd067b6","a637636da6f7f6bc","a637636da6f7f6bc","a637636da6f7f6bc","a637636da6f7f6bc","a637636da6f7f6bc"]}, -{"id":"edge-ctx:107","input":"SELECT \"a\"\"é\" 'c\\q', z\nFROM t","digests":["d007a64b45d568db","d007a64b45d568db","d007a64b45d568db","d007a64b45d568db","e35d9e820303c3ca","e35d9e820303c3ca","e35d9e820303c3ca","e35d9e820303c3ca","e35d9e820303c3ca"]}, +{"id":"edge:107","input":"\"a\"\"é\" 'c\\q'","digests":["379a3de5de0783de","379a3de5de0783de","379a3de5de0783de","379a3de5de0783de","a637636da6f7f6bc","a637636da6f7f6bc","a637636da6f7f6bc","a637636da6f7f6bc","a637636da6f7f6bc"]}, +{"id":"edge-ctx:107","input":"SELECT \"a\"\"é\" 'c\\q', z\nFROM t","digests":["130ebd5c16fe0078","130ebd5c16fe0078","130ebd5c16fe0078","130ebd5c16fe0078","e35d9e820303c3ca","e35d9e820303c3ca","e35d9e820303c3ca","e35d9e820303c3ca","e35d9e820303c3ca"]}, {"id":"edge:108","input":"'a\r\nb' 'c\\q'","digests":["73db9fbfdaf37c05","73db9fbfdaf37c05","73db9fbfdaf37c05","73db9fbfdaf37c05","abadf0637d47955a","abadf0637d47955a","abadf0637d47955a","abadf0637d47955a","abadf0637d47955a"]}, {"id":"edge-ctx:108","input":"SELECT 'a\r\nb' 'c\\q', z\nFROM t","digests":["99b05fe11096e26b","99b05fe11096e26b","99b05fe11096e26b","99b05fe11096e26b","60890844420c3649","60890844420c3649","60890844420c3649","60890844420c3649","60890844420c3649"]}, -{"id":"edge:109","input":"'é'\n'é' \"c\nd\"","digests":["51cd27f3bf6eda7a","51cd27f3bf6eda7a","51cd27f3bf6eda7a","51cd27f3bf6eda7a","51cd27f3bf6eda7a","51cd27f3bf6eda7a","51cd27f3bf6eda7a","51cd27f3bf6eda7a","51cd27f3bf6eda7a"]}, -{"id":"edge-ctx:109","input":"SELECT 'é'\n'é' \"c\nd\", z\nFROM t","digests":["4fa35b89284cebdc","4fa35b89284cebdc","4fa35b89284cebdc","4fa35b89284cebdc","4fa35b89284cebdc","4fa35b89284cebdc","4fa35b89284cebdc","4fa35b89284cebdc","4fa35b89284cebdc"]}, -{"id":"edge:110","input":"SELECT 'a\nb', 'é€' FROM t WHERE x = 'c\\q'","digests":["da54abce30befdfe","da54abce30befdfe","da54abce30befdfe","da54abce30befdfe","a99c1a7c7ddd937c","a99c1a7c7ddd937c","a99c1a7c7ddd937c","a99c1a7c7ddd937c","a99c1a7c7ddd937c"]}, -{"id":"edge-ctx:110","input":"SELECT SELECT 'a\nb', 'é€' FROM t WHERE x = 'c\\q', z\nFROM t","digests":["1f0ae0238f3795cd","1f0ae0238f3795cd","1f0ae0238f3795cd","1f0ae0238f3795cd","63d26bf67bde0e1f","63d26bf67bde0e1f","63d26bf67bde0e1f","63d26bf67bde0e1f","63d26bf67bde0e1f"]}, -{"id":"edge:111","input":"SELECT 'a\nb',\n 'é€𝄞', \"c\nd\" FROM t","digests":["104e195a9454c968","104e195a9454c968","104e195a9454c968","104e195a9454c968","104e195a9454c968","104e195a9454c968","104e195a9454c968","104e195a9454c968","104e195a9454c968"]}, -{"id":"edge-ctx:111","input":"SELECT SELECT 'a\nb',\n 'é€𝄞', \"c\nd\" FROM t, z\nFROM t","digests":["8eeb2b1f1c2f9d94","8eeb2b1f1c2f9d94","8eeb2b1f1c2f9d94","8eeb2b1f1c2f9d94","8eeb2b1f1c2f9d94","8eeb2b1f1c2f9d94","8eeb2b1f1c2f9d94","8eeb2b1f1c2f9d94","8eeb2b1f1c2f9d94"]}, +{"id":"edge:109","input":"'é'\n'é' \"c\nd\"","digests":["16b4987863387940","16b4987863387940","16b4987863387940","16b4987863387940","16b4987863387940","16b4987863387940","16b4987863387940","16b4987863387940","16b4987863387940"]}, +{"id":"edge-ctx:109","input":"SELECT 'é'\n'é' \"c\nd\", z\nFROM t","digests":["ca869cc5b528f5d9","ca869cc5b528f5d9","ca869cc5b528f5d9","ca869cc5b528f5d9","ca869cc5b528f5d9","ca869cc5b528f5d9","ca869cc5b528f5d9","ca869cc5b528f5d9","ca869cc5b528f5d9"]}, +{"id":"edge:110","input":"SELECT 'a\nb', 'é€' FROM t WHERE x = 'c\\q'","digests":["1d403ae80e980d1d","1d403ae80e980d1d","1d403ae80e980d1d","1d403ae80e980d1d","a99c1a7c7ddd937c","a99c1a7c7ddd937c","a99c1a7c7ddd937c","a99c1a7c7ddd937c","a99c1a7c7ddd937c"]}, +{"id":"edge-ctx:110","input":"SELECT SELECT 'a\nb', 'é€' FROM t WHERE x = 'c\\q', z\nFROM t","digests":["7fd58d889b467ed5","7fd58d889b467ed5","7fd58d889b467ed5","7fd58d889b467ed5","63d26bf67bde0e1f","63d26bf67bde0e1f","63d26bf67bde0e1f","63d26bf67bde0e1f","63d26bf67bde0e1f"]}, +{"id":"edge:111","input":"SELECT 'a\nb',\n 'é€𝄞', \"c\nd\" FROM t","digests":["b18dc1a9a296b7e9","b18dc1a9a296b7e9","b18dc1a9a296b7e9","b18dc1a9a296b7e9","b18dc1a9a296b7e9","b18dc1a9a296b7e9","b18dc1a9a296b7e9","b18dc1a9a296b7e9","b18dc1a9a296b7e9"]}, +{"id":"edge-ctx:111","input":"SELECT SELECT 'a\nb',\n 'é€𝄞', \"c\nd\" FROM t, z\nFROM t","digests":["bd77f85ad43f2230","bd77f85ad43f2230","bd77f85ad43f2230","bd77f85ad43f2230","bd77f85ad43f2230","bd77f85ad43f2230","bd77f85ad43f2230","bd77f85ad43f2230","bd77f85ad43f2230"]}, {"id":"lit:../parser/aggregate_orderby_test.go:36","input":"SELECT STRING_AGG(name, ', ' ORDER BY name) FROM users","digests":["6bd396db3883df7c","6bd396db3883df7c","6bd396db3883df7c","6bd396db3883df7c","6bd396db3883df7c","6bd396db3883df7c","6bd396db3883df7c","6bd396db3883df7c","6bd396db3883df7c"]}, {"id":"lit:../parser/aggregate_orderby_test.go:41","input":"SELECT STRING_AGG(name, ', ' ORDER BY name ASC) FROM users","digests":["7e23ff33e80fdc65","7e23ff33e80fdc65","7e23ff33e80fdc65","7e23ff33e80fdc65","7e23ff33e80fdc65","7e23ff33e80fdc65","7e23ff33e80fdc65","7e23ff33e80fdc65","7e23ff33e80fdc65"]}, {"id":"lit:../parser/aggregate_orderby_test.go:46","input":"SELECT STRING_AGG(name, ', ' ORDER BY name DESC) FROM users","digests":["0fee5c1192524a25","0fee5c1192524a25","0fee5c1192524a25","0fee5c1192524a25","0fee5c1192524a25","0fee5c1192524a25","0fee5c1192524a25","0fee5c1192524a25","0fee5c1192524a25"]}, @@ -597,6 +597,7 @@ {"id":"lit:../parser/error_context_issue616_test.go:67","input":"Use INTERVAL '1 day' or INTERVAL 1 DAY syntax","digests":["ff389d9b52125e9a","ff389d9b52125e9a","ff389d9b52125e9a","ff389d9b52125e9a","ff389d9b52125e9a","ff389d9b52125e9a","ff389d9b52125e9a","ff389d9b52125e9a","ff389d9b52125e9a"]}, {"id":"lit:../parser/error_context_issue616_test.go:69","input":"SELECT a::DECIMAL('x') FROM t","digests":["c22171057383c473","c22171057383c473","c22171057383c473","c22171057383c473","c22171057383c473","c22171057383c473","c22171057383c473","c22171057383c473","c22171057383c473"]}, {"id":"lit:../parser/error_context_issue616_test.go:310","input":"%d builder call(s) pass something other than \"\" as sql (use \"\", WithHint for hints, WithCause for wrapped errors):\n%s","digests":["8f9518cf2ab7289a","8f9518cf2ab7289a","8f9518cf2ab7289a","8f9518cf2ab7289a","8f9518cf2ab7289a","8f9518cf2ab7289a","8f9518cf2ab7289a","8f9518cf2ab7289a","8f9518cf2ab7289a"]}, +{"id":"lit:../parser/error_position_test.go:61","input":"SELECT 'é' /* ü */ FROM t WHERE ) x","digests":["d656c234e248b507","d656c234e248b507","d656c234e248b507","d656c234e248b507","d656c234e248b507","d656c234e248b507","d656c234e248b507","d656c234e248b507","d656c234e248b507"]}, {"id":"lit:../parser/error_recovery_test.go:180","input":"Error message should contain '%s', got: %v","digests":["76150695dee54ca7","76150695dee54ca7","76150695dee54ca7","76150695dee54ca7","76150695dee54ca7","76150695dee54ca7","76150695dee54ca7","76150695dee54ca7","76150695dee54ca7"]}, {"id":"lit:../parser/error_recovery_test.go:771","input":"missing FROM doesn't cascade","digests":["4bf5658a44be5ac9","4bf5658a44be5ac9","4bf5658a44be5ac9","4bf5658a44be5ac9","4bf5658a44be5ac9","4bf5658a44be5ac9","4bf5658a44be5ac9","4bf5658a44be5ac9","4bf5658a44be5ac9"]}, {"id":"lit:../parser/error_recovery_test.go:784","input":"missing VALUES doesn't cascade","digests":["cd18059e90379dcd","cd18059e90379dcd","cd18059e90379dcd","cd18059e90379dcd","cd18059e90379dcd","cd18059e90379dcd","cd18059e90379dcd","cd18059e90379dcd","cd18059e90379dcd"]}, @@ -1093,6 +1094,24 @@ {"id":"lit:dollar_quote_test.go:225","input":"expected 'content', got %q","digests":["606c841593002e5b","606c841593002e5b","606c841593002e5b","606c841593002e5b","606c841593002e5b","606c841593002e5b","606c841593002e5b","606c841593002e5b","606c841593002e5b"]}, {"id":"lit:dollar_quoted_test.go:89","input":"$$it's a test with 'quotes'$$","digests":["0e234629bf6dfe75","0e234629bf6dfe75","0e234629bf6dfe75","0e234629bf6dfe75","0e234629bf6dfe75","0e234629bf6dfe75","0e234629bf6dfe75","0e234629bf6dfe75","0e234629bf6dfe75"]}, {"id":"lit:dollar_quoted_test.go:94","input":"it's a test with 'quotes'","digests":["08829a1dc7a2f0d4","08829a1dc7a2f0d4","08829a1dc7a2f0d4","08829a1dc7a2f0d4","08829a1dc7a2f0d4","08829a1dc7a2f0d4","08829a1dc7a2f0d4","08829a1dc7a2f0d4","08829a1dc7a2f0d4"]}, +{"id":"lit:error_location_test.go:54","input":"\n\"\n","digests":["dd1ef4f81a97a2c7","dd1ef4f81a97a2c7","dd1ef4f81a97a2c7","dd1ef4f81a97a2c7","dd1ef4f81a97a2c7","dd1ef4f81a97a2c7","dd1ef4f81a97a2c7","dd1ef4f81a97a2c7","dd1ef4f81a97a2c7"]}, +{"id":"lit:error_location_test.go:54","input":"\"","digests":["d7d7306a5ead7d57","d7d7306a5ead7d57","d7d7306a5ead7d57","d7d7306a5ead7d57","d7d7306a5ead7d57","d7d7306a5ead7d57","d7d7306a5ead7d57","d7d7306a5ead7d57","d7d7306a5ead7d57"]}, +{"id":"lit:error_location_test.go:55","input":"SELECT\t\t\"a\nb\"","digests":["c225e56d5f911395","c225e56d5f911395","c225e56d5f911395","c225e56d5f911395","c225e56d5f911395","c225e56d5f911395","c225e56d5f911395","c225e56d5f911395","c225e56d5f911395"]}, +{"id":"lit:error_location_test.go:56","input":"SELECT 'é€𝄞', \"x\ny\"","digests":["7ec62c435e430047","7ec62c435e430047","7ec62c435e430047","7ec62c435e430047","7ec62c435e430047","7ec62c435e430047","7ec62c435e430047","7ec62c435e430047","7ec62c435e430047"]}, +{"id":"lit:error_location_test.go:57","input":"SELECT 1\nFROM t\nWHERE\t\"a\nb\"","digests":["057578b0e39ca247","057578b0e39ca247","057578b0e39ca247","057578b0e39ca247","057578b0e39ca247","057578b0e39ca247","057578b0e39ca247","057578b0e39ca247","057578b0e39ca247"]}, +{"id":"lit:error_location_test.go:61","input":"SELECT 'é€',\n\t$tag$body\nmore","digests":["da4b270b68582658","da4b270b68582658","da4b270b68582658","da4b270b68582658","da4b270b68582658","da4b270b68582658","da4b270b68582658","da4b270b68582658","da4b270b68582658"]}, +{"id":"lit:error_location_test.go:62","input":"SELECT 'é€𝄞' || $$abc","digests":["50ec2c14476feda7","50ec2c14476feda7","50ec2c14476feda7","50ec2c14476feda7","50ec2c14476feda7","50ec2c14476feda7","50ec2c14476feda7","50ec2c14476feda7","50ec2c14476feda7"]}, +{"id":"lit:error_location_test.go:66","input":"SELECT N'é€',\n\t\t[abc def","digests":["902a75fa6321220d","902a75fa6321220d","902a75fa6321220d","902a75fa6321220d","ec6d7a4d8c5493f7","902a75fa6321220d","902a75fa6321220d","902a75fa6321220d","902a75fa6321220d"]}, +{"id":"lit:error_location_test.go:69","input":"SELECT\t'é\\q'","digests":["a764b5de5e251c76","a764b5de5e251c76","a764b5de5e251c76","a764b5de5e251c76","eab443a4d9c7932a","eab443a4d9c7932a","eab443a4d9c7932a","eab443a4d9c7932a","eab443a4d9c7932a"]}, +{"id":"lit:error_location_test.go:69","input":"q'","digests":["f3aa96ae5440e50e","f3aa96ae5440e50e","f3aa96ae5440e50e","f3aa96ae5440e50e","f3aa96ae5440e50e","f3aa96ae5440e50e","f3aa96ae5440e50e","f3aa96ae5440e50e","f3aa96ae5440e50e"]}, +{"id":"lit:error_location_test.go:70","input":"SELECT 'a\nb€',\n\t'x\\q'","digests":["b9a16ff6aaf6e48c","b9a16ff6aaf6e48c","b9a16ff6aaf6e48c","b9a16ff6aaf6e48c","f03acf1c138b7de5","f03acf1c138b7de5","f03acf1c138b7de5","f03acf1c138b7de5","f03acf1c138b7de5"]}, +{"id":"lit:error_location_test.go:73","input":"SELECT\t'é',\n\t'abc","digests":["2a4ab8f3bd8eda6e","2a4ab8f3bd8eda6e","2a4ab8f3bd8eda6e","2a4ab8f3bd8eda6e","2a4ab8f3bd8eda6e","2a4ab8f3bd8eda6e","2a4ab8f3bd8eda6e","2a4ab8f3bd8eda6e","2a4ab8f3bd8eda6e"]}, +{"id":"lit:error_location_test.go:74","input":"SELECT 'é€'\t/* never closed","digests":["0029d62cda63316a","0029d62cda63316a","0029d62cda63316a","0029d62cda63316a","0029d62cda63316a","0029d62cda63316a","0029d62cda63316a","0029d62cda63316a","0029d62cda63316a"]}, +{"id":"lit:error_location_test.go:75","input":"SELECT\t`a","digests":["97bc9bacbaebdefc","97bc9bacbaebdefc","97bc9bacbaebdefc","97bc9bacbaebdefc","97bc9bacbaebdefc","97bc9bacbaebdefc","97bc9bacbaebdefc","97bc9bacbaebdefc","97bc9bacbaebdefc"]}, +{"id":"lit:error_location_test.go:76","input":"SELECT\t'''abc","digests":["695cfa0ce77b94a0","695cfa0ce77b94a0","695cfa0ce77b94a0","695cfa0ce77b94a0","695cfa0ce77b94a0","695cfa0ce77b94a0","695cfa0ce77b94a0","695cfa0ce77b94a0","695cfa0ce77b94a0"]}, +{"id":"lit:error_location_test.go:123","input":"é\t'abc","digests":["ea77e24b908a07cd","ea77e24b908a07cd","ea77e24b908a07cd","ea77e24b908a07cd","ea77e24b908a07cd","ea77e24b908a07cd","ea77e24b908a07cd","ea77e24b908a07cd","ea77e24b908a07cd"]}, +{"id":"lit:error_location_test.go:124","input":"x\n\t'ab\ncd'","digests":["6d0c22165da41bca","6d0c22165da41bca","6d0c22165da41bca","6d0c22165da41bca","6d0c22165da41bca","6d0c22165da41bca","6d0c22165da41bca","6d0c22165da41bca","6d0c22165da41bca"]}, +{"id":"lit:error_location_test.go:125","input":"\t'é\\q'","digests":["3952d0f0c7c51ec1","3952d0f0c7c51ec1","3952d0f0c7c51ec1","3952d0f0c7c51ec1","21532fa95a2b2677","21532fa95a2b2677","21532fa95a2b2677","21532fa95a2b2677","21532fa95a2b2677"]}, {"id":"lit:json_operators_test.go:34","input":"data -\u003e 'key'","digests":["6852d2308725aa53","6852d2308725aa53","6852d2308725aa53","6852d2308725aa53","6852d2308725aa53","6852d2308725aa53","6852d2308725aa53","6852d2308725aa53","6852d2308725aa53"]}, {"id":"lit:json_operators_test.go:43","input":"data -\u003e\u003e 'key'","digests":["96f6be85c8493e22","96f6be85c8493e22","96f6be85c8493e22","96f6be85c8493e22","96f6be85c8493e22","96f6be85c8493e22","96f6be85c8493e22","96f6be85c8493e22","96f6be85c8493e22"]}, {"id":"lit:json_operators_test.go:52","input":"data -\u003e 'a' -\u003e\u003e 'b'","digests":["2b66f28a031efb5b","2b66f28a031efb5b","2b66f28a031efb5b","2b66f28a031efb5b","2b66f28a031efb5b","2b66f28a031efb5b","2b66f28a031efb5b","2b66f28a031efb5b","2b66f28a031efb5b"]}, @@ -1119,7 +1138,7 @@ {"id":"lit:literal_golden_test.go:206","input":"json:\"comment\"","digests":["00ab88d454394d5e","00ab88d454394d5e","00ab88d454394d5e","00ab88d454394d5e","00ab88d454394d5e","00ab88d454394d5e","00ab88d454394d5e","00ab88d454394d5e","00ab88d454394d5e"]}, {"id":"lit:literal_golden_test.go:207","input":"json:\"dialects\"","digests":["d6c0c6a204852ae9","d6c0c6a204852ae9","d6c0c6a204852ae9","d6c0c6a204852ae9","d6c0c6a204852ae9","d6c0c6a204852ae9","d6c0c6a204852ae9","d6c0c6a204852ae9","d6c0c6a204852ae9"]}, {"id":"lit:literal_golden_test.go:208","input":"json:\"entries\"","digests":["a962f350b7b33633","a962f350b7b33633","a962f350b7b33633","a962f350b7b33633","a962f350b7b33633","a962f350b7b33633","a962f350b7b33633","a962f350b7b33633","a962f350b7b33633"]}, -{"id":"lit:literal_golden_test.go:419","input":",\"entries\":[","digests":["2dd7cc975a561921","2dd7cc975a561921","2dd7cc975a561921","2dd7cc975a561921","3bebad2e7c6b9b67","2dd7cc975a561921","2dd7cc975a561921","2dd7cc975a561921","2dd7cc975a561921"]}, +{"id":"lit:literal_golden_test.go:419","input":",\"entries\":[","digests":["2dd7cc975a561921","2dd7cc975a561921","2dd7cc975a561921","2dd7cc975a561921","0fb009d0a2966d91","2dd7cc975a561921","2dd7cc975a561921","2dd7cc975a561921","2dd7cc975a561921"]}, {"id":"lit:literal_golden_test.go:503","input":"'\"`‘’“”«»","digests":["4e00ba84ec219d05","4e00ba84ec219d05","4e00ba84ec219d05","4e00ba84ec219d05","4e00ba84ec219d05","4e00ba84ec219d05","4e00ba84ec219d05","4e00ba84ec219d05","4e00ba84ec219d05"]}, {"id":"lit:memory_leak_test.go:44","input":"SELECT id, name, email FROM users WHERE age \u003e 21 AND status = 'active' ORDER BY name","digests":["b3c6ae6984cab58c","b3c6ae6984cab58c","b3c6ae6984cab58c","b3c6ae6984cab58c","b3c6ae6984cab58c","b3c6ae6984cab58c","b3c6ae6984cab58c","b3c6ae6984cab58c","b3c6ae6984cab58c"]}, {"id":"lit:memory_leak_test.go:217","input":"SELECT id, name, email, created_at FROM users WHERE status = 'active' AND age BETWEEN 18 AND 65 ORDER BY created_at DESC LIMIT 100","digests":["25af55e1e76d4fbf","25af55e1e76d4fbf","25af55e1e76d4fbf","25af55e1e76d4fbf","25af55e1e76d4fbf","25af55e1e76d4fbf","25af55e1e76d4fbf","25af55e1e76d4fbf","25af55e1e76d4fbf"]}, @@ -1264,7 +1283,6 @@ {"id":"lit:tokenizer_fuzz_test.go:490","input":"a\"\"b\" \"c\nd\"","digests":["502124b83d2cbad0","502124b83d2cbad0","502124b83d2cbad0","502124b83d2cbad0","502124b83d2cbad0","502124b83d2cbad0","502124b83d2cbad0","502124b83d2cbad0","502124b83d2cbad0"]}, {"id":"lit:tokenizer_fuzz_test.go:491","input":"a``b` `c\nd`","digests":["3d1e1e5805268256","3d1e1e5805268256","3d1e1e5805268256","3d1e1e5805268256","3d1e1e5805268256","3d1e1e5805268256","3d1e1e5805268256","3d1e1e5805268256","3d1e1e5805268256"]}, {"id":"lit:tokenizer_fuzz_test.go:492","input":"a\nb''' x","digests":["02df616334cedda6","02df616334cedda6","02df616334cedda6","02df616334cedda6","02df616334cedda6","02df616334cedda6","02df616334cedda6","02df616334cedda6","02df616334cedda6"]}, -{"id":"lit:tokenizer_fuzz_test.go:498","input":"\"","digests":["d7d7306a5ead7d57","d7d7306a5ead7d57","d7d7306a5ead7d57","d7d7306a5ead7d57","d7d7306a5ead7d57","d7d7306a5ead7d57","d7d7306a5ead7d57","d7d7306a5ead7d57","d7d7306a5ead7d57"]}, {"id":"lit:tokenizer_fuzz_test.go:498","input":"“","digests":["3c35396ae06593d5","3c35396ae06593d5","3c35396ae06593d5","3c35396ae06593d5","3c35396ae06593d5","3c35396ae06593d5","3c35396ae06593d5","3c35396ae06593d5","3c35396ae06593d5"]}, {"id":"lit:tokenizer_fuzz_test.go:498","input":"‘","digests":["b14c9e5c4bf82b90","b14c9e5c4bf82b90","b14c9e5c4bf82b90","b14c9e5c4bf82b90","b14c9e5c4bf82b90","b14c9e5c4bf82b90","b14c9e5c4bf82b90","b14c9e5c4bf82b90","b14c9e5c4bf82b90"]}, {"id":"lit:tokenizer_fuzz_test.go:498","input":"«","digests":["5c99dfed8cba44aa","5c99dfed8cba44aa","5c99dfed8cba44aa","5c99dfed8cba44aa","5c99dfed8cba44aa","5c99dfed8cba44aa","5c99dfed8cba44aa","5c99dfed8cba44aa","5c99dfed8cba44aa"]}, diff --git a/pkg/sql/tokenizer/tokenizer.go b/pkg/sql/tokenizer/tokenizer.go index 5abf2e2f4..5cfbee68e 100644 --- a/pkg/sql/tokenizer/tokenizer.go +++ b/pkg/sql/tokenizer/tokenizer.go @@ -415,7 +415,7 @@ func (t *Tokenizer) SetDialect(dialect keywords.SQLDialect) { // } func NewWithKeywords(kw *keywords.Keywords) (*Tokenizer, error) { if kw == nil { - return nil, errors.InvalidSyntaxError("keywords cannot be nil", models.Location{Line: 1, Column: 0}, "") + return nil, errors.InvalidSyntaxError("keywords cannot be nil", models.Location{Line: 1, Column: 1}, "") } return &Tokenizer{ @@ -581,7 +581,7 @@ func (t *Tokenizer) tokenize(ctx context.Context, input []byte, src string, have // Validate input size to prevent DoS attacks. This must stay ahead of the // string copy below so an oversized input is rejected before it is copied. if len(input) > MaxInputSize { - err := errors.InputTooLargeError(int64(len(input)), MaxInputSize, models.Location{Line: 1, Column: 0}) + err := errors.InputTooLargeError(int64(len(input)), MaxInputSize, models.Location{Line: 1, Column: 1}) metrics.RecordTokenization(time.Since(startTime), len(input), err) return nil, err } @@ -1315,7 +1315,7 @@ func (t *Tokenizer) readQuotedIdentifierSlow(startPos Position, opener rune) (mo if r == '\n' { return models.Token{}, errors.UnterminatedStringError( - models.Location{Line: startPos.Line, Column: startPos.Column}, + t.toSQLPosition(startPos), t.src, ) } @@ -1576,9 +1576,12 @@ func (t *Tokenizer) readQuotedStringSlow(startPos Position, originalQuote, quote // others keep it as a literal character (see backslashEscapes). if t.backslashEscapes() { if err := t.handleEscapeSequence(buf); err != nil { + // On failure handleEscapeSequence has consumed only the + // backslash, so t.pos is the offending escape character + // (or end of input), as in the wrapped error. return models.Token{}, errors.InvalidSyntaxError( fmt.Sprintf("invalid escape sequence: %v", err), - models.Location{Line: t.pos.Line, Column: t.pos.Column}, + t.getCurrentPosition(), t.src, ) } @@ -1885,9 +1888,10 @@ func (t *Tokenizer) readPunctuation() (models.Token, error) { } t.pos.AdvanceRune(ch, chSize) } + // Report the opening '[', the byte just before contentStart. return models.Token{}, errors.InvalidSyntaxError( "unterminated bracket identifier", - t.getCurrentPosition(), + t.toSQLPosition(Position{Index: contentStart - 1}), t.src, ) } @@ -2258,9 +2262,10 @@ func (t *Tokenizer) readPunctuation() (models.Token, error) { cr, cs := utf8.DecodeRune(t.input[t.pos.Index:]) t.pos.AdvanceRune(cr, cs) } - // Unterminated dollar-quoted string + // Unterminated dollar-quoted string: report the opening + // tag's leading '$', the byte just before tagStart. return models.Token{}, errors.UnterminatedStringError( - models.Location{Line: t.pos.Line, Column: t.pos.Column}, + t.toSQLPosition(Position{Index: tagStart - 1}), t.src, ) } @@ -2334,16 +2339,20 @@ func (t *Tokenizer) toSQLPosition(pos Position) models.Location { lineStart = t.lineStarts[li] } - // Column is 1-based and never less than 1. - end := min(pos.Index, len(t.input)) - column := max(end-lineStart, 0) + 1 - // Remember the line so the next, usually monotone, request starts there. t.posCacheLine = li + return lineLocation(li, lineStart, pos.Index, len(t.input)) +} + +// lineLocation is the column rule shared by toSQLPosition and locationAt: the +// byte offset pos, clamped to inputLen, on the line with 0-based index li that +// starts at byte lineStart. Columns are 1-based and never less than 1. +func lineLocation(li, lineStart, pos, inputLen int) models.Location { + end := min(pos, inputLen) return models.Location{ Line: li + 1, - Column: column, + Column: max(end-lineStart, 0) + 1, } } @@ -2352,30 +2361,19 @@ func (t *Tokenizer) getCurrentPosition() models.Location { return t.toSQLPosition(t.pos) } -// getLocation produces 1-based {Line, Column} for a given byte offset +// getLocation produces the 1-based {Line, Column} of a byte offset under the +// same rule as toSQLPosition. func (t *Tokenizer) getLocation(pos int) models.Location { - // Find the line containing pos - line := 1 - column := 1 - lineStart := 0 - - // Find the line number and start of the line - for i := 0; i < pos && i < len(t.input); i++ { - if t.input[i] == '\n' { - line++ - lineStart = i + 1 - } - } - - // Calculate column as offset from line start - if pos >= lineStart { - column = pos - lineStart + 1 - } + return locationAt(t.input, pos) +} - return models.Location{ - Line: line, - Column: column, - } +// locationAt computes the 1-based {Line, Column} of byte offset pos in input +// without the tokenizer's line index, using the same rule as toSQLPosition +// (see lineLocation). It is used off the hot path (Position.Location and the +// standalone StringLiteralReader), where no line index is available. +func locationAt(input []byte, pos int) models.Location { + prefix := input[:min(max(pos, 0), len(input))] + return lineLocation(bytes.Count(prefix, []byte{'\n'}), bytes.LastIndexByte(prefix, '\n')+1, pos, len(input)) } func isIdentifierChar(r rune) bool {