From 9ab83e033ff8aea491dbb060fc2b845ecdd0c8c9 Mon Sep 17 00:00:00 2001 From: Ajit Pratap Singh Date: Tue, 27 Oct 2026 14:01:19 +0530 Subject: [PATCH] fix(splitter): agree with the parser on escapes, parentheses, GO and Oracle slash lines - SplitStatements follows each dialect's backslash-escape rule (MySQL, MariaDB, ClickHouse; PostgreSQL E-strings), so 'a\';b' is one string. - A ';' inside parentheses no longer ends a statement (CREATE RULE ... DO ALSO (a; b)); unbalanced nesting falls back to one linear semicolon pass. - GO [count] lines accept trailing line and block comments in both the splitter and the parser, through one shared helper (internal/batch). - Oracle: a line holding only '/' is a SQL*Plus execute command in the parser too, so the parser and the splitter see the same statements; a slash with a comment on its line stays division. - internal/dropcheck compares word counts, not word sets; backslash handling is per dialect. - scripts/dialect-audit and TestRealWorldCorpus share one pipeline (internal/dialectaudit); block-comment-only statements are empty; -dialect is validated; glob errors and empty dialect directories fail. - format_corpus_test maps /sqlserver/ paths to the SQL Server dialect. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DnbRnTFErhoMJdAjVEeCg2 --- CHANGELOG.md | 3 + cmd/gosqlx/cmd/format_corpus_test.go | 30 ++++ internal/batch/batch.go | 4 +- internal/batch/text.go | 72 ++++++++ internal/batch/text_test.go | 45 +++++ internal/dialectaudit/audit.go | 101 +++++++++++ internal/dialectaudit/audit_test.go | 67 +++++++ internal/dropcheck/dropcheck.go | 45 +++-- internal/dropcheck/dropcheck_test.go | 20 +++ pkg/gosqlx/realworld_corpus_test.go | 61 +++---- pkg/gosqlx/splitter.go | 42 ++++- pkg/gosqlx/splitter_batch_comments_test.go | 48 +++++ pkg/gosqlx/splitter_boundaries_test.go | 198 +++++++++++++++++++++ pkg/gosqlx/splitter_proc.go | 22 +-- pkg/sql/parser/oracle_execute.go | 78 ++++++++ pkg/sql/parser/parser.go | 1 + pkg/sql/parser/validate.go | 2 +- scripts/dialect-audit/main.go | 108 ++++------- scripts/dialect-audit/main_test.go | 62 +++++++ 19 files changed, 857 insertions(+), 152 deletions(-) create mode 100644 internal/batch/text.go create mode 100644 internal/batch/text_test.go create mode 100644 internal/dialectaudit/audit.go create mode 100644 internal/dialectaudit/audit_test.go create mode 100644 pkg/gosqlx/splitter_batch_comments_test.go create mode 100644 pkg/gosqlx/splitter_boundaries_test.go create mode 100644 pkg/sql/parser/oracle_execute.go create mode 100644 scripts/dialect-audit/main_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 508e16cd5..de1bb83e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 in dialects without them. ### Fixed +- Statement splitting follows dialect string escapes, parenthesized bodies, + commented SQL Server GO lines and Oracle SQL*Plus execute lines; corpus and + CLI audits share a pipeline and detect repeated dropped words. - **AST walkers** (`ast.Inspect`, `Walk`, the security scanner, linter and table extraction) reach SELECT `TOP` and ClickHouse `ARRAY JOIN`, the ENGINE/ORDER BY/PARTITION BY/TTL/SETTINGS clauses of `CREATE TABLE` and diff --git a/cmd/gosqlx/cmd/format_corpus_test.go b/cmd/gosqlx/cmd/format_corpus_test.go index 315440878..96bf6a603 100644 --- a/cmd/gosqlx/cmd/format_corpus_test.go +++ b/cmd/gosqlx/cmd/format_corpus_test.go @@ -50,6 +50,36 @@ func corpusDialect(rel string) string { } } +func TestCorpusDialectSQLServer(t *testing.T) { + for _, path := range []string{"realworld/sqlserver/01.sql", "realworld/mssql/01.sql"} { + if got := corpusDialect(path); got != "sqlserver" { + t.Errorf("corpusDialect(%q) = %q", path, got) + } + } +} + +func TestCLI_FormatOracleExecuteLines(t *testing.T) { + const input = "BEGIN\n NULL;\nEND;\n/\nBEGIN\n NULL;\nEND;\n/\nSELECT 3 FROM dual;" + // SQL*Plus execute commands do not add statements to the SQL buffer. + reference := strings.ReplaceAll(input, "\n/\n", "\n") + original, err := parseWithDialect(t, reference, "oracle") + if err != nil { + t.Fatalf("reference script does not parse: %v", err) + } + defer ast.ReleaseAST(original) + if len(original.Statements) != 3 { + t.Fatalf("reference script has %d statements, want 3", len(original.Statements)) + } + + t.Cleanup(func() { rootCmd.SetArgs(nil) }) + path := writeTemp(t, "oracle.sql", input) + result := runCLI(t, nil, "format", "--dialect", "oracle", path) + if result.err != nil { + t.Fatalf("format --dialect oracle failed: %v\nstderr:\n%s", result.err, result.stderr) + } + checkCorpusOutput(t, "Oracle execute lines", "CLI", result.stdout, "oracle", original.SQL(), len(original.Statements)) +} + // TestFormatSQL_CorpusRoundTrip formats every parseable testdata/**/*.sql file // with the CLI formatter in several modes and requires that the output // re-parses to an equivalent tree and formats idempotently (#599). diff --git a/internal/batch/batch.go b/internal/batch/batch.go index 994de0255..6b4cfdf0e 100644 --- a/internal/batch/batch.go +++ b/internal/batch/batch.go @@ -51,7 +51,9 @@ func End(tokens []models.TokenWithSpan, i int) (int, bool) { // Line reports whether the token at index i is an unquoted GO alone on its // line, optionally followed on that line by one number token: the shape of a -// sqlcmd batch separator. end is the index of the first token after the line +// sqlcmd batch separator. SQL comments are trivia and may follow GO or its +// count, including a block comment spanning multiple lines (see TextEnd). +// end is the index of the first token after the line // and valid reports whether the number, if any, is a valid repeat count // (GO 0 and GO 1.5 have the shape of a separator but an invalid count). // Tokens without source positions cannot describe a client directive. diff --git a/internal/batch/text.go b/internal/batch/text.go new file mode 100644 index 000000000..79613f18d --- /dev/null +++ b/internal/batch/text.go @@ -0,0 +1,72 @@ +// Copyright 2026 GoSQLX Authors +// SPDX-License-Identifier: Apache-2.0 + +package batch + +// TextEnd returns the end of a GO directive whose GO word ends at wordEnd. +// The caller must check that GO is the first word on its line. As with Line, +// comments are trivia: both line and block comments may follow GO [count]. +// A block comment can span lines; text after its closing delimiter belongs to +// the next line, not the GO line. Repeat counts must be positive whole numbers; +// zero and nonnumeric arguments are rejected rather than silently ignored. +// This is sqlcmd's GO [count] syntax with SQL comments allowed on the line. +func TextEnd(src string, wordEnd int) (int, bool) { + if wordEnd < 0 || wordEnd > len(src) { + return 0, false + } + k := wordEnd + count := false + for k < len(src) { + switch src[k] { + case ' ', '\t', '\r': + k++ + case '\n': + return k + 1, true + case '-': + if k+1 >= len(src) || src[k+1] != '-' { + return 0, false + } + for k < len(src) && src[k] != '\n' { + k++ + } + case '/': + if k+1 >= len(src) || src[k+1] != '*' { + return 0, false + } + k += 2 + depth := 1 + crossedLine := false + for k < len(src) && depth > 0 { + if src[k] == '\n' { + crossedLine = true + } + // Match the tokenizer: block comments do not nest. + if k+1 < len(src) && src[k:k+2] == "*/" { + depth-- + k += 2 + } else { + k++ + } + } + if depth != 0 { + return 0, false + } + if crossedLine { + return k, true + } + default: + if count { + return 0, false + } + start := k + for k < len(src) && src[k] >= '0' && src[k] <= '9' { + k++ + } + if _, valid := Count(src[start:k]); !valid { + return 0, false + } + count = true + } + } + return k, true +} diff --git a/internal/batch/text_test.go b/internal/batch/text_test.go new file mode 100644 index 000000000..3e6d37fb3 --- /dev/null +++ b/internal/batch/text_test.go @@ -0,0 +1,45 @@ +// Copyright 2026 GoSQLX Authors +// SPDX-License-Identifier: Apache-2.0 + +package batch_test + +import ( + "testing" + + "github.com/ajitpratap0/GoSQLX/internal/batch" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" + "github.com/ajitpratap0/GoSQLX/pkg/sql/tokenizer" +) + +func TestCommentedLines(t *testing.T) { + for _, tc := range []struct { + text string + want bool + }{ + {"GO", true}, {"go", true}, {"GO 2", true}, {"GO 2 -- c", true}, + {"GO -- c", true}, {"GO /* c */", true}, {"GO 2 /* c */", true}, + {"GO /* multi\nline */", true}, {"GO 0", false}, {"GO x", false}, + {"GO 2 /* c */ x", false}, {"GO /* c */ 2", true}, + {"GO /* outer /* inner */ x */", false}, + } { + for _, ending := range []string{"\n", "\r\n"} { + t.Run(tc.text+ending, func(t *testing.T) { + src := tc.text + ending + "SELECT 1" + _, got := batch.TextEnd(src, 2) + if got != tc.want { + t.Errorf("TextEnd(%q) = %v, want %v", src, got, tc.want) + } + tkz := tokenizer.GetTokenizer() + defer tokenizer.PutTokenizer(tkz) + tkz.SetDialect(keywords.DialectSQLServer) + tokens, err := tkz.TokenizeString(src) + if err != nil { + t.Fatal(err) + } + if _, got := batch.End(tokens, 0); got != tc.want { + t.Errorf("End(%q) = %v, want %v", src, got, tc.want) + } + }) + } + } +} diff --git a/internal/dialectaudit/audit.go b/internal/dialectaudit/audit.go new file mode 100644 index 000000000..fae872e23 --- /dev/null +++ b/internal/dialectaudit/audit.go @@ -0,0 +1,101 @@ +// Copyright 2026 GoSQLX Authors +// SPDX-License-Identifier: Apache-2.0 + +// Package dialectaudit implements the shared per-statement corpus audit. +package dialectaudit + +import ( + "fmt" + "strings" + + "github.com/ajitpratap0/GoSQLX/internal/dropcheck" + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" +) + +// Result identifies the failed pipeline stage, or ok/empty for passing input. +type Result struct { + Stage string + Error string + Format string +} + +// Check parses, formats, reparses and reformats one statement, then checks for +// dropped words. Callbacks let in-package gosqlx tests use the same pipeline +// as the command without creating an import cycle. +func Check(stmt string, d keywords.SQLDialect, parse func(string, keywords.SQLDialect) (*ast.AST, error), format func(*ast.AST) string) (r Result) { + defer func() { + if p := recover(); p != nil { + r.Stage, r.Error = "panic", fmt.Sprint(p) + } + }() + if Empty(stmt) { + return Result{Stage: "empty"} + } + tree, err := parse(stmt, d) + if err != nil { + return Result{Stage: "parse", Error: err.Error()} + } + defer ast.ReleaseAST(tree) + f1 := format(tree) + r.Format = f1 + tree2, err := parse(f1, d) + if err != nil { + r.Stage, r.Error = "reparse", err.Error() + return r + } + defer ast.ReleaseAST(tree2) + if f2 := format(tree2); f2 != f1 { + r.Stage, r.Error = "idempotent", "format(parse(format(x))) != format(x): "+Compact(f2) + return r + } + if lost := dropcheck.DroppedWords(stmt, f1, string(d)); len(lost) > 0 { + r.Stage, r.Error = "drop", "formatter output lost words: "+strings.Join(lost, " ") + return r + } + return Result{Stage: "ok"} +} + +// Empty reports whether s consists only of whitespace and complete comments. +// Unterminated block comments remain input so the parser can reject them. +func Empty(s string) bool { + for i := 0; i < len(s); { + switch { + case strings.ContainsRune(" \t\r\n\f\v", rune(s[i])): + i++ + case strings.HasPrefix(s[i:], "--"): + if end := strings.IndexByte(s[i:], '\n'); end >= 0 { + i += end + 1 + } else { + return true + } + case strings.HasPrefix(s[i:], "/*"): + depth := 1 + i += 2 + for i < len(s) && depth > 0 { + switch { + case strings.HasPrefix(s[i:], "*/"): + depth-- + i += 2 + default: + i++ + } + } + if depth > 0 { + return false + } + default: + return false + } + } + return true +} + +// Compact produces a bounded single-line diagnostic. +func Compact(s string) string { + s = strings.Join(strings.Fields(s), " ") + if r := []rune(s); len(r) > 300 { + s = string(r[:300]) + "…" + } + return s +} diff --git a/internal/dialectaudit/audit_test.go b/internal/dialectaudit/audit_test.go new file mode 100644 index 000000000..deda98fa2 --- /dev/null +++ b/internal/dialectaudit/audit_test.go @@ -0,0 +1,67 @@ +// Copyright 2026 GoSQLX Authors +// SPDX-License-Identifier: Apache-2.0 + +package dialectaudit + +import ( + "errors" + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" +) + +func TestEmpty(t *testing.T) { + for _, tc := range []struct { + sql string + want bool + }{ + {"", true}, {" -- line\n /* block\n comment */ -- tail", true}, + {"/* outer /* inner */ rest */", false}, {"/* unfinished", false}, + {"/* comment */ SELECT 1", false}, {"'-- literal'", false}, {";", false}, + } { + if got := Empty(tc.sql); got != tc.want { + t.Errorf("Empty(%q) = %v, want %v", tc.sql, got, tc.want) + } + } +} + +func TestCheckStages(t *testing.T) { + for _, stage := range []string{"ok", "empty", "parse", "reparse", "idempotent", "drop", "panic"} { + t.Run(stage, func(t *testing.T) { + calls := 0 + parse := func(string, keywords.SQLDialect) (*ast.AST, error) { + calls++ + if stage == "panic" { + panic("test panic") + } + if stage == "parse" || stage == "reparse" && calls == 2 { + return nil, errors.New("test error") + } + return nil, nil + } + formats := 0 + format := func(*ast.AST) string { + formats++ + if stage == "idempotent" && formats == 2 { + return "SELECT y" + } + return "SELECT x" + } + sql := "SELECT x" + if stage == "drop" { + sql = "SELECT x WHERE x" + } + if stage == "empty" { + sql = "/* comments only */" + } + got := Check(sql, keywords.DialectPostgreSQL, parse, format) + if got.Stage != stage { + t.Fatalf("stage = %q, want %q: %s", got.Stage, stage, got.Error) + } + if stage == "empty" && calls != 0 { + t.Fatal("parsed empty statement") + } + }) + } +} diff --git a/internal/dropcheck/dropcheck.go b/internal/dropcheck/dropcheck.go index f0a1b5535..abfa92009 100644 --- a/internal/dropcheck/dropcheck.go +++ b/internal/dropcheck/dropcheck.go @@ -5,27 +5,30 @@ // drops even though its output re-parses: it reports words of the input // (outside comments and string literals) that are missing from the output. // It is shared by the real-world corpus test and scripts/dialect-audit. +// This is a heuristic: it can miss a moved clause or other semantic changes. package dropcheck import ( "regexp" "strings" + + sqldialect "github.com/ajitpratap0/GoSQLX/pkg/sql/dialect" ) -// DroppedWords reports words of the input (outside comments and string -// literals) that are missing from the formatted output — a cheap detector for -// clauses the formatter silently drops even though its output reparses. -func DroppedWords(in, out string) []string { +// DroppedWords compares word multisets, reporting each missing occurrence in +// input order after canonicalizing synonyms and ignoring optional noise words. +// The optional dialect is passed to Words. +func DroppedWords(in, out string, dialect ...string) []string { in = CanonicalSynonyms(in) - have := map[string]bool{} - for _, w := range Words(out) { - have[w] = true + have := map[string]int{} + for _, w := range Words(out, dialect...) { + have[w]++ } var lost []string - seen := map[string]bool{} - for _, w := range Words(in) { - if !have[w] && !ignorable[w] && !seen[w] { - seen[w] = true + for _, w := range Words(in, dialect...) { + if have[w] > 0 { + have[w]-- + } else if !ignorable[w] { lost = append(lost, w) } } @@ -40,7 +43,7 @@ func DroppedWords(in, out string) []string { // and the function ISNULL(...) are not touched. var synonyms = regexp.MustCompile(`(?i)\b(SELECT\s+ALL|SELECT\s+UNIQUE|DISTINCTROW|ISNULL|NOTNULL)\b`) -// CanonicalSynonyms rewrites the synonyms of synonyms in s +// CanonicalSynonyms rewrites SQL synonyms in s // to the canonical spelling the formatter writes. func CanonicalSynonyms(s string) string { var sb strings.Builder @@ -76,7 +79,14 @@ var ignorable = map[string]bool{"AS": true, "ASC": true, "INNER": true, "OUTER": // Words returns the upper-cased identifier-like words of s, skipping // comments and quoted string literals but keeping quoted identifiers' text. -func Words(s string) []string { +// The optional dialect uses the tokenizer's string escape rules; omitted means +// generic SQL. PostgreSQL E-prefixed strings always allow backslash escapes. +func Words(s string, dialect ...string) []string { + d := sqldialect.Generic + if len(dialect) > 0 { + d = sqldialect.Parse(dialect[0]) + } + backslash := d.StringEscape() != sqldialect.StringEscapeNone var words []string for i := 0; i < len(s); { c := s[i] @@ -93,9 +103,14 @@ func Words(s string) []string { i += end + 4 case c == '$' && dollarQuoteEnd(s, i) > i: i = dollarQuoteEnd(s, i) - case c == '\'': + case c == '\'' || (c == 'E' || c == 'e') && i+1 < len(s) && s[i+1] == '\'' && d == sqldialect.PostgreSQL: + escapes := backslash + if c != '\'' { + i++ + escapes = true + } for i++; i < len(s); i++ { - if s[i] == '\\' { + if s[i] == '\\' && escapes { i++ continue } diff --git a/internal/dropcheck/dropcheck_test.go b/internal/dropcheck/dropcheck_test.go index c4a7b5ae4..8522bd057 100644 --- a/internal/dropcheck/dropcheck_test.go +++ b/internal/dropcheck/dropcheck_test.go @@ -23,6 +23,8 @@ func TestDroppedWordsCanonicalSynonyms(t *testing.T) { {"postfix notnull", "SELECT x NOTNULL FROM users", "SELECT x IS NOT NULL FROM users", nil}, {"isnull function preserved", "SELECT ISNULL(x, y) FROM users", "SELECT x FROM users", []string{"ISNULL", "Y"}}, {"union all retained", "SELECT x FROM a UNION ALL SELECT x FROM b", "SELECT x FROM a UNION SELECT x FROM b", []string{"ALL"}}, + {"one repeated AND dropped", "SELECT x FROM t WHERE a AND b AND c", "SELECT x FROM t WHERE a AND b c", []string{"AND"}}, + {"two repeated words dropped", "SELECT x, x, x FROM t", "SELECT x FROM t", []string{"X", "X"}}, {"dropped clause words", "SELECT x FROM users WHERE enabled = true", "SELECT x FROM users", []string{"WHERE", "ENABLED", "TRUE"}}, {"dollar-quoted literal contents ignored", "SELECT $body$WHERE missing$body$ FROM users", "SELECT 'rewritten' FROM users", nil}, {"quoted and commented words ignored", "SELECT x, 'missing string' FROM users -- missing comment", "SELECT x FROM users", nil}, @@ -36,6 +38,24 @@ func TestDroppedWordsCanonicalSynonyms(t *testing.T) { } } +func TestWordsDialectBackslash(t *testing.T) { + for _, dialect := range []string{"generic", "postgresql", "sqlite", "sqlserver", "oracle", "mysql", "mariadb", "clickhouse", "snowflake", "bigquery", "redshift"} { + t.Run(dialect, func(t *testing.T) { + want := []string{"SELECT", "AND", "B"} + switch dialect { + case "mysql", "mariadb", "clickhouse", "snowflake", "bigquery", "redshift": + want = []string{"SELECT"} + } + if got := Words(`SELECT 'a\' AND b`, dialect); !reflect.DeepEqual(got, want) { + t.Fatalf("Words = %v, want %v", got, want) + } + }) + } + if got := Words(`SELECT E'a\'; b' AND x`, "postgresql"); !reflect.DeepEqual(got, []string{"SELECT", "AND", "X"}) { + t.Fatalf("E string words = %v", got) + } +} + func TestSQLWordsSkipsDollarQuotedLiteral(t *testing.T) { tests := []struct { name string diff --git a/pkg/gosqlx/realworld_corpus_test.go b/pkg/gosqlx/realworld_corpus_test.go index d6082e20a..db1df86d2 100644 --- a/pkg/gosqlx/realworld_corpus_test.go +++ b/pkg/gosqlx/realworld_corpus_test.go @@ -25,7 +25,7 @@ import ( "strings" "testing" - "github.com/ajitpratap0/GoSQLX/internal/dropcheck" + "github.com/ajitpratap0/GoSQLX/internal/dialectaudit" "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" ) @@ -66,7 +66,10 @@ func TestRealWorldCorpus(t *testing.T) { t.Errorf("testdata/realworld/%s: not a valid dialect name", dialect) continue } - files, _ := filepath.Glob(filepath.Join(realworldRoot, dialect, "*.sql")) + files, err := realworldSQLFiles(filepath.Join(realworldRoot, dialect)) + if err != nil { + t.Fatalf("corpus dialect %s: %v", dialect, err) + } for _, file := range files { data, err := os.ReadFile(file) if err != nil { @@ -79,7 +82,11 @@ func TestRealWorldCorpus(t *testing.T) { } key := fmt.Sprintf("%s#%d", base, i) seen[key] = true - stage, detail := realworldCheck(stmt, keywords.SQLDialect(dialect)) + result := dialectaudit.Check(stmt, keywords.SQLDialect(dialect), ParseWithDialect, func(tree *ast.AST) string { return FormatAST(tree) }) + stage, detail := result.Stage, result.Error + if result.Format != "" { + detail += "\n formatted: " + compact(result.Format) + } if stage != "ok" { current = append(current, key+" "+stage) } @@ -116,28 +123,23 @@ func TestRealWorldCorpus(t *testing.T) { } } -func realworldCheck(stmt string, d keywords.SQLDialect) (stage, detail string) { - defer func() { - if p := recover(); p != nil { - stage, detail = "panic", fmt.Sprint(p) - } - }() - tree, err := ParseWithDialect(stmt, d) +func realworldSQLFiles(dir string) ([]string, error) { + files, err := filepath.Glob(filepath.Join(dir, "*.sql")) if err != nil { - return "parse", err.Error() + return nil, fmt.Errorf("glob %s: %w", dir, err) } - f1 := FormatAST(tree) - tree2, err := ParseWithDialect(f1, d) - if err != nil { - return "reparse", err.Error() + "\n formatted: " + compact(f1) + if len(files) == 0 { + return nil, fmt.Errorf("%s: no .sql files", dir) } - if f2 := FormatAST(tree2); f2 != f1 { - return "idempotent", compact(f1) + "\n != " + compact(f2) - } - if lost := dropcheck.DroppedWords(stmt, f1); len(lost) > 0 { - return "drop", "formatter lost " + strings.Join(lost, " ") + "\n formatted: " + compact(f1) + return files, nil +} + +func TestRealworldSQLFilesRejectsMissingCorpus(t *testing.T) { + for _, dir := range []string{t.TempDir(), "["} { + if _, err := realworldSQLFiles(dir); err == nil { + t.Errorf("realworldSQLFiles(%q) succeeded", dir) + } } - return "ok", "" } func writeKnownFailures(t *testing.T, lines []string) { @@ -195,22 +197,9 @@ func loadKnownFailures(t *testing.T) map[string]string { return known } -func onlyComments(s string) bool { - for _, line := range strings.Split(s, "\n") { - if t := strings.TrimSpace(line); t != "" && !strings.HasPrefix(t, "--") { - return false - } - } - return true -} +func onlyComments(s string) bool { return dialectaudit.Empty(s) } -func compact(s string) string { - s = strings.Join(strings.Fields(s), " ") - if r := []rune(s); len(r) > 240 { - s = string(r[:240]) + "…" - } - return s -} +func compact(s string) string { return dialectaudit.Compact(s) } func assertDialectStandInsUntouched(t *testing.T) { t.Helper() diff --git a/pkg/gosqlx/splitter.go b/pkg/gosqlx/splitter.go index 0f8049e9b..3e764697a 100644 --- a/pkg/gosqlx/splitter.go +++ b/pkg/gosqlx/splitter.go @@ -14,7 +14,11 @@ package gosqlx -import "strings" +import ( + "strings" + + sqldialect "github.com/ajitpratap0/GoSQLX/pkg/sql/dialect" +) // SplitStatements splits a SQL source string into top-level statements on // unquoted semicolons. It is dialect-aware: depending on the supplied dialect @@ -55,6 +59,9 @@ import "strings" // downstream), and its only allocation is the result slice. func SplitStatements(sql, dialect string) []string { p := profileForDialect(dialect) + if d := sqldialect.Parse(dialect); d != sqldialect.Unknown { + p.supportsBackslash = d.StringEscape() != sqldialect.StringEscapeNone + } return splitWithProfile(sql, p) } @@ -66,7 +73,9 @@ func SplitStatements(sql, dialect string) []string { // DELIMITER directive for MySQL/MariaDB, and a line holding only "/" for // Oracle. Directive lines are consumed and do not appear in any segment. // Transaction statements (BEGIN; BEGIN TRANSACTION; …) are not blocks, and a -// script whose blocks do not balance falls back to plain semicolon splitting. +// script whose blocks or parentheses do not balance falls back in one extra +// linear pass to plain semicolon splitting, so the parser can reject malformed +// fragments without hiding subsequent statements. // dialectProfile collects the per-dialect feature flags the splitter needs. // Profiles are immutable and shared; one is resolved per call and passed to @@ -76,6 +85,7 @@ type dialectProfile struct { supportsBackticks bool supportsBrackets bool supportsEStrings bool + supportsBackslash bool supportsNestedBlockComments bool // Procedural-script features (see splitter_proc.go). @@ -83,6 +93,7 @@ type dialectProfile struct { oracle bool // DECLARE / IS|AS sections; "/" line terminator delimiter bool // MySQL client DELIMITER directive noBlocks bool // disable BEGIN…END tracking (unbalanced-script fallback) + noParens bool // ignore parentheses during the unbalanced-script fallback } // profileForDialect returns the feature flags for the named dialect. The @@ -124,10 +135,12 @@ func profileForDialect(dialect string) dialectProfile { // reconstruct the logic from a stack trace. func splitWithProfile(src string, p dialectProfile) []string { out, open := splitCore(src, p) - if open && !p.noBlocks { - // Unbalanced BEGIN/CASE/IF: don't let one malformed block swallow - // the rest of the script; fall back to plain semicolon splitting. + if open { + // Retry unbalanced blocks or parentheses once, ignoring nesting. + // Each malformed fragment can then reach the parser for an error + // without hiding later statements. Two linear passes bound the work. p.noBlocks = true + p.noParens = true out, _ = splitCore(src, p) } return out @@ -147,6 +160,7 @@ func splitCore(src string, p dialectProfile) (out []string, open bool) { // start is the index in src where the current segment begins. Every // segment is the contiguous substring src[start:i], so no bytes are copied. start := 0 + parens := 0 // Mutually-exclusive context flags. At most one of (inSingle, inDouble, // inBacktick, inBracket, inLine, dollarTag != "") is ever true at a time. @@ -216,10 +230,10 @@ func splitCore(src string, p dialectProfile) (out []string, open bool) { continue case inSingle: - // PG E-strings: `\` escapes the next byte (commonly \' for a + // Backslash dialects and PG E-strings escape the next byte (commonly \' for a // literal apostrophe). The quote we are escaping must not close // the string. - if eStringActive && c == '\\' && i+1 < len(src) { + if (eStringActive || p.supportsBackslash) && c == '\\' && i+1 < len(src) { i += 2 continue } @@ -322,8 +336,18 @@ func splitCore(src string, p dialectProfile) (out []string, open bool) { } i++ + case c == '(' && !p.noParens: + parens++ + i++ + + case c == ')' && !p.noParens: + if parens > 0 { + parens-- + } + i++ + case c == ';': - if delim != "" || st.depth > 0 || (p.tsql && elseFollows(src, i+1)) { + if delim != "" || parens > 0 || st.depth > 0 || (p.tsql && elseFollows(src, i+1)) { // Inside a custom-delimiter statement or an open block, or // T-SQL IF ... stmt; ELSE stmt. st.semicolon() @@ -383,7 +407,7 @@ func splitCore(src string, p dialectProfile) (out []string, open bool) { } } - open = st.depth > 0 + open = st.depth > 0 || parens > 0 if start < len(src) { out = append(out, src[start:]) } diff --git a/pkg/gosqlx/splitter_batch_comments_test.go b/pkg/gosqlx/splitter_batch_comments_test.go new file mode 100644 index 000000000..7d0e2ca1c --- /dev/null +++ b/pkg/gosqlx/splitter_batch_comments_test.go @@ -0,0 +1,48 @@ +// Copyright 2026 GoSQLX Authors +// SPDX-License-Identifier: Apache-2.0 + +package gosqlx + +import ( + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" +) + +func TestSplitStatementsGOLineComments(t *testing.T) { + for _, tc := range []struct { + line string + split bool + }{ + {"GO", true}, {"go", true}, {"GO 2", true}, {"GO 2 -- c", true}, + {"GO -- c", true}, {"GO /* c */", true}, {"GO 2 /* c */", true}, + {"GO /* multi\nline */", true}, {"GO 0", false}, {"GO x", false}, + } { + for _, ending := range []string{"\n", "\r\n"} { + t.Run(tc.line+ending, func(t *testing.T) { + src := "SELECT 1" + ending + tc.line + ending + "SELECT 2" + want := 1 + if tc.split { + want = 2 + } + if got := SplitStatements(src, "sqlserver"); len(got) != want { + t.Fatalf("SplitStatements(%q) = %#v, want %d statements", src, got, want) + } + if tc.split { + tree, err := ParseWithDialect(src, keywords.DialectSQLServer) + if err != nil { + t.Fatal(err) + } + defer ast.ReleaseAST(tree) + if len(tree.Statements) != 3 { + t.Fatalf("parser produced %d statements, want two SELECTs and a GO separator", len(tree.Statements)) + } + if _, ok := tree.Statements[1].(*ast.BatchSeparator); !ok { + t.Fatalf("expected GO separator, got %T", tree.Statements[1]) + } + } + }) + } + } +} diff --git a/pkg/gosqlx/splitter_boundaries_test.go b/pkg/gosqlx/splitter_boundaries_test.go new file mode 100644 index 000000000..1797ec60c --- /dev/null +++ b/pkg/gosqlx/splitter_boundaries_test.go @@ -0,0 +1,198 @@ +package gosqlx + +import ( + "strings" + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" + "github.com/ajitpratap0/GoSQLX/pkg/sql/parser" + "github.com/ajitpratap0/GoSQLX/pkg/sql/tokenizer" +) + +func TestSplitDialectBoundaries(t *testing.T) { + for _, dialect := range []string{"mysql", "mariadb", "clickhouse", "snowflake", "bigquery", "redshift"} { + t.Run(dialect, func(t *testing.T) { + sql := `SELECT 'a\';b' ; SELECT 3` + if got := nonEmpty(SplitStatements(sql, dialect)); got != 2 { + t.Fatalf("split count = %d", got) + } + tree, err := ParseWithDialect(sql, keywords.SQLDialect(dialect)) + if err != nil { + t.Fatal(err) + } + defer ast.ReleaseAST(tree) + if len(tree.Statements) != 2 { + t.Fatalf("parser count = %d", len(tree.Statements)) + } + }) + } + for _, dialect := range []string{"postgresql", "sqlite", "sqlserver", "oracle", "generic"} { + t.Run(dialect, func(t *testing.T) { + sql := `SELECT 'a\'; SELECT 3` + if got := nonEmpty(SplitStatements(sql, dialect)); got != 2 { + t.Fatalf("split count = %d", got) + } + tree, err := ParseWithDialect(sql, keywords.SQLDialect(dialect)) + if err != nil { + t.Fatal(err) + } + defer ast.ReleaseAST(tree) + if len(tree.Statements) != 2 { + t.Fatalf("parser count = %d", len(tree.Statements)) + } + }) + } +} + +func TestSplitParenthesisBoundaries(t *testing.T) { + for _, tc := range []struct { + name, sql string + want int + }{ + {"rule", `CREATE RULE r AS ON INSERT TO t DO ALSO (INSERT INTO u VALUES (1); INSERT INTO v VALUES (2)); SELECT 3`, 2}, + {"quoted", `SELECT '('; SELECT 3`, 2}, + {"comments", `SELECT 1 /* ( */; SELECT 3`, 2}, + {"unclosed", `SELECT (1; SELECT 2; SELECT 3`, 3}, + {"extra close", `SELECT )1; SELECT 2`, 2}, + {"nested", `SELECT ((1;2)); SELECT 3`, 2}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := nonEmpty(SplitStatements(tc.sql, "postgresql")); got != tc.want { + t.Fatalf("got %d, want %d", got, tc.want) + } + if tc.name == "rule" { + tree, err := ParseWithDialect(tc.sql, keywords.DialectPostgreSQL) + if err != nil { + t.Fatal(err) + } + defer ast.ReleaseAST(tree) + if len(tree.Statements) != tc.want { + t.Fatalf("parser count=%d", len(tree.Statements)) + } + } + }) + } + // A malformed script gets one fallback pass, including many unmatched opens. + for _, n := range []int{100, 400, 1600} { + sql := strings.Repeat("SELECT (1;", n) + if got := nonEmpty(SplitStatements(sql, "postgresql")); got != n { + t.Fatalf("n=%d: got %d", n, got) + } + } +} + +func TestOracleExecuteLine(t *testing.T) { + for _, tc := range []struct { + name, sql, dialect string + want int + }{ + {"execute", "SELECT 4\n/\nSELECT (SELECT 1 FROM dual) FROM dual", "oracle", 2}, + {"crlf", "SELECT 4\r\n / \r\nSELECT 3 FROM dual", "oracle", 2}, + {"right operand", "SELECT 4\n/ 2 FROM dual", "oracle", 1}, + {"line comment", "SELECT 4\n/ -- comment\n2 FROM dual", "oracle", 1}, + {"block comment", "SELECT 4\n/ /* c */\n2 FROM dual", "oracle", 1}, + {"leading comment", "SELECT 4\n/* c */ /\n2 FROM dual", "oracle", 1}, + {"multiline comment", "SELECT 4\n/ /* c\nmore */\n2 FROM dual", "oracle", 1}, + {"comment before execute", "SELECT 4 /* c */\n/\nSELECT 2 FROM dual", "oracle", 2}, + {"quoted slash", "SELECT '/' FROM dual; SELECT 3 FROM dual", "oracle", 2}, + {"left operand", "SELECT 4 /\n2 FROM dual", "oracle", 1}, + {"other dialect", "SELECT 4\n/\n2", "postgresql", 1}, + {"trailing execute", "SELECT 4 FROM dual\n/", "oracle", 1}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := nonEmpty(SplitStatements(tc.sql, tc.dialect)); got != tc.want { + t.Fatalf("split count=%d", got) + } + tree, err := ParseWithDialect(tc.sql, keywords.SQLDialect(tc.dialect)) + if err != nil { + t.Fatal(err) + } + defer ast.ReleaseAST(tree) + if len(tree.Statements) != tc.want { + t.Fatalf("parser count=%d", len(tree.Statements)) + } + }) + } + // The second fragment in the reported example is not a complete query. + // SQL*Plus ends SELECT 4 at '/', so reject that fragment instead of + // silently reading the slash as division in a single valid expression. + sql := "SELECT 4\n/\n(SELECT 1 FROM dual) FROM dual" + if got := nonEmpty(SplitStatements(sql, "oracle")); got != 2 { + t.Fatalf("split count=%d", got) + } + tree, err := ParseWithDialect(sql, keywords.DialectOracle) + if tree != nil { + defer ast.ReleaseAST(tree) + } + if err == nil { + t.Fatal("expected invalid statement after SQL*Plus execute command") + } +} + +func TestOracleExecuteBlockFormatting(t *testing.T) { + for _, ending := range []string{"\n", "\r\n"} { + t.Run(ending, func(t *testing.T) { + sql := strings.ReplaceAll("BEGIN NULL; END;\n/\nBEGIN NULL; END;\n/\nSELECT 3 FROM dual;", "\n", ending) + original, err := ParseWithDialect(sql, keywords.DialectOracle) + if err != nil { + t.Fatal(err) + } + defer ast.ReleaseAST(original) + if len(original.Statements) != 3 { + t.Fatalf("original count=%d", len(original.Statements)) + } + formatted, err := FormatWithDialect(sql, DefaultFormatOptions(), keywords.DialectOracle) + if err != nil { + t.Fatal(err) + } + reparsed, err := ParseWithDialect(formatted, keywords.DialectOracle) + if err != nil { + t.Fatal(err) + } + defer ast.ReleaseAST(reparsed) + if len(reparsed.Statements) != 3 || original.SQL() != reparsed.SQL() { + t.Fatalf("format changed statements:\n%s", formatted) + } + tkz := tokenizer.GetTokenizer() + defer tokenizer.PutTokenizer(tkz) + tkz.SetDialect(keywords.DialectOracle) + tokens, err := tkz.TokenizeString(sql) + if err != nil { + t.Fatal(err) + } + p := parser.NewParser(parser.WithDialect("oracle"), parser.WithStrictMode()) + tree, err := p.ParseFromModelTokens(tokens) + if err != nil { + t.Fatalf("strict block parse: %v", err) + } + defer ast.ReleaseAST(tree) + if len(tree.Statements) != 3 { + t.Fatalf("strict count=%d", len(tree.Statements)) + } + }) + } +} + +// TestOracleExecuteLineFormat checks that SQL*Plus '/' lines after PL/SQL +// blocks and queries neither fail formatting nor drop statements. +func TestOracleExecuteLineFormat(t *testing.T) { + sql := "BEGIN\n NULL;\nEND;\n/\nSELECT 1 FROM dual\n/\nCREATE OR REPLACE PROCEDURE p AS\nBEGIN\n NULL;\nEND;\n/\n" + out, err := FormatWithDialect(sql, DefaultFormatOptions(), keywords.DialectOracle) + if err != nil { + t.Fatalf("FormatWithDialect: %v", err) + } + for _, want := range []string{"BEGIN NULL; END", "SELECT 1", "CREATE OR REPLACE PROCEDURE p"} { + if !strings.Contains(out, want) { + t.Errorf("output lost %q:\n%s", want, out) + } + } + tree, err := ParseWithDialect(sql, keywords.DialectOracle) + if err != nil { + t.Fatalf("ParseWithDialect: %v", err) + } + defer ast.ReleaseAST(tree) + if len(tree.Statements) != 3 { + t.Errorf("statements = %d, want 3", len(tree.Statements)) + } +} diff --git a/pkg/gosqlx/splitter_proc.go b/pkg/gosqlx/splitter_proc.go index fae32e297..b6ffaef52 100644 --- a/pkg/gosqlx/splitter_proc.go +++ b/pkg/gosqlx/splitter_proc.go @@ -532,30 +532,14 @@ func lineOnly(src string, i, j int) (ls, le int, ok bool) { return ls, le, true } -// goLine recognises a T-SQL batch separator: the word GO at src[i:j], alone -// on its line, optionally followed by a repeat count and a -- comment. +// goLine recognises a T-SQL GO [count] line with SQL comments, using the +// same comment and repeat-count rules as the parser's batch.Line helper. func goLine(src string, i, j int) (ls, le int, ok bool) { ls, ok = lineStartOK(src, i) if !ok { return 0, 0, false } - k := skipBlanks(src, j) - countStart := k - for k < len(src) && src[k] >= '0' && src[k] <= '9' { - k++ - } - if k > countStart { - if _, valid := batch.Count(src[countStart:k]); !valid { - return 0, 0, false - } - } - k = skipBlanks(src, k) - if k+1 < len(src) && src[k] == '-' && src[k+1] == '-' { - for k < len(src) && src[k] != '\n' { - k++ - } - } - le, ok = afterLine(src, k) + le, ok = batch.TextEnd(src, j) if !ok { return 0, 0, false } diff --git a/pkg/sql/parser/oracle_execute.go b/pkg/sql/parser/oracle_execute.go new file mode 100644 index 000000000..bf5a6600b --- /dev/null +++ b/pkg/sql/parser/oracle_execute.go @@ -0,0 +1,78 @@ +// Copyright 2026 GoSQLX Authors +// SPDX-License-Identifier: Apache-2.0 + +package parser + +import ( + "strings" + + "github.com/ajitpratap0/GoSQLX/pkg/models" + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" + "github.com/ajitpratap0/GoSQLX/pkg/sql/dialect" +) + +// parseWithSource supplies the original text for SQL*Plus line recognition. +// Other dialects retain their usual preprocessing path. Oracle preprocesses +// directly here so the token-only hook cannot discard a commented slash first. +func (p *Parser) parseWithSource(tokens []models.TokenWithSpan, in sqlText) (*ast.AST, error) { + if p.DialectTyped() != dialect.Oracle { + return p.ParseFromModelTokens(tokens) + } + clear(p.tokenBuf) + p.tokenBuf = preprocessTokensInto(p.tokenBuf, tokens) + p.oracleExecuteLines(in.source()) + return p.parseTokens(p.tokenBuf) +} + +// oracleExecuteLines normalizes SQL*Plus execute commands to terminators before +// expression parsing can consume them as division. SQL*Plus requires '/' alone +// on its line: '/ -- comment', '/ /* c */' and '/* c */ /' are SQL, not execute +// commands. ParseWithDialect checks the original line, matching SplitStatements. +// Token-only entry points have only SQL token positions, so they cannot resolve +// comment-only neighbours; positionless streams cannot represent commands at all. +// A slash after ';' is consumed, not rewritten to an extra empty statement (also +// in strict mode). Commands delimit scripts; they do not reexecute prior SQL. +// Only the parser-owned buffer changes, and source lines are scanned once. +func (p *Parser) oracleExecuteLines(source string) { + if p.DialectTyped() != dialect.Oracle { + return + } + line, offset, n := 1, 0, 0 + for i := range p.tokenBuf { + t := p.tokenBuf[i] + alone := t.Token.Type == models.TokenTypeDiv && t.Start.Line > 0 && t.Start.Line == t.End.Line && + (i == 0 || p.tokenBuf[i-1].End.Line < t.Start.Line) && + (i+1 == len(p.tokenBuf) || p.tokenBuf[i+1].Token.Type == models.TokenTypeEOF || p.tokenBuf[i+1].Start.Line > t.End.Line) + if alone && source != "" { + for line < t.Start.Line && offset < len(source) { + end := strings.IndexByte(source[offset:], '\n') + if end < 0 { + offset = len(source) + break + } + offset += end + 1 + line++ + } + end := strings.IndexByte(source[offset:], '\n') + if end < 0 { + end = len(source) - offset + } + text := source[offset : offset+end] + if line == 1 { + text = strings.TrimPrefix(text, "\xef\xbb\xbf") + } + alone = strings.Trim(text, " \t\r") == "/" + } + if alone { + if n > 0 && p.tokenBuf[n-1].Token.Type == models.TokenTypeSemicolon { + continue + } + t.Token.Type = models.TokenTypeSemicolon + t.Token.Value = ";" + } + p.tokenBuf[n] = t + n++ + } + clear(p.tokenBuf[n:]) + p.tokenBuf = p.tokenBuf[:n] +} diff --git a/pkg/sql/parser/parser.go b/pkg/sql/parser/parser.go index 337b3dedc..167dfcdc1 100644 --- a/pkg/sql/parser/parser.go +++ b/pkg/sql/parser/parser.go @@ -408,6 +408,7 @@ func (p *Parser) preprocess(tokens []models.TokenWithSpan) []models.TokenWithSpa // previous stream so a shorter one cannot leave stale strings in the tail. clear(p.tokenBuf) p.tokenBuf = preprocessTokensInto(p.tokenBuf, tokens) + p.oracleExecuteLines("") return p.tokenBuf } diff --git a/pkg/sql/parser/validate.go b/pkg/sql/parser/validate.go index ddd98fc1f..953595dcb 100644 --- a/pkg/sql/parser/validate.go +++ b/pkg/sql/parser/validate.go @@ -287,5 +287,5 @@ func parseWithDialect(in sqlText, dialect keywords.SQLDialect) (*ast.AST, error) defer PutParser(p) p.ApplyOptions(WithDialect(string(dialect))) - return p.ParseFromModelTokens(tokens) + return p.parseWithSource(tokens, in) } diff --git a/scripts/dialect-audit/main.go b/scripts/dialect-audit/main.go index 274bcee12..39a7d9006 100644 --- a/scripts/dialect-audit/main.go +++ b/scripts/dialect-audit/main.go @@ -15,6 +15,7 @@ // Command dialect-audit runs real-world SQL files through the user-facing // pipeline (split → parse with dialect → format → reparse → reformat) and // reports every statement that fails a stage. +// JSON mode emits failures only, even when -v is also supplied. // // Usage: // @@ -26,11 +27,12 @@ import ( "encoding/json" "flag" "fmt" + "io" "os" - "strings" - "github.com/ajitpratap0/GoSQLX/internal/dropcheck" + "github.com/ajitpratap0/GoSQLX/internal/dialectaudit" "github.com/ajitpratap0/GoSQLX/pkg/gosqlx" + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" ) @@ -43,101 +45,65 @@ type result struct { Format string `json:"formatted,omitempty"` } -func main() { - dialect := flag.String("dialect", "generic", "SQL dialect") - asJSON := flag.Bool("json", false, "emit JSON lines for every failure") - verbose := flag.Bool("v", false, "print passing statements too") - flag.Parse() +func main() { os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) } + +func run(args []string, stdout, stderr io.Writer) int { + flags := flag.NewFlagSet("dialect-audit", flag.ContinueOnError) + flags.SetOutput(stderr) + dialect := flags.String("dialect", "generic", "SQL dialect") + asJSON := flags.Bool("json", false, "emit JSON lines for every failure") + verbose := flags.Bool("v", false, "print passing statements too") + if err := flags.Parse(args); err != nil { + return 2 + } + if !keywords.IsValidDialect(*dialect) { + fmt.Fprintf(stderr, "invalid SQL dialect: %q\n", *dialect) + return 2 + } d := keywords.SQLDialect(*dialect) total, failed := 0, 0 stages := map[string]int{} - enc := json.NewEncoder(os.Stdout) + enc := json.NewEncoder(stdout) - for _, path := range flag.Args() { + for _, path := range flags.Args() { data, err := os.ReadFile(path) if err != nil { - fmt.Fprintf(os.Stderr, "read %s: %v\n", path, err) - os.Exit(2) + fmt.Fprintf(stderr, "read %s: %v\n", path, err) + return 2 } for i, stmt := range gosqlx.SplitStatements(string(data), *dialect) { - if strings.TrimSpace(stripComments(stmt)) == "" { + audit := dialectaudit.Check(stmt, d, gosqlx.ParseWithDialect, func(tree *ast.AST) string { return gosqlx.FormatAST(tree) }) + if audit.Stage == "empty" { continue } total++ - r := check(path, i, stmt, d) + r := result{File: path, Index: i, SQL: stmt, Stage: audit.Stage, Error: audit.Error, Format: audit.Format} stages[r.Stage]++ if r.Stage == "ok" { - if *verbose { - fmt.Printf("ok %s#%d\n", path, i) + if *verbose && !*asJSON { + fmt.Fprintf(stdout, "ok %s#%d\n", path, i) } continue } failed++ if *asJSON { - _ = enc.Encode(r) + if err := enc.Encode(r); err != nil { + fmt.Fprintf(stderr, "encode failure: %v\n", err) + return 2 + } continue } - fmt.Printf("FAIL %s#%d [%s] %s\n sql: %s\n", path, i, r.Stage, r.Error, oneLine(r.SQL)) + fmt.Fprintf(stdout, "FAIL %s#%d [%s] %s\n sql: %s\n", path, i, r.Stage, r.Error, dialectaudit.Compact(r.SQL)) if r.Format != "" { - fmt.Printf(" fmt: %s\n", oneLine(r.Format)) + fmt.Fprintf(stdout, " fmt: %s\n", dialectaudit.Compact(r.Format)) } } } - fmt.Fprintf(os.Stderr, "dialect=%s total=%d passed=%d failed=%d stages=%v\n", + fmt.Fprintf(stderr, "dialect=%s total=%d passed=%d failed=%d stages=%v\n", *dialect, total, total-failed, failed, stages) if failed > 0 { - os.Exit(1) - } -} - -func check(path string, i int, stmt string, d keywords.SQLDialect) (r result) { - r = result{File: path, Index: i, SQL: stmt} - defer func() { - if p := recover(); p != nil { - r.Stage, r.Error = "panic", fmt.Sprint(p) - } - }() - tree, err := gosqlx.ParseWithDialect(stmt, d) - if err != nil { - r.Stage, r.Error = "parse", err.Error() - return r - } - f1 := gosqlx.FormatAST(tree) - r.Format = f1 - tree2, err := gosqlx.ParseWithDialect(f1, d) - if err != nil { - r.Stage, r.Error = "reparse", err.Error() - return r - } - if f2 := gosqlx.FormatAST(tree2); f2 != f1 { - r.Stage, r.Error = "idempotent", "format(parse(format(x))) != format(x): "+oneLine(f2) - return r - } - if lost := dropcheck.DroppedWords(stmt, f1); len(lost) > 0 { - r.Stage, r.Error = "drop", "formatter output lost words: "+strings.Join(lost, " ") - return r - } - r.Stage, r.Format = "ok", "" - return r -} - -func stripComments(s string) string { - var b strings.Builder - for _, line := range strings.Split(s, "\n") { - if t := strings.TrimSpace(line); strings.HasPrefix(t, "--") { - continue - } - b.WriteString(line) - b.WriteByte('\n') - } - return b.String() -} - -func oneLine(s string) string { - s = strings.Join(strings.Fields(s), " ") - if r := []rune(s); len(r) > 300 { - s = string(r[:300]) + "…" + return 1 } - return s + return 0 } diff --git a/scripts/dialect-audit/main_test.go b/scripts/dialect-audit/main_test.go new file mode 100644 index 000000000..c45f58327 --- /dev/null +++ b/scripts/dialect-audit/main_test.go @@ -0,0 +1,62 @@ +// Copyright 2026 GoSQLX Authors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestRunInvalidDialect(t *testing.T) { + var stdout, stderr bytes.Buffer + if code := run([]string{"-dialect", "typo"}, &stdout, &stderr); code != 2 { + t.Fatalf("exit = %d, want 2", code) + } + if stdout.Len() != 0 || !strings.Contains(stderr.String(), "invalid SQL dialect") { + t.Fatalf("stdout=%q stderr=%q", stdout.String(), stderr.String()) + } +} + +func TestRunJSONFailuresOnly(t *testing.T) { + for _, tc := range []struct { + sql string + code int + failures int + }{ + {"/* block only */; SELECT 1; -- line only", 0, 0}, + {"SELECT 1; SELECT FROM", 1, 1}, + } { + file := filepath.Join(t.TempDir(), "input.sql") + if err := os.WriteFile(file, []byte(tc.sql), 0o600); err != nil { + t.Fatal(err) + } + var stdout, stderr bytes.Buffer + if code := run([]string{"-json", "-v", file}, &stdout, &stderr); code != tc.code { + t.Fatalf("exit=%d stderr=%s", code, &stderr) + } + lines := strings.Split(strings.TrimSpace(stdout.String()), "\n") + if tc.failures == 0 { + if stdout.Len() != 0 { + t.Fatalf("stdout=%q", stdout.String()) + } + continue + } + if len(lines) != tc.failures { + t.Fatalf("stdout=%q", stdout.String()) + } + for _, line := range lines { + var r result + if err := json.Unmarshal([]byte(line), &r); err != nil { + t.Fatal(err) + } + if r.Stage == "ok" || r.Stage == "empty" { + t.Fatalf("passing JSON result: %+v", r) + } + } + } +}