diff --git a/.golangci.yml b/.golangci.yml index a7366360..0c0d04e3 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -22,6 +22,10 @@ linters: - fmt.Print - fmt.Printf - fmt.Println + # ast.textWriter is implemented only by *bytes.Buffer and + # *strings.Builder, whose writes never return an error. + - (github.com/ajitpratap0/GoSQLX/pkg/sql/ast.textWriter).WriteString + - (github.com/ajitpratap0/GoSQLX/pkg/sql/ast.textWriter).WriteByte staticcheck: checks: - "all" diff --git a/CHANGELOG.md b/CHANGELOG.md index fed25727..298886dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- `ast.PivotClause.AggregateAlias`, `ast.PivotClause.ExtraAggregates` + (`[]ast.PivotAggregate`), `ast.UnpivotClause.NullHandling`, + `ast.UnpivotClause.ValueColumns` and `ast.UnpivotNullHandling`; rendered by + `SQL()` and `pkg/formatter`. - **Real-world dialect corpus**: `testdata/realworld//` holds 2,788 production-style statements across PostgreSQL, MySQL, MariaDB, SQL Server, Oracle, SQLite, Snowflake and ClickHouse. `TestRealWorldCorpus` runs each @@ -17,8 +21,72 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 listed failure that now passes. `scripts/dialect-audit` runs the same checks on any SQL file; `BenchmarkRealWorldParse` and `BenchmarkRealWorldParseFormat` measure the corpus. +- **Dialect coverage**: every statement of the real-world corpus now parses, + formats, re-parses identically and keeps all its words; `known_failures.txt` + is empty (1,327 failures before). +- **AST representation**: `ExplainStatement` retains the explained statement; + `BatchSeparator` retains SQL Server `GO` and its optional `RepeatCount` without + expanding repeated batches. `SQL()` and `FormatAST` keep GO on its own line. + Callers enumerating `AST.Statements` or recovery results must account for + batch-separator nodes. +- **Literal and identifier metadata**: string literals retain their introducer + (`N`, `X`, `B`, `_utf8mb4`, ...) and escape rules (`LiteralValue.DialectExtras()`, + `EnsureExtras()`, `SetRawBackslash()`). `Identifier.QuoteStyle`, + `Identifier.Quotes` and the statement/table/column `NameQuote` fields retain + identifier delimiters through rendering. + +### Changed +- **Dialect extension fields**: the new dialect-specific clauses live behind + optional pointers so that common statements do not carry them: + `SelectStatement.Extras` (`SelectExtras`: SELECT modifiers, INTO, LIMIT BY, + ROWS EXAMINED, INTERPOLATE, query hints, output formats), + `InsertStatement.Extras`, `UpdateStatement.Extras` / `DeleteStatement.Extras` + (`DMLTargetFields`: modifiers, TOP, ONLY, extra targets, joins, OUTPUT, + ORDER BY / LIMIT, RETURNING INTO), `FunctionCall.Extras`, + `BinaryExpression.Extras` and `TableReference.Modifiers` (including derived + `VALUES`). Read them through `DialectExtras()` / `DialectModifiers()`, which + return a shared read-only empty value when the pointer is nil, and call + `EnsureExtras()` / `EnsureModifiers()` before writing. +- **Transpilation**: string literals and quoted identifiers are rendered using + the target dialect's escape rules and delimiters. Unrepresentable literals + return an error rather than changing their value. SQL Server GO scripts + targeting another dialect return an error because batch scopes and repeat + counts cannot be represented there. +- **Transpiling dialect-specific operators**: `^` is exponentiation in + PostgreSQL and bitwise XOR in MySQL, MariaDB and SQL Server (recorded as + `BinaryExpression.BitwiseXor`). Exponentiation is written `POWER(a, b)` + outside PostgreSQL; XOR keeps `^` with explicit grouping between the MySQL + family and SQL Server, becomes `#` in PostgreSQL, `BITXOR()` in Snowflake and + `bitXor()` in ClickHouse, and returns an error for Oracle, SQLite and the + generic dialect. PostgreSQL `@`, `|/` and `||/` become `ABS()`, `SQRT()` and + `CBRT()` (an error where CBRT is missing); `!!`, SQL Server money literals + (`$5.00`, `$5`) and Snowflake object constants (`{'a': 1}`) return an error + in dialects without them. ### Fixed +- **AST walkers** (`ast.Inspect`, `Walk`, the security scanner, linter and + table extraction) reach SELECT `TOP` and ClickHouse `ARRAY JOIN`, the + ENGINE/ORDER BY/PARTITION BY/TTL/SETTINGS clauses of `CREATE TABLE` and + `CREATE MATERIALIZED VIEW`, Snowflake `CLONE ... AT`, and PostgreSQL + partition bounds. `AST.SQL()` no longer drops `CREATE`/`ALTER`/`DROP + SEQUENCE` statements, and MERGE keeps the source's column alias list + (`USING (VALUES ...) AS s (id, val)`). +- PostgreSQL `@` (absolute value) is always written as `@ x`; `@a`, `@t.a` and + `@abs(a)` re-parsed as placeholders. `@` directly followed by operator + characters (`@-@`, `@-5`) is now rejected, as PostgreSQL lexes it as one + operator. +- A WITH clause before nested parenthesized set operands + (`WITH c ... ((A UNION B) INTERSECT C) EXCEPT D`) stays in front of the whole + operation in `SQL()`, `FormatAST` and `Transpile`. +- Optimizer hints in subqueries stay with their SELECT when the renderer + reorders clauses (Oracle `START WITH` / `CONNECT BY`); when the hinted SELECT + cannot be identified, formatting fails with `ErrUnplaceableComment`. +- LSP document outline no longer lists GO separators, and statements before a + GO line no longer extend to the end of the file. +- `ReleaseAST` returns the pooled nodes under standalone `VALUES` statements and + top-level set operations. +- `gosqlx format --dialect sqlserver` runs the statement-count guard again, + counting GO lines and statements that end without `;`. - **`ExtractColumns` / `ExtractColumnsQualified` no longer report SELECT aliases** (#849): an unqualified name in `ORDER BY` or `HAVING` that matches a SELECT-list alias of the same query block is treated as the alias, not a @@ -44,14 +112,76 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 rendered. This preserves BigQuery backticks and Oracle alias case. - Numeric UNPIVOT row aliases after `AS` are limited to BigQuery and Oracle; Snowflake and SQL Server reject them. - -### Added -- `ast.PivotClause.AggregateAlias`, `ast.PivotClause.ExtraAggregates` - (`[]ast.PivotAggregate`), `ast.UnpivotClause.NullHandling`, - `ast.UnpivotClause.ValueColumns` and `ast.UnpivotNullHandling`; rendered by - `SQL()` and `pkg/formatter`. +- **SQL round trips**: preserve ALTER actions, grouped joins, set-operation + query tails and CTE placement, standalone VALUES, JSON_TABLE/XMLTABLE literal + quoting, and nested optimizer hints. Hint placement uses the source dialect's + lexer so a keyword inside a string or quoted identifier is never a hint slot. +- **Pooled parsing**: internal tokenize/parse paths reuse scratch token storage + without retaining it in returned ASTs. Concurrent retained-tree regression + tests cover the public parse entry points; SELECT release clears all Extras. +- **`SplitStatements` keeps compound statements whole**: `BEGIN ... END` blocks + (trigger, procedure and function bodies, T-SQL `IF ... BEGIN`, `BEGIN TRY`, + PostgreSQL `BEGIN ATOMIC`), `CASE ... END`, `IF`/`LOOP`/`WHILE`/`REPEAT` + inside a block and Oracle `DECLARE`/`IS`/`AS` sections are no longer cut at + the first inner `;`. Transaction statements (`BEGIN;`, `BEGIN TRANSACTION`, + `BEGIN IMMEDIATE`) are not blocks, and a script whose blocks do not balance + falls back to plain `;` splitting. Client batch separators are honoured and + removed from the output: a `GO [n]` line for SQL Server, `DELIMITER` for + MySQL/MariaDB and a lone `/` line for Oracle. +- **`gosqlx.Lint` / `LintWithDialect` report invalid SQL**: SQL that does not + tokenize or parse returns an error wrapping `ErrTokenize` or `ErrSyntax` + together with the violations of the text and token rules. Callers that + ignored the error and treated an empty result as clean must now check it. + `linter.FileResult` gains `TokenizeError`, `ParseError` and `SyntaxError()`. +- **`gosqlx.Format` and `gosqlx format` keep comments and optimizer hints**: + comments stay with their statement (a comment inside a statement moves to the + line above it) and `/*+ ... */` hints directly after a statement's first + keyword stay in place. Input whose hints or MySQL `/*! ... */` comments + cannot be placed faithfully is refused (`formatter.ErrUnplaceableComment`) + instead of being rewritten without them, and `format -i` verifies that the + output holds exactly the input's comments. LSP and C-binding dialect + formatting no longer drop comments either. New `formatter.JoinWithComments`, + `gosqlx.FormatWithDialect` and `gosqlx.WithComments()`. +- **CLI**: `lint` exits 1 on syntax errors and `--auto-fix` judges the fixed + text; `format --no-uppercase` lower-cases clause keywords (it used to upper-case + them; operators such as `AND`, `IN`, `IS NULL`, `DESC` and function names + are still written in upper case); `postgres`, `pg`, `mssql`, `tsql` and `sqlite3` are accepted by every + `--dialect` flag; errors are printed once and usage only for flag and + argument errors; `validate --quiet` exits silently; `validate` names stdin + ``, validates `GO`/`DELIMITER` scripts and files over the token + limit statement by statement; empty and comment-only input is valid for + `validate`, `lint` and `format`; `-o` works for `parse` and `transpile`, + `-f`/`-o` map onto `validate`'s output flags and are rejected where + unsupported; `format` keeps a leading byte order mark. +- **`ParseReaderMultiple`** skips segments that hold only comments. +- **Docs**: removed `parser.NewTokenConverter`, `SetMaxTokens`, + `SetMaxRecursionDepth` and the `parser.max_tokens` config key, none of which + exist; the limits are constants. ### Behavior changes +- A raw NUL or Ctrl-Z byte inside a string literal is an error in dialects + whose literals cannot spell it back (NUL without backslash escapes; both + without a dialect; Ctrl-Z in ClickHouse; both in Snowflake). Formatting used + to drop or rewrite it, changing the value. +- Statement counts (`gosqlx analyze`, `gosqlx parse`, the MCP `parse_sql` + tool and the C binding's `statement_count` / statement types) exclude GO + separators. +- `fingerprint.Normalize` and `Fingerprint` keep identifier quoting + (`SELECT "A" FROM "T"` used to normalize to `SELECT A FROM T`), so + fingerprints of queries with quoted identifiers differ from earlier + releases. Re-compute stored fingerprints after upgrading. +- Outside SQL Server, a utility statement (`SET`, `USE`, `GRANT`, `COPY`, + `CALL`, ...) followed on the next line by another statement without `;` is + an error; only a transaction statement on its own line is accepted without + a separator. Statements whose body may hold other statements (`CREATE + FUNCTION`/`TRIGGER`/`RULE`, `PREPARE ... AS`, `DECLARE ... CURSOR`) still + run to the next `;`, as does a second statement on the same line as a + `GRANT` or `COPY`. +- Transaction statements are checked against each dialect's grammar + (`BEGIN banana` is rejected); `<=>` is accepted only in MySQL, MariaDB and + PostgreSQL (pgvector), `==` only in SQLite and ClickHouse; an unbalanced + SQL Server `BEGIN ... END` block, or an `IF`/`WHILE` without a body, is an + error. - PIVOT / UNPIVOT `IN` lists now require commas between entries and an alias after `AS` in every PIVOT dialect. Where bare aliases are supported, `IN (a b)` is one entry (`a AS b`); otherwise a missing comma is rejected. @@ -2801,4 +2931,4 @@ For questions about upgrading or changelog entries: [1.0.1]: https://github.com/ajitpratap0/GoSQLX/compare/v1.0.0...v1.0.1 [1.0.0]: https://github.com/ajitpratap0/GoSQLX/compare/v0.9.0...v1.0.0 [0.9.0]: https://github.com/ajitpratap0/GoSQLX/compare/v0.8.0...v0.9.0 -[0.8.0]: https://github.com/ajitpratap0/GoSQLX/releases/tag/v0.8.0 \ No newline at end of file +[0.8.0]: https://github.com/ajitpratap0/GoSQLX/releases/tag/v0.8.0 diff --git a/cmd/gosqlx/cmd/analyze.go b/cmd/gosqlx/cmd/analyze.go index 14452628..d0e455a7 100644 --- a/cmd/gosqlx/cmd/analyze.go +++ b/cmd/gosqlx/cmd/analyze.go @@ -67,7 +67,7 @@ This is a basic implementation for CLI foundation.`, func analyzeRun(cmd *cobra.Command, args []string) error { // Reject unknown dialect names early before any parsing. - if err := validateDialectName(analyzeDialect); err != nil { + if err := normalizeDialectFlag(&analyzeDialect); err != nil { return err } diff --git a/cmd/gosqlx/cmd/analyzer.go b/cmd/gosqlx/cmd/analyzer.go index a9a729e7..c9ea47bf 100644 --- a/cmd/gosqlx/cmd/analyzer.go +++ b/cmd/gosqlx/cmd/analyzer.go @@ -85,7 +85,10 @@ func (a *Analyzer) Analyze(input string) (*AnalyzerResult, error) { return result, result.Error } - // Convert TokenWithSpan to Token using centralized converter + if !hasStatementTokens(tokens) { + result.Error = errNoStatements + return result, result.Error + } // Parse with proper error handling for memory management p := parserForDialect(a.Opts.Dialect) diff --git a/cmd/gosqlx/cmd/batch_count_validation_test.go b/cmd/gosqlx/cmd/batch_count_validation_test.go new file mode 100644 index 00000000..018a8a96 --- /dev/null +++ b/cmd/gosqlx/cmd/batch_count_validation_test.go @@ -0,0 +1,35 @@ +// Copyright 2026 GoSQLX Authors +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +import ( + "io" + "testing" +) + +func TestValidatorRejectsInvalidGOBatchCount(t *testing.T) { + for _, count := range []string{"0", "999999999999999999999999999999", "1.5", "-1"} { + t.Run(count, func(t *testing.T) { + v := NewValidator(io.Discard, io.Discard, ValidatorOptions{Dialect: "sqlserver"}) + src := "SELECT 1\nGO " + count + "\nSELECT 2" + result := v.ValidateInput("batch.sql", []byte(src)) + if result.InvalidFiles != 1 || result.ValidFiles != 0 { + t.Fatalf("ValidateInput(%q): valid=%d invalid=%d; want invalid", src, result.ValidFiles, result.InvalidFiles) + } + }) + } +} + +func TestValidatorAcceptsValidGOBatchCount(t *testing.T) { + for _, directive := range []string{"GO", "GO 3", "GO 3 -- repeat", "GO 3 -- repeat\r"} { + t.Run(directive, func(t *testing.T) { + v := NewValidator(io.Discard, io.Discard, ValidatorOptions{Dialect: "sqlserver"}) + src := "SELECT 1\r\n" + directive + "\nSELECT 2" + result := v.ValidateInput("batch.sql", []byte(src)) + if result.ValidFiles != 1 || result.InvalidFiles != 0 { + t.Fatalf("ValidateInput(%q): valid=%d invalid=%d, error=%v; want valid", src, result.ValidFiles, result.InvalidFiles, result.Files[0].Error) + } + }) + } +} diff --git a/cmd/gosqlx/cmd/cli_e2e_test.go b/cmd/gosqlx/cmd/cli_e2e_test.go new file mode 100644 index 00000000..af536e33 --- /dev/null +++ b/cmd/gosqlx/cmd/cli_e2e_test.go @@ -0,0 +1,405 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +// resetCLIFlags returns every flag of every command to its default. Flag +// values live in package variables that survive from one rootCmd.Execute to +// the next, so each run starts from a clean slate. +func resetCLIFlags() { + var walk func(c *cobra.Command) + reset := func(f *pflag.Flag) { + if sv, ok := f.Value.(pflag.SliceValue); ok { + _ = sv.Replace(nil) + } else { + _ = f.Value.Set(f.DefValue) + } + f.Changed = false + } + walk = func(c *cobra.Command) { + c.SilenceUsage = false + c.Flags().VisitAll(reset) + c.PersistentFlags().VisitAll(reset) + for _, sub := range c.Commands() { + walk(sub) + } + } + walk(rootCmd) +} + +// cliResult is what one in-process run of the gosqlx command line produced. +type cliResult struct { + stdout, stderr string + err error +} + +// runCLI runs the command line in process with stdin as standard input. +// stdin is only wired up when non-nil. +func runCLI(t *testing.T, stdin *string, args ...string) cliResult { + t.Helper() + resetCLIFlags() + t.Cleanup(resetCLIFlags) + + if stdin != nil { + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("pipe: %v", err) + } + old := os.Stdin + os.Stdin = r + t.Cleanup(func() { os.Stdin = old; _ = r.Close() }) + go func() { + _, _ = w.WriteString(*stdin) + _ = w.Close() + }() + } + + var out, errOut bytes.Buffer + rootCmd.SetOut(&out) + rootCmd.SetErr(&errOut) + rootCmd.SetArgs(args) + err := rootCmd.Execute() + rootCmd.SetOut(nil) + rootCmd.SetErr(nil) + return cliResult{stdout: out.String(), stderr: errOut.String(), err: err} +} + +func str(s string) *string { return &s } + +func writeTemp(t *testing.T, name, content string) string { + t.Helper() + path := filepath.Join(t.TempDir(), name) + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + return path +} + +func TestCLI_LintExitsNonZeroOnSyntaxErrors(t *testing.T) { + bad := writeTemp(t, "bad.sql", "SELEC FROM\n") + good := writeTemp(t, "good.sql", "SELECT id FROM users WHERE id = 1;\n") + + tests := []struct { + name string + stdin *string + args []string + wantErr bool + }{ + {"stdin syntax error", str("SELEC FROM"), []string{"lint"}, true}, + {"stdin valid", str("SELECT id FROM users"), []string{"lint"}, false}, + {"file syntax error", nil, []string{"lint", bad}, true}, + {"file valid", nil, []string{"lint", good}, false}, + {"inline syntax error", nil, []string{"lint", "SELECT FROM WHERE ("}, true}, + {"inline valid", nil, []string{"lint", "SELECT id FROM users"}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + res := runCLI(t, tt.stdin, tt.args...) + if (res.err != nil) != tt.wantErr { + t.Fatalf("err = %v, wantErr %v\nstdout: %s", res.err, tt.wantErr, res.stdout) + } + if tt.wantErr && strings.Contains(res.stdout, "No violations found") { + t.Fatalf("invalid SQL reported as clean:\n%s", res.stdout) + } + }) + } +} + +func TestCLI_LintAutoFixJudgesTheFixedText(t *testing.T) { + path := writeTemp(t, "ws.sql", "SELECT id FROM users \n") + res := runCLI(t, nil, "lint", "--auto-fix", path) + if res.err != nil { + t.Fatalf("lint --auto-fix left %v after fixing everything\n%s", res.err, res.stdout) + } + if !strings.Contains(res.stdout, "0 violation(s) remain after auto-fix") { + t.Fatalf("missing remaining-violation count:\n%s", res.stdout) + } + got, _ := os.ReadFile(path) + if strings.Contains(string(got), " \n") { + t.Fatalf("file not fixed: %q", got) + } + + res = runCLI(t, str("SELECT id FROM users \n"), "lint", "--auto-fix") + if res.err != nil { + t.Fatalf("stdin lint --auto-fix: %v\n%s", res.err, res.stdout) + } +} + +func TestCLI_NoStatementsIsNotAnError(t *testing.T) { + for _, content := range []string{"", " \n", "-- just a comment\n", "/* a */ -- b\n"} { + path := writeTemp(t, "empty.sql", content) + for _, cmd := range []string{"validate", "lint", "format"} { + if cmd == "lint" && strings.TrimSpace(content) == "" && content != "" { + continue // whitespace-only text legitimately breaks the whitespace rules + } + res := runCLI(t, nil, cmd, path) + if res.err != nil { + t.Errorf("%s %q: %v\n%s%s", cmd, content, res.err, res.stdout, res.stderr) + } + } + } + // Comment-only stdin is the same: valid, and format echoes it. + res := runCLI(t, str("-- keep me\n"), "format") + if res.err != nil || strings.TrimSpace(res.stdout) != "-- keep me" { + t.Fatalf("format of a comment-only stdin = %q, %v", res.stdout, res.err) + } + if res = runCLI(t, str("-- keep me\n"), "validate"); res.err != nil { + t.Fatalf("validate of a comment-only stdin: %v", res.err) + } + // parse and analyze need a statement to show; they say so. + path := writeTemp(t, "c.sql", "-- only a comment\n") + for _, cmd := range []string{"parse", "analyze"} { + res := runCLI(t, nil, cmd, path) + if res.err == nil || !strings.Contains(res.err.Error(), "no SQL statements") { + t.Errorf("%s of a comment-only file: err = %v, want a 'no SQL statements' error", cmd, res.err) + } + } +} + +func TestCLI_DialectAliases(t *testing.T) { + for _, alias := range []string{"postgres", "pg", "mssql", "tsql", "sqlite3", "postgresql", "sqlserver"} { + for _, cmd := range []string{"validate", "format", "lint", "parse", "analyze"} { + t.Run(cmd+"/"+alias, func(t *testing.T) { + res := runCLI(t, nil, cmd, "--dialect", alias, "SELECT id FROM users") + if res.err != nil && strings.Contains(res.err.Error(), "unknown SQL dialect") { + t.Fatalf("--dialect %s rejected: %v", alias, res.err) + } + }) + } + } + res := runCLI(t, nil, "validate", "--dialect", "nosuchsql", "SELECT 1") + if res.err == nil || !strings.Contains(res.err.Error(), "unknown SQL dialect") { + t.Fatalf("unknown dialect: err = %v", res.err) + } + // Case handling for canonical names is unchanged. + if res = runCLI(t, nil, "validate", "--dialect", "MySQL", "SELECT 1"); res.err == nil { + t.Fatal(`--dialect MySQL accepted; names are case-sensitive`) + } +} + +func TestCLI_FormatNoUppercase(t *testing.T) { + res := runCLI(t, str("SELECT a FROM t"), "format", "--no-uppercase") + if res.err != nil { + t.Fatal(res.err) + } + if got := strings.ToLower(res.stdout); got != res.stdout { + t.Fatalf("--no-uppercase output has upper-case keywords: %q", res.stdout) + } + res = runCLI(t, str("select a from t"), "format") + if !strings.Contains(res.stdout, "SELECT") { + t.Fatalf("default output is not upper case: %q", res.stdout) + } + res = runCLI(t, str("select a from t"), "format", "--uppercase", "--no-uppercase") + if res.err == nil { + t.Fatal("--uppercase together with --no-uppercase was accepted") + } +} + +func TestCLI_FormatKeepsCommentsAndHints(t *testing.T) { + res := runCLI(t, str("SELECT /*+ hint */ a FROM t -- note\n"), "format") + if res.err != nil { + t.Fatal(res.err) + } + for _, want := range []string{"SELECT /*+ hint */ a", "-- note"} { + if !strings.Contains(res.stdout, want) { + t.Fatalf("output %q lost %q", res.stdout, want) + } + } + + // A hint that cannot be put back refuses the file and leaves it alone. + original := "SELECT a FROM /*+ FULL(t) */ t -- keep\n" + path := writeTemp(t, "hint.sql", original) + res = runCLI(t, nil, "format", "-i", path) + if res.err == nil { + t.Fatalf("format -i accepted a hint it cannot place\n%s%s", res.stdout, res.stderr) + } + if got, _ := os.ReadFile(path); string(got) != original { + t.Fatalf("file was rewritten despite the refusal: %q", got) + } +} + +func TestCLI_FormatInPlaceNeverDropsComments(t *testing.T) { + original := "-- header\nselect a, /* inline */ b from t; -- after\n-- next\nselect 2\n-- tail\n" + path := writeTemp(t, "c.sql", original) + res := runCLI(t, nil, "format", "-i", path) + if res.err != nil { + t.Fatalf("%v\n%s%s", res.err, res.stdout, res.stderr) + } + got, _ := os.ReadFile(path) + for _, c := range []string{"-- header", "/* inline */", "-- after", "-- next", "-- tail"} { + if !strings.Contains(string(got), c) { + t.Errorf("formatted file lost %q:\n%s", c, got) + } + } +} + +func TestCLI_FormatKeepsByteOrderMark(t *testing.T) { + path := writeTemp(t, "bom.sql", "\uFEFFselect a from t\n") + if res := runCLI(t, nil, "format", "-i", path); res.err != nil { + t.Fatal(res.err) + } + got, _ := os.ReadFile(path) + if !strings.HasPrefix(string(got), "\uFEFFSELECT") { + t.Fatalf("byte order mark lost or output wrong: %q", got) + } + if res := runCLI(t, nil, "validate", path); res.err != nil { + t.Fatalf("validate of a BOM file: %v", res.err) + } +} + +func TestCLI_ValidateStdinIsNamedStdin(t *testing.T) { + res := runCLI(t, str("SELECT 1"), "validate") + if res.err != nil { + t.Fatal(res.err) + } + if !strings.Contains(res.stdout, "") || strings.Contains(res.stdout, os.TempDir()) { + t.Fatalf("stdin result is not labelled :\n%s", res.stdout) + } + res = runCLI(t, str("SELEC"), "validate") + if res.err == nil || !strings.Contains(res.stderr+res.stdout, "") { + t.Fatalf("invalid stdin not reported under : err=%v\n%s%s", res.err, res.stdout, res.stderr) + } +} + +func TestCLI_ErrorsAreNotPrintedByCobraAndUsageIsSilent(t *testing.T) { + res := runCLI(t, str("SELEC"), "validate") + if res.err == nil { + t.Fatal("expected a validation failure") + } + if strings.Contains(res.stderr, "Usage:") || strings.Contains(res.stdout, "Usage:") { + t.Fatalf("usage printed for a syntax error:\n%s%s", res.stdout, res.stderr) + } + if strings.Contains(res.stderr, "Error:") { + t.Fatalf("cobra printed the error that main prints:\n%s", res.stderr) + } + // Argument errors still show usage. + res = runCLI(t, nil, "parse", "a", "b") + if res.err == nil || !strings.Contains(res.stderr+res.stdout, "Usage:") { + t.Fatalf("argument error without usage: err=%v\n%s%s", res.err, res.stdout, res.stderr) + } +} + +func TestCLI_ValidateQuietFailsSilently(t *testing.T) { + res := runCLI(t, str("SELEC"), "validate", "--quiet") + if res.err == nil || !IsSilentError(res.err) { + t.Fatalf("quiet validation failure: err = %v, want a silent error", res.err) + } + if res.stdout != "" || res.stderr != "" { + t.Fatalf("quiet mode printed output: %q %q", res.stdout, res.stderr) + } + // Non-validation errors are still reported. + res = runCLI(t, nil, "validate", "--quiet", "--dialect", "bogus", "SELECT 1") + if res.err == nil || IsSilentError(res.err) { + t.Fatalf("bad dialect under --quiet: err = %v, want a visible error", res.err) + } +} + +func TestCLI_ValidateAcceptsBatchScripts(t *testing.T) { + tsql := writeTemp(t, "batches.sql", "CREATE TABLE a (id INT)\nGO\nSELECT id FROM a\nGO\n") + res := runCLI(t, nil, "validate", "--dialect", "sqlserver", tsql) + if res.err != nil { + t.Fatalf("T-SQL GO script: %v\n%s%s", res.err, res.stdout, res.stderr) + } + // Without the dialect, GO is not a separator and the file is invalid. + if res = runCLI(t, nil, "validate", "--dialect", "postgresql", tsql); res.err == nil { + t.Fatal("GO script accepted for postgresql") + } + + mysql := writeTemp(t, "delim.sql", "DELIMITER //\nSELECT 1//\nSELECT 2//\nDELIMITER ;\nSELECT 3;\n") + if res = runCLI(t, nil, "validate", "--dialect", "mysql", mysql); res.err != nil { + t.Fatalf("DELIMITER script: %v\n%s%s", res.err, res.stdout, res.stderr) + } + + // A genuinely broken statement is still reported, with the whole-file + // position. + broken := writeTemp(t, "broken.sql", "SELECT 1;\nSELEC 2;\n") + res = runCLI(t, nil, "validate", broken) + if res.err == nil || !strings.Contains(res.stderr+res.stdout, "line 2") { + t.Fatalf("broken file: err=%v\n%s%s", res.err, res.stdout, res.stderr) + } +} + +func TestCLI_GlobalFlagsAreHonouredOrRejected(t *testing.T) { + // -f is rejected where there is only one output format. + for _, args := range [][]string{ + {"lint", "-f", "json", "SELECT 1"}, + {"format", "-f", "json", "SELECT 1"}, + {"transpile", "-f", "yaml", "SELECT 1"}, + } { + res := runCLI(t, nil, args...) + if res.err == nil || !strings.Contains(res.err.Error(), "-f/--format") { + t.Errorf("%v: err = %v, want a -f/--format rejection", args, res.err) + } + } + // "-f auto" is the default and is fine. + if res := runCLI(t, nil, "format", "-f", "auto", "SELECT 1"); res.err != nil { + t.Errorf("format -f auto: %v", res.err) + } + + // -o is honoured by parse and transpile. + dir := t.TempDir() + out := filepath.Join(dir, "ast.txt") + if res := runCLI(t, nil, "parse", "-o", out, "SELECT 1"); res.err != nil { + t.Fatal(res.err) + } else if res.stdout != "" { + t.Fatalf("parse -o still wrote to stdout: %q", res.stdout) + } + if b, err := os.ReadFile(out); err != nil || len(b) == 0 { + t.Fatalf("parse -o produced no file: %v", err) + } + out = filepath.Join(dir, "t.sql") + if res := runCLI(t, nil, "transpile", "--from", "mysql", "--to", "postgres", "-o", out, "SELECT 1"); res.err != nil { + t.Fatal(res.err) + } + if b, err := os.ReadFile(out); err != nil || !strings.Contains(string(b), "SELECT 1") { + t.Fatalf("transpile -o file = %q, %v", b, err) + } + + // format -o with several inputs would overwrite the file repeatedly. + a := writeTemp(t, "a.sql", "SELECT 1") + b := writeTemp(t, "b.sql", "SELECT 2") + if res := runCLI(t, nil, "format", "-o", filepath.Join(dir, "o.sql"), a, b); res.err == nil { + t.Fatal("format -o with two inputs accepted") + } + + // validate maps -f json / -o onto its own output flags. + res := runCLI(t, nil, "validate", "-f", "json", "SELECT 1") + if res.err != nil || !strings.Contains(res.stdout, "{") { + t.Fatalf("validate -f json: %v\n%s", res.err, res.stdout) + } + if res = runCLI(t, nil, "validate", "-f", "yaml", "SELECT 1"); res.err == nil { + t.Fatal("validate -f yaml accepted") + } + jsonOut := filepath.Join(dir, "v.json") + if res = runCLI(t, nil, "validate", "-f", "json", "-o", jsonOut, "SELECT 1"); res.err != nil { + t.Fatalf("validate -f json -o: %v", res.err) + } + if b, err := os.ReadFile(jsonOut); err != nil || !strings.Contains(string(b), "{") { + t.Fatalf("validate -o file = %q, %v", b, err) + } + if res = runCLI(t, nil, "validate", "-o", filepath.Join(dir, "x"), "SELECT 1"); res.err == nil { + t.Fatal("validate -o without a machine-readable format accepted") + } +} diff --git a/cmd/gosqlx/cmd/cmd_utils.go b/cmd/gosqlx/cmd/cmd_utils.go index d8383be8..d07b205a 100644 --- a/cmd/gosqlx/cmd/cmd_utils.go +++ b/cmd/gosqlx/cmd/cmd_utils.go @@ -15,12 +15,14 @@ package cmd import ( + "errors" "fmt" "strings" "github.com/spf13/cobra" "github.com/spf13/pflag" + "github.com/ajitpratap0/GoSQLX/pkg/models" "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" "github.com/ajitpratap0/GoSQLX/pkg/sql/parser" "github.com/ajitpratap0/GoSQLX/pkg/sql/tokenizer" @@ -57,7 +59,7 @@ func trackChangedFlags(cmd *cobra.Command) map[string]bool { // Names are matched case-sensitively, as keywords.IsValidDialect does. func validateDialectName(dialect string) error { if dialect != "" && !keywords.IsValidDialect(dialect) { - return fmt.Errorf("unknown SQL dialect %q; valid dialects: %s", dialect, validDialectsText()) + return fmt.Errorf("unknown SQL dialect %q; valid dialects: %s (aliases: %s)", dialect, validDialectsText(), dialectAliasText()) } return nil } @@ -105,3 +107,19 @@ func parserForDialect(dialect string) *parser.Parser { } return parser.NewParser(parser.WithDialect(dialect)) } + +// errNoStatements is returned by commands that need a statement to work on +// (parse, analyze) when the input is empty, whitespace or only comments. +var errNoStatements = errors.New("no SQL statements found (input is empty or contains only comments)") + +// hasStatementTokens reports whether tokens holds anything besides the +// end-of-input marker. Empty, whitespace-only and comment-only input has no +// statements; every command treats it the same way. +func hasStatementTokens(tokens []models.TokenWithSpan) bool { + for i := range tokens { + if tokens[i].Token.Type != models.TokenTypeEOF { + return true + } + } + return false +} diff --git a/cmd/gosqlx/cmd/dialect_alias.go b/cmd/gosqlx/cmd/dialect_alias.go new file mode 100644 index 00000000..9c9f9062 --- /dev/null +++ b/cmd/gosqlx/cmd/dialect_alias.go @@ -0,0 +1,62 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "sort" + "strings" +) + +// dialectAliases maps the alternative names users commonly type to the +// canonical dialect names of keywords.DialectNames. Matching is exact +// (lower case), like the canonical names. +var dialectAliases = map[string]string{ + "postgres": "postgresql", + "pg": "postgresql", + "mssql": "sqlserver", + "tsql": "sqlserver", + "sql-server": "sqlserver", + "sqlite3": "sqlite", +} + +// canonicalDialect returns the canonical dialect name for name: aliases are +// resolved, anything else is returned unchanged for validateDialectName to +// accept or reject. +func canonicalDialect(name string) string { + if canon, ok := dialectAliases[name]; ok { + return canon + } + return name +} + +// normalizeDialectFlag resolves aliases in the dialect flag value *name in +// place and rejects unknown dialects. Every command that takes --dialect +// calls it first, so "postgres", "pg", "mssql" and "tsql" work everywhere, +// not only in transpile. +func normalizeDialectFlag(name *string) error { + *name = canonicalDialect(*name) + return validateDialectName(*name) +} + +// dialectAliasText lists the accepted aliases for error messages, e.g. +// "mssql, pg, postgres, sql-server, sqlite3, tsql". +func dialectAliasText() string { + names := make([]string, 0, len(dialectAliases)) + for alias := range dialectAliases { + names = append(names, alias) + } + sort.Strings(names) + return strings.Join(names, ", ") +} diff --git a/cmd/gosqlx/cmd/format.go b/cmd/gosqlx/cmd/format.go index c394a081..ac6246e7 100644 --- a/cmd/gosqlx/cmd/format.go +++ b/cmd/gosqlx/cmd/format.go @@ -32,6 +32,8 @@ var ( formatCheck bool formatMaxLine int formatDialect string + + formatNoUppercase bool ) // formatCmd represents the format command @@ -44,7 +46,7 @@ Examples: gosqlx format query.sql # Format to stdout gosqlx format -i query.sql # Format in-place gosqlx format --indent 4 query.sql # Use 4-space indentation - gosqlx format --no-uppercase query.sql # Keep original keyword case + gosqlx format --no-uppercase query.sql # Write keywords in lower case gosqlx format --compact query.sql # Compact format (minimal whitespace) gosqlx format --check query.sql # Check if formatting is needed (CI mode) gosqlx format "*.sql" # Format all SQL files @@ -64,9 +66,19 @@ Performance: 100x faster than SQLFluff for equivalent operations`, func formatRun(cmd *cobra.Command, args []string) error { // Reject unknown dialect names early before any parsing. - if err := validateDialectName(formatDialect); err != nil { + if err := normalizeDialectFlag(&formatDialect); err != nil { + return err + } + + if err := rejectGlobalFormat(cmd); err != nil { return err } + if err := resolveUppercaseFlags(cmd); err != nil { + return err + } + if outputFile != "" && len(args) > 1 { + return fmt.Errorf("-o/--output writes one file but %d inputs were given; format them one at a time or use -i", len(args)) + } // Handle stdin input if ShouldReadFromStdin(args) { @@ -251,6 +263,19 @@ func init() { formatCmd.Flags().IntVar(&formatMaxLine, "max-line", 80, "maximum line length (config: format.max_line_length)") formatCmd.Flags().StringVar(&formatDialect, "dialect", "", dialectFlagUsage("default: postgresql")) - // Add negation flags - formatCmd.Flags().BoolVar(&formatUppercase, "no-uppercase", false, "keep original keyword case") + // Negation flag. It has its own variable: sharing formatUppercase made + // --no-uppercase set uppercase to true. + formatCmd.Flags().BoolVar(&formatNoUppercase, "no-uppercase", false, "write keywords in lower case (opposite of --uppercase)") +} + +// resolveUppercaseFlags folds --no-uppercase into formatUppercase. +func resolveUppercaseFlags(cmd *cobra.Command) error { + if !formatNoUppercase { + return nil + } + if flagSet(cmd, "uppercase") && formatUppercase { + return fmt.Errorf("--uppercase and --no-uppercase cannot be used together") + } + formatUppercase = false + return nil } diff --git a/cmd/gosqlx/cmd/format_batch_test.go b/cmd/gosqlx/cmd/format_batch_test.go new file mode 100644 index 00000000..a31b0e1f --- /dev/null +++ b/cmd/gosqlx/cmd/format_batch_test.go @@ -0,0 +1,39 @@ +// Copyright 2026 GoSQLX Authors +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +import ( + "fmt" + "strings" + "testing" +) + +func TestFormatSQLBatchBoundaries(t *testing.T) { + for _, input := range []string{ + "CREATE PROCEDURE p AS SELECT 1\nGO\nSELECT 2\nGO 3", + "CREATE VIEW v AS SELECT 1 AS a\nGO -- next\nSELECT * FROM v", + "CREATE TRIGGER tr ON t AFTER INSERT AS BEGIN SELECT 1; END\nGO\nSELECT 2", + "SET NOCOUNT ON\nSELECT 1\nGO\nSET NOCOUNT OFF\nGO", + "GO\nGO 2\nSELECT 1\nGO", + "SELECT 1;\nGO\n;SELECT 2 -- hi", + } { + for _, compact := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/compact=%t", input, compact), func(t *testing.T) { + opts := CLIFormatterOptions{Dialect: "sqlserver", Compact: compact, Uppercase: true, IndentSize: 2} + out, err := formatWith(input, opts) + if err != nil { + t.Fatal(err) + } + assertRoundTrip(t, input, out, "sqlserver") + if strings.Contains(out, "GO;") || strings.Contains(out, "GO 3;") { + t.Fatalf("GO acquired a semicolon: %s", out) + } + again, err := formatWith(out, opts) + if err != nil || again != out { + t.Fatalf("format not idempotent: %v\n%s\n%s", err, out, again) + } + }) + } + } +} diff --git a/cmd/gosqlx/cmd/format_check_summary_test.go b/cmd/gosqlx/cmd/format_check_summary_test.go index 20335d36..daceca04 100644 --- a/cmd/gosqlx/cmd/format_check_summary_test.go +++ b/cmd/gosqlx/cmd/format_check_summary_test.go @@ -28,11 +28,11 @@ func TestFormat_CheckSummaryOmitsFailedFiles(t *testing.T) { dir := t.TempDir() good := filepath.Join(dir, "good.sql") bad := filepath.Join(dir, "bad.sql") - // EXPLAIN is refused by the formatter, so bad.sql always fails. + // DESCRIBE is refused by the formatter, so bad.sql always fails. if err := os.WriteFile(good, []byte("SELECT a FROM t"), 0600); err != nil { t.Fatal(err) } - if err := os.WriteFile(bad, []byte("EXPLAIN VERBOSE DELETE FROM users"), 0600); err != nil { + if err := os.WriteFile(bad, []byte("DESCRIBE users"), 0600); err != nil { t.Fatal(err) } diff --git a/cmd/gosqlx/cmd/format_comment_starts_test.go b/cmd/gosqlx/cmd/format_comment_starts_test.go new file mode 100644 index 00000000..76aebea2 --- /dev/null +++ b/cmd/gosqlx/cmd/format_comment_starts_test.go @@ -0,0 +1,52 @@ +// Copyright 2026 GoSQLX Authors +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +import ( + "fmt" + "strings" + "testing" +) + +// TestFormatCommentsStayWithTheirStatements checks that comments are not +// attached to the wrong statement when a statement holds ';' (a procedure or +// trigger body) or T-SQL statements are not separated by ';'. +func TestFormatCommentsStayWithTheirStatements(t *testing.T) { + tests := []struct { + dialect string + input string + // order lists pieces of the output in the order they must appear. + order []string + }{ + {"sqlserver", + "-- A\nSELECT 1\nSELECT 2;\n-- B: the proc\nCREATE PROC p AS BEGIN SELECT 1; END;\n", + []string{"-- A", "SELECT 1", "SELECT 2", "-- B: the proc", "CREATE PROC"}}, + {"mysql", + "-- Section one\nSELECT 1 FROM users;\n-- Section two: trigger\nCREATE TRIGGER t AFTER INSERT ON x FOR EACH ROW BEGIN SELECT 1; SELECT 2; END;\n-- Section three\nSELECT 3 FROM orders;\n", + []string{"-- Section one", "SELECT 1", "users", "-- Section two: trigger", "CREATE TRIGGER", "-- Section three", "SELECT 3"}}, + } + for _, tc := range tests { + for _, compact := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/compact=%t", tc.dialect, compact), func(t *testing.T) { + opts := CLIFormatterOptions{Dialect: tc.dialect, Compact: compact, Uppercase: true, IndentSize: 2} + out, err := formatWith(tc.input, opts) + if err != nil { + t.Fatal(err) + } + rest := out + for _, piece := range tc.order { + i := strings.Index(rest, piece) + if i < 0 { + t.Fatalf("%q missing or out of order in:\n%s", piece, out) + } + rest = rest[i+len(piece):] + } + again, err := formatWith(out, opts) + if err != nil || again != out { + t.Fatalf("format not idempotent: %v\n%s\n%s", err, out, again) + } + }) + } + } +} diff --git a/cmd/gosqlx/cmd/format_corpus_test.go b/cmd/gosqlx/cmd/format_corpus_test.go index cb8afde4..31544087 100644 --- a/cmd/gosqlx/cmd/format_corpus_test.go +++ b/cmd/gosqlx/cmd/format_corpus_test.go @@ -30,19 +30,14 @@ import ( // rendering gap. For these the CLI must refuse to format (return an error) // rather than emit changed SQL; once the gap is fixed they simply pass. var corpusFormatKnownGaps = map[string]string{ - "corpus/orm/gorm_joins.sql": "follow-up: a name quoted only for case (\"Company\") renders bare; the AST does not record quoting, so the CLI refuses", - "mssql/13_cross_apply.sql": "follow-up: CROSS APPLY rendered as CROSS APPLY JOIN, TOP lost in derived table", - "mssql/14_outer_apply.sql": "follow-up: OUTER APPLY rendered as OUTER APPLY JOIN, TOP lost in derived table", - "mysql/29_full_text_search.sql": "follow-up: MATCH ... AGAINST rendered as AGAINST AGAINST(...)", - "oracle/08_connect_by_path.sql": "follow-up: START WITH / CONNECT BY not rendered", - "oracle/19_regexp_functions.sql": "follow-up: string literal backslashes doubled for a dialect that keeps them literal", + "realworld/mysql/05_dialect_specific.sql": "DESCRIBE has no renderer, so the CLI refuses the file rather than rewrite it (see TestFormatSQL_RefusesDescribe)", } // corpusDialect maps a testdata path to the dialect its SQL is written in. func corpusDialect(rel string) string { p := "/" + rel switch { - case strings.Contains(p, "/mssql/"): + case strings.Contains(p, "/mssql/"), strings.Contains(p, "/sqlserver/"): return "sqlserver" case strings.Contains(p, "/mysql/"): return "mysql" diff --git a/cmd/gosqlx/cmd/format_explain_test.go b/cmd/gosqlx/cmd/format_explain_test.go index e0e6dfc7..12cea154 100644 --- a/cmd/gosqlx/cmd/format_explain_test.go +++ b/cmd/gosqlx/cmd/format_explain_test.go @@ -20,28 +20,49 @@ import ( "path/filepath" "strings" "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" ) -// TestFormatSQL_RefusesExplainAndDescribe guards against gosqlx format turning -// a non-executing EXPLAIN into an executable statement. The parser splits -// "EXPLAIN VERBOSE|ANALYZE " into a DescribeStatement plus , and -// DescribeStatement drops EXPLAIN options and object kinds, so the CLI must -// refuse it instead of writing "DESCRIBE VERBOSE; DELETE FROM users". -func TestFormatSQL_RefusesExplainAndDescribe(t *testing.T) { +// TestFormatSQL_PreservesExplain checks that EXPLAIN formats as one +// statement that keeps EXPLAIN and its options, so a non-executing EXPLAIN is +// never rewritten into the executable statement. +func TestFormatSQL_PreservesExplain(t *testing.T) { + tests := []struct { + dialect string + in string + want string // compact, upper-cased output + }{ + {"", "EXPLAIN VERBOSE DELETE FROM users", "EXPLAIN VERBOSE DELETE FROM users"}, + {"postgresql", "EXPLAIN VERBOSE DELETE FROM users;\n", "EXPLAIN VERBOSE DELETE FROM users"}, + {"", "explain verbose update accounts set balance = 0", "EXPLAIN VERBOSE UPDATE accounts SET balance = 0"}, + {"", "EXPLAIN VERBOSE INSERT INTO t (a) VALUES (1)", "EXPLAIN VERBOSE INSERT INTO t (a) VALUES (1)"}, + {"", "EXPLAIN ANALYZE DELETE FROM users", "EXPLAIN ANALYZE DELETE FROM users"}, + {"mysql", "EXPLAIN ANALYZE SELECT * FROM users", "EXPLAIN ANALYZE SELECT * FROM users"}, + {"", "EXPLAIN SELECT 1", "EXPLAIN SELECT 1"}, + } + for _, tc := range tests { + out, err := formatWith(tc.in, CLIFormatterOptions{Compact: true, Uppercase: true, Dialect: tc.dialect}) + if err != nil { + t.Fatalf("formatSQL(%q): %v", tc.in, err) + } + if got := strings.TrimSuffix(strings.TrimSpace(out), ";"); got != tc.want { + t.Errorf("formatSQL(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} + +// TestFormatSQL_RefusesDescribe guards statements the formatter cannot +// rewrite faithfully: DESCRIBE drops object kinds, and EXPLAIN of a statement +// the parser does not wrap would otherwise be split in two. +func TestFormatSQL_RefusesDescribe(t *testing.T) { tests := []struct { name string dialect string in string }{ - {name: "explain verbose delete", in: "EXPLAIN VERBOSE DELETE FROM users"}, - {name: "explain verbose delete postgresql", dialect: "postgresql", in: "EXPLAIN VERBOSE DELETE FROM users;\n"}, - {name: "explain verbose update", in: "EXPLAIN VERBOSE UPDATE accounts SET balance = 0"}, {name: "explain verbose drop", in: "EXPLAIN VERBOSE DROP TABLE users"}, {name: "explain verbose truncate", in: "EXPLAIN VERBOSE TRUNCATE TABLE users"}, - {name: "explain verbose insert", in: "EXPLAIN VERBOSE INSERT INTO t (a) VALUES (1)"}, - {name: "explain analyze delete", in: "EXPLAIN ANALYZE DELETE FROM users"}, - {name: "mysql explain analyze select", dialect: "mysql", in: "EXPLAIN ANALYZE SELECT * FROM users"}, - {name: "explain select", in: "EXPLAIN SELECT 1"}, {name: "describe table", in: "DESCRIBE users"}, {name: "snowflake describe table kind", dialect: "snowflake", in: "DESCRIBE TABLE users"}, } @@ -64,10 +85,10 @@ func TestFormatSQL_RefusesExplainAndDescribe(t *testing.T) { } } -// TestFormat_InPlaceLeavesExplainUnchanged checks the end-to-end format -i +// TestFormat_InPlaceLeavesDescribeUnchanged checks the end-to-end format -i // path: the file must be left untouched and counted as failed. -func TestFormat_InPlaceLeavesExplainUnchanged(t *testing.T) { - const original = "EXPLAIN VERBOSE DELETE FROM users;\n" +func TestFormat_InPlaceLeavesDescribeUnchanged(t *testing.T) { + const original = "DESCRIBE users;\n" path := filepath.Join(t.TempDir(), "r.sql") if err := os.WriteFile(path, []byte(original), 0600); err != nil { t.Fatal(err) @@ -129,3 +150,75 @@ func TestFormatSQL_UnsupportedStatementMessage(t *testing.T) { t.Fatalf("want an error naming DescribeStatement, got %v", err) } } + +// TestTSQLStatementBound covers the SQL Server form of the separator guard: +// statements may end without ';', so the bound also counts GO lines and the +// words a T-SQL statement can start with. +func TestTSQLStatementBound(t *testing.T) { + tests := []struct { + in string + want int + }{ + {"SELECT 1", 1}, + {"SELECT 1;", 1}, + {"SELECT 1\nGO\nSELECT 2", 3}, + {"SELECT 1\nGO 2\nSELECT 2 PRINT 'x'", 4}, + {"UPDATE t SET a = 1 WHERE b = 2", 2}, // SET may start a statement + {"SELECT a, b FROM t WHERE c = 1", 1}, + {"SELECT t.select FROM t", 1}, // a qualified name part is not a start + {"SELECT [select] FROM t", 1}, // nor is a quoted name + } + for _, tc := range tests { + tkz, release, err := tokenizerForDialect("sqlserver") + if err != nil { + t.Fatal(err) + } + tokens, err := tkz.TokenizeString(tc.in) + if err != nil { + release() + t.Fatalf("tokenize %q: %v", tc.in, err) + } + got := tsqlStatementBound(tokens) + release() + if got != tc.want { + t.Errorf("tsqlStatementBound(%q) = %d, want %d", tc.in, got, tc.want) + } + } +} + +// TestCheckFormattable_SQLServerGuard checks that the statement-count guard +// runs for SQL Server: a tree with more statements than the input can hold +// (the parser split a statement) is refused, a legitimate one is not. +func TestCheckFormattable_SQLServerGuard(t *testing.T) { + tkz, release, err := tokenizerForDialect("sqlserver") + if err != nil { + t.Fatal(err) + } + defer release() + tokens, err := tkz.TokenizeString("SELECT a b") + if err != nil { + t.Fatal(err) + } + split := &ast.AST{Statements: []ast.Statement{&ast.SelectStatement{}, &ast.SelectStatement{}}} + if err := checkFormattable(split, tokens, "sqlserver"); err == nil { + t.Error("a statement split in two was accepted") + } + whole := &ast.AST{Statements: []ast.Statement{&ast.SelectStatement{}}} + if err := checkFormattable(whole, tokens, "sqlserver"); err != nil { + t.Errorf("one statement refused: %v", err) + } + for _, sql := range []string{ + "SELECT 1\nGO\nSELECT 2\nGO 3", + "SET NOCOUNT ON\nSELECT 1\nUPDATE t SET a = 1\nCOMMIT", + "IF @a = 1 SELECT 1 ELSE SELECT 2", + } { + out, err := formatWith(sql, CLIFormatterOptions{Compact: true, Dialect: "sqlserver"}) + if err != nil { + t.Errorf("format %q: %v", sql, err) + continue + } + if out == "" { + t.Errorf("format %q: empty output", sql) + } + } +} diff --git a/cmd/gosqlx/cmd/format_guard_test.go b/cmd/gosqlx/cmd/format_guard_test.go index aac8cb43..0150d39c 100644 --- a/cmd/gosqlx/cmd/format_guard_test.go +++ b/cmd/gosqlx/cmd/format_guard_test.go @@ -33,14 +33,11 @@ func TestFormatSQL_RefusesLostQuotes(t *testing.T) { dialect string in string }{ - {name: "alter mixed case", in: `ALTER TABLE t ADD COLUMN "Mixed" INT`}, - // Refused by checkMySQLDoubleQuotes: in ALTER, `select` would render as - // "select". (In a SELECT list checkQuotedTokens refuses it first.) - {name: "alter mysql backtick keyword", dialect: "mysql", in: "ALTER TABLE `Users` ADD COLUMN `select` INT"}, - {name: "select mixed case", in: `SELECT "Mixed" FROM t`}, - {name: "select mixed case table", in: `SELECT a FROM "Mixed"`}, - {name: "create table quoted", in: `CREATE TABLE "Mixed" ("order" INT)`}, - {name: "oracle lowercase quoted", dialect: "oracle", in: `SELECT "abc" FROM t`}, + // Names the AST does not record quoting for (a TABLESPACE name, for + // example) still render bare, so the CLI must refuse. (CREATE + // SEQUENCE and ALTER POLICY names are recorded, so their quotes + // survive.) + {name: "tablespace mixed case", in: `CREATE TABLE t (a INT) TABLESPACE "Ts"`}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -70,15 +67,20 @@ func TestFormatSQL_AllowsHarmlessQuotes(t *testing.T) { want string }{ {name: "lowercase quoted", in: `SELECT "users_id" FROM "users"`, want: "SELECT users_id FROM users"}, + {name: "alter policy mixed case", in: `ALTER POLICY "Pol" ON "Tbl" RENAME TO "Q"`, want: `ALTER POLICY "Pol" ON "Tbl" RENAME TO "Q"`}, {name: "alter plain", in: "ALTER TABLE t ADD COLUMN c INT", want: "ALTER TABLE t ADD COLUMN c INT"}, // Reserved words keep their quotes on render (#632), so these format. {name: "alter reserved word", in: `ALTER TABLE t ADD COLUMN "order" INT`, want: `ALTER TABLE t ADD COLUMN "order" INT`}, {name: "alter reserved word postgresql", dialect: "postgresql", in: `ALTER TABLE t ADD COLUMN "order" INT`, want: `ALTER TABLE t ADD COLUMN "order" INT`}, {name: "mysql backtick plain", dialect: "mysql", in: "ALTER TABLE `Users` ADD COLUMN `c` INT", want: "ALTER TABLE Users ADD COLUMN c INT"}, - {name: "sqlserver bracket kept quoted", dialect: "sqlserver", in: "SELECT [Mixed Name] FROM t", want: `SELECT "Mixed Name" FROM t`}, + {name: "sqlserver bracket kept quoted", dialect: "sqlserver", in: "SELECT [Mixed Name] FROM t", want: "SELECT [Mixed Name] FROM t"}, {name: "oracle uppercase quoted", dialect: "oracle", in: `SELECT "ABC" FROM t`, want: "SELECT ABC FROM t"}, {name: "string literal kept", in: "SELECT a FROM t WHERE b = 'it''s'", want: "SELECT a FROM t WHERE b = 'it''s'"}, {name: "mysql escaped literal", dialect: "mysql", in: `SELECT 'a\nb'`, want: `SELECT 'a\nb'`}, + {name: "mysql adjacent literals", dialect: "mysql", in: "SELECT 'a' 'b'", want: "SELECT 'ab'"}, + {name: "postgresql adjacent literals", dialect: "postgresql", in: "SELECT 'a' 'b' || 'c'", want: "SELECT 'ab' || 'c'"}, + {name: "mysql string alias", dialect: "mysql", in: "SELECT a AS 'alias with space', b AS 'plain', 'lit' FROM t", want: "SELECT a AS `alias with space`, b AS plain, 'lit' FROM t"}, + {name: "sqlite string alias", dialect: "sqlite", in: "SELECT a AS 'x y' FROM t", want: `SELECT a AS "x y" FROM t`}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -93,19 +95,37 @@ func TestFormatSQL_AllowsHarmlessQuotes(t *testing.T) { } } -// TestFormatSQL_RefusesBackslashLiteralChange checks that string literals -// with backslashes are never rewritten in dialects where backslash is not an -// escape character (the renderer doubles them); the CLI must refuse. -func TestFormatSQL_RefusesBackslashLiteralChange(t *testing.T) { - for _, d := range []string{"sqlserver", "oracle", "sqlite", "snowflake", "clickhouse"} { - t.Run(d, func(t *testing.T) { - in := `SELECT 'C:\temp\x'` - out, err := formatWith(in, CLIFormatterOptions{Compact: true, Uppercase: true, Dialect: d}) - if err == nil { - t.Fatalf("formatSQL(%q, %s) = %q, want an error", in, d, out) +// TestFormatSQL_KeepsBackslashLiteral checks that a string literal with +// backslashes is written back with the rules of its dialect: untouched where +// the backslash is an ordinary character, re-escaped (and so equal after +// tokenizing) where it escapes. The renderer used to double every backslash, +// so the guard refused these inputs. +func TestFormatSQL_KeepsBackslashLiteral(t *testing.T) { + const in = `SELECT 'C:\temp\x'` + for _, tt := range []struct { + dialect string + want string // literal text expected in the output + }{ + {"sqlserver", `'C:\temp\x'`}, + {"oracle", `'C:\temp\x'`}, + {"sqlite", `'C:\temp\x'`}, + {"postgresql", `'C:\temp\x'`}, + // \t is a tab; \x keeps its backslash, which the output doubles. + // (Snowflake refuses an incomplete \x escape; see + // format_string_escape_test.go.) + {"clickhouse", "'C:\temp\\\\x'"}, + } { + t.Run(tt.dialect, func(t *testing.T) { + out, err := formatWith(in, CLIFormatterOptions{Compact: true, Uppercase: true, Dialect: tt.dialect}) + if err != nil { + t.Fatalf("formatSQL(%q, %s): %v", in, tt.dialect, err) + } + if !strings.Contains(out, tt.want) { + t.Fatalf("formatSQL(%q, %s) = %q, want it to contain %q", in, tt.dialect, out, tt.want) } - if !strings.Contains(err.Error(), "string literal") { - t.Fatalf("error %q does not name the changed string literal", err) + again, err := formatWith(out, CLIFormatterOptions{Compact: true, Uppercase: true, Dialect: tt.dialect}) + if err != nil || again != out { + t.Fatalf("not idempotent: %q then %q (err %v)", out, again, err) } }) } diff --git a/cmd/gosqlx/cmd/format_quoting_test.go b/cmd/gosqlx/cmd/format_quoting_test.go new file mode 100644 index 00000000..b153d121 --- /dev/null +++ b/cmd/gosqlx/cmd/format_quoting_test.go @@ -0,0 +1,67 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import "testing" + +// TestFormatSQL_KeepsNeededQuotes checks that the CLI formats input whose +// quoted names need their quotes (letter case, keywords, MySQL backtick +// aliases) and keeps those quotes, where it used to refuse because the +// renderer wrote them bare or in the wrong delimiters. +func TestFormatSQL_KeepsNeededQuotes(t *testing.T) { + tests := []struct { + name string + dialect string + in string + want string + }{ + {name: "select mixed case", in: `SELECT "Mixed" FROM t`, want: `SELECT "Mixed" FROM t`}, + {name: "select mixed case table", in: `SELECT a FROM "Mixed"`, want: `SELECT a FROM "Mixed"`}, + {name: "qualified mixed case alias", in: `SELECT "C"."name" AS "CName" FROM "companies" AS "C"`, + want: `SELECT "C".name AS "CName" FROM companies "C"`}, + {name: "keyword table name", dialect: "postgresql", in: `SELECT 1 FROM "table"`, want: `SELECT 1 FROM "table"`}, + {name: "user column", dialect: "postgresql", in: `SELECT "user" FROM t`, want: `SELECT "user" FROM t`}, + {name: "oracle lowercase quoted", dialect: "oracle", in: `SELECT "abc" FROM t`, want: `SELECT "abc" FROM t`}, + {name: "oracle mixed case", dialect: "oracle", in: `SELECT "ID", "CreatedAt" FROM "AppUsers"`, + want: `SELECT ID, "CreatedAt" FROM "AppUsers"`}, + {name: "oracle bare hash", dialect: "oracle", in: `SELECT s.serial#, emp# FROM v$session s`, + want: `SELECT s.serial#, emp# FROM v$session s`}, + {name: "create table quoted", in: `CREATE TABLE "Mixed" ("order" INT)`, want: `CREATE TABLE "Mixed" ("order" INT)`}, + {name: "alter add mixed case column", in: `ALTER TABLE t ADD COLUMN "Mixed" INT`, want: `ALTER TABLE t ADD COLUMN "Mixed" INT`}, + {name: "prisma unique index", dialect: "postgresql", in: `CREATE UNIQUE INDEX "User_email_key" ON "User"("email")`, + want: `CREATE UNIQUE INDEX "User_email_key" ON "User" (email)`}, + {name: "prisma foreign key", dialect: "postgresql", + in: `ALTER TABLE "Post" ADD CONSTRAINT "Post_authorId_fkey" FOREIGN KEY ("authorId") REFERENCES "User"("id") ON DELETE RESTRICT ON UPDATE CASCADE`, + want: `ALTER TABLE "Post" ADD CONSTRAINT "Post_authorId_fkey" FOREIGN KEY ("authorId") REFERENCES "User" (id) ON DELETE RESTRICT ON UPDATE CASCADE`}, + {name: "prisma drop table", dialect: "postgresql", in: `DROP TABLE "Post"`, want: `DROP TABLE "Post"`}, + {name: "mysql backtick reserved alias", dialect: "mysql", in: "SELECT EXISTS(SELECT 1) AS `exists`", + want: "SELECT EXISTS (SELECT 1) AS `exists`"}, + {name: "mysql backtick keyword column", dialect: "mysql", in: "ALTER TABLE `Users` ADD COLUMN `select` INT", + want: "ALTER TABLE Users ADD COLUMN `select` INT"}, + {name: "mysql backtick table alias", dialect: "mysql", in: "SELECT `o`.`id` FROM `order` AS `o`", + want: "SELECT o.id FROM `order` o"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + out, err := formatWith(tc.in, CLIFormatterOptions{Compact: true, Uppercase: true, Dialect: tc.dialect}) + if err != nil { + t.Fatalf("formatSQL(%q): %v", tc.in, err) + } + if out != tc.want { + t.Fatalf("formatSQL(%q) = %q, want %q", tc.in, out, tc.want) + } + }) + } +} diff --git a/cmd/gosqlx/cmd/format_string_escape_test.go b/cmd/gosqlx/cmd/format_string_escape_test.go new file mode 100644 index 00000000..0f4f14a6 --- /dev/null +++ b/cmd/gosqlx/cmd/format_string_escape_test.go @@ -0,0 +1,59 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import "testing" + +// TestFormatSQL_StringEscapesByDialect checks the CLI formatter on string +// literals whose backslashes mean different things per dialect. Without +// --dialect the escape cannot be interpreted, so the input is refused rather +// than rewritten (it used to come back with every backslash doubled, which +// PostgreSQL reads as a different value). A NUL escape is written back as \0 +// instead of being dropped. +func TestFormatSQL_StringEscapesByDialect(t *testing.T) { + tests := []struct { + name string + dialect string + in string + want string + wantErr bool + }{ + {name: "no dialect path", in: `SELECT 'C:\dir', 'x\.y' FROM t`, wantErr: true}, + {name: "no dialect NUL", in: `SELECT 'a\0b'`, wantErr: true}, + {name: "no dialect known escapes", in: `SELECT 'a\nb', 'c\\d'`, want: `SELECT 'a\nb', 'c\\d'`}, + {name: "postgresql path", dialect: "postgresql", in: `SELECT 'C:\dir', 'x\.y' FROM t`, want: `SELECT 'C:\dir', 'x\.y' FROM t`}, + {name: "mysql NUL", dialect: "mysql", in: `SELECT 'a\0b'`, want: `SELECT 'a\0b'`}, + {name: "snowflake unknown escape", dialect: "snowflake", in: `SELECT 'C:\temp\q'`, want: "SELECT 'C:\temp" + `q'`}, + {name: "snowflake incomplete hex", dialect: "snowflake", in: `SELECT 'C:\temp\x'`, wantErr: true}, + {name: "postgresql leading quote", dialect: "postgresql", in: `SELECT '''a' FROM t`, want: `SELECT '''a' FROM t`}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + out, err := formatWith(tc.in, CLIFormatterOptions{Compact: true, Uppercase: true, Dialect: tc.dialect}) + if tc.wantErr { + if err == nil { + t.Fatalf("formatSQL(%q) = %q, want an error", tc.in, out) + } + return + } + if err != nil { + t.Fatalf("formatSQL(%q): %v", tc.in, err) + } + if out != tc.want { + t.Fatalf("formatSQL(%q) = %q, want %q", tc.in, out, tc.want) + } + }) + } +} diff --git a/cmd/gosqlx/cmd/formatter.go b/cmd/gosqlx/cmd/formatter.go index 9bbe77de..a89d197e 100644 --- a/cmd/gosqlx/cmd/formatter.go +++ b/cmd/gosqlx/cmd/formatter.go @@ -23,6 +23,7 @@ import ( "time" "github.com/ajitpratap0/GoSQLX/cmd/gosqlx/internal/config" + "github.com/ajitpratap0/GoSQLX/internal/batch" pkgformatter "github.com/ajitpratap0/GoSQLX/pkg/formatter" "github.com/ajitpratap0/GoSQLX/pkg/models" "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" @@ -208,8 +209,25 @@ func (f *Formatter) formatFile(filename string) FileFormatterResult { return result } -// formatSQL formats a SQL string +// utf8BOM is the byte order mark some editors put at the start of a file. +const utf8BOM = "\uFEFF" + +// formatSQL formats a SQL string. A leading UTF-8 byte order mark is not SQL: +// it is set aside and written back in front of the result, so formatting a +// file in place does not strip it. func (f *Formatter) formatSQL(sql string) (string, error) { + if strings.HasPrefix(sql, utf8BOM) { + out, err := f.formatSQLText(strings.TrimPrefix(sql, utf8BOM)) + if err != nil { + return "", err + } + return utf8BOM + out, nil + } + return f.formatSQLText(sql) +} + +// formatSQLText formats SQL text without a byte order mark. +func (f *Formatter) formatSQLText(sql string) (string, error) { // Acquire a tokenizer for the configured dialect (pooled for the default) tkz, releaseTkz, err := tokenizerForDialect(f.Opts.Dialect) if err != nil { @@ -226,8 +244,12 @@ func (f *Formatter) formatSQL(sql string) (string, error) { if len(tokens) == 0 { return "", nil } - - // Convert tokens for parser using centralized converter + if !hasStatementTokens(tokens) { + // Whitespace or comments only: there is nothing to format, and the + // text is returned untouched rather than reported as a syntax error. + return sql, nil + } + comments := tkz.Comments // Parse to AST with proper error handling for memory management p := parserForDialect(f.Opts.Dialect) @@ -242,7 +264,7 @@ func (f *Formatter) formatSQL(sql string) (string, error) { ast.ReleaseAST(parsedAST) }() - if err := checkFormattable(parsedAST, tokens); err != nil { + if err := checkFormattable(parsedAST, tokens, f.Opts.Dialect); err != nil { return "", err } @@ -256,11 +278,18 @@ func (f *Formatter) formatSQL(sql string) (string, error) { Dialect: f.Opts.Dialect, } - // Create formatter and format the AST + // Create formatter and format the AST. Comments are put back around the + // statements they belonged to; one that cannot be put back (an optimizer + // hint away from its statement's first keyword) refuses the whole file. formatter := NewSQLFormatter(formatterOpts) - formatted, err := formatter.Format(parsedAST) + var formatted string + if len(comments) == 0 { + formatted, err = formatter.Format(parsedAST) + } else { + formatted, err = formatter.FormatWithComments(parsedAST, tokens, comments) + } if err != nil { - return "", fmt.Errorf("render: %w", err) + return "", fmt.Errorf("render: %w (input left unchanged)", err) } // Refuse output that fails verifyFormattedSQL, so format, format -i and @@ -283,10 +312,43 @@ func (f *Formatter) formatSQL(sql string) (string, error) { if err := verifyFormattedSQL(parsedAST, tokens, verifyText, f.Opts.Dialect); err != nil { return "", err } + // Comments are not part of the AST, so verify them on the real output. + if err := verifyComments(comments, formatted, f.Opts.Dialect); err != nil { + return "", err + } return formatted, nil } +// verifyComments requires the formatted output to contain exactly the +// comments of the input, so format -i can never delete or invent one. +func verifyComments(input []models.Comment, formatted, dialect string) error { + tkz, release, err := tokenizerForDialect(dialect) + if err != nil { + return fmt.Errorf("tokenizer setup failed: %w", err) + } + defer release() + if _, err := tkz.TokenizeString(formatted); err != nil { + return fmt.Errorf("formatted output does not re-tokenize (formatter bug; input left unchanged): %w", err) + } + pool := make(map[string]int, len(input)) + for _, c := range tkz.Comments { + pool[c.Text]++ + } + for _, c := range input { + if pool[c.Text] == 0 { + return fmt.Errorf("formatted output loses the comment %q (formatter bug; input left unchanged)", c.Text) + } + pool[c.Text]-- + } + for text, n := range pool { + if n > 0 { + return fmt.Errorf("formatted output adds the comment %q (formatter bug; input left unchanged)", text) + } + } + return nil +} + // verifyFormattedSQL re-parses formatted with the same dialect and checks // that it yields the same number of statements, the same AST.SQL(), the same // canonical pkg/formatter rendering of each statement, and the same string @@ -355,14 +417,17 @@ func checkQuotedTokens(input, output []models.TokenWithSpan, dialect string) err } defer release() + input = mergeAdjacentStrings(input, dialect) + output = mergeAdjacentStrings(output, dialect) + pool := make(map[string]int) for _, t := range output { if isStringLiteralToken(t.Token) || bw.isQuotedIdent(t.Token) { pool[t.Token.Value]++ } } - for _, t := range input { - if !isStringLiteralToken(t.Token) { + for i, t := range input { + if !isStringLiteralToken(t.Token) || isStringAlias(input, i, dialect) { continue } if pool[t.Token.Value] == 0 { @@ -370,8 +435,8 @@ func checkQuotedTokens(input, output []models.TokenWithSpan, dialect string) err } pool[t.Token.Value]-- } - for _, t := range input { - if !bw.isQuotedIdent(t.Token) { + for i, t := range input { + if !bw.isQuotedIdent(t.Token) && !isStringAlias(input, i, dialect) { continue } v := t.Token.Value @@ -386,6 +451,64 @@ func checkQuotedTokens(input, output []models.TokenWithSpan, dialect string) err return nil } +// isStringAlias reports whether toks[i] is a string literal that names a +// column, as in "SELECT a AS 'x'" (MySQL, MariaDB, SQLite). The parser reads +// it as the alias x, the same as the delimited name, so the formatter writes +// it as a name rather than as a string. +func isStringAlias(toks []models.TokenWithSpan, i int, dialect string) bool { + switch strings.ToLower(dialect) { + case "mysql", "mariadb", "sqlite": + default: + return false + } + t := toks[i].Token + if i == 0 || (t.Type != models.TokenTypeString && t.Type != models.TokenTypeSingleQuotedString) { + return false + } + return toks[i-1].Token.Type == models.TokenTypeAs +} + +// mergeAdjacentStrings returns toks with each run of adjacent single-quoted +// string tokens merged into one token, for the dialects whose parser reads +// 'a' 'b' as the literal 'ab' (MySQL, MariaDB, PostgreSQL). The formatter +// writes the merged literal, which means the same. Other dialects, and toks +// without such a run, are returned as they are. +func mergeAdjacentStrings(toks []models.TokenWithSpan, dialect string) []models.TokenWithSpan { + switch strings.ToLower(dialect) { + case "mysql", "mariadb", "postgresql", "postgres": + default: + return toks + } + plain := func(t models.Token) bool { + return t.Type == models.TokenTypeString || t.Type == models.TokenTypeSingleQuotedString + } + var out []models.TokenWithSpan + for i := 0; i < len(toks); i++ { + if !plain(toks[i].Token) || i+1 >= len(toks) || !plain(toks[i+1].Token) { + if out != nil { + out = append(out, toks[i]) + } + continue + } + if out == nil { + out = append(make([]models.TokenWithSpan, 0, len(toks)), toks[:i]...) + } + merged := toks[i] + var sb strings.Builder + sb.WriteString(merged.Token.Value) + for i+1 < len(toks) && plain(toks[i+1].Token) { + i++ + sb.WriteString(toks[i].Token.Value) + } + merged.Token.Value = sb.String() + out = append(out, merged) + } + if out == nil { + return toks + } + return out +} + // checkMySQLDoubleQuotes requires MySQL/MariaDB output to keep exactly the // input's double-quoted tokens. MySQL reads "..." as a string literal unless // ANSI_QUOTES is set, while the parser treats it as an identifier, so the @@ -621,33 +744,49 @@ func FormatterOptionsFromConfig(cfg *config.Config, flagsChanged map[string]bool // checkFormattable refuses parse results the CLI cannot rewrite faithfully, so // format, format -i and format -o leave such input unchanged. // -// DescribeStatement keeps only a name: it drops EXPLAIN options, the explained -// query and DESCRIBE object kinds, and "EXPLAIN VERBOSE|ANALYZE " parses -// as a DescribeStatement followed by , which would rewrite a -// non-executing EXPLAIN into the executable statement. UnsupportedStatement is -// rendered as a comment. Both are refused by type. As a general guard, a parse -// with more statements than semicolon-separated segments in the input means -// the parser split a statement, which re-parsing the output cannot detect. -func checkFormattable(tree *ast.AST, tokens []models.TokenWithSpan) error { +// DescribeStatement keeps only a name and drops DESCRIBE object kinds +// (EXPLAIN is an ExplainStatement, which round-trips). +// UnsupportedStatement is rendered as a comment. Both are refused by type. As a general guard, a parse +// with more statements than the input can hold means the parser split a +// statement, which re-parsing the output cannot detect. The input holds at +// most one statement per semicolon-separated segment; T-SQL statements may +// also end without ';', so for SQL Server the bound also counts GO lines and +// the words a statement can start with (see tsqlStatementBound). +func checkFormattable(tree *ast.AST, tokens []models.TokenWithSpan, dialect ...string) error { for _, stmt := range tree.Statements { switch stmt.(type) { case *ast.DescribeStatement, *ast.UnsupportedStatement: return fmt.Errorf("unsupported statement type: %T (input left unchanged)", stmt) } } - if got, segs := len(tree.Statements), statementSegments(tokens); got > segs { - return fmt.Errorf("input parses as %d statements but has %d semicolon-separated statements (input left unchanged)", got, segs) + bound, unit := 0, "semicolon-separated" + if len(dialect) > 0 && isSQLServerName(dialect[0]) { + bound, unit = tsqlStatementBound(tokens), "possible T-SQL" + } else { + bound = statementSegments(tokens) + } + if got := len(tree.Statements); got > bound { + return fmt.Errorf("input parses as %d statements but has %d %s statements (input left unchanged)", got, bound, unit) } return nil } +func isSQLServerName(d string) bool { + switch strings.ToLower(d) { + case "sqlserver", "mssql", "tsql": + return true + } + return false +} + // statementSegments counts the non-empty runs of tokens between semicolons, -// ignoring EOF, i.e. the most statements the input can legitimately hold. +// ignoring EOF, i.e. the most statements the input can legitimately hold +// outside SQL Server. func statementSegments(tokens []models.TokenWithSpan) int { segs := 0 inSeg := false - for _, tok := range tokens { - switch tok.Token.Type { + for i := range tokens { + switch tokens[i].Token.Type { case models.TokenTypeSemicolon: inSeg = false case models.TokenTypeEOF: @@ -660,3 +799,73 @@ func statementSegments(tokens []models.TokenWithSpan) int { } return segs } + +// tsqlStartWords are the words a T-SQL statement can start with when it +// follows another one without ';'. It must cover every word the SQL Server +// parser starts a statement at (its implicit statement starts and the +// statement keywords); a missing word makes the guard refuse valid input, +// never accept a split statement. +var tsqlStartWords = map[string]bool{ + "SELECT": true, "INSERT": true, "UPDATE": true, "DELETE": true, "MERGE": true, + "WITH": true, "CREATE": true, "ALTER": true, "DROP": true, "TRUNCATE": true, + "DECLARE": true, "SET": true, "EXEC": true, "EXECUTE": true, "IF": true, + "ELSE": true, "WHILE": true, "BEGIN": true, "END": true, "PRINT": true, + "THROW": true, "RAISERROR": true, "WAITFOR": true, "RETURN": true, "USE": true, + "DBCC": true, "COMMIT": true, "ROLLBACK": true, "SAVE": true, "GRANT": true, + "REVOKE": true, "DENY": true, "GOTO": true, "BREAK": true, "CONTINUE": true, + "OPEN": true, "CLOSE": true, "FETCH": true, "DEALLOCATE": true, "BULK": true, + "BACKUP": true, "RESTORE": true, "CHECKPOINT": true, "KILL": true, + "RECONFIGURE": true, "SHUTDOWN": true, "REVERT": true, + "ENABLE": true, "DISABLE": true, "VALUES": true, "EXPLAIN": true, "SHOW": true, + "DESCRIBE": true, "CALL": true, "REPLACE": true, +} + +// tsqlStatementBound returns the most statements a SQL Server script can +// hold: one per run of tokens between semicolons and GO lines, one per GO +// line (a retained BatchSeparator is itself an AST statement), and one more +// per word inside a run that can start a statement (tsqlStartWords), since +// T-SQL statements need no ';' between them. +func tsqlStatementBound(tokens []models.TokenWithSpan) int { + bound := 0 + inSeg := false + for i := 0; i < len(tokens); i++ { + if next, ok := batch.End(tokens, i); ok { + bound++ + inSeg = false + i = next - 1 + continue + } + tok := &tokens[i] + switch tok.Token.Type { + case models.TokenTypeSemicolon: + inSeg = false + case models.TokenTypeEOF: + default: + if !inSeg { + bound++ + inSeg = true + } else if tsqlMayStartStatement(tokens, i) { + bound++ + } + } + } + return bound +} + +// tsqlMayStartStatement reports whether tokens[i] is an unquoted word in +// tsqlStartWords that is not part of a qualified name. +func tsqlMayStartStatement(tokens []models.TokenWithSpan, i int) bool { + tok := &tokens[i].Token + if tok.Quote != 0 || tok.Value == "" || len(tok.Value) > len("RECONFIGURE") { + return false + } + switch tok.Type { + case models.TokenTypeString, models.TokenTypeSingleQuotedString, models.TokenTypeDoubleQuotedString, + models.TokenTypeNumber, models.TokenTypePlaceholder: + return false + } + if i > 0 && tokens[i-1].Token.Type == models.TokenTypePeriod { + return false + } + return tsqlStartWords[strings.ToUpper(tok.Value)] +} diff --git a/cmd/gosqlx/cmd/global_flags.go b/cmd/gosqlx/cmd/global_flags.go new file mode 100644 index 00000000..ef65ab11 --- /dev/null +++ b/cmd/gosqlx/cmd/global_flags.go @@ -0,0 +1,60 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "bytes" + "fmt" + "io" + + "github.com/spf13/cobra" +) + +// The global -f/--format and -o/--output flags are inherited by every +// command. A command either honours a flag or rejects it when the user sets +// it; silently ignoring a flag the user typed is never an option. + +// flagSet reports whether the user explicitly passed the named flag. +func flagSet(cmd *cobra.Command, name string) bool { + f := cmd.Flags().Lookup(name) + return f != nil && f.Changed +} + +// formatSet reports whether -f/--format asks for something other than the +// default "auto". The value, not the flag's Changed state, is what matters: +// "-f auto" is the default spelled out. +func formatSet() bool { return format != "" && format != "auto" } + +// rejectGlobalFormat fails when -f/--format was set to anything but "auto" on +// a command that has only one output format. +func rejectGlobalFormat(cmd *cobra.Command) error { + if formatSet() { + return fmt.Errorf("%s does not support -f/--format (got %q)", cmd.CommandPath(), format) + } + return nil +} + +// outputSink returns the writer a command should print its result to and a +// finish function to call once it has printed. With -o the result is buffered +// and written to the file by finish; otherwise it goes straight to stdout. +func outputSink(cmd *cobra.Command) (io.Writer, func() error) { + if outputFile == "" { + return cmd.OutOrStdout(), func() error { return nil } + } + var buf bytes.Buffer + return &buf, func() error { + return WriteOutput(buf.Bytes(), outputFile, cmd.OutOrStdout()) + } +} diff --git a/cmd/gosqlx/cmd/highlight.go b/cmd/gosqlx/cmd/highlight.go index 932dd29c..e0406ca5 100644 --- a/cmd/gosqlx/cmd/highlight.go +++ b/cmd/gosqlx/cmd/highlight.go @@ -146,7 +146,7 @@ func newHighlightConfig(format, theme, dialect, color string, standalone bool) ( if !ok { return highlightConfig{}, fmt.Errorf("unknown theme %q; valid themes: %s", theme, strings.Join(highlight.ThemeNames(), ", ")) } - if err := validateDialectName(dialect); err != nil { + if err := normalizeDialectFlag(&dialect); err != nil { return highlightConfig{}, err } color = strings.ToLower(strings.TrimSpace(color)) diff --git a/cmd/gosqlx/cmd/lint.go b/cmd/gosqlx/cmd/lint.go index 49c7ff3b..e037645a 100644 --- a/cmd/gosqlx/cmd/lint.go +++ b/cmd/gosqlx/cmd/lint.go @@ -75,7 +75,11 @@ Pipeline/Stdin Examples: Exit Codes: 0 - No violations found - 1 - Errors or warnings found (warnings only if --fail-on-warn is set)`, + 1 - Violations found, or the SQL has a syntax error. With --auto-fix the + result is judged after the fixes are applied. + +Input with no statements (empty, whitespace or only comments) is not a syntax +error.`, Args: cobra.MinimumNArgs(0), RunE: lintRun, } @@ -84,7 +88,10 @@ func lintRun(cmd *cobra.Command, args []string) error { cmd.SilenceUsage = true // Reject unknown dialect names early before any parsing. - if err := validateDialectName(lintDialect); err != nil { + if err := normalizeDialectFlag(&lintDialect); err != nil { + return err + } + if err := rejectGlobalFormat(cmd); err != nil { return err } @@ -139,7 +146,8 @@ func lintRun(cmd *cobra.Command, args []string) error { fmt.Fprintln(outWriter, "\nApplying auto-fixes...") fixCount := 0 - for _, fileResult := range result.Files { + fixedIdx := make([]int, 0, len(result.Files)) + for idx, fileResult := range result.Files { if len(fileResult.Violations) == 0 { continue } @@ -185,10 +193,23 @@ func lintRun(cmd *cobra.Command, args []string) error { continue } fixCount++ + fixedIdx = append(fixedIdx, idx) fmt.Fprintf(outWriter, "Fixed: %s\n", fileResult.Filename) } } + // Judge the run by what is left after the fixes, not by the report + // printed before them. + if len(fixedIdx) > 0 { + for _, idx := range fixedIdx { + before := result.Files[idx] + after := l.LintFile(before.Filename) + result.TotalViolations += len(after.Violations) - len(before.Violations) + result.Files[idx] = after + } + fmt.Fprintf(outWriter, "\n%d violation(s) remain after auto-fix\n", result.TotalViolations) + } + fmt.Fprintf(outWriter, "\nAuto-fixed %d file(s)\n", fixCount) } @@ -219,9 +240,12 @@ func lintRun(cmd *cobra.Command, args []string) error { warningCount := 0 infoCount := 0 fileErrorCount := 0 + syntaxErrorCount := 0 for _, fileResult := range result.Files { if fileResult.Error != nil { fileErrorCount++ + } else if fileResult.SyntaxError() != nil { + syntaxErrorCount++ } for _, violation := range fileResult.Violations { switch violation.Severity { @@ -240,6 +264,9 @@ func lintRun(cmd *cobra.Command, args []string) error { } totalViolations := errorCount + warningCount + infoCount + if syntaxErrorCount > 0 { + return fmt.Errorf("%d file(s) contain SQL syntax errors (and %d violation(s))", syntaxErrorCount, totalViolations) + } if totalViolations > 0 { return fmt.Errorf("found %d violation(s): %d error(s), %d warning(s), %d info(s)", totalViolations, errorCount, warningCount, infoCount) @@ -281,13 +308,22 @@ func lintFromStdin(cmd *cobra.Command) error { // Display results fmt.Fprintf(outWriter, "Linting stdin input:\n\n") + synErr := result.SyntaxError() + if synErr != nil { + fmt.Fprintf(outWriter, "Syntax error: %v\n\n", synErr) + } + if len(result.Violations) == 0 { - fmt.Fprintln(outWriter, "No violations found.") + if synErr == nil { + fmt.Fprintln(outWriter, "No violations found.") + } // Write to file if specified if outputFile != "" { - return WriteOutput(outputBuf.Bytes(), outputFile, cmd.OutOrStdout()) + if err := WriteOutput(outputBuf.Bytes(), outputFile, cmd.OutOrStdout()); err != nil { + return err + } } - return nil + return lintOutcomeError(result.Violations, synErr) } fmt.Fprintf(outWriter, "Found %d violation(s):\n\n", len(result.Violations)) @@ -295,7 +331,8 @@ func lintFromStdin(cmd *cobra.Command) error { fmt.Fprintf(outWriter, "%d. %s\n", i+1, linter.FormatViolation(violation)) } - // Apply auto-fix if requested + // Apply auto-fix if requested. The exit status then reflects what is left + // in the fixed text. if lintAutoFix { fmt.Fprintln(outWriter, "\nAuto-fixed output:") fixed := string(content) @@ -308,6 +345,11 @@ func lintFromStdin(cmd *cobra.Command) error { } } fmt.Fprintln(outWriter, fixed) + if fixed != string(content) { + result = l.LintString(fixed, "stdin") + synErr = result.SyntaxError() + fmt.Fprintf(outWriter, "\n%d violation(s) remain after auto-fix\n", len(result.Violations)) + } } // Write to file if specified @@ -317,23 +359,7 @@ func lintFromStdin(cmd *cobra.Command) error { } } - // Exit with error code if there were violations - errorCount := 0 - warningCount := 0 - for _, violation := range result.Violations { - switch violation.Severity { - case linter.SeverityError: - errorCount++ - case linter.SeverityWarning: - warningCount++ - } - } - - if errorCount > 0 || (lintFailOnWarn && warningCount > 0) { - return fmt.Errorf("%d error(s) and %d warning(s) found", errorCount, warningCount) - } - - return nil + return lintOutcomeError(result.Violations, synErr) } // lintInlineSQL lints inline SQL passed as a command argument @@ -347,17 +373,19 @@ func lintInlineSQL(cmd *cobra.Command, sql string) error { outWriter = &outputBuf } - if len(result.Violations) == 0 { - fmt.Fprintln(outWriter, "No violations found.") - if outputFile != "" { - return WriteOutput(outputBuf.Bytes(), outputFile, cmd.OutOrStdout()) - } - return nil + synErr := result.SyntaxError() + if synErr != nil { + fmt.Fprintf(outWriter, "Syntax error: %v\n\n", synErr) } - fmt.Fprintf(outWriter, "Found %d violation(s):\n\n", len(result.Violations)) - for i, violation := range result.Violations { - fmt.Fprintf(outWriter, "%d. %s\n", i+1, linter.FormatViolation(violation)) + switch { + case len(result.Violations) > 0: + fmt.Fprintf(outWriter, "Found %d violation(s):\n\n", len(result.Violations)) + for i, violation := range result.Violations { + fmt.Fprintf(outWriter, "%d. %s\n", i+1, linter.FormatViolation(violation)) + } + case synErr == nil: + fmt.Fprintln(outWriter, "No violations found.") } if outputFile != "" { @@ -366,9 +394,16 @@ func lintInlineSQL(cmd *cobra.Command, sql string) error { } } + return lintOutcomeError(result.Violations, synErr) +} + +// lintOutcomeError maps one lint run of stdin or inline SQL to the command's +// result: invalid SQL and error-severity violations always fail it, warnings +// only with --fail-on-warn. +func lintOutcomeError(violations []linter.Violation, synErr error) error { errorCount := 0 warningCount := 0 - for _, violation := range result.Violations { + for _, violation := range violations { switch violation.Severity { case linter.SeverityError: errorCount++ @@ -377,10 +412,12 @@ func lintInlineSQL(cmd *cobra.Command, sql string) error { } } + if synErr != nil { + return fmt.Errorf("invalid SQL: %w", synErr) + } if errorCount > 0 || (lintFailOnWarn && warningCount > 0) { return fmt.Errorf("%d error(s) and %d warning(s) found", errorCount, warningCount) } - return nil } diff --git a/cmd/gosqlx/cmd/parse.go b/cmd/gosqlx/cmd/parse.go index c32d74e8..ae2b3e32 100644 --- a/cmd/gosqlx/cmd/parse.go +++ b/cmd/gosqlx/cmd/parse.go @@ -60,7 +60,7 @@ Performance: Direct AST inspection without intermediate representations`, func parseRun(cmd *cobra.Command, args []string) error { // Reject unknown dialect names early before any parsing. - if err := validateDialectName(parseDialect); err != nil { + if err := normalizeDialectFlag(&parseDialect); err != nil { return err } @@ -101,7 +101,8 @@ func parseRun(cmd *cobra.Command, args []string) error { }) // Create parser with injectable output writers - parser := NewParser(cmd.OutOrStdout(), cmd.ErrOrStderr(), opts) + out, finish := outputSink(cmd) + parser := NewParser(out, cmd.ErrOrStderr(), opts) // Run parsing result, err := parser.Parse(args[0]) @@ -115,7 +116,10 @@ func parseRun(cmd *cobra.Command, args []string) error { } // Display the result - return parser.Display(result) + if err := parser.Display(result); err != nil { + return err + } + return finish() } // parseFromStdin handles parsing from stdin input @@ -159,7 +163,8 @@ func parseFromStdin(cmd *cobra.Command) error { }) // Create parser - parser := NewParser(cmd.OutOrStdout(), cmd.ErrOrStderr(), opts) + out, finish := outputSink(cmd) + parser := NewParser(out, cmd.ErrOrStderr(), opts) // Parse the stdin content (Parse accepts string input directly) result, err := parser.Parse(string(content)) @@ -173,7 +178,10 @@ func parseFromStdin(cmd *cobra.Command) error { } // Display the result - return parser.Display(result) + if err := parser.Display(result); err != nil { + return err + } + return finish() } func init() { diff --git a/cmd/gosqlx/cmd/parser_cmd.go b/cmd/gosqlx/cmd/parser_cmd.go index 09716e68..84bbc3dd 100644 --- a/cmd/gosqlx/cmd/parser_cmd.go +++ b/cmd/gosqlx/cmd/parser_cmd.go @@ -94,6 +94,11 @@ func (p *Parser) Parse(input string) (*ParserResult, error) { return result, nil } + if !hasStatementTokens(tokens) { + result.Error = errNoStatements + return result, result.Error + } + // Parse to AST with proper error handling for memory management pr := parserForDialect(p.Opts.Dialect) defer pr.Release() @@ -198,8 +203,12 @@ func (p *Parser) displayAST(result *ParserResult) error { } var statements []StatementDisplay + stmtCount := 0 // GO separators are listed but not counted, as in analyze for _, stmt := range result.AST.Statements { statements = append(statements, convertStatement(stmt)) + if _, isBatch := stmt.(*ast.BatchSeparator); !isBatch { + stmtCount++ + } } display := ASTDisplay{ @@ -225,7 +234,7 @@ func (p *Parser) displayAST(result *ParserResult) error { default: fmt.Fprintf(p.Out, "AST Structure:\n") fmt.Fprintf(p.Out, " Type: %s\n", display.Type) - fmt.Fprintf(p.Out, " Statements: %d\n", len(display.Statements)) + fmt.Fprintf(p.Out, " Statements: %d\n", stmtCount) fmt.Fprintf(p.Out, " Token Count: %d\n", display.TokenCount) fmt.Fprintf(p.Out, " SQL Compliance: %s\n", display.Metadata["sql_compliance"]) fmt.Fprintf(p.Out, "\nStatements:\n") @@ -292,7 +301,7 @@ func (p *Parser) displayTree(astObj *ast.AST) error { if len(s.OrderBy) > 0 { fmt.Fprintf(p.Out, "%s├── OrderBy\n", childPrefix) } - if s.Limit != nil { + if s.HasLimit() { fmt.Fprintf(p.Out, "%s└── Limit\n", childPrefix) } } @@ -331,7 +340,7 @@ func convertStatement(stmt ast.Statement) StatementDisplay { if len(s.OrderBy) > 0 { details["has_order_by"] = true } - if s.Limit != nil { + if s.HasLimit() { details["has_limit"] = true } if s.With != nil { diff --git a/cmd/gosqlx/cmd/root.go b/cmd/gosqlx/cmd/root.go index 11a08b3f..fd142440 100644 --- a/cmd/gosqlx/cmd/root.go +++ b/cmd/gosqlx/cmd/root.go @@ -15,6 +15,8 @@ package cmd import ( + "errors" + "github.com/spf13/cobra" "github.com/ajitpratap0/GoSQLX/cmd/gosqlx/internal/actioncmd" @@ -175,6 +177,19 @@ func Execute() error { return rootCmd.Execute() } +// silentError is a failure whose exit status matters but whose message the +// user asked not to see (validate --quiet). +type silentError struct{ error } + +func (e silentError) Unwrap() error { return e.error } + +// IsSilentError reports whether err must be turned into an exit status +// without printing anything (the caller prints every other error once). +func IsSilentError(err error) bool { + var s silentError + return errors.As(err, &s) +} + // init initializes the root command and registers global flags. // // This function is called automatically during package initialization. @@ -194,6 +209,15 @@ func init() { // the library (and the LSP, which reports gosqlx.Version) cannot drift. rootCmd.Version = Version + // Errors are returned to main, which prints each one exactly once, so + // cobra must not print them as well. Usage is printed for flag and + // argument errors (cobra validates those before this hook runs) but not + // for failures of the work itself, e.g. a syntax error in the input. + rootCmd.SilenceErrors = true + rootCmd.PersistentPreRun = func(cmd *cobra.Command, _ []string) { + cmd.SilenceUsage = true + } + // Global flags available to all subcommands rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "enable verbose output") rootCmd.PersistentFlags().StringVarP(&outputFile, "output", "o", "", "output file (default: stdout)") diff --git a/cmd/gosqlx/cmd/sql_analyzer.go b/cmd/gosqlx/cmd/sql_analyzer.go index 186165da..d8926f02 100644 --- a/cmd/gosqlx/cmd/sql_analyzer.go +++ b/cmd/gosqlx/cmd/sql_analyzer.go @@ -507,6 +507,9 @@ func (a *SQLAnalyzer) buildQueryInfo(astObj *ast.AST, sql string) QueryInfo { for _, stmt := range astObj.Statements { switch stmt.(type) { + case *ast.BatchSeparator: + // A GO line is a client directive, not a statement. + continue case *ast.SelectStatement: stmtTypes = append(stmtTypes, "SELECT") case *ast.InsertStatement: @@ -535,7 +538,7 @@ func (a *SQLAnalyzer) buildQueryInfo(astObj *ast.AST, sql string) QueryInfo { return QueryInfo{ Size: size, Lines: lines, - StatementCount: len(astObj.Statements), + StatementCount: len(stmtTypes), StatementTypes: stmtTypes, } } diff --git a/cmd/gosqlx/cmd/sql_analyzer_test.go b/cmd/gosqlx/cmd/sql_analyzer_test.go index 94053044..a8aa6fda 100644 --- a/cmd/gosqlx/cmd/sql_analyzer_test.go +++ b/cmd/gosqlx/cmd/sql_analyzer_test.go @@ -18,7 +18,9 @@ import ( "strings" "testing" + "github.com/ajitpratap0/GoSQLX/pkg/gosqlx" "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" "github.com/ajitpratap0/GoSQLX/pkg/sql/parser" "github.com/ajitpratap0/GoSQLX/pkg/sql/tokenizer" ) @@ -835,3 +837,21 @@ func getIssueIDs(issues []AnalysisIssue) []string { } return ids } + +// TestBuildQueryInfoExcludesGO checks that GO batch separators are not +// counted or typed as statements in the analysis report. +func TestBuildQueryInfoExcludesGO(t *testing.T) { + const sql = "SELECT 1\nGO\nUPDATE t SET a = 1\nGO 2" + tree, err := gosqlx.ParseWithDialect(sql, keywords.DialectSQLServer) + if err != nil { + t.Fatalf("parse: %v", err) + } + defer ast.ReleaseAST(tree) + info := NewSQLAnalyzer().buildQueryInfo(tree, sql) + if info.StatementCount != 2 { + t.Errorf("StatementCount = %d, want 2", info.StatementCount) + } + if got := strings.Join(info.StatementTypes, ","); got != "SELECT,UPDATE" { + t.Errorf("StatementTypes = %s, want SELECT,UPDATE", got) + } +} diff --git a/cmd/gosqlx/cmd/sql_formatter.go b/cmd/gosqlx/cmd/sql_formatter.go index 68e931d8..7a40317f 100644 --- a/cmd/gosqlx/cmd/sql_formatter.go +++ b/cmd/gosqlx/cmd/sql_formatter.go @@ -19,6 +19,7 @@ import ( "strings" pkgformatter "github.com/ajitpratap0/GoSQLX/pkg/formatter" + "github.com/ajitpratap0/GoSQLX/pkg/models" "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" ) @@ -78,19 +79,41 @@ func NewSQLFormatter(opts FormatterOptions) *SQLFormatter { // Format renders every statement of astObj, separated by "; " in compact mode // and ";\n\n" otherwise. No trailing semicolon is added. func (f *SQLFormatter) Format(astObj *ast.AST) (string, error) { - if astObj == nil { - return "", errors.New("cannot format a nil AST") + stmts, err := f.formatStatements(astObj) + if err != nil { + return "", err } - sep := ";\n\n" - if f.compact { - sep = "; " + return pkgformatter.JoinWithComments(stmts, nil, nil, f.layout()) +} + +// FormatWithComments is Format for input that had comments: tokens and +// comments are those of the source text, and the comments are put back around +// the statements they belonged to (see pkgformatter.JoinWithComments). +func (f *SQLFormatter) FormatWithComments(astObj *ast.AST, tokens []models.TokenWithSpan, comments []models.Comment) (string, error) { + stmts, err := f.formatStatements(astObj) + if err != nil { + return "", err } - var sb strings.Builder + layout := f.layout() + layout.Batches = pkgformatter.BatchPositions(astObj) + layout.Starts = pkgformatter.StatementPositions(astObj) + return pkgformatter.JoinWithComments(stmts, tokens, comments, layout) +} + +func (f *SQLFormatter) formatStatements(astObj *ast.AST) ([]string, error) { + if astObj == nil { + return nil, errors.New("cannot format a nil AST") + } + stmts := make([]string, len(astObj.Statements)) for i, stmt := range astObj.Statements { - if i > 0 { - sb.WriteString(sep) - } - sb.WriteString(pkgformatter.FormatStatement(stmt, f.style)) + stmts[i] = pkgformatter.FormatStatement(stmt, f.style) + } + return stmts, nil +} + +func (f *SQLFormatter) layout() pkgformatter.CommentLayout { + if f.compact { + return pkgformatter.CommentLayout{Terminator: ";", Gap: " ", Dialect: f.style.Dialect} } - return sb.String(), nil + return pkgformatter.CommentLayout{Terminator: ";", Gap: "\n\n", Dialect: f.style.Dialect} } diff --git a/cmd/gosqlx/cmd/transpile.go b/cmd/gosqlx/cmd/transpile.go index ba07b949..349004a5 100644 --- a/cmd/gosqlx/cmd/transpile.go +++ b/cmd/gosqlx/cmd/transpile.go @@ -17,7 +17,6 @@ package cmd import ( "fmt" "io" - "os" "strings" "github.com/spf13/cobra" @@ -44,6 +43,9 @@ Examples: gosqlx transpile --from postgres --to sqlite "CREATE TABLE t (id SERIAL PRIMARY KEY, tags TEXT)"`, Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + if err := rejectGlobalFormat(cmd); err != nil { + return err + } fromStr, _ := cmd.Flags().GetString("from") toStr, _ := cmd.Flags().GetString("to") @@ -61,7 +63,7 @@ Examples: sql = args[0] } else { // Read from stdin. - data, readErr := io.ReadAll(os.Stdin) + data, readErr := io.ReadAll(cmd.InOrStdin()) if readErr != nil { return fmt.Errorf("reading stdin: %w", readErr) } @@ -76,8 +78,9 @@ Examples: if err != nil { return fmt.Errorf("transpile: %w", err) } - fmt.Println(result) - return nil + out, finish := outputSink(cmd) + fmt.Fprintln(out, result) + return finish() }, } @@ -92,14 +95,14 @@ func init() { // parseDialectFlag converts a dialect name string to a keywords.SQLDialect value. func parseDialectFlag(s string) (keywords.SQLDialect, error) { - switch strings.ToLower(s) { + switch canonicalDialect(strings.ToLower(s)) { case "mysql": return keywords.DialectMySQL, nil - case "postgres", "postgresql": + case "postgresql": return keywords.DialectPostgreSQL, nil case "sqlite": return keywords.DialectSQLite, nil - case "sqlserver", "mssql": + case "sqlserver": return keywords.DialectSQLServer, nil case "oracle": return keywords.DialectOracle, nil diff --git a/cmd/gosqlx/cmd/validate.go b/cmd/gosqlx/cmd/validate.go index cfcf39ee..36b3ec35 100644 --- a/cmd/gosqlx/cmd/validate.go +++ b/cmd/gosqlx/cmd/validate.go @@ -81,14 +81,15 @@ func validateRun(cmd *cobra.Command, args []string) error { return nil } - // In quiet/check mode, silence all cobra output - only exit code matters - if validateQuiet { - cmd.SilenceErrors = true - cmd.SilenceUsage = true - } + // In quiet/check mode only the exit code matters: failures are returned + // as silent errors, which main turns into exit status 1 without printing. // Reject unknown dialect names early before any parsing. - if err := validateDialectName(validateDialect); err != nil { + if err := normalizeDialectFlag(&validateDialect); err != nil { + return err + } + + if err := applyGlobalValidateFlags(cmd); err != nil { return err } @@ -190,7 +191,7 @@ func validateRun(cmd *cobra.Command, args []string) error { // Return error if there were invalid files if result.InvalidFiles > 0 { - return fmt.Errorf("validation failed: %d invalid file(s)", result.InvalidFiles) + return validationFailed(fmt.Errorf("validation failed: %d invalid file(s)", result.InvalidFiles)) } return nil @@ -209,22 +210,12 @@ func validateFromStdin(cmd *cobra.Command) error { return fmt.Errorf("stdin validation failed: %w", err) } - // Create a temporary file to leverage existing validation logic - tmpFile, err := os.CreateTemp("", "gosqlx-stdin-*.sql") - if err != nil { - return fmt.Errorf("failed to create temporary file: %w", err) - } - defer func() { _ = os.Remove(tmpFile.Name()) }() - defer func() { _ = tmpFile.Close() }() - - // Write stdin content to temp file - if _, err := tmpFile.Write(content); err != nil { - return fmt.Errorf("failed to write to temporary file: %w", err) - } - if err := tmpFile.Close(); err != nil { - return fmt.Errorf("failed to close temporary file: %w", err) - } + return validateContent(cmd, stdinDisplayName, content) +} +// validateContent validates one in-memory input (stdin, inline SQL) and +// reports it under name in the selected output format. +func validateContent(cmd *cobra.Command, name string, content []byte) error { // Load configuration cfg, err := config.LoadDefault() if err != nil { @@ -237,7 +228,7 @@ func validateFromStdin(cmd *cobra.Command) error { // Create validator options quietMode := validateQuiet || validateOutputFormat == OutputFormatSARIF || validateOutputFormat == OutputFormatJSON opts := ValidatorOptionsFromConfig(cfg, flagsChanged, ValidatorFlags{ - Recursive: false, // stdin is always single input + Recursive: false, // a single input Pattern: "", Quiet: quietMode, ShowStats: validateStats && validateOutputFormat == OutputFormatText, // Only show text stats for text output @@ -249,15 +240,8 @@ func validateFromStdin(cmd *cobra.Command) error { // Create validator validator := NewValidator(cmd.OutOrStdout(), cmd.ErrOrStderr(), opts) - // Validate the temporary file - result, err := validator.Validate([]string{tmpFile.Name()}) - if err != nil { - return err - } - - // Update result to show "stdin" instead of temp file path - // The validation has already output results with temp file path - // Different output formats are handled below + // Validate the content in memory; results are reported under name + result := validator.ValidateInput(name, content) // Handle different output formats switch validateOutputFormat { @@ -275,7 +259,7 @@ func validateFromStdin(cmd *cobra.Command) error { fmt.Fprintf(cmd.OutOrStdout(), "SARIF output written to %s\n", validateOutputFile) } case OutputFormatJSON: - jsonData, err := output.FormatValidationJSON(result, []string{"stdin"}, validateStats) + jsonData, err := output.FormatValidationJSON(result, []string{name}, validateStats) if err != nil { return fmt.Errorf("failed to generate JSON output: %w", err) } @@ -291,15 +275,62 @@ func validateFromStdin(cmd *cobra.Command) error { // Return error if validation failed if result.InvalidFiles > 0 { - return fmt.Errorf("validation failed: %d invalid file(s)", result.InvalidFiles) + return validationFailed(fmt.Errorf("validation failed: %d invalid file(s)", result.InvalidFiles)) } return nil } +// applyGlobalValidateFlags maps the global -o/--output and -f/--format flags +// onto validate's own --output-file and --output-format. Both only make sense +// for the machine-readable formats; text output always goes to stdout. +func applyGlobalValidateFlags(cmd *cobra.Command) error { + if formatSet() { + if flagSet(cmd, "output-format") && validateOutputFormat != format { + return fmt.Errorf("-f/--format %q conflicts with --output-format %q", format, validateOutputFormat) + } + switch format { + case OutputFormatJSON, OutputFormatText: + validateOutputFormat = format + default: + return fmt.Errorf("validate does not support -f/--format %q (use json, or --output-format sarif)", format) + } + } + if outputFile != "" { + if validateOutputFile != "" && validateOutputFile != outputFile { + return fmt.Errorf("-o/--output %q conflicts with --output-file %q", outputFile, validateOutputFile) + } + validateOutputFile = outputFile + if validateOutputFormat == "" || validateOutputFormat == OutputFormatText { + return fmt.Errorf("-o/--output needs a machine-readable format: add -f json or --output-format sarif") + } + } + return nil +} + +// inlineDisplayName is how results name SQL given as a command argument. +const inlineDisplayName = "" + +// stdinDisplayName is how results name input read from standard input. +const stdinDisplayName = "" + +// validationFailed wraps a validation failure. With --quiet only the exit +// status matters, so the error is returned silent: main exits 1 and prints +// nothing. +func validationFailed(err error) error { + if validateQuiet { + return silentError{err} + } + return err +} + // validateInlineSQL validates inline SQL passed as a command argument. // Uses the fast-path Validate() which skips full AST construction (#274). func validateInlineSQL(cmd *cobra.Command, sql string) error { + if validateOutputFormat == OutputFormatJSON || validateOutputFormat == OutputFormatSARIF { + // A machine-readable report needs the full result, not the fast path. + return validateContent(cmd, inlineDisplayName, []byte(sql)) + } var err error if validateDialect != "" { err = parser.ValidateWithDialect(sql, keywords.SQLDialect(validateDialect)) @@ -310,7 +341,7 @@ func validateInlineSQL(cmd *cobra.Command, sql string) error { if !validateQuiet { fmt.Fprintf(cmd.ErrOrStderr(), "❌ Invalid SQL: %v\n", err) } - return fmt.Errorf("validation failed: %w", err) + return validationFailed(fmt.Errorf("validation failed: %w", err)) } if sql == "" { diff --git a/cmd/gosqlx/cmd/validator.go b/cmd/gosqlx/cmd/validator.go index 8733e26e..73e64f77 100644 --- a/cmd/gosqlx/cmd/validator.go +++ b/cmd/gosqlx/cmd/validator.go @@ -15,6 +15,7 @@ package cmd import ( + "errors" "fmt" "io" "os" @@ -24,6 +25,8 @@ import ( "github.com/ajitpratap0/GoSQLX/cmd/gosqlx/internal/config" "github.com/ajitpratap0/GoSQLX/cmd/gosqlx/internal/output" + pkgerrors "github.com/ajitpratap0/GoSQLX/pkg/errors" + "github.com/ajitpratap0/GoSQLX/pkg/gosqlx" "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" "github.com/ajitpratap0/GoSQLX/pkg/sql/parser" @@ -242,14 +245,79 @@ func (v *Validator) validateFile(filename string) output.FileValidationResult { return result } - data := inputResult.Content - result.Size = int64(len(data)) + return v.validateData(filename, inputResult.Content) +} +// ValidateInput validates in-memory content (for example stdin) and reports +// it under the display name name instead of a file path. +func (v *Validator) ValidateInput(name string, data []byte) *output.ValidationResult { + start := time.Now() + fileResult := v.validateData(name, data) + result := &output.ValidationResult{ + Files: []output.FileValidationResult{fileResult}, + TotalFiles: 1, + TotalBytes: fileResult.Size, + } + switch { + case fileResult.Error != nil: + if !v.Opts.Quiet { + fmt.Fprintf(v.Err, "❌ %s: %v\n", name, fileResult.Error) // #nosec G705 + } + result.InvalidFiles++ + case fileResult.Valid: + if !v.Opts.Quiet { + fmt.Fprintf(v.Out, "✅ %s: Valid SQL\n", name) // #nosec G705 + } + result.ValidFiles++ + default: + if !v.Opts.Quiet { + fmt.Fprintf(v.Out, "❌ %s: Invalid SQL\n", name) // #nosec G705 + } + result.InvalidFiles++ + } + result.Duration = time.Since(start) + if v.Opts.ShowStats { + v.displayStats(result) + } + return result +} + +// validateData validates the content of one input. Content with no +// statements (empty, whitespace or only comments) is valid. +// +// The whole content is parsed first, so errors carry exact positions. Only +// when that fails is the content split into statements (SplitStatements: GO +// batches, DELIMITER scripts, procedural bodies) and each statement checked +// on its own: a script the parser accepts statement by statement is valid, and +// a file past the tokenizer's token limit is validated without it. +func (v *Validator) validateData(name string, data []byte) output.FileValidationResult { + result := output.FileValidationResult{Path: name, Size: int64(len(data))} if len(data) == 0 { result.Valid = true - return result // Empty inputs are considered valid + return result + } + + empty, wholeErr := v.checkSQL(string(data)) + if empty || wholeErr == nil { + result.Valid = true + return result + } + + if ok, stmtErr := v.checkStatements(string(data)); ok { + result.Valid = true + return result + } else if stmtErr != nil && isTokenLimitError(wholeErr) { + // The whole-file error only says the file is too big to read at once. + result.Error = stmtErr + return result } + result.Error = wholeErr + return result +} +// checkSQL tokenizes and parses src as one unit. empty reports that src holds +// no statements. +func (v *Validator) checkSQL(src string) (empty bool, err error) { // Use pooled tokenizer for performance with dialect support tkz := tokenizer.GetTokenizer() defer tokenizer.PutTokenizer(tkz) @@ -259,35 +327,60 @@ func (v *Validator) validateFile(filename string) output.FileValidationResult { tkz.SetDialect(keywords.SQLDialect(v.Opts.Dialect)) } - // Tokenize - tokens, err := tkz.Tokenize(data) + tokens, err := tkz.TokenizeString(src) if err != nil { - result.Error = fmt.Errorf("tokenization failed: %w", err) - return result + return false, fmt.Errorf("tokenization failed: %w", err) } - - if len(tokens) == 0 { - result.Valid = true - return result + if !hasStatementTokens(tokens) { + return true, nil } - // Convert TokenWithSpan to Token using centralized converter - // Parse to validate syntax with proper error handling for memory management p := parser.NewParser(parser.WithDialect(v.Opts.Dialect)) astObj, err := p.ParseFromModelTokens(tokens) if err != nil { - result.Error = fmt.Errorf("parsing failed: %w", err) - return result + return false, fmt.Errorf("parsing failed: %w", err) } // CRITICAL: Always release AST - defer func() { - ast.ReleaseAST(astObj) - }() + ast.ReleaseAST(astObj) + return false, nil +} - result.Valid = true - return result +// checkStatements validates src statement by statement. ok is true when src +// holds at least one statement and every statement is valid. Otherwise err is +// the first failing statement's error, prefixed with its number and start +// line. +func (v *Validator) checkStatements(src string) (ok bool, err error) { + segments := gosqlx.SplitStatements(src, v.Opts.Dialect) + checked := 0 + cursor := 0 + for _, seg := range segments { + trimmed := strings.TrimSpace(seg) + if trimmed == "" { + continue + } + at := strings.Index(src[cursor:], seg) + line := 1 + if at >= 0 { + line = 1 + strings.Count(src[:cursor+at], "\n") + cursor += at + len(seg) + } + empty, segErr := v.checkSQL(seg) + if segErr != nil { + return false, fmt.Errorf("statement %d (starting at line %d): %w", checked+1, line, segErr) + } + if !empty { + checked++ + } + } + return checked > 0, nil +} + +// isTokenLimitError reports whether err is the tokenizer's token-limit error. +func isTokenLimitError(err error) bool { + var e *pkgerrors.Error + return errors.As(err, &e) && e.Code == pkgerrors.ErrCodeTokenLimitReached } // expandFileArgs expands file arguments (glob patterns, directories) into a list of files diff --git a/cmd/gosqlx/internal/output/json.go b/cmd/gosqlx/internal/output/json.go index 46649dad..54d52af5 100644 --- a/cmd/gosqlx/internal/output/json.go +++ b/cmd/gosqlx/internal/output/json.go @@ -510,7 +510,7 @@ func convertStatementToJSON(stmt ast.Statement) JSONStatement { result.Details["has_where"] = s.Where != nil result.Details["has_group_by"] = len(s.GroupBy) > 0 result.Details["has_order_by"] = len(s.OrderBy) > 0 - result.Details["has_limit"] = s.Limit != nil + result.Details["has_limit"] = s.HasLimit() result.Details["has_distinct"] = s.Distinct if s.With != nil { result.Details["has_with"] = true diff --git a/cmd/gosqlx/main.go b/cmd/gosqlx/main.go index bb467843..43053131 100644 --- a/cmd/gosqlx/main.go +++ b/cmd/gosqlx/main.go @@ -81,7 +81,9 @@ import ( // See package documentation for comprehensive usage information. func main() { if err := cmd.Execute(); err != nil { - fmt.Fprintf(os.Stderr, "Error: %v\n", err) + if !cmd.IsSilentError(err) { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + } os.Exit(1) } } diff --git a/docs/CLI_GUIDE.md b/docs/CLI_GUIDE.md index 5ae118d4..88cbbc34 100644 --- a/docs/CLI_GUIDE.md +++ b/docs/CLI_GUIDE.md @@ -174,6 +174,13 @@ cat query.sql | gosqlx validate && echo "Valid SQL" | 0 | Valid SQL - no syntax errors | | 1 | Invalid SQL - syntax errors detected | +Input with no statements (empty, whitespace only, or only comments) is valid. +A file that fails as a whole is checked again statement by statement, so +scripts that use the T-SQL `GO` batch separator or the MySQL `DELIMITER` +directive validate, and a file over the tokenizer's 1,000,000 token limit +validates when every statement is under it. Results read from stdin are named +``. + #### Performance Target - **Latency**: <10ms for typical queries (50-500 characters) @@ -199,7 +206,7 @@ gosqlx format [file...] [flags] | `--dialect` | string | postgresql | SQL dialect (generic, postgresql, mysql, mariadb, sqlserver, oracle, sqlite, snowflake, bigquery, redshift, clickhouse); unknown names are rejected | | `--indent` | int | 2 | Indentation size in spaces (0-8) | | `--uppercase` | bool | true | Uppercase SQL keywords | -| `--no-uppercase` | bool | false | Keep original keyword case | +| `--no-uppercase` | bool | false | Write keywords in lower case (cannot be combined with `--uppercase`) | | `--compact` | bool | false | Compact format (minimal whitespace) | | `--check` | bool | false | Check if formatting is needed (CI mode) | | `--max-line` | int | 80 | Maximum line length (0-500) | @@ -534,10 +541,14 @@ gosqlx lint - | Code | Meaning | |------|---------| | 0 | No violations found | -| 1 | Any violation found (error, warning, or info) - usable as CI gate | +| 1 | Any violation found (error, warning, or info), or the SQL has a syntax error - usable as CI gate | > Since v1.9.0, `gosqlx lint` exits 1 whenever any violation is found (error, warning, or info). > `--fail-on-warn` is therefore redundant for exit codes. +> +> SQL that does not tokenize or parse is reported as a syntax error and also exits 1; the style +> rules that need no parse tree still report their violations. With `--auto-fix` the exit code +> reflects what is left after the fixes. --- diff --git a/docs/LINTING_RULES.md b/docs/LINTING_RULES.md index ab98549a..ed051e1d 100644 --- a/docs/LINTING_RULES.md +++ b/docs/LINTING_RULES.md @@ -717,7 +717,7 @@ gosqlx lint --fail-on-warn query.sql | Code | Meaning | |------|---------| | 0 | No violations (or info only) | -| 1 | Violations found (errors/warnings) | +| 1 | Violations found (errors/warnings), or the SQL has a syntax error | --- diff --git a/docs/SQL_COMPATIBILITY.md b/docs/SQL_COMPATIBILITY.md index f211305e..8e37a04b 100644 --- a/docs/SQL_COMPATIBILITY.md +++ b/docs/SQL_COMPATIBILITY.md @@ -4,6 +4,42 @@ This matrix documents the comprehensive SQL feature support in GoSQLX across different SQL dialects and standards. The testing was conducted using the comprehensive integration test suite with 700+ test cases covering real-world SQL patterns. +### Real-world dialect audit (October 2026, unreleased) + +The real-world corpus contains 2,788 statements. Every statement passes +split → dialect-aware parse → format → reparse → format idempotency, plus a +check for dropped SQL words. `testdata/realworld/known_failures.txt` lists no +failures. +These results describe this corpus; they do not establish complete dialect +coverage or database execution equivalence. Some routine and utility statements +retain their text rather than receiving a complete semantic AST. + +| Dialect | Passing statements | Known failures | Pass rate | +| --- | ---: | ---: | ---: | +| PostgreSQL | 300 / 300 | 0 | 100% | +| MySQL | 405 / 405 | 0 | 100% | +| MariaDB | 374 / 374 | 0 | 100% | +| SQL Server | 276 / 276 | 0 | 100% | +| Oracle | 368 / 368 | 0 | 100% | +| SQLite | 348 / 348 | 0 | 100% | +| Snowflake | 325 / 325 | 0 | 100% | +| ClickHouse | 392 / 392 | 0 | 100% | + +Reproduce the ratchet and per-dialect summaries from the repository root: + +```sh +go test ./pkg/gosqlx -run '^TestRealWorldCorpus$' -count=1 +for dir in testdata/realworld/*/; do + go run ./scripts/dialect-audit -dialect "$(basename "$dir")" "$dir"*.sql +done +``` + +SQL Server batch boundaries also have whole-script round-trip tests: the AST +retains each `GO` as a `BatchSeparator`, including its optional repeat count, +and both library and CLI rendering keep it on a separate line. The splitter +removes client directives when returning individual SQL statements, so the +statement-by-statement corpus totals above do not count GO directives. + ### Feature Highlights #### Snowflake and ClickHouse diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index 7366894b..a06b537e 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -274,12 +274,11 @@ lsp: diagnostics: max_diagnostics: 50 # Limit diagnostic count debounce_ms: 1000 # Reduce parsing frequency - -parser: - max_recursion_depth: 100 - max_tokens: 50000 ``` +The parser's own limits (`parser.MaxRecursionDepth`, `tokenizer.MaxTokens`) are +fixed constants, not settings; see "Complex Nested Query Parsing" below. + **Monitor Memory Usage:** ```go import ( @@ -867,38 +866,45 @@ func TestJSONOperatorParsing(sql string) { **Symptom:** Deeply nested queries fail with "recursion depth limit" error -**Common Cause:** Parser hits max recursion depth (default 200) +**Common Cause:** The parser hits its recursion limit. `parser.MaxRecursionDepth` +is 100 and, like the tokenizer's `MaxTokens` (1,000,000 tokens per input, +comments included) and `MaxInputSize` (10 MB), it is a compile-time constant: +there is no config key and no setter (`SetMaxRecursionDepth` and `SetMaxTokens` +do not exist). The limits protect the process from stack exhaustion and +runaway memory on hostile input. **Solutions:** -```yaml -# .gosqlx.yml - Increase recursion limit -parser: - max_recursion_depth: 500 # Increase for complex queries - max_tokens: 100000 # Increase token limit if needed -``` -**Code Solution:** -```go -import "github.com/ajitpratap0/GoSQLX/pkg/sql/parser" +- Flatten the query. Replace deeply nested derived tables with CTEs (see + below); each CTE parses at a shallow depth. +- For input over the token or size limit, split it into statements and parse + them one at a time. Each statement gets its own budget: -func ParseComplexQuery(sql string) error { - p := parser.NewParser() +```go +import ( + "fmt" + "strings" - // Increase limits for complex queries - p.SetMaxRecursionDepth(500) - p.SetMaxTokens(100000) + "github.com/ajitpratap0/GoSQLX/pkg/gosqlx" +) - tokens, _ := /* tokenize */ - astTree, err := p.Parse(tokens) - if err != nil { - return err +func ParseLargeScript(script string) error { + for i, stmt := range gosqlx.SplitStatements(script, "postgresql") { + if strings.TrimSpace(stmt) == "" { + continue + } + if _, err := gosqlx.Parse(stmt); err != nil { + return fmt.Errorf("statement %d: %w", i+1, err) + } } - defer ast.ReleaseAST(astTree) - return nil } ``` +`gosqlx validate` does this for you: when a file fails as a whole it re-checks +it statement by statement, so a file past the token limit validates as long as +each statement is under it. + **Refactor Complex Query:** ```sql -- Instead of deep nesting: @@ -1027,11 +1033,11 @@ SELECT * FROM users WHERE name = 'John''s Pizza'; **E1006 - Input Too Large** - **Cause:** SQL input exceeds size limits (DoS protection) -- **Fix:** Split large queries or increase limits if appropriate +- **Fix:** Split the input into smaller statements (the 10 MB limit is a constant) **E1007 - Token Limit Reached** - **Cause:** Too many tokens generated (DoS protection) -- **Fix:** Simplify query or increase limits +- **Fix:** Split the input into smaller statements; the 1,000,000 token limit is a constant (see `gosqlx.SplitStatements`) **E1008 - Tokenizer Panic** - **Cause:** Internal tokenizer error (recovered panic) @@ -1215,10 +1221,6 @@ lsp: incremental_sync: true parser: - # Reduce recursion depth for faster parsing - max_recursion_depth: 200 - max_tokens: 50000 - # Enable parser caching cache_enabled: true cache_ttl_seconds: 300 # 5 minutes diff --git a/docs/USAGE_GUIDE.md b/docs/USAGE_GUIDE.md index 136974ff..ca14ef76 100644 --- a/docs/USAGE_GUIDE.md +++ b/docs/USAGE_GUIDE.md @@ -102,6 +102,7 @@ package main import ( "fmt" "log" + "github.com/ajitpratap0/GoSQLX/pkg/models" "github.com/ajitpratap0/GoSQLX/pkg/sql/tokenizer" ) @@ -139,8 +140,9 @@ package main import ( "fmt" - "github.com/ajitpratap0/GoSQLX/pkg/sql/tokenizer" + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" "github.com/ajitpratap0/GoSQLX/pkg/sql/parser" + "github.com/ajitpratap0/GoSQLX/pkg/sql/tokenizer" ) func ParseSQL(sql string) error { @@ -153,23 +155,18 @@ func ParseSQL(sql string) error { return err } - // Step 2: Convert to parser tokens using the proper converter - converter := parser.NewTokenConverter() - result, err := converter.Convert(tokens) - if err != nil { - return fmt.Errorf("token conversion failed: %w", err) - } - - // Step 3: Parse + // Step 2: Parse the tokenizer's output directly. No conversion step is + // needed: ParseFromModelTokens takes []models.TokenWithSpan. p := parser.NewParser() defer p.Release() - ast, err := p.Parse(result.Tokens) + tree, err := p.ParseFromModelTokens(tokens) if err != nil { return err } + defer ast.ReleaseAST(tree) // CRITICAL: return the AST to its pool - fmt.Printf("Parsed: %T\n", ast) + fmt.Printf("Parsed %d statement(s)\n", len(tree.Statements)) return nil } ``` @@ -1887,19 +1884,16 @@ func ProcessWithPreallocation(sql string) error { return err } - // Convert tokens using the proper converter - converter := parser.NewTokenConverter() - result, err := converter.Convert(tokens) - if err != nil { - return err - } - - // Parse with pre-converted tokens + // Parse the tokenizer's output directly p := parser.NewParser() defer p.Release() - _, err = p.Parse(result.Tokens) - return err + tree, err := p.ParseFromModelTokens(tokens) + if err != nil { + return err + } + ast.ReleaseAST(tree) + return nil } ``` diff --git a/examples/cmd/example_test.go b/examples/cmd/example_test.go index feeb5d98..a17f3634 100644 --- a/examples/cmd/example_test.go +++ b/examples/cmd/example_test.go @@ -143,8 +143,8 @@ func TestExampleErrorCases(t *testing.T) { query: `SELECT * FROM users WHERE name = 'unterminated;`, }, { - name: "invalid Unicode escape sequence", - query: `SELECT * FROM users WHERE name = '\u123';`, // Invalid Unicode escape + name: "incomplete escape sequence", + query: `SELECT * FROM users WHERE name = 'abc\`, // Backslash at end of input }, } diff --git a/internal/batch/batch.go b/internal/batch/batch.go new file mode 100644 index 00000000..994de025 --- /dev/null +++ b/internal/batch/batch.go @@ -0,0 +1,79 @@ +// Copyright 2026 GoSQLX Authors +// SPDX-License-Identifier: Apache-2.0 + +// Package batch recognizes SQL Server client batch separators. Callers must +// enable it only for the SQL Server dialect. +package batch + +import ( + "strconv" + "strings" + + "github.com/ajitpratap0/GoSQLX/pkg/models" +) + +// Count validates a positive decimal GO repeat count without executing it. +func Count(s string) (int, bool) { + if s == "" { + return 0, false + } + for i := range s { + if s[i] < '0' || s[i] > '9' { + return 0, false + } + } + n, err := strconv.Atoi(s) + return n, err == nil && n > 0 +} + +// IsText reports whether s is a rendered GO or GO followed by a repeat count. +func IsText(s string) bool { + fields := strings.Fields(s) + if len(fields) == 0 || !strings.EqualFold(fields[0], "GO") { + return false + } + if len(fields) == 1 { + return true + } + if len(fields) != 2 { + return false + } + _, ok := Count(fields[1]) + return ok +} + +// End returns the first token after an unquoted GO [count] on its own line. +// Tokens without source positions cannot describe a client directive. +func End(tokens []models.TokenWithSpan, i int) (int, bool) { + end, valid, ok := Line(tokens, i) + return end, ok && valid +} + +// Line reports whether the token at index i is an unquoted GO alone on its +// line, optionally followed on that line by one number token: the shape of a +// sqlcmd batch separator. end is the index of the first token after the line +// and valid reports whether the number, if any, is a valid repeat count +// (GO 0 and GO 1.5 have the shape of a separator but an invalid count). +// Tokens without source positions cannot describe a client directive. +func Line(tokens []models.TokenWithSpan, i int) (end int, valid, ok bool) { + if i < 0 || i >= len(tokens) { + return 0, false, false + } + tok := &tokens[i] + if tok.Token.Type != models.TokenTypeIdentifier || tok.Token.Quote != 0 || + !strings.EqualFold(tok.Token.Value, "GO") || tok.Start.Line == 0 || + tok.Start.Line != tok.End.Line || tok.End.Column-tok.Start.Column != 2 || + (i > 0 && tokens[i-1].End.Line >= tok.Start.Line) { + return 0, false, false + } + valid = true + j := i + 1 + if j < len(tokens) && tokens[j].Token.Type == models.TokenTypeNumber && tokens[j].Start.Line == tok.Start.Line { + _, valid = Count(tokens[j].Token.Value) + j++ + } + if j < len(tokens) && tokens[j].Token.Type != models.TokenTypeEOF && tokens[j].Start.Line <= tok.Start.Line { + return 0, false, false + } + return j, valid, true +} diff --git a/internal/batch/batch_test.go b/internal/batch/batch_test.go new file mode 100644 index 00000000..d6cc7c86 --- /dev/null +++ b/internal/batch/batch_test.go @@ -0,0 +1,48 @@ +// Copyright 2026 GoSQLX Authors +// SPDX-License-Identifier: Apache-2.0 + +package batch_test + +import ( + "testing" + + "github.com/ajitpratap0/GoSQLX/internal/batch" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" + "github.com/ajitpratap0/GoSQLX/pkg/sql/tokenizer" +) + +func TestEnd(t *testing.T) { + for _, tc := range []struct { + name, sql string + want bool + }{ + {"bare", "GO", true}, + {"repeat", "go 3 -- repeat\nSELECT 1", true}, + {"quoted", "[GO]", false}, + {"double quoted", `"GO"`, false}, + {"string", "'GO'", false}, + {"fraction", "GO 1.5", false}, + {"zero", "GO 0", false}, + {"trailing word", "GO SELECT 1", false}, + } { + t.Run(tc.name, func(t *testing.T) { + tkz := tokenizer.GetTokenizer() + defer tokenizer.PutTokenizer(tkz) + tkz.SetDialect(keywords.DialectSQLServer) + tokens, err := tkz.TokenizeString(tc.sql) + if err != nil { + t.Fatal(err) + } + if _, got := batch.End(tokens, 0); got != tc.want { + t.Errorf("End(%q) = %v, want %v", tc.sql, got, tc.want) + } + for i := range tokens { + tokens[i].Start.Line = 0 + tokens[i].End.Line = 0 + } + if _, got := batch.End(tokens, 0); got { + t.Error("tokens without spans must not form batch separators") + } + }) + } +} diff --git a/internal/tokscratch/tokscratch.go b/internal/tokscratch/tokscratch.go new file mode 100644 index 00000000..6c6be6c9 --- /dev/null +++ b/internal/tokscratch/tokscratch.go @@ -0,0 +1,65 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package tokscratch lets GoSQLX's own tokenize-then-parse entry points +// tokenize into a buffer owned by the pooled tokenizer instead of a fresh +// slice, without adding that borrowed-result contract to the public +// tokenizer API. +// +// Package tokenizer registers the implementation in its init. Every function +// here takes the *tokenizer.Tokenizer as t (typed any only to avoid an import +// cycle) and returns the same tokens and errors as the matching public method +// (Tokenize, TokenizeString, TokenizeStringContext), with one difference: the +// returned slice, and the Word values its tokens point to, are owned by t. +// They are valid only until the next tokenize call on t, t.Reset or +// tokenizer.PutTokenizer(t). Callers must copy what they keep (the parser's +// preprocess step does) and must never return the slice to a user. Token +// strings are substrings of the input as with the public methods, so they +// stay valid. +package tokscratch + +import ( + "context" + + "github.com/ajitpratap0/GoSQLX/pkg/models" +) + +// Func is the registered implementation. When isString is true, src is the +// SQL and input is ignored; otherwise input is the SQL and is copied once +// into a string as Tokenize does. +type Func func(ctx context.Context, t any, input []byte, src string, isString bool) ([]models.TokenWithSpan, error) + +var impl Func + +// Register installs the implementation. It is called once by package +// tokenizer's init and must not be called by anyone else. +func Register(f Func) { impl = f } + +// Bytes is t.Tokenize(input) into t's scratch buffer. +func Bytes(t any, input []byte) ([]models.TokenWithSpan, error) { + return impl(context.Background(), t, input, "", false) +} + +// String is t.TokenizeString(s) into t's scratch buffer. +func String(t any, s string) ([]models.TokenWithSpan, error) { + return impl(context.Background(), t, nil, s, true) +} + +// StringContext is t.TokenizeStringContext(ctx, s) into t's scratch buffer. +func StringContext(ctx context.Context, t any, s string) ([]models.TokenWithSpan, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + return impl(ctx, t, nil, s, true) +} diff --git a/pkg/advisor/rules_advanced.go b/pkg/advisor/rules_advanced.go index ad116af2..fb327856 100644 --- a/pkg/advisor/rules_advanced.go +++ b/pkg/advisor/rules_advanced.go @@ -117,7 +117,7 @@ func (r *RedundantOrderByInCTERule) Analyze(stmt ast.Statement) []Suggestion { continue } // ORDER BY without LIMIT/FETCH/TOP is redundant in a CTE - hasLimit := cteQuery.Limit != nil || cteQuery.Fetch != nil || cteQuery.Top != nil + hasLimit := cteQuery.Limit != nil || cteQuery.DialectExtras().LimitExpr != nil || cteQuery.Fetch != nil || cteQuery.Top != nil if len(cteQuery.OrderBy) > 0 && !hasLimit { suggestions = append(suggestions, Suggestion{ RuleID: r.ID(), @@ -323,7 +323,7 @@ func (r *MissingLimitRule) Analyze(stmt ast.Statement) []Suggestion { if !ok { return nil } - hasLimit := sel.Limit != nil || sel.Fetch != nil || sel.Top != nil + hasLimit := sel.Limit != nil || sel.DialectExtras().LimitExpr != nil || sel.Fetch != nil || sel.Top != nil if len(sel.OrderBy) > 0 && !hasLimit { return []Suggestion{{ RuleID: r.ID(), diff --git a/pkg/cbinding/cbinding_ext.go b/pkg/cbinding/cbinding_ext.go index f1a59f73..4b397a82 100644 --- a/pkg/cbinding/cbinding_ext.go +++ b/pkg/cbinding/cbinding_ext.go @@ -30,7 +30,6 @@ import ( "strings" goerrors "github.com/ajitpratap0/GoSQLX/pkg/errors" - "github.com/ajitpratap0/GoSQLX/pkg/formatter" "github.com/ajitpratap0/GoSQLX/pkg/gosqlx" "github.com/ajitpratap0/GoSQLX/pkg/linter" lintrules "github.com/ajitpratap0/GoSQLX/pkg/linter/rules" @@ -48,10 +47,6 @@ var errCodeUnsupportedDialect = string(goerrors.ErrCodeUnsupportedDialect) // options_json argument that is not a valid options object. const errCodeInvalidOptions = "INVALID_OPTIONS" -// formatLineWidth is the line width gosqlx.DefaultFormatOptions sets. It is -// spelled out because the FormatOptions field that carries it is deprecated. -const formatLineWidth = 80 - // defaultLintMaxLineLength mirrors the default of `gosqlx lint --max-length`. const defaultLintMaxLineLength = 100 @@ -113,11 +108,14 @@ func parseResultFor(sql, dialect string) ParseResult { stmtTypes := make([]string, 0, len(tree.Statements)) for _, stmt := range tree.Statements { + if _, ok := stmt.(*ast.BatchSeparator); ok { + continue // a GO line is a client directive, not a statement + } stmtTypes = append(stmtTypes, statementTypeName(stmt)) } return ParseResult{ Success: true, - StmtCount: len(tree.Statements), + StmtCount: len(stmtTypes), StmtTypes: stmtTypes, } } @@ -157,9 +155,9 @@ func validationResultFor(sql, dialect string) ValidationResult { } // formatFor formats sql with gosqlx.DefaultFormatOptions. An empty dialect goes -// through gosqlx.Format; otherwise the SQL is parsed with the dialect and -// rendered with the same option mapping gosqlx.Format uses, plus the dialect so -// dialect-specific constructs are rendered for that dialect. +// through gosqlx.Format; otherwise gosqlx.FormatWithDialect parses the SQL with +// the dialect and renders dialect-specific constructs for it. Either way +// comments and optimizer hints are preserved. func formatFor(sql, dialect string) (string, error) { if err := dialectError(dialect); err != nil { return "", err @@ -168,23 +166,7 @@ func formatFor(sql, dialect string) (string, error) { if dialect == "" { return gosqlx.Format(sql, opts) } - tree, err := gosqlx.ParseWithDialect(sql, keywords.SQLDialect(dialect)) - if err != nil { - return "", fmt.Errorf("cannot format invalid SQL: %w", err) - } - kwCase := ast.KeywordPreserve - if opts.UppercaseKeywords { - kwCase = ast.KeywordUpper - } - return formatter.FormatAST(tree, ast.FormatOptions{ - IndentStyle: ast.IndentSpaces, - IndentWidth: opts.IndentSize, - KeywordCase: kwCase, - LineWidth: formatLineWidth, - NewlinePerClause: opts.IndentSize > 0, - AddSemicolon: opts.AddSemicolon, - Dialect: dialect, - }), nil + return gosqlx.FormatWithDialect(sql, opts, keywords.SQLDialect(dialect)) } // formatResultFor builds the gosqlx_format / gosqlx_format_dialect result. diff --git a/pkg/cbinding/cbinding_test.go b/pkg/cbinding/cbinding_test.go index cf58a4ef..b886c07c 100644 --- a/pkg/cbinding/cbinding_test.go +++ b/pkg/cbinding/cbinding_test.go @@ -482,3 +482,33 @@ func TestMetadataResultJSON(t *testing.T) { t.Errorf("expected columns_qualified with u.name, got %v", decoded.ColumnsQualified) } } + +// TestParseResultCountsExcludeGO checks that GO batch separators are not +// counted or typed as statements in the parse result. +func TestParseResultCountsExcludeGO(t *testing.T) { + tests := []struct { + sql string + wantCount int + wantTypes []string + }{ + {"SELECT 1\nGO\nSELECT 2\nGO 3", 2, []string{"SELECT", "SELECT"}}, + {"GO\nGO", 0, []string{}}, + {"SELECT 1", 1, []string{"SELECT"}}, + } + for _, tc := range tests { + t.Run(tc.sql, func(t *testing.T) { + r := parseResultFor(tc.sql, "sqlserver") + if !r.Success { + t.Fatalf("parse failed: %s", r.Error) + } + if r.StmtCount != tc.wantCount || len(r.StmtTypes) != len(tc.wantTypes) { + t.Fatalf("count = %d, types = %v; want %d, %v", r.StmtCount, r.StmtTypes, tc.wantCount, tc.wantTypes) + } + for i := range tc.wantTypes { + if r.StmtTypes[i] != tc.wantTypes[i] { + t.Errorf("types = %v, want %v", r.StmtTypes, tc.wantTypes) + } + } + }) + } +} diff --git a/pkg/errors/builders.go b/pkg/errors/builders.go index 70de257b..579bafba 100644 --- a/pkg/errors/builders.go +++ b/pkg/errors/builders.go @@ -255,7 +255,7 @@ func TokenLimitReachedError(count, maxTokens int, location models.Location, sql ErrCodeTokenLimitReached, fmt.Sprintf("token count %d exceeds limit of %d tokens", count, maxTokens), location, - ).WithContext(sql, 1).WithHint(fmt.Sprintf("Simplify query or adjust MaxTokens limit (currently %d)", maxTokens)) + ).WithContext(sql, 1).WithHint(fmt.Sprintf("Split the input into smaller statements; the limit is fixed at %d tokens per input", maxTokens)) } // TokenizerPanicError creates an E1008 error for a panic that was recovered inside diff --git a/pkg/errors/doc.go b/pkg/errors/doc.go index 6324ef43..07953a2c 100644 --- a/pkg/errors/doc.go +++ b/pkg/errors/doc.go @@ -304,7 +304,7 @@ // // Token count limits // err := errors.TokenLimitReachedError(15000, 10000, location, sqlSource) // // Message: "token count 15000 exceeds limit of 10000 tokens" -// // Hint: "Simplify query or adjust MaxTokens limit (currently 10000)" +// // Hint: "Split the input into smaller statements; the limit is fixed at 10000 tokens per input" // // Panic Recovery: // diff --git a/pkg/formatter/alter_sql_w3_test.go b/pkg/formatter/alter_sql_w3_test.go new file mode 100644 index 00000000..42843811 --- /dev/null +++ b/pkg/formatter/alter_sql_w3_test.go @@ -0,0 +1,191 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package formatter_test + +import ( + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/formatter" + "github.com/ajitpratap0/GoSQLX/pkg/gosqlx" + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" +) + +// TestAlterStatementSQL checks that AST.SQL() and Statement.SQL() write every +// ALTER statement (they used to write nothing for an *ast.AlterStatement) and +// that the output matches the formatter's compact rendering and re-parses to +// the same text. +func TestAlterStatementSQL(t *testing.T) { + tests := []struct { + name string + dialect keywords.SQLDialect + in string + want string + }{ + {"add column", keywords.DialectPostgreSQL, + "ALTER TABLE t ADD COLUMN c INT NOT NULL DEFAULT 0", "ALTER TABLE t ADD COLUMN c INT NOT NULL DEFAULT 0"}, + {"several actions", keywords.DialectPostgreSQL, + "ALTER TABLE IF EXISTS ONLY s.t DROP COLUMN a CASCADE, ALTER COLUMN b SET NOT NULL, RENAME TO u", + "ALTER TABLE IF EXISTS ONLY s.t DROP COLUMN a CASCADE, ALTER COLUMN b SET NOT NULL, RENAME TO u"}, + {"set data type using", keywords.DialectPostgreSQL, + "ALTER TABLE t ALTER COLUMN c TYPE BIGINT USING CAST(c AS BIGINT)", "ALTER TABLE t ALTER COLUMN c TYPE BIGINT USING CAST(c AS BIGINT)"}, + {"quoted names", keywords.DialectPostgreSQL, + `ALTER TABLE "Post" RENAME COLUMN "Old" TO "New"`, `ALTER TABLE "Post" RENAME COLUMN "Old" TO "New"`}, + {"mysql add index", keywords.DialectMySQL, + "ALTER TABLE t ADD UNIQUE KEY idx (a, b)", "ALTER TABLE t ADD UNIQUE KEY idx (a, b)"}, + {"clickhouse on cluster", keywords.DialectClickHouse, + "ALTER TABLE t ON CLUSTER c DELETE WHERE x = 1", "ALTER TABLE t ON CLUSTER c DELETE WHERE x = 1"}, + {"alter policy", keywords.DialectGeneric, + "ALTER POLICY p ON s.t TO a, b USING (x = 1) WITH CHECK (y = 2)", "ALTER POLICY p ON s.t TO a, b USING (x = 1) WITH CHECK (y = 2)"}, + {"alter connector url", keywords.DialectGeneric, + "ALTER CONNECTOR c SET URL 'it''s'", "ALTER CONNECTOR c SET URL 'it''s'"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + tree, err := gosqlx.ParseWithDialect(tc.in, tc.dialect) + if err != nil { + t.Fatalf("parse %q: %v", tc.in, err) + } + defer ast.ReleaseAST(tree) + if len(tree.Statements) != 1 { + t.Fatalf("got %d statements, want 1", len(tree.Statements)) + } + alter, ok := tree.Statements[0].(*ast.AlterStatement) + if !ok { + t.Fatalf("got %T, want *ast.AlterStatement", tree.Statements[0]) + } + if got := alter.SQL(); got != tc.want { + t.Errorf("Statement.SQL() = %q, want %q", got, tc.want) + } + if got := tree.SQL(); got != tc.want { + t.Errorf("AST.SQL() = %q, want %q", got, tc.want) + } + style := ast.CompactStyle() + style.Dialect = string(tc.dialect) + if got := formatter.FormatStatement(alter, style); got != tc.want { + t.Errorf("FormatStatement = %q, want %q", got, tc.want) + } + // The SQL() text parses back to the same statement. + again, err := gosqlx.ParseWithDialect(tc.want, tc.dialect) + if err != nil { + t.Fatalf("re-parse %q: %v", tc.want, err) + } + defer ast.ReleaseAST(again) + if got := again.SQL(); got != tc.want { + t.Errorf("re-parsed SQL() = %q, want %q", got, tc.want) + } + }) + } +} + +// TestAlterStatementSQL_Bare checks that an ALTER statement without an +// operation, and a nil one, never write more than the ALTER keyword. +func TestAlterStatementSQL_Bare(t *testing.T) { + var nilStmt *ast.AlterStatement + if got := nilStmt.SQL(); got != "" { + t.Errorf("nil SQL() = %q, want empty", got) + } + role := &ast.AlterStatement{Type: ast.AlterTypeRole, Name: "r"} + if got := role.SQL(); got != "ALTER" { + t.Errorf("operation-less SQL() = %q, want ALTER", got) + } +} + +// TestAlterNameQuotes_Recorded checks that the delimiters of names in ALTER +// ROLE / POLICY / CONNECTOR, index names, partition names, INHERITS and index +// names inside constraints are recorded and written back when they carry +// meaning (case, keywords). +func TestAlterNameQuotes_Recorded(t *testing.T) { + tests := []struct { + name string + dialect keywords.SQLDialect + in string + want string + }{ + {"policy", keywords.DialectGeneric, + `ALTER POLICY "Pol" ON "Sch"."Tbl" TO "Admin", other`, `ALTER POLICY "Pol" ON "Sch"."Tbl" TO "Admin", other`}, + {"policy rename", keywords.DialectGeneric, + `ALTER POLICY "Pol" ON t RENAME TO "Pol2"`, `ALTER POLICY "Pol" ON t RENAME TO "Pol2"`}, + {"connector", keywords.DialectGeneric, + `ALTER CONNECTOR "Conn" SET OWNER USER "Bob"`, `ALTER CONNECTOR "Conn" SET OWNER USER "Bob"`}, + {"add index name", keywords.DialectMySQL, + "ALTER TABLE t ADD INDEX `Select` (a)", "ALTER TABLE t ADD INDEX `Select` (a)"}, + {"drop index name", keywords.DialectMySQL, + "ALTER TABLE t DROP INDEX `Select`", "ALTER TABLE t DROP INDEX `Select`"}, + {"constraint index name", keywords.DialectMySQL, + "CREATE TABLE t (a INT, KEY `Select` (a), UNIQUE KEY `Order` (a))", + "CREATE TABLE t (a INT, KEY `Select` (a), UNIQUE KEY `Order` (a))"}, + {"fulltext index name", keywords.DialectMySQL, + "CREATE TABLE t (a TEXT, FULLTEXT KEY `Select` (a))", "CREATE TABLE t (a TEXT, FULLTEXT KEY `Select` (a))"}, + {"clickhouse index name", keywords.DialectClickHouse, + "CREATE TABLE t (a Int32, INDEX `Select` a TYPE minmax GRANULARITY 1) ENGINE = MergeTree ORDER BY a", + "CREATE TABLE t (a Int32, INDEX `Select` a TYPE minmax GRANULARITY 1) ENGINE=MergeTree ORDER BY a"}, + {"inherits", keywords.DialectPostgreSQL, + `CREATE TABLE t (a INT) INHERITS ("Parent", "Sch"."Other")`, `CREATE TABLE t (a INT) INHERITS ("Parent", "Sch"."Other")`}, + {"partition name", keywords.DialectMySQL, + "CREATE TABLE t (a INT) PARTITION BY RANGE (a) (PARTITION `Select` VALUES LESS THAN (10))", + "CREATE TABLE t (a INT) PARTITION BY RANGE (a) (PARTITION `Select` VALUES LESS THAN (10))"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + tree, err := gosqlx.ParseWithDialect(tc.in, tc.dialect) + if err != nil { + t.Fatalf("parse %q: %v", tc.in, err) + } + defer ast.ReleaseAST(tree) + style := ast.CompactStyle() + style.Dialect = string(tc.dialect) + got := formatter.FormatStatement(tree.Statements[0], style) + if got != tc.want { + t.Errorf("FormatStatement\n got: %s\nwant: %s", got, tc.want) + } + if sql := tree.SQL(); sql != tc.want { + t.Errorf("SQL()\n got: %s\nwant: %s", sql, tc.want) + } + }) + } +} + +// TestAlterRoleNameQuotes checks that a hand-built ALTER ROLE statement +// writes the delimiters recorded on its names, and only those. +func TestAlterRoleNameQuotes(t *testing.T) { + q := ast.QuotedName('"') + tests := []struct { + name string + stmt *ast.AlterStatement + want string + }{ + {"rename keeps case quotes", &ast.AlterStatement{ + Type: ast.AlterTypeRole, Name: "MyRole", NameQuote: q, + Operation: &ast.AlterRoleOperation{Type: ast.RenameRole, NewName: "NewRole", NewNameQuote: q}, + }, `ALTER ROLE "MyRole" RENAME TO "NewRole"`}, + {"plain names stay bare", &ast.AlterStatement{ + Type: ast.AlterTypeRole, Name: "myrole", + Operation: &ast.AlterRoleOperation{Type: ast.RenameRole, NewName: "newrole"}, + }, `ALTER ROLE myrole RENAME TO newrole`}, + {"add member", &ast.AlterStatement{ + Type: ast.AlterTypeRole, Name: "r", + Operation: &ast.AlterRoleOperation{Type: ast.AddMember, MemberName: "User", MemberNameQuote: q}, + }, `ALTER ROLE r ADD MEMBER "User"`}, + } + for _, tc := range tests { + if got := tc.stmt.SQL(); got != tc.want { + t.Errorf("%s: SQL() = %q, want %q", tc.name, got, tc.want) + } + if got := formatter.FormatStatement(tc.stmt, ast.CompactStyle()); got != tc.want { + t.Errorf("%s: FormatStatement = %q, want %q", tc.name, got, tc.want) + } + } +} diff --git a/pkg/formatter/comment_literal_regression_test.go b/pkg/formatter/comment_literal_regression_test.go new file mode 100644 index 00000000..d5c44545 --- /dev/null +++ b/pkg/formatter/comment_literal_regression_test.go @@ -0,0 +1,87 @@ +// Copyright 2026 GoSQLX Authors +// SPDX-License-Identifier: Apache-2.0 + +package formatter + +import ( + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" + "github.com/ajitpratap0/GoSQLX/pkg/sql/tokenizer" +) + +func TestNestedHintSkipsDialectQuotedText(t *testing.T) { + tests := []struct { + name, source, rendered, want string + dialect keywords.SQLDialect + }{ + {"postgres backslash", `SELECT 'abc\' AS a, (SELECT /*+ H */ 1) AS b, 'SELECT' AS c`, `SELECT 'abc\' AS a, (SELECT 1) AS b, 'SELECT' AS c`, `SELECT 'abc\' AS a, (SELECT /*+ H */ 1) AS b, 'SELECT' AS c`, keywords.DialectPostgreSQL}, + {"sqlserver bracket", `SELECT [SELECT] AS a, (SELECT /*+ H */ 1) AS b`, `SELECT [SELECT] AS a, (SELECT 1) AS b`, `SELECT [SELECT] AS a, (SELECT /*+ H */ 1) AS b`, keywords.DialectSQLServer}, + {"postgres dollar", `SELECT $$SELECT$$ AS a, (SELECT /*+ H */ 1) AS b`, `SELECT $$SELECT$$ AS a, (SELECT 1) AS b`, `SELECT $$SELECT$$ AS a, (SELECT /*+ H */ 1) AS b`, keywords.DialectPostgreSQL}, + {"oracle q quote", `SELECT q'[SELECT]' AS a, (SELECT /*+ H */ 1) AS b`, `SELECT q'[SELECT]' AS a, (SELECT 1) AS b`, `SELECT q'[SELECT]' AS a, (SELECT /*+ H */ 1) AS b`, keywords.DialectOracle}, + {"mysql national", `SELECT N'SELECT' AS a, (SELECT /*+ H */ 1) AS b`, `SELECT N'SELECT' AS a, (SELECT 1) AS b`, `SELECT N'SELECT' AS a, (SELECT /*+ H */ 1) AS b`, keywords.DialectMySQL}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + tkz := tokenizer.GetTokenizer() + defer tokenizer.PutTokenizer(tkz) + tkz.SetDialect(tc.dialect) + tokens, err := tkz.TokenizeString(tc.source) + if err != nil { + t.Fatal(err) + } + got, err := JoinWithComments([]string{tc.rendered}, tokens, tkz.Comments, CommentLayout{Dialect: string(tc.dialect)}) + if err != nil { + t.Fatal(err) + } + if got != tc.want { + t.Fatalf("got %q, want %q", got, tc.want) + } + }) + } +} + +func TestSQLServerBatchJoinKeepsGoOnOwnLine(t *testing.T) { + layout := CommentLayout{Dialect: string(keywords.DialectSQLServer), Terminator: ";", Gap: " "} + got, err := JoinWithComments([]string{"SELECT 1", "GO 2", "SELECT 2"}, nil, nil, layout) + if err != nil || got != "SELECT 1;\nGO 2\nSELECT 2" { + t.Fatalf("plain join = %q, %v", got, err) + } + + tkz := tokenizer.GetTokenizer() + defer tokenizer.PutTokenizer(tkz) + tkz.SetDialect(keywords.DialectSQLServer) + tokens, err := tkz.TokenizeString("SELECT 1\nGO -- batch\nSELECT /*+ H */ 2") + if err != nil { + t.Fatal(err) + } + got, err = JoinWithComments([]string{"SELECT 1", "GO", "SELECT 2"}, tokens, tkz.Comments, layout) + if err != nil || got != "SELECT 1;\nGO -- batch\nSELECT /*+ H */ 2" { + t.Fatalf("comment join = %q, %v", got, err) + } + + tokens, err = tkz.TokenizeString("SELECT 1\nGO 3 -- repeat\nGO\nSELECT 2") + if err != nil { + t.Fatal(err) + } + got, err = JoinWithComments([]string{"SELECT 1", "GO 3", "GO", "SELECT 2"}, tokens, tkz.Comments, layout) + if err != nil || got != "SELECT 1;\nGO 3 -- repeat\nGO\nSELECT 2" { + t.Fatalf("repeat and consecutive join = %q, %v", got, err) + } + + tokens, err = tkz.TokenizeString("SELECT 1;\nGO\n;SELECT 2 -- hi") + if err != nil { + t.Fatal(err) + } + got, err = JoinWithComments([]string{"SELECT 1", "GO", "SELECT 2"}, tokens, tkz.Comments, layout) + if err != nil || got != "SELECT 1;\nGO\nSELECT 2 -- hi" { + t.Fatalf("empty semicolon after GO join = %q, %v", got, err) + } + + other := layout + other.Dialect = string(keywords.DialectPostgreSQL) + got, err = JoinWithComments([]string{"SELECT 1", "GO", "SELECT 2"}, nil, nil, other) + if err != nil || got != "SELECT 1; GO; SELECT 2" { + t.Fatalf("other dialect join = %q, %v", got, err) + } +} diff --git a/pkg/formatter/comments.go b/pkg/formatter/comments.go new file mode 100644 index 00000000..2a299bad --- /dev/null +++ b/pkg/formatter/comments.go @@ -0,0 +1,900 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package formatter + +import ( + "errors" + "fmt" + "sort" + "strings" + + "github.com/ajitpratap0/GoSQLX/internal/batch" + "github.com/ajitpratap0/GoSQLX/pkg/models" + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" + "github.com/ajitpratap0/GoSQLX/pkg/sql/tokenizer" +) + +// ErrUnplaceableComment is returned by JoinWithComments for a comment that +// carries meaning to the database (an optimizer hint such as /*+ FULL(t) */ or +// a MySQL executable comment such as /*!40101 ... */) and cannot be put back +// where it was. Writing such a comment elsewhere, or dropping it, would change +// what the statement does, so the caller must leave the input unformatted. +var ErrUnplaceableComment = errors.New("formatter: cannot preserve comment") + +// CommentLayout describes how JoinWithComments separates statements. +type CommentLayout struct { + // Dialect selects lexical rules for rendered keywords and batch boundaries. + Dialect string + // Terminator is written after every statement except the last. Use ";" + // when the rendered statements carry no terminator, "" when they already + // end with one. + Terminator string + // Gap separates one statement (and its trailing comments) from the next: + // "\n\n", "\n" or " ". It is widened to a line break whenever a line + // comment would otherwise swallow the following text. + Gap string + // Batches holds the source positions of the SQL Server GO separators the + // parser recognized (ast.BatchSeparator.Pos), so that the comment layer + // splits the source at exactly the parser's batch boundaries. When it is + // nil, GO lines are recognized from the tokens. + Batches []models.Location + // Starts holds the source position of the first token of every statement + // the parser produced, one per rendered statement and in the same order + // (GO separators included), as StatementStarts returns them. When it is + // set, the source is cut into statements at exactly these positions, so a + // ';' inside a statement (a procedure or trigger body) does not split it + // and statements without a ';' between them (T-SQL) are told apart. When + // it is nil, statements are found by splitting the tokens at ';'. + Starts []models.Location +} + +// JoinWithComments joins already-rendered statements and puts the comments of +// the original text back. +// +// stmts holds the rendered text of every statement in source order, tokens the +// token stream of the source (EOF is ignored) and comments the comments the +// tokenizer captured from it. Statements are matched to the source at the +// parser's statement start positions (layout.Starts) when they are given, and +// otherwise by splitting tokens at ';'. +// +// Where a comment goes: +// +// - A comment between two statements goes in front of the next statement, +// except one that follows the previous statement on the same line, which +// stays after it (SELECT 1; -- done). +// - A comment inside a statement is moved in front of that statement, in +// source order. The renderer lays statements out afresh, so there is no +// position inside them to return to. +// - A comment after the last statement stays after it. +// - An optimizer hint directly after a SELECT, INSERT, UPDATE, DELETE, +// MERGE or REPLACE keyword (SELECT /*+ ... */, also one inside a CTE, +// subquery or union branch) stays there, provided the rendered statement +// has the same number of that keyword and the hinted one can be told +// apart from the others by its surrounding words when the renderer +// writes them in another order. A hint anywhere else, and a MySQL +// executable comment (/*! ... */) inside a statement, make the call fail +// with ErrUnplaceableComment. An executable comment followed by ';' on its +// own is a statement of its own and is kept verbatim with its ';'. +// +// When the statements cannot be matched to the source (stmts and the runs of +// tokens differ in number, or a run does not begin at its layout.Starts +// position) every comment is put in front of the first statement, under the +// same restriction on hints. A comment is never attached to a statement it +// was not written next to. +// +// No comment is ever dropped: the comments of the result are those of the +// input. +func JoinWithComments(stmts []string, tokens []models.TokenWithSpan, comments []models.Comment, layout CommentLayout) (string, error) { + if len(comments) == 0 { + return joinPlain(stmts, layout), nil + } + items := splitItems(tokens, layout) + var nStmt int + for _, it := range items { + if !it.empty { + nStmt++ + } + } + if nStmt != len(stmts) || len(stmts) == 0 || !batchesAgree(items, stmts, layout.Dialect) || + !startsAgree(items, layout.Starts) { + return joinHoisted(stmts, comments, layout) + } + + // Attach every comment to an item. + type attach struct { + leading []string + trailing []string + hints []placedHint + } + att := make([]attach, len(items)) + occs := make([]map[string]*hintOccs, len(items)) // per item, built on demand + var tail []string + for ci := range comments { + c := comments[ci] + sig := isSignificantComment(c.Text) + idx, role := locateComment(items, c) + switch role { + case roleLeading: + if sig && !items[idx].empty && isExecutable(c.Text) { + // An executable comment in front of a statement, with no ';' + // of its own, runs together with it. + return "", fmt.Errorf("%w: %s", ErrUnplaceableComment, abbreviate(c.Text)) + } + att[idx].leading = append(att[idx].leading, c.Text) + case roleTrailing: + if sig && isExecutable(c.Text) { + return "", fmt.Errorf("%w: %s", ErrUnplaceableComment, abbreviate(c.Text)) + } + att[idx].trailing = append(att[idx].trailing, c.Text) + case roleInner: + it := items[idx] + if sig { + if isHint(c.Text) { + if occs[idx] == nil { + occs[idx] = map[string]*hintOccs{} + } + if h, ok := it.hintSlot(tokens, c, occs[idx]); ok { + att[idx].hints = append(att[idx].hints, h) + continue + } + } + return "", fmt.Errorf("%w: %s", ErrUnplaceableComment, abbreviate(c.Text)) + } + att[idx].leading = append(att[idx].leading, c.Text) + case roleTail: + if sig && isExecutable(c.Text) { + return "", fmt.Errorf("%w: %s", ErrUnplaceableComment, abbreviate(c.Text)) + } + tail = append(tail, c.Text) + } + } + + var sb strings.Builder + prevEndsInLine := false // the text written so far ends in a line comment + prevBatch := false // last emitted item, ignoring empty semicolons + si := 0 // index into stmts + for i, it := range items { + a := att[i] + var chunk strings.Builder + endsInLine := false + if it.empty { + // A bare ';' with comments before it. If one of them is an + // executable comment (/*!...*/;) the comment is the statement and + // keeps its ';'; otherwise they are plain comments and the stray + // ';' is dropped, as the formatter drops every empty statement. + text := strings.Join(a.leading, "\n") + chunk.WriteString(text) + if hasExecutable(a.leading) { + if strings.HasPrefix(lastComment(a.leading), "--") { + chunk.WriteString("\n") + } + chunk.WriteString(";") + } else { + endsInLine = strings.HasPrefix(lastComment(a.leading), "--") + } + } else { + for _, l := range a.leading { + chunk.WriteString(l) + chunk.WriteString("\n") + } + stmt := stmts[si] + si++ + if len(a.hints) > 0 { + var err error + if stmt, err = insertHints(stmt, a.hints, layout.Dialect); err != nil { + return "", err + } + } + chunk.WriteString(stmt) + if i < len(items)-1 && !it.batch { + chunk.WriteString(layout.Terminator) + } + } + for _, t := range a.trailing { + chunk.WriteString(" ") + chunk.WriteString(t) + } + if chunk.Len() == 0 { + continue + } + if sb.Len() > 0 { + gap := layout.Gap + if it.batch || prevBatch { + gap = "\n" + } else if (prevEndsInLine || len(a.leading) > 0) && !strings.Contains(gap, "\n") { + gap = "\n" + } + sb.WriteString(gap) + } + sb.WriteString(chunk.String()) + if len(a.trailing) > 0 { + endsInLine = strings.HasPrefix(lastComment(a.trailing), "--") + } + prevEndsInLine = endsInLine + prevBatch = it.batch + } + for _, t := range tail { + sb.WriteString("\n") + sb.WriteString(t) + } + return sb.String(), nil +} + +func hasExecutable(comments []string) bool { + for _, c := range comments { + if isExecutable(c) { + return true + } + } + return false +} + +func lastComment(c []string) string { + if len(c) == 0 { + return "" + } + return c[len(c)-1] +} + +// joinPlain joins statements without comments. +func joinPlain(stmts []string, layout CommentLayout) string { + var sb strings.Builder + for i, s := range stmts { + if i > 0 { + if !isBatchText(stmts[i-1], layout.Dialect) { + sb.WriteString(layout.Terminator) + } + if isBatchText(s, layout.Dialect) || isBatchText(stmts[i-1], layout.Dialect) { + sb.WriteByte('\n') + } else { + sb.WriteString(layout.Gap) + } + } + sb.WriteString(s) + } + return sb.String() +} + +func isBatchText(s, dialect string) bool { + return isSQLServerDialect(dialect) && batch.IsText(s) +} + +func isSQLServerDialect(d string) bool { + switch strings.ToLower(d) { + case "sqlserver", "mssql", "tsql": + return true + } + return false +} + +// joinHoisted is the fallback used when statements cannot be matched to the +// source: all comments go in front of the first statement. +func joinHoisted(stmts []string, comments []models.Comment, layout CommentLayout) (string, error) { + var sb strings.Builder + for _, c := range comments { + if isSignificantComment(c.Text) { + return "", fmt.Errorf("%w: %s", ErrUnplaceableComment, abbreviate(c.Text)) + } + sb.WriteString(c.Text) + sb.WriteString("\n") + } + sb.WriteString(joinPlain(stmts, layout)) + return sb.String(), nil +} + +// item is one ';'-terminated run of tokens: a statement, or (empty) a bare +// ';' standing where a statement would be. +type item struct { + first, last int // token indexes of the statement (first > last for empty) + term int // index of the terminating ';' token, -1 when absent + empty bool + batch bool + // start and end bound the item in the source: start is the first token + // (or, for an empty item, its ';'), end is just past the terminator (or + // the last token). + start, end models.Location +} + +// batchesAgree reports whether the GO items of the source are the rendered +// GO statements, one for one. +func batchesAgree(items []item, stmts []string, dialect string) bool { + si := 0 + for _, it := range items { + if it.empty { + continue + } + if it.batch != isBatchText(stmts[si], dialect) { + return false + } + si++ + } + return true +} + +// startsAgree reports whether, when the parser's statement starts are known, +// every statement item begins at its start, one for one. +func startsAgree(items []item, starts []models.Location) bool { + if starts == nil { + return true + } + j := 0 + for _, it := range items { + if it.empty { + continue + } + if j >= len(starts) || it.start != starts[j] { + return false + } + j++ + } + return j == len(starts) +} + +// splitItems splits tokens into statements and, in SQL Server, GO lines: those +// at layout.Batches, or those batch.End recognizes when it is nil. Statements +// begin at layout.Starts when it is set (see splitAtStarts) and are otherwise +// cut at every ';'. +func splitItems(tokens []models.TokenWithSpan, layout CommentLayout) []item { + var items []item + sqlServer := isSQLServerDialect(layout.Dialect) + byStarts := layout.Starts != nil + nextBatch, nextStart := 0, 0 + cur := item{first: -1, term: -1} + // pending holds the ';' tokens seen in cur since its last other token + // (byStarts only): the first ends the statement unless another of its + // tokens follows, the others are stray empty statements. + var pending []int + flush := func() { + if cur.first >= 0 { + if len(pending) > 0 { + cur.term = pending[0] + cur.end = tokens[pending[0]].End + } + items = append(items, cur) + for _, k := range pending[min(1, len(pending)):] { + items = append(items, item{first: k, last: k - 1, term: k, empty: true, start: tokens[k].Start, end: tokens[k].End}) + } + } + pending = pending[:0] + cur = item{first: -1, term: -1} + } + for i := 0; i < len(tokens); i++ { + t := tokens[i] + if sqlServer { + end, ok := 0, false + if layout.Batches == nil { + end, ok = batch.End(tokens, i) + } else if nextBatch < len(layout.Batches) && t.Start == layout.Batches[nextBatch] { + nextBatch++ + end, _, ok = batch.Line(tokens, i) + if !ok { + end = i + 1 + } + ok = true + } + if ok { + flush() + if byStarts && nextStart < len(layout.Starts) && t.Start == layout.Starts[nextStart] { + nextStart++ + } + items = append(items, item{first: i, last: end - 1, term: -1, batch: true, start: t.Start, end: tokens[end-1].End}) + i = end - 1 + continue + } + } + if t.Token.Type == models.TokenTypeEOF { + continue + } + if byStarts && nextStart < len(layout.Starts) && t.Start == layout.Starts[nextStart] { + nextStart++ + flush() + cur.first, cur.start = i, t.Start + } + if t.Token.Type == models.TokenTypeSemicolon { + switch { + case cur.first < 0: + items = append(items, item{first: i, last: i - 1, term: i, empty: true, start: t.Start, end: t.End}) + case byStarts: + pending = append(pending, i) + default: + cur.term = i + cur.end = t.End + flush() + } + continue + } + if cur.first < 0 { + cur.first = i + cur.start = t.Start + } + pending = pending[:0] + cur.last = i + cur.end = t.End + } + flush() + return items +} + +// placedHint is an optimizer hint written right after a statement keyword: +// the ordinal-th occurrence (0-based) of keyword among the total occurrences +// of that word in the statement, e.g. the hint in +// "WITH c AS (SELECT /*+ MATERIALIZE */ ...) SELECT ..." is the first of two +// SELECT keywords. +type placedHint struct { + text string + keyword string + ordinal int + total int + // sigs holds the context signature (see occurrenceSig) of every + // occurrence of keyword in the source statement, in source order. The + // rendered statement must show the same occurrences, possibly reordered + // by the renderer (Oracle START WITH before CONNECT BY), for the hint to + // be placed. nil skips the check. + sigs []string +} + +// hintOccs lists the occurrences of one hint keyword in a source statement: +// their token indexes and context signatures (occurrenceSig), in source +// order. It is computed once per statement and keyword and shared by every +// hint on that keyword. +type hintOccs struct { + idx []int + sigs []string +} + +// hintSlot reports whether c lies directly after a keyword that can carry a +// hint (SELECT, INSERT, UPDATE, DELETE, MERGE, REPLACE) of the statement, and +// where that keyword is among the statement's occurrences of the same word. +// cache holds the occurrences of each keyword in the statement, filled on +// first use. +func (it item) hintSlot(tokens []models.TokenWithSpan, c models.Comment, cache map[string]*hintOccs) (placedHint, bool) { + if it.empty || it.first < 0 || it.first == it.last { + return placedHint{}, false + } + // j is the last token of the statement that starts before the comment; + // the comment must lie between it and the next token. + n := sort.Search(it.last-it.first+1, func(k int) bool { + return before(c.Start, tokens[it.first+k].Start) + }) + j := it.first + n - 1 + if j < it.first || j >= it.last || before(c.Start, tokens[j].End) || !before(c.Start, tokens[j+1].Start) { + return placedHint{}, false + } + kw, ok := hintKeyword(tokens[j].Token) + if !ok { + return placedHint{}, false + } + occ := cache[kw] + if occ == nil { + occ = &hintOccs{} + span := tokens[it.first : it.last+1] + for k := it.first; k <= it.last; k++ { + if w, isWord := wordToken(tokens[k].Token); isWord && strings.EqualFold(w, kw) { + occ.idx = append(occ.idx, k) + occ.sigs = append(occ.sigs, occurrenceSig(span, k-it.first)) + } + } + cache[kw] = occ + } + ordinal := sort.SearchInts(occ.idx, j) + return placedHint{text: c.Text, keyword: kw, ordinal: ordinal, total: len(occ.idx), sigs: occ.sigs}, true +} + +// occurrenceSig returns a signature of the context of the keyword at +// tokens[k]: the last few words before it and the first few words after it +// within its parenthesized scope, upper-cased, ignoring punctuation, string +// literals and the optional AS keyword. The signature tells apart nested +// SELECTs that the renderer may write in another order than the source, as +// in "CONNECT BY ... IN (SELECT 1 FROM u) START WITH ... IN (SELECT 2 ...)". +func occurrenceSig(tokens []models.TokenWithSpan, k int) string { + const nBefore, nAfter = 3, 4 + var before, after []string + for i := k - 1; i >= 0 && len(before) < nBefore; i-- { + if w, ok := sigWord(tokens[i].Token); ok { + before = append(before, w) + } + } + depth := 0 + for i := k + 1; i < len(tokens) && len(after) < nAfter; i++ { + switch tokens[i].Token.Type { + case models.TokenTypeLParen: + depth++ + continue + case models.TokenTypeRParen: + depth-- + continue + case models.TokenTypeSemicolon, models.TokenTypeEOF: + depth = -1 + } + if depth < 0 { + break + } + if w, ok := sigWord(tokens[i].Token); ok { + after = append(after, w) + } + } + var sb strings.Builder + for i := len(before) - 1; i >= 0; i-- { + sb.WriteString(before[i]) + sb.WriteByte(' ') + } + sb.WriteByte('|') + for _, w := range after { + sb.WriteByte(' ') + sb.WriteString(w) + } + return sb.String() +} + +// sigWord returns the upper-case text of a word, quoted name or number token +// for occurrenceSig. String literals (whose escaping a renderer may change), +// punctuation and AS (which a renderer may add before an alias) are skipped. +func sigWord(tok models.Token) (string, bool) { + if tok.Type == models.TokenTypeNumber { + return tok.Value, tok.Value != "" + } + if tok.Quote != 0 && tok.Value != "" && (tok.Type == models.TokenTypeIdentifier || tok.Type == models.TokenTypeDoubleQuotedString) { + return strings.ToUpper(tok.Value), true + } + w, ok := wordToken(tok) + if !ok || strings.EqualFold(w, "AS") { + return "", false + } + return strings.ToUpper(w), true +} + +// hintKeyword returns the upper-case keyword of tok when it is an unquoted +// statement keyword that can carry an optimizer hint. +func hintKeyword(tok models.Token) (string, bool) { + w, ok := wordToken(tok) + if !ok { + return "", false + } + switch kw := strings.ToUpper(w); kw { + case "SELECT", "INSERT", "UPDATE", "DELETE", "MERGE", "REPLACE": + return kw, true + } + return "", false +} + +// wordToken returns the text of tok when it is an unquoted word (a keyword or +// bare identifier), not a string literal or quoted name. +func wordToken(tok models.Token) (string, bool) { + if tok.Quote != 0 || tok.Value == "" { + return "", false + } + switch tok.Type { + case models.TokenTypeString, models.TokenTypeSingleQuotedString, + models.TokenTypeTripleSingleQuotedString, models.TokenTypeTripleDoubleQuotedString, + models.TokenTypeDollarQuotedString, models.TokenTypeByteStringLiteral, + models.TokenTypeNationalStringLiteral, models.TokenTypeEscapedStringLiteral, + models.TokenTypeUnicodeStringLiteral, models.TokenTypeHexStringLiteral, + models.TokenTypeNumber: + return "", false + } + for i := 0; i < len(tok.Value); i++ { + if !isWordByte(tok.Value[i]) { + return "", false + } + } + return tok.Value, true +} + +func isWordByte(b byte) bool { + return b >= 'A' && b <= 'Z' || b >= 'a' && b <= 'z' || b >= '0' && b <= '9' || b == '_' || b == '$' || b >= 0x80 +} + +type commentRole int + +const ( + roleLeading commentRole = iota + roleTrailing + roleInner + roleTail +) + +// locateComment finds the item a comment belongs to and its role there. +// Items are in source order and do not overlap, so the item is found by +// binary search: the first one that ends after the comment starts. +func locateComment(items []item, c models.Comment) (int, commentRole) { + if len(items) == 0 { + return 0, roleTail + } + i := sort.Search(len(items), func(i int) bool { return before(c.Start, items[i].end) }) + if i < len(items) { + it := items[i] + if before(c.Start, it.start) { + // Between the previous item and this one (or before the first). + if i > 0 && !it.empty && c.Start.Line == items[i-1].end.Line && + (items[i-1].term >= 0 || items[i-1].batch) { + return i - 1, roleTrailing + } + return i, roleLeading + } + return i, roleInner + } + last := items[len(items)-1] + if c.Start.Line == last.end.Line { + return len(items) - 1, roleTrailing + } + return len(items) - 1, roleTail +} + +// before reports whether a is strictly earlier than b. +func before(a, b models.Location) bool { + if a.Line != b.Line { + return a.Line < b.Line + } + return a.Column < b.Column +} + +// isHint reports whether text is an optimizer hint comment, /*+ ... */. +func isHint(text string) bool { return strings.HasPrefix(text, "/*+") } + +// isExecutable reports whether text is a MySQL/MariaDB executable comment, +// /*! ... */ or /*M! ... */, whose content the server runs. +func isExecutable(text string) bool { + return strings.HasPrefix(text, "/*!") || strings.HasPrefix(text, "/*M!") +} + +// isLineHint reports whether text is an Oracle line hint, --+ ... . It is +// significant like /*+ */ but cannot be written inline after a keyword (it +// would swallow the rest of the line), so it is never placed, only refused +// when it sits inside a statement. +func isLineHint(text string) bool { return strings.HasPrefix(text, "--+") } + +// isSignificantComment reports whether a comment affects how the database +// runs the statement. +func isSignificantComment(text string) bool { + return isHint(text) || isLineHint(text) || isExecutable(text) +} + +// insertHints writes each hint after the keyword it followed in the source: +// "SELECT a FROM t" becomes "SELECT /*+ H */ a FROM t". A hint on the first +// word of the statement needs the rendered statement to start with that word. +// A hint on a later keyword (a SELECT inside a CTE or subquery) needs the +// rendered statement to have as many occurrences of the word as the source, +// and goes after the same one; otherwise the hint has no place to go and the +// call fails with ErrUnplaceableComment. +func insertHints(stmt string, hints []placedHint, dialect ...string) (string, error) { + type insertion struct { + at int // byte offset just after the keyword + text string + } + ins := make([]insertion, 0, len(hints)) + words := map[string][]wordOcc{} // upper-case word -> occurrences in stmt + matchers := map[string]*occMatcher{} + scanned := false + for _, h := range hints { + if h.ordinal == 0 && h.total >= 1 { + if end, ok := leadingWordEnd(stmt, h.keyword); ok { + ins = append(ins, insertion{end, h.text}) + continue + } + } + if !scanned { + var d string + if len(dialect) > 0 { + d = dialect[0] + } + var err error + words, err = wordEnds(stmt, d) + if err != nil { + return "", fmt.Errorf("%w: rendered statement: %v", ErrUnplaceableComment, err) + } + scanned = true + } + occs := words[h.keyword] + if len(occs) != h.total || h.ordinal >= len(occs) { + return "", fmt.Errorf("%w: %s (rendered statement does not keep the %s it follows)", ErrUnplaceableComment, abbreviate(h.text), h.keyword) + } + m := matchers[h.keyword] + if m == nil || !m.covers(h.sigs) { + m = newOccMatcher(h.sigs, occs) + matchers[h.keyword] = m + } + at, ok := m.match(h, occs) + if !ok { + return "", fmt.Errorf("%w: %s (rendered statement reorders the %s it follows)", ErrUnplaceableComment, abbreviate(h.text), h.keyword) + } + ins = append(ins, insertion{at, h.text}) + } + // Write the statement once with every hint in place; hints at one offset + // keep their source order. + sort.SliceStable(ins, func(a, b int) bool { return ins[a].at < ins[b].at }) + var sb strings.Builder + n := len(stmt) + for _, in := range ins { + n += len(in.text) + 1 + } + sb.Grow(n) + prev := 0 + for _, in := range ins { + sb.WriteString(stmt[prev:in.at]) + sb.WriteByte(' ') + sb.WriteString(in.text) + prev = in.at + } + sb.WriteString(stmt[prev:]) + return sb.String(), nil +} + +// occMatcher finds, for the hints on one keyword of one statement, the +// rendered occurrence of the keyword each hint followed in the source. When +// the rendered occurrences have the same context signatures in the same order +// as the source ones, that is the one with the same ordinal. When the renderer +// wrote them in another order, it is the one whose signature matches, +// provided that signature is unique on both sides; otherwise the keyword +// cannot be identified. The signatures are compared once per statement and +// keyword, not once per hint. +type occMatcher struct { + sigs []string // the source signatures it was built for + same bool // the rendered occurrences have the same signatures in order + // For a reordered statement: how often each signature occurs in the + // source, and how often and (when once) where in the rendered statement. + inSource map[string]int + rendered map[string]int + at map[string]int +} + +func newOccMatcher(sigs []string, occs []wordOcc) *occMatcher { + m := &occMatcher{sigs: sigs} + if sigs == nil || len(sigs) != len(occs) { + return m + } + m.same = true + for i := range occs { + if occs[i].sig != sigs[i] { + m.same = false + break + } + } + if m.same { + return m + } + m.inSource = make(map[string]int, len(sigs)) + for _, s := range sigs { + m.inSource[s]++ + } + m.rendered = make(map[string]int, len(occs)) + m.at = make(map[string]int, len(occs)) + for _, o := range occs { + m.rendered[o.sig]++ + m.at[o.sig] = o.end + } + return m +} + +// covers reports whether m was built for exactly sigs (the same shared +// slice, as every hint on a keyword of one statement carries). +func (m *occMatcher) covers(sigs []string) bool { + if len(sigs) != len(m.sigs) { + return false + } + return len(sigs) == 0 || &sigs[0] == &m.sigs[0] +} + +// match returns the end offset of the rendered occurrence that h followed, +// for a hint whose sigs m covers; ok is false when it cannot be identified. +func (m *occMatcher) match(h placedHint, occs []wordOcc) (at int, ok bool) { + if h.sigs == nil { + return occs[h.ordinal].end, true + } + if len(h.sigs) != len(occs) { + return 0, false + } + if m.same { + return occs[h.ordinal].end, true + } + want := h.sigs[h.ordinal] + if m.inSource[want] != 1 || m.rendered[want] != 1 { + return 0, false + } + return m.at[want], true +} + +// leadingWordEnd returns the end offset of the first word of stmt when it is +// keyword. +func leadingWordEnd(stmt, keyword string) (int, bool) { + i := 0 + for i < len(stmt) && (stmt[i] == ' ' || stmt[i] == '\t' || stmt[i] == '\n' || stmt[i] == '\r') { + i++ + } + j := i + for j < len(stmt) && (stmt[j] >= 'A' && stmt[j] <= 'Z' || stmt[j] >= 'a' && stmt[j] <= 'z') { + j++ + } + return j, strings.EqualFold(stmt[i:j], keyword) +} + +// wordOcc is one occurrence of a hint keyword in a rendered statement: the +// byte offset just after it and its context signature (occurrenceSig). +type wordOcc struct { + end int + sig string +} + +// wordEnds uses the active dialect's lexer so quote, escape, and delimited +// identifier rules are identical to those used to parse the statement. +func wordEnds(stmt, dialect string) (map[string][]wordOcc, error) { + out := map[string][]wordOcc{} + tkz := tokenizer.GetTokenizer() + defer tokenizer.PutTokenizer(tkz) + if dialect != "" { + tkz.SetDialect(keywords.SQLDialect(dialect)) + } + tokens, err := tkz.TokenizeString(stmt) + if err != nil { + return nil, err + } + lineStart := []int{0} + for i := range stmt { + if stmt[i] == '\n' { + lineStart = append(lineStart, i+1) + } + } + for k, tok := range tokens { + w, ok := hintKeyword(tok.Token) + if !ok || tok.Start.Line < 1 || tok.Start.Line > len(lineStart) || + tok.End.Line < 1 || tok.End.Line > len(lineStart) { + continue + } + start := lineStart[tok.Start.Line-1] + tok.Start.Column - 1 + end := lineStart[tok.End.Line-1] + tok.End.Column - 1 + if start < 0 || end > len(stmt) || start >= end { + continue + } + if start > 0 && (stmt[start-1] == '.' || stmt[start-1] == '@' || stmt[start-1] == ':') { + continue + } + out[w] = append(out[w], wordOcc{end: end, sig: occurrenceSig(tokens, k)}) + } + return out, nil +} + +func abbreviate(s string) string { + if len(s) > 40 { + return s[:37] + "..." + } + return s +} + +// BatchPositions returns the source positions of the GO separators of tree +// for CommentLayout.Batches. The result is non-nil, also for a tree without +// separators, so that the comment layer follows the parser's boundaries. +func BatchPositions(tree *ast.AST) []models.Location { + out := []models.Location{} + if tree == nil { + return out + } + for _, stmt := range tree.Statements { + if sep, ok := stmt.(*ast.BatchSeparator); ok && sep != nil { + out = append(out, sep.Pos) + } + } + return out +} + +// StatementPositions returns the source start position of every statement of +// tree (ast.AST.StatementStarts) for CommentLayout.Starts, or nil when the +// tree does not carry one per statement (it was built by hand or from tokens +// without positions), in which case the comment layer splits at ';'. +func StatementPositions(tree *ast.AST) []models.Location { + if tree == nil || len(tree.StatementStarts) != len(tree.Statements) { + return nil + } + return tree.StatementStarts +} diff --git a/pkg/formatter/comments_test.go b/pkg/formatter/comments_test.go new file mode 100644 index 00000000..19b5ef1e --- /dev/null +++ b/pkg/formatter/comments_test.go @@ -0,0 +1,480 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package formatter + +import ( + "errors" + "sort" + "strings" + "testing" + "time" + + "github.com/ajitpratap0/GoSQLX/pkg/models" + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" + "github.com/ajitpratap0/GoSQLX/pkg/sql/parser" + "github.com/ajitpratap0/GoSQLX/pkg/sql/tokenizer" +) + +// renderJoin formats every statement of sql compactly and re-attaches its +// comments with JoinWithComments. +func renderJoin(t *testing.T, sql string, layout CommentLayout) (string, error) { + t.Helper() + tkz := tokenizer.GetTokenizer() + defer tokenizer.PutTokenizer(tkz) + tokens, err := tkz.TokenizeString(sql) + if err != nil { + t.Fatalf("tokenize %q: %v", sql, err) + } + p := parser.GetParser() + defer parser.PutParser(p) + tree, err := p.ParseFromModelTokens(tokens) + if err != nil { + t.Fatalf("parse %q: %v", sql, err) + } + defer ast.ReleaseAST(tree) + + style := ast.CompactStyle() + style.AddSemicolon = false + stmts := make([]string, len(tree.Statements)) + for i, s := range tree.Statements { + stmts[i] = FormatStatement(s, style) + } + return JoinWithComments(stmts, tokens, tkz.Comments, layout) +} + +func TestJoinWithComments(t *testing.T) { + layout := CommentLayout{Terminator: ";", Gap: "\n"} + tests := []struct { + name string + sql string + want string + }{ + {"no comments", "select a from t; select b from u", "SELECT a FROM t;\nSELECT b FROM u"}, + {"leading line comment", "-- who\nselect a from t", "-- who\nSELECT a FROM t"}, + {"trailing line comment", "select a from t -- why", "SELECT a FROM t -- why"}, + {"trailing comment after semicolon stays with its statement", + "select a from t; -- one\nselect b from u", "SELECT a FROM t; -- one\nSELECT b FROM u"}, + {"comment between statements goes before the next", + "select a from t;\n-- two\nselect b from u", "SELECT a FROM t;\n-- two\nSELECT b FROM u"}, + {"comment after the last statement on its own line", + "select a from t;\n-- end", "SELECT a FROM t\n-- end"}, + {"inner comment moves above its statement", + "select a, /* b is new */ b from t", "/* b is new */\nSELECT a, b FROM t"}, + {"inner line comment moves above its statement", + "select a, -- first\n b from t", "-- first\nSELECT a, b FROM t"}, + {"hint after first keyword stays", "select /*+ FULL(t) */ a from t", "SELECT /*+ FULL(t) */ a FROM t"}, + {"hint plus comment", "select /*+ FULL(t) */ a from t -- note", "SELECT /*+ FULL(t) */ a FROM t -- note"}, + {"hint on insert", "insert /*+ APPEND */ into t (a) values (1)", "INSERT /*+ APPEND */ INTO t (a) VALUES (1)"}, + {"hint in subquery", "select * from (select /*+ x */ 1) s", "SELECT * FROM (SELECT /*+ x */ 1) s"}, + {"hint in union branch", "select 1 union select /*+ x */ 2", "SELECT 1 union SELECT /*+ x */ 2"}, + {"hint inside a CTE body", + "with c as (select /*+ MATERIALIZE */ * from big where f = 'select') select count(*) from c", + "WITH c AS (SELECT /*+ MATERIALIZE */ * FROM big WHERE f = 'select') SELECT count(*) FROM c"}, + {"hints on the CTE and the main query", + "with c as (select /*+ a */ 1) select /*+ b */ * from c", + "WITH c AS (SELECT /*+ a */ 1) SELECT /*+ b */ * FROM c"}, + {"two hints on one keyword keep their order", "select /*+ a */ /*+ b */ 1", "SELECT /*+ a */ /*+ b */ 1"}, + {"hint on second statement", "select 1; select /*+ x */ 2", "SELECT 1;\nSELECT /*+ x */ 2"}, + {"executable comment statement kept with its semicolon", + "/*!40101 SET x = 1 */;\nselect 1", "/*!40101 SET x = 1 */;\nSELECT 1"}, + {"comment-only statement between statements", + "select 1;\n/*!40101 SET x = 1 */;\nselect 2", "SELECT 1;\n/*!40101 SET x = 1 */;\nSELECT 2"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := renderJoin(t, tt.sql, layout) + if err != nil { + t.Fatalf("JoinWithComments: %v", err) + } + if got != tt.want { + t.Fatalf("got\n%s\nwant\n%s", got, tt.want) + } + }) + } +} + +func TestJoinWithComments_RefusesWhatItCannotPlace(t *testing.T) { + layout := CommentLayout{Terminator: ";", Gap: "\n"} + for name, sql := range map[string]string{ + "hint after from": "select a from /*+ x */ t", + "executable in stmt": "select 1 /*!50000 + 1 */", + "executable before stmt": "/*!40101 SET x = 1 */ select 1", + "oracle line hint": "select --+ FULL(t)\n a from t", + } { + t.Run(name, func(t *testing.T) { + got, err := renderJoin(t, sql, layout) + if !errors.Is(err, ErrUnplaceableComment) { + t.Fatalf("err = %v (output %q), want ErrUnplaceableComment", err, got) + } + }) + } +} + +func TestJoinWithComments_NeverLosesAComment(t *testing.T) { + inputs := []string{ + "-- a\nselect 1 -- b\n; /* c */ select 2 /* d */ -- e\n-- f", + "select /* x */ 1, /* y */ 2 from /* z */ t where a = 1 -- w", + ";;-- only\n;select 1", + "select 1;\n\n\n/* t1 */\n/* t2 */", + } + for _, sql := range inputs { + for _, layout := range []CommentLayout{ + {Terminator: ";", Gap: "\n"}, + {Terminator: ";", Gap: "\n\n"}, + {Terminator: ";", Gap: " "}, + } { + got, err := renderJoin(t, sql, layout) + if err != nil { + t.Fatalf("%q: %v", sql, err) + } + if a, b := commentTexts(t, sql), commentTexts(t, got); strings.Join(a, "|") != strings.Join(b, "|") { + t.Errorf("comments differ for %q (layout %+v)\n in: %q\n out: %q\n got: %q", sql, layout, a, b, got) + } + // A line comment must never swallow SQL that follows it. + if _, err := reparseCount(t, got); err != nil { + t.Errorf("output %q does not parse: %v", got, err) + } + } + } +} + +func commentTexts(t *testing.T, sql string) []string { + t.Helper() + tkz := tokenizer.GetTokenizer() + defer tokenizer.PutTokenizer(tkz) + if _, err := tkz.TokenizeString(sql); err != nil { + t.Fatalf("tokenize %q: %v", sql, err) + } + out := make([]string, 0, len(tkz.Comments)) + for _, c := range tkz.Comments { + out = append(out, c.Text) + } + sort.Strings(out) + return out +} + +func reparseCount(t *testing.T, sql string) (int, error) { + t.Helper() + tkz := tokenizer.GetTokenizer() + defer tokenizer.PutTokenizer(tkz) + tokens, err := tkz.TokenizeString(sql) + if err != nil { + return 0, err + } + p := parser.GetParser() + defer parser.PutParser(p) + tree, err := p.ParseFromModelTokens(tokens) + if err != nil { + return 0, err + } + defer ast.ReleaseAST(tree) + return len(tree.Statements), nil +} + +func TestJoinWithComments_Fallback(t *testing.T) { + // A statement count that does not match the semicolon-separated runs of + // tokens (here: the caller passes one fewer rendered statement) puts the + // comments in front of the first statement instead of losing them. + tkz := tokenizer.GetTokenizer() + defer tokenizer.PutTokenizer(tkz) + tokens, err := tkz.TokenizeString("select 1; -- two\nselect 2") + if err != nil { + t.Fatal(err) + } + got, err := JoinWithComments([]string{"SELECT 1"}, tokens, tkz.Comments, CommentLayout{Terminator: ";", Gap: "\n"}) + if err != nil { + t.Fatal(err) + } + if want := "-- two\nSELECT 1"; got != want { + t.Fatalf("got %q, want %q", got, want) + } +} + +func TestJoinWithComments_NoComments(t *testing.T) { + got, err := JoinWithComments([]string{"A", "B", "C"}, nil, nil, CommentLayout{Terminator: ";", Gap: " "}) + if err != nil || got != "A; B; C" { + t.Fatalf("got %q, %v", got, err) + } + got, _ = JoinWithComments([]string{"A;", "B;"}, nil, nil, CommentLayout{Gap: "\n"}) + if got != "A;\nB;" { + t.Fatalf("got %q", got) + } +} + +// TestInsertHints_RefusesWhenKeywordMoved checks that a hint on a nested +// keyword is only placed when the rendered statement has the same number of +// that keyword, so it can never land on a different one. +func TestInsertHints_RefusesWhenKeywordMoved(t *testing.T) { + h := []placedHint{{text: "/*+ x */", keyword: "SELECT", ordinal: 1, total: 2}} + tests := []struct { + name string + stmt string + want string + ok bool + }{ + {"same count", "WITH c AS (SELECT 1) SELECT 2", "WITH c AS (SELECT 1) SELECT /*+ x */ 2", true}, + {"words in strings and comments are skipped", "WITH c AS (SELECT 'select') /* select */ SELECT 2", "WITH c AS (SELECT 'select') /* select */ SELECT /*+ x */ 2", true}, + {"quoted names are skipped", `WITH "select" AS (SELECT 1) SELECT 2`, `WITH "select" AS (SELECT 1) SELECT /*+ x */ 2`, true}, + {"fewer keywords", "SELECT 2", "", false}, + {"more keywords", "SELECT 1 UNION SELECT 2 UNION SELECT 3", "", false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := insertHints(tc.stmt, h) + if !tc.ok { + if !errors.Is(err, ErrUnplaceableComment) { + t.Fatalf("err = %v (output %q), want ErrUnplaceableComment", err, got) + } + return + } + if err != nil || got != tc.want { + t.Fatalf("insertHints = %q, %v; want %q", got, err, tc.want) + } + }) + } +} + +// TestOccMatcher checks how a nested hint finds its keyword in the +// rendered statement: by ordinal when the occurrences keep their order, by +// signature when the renderer reordered them, and not at all when the +// occurrences cannot be told apart or were rendered differently. +func TestOccMatcher(t *testing.T) { + occ := func(sigs ...string) []wordOcc { + out := make([]wordOcc, len(sigs)) + for i, s := range sigs { + out[i] = wordOcc{end: (i + 1) * 10, sig: s} + } + return out + } + tests := []struct { + name string + sigs []string + ordinal int + occs []wordOcc + want int + ok bool + }{ + {"same order", []string{"a", "b", "c"}, 1, occ("a", "b", "c"), 20, true}, + {"reordered, unique", []string{"a", "b", "c"}, 1, occ("a", "c", "b"), 30, true}, + {"reordered, duplicate signature", []string{"a", "b", "b", "c"}, 1, occ("a", "c", "b", "b"), 0, false}, + {"hinted occurrence rendered differently", []string{"a", "b", "c"}, 1, occ("a", "x", "c"), 0, false}, + {"other occurrence rendered differently", []string{"a", "b", "c"}, 1, occ("x", "c", "b"), 30, true}, + {"duplicate signature in the output", []string{"a", "b", "c"}, 1, occ("b", "c", "b"), 0, false}, + {"unchecked", nil, 1, occ("a", "x"), 20, true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + h := placedHint{text: "/*+ h */", keyword: "SELECT", ordinal: tc.ordinal, total: len(tc.occs), sigs: tc.sigs} + got, ok := newOccMatcher(h.sigs, tc.occs).match(h, tc.occs) + if ok != tc.ok || (ok && got != tc.want) { + t.Fatalf("match = %d, %v; want %d, %v", got, ok, tc.want, tc.ok) + } + }) + } +} + +// TestJoinWithComments_FollowsParserBatches checks that the comment layer +// splits the source at the GO separators the parser reported in +// CommentLayout.Batches, and falls back to putting comments in front when +// the rendered statements and the source batches do not line up. +func TestJoinWithComments_FollowsParserBatches(t *testing.T) { + const src = "SELECT 1 -- one\nGO\n-- two\nSELECT 2" + tkz := tokenizer.GetTokenizer() + defer tokenizer.PutTokenizer(tkz) + tkz.SetDialect(keywords.DialectSQLServer) + tokens, err := tkz.TokenizeString(src) + if err != nil { + t.Fatal(err) + } + comments := tkz.Comments + var goPos models.Location + for _, tok := range tokens { + if tok.Token.Value == "GO" { + goPos = tok.Start + } + } + tests := []struct { + name string + stmts []string + batches []models.Location + want string + }{ + {"parser positions", []string{"SELECT 1", "GO", "SELECT 2"}, []models.Location{goPos}, "SELECT 1;\n-- one\nGO\n-- two\nSELECT 2"}, + {"recognized from tokens", []string{"SELECT 1", "GO", "SELECT 2"}, nil, "SELECT 1;\n-- one\nGO\n-- two\nSELECT 2"}, + {"parser saw no separator", []string{"SELECT 1", "GO", "SELECT 2"}, []models.Location{}, "-- one\n-- two\nSELECT 1;\nGO\nSELECT 2"}, + {"rendered kinds differ", []string{"SELECT 1", "SELECT 9", "SELECT 2"}, []models.Location{goPos}, "-- one\n-- two\nSELECT 1;\nSELECT 9;\nSELECT 2"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + layout := CommentLayout{Dialect: "sqlserver", Terminator: ";", Gap: "\n", Batches: tc.batches} + got, err := JoinWithComments(tc.stmts, tokens, comments, layout) + if err != nil { + t.Fatal(err) + } + if got != tc.want { + t.Errorf("got %q\nwant %q", got, tc.want) + } + }) + } +} + +// renderJoinDialect is renderJoin in a dialect, with the parser's statement +// starts and GO positions in the layout, as the formatting entry points pass +// them. +func renderJoinDialect(t *testing.T, sql string, dialect keywords.SQLDialect, layout CommentLayout) (string, error) { + t.Helper() + tkz := tokenizer.GetTokenizer() + defer tokenizer.PutTokenizer(tkz) + tkz.SetDialect(dialect) + tokens, err := tkz.TokenizeString(sql) + if err != nil { + t.Fatalf("tokenize %q: %v", sql, err) + } + p := parser.GetParser() + defer parser.PutParser(p) + p.ApplyOptions(parser.WithDialect(string(dialect))) + tree, err := p.ParseFromModelTokens(tokens) + if err != nil { + t.Fatalf("parse %q: %v", sql, err) + } + defer ast.ReleaseAST(tree) + + style := ast.CompactStyle() + style.AddSemicolon = false + style.Dialect = string(dialect) + stmts := make([]string, len(tree.Statements)) + for i, s := range tree.Statements { + stmts[i] = FormatStatement(s, style) + } + layout.Dialect = string(dialect) + layout.Batches = BatchPositions(tree) + layout.Starts = StatementPositions(tree) + return JoinWithComments(stmts, tokens, tkz.Comments, layout) +} + +// TestJoinWithComments_FollowsParserStatementStarts checks that comments stay +// with their statements when a statement holds ';' (a procedure or trigger +// body) or statements are not separated by ';' (T-SQL): the source is cut at +// the parser's statement starts, not at every ';'. +func TestJoinWithComments_FollowsParserStatementStarts(t *testing.T) { + layout := CommentLayout{Terminator: ";", Gap: "\n"} + tests := []struct { + name string + dialect keywords.SQLDialect + sql string + want string + }{ + {"t-sql without ';' before a procedure body", keywords.DialectSQLServer, + "-- A\nSELECT 1\nSELECT 2;\n-- B: the proc\nCREATE PROC p AS BEGIN SELECT 1; END;\n", + "-- A\nSELECT 1;\nSELECT 2;\n-- B: the proc\nCREATE PROC p AS BEGIN SELECT 1; END"}, + {"trigger body", keywords.DialectMySQL, + "-- Section one\nSELECT 1 FROM users;\n-- Section two: trigger\nCREATE TRIGGER t AFTER INSERT ON x FOR EACH ROW BEGIN SELECT 1; SELECT 2; END;\n-- Section three\nSELECT 3 FROM orders;\n", + "-- Section one\nSELECT 1 FROM users;\n-- Section two: trigger\nCREATE TRIGGER t AFTER INSERT ON x FOR EACH ROW BEGIN SELECT 1; SELECT 2; END;\n-- Section three\nSELECT 3 FROM orders"}, + {"comment inside a body moves above its statement", keywords.DialectMySQL, + "SELECT 0;\nCREATE TRIGGER t AFTER INSERT ON x FOR EACH ROW BEGIN SELECT 1; -- inner\nSELECT 2; END; -- after\nSELECT 3", + "SELECT 0;\n-- inner\nCREATE TRIGGER t AFTER INSERT ON x FOR EACH ROW BEGIN SELECT 1; SELECT 2; END; -- after\nSELECT 3"}, + // As without starts: a comment after a stray ';' leads the next + // statement. + {"stray semicolons stay empty statements", keywords.DialectPostgreSQL, + "SELECT 1;; -- one\n;\n-- two\nSELECT 2", + "SELECT 1;\n-- one\n-- two\nSELECT 2"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := renderJoinDialect(t, tc.sql, tc.dialect, layout) + if err != nil { + t.Fatal(err) + } + if got != tc.want { + t.Errorf("got %q\nwant %q", got, tc.want) + } + }) + } +} + +// TestJoinWithComments_StartsDisagree checks that statement starts that do +// not line up with the source fall back to hoisting every comment instead of +// attaching one to the wrong statement. +func TestJoinWithComments_StartsDisagree(t *testing.T) { + tkz := tokenizer.GetTokenizer() + defer tokenizer.PutTokenizer(tkz) + tokens, err := tkz.TokenizeString("select 1; -- one\nselect 2;\n-- two\nselect 3") + if err != nil { + t.Fatal(err) + } + var starts []models.Location + for _, tok := range tokens { + if strings.EqualFold(tok.Token.Value, "select") { + starts = append(starts, tok.Start) + } + } + stmts := []string{"SELECT 1", "SELECT 2", "SELECT 3"} + tests := []struct { + name string + starts []models.Location + want string + }{ + {"agree", starts, "SELECT 1; -- one\nSELECT 2;\n-- two\nSELECT 3"}, + {"one missing", starts[:2], "-- one\n-- two\nSELECT 1;\nSELECT 2;\nSELECT 3"}, + {"shifted", []models.Location{starts[0], {Line: starts[1].Line, Column: starts[1].Column + 1}, starts[2]}, "-- one\n-- two\nSELECT 1;\nSELECT 2;\nSELECT 3"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := JoinWithComments(stmts, tokens, tkz.Comments, CommentLayout{Terminator: ";", Gap: "\n", Starts: tc.starts}) + if err != nil { + t.Fatal(err) + } + if got != tc.want { + t.Errorf("got %q\nwant %q", got, tc.want) + } + }) + } +} + +// TestJoinWithComments_ScalesLinearly guards against the quadratic hint +// placement and comment lookup: thousands of hints in one statement and +// thousands of commented statements must format quickly. The bounds are +// generous so that the race detector and slow CI machines pass; the +// quadratic versions took seconds. +func TestJoinWithComments_ScalesLinearly(t *testing.T) { + if testing.Short() { + t.Skip("timing test") + } + tests := []struct { + name string + sql string + hint int // hints expected in the output + }{ + {"hints", strings.Repeat("SELECT /*+ H */ 1 UNION ALL ", 3000) + "SELECT /*+ H */ 1", 3001}, + {"comments", strings.Repeat("SELECT 1; -- c\n", 20000), 0}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + start := time.Now() + got, err := renderJoinDialect(t, tc.sql, keywords.DialectPostgreSQL, CommentLayout{Terminator: ";", Gap: "\n"}) + elapsed := time.Since(start) + if err != nil { + t.Fatal(err) + } + if n := strings.Count(got, "SELECT /*+ H */ 1"); n != tc.hint { + t.Errorf("%d hints in place, want %d", n, tc.hint) + } + if limit := 3 * time.Second; elapsed > limit { + t.Errorf("took %v, want under %v", elapsed, limit) + } + }) + } +} diff --git a/pkg/formatter/formatter.go b/pkg/formatter/formatter.go index eb5da4c5..1b8dc15c 100644 --- a/pkg/formatter/formatter.go +++ b/pkg/formatter/formatter.go @@ -95,6 +95,7 @@ import ( "github.com/ajitpratap0/GoSQLX/pkg/models" "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" "github.com/ajitpratap0/GoSQLX/pkg/sql/parser" "github.com/ajitpratap0/GoSQLX/pkg/sql/tokenizer" ) @@ -126,8 +127,17 @@ func (f *Formatter) Format(sql string) (string, error) { return "", nil } + // The dialect selects the tokenizer and parser rules (string escapes, + // quoting, dialect-only syntax) as well as the rendering. An unknown + // dialect name is ignored for tokenizing and parsing, as before. + dialect := keywords.SQLDialect(strings.ToLower(f.opts.Dialect)) + useDialect := dialect != "" && keywords.IsValidDialect(string(dialect)) + tkz := tokenizer.GetTokenizer() defer tokenizer.PutTokenizer(tkz) + if useDialect { + tkz.SetDialect(dialect) + } tokens, err := tkz.TokenizeString(sql) if err != nil { @@ -143,6 +153,9 @@ func (f *Formatter) Format(sql string) (string, error) { p := parser.GetParser() defer parser.PutParser(p) + if useDialect { + p.ApplyOptions(parser.WithDialect(string(dialect))) + } parsedAST, err := p.ParseFromModelTokens(tokens) if err != nil { return "", fmt.Errorf("parsing failed: %w", err) diff --git a/pkg/formatter/formatter_dialect_test.go b/pkg/formatter/formatter_dialect_test.go new file mode 100644 index 00000000..95dc3b25 --- /dev/null +++ b/pkg/formatter/formatter_dialect_test.go @@ -0,0 +1,51 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package formatter_test + +import ( + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/formatter" +) + +// TestFormatter_Format_UsesDialect checks that Formatter.Format tokenizes and +// parses with Options.Dialect, not only renders with it: strings and syntax +// that are valid in the dialect must not fail in the no-dialect front end. +func TestFormatter_Format_UsesDialect(t *testing.T) { + tests := []struct { + name string + dialect string + in string + want string + }{ + {"postgres backslash literal", "postgresql", `SELECT 'C:\dir'`, `SELECT 'C:\dir'`}, + {"mysql escaped quote", "mysql", `SELECT 'it\'s'`, `SELECT 'it''s'`}, + {"mysql dialect name case", "MySQL", `SELECT 'it\'s'`, `SELECT 'it''s'`}, + {"clickhouse engine", "clickhouse", `CREATE TABLE t (a Int32) ENGINE = MergeTree ORDER BY a`, `CREATE TABLE t (a Int32) ENGINE=MergeTree ORDER BY a`}, + {"unknown dialect ignored", "nosuchdb", `SELECT a FROM t`, `SELECT a FROM t`}, + {"no dialect", "", `SELECT a FROM t`, `SELECT a FROM t`}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := formatter.New(formatter.Options{Compact: true, Dialect: tc.dialect}).Format(tc.in) + if err != nil { + t.Fatalf("Format(%q) with dialect %q: %v", tc.in, tc.dialect, err) + } + if got != tc.want { + t.Errorf("Format(%q) with dialect %q\n got: %s\nwant: %s", tc.in, tc.dialect, got, tc.want) + } + }) + } +} diff --git a/pkg/formatter/join_group_render_test.go b/pkg/formatter/join_group_render_test.go new file mode 100644 index 00000000..d8e62ab5 --- /dev/null +++ b/pkg/formatter/join_group_render_test.go @@ -0,0 +1,64 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package formatter + +import ( + "strings" + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" + "github.com/ajitpratap0/GoSQLX/pkg/sql/parser" +) + +// TestJoinGroupRender checks that a parenthesized join group used as a join +// operand is rendered whole, in compact and readable layouts and with +// lowercase keywords, and that the output parses back to the same text. +func TestJoinGroupRender(t *testing.T) { + const in = "SELECT * FROM a LEFT JOIN (b JOIN c ON b.id = c.b_id, d) ON a.id = b.a_id WHERE a.x = 1" + tree, err := parser.ParseWithDialect(in, keywords.DialectMySQL) + if err != nil { + t.Fatal(err) + } + defer ast.ReleaseAST(tree) + + styles := map[string]ast.FormatOptions{ + "compact": ast.CompactStyle(), + "readable": ast.ReadableStyle(), + } + lower := ast.CompactStyle() + lower.KeywordCase = ast.KeywordLower + styles["lower"] = lower + + for name, opts := range styles { + opts.Dialect = "mysql" + out := FormatAST(tree, opts) + if !strings.Contains(strings.ToUpper(out), "(B INNER JOIN C ON B.ID = C.B_ID, D)") && + !strings.Contains(strings.ToUpper(out), "(B INNER JOIN C ON B.ID = C.B_ID , D)") { + t.Errorf("%s: group not rendered whole: %s", name, out) + } + tree2, err := parser.ParseWithDialect(out, keywords.DialectMySQL) + if err != nil { + t.Errorf("%s: output %q does not re-parse: %v", name, out, err) + continue + } + compact := ast.CompactStyle() + compact.Dialect = "mysql" + if got, want := FormatAST(tree2, compact), FormatAST(tree, compact); got != want { + t.Errorf("%s: output re-parses to %q, want %q", name, got, want) + } + ast.ReleaseAST(tree2) + } +} diff --git a/pkg/formatter/name_quote.go b/pkg/formatter/name_quote.go new file mode 100644 index 00000000..40c28e8a --- /dev/null +++ b/pkg/formatter/name_quote.go @@ -0,0 +1,74 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package formatter + +import ( + "strings" + + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" +) + +// dialectDelimiter returns the identifier delimiter a dialect requires +// instead of ANSI double quotes, or 0 when double quotes are fine. MySQL and +// MariaDB read "..." as a string literal unless ANSI_QUOTES is set, so a name +// they need delimited must use backticks. +func dialectDelimiter(dialect string) rune { + switch strings.ToLower(dialect) { + case "mysql", "mariadb": + return '`' + } + return 0 +} + +// nq returns q with the target dialect's delimiter as its default style, so +// a name that needs delimiters only because of its content is written with +// the delimiters the dialect accepts (see dialectDelimiter). A style recorded +// from the source is kept. +func (f *nodeFormatter) nq(q ast.NameQuote) ast.NameQuote { + if d := dialectDelimiter(f.opts.Dialect); d != 0 { + return q.WithDefaultStyle(d) + } + return q +} + +// quoteNamesWith renders a CTE or view column list like quoteNames, +// honouring each name's recorded delimiters (qs by index, may be nil). +func (f *nodeFormatter) quoteNamesWith(names []string, qs []ast.NameQuote) string { + return joinQuoted(names, qs, ast.QuoteNameWith, f.nq) +} + +// quoteColumnNamesWith joins single-part DDL column names like +// quoteColumnNames, honouring each name's recorded delimiters (qs by index, +// may be nil) and the target dialect's delimiter. +func (f *nodeFormatter) quoteColumnNamesWith(names []string, qs []ast.NameQuote) string { + return joinQuoted(names, qs, ast.QuoteColumnNameWith, f.nq) +} + +// joinQuoted joins names with ", ", each rendered by quote with its +// NameQuote from qs (by index; may be nil), adjusted by nq when not nil. +func joinQuoted(names []string, qs []ast.NameQuote, quote func(string, ast.NameQuote) string, nq func(ast.NameQuote) ast.NameQuote) string { + parts := make([]string, len(names)) + for i, n := range names { + var q ast.NameQuote + if i < len(qs) { + q = qs[i] + } + if nq != nil { + q = nq(q) + } + parts[i] = quote(n, q) + } + return strings.Join(parts, ", ") +} diff --git a/pkg/formatter/name_quote_test.go b/pkg/formatter/name_quote_test.go new file mode 100644 index 00000000..3df21765 --- /dev/null +++ b/pkg/formatter/name_quote_test.go @@ -0,0 +1,134 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package formatter + +import ( + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" + "github.com/ajitpratap0/GoSQLX/pkg/sql/parser" +) + +// TestNameQuote_FormatKeepsCaseQuotes checks that FormatStatement keeps the +// quotes a name needs for its letter case where the dialect folds unquoted +// names, in the clauses the formatter renders itself (FROM and JOIN tables +// and aliases, CTE names and columns, DML targets, CREATE TABLE / VIEW), and +// drops them where the dialect does not fold. +func TestNameQuote_FormatKeepsCaseQuotes(t *testing.T) { + tests := []struct { + name string + dialect keywords.SQLDialect + in string + want string + }{ + {"pg select", keywords.DialectPostgreSQL, + `SELECT "C"."name" AS "CName" FROM "companies" AS "C" JOIN "Users" "U" ON "U".cid = "C".id`, + `SELECT "C".name AS "CName" FROM companies "C" INNER JOIN "Users" "U" ON "U".cid = "C".id`}, + {"pg window names", keywords.DialectPostgreSQL, + `SELECT SUM(x) OVER "W" FROM t WINDOW "W" AS (ORDER BY x), "W2" AS ("W")`, + `SELECT SUM(x) OVER "W" FROM t WINDOW "W" AS (ORDER BY x), "W2" AS ("W")`}, + {"pg cte", keywords.DialectPostgreSQL, + `WITH "Cte"("Total") AS (SELECT 1) SELECT "Total" FROM "Cte"`, + `WITH "Cte" ("Total") AS (SELECT 1) SELECT "Total" FROM "Cte"`}, + {"pg update", keywords.DialectPostgreSQL, + `UPDATE "MyTable" AS "T" SET "ColA" = 1 WHERE "T"."ColB" = 2`, + `UPDATE "MyTable" "T" SET "ColA" = 1 WHERE "T"."ColB" = 2`}, + {"pg delete", keywords.DialectPostgreSQL, `DELETE FROM "MyTable" WHERE "Id" = 1`, `DELETE FROM "MyTable" WHERE "Id" = 1`}, + {"pg insert", keywords.DialectPostgreSQL, `INSERT INTO "MyTable" ("ColA") VALUES (1)`, `INSERT INTO "MyTable" ("ColA") VALUES (1)`}, + {"pg merge", keywords.DialectPostgreSQL, + `MERGE INTO t AS "Tgt" USING s AS "Src" ON "Tgt".id = "Src".id WHEN MATCHED THEN UPDATE SET "Val" = 1 WHEN NOT MATCHED THEN INSERT ("Id") VALUES (1)`, + `MERGE INTO t "Tgt" USING s "Src" ON "Tgt".id = "Src".id WHEN MATCHED THEN UPDATE SET "Val" = 1 WHEN NOT MATCHED THEN INSERT ("Id") VALUES (1)`}, + {"pg create table", keywords.DialectPostgreSQL, `CREATE TABLE "MyTable" ("ColA" INT)`, `CREATE TABLE "MyTable" ("ColA" INT)`}, + {"pg create view", keywords.DialectPostgreSQL, `CREATE VIEW "V" ("A") AS SELECT 1`, `CREATE VIEW "V" ("A") AS SELECT 1`}, + {"pg create index", keywords.DialectPostgreSQL, `CREATE INDEX "Idx" ON "T" ("Col", b)`, `CREATE INDEX "Idx" ON "T" ("Col", b)`}, + {"pg drop", keywords.DialectPostgreSQL, `DROP TABLE "Post", other`, `DROP TABLE "Post", other`}, + {"pg alter add constraint", keywords.DialectPostgreSQL, + `ALTER TABLE "Post" ADD CONSTRAINT "Post_fk" FOREIGN KEY ("authorId") REFERENCES "User"("Id")`, + `ALTER TABLE "Post" ADD CONSTRAINT "Post_fk" FOREIGN KEY ("authorId") REFERENCES "User" ("Id")`}, + {"pg alter rename column", keywords.DialectPostgreSQL, + `ALTER TABLE "Post" RENAME COLUMN "Old" TO "New"`, `ALTER TABLE "Post" RENAME COLUMN "Old" TO "New"`}, + {"pg alter rename table", keywords.DialectPostgreSQL, `ALTER TABLE "Post" RENAME TO "Posts"`, `ALTER TABLE "Post" RENAME TO "Posts"`}, + {"pg alter drop column and constraint", keywords.DialectPostgreSQL, + `ALTER TABLE t DROP COLUMN "Old"`, `ALTER TABLE t DROP COLUMN "Old"`}, + {"pg alter drop keyword constraint", keywords.DialectPostgreSQL, + `ALTER TABLE t DROP CONSTRAINT "key"`, `ALTER TABLE t DROP CONSTRAINT "key"`}, + // Names the ALTER renderer used to write verbatim now get quoted + // when their content needs it. + {"pg alter spaced names", keywords.DialectPostgreSQL, + `ALTER TABLE "my table" DROP COLUMN "a b"`, `ALTER TABLE "my table" DROP COLUMN "a b"`}, + {"mysql alter backticks", keywords.DialectMySQL, + "ALTER TABLE `my table` RENAME COLUMN `a b` TO `select`", "ALTER TABLE `my table` RENAME COLUMN `a b` TO `select`"}, + {"oracle lower case", keywords.DialectOracle, `SELECT "ID", "id" FROM "T" "t"`, `SELECT ID, "id" FROM T "t"`}, + {"mysql case not needed", keywords.DialectMySQL, "SELECT `Id` FROM `Users` AS `U`", "SELECT Id FROM Users U"}, + {"mysql reserved alias", keywords.DialectMySQL, "SELECT 1 AS `exists` FROM `order` AS `group`", "SELECT 1 AS `exists` FROM `order` `group`"}, + {"sqlserver reserved", keywords.DialectSQLServer, "SELECT [Group] FROM [order] [Key]", "SELECT [Group] FROM [order] [Key]"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + tree, err := parser.ParseWithDialect(tc.in, tc.dialect) + if err != nil { + t.Fatalf("parse %q: %v", tc.in, err) + } + defer ast.ReleaseAST(tree) + style := ast.CompactStyle() + style.Dialect = string(tc.dialect) + got := FormatStatement(tree.Statements[0], style) + if got != tc.want { + t.Fatalf("FormatStatement mismatch\n got: %s\nwant: %s", got, tc.want) + } + }) + } +} + +// TestNameQuote_FormatDialectDelimiter checks that a name needing delimiters +// because of its content, with no delimiter recorded (a hand-built AST), is +// written with backticks when the target dialect is MySQL or MariaDB, which +// read "..." as a string, and with ANSI double quotes otherwise. +func TestNameQuote_FormatDialectDelimiter(t *testing.T) { + stmt := &ast.SelectStatement{ + Columns: []ast.Expression{&ast.Identifier{Name: "a"}}, + From: []ast.TableReference{{Name: "my db.order", Alias: "select"}}, + } + tests := []struct { + dialect string + want string + }{ + {"mysql", "SELECT a FROM `my db`.`order` `select`"}, + {"MariaDB", "SELECT a FROM `my db`.`order` `select`"}, + {"postgresql", `SELECT a FROM "my db"."order" "select"`}, + {"", `SELECT a FROM "my db"."order" "select"`}, + } + for _, tc := range tests { + style := ast.CompactStyle() + style.Dialect = tc.dialect + if got := FormatStatement(stmt, style); got != tc.want { + t.Errorf("dialect %q: got %s, want %s", tc.dialect, got, tc.want) + } + } +} + +// TestNameQuote_FormatNoAllocForPlainNames checks that the dialect-aware name +// quoting does not allocate for names that need no delimiters. +func TestNameQuote_FormatNoAllocForPlainNames(t *testing.T) { + q := ast.NameQuoteOf('`') + if n := testing.AllocsPerRun(100, func() { + _ = ast.QuoteQualifiedNameWith("db.users", q) + _ = ast.QuoteTableAliasWith("u", q) + _ = ast.QuoteNameWith("total", q) + }); n != 0 { + t.Errorf("plain names allocate %v times, want 0", n) + } +} diff --git a/pkg/formatter/nested_semicolon_test.go b/pkg/formatter/nested_semicolon_test.go index bc06fd9f..ecc12333 100644 --- a/pkg/formatter/nested_semicolon_test.go +++ b/pkg/formatter/nested_semicolon_test.go @@ -127,12 +127,6 @@ func TestFormatAST_ReadableSemicolonOnlyTerminatesStatements(t *testing.T) { if werr != nil { return werr } - // The real-world corpus has its own ratchet (TestRealWorldCorpus) and - // still holds statements whose ';' placement is wrong; the dialect - // fixes that repair them remove this skip. - if d.IsDir() && d.Name() == "realworld" { - return filepath.SkipDir - } if d.IsDir() || !strings.HasSuffix(path, ".sql") { return nil } diff --git a/pkg/formatter/operators_c1_render_test.go b/pkg/formatter/operators_c1_render_test.go new file mode 100644 index 00000000..f381f5b4 --- /dev/null +++ b/pkg/formatter/operators_c1_render_test.go @@ -0,0 +1,83 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package formatter_test + +import ( + "strings" + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" +) + +// TestOperatorsC1_RenderRoundTrip formats the operator and predicate +// constructs through every renderer: each output must re-parse to the same +// statement and keep the operator words (nothing is dropped or reworded). +func TestOperatorsC1_RenderRoundTrip(t *testing.T) { + tests := []struct { + d keywords.SQLDialect + in string + want string // must appear in the compact rendering + }{ + {keywords.DialectPostgreSQL, "SELECT 1 FROM t WHERE id = ANY($1)", "id = ANY ($1)"}, + {keywords.DialectPostgreSQL, "SELECT 1 FROM t WHERE id <> ALL(ARRAY[1, 2])", "id <> ALL (ARRAY[1, 2])"}, + {keywords.DialectPostgreSQL, "SELECT 1 FROM t WHERE 'x' = SOME(tags)", "'x' = SOME (tags)"}, + {keywords.DialectPostgreSQL, "SELECT u.id FROM u WHERE u.id IS DISTINCT FROM 5", "u.id IS DISTINCT FROM 5"}, + {keywords.DialectSnowflake, "SELECT u.id FROM u WHERE u.id IS NOT DISTINCT FROM 5", "u.id IS NOT DISTINCT FROM 5"}, + {keywords.DialectPostgreSQL, "SELECT x IS NOT FALSE, y IS UNKNOWN FROM t", "x IS NOT FALSE, y IS UNKNOWN"}, + {keywords.DialectSQLite, "SELECT * FROM t WHERE a IS NOT b", "a IS NOT b"}, + {keywords.DialectPostgreSQL, "SELECT created_at AT TIME ZONE 'UTC' FROM t", "created_at AT TIME ZONE 'UTC'"}, + {keywords.DialectPostgreSQL, `SELECT col COLLATE "C" FROM t ORDER BY name COLLATE "en_US"`, `col COLLATE "C"`}, + {keywords.DialectMySQL, "SELECT x FROM t WHERE x COLLATE utf8mb4_bin = 'a'", "x COLLATE utf8mb4_bin = 'a'"}, + {keywords.DialectPostgreSQL, `SELECT a COLLATE "en_US.utf8", b COLLATE pg_catalog."C" FROM t`, `a COLLATE "en_US.utf8", b COLLATE pg_catalog."C"`}, + {keywords.DialectPostgreSQL, `CREATE INDEX i ON t (a COLLATE "C" DESC)`, `(a COLLATE "C" DESC)`}, + {keywords.DialectPostgreSQL, "SELECT x FROM t WHERE tsv @@ to_tsquery('a') AND a && b", "tsv @@ to_tsquery('a') AND a && b"}, + {keywords.DialectSQLServer, "SELECT a & b, ~c FROM t", "a & b, ~c"}, + {keywords.DialectMySQL, "SELECT a DIV 3, b MOD 2 FROM t WHERE x NOT REGEXP 'a' AND y SOUNDS LIKE z", "a DIV 3, b MOD 2"}, + {keywords.DialectMySQL, "SELECT * FROM t WHERE a <=> NULL", "a <=> NULL"}, + {keywords.DialectSQLite, "SELECT * FROM t WHERE p GLOB '*.txt' AND a MATCH 'foo' AND b == 1", "p GLOB '*.txt' AND a MATCH 'foo' AND b == 1"}, + {keywords.DialectPostgreSQL, "SELECT a FROM t WHERE n SIMILAR TO 'x%' AND m NOT SIMILAR TO 'y%'", "n SIMILAR TO 'x%' AND m NOT SIMILAR TO 'y%'"}, + {keywords.DialectOracle, "SELECT a FROM t WHERE a LIKE 'x' ESCAPE 'y'", "a LIKE 'x' ESCAPE 'y'"}, + {keywords.DialectSQLite, "SELECT a IN (1, 2) = 1 FROM t WHERE a NOT IN () AND b IN u", "a IN (1, 2) = 1"}, + {keywords.DialectClickHouse, "SELECT a FROM t WHERE a GLOBAL IN (SELECT 1) AND b GLOBAL NOT IN (SELECT 2)", "a GLOBAL IN (SELECT 1) AND b GLOBAL NOT IN (SELECT 2)"}, + {keywords.DialectClickHouse, "SELECT a ? 1 : 2", "a ? 1 : 2"}, + {keywords.DialectPostgreSQL, "SELECT x::int[], CAST(a AS text[]), x::double precision FROM t", "CAST(x AS int[]), CAST(a AS text[]), CAST(x AS double precision)"}, + {keywords.DialectMySQL, "SELECT CAST(x AS SIGNED INTEGER), CAST(y AS CHAR CHARACTER SET utf8mb4) FROM t", "CAST(x AS SIGNED INTEGER), CAST(y AS CHAR CHARACTER SET utf8mb4)"}, + } + for _, tc := range tests { + t.Run(string(tc.d)+"/"+tc.in, func(t *testing.T) { + rs := assertRoundTrip(t, tc.d, tc.in) + for _, name := range []string{"SQL", "FmtCompact"} { + if !strings.Contains(rs[name], tc.want) { + t.Errorf("%s = %q, want it to contain %q", name, rs[name], tc.want) + } + } + }) + } +} + +// TestOperatorsC1_LogicalOperatorsUpperCase pins that the formatter writes +// AND and OR in upper case whatever their source spelling (the CLI used to +// leave them as typed). +func TestOperatorsC1_LogicalOperatorsUpperCase(t *testing.T) { + rs := assertRoundTrip(t, keywords.DialectGeneric, "select a from t where a=1 and b=2 or c=3") + for name, out := range rs { + if strings.Contains(out, " and ") || strings.Contains(out, " or ") { + t.Errorf("%s = %q, want AND / OR in upper case", name, out) + } + if !strings.Contains(out, " AND ") || !strings.Contains(out, " OR ") { + t.Errorf("%s = %q, want it to contain AND and OR", name, out) + } + } +} diff --git a/pkg/formatter/quote_names.go b/pkg/formatter/quote_names.go index 31869a8e..634ce348 100644 --- a/pkg/formatter/quote_names.go +++ b/pkg/formatter/quote_names.go @@ -20,22 +20,6 @@ import ( "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" ) -// quoteNames renders a CTE or view column list as a comma-separated list, -// double-quoting each name that would not re-parse bare (see ast.QuoteName). -func quoteNames(names []string) string { - switch len(names) { - case 0: - return "" - case 1: - return ast.QuoteName(names[0]) - } - parts := make([]string, len(names)) - for i, n := range names { - parts[i] = ast.QuoteName(n) - } - return strings.Join(parts, ", ") -} - // quoteQualifiedNames joins table-like names with ", ", quoting each part // that needs it (see ast.QuoteQualifiedName). func quoteQualifiedNames(names []string) string { diff --git a/pkg/formatter/quoted_identifier_render_test.go b/pkg/formatter/quoted_identifier_render_test.go index 53e945f3..cf0651fc 100644 --- a/pkg/formatter/quoted_identifier_render_test.go +++ b/pkg/formatter/quoted_identifier_render_test.go @@ -42,8 +42,10 @@ func TestQuotedIdentifier_FormatAST_Issue611(t *testing.T) { "CREATE VIEW v (\"select\") AS\nSELECT 1"}, {"select_reserved", `SELECT "select" FROM t`, "", "SELECT \"select\"\nFROM t"}, - {"select_space_case_table_space", `SELECT "a b", "Mixed" FROM "my table"`, `SELECT "a b", Mixed FROM "my table"`, - "SELECT \"a b\", Mixed\nFROM \"my table\""}, + // Quotes kept for letter case depend on the dialect's folding; see + // TestNameQuote_FormatKeepsCaseQuotes. + {"select_space_table_space", `SELECT "a b", "c d" FROM "my table"`, "", + "SELECT \"a b\", \"c d\"\nFROM \"my table\""}, {"mview_column_reserved", `CREATE MATERIALIZED VIEW mv ("select") AS SELECT 1`, "", "CREATE MATERIALIZED VIEW mv (\"select\") AS\nSELECT 1"}, {"cte_name_reserved", `WITH "select" AS (SELECT 1) SELECT * FROM "select"`, "", diff --git a/pkg/formatter/render.go b/pkg/formatter/render.go index e87bff7f..95b3a4fb 100644 --- a/pkg/formatter/render.go +++ b/pkg/formatter/render.go @@ -83,15 +83,43 @@ func FormatAST(a *ast.AST, opts ast.FormatOptions) string { return "" } parts := make([]string, 0, len(a.Statements)) + hasBatch := false for _, stmt := range a.Statements { - parts = append(parts, FormatStatement(stmt, opts)) + part := FormatStatement(stmt, opts) + // Renderers that keep a statement as written (UtilityStatement and + // the stmtSQL fallback) do not add the ";" themselves; without it the + // next statement would be read back as part of their text. + _, isBatch := stmt.(*ast.BatchSeparator) + hasBatch = hasBatch || isBatch + if opts.AddSemicolon && !isBatch && part != "" && !strings.HasSuffix(part, ";") { + part += ";" + } + parts = append(parts, part) } sep := ";\n" if opts.AddSemicolon { - // Each statement already ends with ";" + // Each statement now ends with ";" sep = "\n" } - result := strings.Join(parts, sep) + var result string + if !hasBatch { + result = strings.Join(parts, sep) + } else { + var joined strings.Builder + for i, part := range parts { + if i > 0 { + _, prevBatch := a.Statements[i-1].(*ast.BatchSeparator) + _, curBatch := a.Statements[i].(*ast.BatchSeparator) + if prevBatch || curBatch { + joined.WriteByte('\n') + } else { + joined.WriteString(sep) + } + } + joined.WriteString(part) + } + result = joined.String() + } // Emit preserved comments if len(a.Comments) > 0 { @@ -166,6 +194,8 @@ func FormatStatement(s ast.Statement, opts ast.FormatOptions) string { return renderShow(v, opts) case *ast.DescribeStatement: return renderDescribe(v, opts) + case *ast.ExplainStatement: + return renderExplain(v, opts) case *ast.UnsupportedStatement: return renderUnsupported(v, opts) default: @@ -210,6 +240,9 @@ func renderSelect(s *ast.SelectStatement, opts ast.FormatOptions) string { f := newNodeFormatter(opts) sb := f.sb + // Most rendered SELECTs are under 128 bytes; one up-front allocation + // replaces the builder's 8-16-32-64-128 growth steps. + sb.Grow(128) if s.With != nil { sb.WriteString(renderWith(s.With, f)) @@ -229,24 +262,16 @@ func renderSelect(s *ast.SelectStatement, opts ast.FormatOptions) string { sb.WriteString(" ") } - // Render TOP clause (SQL Server). Emitted between SELECT [DISTINCT] and columns. - if s.Top != nil { - sb.WriteString(f.kw("TOP")) - sb.WriteString(" ") - sb.WriteString(FormatExpression(s.Top.Count, opts)) - if s.Top.IsPercent { - sb.WriteString(" ") - sb.WriteString(f.kw("PERCENT")) - } - if s.Top.WithTies { - sb.WriteString(" ") - sb.WriteString(f.kw("WITH TIES")) - } - sb.WriteString(" ") - } + // MySQL select modifiers and TOP, between SELECT [DISTINCT] and columns. + writeSelectPrologue(sb, s, f) sb.WriteString(exprListSQL(s.Columns)) + if into := selectExtras(s).Into; into != nil && !into.Trailing { + sb.WriteString(" ") + sb.WriteString(selectIntoSQL(into, f)) + } + if len(s.From) > 0 { sb.WriteString(f.clauseSep()) sb.WriteString(f.kw("FROM")) @@ -264,11 +289,23 @@ func renderSelect(s *ast.SelectStatement, opts ast.FormatOptions) string { sb.WriteString(joinSQL(&j, f)) } + if s.ArrayJoin != nil { + sb.WriteString(f.clauseSep()) + sb.WriteString(arrayJoinSQL(s.ArrayJoin, f)) + } + if s.Sample != nil { sb.WriteString(f.clauseSep()) sb.WriteString(sampleSQL(s.Sample, f)) } + if s.PrewhereClause != nil { + sb.WriteString(f.clauseSep()) + sb.WriteString(f.kw("PREWHERE")) + sb.WriteString(" ") + sb.WriteString(FormatExpression(s.PrewhereClause, opts)) + } + if s.Where != nil { sb.WriteString(f.clauseSep()) sb.WriteString(f.kw("WHERE")) @@ -276,11 +313,9 @@ func renderSelect(s *ast.SelectStatement, opts ast.FormatOptions) string { sb.WriteString(FormatExpression(s.Where, opts)) } - if len(s.GroupBy) > 0 { + if gb := groupBySQL(s, f); gb != "" { sb.WriteString(f.clauseSep()) - sb.WriteString(f.kw("GROUP BY")) - sb.WriteString(" ") - sb.WriteString(exprListSQL(s.GroupBy)) + sb.WriteString(gb) } if s.Having != nil { @@ -290,52 +325,52 @@ func renderSelect(s *ast.SelectStatement, opts ast.FormatOptions) string { sb.WriteString(FormatExpression(s.Having, opts)) } + // QUALIFY, START WITH, CONNECT BY + writeHierarchical(sb, s, f) + if len(s.Windows) > 0 { sb.WriteString(f.clauseSep()) sb.WriteString(f.kw("WINDOW")) sb.WriteString(" ") wins := make([]string, len(s.Windows)) for i := range s.Windows { - wins[i] = ast.QuoteName(s.Windows[i].Name) + " AS (" + windowSpecSQL(&s.Windows[i]) + ")" + wins[i] = ast.QuoteNameWith(s.Windows[i].Name, f.nq(s.Windows[i].NameQuote)) + " AS (" + windowSpecSQL(&s.Windows[i]) + ")" } sb.WriteString(strings.Join(wins, ", ")) } if len(s.OrderBy) > 0 { sb.WriteString(f.clauseSep()) - sb.WriteString(f.kw("ORDER BY")) + if s.DialectExtras().OrderSiblings { + sb.WriteString(f.kw("ORDER SIBLINGS BY")) + } else { + sb.WriteString(f.kw("ORDER BY")) + } sb.WriteString(" ") sb.WriteString(orderBySQL(s.OrderBy)) } - - if s.Limit != nil { - sb.WriteString(f.clauseSep()) - fmt.Fprintf(sb, "%s %d", f.kw("LIMIT"), *s.Limit) + if interp := selectExtras(s).Interpolate; interp != nil { + sb.WriteString(interpolateSQL(interp, f)) } - if s.Offset != nil { - sb.WriteString(f.clauseSep()) - fmt.Fprintf(sb, "%s %d", f.kw("OFFSET"), *s.Offset) - } + // LIMIT BY, LIMIT, OFFSET + writeLimitOffsetClauses(sb, s, f) if s.Fetch != nil { sb.WriteString(fetchSQL(s.Fetch, f)) } + if into := selectExtras(s).Into; into != nil && into.Trailing { + sb.WriteString(f.clauseSep()) + sb.WriteString(selectIntoSQL(into, f)) + } + if s.For != nil { sb.WriteString(forSQL(s.For)) } - if len(s.Settings) > 0 { - sb.WriteString(f.clauseSep()) - sb.WriteString(f.kw("SETTINGS")) - sb.WriteString(" ") - parts := make([]string, len(s.Settings)) - for i := range s.Settings { - parts[i] = s.Settings[i].SQL() - } - sb.WriteString(strings.Join(parts, ", ")) - } + // FOR JSON/XML, SETTINGS, OPTION (...), FORMAT + writeSelectEpilogue(sb, s, f) if opts.AddSemicolon { sb.WriteString(";") @@ -349,54 +384,20 @@ func renderSelect(s *ast.SelectStatement, opts ast.FormatOptions) string { // shallow copy of the statement. This keeps the rendering code simple — // each clause renderer only handles its own field. func normalizeSelectForDialect(s *ast.SelectStatement, dialect string) { - switch dialect { - case "sqlserver": - if s.Top == nil && s.Limit != nil { - if s.Offset != nil || len(s.OrderBy) > 0 { - // SQL Server 2012+ OFFSET/FETCH syntax (requires ORDER BY in practice, - // but we emit it faithfully and let the database validate). - fetchVal := int64(*s.Limit) - s.Fetch = &ast.FetchClause{ - FetchValue: &fetchVal, - FetchType: "NEXT", - } - offsetVal := int64(0) - if s.Offset != nil { - offsetVal = int64(*s.Offset) - } - s.Fetch.OffsetValue = &offsetVal - s.Limit = nil - s.Offset = nil - } else { - // Simple TOP N - s.Top = &ast.TopClause{ - Count: &ast.LiteralValue{Value: *s.Limit, Type: "int"}, - } - s.Limit = nil - } - } - - case "oracle": - if s.Fetch == nil && s.Limit != nil { - fetchVal := int64(*s.Limit) - s.Fetch = &ast.FetchClause{ - FetchValue: &fetchVal, - FetchType: "FIRST", - } - if s.Offset != nil { - offsetVal := int64(*s.Offset) - s.Fetch.OffsetValue = &offsetVal - s.Offset = nil - } - s.Limit = nil - } - } + normalizeLimitForDialect(s, dialect) } func renderInsert(i *ast.InsertStatement, opts ast.FormatOptions) string { if i == nil { return "" } + if i.DialectExtras().MultiTable != nil { + // Oracle INSERT ALL / FIRST: rendered on one line by the AST. + if opts.AddSemicolon { + return i.SQL() + ";" + } + return i.SQL() + } f := newNodeFormatter(opts) sb := f.sb @@ -405,9 +406,7 @@ func renderInsert(i *ast.InsertStatement, opts ast.FormatOptions) string { sb.WriteString(f.clauseSep()) } - sb.WriteString(f.kw("INSERT INTO")) - sb.WriteString(" ") - sb.WriteString(ast.QuoteQualifiedName(i.TableName)) + writeInsertHead(sb, f, i) // MySQL INSERT ... SET form: Columns[k] = Values[0][k]. setForm := i.SetSyntax && i.Query == nil && len(i.Columns) > 0 && @@ -419,11 +418,9 @@ func renderInsert(i *ast.InsertStatement, opts ast.FormatOptions) string { sb.WriteString(")") } - if len(i.Output) > 0 && !setForm { - sb.WriteString(f.clauseSep()) - sb.WriteString(f.kw("OUTPUT")) - sb.WriteString(" ") - sb.WriteString(exprListSQL(i.Output)) + if !setForm { + writeDMLOutput(sb, f, i.Output, i.DialectExtras().OutputInto) + writeInsertBeforeSource(sb, f, i) } if setForm { @@ -455,6 +452,7 @@ func renderInsert(i *ast.InsertStatement, opts ast.FormatOptions) string { sb.WriteString(f.clauseSep()) sb.WriteString(f.kw("DEFAULT VALUES")) } + writeInsertAfterSource(sb, f, i) if i.OnConflict != nil { sb.WriteString(onConflictSQL(i.OnConflict)) @@ -471,12 +469,7 @@ func renderInsert(i *ast.InsertStatement, opts ast.FormatOptions) string { sb.WriteString(strings.Join(upds, ", ")) } - if len(i.Returning) > 0 { - sb.WriteString(f.clauseSep()) - sb.WriteString(f.kw("RETURNING")) - sb.WriteString(" ") - sb.WriteString(exprListSQL(i.Returning)) - } + writeInsertTail(sb, f, i) if opts.AddSemicolon { sb.WriteString(";") @@ -497,12 +490,14 @@ func renderUpdate(u *ast.UpdateStatement, opts ast.FormatOptions) string { sb.WriteString(f.clauseSep()) } + x := u.DialectExtras() sb.WriteString(f.kw("UPDATE")) sb.WriteString(" ") - sb.WriteString(ast.QuoteQualifiedName(u.TableName)) - if u.Alias != "" { - sb.WriteString(" ") - sb.WriteString(ast.QuoteTableAlias(u.Alias)) + writeDMLModifiers(sb, f, x.Modifiers) + writeDMLTop(sb, f, x.Top) + writeDMLTargets(sb, f, x, u.TableName, u.TableNameQuote, u.Alias, u.AliasQuote) + if len(u.From) == 0 { + writeDMLJoins(sb, f, x.Joins) } sb.WriteString(f.clauseSep()) @@ -513,6 +508,7 @@ func renderUpdate(u *ast.UpdateStatement, opts ast.FormatOptions) string { upds[i] = ast.ColumnRefSQL(upd.Column) + " = " + exprSQL(upd.Value) } sb.WriteString(strings.Join(upds, ", ")) + writeDMLOutput(sb, f, x.Output, x.OutputInto) if len(u.From) > 0 { sb.WriteString(f.clauseSep()) @@ -523,6 +519,7 @@ func renderUpdate(u *ast.UpdateStatement, opts ast.FormatOptions) string { froms[i] = tableRefSQL(&u.From[i], f) } sb.WriteString(strings.Join(froms, ", ")) + writeDMLJoins(sb, f, x.Joins) } if u.Where != nil { @@ -531,13 +528,8 @@ func renderUpdate(u *ast.UpdateStatement, opts ast.FormatOptions) string { sb.WriteString(" ") sb.WriteString(FormatExpression(u.Where, opts)) } - - if len(u.Returning) > 0 { - sb.WriteString(f.clauseSep()) - sb.WriteString(f.kw("RETURNING")) - sb.WriteString(" ") - sb.WriteString(exprListSQL(u.Returning)) - } + writeDMLOrderLimit(sb, f, x) + writeReturning(sb, f, u.Returning, x.ReturningInto) if opts.AddSemicolon { sb.WriteString(";") @@ -558,12 +550,33 @@ func renderDelete(d *ast.DeleteStatement, opts ast.FormatOptions) string { sb.WriteString(f.clauseSep()) } - sb.WriteString(f.kw("DELETE FROM")) + x := d.DialectExtras() + sb.WriteString(f.kw("DELETE")) sb.WriteString(" ") - sb.WriteString(ast.QuoteQualifiedName(d.TableName)) - if d.Alias != "" { + writeDMLModifiers(sb, f, x.Modifiers) + writeDMLTop(sb, f, x.Top) + if len(d.From) == 0 { + sb.WriteString(f.kw("FROM")) sb.WriteString(" ") - sb.WriteString(ast.QuoteTableAlias(d.Alias)) + } + writeDMLTargets(sb, f, x, d.TableName, d.TableNameQuote, d.Alias, d.AliasQuote) + if d.BeforeSystemTime != nil { + sb.WriteString(" ") + sb.WriteString(f.kw("BEFORE SYSTEM_TIME")) + sb.WriteString(" ") + sb.WriteString(exprSQL(d.BeforeSystemTime)) + } + writeDMLOutput(sb, f, x.Output, x.OutputInto) + + if len(d.From) > 0 { + sb.WriteString(f.clauseSep()) + sb.WriteString(f.kw("FROM")) + sb.WriteString(" ") + froms := make([]string, len(d.From)) + for i := range d.From { + froms[i] = tableRefSQL(&d.From[i], f) + } + sb.WriteString(strings.Join(froms, ", ")) } if len(d.Using) > 0 { @@ -576,6 +589,7 @@ func renderDelete(d *ast.DeleteStatement, opts ast.FormatOptions) string { } sb.WriteString(strings.Join(usings, ", ")) } + writeDMLJoins(sb, f, x.Joins) if d.Where != nil { sb.WriteString(f.clauseSep()) @@ -583,13 +597,8 @@ func renderDelete(d *ast.DeleteStatement, opts ast.FormatOptions) string { sb.WriteString(" ") sb.WriteString(FormatExpression(d.Where, opts)) } - - if len(d.Returning) > 0 { - sb.WriteString(f.clauseSep()) - sb.WriteString(f.kw("RETURNING")) - sb.WriteString(" ") - sb.WriteString(exprListSQL(d.Returning)) - } + writeDMLOrderLimit(sb, f, x) + writeReturning(sb, f, d.Returning, x.ReturningInto) if opts.AddSemicolon { sb.WriteString(";") @@ -599,92 +608,26 @@ func renderDelete(d *ast.DeleteStatement, opts ast.FormatOptions) string { } func renderCreateTable(c *ast.CreateTableStatement, opts ast.FormatOptions) string { - if c == nil { - return "" - } - f := newNodeFormatter(opts) - sb := f.sb - - sb.WriteString(f.kw("CREATE")) - sb.WriteString(" ") - if c.Temporary { - sb.WriteString(f.kw("TEMPORARY")) - sb.WriteString(" ") - } - sb.WriteString(f.kw("TABLE")) - sb.WriteString(" ") - if c.IfNotExists { - sb.WriteString(f.kw("IF NOT EXISTS")) - sb.WriteString(" ") - } - sb.WriteString(ast.QuoteQualifiedName(c.Name)) - - if opts.NewlinePerClause { - sb.WriteString(" (\n") - f.depth++ - parts := make([]string, 0, len(c.Columns)+len(c.Constraints)) - for _, col := range c.Columns { - col := col - parts = append(parts, f.indentStr()+columnDefSQL(&col)) - } - for _, con := range c.Constraints { - con := con - parts = append(parts, f.indentStr()+tableConstraintSQL(&con)) - } - sb.WriteString(strings.Join(parts, ",\n")) - f.depth-- - sb.WriteString("\n") - sb.WriteString(f.indentStr()) - sb.WriteString(")") - } else { - sb.WriteString(" (") - parts := make([]string, 0, len(c.Columns)+len(c.Constraints)) - for _, col := range c.Columns { - col := col - parts = append(parts, columnDefSQL(&col)) - } - for _, con := range c.Constraints { - con := con - parts = append(parts, tableConstraintSQL(&con)) - } - sb.WriteString(strings.Join(parts, ", ")) - sb.WriteString(")") - } - - if len(c.Inherits) > 0 { - sb.WriteString(" ") - sb.WriteString(f.kw("INHERITS")) - sb.WriteString(" (") - sb.WriteString(quoteQualifiedNames(c.Inherits)) - sb.WriteString(")") - } - - if c.PartitionBy != nil { - sb.WriteString(" ") - sb.WriteString(f.kw("PARTITION BY")) - // An unknown method is left out: "PARTITION BY (cols)" does not parse. - fmt.Fprintf(sb, " %s (%s)", ast.PartitionByText(c.PartitionBy.Type), quoteColumnNames(c.PartitionBy.Columns)) - } - - for _, opt := range c.Options { - // Only known option names (ENGINE, CHARSET) are written. - if ast.TableOptionNameText(opt.Name) == "" { - continue - } - fmt.Fprintf(sb, " %s=%s", opt.Name, ast.QuoteTableOptionValue(opt.Value)) - } - - if opts.AddSemicolon { - sb.WriteString(";") - } - - return f.result() + return renderCreateTableStmt(c, opts) } func renderSetOperation(s *ast.SetOperation, opts ast.FormatOptions) string { if s == nil { return "" } + // A WITH clause that scopes over the whole operation but would end up + // inside the parentheses of a left operand is written before the chain. + if with, body := ast.HoistLeadingWith(s, opts.Dialect); with != nil { + f := newNodeFormatter(opts) + f.sb.WriteString(renderWith(with, f)) + f.sb.WriteString(f.clauseSep()) + f.sb.WriteString(renderSetOperationBody(body, opts)) + return f.result() + } + return renderSetOperationBody(s, opts) +} + +func renderSetOperationBody(s *ast.SetOperation, opts ast.FormatOptions) string { f := newNodeFormatter(opts) sb := f.sb inner := nestedOpts(opts) @@ -700,7 +643,7 @@ func renderSetOperation(s *ast.SetOperation, opts ast.FormatOptions) string { spine := []*ast.SetOperation{s} for { last := spine[len(spine)-1] - if last.LeftParenthesized { + if ast.SetOperandNeedsParens(last, false, opts.Dialect) { // A parenthesized left operand is rendered whole, in parentheses. break } @@ -712,7 +655,7 @@ func renderSetOperation(s *ast.SetOperation, opts ast.FormatOptions) string { } if innermost := spine[len(spine)-1]; innermost.Left != nil { - if innermost.LeftParenthesized { + if ast.SetOperandNeedsParens(innermost, false, opts.Dialect) { sb.WriteString("(") sb.WriteString(FormatStatement(innermost.Left, inner)) sb.WriteString(")") @@ -726,17 +669,14 @@ func renderSetOperation(s *ast.SetOperation, opts ast.FormatOptions) string { // The operator is UNION, EXCEPT or INTERSECT; anything else can only // come from a hand-built AST and is written as UNION followed by the // double-quoted text, which fails to parse (see ast.SetOperatorSQL). - op, known := ast.SetOperatorSQL(level.Operator) + op, known := ast.SetOperationKeyword(level, opts.Dialect) if known { op = f.kw(op) } - if level.All { - op += " " + f.kw("ALL") - } sb.WriteString(op) sb.WriteString(f.clauseSep()) if level.Right != nil { - if rightSetOp, ok := level.Right.(*ast.SetOperation); level.RightParenthesized || (ok && rightSetOp != nil) { + if ast.SetOperandNeedsParens(level, true, opts.Dialect) { sb.WriteString("(") sb.WriteString(FormatStatement(level.Right, inner)) sb.WriteString(")") @@ -745,6 +685,20 @@ func renderSetOperation(s *ast.SetOperation, opts ast.FormatOptions) string { } } } + if t := s.Tail; t != nil { + // ORDER BY / LIMIT / OFFSET / FETCH of the whole operation. + view := t.Select() + if len(view.OrderBy) > 0 { + sb.WriteString(f.clauseSep()) + sb.WriteString(f.kw("ORDER BY")) + sb.WriteString(" ") + sb.WriteString(orderBySQL(view.OrderBy)) + } + writeLimitOffsetClauses(sb, view, f) + if view.Fetch != nil { + sb.WriteString(fetchSQL(view.Fetch, f)) + } + } if opts.AddSemicolon { sb.WriteString(";") } @@ -789,169 +743,15 @@ func renderAlterTable(a *ast.AlterTableStatement, opts ast.FormatOptions) string } func renderCreateIndex(c *ast.CreateIndexStatement, opts ast.FormatOptions) string { - if c == nil { - return "" - } - f := newNodeFormatter(opts) - sb := f.sb - - sb.WriteString(f.kw("CREATE")) - sb.WriteString(" ") - if c.Unique { - sb.WriteString(f.kw("UNIQUE")) - sb.WriteString(" ") - } - sb.WriteString(f.kw("INDEX")) - sb.WriteString(" ") - if c.IfNotExists { - sb.WriteString(f.kw("IF NOT EXISTS")) - sb.WriteString(" ") - } - sb.WriteString(ast.QuoteQualifiedName(c.Name)) - sb.WriteString(" ") - sb.WriteString(f.kw("ON")) - sb.WriteString(" ") - sb.WriteString(ast.QuoteQualifiedName(c.Table)) - - if c.Using != "" { - sb.WriteString(" ") - sb.WriteString(f.kw("USING")) - sb.WriteString(" ") - sb.WriteString(ast.QuoteColumnName(c.Using)) - } - - sb.WriteString(" (") - cols := make([]string, len(c.Columns)) - for i, col := range c.Columns { - s := ast.QuoteColumnName(col.Column) - if col.Collate != "" { - s += " " + f.kw("COLLATE") + " " + ast.QuoteColumnName(col.Collate) - } - if dir := ast.SortDirectionText(col.Direction); dir != "" { - s += " " + f.kw(dir) - } - if col.NullsLast { - s += " " + f.kw("NULLS LAST") - } - cols[i] = s - } - sb.WriteString(strings.Join(cols, ", ")) - sb.WriteString(")") - - if c.Where != nil { - sb.WriteString(f.clauseSep()) - sb.WriteString(f.kw("WHERE")) - sb.WriteString(" ") - sb.WriteString(FormatExpression(c.Where, opts)) - } - - if opts.AddSemicolon { - sb.WriteString(";") - } - - return f.result() + return renderCreateIndexStmt(c, opts) } func renderCreateView(c *ast.CreateViewStatement, opts ast.FormatOptions) string { - if c == nil { - return "" - } - f := newNodeFormatter(opts) - sb := f.sb - - sb.WriteString(f.kw("CREATE")) - sb.WriteString(" ") - if c.OrReplace { - sb.WriteString(f.kw("OR REPLACE")) - sb.WriteString(" ") - } - if c.Temporary { - sb.WriteString(f.kw("TEMPORARY")) - sb.WriteString(" ") - } - sb.WriteString(f.kw("VIEW")) - sb.WriteString(" ") - if c.IfNotExists { - sb.WriteString(f.kw("IF NOT EXISTS")) - sb.WriteString(" ") - } - sb.WriteString(ast.QuoteQualifiedName(c.Name)) - - if len(c.Columns) > 0 { - sb.WriteString(" (") - sb.WriteString(quoteNames(c.Columns)) - sb.WriteString(")") - } - - sb.WriteString(" ") - sb.WriteString(f.kw("AS")) - sb.WriteString(f.clauseSep()) - sb.WriteString(FormatStatement(c.Query, nestedOpts(opts))) - - // WithOption is CHECK OPTION, CASCADED or LOCAL CHECK OPTION, written - // after WITH; anything else can only come from a hand-built AST and is - // dropped rather than written raw. - if ast.ViewWithOptionText(c.WithOption) != "" { - sb.WriteString(f.clauseSep()) - sb.WriteString(f.kw("WITH")) - sb.WriteString(" ") - sb.WriteString(f.kw(c.WithOption)) - } - - if opts.AddSemicolon { - sb.WriteString(";") - } - - return f.result() + return renderCreateViewStmt(c, opts) } func renderCreateMaterializedView(c *ast.CreateMaterializedViewStatement, opts ast.FormatOptions) string { - if c == nil { - return "" - } - f := newNodeFormatter(opts) - sb := f.sb - - sb.WriteString(f.kw("CREATE MATERIALIZED VIEW")) - sb.WriteString(" ") - if c.IfNotExists { - sb.WriteString(f.kw("IF NOT EXISTS")) - sb.WriteString(" ") - } - sb.WriteString(ast.QuoteQualifiedName(c.Name)) - - if len(c.Columns) > 0 { - sb.WriteString(" (") - sb.WriteString(quoteNames(c.Columns)) - sb.WriteString(")") - } - - if c.Tablespace != "" { - sb.WriteString(" ") - sb.WriteString(f.kw("TABLESPACE")) - sb.WriteString(" ") - sb.WriteString(ast.QuoteColumnName(c.Tablespace)) - } - - sb.WriteString(" ") - sb.WriteString(f.kw("AS")) - sb.WriteString(f.clauseSep()) - sb.WriteString(FormatStatement(c.Query, nestedOpts(opts))) - - if c.WithData != nil { - sb.WriteString(f.clauseSep()) - if *c.WithData { - sb.WriteString(f.kw("WITH DATA")) - } else { - sb.WriteString(f.kw("WITH NO DATA")) - } - } - - if opts.AddSemicolon { - sb.WriteString(";") - } - - return f.result() + return renderCreateMaterializedViewStmt(c, opts) } func renderRefreshMaterializedView(r *ast.RefreshMaterializedViewStatement, opts ast.FormatOptions) string { @@ -986,66 +786,11 @@ func renderRefreshMaterializedView(r *ast.RefreshMaterializedViewStatement, opts } func renderDrop(d *ast.DropStatement, opts ast.FormatOptions) string { - if d == nil { - return "" - } - f := newNodeFormatter(opts) - sb := f.sb - - sb.WriteString(f.kw("DROP")) - sb.WriteString(" ") - // An unknown object type is left out: "DROP name" does not parse. - sb.WriteString(f.kw(ast.DropObjectTypeText(d.ObjectType))) - sb.WriteString(" ") - if d.IfExists { - sb.WriteString(f.kw("IF EXISTS")) - sb.WriteString(" ") - } - sb.WriteString(quoteQualifiedNames(d.Names)) - - if c := ast.CascadeText(d.CascadeType); c != "" { - sb.WriteString(" ") - sb.WriteString(f.kw(c)) - } - - if opts.AddSemicolon { - sb.WriteString(";") - } - - return f.result() + return renderDropStmt(d, opts) } func renderTruncate(t *ast.TruncateStatement, opts ast.FormatOptions) string { - if t == nil { - return "" - } - f := newNodeFormatter(opts) - sb := f.sb - - sb.WriteString(f.kw("TRUNCATE")) - sb.WriteString(" ") - sb.WriteString(f.kw("TABLE")) - sb.WriteString(" ") - sb.WriteString(quoteQualifiedNames(t.Tables)) - - if t.RestartIdentity { - sb.WriteString(" ") - sb.WriteString(f.kw("RESTART IDENTITY")) - } else if t.ContinueIdentity { - sb.WriteString(" ") - sb.WriteString(f.kw("CONTINUE IDENTITY")) - } - - if c := ast.CascadeText(t.CascadeType); c != "" { - sb.WriteString(" ") - sb.WriteString(f.kw(c)) - } - - if opts.AddSemicolon { - sb.WriteString(";") - } - - return f.result() + return renderTruncateStmt(t, opts) } func renderMerge(m *ast.MergeStatement, opts ast.FormatOptions) string { @@ -1060,7 +805,7 @@ func renderMerge(m *ast.MergeStatement, opts ast.FormatOptions) string { sb.WriteString(tableRefSQL(&m.TargetTable, f)) if m.TargetAlias != "" { sb.WriteString(" ") - sb.WriteString(ast.QuoteTableAlias(m.TargetAlias)) + sb.WriteString(ast.QuoteTableAliasWith(m.TargetAlias, f.nq(m.TargetAliasQuote))) } sb.WriteString(f.clauseSep()) @@ -1069,7 +814,8 @@ func renderMerge(m *ast.MergeStatement, opts ast.FormatOptions) string { sb.WriteString(tableRefSQL(&m.SourceTable, f)) if m.SourceAlias != "" { sb.WriteString(" ") - sb.WriteString(ast.QuoteTableAlias(m.SourceAlias)) + sb.WriteString(ast.QuoteTableAliasWith(m.SourceAlias, f.nq(m.SourceAliasQuote))) + sb.WriteString(ast.TableColumnAliasesSQLWith(&m.SourceTable, f.nq)) } sb.WriteString(f.clauseSep()) @@ -1088,6 +834,8 @@ func renderMerge(m *ast.MergeStatement, opts ast.FormatOptions) string { sb.WriteString(f.kw("NOT MATCHED")) case "NOT_MATCHED_BY_SOURCE": sb.WriteString(f.kw("NOT MATCHED BY SOURCE")) + case "NOT_MATCHED_BY_TARGET": + sb.WriteString(f.kw("NOT MATCHED BY TARGET")) default: // An unknown kind is left out: "WHEN THEN" does not parse. sb.WriteString(f.kw(ast.MergeWhenText(when.Type))) @@ -1111,7 +859,7 @@ func renderMerge(m *ast.MergeStatement, opts ast.FormatOptions) string { sb.WriteString(" ") sets := make([]string, len(when.Action.SetClauses)) for i, sc := range when.Action.SetClauses { - sets[i] = ast.QuoteQualifiedName(sc.Column) + " = " + exprSQL(sc.Value) + sets[i] = ast.QuoteQualifiedNameWith(sc.Column, f.nq(sc.ColumnQuote)) + " = " + exprSQL(sc.Value) } sb.WriteString(strings.Join(sets, ", ")) case "DELETE": @@ -1124,7 +872,7 @@ func renderMerge(m *ast.MergeStatement, opts ast.FormatOptions) string { } else { if len(when.Action.Columns) > 0 { sb.WriteString(" (") - sb.WriteString(quoteColumnNames(when.Action.Columns)) + sb.WriteString(f.quoteColumnNamesWith(when.Action.Columns, when.Action.ColumnQuotes)) sb.WriteString(")") } if len(when.Action.Values) > 0 { @@ -1139,9 +887,13 @@ func renderMerge(m *ast.MergeStatement, opts ast.FormatOptions) string { sb.WriteString(")") } } + default: + sb.WriteString(f.kw(ast.MergeActionText(when.Action.ActionType))) } + writeMergeActionWhere(sb, f, when.Action) } } + writeMergeTail(sb, f, m) if opts.AddSemicolon { sb.WriteString(";") @@ -1219,11 +971,21 @@ func renderIn(i *ast.InExpression, opts ast.FormatOptions) string { sb.WriteString(FormatExpression(i.Expr, opts)) sb.WriteString(" ") + if i.Global { + sb.WriteString(f.kw("GLOBAL")) + sb.WriteString(" ") + } if i.Not { sb.WriteString(f.kw("NOT")) sb.WriteString(" ") } sb.WriteString(f.kw("IN")) + if i.Bare && i.Subquery == nil { + // SQLite "a IN tbl": no parentheses around the table reference. + sb.WriteString(" ") + sb.WriteString(exprListSQL(i.List)) + return f.result() + } sb.WriteString(" (") if i.Subquery != nil { sb.WriteString(FormatStatement(i.Subquery, nestedOpts(opts))) @@ -1293,14 +1055,7 @@ func renderWith(w *ast.WithClause, f *nodeFormatter) string { } ctes := make([]string, len(w.CTEs)) for i, cte := range w.CTEs { - s := ast.QuoteName(cte.Name) + " " - if len(cte.Columns) > 0 { - s += "(" + quoteNames(cte.Columns) + ") " - } - s += f.kw("AS") + " (" - s += FormatStatement(cte.Statement, nestedOpts(f.opts)) - s += ")" - ctes[i] = s + ctes[i] = cteSQL(cte, f) } sb.WriteString(strings.Join(ctes, ", ")) return sb.String() @@ -1342,7 +1097,9 @@ func orderBySQL(orders []ast.OrderByExpression) string { parts := make([]string, len(orders)) for i, o := range orders { s := exprSQL(o.Expression) - if !o.Ascending { + if op := ast.OrderUsingOperatorText(o.UsingOperator); op != "" { + s += " USING " + op + } else if !o.Ascending { s += " DESC" } if o.NullsFirst != nil { @@ -1373,6 +1130,21 @@ func orderBySQL(orders []ast.OrderByExpression) string { // so PIVOT/UNPIVOT/FOR/IN/AS/LATERAL keywords honor the caller's case policy. func tableRefSQL(t *ast.TableReference, f *nodeFormatter) string { var sb strings.Builder + if g := ast.TableRefJoinGroup(t); g != nil { + sb.WriteString("(") + for i := range g.Tables { + if i > 0 { + sb.WriteString(", ") + } + sb.WriteString(tableRefSQL(&g.Tables[i], f)) + } + for i := range g.Joins { + sb.WriteString(" ") + sb.WriteString(joinSQL(&g.Joins[i], f)) + } + sb.WriteString(")") + return sb.String() + } if t.Lateral { sb.WriteString(f.kw("LATERAL")) sb.WriteString(" ") @@ -1391,23 +1163,28 @@ func tableRefSQL(t *ast.TableReference, f *nodeFormatter) string { sb.WriteString("(") sb.WriteString(stmtSQL(t.SetSubquery)) sb.WriteString(")") + } else if t.DialectModifiers().Values != nil { + sb.WriteString("(") + sb.WriteString(f.kw("VALUES")) + sb.WriteString(strings.TrimPrefix(t.DialectModifiers().Values.SQL(), "VALUES")) + sb.WriteString(")") } else { - sb.WriteString(ast.QuoteQualifiedName(t.Name)) + if t.Only { + sb.WriteString(f.kw("ONLY")) + sb.WriteString(" ") + } + sb.WriteString(ast.QuoteQualifiedNameWith(t.Name, f.nq(t.NameQuote))) + sb.WriteString(ast.TableRefNameSuffixSQL(t)) } - if t.Alias != "" && t.Pivot == nil && t.Unpivot == nil { + sb.WriteString(ast.TableRefPreAliasSQL(t, f.kw)) + if t.Alias != "" && !ast.TableAliasFollowsClause(t) { sb.WriteString(" ") - sb.WriteString(ast.QuoteTableAlias(t.Alias)) - if t.ColumnAliases != nil && len(t.ColumnAliases.Names) > 0 && t.TableFunc != nil { - sb.WriteString("(") - for i, c := range t.ColumnAliases.Names { - if i > 0 { - sb.WriteString(", ") - } - sb.WriteString(ast.QuoteColumnName(c)) - } - sb.WriteString(")") - } + sb.WriteString(ast.QuoteTableAliasWith(t.Alias, f.nq(t.AliasQuote))) + sb.WriteString(ast.TableColumnAliasesSQLWith(t, f.nq)) + } else { + sb.WriteString(ast.TableRefSourceAliasSQLWith(t, f.nq)) } + sb.WriteString(ast.TableRefPostAliasSQL(t, f.kw)) // SQL Server table hints follow the alias and precede PIVOT/UNPIVOT (#604). if len(t.TableHints) > 0 { sb.WriteString(" ") @@ -1417,103 +1194,27 @@ func tableRefSQL(t *ast.TableReference, f *nodeFormatter) string { if i > 0 { sb.WriteString(", ") } - // Bare-word hints honor the keyword case policy; anything else + // Hint keywords honor the keyword case policy; anything else // (only possible in a hand-built AST) is double-quoted. - if q := ast.QuoteTableHint(h); q != h { - sb.WriteString(q) - } else { - sb.WriteString(f.kw(h)) - } + sb.WriteString(ast.TableHintSQL(h, f.kw)) } sb.WriteString(")") } - if t.Pivot != nil { - sb.WriteString(" ") - sb.WriteString(f.kw("PIVOT")) - sb.WriteString(" (") - sb.WriteString(exprSQL(t.Pivot.AggregateFunction)) - writePivotAggregateAlias(&sb, f, t.Pivot.AggregateAlias) - for _, a := range t.Pivot.ExtraAggregates { - sb.WriteString(", ") - sb.WriteString(exprSQL(a.Function)) - writePivotAggregateAlias(&sb, f, a.Alias) - } - sb.WriteString(" ") - sb.WriteString(f.kw("FOR")) - sb.WriteString(" ") - sb.WriteString(ast.QuotePivotColumn(t.Pivot.PivotColumn)) - sb.WriteString(" ") - sb.WriteString(f.kw("IN")) - sb.WriteString(" (") - writePivotItems(&sb, t.Pivot.InValues) - sb.WriteString("))") - } - if t.Unpivot != nil { - sb.WriteString(" ") - sb.WriteString(f.kw("UNPIVOT")) - if nh := ast.UnpivotNullHandling(t.Unpivot.NullHandling); nh != "" { - sb.WriteString(" ") - sb.WriteString(f.kw(nh)) - } - sb.WriteString(" (") - if len(t.Unpivot.ValueColumns) > 0 { - sb.WriteString("(") - for i, c := range t.Unpivot.ValueColumns { - if i > 0 { - sb.WriteString(", ") - } - sb.WriteString(ast.QuoteUnpivotValueColumn(c)) - } - sb.WriteString(")") - } else { - sb.WriteString(ast.QuoteUnpivotValueColumn(t.Unpivot.ValueColumn)) - } - sb.WriteString(" ") - sb.WriteString(f.kw("FOR")) - sb.WriteString(" ") - sb.WriteString(ast.QuotePivotColumn(t.Unpivot.NameColumn)) - sb.WriteString(" ") - sb.WriteString(f.kw("IN")) - sb.WriteString(" (") - writePivotItems(&sb, t.Unpivot.InColumns) - sb.WriteString("))") - } - if t.Alias != "" && (t.Pivot != nil || t.Unpivot != nil) { + sb.WriteString(ast.PivotSQL(t.Pivot, f.kw)) + sb.WriteString(ast.UnpivotSQL(t.Unpivot, f.kw)) + sb.WriteString(ast.MatchRecognizeSQL(t.MatchRecognize, f.kw)) + if t.Alias != "" && ast.TableAliasFollowsClause(t) { // PIVOT/UNPIVOT aliases conventionally use AS to avoid ambiguity // with the closing paren of the clause. sb.WriteString(" ") sb.WriteString(f.kw("AS")) sb.WriteString(" ") - sb.WriteString(ast.QuoteTableAlias(t.Alias)) + sb.WriteString(ast.QuoteTableAliasWith(t.Alias, f.nq(t.AliasQuote))) + sb.WriteString(ast.TableColumnAliasesSQLWith(t, f.nq)) } return sb.String() } -// writePivotAggregateAlias writes " AS alias" for a PIVOT aggregate alias, -// preserving source delimiters or quoting it like a table alias; an empty -// alias writes nothing. -func writePivotAggregateAlias(sb *strings.Builder, f *nodeFormatter, alias string) { - if alias == "" { - return - } - sb.WriteString(" ") - sb.WriteString(f.kw("AS")) - sb.WriteString(" ") - sb.WriteString(ast.QuotePivotAggregateAlias(alias)) -} - -// writePivotItems writes PIVOT IN values or UNPIVOT IN columns separated by -// ", ", each in its stored source form when well formed and double-quoted -// otherwise (see ast.QuotePivotItem). -func writePivotItems(sb *strings.Builder, items []string) { - for i, item := range items { - if i > 0 { - sb.WriteString(", ") - } - sb.WriteString(ast.QuotePivotItem(item)) - } -} - // sampleSQL renders a ClickHouse SAMPLE clause. func sampleSQL(s *ast.SampleClause, f *nodeFormatter) string { var sb strings.Builder @@ -1544,14 +1245,22 @@ func sampleSQL(s *ast.SampleClause, f *nodeFormatter) string { // joinSQL renders a JOIN clause. func joinSQL(j *ast.JoinClause, f *nodeFormatter) string { var sb strings.Builder - // An unknown join type is dropped, leaving a plain JOIN. - if t := ast.JoinTypeText(j.Type); t != "" { - sb.WriteString(f.kw(t)) - sb.WriteString(" ") + if j.Comma { + sb.WriteString(", ") + sb.WriteString(tableRefSQL(&j.Right, f)) + return sb.String() } - sb.WriteString(f.kw("JOIN")) + // An unknown join type is dropped, leaving a plain JOIN. + sb.WriteString(joinKeywordsSQL(j, f)) sb.WriteString(" ") sb.WriteString(tableRefSQL(&j.Right, f)) + if j.MatchCondition != nil { + sb.WriteString(" ") + sb.WriteString(f.kw("MATCH_CONDITION")) + sb.WriteString(" (") + sb.WriteString(exprSQL(j.MatchCondition)) + sb.WriteString(")") + } if len(j.Using) > 0 { sb.WriteString(" ") sb.WriteString(f.kw("USING")) @@ -1578,7 +1287,7 @@ func joinSQL(j *ast.JoinClause, f *nodeFormatter) string { func windowSpecSQL(w *ast.WindowSpec) string { var parts []string if w.BaseWindow != "" { - parts = append(parts, ast.QuoteName(w.BaseWindow)) + parts = append(parts, ast.QuoteNameWith(w.BaseWindow, w.BaseWindowQuote)) } if len(w.PartitionBy) > 0 { parts = append(parts, "PARTITION BY "+exprListSQL(w.PartitionBy)) @@ -1627,13 +1336,16 @@ func fetchSQL(fc *ast.FetchClause, f *nodeFormatter) string { fmt.Fprintf(&sb, " %s %s", f.kw("FETCH"), f.kw(fetchType)) if fc.FetchValue != nil { fmt.Fprintf(&sb, " %d", *fc.FetchValue) + } else if fc.FetchExpr != nil { + sb.WriteString(" ") + sb.WriteString(exprSQL(fc.FetchExpr)) } if fc.IsPercent { sb.WriteString(" ") sb.WriteString(f.kw("PERCENT")) } sb.WriteString(" ") - sb.WriteString(f.kw("ROWS")) + sb.WriteString(f.kw(ast.FetchRowsKeyword(fc))) if fc.WithTies { sb.WriteString(" ") sb.WriteString(f.kw("WITH TIES")) @@ -1647,6 +1359,17 @@ func fetchSQL(fc *ast.FetchClause, f *nodeFormatter) string { // forSQL renders a FOR UPDATE/SHARE clause. func forSQL(f *ast.ForClause) string { var sb strings.Builder + if f.LockInShareMode { + sb.WriteString(" LOCK IN SHARE MODE") + if f.NoWait { + sb.WriteString(" NOWAIT") + } + if f.Wait != nil { + sb.WriteString(" WAIT ") + sb.WriteString(exprSQL(f.Wait)) + } + return sb.String() + } sb.WriteString(" FOR ") sb.WriteString(ast.LockTypeText(f.LockType)) if len(f.Tables) > 0 { @@ -1659,6 +1382,10 @@ func forSQL(f *ast.ForClause) string { if f.SkipLocked { sb.WriteString(" SKIP LOCKED") } + if f.Wait != nil { + sb.WriteString(" WAIT ") + sb.WriteString(exprSQL(f.Wait)) + } return sb.String() } @@ -1671,6 +1398,10 @@ func onConflictSQL(oc *ast.OnConflict) string { sb.WriteString(ast.ColumnRefListSQL(oc.Target)) sb.WriteString(")") } + if oc.TargetWhere != nil { + sb.WriteString(" WHERE ") + sb.WriteString(exprSQL(oc.TargetWhere)) + } if oc.Constraint != "" { sb.WriteString(" ON CONSTRAINT ") sb.WriteString(ast.QuoteColumnName(oc.Constraint)) @@ -1691,97 +1422,3 @@ func onConflictSQL(oc *ast.OnConflict) string { } return sb.String() } - -// columnDefSQL renders a column definition. -func columnDefSQL(c *ast.ColumnDef) string { - var sb strings.Builder - sb.WriteString(ast.QuoteColumnName(c.Name)) - sb.WriteString(" ") - sb.WriteString(ast.QuoteDataType(c.Type)) - for _, con := range c.Constraints { - con := con - sb.WriteString(" ") - sb.WriteString(columnConstraintSQL(&con)) - } - return sb.String() -} - -// columnConstraintSQL renders a column constraint. -func columnConstraintSQL(c *ast.ColumnConstraint) string { - switch c.Type { - case "NOT NULL", "UNIQUE", "PRIMARY KEY": - return c.Type - case "DEFAULT": - return "DEFAULT " + exprSQL(c.Default) - case "REFERENCES": - if c.References != nil { - return referenceSQL(c.References) - } - return "REFERENCES" - case "CHECK": - return "CHECK (" + exprSQL(c.Check) + ")" - default: - if c.AutoIncrement { - return "AUTO_INCREMENT" - } - // An unknown constraint kind is dropped. - return ast.ColumnConstraintText(c.Type) - } -} - -// tableConstraintSQL renders a table-level constraint. -func tableConstraintSQL(tc *ast.TableConstraint) string { - var sb strings.Builder - if tc.Name != "" { - sb.WriteString("CONSTRAINT ") - sb.WriteString(ast.QuoteConstraintName(tc.Name)) - sb.WriteString(" ") - } - switch tc.Type { - case "PRIMARY KEY": - sb.WriteString("PRIMARY KEY (") - sb.WriteString(quoteColumnNames(tc.Columns)) - sb.WriteString(")") - case "UNIQUE": - sb.WriteString("UNIQUE (") - sb.WriteString(quoteColumnNames(tc.Columns)) - sb.WriteString(")") - case "FOREIGN KEY": - sb.WriteString("FOREIGN KEY (") - sb.WriteString(quoteColumnNames(tc.Columns)) - sb.WriteString(") ") - if tc.References != nil { - sb.WriteString(referenceSQL(tc.References)) - } - case "CHECK": - sb.WriteString("CHECK (") - sb.WriteString(exprSQL(tc.Check)) - sb.WriteString(")") - default: - // The parser only stores the kinds above; any other kind can only - // come from a hand-built AST and is dropped, leaving a constraint - // without a kind, which does not parse. - } - return sb.String() -} - -// referenceSQL renders a REFERENCES clause. -func referenceSQL(r *ast.ReferenceDefinition) string { - var sb strings.Builder - sb.WriteString("REFERENCES ") - sb.WriteString(ast.QuoteQualifiedName(r.Table)) - if len(r.Columns) > 0 { - sb.WriteString(" (") - sb.WriteString(quoteColumnNames(r.Columns)) - sb.WriteString(")") - } - if a := ast.ReferentialActionText(r.OnDelete); a != "" { - sb.WriteString(" ON DELETE ") - sb.WriteString(a) - } - if a := ast.ReferentialActionText(r.OnUpdate); a != "" { - sb.WriteString(" ON UPDATE ") - sb.WriteString(a) - } - return sb.String() -} diff --git a/pkg/formatter/render_alias_quoting_632_test.go b/pkg/formatter/render_alias_quoting_632_test.go index d0769886..4ea8bd43 100644 --- a/pkg/formatter/render_alias_quoting_632_test.go +++ b/pkg/formatter/render_alias_quoting_632_test.go @@ -215,7 +215,7 @@ func TestFormat_AliasQuoting_PR632(t *testing.T) { {`SELECT * FROM t AS "u; DROP TABLE x; --"`, "SELECT *\nFROM t \"u; DROP TABLE x; --\""}, {`SELECT amount AS total FROM t`, "SELECT amount AS total\nFROM t"}, {`SELECT * FROM t AS u`, "SELECT *\nFROM t u"}, - {`SELECT amount AS "Total" FROM t ORDER BY "Total"`, "SELECT amount AS Total\nFROM t\nORDER BY Total"}, + {`SELECT amount AS "Total" FROM t ORDER BY "Total"`, "SELECT amount AS \"Total\"\nFROM t\nORDER BY \"Total\""}, {`SELECT amount AS "select" FROM t ORDER BY "select"`, "SELECT amount AS \"select\"\nFROM t\nORDER BY \"select\""}, } for _, tc := range cases { diff --git a/pkg/formatter/render_alter.go b/pkg/formatter/render_alter.go index 8226c873..69273171 100644 --- a/pkg/formatter/render_alter.go +++ b/pkg/formatter/render_alter.go @@ -14,102 +14,20 @@ package formatter -import "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" +import ( + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" +) -// renderAlterStatement renders an *ast.AlterStatement. ALTER TABLE operations -// produced by the parser (ADD COLUMN, ADD CONSTRAINT, DROP COLUMN, DROP -// CONSTRAINT, RENAME TO, RENAME COLUMN, ALTER COLUMN) are rendered in full; -// other ALTER kinds (ROLE, POLICY, CONNECTOR) keep the previous stmtSQL output. +// renderAlterStatement renders an *ast.AlterStatement. ALTER TABLE statements +// (every action the parser produces, comma-separated) are rendered in full, as +// are ALTER ROLE, POLICY and CONNECTOR; a statement without an operation is +// the bare ALTER keyword. The clause writers are shared with +// ast.AlterStatement.SQL() (ast.AlterSQL). func renderAlterStatement(a *ast.AlterStatement, opts ast.FormatOptions) string { if a == nil { return "" } - if a.Type != ast.AlterTypeTable { - return stmtSQL(a) - } f := newNodeFormatter(opts) - f.sb.WriteString(f.kw("ALTER TABLE")) - f.sb.WriteString(" ") - f.sb.WriteString(a.Name) - if op, ok := a.Operation.(*ast.AlterTableOperation); ok && op != nil { - f.sb.WriteString(f.clauseSep()) - writeAlterTableOperation(f, op) - } - if opts.AddSemicolon { - f.sb.WriteString(";") - } - return f.result() -} - -// alterIdentName returns the name of i, or "" for nil. -func alterIdentName(i *ast.Ident) string { - if i == nil { - return "" - } - return i.Name -} - -// writeAlterTableOperation renders a single ALTER TABLE operation. Operation -// types the ALTER TABLE parser does not produce are left empty. -func writeAlterTableOperation(f *nodeFormatter, op *ast.AlterTableOperation) { - sb := f.sb - switch op.Type { - case ast.AddColumn: - sb.WriteString(f.kw("ADD COLUMN")) - if op.ColumnDef != nil { - sb.WriteString(" ") - sb.WriteString(columnDefSQL(op.ColumnDef)) - } - case ast.AddConstraint: - sb.WriteString(f.kw("ADD")) - if op.Constraint != nil { - // tableConstraintSQL prints "CONSTRAINT name" itself when named. - if op.Constraint.Name == "" { - sb.WriteString(" ") - sb.WriteString(f.kw("CONSTRAINT")) - } - sb.WriteString(" ") - sb.WriteString(tableConstraintSQL(op.Constraint)) - } - case ast.DropColumn: - sb.WriteString(f.kw("DROP COLUMN")) - sb.WriteString(" ") - sb.WriteString(alterIdentName(op.ColumnName)) - if op.CascadeDrops { - sb.WriteString(" ") - sb.WriteString(f.kw("CASCADE")) - } - case ast.DropConstraint: - sb.WriteString(f.kw("DROP CONSTRAINT")) - sb.WriteString(" ") - sb.WriteString(alterIdentName(op.ConstraintName)) - if op.CascadeDrops { - sb.WriteString(" ") - sb.WriteString(f.kw("CASCADE")) - } - case ast.RenameTable: - sb.WriteString(f.kw("RENAME TO")) - sb.WriteString(" ") - sb.WriteString(op.NewTableName.String()) - case ast.RenameColumn: - oldName := op.ColumnName - if oldName == nil { - oldName = op.OldColumnName - } - sb.WriteString(f.kw("RENAME COLUMN")) - sb.WriteString(" ") - sb.WriteString(alterIdentName(oldName)) - sb.WriteString(" ") - sb.WriteString(f.kw("TO")) - sb.WriteString(" ") - sb.WriteString(alterIdentName(op.NewColumnName)) - case ast.AlterColumn: - sb.WriteString(f.kw("ALTER COLUMN")) - sb.WriteString(" ") - sb.WriteString(alterIdentName(op.ColumnName)) - if op.ColumnDef != nil { - sb.WriteString(" ") - sb.WriteString(columnDefSQL(op.ColumnDef)) - } - } + f.sb.WriteString(ast.AlterSQL(a, f.ddlStyle())) + return f.finishDDL() } diff --git a/pkg/formatter/render_alter_ext_test.go b/pkg/formatter/render_alter_ext_test.go new file mode 100644 index 00000000..cd2a6726 --- /dev/null +++ b/pkg/formatter/render_alter_ext_test.go @@ -0,0 +1,256 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package formatter_test + +import ( + "strings" + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/formatter" + "github.com/ajitpratap0/GoSQLX/pkg/gosqlx" + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" +) + +// formatAlter parses sql in dialect and renders its only statement with opts. +// ALTER statements other than ALTER TABLE come back as utility statements. +func formatAlter(t *testing.T, sql string, dialect keywords.SQLDialect, opts ast.FormatOptions) string { + t.Helper() + tree, err := gosqlx.ParseWithDialect(sql, dialect) + if err != nil { + t.Fatalf("parse %q (%s): %v", sql, dialect, err) + } + if len(tree.Statements) != 1 { + t.Fatalf("parse %q: expected 1 statement, got %d", sql, len(tree.Statements)) + } + return formatter.FormatStatement(tree.Statements[0], opts) +} + +// TestFormatStatement_AlterTableDialects checks that the ALTER TABLE forms of +// each dialect render with their meaning intact (nothing dropped, nothing +// added) and that the rendering re-parses to the same text. +func TestFormatStatement_AlterTableDialects(t *testing.T) { + opts := ast.CompactStyle() + opts.KeywordCase = ast.KeywordUpper + + tests := []struct { + name string + dialect keywords.SQLDialect + in string + want string + }{ + // PostgreSQL + {"pg add column if not exists", keywords.DialectPostgreSQL, "ALTER TABLE users ADD COLUMN IF NOT EXISTS phone VARCHAR(32)", "ALTER TABLE users ADD COLUMN IF NOT EXISTS phone VARCHAR(32)"}, + {"pg only and if exists", keywords.DialectPostgreSQL, "alter table if exists only public.t drop column if exists a cascade", "ALTER TABLE IF EXISTS ONLY public.t DROP COLUMN IF EXISTS a CASCADE"}, + {"pg several actions", keywords.DialectPostgreSQL, "ALTER TABLE t ADD COLUMN a int, ADD COLUMN b int", "ALTER TABLE t ADD COLUMN a int, ADD COLUMN b int"}, + {"pg set default", keywords.DialectPostgreSQL, "ALTER TABLE t ALTER COLUMN a SET DEFAULT 5", "ALTER TABLE t ALTER COLUMN a SET DEFAULT 5"}, + {"pg drop default", keywords.DialectPostgreSQL, "ALTER TABLE t ALTER COLUMN a DROP DEFAULT", "ALTER TABLE t ALTER COLUMN a DROP DEFAULT"}, + {"pg alter without column keyword", keywords.DialectPostgreSQL, "ALTER TABLE t ALTER a SET NOT NULL", "ALTER TABLE t ALTER a SET NOT NULL"}, + {"pg type using", keywords.DialectPostgreSQL, "ALTER TABLE t ALTER COLUMN a TYPE numeric(10,2) USING a::numeric", "ALTER TABLE t ALTER COLUMN a TYPE numeric(10,2) USING CAST(a AS numeric)"}, + {"pg rls", keywords.DialectPostgreSQL, "ALTER TABLE t ENABLE ROW LEVEL SECURITY", "ALTER TABLE t ENABLE ROW LEVEL SECURITY"}, + {"pg owner", keywords.DialectPostgreSQL, "ALTER TABLE t OWNER TO bob", "ALTER TABLE t OWNER TO bob"}, + {"pg bare primary key", keywords.DialectPostgreSQL, "ALTER TABLE t ADD PRIMARY KEY (a)", "ALTER TABLE t ADD PRIMARY KEY (a)"}, + {"pg named check not valid", keywords.DialectPostgreSQL, "ALTER TABLE t ADD CONSTRAINT c CHECK (a > 0) NOT VALID", "ALTER TABLE t ADD CONSTRAINT c CHECK (a > 0) NOT VALID"}, + {"pg drop constraint restrict", keywords.DialectPostgreSQL, "ALTER TABLE t DROP CONSTRAINT IF EXISTS c RESTRICT", "ALTER TABLE t DROP CONSTRAINT IF EXISTS c RESTRICT"}, + {"pg rename column without keyword", keywords.DialectPostgreSQL, "ALTER TABLE t RENAME a TO b", "ALTER TABLE t RENAME a TO b"}, + {"pg rename to qualified", keywords.DialectPostgreSQL, "ALTER TABLE t RENAME TO s.u", "ALTER TABLE t RENAME TO s.u"}, + {"pg quoted names", keywords.DialectPostgreSQL, `ALTER TABLE "My Table" ADD COLUMN "Col X" int`, `ALTER TABLE "My Table" ADD COLUMN "Col X" int`}, + {"pg storage parameters", keywords.DialectPostgreSQL, "ALTER TABLE t SET (fillfactor = 70)", "ALTER TABLE t SET (fillfactor = 70)"}, + // MySQL / MariaDB + {"mysql modify after", keywords.DialectMySQL, "ALTER TABLE t MODIFY COLUMN c INT NOT NULL AFTER b", "ALTER TABLE t MODIFY COLUMN c INT NOT NULL AFTER b"}, + {"mysql change first", keywords.DialectMySQL, "ALTER TABLE t CHANGE a b INT FIRST", "ALTER TABLE t CHANGE a b INT FIRST"}, + {"mysql add index", keywords.DialectMySQL, "ALTER TABLE t ADD UNIQUE INDEX uq (a, b)", "ALTER TABLE t ADD UNIQUE INDEX uq (a, b)"}, + {"mysql drop index", keywords.DialectMySQL, "ALTER TABLE t DROP INDEX ix", "ALTER TABLE t DROP INDEX ix"}, + {"mysql drop primary key", keywords.DialectMySQL, "ALTER TABLE t DROP PRIMARY KEY", "ALTER TABLE t DROP PRIMARY KEY"}, + {"mysql drop foreign key", keywords.DialectMySQL, "ALTER TABLE t DROP FOREIGN KEY fk", "ALTER TABLE t DROP FOREIGN KEY fk"}, + {"mysql drop without column keyword", keywords.DialectMySQL, "ALTER TABLE t DROP c", "ALTER TABLE t DROP c"}, + {"mysql options", keywords.DialectMySQL, "ALTER TABLE t ENGINE=InnoDB, ALGORITHM=INPLACE, LOCK=NONE", "ALTER TABLE t ENGINE=InnoDB, ALGORITHM=INPLACE, LOCK=NONE"}, + {"mysql convert", keywords.DialectMySQL, "ALTER TABLE t CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci", "ALTER TABLE t CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci"}, + {"mysql rename without to", keywords.DialectMySQL, "ALTER TABLE t RENAME u", "ALTER TABLE t RENAME TO u"}, + {"mysql parenthesized add", keywords.DialectMySQL, "ALTER TABLE t ADD (a INT, b INT)", "ALTER TABLE t ADD (a INT, b INT)"}, + {"mysql order by", keywords.DialectMySQL, "ALTER TABLE t ORDER BY a, b DESC", "ALTER TABLE t ORDER BY a, b DESC"}, + {"mysql drop partitions", keywords.DialectMySQL, "ALTER TABLE t DROP PARTITION p1, p2", "ALTER TABLE t DROP PARTITION p1, p2"}, + {"mariadb online", keywords.DialectMariaDB, "ALTER ONLINE TABLE t ADD COLUMN c INT", "ALTER ONLINE TABLE t ADD COLUMN c INT"}, + {"mariadb drop index if exists", keywords.DialectMariaDB, "ALTER TABLE t DROP INDEX IF EXISTS ix", "ALTER TABLE t DROP INDEX IF EXISTS ix"}, + {"mariadb add if not exists without column", keywords.DialectMariaDB, "ALTER TABLE t ADD IF NOT EXISTS c INT", "ALTER TABLE t ADD IF NOT EXISTS c INT"}, + {"mariadb system versioning", keywords.DialectMariaDB, "ALTER TABLE t ADD SYSTEM VERSIONING", "ALTER TABLE t ADD SYSTEM VERSIONING"}, + // SQL Server + {"mssql add without column keyword", keywords.DialectSQLServer, "ALTER TABLE dbo.t ADD c INT NULL", "ALTER TABLE dbo.t ADD c INT NULL"}, + {"mssql add list", keywords.DialectSQLServer, "ALTER TABLE dbo.t ADD a INT NULL, b INT NOT NULL", "ALTER TABLE dbo.t ADD a INT NULL, b INT NOT NULL"}, + {"mssql drop column list", keywords.DialectSQLServer, "ALTER TABLE dbo.t DROP COLUMN a, b", "ALTER TABLE dbo.t DROP COLUMN a, b"}, + {"mssql alter column definition", keywords.DialectSQLServer, "ALTER TABLE dbo.t ALTER COLUMN Name NVARCHAR(400) NOT NULL", "ALTER TABLE dbo.t ALTER COLUMN Name NVARCHAR(400) NOT NULL"}, + {"mssql with nocheck", keywords.DialectSQLServer, "ALTER TABLE dbo.t WITH NOCHECK ADD CONSTRAINT c CHECK (a >= 0)", "ALTER TABLE dbo.t WITH NOCHECK ADD CONSTRAINT c CHECK (a >= 0)"}, + {"mssql nonclustered", keywords.DialectSQLServer, "ALTER TABLE dbo.t ADD CONSTRAINT uq UNIQUE NONCLUSTERED (a, b)", "ALTER TABLE dbo.t ADD CONSTRAINT uq UNIQUE NONCLUSTERED (a, b)"}, + {"mssql nocheck constraint", keywords.DialectSQLServer, "ALTER TABLE dbo.t NOCHECK CONSTRAINT fk", "ALTER TABLE dbo.t NOCHECK CONSTRAINT fk"}, + // Oracle + {"oracle parenthesized add", keywords.DialectOracle, "ALTER TABLE t ADD (a NUMBER, b VARCHAR2(10))", "ALTER TABLE t ADD (a NUMBER, b VARCHAR2(10))"}, + {"oracle modify list", keywords.DialectOracle, "ALTER TABLE t MODIFY (a VARCHAR2(20) NOT NULL)", "ALTER TABLE t MODIFY (a VARCHAR2(20) NOT NULL)"}, + {"oracle enable novalidate", keywords.DialectOracle, "ALTER TABLE t ENABLE NOVALIDATE CONSTRAINT c", "ALTER TABLE t ENABLE NOVALIDATE CONSTRAINT c"}, + {"oracle add partition", keywords.DialectOracle, "ALTER TABLE t ADD PARTITION p VALUES LESS THAN (TO_DATE('2026-01-01', 'YYYY-MM-DD'))", "ALTER TABLE t ADD PARTITION p VALUES LESS THAN (TO_DATE('2026-01-01', 'YYYY-MM-DD'))"}, + {"oracle move tablespace", keywords.DialectOracle, "ALTER TABLE t MOVE TABLESPACE users2", "ALTER TABLE t MOVE TABLESPACE users2"}, + // Snowflake + {"snowflake qualified", keywords.DialectSnowflake, "ALTER TABLE a.b.c ADD COLUMN IF NOT EXISTS x INT", "ALTER TABLE a.b.c ADD COLUMN IF NOT EXISTS x INT"}, + {"snowflake swap", keywords.DialectSnowflake, "ALTER TABLE a.b SWAP WITH a.c", "ALTER TABLE a.b SWAP WITH a.c"}, + {"snowflake cluster by", keywords.DialectSnowflake, "ALTER TABLE t CLUSTER BY (a, b)", "ALTER TABLE t CLUSTER BY (a, b)"}, + {"snowflake set properties", keywords.DialectSnowflake, "ALTER TABLE t SET DATA_RETENTION_TIME_IN_DAYS = 7, COMMENT = 'x'", "ALTER TABLE t SET DATA_RETENTION_TIME_IN_DAYS = 7, COMMENT = 'x'"}, + {"snowflake set data type", keywords.DialectSnowflake, "ALTER TABLE t ALTER COLUMN a SET DATA TYPE VARCHAR(10)", "ALTER TABLE t ALTER COLUMN a SET DATA TYPE VARCHAR(10)"}, + {"snowflake masking policy", keywords.DialectSnowflake, "ALTER TABLE t MODIFY COLUMN email SET MASKING POLICY email_mask", "ALTER TABLE t MODIFY COLUMN email SET MASKING POLICY email_mask"}, + // ClickHouse + {"clickhouse on cluster", keywords.DialectClickHouse, "ALTER TABLE db.t ON CLUSTER c ADD COLUMN x UInt8 AFTER y", "ALTER TABLE db.t ON CLUSTER c ADD COLUMN x UInt8 AFTER y"}, + {"clickhouse update", keywords.DialectClickHouse, "ALTER TABLE t UPDATE a = 1, b = 'x' IN PARTITION '202401' WHERE id = 1", "ALTER TABLE t UPDATE a = 1, b = 'x' IN PARTITION '202401' WHERE id = 1"}, + {"clickhouse delete", keywords.DialectClickHouse, "ALTER TABLE t DELETE WHERE ts < now() - INTERVAL 90 DAY", "ALTER TABLE t DELETE WHERE ts < now() - INTERVAL 90 DAY"}, + {"clickhouse modify ttl", keywords.DialectClickHouse, "ALTER TABLE t MODIFY TTL ts + INTERVAL 6 MONTH", "ALTER TABLE t MODIFY TTL ts + INTERVAL 6 MONTH"}, + {"clickhouse add index", keywords.DialectClickHouse, "ALTER TABLE t ADD INDEX i a TYPE bloom_filter GRANULARITY 3", "ALTER TABLE t ADD INDEX i a TYPE bloom_filter GRANULARITY 3"}, + {"clickhouse drop partition id", keywords.DialectClickHouse, "ALTER TABLE t DROP PARTITION ID '20240101'", "ALTER TABLE t DROP PARTITION ID '20240101'"}, + {"clickhouse modify setting", keywords.DialectClickHouse, "ALTER TABLE t MODIFY SETTING a = 1, b = 2", "ALTER TABLE t MODIFY SETTING a = 1, b = 2"}, + // SQLite + {"sqlite add column", keywords.DialectSQLite, "ALTER TABLE t ADD COLUMN a INT", "ALTER TABLE t ADD COLUMN a INT"}, + {"sqlite drop column", keywords.DialectSQLite, "ALTER TABLE t DROP COLUMN a", "ALTER TABLE t DROP COLUMN a"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := formatAlter(t, tc.in, tc.dialect, opts) + if got != tc.want { + t.Errorf("format(%q):\n got: %q\nwant: %q", tc.in, got, tc.want) + } + // The rendering must re-parse and render identically. + if again := formatAlter(t, got, tc.dialect, opts); again != got { + t.Errorf("render is not stable across re-parse:\nfirst: %q\nsecond: %q", got, again) + } + }) + } +} + +// TestFormatStatement_AlterTableLayout checks the readable layout of an ALTER +// TABLE with several actions and the keyword casing of the new forms. +func TestFormatStatement_AlterTableLayout(t *testing.T) { + const in = "alter table s.t add column if not exists a int, drop column b, alter column c set default 1" + readable := formatAlter(t, in, keywords.DialectPostgreSQL, ast.ReadableStyle()) + const wantReadable = "ALTER TABLE s.t\nADD COLUMN IF NOT EXISTS a int,\nDROP COLUMN b,\nALTER COLUMN c SET DEFAULT 1;" + if readable != wantReadable { + t.Errorf("readable:\n got: %q\nwant: %q", readable, wantReadable) + } + + lower := ast.CompactStyle() + lower.KeywordCase = ast.KeywordLower + const wantLower = "alter table s.t add column if not exists a int, drop column b, alter column c set default 1" + if got := formatAlter(t, in, keywords.DialectPostgreSQL, lower); got != wantLower { + t.Errorf("lower:\n got: %q\nwant: %q", got, wantLower) + } + + // The action keywords and the keyword phrase of generic actions follow + // the casing; the text after a phrase is kept as written. + const gen = "ALTER TABLE t ENABLE ROW LEVEL SECURITY, OWNER TO Bob" + if got := formatAlter(t, gen, keywords.DialectPostgreSQL, lower); got != "alter table t enable row level security, owner to Bob" { + t.Errorf("lower generic: got %q", got) + } +} + +// TestFormatStatement_AlterObjects checks that ALTER statements are +// written back as they were. +func TestFormatStatement_AlterObjects(t *testing.T) { + tests := []struct { + dialect keywords.SQLDialect + sql string + }{ + {keywords.DialectPostgreSQL, "ALTER INDEX idx_users_email RENAME TO idx_users_email_v2"}, + {keywords.DialectSQLServer, "ALTER INDEX ALL ON dbo.Orders REBUILD WITH (ONLINE = ON)"}, + {keywords.DialectOracle, "ALTER SESSION SET NLS_LANGUAGE = 'AMERICAN' NLS_TERRITORY = 'AMERICA'"}, + {keywords.DialectOracle, "ALTER SYSTEM KILL SESSION '123,4567' IMMEDIATE"}, + {keywords.DialectSnowflake, "ALTER WAREHOUSE etl_wh SET WAREHOUSE_SIZE = 'LARGE'"}, + {keywords.DialectSnowflake, "ALTER SESSION SET QUERY_TAG = 'nightly_etl'"}, + {keywords.DialectMySQL, "ALTER DATABASE app CHARACTER SET utf8mb4"}, + } + for _, tc := range tests { + t.Run(tc.sql, func(t *testing.T) { + if got := formatAlter(t, tc.sql, tc.dialect, ast.CompactStyle()); got != tc.sql { + t.Errorf("got %q, want %q", got, tc.sql) + } + }) + } +} + +// TestFormatStatement_AlterRolePolicyConnector checks that ALTER ROLE, +// ALTER POLICY and ALTER CONNECTOR no longer render as a bare "ALTER". +func TestFormatStatement_AlterRolePolicyConnector(t *testing.T) { + opts := ast.CompactStyle() + opts.KeywordCase = ast.KeywordUpper + tests := []struct { + name string + dialect keywords.SQLDialect + in string + want string + }{ + {"role rename", keywords.DialectGeneric, "ALTER ROLE r RENAME TO s", "ALTER ROLE r RENAME TO s"}, + {"role add member", keywords.DialectGeneric, "ALTER ROLE r ADD MEMBER m", "ALTER ROLE r ADD MEMBER m"}, + {"role drop member", keywords.DialectGeneric, "ALTER ROLE r DROP MEMBER m", "ALTER ROLE r DROP MEMBER m"}, + {"role set", keywords.DialectGeneric, "ALTER ROLE r SET search_path TO public", "ALTER ROLE r SET search_path TO public"}, + {"role reset all", keywords.DialectGeneric, "ALTER ROLE r RESET ALL", "ALTER ROLE r RESET ALL"}, + {"role options", keywords.DialectGeneric, "ALTER ROLE r WITH SUPERUSER, VALID UNTIL '2030-01-01'", "ALTER ROLE r WITH SUPERUSER, VALID UNTIL '2030-01-01'"}, + {"role password", keywords.DialectGeneric, "ALTER ROLE r WITH LOGIN, PASSWORD 'x'", "ALTER ROLE r WITH LOGIN, PASSWORD 'x'"}, + {"policy rename", keywords.DialectPostgreSQL, "ALTER POLICY p ON s.t RENAME TO q", "ALTER POLICY p ON s.t RENAME TO q"}, + {"policy modify", keywords.DialectPostgreSQL, "ALTER POLICY p ON t TO a, b USING (x > 1) WITH CHECK (y > 1)", "ALTER POLICY p ON t TO a, b USING (x > 1) WITH CHECK (y > 1)"}, + {"connector url", keywords.DialectGeneric, "ALTER CONNECTOR c SET URL 'jdbc:x'", "ALTER CONNECTOR c SET URL 'jdbc:x'"}, + {"connector properties", keywords.DialectGeneric, "ALTER CONNECTOR c SET DCPROPERTIES ('b' = 'it''s', a = '1')", "ALTER CONNECTOR c SET DCPROPERTIES ('a' = '1', 'b' = 'it''s')"}, + {"connector owner", keywords.DialectGeneric, "ALTER CONNECTOR c SET OWNER USER bob", "ALTER CONNECTOR c SET OWNER USER bob"}, + {"connector owner role", keywords.DialectGeneric, "ALTER CONNECTOR c SET OWNER ROLE admins", "ALTER CONNECTOR c SET OWNER ROLE admins"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := formatAlter(t, tc.in, tc.dialect, opts) + if got != tc.want { + t.Errorf("format(%q):\n got: %q\nwant: %q", tc.in, got, tc.want) + } + if again := formatAlter(t, got, tc.dialect, opts); again != got { + t.Errorf("render is not stable across re-parse:\nfirst: %q\nsecond: %q", got, again) + } + }) + } +} + +// TestFormatStatement_AlterHandBuiltKeywords_PR632 checks that the keyword +// fields of a hand-built ALTER TABLE are not written verbatim: a value that +// is not a bare keyword phrase is dropped, and an ON CLUSTER target that is +// not a well-formed name or string is quoted. +func TestFormatStatement_AlterHandBuiltKeywords_PR632(t *testing.T) { + const evil = "x; DROP TABLE y; --" + col := &ast.ColumnDef{Name: "a", Type: "INT"} + tests := []struct { + name string + stmt *ast.AlterStatement + }{ + {"other keyword", &ast.AlterStatement{Type: ast.AlterTypeTable, Name: "t", + Operation: &ast.AlterTableOperation{Type: ast.AlterTableOther, Keyword: evil}}}, + {"index kind", &ast.AlterStatement{Type: ast.AlterTypeTable, Name: "t", + Operation: &ast.AlterTableOperation{Type: ast.AddIndex, IndexKind: evil, IndexKeyword: "INDEX", IndexName: "i", Text: "(a)"}}}, + {"drop index keyword", &ast.AlterStatement{Type: ast.AlterTypeTable, Name: "t", + Operation: &ast.AlterTableOperation{Type: ast.DropIndex, IndexKeyword: evil, IndexName: "i"}}}, + {"constraint kind", &ast.AlterStatement{Type: ast.AlterTypeTable, Name: "t", + Operation: &ast.AlterTableOperation{Type: ast.DropConstraint, ConstraintKind: evil, ConstraintName: &ast.Ident{Name: "c"}}}}, + {"action modifier", &ast.AlterStatement{Type: ast.AlterTypeTable, Name: "t", + Operation: &ast.AlterTableOperation{Type: ast.AddColumn, Modifier: evil, ColumnDef: col}}}, + {"statement modifier and cluster", &ast.AlterStatement{Type: ast.AlterTypeTable, Name: "t", + Modifiers: []string{evil}, OnCluster: evil, + Operation: &ast.AlterTableOperation{Type: ast.DropColumn, ColumnName: &ast.Ident{Name: "a"}}}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := formatter.FormatStatement(tc.stmt, ast.CompactStyle()) + if strings.Contains(got, "; DROP") && !strings.Contains(got, `"x; DROP TABLE y; --"`) { + t.Errorf("hand-built keyword written verbatim: %s", got) + } + }) + } +} diff --git a/pkg/formatter/render_columns.go b/pkg/formatter/render_columns.go new file mode 100644 index 00000000..abeb9e02 --- /dev/null +++ b/pkg/formatter/render_columns.go @@ -0,0 +1,27 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package formatter + +import "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" + +// Column definitions and table constraints render exactly as the AST's own +// SQL() does (ast/sql_columns.go): the constraint kinds, their quoting and the +// keyword allow-lists live in one place so the two renderers cannot drift. + +// columnDefSQL renders a column definition. +func columnDefSQL(c *ast.ColumnDef) string { return ast.ColumnDefSQL(c) } + +// tableConstraintSQL renders a table-level constraint or index element. +func tableConstraintSQL(tc *ast.TableConstraint) string { return ast.TableConstraintSQL(tc) } diff --git a/pkg/formatter/render_columns_test.go b/pkg/formatter/render_columns_test.go new file mode 100644 index 00000000..a45a4b10 --- /dev/null +++ b/pkg/formatter/render_columns_test.go @@ -0,0 +1,98 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package formatter_test + +import ( + "strings" + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/gosqlx" + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" +) + +// TestFormat_ColumnDefinitionsKeepTheirMeaning formats CREATE TABLE statements +// that use dialect column attributes and table constraints, and checks that +// the formatter keeps every attribute (nothing is dropped), writes the dialect +// spelling the source used, and that the output parses again and formats to +// itself. +func TestFormat_ColumnDefinitionsKeepTheirMeaning(t *testing.T) { + cases := []struct { + name string + dialect keywords.SQLDialect + sql string + want []string // substrings the formatted text must contain + notWant []string + }{ + {"sqlite autoincrement", keywords.DialectSQLite, + "CREATE TABLE t (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL)", + []string{"id INTEGER PRIMARY KEY AUTOINCREMENT"}, []string{"AUTO_INCREMENT"}}, + {"mysql auto_increment", keywords.DialectMySQL, + "CREATE TABLE t (id INT UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'pk', PRIMARY KEY (id))", + []string{"id INT UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'pk'", "PRIMARY KEY (id)"}, []string{"AUTOINCREMENT"}}, + {"mysql attributes", keywords.DialectMySQL, + "CREATE TABLE t (s VARCHAR(10) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, ts DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, g INT AS (1 + 1) STORED, v INT INVISIBLE)", + []string{"CHARACTER SET utf8mb4 COLLATE utf8mb4_bin", "ON UPDATE CURRENT_TIMESTAMP", "AS (1 + 1) STORED", "INVISIBLE"}, nil}, + {"mysql indexes", keywords.DialectMySQL, + "CREATE TABLE t (id INT, a TEXT, UNIQUE KEY uq (id), FULLTEXT KEY ft (a), KEY k (id), KEY f ((id + 1)))", + []string{"UNIQUE KEY uq (id)", "FULLTEXT KEY ft (a)", "KEY k (id)", "KEY f ((id + 1))"}, nil}, + {"postgres identity and types", keywords.DialectPostgreSQL, + "CREATE TABLE t (id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, tags text[], at timestamp with time zone, d double precision)", + []string{"GENERATED ALWAYS AS IDENTITY PRIMARY KEY", "tags text[]", "at timestamp with time zone", "d double precision"}, nil}, + {"postgres deferrable", keywords.DialectPostgreSQL, + "CREATE TABLE t (a int REFERENCES u (id) DEFERRABLE INITIALLY DEFERRED, b int, UNIQUE (b) DEFERRABLE, LIKE v INCLUDING ALL)", + []string{"DEFERRABLE INITIALLY DEFERRED", "UNIQUE (b) DEFERRABLE", "LIKE v INCLUDING ALL"}, nil}, + {"sqlserver identity", keywords.DialectSQLServer, + "CREATE TABLE t (id INT IDENTITY(1, 1) NOT NULL, CONSTRAINT pk PRIMARY KEY CLUSTERED (id))", + []string{"IDENTITY(1, 1) NOT NULL", "CONSTRAINT pk PRIMARY KEY CLUSTERED (id)"}, nil}, + {"snowflake autoincrement", keywords.DialectSnowflake, + "CREATE TABLE t (id NUMBER AUTOINCREMENT START 1 INCREMENT 1)", + []string{"AUTOINCREMENT START 1 INCREMENT 1"}, []string{"AUTO_INCREMENT"}}, + {"sqlite typeless", keywords.DialectSQLite, + "CREATE TABLE t (a, b NOT NULL, c COLLATE NOCASE)", + []string{"a,", "b NOT NULL", "c COLLATE NOCASE"}, nil}, + {"clickhouse clauses", keywords.DialectClickHouse, + "CREATE TABLE t (a UInt8, b UInt8 MATERIALIZED a + 1, c UInt8 DEFAULT 1 CODEC(ZSTD), INDEX i a TYPE minmax GRANULARITY 1, CONSTRAINT k CHECK a > 0) ENGINE = MergeTree ORDER BY a", + []string{"b UInt8 MATERIALIZED a + 1", "c UInt8 DEFAULT 1 CODEC(ZSTD)", "INDEX i a TYPE minmax GRANULARITY 1", "CONSTRAINT k CHECK (a > 0)"}, nil}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + tree, err := gosqlx.ParseWithDialect(tc.sql, tc.dialect) + if err != nil { + t.Fatalf("parse: %v", err) + } + defer ast.ReleaseAST(tree) + out := gosqlx.FormatAST(tree) + for _, w := range tc.want { + if !strings.Contains(out, w) { + t.Errorf("formatted output lacks %q:\n%s", w, out) + } + } + for _, w := range tc.notWant { + if strings.Contains(out, w) { + t.Errorf("formatted output contains %q:\n%s", w, out) + } + } + tree2, err := gosqlx.ParseWithDialect(out, tc.dialect) + if err != nil { + t.Fatalf("formatted output does not parse: %v\n%s", err, out) + } + defer ast.ReleaseAST(tree2) + if again := gosqlx.FormatAST(tree2); again != out { + t.Errorf("formatting is not idempotent:\n%s\n%s", out, again) + } + }) + } +} diff --git a/pkg/formatter/render_ddl.go b/pkg/formatter/render_ddl.go index 76ac2f99..de3cf7cc 100644 --- a/pkg/formatter/render_ddl.go +++ b/pkg/formatter/render_ddl.go @@ -121,7 +121,7 @@ func renderDescribe(s *ast.DescribeStatement, opts ast.FormatOptions) string { f := newNodeFormatter(opts) sb := f.sb - sb.WriteString(f.kw("DESCRIBE")) + sb.WriteString(f.kw(s.Prefix())) if s.TableName != "" { sb.WriteString(" ") sb.WriteString(ast.QuoteQualifiedName(s.TableName)) @@ -129,6 +129,23 @@ func renderDescribe(s *ast.DescribeStatement, opts ast.FormatOptions) string { return sb.String() } +// renderExplain renders EXPLAIN / DESCRIBE . +func renderExplain(s *ast.ExplainStatement, opts ast.FormatOptions) string { + f := newNodeFormatter(opts) + sb := f.sb + + sb.WriteString(f.kw(s.Keyword)) + if s.Options != "" { + sb.WriteString(" ") + sb.WriteString(s.Options) + } + if s.Statement != nil { + sb.WriteString(" ") + sb.WriteString(FormatStatement(s.Statement, opts)) + } + return sb.String() +} + // renderUnsupported renders an UnsupportedStatement as a SQL comment // preserving the original SQL fragment. This prevents silently producing // corrupt SQL for statement types the formatter cannot handle. The fragment diff --git a/pkg/formatter/render_ddl_name_quoting_632_test.go b/pkg/formatter/render_ddl_name_quoting_632_test.go index b647f6e3..4c207d5f 100644 --- a/pkg/formatter/render_ddl_name_quoting_632_test.go +++ b/pkg/formatter/render_ddl_name_quoting_632_test.go @@ -478,9 +478,9 @@ func TestFormat_UnsupportedRawSQLStaysCommented_PR632(t *testing.T) { } } - // The same through the parser: the raw text of an unsupported Snowflake - // statement keeps a string literal's line break (without its quotes), and - // must not turn into a live statement after formatting. + // The same through the parser: a Snowflake CREATE STAGE is kept verbatim + // with its string literal (line break included) quoted, so the text in + // the literal never turns into a live statement after formatting. const sql = "CREATE STAGE s COMMENT = 'a\nDROP TABLE x'" tree, err := gosqlx.ParseWithDialect(sql, keywords.DialectSnowflake) if err != nil { @@ -489,11 +489,14 @@ func TestFormat_UnsupportedRawSQLStaysCommented_PR632(t *testing.T) { defer ast.ReleaseAST(tree) out := formatter.FormatAST(tree, ast.ReadableStyle()) tree2, err := gosqlx.ParseWithDialect(out, keywords.DialectSnowflake) - if err == nil { - n := len(tree2.Statements) - ast.ReleaseAST(tree2) - if n != 0 { - t.Fatalf("formatted unsupported statement re-parses to %d live statements: %q", n, out) - } + if err != nil { + t.Fatalf("re-parse %q: %v", out, err) + } + defer ast.ReleaseAST(tree2) + if len(tree2.Statements) != 1 { + t.Fatalf("formatted statement re-parses to %d statements: %q", len(tree2.Statements), out) + } + if _, ok := tree2.Statements[0].(*ast.UtilityStatement); !ok { + t.Fatalf("re-parsed statement is %T, want *ast.UtilityStatement: %q", tree2.Statements[0], out) } } diff --git a/pkg/formatter/render_ddl_objects.go b/pkg/formatter/render_ddl_objects.go new file mode 100644 index 00000000..2556c272 --- /dev/null +++ b/pkg/formatter/render_ddl_objects.go @@ -0,0 +1,137 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// render_ddl_objects.go - formatter render handlers for CREATE TABLE, +// CREATE INDEX, CREATE [MATERIALIZED] VIEW, DROP and TRUNCATE. The clause +// writers are shared with the AST SQL() methods (ast.DDLStyle); this file +// supplies keyword casing, nested formatting and the CREATE TABLE column +// layout. + +package formatter + +import ( + "strings" + + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" +) + +// ddlStyle returns the shared-writer style for this formatter: keyword +// casing, formatted nested expressions and queries, and the clause +// separator. +func (f *nodeFormatter) ddlStyle() ast.DDLStyle { + opts := f.opts + return ast.DDLStyle{ + Keyword: f.kw, + Expr: func(e ast.Expression) string { return FormatExpression(e, opts) }, + Stmt: func(s ast.Statement) string { return FormatStatement(s, nestedOpts(opts)) }, + ClauseSep: f.clauseSep(), + NameQuote: f.nq, + } +} + +// finishDDL appends the optional semicolon and returns the result. +func (f *nodeFormatter) finishDDL() string { + if f.opts.AddSemicolon { + f.sb.WriteString(";") + } + return f.result() +} + +func renderCreateTableStmt(c *ast.CreateTableStatement, opts ast.FormatOptions) string { + if c == nil { + return "" + } + f := newNodeFormatter(opts) + sb := f.sb + st := f.ddlStyle() + + sb.WriteString(ast.CreateTableHeadSQL(c, st)) + + if c.HasColumnList() { + parts := make([]string, 0, len(c.Columns)+len(c.PeriodDefinitions)+len(c.Constraints)) + for i := range c.Columns { + parts = append(parts, columnDefSQL(&c.Columns[i])) + } + for _, pd := range c.PeriodDefinitions { + parts = append(parts, f.kw("PERIOD FOR")+strings.TrimPrefix(pd.SQL(), "PERIOD FOR")) + } + for i := range c.Constraints { + parts = append(parts, tableConstraintSQL(&c.Constraints[i])) + } + if opts.NewlinePerClause { + sb.WriteString(" (\n") + f.depth++ + for i := range parts { + parts[i] = f.indentStr() + parts[i] + } + sb.WriteString(strings.Join(parts, ",\n")) + f.depth-- + sb.WriteString("\n") + sb.WriteString(f.indentStr()) + sb.WriteString(")") + } else { + sb.WriteString(" (") + sb.WriteString(strings.Join(parts, ", ")) + sb.WriteString(")") + } + } + + sb.WriteString(ast.CreateTableTailSQL(c, st)) + return f.finishDDL() +} + +func renderCreateIndexStmt(c *ast.CreateIndexStatement, opts ast.FormatOptions) string { + if c == nil { + return "" + } + f := newNodeFormatter(opts) + f.sb.WriteString(ast.CreateIndexSQL(c, f.ddlStyle())) + return f.finishDDL() +} + +func renderCreateViewStmt(c *ast.CreateViewStatement, opts ast.FormatOptions) string { + if c == nil { + return "" + } + f := newNodeFormatter(opts) + f.sb.WriteString(ast.CreateViewSQL(c, f.ddlStyle())) + return f.finishDDL() +} + +func renderCreateMaterializedViewStmt(c *ast.CreateMaterializedViewStatement, opts ast.FormatOptions) string { + if c == nil { + return "" + } + f := newNodeFormatter(opts) + f.sb.WriteString(ast.CreateMaterializedViewSQL(c, f.ddlStyle())) + return f.finishDDL() +} + +func renderDropStmt(d *ast.DropStatement, opts ast.FormatOptions) string { + if d == nil { + return "" + } + f := newNodeFormatter(opts) + f.sb.WriteString(ast.DropSQL(d, f.ddlStyle())) + return f.finishDDL() +} + +func renderTruncateStmt(t *ast.TruncateStatement, opts ast.FormatOptions) string { + if t == nil { + return "" + } + f := newNodeFormatter(opts) + f.sb.WriteString(ast.TruncateSQL(t, f.ddlStyle())) + return f.finishDDL() +} diff --git a/pkg/formatter/render_ddl_objects_test.go b/pkg/formatter/render_ddl_objects_test.go new file mode 100644 index 00000000..8c80b43b --- /dev/null +++ b/pkg/formatter/render_ddl_objects_test.go @@ -0,0 +1,122 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package formatter_test + +import ( + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/formatter" + "github.com/ajitpratap0/GoSQLX/pkg/gosqlx" + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" +) + +// TestFormatDDLObjects checks the formatter output of the CREATE TABLE, +// CREATE INDEX, view, DROP and TRUNCATE extensions under the readable +// (newline-per-clause, upper-case) and lower-case compact styles, and that +// each output re-parses and formats idempotently. +func TestFormatDDLObjects(t *testing.T) { + lower := ast.CompactStyle() + lower.KeywordCase = ast.KeywordLower + tests := []struct { + name string + dialect keywords.SQLDialect + sql string + opts ast.FormatOptions + want string + }{ + {"ctas readable", keywords.DialectPostgreSQL, "CREATE TABLE t AS SELECT a FROM u WHERE a > 1", ast.ReadableStyle(), + "CREATE TABLE t AS\nSELECT a\nFROM u\nWHERE a > 1;"}, + {"ctas lower", keywords.DialectPostgreSQL, "CREATE TABLE t AS SELECT a FROM u WITH NO DATA", lower, + "create table t as select a from u with no data"}, + {"clickhouse engine lower", keywords.DialectClickHouse, + "CREATE TABLE e (a UInt8) ENGINE = MergeTree ORDER BY a SETTINGS index_granularity = 1", lower, + "create table e (a UInt8) engine=MergeTree order by a settings index_granularity = 1"}, + {"columns readable with options", keywords.DialectMySQL, + "CREATE TABLE t (id INT, PRIMARY KEY (id)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", ast.ReadableStyle(), + "CREATE TABLE t (\n id INT,\n PRIMARY KEY (id)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;"}, + {"period readable", keywords.DialectMariaDB, + "CREATE TABLE p (s TIMESTAMP(6), e TIMESTAMP(6), PERIOD FOR SYSTEM_TIME (s, e)) WITH SYSTEM VERSIONING", ast.ReadableStyle(), + "CREATE TABLE p (\n s TIMESTAMP(6),\n e TIMESTAMP(6),\n PERIOD FOR SYSTEM_TIME (s, e)\n) WITH SYSTEM VERSIONING;"}, + {"index readable", keywords.DialectSQLServer, + "CREATE NONCLUSTERED INDEX ix ON t (a) INCLUDE (b) WHERE a > 1 WITH (FILLFACTOR = 90)", ast.ReadableStyle(), + "CREATE NONCLUSTERED INDEX ix ON t (a) INCLUDE (b)\nWHERE a > 1 WITH (FILLFACTOR = 90);"}, + {"view lower", keywords.DialectOracle, "CREATE OR REPLACE FORCE VIEW v AS SELECT a FROM t WITH READ ONLY", lower, + "create or replace force view v as select a from t with read only"}, + {"mv readable", keywords.DialectClickHouse, "CREATE MATERIALIZED VIEW mv TO d AS SELECT a FROM s", ast.ReadableStyle(), + "CREATE MATERIALIZED VIEW mv TO d AS\nSELECT a\nFROM s;"}, + {"drop lower", keywords.DialectMySQL, "DROP INDEX ix ON t", lower, "drop index ix on t"}, + {"truncate", keywords.DialectOracle, "TRUNCATE TABLE t DROP STORAGE", ast.ReadableStyle(), "TRUNCATE TABLE t DROP STORAGE;"}, + {"verbatim object", keywords.DialectPostgreSQL, "CREATE SCHEMA IF NOT EXISTS analytics", lower, + "CREATE SCHEMA IF NOT EXISTS analytics"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tree, err := gosqlx.ParseWithDialect(tt.sql, tt.dialect) + if err != nil { + t.Fatalf("parse: %v", err) + } + defer ast.ReleaseAST(tree) + got := formatter.FormatAST(tree, tt.opts) + if got != tt.want { + t.Fatalf("format\n got: %q\nwant: %q", got, tt.want) + } + tree2, err := gosqlx.ParseWithDialect(got, tt.dialect) + if err != nil { + t.Fatalf("re-parse %q: %v", got, err) + } + defer ast.ReleaseAST(tree2) + if again := formatter.FormatAST(tree2, tt.opts); again != got { + t.Fatalf("not idempotent\nfirst: %q\nsecond: %q", got, again) + } + }) + } +} + +// TestFormatAST_VerbatimStatementsKeepSeparators checks that statements kept +// as written (CREATE SCHEMA, DROP TYPE, GRANT, ...) still end with ";" when +// AddSemicolon is set, so the next statement is not read back as part of +// their text. +func TestFormatAST_VerbatimStatementsKeepSeparators(t *testing.T) { + tests := []struct { + dialect keywords.SQLDialect + sql string + }{ + {keywords.DialectPostgreSQL, "DROP TYPE mood; DROP SCHEMA s CASCADE"}, + {keywords.DialectPostgreSQL, "CREATE SCHEMA s; CREATE TABLE s.t (a INT)"}, + {keywords.DialectPostgreSQL, "CREATE FUNCTION f() RETURNS int AS $$ SELECT 1 $$ LANGUAGE sql; SELECT 1"}, + {keywords.DialectPostgreSQL, "GRANT SELECT ON t TO u; SELECT 1"}, + {keywords.DialectSnowflake, "CREATE STAGE s URL = 's3://b/p'; DROP WAREHOUSE w"}, + } + for _, tt := range tests { + t.Run(tt.sql, func(t *testing.T) { + tree, err := gosqlx.ParseWithDialect(tt.sql, tt.dialect) + if err != nil { + t.Fatalf("parse: %v", err) + } + defer ast.ReleaseAST(tree) + want := len(tree.Statements) + out := formatter.FormatAST(tree, ast.ReadableStyle()) + tree2, err := gosqlx.ParseWithDialect(out, tt.dialect) + if err != nil { + t.Fatalf("re-parse %q: %v", out, err) + } + defer ast.ReleaseAST(tree2) + if got := len(tree2.Statements); got != want { + t.Fatalf("readable output re-parses to %d statements, want %d:\n%s", got, want, out) + } + }) + } +} diff --git a/pkg/formatter/render_dml.go b/pkg/formatter/render_dml.go new file mode 100644 index 00000000..7eeb7ad1 --- /dev/null +++ b/pkg/formatter/render_dml.go @@ -0,0 +1,252 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package formatter + +import ( + "strings" + + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" +) + +// writeDMLModifiers writes each known statement modifier followed by a space. +func writeDMLModifiers(sb *strings.Builder, f *nodeFormatter, mods []string) { + for _, m := range mods { + if t := ast.DMLModifierText(m); t != "" { + sb.WriteString(f.kw(t)) + sb.WriteString(" ") + } + } +} + +// writeDMLTop writes "TOP (n) [PERCENT] "; DML requires the parentheses. +func writeDMLTop(sb *strings.Builder, f *nodeFormatter, top *ast.TopClause) { + if top == nil || top.Count == nil { + return + } + sb.WriteString(f.kw("TOP")) + sb.WriteString(" (") + sb.WriteString(exprSQL(top.Count)) + sb.WriteString(") ") + if top.IsPercent { + sb.WriteString(f.kw("PERCENT")) + sb.WriteString(" ") + } +} + +// writeDMLTargets writes the UPDATE / DELETE target list. +func writeDMLTargets(sb *strings.Builder, f *nodeFormatter, d *ast.DMLTargetFields, name string, nameQ ast.NameQuote, alias string, aliasQ ast.NameQuote) { + if d.Only { + sb.WriteString(f.kw("ONLY")) + sb.WriteString(" ") + } + first := true + if name != "" { + sb.WriteString(ast.QuoteQualifiedNameWith(name, f.nq(nameQ))) + if alias != "" { + sb.WriteString(" ") + sb.WriteString(ast.QuoteTableAliasWith(alias, f.nq(aliasQ))) + } + first = false + } + for i := range d.Tables { + if !first { + sb.WriteString(", ") + } + sb.WriteString(tableRefSQL(&d.Tables[i], f)) + first = false + } +} + +// writeDMLJoins writes the JOIN clauses of a multi-table UPDATE / DELETE. +func writeDMLJoins(sb *strings.Builder, f *nodeFormatter, joins []ast.JoinClause) { + for i := range joins { + sb.WriteString(f.clauseSep()) + sb.WriteString(joinSQL(&joins[i], f)) + } +} + +// writeDMLOutput writes the SQL Server OUTPUT [INTO] clause. +func writeDMLOutput(sb *strings.Builder, f *nodeFormatter, output []ast.Expression, into *ast.OutputInto) { + if len(output) == 0 { + return + } + sb.WriteString(f.clauseSep()) + sb.WriteString(f.kw("OUTPUT")) + sb.WriteString(" ") + sb.WriteString(exprListSQL(output)) + if into != nil && into.Target != "" { + sb.WriteString(" ") + sb.WriteString(f.kw("INTO")) + sb.WriteString(" ") + sb.WriteString(ast.QuoteQualifiedName(into.Target)) + if len(into.Columns) > 0 { + sb.WriteString(" (") + sb.WriteString(ast.ColumnRefListSQL(into.Columns)) + sb.WriteString(")") + } + } +} + +// writeDMLOrderLimit writes the single-table ORDER BY ... LIMIT n clauses. +func writeDMLOrderLimit(sb *strings.Builder, f *nodeFormatter, d *ast.DMLTargetFields) { + if len(d.OrderBy) > 0 { + sb.WriteString(f.clauseSep()) + sb.WriteString(f.kw("ORDER BY")) + sb.WriteString(" ") + sb.WriteString(orderBySQL(d.OrderBy)) + } + if d.Limit != nil { + sb.WriteString(f.clauseSep()) + sb.WriteString(f.kw("LIMIT")) + sb.WriteString(" ") + sb.WriteString(exprSQL(d.Limit)) + } +} + +// writeReturning writes RETURNING items [INTO targets]. +func writeReturning(sb *strings.Builder, f *nodeFormatter, returning, into []ast.Expression) { + if len(returning) == 0 { + return + } + sb.WriteString(f.clauseSep()) + sb.WriteString(f.kw("RETURNING")) + sb.WriteString(" ") + sb.WriteString(exprListSQL(returning)) + if len(into) > 0 { + sb.WriteString(" ") + sb.WriteString(f.kw("INTO")) + sb.WriteString(" ") + sb.WriteString(exprListSQL(into)) + } +} + +// writeInsertHead writes "INSERT [modifiers] INTO target". +func writeInsertHead(sb *strings.Builder, f *nodeFormatter, i *ast.InsertStatement) { + sb.WriteString(f.kw("INSERT")) + sb.WriteString(" ") + writeDMLModifiers(sb, f, i.DialectExtras().Modifiers) + sb.WriteString(f.kw("INTO")) + sb.WriteString(" ") + if i.DialectExtras().TableFunction != nil { + sb.WriteString(f.kw("FUNCTION")) + sb.WriteString(" ") + sb.WriteString(exprSQL(i.DialectExtras().TableFunction)) + return + } + sb.WriteString(ast.QuoteQualifiedNameWith(i.TableName, f.nq(i.DialectExtras().TableNameQuote))) +} + +// writeInsertBeforeSource writes ClickHouse SETTINGS and PostgreSQL +// OVERRIDING ... VALUE, which sit between the column list and the source. +func writeInsertBeforeSource(sb *strings.Builder, f *nodeFormatter, i *ast.InsertStatement) { + if len(i.DialectExtras().Settings) > 0 { + sb.WriteString(" ") + sb.WriteString(f.kw("SETTINGS")) + sb.WriteString(" ") + for k := range i.DialectExtras().Settings { + if k > 0 { + sb.WriteString(", ") + } + sb.WriteString(i.DialectExtras().Settings[k].SQL()) + } + } + if t := ast.OverridingText(i.DialectExtras().Overriding); t != "" { + sb.WriteString(" ") + sb.WriteString(f.kw("OVERRIDING " + t + " VALUE")) + } +} + +// writeInsertAfterSource writes the MySQL row alias AS name [(cols)]. +func writeInsertAfterSource(sb *strings.Builder, f *nodeFormatter, i *ast.InsertStatement) { + if i.DialectExtras().RowAlias == "" { + return + } + sb.WriteString(" ") + sb.WriteString(f.kw("AS")) + sb.WriteString(" ") + sb.WriteString(ast.QuoteTableAlias(i.DialectExtras().RowAlias)) + if len(i.DialectExtras().RowAliasColumns) > 0 { + sb.WriteString(" (") + sb.WriteString(quoteColumnNames(i.DialectExtras().RowAliasColumns)) + sb.WriteString(")") + } +} + +// writeInsertTail writes RETURNING [INTO] and the ClickHouse FORMAT name. +func writeInsertTail(sb *strings.Builder, f *nodeFormatter, i *ast.InsertStatement) { + writeReturning(sb, f, i.Returning, i.DialectExtras().ReturningInto) + if l := i.DialectExtras().LogErrors; l != nil { + sb.WriteString(f.clauseSep()) + sb.WriteString(f.kw("LOG ERRORS")) + if l.Into != "" { + sb.WriteString(" ") + sb.WriteString(f.kw("INTO")) + sb.WriteString(" ") + sb.WriteString(ast.QuoteQualifiedNameWith(l.Into, f.nq(l.IntoQuote))) + } + if l.Tag != nil { + sb.WriteString(" (") + sb.WriteString(exprSQL(l.Tag)) + sb.WriteString(")") + } + if lim := ast.LogErrorsRejectLimitText(l.RejectLimit); lim != "" { + sb.WriteString(" ") + sb.WriteString(f.kw("REJECT LIMIT")) + sb.WriteString(" ") + sb.WriteString(f.kw(lim)) + } + } + if i.DialectExtras().DataFormat != "" { + sb.WriteString(f.clauseSep()) + sb.WriteString(f.kw("FORMAT")) + sb.WriteString(" ") + sb.WriteString(formatNameSQL(i.DialectExtras().DataFormat)) + } +} + +// formatNameSQL writes a ClickHouse format name (CSV, JSONEachRow, Values) +// bare when it is a plain word and double-quoted otherwise. +func formatNameSQL(name string) string { + for k := 0; k < len(name); k++ { + c := name[k] + if c != '_' && (c < '0' || c > '9') && (c|0x20 < 'a' || c|0x20 > 'z') { + return ast.QuoteName(name) + } + } + return name +} + +// writeMergeTail writes the MERGE OUTPUT and RETURNING clauses. +func writeMergeTail(sb *strings.Builder, f *nodeFormatter, m *ast.MergeStatement) { + writeDMLOutput(sb, f, m.Output, m.OutputInto) + writeReturning(sb, f, m.Returning, nil) +} + +// writeMergeActionWhere writes the Oracle WHERE / DELETE WHERE predicates of +// a MERGE action. +func writeMergeActionWhere(sb *strings.Builder, f *nodeFormatter, a *ast.MergeAction) { + if a.Where != nil { + sb.WriteString(" ") + sb.WriteString(f.kw("WHERE")) + sb.WriteString(" ") + sb.WriteString(exprSQL(a.Where)) + } + if a.DeleteWhere != nil { + sb.WriteString(" ") + sb.WriteString(f.kw("DELETE WHERE")) + sb.WriteString(" ") + sb.WriteString(exprSQL(a.DeleteWhere)) + } +} diff --git a/pkg/formatter/render_injection_632_test.go b/pkg/formatter/render_injection_632_test.go index c2b3aa1e..327f91be 100644 --- a/pkg/formatter/render_injection_632_test.go +++ b/pkg/formatter/render_injection_632_test.go @@ -101,8 +101,8 @@ func TestQuotedIdentifier_Format_ConsistentWithDefinitions_PR632(t *testing.T) { sql string want string }{ - {"order_by_alias", `SELECT amount AS "Total" FROM t ORDER BY "Total"`, "SELECT amount AS Total\nFROM t\nORDER BY Total"}, - {"cte_column", `WITH cte (a, "B") AS (SELECT 1, 2) SELECT "B" FROM cte`, "WITH cte (a, B) AS (SELECT 1, 2)\nSELECT B\nFROM cte"}, + {"order_by_alias", `SELECT amount AS "Total" FROM t ORDER BY "Total"`, "SELECT amount AS \"Total\"\nFROM t\nORDER BY \"Total\""}, + {"cte_column", `WITH cte (a, "B") AS (SELECT 1, 2) SELECT "B" FROM cte`, "WITH cte (a, \"B\") AS (SELECT 1, 2)\nSELECT \"B\"\nFROM cte"}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { diff --git a/pkg/formatter/render_keyword_allowlist_632_test.go b/pkg/formatter/render_keyword_allowlist_632_test.go index ef2c9e00..981d1851 100644 --- a/pkg/formatter/render_keyword_allowlist_632_test.go +++ b/pkg/formatter/render_keyword_allowlist_632_test.go @@ -241,7 +241,7 @@ func TestKeywordFieldsKnownValuesRoundTrip_PR632(t *testing.T) { want string }{ {keywords.DialectGeneric, "SELECT a FROM t INTERSECT ALL SELECT b FROM u", "INTERSECT ALL"}, - {keywords.DialectOracle, "SELECT a FROM t MINUS SELECT b FROM u", "EXCEPT"}, + {keywords.DialectOracle, "SELECT a FROM t MINUS SELECT b FROM u", "MINUS"}, {keywords.DialectGeneric, "SELECT a FROM t NATURAL JOIN u", "NATURAL JOIN"}, {keywords.DialectGeneric, "SELECT a FROM t NATURAL FULL JOIN u", "NATURAL FULL JOIN"}, {keywords.DialectGeneric, "DROP INDEX i", "DROP INDEX i"}, diff --git a/pkg/formatter/render_keyword_fields_632_test.go b/pkg/formatter/render_keyword_fields_632_test.go index 7b78ace5..3828d5f3 100644 --- a/pkg/formatter/render_keyword_fields_632_test.go +++ b/pkg/formatter/render_keyword_fields_632_test.go @@ -261,8 +261,10 @@ func TestQuotedConstraintNamesRoundTrip_PR632(t *testing.T) { {`CREATE TABLE t (a INT, CONSTRAINT "default" CHECK (a > 0))`, `CONSTRAINT "default" CHECK`}, {`CREATE TABLE t (a INT, CONSTRAINT "table" PRIMARY KEY (a))`, `CONSTRAINT "table" PRIMARY KEY`}, {`CREATE TABLE t (a INT, CONSTRAINT "owner" PRIMARY KEY (a))`, `CONSTRAINT "owner" PRIMARY KEY`}, - // Quoted only for case: rendered bare, like Identifier.SQL(). - {`CREATE TABLE t (a INT, CONSTRAINT "MixedCase" PRIMARY KEY (a))`, `CONSTRAINT MixedCase PRIMARY KEY`}, + // Quoted only for case: the quotes are kept where the dialect folds + // unquoted names (generic, PostgreSQL) and dropped in SQLite, so only + // the name is checked here; the re-parse check below pins the name. + {`CREATE TABLE t (a INT, CONSTRAINT "MixedCase" PRIMARY KEY (a))`, `MixedCase`}, {`CREATE TABLE t (a INT, CONSTRAINT PK_USERS PRIMARY KEY (a))`, `CONSTRAINT PK_USERS PRIMARY KEY`}, {`CREATE TABLE t (a INT, CONSTRAINT pk_t PRIMARY KEY (a))`, `CONSTRAINT pk_t PRIMARY KEY`}, } diff --git a/pkg/formatter/render_query_ext.go b/pkg/formatter/render_query_ext.go new file mode 100644 index 00000000..d3ca3a35 --- /dev/null +++ b/pkg/formatter/render_query_ext.go @@ -0,0 +1,64 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// render_query_ext.go renders the query-expression parts that the formatter +// used to drop: CTE MATERIALIZED / NOT MATERIALIZED, ClickHouse scalar CTEs, +// and the table-factor modifiers of a FROM item. + +package formatter + +import ( + "strings" + + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" +) + +// cteSQL renders one common table expression of a WITH clause. +func cteSQL(cte *ast.CommonTableExpr, f *nodeFormatter) string { + if cte == nil { + return "" + } + var sb strings.Builder + // ClickHouse scalar CTE: WITH AS + if cte.ScalarExpr != nil { + sb.WriteString(FormatExpression(cte.ScalarExpr, f.opts)) + sb.WriteString(" ") + sb.WriteString(f.kw("AS")) + sb.WriteString(" ") + sb.WriteString(ast.QuoteTableAliasWith(cte.Name, f.nq(cte.NameQuote))) + return sb.String() + } + sb.WriteString(ast.QuoteTableAliasWith(cte.Name, f.nq(cte.NameQuote))) + sb.WriteString(" ") + if len(cte.Columns) > 0 { + sb.WriteString("(") + sb.WriteString(f.quoteNamesWith(cte.Columns, cte.ColumnQuotes)) + sb.WriteString(") ") + } + sb.WriteString(f.kw("AS")) + sb.WriteString(" ") + if cte.Materialized != nil { + if !*cte.Materialized { + sb.WriteString(f.kw("NOT")) + sb.WriteString(" ") + } + sb.WriteString(f.kw("MATERIALIZED")) + sb.WriteString(" ") + } + sb.WriteString("(") + sb.WriteString(FormatStatement(cte.Statement, nestedOpts(f.opts))) + sb.WriteString(")") + sb.WriteString(ast.CTESearchCycleSQL(cte, f.kw)) + return sb.String() +} diff --git a/pkg/formatter/render_select_ext.go b/pkg/formatter/render_select_ext.go new file mode 100644 index 00000000..9598cd15 --- /dev/null +++ b/pkg/formatter/render_select_ext.go @@ -0,0 +1,486 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// render_select_ext.go renders the SELECT clauses that renderSelect used to +// drop or mangle: select modifiers, TOP parentheses, INTO, ARRAY JOIN, +// PREWHERE, GROUP BY modifiers, QUALIFY, START WITH / CONNECT BY, ORDER +// SIBLINGS BY, LIMIT BY, LIMIT/OFFSET expressions, FOR JSON/XML, OPTION (...) +// and FORMAT. + +package formatter + +import ( + "strconv" + "strings" + + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" +) + +// noSelectExtras is the read-only stand-in selectExtras returns for a +// statement without Extras. Nothing may write through it. +var noSelectExtras ast.SelectExtras + +// selectExtras returns s.Extras, or a read-only empty SelectExtras when it is +// nil, so the renderers can read the rarer clauses without nil checks. +func selectExtras(s *ast.SelectStatement) *ast.SelectExtras { + if s.Extras != nil { + return s.Extras + } + return &noSelectExtras +} + +// writeSelectPrologue writes the MySQL select modifiers and the TOP clause, +// each followed by a space. +func writeSelectPrologue(sb *strings.Builder, s *ast.SelectStatement, f *nodeFormatter) { + for _, m := range selectExtras(s).SelectModifiers { + if t := ast.SelectModifierText(m); t != "" { + sb.WriteString(f.kw(t)) + sb.WriteString(" ") + } + } + if s.Top == nil || s.Top.Count == nil { + return + } + sb.WriteString(f.kw("TOP")) + sb.WriteString(" ") + if ast.TopNeedsParens(s.Top) { + sb.WriteString("(") + sb.WriteString(FormatExpression(s.Top.Count, f.opts)) + sb.WriteString(")") + } else { + sb.WriteString(FormatExpression(s.Top.Count, f.opts)) + } + if s.Top.IsPercent { + sb.WriteString(" ") + sb.WriteString(f.kw("PERCENT")) + } + if s.Top.WithTies { + sb.WriteString(" ") + sb.WriteString(f.kw("WITH TIES")) + } + sb.WriteString(" ") +} + +// selectIntoSQL renders "INTO ..." (without a leading separator). +func selectIntoSQL(in *ast.SelectInto, f *nodeFormatter) string { + var sb strings.Builder + sb.WriteString(f.kw("INTO")) + sb.WriteString(" ") + switch in.Kind { + case ast.SelectIntoVariables: + sb.WriteString(exprListSQL(in.Targets)) + case ast.SelectIntoOutfile, ast.SelectIntoDumpfile: + sb.WriteString(f.kw(in.Kind)) + sb.WriteString(" ") + sb.WriteString(exprSQL(in.File)) + if in.Kind == ast.SelectIntoOutfile { + if in.CharacterSet != "" { + sb.WriteString(" ") + sb.WriteString(f.kw("CHARACTER SET")) + sb.WriteString(" ") + sb.WriteString(ast.QuoteName(in.CharacterSet)) + } + writeIntoFileOptions(&sb, "FIELDS", in.FieldsOptions, f) + writeIntoFileOptions(&sb, "LINES", in.LinesOptions, f) + } + default: + if t := ast.SelectIntoTemporaryText(in.Temporary); t != "" { + sb.WriteString(f.kw(t)) + sb.WriteString(" ") + } + if in.TableKeyword { + sb.WriteString(f.kw("TABLE")) + sb.WriteString(" ") + } + sb.WriteString(ast.QuoteQualifiedName(in.Table)) + } + return sb.String() +} + +func writeIntoFileOptions(sb *strings.Builder, clause string, opts []ast.IntoFileOption, f *nodeFormatter) { + wrote := false + for _, o := range opts { + kw := ast.IntoFileOptionText(o.Option) + if kw == "" || o.Value == nil { + continue + } + if !wrote { + sb.WriteString(" ") + sb.WriteString(f.kw(clause)) + wrote = true + } + sb.WriteString(" ") + sb.WriteString(f.kw(kw)) + sb.WriteString(" ") + sb.WriteString(exprSQL(o.Value)) + } +} + +// arrayJoinSQL renders a ClickHouse [LEFT] ARRAY JOIN clause. +func arrayJoinSQL(a *ast.ArrayJoinClause, f *nodeFormatter) string { + var sb strings.Builder + if a.Left { + sb.WriteString(f.kw("LEFT ARRAY JOIN")) + } else { + sb.WriteString(f.kw("ARRAY JOIN")) + } + sb.WriteString(" ") + for i, e := range a.Elements { + if i > 0 { + sb.WriteString(", ") + } + sb.WriteString(exprSQL(e.Expr)) + if e.Alias != "" { + sb.WriteString(" ") + sb.WriteString(f.kw("AS")) + sb.WriteString(" ") + sb.WriteString(ast.QuoteName(e.Alias)) + } + } + return sb.String() +} + +// groupBySQL renders the GROUP BY clause with its modifiers, or "" when the +// statement has none. +func groupBySQL(s *ast.SelectStatement, f *nodeFormatter) string { + m := s.DialectExtras().GroupByModifiers + if len(s.GroupBy) == 0 && !m.All { + return "" + } + var sb strings.Builder + sb.WriteString(f.kw("GROUP BY")) + sb.WriteString(" ") + if m.All { + sb.WriteString(f.kw("ALL")) + } else { + sb.WriteString(exprListSQL(s.GroupBy)) + } + switch { + case m.WithRollup: + sb.WriteString(" ") + sb.WriteString(f.kw("WITH ROLLUP")) + case m.WithCube: + sb.WriteString(" ") + sb.WriteString(f.kw("WITH CUBE")) + } + if m.WithTotals { + sb.WriteString(" ") + sb.WriteString(f.kw("WITH TOTALS")) + } + return sb.String() +} + +// writeHierarchical writes QUALIFY and the Oracle / MariaDB START WITH and +// CONNECT BY clauses, each preceded by a clause separator. +func writeHierarchical(sb *strings.Builder, s *ast.SelectStatement, f *nodeFormatter) { + if s.Qualify != nil { + sb.WriteString(f.clauseSep()) + sb.WriteString(f.kw("QUALIFY")) + sb.WriteString(" ") + sb.WriteString(FormatExpression(s.Qualify, f.opts)) + } + if s.StartWith != nil { + sb.WriteString(f.clauseSep()) + sb.WriteString(f.kw("START WITH")) + sb.WriteString(" ") + sb.WriteString(FormatExpression(s.StartWith, f.opts)) + } + if s.ConnectBy != nil { + sb.WriteString(f.clauseSep()) + sb.WriteString(f.kw("CONNECT BY")) + sb.WriteString(" ") + if s.ConnectBy.NoCycle { + sb.WriteString(f.kw("NOCYCLE")) + sb.WriteString(" ") + } + sb.WriteString(FormatExpression(s.ConnectBy.Condition, f.opts)) + } +} + +// writeLimitOffsetClauses writes LIMIT BY, LIMIT and OFFSET, each preceded by +// a clause separator. OFFSET keeps ROWS when it was written or a FETCH +// clause follows. +func writeLimitOffsetClauses(sb *strings.Builder, s *ast.SelectStatement, f *nodeFormatter) { + if lb := selectExtras(s).LimitBy; lb != nil && lb.Count != nil && len(lb.Exprs) > 0 { + sb.WriteString(f.clauseSep()) + sb.WriteString(f.kw("LIMIT")) + sb.WriteString(" ") + sb.WriteString(exprSQL(lb.Count)) + if lb.Offset != nil { + sb.WriteString(" ") + sb.WriteString(f.kw("OFFSET")) + sb.WriteString(" ") + sb.WriteString(exprSQL(lb.Offset)) + } + sb.WriteString(" ") + sb.WriteString(f.kw("BY")) + sb.WriteString(" ") + sb.WriteString(exprListSQL(lb.Exprs)) + } + if s.HasLimit() { + sb.WriteString(f.clauseSep()) + sb.WriteString(f.kw("LIMIT")) + sb.WriteString(" ") + switch { + case s.Limit != nil: + sb.WriteString(strconv.Itoa(*s.Limit)) + case s.DialectExtras().LimitExpr != nil: + sb.WriteString(exprSQL(s.DialectExtras().LimitExpr)) + default: + sb.WriteString(f.kw("ALL")) + } + if s.DialectExtras().LimitWithTies { + sb.WriteString(" ") + sb.WriteString(f.kw("WITH TIES")) + } + } + if s.HasOffset() { + sb.WriteString(f.clauseSep()) + sb.WriteString(f.kw("OFFSET")) + sb.WriteString(" ") + if s.Offset != nil { + sb.WriteString(strconv.Itoa(*s.Offset)) + } else { + sb.WriteString(exprSQL(s.DialectExtras().OffsetExpr)) + } + if s.DialectExtras().OffsetRows || s.Fetch != nil { + sb.WriteString(" ") + sb.WriteString(f.kw(ast.OffsetRowsKeyword(s))) + } + } + if x := s.Extras; x != nil && x.RowsExamined != nil { + sb.WriteString(f.clauseSep()) + sb.WriteString(f.kw("ROWS EXAMINED")) + sb.WriteString(" ") + sb.WriteString(exprSQL(x.RowsExamined)) + } +} + +// forOutputSQL renders " FOR JSON|XML|BROWSE ..." (with a leading space), or +// "" for an unknown kind. +func forOutputSQL(fo *ast.ForOutputClause, f *nodeFormatter) string { + kind := ast.ForOutputKindText(fo.Kind) + if kind == "" { + return "" + } + var sb strings.Builder + sb.WriteString(" ") + sb.WriteString(f.kw("FOR")) + sb.WriteString(" ") + sb.WriteString(f.kw(kind)) + if mode := ast.ForOutputModeText(fo.Mode); mode != "" { + sb.WriteString(" ") + sb.WriteString(f.kw(mode)) + if fo.ModeArg != nil { + sb.WriteString("(") + sb.WriteString(exprSQL(fo.ModeArg)) + sb.WriteString(")") + } + } + for _, o := range fo.Options { + name := ast.ForOutputOptionText(o.Name) + if name == "" { + continue + } + sb.WriteString(", ") + sb.WriteString(f.kw(name)) + if o.Arg != nil { + sb.WriteString("(") + sb.WriteString(exprSQL(o.Arg)) + sb.WriteString(")") + } + } + return sb.String() +} + +// queryHintsSQL renders "OPTION (hint, ...)" (without a leading separator). +func queryHintsSQL(hints []ast.QueryHint, f *nodeFormatter) string { + var sb strings.Builder + sb.WriteString(f.kw("OPTION")) + sb.WriteString(" (") + first := true + for i := range hints { + h := &hints[i] + name := ast.QueryHintNameText(h.Name) + if name == "" { + continue + } + if !first { + sb.WriteString(", ") + } + first = false + sb.WriteString(f.kw(name)) + if h.Value != nil { + if h.Equals { + sb.WriteString(" = ") + } else { + sb.WriteString(" ") + } + sb.WriteString(exprSQL(h.Value)) + } + if h.HasArgs { + sb.WriteString(" (") + for j, a := range h.Args { + if j > 0 { + sb.WriteString(", ") + } + sb.WriteString(exprSQL(a.Expr)) + if a.Unknown { + sb.WriteString(" ") + sb.WriteString(f.kw("UNKNOWN")) + } + } + sb.WriteString(")") + } + } + sb.WriteString(")") + return sb.String() +} + +// writeSelectEpilogue writes the clauses after FOR UPDATE: FOR JSON/XML, +// SETTINGS, OPTION (...) and FORMAT. +func writeSelectEpilogue(sb *strings.Builder, s *ast.SelectStatement, f *nodeFormatter) { + x := selectExtras(s) + if x.ForOutput != nil { + sb.WriteString(forOutputSQL(x.ForOutput, f)) + } + if len(s.Settings) > 0 { + sb.WriteString(f.clauseSep()) + sb.WriteString(f.kw("SETTINGS")) + sb.WriteString(" ") + parts := make([]string, len(s.Settings)) + for i := range s.Settings { + parts[i] = s.Settings[i].SQL() + } + sb.WriteString(strings.Join(parts, ", ")) + } + if len(x.QueryHints) > 0 { + sb.WriteString(f.clauseSep()) + sb.WriteString(queryHintsSQL(x.QueryHints, f)) + } + if n := ast.FormatNameText(x.OutputFormat); n != "" { + sb.WriteString(f.clauseSep()) + sb.WriteString(f.kw("FORMAT")) + sb.WriteString(" ") + sb.WriteString(n) + } +} + +// joinKeywordsSQL renders the keywords of a join ("LEFT JOIN", "CROSS +// APPLY", "GLOBAL ANY LEFT JOIN", ...) with the formatter's keyword case. +func joinKeywordsSQL(j *ast.JoinClause, f *nodeFormatter) string { + return f.kw(ast.JoinKeywordsText(j)) +} + +// interpolateSQL renders " INTERPOLATE [(col [AS expr], ...)]". +func interpolateSQL(c *ast.InterpolateClause, f *nodeFormatter) string { + var sb strings.Builder + sb.WriteString(" ") + sb.WriteString(f.kw("INTERPOLATE")) + if len(c.Items) == 0 { + return sb.String() + } + sb.WriteString(" (") + for i, it := range c.Items { + if i > 0 { + sb.WriteString(", ") + } + sb.WriteString(ast.QuoteName(it.Column)) + if it.Expr != nil { + sb.WriteString(" ") + sb.WriteString(f.kw("AS")) + sb.WriteString(" ") + sb.WriteString(exprSQL(it.Expr)) + } + } + sb.WriteString(")") + return sb.String() +} + +// normalizeLimitForDialect rewrites LIMIT / OFFSET (integer or expression +// values) into TOP or OFFSET ... FETCH for SQL Server and into FETCH FIRST +// for Oracle, on the caller's shallow copy of the statement. LIMIT ALL means +// no limit and is dropped for those dialects. +func normalizeLimitForDialect(s *ast.SelectStatement, dialect string) { + if dialect != "sqlserver" && dialect != "oracle" { + return + } + if s.Extras != nil { + // renderSelect copied the statement; detach metadata before normalization. + x := *s.Extras + s.Extras = &x + s.Extras.LimitAll = false + } + if s.Limit == nil && s.DialectExtras().LimitExpr == nil { + return + } + newFetch := func(fetchType string) *ast.FetchClause { + fc := &ast.FetchClause{FetchType: fetchType} + if s.Limit != nil { + v := int64(*s.Limit) + fc.FetchValue = &v + } else { + fc.FetchExpr = s.DialectExtras().LimitExpr + } + return fc + } + clearLimit := func() { + s.Limit = nil + if s.Extras != nil { + s.Extras.LimitExpr, s.Extras.LimitWithTies = nil, false + } + } + // moveOffset moves an integer OFFSET into the FETCH clause; an OFFSET + // expression stays on the statement and renders as OFFSET expr ROWS. + moveOffset := func(fc *ast.FetchClause, defaultZero bool) { + switch { + case s.Offset != nil: + v := int64(*s.Offset) + fc.OffsetValue = &v + s.Offset = nil + case s.DialectExtras().OffsetExpr == nil && defaultZero: + var zero int64 + fc.OffsetValue = &zero + } + } + + switch dialect { + case "sqlserver": + if s.Top != nil { + return + } + if s.HasOffset() || len(s.OrderBy) > 0 { + // SQL Server 2012+ OFFSET/FETCH syntax (requires ORDER BY in practice, + // but we emit it faithfully and let the database validate). + s.Fetch = newFetch("NEXT") + moveOffset(s.Fetch, true) + } else { + // Simple TOP N + var count ast.Expression = s.DialectExtras().LimitExpr + if s.Limit != nil { + count = &ast.LiteralValue{Value: *s.Limit, Type: "int"} + } + s.Top = &ast.TopClause{Count: count} + } + clearLimit() + + case "oracle": + if s.Fetch != nil { + return + } + s.Fetch = newFetch("FIRST") + moveOffset(s.Fetch, false) + clearLimit() + } +} diff --git a/pkg/formatter/select_ext_dialect_test.go b/pkg/formatter/select_ext_dialect_test.go new file mode 100644 index 00000000..0322198f --- /dev/null +++ b/pkg/formatter/select_ext_dialect_test.go @@ -0,0 +1,71 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package formatter + +import ( + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" +) + +// TestNormalizeLimitForDialect checks that LIMIT / OFFSET values, integer or +// expression, become TOP / OFFSET ... FETCH for SQL Server and FETCH FIRST +// for Oracle when formatting for those dialects. +func TestNormalizeLimitForDialect(t *testing.T) { + ph := func(v string) *ast.LiteralValue { return &ast.LiteralValue{Value: v, Type: "placeholder"} } + intp := func(v int) *int { return &v } + order := []ast.OrderByExpression{{Expression: &ast.Identifier{Name: "a"}, Ascending: true}} + cases := []struct { + name string + dialect string + mod func(s *ast.SelectStatement) + want string + }{ + {"sqlserver limit expr to top", "sqlserver", func(s *ast.SelectStatement) { s.EnsureExtras().LimitExpr = ph("@n") }, "SELECT TOP (@n) a FROM t"}, + {"sqlserver limit int to top", "sqlserver", func(s *ast.SelectStatement) { s.Limit = intp(5) }, "SELECT TOP 5 a FROM t"}, + {"sqlserver limit expr offset expr", "sqlserver", func(s *ast.SelectStatement) { + s.OrderBy = order + s.EnsureExtras().LimitExpr = ph("@n") + s.EnsureExtras().OffsetExpr = ph("@s") + }, "SELECT a FROM t ORDER BY a OFFSET @s ROWS FETCH NEXT @n ROWS ONLY"}, + {"sqlserver limit int offset int", "sqlserver", func(s *ast.SelectStatement) { + s.OrderBy = order + s.Limit = intp(10) + s.Offset = intp(20) + }, "SELECT a FROM t ORDER BY a OFFSET 20 ROWS FETCH NEXT 10 ROWS ONLY"}, + {"sqlserver limit all dropped", "sqlserver", func(s *ast.SelectStatement) { s.EnsureExtras().LimitAll = true }, "SELECT a FROM t"}, + {"oracle limit expr", "oracle", func(s *ast.SelectStatement) { s.EnsureExtras().LimitExpr = ph(":n") }, "SELECT a FROM t FETCH FIRST :n ROWS ONLY"}, + {"oracle limit with offset expr", "oracle", func(s *ast.SelectStatement) { + s.EnsureExtras().LimitExpr = ph(":n") + s.EnsureExtras().OffsetExpr = ph(":o") + }, "SELECT a FROM t OFFSET :o ROWS FETCH FIRST :n ROWS ONLY"}, + {"postgres untouched", "postgresql", func(s *ast.SelectStatement) { s.EnsureExtras().LimitExpr = ph("$1") }, "SELECT a FROM t LIMIT $1"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s := &ast.SelectStatement{Columns: []ast.Expression{&ast.Identifier{Name: "a"}}, From: []ast.TableReference{{Name: "t"}}} + tc.mod(s) + opts := ast.CompactStyle() + opts.Dialect = tc.dialect + got := FormatStatement(s, opts) + if got != tc.want { + t.Errorf("got: %s\nwant: %s", got, tc.want) + } + if s.DialectExtras().LimitExpr == nil && s.Limit == nil && !s.DialectExtras().LimitAll && tc.dialect != "postgresql" { + t.Errorf("FormatStatement mutated the input statement") + } + }) + } +} diff --git a/pkg/formatter/select_ext_render_test.go b/pkg/formatter/select_ext_render_test.go new file mode 100644 index 00000000..393078a0 --- /dev/null +++ b/pkg/formatter/select_ext_render_test.go @@ -0,0 +1,292 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package formatter_test + +import ( + "reflect" + "strings" + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/formatter" + "github.com/ajitpratap0/GoSQLX/pkg/gosqlx" + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" +) + +// TestRender_SelectBodyClauses checks that SELECT-body clauses survive the +// formatter and tree.SQL(): the compact output must equal want, re-parse with +// the same dialect, and format to itself again. +func TestRender_SelectBodyClauses(t *testing.T) { + cases := []struct { + name string + dialect keywords.SQLDialect + sql string + want string + }{ + {"limit placeholder", keywords.DialectPostgreSQL, "SELECT 1 FROM t LIMIT $1 OFFSET $2", "SELECT 1 FROM t LIMIT $1 OFFSET $2"}, + {"limit all offset rows", keywords.DialectPostgreSQL, "SELECT * FROM t LIMIT ALL OFFSET 5 ROWS", "SELECT * FROM t LIMIT ALL OFFSET 5 ROWS"}, + {"offset before limit", keywords.DialectPostgreSQL, "SELECT * FROM t OFFSET 10 LIMIT 5", "SELECT * FROM t LIMIT 5 OFFSET 10"}, + {"limit expression", keywords.DialectPostgreSQL, "SELECT * FROM t LIMIT (SELECT 1)", "SELECT * FROM t LIMIT (SELECT 1)"}, + {"fetch first row", keywords.DialectPostgreSQL, "SELECT * FROM t FETCH FIRST ROW ONLY", "SELECT * FROM t FETCH FIRST ROW ONLY"}, + {"order by using", keywords.DialectPostgreSQL, "SELECT a FROM t ORDER BY a USING <", "SELECT a FROM t ORDER BY a USING <"}, + {"select into postgres", keywords.DialectPostgreSQL, "SELECT * INTO TEMP TABLE n FROM o WHERE x > 1", "SELECT * INTO TEMP TABLE n FROM o WHERE x > 1"}, + {"where without from", keywords.DialectPostgreSQL, "SELECT 1 WHERE 1 = 1", "SELECT 1 WHERE 1 = 1"}, + {"mysql limit offset count", keywords.DialectMySQL, "SELECT * FROM t LIMIT ?, ?", "SELECT * FROM t LIMIT ? OFFSET ?"}, + {"mysql with rollup", keywords.DialectMySQL, "SELECT a, b FROM t GROUP BY a, b WITH ROLLUP", "SELECT a, b FROM t GROUP BY a, b WITH ROLLUP"}, + {"mysql modifiers", keywords.DialectMySQL, "SELECT SQL_CALC_FOUND_ROWS * FROM t", "SELECT SQL_CALC_FOUND_ROWS * FROM t"}, + {"mysql straight join", keywords.DialectMySQL, "SELECT * FROM a STRAIGHT_JOIN b ON a.id = b.id", "SELECT * FROM a STRAIGHT_JOIN b ON a.id = b.id"}, + {"mysql join without on", keywords.DialectMySQL, "SELECT * FROM a JOIN b", "SELECT * FROM a INNER JOIN b"}, + {"mysql comma after join", keywords.DialectMySQL, "SELECT * FROM a LEFT JOIN b ON a.id = b.id, t3", "SELECT * FROM a LEFT JOIN b ON a.id = b.id , t3"}, + {"mysql lock in share mode", keywords.DialectMySQL, "SELECT * FROM t WHERE id = 1 LOCK IN SHARE MODE", "SELECT * FROM t WHERE id = 1 LOCK IN SHARE MODE"}, + {"mysql into vars", keywords.DialectMySQL, "SELECT a, b INTO @x, @y FROM t", "SELECT a, b INTO @x, @y FROM t"}, + {"mysql outfile", keywords.DialectMySQL, "SELECT * FROM o INTO OUTFILE '/tmp/o.csv' FIELDS TERMINATED BY ',' LINES TERMINATED BY ';'", "SELECT * FROM o INTO OUTFILE '/tmp/o.csv' FIELDS TERMINATED BY ',' LINES TERMINATED BY ';'"}, + {"mariadb wait", keywords.DialectMariaDB, "SELECT * FROM t FOR UPDATE WAIT 5", "SELECT * FROM t FOR UPDATE WAIT 5"}, + {"sqlite negative limit", keywords.DialectSQLite, "SELECT a FROM t LIMIT -1 OFFSET 5", "SELECT a FROM t LIMIT -1 OFFSET 5"}, + {"sqlserver offset rows fetch", keywords.DialectSQLServer, "SELECT a FROM t ORDER BY a OFFSET 5 ROWS FETCH NEXT 5 ROWS ONLY", "SELECT a FROM t ORDER BY a OFFSET 5 ROWS FETCH NEXT 5 ROWS ONLY"}, + {"sqlserver offset fetch vars", keywords.DialectSQLServer, "SELECT * FROM t ORDER BY id OFFSET @s ROWS FETCH NEXT @n ROWS ONLY", "SELECT * FROM t ORDER BY id OFFSET @s ROWS FETCH NEXT @n ROWS ONLY"}, + {"sqlserver top in derived table", keywords.DialectSQLServer, "SELECT * FROM (SELECT TOP (10) a FROM t ORDER BY a) AS s", "SELECT * FROM (SELECT TOP (10) a FROM t ORDER BY a) s"}, + {"sqlserver top variable", keywords.DialectSQLServer, "SELECT TOP (@n) * FROM t", "SELECT TOP (@n) * FROM t"}, + {"sqlserver cross apply", keywords.DialectSQLServer, "SELECT * FROM a CROSS APPLY (SELECT 1 AS x) AS t", "SELECT * FROM a CROSS APPLY (SELECT 1 AS x) t"}, + {"sqlserver apply table function", keywords.DialectSQLServer, "SELECT t.text FROM qs OUTER APPLY sys.dm_exec_sql_text(qs.h) AS t", "SELECT t.text FROM qs OUTER APPLY sys.dm_exec_sql_text(qs.h) t"}, + {"sqlserver hash join", keywords.DialectSQLServer, "SELECT o.Id FROM o INNER HASH JOIN c ON c.Id = o.CId", "SELECT o.Id FROM o INNER HASH JOIN c ON c.Id = o.CId"}, + {"sqlserver into temp", keywords.DialectSQLServer, "SELECT * INTO #t FROM u", "SELECT * INTO #t FROM u"}, + {"sqlserver for json", keywords.DialectSQLServer, "SELECT a FROM t FOR JSON PATH, ROOT('r'), INCLUDE_NULL_VALUES", "SELECT a FROM t FOR JSON PATH, ROOT('r'), INCLUDE_NULL_VALUES"}, + {"sqlserver for xml", keywords.DialectSQLServer, "SELECT a FROM t FOR XML PATH('x'), ROOT('y'), TYPE", "SELECT a FROM t FOR XML PATH('x'), ROOT('y'), TYPE"}, + {"sqlserver option", keywords.DialectSQLServer, "SELECT * FROM t WHERE c = @c OPTION (MAXDOP 1, RECOMPILE, OPTIMIZE FOR (@c UNKNOWN))", "SELECT * FROM t WHERE c = @c OPTION (MAXDOP 1, RECOMPILE, OPTIMIZE FOR (@c UNKNOWN))"}, + {"sqlserver with rollup", keywords.DialectSQLServer, "SELECT a, SUM(b) FROM t GROUP BY a WITH ROLLUP", "SELECT a, SUM(b) FROM t GROUP BY a WITH ROLLUP"}, + {"snowflake qualify", keywords.DialectSnowflake, "SELECT * FROM t QUALIFY ROW_NUMBER() OVER (ORDER BY a) = 1", "SELECT * FROM t QUALIFY ROW_NUMBER() OVER (ORDER BY a) = 1"}, + {"snowflake top", keywords.DialectSnowflake, "SELECT TOP 10 a FROM t", "SELECT TOP 10 a FROM t"}, + {"snowflake star modifiers", keywords.DialectSnowflake, "SELECT * EXCLUDE (a, b) REPLACE (UPPER(n) AS n) FROM t", "SELECT * EXCLUDE (a, b) REPLACE (UPPER(n) AS n) FROM t"}, + {"snowflake exclude qualified name", keywords.DialectSnowflake, "SELECT t.* EXCLUDE (t.a, \"my col\") FROM t", "SELECT t.* EXCLUDE (t.a, \"my col\") FROM t"}, + {"snowflake ilike rename", keywords.DialectSnowflake, "SELECT * ILIKE '%id%', b.* RENAME (id AS b_id) FROM b", "SELECT * ILIKE '%id%', b.* RENAME (id AS b_id) FROM b"}, + {"snowflake group by all", keywords.DialectSnowflake, "SELECT a, COUNT(*) FROM t GROUP BY ALL", "SELECT a, COUNT(*) FROM t GROUP BY ALL"}, + {"snowflake asof", keywords.DialectSnowflake, "SELECT * FROM t1 ASOF JOIN t2 MATCH_CONDITION (t1.ts >= t2.ts) ON t1.sym = t2.sym", "SELECT * FROM t1 ASOF JOIN t2 MATCH_CONDITION (t1.ts >= t2.ts) ON t1.sym = t2.sym"}, + {"oracle connect by", keywords.DialectOracle, "SELECT id FROM e START WITH id = 1 CONNECT BY PRIOR id = m", "SELECT id FROM e START WITH id = 1 CONNECT BY PRIOR id = m"}, + {"oracle row generator", keywords.DialectOracle, "SELECT LEVEL FROM dual CONNECT BY LEVEL <= 10", "SELECT LEVEL FROM dual CONNECT BY LEVEL <= 10"}, + {"oracle siblings", keywords.DialectOracle, "SELECT n FROM e START WITH m IS NULL CONNECT BY PRIOR id = m ORDER SIBLINGS BY n", "SELECT n FROM e START WITH m IS NULL CONNECT BY PRIOR id = m ORDER SIBLINGS BY n"}, + {"oracle fetch first 1 row", keywords.DialectOracle, "SELECT a FROM t FETCH FIRST 1 ROW ONLY", "SELECT a FROM t FETCH FIRST 1 ROW ONLY"}, + {"oracle outer apply", keywords.DialectOracle, "SELECT d.n FROM d OUTER APPLY (SELECT l FROM e WHERE e.d = d.id) e", "SELECT d.n FROM d OUTER APPLY (SELECT l FROM e WHERE e.d = d.id) e"}, + {"clickhouse prewhere", keywords.DialectClickHouse, "SELECT * FROM t PREWHERE a = 1 WHERE b = 2", "SELECT * FROM t PREWHERE a = 1 WHERE b = 2"}, + {"clickhouse array join", keywords.DialectClickHouse, "SELECT id, tag FROM t LEFT ARRAY JOIN tags AS tag", "SELECT id, tag FROM t LEFT ARRAY JOIN tags AS tag"}, + {"clickhouse limit by", keywords.DialectClickHouse, "SELECT u, p FROM e ORDER BY ts DESC LIMIT 2 BY u, p LIMIT 100", "SELECT u, p FROM e ORDER BY ts DESC LIMIT 2 BY u, p LIMIT 100"}, + {"clickhouse with totals", keywords.DialectClickHouse, "SELECT d, sum(x) FROM s GROUP BY d WITH TOTALS ORDER BY d", "SELECT d, sum(x) FROM s GROUP BY d WITH TOTALS ORDER BY d"}, + {"clickhouse global any left join", keywords.DialectClickHouse, "SELECT * FROM a GLOBAL ANY LEFT JOIN b USING (id)", "SELECT * FROM a GLOBAL ANY LEFT JOIN b USING (id)"}, + {"clickhouse asof join", keywords.DialectClickHouse, "SELECT a FROM t ASOF JOIN u ON t.a = u.a AND t.ts >= u.ts", "SELECT a FROM t ASOF JOIN u ON t.a = u.a AND t.ts >= u.ts"}, + {"clickhouse left anti join", keywords.DialectClickHouse, "SELECT * FROM a LEFT ANTI JOIN b ON a.id = b.id", "SELECT * FROM a ANTI LEFT JOIN b ON a.id = b.id"}, + {"clickhouse star except", keywords.DialectClickHouse, "SELECT * EXCEPT (a) FROM t", "SELECT * EXCEPT (a) FROM t"}, + {"clickhouse format", keywords.DialectClickHouse, "SELECT a FROM t SETTINGS max_threads = 2 FORMAT JSONEachRow", "SELECT a FROM t SETTINGS max_threads = 2 FORMAT JSONEachRow"}, + {"clickhouse limit with ties", keywords.DialectClickHouse, "SELECT x FROM t ORDER BY x LIMIT 10 WITH TIES", "SELECT x FROM t ORDER BY x LIMIT 10 WITH TIES"}, + {"clickhouse interpolate", keywords.DialectClickHouse, "SELECT d, c FROM t ORDER BY d WITH FILL STEP 1 INTERPOLATE (c AS c + 1)", "SELECT d, c FROM t ORDER BY d WITH FILL STEP 1 INTERPOLATE (c AS c + 1)"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + tree, err := gosqlx.ParseWithDialect(tc.sql, tc.dialect) + if err != nil { + t.Fatalf("parse %q: %v", tc.sql, err) + } + got := formatter.FormatAST(tree, ast.CompactStyle()) + if got != tc.want { + t.Errorf("formatter:\n got: %s\nwant: %s", got, tc.want) + } + if sql := tree.SQL(); sql != tc.want { + t.Errorf("SQL():\n got: %s\nwant: %s", sql, tc.want) + } + tree2, err := gosqlx.ParseWithDialect(got, tc.dialect) + if err != nil { + t.Fatalf("re-parse %q: %v", got, err) + } + if again := formatter.FormatAST(tree2, ast.CompactStyle()); again != got { + t.Errorf("not idempotent:\n first: %s\nsecond: %s", got, again) + } + readable := formatter.FormatAST(tree, ast.ReadableStyle()) + if _, err := gosqlx.ParseWithDialect(readable, tc.dialect); err != nil { + t.Errorf("readable output does not re-parse: %v\n%s", err, readable) + } + }) + } +} + +// selectFieldSamples gives a non-zero sample for every exported +// SelectStatement field that must be rendered. A new field fails +// TestRender_SelectRendersEveryField until it is added here (and rendered) +// or to selectFieldsNotRendered. +func selectFieldSamples() map[string]func(*ast.SelectStatement) { + ident := func(n string) *ast.Identifier { return &ast.Identifier{Name: n} } + n := func(v int) *int { return &v } + i64 := func(v int64) *int64 { return &v } + return map[string]func(*ast.SelectStatement){ + "With": func(s *ast.SelectStatement) { + s.With = &ast.WithClause{CTEs: []*ast.CommonTableExpr{{Name: "c", Statement: &ast.SelectStatement{Columns: []ast.Expression{ident("x")}}}}} + }, + "Distinct": func(s *ast.SelectStatement) { s.Distinct = true }, + "DistinctOnColumns": func(s *ast.SelectStatement) { s.DistinctOnColumns = []ast.Expression{ident("d")} }, + "Top": func(s *ast.SelectStatement) { + s.Top = &ast.TopClause{Count: &ast.LiteralValue{Value: "5", Type: "int"}} + }, + "Columns": func(s *ast.SelectStatement) { s.Columns = append(s.Columns, ident("extra")) }, + "From": func(s *ast.SelectStatement) { s.From = append(s.From, ast.TableReference{Name: "u"}) }, + "Joins": func(s *ast.SelectStatement) { + s.Joins = []ast.JoinClause{{Type: "LEFT", Right: ast.TableReference{Name: "j"}, Condition: ident("ok")}} + }, + "ArrayJoin": func(s *ast.SelectStatement) { + s.ArrayJoin = &ast.ArrayJoinClause{Elements: []ast.ArrayJoinElement{{Expr: ident("arr")}}} + }, + "PrewhereClause": func(s *ast.SelectStatement) { s.PrewhereClause = ident("pw") }, + "Sample": func(s *ast.SelectStatement) { s.Sample = &ast.SampleClause{Value: "0.1"} }, + "Where": func(s *ast.SelectStatement) { s.Where = ident("w") }, + "GroupBy": func(s *ast.SelectStatement) { s.GroupBy = []ast.Expression{ident("g")} }, + "Having": func(s *ast.SelectStatement) { s.Having = ident("h") }, + "Qualify": func(s *ast.SelectStatement) { s.Qualify = ident("q") }, + "StartWith": func(s *ast.SelectStatement) { s.StartWith = ident("sw") }, + "ConnectBy": func(s *ast.SelectStatement) { s.ConnectBy = &ast.ConnectByClause{Condition: ident("cb")} }, + "Windows": func(s *ast.SelectStatement) { s.Windows = []ast.WindowSpec{{Name: "win"}} }, + "OrderBy": func(s *ast.SelectStatement) { + s.OrderBy = []ast.OrderByExpression{{Expression: ident("o"), Ascending: true}} + }, + "Limit": func(s *ast.SelectStatement) { s.Limit = n(7) }, + "Offset": func(s *ast.SelectStatement) { s.Offset = n(8) }, + "Fetch": func(s *ast.SelectStatement) { s.Fetch = &ast.FetchClause{FetchType: "FIRST", FetchValue: i64(3)} }, + "For": func(s *ast.SelectStatement) { s.For = &ast.ForClause{LockType: "UPDATE"} }, + "Settings": func(s *ast.SelectStatement) { + s.Settings = []ast.QuerySetting{{Name: "max_threads", Value: &ast.LiteralValue{Value: "2", Type: "int"}}} + }, + "SelectModifiers": func(s *ast.SelectStatement) { s.EnsureExtras().SelectModifiers = []string{"SQL_NO_CACHE"} }, + "Into": func(s *ast.SelectStatement) { + s.EnsureExtras().Into = &ast.SelectInto{Kind: ast.SelectIntoTable, Table: "dest"} + }, + "GroupByModifiers": func(s *ast.SelectStatement) { + s.GroupBy = []ast.Expression{ident("g")} + s.EnsureExtras().GroupByModifiers = ast.GroupByModifiers{WithRollup: true} + }, + "OrderSiblings": func(s *ast.SelectStatement) { + s.OrderBy = []ast.OrderByExpression{{Expression: ident("o"), Ascending: true}} + s.EnsureExtras().OrderSiblings = true + }, + "Interpolate": func(s *ast.SelectStatement) { + s.OrderBy = []ast.OrderByExpression{{Expression: ident("o"), Ascending: true}} + s.EnsureExtras().Interpolate = &ast.InterpolateClause{Items: []ast.InterpolateItem{{Column: "c"}}} + }, + "LimitExpr": func(s *ast.SelectStatement) { s.EnsureExtras().LimitExpr = &ast.Identifier{Name: "$1"} }, + "OffsetExpr": func(s *ast.SelectStatement) { s.EnsureExtras().OffsetExpr = &ast.Identifier{Name: "$2"} }, + "LimitAll": func(s *ast.SelectStatement) { s.EnsureExtras().LimitAll = true }, + "OffsetRows": func(s *ast.SelectStatement) { + s.Offset = n(8) + s.EnsureExtras().OffsetRows = true + }, + "LimitWithTies": func(s *ast.SelectStatement) { + s.Limit = n(7) + s.EnsureExtras().LimitWithTies = true + }, + "RowsExamined": func(s *ast.SelectStatement) { + s.Limit = n(7) + s.EnsureExtras().RowsExamined = &ast.LiteralValue{Value: "1000", Type: "int"} + }, + "LimitBy": func(s *ast.SelectStatement) { + s.EnsureExtras().LimitBy = &ast.LimitByClause{Count: &ast.LiteralValue{Value: "1", Type: "int"}, Exprs: []ast.Expression{ident("lb")}} + }, + "ForOutput": func(s *ast.SelectStatement) { + s.EnsureExtras().ForOutput = &ast.ForOutputClause{Kind: "JSON", Mode: "AUTO"} + }, + "QueryHints": func(s *ast.SelectStatement) { s.EnsureExtras().QueryHints = []ast.QueryHint{{Name: "RECOMPILE"}} }, + "OutputFormat": func(s *ast.SelectStatement) { s.EnsureExtras().OutputFormat = "JSONEachRow" }, + } +} + +// selectFieldsNotRendered are SelectStatement fields that carry no SQL text. +var selectFieldsNotRendered = map[string]string{ + "TableName": "pool/compatibility mirror of From[0].Name", + "Pos": "source position", +} + +// TestRender_SelectRendersEveryField sets each exported SelectStatement +// field in turn and asserts that both pkg/formatter and SelectStatement.SQL() +// output change, so a parsed clause cannot be silently dropped by either +// renderer. +func TestRender_SelectRendersEveryField(t *testing.T) { + base := func() *ast.SelectStatement { + return &ast.SelectStatement{ + Columns: []ast.Expression{&ast.Identifier{Name: "a"}}, + From: []ast.TableReference{{Name: "t"}}, + } + } + // Some samples need a companion field set in the baseline too, so that + // only the field under test differs. + baseFor := map[string]func(*ast.SelectStatement){ + "GroupByModifiers": func(s *ast.SelectStatement) { s.GroupBy = []ast.Expression{&ast.Identifier{Name: "g"}} }, + "OrderSiblings": func(s *ast.SelectStatement) { + s.OrderBy = []ast.OrderByExpression{{Expression: &ast.Identifier{Name: "o"}, Ascending: true}} + }, + "Interpolate": func(s *ast.SelectStatement) { + s.OrderBy = []ast.OrderByExpression{{Expression: &ast.Identifier{Name: "o"}, Ascending: true}} + }, + "OffsetRows": func(s *ast.SelectStatement) { v := 8; s.Offset = &v }, + "LimitWithTies": func(s *ast.SelectStatement) { v := 7; s.Limit = &v }, + "RowsExamined": func(s *ast.SelectStatement) { v := 7; s.Limit = &v }, + } + samples := selectFieldSamples() + // The fields of SelectStatement.Extras are checked like the statement's + // own fields; Extras itself is only their holder. + var fields []reflect.StructField + for _, typ := range []reflect.Type{reflect.TypeOf(ast.SelectStatement{}), reflect.TypeOf(ast.SelectExtras{})} { + for i := 0; i < typ.NumField(); i++ { + if f := typ.Field(i); f.IsExported() && f.Name != "Extras" { + fields = append(fields, f) + } + } + } + for _, field := range fields { + name := field.Name + if _, skip := selectFieldsNotRendered[name]; skip { + continue + } + set, ok := samples[name] + if !ok { + t.Errorf("SelectStatement.%s has no render sample: add one to selectFieldSamples and render it", name) + continue + } + t.Run(name, func(t *testing.T) { + before, after := base(), base() + if prep := baseFor[name]; prep != nil { + prep(before) + prep(after) + } + set(after) + opts := ast.CompactStyle() + if f0, f1 := formatter.FormatStatement(before, opts), formatter.FormatStatement(after, opts); f0 == f1 { + t.Errorf("formatter ignores %s: %s", name, f1) + } + if s0, s1 := before.SQL(), after.SQL(); s0 == s1 { + t.Errorf("SQL() ignores %s: %s", name, s1) + } + }) + } +} + +// TestRender_SelectExtKeywordCase checks that the new clause keywords follow +// the formatter's keyword case. +func TestRender_SelectExtKeywordCase(t *testing.T) { + tree, err := gosqlx.ParseWithDialect("SELECT a FROM t GROUP BY a WITH ROLLUP ORDER BY a OFFSET 5 ROWS FETCH NEXT 5 ROWS ONLY", keywords.DialectSQLServer) + if err != nil { + t.Fatalf("parse: %v", err) + } + opts := ast.CompactStyle() + opts.KeywordCase = ast.KeywordLower + got := formatter.FormatAST(tree, opts) + for _, kw := range []string{"with rollup", "offset 5 rows", "fetch next 5 rows only"} { + if !strings.Contains(got, kw) { + t.Errorf("lower-case output %q lacks %q", got, kw) + } + } +} diff --git a/pkg/formatter/setop_chain_test.go b/pkg/formatter/setop_chain_test.go index 14cc0545..fe9251e5 100644 --- a/pkg/formatter/setop_chain_test.go +++ b/pkg/formatter/setop_chain_test.go @@ -15,10 +15,11 @@ package formatter_test import ( + "fmt" "math/rand" + "runtime" "strings" "testing" - "time" "github.com/ajitpratap0/GoSQLX/pkg/formatter" "github.com/ajitpratap0/GoSQLX/pkg/gosqlx" @@ -29,8 +30,9 @@ import ( // render(Left) + sep + op + sep + render(Right), recursing on both sides with // operands rendered without ";", and a single trailing ";" after the whole // top-level set operation when AddSemicolon is set (#602). An operand is -// wrapped in parentheses when its Parenthesized flag is set or, for Right, -// when it is a non-nil *SetOperation (#612). +// wrapped in parentheses when its Parenthesized flag is set or when it is a +// non-nil *SetOperation that standard precedence would not group as the +// tree does (#612); see ast.SetOperandNeedsParens. func setOpReference(s *ast.SetOperation, opts ast.FormatOptions) string { if s == nil { return "" @@ -60,8 +62,9 @@ func setOpReferenceBody(s *ast.SetOperation, opts ast.FormatOptions) string { } return "(" + render(st) + ")" } - rightSetOp, ok := s.Right.(*ast.SetOperation) - rightParens := s.RightParenthesized || (ok && rightSetOp != nil) + // A right set operation is parenthesized unless standard precedence + // already groups it (A UNION B INTERSECT C). + rightParens := ast.SetOperandNeedsParens(s, true, opts.Dialect) sep := " " if opts.NewlinePerClause { sep = "\n" @@ -78,7 +81,7 @@ func setOpReferenceBody(s *ast.SetOperation, opts ast.FormatOptions) string { } var sb strings.Builder if s.Left != nil { - sb.WriteString(operand(s.Left, s.LeftParenthesized)) + sb.WriteString(operand(s.Left, ast.SetOperandNeedsParens(s, false, opts.Dialect))) } sb.WriteString(sep + op + sep) if s.Right != nil { @@ -147,26 +150,86 @@ func TestFormatStatement_SetOperationMatchesReference(t *testing.T) { } } -// TestFormatAST_SetOperationChainLinearTime checks that a long UNION chain -// (a left-deep tree) is formatted in time linear in its length. Re-copying -// the rendered left side at every level takes seconds for this input; the -// linear rendering takes milliseconds, so the bound below is generous. -func TestFormatAST_SetOperationChainLinearTime(t *testing.T) { +// TestFormatAST_SetOperationChainLinearGrowth catches repeated copying of +// the rendered prefix: four times as many UNIONs must allocate substantially +// less than the sixteen-fold growth of quadratic copying. Byte growth, rather +// than a wall-clock limit, also works under race and atomic coverage overhead. +// The full 80,000-UNION output must still match exactly. Allocation-free CPU +// regressions can be measured separately with BenchmarkFormatAST_SetOperationChain. +func TestFormatAST_SetOperationChainLinearGrowth(t *testing.T) { + const small, large = 1024, 4096 + const noiseAllowance = 1 << 20 + var smallBytes uint64 + for _, n := range []int{small, large} { + sql := "SELECT 1" + strings.Repeat(" UNION SELECT 1", n) + tree, err := gosqlx.Parse(sql) + if err != nil { + t.Fatalf("Parse %d UNIONs: %v", n, err) + } + defer ast.ReleaseAST(tree) + out, allocated := setOperationFormatBytes(tree) + if out != sql { + t.Fatalf("formatted %d UNIONs differ from input", n) + } + t.Logf("%d UNIONs: %d allocated bytes", n, allocated) + // This permits well over the current per-node cost while rejecting + // repeated prefix copies at these bounded sizes, before the big case. + if budget := uint64(n)*2048 + noiseAllowance; allocated > budget { + t.Fatalf("formatting %d UNIONs allocated %d bytes, limit %d", n, allocated, budget) + } + if n == small { + smallBytes = allocated + } else if budget := 8*smallBytes + noiseAllowance; allocated > budget { + t.Fatalf("fourfold UNION growth allocated %d -> %d bytes, limit %d", smallBytes, allocated, budget) + } + } + const n = 80000 sql := "SELECT 1" + strings.Repeat(" UNION SELECT 1", n) tree, err := gosqlx.Parse(sql) if err != nil { t.Fatalf("Parse: %v", err) } + defer ast.ReleaseAST(tree) - start := time.Now() out := formatter.FormatAST(tree, ast.FormatOptions{}) - elapsed := time.Since(start) if out != sql { t.Fatalf("formatted output differs from input (len %d vs %d)", len(out), len(sql)) } - if elapsed > 2*time.Second { - t.Fatalf("formatting %d UNIONs took %v, want well under 2s", n, elapsed) +} + +// The caller runs sequentially: TotalAlloc includes every goroutine in this +// process. Parsing and a warm-up render are excluded from the measurement. +func setOperationFormatBytes(tree *ast.AST) (string, uint64) { + opts := ast.FormatOptions{} + _ = formatter.FormatAST(tree, opts) + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + out := formatter.FormatAST(tree, opts) + runtime.ReadMemStats(&after) + runtime.KeepAlive(out) + return out, after.TotalAlloc - before.TotalAlloc +} + +func BenchmarkFormatAST_SetOperationChain(b *testing.B) { + for _, n := range []int{1024, 4096, 80000} { + b.Run(fmt.Sprint(n), func(b *testing.B) { + sql := "SELECT 1" + strings.Repeat(" UNION SELECT 1", n) + tree, err := gosqlx.Parse(sql) + if err != nil { + b.Fatal(err) + } + defer ast.ReleaseAST(tree) + b.SetBytes(int64(len(sql))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if out := formatter.FormatAST(tree, ast.FormatOptions{}); len(out) != len(sql) { + b.Fatalf("formatted length %d, want %d", len(out), len(sql)) + } + } + b.StopTimer() + }) } } diff --git a/pkg/formatter/setop_with_hoist_test.go b/pkg/formatter/setop_with_hoist_test.go new file mode 100644 index 00000000..d4df5e1d --- /dev/null +++ b/pkg/formatter/setop_with_hoist_test.go @@ -0,0 +1,227 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package formatter + +import ( + "strings" + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" + "github.com/ajitpratap0/GoSQLX/pkg/sql/parser" +) + +// TestSetOpWithHoist checks that a WITH clause scoping over a whole set +// operation stays in front of it when the left operand needs parentheses in +// the target dialect: the clause is hoisted out of the parentheses, so the +// other operands still see its CTEs and the grouping is kept. +func TestSetOpWithHoist(t *testing.T) { + const with = "WITH c AS (SELECT 1 AS a) " + tests := []struct { + name string + parse keywords.SQLDialect + in string + render string // target dialect + want string + reparse bool // the output parses back with the target dialect to the same text + }{ + { + name: "equal precedence tree written for standard precedence", + parse: keywords.DialectSQLite, render: "postgresql", + in: with + "SELECT a FROM c UNION SELECT a FROM c INTERSECT SELECT a FROM c", + want: with + "(SELECT a FROM c UNION SELECT a FROM c) INTERSECT SELECT a FROM c", + reparse: true, + }, + { + name: "equal precedence tree written for its own dialect", + parse: keywords.DialectSQLite, render: "sqlite", + in: with + "SELECT a FROM c UNION SELECT a FROM c INTERSECT SELECT a FROM c", + want: with + "SELECT a FROM c UNION SELECT a FROM c INTERSECT SELECT a FROM c", + reparse: true, + }, + { + name: "no WITH keeps plain parentheses", + parse: keywords.DialectSQLite, render: "postgresql", + in: "SELECT a FROM c UNION SELECT a FROM c INTERSECT SELECT a FROM c", + want: "(SELECT a FROM c UNION SELECT a FROM c) INTERSECT SELECT a FROM c", + reparse: true, + }, + { + name: "WITH written before a parenthesized operand", + parse: keywords.DialectPostgreSQL, render: "postgresql", + in: with + "(SELECT a FROM c UNION SELECT a FROM c) INTERSECT SELECT a FROM c", + want: with + "(SELECT a FROM c UNION SELECT a FROM c) INTERSECT SELECT a FROM c", + reparse: true, + }, + { + name: "WITH before a single parenthesized operand", + parse: keywords.DialectPostgreSQL, render: "postgresql", + in: with + "(SELECT a FROM c) UNION SELECT a FROM c", + want: with + "(SELECT a FROM c) UNION SELECT a FROM c", + reparse: true, + }, + { + name: "WITH before a parenthesized query without set operation", + parse: keywords.DialectPostgreSQL, render: "postgresql", + in: with + "(SELECT a FROM c)", + want: with + "SELECT a FROM c", + }, + { + name: "WITH inside the parentheses stays inside", + parse: keywords.DialectPostgreSQL, render: "postgresql", + in: "(" + with + "SELECT a FROM c UNION SELECT 2) INTERSECT SELECT 3", + want: "(" + with + "SELECT a FROM c UNION SELECT 2) INTERSECT SELECT 3", + reparse: true, + }, + { + name: "chain with standard precedence keeps its WITH in place", + parse: keywords.DialectPostgreSQL, render: "postgresql", + in: with + "SELECT a FROM c UNION SELECT a FROM c INTERSECT SELECT a FROM c", + want: with + "SELECT a FROM c UNION SELECT a FROM c INTERSECT SELECT a FROM c", + reparse: true, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + tree, err := parser.ParseWithDialect(tc.in, tc.parse) + if err != nil { + t.Fatalf("parse %q: %v", tc.in, err) + } + defer ast.ReleaseAST(tree) + opts := ast.CompactStyle() + opts.Dialect = tc.render + if got := FormatAST(tree, opts); got != tc.want { + t.Errorf("FormatAST(%q)\n got: %s\nwant: %s", tc.in, got, tc.want) + } + if tc.render == "postgresql" { + // SQL() writes with standard precedence. + if got := tree.SQL(); got != tc.want { + t.Errorf("tree.SQL(%q)\n got: %s\nwant: %s", tc.in, got, tc.want) + } + } + if !tc.reparse { + return + } + out := FormatAST(tree, opts) + tree2, err := parser.ParseWithDialect(out, keywords.SQLDialect(tc.render)) + if err != nil { + t.Fatalf("output %q does not re-parse: %v", out, err) + } + defer ast.ReleaseAST(tree2) + if got := FormatAST(tree2, opts); got != out { + t.Errorf("output %q re-renders as %q", out, got) + } + }) + } +} + +// TestParseWithParenthesizedQuery_Rejects checks the forms the parser refuses +// instead of misplacing a WITH clause. +func TestParseWithParenthesizedQuery_Rejects(t *testing.T) { + for _, in := range []string{ + "WITH c AS (SELECT 1) (WITH d AS (SELECT 2) SELECT 3) UNION SELECT 4", + "WITH c AS (SELECT 1) (VALUES (1)) UNION SELECT 4", + } { + tree, err := parser.ParseWithDialect(in, keywords.DialectPostgreSQL) + if err == nil { + ast.ReleaseAST(tree) + t.Errorf("parse %q: expected an error", in) + } + } +} + +// TestSetOperationTailRender checks that the ORDER BY / LIMIT after a set +// operation whose last operand has its own clauses is written after the whole +// operation in every layout, and that the output parses back to the same text. +func TestSetOperationTailRender(t *testing.T) { + const in = "SELECT a FROM t UNION ALL (SELECT b FROM u ORDER BY b LIMIT 1) ORDER BY 1 LIMIT 3 OFFSET 2" + tree, err := parser.ParseWithDialect(in, keywords.DialectPostgreSQL) + if err != nil { + t.Fatal(err) + } + defer ast.ReleaseAST(tree) + + compact := ast.CompactStyle() + compact.Dialect = "postgresql" + if got := FormatAST(tree, compact); got != in { + t.Errorf("compact = %q, want %q", got, in) + } + for name, opts := range map[string]ast.FormatOptions{"readable": ast.ReadableStyle(), "compact+semicolon": func() ast.FormatOptions { + o := ast.CompactStyle() + o.AddSemicolon = true + return o + }()} { + opts.Dialect = "postgresql" + out := FormatAST(tree, opts) + if !strings.Contains(out, "ORDER BY 1") || strings.Index(out, "ORDER BY 1") < strings.Index(out, ")") { + t.Errorf("%s: tail is not after the parenthesized operand: %s", name, out) + } + tree2, err := parser.ParseWithDialect(out, keywords.DialectPostgreSQL) + if err != nil { + t.Errorf("%s: output %q does not re-parse: %v", name, out, err) + continue + } + if got := FormatAST(tree2, compact); got != in { + t.Errorf("%s: output re-parses to %q, want %q", name, got, in) + } + ast.ReleaseAST(tree2) + } +} + +// TestSetOpWithHoistNested checks that a WITH clause written before a +// parenthesized left operand that has parenthesized operands of its own is +// written before the whole set operation, not inside the parentheses where +// its CTEs would be hidden from the other operands. +func TestSetOpWithHoistNested(t *testing.T) { + const with = "WITH c AS (SELECT a FROM t) " + inputs := []struct { + name string + in string + }{ + {"two levels", with + "((SELECT a FROM c UNION SELECT a FROM c) INTERSECT SELECT a FROM c) EXCEPT SELECT a FROM c"}, + {"three levels", with + "(((SELECT a FROM c UNION SELECT a FROM c) INTERSECT SELECT a FROM c) EXCEPT SELECT a FROM c) UNION ALL SELECT a FROM c"}, + {"inner single operand", with + "((SELECT a FROM c) UNION SELECT a FROM c) EXCEPT SELECT a FROM c"}, + } + dialects := []keywords.SQLDialect{"", keywords.DialectPostgreSQL, keywords.DialectOracle, keywords.DialectSQLite} + for _, d := range dialects { + for _, tc := range inputs { + t.Run(string(d)+"/"+tc.name, func(t *testing.T) { + tree, err := parser.ParseWithDialect(tc.in, d) + if err != nil { + t.Fatalf("parse %q: %v", tc.in, err) + } + defer ast.ReleaseAST(tree) + opts := ast.CompactStyle() + opts.Dialect = string(d) + outputs := map[string]string{"SQL()": tree.SQL(), "FormatAST": FormatAST(tree, opts)} + for name, out := range outputs { + if !strings.HasPrefix(out, with) { + t.Errorf("%s: WITH clause is not in front of the set operation:\n%s", name, out) + continue + } + tree2, err := parser.ParseWithDialect(out, d) + if err != nil { + t.Errorf("%s: output %q does not re-parse: %v", name, out, err) + continue + } + if got := FormatAST(tree2, opts); got != outputs["FormatAST"] { + t.Errorf("%s: output %q re-renders as %q, want %q", name, out, got, outputs["FormatAST"]) + } + ast.ReleaseAST(tree2) + } + }) + } + } +} diff --git a/pkg/formatter/table_factor_render_test.go b/pkg/formatter/table_factor_render_test.go new file mode 100644 index 00000000..ea52c1dc --- /dev/null +++ b/pkg/formatter/table_factor_render_test.go @@ -0,0 +1,370 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package formatter_test + +import ( + "reflect" + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/formatter" + "github.com/ajitpratap0/GoSQLX/pkg/gosqlx" + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" +) + +// TestRender_QueryExpressionsAndTableFactors checks that set operations, +// CTE modifiers and table-factor modifiers survive the formatter and +// tree.SQL(): the compact output must equal want, re-parse with the same +// dialect and format to itself again. +func TestRender_QueryExpressionsAndTableFactors(t *testing.T) { + cases := []struct { + name string + dialect keywords.SQLDialect + sql string + want string + }{ + // Set operations + {"intersect precedence", keywords.DialectPostgreSQL, "SELECT 1 UNION SELECT 2 INTERSECT SELECT 3", "SELECT 1 UNION SELECT 2 INTERSECT SELECT 3"}, + {"intersect precedence trailing order by", keywords.DialectPostgreSQL, "SELECT 1 UNION SELECT 2 INTERSECT SELECT 3 ORDER BY 1 LIMIT 1", "SELECT 1 UNION SELECT 2 INTERSECT SELECT 3 ORDER BY 1 LIMIT 1"}, + {"explicit grouping kept", keywords.DialectPostgreSQL, "(SELECT 1 UNION SELECT 2) INTERSECT SELECT 3", "(SELECT 1 UNION SELECT 2) INTERSECT SELECT 3"}, + {"right grouping kept", keywords.DialectPostgreSQL, "SELECT 1 INTERSECT (SELECT 2 UNION SELECT 3)", "SELECT 1 INTERSECT (SELECT 2 UNION SELECT 3)"}, + {"intersect distinct", keywords.DialectMariaDB, "SELECT a FROM t1 INTERSECT DISTINCT SELECT a FROM t2", "SELECT a FROM t1 INTERSECT DISTINCT SELECT a FROM t2"}, + {"union distinct", keywords.DialectMySQL, "SELECT a FROM t1 UNION DISTINCT SELECT a FROM t2", "SELECT a FROM t1 UNION DISTINCT SELECT a FROM t2"}, + {"oracle minus", keywords.DialectOracle, "SELECT a FROM t MINUS SELECT a FROM u", "SELECT a FROM t MINUS SELECT a FROM u"}, + {"values operand", keywords.DialectPostgreSQL, "SELECT 1 UNION VALUES (2)", "SELECT 1 UNION VALUES (2)"}, + // CTEs + {"not materialized", keywords.DialectPostgreSQL, "WITH a AS NOT MATERIALIZED (SELECT 1 AS x) SELECT * FROM a", "WITH a AS NOT MATERIALIZED (SELECT 1 AS x) SELECT * FROM a"}, + {"materialized sqlite", keywords.DialectSQLite, "WITH a AS MATERIALIZED (SELECT 1 AS x) SELECT * FROM a", "WITH a AS MATERIALIZED (SELECT 1 AS x) SELECT * FROM a"}, + {"clickhouse scalar cte", keywords.DialectClickHouse, "WITH (SELECT max(ts) FROM t) AS latest SELECT 1", "WITH (SELECT max(ts) FROM t) AS latest SELECT 1"}, + {"cte values body", keywords.DialectSQLite, "WITH c AS (VALUES (1), (2)) SELECT * FROM c", "WITH c AS (VALUES (1), (2)) SELECT * FROM c"}, + {"recursive cte seeded by values", keywords.DialectPostgreSQL, "WITH RECURSIVE r(n) AS (VALUES (1) UNION ALL SELECT n + 1 FROM r WHERE n < 5) SELECT * FROM r", "WITH RECURSIVE r (n) AS (VALUES (1) UNION ALL SELECT n + 1 FROM r WHERE n < 5) SELECT * FROM r"}, + // Table-factor modifiers + {"clickhouse final", keywords.DialectClickHouse, "SELECT * FROM t AS x FINAL WHERE a = 1", "SELECT * FROM t x FINAL WHERE a = 1"}, + {"snowflake at", keywords.DialectSnowflake, "SELECT * FROM t AT (OFFSET => -60)", "SELECT * FROM t AT (OFFSET => -60)"}, + {"snowflake changes", keywords.DialectSnowflake, "SELECT * FROM t CHANGES (INFORMATION => DEFAULT) AT (OFFSET => -1) x", "SELECT * FROM t CHANGES (INFORMATION => DEFAULT) AT (OFFSET => -1) x"}, + {"mariadb system time", keywords.DialectMariaDB, "SELECT * FROM accounts FOR SYSTEM_TIME AS OF TIMESTAMP '2025-01-01 00:00:00' WHERE id = 1", "SELECT * FROM accounts FOR SYSTEM_TIME AS OF TIMESTAMP '2025-01-01 00:00:00' WHERE id = 1"}, + {"sqlserver system time", keywords.DialectSQLServer, "SELECT * FROM t FOR SYSTEM_TIME CONTAINED IN ('2024-01-01', '2025-01-01') AS x", "SELECT * FROM t FOR SYSTEM_TIME CONTAINED IN ('2024-01-01', '2025-01-01') x"}, + {"match recognize", keywords.DialectSnowflake, + "SELECT * FROM ticker MATCH_RECOGNIZE (PARTITION BY symbol ORDER BY ts MEASURES MATCH_NUMBER() AS m ALL ROWS PER MATCH AFTER MATCH SKIP PAST LAST ROW PATTERN (UP+ DOWN*?) DEFINE UP AS price > PREV(price), DOWN AS price < PREV(price)) mr", + "SELECT * FROM ticker MATCH_RECOGNIZE (PARTITION BY symbol ORDER BY ts MEASURES MATCH_NUMBER() AS m ALL ROWS PER MATCH AFTER MATCH SKIP PAST LAST ROW PATTERN (UP+ DOWN*?) DEFINE UP AS price > PREV(price), DOWN AS price < PREV(price)) AS mr"}, + {"pg tablesample", keywords.DialectPostgreSQL, "SELECT * FROM sales s TABLESAMPLE BERNOULLI (10) REPEATABLE (1)", "SELECT * FROM sales s TABLESAMPLE BERNOULLI (10) REPEATABLE (1)"}, + {"sqlserver tablesample", keywords.DialectSQLServer, "SELECT * FROM sales TABLESAMPLE (10 PERCENT)", "SELECT * FROM sales TABLESAMPLE (10 PERCENT)"}, + {"snowflake sample", keywords.DialectSnowflake, "SELECT * FROM t SAMPLE ROW (10 ROWS) SEED (7)", "SELECT * FROM t SAMPLE ROW (10 ROWS) SEED (7)"}, + {"oracle sample", keywords.DialectOracle, "SELECT * FROM t SAMPLE BLOCK (10, 2) SEED (5) x", "SELECT * FROM t SAMPLE BLOCK (10, 2) SEED (5) x"}, + {"pg only", keywords.DialectPostgreSQL, "SELECT * FROM ONLY parent_tbl p JOIN ONLY c ON p.id = c.id", "SELECT * FROM ONLY parent_tbl p INNER JOIN ONLY c ON p.id = c.id"}, + {"oracle flashback", keywords.DialectOracle, "SELECT * FROM t AS OF SCN 123 x", "SELECT * FROM t AS OF SCN 123 x"}, + {"oracle versions", keywords.DialectOracle, "SELECT * FROM t VERSIONS BETWEEN SCN MINVALUE AND MAXVALUE", "SELECT * FROM t VERSIONS BETWEEN SCN MINVALUE AND MAXVALUE"}, + {"oracle db link", keywords.DialectOracle, "SELECT * FROM t@lnk.example.com x", "SELECT * FROM t@lnk.example.com x"}, + {"mysql partition", keywords.DialectMySQL, "SELECT * FROM t PARTITION (p0, p1) AS x", "SELECT * FROM t PARTITION (p0, p1) x"}, + {"oracle partition", keywords.DialectOracle, "SELECT * FROM sales PARTITION (sales_q1) s", "SELECT * FROM sales PARTITION (sales_q1) s"}, + {"mysql index hints", keywords.DialectMySQL, "SELECT * FROM t x FORCE INDEX FOR JOIN (ix, PRIMARY) USE KEY ()", "SELECT * FROM t x FORCE INDEX FOR JOIN (ix, PRIMARY) USE KEY ()"}, + {"sqlite indexed by", keywords.DialectSQLite, "SELECT * FROM t AS x INDEXED BY i WHERE a = 1", "SELECT * FROM t x INDEXED BY i WHERE a = 1"}, + {"sqlite not indexed", keywords.DialectSQLite, "SELECT * FROM t NOT INDEXED", "SELECT * FROM t NOT INDEXED"}, + {"sqlserver bare hints", keywords.DialectSQLServer, "SELECT * FROM t x (NOLOCK)", "SELECT * FROM t x WITH (NOLOCK)"}, + {"sqlserver index hint", keywords.DialectSQLServer, "SELECT * FROM t WITH (NOLOCK, INDEX = ix_a)", "SELECT * FROM t WITH (NOLOCK, INDEX(ix_a))"}, + {"values derived table", keywords.DialectPostgreSQL, "SELECT * FROM (VALUES (1, 'a'), (2, 'b')) AS v(id, name)", "SELECT * FROM (VALUES (1, 'a'), (2, 'b')) v(id, name)"}, + {"mysql values row", keywords.DialectMySQL, "SELECT * FROM (VALUES ROW(1, 2), ROW(3, 4)) AS v(a, b)", "SELECT * FROM (VALUES ROW(1, 2), ROW(3, 4)) v(a, b)"}, + {"derived column aliases", keywords.DialectPostgreSQL, "SELECT * FROM (SELECT 1 AS x) AS s(y)", "SELECT * FROM (SELECT 1 AS x) s(y)"}, + {"table column aliases", keywords.DialectSnowflake, "SELECT * FROM t AS x(a, b)", "SELECT * FROM t x(a, b)"}, + {"typed record aliases", keywords.DialectPostgreSQL, "SELECT * FROM jsonb_to_recordset(j) AS t(a int, b text)", "SELECT * FROM jsonb_to_recordset(j) t(a int, b text)"}, + {"sqlserver table function", keywords.DialectSQLServer, "SELECT * FROM STRING_SPLIT('a,b', ',')", "SELECT * FROM STRING_SPLIT('a,b', ',')"}, + {"sqlite table function", keywords.DialectSQLite, "SELECT * FROM t, json_each(t.a)", "SELECT * FROM t, json_each(t.a)"}, + {"flatten keyword arguments", keywords.DialectSnowflake, "SELECT * FROM t, LATERAL FLATTEN(input => t.a, outer => TRUE) f", "SELECT * FROM t, LATERAL FLATTEN(input => t.a, outer => TRUE) f"}, + // CTE names and SEARCH / CYCLE + {"keyword cte names", keywords.DialectSnowflake, "WITH source AS (SELECT 1 AS a), target AS (SELECT a FROM source) SELECT * FROM target", "WITH source AS (SELECT 1 AS a), target AS (SELECT a FROM source) SELECT * FROM target"}, + {"final cte name", keywords.DialectSnowflake, "WITH final AS (SELECT 1 AS a) SELECT * FROM final", "WITH final AS (SELECT 1 AS a) SELECT * FROM final"}, + {"cte search", keywords.DialectPostgreSQL, "WITH RECURSIVE r(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM r WHERE n < 5) SEARCH BREADTH FIRST BY n SET ord SELECT * FROM r ORDER BY ord", "WITH RECURSIVE r (n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM r WHERE n < 5) SEARCH BREADTH FIRST BY n SET ord SELECT * FROM r ORDER BY ord"}, + {"cte cycle using", keywords.DialectPostgreSQL, "WITH RECURSIVE r AS (SELECT 1 AS n UNION SELECT n + 1 FROM r) CYCLE n SET is_cycle USING path SELECT * FROM r", "WITH RECURSIVE r AS (SELECT 1 AS n UNION SELECT n + 1 FROM r) CYCLE n SET is_cycle USING path SELECT * FROM r"}, + {"oracle search and cycle", keywords.DialectOracle, "WITH b (id, pid) AS (SELECT id, pid FROM p UNION ALL SELECT p.id, p.pid FROM p JOIN b ON p.pid = b.id) SEARCH DEPTH FIRST BY id SET ord CYCLE id SET is_cycle TO 'Y' DEFAULT 'N' SELECT * FROM b", "WITH b (id, pid) AS (SELECT id, pid FROM p UNION ALL SELECT p.id, p.pid FROM p INNER JOIN b ON p.pid = b.id) SEARCH DEPTH FIRST BY id SET ord CYCLE id SET is_cycle TO 'Y' DEFAULT 'N' SELECT * FROM b"}, + // Snowflake FROM forms + {"snowflake bare values", keywords.DialectSnowflake, "SELECT * FROM VALUES (1, 'a'), (2, 'b') AS v (id, name)", "SELECT * FROM (VALUES (1, 'a'), (2, 'b')) v(id, name)"}, + {"snowflake stage file", keywords.DialectSnowflake, "SELECT $1, $2 FROM @my_stage/data.csv (FILE_FORMAT => 'my_csv')", "SELECT $1, $2 FROM @my_stage/data.csv (FILE_FORMAT => 'my_csv')"}, + {"snowflake qualified positional column", keywords.DialectSnowflake, "SELECT t.$1 AS id FROM @my_stage/users/ (FILE_FORMAT => 'my_csv') t", "SELECT t.$1 AS id FROM @my_stage/users/ (FILE_FORMAT => 'my_csv') t"}, + {"sqlserver default schema", keywords.DialectSQLServer, "SELECT x.a FROM srv.db..t AS x", "SELECT x.a FROM srv.db..t x"}, + {"sqlserver openjson schema", keywords.DialectSQLServer, "SELECT * FROM OPENJSON(@json) WITH (Id INT '$.id', Raw NVARCHAR(MAX) '$.raw' AS JSON) AS j", "SELECT * FROM OPENJSON(@json) WITH (Id INT '$.id', Raw NVARCHAR(MAX) '$.raw' AS JSON) j"}, + // PIVOT / UNPIVOT + {"oracle xml quoted aggregates and source alias", keywords.DialectOracle, `SELECT * FROM sales s PIVOT XML (SUM(amount) AS "Total", COUNT(*) AS "Count" FOR product IN (ANY)) AS p`, `SELECT * FROM sales s PIVOT XML (SUM(amount) AS "Total", COUNT(*) AS "Count" FOR product IN (ANY)) AS p`}, + {"bigquery multiple aggregates and source alias", keywords.DialectBigQuery, "SELECT * FROM sales s PIVOT (SUM(amount) AS `Total`, COUNT(*) AS `Count` FOR product IN ('A')) AS p", "SELECT * FROM sales s PIVOT (SUM(amount) AS `Total`, COUNT(*) AS `Count` FOR product IN ('A')) AS p"}, + {"bigquery quoted unpivot columns and source alias", keywords.DialectBigQuery, "SELECT * FROM sales s UNPIVOT INCLUDE NULLS ((`First`, `Second`) FOR product IN ((q1, q2) AS 'H1')) AS u", "SELECT * FROM sales s UNPIVOT INCLUDE NULLS ((`First`, `Second`) FOR product IN ((q1, q2) AS 'H1')) AS u"}, + {"oracle multi aggregate pivot", keywords.DialectOracle, "SELECT * FROM (SELECT a, b, c FROM t) PIVOT (SUM(c) AS total, COUNT(*) AS cnt FOR a IN (1 AS x, 2 AS y))", "SELECT * FROM (SELECT a, b, c FROM t) PIVOT (SUM(c) AS total, COUNT(*) AS cnt FOR a IN (1 AS x, 2 AS y))"}, + {"pivot source alias", keywords.DialectSQLServer, "SELECT * FROM (SELECT a, q FROM t) s PIVOT (SUM(a) FOR q IN ([Q1], [Q2])) AS p", "SELECT * FROM (SELECT a, q FROM t) s PIVOT (SUM(a) FOR q IN ([Q1], [Q2])) AS p"}, + {"pivot xml any", keywords.DialectOracle, "SELECT * FROM sales PIVOT XML (SUM(amount) FOR product IN (ANY))", "SELECT * FROM sales PIVOT XML (SUM(amount) FOR product IN (ANY))"}, + {"pivot any order by", keywords.DialectSnowflake, "SELECT * FROM q PIVOT (AVG(amount) FOR quarter IN (ANY ORDER BY quarter))", "SELECT * FROM q PIVOT (AVG(amount) FOR quarter IN (ANY ORDER BY quarter))"}, + {"pivot in subquery", keywords.DialectSnowflake, "SELECT * FROM q PIVOT (AVG(amount) FOR quarter IN (SELECT DISTINCT quarter FROM qs))", "SELECT * FROM q PIVOT (AVG(amount) FOR quarter IN (SELECT DISTINCT quarter FROM qs))"}, + {"pivot alias columns", keywords.DialectSnowflake, "SELECT * FROM m PIVOT (SUM(amount) FOR month IN ('JAN', 'FEB')) AS p (emp_id, jan, feb)", "SELECT * FROM m PIVOT (SUM(amount) FOR month IN ('JAN', 'FEB')) AS p(emp_id, jan, feb)"}, + {"unpivot include nulls", keywords.DialectSnowflake, "SELECT * FROM m UNPIVOT INCLUDE NULLS (sales FOR month IN (jan, feb)) AS u", "SELECT * FROM m UNPIVOT INCLUDE NULLS (sales FOR month IN (jan, feb)) AS u"}, + {"multi-column unpivot", keywords.DialectOracle, "SELECT * FROM w UNPIVOT ((units, revenue) FOR period IN ((u1, r1) AS 'P1', (u2, r2) AS 'P2'))", "SELECT * FROM w UNPIVOT ((units, revenue) FOR period IN ((u1, r1) AS 'P1', (u2, r2) AS 'P2'))"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + tree, err := gosqlx.ParseWithDialect(tc.sql, tc.dialect) + if err != nil { + t.Fatalf("parse: %v", err) + } + defer ast.ReleaseAST(tree) + got := formatter.FormatAST(tree, ast.CompactStyle()) + if got != tc.want { + t.Errorf("formatter:\n got: %s\nwant: %s", got, tc.want) + } + if sql := tree.SQL(); sql != tc.want { + t.Errorf("SQL():\n got: %s\nwant: %s", sql, tc.want) + } + tree2, err := gosqlx.ParseWithDialect(got, tc.dialect) + if err != nil { + t.Fatalf("re-parse %q: %v", got, err) + } + defer ast.ReleaseAST(tree2) + if again := formatter.FormatAST(tree2, ast.CompactStyle()); again != got { + t.Errorf("not idempotent:\n first: %s\nsecond: %s", got, again) + } + readable := formatter.FormatAST(tree, ast.ReadableStyle()) + if _, err := gosqlx.ParseWithDialect(readable, tc.dialect); err != nil { + t.Errorf("readable output does not re-parse: %v\n%s", err, readable) + } + }) + } +} + +// TestRender_SetOperationTargetDialect checks the dialect-dependent parts of +// set-operation rendering: MINUS is written as EXCEPT for a dialect without +// MINUS, and a right-nested INTERSECT keeps its parentheses for a dialect +// that gives all set operators equal precedence. +func TestRender_SetOperationTargetDialect(t *testing.T) { + cases := []struct { + name string + parseIn keywords.SQLDialect + sql string + target string + want string + }{ + {"minus to postgres", keywords.DialectOracle, "SELECT a FROM t MINUS SELECT a FROM u", "postgresql", "SELECT a FROM t EXCEPT SELECT a FROM u"}, + {"minus to snowflake", keywords.DialectOracle, "SELECT a FROM t MINUS SELECT a FROM u", "snowflake", "SELECT a FROM t MINUS SELECT a FROM u"}, + {"precedence to sqlite", keywords.DialectPostgreSQL, "SELECT 1 UNION SELECT 2 INTERSECT SELECT 3", "sqlite", "SELECT 1 UNION (SELECT 2 INTERSECT SELECT 3)"}, + {"precedence to oracle", keywords.DialectPostgreSQL, "SELECT 1 UNION SELECT 2 INTERSECT SELECT 3", "oracle", "SELECT 1 UNION (SELECT 2 INTERSECT SELECT 3)"}, + {"precedence to mysql", keywords.DialectPostgreSQL, "SELECT 1 UNION SELECT 2 INTERSECT SELECT 3", "mysql", "SELECT 1 UNION SELECT 2 INTERSECT SELECT 3"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + tree, err := gosqlx.ParseWithDialect(tc.sql, tc.parseIn) + if err != nil { + t.Fatalf("parse: %v", err) + } + defer ast.ReleaseAST(tree) + opts := ast.CompactStyle() + opts.Dialect = tc.target + if got := formatter.FormatAST(tree, opts); got != tc.want { + t.Errorf("got %s\nwant %s", got, tc.want) + } + }) + } +} + +// TestRender_TableFactorKeywordCase checks that the table-factor and CTE +// keywords follow the formatter's keyword case. +func TestRender_TableFactorKeywordCase(t *testing.T) { + cases := []struct { + dialect keywords.SQLDialect + sql string + want string + }{ + {keywords.DialectPostgreSQL, "WITH a AS NOT MATERIALIZED (SELECT 1) SELECT * FROM ONLY a TABLESAMPLE SYSTEM (5) REPEATABLE (2)", + "with a as not materialized (select 1) select * from only a tablesample system (5) repeatable (2)"}, + {keywords.DialectMySQL, "SELECT * FROM t PARTITION (P0) USE INDEX FOR ORDER BY (Ix)", + "select * from t partition (P0) use index for order by (Ix)"}, + {keywords.DialectSQLServer, "SELECT * FROM t FOR SYSTEM_TIME ALL WITH (NOLOCK, INDEX(Ix))", + "select * from t for system_time all with (nolock, index(Ix))"}, + {keywords.DialectOracle, "SELECT a FROM t AS OF SCN 1 MINUS SELECT a FROM u", + "select a from t as of scn 1 minus select a from u"}, + } + for _, tc := range cases { + tree, err := gosqlx.ParseWithDialect(tc.sql, tc.dialect) + if err != nil { + t.Fatalf("parse %q: %v", tc.sql, err) + } + opts := ast.CompactStyle() + opts.KeywordCase = ast.KeywordLower + got := formatter.FormatAST(tree, opts) + ast.ReleaseAST(tree) + if got != tc.want { + t.Errorf("got %s\nwant %s", got, tc.want) + } + } +} + +// tableRefFieldSamples gives a non-zero sample for every exported +// TableReference field. A new field fails TestRender_TableReferenceRendersEveryField +// until it is added here (and rendered) or to tableRefFieldsNotRendered. +func tableRefFieldSamples() map[string]func(*ast.TableReference) { + ident := func(n string) *ast.Identifier { return &ast.Identifier{Name: n} } + num := func(v string) *ast.LiteralValue { return &ast.LiteralValue{Value: v, Type: "int"} } + sub := func() *ast.SelectStatement { return &ast.SelectStatement{Columns: []ast.Expression{ident("s")}} } + return map[string]func(*ast.TableReference){ + "Name": func(r *ast.TableReference) { r.Name = "other" }, + "Alias": func(r *ast.TableReference) { r.Alias = "y" }, + // A recorded NameQuote keeps the base name "t" / alias "x" quoted. + "NameQuote": func(r *ast.TableReference) { r.NameQuote = ast.QuotedName('"') }, + "AliasQuote": func(r *ast.TableReference) { r.AliasQuote = ast.QuotedName('"') }, + "Subquery": func(r *ast.TableReference) { r.Subquery = sub() }, + "Lateral": func(r *ast.TableReference) { r.Lateral = true }, + "Final": func(r *ast.TableReference) { r.Final = true }, + "WithOrdinality": func(r *ast.TableReference) { + r.WithOrdinality = true + }, + "TableHints": func(r *ast.TableReference) { r.TableHints = []string{"NOLOCK"} }, + "TableFunc": func(r *ast.TableReference) { + r.TableFunc = &ast.FunctionCall{Name: "f", Arguments: []ast.Expression{num("1")}} + }, + "ColumnAliases": func(r *ast.TableReference) { r.ColumnAliases = &ast.ColumnAliasList{Names: []string{"c"}} }, + "TimeTravel": func(r *ast.TableReference) { + r.TimeTravel = &ast.TimeTravelClause{Kind: "AT", Named: map[string]ast.Expression{"OFFSET": num("1")}} + }, + "ForSystemTime": func(r *ast.TableReference) { + r.ForSystemTime = &ast.ForSystemTimeClause{Type: ast.SystemTimeAll} + }, + "Pivot": func(r *ast.TableReference) { + r.Pivot = &ast.PivotClause{AggregateFunction: &ast.FunctionCall{Name: "SUM", Arguments: []ast.Expression{ident("v")}}, PivotColumn: "k", InValues: []string{"1"}} + }, + "Unpivot": func(r *ast.TableReference) { + r.Unpivot = &ast.UnpivotClause{ValueColumn: "v", NameColumn: "k", InColumns: []string{"a"}} + }, + "MatchRecognize": func(r *ast.TableReference) { + r.MatchRecognize = &ast.MatchRecognizeClause{Pattern: "A+", Definitions: []ast.PatternDef{{Name: "A", Condition: ident("ok")}}} + }, + "SetSubquery": func(r *ast.TableReference) { + r.SetSubquery = &ast.SetOperation{Left: sub(), Operator: "UNION", Right: sub()} + }, + "Only": func(r *ast.TableReference) { r.Only = true }, + "Modifiers": func(r *ast.TableReference) { + r.Modifiers = &ast.TableModifiers{ + TableSample: &ast.TableSampleClause{Keyword: "TABLESAMPLE", Method: "SYSTEM", Args: []ast.Expression{num("5")}, AfterAlias: true}, + Flashback: &ast.FlashbackClause{Kind: "SCN", Point: num("9")}, + DBLink: "lnk", + Partitions: []string{"p0"}, + IndexHints: []ast.IndexHint{{Action: "USE", Keyword: "INDEX", Names: []string{"ix"}}}, + IndexedBy: "i", + } + }, + } +} + +// tableRefFieldsNotRendered are TableReference fields that carry no SQL text. +var tableRefFieldsNotRendered = map[string]string{} + +// TestRender_TableReferenceRendersEveryField sets each exported +// TableReference field in turn and asserts that both pkg/formatter and +// SelectStatement.SQL() output change, so a parsed table modifier cannot be +// silently dropped by either renderer. +func TestRender_TableReferenceRendersEveryField(t *testing.T) { + base := func() *ast.SelectStatement { + return &ast.SelectStatement{ + Columns: []ast.Expression{&ast.Identifier{Name: "a"}}, + From: []ast.TableReference{{Name: "t", Alias: "x"}}, + } + } + // WITH ORDINALITY is written only for a table function. + baseFor := map[string]func(*ast.TableReference){ + "WithOrdinality": func(r *ast.TableReference) { + r.TableFunc = &ast.FunctionCall{Name: "f"} + }, + } + samples := tableRefFieldSamples() + typ := reflect.TypeOf(ast.TableReference{}) + for i := 0; i < typ.NumField(); i++ { + field := typ.Field(i) + if !field.IsExported() { + continue + } + name := field.Name + if _, skip := tableRefFieldsNotRendered[name]; skip { + continue + } + set, ok := samples[name] + if !ok { + t.Errorf("TableReference.%s has no render sample: add one to tableRefFieldSamples and render it", name) + continue + } + t.Run(name, func(t *testing.T) { + before, after := base(), base() + if prep := baseFor[name]; prep != nil { + prep(&before.From[0]) + prep(&after.From[0]) + } + set(&after.From[0]) + opts := ast.CompactStyle() + f0, f1 := formatter.FormatStatement(before, opts), formatter.FormatStatement(after, opts) + if f0 == f1 { + t.Errorf("formatter ignores %s: %s", name, f1) + } + if s0, s1 := before.SQL(), after.SQL(); s0 == s1 { + t.Errorf("SQL() ignores %s: %s", name, s1) + } + }) + } +} + +// TestRender_TableModifiersRendersEveryField sets each exported +// TableModifiers field in turn and asserts that both renderers write it. +func TestRender_TableModifiersRendersEveryField(t *testing.T) { + num := func(v string) *ast.LiteralValue { return &ast.LiteralValue{Value: v, Type: "int"} } + samples := map[string]func(*ast.TableModifiers){ + "Values": func(m *ast.TableModifiers) { + m.Values = &ast.Values{Rows: [][]ast.Expression{{num("1")}}} + }, + "TableSample": func(m *ast.TableModifiers) { + m.TableSample = &ast.TableSampleClause{Keyword: "SAMPLE", Args: []ast.Expression{num("5")}} + }, + "Flashback": func(m *ast.TableModifiers) { m.Flashback = &ast.FlashbackClause{Kind: "SCN", Point: num("9")} }, + "DBLink": func(m *ast.TableModifiers) { m.DBLink = "lnk" }, + "Partitions": func(m *ast.TableModifiers) { m.Partitions = []string{"p0"} }, + "IndexHints": func(m *ast.TableModifiers) { + m.IndexHints = []ast.IndexHint{{Action: "FORCE", Keyword: "INDEX", Names: []string{"ix"}}} + }, + "IndexedBy": func(m *ast.TableModifiers) { m.IndexedBy = "i" }, + "NotIndexed": func(m *ast.TableModifiers) { m.NotIndexed = true }, + "PartitionFor": func(m *ast.TableModifiers) { + m.PartitionFor = &ast.PartitionForClause{Values: []ast.Expression{num("1")}} + }, + "Group": func(m *ast.TableModifiers) { + m.Group = &ast.JoinGroup{Tables: []ast.TableReference{{Name: "u"}, {Name: "v"}}} + }, + "WithSchema": func(m *ast.TableModifiers) { + m.WithSchema = []ast.OpenJSONColumn{{Name: "id", Type: "INT", Path: "$.id"}} + }, + } + typ := reflect.TypeOf(ast.TableModifiers{}) + for i := 0; i < typ.NumField(); i++ { + name := typ.Field(i).Name + set, ok := samples[name] + if !ok { + t.Errorf("TableModifiers.%s has no render sample", name) + continue + } + t.Run(name, func(t *testing.T) { + stmt := &ast.SelectStatement{ + Columns: []ast.Expression{&ast.Identifier{Name: "a"}}, + From: []ast.TableReference{{Name: "t", Alias: "x", Modifiers: &ast.TableModifiers{}}}, + } + before := formatter.FormatStatement(stmt, ast.CompactStyle()) + beforeSQL := stmt.SQL() + set(stmt.From[0].Modifiers) + if after := formatter.FormatStatement(stmt, ast.CompactStyle()); after == before { + t.Errorf("formatter ignores %s: %s", name, after) + } + if after := stmt.SQL(); after == beforeSQL { + t.Errorf("SQL() ignores %s: %s", name, after) + } + }) + } +} diff --git a/pkg/formatter/type_args_quoting_test.go b/pkg/formatter/type_args_quoting_test.go new file mode 100644 index 00000000..d3887e61 --- /dev/null +++ b/pkg/formatter/type_args_quoting_test.go @@ -0,0 +1,63 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package formatter_test + +import ( + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/formatter" + "github.com/ajitpratap0/GoSQLX/pkg/gosqlx" + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" +) + +// TestTypeArgsKeepQuoteKinds checks that parenthesised argument text the +// parser keeps as written (ClickHouse PROJECTION bodies, engine arguments, +// type arguments) keeps the delimiters of its names and the escapes of its +// strings: "x" must not become the string 'x', and `x` must not lose its +// backticks. +func TestTypeArgsKeepQuoteKinds(t *testing.T) { + tests := []struct { + name string + in string + }{ + {"projection double quotes", `CREATE TABLE t (a Int32, PROJECTION p (SELECT "a", sum("a") GROUP BY "a")) ENGINE=MergeTree ORDER BY a`}, + {"projection backticks", "CREATE TABLE t (a Int32, PROJECTION p (SELECT `a`, sum(`a`) GROUP BY `a`)) ENGINE=MergeTree ORDER BY a"}, + {"projection string stays string", `CREATE TABLE t (a Int32, PROJECTION p (SELECT 'a', "a" GROUP BY "a")) ENGINE=MergeTree ORDER BY a`}, + {"projection doubled delimiters", "CREATE TABLE t (a Int32, PROJECTION p (SELECT `x``y` GROUP BY `x``y`)) ENGINE=MergeTree ORDER BY a"}, + {"engine args with identifiers", "CREATE TABLE t (a Int32) ENGINE=Distributed(cluster, `db`, \"t\", rand())"}, + {"engine args escaped quote", "CREATE TABLE t (a Int32) ENGINE=ReplicatedMergeTree('/p/it''s', '{replica}') ORDER BY a"}, + {"engine args empty string", "CREATE TABLE t (a Int32) ENGINE=Foo('', 'x')"}, + {"named tuple elements", "CREATE TABLE t (a Tuple(`x y` UInt8, \"z\" String)) ENGINE=Memory"}, + {"enum with empty member", "CREATE TABLE t (a Enum8('' = 0, 'b' = 1)) ENGINE=Memory"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + tree, err := gosqlx.ParseWithDialect(tc.in, keywords.DialectClickHouse) + if err != nil { + t.Fatalf("parse %q: %v", tc.in, err) + } + defer ast.ReleaseAST(tree) + if got := tree.SQL(); got != tc.in { + t.Errorf("SQL()\n got: %s\nwant: %s", got, tc.in) + } + style := ast.CompactStyle() + style.Dialect = string(keywords.DialectClickHouse) + if got := formatter.FormatStatement(tree.Statements[0], style); got != tc.in { + t.Errorf("FormatStatement\n got: %s\nwant: %s", got, tc.in) + } + }) + } +} diff --git a/pkg/gosqlx/batch_count_validation_test.go b/pkg/gosqlx/batch_count_validation_test.go new file mode 100644 index 00000000..de0d2ce3 --- /dev/null +++ b/pkg/gosqlx/batch_count_validation_test.go @@ -0,0 +1,32 @@ +// Copyright 2026 GoSQLX Authors +// SPDX-License-Identifier: Apache-2.0 + +package gosqlx + +import ( + "reflect" + "testing" +) + +func TestSplitStatementsPreservesInvalidGOBatchCount(t *testing.T) { + for _, count := range []string{"0", "999999999999999999999999999999", "1.5", "-1"} { + t.Run(count, func(t *testing.T) { + src := "SELECT 1\nGO " + count + "\nSELECT 2" + if got := SplitStatements(src, "sqlserver"); !reflect.DeepEqual(got, []string{src}) { + t.Fatalf("SplitStatements(%q) = %q; invalid GO count must reach the parser", src, got) + } + }) + } +} + +func TestSplitStatementsConsumesValidGOBatchCount(t *testing.T) { + for _, directive := range []string{"GO", "GO 3", "GO 3 -- repeat", "GO 3 -- repeat\r"} { + t.Run(directive, func(t *testing.T) { + src := "SELECT 1\r\n" + directive + "\nSELECT 2" + want := []string{"SELECT 1\r\n", "SELECT 2"} + if got := SplitStatements(src, "sqlserver"); !reflect.DeepEqual(got, want) { + t.Fatalf("SplitStatements(%q) = %q, want %q", src, got, want) + } + }) + } +} diff --git a/pkg/gosqlx/batch_separator_test.go b/pkg/gosqlx/batch_separator_test.go new file mode 100644 index 00000000..3511d816 --- /dev/null +++ b/pkg/gosqlx/batch_separator_test.go @@ -0,0 +1,83 @@ +// Copyright 2026 GoSQLX Authors +// SPDX-License-Identifier: Apache-2.0 + +package gosqlx + +import ( + "strings" + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/formatter" + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" +) + +func TestBatchSeparatorRoundTrip(t *testing.T) { + tests := []struct{ name, sql string }{ + {"procedure", "CREATE PROCEDURE p AS SELECT 1\nGO\nSELECT 2\nGO 3"}, + {"view", "CREATE VIEW v AS SELECT 1 AS a\nGO\nSELECT * FROM v"}, + {"trigger", "CREATE TRIGGER tr ON t AFTER INSERT AS BEGIN SELECT 1; END\nGO\nSELECT 2"}, + {"empty batches", "GO\nGO 2\nSELECT 1\nGO\nGO"}, + {"comments", "-- lead\nCREATE PROCEDURE p AS SELECT 1;\nGO -- batch\n/* next */ SELECT 2\nGO 3 -- end"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + tree, err := ParseWithDialect(tc.sql, keywords.DialectSQLServer) + if err != nil { + t.Fatal(err) + } + defer ast.ReleaseAST(tree) + separators := 0 + for _, s := range tree.Statements { + if sep, ok := s.(*ast.BatchSeparator); ok { + if sep.RepeatCount < 0 || sep.TokenLiteral() != "GO" || len(sep.Children()) != 0 { + t.Errorf("invalid batch separator: %#v", sep) + } + separators++ + } + } + if separators == 0 { + t.Fatal("parser discarded GO batch boundaries") + } + outputs := []string{tree.SQL(), FormatAST(tree)} + for _, semicolons := range []bool{false, true} { + opts := ast.CompactStyle() + opts.Dialect, opts.AddSemicolon = "sqlserver", semicolons + outputs = append(outputs, formatter.FormatAST(tree, opts)) + options := DefaultFormatOptions() + options.AddSemicolon = semicolons + out, err := FormatWithDialect(tc.sql, options, keywords.DialectSQLServer) + if err != nil { + t.Fatalf("FormatWithDialect: %v", err) + } + outputs = append(outputs, out) + } + for _, out := range outputs { + if strings.Contains(out, "GO;") || strings.Contains(out, "GO 3;") { + t.Errorf("batch separator gained semicolon: %s", out) + } + again, err := ParseWithDialect(out, keywords.DialectSQLServer) + if err != nil { + t.Errorf("reparse %q: %v", out, err) + continue + } + if again.SQL() != tree.SQL() { + t.Errorf("batch meaning changed:\nwant %s\ngot %s", tree.SQL(), again.SQL()) + } + ast.ReleaseAST(again) + } + }) + } +} + +func TestBatchSeparatorInvalidCounts(t *testing.T) { + for _, count := range []string{"0", "-1", "1.5", "999999999999999999999999999999"} { + t.Run(count, func(t *testing.T) { + tree, err := ParseWithDialect("SELECT 1\nGO "+count+"\nSELECT 2", keywords.DialectSQLServer) + if err == nil { + ast.ReleaseAST(tree) + t.Fatal("accepted invalid GO repeat count") + } + }) + } +} diff --git a/pkg/gosqlx/children_reach_audit_test.go b/pkg/gosqlx/children_reach_audit_test.go new file mode 100644 index 00000000..aef2dba2 --- /dev/null +++ b/pkg/gosqlx/children_reach_audit_test.go @@ -0,0 +1,230 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gosqlx + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "sort" + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" +) + +// collectIdents gathers every *Identifier reachable from v through exported +// fields, with the field path that holds it. +func collectIdents(v reflect.Value, path string, seen map[uintptr]bool, out map[*ast.Identifier]string) { + switch v.Kind() { + case reflect.Ptr: + if v.IsNil() { + return + } + if id, ok := v.Interface().(*ast.Identifier); ok { + if _, dup := out[id]; !dup { + out[id] = path + } + return + } + if seen[v.Pointer()] { + return + } + seen[v.Pointer()] = true + collectIdents(v.Elem(), path, seen, out) + case reflect.Interface: + if !v.IsNil() { + collectIdents(v.Elem(), path, seen, out) + } + case reflect.Struct: + t := v.Type() + for i := 0; i < v.NumField(); i++ { + if !t.Field(i).IsExported() { + continue + } + collectIdents(v.Field(i), path+"."+t.Field(i).Name, seen, out) + } + case reflect.Slice, reflect.Array: + for i := 0; i < v.Len(); i++ { + collectIdents(v.Index(i), path, seen, out) + } + } +} + +func TestInspectReachesEveryIdentifierCorpus(t *testing.T) { + dirs, _ := os.ReadDir(realworldRoot) + gaps := map[string]string{} + for _, d := range dirs { + if !d.IsDir() { + continue + } + files, _ := filepath.Glob(filepath.Join(realworldRoot, d.Name(), "*.sql")) + for _, f := range files { + data, _ := os.ReadFile(f) + for _, s := range SplitStatements(string(data), d.Name()) { + tree, err := ParseWithDialect(s, keywords.SQLDialect(d.Name())) + if err != nil { + continue + } + for _, stmt := range tree.Statements { + all := map[*ast.Identifier]string{} + collectIdents(reflect.ValueOf(stmt), fmt.Sprintf("%T", stmt), map[uintptr]bool{}, all) + vis := map[*ast.Identifier]bool{} + ast.Inspect(stmt, func(n ast.Node) bool { + if id, ok := n.(*ast.Identifier); ok { + vis[id] = true + } + return true + }) + for id, p := range all { + if !vis[id] { + gaps[p] = s + } + } + } + } + } + } + var keys []string + for k := range gaps { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + t.Errorf("field not reached by ast.Inspect: %s, e.g. %.120s", k, gaps[k]) + } +} + +func TestInspectReachesEveryIdentifierCases(t *testing.T) { + cases := []struct{ d, q string }{ + {"sqlserver", "SELECT * FROM t PIVOT (SUM(mk1) FOR mk2 IN ([a],[b])) AS p"}, + {"sqlserver", "SELECT * FROM t UNPIVOT (mk1 FOR mk2 IN (mk3, mk4)) AS p"}, + {"snowflake", "SELECT * FROM t PIVOT (SUM(mk1) FOR mk2 IN (SELECT mk3 FROM u)) AS p"}, + {"snowflake", "SELECT * FROM t PIVOT (SUM(mk1) FOR mk2 IN (ANY ORDER BY mk3)) AS p"}, + {"snowflake", "SELECT * FROM t PIVOT (SUM(mk1) FOR mk2 IN ('a') DEFAULT ON NULL (mk3)) AS p"}, + {"postgresql", "INSERT INTO t (a) VALUES (1) ON CONFLICT (a) DO UPDATE SET a = mk1 WHERE mk2 > 1"}, + {"postgresql", "INSERT INTO t (a) VALUES (1) ON CONFLICT (a) WHERE mk3 > 1 DO NOTHING"}, + {"snowflake", "SELECT * FROM t MATCH_RECOGNIZE (PARTITION BY mk1 ORDER BY mk2 MEASURES mk3 AS m ALL ROWS PER MATCH PATTERN (A B+) DEFINE A AS mk4 > 1, B AS mk5 < 2)"}, + {"clickhouse", "SELECT a FROM t ORDER BY a WITH FILL INTERPOLATE (mk1 AS mk2 + 1)"}, + {"clickhouse", "SELECT a FROM t LIMIT 1 BY mk1 SETTINGS x = mk2"}, + {"clickhouse", "SELECT a FROM t PREWHERE mk1 > 1 FINAL SAMPLE 0.1"}, + {"mysql", "SELECT a INTO OUTFILE 'f' FIELDS TERMINATED BY ',' FROM t WHERE mk1 > 1"}, + {"postgresql", "SELECT a FROM t FOR UPDATE OF mk1 SKIP LOCKED"}, + {"snowflake", "SELECT v:mk1[0].b, v[mk2] FROM t"}, + {"snowflake", "SELECT {'a': mk1, 'b': mk2}"}, + {"sqlserver", "DELETE TOP (mk1) FROM t OUTPUT deleted.mk2 WHERE mk3 > 1"}, + {"sqlserver", "UPDATE TOP (mk1) t SET a = 1 OUTPUT inserted.mk2 WHERE mk3 > 1"}, + {"sqlserver", "INSERT TOP (mk1) INTO t (a) OUTPUT inserted.mk2 VALUES (1)"}, + {"mysql", "DELETE FROM t WHERE mk1 > 1 ORDER BY mk2 LIMIT 3"}, + {"mysql", "UPDATE t SET a = 1 WHERE mk1 > 1 ORDER BY mk2 LIMIT 3"}, + {"mysql", "DELETE t1, t2 FROM t1 JOIN t2 ON t1.mk1 = t2.mk2 WHERE mk3 > 1"}, + {"oracle", "DELETE FROM t WHERE mk1 > 1 RETURNING a INTO mk2"}, + {"oracle", "INSERT INTO t (a) VALUES (1) RETURNING a INTO mk1"}, + {"oracle", "INSERT ALL WHEN mk1 > 1 THEN INTO t (a) VALUES (mk2) ELSE INTO u (b) VALUES (mk3) SELECT * FROM d"}, + {"postgresql", "SELECT mk1 LIKE 'x' ESCAPE mk2 FROM t"}, + {"mariadb", "DELETE HISTORY FROM t BEFORE SYSTEM_TIME mk1"}, + {"mysql", "INSERT INTO t (a) VALUES (1) ON DUPLICATE KEY UPDATE a = mk1"}, + {"mysql", "INSERT INTO t SET a = mk1"}, + {"clickhouse", "INSERT INTO t SETTINGS x = mk1 VALUES (1)"}, + {"clickhouse", "CREATE TABLE t (a Int, INDEX i mk1 TYPE minmax GRANULARITY 1) ENGINE = MergeTree ORDER BY a"}, + {"clickhouse", "CREATE TABLE t (a Int DEFAULT mk1, b Int MATERIALIZED mk2, c Int ALIAS mk3 TTL mk4) ENGINE = MergeTree ORDER BY a"}, + {"clickhouse", "CREATE TABLE t (a Int) ENGINE = MergeTree ORDER BY a COMMENT 'x'"}, + {"postgresql", "CREATE TABLE t (a int, b int GENERATED ALWAYS AS (mk1 + 1) STORED, c int GENERATED ALWAYS AS IDENTITY, CHECK (mk2 > 1), EXCLUDE USING gist (mk3 WITH &&))"}, + {"postgresql", "CREATE TABLE t (a int) PARTITION BY RANGE (mk1)"}, + {"mysql", "CREATE TABLE t (a int) PARTITION BY RANGE (mk1) (PARTITION p0 VALUES LESS THAN (mk2))"}, + {"postgresql", "CREATE INDEX i ON t (mk1, (mk2 + 1)) INCLUDE (c) WHERE mk3 > 1"}, + {"postgresql", "CREATE VIEW v AS SELECT mk1 FROM t"}, + {"postgresql", "CREATE TABLE t AS SELECT mk1 FROM u"}, + {"postgresql", "ALTER TABLE t ADD COLUMN a int DEFAULT mk1"}, + {"postgresql", "ALTER TABLE t ADD CONSTRAINT c CHECK (mk1 > 1)"}, + {"postgresql", "ALTER TABLE t ALTER COLUMN a SET DEFAULT mk1"}, + {"postgresql", "ALTER TABLE t ALTER COLUMN a TYPE int USING mk1::int"}, + {"mysql", "ALTER TABLE t PARTITION BY RANGE (mk1) (PARTITION p VALUES LESS THAN (mk2))"}, + {"mysql", "ALTER TABLE t ADD PARTITION (PARTITION p VALUES LESS THAN (mk1))"}, + {"postgresql", "ALTER TABLE t ATTACH PARTITION p FOR VALUES FROM (mk1) TO (mk2)"}, + {"postgresql", "SELECT mk1 FROM t TABLESAMPLE BERNOULLI (mk2)"}, + {"postgresql", "SELECT * FROM generate_series(1, mk1) WITH ORDINALITY AS g(a, b)"}, + {"postgresql", "SELECT * FROM t, LATERAL (SELECT mk1) s"}, + {"postgresql", "SELECT * FROM ROWS FROM (f(mk1), g(mk2)) AS r"}, + {"postgresql", "SELECT * FROM t WHERE a = ANY(ARRAY[mk1, mk2])"}, + {"postgresql", "SELECT ARRAY[mk1][mk2], mk3[1:mk4] FROM t"}, + {"postgresql", "SELECT mk1 AT TIME ZONE mk2, EXTRACT(year FROM mk3), CAST(mk4 AS int), POSITION(mk5 IN mk6), SUBSTRING(mk7 FROM mk8 FOR mk9), TRIM(BOTH mk10 FROM mk11), OVERLAY(mk12 PLACING mk13 FROM mk14)"}, + {"postgresql", "SELECT COUNT(*) FILTER (WHERE mk1 > 1) OVER (PARTITION BY mk2 ORDER BY mk3 ROWS BETWEEN mk4 PRECEDING AND mk5 FOLLOWING), string_agg(mk6, ',' ORDER BY mk7) FROM t"}, + {"postgresql", "SELECT mk1 FROM t GROUP BY GROUPING SETS ((mk2), ROLLUP(mk3), CUBE(mk4))"}, + {"postgresql", "SELECT * FROM t JOIN u USING (mk1) JOIN v ON mk2 = 1"}, + {"postgresql", "SELECT * FROM t FETCH FIRST mk1 ROWS WITH TIES"}, + {"postgresql", "SELECT mk1 FROM t WINDOW w AS (PARTITION BY mk2)"}, + {"postgresql", "WITH RECURSIVE c AS (SELECT mk1) CYCLE mk2 SET mk3 USING mk4 SELECT * FROM c"}, + {"postgresql", "SELECT CASE mk1 WHEN mk2 THEN mk3 ELSE mk4 END"}, + {"postgresql", "SELECT mk1 IS DISTINCT FROM mk2, mk3 BETWEEN mk4 AND mk5, mk6 IN (mk7, mk8), EXISTS (SELECT mk9)"}, + {"postgresql", "CREATE SEQUENCE s START WITH 1"}, + {"postgresql", "COMMENT ON TABLE t IS 'x'"}, + {"postgresql", "COPY t (a) FROM STDIN"}, + {"postgresql", "CREATE FUNCTION f(a int) RETURNS int AS $$ SELECT mk1 $$ LANGUAGE sql"}, + {"postgresql", "CREATE TRIGGER tr BEFORE INSERT ON t FOR EACH ROW WHEN (mk1 > 1) EXECUTE FUNCTION f()"}, + {"postgresql", "EXPLAIN SELECT mk1 FROM t"}, + {"postgresql", "PREPARE p AS SELECT mk1 FROM t"}, + {"mysql", "SELECT * FROM t PARTITION (p0) WHERE mk1 > 1"}, + {"mysql", "REPLACE INTO t (a) VALUES (mk1)"}, + {"mysql", "SELECT a FROM t FORCE INDEX (i) WHERE mk1 > 1"}, + {"sqlserver", "SELECT * FROM t WITH (NOLOCK) CROSS APPLY f(mk1) AS x"}, + {"sqlserver", "SELECT mk1 FROM t FOR XML PATH('x')"}, + {"sqlserver", "SELECT mk1 FROM t OPTION (MAXDOP mk2)"}, + {"sqlserver", "DECLARE @x int = mk1"}, + {"sqlserver", "IF mk1 > 1 SELECT 1"}, + {"sqlserver", "SELECT mk1 FROM t FOR SYSTEM_TIME AS OF mk2"}, + {"oracle", "SELECT mk1 FROM t START WITH mk2 = 1 CONNECT BY PRIOR mk3 = mk4"}, + {"oracle", "SELECT mk1 FROM t OFFSET mk2 ROWS FETCH NEXT mk3 ROWS ONLY"}, + {"snowflake", "SELECT mk1 FROM t QUALIFY ROW_NUMBER() OVER (ORDER BY mk2) = 1"}, + {"snowflake", "SELECT * FROM t, LATERAL FLATTEN(input => mk1) f"}, + {"snowflake", "CREATE TABLE t CLONE s BEFORE (STATEMENT => mk1)"}, + {"bigquery", "SELECT mk1 FROM t, UNNEST(mk2) AS x"}, + {"clickhouse", "SELECT mk1 FROM t LEFT ARRAY JOIN mk2 AS x, mk3 AS y"}, + {"clickhouse", "CREATE MATERIALIZED VIEW v TO d (a Int) AS SELECT mk1 FROM t"}, + {"clickhouse", "CREATE MATERIALIZED VIEW v ENGINE = MergeTree() ORDER BY mk1 TTL mk2 SETTINGS a = mk3 POPULATE AS SELECT 1"}, + {"clickhouse", "CREATE MATERIALIZED VIEW v ENGINE = ReplicatedMergeTree(mk1, mk2) ORDER BY a AS SELECT 1"}, + {"clickhouse", "CREATE VIEW v AS SELECT mk1"}, + {"oracle", "CREATE MATERIALIZED VIEW v REFRESH FAST AS SELECT mk1 FROM t"}, + {"postgresql", "CREATE TABLE t (a int) WITH (fillfactor = 70)"}, + {"mysql", "CREATE TABLE t (a int) ENGINE=InnoDB"}, + {"mysql", "CREATE TABLE t (a int, KEY k (mk1))"}, + {"postgresql", "CREATE TABLE t (LIKE u INCLUDING ALL)"}, + {"postgresql", "CREATE UNIQUE INDEX CONCURRENTLY i ON t USING btree (mk1 DESC NULLS LAST) WITH (fillfactor = 70)"}, + } + for _, c := range cases { + tree, err := ParseWithDialect(c.q, keywords.SQLDialect(c.d)) + if err != nil { + t.Logf("skipped (unsupported syntax) %s: %.100s: %v", c.d, c.q, err) + continue + } + for _, stmt := range tree.Statements { + all := map[*ast.Identifier]string{} + collectIdents(reflect.ValueOf(stmt), fmt.Sprintf("%T", stmt), map[uintptr]bool{}, all) + vis := map[*ast.Identifier]bool{} + ast.Inspect(stmt, func(n ast.Node) bool { + if id, ok := n.(*ast.Identifier); ok { + vis[id] = true + } + return true + }) + for id, p := range all { + if !vis[id] { + t.Errorf("not reached by ast.Inspect: %s %s (%s) in %.100s", c.d, p, id.Name, c.q) + } + } + } + } +} diff --git a/pkg/gosqlx/children_reach_test.go b/pkg/gosqlx/children_reach_test.go new file mode 100644 index 00000000..02db1222 --- /dev/null +++ b/pkg/gosqlx/children_reach_test.go @@ -0,0 +1,117 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gosqlx + +import ( + "strings" + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" +) + +// TestChildrenReachEveryExpression parses statements that carry marker +// identifiers (mkN) in clauses stored outside the main column/where fields +// and checks ast.Inspect reaches every one, so walkers (security scanner, +// linter, table extraction) do not skip those subtrees. +func TestChildrenReachEveryExpression(t *testing.T) { + tests := []struct { + name string + dialect keywords.SQLDialect + sql string + markers int + }{ + {"clickhouse engine clauses", keywords.DialectClickHouse, + "CREATE TABLE t (a Int) ENGINE=MergeTree ORDER BY mk1 PARTITION BY mk2 SAMPLE BY mk3 TTL mk4 + INTERVAL 1 DAY SETTINGS x = mk5", 5}, + {"clickhouse engine args", keywords.DialectClickHouse, + "CREATE TABLE t (a Int) ENGINE = Distributed(mk1, mk2, t, rand())", 2}, + {"clickhouse materialized view", keywords.DialectClickHouse, + "CREATE MATERIALIZED VIEW v ENGINE=MergeTree ORDER BY mk1 PARTITION BY mk2 AS SELECT 1", 2}, + {"clickhouse array join", keywords.DialectClickHouse, + "SELECT * FROM t ARRAY JOIN mk1 AS x", 1}, + {"snowflake clone point", keywords.DialectSnowflake, + "CREATE TABLE c CLONE s AT (TIMESTAMP => mk1)", 1}, + {"postgres partition from to", keywords.DialectPostgreSQL, + "CREATE TABLE p PARTITION OF q FOR VALUES FROM (mk1) TO (mk2)", 2}, + {"postgres partition in", keywords.DialectPostgreSQL, + "CREATE TABLE p PARTITION OF q FOR VALUES IN (mk1, mk2)", 2}, + {"postgres partition hash", keywords.DialectPostgreSQL, + "CREATE TABLE p PARTITION OF q FOR VALUES WITH (MODULUS mk1, REMAINDER mk2)", 2}, + {"sqlserver top", keywords.DialectSQLServer, + "SELECT TOP (mk1) a FROM t", 1}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + tree, err := ParseWithDialect(tc.sql, tc.dialect) + if err != nil { + t.Fatalf("parse %q: %v", tc.sql, err) + } + seen := map[string]bool{} + for _, stmt := range tree.Statements { + ast.Inspect(stmt, func(n ast.Node) bool { + if id, ok := n.(*ast.Identifier); ok && strings.HasPrefix(id.Name, "mk") { + seen[id.Name] = true + } + return true + }) + } + for i := 1; i <= tc.markers; i++ { + name := "mk" + string(rune('0'+i)) + if !seen[name] { + t.Errorf("%s not reached by ast.Inspect in %q", name, tc.sql) + } + } + }) + } +} + +// TestMergeSourceColumnAliasesRender checks the column alias list of a MERGE +// derived-table source survives rendering. +func TestMergeSourceColumnAliasesRender(t *testing.T) { + tests := []struct { + dialect keywords.SQLDialect + sql string + want string + }{ + {keywords.DialectSQLServer, + "MERGE dbo.Target AS t USING (VALUES (1,N'a'),(2,N'b')) AS s (Id, Val) ON t.Id = s.Id WHEN MATCHED THEN UPDATE SET Val = s.Val", + "s (Id, Val) ON"}, + {keywords.DialectPostgreSQL, + "MERGE INTO t USING (SELECT 1,2) AS s(c1,c2) ON t.id = s.c1 WHEN MATCHED THEN DELETE", + "s(c1, c2) ON"}, + } + for _, tc := range tests { + tree, err := ParseWithDialect(tc.sql, tc.dialect) + if err != nil { + t.Fatalf("parse: %v", err) + } + got := tree.SQL() + if !strings.Contains(strings.ReplaceAll(got, " (", "("), strings.ReplaceAll(tc.want, " (", "(")) { + t.Errorf("%s: rendered %q lacks column aliases %q", tc.dialect, got, tc.want) + } + } +} + +// TestASTSQLFallsBackToToSQL covers statements that only implement ToSQL. +func TestASTSQLFallsBackToToSQL(t *testing.T) { + tree, err := ParseWithDialect("CREATE SEQUENCE seq_orders START WITH 1 INCREMENT BY 1", keywords.DialectMariaDB) + if err != nil { + t.Fatalf("parse: %v", err) + } + got := tree.SQL() + if !strings.Contains(strings.ToUpper(got), "CREATE SEQUENCE") || !strings.Contains(got, "seq_orders") { + t.Errorf("AST.SQL() = %q, want the CREATE SEQUENCE statement", got) + } +} diff --git a/pkg/gosqlx/concurrent_retained_ast_test.go b/pkg/gosqlx/concurrent_retained_ast_test.go new file mode 100644 index 00000000..1be8d09c --- /dev/null +++ b/pkg/gosqlx/concurrent_retained_ast_test.go @@ -0,0 +1,110 @@ +// Copyright 2026 GoSQLX Authors +// SPDX-License-Identifier: Apache-2.0 + +package gosqlx + +import ( + "context" + "encoding/json" + "fmt" + "sync" + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" + "github.com/ajitpratap0/GoSQLX/pkg/sql/parser" +) + +// Retaining every tree while other goroutines churn tokenizer/parser pools +// checks both scratch-token ownership and premature pooled-node reuse. +func TestConcurrentRetainedASTs(t *testing.T) { + tests := []struct { + name string + parse func(string) (*ast.AST, error) + tail string + }{ + {"Parse", Parse, ""}, + {"ParseBytes", func(s string) (*ast.AST, error) { return ParseBytes([]byte(s)) }, ""}, + {"ParseWithContext", func(s string) (*ast.AST, error) { return ParseWithContext(context.Background(), s) }, ""}, + {"ParseWithDialect", func(s string) (*ast.AST, error) { return ParseWithDialect(s, keywords.DialectSQLServer) }, " FOR JSON PATH"}, + {"ParserBytesWithDialect", func(s string) (*ast.AST, error) { + return parser.ParseBytesWithDialect([]byte(s), keywords.DialectSQLServer) + }, " FOR JSON PATH"}, + {"ParseTree", func(s string) (*ast.AST, error) { + tree, err := ParseTree(context.Background(), s, WithDialect("sqlserver"), WithComments()) + if err != nil { + return nil, err + } + return tree.Raw(), nil + }, " FOR JSON PATH"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + type snapshot struct { + tree *ast.AST + json string + sql string + } + const workers, retained, churn = 8, 16, 64 + var ready, done sync.WaitGroup + start := make(chan struct{}) + ready.Add(workers) + done.Add(workers) + for worker := 0; worker < workers; worker++ { + go func() { + defer done.Done() + var held []snapshot + defer func() { + for _, s := range held { + ast.ReleaseAST(s.tree) + } + }() + query := func(i int) string { + return fmt.Sprintf(`/* worker %d query %d */ SELECT "Mixed%d", N'o''clock%d', COUNT(*) OVER (PARTITION BY g ORDER BY id) FROM "Table%d" WHERE a IS NOT NULL ORDER BY id`, worker, i, i, i, worker) + tc.tail + } + for i := 0; i < retained; i++ { + tree, err := tc.parse(query(i)) + if err != nil { + t.Errorf("retain worker %d: %v", worker, err) + break + } + data, err := json.Marshal(tree) + if err != nil { + ast.ReleaseAST(tree) + t.Errorf("snapshot: %v", err) + break + } + held = append(held, snapshot{tree, string(data), FormatAST(tree)}) + } + ready.Done() + <-start + for i := 0; i < churn; i++ { + tree, err := tc.parse(query(i + retained)) + if err != nil { + t.Errorf("churn worker %d: %v", worker, err) + return + } + ast.ReleaseAST(tree) + if err := Validate(fmt.Sprintf("SELECT churn%d FROM other%d WHERE a = 'fresh'", i, worker)); err != nil { + t.Errorf("validate churn: %v", err) + return + } + } + for i, s := range held { + data, err := json.Marshal(s.tree) + if err != nil { + t.Errorf("marshal retained AST: %v", err) + continue + } + if string(data) != s.json || FormatAST(s.tree) != s.sql { + t.Errorf("retained AST %d in worker %d changed after concurrent pool reuse", i, worker) + } + } + }() + } + ready.Wait() + close(start) + done.Wait() + }) + } +} diff --git a/pkg/gosqlx/dml_grammar_test.go b/pkg/gosqlx/dml_grammar_test.go new file mode 100644 index 00000000..922eeb7f --- /dev/null +++ b/pkg/gosqlx/dml_grammar_test.go @@ -0,0 +1,242 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gosqlx + +import ( + "reflect" + "sort" + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" +) + +// dmlRoundTripCases are the DML grammar forms of UPDATE / DELETE / INSERT / +// REPLACE / MERGE: each must parse in its dialect, format to want with +// FormatAST and with the statement's own SQL(), and re-parse to the same +// text. want differs from sql only by normalization (AS dropped before a +// table alias, INNER added to a plain JOIN, INTO added where optional). +var dmlRoundTripCases = []struct { + name string + dialect keywords.SQLDialect + sql string + want string // "" means want == sql +}{ + // UPDATE / DELETE row limit (the LIMIT used to be parsed and dropped). + {"update limit", keywords.DialectMySQL, "UPDATE t SET a = 1 WHERE b = 2 LIMIT 5", ""}, + {"delete limit", keywords.DialectMySQL, "DELETE FROM t WHERE b = 2 LIMIT 5", ""}, + {"delete limit mariadb", keywords.DialectMariaDB, "DELETE FROM t WHERE b = 2 LIMIT 5", ""}, + {"delete order limit", keywords.DialectMySQL, "DELETE FROM t ORDER BY id LIMIT 5", ""}, + {"update order limit", keywords.DialectMySQL, "UPDATE t SET a = 1 WHERE b = 2 ORDER BY id DESC LIMIT 5", ""}, + {"sqlite delete order limit", keywords.DialectSQLite, "DELETE FROM t WHERE a = 1 ORDER BY id LIMIT 3", ""}, + + // Target aliases. + {"update as alias", keywords.DialectPostgreSQL, "UPDATE t AS x SET a = 1", "UPDATE t x SET a = 1"}, + {"update bare alias", keywords.DialectOracle, "UPDATE t x SET a = 1", ""}, + {"update alias sqlite from", keywords.DialectSQLite, "UPDATE orders AS o SET status = 'vip' FROM customers AS c WHERE c.id = o.customer_id", + "UPDATE orders o SET status = 'vip' FROM customers c WHERE c.id = o.customer_id"}, + {"delete bare alias", keywords.DialectPostgreSQL, "DELETE FROM t x WHERE x.a = 1", ""}, + {"delete as alias", keywords.DialectPostgreSQL, "DELETE FROM t AS x WHERE x.a = 1", "DELETE FROM t x WHERE x.a = 1"}, + {"update only", keywords.DialectPostgreSQL, "UPDATE ONLY t SET x = 1 WHERE d > 0", ""}, + {"delete only", keywords.DialectPostgreSQL, "DELETE FROM ONLY t WHERE a = 1", ""}, + + // DELETE ... USING and multi-table forms. + {"delete using", keywords.DialectPostgreSQL, "DELETE FROM a USING b WHERE a.id = b.id RETURNING a.id", ""}, + {"delete using snowflake", keywords.DialectSnowflake, "DELETE FROM a USING b WHERE a.id = b.id", ""}, + {"mysql delete using join", keywords.DialectMySQL, "DELETE FROM a, b USING a INNER JOIN b ON a.id = b.a_id WHERE a.flag = 0", ""}, + {"mysql delete targets from", keywords.DialectMySQL, "DELETE o FROM o JOIN c ON c.id = o.cid", "DELETE o FROM o INNER JOIN c ON c.id = o.cid"}, + {"mysql delete two targets", keywords.DialectMySQL, "DELETE a, b FROM p a LEFT JOIN m b ON b.pid = a.id WHERE a.k = 'r'", ""}, + {"sqlserver delete target from", keywords.DialectSQLServer, "DELETE o FROM dbo.Orders o LEFT JOIN dbo.Customers c ON c.Id = o.CustomerId WHERE c.Id IS NULL", ""}, + {"oracle delete without from", keywords.DialectOracle, "DELETE employees WHERE employee_id = 999", "DELETE FROM employees WHERE employee_id = 999"}, + {"oracle delete derived", keywords.DialectOracle, "DELETE FROM (SELECT * FROM e WHERE d = 10) WHERE s < 1000", ""}, + + // Multi-table UPDATE. + {"mysql update join", keywords.DialectMySQL, "UPDATE t JOIN u ON t.id = u.id SET t.a = 1", "UPDATE t INNER JOIN u ON t.id = u.id SET t.a = 1"}, + {"mysql update comma", keywords.DialectMySQL, "UPDATE t, u SET t.a = u.b WHERE t.id = u.id", ""}, + {"mysql update join derived", keywords.DialectMySQL, "UPDATE c INNER JOIN (SELECT id, MAX(x) AS m FROM o GROUP BY id) s ON s.id = c.id SET c.m = s.m", ""}, + {"sqlserver update from join", keywords.DialectSQLServer, "UPDATE t SET t.a = s.a FROM dbo.Orders t INNER JOIN dbo.S s ON s.id = t.id WHERE t.b = 1", ""}, + {"postgres update from join", keywords.DialectPostgreSQL, "UPDATE t SET a = s.b FROM s INNER JOIN u ON s.id = u.id WHERE t.id = s.id", ""}, + {"oracle update derived", keywords.DialectOracle, "UPDATE (SELECT e.s AS o, n.s AS n FROM e INNER JOIN n ON e.id = n.id) SET o = n", ""}, + + // Modifiers. + {"insert ignore", keywords.DialectMySQL, "INSERT IGNORE INTO t (a) VALUES (1)", ""}, + {"insert low priority ignore", keywords.DialectMySQL, "INSERT LOW_PRIORITY IGNORE INTO t (a) VALUES (1)", ""}, + {"update modifiers", keywords.DialectMySQL, "UPDATE LOW_PRIORITY IGNORE t SET x = 1", ""}, + {"delete modifiers", keywords.DialectMariaDB, "DELETE LOW_PRIORITY QUICK IGNORE FROM logs WHERE ts < 5", ""}, + {"table named like modifier", keywords.DialectMySQL, "UPDATE ignore SET a = 1", ""}, + {"sqlite insert or replace", keywords.DialectSQLite, "INSERT OR REPLACE INTO t (a) VALUES (1)", ""}, + {"sqlite insert or rollback", keywords.DialectSQLite, "INSERT OR ROLLBACK INTO t (a) VALUES (1)", ""}, + {"sqlite update or ignore", keywords.DialectSQLite, "UPDATE OR IGNORE t SET a = 1 WHERE id = 3", ""}, + {"snowflake insert overwrite", keywords.DialectSnowflake, "INSERT OVERWRITE INTO d SELECT * FROM s", ""}, + {"sqlserver update top", keywords.DialectSQLServer, "UPDATE TOP (500) dbo.Queue SET Status = 1 WHERE Status = 0", ""}, + {"sqlserver delete top percent", keywords.DialectSQLServer, "DELETE TOP (10) PERCENT FROM t", ""}, + {"sqlserver insert without into", keywords.DialectSQLServer, "INSERT t (a) VALUES (1)", "INSERT INTO t (a) VALUES (1)"}, + + // SET list targets and values. + {"set tuple", keywords.DialectPostgreSQL, "UPDATE t SET (a, b) = (1, 2) WHERE id = 5", ""}, + {"set tuple subquery", keywords.DialectOracle, "UPDATE t SET (a, b) = (SELECT x, y FROM u WHERE u.id = t.id) WHERE id = 6", ""}, + {"set qualified target", keywords.DialectSQLServer, "UPDATE o SET o.a = 1 FROM dbo.Orders AS o", "UPDATE o SET o.a = 1 FROM dbo.Orders o"}, + {"set default", keywords.DialectPostgreSQL, "UPDATE t SET a = DEFAULT, b = 2", ""}, + {"set keyword column", keywords.DialectSQLite, "UPDATE OR REPLACE s SET key = 'k2' WHERE id = 1", ""}, + {"values default", keywords.DialectPostgreSQL, "INSERT INTO t (a, b) VALUES (DEFAULT, 1), (2, DEFAULT)", ""}, + {"values empty mysql", keywords.DialectMySQL, "INSERT INTO t () VALUES ()", "INSERT INTO t VALUES ()"}, + {"keyword insert columns", keywords.DialectSQLite, "INSERT INTO settings (key, value) VALUES ('a', '1')", ""}, + + // INSERT variants. + {"mysql row alias", keywords.DialectMySQL, "INSERT INTO t (a) VALUES (1) AS n ON DUPLICATE KEY UPDATE a = n.a", ""}, + {"mysql row alias columns", keywords.DialectMySQL, "INSERT INTO t (a) VALUES (1) AS n (x) ON DUPLICATE KEY UPDATE a = n.x", ""}, + {"on conflict where", keywords.DialectPostgreSQL, "INSERT INTO t (a) VALUES (1) ON CONFLICT (a) WHERE b > 0 DO NOTHING", ""}, + {"on conflict expression", keywords.DialectPostgreSQL, "INSERT INTO t (a) VALUES (1) ON CONFLICT (lower(a)) DO NOTHING", ""}, + {"on conflict keyword column", keywords.DialectSQLite, "INSERT INTO s (key) VALUES ('p') ON CONFLICT (key) DO UPDATE SET key = excluded.key WHERE s.key <> 'x'", ""}, + {"overriding system value", keywords.DialectPostgreSQL, "INSERT INTO t (id, name) OVERRIDING SYSTEM VALUE VALUES (1, 'x')", ""}, + {"overriding user value", keywords.DialectPostgreSQL, "INSERT INTO t (id) OVERRIDING USER VALUE SELECT 1", ""}, + {"returning alias", keywords.DialectPostgreSQL, "INSERT INTO t (a) VALUES (1) RETURNING a AS x, (xmax = 0) AS inserted", ""}, + {"delete returning alias", keywords.DialectPostgreSQL, "DELETE FROM t x WHERE x.c < 1 RETURNING x.id AS deleted_id", ""}, + {"oracle returning into", keywords.DialectOracle, "INSERT INTO t (a) VALUES (1) RETURNING a INTO :x", ""}, + {"oracle update returning into", keywords.DialectOracle, "UPDATE t SET b = b - :amt WHERE id = :id RETURNING b INTO :nb", ""}, + {"oracle delete returning into", keywords.DialectOracle, "DELETE FROM t WHERE d < SYSDATE RETURNING COUNT(*) INTO :cnt", ""}, + {"oracle insert all", keywords.DialectOracle, "INSERT ALL INTO a (x) VALUES (1) INTO b (x) VALUES (2) SELECT 1 FROM dual", ""}, + {"oracle insert all when", keywords.DialectOracle, "INSERT ALL WHEN r = 'E' THEN INTO e VALUES (id, amt) WHEN r = 'W' THEN INTO w VALUES (id, amt) ELSE INTO o VALUES (id, amt) SELECT id, r, amt FROM s", ""}, + {"oracle insert first", keywords.DialectOracle, "INSERT FIRST WHEN amt > 10 THEN INTO big VALUES (id, amt) ELSE INTO small VALUES (id, amt) SELECT id, amt FROM s", ""}, + {"clickhouse insert settings", keywords.DialectClickHouse, "INSERT INTO t (a, b) SETTINGS async_insert = 1, wait_for_async_insert = 0 VALUES (1, 'x')", ""}, + {"clickhouse insert settings format", keywords.DialectClickHouse, "INSERT INTO t SETTINGS async_insert = 1 FORMAT CSV", ""}, + {"clickhouse insert function", keywords.DialectClickHouse, "INSERT INTO FUNCTION s3('https://b.s3.amazonaws.com/o.parquet', 'Parquet') SELECT * FROM events", ""}, + {"clickhouse insert format", keywords.DialectClickHouse, "INSERT INTO t (a) FORMAT JSONEachRow", ""}, + {"clickhouse values format", keywords.DialectClickHouse, "INSERT INTO t VALUES (1) FORMAT Values", ""}, + + // REPLACE. + {"replace select", keywords.DialectMySQL, "REPLACE INTO t (a) SELECT a FROM u", ""}, + {"replace set", keywords.DialectMySQL, "REPLACE t SET a = 1, b = 'x'", "REPLACE INTO t SET a = 1, b = 'x'"}, + {"replace returning", keywords.DialectMariaDB, "REPLACE INTO s (k, v) VALUES ('a', '1') RETURNING k, v", ""}, + {"replace keyword columns", keywords.DialectSQLite, "REPLACE INTO settings (key, value) VALUES ('lang', 'en')", ""}, + + // OUTPUT. + {"delete output", keywords.DialectSQLServer, "DELETE FROM t OUTPUT DELETED.id WHERE a = 1", ""}, + {"delete output into", keywords.DialectSQLServer, "DELETE FROM dbo.O OUTPUT DELETED.Id, DELETED.T INTO dbo.A (Id, T) WHERE c < '2020-01-01'", ""}, + {"update output alias", keywords.DialectSQLServer, "UPDATE dbo.O SET S = 'x' OUTPUT DELETED.S AS OldS, INSERTED.S AS NewS WHERE Id = 1", ""}, + {"insert output into variable", keywords.DialectSQLServer, "INSERT INTO dbo.Log (Msg) OUTPUT INSERTED.Id INTO @Ids (Id) VALUES ('x')", ""}, + {"delete output then from", keywords.DialectSQLServer, "DELETE FROM t OUTPUT deleted.* FROM t JOIN u ON u.id = t.id", "DELETE t OUTPUT deleted.* FROM t INNER JOIN u ON u.id = t.id"}, + + // MERGE. + {"merge derived source", keywords.DialectSnowflake, "MERGE INTO t USING (SELECT k FROM s2) s ON t.k = s.k WHEN NOT MATCHED THEN INSERT (k) VALUES (s.k)", ""}, + {"merge derived source as", keywords.DialectSQLServer, "MERGE INTO dbo.I AS t USING (SELECT Sku, SUM(Qty) AS Qty FROM dbo.R GROUP BY Sku) AS s ON t.Sku = s.Sku WHEN MATCHED THEN UPDATE SET t.OnHand = t.OnHand + s.Qty", + "MERGE INTO dbo.I t USING (SELECT Sku, SUM(Qty) AS Qty FROM dbo.R GROUP BY Sku) s ON t.Sku = s.Sku WHEN MATCHED THEN UPDATE SET t.OnHand = t.OnHand + s.Qty"}, + {"merge do nothing returning", keywords.DialectPostgreSQL, "MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DO NOTHING WHEN NOT MATCHED THEN INSERT (id) VALUES (s.id) RETURNING merge_action(), t.*", ""}, + {"merge by target", keywords.DialectSQLServer, "MERGE INTO dbo.T tgt USING dbo.S src ON tgt.Id = src.Id WHEN NOT MATCHED BY TARGET THEN INSERT (Id) VALUES (src.Id) WHEN NOT MATCHED BY SOURCE THEN DELETE", ""}, + {"merge oracle delete where", keywords.DialectOracle, "MERGE INTO t USING s ON (t.a = s.a) WHEN MATCHED THEN UPDATE SET b = s.b DELETE WHERE (s.c = 1)", ""}, + {"merge oracle update and insert where", keywords.DialectOracle, "MERGE INTO e USING u ON (e.id = u.id) WHEN MATCHED THEN UPDATE SET e.s = u.s WHERE e.s <> u.s WHEN NOT MATCHED THEN INSERT (id, s) VALUES (u.id, u.s) WHERE u.s > 0", ""}, + {"merge output", keywords.DialectSQLServer, "MERGE INTO t x USING u y ON x.id = y.id WHEN MATCHED THEN UPDATE SET a = 1 OUTPUT $action, INSERTED.id", ""}, +} + +func TestDMLGrammarRoundTrip(t *testing.T) { + for _, tc := range dmlRoundTripCases { + t.Run(tc.name, func(t *testing.T) { + want := tc.want + if want == "" { + want = tc.sql + } + tree, err := ParseWithDialect(tc.sql, tc.dialect) + if err != nil { + t.Fatalf("parse %q (%s): %v", tc.sql, tc.dialect, err) + } + defer ast.ReleaseAST(tree) + if len(tree.Statements) != 1 { + t.Fatalf("expected 1 statement, got %d", len(tree.Statements)) + } + if got := FormatAST(tree, WithIndent(0)); got != want { + t.Errorf("FormatAST:\n got: %s\nwant: %s", got, want) + } + sqler, ok := tree.Statements[0].(interface{ SQL() string }) + if !ok { + t.Fatalf("%T has no SQL()", tree.Statements[0]) + } + if got := sqler.SQL(); got != want { + t.Errorf("SQL():\n got: %s\nwant: %s", got, want) + } + // The readable (one clause per line) form must re-parse to itself. + readable := FormatAST(tree) + tree2, err := ParseWithDialect(readable, tc.dialect) + if err != nil { + t.Fatalf("reparse %q: %v", readable, err) + } + defer ast.ReleaseAST(tree2) + if got := FormatAST(tree2); got != readable { + t.Errorf("not idempotent:\n got: %s\nwant: %s", got, readable) + } + if got := FormatAST(tree2, WithIndent(0)); got != want { + t.Errorf("reparsed compact form:\n got: %s\nwant: %s", got, want) + } + }) + } +} + +// TestDMLGrammarDialectGates checks that dialect-specific DML syntax is +// rejected outside its dialects instead of being misread. +func TestDMLGrammarDialectGates(t *testing.T) { + tests := []struct { + name string + dialect keywords.SQLDialect + sql string + }{ + {"order by on delete in postgres", keywords.DialectPostgreSQL, "DELETE FROM t ORDER BY id"}, + {"mysql modifier in postgres", keywords.DialectPostgreSQL, "INSERT IGNORE INTO t (a) VALUES (1)"}, + {"sqlite conflict modifier in mysql", keywords.DialectMySQL, "INSERT OR REPLACE INTO t (a) VALUES (1)"}, + {"multi-table update in postgres", keywords.DialectPostgreSQL, "UPDATE t JOIN u ON t.id = u.id SET a = 1"}, + {"delete without from in postgres", keywords.DialectPostgreSQL, "DELETE t WHERE a = 1"}, + {"insert without into in postgres", keywords.DialectPostgreSQL, "INSERT t (a) VALUES (1)"}, + {"multi-table delete without from", keywords.DialectMySQL, "DELETE a, b WHERE a.id = 1"}, + {"insert all outside oracle", keywords.DialectPostgreSQL, "INSERT ALL INTO a VALUES (1) SELECT 1"}, + {"insert first needs when", keywords.DialectOracle, "INSERT FIRST INTO a VALUES (1) SELECT 1 FROM dual"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + tree, err := ParseWithDialect(tc.sql, tc.dialect) + if err == nil { + ast.ReleaseAST(tree) + t.Fatalf("expected an error for %q in %s", tc.sql, tc.dialect) + } + }) + } +} + +// TestDMLGrammarExtractTables checks that ExtractTables sees the tables of +// multi-table UPDATE and DELETE statements. +func TestDMLGrammarExtractTables(t *testing.T) { + tests := []struct { + name string + dialect keywords.SQLDialect + sql string + want []string + }{ + {"update join", keywords.DialectMySQL, "UPDATE a JOIN b ON a.id = b.id SET a.x = b.x", []string{"a", "b"}}, + {"update comma", keywords.DialectMySQL, "UPDATE a, b SET a.x = b.x WHERE a.id = b.id", []string{"a", "b"}}, + {"delete targets from", keywords.DialectMySQL, "DELETE o FROM orders o JOIN users u ON u.id = o.uid", []string{"orders", "users"}}, + {"delete using join", keywords.DialectMySQL, "DELETE FROM a USING a JOIN b ON a.id = b.id", []string{"a", "b"}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + tree, err := ParseWithDialect(tc.sql, tc.dialect) + if err != nil { + t.Fatalf("parse: %v", err) + } + defer ast.ReleaseAST(tree) + got := ExtractTables(tree) + sort.Strings(got) + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("ExtractTables = %v, want %v", got, tc.want) + } + }) + } +} diff --git a/pkg/gosqlx/extract.go b/pkg/gosqlx/extract.go index 4e807313..6fcbce18 100644 --- a/pkg/gosqlx/extract.go +++ b/pkg/gosqlx/extract.go @@ -375,6 +375,35 @@ func (c *metadataCollector) addTable(name string) { } } +// addTableRef adds the table a reference names; a parenthesized join group +// adds the tables inside it. +func (c *metadataCollector) addTableRef(t *ast.TableReference) { + if g := ast.TableRefJoinGroup(t); g != nil { + for i := range g.Tables { + c.addTableRef(&g.Tables[i]) + } + for i := range g.Joins { + c.addTableRef(&g.Joins[i].Right) + } + return + } + c.addTable(t.Name) +} + +// addDMLTargetTables adds the extra UPDATE / DELETE targets and the joined +// tables of a multi-table statement. +func (c *metadataCollector) addDMLTargetTables(d *ast.DMLTargetFields) { + if d == nil { + return + } + for i := range d.Tables { + c.addTable(d.Tables[i].Name) + } + for i := range d.Joins { + c.addTableRef(&d.Joins[i].Right) + } +} + func (c *metadataCollector) addColumn(table, name string, want extractKind, aliases map[string]struct{}) { if name == "" || name == "*" { return @@ -427,11 +456,11 @@ func (c *metadataCollector) collectFromNode(node ast.Node, aliases map[string]st } } } - for _, from := range n.From { - c.addTable(from.Name) + for i := range n.From { + c.addTableRef(&n.From[i]) } - for _, join := range n.Joins { - c.addTable(join.Right.Name) + for i := range n.Joins { + c.addTableRef(&n.Joins[i].Right) } for _, col := range n.Columns { c.collectFromExpression(col, c.want, nil) @@ -479,6 +508,7 @@ func (c *metadataCollector) collectFromNode(node ast.Node, aliases map[string]st for _, from := range n.From { c.addTable(from.Name) } + c.addDMLTargetTables(n.Extras) for i := range n.Assignments { c.collectFromNode(&n.Assignments[i], nil) } @@ -487,10 +517,24 @@ func (c *metadataCollector) collectFromNode(node ast.Node, aliases map[string]st c.collectFromNode(n.With, nil) } case *ast.DeleteStatement: - c.addTable(n.TableName) + // In DELETE t1 FROM t1 JOIN t2 the targets name FROM entries + // (often by alias), so the tables are the FROM list. + if len(n.From) == 0 { + c.addTable(n.TableName) + } for _, using := range n.Using { c.addTable(using.Name) } + for _, from := range n.From { + c.addTable(from.Name) + } + if len(n.From) == 0 { + c.addDMLTargetTables(n.Extras) + } else if n.Extras != nil { + for i := range n.Extras.Joins { + c.addTableRef(&n.Extras.Joins[i].Right) + } + } c.collectFromExpression(n.Where, c.want, nil) if n.With != nil { c.collectFromNode(n.With, nil) @@ -586,6 +630,24 @@ func (c *metadataCollector) collectFromExpression(expr ast.Expression, want extr case *ast.AliasedExpression: // Unwrap the aliased expression and collect from inner expression c.collectFromExpression(e.Expr, want, aliases) + case *ast.KeywordArgument: + c.collectFromExpression(e.Value, want, aliases) + case *ast.MatchAgainstExpression: + if want&extractFunctions != 0 { + c.addFunction("MATCH") + } + for _, col := range e.Columns { + c.collectFromExpression(col, want, aliases) + } + c.collectFromExpression(e.Query, want, aliases) + case *ast.OuterJoinMarker: + c.collectFromExpression(e.Expr, want, aliases) + case *ast.ConnectByRootExpression: + c.collectFromExpression(e.Expr, want, aliases) + case *ast.AssignmentExpression: + c.collectFromExpression(e.Value, want, aliases) + case *ast.IntervalExpression: + c.collectFromExpression(e.ValueExpr, want, aliases) } } diff --git a/pkg/gosqlx/format_comments_test.go b/pkg/gosqlx/format_comments_test.go new file mode 100644 index 00000000..bb157b93 --- /dev/null +++ b/pkg/gosqlx/format_comments_test.go @@ -0,0 +1,254 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gosqlx + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/ajitpratap0/GoSQLX/pkg/formatter" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" +) + +func TestFormat_PreservesCommentsAndHints(t *testing.T) { + opts := DefaultFormatOptions() + opts.UppercaseKeywords = true + compact := opts + compact.IndentSize = 0 + + tests := []struct { + name string + sql string + opts FormatOptions + want string + }{ + {"hint and trailing comment", "select /*+ hint */ a from t -- note", compact, "SELECT /*+ hint */ a FROM t -- note"}, + {"oracle full hint", "select /*+ FULL(t) */ a from t", compact, "SELECT /*+ FULL(t) */ a FROM t"}, + {"leading comment", "-- report\nselect a from t", compact, "-- report\nSELECT a FROM t"}, + {"comment between statements", "select 1; -- one\n-- two\nselect 2", compact, "SELECT 1; -- one\n-- two\nSELECT 2"}, + {"inner comment moves up", "select a, /* new */ b from t", compact, "/* new */\nSELECT a, b FROM t"}, + {"no comments unchanged path", "select a from t", compact, "SELECT a FROM t"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := Format(tt.sql, tt.opts) + if err != nil { + t.Fatalf("Format: %v", err) + } + if got != tt.want { + t.Fatalf("got %q, want %q", got, tt.want) + } + }) + } +} + +func TestFormat_RefusesToDropAnOptimizerHint(t *testing.T) { + _, err := Format("select a from /*+ FULL(t) */ t", DefaultFormatOptions()) + if !errors.Is(err, formatter.ErrUnplaceableComment) { + t.Fatalf("err = %v, want it to wrap formatter.ErrUnplaceableComment", err) + } +} + +func TestFormat_KeepsHintInSubquery(t *testing.T) { + got, err := Format("select * from (select /*+ FULL(t) */ a from t) s", DefaultFormatOptions()) + if err != nil { + t.Fatalf("Format: %v", err) + } + if !strings.Contains(got, "SELECT /*+ FULL(t) */ a") { + t.Fatalf("got %q, lost the hint after the inner SELECT", got) + } +} + +func TestFormat_AddSemicolonWithComments(t *testing.T) { + opts := DefaultFormatOptions() + opts.AddSemicolon = true + opts.IndentSize = 0 + got, err := Format("select 1 -- a\n; select 2 -- b", opts) + if err != nil { + t.Fatalf("Format: %v", err) + } + if strings.Count(got, ";") != 2 { + t.Fatalf("got %q, want exactly two semicolons", got) + } + for _, c := range []string{"-- a", "-- b"} { + if !strings.Contains(got, c) { + t.Fatalf("got %q, lost %q", got, c) + } + } +} + +func TestParseTree_WithCommentsKeepsThemForFormatTree(t *testing.T) { + const sql = "-- header\nSELECT a FROM t -- trailing" + plain, err := ParseTree(context.Background(), sql) + if err != nil { + t.Fatal(err) + } + if out := FormatTree(plain); strings.Contains(out, "--") { + t.Fatalf("comments kept without WithComments: %q", out) + } + withC, err := ParseTree(context.Background(), sql, WithComments()) + if err != nil { + t.Fatal(err) + } + out := FormatTree(withC) + for _, c := range []string{"-- header", "-- trailing"} { + if !strings.Contains(out, c) { + t.Fatalf("FormatTree output %q lost %q", out, c) + } + } +} + +func TestFormatWithDialect(t *testing.T) { + opts := DefaultFormatOptions() + opts.IndentSize = 0 + opts.UppercaseKeywords = true + + got, err := FormatWithDialect("select /*+ MAX_EXECUTION_TIME(1000) */ a from t -- why", opts, "mysql") + if err != nil { + t.Fatalf("FormatWithDialect: %v", err) + } + if want := "SELECT /*+ MAX_EXECUTION_TIME(1000) */ a FROM t -- why"; got != want { + t.Fatalf("got %q, want %q", got, want) + } + + // An empty dialect is Format. + a, errA := FormatWithDialect("select 1 -- x", opts, "") + b, errB := Format("select 1 -- x", opts) + if errA != nil || errB != nil || a != b { + t.Fatalf("empty dialect: %q (%v) vs Format %q (%v)", a, errA, b, errB) + } + + if _, err := FormatWithDialect("select 1", opts, "nosuchsql"); !errors.Is(err, ErrUnsupportedDialect) { + t.Fatalf("unknown dialect: err = %v, want ErrUnsupportedDialect", err) + } + if _, err := FormatWithDialect("selec 1", opts, "mysql"); !errors.Is(err, ErrSyntax) { + t.Fatalf("invalid SQL: err = %v, want ErrSyntax", err) + } +} + +// TestFormat_NestedHintsFollowTheirSelect checks that a hint after a nested +// SELECT stays with that SELECT when the renderer writes the clauses holding +// the subqueries in a different order (Oracle START WITH before CONNECT BY), +// (see formatter.TestOccMatcher for the cases that are refused). +func TestFormat_NestedHintsFollowTheirSelect(t *testing.T) { + opts := DefaultFormatOptions() + opts.IndentSize = 0 + tests := []struct { + name string + sql string + want []string // fragments of the output + }{ + { + name: "distinct subqueries swap places", + sql: "SELECT a FROM t CONNECT BY PRIOR id = pid AND x IN (SELECT /*+ H1 */ 1 FROM u) START WITH id IN (SELECT /*+ H2 */ 2 FROM v)", + want: []string{"x IN (SELECT /*+ H1 */ 1 FROM u)", "id IN (SELECT /*+ H2 */ 2 FROM v)"}, + }, + { + name: "one hinted subquery moves", + sql: "SELECT a FROM t CONNECT BY PRIOR id = pid AND x IN (SELECT /*+ H1 */ 1 FROM u) START WITH id IN (SELECT 1 FROM u)", + want: []string{"x IN (SELECT /*+ H1 */ 1 FROM u)", "id IN (SELECT 1 FROM u)"}, + }, + { + name: "clauses already in rendered order", + sql: "SELECT a FROM t START WITH id IN (SELECT /*+ H2 */ 2 FROM v) CONNECT BY PRIOR id = pid AND x IN (SELECT /*+ H1 */ 1 FROM u)", + want: []string{"x IN (SELECT /*+ H1 */ 1 FROM u)", "id IN (SELECT /*+ H2 */ 2 FROM v)"}, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := FormatWithDialect(tc.sql, opts, "oracle") + if err != nil { + t.Fatalf("FormatWithDialect: %v", err) + } + for _, w := range tc.want { + if !strings.Contains(got, w) { + t.Errorf("output lacks %q:\n%s", w, got) + } + } + }) + } +} + +// TestFormatWithDialect_CommentsFollowStatementStarts checks that comments +// stay with their statements when a procedure or trigger body holds ';' or +// T-SQL statements have no ';' between them, and that the result is stable. +func TestFormatWithDialect_CommentsFollowStatementStarts(t *testing.T) { + opts := DefaultFormatOptions() + opts.IndentSize = 0 + opts.UppercaseKeywords = true + tests := []struct { + dialect string + sql string + want string + }{ + {"sqlserver", + "-- A\nSELECT 1\nSELECT 2;\n-- B: the proc\nCREATE PROC p AS BEGIN SELECT 1; END;\n", + "-- A\nSELECT 1;\nSELECT 2;\n-- B: the proc\nCREATE PROC p AS BEGIN SELECT 1; END"}, + {"mysql", + "-- Section one\nSELECT 1 FROM users;\n-- Section two: trigger\nCREATE TRIGGER t AFTER INSERT ON x FOR EACH ROW BEGIN SELECT 1; SELECT 2; END;\n-- Section three\nSELECT 3 FROM orders;\n", + "-- Section one\nSELECT 1 FROM users;\n-- Section two: trigger\nCREATE TRIGGER t AFTER INSERT ON x FOR EACH ROW BEGIN SELECT 1; SELECT 2; END;\n-- Section three\nSELECT 3 FROM orders"}, + } + for _, tc := range tests { + t.Run(tc.dialect, func(t *testing.T) { + got, err := FormatWithDialect(tc.sql, opts, keywords.SQLDialect(tc.dialect)) + if err != nil { + t.Fatal(err) + } + if got != tc.want { + t.Fatalf("got %q\nwant %q", got, tc.want) + } + again, err := FormatWithDialect(got, opts, keywords.SQLDialect(tc.dialect)) + if err != nil || again != got { + t.Fatalf("not idempotent: %v\n%q\n%q", err, got, again) + } + }) + } +} + +// TestFormat_CommentsScaleLinearly guards against quadratic optimizer-hint +// placement and comment lookup (seconds and gigabytes before the fix). The +// bounds are generous so that -race and slow CI machines pass. +func TestFormat_CommentsScaleLinearly(t *testing.T) { + if testing.Short() { + t.Skip("timing test") + } + tests := []struct { + name string + sql string + }{ + {"hints", strings.Repeat("SELECT /*+ H */ 1 UNION ALL ", 3000) + "SELECT /*+ H */ 1"}, + {"comments", strings.Repeat("SELECT 1; -- c\n", 40000)}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + start := time.Now() + out, err := Format(tc.sql, DefaultFormatOptions()) + elapsed := time.Since(start) + if err != nil { + t.Fatal(err) + } + if strings.Count(out, "/*+ H */") != strings.Count(tc.sql, "/*+ H */") || + strings.Count(out, "-- c") != strings.Count(tc.sql, "-- c") { + t.Fatal("comments lost") + } + if limit := 3 * time.Second; elapsed > limit { + t.Fatalf("took %v, want under %v", elapsed, limit) + } + }) + } +} diff --git a/pkg/gosqlx/format_options_alloc_test.go b/pkg/gosqlx/format_options_alloc_test.go new file mode 100644 index 00000000..e5d14c85 --- /dev/null +++ b/pkg/gosqlx/format_options_alloc_test.go @@ -0,0 +1,65 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gosqlx + +import "testing" + +// TestFormatAST_Options checks how FormatAST applies its options, including +// none and nil ones, which take the allocation-free default path. +func TestFormatAST_Options(t *testing.T) { + tree, err := Parse("select a, b from t where c = 1") + if err != nil { + t.Fatalf("Parse: %v", err) + } + def := FormatAST(tree) + tests := []struct { + name string + opts []FormatOption + want string + }{ + {"none", nil, def}, + {"nil option", []FormatOption{nil}, def}, + {"default indent", []FormatOption{WithIndent(2)}, def}, + {"compact", []FormatOption{WithIndent(0)}, "SELECT a, b FROM t WHERE c = 1"}, + {"uppercase compact", []FormatOption{WithIndent(0), WithUppercaseKeywords(true)}, "SELECT a, b FROM t WHERE c = 1"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := FormatAST(tree, tc.opts...); got != tc.want { + t.Errorf("FormatAST = %q, want %q", got, tc.want) + } + }) + } + if def == "SELECT a, b FROM t WHERE c = 1" { + t.Errorf("default FormatAST = %q, want the indented layout", def) + } +} + +// TestFormatAST_NoOptionsAllocatesLess checks that a call without options +// does not pay for the option bag a call with options needs. +func TestFormatAST_NoOptionsAllocatesLess(t *testing.T) { + if testing.CoverMode() != "" { + t.Skip("allocation counts are not stable under coverage instrumentation") + } + tree, err := Parse("SELECT a FROM t") + if err != nil { + t.Fatalf("Parse: %v", err) + } + plain := testing.AllocsPerRun(50, func() { _ = FormatAST(tree) }) + withOpt := testing.AllocsPerRun(50, func() { _ = FormatAST(tree, WithIndent(2)) }) + if plain >= withOpt { + t.Errorf("FormatAST allocs: no options %v, with an option %v; want fewer without", plain, withOpt) + } +} diff --git a/pkg/gosqlx/gosqlx.go b/pkg/gosqlx/gosqlx.go index 73b1e268..9b4f114a 100644 --- a/pkg/gosqlx/gosqlx.go +++ b/pkg/gosqlx/gosqlx.go @@ -20,6 +20,7 @@ import ( "strings" "time" + "github.com/ajitpratap0/GoSQLX/internal/tokscratch" "github.com/ajitpratap0/GoSQLX/pkg/fingerprint" "github.com/ajitpratap0/GoSQLX/pkg/formatter" "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" @@ -111,8 +112,9 @@ func Parse(sql string) (*ast.AST, error) { tkz := tokenizer.GetTokenizer() defer tokenizer.PutTokenizer(tkz) - // Step 2: Tokenize SQL - tokens, err := tkz.TokenizeString(sql) + // Step 2: Tokenize SQL into the tokenizer's scratch buffer: the parser + // copies the tokens while preprocessing and they are not used after. + tokens, err := tokscratch.String(tkz, sql) if err != nil { return nil, fmt.Errorf("%w: %w", ErrTokenize, err) } @@ -199,7 +201,7 @@ func ParseWithContext(ctx context.Context, sql string) (*ast.AST, error) { defer tokenizer.PutTokenizer(tkz) // Step 2: Tokenize SQL with context support - tokens, err := tkz.TokenizeStringContext(ctx, sql) + tokens, err := tokscratch.StringContext(ctx, tkz, sql) if err != nil { if ctxErr := ctx.Err(); ctxErr != nil { return nil, fmt.Errorf("%w: %w", ErrTimeout, ctxErr) @@ -277,7 +279,7 @@ func Validate(sql string) error { } // Use the dedicated validation fast-path that avoids building a full AST - err := parser.ValidateBytes([]byte(sql)) + err := parser.Validate(sql) if err != nil { return fmt.Errorf("%w: %w", ErrSyntax, err) } @@ -303,7 +305,7 @@ func ParseBytes(sql []byte) (*ast.AST, error) { tkz := tokenizer.GetTokenizer() defer tokenizer.PutTokenizer(tkz) - tokens, err := tkz.Tokenize(sql) + tokens, err := tokscratch.Bytes(tkz, sql) if err != nil { return nil, fmt.Errorf("%w: %w", ErrTokenize, err) } @@ -427,7 +429,7 @@ func ParseMultiple(queries []string) ([]*ast.AST, error) { p.Reset() // Tokenize - tokens, err := tkz.TokenizeString(sql) + tokens, err := tokscratch.String(tkz, sql) if err != nil { releaseASTs(results) return nil, fmt.Errorf("query %d: %w: %w", i, ErrTokenize, err) @@ -472,7 +474,7 @@ func ValidateMultiple(queries []string) error { p.Reset() // Tokenize - tokens, err := tkz.TokenizeString(sql) + tokens, err := tokscratch.String(tkz, sql) if err != nil { return fmt.Errorf("query %d: %w: %w", i, ErrTokenize, err) } @@ -608,13 +610,56 @@ func DefaultFormatOptions() FormatOptions { // opts.UppercaseKeywords = true // formatted, err := gosqlx.Format(sql, opts) // +// Comments are preserved: a comment between statements stays between them, a +// trailing comment stays after its statement, and a comment inside a statement +// moves to the line above that statement. An optimizer hint (/*+ ... */) +// directly after the first keyword of a statement is kept there. SQL whose +// comments cannot be kept faithfully, such as a hint inside a subquery, is +// rejected with an error wrapping formatter.ErrUnplaceableComment instead of +// being rewritten without them. +// // Returns the formatted SQL string or an error if SQL is invalid. func Format(sql string, options FormatOptions) (string, error) { - // Parse SQL into AST - parsedAST, err := Parse(sql) + return formatSQL(sql, options, "") +} + +// FormatWithDialect is Format for SQL in a specific dialect: the SQL is +// tokenized and parsed with the dialect's rules, and dialect-specific +// constructs are rendered for that dialect (for example TOP versus LIMIT). +// Comments and optimizer hints are preserved exactly as in Format. +// +// An empty dialect behaves like Format. An unknown dialect returns an error +// wrapping ErrUnsupportedDialect. +func FormatWithDialect(sql string, options FormatOptions, dialect keywords.SQLDialect) (string, error) { + if dialect != "" && !keywords.IsValidDialect(string(dialect)) { + return "", fmt.Errorf("%w: %q", ErrUnsupportedDialect, dialect) + } + return formatSQL(sql, options, dialect) +} + +func formatSQL(sql string, options FormatOptions, dialect keywords.SQLDialect) (string, error) { + tkz := tokenizer.GetTokenizer() + defer tokenizer.PutTokenizer(tkz) + if dialect != "" { + tkz.SetDialect(dialect) + } + + tokens, err := tokscratch.String(tkz, sql) if err != nil { - return "", fmt.Errorf("cannot format invalid SQL: %w", err) + return "", fmt.Errorf("cannot format invalid SQL: %w: %w", ErrTokenize, err) } + comments := tkz.Comments + + p := parser.GetParser() + defer parser.PutParser(p) + if dialect != "" { + p.ApplyOptions(parser.WithDialect(string(dialect))) + } + parsedAST, err := p.ParseFromModelTokens(tokens) + if err != nil { + return "", fmt.Errorf("cannot format invalid SQL: %w: %w", ErrSyntax, err) + } + defer ast.ReleaseAST(parsedAST) // Convert gosqlx FormatOptions to ast.FormatOptions kwCase := ast.KeywordPreserve @@ -628,10 +673,30 @@ func Format(sql string, options FormatOptions) (string, error) { LineWidth: options.SingleLineLimit, NewlinePerClause: options.IndentSize > 0, AddSemicolon: options.AddSemicolon, + Dialect: string(dialect), } - result := formatter.FormatAST(parsedAST, astOpts) - return result, nil + if len(comments) == 0 { + return formatter.FormatAST(parsedAST, astOpts), nil + } + + // Comments go back around the statements they belonged to. A comment that + // cannot be put back faithfully (an optimizer hint that is not directly + // after its statement's first keyword) is an error rather than silently + // dropped or moved: either would change what the statement does. + stmts := make([]string, len(parsedAST.Statements)) + for i, stmt := range parsedAST.Statements { + stmts[i] = formatter.FormatStatement(stmt, astOpts) + } + layout := formatter.CommentLayout{Terminator: ";", Gap: "\n", Dialect: string(dialect), Batches: formatter.BatchPositions(parsedAST), Starts: formatter.StatementPositions(parsedAST)} + if options.AddSemicolon { + layout.Terminator = "" // each rendered statement already ends with ';' + } + out, err := formatter.JoinWithComments(stmts, tokens, comments, layout) + if err != nil { + return "", fmt.Errorf("cannot format SQL: %w", err) + } + return out, nil } // ParseWithRecovery tokenizes and parses SQL with error recovery, returning @@ -645,7 +710,7 @@ func ParseWithRecovery(sql string) ([]ast.Statement, []error) { tkz := tokenizer.GetTokenizer() defer tokenizer.PutTokenizer(tkz) - tokens, err := tkz.TokenizeString(sql) + tokens, err := tokscratch.String(tkz, sql) if err != nil { return nil, []error{fmt.Errorf("%w: %w", ErrTokenize, err)} } diff --git a/pkg/gosqlx/join_group_test.go b/pkg/gosqlx/join_group_test.go new file mode 100644 index 00000000..d9ad3023 --- /dev/null +++ b/pkg/gosqlx/join_group_test.go @@ -0,0 +1,50 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gosqlx + +import ( + "reflect" + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" +) + +// TestExtractTables_JoinGroup checks that ExtractTables lists the tables +// inside a parenthesized join group. +func TestExtractTables_JoinGroup(t *testing.T) { + tests := []struct { + name string + sql string + want []string + }{ + {"group with join", "SELECT * FROM a LEFT JOIN (b JOIN c ON b.id = c.b_id) ON a.id = b.a_id", []string{"a", "b", "c"}}, + {"comma list", "SELECT * FROM t1 LEFT JOIN (t2, t3) ON t1.id = t2.id", []string{"t1", "t2", "t3"}}, + {"nested", "SELECT * FROM t1 JOIN (t2 LEFT JOIN (t3 JOIN t4 ON t3.a = t4.a) ON t2.b = t3.b) ON t1.c = t2.c", []string{"t1", "t2", "t3", "t4"}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + tree, err := ParseWithDialect(tc.sql, keywords.DialectMySQL) + if err != nil { + t.Fatal(err) + } + defer ast.ReleaseAST(tree) + got := ExtractTables(tree) + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("ExtractTables = %v, want %v", got, tc.want) + } + }) + } +} diff --git a/pkg/gosqlx/lint.go b/pkg/gosqlx/lint.go index 2cb228d9..8f793447 100644 --- a/pkg/gosqlx/lint.go +++ b/pkg/gosqlx/lint.go @@ -46,6 +46,12 @@ var defaultLinter = sync.OnceValue(func() *linter.Linter { // Lint checks SQL for style, safety, and performance issues using all 30 built-in // lint rules (L001-L030). Returns violations found and any error encountered. // +// SQL that does not tokenize or parse is reported as an error wrapping +// ErrTokenize or ErrSyntax, so a caller that only checks err cannot mistake +// invalid SQL for clean SQL. The returned *LintResult is still non-nil in that +// case and holds the violations of the text and token rules, which run on +// invalid input; callers that want only those can ignore the error. +// // This is a convenience wrapper around the pkg/linter package that uses a // shared, lazily-built linter with all rules enabled using sensible defaults. // For fine-grained control over which rules to enable or their configuration, @@ -64,11 +70,24 @@ var defaultLinter = sync.OnceValue(func() *linter.Linter { // } func Lint(sql string) (*LintResult, error) { l := defaultLinter() - fileResult := l.LintString(sql, "") + return lintResult(l.LintString(sql, "")) +} + +// lintResult converts a linter.FileResult into the Lint return values: a rule +// failure returns no result; invalid SQL returns the result together with an +// error wrapping ErrTokenize or ErrSyntax. +func lintResult(fileResult linter.FileResult) (*LintResult, error) { if fileResult.Error != nil { return nil, fileResult.Error } - return &LintResult{Violations: fileResult.Violations}, nil + res := &LintResult{Violations: fileResult.Violations} + switch { + case fileResult.TokenizeError != nil: + return res, fmt.Errorf("%w: %w", ErrTokenize, fileResult.TokenizeError) + case fileResult.ParseError != nil: + return res, fmt.Errorf("%w: %w", ErrSyntax, fileResult.ParseError) + } + return res, nil } // dialectLinters caches one linter per dialect for LintWithDialect. Only @@ -96,8 +115,9 @@ func linterForDialect(dialect sqlkeywords.SQLDialect) *linter.Linter { // LintWithDialect is like Lint but tokenizes and parses the SQL with the given // dialect, so the AST-based rules (L011-L030) also run on dialect-specific // syntax such as SQL Server's TOP or [bracketed] identifiers. An empty dialect -// behaves exactly like Lint. An unknown dialect returns an error wrapping -// ErrUnsupportedDialect, matching ParseWithDialect. +// behaves exactly like Lint, including its error for invalid SQL. An unknown +// dialect returns an error wrapping ErrUnsupportedDialect, matching +// ParseWithDialect. // // Thread Safety: safe for concurrent use. func LintWithDialect(sql string, dialect sqlkeywords.SQLDialect) (*LintResult, error) { @@ -107,11 +127,7 @@ func LintWithDialect(sql string, dialect sqlkeywords.SQLDialect) (*LintResult, e if !sqlkeywords.IsValidDialect(string(dialect)) { return nil, fmt.Errorf("%w: %q", ErrUnsupportedDialect, dialect) } - fileResult := linterForDialect(dialect).LintString(sql, "") - if fileResult.Error != nil { - return nil, fileResult.Error - } - return &LintResult{Violations: fileResult.Violations}, nil + return lintResult(linterForDialect(dialect).LintString(sql, "")) } // allRules returns all 30 built-in lint rules with sensible defaults: the diff --git a/pkg/gosqlx/lint_dialect_test.go b/pkg/gosqlx/lint_dialect_test.go index 17a24df8..4007c9fa 100644 --- a/pkg/gosqlx/lint_dialect_test.go +++ b/pkg/gosqlx/lint_dialect_test.go @@ -46,6 +46,10 @@ func TestLintWithDialect(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { res, err := LintWithDialect(sql, tt.dialect) + if err != nil && res != nil && errors.Is(err, ErrSyntax) && tt.dialect == "" { + // The default dialect does not parse TOP; that is the point. + err = nil + } if err != nil { t.Fatalf("LintWithDialect: %v", err) } @@ -56,7 +60,7 @@ func TestLintWithDialect(t *testing.T) { } // The shared default linter must stay on the default dialect. - res, err := Lint(sql) + res, err := lintTolerant(sql) if err != nil { t.Fatalf("Lint: %v", err) } diff --git a/pkg/gosqlx/lint_syntax_test.go b/pkg/gosqlx/lint_syntax_test.go new file mode 100644 index 00000000..18f4d10a --- /dev/null +++ b/pkg/gosqlx/lint_syntax_test.go @@ -0,0 +1,87 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gosqlx + +import ( + "errors" + "testing" + + sqlkeywords "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" +) + +// lintTolerant runs Lint for tests that exercise the rules on SQL the parser +// does not accept: the syntax error is expected, the violations are what is +// being checked. +func lintTolerant(sql string) (*LintResult, error) { + res, err := Lint(sql) + if err != nil && res != nil && (errors.Is(err, ErrSyntax) || errors.Is(err, ErrTokenize)) { + return res, nil + } + return res, err +} + +func TestLint_SyntaxErrorsAreReported(t *testing.T) { + tests := []struct { + name string + sql string + dialect sqlkeywords.SQLDialect + wantErr error + }{ + {"misspelled keyword", "SELEC FROM", "", ErrSyntax}, + {"incomplete statement", "SELECT FROM WHERE (", "", ErrSyntax}, + {"unterminated string", "SELECT 'abc", "", ErrTokenize}, + {"dialect: syntax error", "SELEC FROM", sqlkeywords.DialectPostgreSQL, ErrSyntax}, + {"dialect: tokenize error", "SELECT 'abc", sqlkeywords.DialectMySQL, ErrTokenize}, + {"valid SQL", "SELECT id FROM users WHERE id = 1", "", nil}, + {"valid SQL with dialect", "SELECT TOP 5 id FROM users", sqlkeywords.DialectSQLServer, nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var ( + res *LintResult + err error + ) + if tt.dialect == "" { + res, err = Lint(tt.sql) + } else { + res, err = LintWithDialect(tt.sql, tt.dialect) + } + if tt.wantErr == nil { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + return + } + if !errors.Is(err, tt.wantErr) { + t.Fatalf("err = %v, want it to wrap %v", err, tt.wantErr) + } + if res == nil { + t.Fatal("result is nil; the violations of the text and token rules must still be returned") + } + }) + } +} + +func TestLint_SyntaxErrorKeepsTextRuleViolations(t *testing.T) { + // L001 (trailing whitespace) is a text rule and must still fire on SQL + // that does not parse. + res, err := Lint("SELEC FROM \n") + if !errors.Is(err, ErrSyntax) { + t.Fatalf("err = %v, want ErrSyntax", err) + } + if res == nil || !hasRule(res, "L001") { + t.Fatalf("expected L001 alongside the syntax error, got %+v", res) + } +} diff --git a/pkg/gosqlx/lint_test.go b/pkg/gosqlx/lint_test.go index 67a9ee1e..626f0e6a 100644 --- a/pkg/gosqlx/lint_test.go +++ b/pkg/gosqlx/lint_test.go @@ -105,7 +105,7 @@ func TestLint_ConcurrentMatchesSerial(t *testing.T) { } want := make([][]linter.Violation, len(inputs)) for i, in := range inputs { - res, err := Lint(in) + res, err := lintTolerant(in) if err != nil { t.Fatalf("serial Lint(%q): %v", in, err) } @@ -124,7 +124,7 @@ func TestLint_ConcurrentMatchesSerial(t *testing.T) { defer wg.Done() for i := 0; i < iterations; i++ { idx := (g + i) % len(inputs) - res, err := Lint(inputs[idx]) + res, err := lintTolerant(inputs[idx]) if err != nil { mu.Lock() errs = append(errs, fmt.Sprintf("Lint(%q): %v", inputs[idx], err)) @@ -191,7 +191,7 @@ func TestLint_DeterministicL006L009(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { for i := 0; i < 100; i++ { - res, err := Lint(tt.sql) + res, err := lintTolerant(tt.sql) if err != nil { t.Fatalf("iteration %d: Lint: %v", i, err) } diff --git a/pkg/gosqlx/merge_output_into_test.go b/pkg/gosqlx/merge_output_into_test.go new file mode 100644 index 00000000..2e441c58 --- /dev/null +++ b/pkg/gosqlx/merge_output_into_test.go @@ -0,0 +1,57 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gosqlx + +import ( + "strings" + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" +) + +// The formatter keeps the INTO target of a SQL Server MERGE ... OUTPUT. +func TestFormatMergeOutputInto(t *testing.T) { + tests := []struct { + name string + sql string + want string + }{ + {"table variable", + "MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE OUTPUT deleted.id INTO @t", + "INTO @t"}, + {"column list", + "MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE OUTPUT deleted.id INTO dbo.log (id)", + "INTO dbo.log (id)"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + tree, err := ParseWithDialect(tc.sql, keywords.DialectSQLServer) + if err != nil { + t.Fatalf("parse: %v", err) + } + out := FormatAST(tree) + if !strings.Contains(out, tc.want) { + t.Errorf("formatted output %q does not contain %q", out, tc.want) + } + tree2, err := ParseWithDialect(out, keywords.DialectSQLServer) + if err != nil { + t.Fatalf("reparse %q: %v", out, err) + } + if out2 := FormatAST(tree2); out2 != out { + t.Errorf("not idempotent:\n%q\n%q", out, out2) + } + }) + } +} diff --git a/pkg/gosqlx/options.go b/pkg/gosqlx/options.go index 68eae484..8ef64c33 100644 --- a/pkg/gosqlx/options.go +++ b/pkg/gosqlx/options.go @@ -65,6 +65,10 @@ type parseOptions struct { // Inputs that exceed the cap are rejected with ErrTooLarge before any // parsing work begins. maxBytes int64 + + // comments attaches the comments the tokenizer captured to the parsed + // AST (ast.AST.Comments), so FormatTree and FormatAST write them back. + comments bool } // defaultParseOptions returns the baseline configuration used when no options @@ -195,3 +199,25 @@ func WithRecovery() Option { o.recover = true } } + +// WithComments keeps the comments of the SQL text: the tokenizer's comments +// are attached to the parsed ast.AST (AST.Comments), and FormatTree and +// FormatAST write them back out. Without it comments are discarded, which is +// the default because they cost an allocation and most callers only inspect +// the tree. +// +// FormatTree and FormatAST place comments without knowing where in the text +// they stood: comments on their own line go in front of the query, comments +// that shared a line with SQL after it. To keep each comment next to its +// statement, and to keep optimizer hints in place, format the text with +// Format instead. +// +// Example: +// +// tree, err := gosqlx.ParseTree(ctx, "SELECT 1 -- one", gosqlx.WithComments()) +// out := gosqlx.FormatTree(tree) // "SELECT 1 -- one" +func WithComments() Option { + return func(o *parseOptions) { + o.comments = true + } +} diff --git a/pkg/gosqlx/primary_forms_roundtrip_test.go b/pkg/gosqlx/primary_forms_roundtrip_test.go new file mode 100644 index 00000000..bb746339 --- /dev/null +++ b/pkg/gosqlx/primary_forms_roundtrip_test.go @@ -0,0 +1,132 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gosqlx + +import ( + "strings" + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" +) + +// TestPrimaryForms_RoundTrip checks that primary expressions and special +// function-call forms survive format → parse → format unchanged and that the +// formatted text keeps their meaning (want is a substring of the output). +func TestPrimaryForms_RoundTrip(t *testing.T) { + tests := []struct { + dialect keywords.SQLDialect + sql string + want string + }{ + // Keyword-named and qualified function calls are written bare. + {keywords.DialectMySQL, "SELECT LEFT(name, 2), RIGHT(name, 2) FROM t", "LEFT(name, 2), RIGHT(name, 2)"}, + {keywords.DialectPostgreSQL, "SELECT GROUPING(a) FROM t GROUP BY a", "GROUPING(a)"}, + {keywords.DialectMySQL, "SELECT TRUNCATE(x, 1), DEFAULT(c) FROM t", "TRUNCATE(x, 1), DEFAULT(c)"}, + {keywords.DialectClickHouse, "SELECT any(a), range(1, 2) FROM t", "any(a), range(1, 2)"}, + {keywords.DialectSnowflake, "SELECT HASH(a) FROM t", "HASH(a)"}, + {keywords.DialectPostgreSQL, "SELECT ROW(1, 2) = ROW(1, 2)", "ROW(1, 2) = ROW(1, 2)"}, + {keywords.DialectPostgreSQL, "SELECT pg_catalog.now(), auth.uid()", "pg_catalog.now(), auth.uid()"}, + {keywords.DialectOracle, "SELECT DBMS_RANDOM.VALUE(1, 100) FROM dual", "DBMS_RANDOM.VALUE(1, 100)"}, + // Literals. + {keywords.DialectPostgreSQL, "SELECT DATE '2024-01-01', inet '10.0.0.1'", "DATE '2024-01-01', inet '10.0.0.1'"}, + {keywords.DialectOracle, "SELECT a FROM t WHERE d >= DATE '2024-01-01'", "d >= DATE '2024-01-01'"}, + {keywords.DialectSQLServer, "SELECT a FROM t WHERE b = N'x''y' AND c IN (N'a', N'b')", "b = N'x''y' AND c IN (N'a', N'b')"}, + {keywords.DialectMySQL, "SELECT 'a' 'b'", "'ab'"}, + // Special forms keep their keywords. + {keywords.DialectPostgreSQL, "SELECT substring(name FROM 2 FOR 3), position('a' IN b) FROM t", "substring(name FROM 2 FOR 3), position('a' IN b)"}, + {keywords.DialectPostgreSQL, "SELECT trim(BOTH ' ' FROM x), trim(LEADING FROM y) FROM t", "trim(BOTH ' ' FROM x), trim(LEADING FROM y)"}, + {keywords.DialectPostgreSQL, "SELECT overlay(s PLACING 'ab' FROM 2 FOR 3) FROM t", "overlay(s PLACING 'ab' FROM 2 FOR 3)"}, + {keywords.DialectMySQL, "SELECT CONVERT(x USING utf8mb4) FROM t", "CONVERT(x USING utf8mb4)"}, + {keywords.DialectSQLServer, "SELECT TRY_PARSE(N'1' AS INT)", "TRY_PARSE(N'1' AS INT)"}, + {keywords.DialectMariaDB, "SELECT COLUMN_GET(d, 'k' AS CHAR) FROM t", "COLUMN_GET(d, 'k' AS CHAR)"}, + {keywords.DialectOracle, "SELECT LISTAGG(a, ',' ON OVERFLOW TRUNCATE '...' WITH COUNT) WITHIN GROUP (ORDER BY a) FROM t", + "LISTAGG(a, ',' ON OVERFLOW TRUNCATE '...' WITH COUNT) WITHIN GROUP (ORDER BY a)"}, + {keywords.DialectOracle, "SELECT MAX(a) KEEP (DENSE_RANK LAST ORDER BY b) FROM t", "MAX(a) KEEP (DENSE_RANK LAST ORDER BY b)"}, + // GROUP_CONCAT SEPARATOR is not an extra argument. + {keywords.DialectMySQL, "SELECT GROUP_CONCAT(a SEPARATOR '|') FROM t", "GROUP_CONCAT(a SEPARATOR '|')"}, + {keywords.DialectMariaDB, "SELECT GROUP_CONCAT(DISTINCT a ORDER BY a DESC SEPARATOR ',') FROM t", + "GROUP_CONCAT(DISTINCT a ORDER BY a DESC SEPARATOR ',')"}, + // INTERVAL keeps an unquoted value unquoted. + {keywords.DialectMySQL, "SELECT NOW() - INTERVAL 1 DAY", "NOW() - INTERVAL 1 DAY"}, + {keywords.DialectMySQL, "SELECT DATE_ADD(d, INTERVAL 1 MONTH) FROM t", "DATE_ADD(d, INTERVAL 1 MONTH)"}, + {keywords.DialectClickHouse, "SELECT now() - INTERVAL x DAY", "now() - INTERVAL x DAY"}, + {keywords.DialectMySQL, "SELECT INTERVAL(5, 1, 10)", "INTERVAL(5, 1, 10)"}, + {keywords.DialectPostgreSQL, "SELECT INTERVAL '1 day'", "INTERVAL '1 day'"}, + // MATCH ... AGAINST. + {keywords.DialectMySQL, "SELECT * FROM t WHERE MATCH(a) AGAINST ('x' IN BOOLEAN MODE)", "MATCH(a) AGAINST('x' IN BOOLEAN MODE)"}, + // Oracle. + {keywords.DialectOracle, "SELECT * FROM a, b WHERE a.id = b.id(+)", "a.id = b.id(+)"}, + {keywords.DialectOracle, "SELECT CONNECT_BY_ROOT name, PRIOR name AS m FROM e", "CONNECT_BY_ROOT name, PRIOR name AS m"}, + // Sequences and variables. + {keywords.DialectSQLServer, "SELECT NEXT VALUE FOR dbo.s", "NEXT VALUE FOR dbo.s"}, + {keywords.DialectMySQL, "SELECT @@version, @@session.sql_mode", "@@version, @@session.sql_mode"}, + {keywords.DialectMySQL, "SELECT @rn := @rn + 1 AS rn FROM t", "@rn := @rn + 1 AS rn"}, + // Arrays. + {keywords.DialectPostgreSQL, "SELECT ARRAY[[1, 2], [3, 4]]", "ARRAY[[1, 2], [3, 4]]"}, + {keywords.DialectPostgreSQL, "SELECT ARRAY(SELECT 1 UNION SELECT 2)", "ARRAY(SELECT 1 UNION SELECT 2)"}, + {keywords.DialectSnowflake, "SELECT [1, 2, 3] AS ar", "[1, 2, 3] AS ar"}, + {keywords.DialectClickHouse, "SELECT [1, 2, 3][2], (1, 'a').1", "[1, 2, 3][2], (1, 'a').1"}, + // CAST forms. + {keywords.DialectClickHouse, "SELECT CAST(x, 'UInt32')", "CAST(x, 'UInt32')"}, + {keywords.DialectSnowflake, "SELECT TRY_CAST(x AS INT)", "TRY_CAST(x AS INT)"}, + {keywords.DialectPostgreSQL, "SELECT CAST(a AS int[]), CAST(b AS double precision)", "CAST(a AS int[]), CAST(b AS double precision)"}, + } + for _, tt := range tests { + t.Run(string(tt.dialect)+"/"+tt.sql, func(t *testing.T) { + tree, err := ParseWithDialect(tt.sql, tt.dialect) + if err != nil { + t.Fatalf("parse: %v", err) + } + f1 := FormatAST(tree) + if !strings.Contains(strings.Join(strings.Fields(f1), " "), tt.want) { + t.Errorf("formatted output\n %s\ndoes not contain\n %s", f1, tt.want) + } + tree2, err := ParseWithDialect(f1, tt.dialect) + if err != nil { + t.Fatalf("reparse of %q: %v", f1, err) + } + if f2 := FormatAST(tree2); f2 != f1 { + t.Errorf("not idempotent:\n %s\n %s", f1, f2) + } + }) + } +} + +// TestPrimaryForms_ExtractColumns checks that column extraction descends into +// the new primary-expression nodes. +func TestPrimaryForms_ExtractColumns(t *testing.T) { + tests := []struct { + dialect keywords.SQLDialect + sql string + want []string + }{ + {keywords.DialectMySQL, "SELECT id FROM t WHERE MATCH(title, body) AGAINST ('x' IN BOOLEAN MODE)", []string{"body", "id", "title"}}, + {keywords.DialectPostgreSQL, "SELECT substring(name FROM pos FOR len) FROM t", []string{"len", "name", "pos"}}, + {keywords.DialectOracle, "SELECT a.x FROM a, b WHERE a.id = b.id(+)", []string{"id", "x"}}, + {keywords.DialectMySQL, "SELECT d + INTERVAL n DAY FROM t", []string{"d", "n"}}, + } + for _, tt := range tests { + t.Run(tt.sql, func(t *testing.T) { + tree, err := ParseWithDialect(tt.sql, tt.dialect) + if err != nil { + t.Fatalf("parse: %v", err) + } + got := ExtractColumns(tree) + if strings.Join(got, ",") != strings.Join(tt.want, ",") { + t.Errorf("ExtractColumns = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/gosqlx/reader.go b/pkg/gosqlx/reader.go index ebf9208e..6e87f789 100644 --- a/pkg/gosqlx/reader.go +++ b/pkg/gosqlx/reader.go @@ -98,9 +98,10 @@ func ParseReader(ctx context.Context, r io.Reader, opts ...Option) (*Tree, error // splitting or parsing work begins. // // Empty segments (whitespace between consecutive semicolons, trailing -// whitespace after the last ';', etc.) are skipped. The first segment that -// fails to parse short-circuits the call and returns its error wrapped in -// the usual ParseTree sentinels, prefixed with the 1-based statement index. +// whitespace after the last ';', a trailing comment, etc.) are skipped. The +// first segment that fails to parse short-circuits the call and returns its +// error wrapped in the usual ParseTree sentinels, prefixed with the 1-based +// statement index. // // Example: // @@ -136,7 +137,7 @@ func ParseReaderMultiple(ctx context.Context, r io.Reader, opts ...Option) ([]*T trees := make([]*Tree, 0, len(segments)) for i, seg := range segments { seg = strings.TrimSpace(seg) - if seg == "" { + if seg == "" || commentsOnly(seg) { continue } tree, err := ParseTree(ctx, seg, opts...) diff --git a/pkg/gosqlx/reader_test.go b/pkg/gosqlx/reader_test.go index 0c4d7dc1..03a7c89e 100644 --- a/pkg/gosqlx/reader_test.go +++ b/pkg/gosqlx/reader_test.go @@ -305,3 +305,29 @@ func TestSplitSQLStatements(t *testing.T) { }) } } + +// TestParseReaderMultiple_SkipsTrailingComment guards the common script shape +// "stmt; -- note": the comment after the last ';' is not a statement. +func TestParseReaderMultiple_SkipsTrailingComment(t *testing.T) { + trees, err := ParseReaderMultiple(context.Background(), + strings.NewReader("SELECT 1; -- one\nSELECT 2;\n/* done */\n-- bye\n")) + if err != nil { + t.Fatalf("ParseReaderMultiple: %v", err) + } + if len(trees) != 2 { + t.Fatalf("got %d trees, want 2", len(trees)) + } +} + +// TestParseReaderMultiple_BatchScript checks the splitter's GO handling end +// to end through the reader. +func TestParseReaderMultiple_BatchScript(t *testing.T) { + src := "SELECT 1\nGO\nSELECT 2\nGO\n" + trees, err := ParseReaderMultiple(context.Background(), strings.NewReader(src), WithDialect("sqlserver")) + if err != nil { + t.Fatalf("ParseReaderMultiple: %v", err) + } + if len(trees) != 2 { + t.Fatalf("got %d trees, want 2", len(trees)) + } +} diff --git a/pkg/gosqlx/realworld_corpus_test.go b/pkg/gosqlx/realworld_corpus_test.go index b915f8fd..d6082e20 100644 --- a/pkg/gosqlx/realworld_corpus_test.go +++ b/pkg/gosqlx/realworld_corpus_test.go @@ -20,11 +20,13 @@ import ( "fmt" "os" "path/filepath" + "reflect" "sort" "strings" "testing" "github.com/ajitpratap0/GoSQLX/internal/dropcheck" + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" ) @@ -46,6 +48,11 @@ func TestRealWorldCorpus(t *testing.T) { if err != nil { t.Fatalf("read %s: %v", realworldRoot, err) } + // After every statement has been parsed, formatted and re-parsed, the + // shared read-only stand-ins returned by the DialectExtras/DialectModifiers + // accessors must still be empty: a write through one would leak into every + // AST that lacks its own extras. + t.Cleanup(func() { assertDialectStandInsUntouched(t) }) known := loadKnownFailures(t) seen := map[string]bool{} var current []string // "key stage" for every failing statement @@ -204,3 +211,25 @@ func compact(s string) string { } return s } + +func assertDialectStandInsUntouched(t *testing.T) { + t.Helper() + checks := []struct { + name string + got, want any + }{ + {"FunctionCall", *(&ast.FunctionCall{}).DialectExtras(), ast.FunctionExtras{}}, + {"BinaryExpression", *(&ast.BinaryExpression{}).DialectExtras(), ast.BinaryExtras{}}, + {"InsertStatement", *(&ast.InsertStatement{}).DialectExtras(), ast.InsertDialectFields{}}, + {"UpdateStatement", *(&ast.UpdateStatement{}).DialectExtras(), ast.DMLTargetFields{}}, + {"DeleteStatement", *(&ast.DeleteStatement{}).DialectExtras(), ast.DMLTargetFields{}}, + {"SelectStatement", *(&ast.SelectStatement{}).DialectExtras(), ast.SelectExtras{}}, + {"TableReference", *(&ast.TableReference{}).DialectModifiers(), ast.TableModifiers{}}, + {"LiteralValue", (&ast.LiteralValue{}).DialectExtras(), ast.LiteralExtras{}}, + } + for _, c := range checks { + if !reflect.DeepEqual(c.got, c.want) { + t.Errorf("%s: shared read-only dialect stand-in was mutated: %+v", c.name, c.got) + } + } +} diff --git a/pkg/gosqlx/scratch_tokens_test.go b/pkg/gosqlx/scratch_tokens_test.go new file mode 100644 index 00000000..0a8b1df4 --- /dev/null +++ b/pkg/gosqlx/scratch_tokens_test.go @@ -0,0 +1,139 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gosqlx + +import ( + "context" + "fmt" + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" +) + +// TestEntryPoints_ASTOutlivesTokenizerReuse checks every entry point that +// tokenizes into the pooled tokenizer's scratch buffer: the AST it returns +// must stay intact after the same tokenizer is reused by every other such +// entry point, i.e. nothing in the AST may point into the scratch tokens. +func TestEntryPoints_ASTOutlivesTokenizerReuse(t *testing.T) { + const sql = `/* lead */ SELECT "Mixed", b AS x, COUNT(*) FROM s."T" t ` + + `LEFT JOIN u ON t.id = u.id WHERE c > 1 AND d = 'v' GROUP BY 1, 2 ORDER BY 3 DESC LIMIT 5` + ctx := context.Background() + tests := []struct { + name string + parse func() (*ast.AST, error) + }{ + {"Parse", func() (*ast.AST, error) { return Parse(sql) }}, + {"ParseBytes", func() (*ast.AST, error) { return ParseBytes([]byte(sql)) }}, + {"ParseWithContext", func() (*ast.AST, error) { return ParseWithContext(ctx, sql) }}, + {"ParseWithDialect", func() (*ast.AST, error) { return ParseWithDialect(sql, "postgresql") }}, + {"ParseMultiple", func() (*ast.AST, error) { + trees, err := ParseMultiple([]string{sql}) + if err != nil { + return nil, err + } + return trees[0], nil + }}, + {"ParseTree", func() (*ast.AST, error) { + tree, err := ParseTree(ctx, sql, WithDialect("postgresql"), WithComments()) + if err != nil { + return nil, err + } + return tree.Raw(), nil + }}, + {"ParseTreeRecovery", func() (*ast.AST, error) { + tree, err := ParseTree(ctx, sql, WithRecovery()) + if err != nil { + return nil, err + } + return tree.Raw(), nil + }}, + {"ParseWithRecovery", func() (*ast.AST, error) { + stmts, errs := ParseWithRecovery(sql) + if len(errs) > 0 { + return nil, errs[0] + } + return &ast.AST{Statements: stmts}, nil + }}, + } + churn := func(i int) error { + other := fmt.Sprintf("SELECT zz%d, yy FROM qq%d -- c%d\nWHERE ww = 'v%d'", i, i, i, i) + if err := Validate(other); err != nil { + return err + } + if err := ValidateMultiple([]string{other, other}); err != nil { + return err + } + if _, err := Format(other, DefaultFormatOptions()); err != nil { + return err + } + for _, tc := range tests { + if _, err := tc.parse(); err != nil { + return err + } + } + return nil + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + tree, err := tc.parse() + if err != nil { + t.Fatalf("parse: %v", err) + } + want := FormatAST(tree) + for i := 0; i < 20; i++ { + if err := churn(i); err != nil { + t.Fatalf("churn %d: %v", i, err) + } + } + if got := FormatAST(tree); got != want { + t.Errorf("AST changed after tokenizer reuse:\n got %s\nwant %s", got, want) + } + }) + } +} + +// TestFormat_CommentsWithScratchTokens checks that Format, which reads the +// scratch tokens again after parsing to place comments, keeps placing them +// the same way when the tokenizer is reused between calls. +func TestFormat_CommentsWithScratchTokens(t *testing.T) { + tests := []struct { + name string + sql string + }{ + {"leading", "-- head\nSELECT a FROM t"}, + {"between statements", "SELECT a FROM t; /* mid */ SELECT b FROM u"}, + {"trailing", "SELECT a FROM t -- tail"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + first, err := Format(tc.sql, DefaultFormatOptions()) + if err != nil { + t.Fatalf("Format: %v", err) + } + for i := 0; i < 5; i++ { + if _, err := Format(fmt.Sprintf("SELECT x%d /* c */ FROM y", i), DefaultFormatOptions()); err != nil { + t.Fatalf("Format churn: %v", err) + } + again, err := Format(tc.sql, DefaultFormatOptions()) + if err != nil { + t.Fatalf("Format: %v", err) + } + if again != first { + t.Errorf("Format changed on reuse:\n got %q\nwant %q", again, first) + } + } + }) + } +} diff --git a/pkg/gosqlx/splitter.go b/pkg/gosqlx/splitter.go index dc175649..0f8049e9 100644 --- a/pkg/gosqlx/splitter.go +++ b/pkg/gosqlx/splitter.go @@ -29,11 +29,16 @@ import "strings" // "postgresql", "postgres", "pg" — dollar-quotes, E-strings, nested /*…*/ // "mysql", "mariadb" — backtick identifiers // "clickhouse" — backtick identifiers -// "sqlserver", "mssql", "tsql" — [bracketed identifiers] +// "sqlserver", "mssql", "tsql" — [bracketed identifiers], GO batch separator +// "oracle" — DECLARE / IS|AS sections, "/" line terminator // "" — ANSI only (single/double quotes, line/block comments) // +// Compound statements stay in one piece in every dialect (see the notes on +// procedural scripts below): a BEGIN … END body is not cut at its inner ';'. +// // The returned slice preserves the original byte ranges between semicolons -// (the terminating ';' itself is stripped). Empty or whitespace-only segments +// (the terminating ';' itself is stripped). Client directive lines (GO, +// DELIMITER, a lone "/") are consumed and appear in no segment. Empty or whitespace-only segments // are NOT filtered out here; callers such as ParseReaderMultiple trim and // skip them. This mirrors the long-standing behaviour of the splitter that // this function replaces. @@ -53,6 +58,16 @@ func SplitStatements(sql, dialect string) []string { return splitWithProfile(sql, p) } +// Procedural scripts. Beyond plain ';' separation the splitter keeps compound +// statements whole: BEGIN … END blocks (triggers, procedures, T-SQL batches), +// CASE … END, IF … END IF and loops inside a block, Oracle DECLARE / IS|AS +// sections, and PostgreSQL BEGIN ATOMIC bodies. It also honours the batch +// separators of client tools: a line holding only GO [n] for SQL Server, a +// DELIMITER directive for MySQL/MariaDB, and a line holding only "/" for +// Oracle. Directive lines are consumed and do not appear in any segment. +// Transaction statements (BEGIN; BEGIN TRANSACTION; …) are not blocks, and a +// script whose blocks do not balance falls back to plain semicolon splitting. + // dialectProfile collects the per-dialect feature flags the splitter needs. // Profiles are immutable and shared; one is resolved per call and passed to // the state machine by value-ish (read-only) pointer. @@ -62,6 +77,12 @@ type dialectProfile struct { supportsBrackets bool supportsEStrings bool supportsNestedBlockComments bool + + // Procedural-script features (see splitter_proc.go). + tsql bool // GO batches; IF/WHILE/LOOP are not delimited by END IF etc. + oracle bool // DECLARE / IS|AS sections; "/" line terminator + delimiter bool // MySQL client DELIMITER directive + noBlocks bool // disable BEGIN…END tracking (unbalanced-script fallback) } // profileForDialect returns the feature flags for the named dialect. The @@ -76,11 +97,13 @@ func profileForDialect(dialect string) dialectProfile { supportsNestedBlockComments: true, } case "mysql", "mariadb": - return dialectProfile{supportsBackticks: true} + return dialectProfile{supportsBackticks: true, delimiter: true} case "clickhouse": return dialectProfile{supportsBackticks: true} case "sqlserver", "mssql", "tsql": - return dialectProfile{supportsBrackets: true} + return dialectProfile{supportsBrackets: true, tsql: true} + case "oracle": + return dialectProfile{oracle: true} default: return dialectProfile{} } @@ -100,14 +123,27 @@ func profileForDialect(dialect string) dialectProfile { // documents the transitions it performs so future readers do not have to // reconstruct the logic from a stack trace. func splitWithProfile(src string, p dialectProfile) []string { + out, open := splitCore(src, p) + if open && !p.noBlocks { + // Unbalanced BEGIN/CASE/IF: don't let one malformed block swallow + // the rest of the script; fall back to plain semicolon splitting. + p.noBlocks = true + out, _ = splitCore(src, p) + } + return out +} + +// splitCore runs the state machine. open reports that the input ended inside +// an unclosed procedural block. +func splitCore(src string, p dialectProfile) (out []string, open bool) { if len(src) == 0 { - return nil + return nil, false } // Every ';' that terminates a segment is a literal ';' in src, so the // count is an upper bound on the number of emitted segments (semicolons // inside strings/comments are counted too). Presizing means the result // slice is the only allocation and never regrows. - out := make([]string, 0, strings.Count(src, ";")+1) + out = make([]string, 0, strings.Count(src, ";")+1) // start is the index in src where the current segment begins. Every // segment is the contiguous substring src[start:i], so no bytes are copied. start := 0 @@ -129,7 +165,19 @@ func splitWithProfile(src string, p dialectProfile) []string { // escapes the following byte, including '\''. eStringActive := false + // Procedural-script state (splitter_proc.go). st tracks block nesting; + // delim is the active MySQL client delimiter ("" means the default ';'). + var st procState + st.init(p) + delim := "" + // qMiss memoises failed Oracle q-quote searches so unterminated literals + // cost O(n) in total instead of O(n) each (see oracleQuoteEnd). + var qMiss [256]int + i := 0 + if strings.HasPrefix(src, "\xef\xbb\xbf") { + i = 3 // a UTF-8 byte order mark is not part of the first word + } for i < len(src) { c := src[i] @@ -211,6 +259,14 @@ func splitWithProfile(src string, p dialectProfile) []string { } // ─── Top-level: look for openers and semicolons ─────────────────── + if delim != "" && strings.HasPrefix(src[i:], delim) { + // MySQL client DELIMITER in force: only the custom delimiter + // terminates a statement; ';' is ordinary text. + out = append(out, src[start:i]) + i += len(delim) + start = i + continue + } switch { case c == '-' && i+1 < len(src) && src[i+1] == '-': inLine = true @@ -220,6 +276,17 @@ func splitWithProfile(src string, p dialectProfile) []string { blockCommentDepth = 1 i += 2 + case c == '/' && p.oracle && delim == "": + // SQL*Plus "/" on a line of its own ends the current statement + // or PL/SQL block. + if ls, le, ok := lineOnly(src, i, i+1); ok { + out = appendNonBlank(out, src[start:ls]) + start, i = le, le + st.reset() + continue + } + i++ + case p.supportsEStrings && (c == 'E' || c == 'e') && i+1 < len(src) && src[i+1] == '\'': // PG E'...' @@ -256,19 +323,71 @@ func splitWithProfile(src string, p dialectProfile) []string { i++ case c == ';': + if delim != "" || st.depth > 0 || (p.tsql && elseFollows(src, i+1)) { + // Inside a custom-delimiter statement or an open block, or + // T-SQL IF ... stmt; ELSE stmt. + st.semicolon() + i++ + continue + } out = append(out, src[start:i]) i++ start = i + st.semicolon() + + case isWordByte(c): + j := i + 1 + for j < len(src) && isWordByte(src[j]) { + j++ + } + if c >= '0' && c <= '9' || prevIsQualifier(src, i) { + i = j + continue + } + word := src[i:j] + if p.oracle && j < len(src) && src[j] == '\'' { + // Oracle alternative quoting q'[...]' / nq'[...]': the body can + // hold quotes and semicolons. + if end, ok := oracleQuoteEnd(src, word, j, &qMiss); ok { + i = end + continue + } + } + switch { + case p.tsql && len(word) == 2 && asciiFold(word[0]) == 'g' && asciiFold(word[1]) == 'o': + if ls, le, ok := goLine(src, i, j); ok { + out = appendNonBlank(out, src[start:ls]) + start, i = le, le + st.reset() + continue + } + case p.delimiter && len(word) == 9 && equalFoldASCII(word, "delimiter"): + if ls, le, d, ok := delimiterLine(src, i, j); ok { + out = appendNonBlank(out, src[start:ls]) + start, i = le, le + delim = d + st.reset() + continue + } + } + if st.on && delim == "" { + j = st.word(src, i, j) + } + i = j default: + if st.on { + st.punct(src, i) + } i++ } } + open = st.depth > 0 if start < len(src) { out = append(out, src[start:]) } - return out + return out, open } // readDollarTag attempts to read a PostgreSQL dollar-quote opener starting at @@ -324,3 +443,51 @@ func isDollarTagStart(c byte) bool { func isDollarTagCont(c byte) bool { return isDollarTagStart(c) || (c >= '0' && c <= '9') } + +// oracleQuoteEnd reports where an Oracle alternative-quoting literal ends. word +// is the q or nq introducer (any case) and src[quote] the opening apostrophe +// right after it. The literal q'...' takes the delimiter d from the +// character after the apostrophe; ( [ { < pair with ) ] } >, any other +// non-blank character closes itself. The result is the index just past the +// closing apostrophe. +// +// miss remembers, per closing byte, the first start offset whose search for +// "'" reached the end of input without a match. Any later search for +// the same closer starts at or after that offset and must fail too, so it +// returns at once; an unterminated literal therefore costs one scan in total +// rather than one per occurrence. A failed literal is not a literal: the +// caller treats q as an ordinary word, exactly as before. +func oracleQuoteEnd(src, word string, quote int, miss *[256]int) (int, bool) { + if !equalFoldASCII(word, "q") && !equalFoldASCII(word, "nq") { + return 0, false + } + if quote+2 >= len(src) { + return 0, false + } + open := src[quote+1] + if open == ' ' || open == '\t' || open == '\n' || open == '\r' { + return 0, false + } + closer := open + switch open { + case '(': + closer = ')' + case '[': + closer = ']' + case '{': + closer = '}' + case '<': + closer = '>' + } + from := quote + 2 + if m := miss[closer]; m != 0 && from >= m { + return 0, false + } + for k := from; k+1 < len(src); k++ { + if src[k] == closer && src[k+1] == '\'' { + return k + 2, true + } + } + miss[closer] = from + return 0, false +} diff --git a/pkg/gosqlx/splitter_fuzz_test.go b/pkg/gosqlx/splitter_fuzz_test.go index d6487219..1f4d4303 100644 --- a/pkg/gosqlx/splitter_fuzz_test.go +++ b/pkg/gosqlx/splitter_fuzz_test.go @@ -47,6 +47,11 @@ func FuzzSplitStatements(f *testing.F) { {"E'", "postgresql"}, {"'", ""}, {"--", ""}, + {"SELECT 1\nGO\nSELECT 2", "sqlserver"}, + {"DELIMITER //\nSELECT 1;SELECT 2//\nDELIMITER ;\n", "mysql"}, + {"BEGIN SELECT 1; END; BEGIN; COMMIT", "sqlite"}, + {"DECLARE x NUMBER; BEGIN NULL; END;\n/\n", "oracle"}, + {"CREATE PROCEDURE p() BEGIN IF a THEN SELECT 1; END IF; END", "mysql"}, } for _, s := range seeds { f.Add(s.sql, s.dialect) @@ -54,7 +59,7 @@ func FuzzSplitStatements(f *testing.F) { f.Fuzz(func(t *testing.T, src, dialect string) { segs := SplitStatements(src, dialect) - if msg, ok := checkTiling(src, segs); !ok { + if msg, ok := checkTilingFor(src, segs, dialect); !ok { t.Fatalf("SplitStatements(%q, %q) = %q: %s", src, dialect, segs, msg) } }) diff --git a/pkg/gosqlx/splitter_oracle_quote_test.go b/pkg/gosqlx/splitter_oracle_quote_test.go new file mode 100644 index 00000000..8ddb7169 --- /dev/null +++ b/pkg/gosqlx/splitter_oracle_quote_test.go @@ -0,0 +1,118 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gosqlx + +import ( + "reflect" + "strings" + "testing" + "time" +) + +// TestSplitStatements_OracleAlternativeQuoting checks that the body of an +// Oracle q'[...]' literal, which may hold apostrophes and semicolons, does not +// end or hide a statement. +func TestSplitStatements_OracleAlternativeQuoting(t *testing.T) { + tests := []struct { + name string + sql string + dialect string + want []string + }{ + {"apostrophe in body", "SELECT q'[It's quoted]' FROM dual; SELECT 'O''Brien' FROM dual;", "oracle", + []string{"SELECT q'[It's quoted]' FROM dual", " SELECT 'O''Brien' FROM dual"}}, + {"semicolon in body", "SELECT q'(a;b)' FROM dual; SELECT 2 FROM dual", "oracle", + []string{"SELECT q'(a;b)' FROM dual", " SELECT 2 FROM dual"}}, + {"national and other delimiters", "SELECT nq'#x;y#', Q'{z}' FROM dual; SELECT 3 FROM dual", "oracle", + []string{"SELECT nq'#x;y#', Q'{z}' FROM dual", " SELECT 3 FROM dual"}}, + {"identifier ending in q is not a quote", "SELECT seq'a;b' FROM dual", "oracle", + []string{"SELECT seq'a;b' FROM dual"}}, + {"other dialects keep their rules", "SELECT q'x'; SELECT 1", "postgresql", + []string{"SELECT q'x'", " SELECT 1"}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := SplitStatements(tc.sql, tc.dialect); !reflect.DeepEqual(got, tc.want) { + t.Errorf("SplitStatements(%q, %q)\n got: %q\nwant: %q", tc.sql, tc.dialect, got, tc.want) + } + }) + } +} + +// TestSplitStatements_OracleQuoteUnterminatedLinear guards against quadratic +// behaviour on unterminated q-quotes (each used to rescan to end of input). +func TestSplitStatements_OracleQuoteUnterminatedLinear(t *testing.T) { + if testing.Short() { + t.Skip("timing test") + } + assertSplitLinear(t, "bare", func(n int) string { return strings.Repeat("q'(a';", n) }, "oracle", 10000) + assertSplitLinear(t, "select", func(n int) string { return strings.Repeat("SELECT q'(a' FROM dual;\n", n) }, "oracle", 10000) +} + +// assertSplitLinear fails when splitting gen(4n) takes much more than four +// times as long as gen(n): linear work grows about 4x, quadratic work 16x. A +// ratio, unlike a wall-clock bound, holds on slow or race-instrumented CI +// runners. Each size is timed three times and the fastest run counts. +func assertSplitLinear(t *testing.T, name string, gen func(n int) string, dialect string, n int) { + t.Helper() + measure := func(src string) time.Duration { + best := time.Duration(1<<63 - 1) + for i := 0; i < 3; i++ { + start := time.Now() + SplitStatements(src, dialect) + if d := time.Since(start); d < best { + best = d + } + } + return best + } + small, large := measure(gen(n)), measure(gen(4*n)) + if large < 20*time.Millisecond { + return // too fast to be quadratic at this size, and too fast to time reliably + } + if ratio := float64(large) / float64(small); ratio > 10 { + t.Errorf("%s: %v for n=%d but %v for 4n (%.1fx), want roughly linear growth", name, small, n, large, ratio) + } +} + +// TestSplitStatements_OracleQuoteMemoDoesNotHideLiterals checks that a failed +// search for one closer does not suppress later valid literals with another. +func TestSplitStatements_OracleQuoteMemoDoesNotHideLiterals(t *testing.T) { + got := SplitStatements("SELECT q'(a' FROM t; SELECT q'[x;y]' FROM dual; SELECT 1", "oracle") + want := []string{"SELECT q'(a' FROM t", " SELECT q'[x;y]' FROM dual", " SELECT 1"} + if !reflect.DeepEqual(got, want) { + t.Errorf("got %q, want %q", got, want) + } + // Terminated literal first, unterminated of the same closer afterwards. + got = SplitStatements("SELECT q'(a;b)' FROM dual; SELECT q'(c' FROM dual; SELECT 2", "oracle") + if len(got) < 2 || got[0] != "SELECT q'(a;b)' FROM dual" { + t.Errorf("terminated literal split wrongly: %q", got) + } +} + +// TestSplitStatements_DelimiterLength covers the MySQL DELIMITER length cap. +func TestSplitStatements_DelimiterLength(t *testing.T) { + got := SplitStatements("DELIMITER //\nSELECT 1; SELECT 2//\nDELIMITER ;\nSELECT 3;", "mysql") + want := []string{"SELECT 1; SELECT 2", "SELECT 3"} + if !reflect.DeepEqual(got, want) { + t.Errorf("normal DELIMITER: got %q, want %q", got, want) + } + if testing.Short() { + return + } + assertSplitLinear(t, "huge delimiter", func(n int) string { + return "DELIMITER " + strings.Repeat("(", n) + "x\n" + strings.Repeat("(", 3*n) + }, "mysql", 60000) +} diff --git a/pkg/gosqlx/splitter_parser_agree_test.go b/pkg/gosqlx/splitter_parser_agree_test.go new file mode 100644 index 00000000..83bae46a --- /dev/null +++ b/pkg/gosqlx/splitter_parser_agree_test.go @@ -0,0 +1,105 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gosqlx + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" +) + +// splitDialect maps a SplitStatements dialect name to the parser dialect. +func splitDialect(d string) keywords.SQLDialect { + switch d { + case "mssql", "tsql": + return keywords.DialectSQLServer + } + return keywords.SQLDialect(d) +} + +// assertOneStatementPerPiece parses every piece SplitStatements returns for +// sql and requires each piece that parses to hold exactly one statement: the +// splitter and the parser must agree on where statements end. Pieces the +// parser rejects are left to the tests of the features they use. +func assertOneStatementPerPiece(t *testing.T, label, sql, dialect string) { + t.Helper() + for i, piece := range SplitStatements(sql, dialect) { + if onlyComments(piece) { + continue + } + tree, err := ParseWithDialect(piece, splitDialect(dialect)) + if err != nil { + continue + } + if n := len(tree.Statements); n != 1 { + t.Errorf("%s#%d: SplitStatements returned one statement, the parser found %d: %s", + label, i, n, compact(piece)) + } + } +} + +// multiStatementPieces are the procedural split cases whose pieces hold +// several statements by design. +var multiStatementPieces = map[string]string{ + "tsql end then if without semicolon": "T-SQL needs no ';' between statements", + "go not alone on line": "GO is an alias there, and T-SQL needs no ';'", + "delimiter lowercase and double semicolon": "the client sends SELECT 1; SELECT 2 as one batch", +} + +// TestSplitStatements_AgreesWithParser checks that every piece of the +// procedural split cases and of the real-world corpus parses as exactly one +// statement. Files whose name says they are written without semicolons hold +// several statements per batch by design and are only required to parse +// (TestRealWorldCorpus checks that). +func TestSplitStatements_AgreesWithParser(t *testing.T) { + for _, tc := range proceduralSplitCases { + if tc.dialect == "" || multiStatementPieces[tc.name] != "" { + continue // the generic dialect parses neither PL blocks nor GO + } + assertOneStatementPerPiece(t, tc.name, tc.sql, tc.dialect) + } + + dirs, err := os.ReadDir(realworldRoot) + if os.IsNotExist(err) { + t.Skip("testdata/realworld not present") + } + if err != nil { + t.Fatalf("read %s: %v", realworldRoot, err) + } + checked := 0 + for _, d := range dirs { + if !d.IsDir() { + continue + } + files, _ := filepath.Glob(filepath.Join(realworldRoot, d.Name(), "*.sql")) + for _, file := range files { + if strings.Contains(filepath.Base(file), "no_semicolons") { + continue + } + data, err := os.ReadFile(file) + if err != nil { + t.Fatalf("read %s: %v", file, err) + } + assertOneStatementPerPiece(t, d.Name()+"/"+filepath.Base(file), string(data), d.Name()) + checked++ + } + } + if checked == 0 { + t.Fatal("no corpus files checked") + } +} diff --git a/pkg/gosqlx/splitter_proc.go b/pkg/gosqlx/splitter_proc.go new file mode 100644 index 00000000..fae32e29 --- /dev/null +++ b/pkg/gosqlx/splitter_proc.go @@ -0,0 +1,615 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gosqlx + +import ( + "strings" + + "github.com/ajitpratap0/GoSQLX/internal/batch" +) + +// This file holds the procedural-script half of SplitStatements: block +// nesting (BEGIN … END and friends) and the client-tool batch separators +// (T-SQL GO, MySQL DELIMITER, SQL*Plus "/"). The state machine in splitter.go +// calls into it once per identifier-like word, so everything here is +// allocation-free. + +// Word kinds the block tracker cares about. +const ( + kwOther uint8 = iota + kwBegin + kwEnd + kwCase + kwIf + kwLoop + kwWhile + kwFor + kwRepeat + kwThen + kwElse + kwDo + kwDeclare + kwCreate + kwRoutine // PROCEDURE, FUNCTION, PACKAGE + kwTrigger + kwIsAs +) + +// procState tracks BEGIN/CASE/IF/LOOP nesting for one split. depth is the +// number of open blocks; declMask marks (for levels 1..64) the Oracle +// declaration sections that are still waiting for their BEGIN. +type procState struct { + on, tsql, oracle bool + depth int + declMask uint64 + // stmtStart is true while the next word begins a statement (after ';', + // BEGIN, THEN, ELSE, LOOP, a label, …). IF/WHILE/FOR/REPEAT only open a + // block there, which keeps "DROP TABLE IF EXISTS" and IF() calls out. + stmtStart bool + loopHeader bool // WHILE/FOR seen, its LOOP/DO opener not yet + createN int // >0 while within the first words of an Oracle CREATE + routineHdr bool // Oracle PROCEDURE/FUNCTION/PACKAGE header awaiting IS/AS + trigHdr bool // inside an Oracle CREATE TRIGGER header (DECLARE may follow) +} + +func (s *procState) init(p dialectProfile) { + s.on = !p.noBlocks + s.tsql = p.tsql + s.oracle = p.oracle + s.stmtStart = true +} + +// reset discards any open block (a batch separator ends them all). +func (s *procState) reset() { + *s = procState{on: s.on, tsql: s.tsql, oracle: s.oracle, stmtStart: true} +} + +func (s *procState) semicolon() { + s.stmtStart = true + s.routineHdr = false + s.loopHeader = false + if s.depth == 0 { + s.createN = 0 + s.trigHdr = false + } +} + +func (s *procState) push(decl bool) { + s.depth++ + if decl && s.depth <= 64 { + s.declMask |= 1 << uint(s.depth-1) + } +} + +func (s *procState) pop() { + if s.depth == 0 { + return + } + if s.depth <= 64 { + s.declMask &^= 1 << uint(s.depth-1) + } + s.depth-- +} + +// absorbBegin consumes the BEGIN that follows an Oracle declaration section: +// the section and its body are one level. +func (s *procState) absorbBegin() bool { + if s.depth == 0 || s.depth > 64 { + return false + } + bit := uint64(1) << uint(s.depth-1) + if s.declMask&bit == 0 { + return false + } + s.declMask &^= bit + return true +} + +// punct handles the few non-word tokens that start a statement: a label's +// ':' (MySQL) and the closing '>>' of a <