diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index cffbe09c8..a59cc1477 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -10,7 +10,7 @@ "name": "gosqlx", "source": "./plugins/gosqlx", "description": "Validate, format, lint, analyze and security-scan SQL with GoSQLX (MCP tools, skills, slash commands, a SQL reviewer agent and a .sql validation hook).", - "version": "1.16.0", + "version": "1.17.0", "category": "development", "tags": ["sql", "database", "linting", "formatting", "security", "mcp"], "homepage": "https://gosqlx.dev", diff --git a/.github/lighthouse/README.md b/.github/lighthouse/README.md index 5dc23f6bf..851143c7f 100644 --- a/.github/lighthouse/README.md +++ b/.github/lighthouse/README.md @@ -14,6 +14,7 @@ Budgets are the measured value plus roughly 20-30% headroom for CI variance. | `/` | 0.63, 0.80, 0.80 | 339 KB | 114 KB | 18 KB | 2.90 MB (2.4 MB is non-script/non-font assets) | | `/compare/` | 0.87, 0.89, 0.89 | 329 KB | 114 KB | 18 KB | 0.49 MB | | `/playground/` (CI median, 2026-10-20, PR #825) | - | 420 KB | - | - | - | +| `/playground/` (CI median, 2026-10-29, PR #877) | - | - | - | - | 3.60 MB (mostly the GoSQLX WASM module) | The other audited routes (`/playground/`, `/docs/`, `/docs/getting-started/`, `/benchmarks/`, `/changelog/`) share the same layout chunks; their per-route numbers are in the uploaded Lighthouse artifacts. The local run was cut short by @@ -27,10 +28,11 @@ a full disk, so only `/` and `/compare/` were measured locally. | accessibility, seo | error | 0.95 | | best-practices | warn | 0.85 | | resource-summary script / font / stylesheet / total | error | 420 KB / 150 KB / 25 KB / 3.6 MB | -| resource-summary script, `/playground/` only | error | 480 KB | +| resource-summary script / total, `/playground/` only | error | 480 KB / 4.2 MB | `/playground/` loads the CodeMirror editor and the GoSQLX highlighting bridge, so it gets its own script budget -via `assertMatrix` (the measured 420 KB plus ~15% headroom). Every other route keeps the shared 420 KB budget, and +via `assertMatrix` (the measured 420 KB plus ~15% headroom). It also loads the GoSQLX WASM module, which grows +with the parser, so its total budget is the measured 3.60 MB (v1.17.0) plus ~15% headroom. Every other route keeps the shared 420 KB budget, and the two matrix entries are otherwise identical: change an assertion in both when editing thresholds. Re-measure and update this table whenever the budgets are changed. diff --git a/.github/lighthouse/lighthouserc.json b/.github/lighthouse/lighthouserc.json index d8deea079..16329f9c9 100644 --- a/.github/lighthouse/lighthouserc.json +++ b/.github/lighthouse/lighthouserc.json @@ -25,7 +25,7 @@ "resource-summary:script:size": ["error", { "maxNumericValue": 480000, "aggregationMethod": "median" }], "resource-summary:font:size": ["error", { "maxNumericValue": 150000, "aggregationMethod": "median" }], "resource-summary:stylesheet:size": ["error", { "maxNumericValue": 25000, "aggregationMethod": "median" }], - "resource-summary:total:size": ["error", { "maxNumericValue": 3600000, "aggregationMethod": "median" }] + "resource-summary:total:size": ["error", { "maxNumericValue": 4200000, "aggregationMethod": "median" }] } } ] diff --git a/CHANGELOG.md b/CHANGELOG.md index c6be899ba..835bc84cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,59 +7,211 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.17.0] - 2026-10-29 — Real-World Dialect Corpus, Dialect-Faithful Round Trips, Security Scanner & LSP Hardening + +Covers everything merged since v1.16.0, mostly the October 2026 dialect audit: +a 2,813-statement real-world corpus across eight dialects that now parses, +formats, re-parses identically and keeps every word, guarded by a ratchet test; +new syntax across Snowflake, BigQuery, ClickHouse, SQL Server, MySQL/MariaDB +and PostgreSQL; per-dialect lexing and rendering (numeric and Unicode-escape +literals, `#` comments, quoted identifiers, SQL Server `GO` and Oracle `/` +lines); safer transpilation; a hardened `ScanSQL` and an AST scanner that +walks the whole tree; and an LSP server that advertises the spec's +`textDocumentSync` key and survives malformed input. +The release is source-compatible: exported signatures are unchanged apart from +additions. Runtime behaviour does change (parse acceptance per dialect, SQL +Server `GO` and `top`, statement termination, numeric lexing, fingerprints, +statement counts, LSP capabilities), so read **Behavior changes** before +upgrading. + +### Behavior changes +- **SQL Server `GO` inside a statement is an error**: a line holding only + `GO [n]` inside a statement is now a parse error ("GO batch separator inside + a statement"), matching sqlcmd, instead of a column or alias named `go`. + `GO 0` / `GO 1.5` alone on a line report "invalid GO repeat count". (#868) +- **`AST.Statements` holds batch separators**: SQL Server `GO` is kept as an + `*ast.BatchSeparator` node (with its optional `RepeatCount`; repeated batches + are not expanded) instead of being discarded. Code that iterates + `AST.Statements` or recovery results must skip these nodes. Statement counts + (`gosqlx analyze`, `gosqlx parse`, the MCP `parse_sql` tool and the C + binding's `statement_count` / statement types) exclude them. (#867) +- **SQL Server `top` is reserved**: unquoted `top` (any case) is a syntax + error where a table name, INSERT/UPDATE/DELETE/MERGE target, table alias or + select-item alias is expected (`SELECT * FROM top`, `INSERT INTO top ...`); + quote it as `[top]`. (#880) +- **Statements end at their grammar**: outside SQL Server, a utility statement + (`SET`, `USE`, `GRANT`, `COPY`, `CALL`, ...) followed on the next line by + another statement without `;` is an error; only a transaction statement on + its own line is accepted without a separator. Statements whose definition + holds a statement (`PREPARE ... AS`, `DECLARE ... CURSOR FOR`, `CREATE + FUNCTION`/`PROCEDURE`/`TRIGGER`/`EVENT`/`RULE`/`TASK`/`VIEW` with a + single-statement, `RETURN` or `BEGIN ... END` body, `LOAD DATA`) end with + that definition, and `GRANT`, `REVOKE`, PostgreSQL `COPY`, `CALL`, `VACUUM`, + `ANALYZE`, `LISTEN`, `UNLISTEN` and `NOTIFY` end with their grammar, so + trailing tokens or a following statement without `;` are an error on the + same line as well (`VACUUM t junk`, `LISTEN c extra`). MySQL / MariaDB / + Oracle `ANALYZE TABLE` keeps its as-written handling. (#867, #872, #878) +- **Utility statements are gated per dialect**: they are accepted only in the + dialects that have them (`VACUUM` in PostgreSQL/SQLite, + `LISTEN`/`NOTIFY`/`CLUSTER`/`DISCARD` in PostgreSQL, + `FLUSH`/`OPTIMIZE`/`LOCK TABLES`/`KILL` in the MySQL family, `PRAGMA` in + SQLite, `DO $$ ... $$` in PostgreSQL, `LOAD DATA` in the MySQL family and + BigQuery, `SHOW` not in SQLite/SQL Server/Oracle, no `SET`/`GRANT` in + SQLite, ...); a parser without a dialect still accepts all of them. (#872) +- **Per-dialect grammar checks**: transaction statements are checked against + each dialect's grammar (`BEGIN banana` is rejected); `<=>` is accepted only + in MySQL, MariaDB and PostgreSQL (pgvector) and `==` only in SQLite and + ClickHouse; an unbalanced SQL Server `BEGIN ... END` block, or an + `IF`/`WHILE` without a body, is an error. MySQL / MariaDB `:=` inside an + expression assigns only to a user variable (`SELECT @v := 1`); `f(a := 1)` + and `arr[1:=2]` are errors. PostgreSQL `f(a := 1)` parses as a named + argument, written back as `f(a => 1)`. (#867, #872) +- **Numeric literals are lexed per dialect**: numeric words without + whitespace no longer silently become a number plus a column alias, and + numeric spelling is retained. Differences from the previous lexer (#874): + + | Dialect | Numeric changes | Unicode-escape changes | + | --- | --- | --- | + | PostgreSQL | `1_000`, `1_000.000_1`, `0o17`, `0b101`, and base-prefix separators such as `0x_FF` are single numbers. Malformed forms (`1x`, `0x`, `1__0`, `1_`, `1FROM`) error. | `U&'...'` with optional `UESCAPE` (comments allowed before it) is validated and preserved as one literal; `U&"..."` decodes to a quoted identifier usable wherever a name is. Malformed escapes error. | + | MySQL / MariaDB | `0b101` is a single bit literal; `1x`, `1_000`, and bare `0x` are identifiers rather than number/alias pairs. A number with a fraction still ends before a name (`1.5a` is `1.5 AS a`). | Unchanged: `U&'x'` remains separate `U`, `&`, and string tokens. | + | SQLite | `1_000` is one number; numeric trailing junk and unsupported `0b101` error. | Unchanged. | + | ClickHouse | `1_000` and `0b101` are single numbers; numeric trailing junk errors. | Unchanged. | + | SQL Server | Numeric trailing junk, including `1_000` and `0b101`, errors; a keyword written against a number (`1AS x`) is still accepted. | Unchanged. | + | Oracle | `1d`, `1.5f`, `1e5D` (BINARY_DOUBLE/BINARY_FLOAT) are single numbers. Storage sizes (`100M`, `8K`, `2G`) keep their units wherever a size appears; keywords against a number (`1FROM`) are accepted; other trailing junk, `1_000` and `0b101` error. | Unchanged. | + | Snowflake / BigQuery / Redshift | Numeric trailing junk, `0b101` and `1_000` error. | Unchanged. | + | generic | As Oracle for storage sizes and keywords; `1d`, `1x`, `1_000`, `0b101` error. | Unchanged. | + + `0x1F` remains a single hex literal in every dialect. +- **MySQL / MariaDB `#` line comments**: with the MySQL or MariaDB dialect, `#` + starts a line comment, like `--`. The tokenizer records it in `Comments` + (re-emitted by the formatter), `SplitStatements` no longer splits on a `;` + inside it, and transpiling to another dialect drops it instead of emitting + `#` (PostgreSQL XOR). Previously `WHERE x = 1 # c` parsed `#` as an operator + and `SELECT a FROM t # c` failed with E2002. Other dialects keep their `#` + meaning (PostgreSQL `#`, `#>`, `#>>`, `#-`; SQL Server `#temp`; Oracle + `emp#`). (#881) +- **Unrepresentable string bytes are errors**: a raw NUL or Ctrl-Z byte inside + a string literal is an error in dialects whose literals cannot spell it back + (NUL without backslash escapes; both without a dialect; Ctrl-Z in + ClickHouse; both in Snowflake). Formatting used to drop or rewrite it, + changing the value. (#867) +- **Fingerprints keep identifier quoting**: `fingerprint.Normalize` and + `Fingerprint` keep identifier quoting (`SELECT "A" FROM "T"` used to + normalize to `SELECT A FROM T`), so fingerprints of queries with quoted + identifiers differ from earlier releases. Re-compute stored fingerprints + after upgrading. (#867) +- **`gosqlx.Lint` / `LintWithDialect` report invalid SQL**: SQL that does not + tokenize or parse returns an error wrapping `ErrTokenize` or `ErrSyntax` + together with the violations of the text and token rules. Callers that + ignored the error and treated an empty result as clean must now check it. + `linter.FileResult` gains `TokenizeError`, `ParseError` and `SyntaxError()`; + `gosqlx lint` exits 1 on syntax errors. (#867) +- **Formatting refuses to drop comments**: `gosqlx.Format` and `gosqlx format` + keep comments and optimizer hints (see Fixed); input whose hints or MySQL + `/*! ... */` comments cannot be placed faithfully is refused with + `formatter.ErrUnplaceableComment` instead of being rewritten without them. + (#867) +- **Transpilation refuses lossy output**: string literals and quoted + identifiers are rendered with the target dialect's escape rules and + delimiters, and unrepresentable literals return an error rather than + changing their value. SQL Server `GO` scripts targeting another dialect, + operators the target cannot express, `LIMIT` on a set operation without + `ORDER BY` for SQL Server, `TOP ... PERCENT` / `WITH TIES` where the target + has no equivalent, and rewrites nested deeper than 512 levels return an + error instead of emitting changed or invalid SQL. (#867, #874, #876, #883, + #887) +- **PIVOT / UNPIVOT `IN` lists** require commas between entries and an alias + after `AS` in every PIVOT dialect. Where bare aliases are supported, + `IN (a b)` is one entry (`a AS b`); otherwise a missing comma is rejected. + `IN ('a' AS)` no longer silently drops the `AS`. Numeric UNPIVOT row aliases + after `AS` are limited to BigQuery and Oracle. (#857) +- **LSP capabilities key**: `initialize` advertises `textDocumentSync`, as the + LSP specification requires, instead of the non-standard + `textDocumentSyncOptions`; unknown methods are answered with `-32601` + (MethodNotFound) instead of `-32603`. Clients that read the old key must + switch. (#882, #887) + +### Security +- **`ScanSQL` bypasses closed**: the string scanner (also used by MCP, the C + binding and WASM) matches comments between keywords (`UNION/**/SELECT`, + `SLEEP/**/(5)`, `;--\nDROP`, `#` comments, `/*/`, MySQL `1--1`) through + comment-free views of the text under each quoting and escaping convention, + and scans MySQL/MariaDB `/*! */` and `/*M! */` bodies. Dollar-quote stripping + no longer runs inside quoted strings or identifiers. Tautology and comment + checks cover the whole input instead of the first 10,000 bytes, and inputs + above 10 MiB get a CRITICAL `INPUT_TOO_LARGE` finding whatever the severity + filter. (#885) +- **`Scanner.Scan(tree)` walks the whole AST**: time-based, out-of-band and + tautology patterns are detected inside CTEs, derived tables and subqueries, + CASE/CAST/IN/BETWEEN/ANY, ORDER BY/GROUP BY, JOIN ON, window specs, FILTER, + LIMIT, LATERAL/APPLY, PREWHERE, QUALIFY, CONNECT BY, RETURNING, ON CONFLICT, + INSERT ... SELECT, MERGE, CREATE TABLE AS, CREATE [MATERIALIZED] VIEW and + EXPLAIN, matching `ScanSQL`. UNION injection checks also run on set + operations nested in CTEs and subqueries, and the UNION NULL-padding check + matches the parser's NULL literal. Qualified column comparisons such as + `a.id = b.id` are no longer reported as tautologies. (#879, #885) +- **Raw-text statements are scanned**: raw dialect statements and standalone + `VALUES` are scanned for injection patterns. (#871) +- **CLI security scan merges both scanners**: `gosqlx analyze --security` and + `gosqlx lint`'s security pass run `ScanSQL` as well as the AST scan and merge + the findings, so `SELECT 1; DROP TABLE users` is reported as CRITICAL instead + of clean. (#887) +- **C binding**: every C export recovers from panics and returns an error + instead of crashing the host process. (#871) + ### Added -- Parse and faithfully render Snowflake PIVOT defaults, ClickHouse FINAL/SAMPLE - (also on joined tables, `TableModifiers.Sample`), SQL Server INSERT TOP, MySQL / - MariaDB ALTER TABLE partition actions (ADD/DROP [IF [NOT] EXISTS], TRUNCATE, - REORGANIZE, COALESCE, EXCHANGE, ANALYZE/CHECK/OPTIMIZE/REBUILD/REPAIR, - DISCARD/IMPORT, REMOVE/UPGRADE PARTITIONING, ALL), PostgreSQL ROWS FROM and - array subscripts, and ROLLUP/CUBE/nested GROUPING SETS inside GROUPING SETS; - cover identity constraints. Partition actions outside the structured grammar - still parse as as-written `AlterTableOther` text. -- `ast.PivotClause.AggregateAlias`, `ast.PivotClause.ExtraAggregates` - (`[]ast.PivotAggregate`), `ast.UnpivotClause.NullHandling`, - `ast.UnpivotClause.ValueColumns` and `ast.UnpivotNullHandling`; rendered by - `SQL()` and `pkg/formatter`. -- **Real-world dialect corpus**: `testdata/realworld//` holds 2,788 +- **Real-world dialect corpus**: `testdata/realworld//` holds 2,813 production-style statements across PostgreSQL, MySQL, MariaDB, SQL Server, Oracle, SQLite, Snowflake and ClickHouse. `TestRealWorldCorpus` runs each through split, parse, format, reparse, idempotency and dropped-word checks against a ratchet list (`testdata/realworld/known_failures.txt`, regenerate with `-update-known-failures`): a new failure fails the test, and so does a - listed failure that now passes. `scripts/dialect-audit` runs the same checks + listed failure that now passes. Every statement now passes and the list is + empty (1,327 failures before). `scripts/dialect-audit` runs the same checks on any SQL file; `BenchmarkRealWorldParse` and `BenchmarkRealWorldParseFormat` - measure the corpus. -- **Dialect coverage**: every statement of the real-world corpus now parses, - formats, re-parses identically and keeps all its words; `known_failures.txt` - is empty (1,327 failures before). -- **AST representation**: `ExplainStatement` retains the explained statement; - `BatchSeparator` retains SQL Server `GO` and its optional `RepeatCount` without - expanding repeated batches. `SQL()` and `FormatAST` keep GO on its own line. - Callers enumerating `AST.Statements` or recovery results must account for - batch-separator nodes. -- **Literal and identifier metadata**: string literals retain their introducer - (`N`, `X`, `B`, `_utf8mb4`, ...) and escape rules (`LiteralValue.DialectExtras()`, - `EnsureExtras()`, `SetRawBackslash()`). `Identifier.QuoteStyle`, - `Identifier.Quotes` and the statement/table/column `NameQuote` fields retain - identifier delimiters through rendering. - -### Changed -- **MySQL / MariaDB `#` line comments**: with the MySQL or MariaDB dialect, `#` - now starts a line comment, like `--`. The tokenizer records it in `Comments` - (re-emitted by the formatter), `SplitStatements` no longer splits on a `;` - inside it, and transpiling to another dialect drops it instead of emitting `#` - (PostgreSQL XOR). Previously `WHERE x = 1 # c` parsed `#` as an operator and - `SELECT a FROM t # c` failed with E2002. Other dialects keep their `#` - meaning (PostgreSQL `#`, `#>`, `#>>`, `#-`; SQL Server `#temp`; Oracle `emp#`). -- Parsed ClickHouse SAMPLE sizes and offsets now use inspectable expression - fields, which take rendering precedence over the retained legacy string fields. -- ClickHouse SAMPLE accepts only decimal numbers (hex such as `0x10` is - rejected) and renders before ARRAY JOIN; a SAMPLE clause with no valid size is - omitted instead of rendered as a bare `SAMPLE`. -- A column with more than one of DEFAULT, an identity clause and a generation - expression is rejected; signed numbers in identity options render as `-1`. -- **Dialect extension fields**: the new dialect-specific clauses live behind - optional pointers so that common statements do not carry them: + measure the corpus. (#866, #867) +- **New syntax**: Snowflake PIVOT `DEFAULT ON NULL`, ClickHouse FINAL/SAMPLE + (also on joined tables, `TableModifiers.Sample`; decimal sizes and offsets + only, held in inspectable expression fields), SQL Server `INSERT TOP`, MySQL + / MariaDB `ALTER TABLE` partition actions (ADD/DROP [IF [NOT] EXISTS], + TRUNCATE, REORGANIZE, COALESCE, EXCHANGE, ANALYZE/CHECK/OPTIMIZE/REBUILD/ + REPAIR, DISCARD/IMPORT, REMOVE/UPGRADE PARTITIONING, ALL), PostgreSQL + `ROWS FROM` and array subscripts, ROLLUP/CUBE/nested GROUPING SETS inside + GROUPING SETS, and identity constraints (a column with more than one of + DEFAULT, an identity clause and a generation expression is rejected). + Partition actions outside the structured grammar still parse as as-written + `AlterTableOther` text. (#873) +- **BigQuery PIVOT / UNPIVOT**: the parser gated PIVOT/UNPIVOT on a hard-coded + SQL Server / Oracle / Snowflake list, so BigQuery rejected them with E2002 + although its capabilities advertise `SupportsPivotUnpivot`. The gate is now + `Capabilities().SupportsPivotUnpivot` (true for exactly SQL Server, Oracle, + Snowflake and BigQuery). Supported BigQuery forms: aggregate aliases and + multiple aggregates, `IN` value aliases with or without `AS`, + `UNPIVOT INCLUDE NULLS` / `EXCLUDE NULLS`, row-value aliases with or without + `AS` (string or INT64) and multi-column UNPIVOT. Oracle also gets + aliased/multiple aggregates, the NULLS modifier and multi-column UNPIVOT; + Snowflake gets the NULLS modifier and bare `IN` value aliases. New + `ast.PivotClause.AggregateAlias`, `ast.PivotClause.ExtraAggregates` + (`[]ast.PivotAggregate`), `ast.UnpivotClause.NullHandling`, + `ast.UnpivotClause.ValueColumns` and `ast.UnpivotNullHandling`; quoted + aliases and value columns keep their source delimiters when rendered. + (#857, fixes #854) +- **More statements**: MariaDB `DELETE HISTORY FROM t [PARTITION (p, ...)] + [BEFORE SYSTEM_TIME [TIMESTAMP | TRANSACTION] expr]` (held in + `ast.DMLTargetFields.History`, `ast.DeleteHistoryOptions`); PostgreSQL + `ALTER DEFAULT PRIVILEGES [FOR ROLE r] [IN SCHEMA s] GRANT | REVOKE ...`. + (#872, #880) +- **AST metadata for faithful rendering**: `ExplainStatement` retains the + explained statement; `ast.BatchSeparator` retains SQL Server `GO`; string + literals retain their introducer (`N`, `X`, `B`, `_utf8mb4`, ...) and escape + rules (`LiteralValue.DialectExtras()`, `EnsureExtras()`, + `SetRawBackslash()`); `Identifier.QuoteStyle`, `Identifier.Quotes` and the + statement/table/column `NameQuote` fields retain identifier delimiters. + `AST.StatementEnds`, `AST.ExecuteSlash` / `ExecuteSlashPositions` (Oracle + SQL*Plus `/` lines) and `AST.UnicodeIdentifiers` (PostgreSQL `U&"..."` + spellings) record source layout for the formatter. (#867, #869, #886) +- **Dialect extension fields**: dialect-specific clauses live behind optional + pointers so that common statements do not carry them: `SelectStatement.Extras` (`SelectExtras`: SELECT modifiers, INTO, LIMIT BY, ROWS EXAMINED, INTERPOLATE, query hints, output formats), `InsertStatement.Extras`, `UpdateStatement.Extras` / `DeleteStatement.Extras` @@ -68,12 +220,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `BinaryExpression.Extras` and `TableReference.Modifiers` (including derived `VALUES`). Read them through `DialectExtras()` / `DialectModifiers()`, which return values for read access without shared mutable stand-ins; call - `EnsureExtras()` / `EnsureModifiers()` before writing. -- **Transpilation**: string literals and quoted identifiers are rendered using - the target dialect's escape rules and delimiters. Unrepresentable literals - return an error rather than changing their value. SQL Server GO scripts - targeting another dialect return an error because batch scopes and repeat - counts cannot be represented there. + `EnsureExtras()` / `EnsureModifiers()` before writing. (#867) +- **Formatter API**: `formatter.FormatASTChecked`, `FormatStatementChecked` + and `FormatExpressionChecked` return an error for nodes they cannot render + faithfully; `formatter.JoinWithComments`, `JoinASTWithComments`, + `CommentLayout`, `BatchPositions` and `StatementPositions` re-attach + comments to formatted statements; `EnsureStatementTerminator`, + `ExecuteSlashStatement` and `RestoreUnicodeIdentifiers` keep terminators, + Oracle `/` lines and `U&` spellings. `gosqlx.FormatWithDialect`, + `gosqlx.WithComments()`, `gosqlx.FormatASTChecked`, + `gosqlx.FormatTreeChecked`, `Tree.FormatChecked` and + `transform.FormatSQLChecked` expose the same at the higher levels. + (#867, #884, #886) +- **MCP**: the `parse_sql` tool accepts an optional `dialect`. The hosted + server is described by `server.json`, and tagged releases publish it to the + MCP Registry (`publish-mcp.yml`, `docs/MCP_REGISTRY.md`). (#859, #876) + +### Changed - **Transpiling dialect-specific operators**: `^` is exponentiation in PostgreSQL and bitwise XOR in MySQL, MariaDB and SQL Server (recorded as `BinaryExpression.BitwiseXor`). Exponentiation is written `POWER(a, b)` @@ -81,229 +244,143 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 family and SQL Server, becomes `#` in PostgreSQL, `BITXOR()` in Snowflake and `bitXor()` in ClickHouse, and returns an error for Oracle, SQLite and the generic dialect. PostgreSQL `@`, `|/` and `||/` become `ABS()`, `SQRT()` and - `CBRT()` (an error where CBRT is missing); `!!`, SQL Server money literals + `CBRT()` (an error where CBRT is missing); PostgreSQL `#` (XOR), `~`, + `<<`/`>>`, `&`/`|` and `||` (CONCAT in MySQL/MariaDB) are rewritten into the + target's form; `!!`, `@>`, `?`, regex `~`, SQL Server money literals (`$5.00`, `$5`) and Snowflake object constants (`{'a': 1}`) return an error - in dialects without them. - -### Fixed -- **CLI security scan**: `gosqlx analyze --security` and `gosqlx lint`'s security pass now run the string scanner (`ScanSQL`) as well as the AST scan and merge the findings, so `SELECT 1; DROP TABLE users` is reported as CRITICAL instead of clean. Inline SQL that contains `/` (for example `UNION/**/SELECT` or `4 / 2`) is no longer mistaken for a missing file path. -- **Transpile row limits**: for SQL Server, `LIMIT n OFFSET m` and bare `OFFSET` without ORDER BY add `ORDER BY (SELECT NULL)`, and `FETCH FIRST` gains the required `OFFSET 0 ROWS`. Row limits inside derived tables, JOIN operands and scalar subqueries are now rewritten for the target dialect (Oracle kept `LIMIT` there). `WITH TIES` survives set-operation tails, and targets that cannot express it return an error instead of dropping it. -- **LSP**: requests for unknown methods are answered with `-32601` (MethodNotFound) instead of `-32603`. -- **Formatter fidelity**: Oracle SQL*Plus `/` execute lines are kept by FormatAST, `Formatter.Format`, `FormatWithDialect` and `gosqlx format` (they were dropped, and the CLI refused such scripts). Statement counts do not change: the AST records which statements a `/` follows in `AST.ExecuteSlash`. PostgreSQL `U&"..."` identifiers keep their original spelling when formatting for PostgreSQL or no target (other targets get the decoded name; AST name fields stay decoded), so names with escaped control characters no longer produce output that fails to re-tokenize. Comments next to SQL Server `GO` lines and trailing comments of statements written without `;` stay where they were, and `GO 2 /* c */` keeps both count and comment on every entry point. -- **Formatter performance**: lower-case keyword mode streams into a single builder instead of copying and re-rendering subtrees; it went from 3.5–10x to 1.2–1.5x the cost of upper-case output, with 5x to 500x fewer allocations. -- **Security (`ScanSQL`)**: closed several ways to slip injection past the string scanner (also used by MCP, the C binding and WASM). Dollar-quote stripping no longer runs inside quoted strings or identifiers (`SELECT '$x$'; DROP TABLE users; SELECT '$x$'` was clean). Tautology and comment checks now cover the whole input instead of the first 10,000 bytes; inputs above 10 MiB get a CRITICAL `INPUT_TOO_LARGE` finding whatever the severity filter. Comments between keywords (`UNION/**/SELECT`, `SLEEP/**/(5)`, `;--\nDROP`, `#` comments, `/*/`, MySQL `1--1`) are matched through comment-free views of the text under each quoting and escaping convention, and MySQL/MariaDB `/*! */` and `/*M! */` bodies are scanned. The AST UNION NULL-padding check now matches the parser's NULL literal. The tautology check no longer allocates per candidate (1 MiB of `1=2` went from 50 MB to 1 MB). -- **Format**: with `AddSemicolon` and a comment anywhere in the input, raw-text statements (PostgreSQL `DO $$...$$`, `CREATE FUNCTION` bodies, Oracle `BEGIN ... END` blocks) lost their `;`, so the output no longer parsed or merged two statements. The comment-preserving path now adds terminators the same way as the plain path (`formatter.EnsureStatementTerminator`). -- **Extract**: `gosqlx.Extract*` and `ExtractMetadata` (and the MCP `extract_metadata`/`analyze_sql` tools) took exponential time on nested CTEs and UNION chains because CTE, set-operation and DML children were visited twice. Each node is now visited once; nested CTEs at depth 10 took about 59 s and now finish in microseconds. -- **Transpile**: a LIMIT on a whole UNION/INTERSECT/EXCEPT no longer becomes `TOP` on the last branch when targeting SQL Server (that silently changed results); it is emitted as `ORDER BY ... OFFSET n ROWS FETCH NEXT m ROWS ONLY`, and without ORDER BY Transpile returns an error. Very long operator chains (for example 490,000 `OR` terms) no longer crash the process with an unrecoverable stack overflow: the operator and string-literal rewrites walk the tree iteratively, and rewrites that would nest output more than 512 levels deep return an error. -- **LSP (`pkg/lsp`)**: `initialize` now advertises `textDocumentSync` (it sent the non-standard `textDocumentSyncOptions`, so VS Code never sent open/change/close events). A negative `Content-Length` and a `didChange` range past the end of the document no longer crash the server; oversized messages are drained so framing survives; `didChange` character offsets are read as UTF-16 code units and CRLF line ends are respected. A panic in a request handler or in background diagnostics is now logged and answered with `InternalError` instead of ending the process. -- PostgreSQL / Redshift / SQLite `VACUUM` and `ANALYZE`, and PostgreSQL `LISTEN`, - `UNLISTEN` and `NOTIFY`, now end where their grammar ends: trailing same-line - tokens (`VACUUM t junk`, `LISTEN c extra`) are a parse error instead of being - folded into the statement text. MySQL / MariaDB / Oracle `ANALYZE TABLE` keeps - its as-written handling. -- SQL Server: a delimited `[top]` / `"top"` is a plain name (also in the select - list, where it was read as the TOP clause) and keeps its delimiters on output; - bare `TOP` is rejected as a table, DML target or alias name, as T-SQL reserves it. -- MariaDB `DELETE HISTORY FROM t [PARTITION (p, ...)] [BEFORE SYSTEM_TIME - [TIMESTAMP | TRANSACTION] expr]` parses and round-trips the partition list, the - unit keyword and any point expression (e.g. `NOW() - INTERVAL 1 DAY`), held in - the new `ast.DMLTargetFields.History` (`ast.DeleteHistoryOptions`). -- **Security scanner**: `Scanner.Scan(tree)` now walks the whole AST, so time-based, - out-of-band and tautology patterns are detected inside CTEs, derived tables and - subqueries, CASE/CAST/IN/BETWEEN/ANY, ORDER BY/GROUP BY, JOIN ON, window specs, - FILTER, LIMIT, LATERAL/APPLY, PREWHERE, QUALIFY, CONNECT BY, RETURNING, ON CONFLICT, - INSERT ... SELECT, MERGE, CREATE TABLE AS, CREATE [MATERIALIZED] VIEW and EXPLAIN, - matching `ScanSQL`. UNION injection checks also run on set operations nested in CTEs - and subqueries. Qualified column comparisons such as `a.id = b.id` are no longer - reported as tautologies. -- `GRANT ... TO [`, `TO [AppUser`, `TO CURRENT_USER(`, MariaDB - `IDENTIFIED VIA ... USING PASSWORD(` and `COPY (` with an unclosed group no - longer panic with `index out of range [-1]`; they return an E2002 error - (expected `]` or `)`) in every dialect. -- Watch now respects `--dialect`, re-validates changes and formats files in place - with the CLI safety checks; MCP parsing accepts a dialect and documents comment preservation. -- Correct transpile dialect help, avoid duplicate lint syntax errors, and rewrite - SQL Server SELECT TOP limits across dialects, refusing PERCENT and WITH TIES. -- Preserve dialect numeric and PostgreSQL Unicode-escape literals and quoted type names; correct JSON/power precedence and retain operator meaning during target-dialect formatting. Hand-built expression trees, including hand-built operands inserted into parsed trees, now render necessary parentheses. -- Formatting or transpiling to another dialect rewrites PostgreSQL `#` (XOR), - `~`, `<<`/`>>`, `&`/`|` and `||` (CONCAT in MySQL/MariaDB) into the target's - form, and refuses with an error where the target has none (for example `#` - in Oracle/SQLite, `@>`/`?`/regex `~` outside PostgreSQL), instead of - emitting them unchanged (`a # b` read as `a` plus a comment in MySQL). -- Scan raw dialect statements and standalone VALUES for injection patterns, recover - panics at every C export, and guard unlinked tokenizer scratch calls. -- Formatter lower-case mode covers expression and built-in function keywords; - comments use LF endings and keep SQL Server batch boundaries. Fix MySQL join - comma spacing, SELECT alias qualifiers, and TOP/FETCH limit transforms. -- Lower-case expression formatting avoids repeated subtree rendering and copying. - Raw `JSON_TABLE`/`XMLTABLE` clauses retain their source case in either mode. -- Statement splitting follows dialect string escapes, parenthesized bodies, - commented SQL Server GO lines and Oracle SQL*Plus execute lines; corpus and - CLI audits share a pipeline and detect repeated dropped words. -- **AST walkers** (`ast.Inspect`, `Walk`, the security scanner, linter and - table extraction) reach SELECT `TOP` and ClickHouse `ARRAY JOIN`, the - ENGINE/ORDER BY/PARTITION BY/TTL/SETTINGS clauses of `CREATE TABLE` and - `CREATE MATERIALIZED VIEW`, Snowflake `CLONE ... AT`, and PostgreSQL - partition bounds. `AST.SQL()` no longer drops `CREATE`/`ALTER`/`DROP - SEQUENCE` statements, and MERGE keeps the source's column alias list - (`USING (VALUES ...) AS s (id, val)`). -- PostgreSQL `@` (absolute value) is always written as `@ x`; `@a`, `@t.a` and - `@abs(a)` re-parsed as placeholders. `@` directly followed by operator - characters (`@-@`, `@-5`) is now rejected, as PostgreSQL lexes it as one - operator. -- A WITH clause before nested parenthesized set operands - (`WITH c ... ((A UNION B) INTERSECT C) EXCEPT D`) stays in front of the whole - operation in `SQL()`, `FormatAST` and `Transpile`. -- Optimizer hints in subqueries stay with their SELECT when the renderer - reorders clauses (Oracle `START WITH` / `CONNECT BY`); when the hinted SELECT - cannot be identified, formatting fails with `ErrUnplaceableComment`. -- LSP document outline no longer lists GO separators, and statements before a - GO line no longer extend to the end of the file. -- `ReleaseAST` returns the pooled nodes under standalone `VALUES` statements and - top-level set operations. -- `gosqlx format --dialect sqlserver` runs the statement-count guard again, - counting GO lines and statements that end without `;`. -- **`ExtractColumns` / `ExtractColumnsQualified` no longer report SELECT aliases** - (#849): an unqualified name in `ORDER BY` or `HAVING` that matches a - SELECT-list alias of the same query block is treated as the alias, not a - column (e.g. `... AS orders ... ORDER BY orders` no longer lists `orders`). - Qualified references stay columns, and aliases never cross subquery, CTE or - UNION boundaries. This also corrects the MCP `extract_metadata` / - `analyze_sql` tools. -- **BigQuery PIVOT / UNPIVOT** (#854): the parser gated PIVOT/UNPIVOT on a - hard-coded SQL Server / Oracle / Snowflake list, so BigQuery rejected them - with E2002 although its capabilities advertise `SupportsPivotUnpivot`. The - gate is now `Capabilities().SupportsPivotUnpivot` (true for exactly SQL - Server, Oracle, Snowflake and BigQuery; the default, Generic, PostgreSQL, - MySQL, MariaDB, SQLite, ClickHouse and Redshift dialects still treat `pivot` - / `unpivot` as identifiers). Supported BigQuery forms: aggregate aliases and - multiple aggregates, `IN` value aliases with or without `AS`, - `UNPIVOT INCLUDE NULLS` / `EXCLUDE NULLS`, row-value aliases with or without - `AS` (string or INT64) and multi-column UNPIVOT. The extensions follow each - dialect's grammar: Oracle also gets aliased/multiple aggregates, the NULLS - modifier and multi-column UNPIVOT; Snowflake gets the NULLS modifier and - bare `IN` value aliases; SQL Server is unchanged. -- PIVOT/UNPIVOT quoted aggregate aliases, UNPIVOT value columns, and - backtick-quoted `IN` names and aliases keep their source delimiters when - rendered. This preserves BigQuery backticks and Oracle alias case. -- Numeric UNPIVOT row aliases after `AS` are limited to BigQuery and Oracle; - Snowflake and SQL Server reject them. -- **SQL round trips**: preserve ALTER actions, grouped joins, set-operation - query tails and CTE placement, standalone VALUES, JSON_TABLE/XMLTABLE literal - quoting, and nested optimizer hints. Hint placement uses the source dialect's - lexer so a keyword inside a string or quoted identifier is never a hint slot. -- **Pooled parsing**: internal tokenize/parse paths reuse scratch token storage - without retaining it in returned ASTs. Concurrent retained-tree regression - tests cover the public parse entry points; SELECT release clears all Extras. -- **`SplitStatements` keeps compound statements whole**: `BEGIN ... END` blocks - (trigger, procedure and function bodies, T-SQL `IF ... BEGIN`, `BEGIN TRY`, - PostgreSQL `BEGIN ATOMIC`), `CASE ... END`, `IF`/`LOOP`/`WHILE`/`REPEAT` - inside a block and Oracle `DECLARE`/`IS`/`AS` sections are no longer cut at - the first inner `;`. Transaction statements (`BEGIN;`, `BEGIN TRANSACTION`, - `BEGIN IMMEDIATE`) are not blocks, and a script whose blocks do not balance - falls back to plain `;` splitting. Client batch separators are honoured and - removed from the output: a `GO [n]` line for SQL Server, `DELIMITER` for - MySQL/MariaDB and a lone `/` line for Oracle. -- **`gosqlx.Lint` / `LintWithDialect` report invalid SQL**: SQL that does not - tokenize or parse returns an error wrapping `ErrTokenize` or `ErrSyntax` - together with the violations of the text and token rules. Callers that - ignored the error and treated an empty result as clean must now check it. - `linter.FileResult` gains `TokenizeError`, `ParseError` and `SyntaxError()`. -- **`gosqlx.Format` and `gosqlx format` keep comments and optimizer hints**: - comments stay with their statement (a comment inside a statement moves to the - line above it) and `/*+ ... */` hints directly after a statement's first - keyword stay in place. Input whose hints or MySQL `/*! ... */` comments - cannot be placed faithfully is refused (`formatter.ErrUnplaceableComment`) - instead of being rewritten without them, and `format -i` verifies that the - output holds exactly the input's comments. LSP and C-binding dialect - formatting no longer drop comments either. New `formatter.JoinWithComments`, - `gosqlx.FormatWithDialect` and `gosqlx.WithComments()`. -- **CLI**: `lint` exits 1 on syntax errors and `--auto-fix` judges the fixed - text; `format --no-uppercase` lower-cases keywords, keyword operators - and built-in function keywords (raw `JSON_TABLE`/`XMLTABLE` clauses retain source - case); `postgres`, `pg`, `mssql`, `tsql` and `sqlite3` are accepted by every + in dialects without them. PostgreSQL `@` (absolute value) is always written + as `@ x`, and `@` directly followed by operator characters (`@-@`, `@-5`) is + rejected, as PostgreSQL lexes it as one operator. (#867, #874) +- **SQL Server row limits when transpiling**: `SELECT TOP` limits are + rewritten across dialects (refusing `PERCENT` and `WITH TIES` where the + target cannot express them); `LIMIT n OFFSET m` and bare `OFFSET` without + ORDER BY add `ORDER BY (SELECT NULL)`, and `FETCH FIRST` gains the required + `OFFSET 0 ROWS`. A LIMIT on a whole UNION/INTERSECT/EXCEPT is emitted as + `ORDER BY ... OFFSET n ROWS FETCH NEXT m ROWS ONLY` instead of `TOP` on the + last branch (which silently changed results). Row limits inside derived + tables, JOIN operands and scalar subqueries are rewritten for the target + dialect, and `WITH TIES` survives set-operation tails. (#876, #883, #887) +- **SQL Server `[top]`**: a delimited `[top]` / `"top"` is a plain name (also + in the select list, where it was read as the TOP clause) and keeps its + delimiters on output. (#880) +- **`gosqlx watch`** respects `--dialect`, re-validates changed files and + formats files in place with the CLI safety checks. (#876) +- **CLI**: `lint --auto-fix` judges the fixed text; `format --no-uppercase` + lower-cases keywords, keyword operators and built-in function keywords; + `postgres`, `pg`, `mssql`, `tsql` and `sqlite3` are accepted by every `--dialect` flag; errors are printed once and usage only for flag and argument errors; `validate --quiet` exits silently; `validate` names stdin - ``, validates `GO`/`DELIMITER` scripts and files over the token + `` and validates `GO`/`DELIMITER` scripts and files over the token limit statement by statement; empty and comment-only input is valid for `validate`, `lint` and `format`; `-o` works for `parse` and `transpile`, `-f`/`-o` map onto `validate`'s output flags and are rejected where - unsupported; `format` keeps a leading byte order mark. -- **`ParseReaderMultiple`** skips segments that hold only comments. -- **Docs**: removed `parser.NewTokenConverter`, `SetMaxTokens`, - `SetMaxRecursionDepth` and the `parser.max_tokens` config key, none of which - exist; the limits are constants. -- **Parser**: PostgreSQL `ALTER DEFAULT PRIVILEGES [FOR ROLE r] [IN SCHEMA s] - GRANT | REVOKE ...` parses and round-trips; MariaDB ``DELETE `history` FROM - history`` stays a multi-table delete when formatted (it was written back as - `DELETE HISTORY`); a T-SQL `IF ... ELSE IF ...` chain of any length is one - statement, parsed in linear time (chains over ~100 branches hit the - recursion limit and swallowed the next statement); MySQL `ANALYZE TABLE t - UPDATE HISTOGRAM ...` and Oracle `ANALYZE TABLE t DELETE STATISTICS` are no - longer read as MariaDB's `ANALYZE `. + unsupported; `format` keeps a leading byte order mark; inline SQL that + contains `/` (`UNION/**/SELECT`, `4 / 2`) is no longer mistaken for a file + path; the transpile dialect help is corrected. (#867, #876, #887) -### Behavior changes -- SQL Server: unquoted `top` (any case) is now a syntax error where a table - name, INSERT/UPDATE/DELETE/MERGE target, table alias or select-item alias is - expected (`SELECT * FROM top`, `INSERT INTO top ...`); quote it as `[top]`. -- Numeric words without whitespace no longer silently become a number plus a - column alias. Numeric spelling is retained. Differences from the previous lexer: - - | Dialect | Numeric changes | Unicode-escape changes | - | --- | --- | --- | - | PostgreSQL | `1_000`, `1_000.000_1`, `0o17`, `0b101`, and base-prefix separators such as `0x_FF` are single numbers. Malformed forms (`1x`, `0x`, `1__0`, `1_`, `1FROM`) error. | `U&'...'` with optional `UESCAPE` (comments allowed before it) is validated and preserved as one literal; `U&"..."` decodes to a quoted identifier usable wherever a name is. Malformed escapes error. | - | MySQL / MariaDB | `0b101` is a single bit literal; `1x`, `1_000`, and bare `0x` are identifiers rather than number/alias pairs. A number with a fraction still ends before a name (`1.5a` is `1.5 AS a`). | Unchanged: `U&'x'` remains separate `U`, `&`, and string tokens. | - | SQLite | `1_000` is one number; numeric trailing junk and unsupported `0b101` error. | Unchanged. | - | ClickHouse | `1_000` and `0b101` are single numbers; numeric trailing junk errors. | Unchanged. | - | SQL Server | Numeric trailing junk, including `1_000` and `0b101`, errors; a keyword written against a number (`1AS x`) is still accepted. | Unchanged. | - | Oracle | `1d`, `1.5f`, `1e5D` (BINARY_DOUBLE/BINARY_FLOAT) are single numbers. Storage sizes (`100M`, `8K`, `2G`) keep their units wherever a size appears; keywords against a number (`1FROM`) are accepted; other trailing junk, `1_000` and `0b101` error. | Unchanged. | - | Snowflake / BigQuery / Redshift | Numeric trailing junk, `0b101` and `1_000` error. | Unchanged. | - | generic | As Oracle for storage sizes and keywords; `1d`, `1x`, `1_000`, `0b101` error. | Unchanged. | +### Fixed +- **Formatter fidelity**: `gosqlx.Format` and `gosqlx format` keep comments + (a comment inside a statement moves to the line above it) and `/*+ ... */` + hints directly after a statement's first keyword; `format -i` verifies that + the output holds exactly the input's comments; LSP and C-binding dialect + formatting no longer drop comments. Oracle SQL*Plus `/` execute lines are + kept by `FormatAST`, `Formatter.Format`, `FormatWithDialect` and `gosqlx + format` (they were dropped, and the CLI refused such scripts). PostgreSQL + `U&"..."` identifiers keep their original spelling when formatting for + PostgreSQL or no target (other targets get the decoded name), so names with + escaped control characters no longer produce output that fails to + re-tokenize. Comments next to SQL Server `GO` lines and trailing comments of + statements written without `;` stay where they were, and `GO 2 /* c */` + keeps both count and comment. Lower-case mode covers expression and + built-in function keywords, and raw `JSON_TABLE`/`XMLTABLE` clauses retain + their source case. (#867, #870, #886) +- **Format terminators**: with `AddSemicolon` and a comment anywhere in the + input, raw-text statements (PostgreSQL `DO $$...$$`, `CREATE FUNCTION` + bodies, Oracle `BEGIN ... END` blocks) lost their `;`, so the output no + longer parsed or merged two statements. The comment-preserving path now adds + terminators the same way as the plain path. (#884) +- **SQL round trips**: `SQL()`, `FormatAST` and `Transpile` preserve ALTER + actions, grouped joins, set-operation query tails, CTE placement (a WITH + clause before nested parenthesized set operands stays in front of the whole + operation), standalone VALUES, JSON_TABLE/XMLTABLE literal quoting, quoted + type names, and optimizer hints in subqueries (when the hinted SELECT cannot + be identified, formatting fails with `ErrUnplaceableComment`). Hint + placement uses the source dialect's lexer, so a keyword inside a string or + quoted identifier is never a hint slot. `AST.SQL()` no longer drops + `CREATE`/`ALTER`/`DROP SEQUENCE` statements, MERGE keeps the source's column + alias list (`USING (VALUES ...) AS s (id, val)`), JSON/power operator + precedence is correct, and hand-built expression trees (including hand-built + operands inserted into parsed trees) render the parentheses they need. + Formatting fixes MySQL join comma spacing and SELECT alias qualifiers. + (#867, #870, #874) +- **Statement splitting**: `SplitStatements` keeps compound statements whole: + `BEGIN ... END` blocks (trigger, procedure and function bodies, T-SQL + `IF ... BEGIN`, `BEGIN TRY`, PostgreSQL `BEGIN ATOMIC`), `CASE ... END`, + `IF`/`LOOP`/`WHILE`/`REPEAT` inside a block and Oracle `DECLARE`/`IS`/`AS` + sections are no longer cut at the first inner `;`. Transaction statements + (`BEGIN;`, `BEGIN TRANSACTION`, `BEGIN IMMEDIATE`) are not blocks, and a + script whose blocks do not balance falls back to plain `;` splitting. Client + batch separators are honoured and removed from the output: a `GO [n]` line + for SQL Server, `DELIMITER` for MySQL/MariaDB and a lone `/` line for + Oracle. Splitting follows dialect string escapes, parenthesized bodies and + commented `GO` lines, agreeing with the parser. `ParseReaderMultiple` skips + segments that hold only comments. (#867, #869) +- **Parser**: a T-SQL `IF ... ELSE IF ...` chain of any length is one + statement, parsed in linear time (chains over ~100 branches hit the + recursion limit and swallowed the next statement); MariaDB ``DELETE + `history` FROM history`` stays a multi-table delete when formatted; MySQL + `ANALYZE TABLE t UPDATE HISTOGRAM ...` and Oracle `ANALYZE TABLE t DELETE + STATISTICS` are no longer read as MariaDB's `ANALYZE `; + `GRANT ... TO [`, `TO [AppUser`, `TO CURRENT_USER(`, MariaDB `IDENTIFIED VIA + ... USING PASSWORD(` and `COPY (` with an unclosed group return an E2002 + error instead of panicking with `index out of range [-1]`. (#872, #880, + #881) +- **`ExtractColumns` / `ExtractColumnsQualified` no longer report SELECT + aliases**: an unqualified name in `ORDER BY` or `HAVING` that matches a + SELECT-list alias of the same query block is treated as the alias, not a + column. Qualified references stay columns, and aliases never cross + subquery, CTE or UNION boundaries. This also corrects the MCP + `extract_metadata` / `analyze_sql` tools. (#865, fixes #849) +- **Extract performance**: `gosqlx.Extract*` and `ExtractMetadata` (and the + MCP `extract_metadata` / `analyze_sql` tools) took exponential time on nested + CTEs and UNION chains because CTE, set-operation and DML children were + visited twice. Each node is now visited once; nested CTEs at depth 10 took + about 59 s and now finish in microseconds. (#884) +- **AST walkers** (`ast.Inspect`, `Walk`, the security scanner, linter and + table extraction) reach SELECT `TOP` and ClickHouse `ARRAY JOIN`, the + ENGINE/ORDER BY/PARTITION BY/TTL/SETTINGS clauses of `CREATE TABLE` and + `CREATE MATERIALIZED VIEW`, Snowflake `CLONE ... AT`, and PostgreSQL + partition bounds. `ReleaseAST` returns the pooled nodes under standalone + `VALUES` statements and top-level set operations. (#867) +- **Transpiler robustness**: very long operator chains (for example 490,000 + `OR` terms) no longer crash the process with an unrecoverable stack + overflow; the operator and string-literal rewrites walk the tree + iteratively. (#883) +- **LSP**: a negative `Content-Length` and a `didChange` range past the end + of the document no longer crash the server; oversized messages are drained + so framing survives; `didChange` character offsets are read as UTF-16 code + units and CRLF line ends are respected; a panic in a request handler or in + background diagnostics is logged and answered with `InternalError` instead + of ending the process. The document outline no longer lists GO separators, + and statements before a GO line no longer extend to the end of the file. + (#867, #882) +- **CLI**: `gosqlx format --dialect sqlserver` runs the statement-count guard + again, counting GO lines and statements that end without `;`; `lint` no + longer reports a syntax error twice. (#867, #876) - `0x1F` remains a single hex literal in every dialect. -- SQL Server: a line holding only `GO [n]` inside a statement is now a parse - error ("GO batch separator inside a statement"), matching sqlcmd, instead of - a column or alias named `go`. `GO 0` / `GO 1.5` alone on a line report - "invalid GO repeat count". -- A raw NUL or Ctrl-Z byte inside a string literal is an error in dialects - whose literals cannot spell it back (NUL without backslash escapes; both - without a dialect; Ctrl-Z in ClickHouse; both in Snowflake). Formatting used - to drop or rewrite it, changing the value. -- Statement counts (`gosqlx analyze`, `gosqlx parse`, the MCP `parse_sql` - tool and the C binding's `statement_count` / statement types) exclude GO - separators. -- `fingerprint.Normalize` and `Fingerprint` keep identifier quoting - (`SELECT "A" FROM "T"` used to normalize to `SELECT A FROM T`), so - fingerprints of queries with quoted identifiers differ from earlier - releases. Re-compute stored fingerprints after upgrading. -- Outside SQL Server, a utility statement (`SET`, `USE`, `GRANT`, `COPY`, - `CALL`, ...) followed on the next line by another statement without `;` is - an error; only a transaction statement on its own line is accepted without - a separator. Statements whose definition holds a statement (`PREPARE ... - AS`, `DECLARE ... CURSOR FOR`, `CREATE FUNCTION`/`PROCEDURE`/`TRIGGER`/ - `EVENT`/`RULE`/`TASK`/`VIEW` with a single-statement, `RETURN` or - `BEGIN ... END` body, `LOAD DATA`) end with that definition, and `GRANT`, - `REVOKE`, PostgreSQL `COPY`, `CALL`, `VACUUM`, `ANALYZE`, `LISTEN`, ... end - with their grammar, so a statement after them without `;` is an error on the - same line as well. -- Utility statements are accepted only in the dialects that have them - (`VACUUM` in PostgreSQL/SQLite, `LISTEN`/`NOTIFY`/`CLUSTER`/`DISCARD` in - PostgreSQL, `FLUSH`/`OPTIMIZE`/`LOCK TABLES`/`KILL` in MySQL-family, `PRAGMA` - in SQLite, `DO $$ ... $$` in PostgreSQL, `LOAD DATA` in MySQL-family and - BigQuery, `SHOW` not in SQLite/SQL Server/Oracle, no `SET`/`GRANT` in SQLite, ...); a parser - without a dialect still accepts all of them. -- MySQL / MariaDB: `:=` inside an expression assigns only to a user variable - (`SELECT @v := 1`); `f(a := 1)` and `arr[1:=2]` are errors. PostgreSQL - `f(a := 1)` parses as a named argument, written back as `f(a => 1)`. -- Transaction statements are checked against each dialect's grammar - (`BEGIN banana` is rejected); `<=>` is accepted only in MySQL, MariaDB and - PostgreSQL (pgvector), `==` only in SQLite and ClickHouse; an unbalanced - SQL Server `BEGIN ... END` block, or an `IF`/`WHILE` without a body, is an - error. -- PIVOT / UNPIVOT `IN` lists now require commas between entries and an alias - after `AS` in every PIVOT dialect. Where bare aliases are supported, - `IN (a b)` is one entry (`a AS b`); otherwise a missing comma is rejected. - `IN ('a' AS)` no longer silently drops the `AS`. +### Performance +- **Lower-case formatting** streams into a single builder instead of copying + and re-rendering subtrees: it went from 3.5–10x to 1.2–1.5x the cost of + upper-case output, with 5x to 500x fewer allocations. (#870, #886) +- **`ScanSQL` tautology check** no longer allocates per candidate (1 MiB of + `1=2` went from 50 MB to 1 MB). (#885) +- **Costs of the correctness work** (Apple M4, `-count=6`, against v1.16.0): + `Transpile` of a short MySQL `CREATE TABLE` to PostgreSQL takes about + 5.2 µs instead of 2.0 µs (52 instead of 32 allocations), mostly in the new + operator-retargeting and `||`-meaning passes that walk every statement. + The parse-stage baselines are about 6% slower with 8% fewer allocations and + about 24% more bytes per parse, from the new AST metadata; all stay within + the CI regression baselines. ## [1.16.0] - 2026-10-22 — Performance Audit, SQL Highlighting, Dialect-Aware LSP, Claude Code Plugin & ChatGPT App, Correctness Sweep diff --git a/CLAUDE.md b/CLAUDE.md index 29e8b24c3..590045925 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,7 +8,7 @@ GoSQLX is a **production-ready**, **race-free**, high-performance SQL parsing SD **Requirements**: Go 1.26+ (upgraded from 1.23 to fix stdlib vulnerabilities; `mark3labs/mcp-go` requires 1.23) -**Production Status**: ✅ Validated for production deployment (v1.6.0+, current: v1.16.0) +**Production Status**: ✅ Validated for production deployment (v1.6.0+, current: v1.17.0) - Thread-safe with zero race conditions (20,000+ concurrent operations tested) - 1.38M+ ops/sec sustained, 1.5M peak with memory-efficient object pooling - ~80-85% SQL-99 compliance (window functions, CTEs, set operations, MERGE, etc.) diff --git a/cmd/gosqlx/cmd/doc.go b/cmd/gosqlx/cmd/doc.go index 0d4a8c106..f0f4b22f4 100644 --- a/cmd/gosqlx/cmd/doc.go +++ b/cmd/gosqlx/cmd/doc.go @@ -367,7 +367,7 @@ // // Version information: // -// Version = "1.16.0" - Current CLI version +// Version = "1.17.0" - Current CLI version // // # Dependencies // diff --git a/cmd/gosqlx/cmd/root.go b/cmd/gosqlx/cmd/root.go index fd142440b..845975184 100644 --- a/cmd/gosqlx/cmd/root.go +++ b/cmd/gosqlx/cmd/root.go @@ -31,6 +31,14 @@ import ( // This version tracks feature releases and compatibility. // Format: MAJOR.MINOR.PATCH (Semantic Versioning 2.0.0) // +// Version 1.17.0 includes: +// - Real-world dialect corpus: 2,813 statements across 8 dialects checked for parse, format and round trip +// - Dialect-faithful formatting: Oracle / lines, U& identifiers, T-SQL GO comments, lower-case keywords +// - Transpiler fixes: SQL Server set-operation LIMIT to OFFSET/FETCH, nested operator rewrites +// - ScanSQL hardening against comment, quoting and dollar-quote bypasses; analyze and lint report its findings +// - LSP hardening: spec textDocumentSync key, malformed-message and didChange safety, -32601 for unknown methods +// - A GO line inside a SQL Server statement is now a parse error +// // Version 1.16.0 includes: // - SQL syntax highlighting (pkg/highlight) for API, CLI (gosqlx highlight), LSP, WASM and playground // - Dialect-aware LSP (diagnostics, formatting, completion, didChangeConfiguration) diff --git a/cmd/gosqlx/doc.go b/cmd/gosqlx/doc.go index a14a13b23..c97a35302 100644 --- a/cmd/gosqlx/doc.go +++ b/cmd/gosqlx/doc.go @@ -24,7 +24,7 @@ // // # Version // -// Current version: 1.16.0 +// Current version: 1.17.0 // // # Architecture // diff --git a/docs/CLI_GUIDE.md b/docs/CLI_GUIDE.md index 08103db71..9ffa2526d 100644 --- a/docs/CLI_GUIDE.md +++ b/docs/CLI_GUIDE.md @@ -15,9 +15,13 @@ The GoSQLX Command Line Interface (CLI) provides high-performance SQL parsing, v - [lint](#gosqlx-lint---style-and-quality-checking) - [lsp](#gosqlx-lsp---language-server-protocol) - [highlight](#gosqlx-highlight---syntax-highlighting) + - [transpile](#gosqlx-transpile---convert-sql-between-dialects) + - [stats](#gosqlx-stats---object-pool-counters) + - [watch](#gosqlx-watch---re-run-on-file-changes) - [config](#gosqlx-config---configuration-management) - [completion](#gosqlx-completion---shell-autocompletion) - [Global Flags](#global-flags) +- [Dialect Names and Aliases](#dialect-names-and-aliases) - [Configuration](#configuration) - [Input Methods](#input-methods) - [Output Formats](#output-formats) @@ -88,7 +92,7 @@ gosqlx config init ### `gosqlx validate` - Ultra-Fast SQL Validation -Validate SQL syntax with <10ms typical latency and 100+ files/second throughput. +Validate SQL syntax for one or more files, a SQL string or stdin. Measured parse and validation numbers are on the [benchmarks page](/benchmarks/). #### Syntax @@ -103,11 +107,13 @@ gosqlx validate [file...] [flags] | `--dialect` | string | postgresql | SQL dialect (generic, postgresql, mysql, mariadb, sqlserver, oracle, sqlite, snowflake, bigquery, redshift, clickhouse) | | `-r, --recursive` | bool | false | Recursively process directories | | `-p, --pattern` | string | `*.sql` | File pattern for recursive processing | -| `-q, --quiet` | bool | false | Quiet mode (exit code only) | +| `-q, --quiet` | bool | false | Quiet mode: print nothing, report the result through the exit code only | +| `--check` | bool | false | Alias for `--quiet` | | `-s, --stats` | bool | false | Show performance statistics | | `--strict` | bool | false | Enable strict validation mode | | `--output-format` | string | text | Output format (text, json, sarif) | | `--output-file` | string | stdout | Output file path | +| `--list-dialects` | bool | false | Print all valid SQL dialects and exit | #### Examples @@ -174,23 +180,26 @@ cat query.sql | gosqlx validate && echo "Valid SQL" | 0 | Valid SQL - no syntax errors | | 1 | Invalid SQL - syntax errors detected | +`validate --quiet` (or `--check`) still exits 1 on validation failure, without +printing the failure to stdout or stderr. + Input with no statements (empty, whitespace only, or only comments) is valid. A file that fails as a whole is checked again statement by statement, so scripts that use the T-SQL `GO` batch separator or the MySQL `DELIMITER` directive validate, and a file over the tokenizer's 1,000,000 token limit validates when every statement is under it. Results read from stdin are named -``. - -#### Performance Target +``; stdin with no bytes at all is reported as an error. `--quiet` prints nothing, not even errors. -- **Latency**: <10ms for typical queries (50-500 characters) -- **Throughput**: 100+ files/second in batch mode +```bash +printf 'SELECT 1\nGO\nSELECT 2\nGO\n' | gosqlx validate --dialect mssql +# ✅ : Valid SQL (exit code 0) +``` --- ### `gosqlx format` - High-Performance SQL Formatting -Format SQL queries with intelligent indentation and style - 100x faster than SQLFluff. +Format SQL queries with consistent indentation and keyword style. Comments, optimizer hints and a leading byte order mark are kept. #### Syntax @@ -214,6 +223,53 @@ gosqlx format [file...] [flags] Dialect clauses stored verbatim inside `JSON_TABLE` and `XMLTABLE` retain their source case, including their `COLUMNS` definitions, to preserve names and literals. +#### Comments, hints and keyword case + +`gosqlx format` keeps comments. +A comment between statements stays between them, a trailing comment stays after +its statement, and a comment inside a statement moves to the line above it. An +optimizer hint (`/*+ ... */`) directly after a statement's first keyword stays in +place. Input whose hints or MySQL `/*! ... */` comments cannot be placed faithfully +is refused with an error and left unchanged, and before `-i` rewrites a file the +command checks that the output holds exactly the input's comments. A leading byte +order mark is kept. + +```bash +printf -- '-- top users\nselect id, /* pk */ email from users where id = 1; -- one user\nselect 2\n' | gosqlx format +``` + +Output: + +```sql +-- top users +/* pk */ +SELECT id, email +FROM users +WHERE id = 1; -- one user + +SELECT 2 +``` + +`--no-uppercase` writes every keyword in lower case, including keyword operators +(`AND`, `IN`, `IS NULL`, `BETWEEN`, `CASE`, `DESC`), data types and built-in +function keywords. Identifiers, quoted names and literals keep their spelling: + +```bash +echo "SELECT a FROM t WHERE b IS NULL AND c IN (1,2) ORDER BY CAST(a AS INTEGER) DESC" | gosqlx format --no-uppercase +``` + +Output: + +```sql +select a +from t +where b is null and c in (1, 2) +order by cast(a as integer) desc +``` + +With `--dialect sqlserver`, the statement-count guard counts `GO` lines and +statements that end without `;`, and comments stay on their side of a `GO` line. + #### Examples ```bash @@ -229,7 +285,7 @@ gosqlx format -i query1.sql query2.sql # Custom indentation (4 spaces) gosqlx format --indent 4 query.sql -# Keep original keyword case +# Lower-case every keyword gosqlx format --no-uppercase query.sql # Compact format (minimal whitespace) @@ -250,8 +306,8 @@ gosqlx format -o formatted.sql query.sql # Format with custom line length gosqlx format --max-line 120 query.sql -# Format with lowercase keywords -gosqlx format --no-uppercase query.sql +# Format T-SQL scripts that use GO +gosqlx format --dialect mssql script.sql ``` #### Pipeline/Stdin Examples @@ -283,11 +339,6 @@ cat query.sql | gosqlx format | gosqlx validate | 0 | File is already formatted correctly | | 1 | File needs formatting | -#### Performance - -- **100x faster** than SQLFluff for equivalent operations -- Handles complex queries with CTEs, window functions, JOINs - --- ### `gosqlx parse` - AST Structure Inspection @@ -338,8 +389,14 @@ gosqlx parse -f table query.sql # Combine with other tools gosqlx parse -f json query.sql | jq '.Statements[0]' + +# Write the output to a file +gosqlx parse -f json -o ast.json query.sql ``` +With `--dialect sqlserver`, `GO` lines appear in the statement list as +`BatchSeparator` entries but are not counted in the `Statements` total. + #### Pipeline/Stdin Examples ```bash @@ -385,6 +442,10 @@ gosqlx analyze [file|query] [flags] | `--complexity` | bool | false | Focus on complexity metrics | | `--all` | bool | false | Comprehensive analysis (all features) | +`analyze --security` runs both the AST scanner (`Scan`) and the full-input text +scanner (`ScanSQL`). It merges their findings and removes duplicate reports +with the same pattern, severity and description. + #### Examples ```bash @@ -477,6 +538,12 @@ gosqlx lint [file...] [flags] | `--auto-fix` | bool | false | Automatically fix violations where possible | | `--max-length` | int | 100 | Maximum line length for L005 rule | | `--fail-on-warn` | bool | false | Exit with error code on warnings | +| `--security` | bool | false | Also run the SQL security scanner | + +For files that parse successfully, `lint --security` merges findings from the +AST scanner (`Scan`) and the full-input text scanner (`ScanSQL`), removing +duplicates with the same pattern, severity and description. Security findings +cause a non-zero exit status. #### Available Lint Rules @@ -551,7 +618,14 @@ gosqlx lint - > > SQL that does not tokenize or parse is reported as a syntax error and also exits 1; the style > rules that need no parse tree still report their violations. With `--auto-fix` the exit code -> reflects what is left after the fixes. +> reflects what is left after the fixes. An empty or comment-only file, and comment-only stdin, +> has no violations and exits 0. + +```bash +echo 'SELEC 1' | gosqlx lint +# Syntax error: Error E2002 at line 1, column 1: expected statement, got IDENTIFIER +# (exit code 1) +``` --- @@ -706,6 +780,154 @@ An inline query is detected by the same rules as the other commands (see [Input --- +### `gosqlx transpile` - Convert SQL Between Dialects + +Parse SQL in one dialect and write it for another, applying the rewrite rules +registered for that pair. The same engine backs `gosqlx.Transpile` in Go. + +#### Syntax + +```bash +gosqlx transpile [SQL] [flags] +``` + +Pass the SQL as one argument or pipe it on stdin. + +#### Flags + +| Flag | Type | Default | Description | +|------|------|---------|-------------| +| `--from` | string | `mysql` | Source dialect: `mysql`, `postgres`, `sqlite`, `sqlserver`, `oracle`, `snowflake`, `clickhouse`, `mariadb` | +| `--to` | string | `postgres` | Target dialect, same names as `--from` | +| `-o, --output` | string | stdout | Write the result to a file (global flag) | + +Both flags also accept `postgresql`, `pg`, `mssql`, `tsql`, `sql-server` and +`sqlite3`. `-f, --format` is not supported by this command. + +#### Examples + +```bash +# MySQL to PostgreSQL +gosqlx transpile --from mysql --to postgres "CREATE TABLE t (id INT AUTO_INCREMENT PRIMARY KEY)" +# CREATE TABLE t (id SERIAL PRIMARY KEY) + +# Read from stdin +echo "SELECT * FROM users WHERE name ILIKE '%alice%'" | gosqlx transpile --from postgres --to mysql + +# Write to a file +gosqlx transpile --from mysql --to postgres -o out.sql "SELECT 1" +``` + +Strings and quoted identifiers +are written with the target dialect's escape rules and delimiters, and +dialect-specific operators are mapped. `^` is exponentiation in PostgreSQL and +bitwise XOR in MySQL, MariaDB and SQL Server; PostgreSQL `@` and `|/` become +`ABS()` and `SQRT()`. PostgreSQL `||/` becomes `CBRT()` only for Snowflake and +ClickHouse; other targets, MySQL included, have no `CBRT()` and return an +error. Something the target cannot express +returns an error and exit code 1 instead of SQL with a different meaning: + +```bash +gosqlx transpile --from mysql --to postgres "SELECT \`Order Id\` FROM t WHERE n = 'it\\'s'" +# SELECT "Order Id" FROM t WHERE n = 'it''s' + +gosqlx transpile --from pg --to mysql "SELECT 2 ^ 3" +# SELECT POWER(2, 3) + +gosqlx transpile --from mysql --to postgres "SELECT a ^ b FROM t" +# SELECT a # b FROM t + +gosqlx transpile --from mysql --to snowflake "SELECT a ^ b FROM t" +# SELECT BITXOR(a, b) FROM t + +gosqlx transpile --from pg --to mysql "SELECT @ x, |/ y FROM t" +# SELECT ABS(x), SQRT(y) FROM t + +gosqlx transpile --from mysql --to oracle "SELECT a ^ b FROM t" +# Error: transpile: rewrite: the bitwise XOR operator ^ has no equivalent in oracle +``` + +A SQL Server script with `GO` separators returns an error for any other target +dialect, because batch scopes and repeat counts cannot be represented there. + +--- + +### `gosqlx stats` - Object Pool Counters + +Print the object pool counters of the current process: gets (pool +retrievals), puts (pool returns) and active (currently borrowed) for the +`tokenizer`, `parser` and `ast` pools. A fresh `gosqlx stats` process has done +no parsing, so its counters start at zero. + +#### Syntax + +```bash +gosqlx stats [flags] +``` + +#### Flags + +| Flag | Type | Default | Description | +|------|------|---------|-------------| +| `--json` | bool | false | Output as JSON | + +#### Examples + +```bash +gosqlx stats +# POOL GETS PUTS ACTIVE +# ---- ---- ---- ------ +# ast 0 0 0 +# parser 0 0 0 +# tokenizer 0 0 0 + +gosqlx stats --json +``` + +--- + +### `gosqlx watch` - Re-run on File Changes + +Watch `.sql` files, directories (recursively) or glob patterns. Every watched +file is processed once at start-up, then again whenever it is written or +created, until you press Ctrl+C. + +- `validate` mode (default) parses each file and prints a timestamped `Valid`, + `Invalid` or error line per file. +- `format` mode rewrites each file in place through the same checked path as + `gosqlx format -i`, printing `Formatted` or `No changes needed`. + +#### Syntax + +```bash +gosqlx watch [file|dir...] [flags] +``` + +#### Flags + +| Flag | Type | Default | Description | +|------|------|---------|-------------| +| `--mode` | string | validate | `validate` or `format`; any other value is an error | +| `--dialect` | string | postgresql | SQL dialect used to parse (and format) files | +| `--debounce` | int | 300 | Delay in milliseconds before a change event is handled | +| `--clear` | bool | false | Clear the screen when a change is handled | +| `--watch-verbose` | bool | false | List the watched files and print watcher errors | + +#### Examples + +```bash +# Watch the .sql files in the current directory +gosqlx watch *.sql + +# Re-format SQL Server files in place as they change +gosqlx watch --mode format --dialect sqlserver queries/ + +# Watch a schema file and a directory, with a longer debounce +gosqlx watch --debounce 500 schema.sql migrations/ +``` + +--- + ### `gosqlx config` - Configuration Management Manage GoSQLX configuration files for persistent settings. @@ -851,6 +1073,30 @@ Available for all commands: | `-h, --help` | bool | false | Help for any command | | `--version` | bool | false | Show version information | +`-o` also works for `parse` and `transpile`. +`-f`/`-o` map onto `validate`'s `--output-format`/`--output-file`, and commands +that cannot honour them reject them instead of ignoring them. + +--- + +## Dialect Names and Aliases + +Every `--dialect` flag takes a canonical dialect name (`gosqlx validate +--list-dialects` prints them). These aliases are accepted too: + +| Alias | Dialect | +|-------|---------| +| `postgres`, `pg` | `postgresql` | +| `mssql`, `tsql`, `sql-server` | `sqlserver` | +| `sqlite3` | `sqlite` | + +```bash +gosqlx validate --dialect pg "SELECT 1" +gosqlx format --dialect tsql script.sql +``` + +Unknown names are rejected with an error that lists the valid names. + --- ## Configuration @@ -1220,28 +1466,9 @@ Run with: `task sql:validate`, `task sql:format`, or `task sql:check` ## Performance -GoSQLX CLI delivers exceptional performance for all operations. - -### Benchmark Results - -| Operation | Throughput | Latency | Performance vs Competitors | -|-----------|------------|---------|----------------------------| -| **Validation** | 100+ files/sec | <10ms per query | 100-1000x faster | -| **Formatting** | - | <5ms per file | 100x faster than SQLFluff | -| **Analysis** | 1.38M+ ops/sec | <1μs per query | Production-ready | -| **Parsing** | 1.5M+ ops/sec | <1μs per query | Direct AST inspection | -| **Linting** | 50+ files/sec | <20ms per file | High-performance | - -### Core Library Performance - -- **1.38M+ operations/second** sustained throughput -- **1.5M peak** with memory-efficient object pooling -- **60-80% memory reduction** through object pooling -- **Zero-copy tokenization** for maximum efficiency -- **Concurrent processing** support with linear scaling -- **Race-free implementation** validated through comprehensive testing - -### Performance Tips +Measured numbers for parsing, formatting and validation, with the hardware and +method behind them, are published on the [benchmarks page](/benchmarks/). For +faster runs: 1. **Use batch processing** for multiple files with glob patterns 2. **Enable verbose output only when needed** (`-v` adds overhead) diff --git a/docs/FEATURE_TOUR.md b/docs/FEATURE_TOUR.md index bf4664c02..817d94355 100644 --- a/docs/FEATURE_TOUR.md +++ b/docs/FEATURE_TOUR.md @@ -1,6 +1,8 @@ # Feature Tour -This page walks through the feature examples from the Getting Started guide: PostgreSQL extensions, security scanning, and linting. Install GoSQLX first by following [Getting Started](GETTING_STARTED.md). +This page walks through short examples of GoSQLX features: PostgreSQL extensions, security scanning, linting, formatting that keeps comments, transforms, fingerprinting and transpilation. Install GoSQLX first by following [Getting Started](GETTING_STARTED.md). + +The formatting, transform, fingerprinting and transpilation examples show behaviour released in v1.17.0. ## Feature Examples @@ -137,6 +139,179 @@ func main() { } ``` +### Formatting That Keeps Comments + +`gosqlx.Format` keeps comments. A comment inside a statement moves to the line above it, and optimizer hints (`/*+ ... */`) after a statement's first keyword stay in place. Input whose hints cannot be put back faithfully returns an error wrapping `formatter.ErrUnplaceableComment` instead of losing them. See [Format SQL in Go](guides/format-sql-in-go.md). + +```go +package main + +import ( + "fmt" + "log" + + "github.com/ajitpratap0/GoSQLX/pkg/gosqlx" +) + +func main() { + sql := "-- active users\nselect id, /* primary key */ email from users where active = true; -- first query\nselect count(*) from orders" + + out, err := gosqlx.Format(sql, gosqlx.DefaultFormatOptions()) + if err != nil { + log.Fatal(err) + } + fmt.Println(out) +} +``` + +Output: + +```sql +-- active users +/* primary key */ +SELECT id, email +FROM users +WHERE active = true; -- first query +SELECT count(*) +FROM orders +``` + +### Transforms + +The `pkg/transform` package rewrites parsed statements with composable rules. `SetLimit` and `SetOffset` edit an existing SQL Server `TOP` or a `FETCH FIRST/NEXT` clause instead of adding `LIMIT` (a `PERCENT` or `WITH TIES` row limit returns an error), and `AddTableAlias` rewrites the qualifiers that used the table name. + +```go +package main + +import ( + "fmt" + "log" + + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" + "github.com/ajitpratap0/GoSQLX/pkg/transform" +) + +func main() { + // SetLimit edits an existing SQL Server TOP instead of adding LIMIT. + tree, err := transform.ParseSQLWithDialect("SELECT TOP 10 * FROM users", keywords.DialectSQLServer) + if err != nil { + log.Fatal(err) + } + if err := transform.Apply(tree.Statements[0], transform.SetLimit(5)); err != nil { + log.Fatal(err) + } + fmt.Println(transform.FormatSQLWithDialect(tree.Statements[0], keywords.DialectSQLServer)) + + // AddTableAlias also rewrites the qualifiers that use the table name. + tree, err = transform.ParseSQL("SELECT users.name FROM users WHERE users.id = 1") + if err != nil { + log.Fatal(err) + } + if err := transform.Apply(tree.Statements[0], transform.AddTableAlias("users", "u")); err != nil { + log.Fatal(err) + } + fmt.Println(transform.FormatSQL(tree.Statements[0])) +} +``` + +Output: + +```text +SELECT TOP 5 * FROM users +SELECT u.name FROM users u WHERE u.id = 1 +``` + +### Fingerprinting + +`fingerprint.Normalize` replaces literals with `?`, and `fingerprint.Fingerprint` hashes the normalized text, so queries that differ only in their values share a fingerprint. Identifier quotes are kept: `SELECT "A" FROM "T"` is no longer folded to `SELECT A FROM T`, so fingerprints of queries with quoted identifiers differ from those of earlier releases. Re-compute stored fingerprints after upgrading. + +```go +package main + +import ( + "fmt" + "log" + + "github.com/ajitpratap0/GoSQLX/pkg/fingerprint" +) + +func main() { + a, err := fingerprint.Fingerprint(`SELECT "A" FROM "T" WHERE id = 5`) + if err != nil { + log.Fatal(err) + } + b, err := fingerprint.Fingerprint(`SELECT "A" FROM "T" WHERE id = 42`) + if err != nil { + log.Fatal(err) + } + fmt.Println(a == b) // same query shape, different literal + + normalized, err := fingerprint.Normalize(`SELECT "A" FROM "T" WHERE id = 5`) + if err != nil { + log.Fatal(err) + } + fmt.Println(normalized) +} +``` + +Output: + +```text +true +SELECT "A" FROM "T" WHERE id = ? +``` + +### Transpilation + +`gosqlx.Transpile` parses SQL in one dialect and writes it for another. Strings and quoted identifiers use the target's escape rules and delimiters, and operators are mapped by meaning: `^` is exponentiation in PostgreSQL but bitwise XOR in MySQL, MariaDB and SQL Server, and PostgreSQL `@` and `|/` become `ABS()` and `SQRT()`. The same engine runs behind `gosqlx transpile` in the [CLI](CLI_GUIDE.md#gosqlx-transpile---convert-sql-between-dialects). + +```go +package main + +import ( + "fmt" + "log" + + "github.com/ajitpratap0/GoSQLX/pkg/gosqlx" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" +) + +func main() { + examples := []struct { + sql string + from, to keywords.SQLDialect + }{ + {"SELECT `Order Id` FROM t WHERE n = 'it\\'s'", keywords.DialectMySQL, keywords.DialectPostgreSQL}, + {"SELECT 2 ^ 3", keywords.DialectPostgreSQL, keywords.DialectMySQL}, + {"SELECT a ^ b FROM t", keywords.DialectMySQL, keywords.DialectPostgreSQL}, + {"SELECT a ^ b FROM t", keywords.DialectMySQL, keywords.DialectClickHouse}, + {"SELECT @ x, |/ y FROM t", keywords.DialectPostgreSQL, keywords.DialectMySQL}, + } + for _, e := range examples { + out, err := gosqlx.Transpile(e.sql, e.from, e.to) + if err != nil { + log.Fatal(err) + } + fmt.Println(out) + } + + // SQL the target cannot express is an error, not a silent change. + _, err := gosqlx.Transpile("SELECT a ^ b FROM t", keywords.DialectMySQL, keywords.DialectOracle) + fmt.Println(err) +} +``` + +Output: + +```text +SELECT "Order Id" FROM t WHERE n = 'it''s' +SELECT POWER(2, 3) +SELECT a # b FROM t +SELECT bitXor(a, b) FROM t +SELECT ABS(x), SQRT(y) FROM t +rewrite: the bitwise XOR operator ^ has no equivalent in oracle +``` + ### More Quick Examples ```go diff --git a/docs/LSP_GUIDE.md b/docs/LSP_GUIDE.md index 8f112ede7..dd8e52951 100644 --- a/docs/LSP_GUIDE.md +++ b/docs/LSP_GUIDE.md @@ -170,12 +170,21 @@ Markdown documentation for 70+ SQL keywords including: } ``` +Formatting keeps comments and optimizer hints, in the configured dialect as +well as the default one: a comment inside a statement moves to the line above +it, and a `/*+ ... */` hint after a statement's first keyword stays in place. +When the AST formatter cannot place a document's hints, the server falls back to +the same basic line-by-line formatting it uses for SQL that does not parse. + ### 6. Document Symbols Lists all SQL statements with categorization: - Method (DML: SELECT, INSERT, UPDATE, DELETE) - Struct (DDL: CREATE, DROP, ALTER) +SQL Server `GO` batch separators do not appear in the outline, and a statement +before a `GO` line ends at that line instead of running to the end of the file. + ### 7. Signature Help **Trigger Characters**: `(` and `,` @@ -448,7 +457,7 @@ Using eglot: ```json { "capabilities": { - "textDocumentSyncOptions": { + "textDocumentSync": { "openClose": true, "change": 2, "save": {"includeText": true} @@ -469,6 +478,18 @@ Using eglot: } ``` +### Protocol Error Handling + +Unknown request methods return JSON-RPC error `-32601` (`MethodNotFound`). +The server logs malformed `Content-Length` headers and invalid JSON, then +continues processing subsequent well-framed messages. Invalid input does not +require restarting the server. + +Incremental `textDocument/didChange` ranges use UTF-16 positions. Negative +positions are rejected; positions beyond a line or the document are clamped +to its end, and reversed ranges have their endpoints swapped. The server +continues serving requests after these changes. + ### Server Limits | Limit | Value | diff --git a/docs/MCP_GUIDE.md b/docs/MCP_GUIDE.md index 43fe15a40..64606c259 100644 --- a/docs/MCP_GUIDE.md +++ b/docs/MCP_GUIDE.md @@ -392,21 +392,28 @@ curl -s -X POST http://127.0.0.1:8080/mcp \ ### parse\_sql -**Description**: Parse SQL and return an AST summary: statement count and types. +**Description**: Parse SQL and return an AST summary: statement count and types, optionally for a specific dialect. #### Parameters | Parameter | Type | Required | Description | |-----------|------|----------|-------------| | `sql` | string | yes | The SQL string to parse | +| `dialect` | string | no | Defaults to `generic`; one of `generic`, `postgresql`, `mysql`, `mariadb`, `sqlserver`, `oracle`, `sqlite`, `snowflake`, `bigquery`, `redshift`, `clickhouse` | #### Response | Field | Type | Description | |-------|------|-------------| -| `statement_count` | integer | Number of statements parsed | +| `statement_count` | integer | Number of SQL statements parsed, excluding `GO` batch separators | | `statement_types` | array of string | Go type names of each parsed statement (e.g. `*ast.SelectStatement`) | +SQL Server `GO` batch separators +(`*ast.BatchSeparator`) are left out of `statement_count` and `statement_types`. +`parse_sql` defaults to the generic dialect. Set `"dialect": "sqlserver"` to +parse T-SQL scripts containing `GO`; repeat counts on `GO` do not increase +`statement_count`. + #### Example ```bash @@ -461,6 +468,12 @@ curl -s -X POST http://127.0.0.1:8080/mcp \ } ``` +Since v1.17.0 (issue #849), an unqualified name +in `ORDER BY` or `HAVING` that matches a SELECT-list alias is treated as the +alias, so `extract_metadata` and the metadata part of `analyze_sql` no longer +report SELECT aliases as columns. For `SELECT count(*) AS orders FROM t GROUP BY +customer_id ORDER BY orders`, `columns` holds only `customer_id`. + --- ### security\_scan diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md index dcbcb9512..596d9fd36 100644 --- a/docs/MIGRATION.md +++ b/docs/MIGRATION.md @@ -1,5 +1,51 @@ # Migration Guide +## v1.16.0 → v1.17.0 (2026-10-29) + +**Source-compatible upgrade**: exported signatures are unchanged apart from additions, so code that compiled against v1.16.0 compiles against v1.17.0. Parse acceptance, rendered SQL, security findings, transpiler errors and some CLI and LSP behaviour change. Work through the checklist below; the [CHANGELOG](../CHANGELOG.md) has the full list. + +### Why upgrade + +- **Real-world dialect corpus**: 2,813 production-style statements across all 8 dialects now parse, format, re-parse identically and keep every word (1,327 failures before). +- **Faithful formatting**: `gosqlx.Format`, `gosqlx format`, LSP and C-binding formatting keep comments, optimizer hints, SQL Server `GO` lines, Oracle `/` lines and PostgreSQL `U&"..."` spellings. +- **Security**: `ScanSQL` bypasses through inline comments, quoted dollar tags and input truncation are closed, `Scanner.Scan` walks the whole AST, and the CLI merges both scanners. +- **Robustness**: the LSP server, C exports and transpiler no longer crash on malformed input, very long operator chains or panics, and metadata extraction is linear on nested CTEs. + +### New APIs to adopt + +- `gosqlx.FormatWithDialect`, `gosqlx.WithComments()`, and the `*Checked` format variants (`gosqlx.FormatASTChecked`, `gosqlx.FormatTreeChecked`, `Tree.FormatChecked`, `formatter.FormatASTChecked` / `FormatStatementChecked` / `FormatExpressionChecked`, `transform.FormatSQLChecked` / `FormatSQLWithDialectChecked`), which return an error for nodes the target dialect cannot render instead of emitting changed SQL. +- `formatter.JoinWithComments`, `formatter.JoinASTWithComments`, `formatter.CommentLayout` and `formatter.ErrUnplaceableComment`. +- `linter.FileResult.TokenizeError`, `ParseError` and `SyntaxError()`. +- `security.MaxScanInputSize` and `security.PatternInputTooLarge`; `dialect.StringEscape`; `tokenizer.IsKeyword`. +- AST: `ast.BatchSeparator` (SQL Server `GO [n]`), `AST.ExecuteSlash` / `ExecuteSlashPositions`, `AST.StatementEnds`, `AST.UnicodeIdentifiers`, identifier `NameQuote` / `QuoteStyle` fields, `LiteralValue.DialectExtras()`, `SelectStatement.Extras` / `DMLTargetFields` and the other fields listed in the CHANGELOG. + +### Behavior changes to act on + +- [ ] **SQL Server `GO`**: a line holding only `GO [n]` inside a statement is a parse error, and `GO 0` / `GO 1.5` report "invalid GO repeat count". `AST.Statements` now contains `*ast.BatchSeparator` nodes: skip them when counting or indexing statements. Statement counts in `gosqlx analyze` / `parse`, MCP `parse_sql` and the C binding exclude GO. +- [ ] **SQL Server `top`**: unquoted `top` is rejected as a table, target or alias name. Write `[top]`. +- [ ] **Numeric lexing** is per dialect: `1_000`, `0b101`, `1x` and similar forms are one literal or an error instead of a number plus an alias. See the table in the CHANGELOG. +- [ ] **MySQL / MariaDB**: `#` starts a line comment; `:=` assigns only to user variables. +- [ ] **Statement boundaries**: outside SQL Server, a utility statement (`SET`, `GRANT`, `COPY`, `VACUUM`, ...) followed by another statement without `;` is an error. Utility statements are accepted only in dialects that have them; a parser without a dialect still accepts all of them. +- [ ] **`gosqlx.Lint` / `LintWithDialect`** return an error wrapping `ErrTokenize` or `ErrSyntax` for invalid SQL. Check `err` instead of treating an empty result as clean. +- [ ] **Formatting** keeps comments, hints, Oracle `/` lines and `U&"..."` spellings; input whose hints cannot be placed returns `formatter.ErrUnplaceableComment`. Regenerate golden files. +- [ ] **Fingerprints** keep identifier quoting. Recompute stored fingerprints. +- [ ] **`ScanSQL`** reports more findings (comment-obfuscated keywords, tautology checks over the whole input) and always adds a CRITICAL `INPUT_TOO_LARGE` finding above 10 MiB. `a.id = b.id` is no longer reported as a tautology. +- [ ] **Transpile to SQL Server**: LIMIT / OFFSET on a UNION, INTERSECT or EXCEPT becomes `ORDER BY ... OFFSET ... FETCH`; without ORDER BY it returns an error. Unrepresentable literals and operators, `TOP PERCENT` / `WITH TIES` where the target has no equivalent, and rewrites nested more than 512 levels deep return errors. +- [ ] **LSP**: capabilities use the standard `textDocumentSync` key, so custom clients that read `textDocumentSyncOptions` must switch; unknown methods return `-32601`; `didChange` columns are UTF-16 code units. +- [ ] **CLI**: `gosqlx lint` exits 1 on syntax errors; `analyze --security` and `lint` include `ScanSQL` findings; empty and comment-only input is valid for `validate`, `lint` and `format`; `postgres`, `pg`, `mssql`, `tsql` and `sqlite3` work for every `--dialect`. +- [ ] **C binding**: the ABI is unchanged. A recovered panic returns a JSON `"internal error"` result, and `gosqlx_version()` returns NULL in that case, so handle NULL. + +### Companion release versions + +| Component | v1.16.0 | v1.17.0 | +|-----------|---------|---------| +| Library (`github.com/ajitpratap0/GoSQLX`) | 1.16.0 | **1.17.0** | +| CLI, MCP server and LSP `serverInfo.version` | 1.16.0 | **1.17.0** | +| VS Code extension and Claude Code plugin | 1.16.0 | **1.17.0** | +| C binding (`libgosqlx`) | 1.16.0 exports | **1.17.0**: same exports, panic-safe | +| Python bindings (`pygosqlx`) | 1.16.0 | **1.17.0** | +| Integrations (`integrations/gorm`, `integrations/opentelemetry`) | 1.16.0 | updated to **1.17.0** after the tag | + ## v1.15.0 → v1.16.0 (2026-10-22) **Source-compatible upgrade**: exported signatures are unchanged apart from additions, so code that compiled against v1.15.0 compiles against v1.16.0. Runtime behaviour does change in several places (token positions, rendered SQL, parse acceptance, CLI refusals, MCP HTTP limits). Read the checklist below before upgrading. The [CHANGELOG](../CHANGELOG.md) has the full list. diff --git a/docs/SQL_COMPATIBILITY.md b/docs/SQL_COMPATIBILITY.md index 8e37a04bd..d3d2fbb48 100644 --- a/docs/SQL_COMPATIBILITY.md +++ b/docs/SQL_COMPATIBILITY.md @@ -4,9 +4,11 @@ This matrix documents the comprehensive SQL feature support in GoSQLX across different SQL dialects and standards. The testing was conducted using the comprehensive integration test suite with 700+ test cases covering real-world SQL patterns. -### Real-world dialect audit (October 2026, unreleased) +### Real-world dialect audit (October 2026) -The real-world corpus contains 2,788 statements. Every statement passes +The real-world corpus contains 2,813 production-style statements: SQL written +by hand to imitate what ORMs, migration tools and applications send, not SQL +collected from production systems. Every statement passes split → dialect-aware parse → format → reparse → format idempotency, plus a check for dropped SQL words. `testdata/realworld/known_failures.txt` lists no failures. @@ -16,16 +18,32 @@ retain their text rather than receiving a complete semantic AST. | Dialect | Passing statements | Known failures | Pass rate | | --- | ---: | ---: | ---: | -| PostgreSQL | 300 / 300 | 0 | 100% | -| MySQL | 405 / 405 | 0 | 100% | -| MariaDB | 374 / 374 | 0 | 100% | -| SQL Server | 276 / 276 | 0 | 100% | +| PostgreSQL | 305 / 305 | 0 | 100% | +| MySQL | 414 / 414 | 0 | 100% | +| MariaDB | 380 / 380 | 0 | 100% | +| SQL Server | 277 / 277 | 0 | 100% | | Oracle | 368 / 368 | 0 | 100% | | SQLite | 348 / 348 | 0 | 100% | -| Snowflake | 325 / 325 | 0 | 100% | -| ClickHouse | 392 / 392 | 0 | 100% | +| Snowflake | 327 / 327 | 0 | 100% | +| ClickHouse | 394 / 394 | 0 | 100% | -Reproduce the ratchet and per-dialect summaries from the repository root: +The table totals 2,813 statements and was checked with `scripts/dialect-audit` +at commit `fda9a5ef`. Reproduce one dialect's row from the repository root +(replace `` with a dialect string such as `mysql`); the last line of output +reads, for example: + +```text +dialect=mysql total=405 passed=405 failed=0 stages=map[ok:405] +``` + +The command is: + +```sh +go run ./scripts/dialect-audit -dialect testdata/realworld//*.sql +``` + +The same command audits your own SQL files. To reproduce the ratchet and every +per-dialect summary at once: ```sh go test ./pkg/gosqlx -run '^TestRealWorldCorpus$' -count=1 @@ -34,6 +52,12 @@ for dir in testdata/realworld/*/; do done ``` +`TestRealWorldCorpus` is a ratchet against `known_failures.txt`: a statement +that fails and is not listed fails the test, and so does a listed statement +that now passes, so the list can only shrink. Regenerate it with +`go test ./pkg/gosqlx -run TestRealWorldCorpus -update-known-failures`. +`BenchmarkRealWorldParse` and `BenchmarkRealWorldParseFormat` measure the corpus. + SQL Server batch boundaries also have whole-script round-trip tests: the AST retains each `GO` as a `BatchSeparator`, including its optional repeat count, and both library and CLI rendering keep it on a separate line. The splitter @@ -470,6 +494,35 @@ MariaDB inherits all MySQL features (SHOW, DESCRIBE, REPLACE INTO, ON DUPLICATE | **WITHOUT ROWID** | ✅ Full | ✅ Full | 85% | CREATE TABLE option fully supported | | **Simplified syntax** | ✅ Full | ✅ Full | 85% | SQLite variations | +### Statement syntax + +These forms are available in v1.17.0. Each example was parsed with +`gosqlx.ParseWithDialect` and formatted with `gosqlx.FormatWithDialect` in the +named dialect. + +| Syntax | Dialects | GoSQLX Parser | Notes | +|--------|----------|---------------|-------| +| **Multi-table UPDATE** | MySQL, MariaDB | ✅ Full | `UPDATE a JOIN b ON a.id = b.id SET a.x = b.x`; joins and extra targets live in `UpdateStatement.Extras` | +| **Multi-table DELETE** | MySQL, MariaDB | ✅ Full | `DELETE a, b FROM a JOIN b ON a.id = b.id WHERE ...`; `DeleteStatement.Extras` | +| **UPDATE / DELETE TOP** | SQL Server | ✅ Full | `UPDATE TOP (10) t SET ...`, `DELETE TOP (5) FROM t ...` | +| **OUTPUT clause** | SQL Server | ✅ Full | `UPDATE ... OUTPUT inserted.id WHERE ...`, `DELETE ... OUTPUT deleted.id` | +| **GO [n] batch separator** | SQL Server | ✅ Full | Kept as `ast.BatchSeparator` with its optional `RepeatCount`; repeated batches are not expanded. `SQL()` and `FormatAST` keep GO on its own line. Statement counts reported by the CLI, the MCP server and the C binding exclude GO, but `AST.Statements` contains a `BatchSeparator` node for each GO. A `GO` line inside a statement is an error. Transpiling a GO script to another dialect returns an error | +| **RETURNING ... INTO** | Oracle | ✅ Full | `UPDATE t SET x = 1 WHERE id = 2 RETURNING x INTO :v` | +| **ARRAY JOIN / LIMIT BY** | ClickHouse | ✅ Full | `SELECT a FROM t ARRAY JOIN arr AS x LIMIT 1 BY a`. Parsed since earlier releases; AST walkers (`ast.Inspect`, `Walk`, security scanner, linter, table extraction) now reach `ARRAY JOIN` | +| **MERGE with a VALUES source** | PostgreSQL, SQL Server | ✅ Full | `MERGE INTO t USING (VALUES (1, 'a')) AS s (id, val) ON ...`; the source's column alias list is kept when rendering | +| **CREATE TABLE ... CLONE ... AT** | Snowflake | ✅ Full | `CREATE TABLE t2 CLONE t1 AT (TIMESTAMP => ...)`; walkers now reach the clone clause | +| **CREATE / ALTER / DROP SEQUENCE** | PostgreSQL, Oracle, Snowflake | ⚠️ Partial | Parsed and kept as statement text (`UtilityStatement`); `AST.SQL()` no longer drops them. MariaDB has a full `CreateSequenceStatement` (see MariaDB-Specific Features) | +| **EXPLAIN** | PostgreSQL, MySQL, SQLite | ✅ Full | `ExplainStatement` now retains the explained statement as an AST (`EXPLAIN ANALYZE ...`, SQLite `EXPLAIN QUERY PLAN ...`) | +| **SQL\*Plus `/` line** | Oracle | ✅ Full | A line holding only `/` ends a statement in the parser and in `SplitStatements`; `/` with other SQL on the line is still division. Known gap: formatting drops the `/` lines | +| **`<=>` operator** | MySQL, MariaDB, PostgreSQL | ✅ Full | Null-safe equality (MySQL family) and pgvector distance (PostgreSQL). Accepted only in these dialects and rejected in the others; v1.16.0 accepted `<=>` in no dialect | +| **`==` operator** | SQLite, ClickHouse | ✅ Full | Accepted only in these dialects and rejected in the others; v1.16.0 accepted `==` in no dialect | + +Related grammar checks: outside SQL Server, a utility statement +(`SET`, `USE`, `GRANT`, `COPY`, `CALL`, ...) followed on the next line by +another statement without `;` is an error (E2002); transaction statements are +checked against each dialect's grammar (`BEGIN banana` is rejected); an +unbalanced SQL Server `BEGIN ... END` block is an error. + ## SQL Standards Compliance ### SQL-92 (Entry Level) @@ -752,7 +805,7 @@ gosqlx format --dialect mysql query.sql - 30+ ClickHouse-specific keywords: TTL, CODEC, FORMAT, SETTINGS, DISTRIBUTED, etc. #### BigQuery -- PIVOT / UNPIVOT (unreleased, #854): aggregate aliases and multiple aggregates (`PIVOT (SUM(s) AS total, COUNT(*) AS n FOR q IN ('Q1' AS q1, 'Q2' q2))`), `UNPIVOT INCLUDE NULLS` / `EXCLUDE NULLS`, row-value aliases with or without `AS` (string or INT64), and multi-column UNPIVOT (`UNPIVOT ((a, b) FOR k IN ((c1, c2) AS 'x', (c3, c4) AS 'y'))`) +- PIVOT / UNPIVOT (#854): aggregate aliases and multiple aggregates (`PIVOT (SUM(s) AS total, COUNT(*) AS n FOR q IN ('Q1' AS q1, 'Q2' q2))`), `UNPIVOT INCLUDE NULLS` / `EXCLUDE NULLS`, row-value aliases with or without `AS` (string or INT64), and multi-column UNPIVOT (`UNPIVOT ((a, b) FOR k IN ((c1, c2) AS 'x', (c3, c4) AS 'y'))`) - PIVOT `IN` values must be literals or names; constant expressions such as `IN (1 + 1)` or `IN (DATE '2024-01-01')` are not parsed - QUALIFY, `UNNEST` in FROM and `(struct).field` access are capability-gated; BigQuery has no dedicated keyword set - Quoted PIVOT aggregate aliases and UNPIVOT value-column names retain their source delimiters, including BigQuery backticks and Oracle double quotes diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index a06b537ef..3877a02d4 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -411,6 +411,36 @@ func AutoFixFile(filename string) error { } ``` +#### Issue: Lint Returns an Error for Invalid SQL + +**Symptom:** `gosqlx.Lint` or `gosqlx.LintWithDialect` returns a non-nil error +together with a result, or `gosqlx lint` exits 1 although no style rule fired. + +**Cause:** SQL that does not +tokenize or parse is reported as an error instead of an empty, clean-looking +result. The error wraps `gosqlx.ErrTokenize` or `gosqlx.ErrSyntax`, and the +returned `*LintResult` still holds the violations of the text and token rules. +The CLI prints the syntax error and exits 1. + +**Solution:** Check the error before treating an empty `Violations` list as clean: + +```go +res, err := gosqlx.Lint(sql) +switch { +case errors.Is(err, gosqlx.ErrSyntax), errors.Is(err, gosqlx.ErrTokenize): + return fmt.Errorf("invalid SQL: %w", err) // res may still list text-rule violations +case err != nil: + return err +} +for _, v := range res.Violations { + fmt.Println(v.Rule, v.Message) +} +``` + +When you use `pkg/linter` directly, `linter.FileResult` carries the same +information in `TokenizeError` and `ParseError`; `SyntaxError()` returns +whichever is set. + #### Issue: Rules Not Detecting Violations **Symptom:** Expected violations not reported by linter @@ -905,6 +935,15 @@ func ParseLargeScript(script string) error { it statement by statement, so a file past the token limit validates as long as each statement is under it. +`SplitStatements` keeps compound +statements whole: a `BEGIN ... END` body of a trigger, procedure or function, +`CASE ... END`, T-SQL `BEGIN TRY` and Oracle `DECLARE` sections are not cut at an +inner `;`. Client batch separators are honoured and removed from the output: a +line holding only `GO [n]` for `sqlserver`, `DELIMITER` for `mysql` and +`mariadb`, and a line holding only `/` for `oracle`. Pass the script's dialect so +that these rules and the dialect's string escapes apply. If a script's blocks +do not balance, the splitter falls back to plain `;` splitting. + **Refactor Complex Query:** ```sql -- Instead of deep nesting: diff --git a/docs/USAGE_GUIDE.md b/docs/USAGE_GUIDE.md index ca14ef76b..67eb72520 100644 --- a/docs/USAGE_GUIDE.md +++ b/docs/USAGE_GUIDE.md @@ -200,6 +200,64 @@ func BatchProcess(queries []string) ([][]models.TokenWithSpan, error) { } ``` +### Splitting SQL Scripts + +`gosqlx.SplitStatements(sql, dialect)` splits a script into statements so you can +parse, validate or execute them one at a time. It ignores semicolons inside +strings, quoted identifiers and comments, and returns segments without the +terminating `;`; skip the empty ones. + +It also keeps compound +statements whole (`BEGIN ... END` bodies of triggers, procedures and functions, +`CASE ... END`, T-SQL `BEGIN TRY`, PostgreSQL `BEGIN ATOMIC`, Oracle `DECLARE` +sections) and honours client batch separators, which it removes from the +output: a line holding only `GO [n]` for `sqlserver`, `DELIMITER` for `mysql` and +`mariadb`, and a line holding only `/` for `oracle`. Backslash escapes in strings +follow the dialect you pass. + +```go +package main + +import ( + "fmt" + "log" + "strings" + + "github.com/ajitpratap0/GoSQLX/pkg/gosqlx" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" +) + +func main() { + script := `CREATE TABLE audit (n INT); +DELIMITER // +CREATE TRIGGER trg AFTER INSERT ON t FOR EACH ROW +BEGIN + UPDATE audit SET n = n + 1; +END// +DELIMITER ; +SELECT n FROM audit;` + + for i, stmt := range gosqlx.SplitStatements(script, "mysql") { + stmt = strings.TrimSpace(stmt) + if stmt == "" { + continue + } + if _, err := gosqlx.ParseWithDialect(stmt, keywords.DialectMySQL); err != nil { + log.Fatalf("statement %d: %v", i+1, err) + } + fmt.Printf("%d: %s\n", i+1, strings.SplitN(stmt, "\n", 2)[0]) + } +} +``` + +Output: + +```text +1: CREATE TABLE audit (n INT) +2: CREATE TRIGGER trg AFTER INSERT ON t FOR EACH ROW +3: SELECT n FROM audit +``` + ### Concurrent Processing Handle multiple queries concurrently: @@ -987,6 +1045,23 @@ func LintSQL(sql string) error { } ``` +`gosqlx.Lint` and +`gosqlx.LintWithDialect` return an error for SQL that does not tokenize or parse. +The error wraps `gosqlx.ErrTokenize` or `gosqlx.ErrSyntax`, and the result is +still returned with the violations of the text and token rules, so the function +above reports invalid SQL instead of calling it clean. To keep those violations, +check for the two errors before returning: + +```go +result, err := gosqlx.Lint("SELEC 1") +if errors.Is(err, gosqlx.ErrSyntax) || errors.Is(err, gosqlx.ErrTokenize) { + fmt.Println("invalid SQL:", err) + // result is not nil here and may still hold text-rule violations +} +``` + +`gosqlx lint` on the command line exits 1 for such input. + ### Rule Categories | Category | Rules | Examples | @@ -1319,7 +1394,8 @@ gosqlx validate query1.sql query2.sql query3.sql ### Format Command -Format SQL with intelligent indentation: +Format SQL with consistent indentation. The formatter keeps comments and optimizer +hints, and `--no-uppercase` writes every keyword in lower case; see the [CLI Guide](CLI_GUIDE.md#comments-hints-and-keyword-case). ```bash # Format and print to stdout diff --git a/docs/api/ast.md b/docs/api/ast.md index fc0fd6c98..5d8ac6230 100644 --- a/docs/api/ast.md +++ b/docs/api/ast.md @@ -38,30 +38,65 @@ type Expression interface { } ``` +## AST + +`AST` is the root container returned by the parser. Return pooled trees with `ast.ReleaseAST` when finished. + +```go +type AST struct { + UnicodeIdentifiers map[int][]UnicodeIdentifierSpelling + Statements []Statement + Comments []models.Comment + StatementStarts []models.Location + StatementEnds []models.Location + ExecuteSlash []bool + ExecuteSlashPositions []models.Location +} +``` + +Since v1.17.0, the container carries source metadata for script formatting: + +| Field | Contents and producer | +|---|---| +| `StatementStarts` | The parser records the first token location for each entry of `Statements`, including the `GO` keyword of a `BatchSeparator`. Locations are zero when parsing tokens without positions. | +| `StatementEnds` | The parser records each statement's last token end, including an optional semicolon and excluding Oracle SQL*Plus execute lines. | +| `ExecuteSlash` | The Oracle parser marks statements followed by a SQL*Plus `/` execute line. When present, this slice is parallel to `Statements`; execute lines do not add statements. | +| `ExecuteSlashPositions` | The Oracle parser records the execute line locations, parallel to `ExecuteSlash`; nil when no execute commands occur. | +| `UnicodeIdentifiers` | The parser records original PostgreSQL `U&` identifier spellings by statement index, plus ordinary name occurrences needed to distinguish equal decoded names. Nil unless Unicode identifiers occur. Semantic name fields on nodes remain decoded. | + +Treat starts and ends as absent unless their lengths equal `len(tree.Statements)`; hand-built trees may omit them. `UnicodeIdentifierSpelling` has `Name string`, `Spelling string` (empty for ordinary names) and `Quoted bool` (whether rendering retains double quotes). + +`Statements` also contains `*ast.BatchSeparator` for T-SQL `GO`. Skip these nodes when counting or processing executable SQL statements; see [BatchSeparator](#batchseparator). + ## DML Statement Types ### `SelectStatement` ```go type SelectStatement struct { - With *WithClause // CTE support - Distinct bool // DISTINCT keyword - DistinctOn []Expression // PostgreSQL DISTINCT ON - Columns []Expression - From []TableReference - Joins []JoinClause - Where Expression - GroupBy []Expression // Supports ROLLUP, CUBE, GROUPING SETS - Having Expression - Windows []WindowSpec // Window function specifications - OrderBy []OrderByExpression // Supports NULLS FIRST/LAST - Limit *int - Offset *int - FetchClause *FetchClause // SQL-99 FETCH FIRST support + With *WithClause // CTE support + Distinct bool // DISTINCT keyword + DistinctOnColumns []Expression // PostgreSQL DISTINCT ON + Top *TopClause // SQL Server TOP N [PERCENT] + Columns []Expression + From []TableReference + Joins []JoinClause + Where Expression + GroupBy []Expression // Supports ROLLUP, CUBE, GROUPING SETS + Having Expression + Windows []WindowSpec // Window function specifications + OrderBy []OrderByExpression // Supports NULLS FIRST/LAST + Limit *int + Offset *int + Fetch *FetchClause // SQL-99 FETCH FIRST/NEXT + Extras *SelectExtras // Rarer dialect clauses; nil when absent + // ... dialect clauses such as ArrayJoin, Sample, Qualify, For, Settings } ``` **PostgreSQL-Specific Features:** -- `DistinctOn` - SELECT DISTINCT ON (column1, column2) +- `DistinctOnColumns` - SELECT DISTINCT ON (column1, column2) + +`Extras` is described under [Dialect extension fields](#dialect-extension-fields). **Example:** ```go @@ -69,7 +104,7 @@ type SelectStatement struct { sql := "SELECT DISTINCT ON (dept_id) dept_id, name, salary FROM employees ORDER BY dept_id, salary DESC" ast, _ := gosqlx.Parse(sql) if stmt, ok := ast.Statements[0].(*ast.SelectStatement); ok { - fmt.Printf("DISTINCT ON columns: %d\n", len(stmt.DistinctOn)) + fmt.Printf("DISTINCT ON columns: %d\n", len(stmt.DistinctOnColumns)) } ``` @@ -259,36 +294,80 @@ type DropStatement struct { } ``` -## CTE and Set Operations +## Utility Statements + +### `ExplainStatement` +`EXPLAIN`, `DESCRIBE` or `DESC` applied to a statement, such as `EXPLAIN ANALYZE SELECT ...`, `EXPLAIN (FORMAT JSON) ...` or SQLite `EXPLAIN QUERY PLAN ...`. `DESCRIBE` of a table name is still a `DescribeStatement`. -### `WithClause` ```go -type WithClause struct { - Recursive bool - Materialized *bool // MATERIALIZED/NOT MATERIALIZED hint - CTEs []CommonTableExpr +type ExplainStatement struct { + Keyword string // EXPLAIN, DESCRIBE or DESC, upper-cased + Options string // text between the keyword and the statement, e.g. "ANALYZE VERBOSE" + Statement Statement // the explained statement } ``` -**Enhancements:** -- `Materialized` - PostgreSQL optimization hints (MATERIALIZED, NOT MATERIALIZED) +`EXPLAIN SELECT 1` parses into an `*ast.ExplainStatement` whose `Statement` is the `*ast.SelectStatement`, and `SQL()` writes it back as `EXPLAIN SELECT 1`. Before v1.17.0, it became a `DescribeStatement`, so code that matched `EXPLAIN` through `*ast.DescribeStatement` must handle `*ast.ExplainStatement` instead. `Children()` returns the explained statement, so `ast.Inspect` and `ast.Walk` reach it. -**Example:** ```go -// Materialized CTE -sql := "WITH cte AS MATERIALIZED (SELECT * FROM large_table WHERE expensive_filter = true) SELECT * FROM cte" -ast, _ := gosqlx.Parse(sql) +tree, _ := gosqlx.ParseWithDialect("EXPLAIN SELECT 1", keywords.DialectPostgreSQL) +if e, ok := tree.Statements[0].(*ast.ExplainStatement); ok { + fmt.Printf("%s %T\n", e.Keyword, e.Statement) // EXPLAIN *ast.SelectStatement +} +``` + +### `BatchSeparator` +A SQL Server `GO` line. It sits in `AST.Statements` between the statements of a T-SQL script. + +```go +type BatchSeparator struct { + RepeatCount int // n in "GO n"; 0 when no count was written + Pos models.Location // source position of the GO keyword +} +``` + +Scripts that use `GO` and `GO ` parse with the SQL Server dialect, also without semicolons. Repeated batches are not expanded, and `SQL()` and `FormatAST` keep each `GO` on its own line. Code that enumerates `AST.Statements` must skip `*ast.BatchSeparator` nodes when it counts statements: + +```go +tree, _ := gosqlx.ParseWithDialect("SELECT 1\nGO\nSELECT 2\nGO 3", keywords.DialectSQLServer) +for _, stmt := range tree.Statements { + if sep, ok := stmt.(*ast.BatchSeparator); ok { + fmt.Println("GO", sep.RepeatCount) // GO 0, then GO 3 + continue + } + fmt.Printf("%T\n", stmt) // *ast.SelectStatement +} +``` + +## CTE and Set Operations + +### `WithClause` +```go +type WithClause struct { + Recursive bool + CTEs []*CommonTableExpr + Pos models.Location +} ``` ### `CommonTableExpr` ```go type CommonTableExpr struct { - Name string - Columns []string - Query *SelectStatement + Name string + Columns []string + Statement Statement + ScalarExpr Expression // ClickHouse scalar CTE + Materialized *bool // nil = default, true = MATERIALIZED, false = NOT MATERIALIZED + Pos models.Location + Search *CTESearchClause + Cycle *CTECycleClause + NameQuote NameQuote + ColumnQuotes []NameQuote } ``` +`Search` and `Cycle` hold recursive CTE clauses for PostgreSQL and Oracle, or are nil when absent. `CTESearchClause` exposes `Breadth bool` (false means DEPTH FIRST), `By []string`, `Set string` and `Pos models.Location`. `CTECycleClause` exposes `Columns []string`, `Set string`, `To Expression`, `Default Expression`, `Using string` (PostgreSQL path column) and `Pos models.Location`. + ### `SetOperation` ```go type SetOperation struct { @@ -377,28 +456,62 @@ ast, _ := gosqlx.Parse(sql) ### `Identifier` ```go type Identifier struct { - Name string - Value string // Alias for Name + Name string + Table string // optional qualifier; "db.t" for db.t.col + Pos models.Location + Quoted bool // Name was an ANSI double-quoted identifier + Quotes NameQuote // which parts were delimited and must stay delimited + QuoteStyle rune // '"', '`' or '[' as written; 0 when nothing was delimited } ``` +`QuoteStyle` records the opening delimiter the source used: `"` (ANSI), `` ` `` (MySQL, MariaDB, ClickHouse) or `[` (SQL Server). `SQL()`, the formatter and fingerprints write quoted identifiers back with the same delimiters, so a MySQL `` `select` `` stays a name. Statement, table and column names carry the same information in `NameQuote` fields, such as `TableReference.NameQuote` and `TableReference.AliasQuote`. + +`NameQuote` is the compact metadata type used by `Identifier.Quotes` and name fields. It records delimiter style and which name parts must stay delimited; the parser sets it. Its recorded style takes precedence over `Identifier.QuoteStyle`. + ### `LiteralValue` ```go type LiteralValue struct { - Type string // STRING, NUMBER, BOOLEAN, NULL - Value string + Value interface{} + Type string // INTEGER, FLOAT, STRING, BOOLEAN, NULL, ... + // unexported extras: introducer and escape metadata +} +``` + +String literals keep their introducer and escape rules. Read them with `DialectExtras()`, which returns a `LiteralExtras` value, and change them through `EnsureExtras()` or `SetRawBackslash()`: + +```go +type LiteralExtras struct { + Prefix string // N, X, B or a MySQL introducer such as _utf8mb4; "" for a plain string + RawBackslash bool // backslash is an ordinary character in the source dialect } ``` +```go +tree, _ := gosqlx.ParseWithDialect("SELECT N'abc', X'0F', _utf8mb4'hi' FROM t", keywords.DialectMySQL) +sel := tree.Statements[0].(*ast.SelectStatement) +lit := sel.Columns[2].(*ast.LiteralValue) +fmt.Println(lit.DialectExtras().Prefix) // _utf8mb4 +fmt.Println(tree.SQL()) // SELECT N'abc', X'0F', _utf8mb4'hi' FROM t +``` + ### `BinaryExpression` ```go type BinaryExpression struct { - Left Expression - Operator string - Right Expression + Left Expression + Operator string + Right Expression + Not bool + PipesOr bool // MySQL || meaning OR + BitwiseXor bool // ^ is bitwise XOR (MySQL, MariaDB, SQL Server), not exponentiation + CustomOp *CustomBinaryOperator // PostgreSQL custom operators + Extras *BinaryExtras // LIKE ... ESCAPE; nil when absent + Pos models.Location } ``` +`^` is exponentiation in PostgreSQL and bitwise XOR in MySQL, MariaDB and SQL Server. The parser records the XOR meaning in `BitwiseXor`, and `gosqlx.Transpile` uses it to write `POWER()`, `#`, `BITXOR()` or `bitXor()` for the target dialect. + **Supported Operators:** - Arithmetic: `+`, `-`, `*`, `/`, `%` - Comparison: `=`, `!=`, `<>`, `<`, `>`, `<=`, `>=` @@ -512,12 +625,22 @@ ast, _ := gosqlx.Parse(sql) type TableReference struct { Name string Alias string - Lateral bool // PostgreSQL LATERAL JOIN Subquery *SelectStatement // Derived table: (SELECT ...) alias + Lateral bool // PostgreSQL LATERAL JOIN + NameQuote NameQuote // delimited parts of Name + AliasQuote NameQuote // delimited parts of Alias + Pivot *PivotClause // PIVOT (...) + Unpivot *UnpivotClause // UNPIVOT (...) SetSubquery *SetOperation // Derived table whose body is a set operation + Modifiers *TableModifiers // Rarer table modifiers; nil when absent + // ... Final, Only, TableHints, TableFunc, TimeTravel and other dialect fields } ``` +`Modifiers` holds TABLESAMPLE / SAMPLE, Oracle flashback and `@dblink`, MySQL `PARTITION` selection and index hints, SQLite `INDEXED BY`, the SQL Server `OPENJSON ... WITH` schema and a derived `VALUES` table. Read it with `DialectModifiers()` and write it after `EnsureModifiers()`; see [Dialect extension fields](#dialect-extension-fields). + +`TableModifiers.Sample *SampleClause` holds ClickHouse `SAMPLE k [OFFSET m]` on a table in a FROM list with joins or several tables. A lone FROM table keeps it on `SelectStatement.Sample`. This differs from `TableModifiers.TableSample`, which holds TABLESAMPLE / SAMPLE row-sampling clauses. + **Enhancement:** - `Lateral` - PostgreSQL LATERAL JOIN support @@ -536,9 +659,12 @@ ast, _ := gosqlx.Parse(sql) ### `FetchClause` ```go type FetchClause struct { - Count *int64 // Number of rows - Percent bool // PERCENT keyword - WithTies bool // WITH TIES keyword + OffsetValue *int64 // OFFSET n ROWS + FetchValue *int64 // FETCH n ROWS + FetchType string // "FIRST" or "NEXT" + IsPercent bool // PERCENT keyword + WithTies bool // WITH TIES keyword + FetchExpr Expression // row count that is not a plain integer } ``` @@ -549,6 +675,60 @@ sql := "SELECT * FROM users ORDER BY score DESC FETCH FIRST 10 ROWS WITH TIES" ast, _ := gosqlx.Parse(sql) ``` +### `PivotClause` and `UnpivotClause` +```go +type PivotClause struct { + AggregateFunction Expression // e.g. SUM(sales) + AggregateAlias string // alias of AggregateFunction + ExtraAggregates []PivotAggregate // second and later aggregates (Oracle, BigQuery) + PivotColumn string + InValues []string + Pos models.Location + PivotExtensions // XML, dynamic IN lists, source aliases +} + +type PivotAggregate struct { + Function Expression + Alias string +} + +type UnpivotClause struct { + NullHandling string // "", "INCLUDE NULLS" or "EXCLUDE NULLS" + ValueColumn string + ValueColumns []string // multi-column UNPIVOT + NameColumn string + InColumns []string + Pos models.Location + SourceAlias string + SourceAliasQuote NameQuote +} +``` + +`AggregateAlias`, `ExtraAggregates`, `NullHandling` and `ValueColumns` are additions in v1.17.0. `NullHandling` has type `string`. In multi-column UNPIVOT, `ValueColumns` holds the target column set, `ValueColumn` is empty, and each `InColumns` entry holds a parenthesized source column set with its optional label. `ast.UnpivotNullHandling(s)` normalizes a `NullHandling` value. Quoted aliases and value columns keep their source delimiters when rendered. + +## Dialect Extension Fields + +Dialect-specific clauses that most statements never use live behind optional pointers, so common statements do not carry them. These are: + +| Field | Type | Holds | +|---|---|---| +| `SelectStatement.Extras` | `*SelectExtras` | SELECT modifiers, INTO, LIMIT BY, ROWS EXAMINED, INTERPOLATE, query hints, output formats | +| `InsertStatement.Extras` | `*InsertDialectFields` | INSERT modifiers (MySQL `IGNORE`, SQLite `OR REPLACE`, Snowflake `OVERWRITE`), PostgreSQL `OVERRIDING`, the MySQL row alias and other INSERT clauses | +| `UpdateStatement.Extras`, `DeleteStatement.Extras` | `*DMLTargetFields` | modifiers, TOP, ONLY, extra targets, joins, OUTPUT, ORDER BY / LIMIT, RETURNING INTO | +| `FunctionCall.Extras` | `*FunctionExtras` | `GROUP_CONCAT ... SEPARATOR`, Oracle KEEP | +| `BinaryExpression.Extras` | `*BinaryExtras` | `LIKE ... ESCAPE` | +| `TableReference.Modifiers` | `*TableModifiers` | sampling, partitions, index hints, derived `VALUES` | + +Read them through `DialectExtras()` (or `DialectModifiers()` for a table reference). It returns a copy of the struct by value (the zero value when the pointer is nil), so callers need no nil check, and assigning to a field of the copy does not change the node. The copy is shallow: slices and pointers inside it still refer to the node's data, so treat them as read-only. To change a node, call `EnsureExtras()` (or `EnsureModifiers()`); it allocates the struct when needed and returns a pointer to it: + +```go +sel := tree.Statements[0].(*ast.SelectStatement) +if sel.DialectExtras().Into != nil { // safe when sel.Extras is nil + // SELECT ... INTO +} +sel.EnsureExtras().OutputFormat = "JSON" // allocates sel.Extras if needed +``` + ## Object Pool Functions ### `NewAST() *AST` diff --git a/docs/api/formatter.md b/docs/api/formatter.md new file mode 100644 index 000000000..bca044802 --- /dev/null +++ b/docs/api/formatter.md @@ -0,0 +1,97 @@ +# formatter package + +Format SQL text or render an existing AST, with dialect-aware conversions and comment placement. + +Import path: `github.com/ajitpratap0/GoSQLX/pkg/formatter` ([pkg.go.dev](https://pkg.go.dev/github.com/ajitpratap0/GoSQLX/pkg/formatter)) + +Part of the [GoSQLX API reference](../API_REFERENCE.md). For the convenience API, see [gosqlx](gosqlx.md#formatting-functions). + +## Formatting SQL Text + +```go +type Options struct { + IndentSize int // spaces per level; New defaults non-positive values to 2 + Uppercase bool // uppercase SQL keywords + Compact bool // single-line layout + Dialect string // input and target dialect; empty = generic +} + +func New(opts Options) *Formatter +func (f *Formatter) Format(sql string) (string, error) +func FormatString(sql string) (string, error) +``` + +`Format` tokenizes, parses and renders SQL, returning tokenization, parsing, conversion or comment-placement errors. It keeps comments and optimizer hints. Whitespace-only input returns `"", nil`. A recognized dialect selects lexical and parser rules as well as rendering; an unknown dialect is ignored for tokenization and parsing. `FormatString` uses `Options{}`. + +```go +f := formatter.New(formatter.Options{Uppercase: true, Compact: true}) +out, err := f.Format("select id from users") +if err != nil { + return err +} +fmt.Println(out) +``` + +## Rendering an AST + +```go +func FormatAST(a *ast.AST, opts ast.FormatOptions) string +func FormatStatement(s ast.Statement, opts ast.FormatOptions) string +func FormatExpression(e ast.Expression, opts ast.FormatOptions) string + +func FormatASTChecked(a *ast.AST, opts ast.FormatOptions) (string, error) +func FormatStatementChecked(s ast.Statement, opts ast.FormatOptions) (string, error) +func FormatExpressionChecked(e ast.Expression, opts ast.FormatOptions) (string, error) +``` + +Start with `ast.ReadableStyle()` or `ast.CompactStyle()` and adjust `ast.FormatOptions`. These functions render existing nodes without parsing SQL again. The checked variants return errors for unsupported target-dialect semantic conversions, including operators nested inside expressions, and do not mutate the caller's nodes. Nil inputs return an empty string without an error. The string-only variants discard errors and return an empty string when rendering fails; prefer checked calls when selecting a target dialect. + +Since v1.17.0, checked rendering lets callers handle these conversion failures explicitly. `FormatASTChecked` can also report comment-placement errors when the tree carries comments and statement spans. + +## Joining Comments and Statements + +```go +type CommentLayout struct { + Dialect string + Terminator string + Gap string + Batches []models.Location + Starts []models.Location +} + +func JoinWithComments(stmts []string, tokens []models.TokenWithSpan, comments []models.Comment, layout CommentLayout) (string, error) +func JoinASTWithComments(tree *ast.AST, stmts []string, tokens []models.TokenWithSpan, comments []models.Comment, layout CommentLayout) (string, error) + +var ErrUnplaceableComment = errors.New("formatter: cannot preserve comment") +``` + +Pass rendered statements in source order and tokens and comments from the original input. `Terminator` is inserted between statements: use `";"` if the parts have no terminators, or `""` if they already do. `Gap` separates statements and their trailing comments, usually `"\n"`, `"\n\n"` or `" "`; line comments force a line break when needed. + +`Starts` supplies one source start per rendered statement, including `GO` entries, to avoid splitting procedure bodies at internal semicolons. Without it, token semicolons determine statement boundaries. `Batches` supplies parser-recognized `GO` positions; nil enables token-based recognition. `Dialect` selects lexical rules and batch boundaries. + +Ordinary comments inside a statement move above it. Between statements, comments lead the next statement unless they shared a line with the preceding statement; trailing comments remain trailing. Optimizer hints directly after a supported statement keyword stay there when the rendered occurrence can be identified. If a meaningful hint or MySQL executable comment cannot be restored faithfully, the join returns an error wrapping `ErrUnplaceableComment`. Use `errors.Is` to check it and retain the original SQL on failure. When source and rendered statements cannot be matched, ordinary comments move before the first statement; hints retain the same error checks. + +`JoinASTWithComments` uses parser statement spans and Oracle execute-line metadata. Supply one rendered part per `AST.Statements` entry, without execute lines; the helper inserts those lines without changing statement counts. Tokens, when available, allow hints inside statements to be placed. + +### Source positions + +```go +func BatchPositions(tree *ast.AST) []models.Location +func StatementPositions(tree *ast.AST) []models.Location +``` + +`BatchPositions` collects `BatchSeparator.Pos` for `CommentLayout.Batches`. It returns a non-nil slice even for nil trees or scripts without `GO`, telling the joiner to follow the parser's boundaries. `StatementPositions` returns `AST.StatementStarts` for `CommentLayout.Starts`, or nil when the tree is nil or the slice length does not match the statement count. + +## Script Rendering Helpers + +```go +func EnsureStatementTerminator(stmt ast.Statement, part string, addSemicolon bool) string +func ExecuteSlashStatement(stmt ast.Statement, part string) string +func RestoreUnicodeIdentifiers(tree *ast.AST, statement int, sql, dialect string) string +``` + +`EnsureStatementTerminator` appends `;` when requested, unless the rendered part is empty, already ends in `;`, or represents a `BatchSeparator`. + +`ExecuteSlashStatement` appends `"\n/"` for an Oracle SQL*Plus execute command without adding a SQL terminator to an ordinary statement. It retains the required ending semicolon for recognized PL/SQL utility blocks. + +`RestoreUnicodeIdentifiers` restores PostgreSQL `U&` source spellings for one statement index using `AST.UnicodeIdentifiers`; semantic names on the AST stay decoded. It restores spellings only for unset, generic or PostgreSQL target dialects. Other targets keep decoded identifiers. A nil tree or absent metadata leaves the rendered SQL unchanged. diff --git a/docs/api/gosqlx.md b/docs/api/gosqlx.md index 1990c32e2..eef38baca 100644 --- a/docs/api/gosqlx.md +++ b/docs/api/gosqlx.md @@ -94,3 +94,140 @@ if err := gosqlx.ValidateMultiple(queries); err != nil { log.Fatal("Validation failed:", err) } ``` + +## Formatting Functions + +### `Format(sql string, options FormatOptions) (string, error)` +Parse SQL with the default dialect and render it back as consistently styled text. `DefaultFormatOptions()` returns the defaults (`IndentSize` 2, `UppercaseKeywords` false, `AddSemicolon` false). Invalid SQL returns an error wrapping `ErrTokenize` or `ErrSyntax`. + +```go +opts := gosqlx.DefaultFormatOptions() +opts.AddSemicolon = true +out, err := gosqlx.Format("select id from users where id = 1", opts) +// SELECT id +// FROM users +// WHERE id = 1; +``` + +`Format` keeps comments and optimizer hints. A comment inside a statement moves to the line above it, and a `/*+ ... */` hint directly after a statement's first keyword stays there. SQL whose hints or MySQL `/*! ... */` comments cannot be put back faithfully returns an error wrapping `formatter.ErrUnplaceableComment` (package `pkg/formatter`) instead of being rewritten without them: + +```go +_, err := gosqlx.Format("select a from /*+ FULL(t) */ t", gosqlx.DefaultFormatOptions()) +errors.Is(err, formatter.ErrUnplaceableComment) // true +``` + +See [Format SQL in Go](../guides/format-sql-in-go.md#comments-and-optimizer-hints) for examples. + +### `FormatWithDialect(sql string, options FormatOptions, dialect keywords.SQLDialect) (string, error)` +`Format` for SQL in a specific dialect: the SQL is tokenized and parsed with that dialect's rules, and comments and hints are kept as in `Format`. An empty dialect behaves like `Format`; an unknown dialect returns an error wrapping `ErrUnsupportedDialect`. + +```go +out, err := gosqlx.FormatWithDialect( + "select top 10 [order id] from dbo.orders -- latest\nwhere total > 100", + gosqlx.DefaultFormatOptions(), + keywords.DialectSQLServer, +) +// -- latest +// SELECT TOP 10 [order id] +// FROM dbo.orders +// WHERE total > 100 +``` + +### `WithComments() Option` +A `ParseTree` option that keeps the comments of the input on the tree so that `FormatTree` and `FormatAST` write them back. Without it comments are discarded. Parser-produced statement spans let formatting keep comments next to their statements; comments inside a statement move above it. Use `Format` on the original text when preserving optimizer hints, since it also retains the token stream needed to place hints inside statements. + +```go +tree, err := gosqlx.ParseTree(ctx, "SELECT 1 -- one", gosqlx.WithComments()) +out := gosqlx.FormatTree(tree) // "SELECT 1 -- one" +``` + +### Checked tree formatting + +Since v1.17.0, use these variants to receive semantic rendering errors, including target-dialect operators that cannot be represented: + +```go +func FormatASTChecked(a *ast.AST, opts ...FormatOption) (string, error) +func FormatTreeChecked(t *Tree, opts ...FormatOption) (string, error) +func (t *Tree) FormatChecked(opts ...FormatOption) (string, error) +``` + +They render an existing tree without parsing it again, return `"", nil` for a nil tree, and do not mutate the caller's AST. The corresponding `FormatAST`, `FormatTree` and `Tree.Format` helpers discard rendering errors and return an empty string. Check the returned error before using formatted SQL. For low-level rendering options and comment placement, see the [formatter package](formatter.md). + +## Linting Functions + +### `Lint(sql string) (*LintResult, error)` +Run all 30 built-in rules (L001-L030) and return the violations in `LintResult.Violations`. + +### `LintWithDialect(sql string, dialect keywords.SQLDialect) (*LintResult, error)` +`Lint` with the SQL tokenized and parsed in the given dialect, so the AST-based rules (L011-L030) also run on dialect-specific syntax. An empty dialect behaves like `Lint`; an unknown dialect returns an error wrapping `ErrUnsupportedDialect`. + +Since v1.17.0, both functions report SQL that does not tokenize or parse as an error: the error wraps `ErrTokenize` or `ErrSyntax`, and the returned `*LintResult` still holds the violations of the text and token rules. Code that ignored the error and treated an empty result as clean must check it: + +```go +res, err := gosqlx.LintWithDialect("SELEC 1", keywords.DialectPostgreSQL) +if errors.Is(err, gosqlx.ErrSyntax) { + // invalid SQL; res is not nil and may still list text-rule violations +} +_ = res +``` + +The lower-level `linter.FileResult` (package `pkg/linter`) gains the fields `TokenizeError` and `ParseError` and the method `SyntaxError()`, which returns whichever of the two is set. + +## Script Splitting + +### `SplitStatements(sql, dialect string) []string` +Split a script into statements at top-level semicolons, ignoring semicolons inside strings, quoted identifiers and comments. The dialect selects the quoting rules, such as PostgreSQL dollar quotes, MySQL backticks and SQL Server brackets; an unknown dialect falls back to ANSI quoting. The terminating `;` is removed and empty segments are kept, so trim and skip them. Segments are substrings of the input, so no bytes are copied. + +The splitter also: + +- keeps compound statements whole: `BEGIN ... END` bodies of triggers, procedures and functions, T-SQL `BEGIN TRY` and `IF ... BEGIN`, PostgreSQL `BEGIN ATOMIC`, `CASE ... END`, `IF`/`LOOP`/`WHILE`/`REPEAT` inside a block, and Oracle `DECLARE` sections. Transaction statements such as `BEGIN;` are not blocks, and a script whose blocks do not balance falls back to plain `;` splitting; +- honours client batch separators and removes them from the output: a line holding only `GO [n]` for SQL Server, `DELIMITER` for MySQL and MariaDB, and a line holding only `/` for Oracle; +- follows each dialect's backslash escapes in strings, and does not end a statement at a `;` inside parentheses. + +```go +parts := gosqlx.SplitStatements("DELIMITER //\nCREATE PROCEDURE p() BEGIN SELECT 1; END//\nDELIMITER ;\nSELECT 2;", "mysql") +// ["CREATE PROCEDURE p() BEGIN SELECT 1; END", "SELECT 2"] + +parts = gosqlx.SplitStatements("SELECT 1\nGO\nSELECT 2\nGO 3\n", "sqlserver") +// ["SELECT 1\n", "SELECT 2\n"] +``` + +## Transpilation + +### `Transpile(sql string, from, to keywords.SQLDialect) (string, error)` +Parse SQL in the `from` dialect and render it for the `to` dialect, applying the rewrite rules registered for that pair. + +```go +out, err := gosqlx.Transpile( + "CREATE TABLE t (id INT AUTO_INCREMENT PRIMARY KEY)", + keywords.DialectMySQL, + keywords.DialectPostgreSQL, +) +// CREATE TABLE t (id SERIAL PRIMARY KEY) +``` + +String literals and quoted identifiers are written with the target dialect's escape rules and delimiters, and dialect-specific operators are mapped for the target: `^` is exponentiation in PostgreSQL and bitwise XOR in MySQL, MariaDB and SQL Server, and PostgreSQL `@` and `|/` become `ABS()` and `SQRT()`. PostgreSQL `||/` becomes `CBRT()` only for Snowflake and ClickHouse; every other target (MySQL, for example) has no `CBRT()` and returns an error. A literal, operator or SQL Server `GO` script the target cannot represent returns an error instead of SQL with a different meaning: + +```go +gosqlx.Transpile("SELECT `Order Id` FROM t WHERE n = 'it\\'s'", keywords.DialectMySQL, keywords.DialectPostgreSQL) +// SELECT "Order Id" FROM t WHERE n = 'it''s' + +gosqlx.Transpile("SELECT 2 ^ 3", keywords.DialectPostgreSQL, keywords.DialectMySQL) +// SELECT POWER(2, 3) + +gosqlx.Transpile("SELECT a ^ b FROM t", keywords.DialectMySQL, keywords.DialectOracle) +// error: rewrite: the bitwise XOR operator ^ has no equivalent in oracle +``` + +## Metadata Extraction + +### `ExtractColumns(astNode *ast.AST) []string` +Return the deduplicated, sorted column names referenced in SELECT lists, WHERE, GROUP BY, ORDER BY, JOIN conditions and HAVING, without table qualifiers. + +Since v1.17.0 (issue #849), an unqualified name in `ORDER BY` or `HAVING` that matches a SELECT-list alias of the same query block is treated as the alias, not reported as a column. Qualified references stay columns, and aliases never cross subquery, CTE or UNION boundaries. `ExtractColumnsQualified` follows the same rule. + +```go +tree, _ := gosqlx.Parse("SELECT count(*) AS orders FROM t GROUP BY customer_id ORDER BY orders") +cols := gosqlx.ExtractColumns(tree) +// [customer_id] +``` diff --git a/docs/api/linter.md b/docs/api/linter.md index 320275cb4..45bfe5af8 100644 --- a/docs/api/linter.md +++ b/docs/api/linter.md @@ -67,14 +67,36 @@ type Result struct { ### `FileResult` ```go type FileResult struct { - Filename string - Violations []Violation - Error error + Filename string + Violations []Violation + Error error // File read or rule execution failure + TokenizeError error // SQL could not be tokenized + ParseError error // Tokenized SQL could not be parsed } ``` ## Functions +### `(r FileResult) SyntaxError() error` +Returns `TokenizeError` when present, otherwise `ParseError`; returns nil when +the input tokenized and parsed. Empty, whitespace-only and comment-only input +is not a syntax error. + +`Error` reports a failed lint operation and may accompany partial violations. +Syntax errors are recorded separately: applicable text and token rules still +run and report violations. Check both operation and syntax errors when using +linting as a validation gate. + +```go +result := l.LintString(sql, "query.sql") +if result.Error != nil { + log.Fatal(result.Error) +} +if err := result.SyntaxError(); err != nil { + log.Fatal(err) +} +``` + ### `New(rules ...Rule) *Linter` Create linter with specified rules. diff --git a/docs/api/lsp.md b/docs/api/lsp.md index f84cdf892..c7dc78e1d 100644 --- a/docs/api/lsp.md +++ b/docs/api/lsp.md @@ -67,6 +67,31 @@ if err := server.Run(); err != nil { The GoSQLX LSP server implements the following LSP features: +The initialization response advertises incremental synchronization under +`capabilities.textDocumentSync`: + +```json +{ + "capabilities": { + "textDocumentSync": { + "openClose": true, + "change": 2, + "save": {"includeText": true} + } + } +} +``` + +### Protocol error handling + +Unknown request methods return JSON-RPC error `-32601` (`MethodNotFound`). +Malformed `Content-Length` headers and invalid JSON are logged without stopping +the server; it continues processing subsequent well-framed messages. +Incremental `textDocument/didChange` ranges use UTF-16 positions: negative +positions are rejected, positions past the line or document end are clamped, +and reversed ranges have their endpoints swapped. These ranges do not crash +the server. + ### **1. Text Document Synchronization** - `textDocument/didOpen` - Document opened notification - `textDocument/didChange` - Document changed notification (incremental sync) diff --git a/docs/api/mcp.md b/docs/api/mcp.md index 2d70461ea..227a71865 100644 --- a/docs/api/mcp.md +++ b/docs/api/mcp.md @@ -66,10 +66,15 @@ How long an idle MCP session's transport state is kept before the sweeper frees |------|-------------| | `validate_sql` | SQL syntax validation with optional dialect | | `format_sql` | SQL formatting (indent, keyword case, semicolon) | -| `parse_sql` | AST summary - statement count and types | +| `parse_sql` | AST summary with optional dialect - statement count and types | | `extract_metadata` | Tables, columns, functions referenced | | `security_scan` | Injection pattern detection with severity | | `lint_sql` | Style rule enforcement (L001–L010) | | `analyze_sql` | Concurrent composite of all 6 above | +`parse_sql` accepts a required `sql` string and an optional `dialect` string (default: `generic`). +Use `"dialect": "sqlserver"` for scripts containing `GO`. The response keys +`statement_count` and `statement_types` exclude `GO` batch separators +(`*ast.BatchSeparator`), including separators with repeat counts. + See [MCP Server Guide](../MCP_GUIDE.md) for complete tool schemas and JSON response formats. diff --git a/docs/api/security.md b/docs/api/security.md index 7cf9c94df..a3f3f3321 100644 --- a/docs/api/security.md +++ b/docs/api/security.md @@ -34,6 +34,9 @@ const ( type PatternType string const ( + PatternInputTooLarge PatternType = "INPUT_TOO_LARGE" + PatternUnionInjection PatternType = "UNION_INJECTION" + PatternUnionGeneric PatternType = "UNION_GENERIC" PatternTautology PatternType = "TAUTOLOGY" PatternComment PatternType = "COMMENT_BYPASS" PatternStackedQuery PatternType = "STACKED_QUERY" @@ -89,12 +92,18 @@ scanner, err := security.NewScannerWithSeverity(security.SeverityHigh) ``` ### `(s *Scanner) Scan(ast *ast.AST) *ScanResult` -Scan parsed AST for security issues. +Scan the full parsed AST, including nested queries, CTEs, expressions, joins, +DML conditions and table functions. Statements retained as raw text (such as +utility statements and unsupported statements) are checked through `ScanSQL`. +A nil AST returns an empty result. ```go scanner := security.NewScanner() -ast, _ := gosqlx.Parse(sql) -result := scanner.Scan(ast) +tree, err := gosqlx.Parse(sql) +if err != nil { + log.Fatal(err) +} +result := scanner.Scan(tree) for _, finding := range result.Findings { fmt.Printf("[%s] %s: %s\n", finding.Severity, finding.Pattern, finding.Description) @@ -102,7 +111,23 @@ for _, finding := range result.Findings { ``` ### `(s *Scanner) ScanSQL(sql string) *ScanResult` -Scan raw SQL string for injection patterns (useful for patterns not in AST). +Scan the complete raw SQL input for patterns, including comments and statements +that may not be represented as expression nodes in the AST. Matching combines +the raw view with comment-normalised views that account for quoted strings and +identifiers, backslash escape modes, nested block comments, MySQL/MariaDB comments +and executable comments. This method has no dialect parameter and considers +multiple lexical interpretations. + +Recognised dollar-quoted string bodies (`$$...$$` and `$tag$...$tag$`) are +excluded from matching when their openers occur outside strings, identifiers +and comments. This reduces findings on literal content; use the AST scan as +well when dialect-specific structure matters. + +Inputs larger than `MaxScanInputSize` (10 MiB, measured in bytes) are not scanned. +They produce one `PatternInputTooLarge` finding with `SeverityCritical`, even +when a minimum-severity filter is configured. `IsClean()` is false and +`HasCritical()` is true for this result. Inputs within the limit are scanned +in full, without prefix truncation. ```go scanner := security.NewScanner() @@ -120,7 +145,7 @@ Returns true if no findings exist. ## Detection Patterns -The scanner detects 8 pattern types: +The scanner reports the following pattern categories: 1. **Tautologies** - Always-true conditions (e.g., `1=1`, `'a'='a'`) 2. **Comment Bypasses** - SQL comment-based injection (`--`, `/**/`, `#`) @@ -129,7 +154,10 @@ The scanner detects 8 pattern types: 5. **Time-Based Blind** - Time delay functions (SLEEP, WAITFOR DELAY, pg_sleep, BENCHMARK) 6. **Out-of-Band** - Data exfiltration (xp_cmdshell, LOAD_FILE, UTL_HTTP) 7. **Dangerous Functions** - Dynamic SQL execution (EXEC, sp_executesql, PREPARE FROM) -8. **Boolean-Based** - Conditional logic exploitation +8. **Boolean-Based** - The exported `PatternBooleanBased` category; the current scanner does not emit it +9. **UNION Injection Fingerprints** (`PatternUnionInjection`) - Raw-SQL UNION patterns involving system tables or NULL padding; CRITICAL severity +10. **Generic UNION** (`PatternUnionGeneric`) - Raw-SQL UNION SELECT; HIGH severity and may be legitimate SQL requiring review +11. **Input Too Large** (`PatternInputTooLarge`) - Input exceeded the scan limit and was not inspected **Example:** ```go @@ -141,17 +169,17 @@ result := scanner.ScanSQL(sql) // Finding: TAUTOLOGY, Severity: CRITICAL // Detect UNION injection -sql := "SELECT * FROM products WHERE id = 1 UNION SELECT NULL, username, password FROM users" +sql = "SELECT * FROM products WHERE id = 1 UNION SELECT NULL, NULL, NULL" result = scanner.ScanSQL(sql) -// Finding: UNION_BASED, Severity: CRITICAL +// Findings include UNION_INJECTION (CRITICAL) and UNION_GENERIC (HIGH) // Detect time-based injection -sql := "SELECT * FROM orders WHERE id = 1; SELECT SLEEP(5)" +sql = "SELECT * FROM orders WHERE id = 1; SELECT SLEEP(5)" result = scanner.ScanSQL(sql) // Finding: TIME_BASED, Severity: HIGH // Clean SQL -sql := "SELECT * FROM users WHERE id = $1" +sql = "SELECT * FROM users WHERE id = $1" result = scanner.ScanSQL(sql) // result.IsClean() == true ``` @@ -160,4 +188,6 @@ result = scanner.ScanSQL(sql) - Pre-compiled regex patterns for performance - Thread-safe pattern compilation (sync.Once) - Precise system table matching (avoids false positives) -- 100% detection rate on common injection patterns + +Findings are heuristics for review, not proof that SQL is safe or malicious. +Use parameterised queries for untrusted values. diff --git a/docs/api/tokenizer.md b/docs/api/tokenizer.md index e3046b92d..83588bf8c 100644 --- a/docs/api/tokenizer.md +++ b/docs/api/tokenizer.md @@ -37,6 +37,34 @@ defer cancel() tokens, err := tkz.TokenizeContext(ctx, []byte("SELECT * FROM users")) ``` +## Dialect-specific lexing + +Use `NewWithDialect(dialect keywords.SQLDialect) (*Tokenizer, error)` or +`(*Tokenizer).SetDialect(dialect keywords.SQLDialect)` before tokenizing. +An empty dialect or `keywords.DialectUnknown` selects PostgreSQL keyword +recognition, with strict string escapes; an explicit PostgreSQL dialect uses +standard strings with literal backslashes. + +- Numeric tokens retain their source spelling. Decimal and exponent forms are + supported; `0x` hexadecimal forms are accepted in every dialect. PostgreSQL, + MySQL/MariaDB and ClickHouse also accept `0b` binary forms; PostgreSQL accepts + `0o` octal forms. PostgreSQL, SQLite and ClickHouse allow single underscores + between digits, and PostgreSQL also allows one immediately after a base prefix. + Oracle accepts floating-point suffixes such as `1.5f` and `1e5D`. +- PostgreSQL `U&'...'` strings and `U&"..."` identifiers support four-digit and + `+` six-digit Unicode escapes, surrogate pairs and an optional `UESCAPE` + character. Invalid Unicode escapes return an error. Strings retain validated + source spelling; identifiers expose decoded names. +- String backslash handling follows the selected dialect: PostgreSQL standard + strings, Oracle, SQLite, SQL Server and generic SQL keep backslashes literal; + MySQL/MariaDB decode escapes while preserving `\%` and `\_`; ClickHouse and + BigQuery decode supported escapes and keep unknown ones; Snowflake and Redshift + decode supported escapes and drop the backslash on unknown ones. An unconfigured + tokenizer uses strict escapes and rejects dialect-dependent escape sequences. +- MySQL and MariaDB treat `#` through the end of the line as a comment. In other + dialects it remains available to operators or identifiers according to their + lexical rules. + ## Supported Token Types | Token Type | Example | Notes | @@ -63,7 +91,7 @@ tokens, err := tkz.TokenizeContext(ctx, []byte("SELECT * FROM users")) | `TokenTypeAtGreater` | `@>` | ✅ PostgreSQL containment | | `TokenTypeLessAt` | `<@` | ✅ PostgreSQL contained by | | `TokenTypeQuestionMark` | `?` | ✅ PostgreSQL key exists | -| `TokenTypeQuestionPipe` | `?|` | ✅ PostgreSQL any key exists | +| `TokenTypeQuestionPipe` | `?\|` | ✅ PostgreSQL any key exists | | `TokenTypeQuestionAmpersand` | `?&` | ✅ PostgreSQL all keys exist | | `TokenTypeHashMinus` | `#-` | ✅ PostgreSQL delete at path | @@ -75,4 +103,4 @@ tokens, err := tkz.TokenizeContext(ctx, []byte("SELECT * FROM users")) - Position tracking (line, column, offset) - Scientific notation support (1.23e4, 1.23E+4) - Triple-quoted strings (Python-style) -- Escape sequences (\n, \t, \r, \\, \', \") +- Escape sequences follow the configured dialect diff --git a/docs/guides/format-sql-in-go.md b/docs/guides/format-sql-in-go.md index 50d721e78..3e4031fe4 100644 --- a/docs/guides/format-sql-in-go.md +++ b/docs/guides/format-sql-in-go.md @@ -75,22 +75,79 @@ fmt.Println(err) Use `errors.Is(err, gosqlx.ErrSyntax)` and `gosqlx.ErrorLocation(err)` to report the position, as shown in [Parse PostgreSQL in Go](parse-postgresql-in-go.md#handle-syntax-errors). -## Comments are not preserved +## Comments and optimizer hints -Because the output is generated from the AST, SQL comments (`-- ...` and `/* ... */`) are dropped: +`gosqlx.Format` keeps comments. A comment between statements stays between them, a trailing comment stays after its statement, and a comment inside a statement moves to the line above that statement: ```go -out, _ := gosqlx.Format("-- top users\nselect id /* pk */ from users where id = 1", gosqlx.DefaultFormatOptions()) -// SELECT id -// FROM users -// WHERE id = 1 +sql := "-- top users\nselect id, /* pk */ email from users where id = 1; -- one user\nselect 2" +out, err := gosqlx.Format(sql, gosqlx.DefaultFormatOptions()) +if err != nil { + log.Fatal(err) +} +fmt.Println(out) ``` -If your files carry important comments, run the formatter in check mode in CI rather than rewriting files automatically, or keep comments outside the statements you format. +Output: + +```sql +-- top users +/* pk */ +SELECT id, email +FROM users +WHERE id = 1; -- one user +SELECT 2 +``` + +An optimizer hint (`/*+ ... */`) directly after the first keyword of a statement stays where it is, because moving it would change what the database does: + +```go +out, _ := gosqlx.Format("select /*+ FULL(t) */ a from t", gosqlx.DefaultFormatOptions()) +// SELECT /*+ FULL(t) */ a +// FROM t +``` + +When a hint or a MySQL `/*! ... */` comment cannot be put back faithfully, `Format` returns an error wrapping `formatter.ErrUnplaceableComment` (from `github.com/ajitpratap0/GoSQLX/pkg/formatter`) instead of rewriting the SQL without it. Leave such input unformatted: + +```go +_, err := gosqlx.Format("select a from /*+ FULL(t) */ t", gosqlx.DefaultFormatOptions()) +if errors.Is(err, formatter.ErrUnplaceableComment) { + // keep the original text +} +fmt.Println(err) +// cannot format SQL: formatter: cannot preserve comment: /*+ FULL(t) */ +``` + +If you work with a parsed tree instead of text, pass `gosqlx.WithComments()` to `gosqlx.ParseTree` so that `gosqlx.FormatTree` writes the comments back. A tree does not record where each comment stood, so comments on their own line go in front of the query and comments that shared a line with SQL go after it. Use `Format` or `FormatWithDialect` on the text when the placement matters. ## Format dialect-specific SQL -`gosqlx.Format` parses with the default dialect. To format SQL that only parses under a specific dialect, parse it into a `Tree` with that dialect and call `Format` on the tree: +`gosqlx.Format` parses with the default dialect. To format SQL that only parses under a specific dialect, call `gosqlx.FormatWithDialect`, which tokenizes and parses with that dialect's rules and keeps comments and hints the same way as `Format`. Use it as follows: + +```go +out, err := gosqlx.FormatWithDialect( + "select top 10 [order id] from dbo.orders -- latest\nwhere total > 100", + gosqlx.DefaultFormatOptions(), + keywords.DialectSQLServer, // github.com/ajitpratap0/GoSQLX/pkg/sql/keywords +) +if err != nil { + log.Fatal(err) +} +fmt.Println(out) +``` + +Output: + +```sql +-- latest +SELECT TOP 10 [order id] +FROM dbo.orders +WHERE total > 100 +``` + +An empty dialect behaves like `Format`, and an unknown dialect returns an error wrapping `gosqlx.ErrUnsupportedDialect`. + +If you already hold a parsed tree, `tree.Format` takes functional options instead of `FormatOptions`: ```go tree, err := gosqlx.ParseTree(ctx, @@ -139,6 +196,21 @@ gosqlx format --indent 4 --dialect mysql report.sql `--check` prints the files that need formatting and exits with status 1, which makes it a good CI gate. Defaults can be set in a `.gosqlx.yml` file; see [Configuration](../CONFIGURATION.md). +The CLI keeps comments and a leading byte order mark. Before it writes a file with `-i`, it checks that the formatted output holds exactly the comments of the input, and it leaves the file unchanged when it does not. `--no-uppercase` writes every keyword in lower case, including keyword operators such as `AND`, `IN` and `IS NULL`, data types and built-in function keywords such as `CAST`; identifiers, quoted names and literals keep their spelling: + +```bash +echo "SELECT a FROM t WHERE b IS NULL AND c IN (1,2) ORDER BY CAST(a AS INTEGER) DESC" | gosqlx format --no-uppercase +``` + +Output: + +```sql +select a +from t +where b is null and c in (1, 2) +order by cast(a as integer) desc +``` + ## Next steps - [Lint SQL in CI](lint-sql-in-ci.md): enforce formatting and style on every pull request diff --git a/docs/guides/lint-sql-in-ci.md b/docs/guides/lint-sql-in-ci.md index 0dbc0c811..effc7cdf0 100644 --- a/docs/guides/lint-sql-in-ci.md +++ b/docs/guides/lint-sql-in-ci.md @@ -179,7 +179,7 @@ To apply only some rules or change their settings, build a linter from `pkg/lint Turning on every rule at once on a large SQL directory usually produces hundreds of findings. A gradual path: 1. Start with validation only (`gosqlx validate`). Broken SQL is never intentional. -2. Add `gosqlx format --check` after one pull request that runs `gosqlx format -i` over the whole tree. Note that the formatter drops SQL comments; see [Format SQL in Go](format-sql-in-go.md#comments-are-not-preserved). +2. Add `gosqlx format --check` after one pull request that runs `gosqlx format -i` over the whole tree. Releases before v1.17.0 drop SQL comments when formatting; v1.17.0 keeps them. See [Format SQL in Go](format-sql-in-go.md#comments-and-optimizer-hints). 3. Enable lint rules a few at a time with `--rules`, fixing existing findings as you go. `gosqlx lint --auto-fix` fixes the whitespace and keyword-case rules automatically. ## Next steps diff --git a/examples/README.md b/examples/README.md index a13cb528c..2192c9408 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,6 +1,6 @@ # GoSQLX Examples -**Version**: v1.16.0 +**Version**: v1.17.0 This directory contains various examples demonstrating how to use the GoSQLX SQL parsing SDK. diff --git a/glama.json b/glama.json index d49609344..28b72b902 100644 --- a/glama.json +++ b/glama.json @@ -8,7 +8,7 @@ "description": "SQL parsing, formatting, linting, security scanning and metadata extraction, built on the GoSQLX Go SDK. 7 MCP tools: validate_sql, format_sql, parse_sql, extract_metadata, security_scan, lint_sql and analyze_sql. Dialect-aware validation for PostgreSQL, MySQL, MariaDB, SQL Server, Oracle, SQLite, Snowflake, BigQuery, Redshift and ClickHouse. Zero-copy tokenizer, race-free, with object pooling.", "repository": "https://github.com/ajitpratap0/GoSQLX", "homepage": "https://gosqlx.dev", - "version": "1.16.0", + "version": "1.17.0", "license": "Apache-2.0", "author": { "name": "Ajit Pratap Singh", diff --git a/llms.txt b/llms.txt index 5b2e8d06d..4a214eb33 100644 --- a/llms.txt +++ b/llms.txt @@ -4,7 +4,7 @@ GoSQLX is a production-ready SQL parsing library for Go with zero-copy tokenization, recursive-descent parsing and AST generation backed by object pooling. It supports 8 dialects with dedicated grammar (PostgreSQL, MySQL, MariaDB, SQL Server, Oracle, SQLite, Snowflake, ClickHouse); the `--dialect` flag also accepts `generic`, `bigquery` and `redshift` (generic keyword set, limited dialect-gated grammar). It ships a CLI (`gosqlx`) for validation, formatting, linting, transpilation and security analysis. Apache-2.0 licensed. Requires Go 1.26+. -Current stable version: v1.16.0 +Current stable version: v1.17.0 Website: https://gosqlx.dev Interactive Playground: https://gosqlx.dev/playground/ diff --git a/performance_baselines.json b/performance_baselines.json index aa07ad55d..e5a00266d 100644 --- a/performance_baselines.json +++ b/performance_baselines.json @@ -1,5 +1,5 @@ { - "version": "1.16.0", + "version": "1.17.0", "updated": "2026-10-22", "baselines": { "SimpleSelect": { diff --git a/pkg/gosqlx/gosqlx.go b/pkg/gosqlx/gosqlx.go index 6a08fb7c6..455a2e6af 100644 --- a/pkg/gosqlx/gosqlx.go +++ b/pkg/gosqlx/gosqlx.go @@ -31,7 +31,7 @@ import ( ) // Version is the current GoSQLX library version. -const Version = "1.16.0" +const Version = "1.17.0" // Parse tokenizes and parses SQL in one call, returning an Abstract Syntax Tree (AST). // diff --git a/plugins/gosqlx/.claude-plugin/plugin.json b/plugins/gosqlx/.claude-plugin/plugin.json index a658fbce1..935d3cba2 100644 --- a/plugins/gosqlx/.claude-plugin/plugin.json +++ b/plugins/gosqlx/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "gosqlx", "displayName": "GoSQLX", - "version": "1.16.0", + "version": "1.17.0", "description": "SQL validation, formatting, linting, injection scanning and analysis for Claude Code, powered by the GoSQLX parser (multi-dialect).", "author": { "name": "Ajit Pratap Singh", diff --git a/python/pygosqlx/__init__.py b/python/pygosqlx/__init__.py index 52823ea52..3d5be99eb 100644 --- a/python/pygosqlx/__init__.py +++ b/python/pygosqlx/__init__.py @@ -89,7 +89,7 @@ ValidationError, ) -__version__ = "1.16.0" +__version__ = "1.17.0" __all__ = [ # Functions "parse", diff --git a/python/pyproject.toml b/python/pyproject.toml index 5ef9abc57..682079e96 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" [project] name = "pygosqlx" -version = "1.16.0" +version = "1.17.0" description = "Python bindings for GoSQLX - High-performance SQL parser" readme = "README.md" requires-python = ">=3.10" diff --git a/server.json b/server.json index 2490351b7..0df62744e 100644 --- a/server.json +++ b/server.json @@ -3,7 +3,7 @@ "name": "io.github.ajitpratap0/gosqlx", "title": "GoSQLX", "description": "SQL validation, formatting, parsing, metadata extraction, linting, and security analysis.", - "version": "1.16.0", + "version": "1.17.0", "repository": { "url": "https://github.com/ajitpratap0/GoSQLX", "source": "github" diff --git a/vscode-extension/CHANGELOG.md b/vscode-extension/CHANGELOG.md index ae3322582..da2b71d81 100644 --- a/vscode-extension/CHANGELOG.md +++ b/vscode-extension/CHANGELOG.md @@ -2,6 +2,34 @@ All notable changes to the "GoSQLX" extension will be documented in this file. +## [1.17.0] - 2026-10-29 + +Bundles the GoSQLX v1.17.0 language server. See the +[main CHANGELOG](https://github.com/ajitpratap0/GoSQLX/blob/main/CHANGELOG.md) +for the full list. + +### Changed +- **Dialect-faithful parsing and formatting**: the bundled server picks up the + v1.17.0 parser and formatter. Every statement of an 8-dialect real-world + corpus now parses and formats without losing words; formatting keeps + comments, optimizer hints, SQL Server `GO` lines, Oracle `/` lines and + PostgreSQL `U&"..."` spellings. Some SQL that parsed before is now an error, + for example a `GO` line inside a SQL Server statement or unquoted `top` as a + SQL Server name; see the main CHANGELOG's Behavior changes. + +### Fixed +- **Document sync**: the server advertises the standard `textDocumentSync` + capability (it sent the non-standard `textDocumentSyncOptions`), so clients + that follow the specification send open, change and close events. +- **Crash safety**: a malformed `Content-Length`, an out-of-range + `didChange` range or a panic in a request handler or background diagnostics + no longer stops the server. +- **Edits with non-ASCII text**: `didChange` character offsets are read as + UTF-16 code units, and CRLF line endings are respected. +- **Unknown requests** are answered with `-32601` (MethodNotFound). +- **Outline**: the document outline no longer lists `GO` separators, and + statements before a `GO` line no longer extend to the end of the file. + ## [1.16.0] - 2026-10-22 Bundles the GoSQLX v1.16.0 language server. See the diff --git a/vscode-extension/package-lock.json b/vscode-extension/package-lock.json index 170d49c99..1d8a01c15 100644 --- a/vscode-extension/package-lock.json +++ b/vscode-extension/package-lock.json @@ -1,12 +1,12 @@ { "name": "gosqlx", - "version": "1.16.0", + "version": "1.17.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "gosqlx", - "version": "1.16.0", + "version": "1.17.0", "license": "Apache-2.0", "dependencies": { "vscode-languageclient": "^9.0.1" diff --git a/vscode-extension/package.json b/vscode-extension/package.json index c92192632..e7b438d3b 100644 --- a/vscode-extension/package.json +++ b/vscode-extension/package.json @@ -2,7 +2,7 @@ "name": "gosqlx", "displayName": "GoSQLX", "description": "High-performance SQL parsing, validation, formatting, and analysis powered by GoSQLX", - "version": "1.16.0", + "version": "1.17.0", "publisher": "ajitpratap0", "license": "Apache-2.0", "repository": { diff --git a/wasm/internal/bridge/bridge.go b/wasm/internal/bridge/bridge.go index 915a1400d..f400aa0d8 100644 --- a/wasm/internal/bridge/bridge.go +++ b/wasm/internal/bridge/bridge.go @@ -28,6 +28,7 @@ package bridge import ( "errors" + "fmt" "github.com/ajitpratap0/GoSQLX/pkg/advisor" "github.com/ajitpratap0/GoSQLX/pkg/gosqlx" @@ -55,6 +56,70 @@ func Parse(sql, dialect string) (*ast.AST, error) { return gosqlx.Parse(sql) } +// TextResponse is the result of Format and Transpile: the rewritten SQL, or +// the reason there is none. +// +// Exactly one of Result and Error is set. ErrorSpan locates Error in the +// submitted SQL when the error carries a position (a tokenizer or parser +// error); errors such as formatter.ErrUnplaceableComment, an unsupported +// dialect or a construct the transpile target cannot express have none. +// +// ErrorSpan is part of the JSON contract for callers that want it. The +// website's WasmLoader.parseResult treats a non-empty "error" like every other +// export does and throws it as an exception, so the playground's Format and +// Transpile tabs show only the error message, not the span. +type TextResponse struct { + Result string `json:"result"` + Error string `json:"error,omitempty"` + ErrorSpan *ErrorSpan `json:"error_span,omitempty"` +} + +func textResponse(sql, result string, err error) TextResponse { + if err != nil { + return TextResponse{Error: err.Error(), ErrorSpan: errorSpan(sql, err)} + } + return TextResponse{Result: result} +} + +// Format formats sql with gosqlx.FormatWithDialect and the default format +// options, tokenizing, parsing and rendering it for the selected playground +// dialect ("generic", empty or unknown names use the generic rules). +// Comments and optimizer hints are kept; SQL whose comments cannot be placed +// faithfully is refused with an error wrapping formatter.ErrUnplaceableComment. +func Format(sql, dialect string) TextResponse { + out, err := gosqlx.FormatWithDialect(sql, gosqlx.DefaultFormatOptions(), Dialect(dialect)) + return textResponse(sql, out, err) +} + +// transpileDialect resolves a transpile source or target. Unlike Dialect, an +// unknown name is an error: silently transpiling for a different dialect than +// the one asked for would show a misleading result. Empty means generic. +func transpileDialect(role, name string) (sqlkeywords.SQLDialect, error) { + if name == "" { + return sqlkeywords.DialectGeneric, nil + } + if !sqlkeywords.IsValidDialect(name) { + return "", fmt.Errorf("unsupported %s dialect %q", role, name) + } + return sqlkeywords.SQLDialect(name), nil +} + +// Transpile rewrites sql from one playground dialect to another with +// gosqlx.Transpile. SQL that does not parse in the source dialect, or that +// uses a construct the target cannot express, yields an error. +func Transpile(sql, from, to string) TextResponse { + src, err := transpileDialect("source", from) + if err != nil { + return textResponse(sql, "", err) + } + dst, err := transpileDialect("target", to) + if err != nil { + return textResponse(sql, "", err) + } + out, err := gosqlx.Transpile(sql, src, dst) + return textResponse(sql, out, err) +} + // LintViolation is one lint finding as the playground renders it. type LintViolation struct { Rule string `json:"rule"` diff --git a/wasm/internal/bridge/bridge_test.go b/wasm/internal/bridge/bridge_test.go index 2c5f7aca7..a3f2eb502 100644 --- a/wasm/internal/bridge/bridge_test.go +++ b/wasm/internal/bridge/bridge_test.go @@ -19,6 +19,7 @@ import ( "strings" "testing" + "github.com/ajitpratap0/GoSQLX/pkg/formatter" sqlkeywords "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" ) @@ -223,3 +224,125 @@ func TestDialect(t *testing.T) { } } } + +func TestFormat(t *testing.T) { + tests := []struct { + name string + sql string + dialect string + want string + }{ + {name: "sqlserver keeps TOP", sql: "SELECT TOP 10 * FROM t", dialect: "sqlserver", want: "SELECT TOP 10 *\nFROM t"}, + {name: "leading comment kept", sql: "-- c\nSELECT 1", dialect: "generic", want: "-- c\nSELECT 1"}, + {name: "inline comment moves above the statement", sql: "-- top\nSELECT /* inline */ a FROM t", dialect: "sqlserver", want: "-- top\n/* inline */\nSELECT a\nFROM t"}, + {name: "empty dialect is generic", sql: "select id,name from users where id>5", dialect: "", want: "SELECT id, name\nFROM users\nWHERE id > 5"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resp := Format(tt.sql, tt.dialect) + if resp.Error != "" { + t.Fatalf("unexpected error: %s", resp.Error) + } + if resp.Result != tt.want { + t.Fatalf("Format(%q, %q) = %q, want %q", tt.sql, tt.dialect, resp.Result, tt.want) + } + }) + } +} + +// TestFormat_UsesDialect guards against formatting with the generic rules: +// TOP does not parse there, so the result depends on the selected dialect. +func TestFormat_UsesDialect(t *testing.T) { + resp := Format("SELECT TOP 10 * FROM t", "generic") + if resp.Error == "" { + t.Fatalf("generic should reject TOP; got %q", resp.Result) + } + if resp.ErrorSpan == nil || resp.ErrorSpan.Line != 1 { + t.Fatalf("parse error should carry a span, got %+v", resp.ErrorSpan) + } +} + +func TestFormat_UnplaceableComment(t *testing.T) { + sql := "SELECT a FROM t /*+ hint */ WHERE x = 1" + resp := Format(sql, "generic") + if resp.Result != "" || !strings.Contains(resp.Error, formatter.ErrUnplaceableComment.Error()) { + t.Fatalf("expected an unplaceable-comment error, got %+v", resp) + } + if resp.ErrorSpan != nil { + t.Fatalf("a comment error has no parser position, got span %+v", resp.ErrorSpan) + } +} + +func TestTranspile(t *testing.T) { + tests := []struct { + name string + sql string + from, to string + want string + }{ + {name: "backticks to double quotes", sql: "SELECT `Order Id` FROM t", from: "mysql", to: "postgresql", want: `SELECT "Order Id" FROM t`}, + {name: "string escapes retargeted", sql: `SELECT a FROM t WHERE n = 'it\'s'`, from: "mysql", to: "postgresql", want: "SELECT a FROM t WHERE n = 'it''s'"}, + {name: "pg power to POWER()", sql: "SELECT 2 ^ 3", from: "postgresql", to: "mysql", want: "SELECT POWER(2, 3)"}, + {name: "mysql xor to pg #", sql: "SELECT a ^ b FROM t", from: "mysql", to: "postgresql", want: "SELECT a # b FROM t"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resp := Transpile(tt.sql, tt.from, tt.to) + if resp.Error != "" { + t.Fatalf("unexpected error: %s", resp.Error) + } + if resp.Result != tt.want { + t.Fatalf("Transpile(%q, %s, %s) = %q, want %q", tt.sql, tt.from, tt.to, resp.Result, tt.want) + } + }) + } +} + +func TestTranspile_Errors(t *testing.T) { + tests := []struct { + name string + sql string + from, to string + contains string + span bool + }{ + {name: "xor has no oracle form", sql: "SELECT a ^ b FROM t", from: "mysql", to: "oracle", contains: "no equivalent in oracle"}, + {name: "source parse error", sql: invalidSQL, from: "mysql", to: "postgresql", contains: "E2002", span: true}, + {name: "unknown source", sql: "SELECT 1", from: "nope", to: "postgresql", contains: `source dialect "nope"`}, + {name: "unknown target", sql: "SELECT 1", from: "mysql", to: "nope", contains: `target dialect "nope"`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resp := Transpile(tt.sql, tt.from, tt.to) + if resp.Result != "" || !strings.Contains(resp.Error, tt.contains) { + t.Fatalf("expected an error containing %q, got %+v", tt.contains, resp) + } + if (resp.ErrorSpan != nil) != tt.span { + t.Fatalf("span = %+v, want present=%v", resp.ErrorSpan, tt.span) + } + }) + } +} + +func TestTextResponse_JSONContract(t *testing.T) { + ok, err := json.Marshal(Transpile("SELECT 1", "", "")) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if string(ok) != `{"result":"SELECT 1"}` { + t.Fatalf("success payload = %s", ok) + } + bad, err := json.Marshal(Format(invalidSQL, "generic")) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var raw map[string]any + if err := json.Unmarshal(bad, &raw); err != nil { + t.Fatalf("unmarshal: %v", err) + } + for _, key := range []string{"result", "error", "error_span"} { + if _, ok := raw[key]; !ok { + t.Errorf("error payload missing %q: %s", key, bad) + } + } +} diff --git a/wasm/internal/bridge/contract_test.go b/wasm/internal/bridge/contract_test.go index 265e0cd05..75899d7ec 100644 --- a/wasm/internal/bridge/contract_test.go +++ b/wasm/internal/bridge/contract_test.go @@ -36,6 +36,8 @@ func contractSample() any { Lint LintResponse `json:"lint"` Analyze AnalyzeResponse `json:"analyze"` Highlight HighlightResponse `json:"highlight"` + Format TextResponse `json:"format"` + Transpile TextResponse `json:"transpile"` }{ Lint: LintResponse{ ParseError: "parse error", @@ -67,6 +69,8 @@ func contractSample() any { Highlight: HighlightResponse{ Spans: []HighlightSpan{{Start: 0, End: 6, Category: "keyword"}}, }, + Format: TextResponse{Result: "SELECT 1", Error: "format error", ErrorSpan: sampleSpan}, + Transpile: TextResponse{Result: "SELECT 1", Error: "transpile error", ErrorSpan: sampleSpan}, } } diff --git a/wasm/main.go b/wasm/main.go index 1091a2a6f..0a87539cb 100644 --- a/wasm/main.go +++ b/wasm/main.go @@ -20,7 +20,6 @@ import ( "encoding/json" "syscall/js" - "github.com/ajitpratap0/GoSQLX/pkg/formatter" "github.com/ajitpratap0/GoSQLX/pkg/gosqlx" "github.com/ajitpratap0/GoSQLX/wasm/internal/bridge" ) @@ -49,18 +48,28 @@ func parse(_ js.Value, args []js.Value) any { return string(b) } +// format formats the SQL for the selected dialect, keeping comments and +// optimizer hints. It returns {"result": "..."} or {"error": "...", +// "error_span": {...}} (bridge.TextResponse). func format(_ js.Value, args []js.Value) any { if len(args) == 0 { return jsonError("no SQL provided") } - sql := args[0].String() - // dialect is accepted but formatter currently uses generic formatting + return jsonResult(bridge.Format(args[0].String(), dialectArg(args))) +} - result, err := formatter.FormatString(sql) - if err != nil { - return jsonError(err.Error()) +// transpile rewrites the SQL from the dialect in the second argument to the +// one in the third. It returns the same envelope as format. +func transpile(_ js.Value, args []js.Value) any { + if len(args) < 3 { + return jsonError("transpile needs SQL, a source dialect and a target dialect") + } + for _, a := range args[:3] { + if a.Type() != js.TypeString { + return jsonError("transpile arguments must be strings") + } } - return result + return jsonResult(bridge.Transpile(args[0].String(), args[1].String(), args[2].String())) } // lint runs the full built-in rule set (L001-L030, the same rules as @@ -129,6 +138,7 @@ func main() { js.Global().Set("gosqlxValidate", js.FuncOf(validate)) js.Global().Set("gosqlxAnalyze", js.FuncOf(analyze)) js.Global().Set("gosqlxHighlight", js.FuncOf(highlight)) + js.Global().Set("gosqlxTranspile", js.FuncOf(transpile)) // Keep the Go program running select {} diff --git a/website/src/app/playground/page.tsx b/website/src/app/playground/page.tsx index 082140149..530214d85 100644 --- a/website/src/app/playground/page.tsx +++ b/website/src/app/playground/page.tsx @@ -9,7 +9,7 @@ import { PROJECT } from '@/data/metrics'; export const metadata: Metadata = pageMeta({ title: 'SQL Playground', description: - 'Try GoSQLX in your browser: paste SQL to parse it into an AST, format, lint and analyze it across dialects, with no install and no server round trip.', + 'Try GoSQLX in your browser: parse SQL into an AST, format, lint and analyze it, or transpile it between dialects, with no install and no server round trip.', path: '/playground/', }); @@ -52,10 +52,11 @@ export default function PlaygroundPage() { Online Go SQL Parser Playground

- Parse, format, lint, and analyze SQL right in your browser with - GoSQLX compiled to WebAssembly. Pick a dialect (PostgreSQL, MySQL, + Parse, format, lint, analyze and transpile SQL right in your browser + with GoSQLX compiled to WebAssembly. Pick a dialect (PostgreSQL, MySQL, SQL Server, Oracle, SQLite, Snowflake, ClickHouse), paste a query, and - inspect the tokens and AST. Nothing leaves your machine. + inspect the tokens and AST, or convert it to another dialect. Nothing + leaves your machine.