fix(tokenizer,ast): lex literals per dialect; keep operator meaning across dialects

CHANGELOG.md

@@ -79,6 +79,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   in dialects without them.
 
 ### Fixed
+- Preserve dialect numeric and PostgreSQL Unicode-escape literals and quoted type names; correct JSON/power precedence and retain operator meaning during target-dialect formatting. Hand-built expression trees, including hand-built operands inserted into parsed trees, now render necessary parentheses.
+- Formatting or transpiling to another dialect rewrites PostgreSQL `#` (XOR),
+  `~`, `<<`/`>>`, `&`/`|` and `||` (CONCAT in MySQL/MariaDB) into the target's
+  form, and refuses with an error where the target has none (for example `#`
+  in Oracle/SQLite, `@>`/`?`/regex `~` outside PostgreSQL), instead of
+  emitting them unchanged (`a # b` read as `a` plus a comment in MySQL).
 - Scan raw dialect statements and standalone VALUES for injection patterns, recover
   panics at every C export, and guard unlinked tokenizer scratch calls.
 - Formatter lower-case mode covers expression and built-in function keywords;
@@ -192,6 +198,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   longer read as MariaDB's `ANALYZE <statement>`.
 
 ### Behavior changes
+- Numeric words without whitespace no longer silently become a number plus a
+  column alias. Numeric spelling is retained. Differences from the previous lexer:
+
+  | Dialect | Numeric changes | Unicode-escape changes |
+  | --- | --- | --- |
+  | PostgreSQL | `1_000`, `1_000.000_1`, `0o17`, `0b101`, and base-prefix separators such as `0x_FF` are single numbers. Malformed forms (`1x`, `0x`, `1__0`, `1_`, `1FROM`) error. | `U&'...'` with optional `UESCAPE` (comments allowed before it) is validated and preserved as one literal; `U&"..."` decodes to a quoted identifier usable wherever a name is. Malformed escapes error. |
+  | MySQL / MariaDB | `0b101` is a single bit literal; `1x`, `1_000`, and bare `0x` are identifiers rather than number/alias pairs. A number with a fraction still ends before a name (`1.5a` is `1.5 AS a`). | Unchanged: `U&'x'` remains separate `U`, `&`, and string tokens. |
+  | SQLite | `1_000` is one number; numeric trailing junk and unsupported `0b101` error. | Unchanged. |
+  | ClickHouse | `1_000` and `0b101` are single numbers; numeric trailing junk errors. | Unchanged. |
+  | SQL Server | Numeric trailing junk, including `1_000` and `0b101`, errors; a keyword written against a number (`1AS x`) is still accepted. | Unchanged. |
+  | Oracle | `1d`, `1.5f`, `1e5D` (BINARY_DOUBLE/BINARY_FLOAT) are single numbers. Storage sizes (`100M`, `8K`, `2G`) keep their units wherever a size appears; keywords against a number (`1FROM`) are accepted; other trailing junk, `1_000` and `0b101` error. | Unchanged. |
+  | Snowflake / BigQuery / Redshift | Numeric trailing junk, `0b101` and `1_000` error. | Unchanged. |
+  | generic | As Oracle for storage sizes and keywords; `1d`, `1x`, `1_000`, `0b101` error. | Unchanged. |
+
+  `0x1F` remains a single hex literal in every dialect.
 - SQL Server: a line holding only `GO [n]` inside a statement is now a parse
   error ("GO batch separator inside a statement"), matching sqlcmd, instead of
   a column or alias named `go`. `GO 0` / `GO 1.5` alone on a line report

cmd/gosqlx/cmd/format_errors_test.go

@@ -0,0 +1,33 @@
+// Copyright 2026 GoSQLX Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package cmd
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/ajitpratap0/GoSQLX/pkg/gosqlx"
+	"github.com/ajitpratap0/GoSQLX/pkg/sql/ast"
+	"github.com/ajitpratap0/GoSQLX/pkg/sql/keywords"
+)
+
+func TestSQLFormatterRejectsUnsupportedXor(t *testing.T) {
+	tree, err := gosqlx.ParseWithDialect("SELECT a ^ b", keywords.DialectMySQL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ast.ReleaseAST(tree)
+	f := NewSQLFormatter(FormatterOptions{Dialect: "sqlite"})
+	for _, comments := range []bool{false, true} {
+		var out string
+		if comments {
+			out, err = f.FormatWithComments(tree, nil, nil)
+		} else {
+			out, err = f.Format(tree)
+		}
+		if err == nil || !strings.Contains(err.Error(), "XOR") || out != "" {
+			t.Fatalf("comments=%v: got %q, %v; want XOR error", comments, out, err)
+		}
+	}
+}

cmd/gosqlx/cmd/formatter.go

@@ -401,8 +401,14 @@ func verifyFormattedSQL(original *ast.AST, inputTokens []models.TokenWithSpan, f
 	}
 	canonical := NewSQLFormatter(FormatterOptions{Compact: true, UppercaseKw: true, Dialect: dialect}).style
 	for i, stmt := range original.Statements {
-		want := pkgformatter.FormatStatement(stmt, canonical)
-		got := pkgformatter.FormatStatement(reparsed.Statements[i], canonical)
+		want, err := pkgformatter.FormatStatementChecked(stmt, canonical)
+		if err != nil {
+			return fmt.Errorf("cannot format original statement %d: %w", i+1, err)
+		}
+		got, err := pkgformatter.FormatStatementChecked(reparsed.Statements[i], canonical)
+		if err != nil {
+			return fmt.Errorf("cannot format reparsed statement %d: %w", i+1, err)
+		}
 		if !strings.EqualFold(want, got) {
 			return fmt.Errorf("formatted output changes statement %d (formatter bug; input left unchanged)", i+1)
 		}

cmd/gosqlx/cmd/sql_formatter.go

@@ -16,6 +16,7 @@ package cmd
 
 import (
 	"errors"
+	"fmt"
 	"strings"
 
 	pkgformatter "github.com/ajitpratap0/GoSQLX/pkg/formatter"
@@ -26,7 +27,7 @@ import (
 // SQLFormatter renders a parsed AST as SQL text for the format command.
 //
 // It is a thin adapter over pkg/formatter: every statement is rendered with
-// pkgformatter.FormatStatement, so the CLI and the library share a single
+// pkgformatter.FormatStatementChecked, so the CLI and the library share a single
 // renderer and the CLI has no renderer of its own to fall out of sync (#599).
 // Rendering gaps in pkg/formatter are caught, where detectable, by
 // verifyFormattedSQL.
@@ -106,7 +107,11 @@ func (f *SQLFormatter) formatStatements(astObj *ast.AST) ([]string, error) {
 	}
 	stmts := make([]string, len(astObj.Statements))
 	for i, stmt := range astObj.Statements {
-		stmts[i] = pkgformatter.FormatStatement(stmt, f.style)
+		formatted, err := pkgformatter.FormatStatementChecked(stmt, f.style)
+		if err != nil {
+			return nil, fmt.Errorf("cannot format statement %d: %w", i+1, err)
+		}
+		stmts[i] = formatted
 	}
 	return stmts, nil
 }

pkg/cbinding/format_errors_test.go

@@ -0,0 +1,28 @@
+// Copyright 2026 GoSQLX Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+)
+
+func TestFormatResultReportsNumericTrailingJunk(t *testing.T) {
+	for _, dialect := range []string{"", "postgresql"} {
+		result := formatResultFor("SELECT 1x", dialect)
+		if result.Success || result.Formatted != "" || !strings.Contains(result.Error, "numeric literal") {
+			t.Fatalf("dialect=%q: %+v; want error result", dialect, result)
+		}
+		raw, err := json.Marshal(result)
+		if err != nil {
+			t.Fatal(err)
+		}
+		decoded := decodeMap(t, string(raw))
+		success, ok := decoded["success"].(bool)
+		if !ok || success || errorOf(decoded) == "" {
+			t.Fatalf("error JSON = %s", raw)
+		}
+	}
+}

pkg/fingerprint/fingerprint.go

@@ -117,7 +117,11 @@ func Normalize(sql string) (string, error) {
 	opts := ast.CompactStyle()
 	var parts []string
 	for _, stmt := range astObj.Statements {
-		parts = append(parts, formatter.FormatStatement(stmt, opts))
+		formatted, err := formatter.FormatStatementChecked(stmt, opts)
+		if err != nil {
+			return "", fmt.Errorf("fingerprint: formatting failed: %w", err)
+		}
+		parts = append(parts, formatted)
 	}
 
 	return strings.Join(parts, "; "), nil

pkg/fingerprint/format_errors_test.go

@@ -0,0 +1,18 @@
+// Copyright 2026 GoSQLX Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package fingerprint_test
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/ajitpratap0/GoSQLX/pkg/fingerprint"
+)
+
+func TestNormalizeRejectsNumericTrailingJunk(t *testing.T) {
+	out, err := fingerprint.Normalize("SELECT 1x")
+	if err == nil || !strings.Contains(err.Error(), "tokenization failed") || out != "" {
+		t.Fatalf("got %q, %v; want tokenization error", out, err)
+	}
+}

pkg/formatter/caret_dialect_test.go

@@ -0,0 +1,141 @@
+// Copyright 2026 GoSQLX Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package formatter_test
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/ajitpratap0/GoSQLX/pkg/formatter"
+	"github.com/ajitpratap0/GoSQLX/pkg/gosqlx"
+	"github.com/ajitpratap0/GoSQLX/pkg/sql/ast"
+	"github.com/ajitpratap0/GoSQLX/pkg/sql/keywords"
+)
+
+func TestCaretTargetDialect(t *testing.T) {
+	tests := []struct {
+		source, target keywords.SQLDialect
+		in, contains   string
+		wantError      bool
+	}{
+		{"postgresql", "mysql", "SELECT a ^ (b + c) * d", "POWER(a, (b + c)) * d", false},
+		{"postgresql", "sqlserver", "SELECT a ^ b", "POWER(a, b)", false},
+		{"postgresql", "oracle", "SELECT a ^ b", "POWER(a, b)", false},
+		{"postgresql", "sqlite", "SELECT a ^ b", "POWER(a, b)", false},
+		{"generic", "postgresql", "SELECT a -> b ^ 2", "(a -> b) ^ 2", false},
+		{"generic", "postgresql", "SELECT a ^ b -> c", "a ^ (b -> c)", false},
+		{"mysql", "postgresql", "SELECT a * b ^ c", "a * (b # c)", false},
+		{"mysql", "sqlserver", "SELECT a * b ^ c", "a * (b ^ c)", false},
+		{"sqlserver", "mysql", "SELECT a + b ^ c", "(a + b) ^ c", false},
+		{"sqlserver", "snowflake", "SELECT a ^ b", "BITXOR(a, b)", false},
+		{"mysql", "clickhouse", "SELECT a ^ b", "bitXor(a, b)", false},
+		{"mysql", "sqlite", "SELECT a ^ b", "", true},
+		{"mysql", "generic", "SELECT a ^ b", "", true},
+		{"postgresql", "mysql", `SELECT U&'d\0061t'`, "", true},
+		// PostgreSQL # (XOR), ~, shifts, || and PostgreSQL-only operators.
+		{"postgresql", "mysql", "SELECT a # b FROM t", "SELECT a ^ b FROM t", false},
+		{"postgresql", "mysql", "SELECT a # b * c", "a ^ (b * c)", false},
+		{"postgresql", "mariadb", "SELECT a * (b # c)", "a * (b ^ c)", false},
+		{"postgresql", "sqlserver", "SELECT a # b + c", "a ^ (b + c)", false},
+		{"postgresql", "snowflake", "SELECT a # b, ~a, a << 2, a | b", "BITXOR(a, b), BITNOT(a), BITSHIFTLEFT(a, 2), BITOR(a, b)", false},
+		{"postgresql", "clickhouse", "SELECT a # b, ~a, a >> 2, a & b", "bitXor(a, b), bitNot(a), bitShiftRight(a, 2), bitAnd(a, b)", false},
+		{"postgresql", "sqlserver", "SELECT a << 2", "LEFT_SHIFT(a, 2)", false},
+		{"postgresql", "oracle", "SELECT a & b FROM t", "BITAND(a, b)", false},
+		{"postgresql", "mysql", "SELECT a || b || c", "CONCAT(a, b, c)", false},
+		{"postgresql", "oracle", "SELECT a # b FROM t", "", true},
+		{"postgresql", "oracle", "SELECT ~a FROM t", "", true},
+		{"postgresql", "oracle", "SELECT a << 2 FROM t", "", true},
+		{"postgresql", "sqlite", "SELECT a # b", "", true},
+		{"postgresql", "mysql", "SELECT a @> b", "", true},
+		{"postgresql", "snowflake", "SELECT a ~ 'x'", "", true},
+	}
+	for _, tt := range tests {
+		t.Run(string(tt.source)+"/"+string(tt.target)+"/"+tt.in, func(t *testing.T) {
+			a, err := gosqlx.ParseWithDialect(tt.in, tt.source)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer ast.ReleaseAST(a)
+			before := a.SQL()
+			opts := ast.CompactStyle()
+			opts.Dialect = string(tt.target)
+			got, err := formatter.FormatASTChecked(a, opts)
+			if (err != nil) != tt.wantError {
+				t.Fatalf("output=%q error=%v", got, err)
+			}
+			if tt.wantError {
+				if legacy := formatter.FormatAST(a, opts); legacy != "" {
+					t.Errorf("unchecked returned unsafe SQL %q", legacy)
+				}
+				if trans, err := gosqlx.Transpile(tt.in, tt.source, tt.target); err == nil {
+					t.Errorf("transpiler returned %q, want error", trans)
+				}
+				return
+			}
+			if !strings.Contains(got, tt.contains) {
+				t.Errorf("got %q want fragment %q", got, tt.contains)
+			}
+			again, err := gosqlx.ParseWithDialect(got, tt.target)
+			if err != nil {
+				t.Fatalf("target reparse %q: %v", got, err)
+			}
+			defer ast.ReleaseAST(again)
+			if a.SQL() != before {
+				t.Errorf("source mutated: %q -> %q", before, a.SQL())
+			}
+			trans, err := gosqlx.Transpile(tt.in, tt.source, tt.target)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !strings.Contains(trans, tt.contains) {
+				t.Errorf("transpiler disagrees: %q", trans)
+			}
+		})
+	}
+}
+
+func TestFormatQuotedTypesForTarget(t *testing.T) {
+	for _, in := range []string{`SELECT CAST(x AS "MyType"), x::"char"`, `CREATE TABLE t (a "MyDomain")`} {
+		a, err := gosqlx.ParseWithDialect(in, keywords.DialectPostgreSQL)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer ast.ReleaseAST(a)
+		before := a.SQL()
+		opts := ast.CompactStyle()
+		opts.Dialect = "mysql"
+		out, err := formatter.FormatASTChecked(a, opts)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if strings.Contains(out, `"My`) || !strings.Contains(out, "`") {
+			t.Fatalf("target type quotes lost: %s", out)
+		}
+		b, err := gosqlx.ParseWithDialect(out, keywords.DialectMySQL)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer ast.ReleaseAST(b)
+		if a.SQL() != before {
+			t.Fatal("source mutated")
+		}
+	}
+}
+
+func TestFormatTargetDoesNotMutateMapExpressions(t *testing.T) {
+	a, err := gosqlx.ParseWithDialect(`SELECT * FROM t AT (TIMESTAMP => CAST('x' AS "MyType"))`, keywords.DialectSnowflake)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ast.ReleaseAST(a)
+	before := a.SQL()
+	opts := ast.CompactStyle()
+	opts.Dialect = "mysql"
+	if _, err := formatter.FormatASTChecked(a, opts); err != nil {
+		t.Fatal(err)
+	}
+	if got := a.SQL(); got != before {
+		t.Fatalf("source mutated: %q -> %q", before, got)
+	}
+}