test(realworld): add multi-dialect real-world corpus with ratchet

CHANGELOG.md

@@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+- **Real-world dialect corpus**: `testdata/realworld/<dialect>/` holds 2,788
+  production-style statements across PostgreSQL, MySQL, MariaDB, SQL Server,
+  Oracle, SQLite, Snowflake and ClickHouse. `TestRealWorldCorpus` runs each
+  through split, parse, format, reparse, idempotency and dropped-word checks
+  against a ratchet list (`testdata/realworld/known_failures.txt`, regenerate
+  with `-update-known-failures`): a new failure fails the test, and so does a
+  listed failure that now passes. `scripts/dialect-audit` runs the same checks
+  on any SQL file; `BenchmarkRealWorldParse` and `BenchmarkRealWorldParseFormat`
+  measure the corpus.
+
 ### Fixed
 - **`ExtractColumns` / `ExtractColumnsQualified` no longer report SELECT aliases**
   (#849): an unqualified name in `ORDER BY` or `HAVING` that matches a

internal/dropcheck/dropcheck.go

@@ -0,0 +1,148 @@
+// Copyright 2026 GoSQLX Authors
+// SPDX-License-Identifier: Apache-2.0
+
+// Package dropcheck is a cheap detector for clauses a formatter silently
+// drops even though its output re-parses: it reports words of the input
+// (outside comments and string literals) that are missing from the output.
+// It is shared by the real-world corpus test and scripts/dialect-audit.
+package dropcheck
+
+import (
+	"regexp"
+	"strings"
+)
+
+// DroppedWords reports words of the input (outside comments and string
+// literals) that are missing from the formatted output — a cheap detector for
+// clauses the formatter silently drops even though its output reparses.
+func DroppedWords(in, out string) []string {
+	in = CanonicalSynonyms(in)
+	have := map[string]bool{}
+	for _, w := range Words(out) {
+		have[w] = true
+	}
+	var lost []string
+	seen := map[string]bool{}
+	for _, w := range Words(in) {
+		if !have[w] && !ignorable[w] && !seen[w] {
+			seen[w] = true
+			lost = append(lost, w)
+		}
+	}
+	return lost
+}
+
+// synonyms matches spellings the formatter writes in their canonical
+// form because the dialects define them as exact synonyms: SELECT ALL (ALL is
+// the default quantifier), DISTINCTROW (MySQL) and SELECT UNIQUE (Oracle) for
+// DISTINCT, and the postfix
+// ISNULL / NOTNULL (PostgreSQL, SQLite) for IS NULL / IS NOT NULL. UNION ALL
+// and the function ISNULL(...) are not touched.
+var synonyms = regexp.MustCompile(`(?i)\b(SELECT\s+ALL|SELECT\s+UNIQUE|DISTINCTROW|ISNULL|NOTNULL)\b`)
+
+// CanonicalSynonyms rewrites the synonyms of synonyms in s
+// to the canonical spelling the formatter writes.
+func CanonicalSynonyms(s string) string {
+	var sb strings.Builder
+	last := 0
+	for _, m := range synonyms.FindAllStringIndex(s, -1) {
+		word := strings.ToUpper(s[m[0]:m[1]])
+		rest := strings.TrimLeft(s[m[1]:], " \t\r\n")
+		var repl string
+		switch {
+		case strings.HasPrefix(word, "SELECT") && strings.HasSuffix(word, "UNIQUE"):
+			repl = "SELECT DISTINCT"
+		case strings.HasPrefix(word, "SELECT"):
+			repl = "SELECT"
+		case word == "DISTINCTROW":
+			repl = "DISTINCT"
+		case strings.HasPrefix(rest, "("): // the ISNULL(...) function
+			continue
+		case word == "ISNULL":
+			repl = "IS NULL"
+		default:
+			repl = "IS NOT NULL"
+		}
+		sb.WriteString(s[last:m[0]])
+		sb.WriteString(repl)
+		last = m[1]
+	}
+	sb.WriteString(s[last:])
+	return sb.String()
+}
+
+// ignorable are noise words the formatter may legitimately add or drop.
+var ignorable = map[string]bool{"AS": true, "ASC": true, "INNER": true, "OUTER": true}
+
+// Words returns the upper-cased identifier-like words of s, skipping
+// comments and quoted string literals but keeping quoted identifiers' text.
+func Words(s string) []string {
+	var words []string
+	for i := 0; i < len(s); {
+		c := s[i]
+		switch {
+		case c == '-' && i+1 < len(s) && s[i+1] == '-':
+			for i < len(s) && s[i] != '\n' {
+				i++
+			}
+		case c == '/' && i+1 < len(s) && s[i+1] == '*':
+			end := strings.Index(s[i+2:], "*/")
+			if end < 0 {
+				return words
+			}
+			i += end + 4
+		case c == '$' && dollarQuoteEnd(s, i) > i:
+			i = dollarQuoteEnd(s, i)
+		case c == '\'':
+			for i++; i < len(s); i++ {
+				if s[i] == '\\' {
+					i++
+					continue
+				}
+				if s[i] == '\'' {
+					if i+1 < len(s) && s[i+1] == '\'' {
+						i++
+						continue
+					}
+					break
+				}
+			}
+			i++
+		case c == '_' || c >= 'A' && c <= 'Z' || c >= 'a' && c <= 'z' || c >= 0x80:
+			j := i
+			for j < len(s) && (s[j] == '_' || s[j] == '$' || s[j] >= '0' && s[j] <= '9' ||
+				s[j] >= 'A' && s[j] <= 'Z' || s[j] >= 'a' && s[j] <= 'z' || s[j] >= 0x80) {
+				j++
+			}
+			words = append(words, strings.ToUpper(s[i:j]))
+			i = j
+		case c >= '0' && c <= '9':
+			for i < len(s) && (s[i] == '.' || s[i] == '_' || s[i] >= '0' && s[i] <= '9' ||
+				s[i] >= 'A' && s[i] <= 'Z' || s[i] >= 'a' && s[i] <= 'z') {
+				i++
+			}
+		default:
+			i++
+		}
+	}
+	return words
+}
+
+// dollarQuoteEnd returns the offset just past the dollar-quoted
+// string ($$...$$ or $tag$...$tag$) that starts at s[i], or i when s[i] does
+// not start one.
+func dollarQuoteEnd(s string, i int) int {
+	j := i + 1
+	for j < len(s) && (s[j] == '_' || s[j] >= '0' && s[j] <= '9' || s[j] >= 'A' && s[j] <= 'Z' || s[j] >= 'a' && s[j] <= 'z') {
+		j++
+	}
+	if j >= len(s) || s[j] != '$' || (j > i+1 && s[i+1] >= '0' && s[i+1] <= '9') {
+		return i
+	}
+	tag := s[i : j+1]
+	end := strings.Index(s[j+1:], tag)
+	if end < 0 {
+		return i
+	}
+	return j + 1 + end + len(tag)
+}

internal/dropcheck/dropcheck_test.go

@@ -0,0 +1,85 @@
+// Copyright 2026 GoSQLX Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package dropcheck
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestDroppedWordsCanonicalSynonyms(t *testing.T) {
+	tests := []struct {
+		name string
+		in   string
+		out  string
+		want []string
+	}{
+		{"select all", "SELECT ALL name FROM users", "SELECT name FROM users", nil},
+		{"select unique", "SELECT UNIQUE name FROM users", "SELECT DISTINCT name FROM users", nil},
+		{"distinctrow", "SELECT DISTINCTROW name FROM users", "SELECT DISTINCT name FROM users", nil},
+		{"postfix isnull", "SELECT x ISNULL FROM users", "SELECT x IS NULL FROM users", nil},
+		{"postfix notnull", "SELECT x NOTNULL FROM users", "SELECT x IS NOT NULL FROM users", nil},
+		{"isnull function preserved", "SELECT ISNULL(x, y) FROM users", "SELECT x FROM users", []string{"ISNULL", "Y"}},
+		{"union all retained", "SELECT x FROM a UNION ALL SELECT x FROM b", "SELECT x FROM a UNION SELECT x FROM b", []string{"ALL"}},
+		{"dropped clause words", "SELECT x FROM users WHERE enabled = true", "SELECT x FROM users", []string{"WHERE", "ENABLED", "TRUE"}},
+		{"dollar-quoted literal contents ignored", "SELECT $body$WHERE missing$body$ FROM users", "SELECT 'rewritten' FROM users", nil},
+		{"quoted and commented words ignored", "SELECT x, 'missing string' FROM users -- missing comment", "SELECT x FROM users", nil},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := DroppedWords(tt.in, tt.out); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("DroppedWords(%q, %q) = %v, want %v", tt.in, tt.out, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestSQLWordsSkipsDollarQuotedLiteral(t *testing.T) {
+	tests := []struct {
+		name string
+		sql  string
+		want []string
+	}{
+		{"untagged", "SELECT $$WHERE missing$$ FROM users", []string{"SELECT", "FROM", "USERS"}},
+		{"tagged", "SELECT $body$WHERE missing$body$ FROM users", []string{"SELECT", "FROM", "USERS"}},
+		{"regular identifier", "SELECT $1, my$column FROM users", []string{"SELECT", "MY$COLUMN", "FROM", "USERS"}},
+		{"quoted identifier retained", "SELECT \"WHERE\" FROM users", []string{"SELECT", "WHERE", "FROM", "USERS"}},
+		{"ordinary string and comments", "SELECT 'WHERE' FROM users /* missing */ -- other\nWHERE live", []string{"SELECT", "FROM", "USERS", "WHERE", "LIVE"}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := Words(tt.sql); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("Words(%q) = %v, want %v", tt.sql, got, tt.want)
+			}
+		})
+	}
+}
+
+// TestDroppedWordsCorpusCases pins what the corpus drop detector accepts as a
+// canonical spelling and what it still reports.
+func TestDroppedWordsCorpusCases(t *testing.T) {
+	tests := []struct {
+		name, in, out string
+		lost          string // space-separated, "" when nothing is lost
+	}{
+		{"select all", "SELECT ALL a FROM t", "SELECT a FROM t", ""},
+		{"union all is not a synonym", "SELECT a FROM t UNION ALL SELECT a FROM u", "SELECT a FROM t UNION SELECT a FROM u", "ALL"},
+		{"oracle select unique", "SELECT UNIQUE a FROM t", "SELECT DISTINCT a FROM t", ""},
+		{"distinctrow", "SELECT DISTINCTROW a FROM t", "SELECT DISTINCT a FROM t", ""},
+		{"postfix isnull", "SELECT a FROM t WHERE a ISNULL", "SELECT a FROM t WHERE a IS NULL", ""},
+		{"postfix notnull", "SELECT a FROM t WHERE a NOTNULL", "SELECT a FROM t WHERE a IS NOT NULL", ""},
+		{"isnull function is not a synonym", "SELECT ISNULL(a, 0) FROM t", "SELECT a FROM t", "ISNULL"},
+		{"dollar quoted strings", "SELECT $$dollar quoted$$, $tag$ body 'with' quotes $tag$", "SELECT 'dollar quoted', ' body ''with'' quotes '", ""},
+		{"dropped clause", "SELECT a FROM t WHERE b = 1", "SELECT a FROM t", "WHERE B"},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			got := strings.Join(DroppedWords(tc.in, tc.out), " ")
+			if got != tc.lost {
+				t.Errorf("lost words = %q, want %q", got, tc.lost)
+			}
+		})
+	}
+}

pkg/formatter/nested_semicolon_test.go

@@ -127,6 +127,12 @@ func TestFormatAST_ReadableSemicolonOnlyTerminatesStatements(t *testing.T) {
 		if werr != nil {
 			return werr
 		}
+		// The real-world corpus has its own ratchet (TestRealWorldCorpus) and
+		// still holds statements whose ';' placement is wrong; the dialect
+		// fixes that repair them remove this skip.
+		if d.IsDir() && d.Name() == "realworld" {
+			return filepath.SkipDir
+		}
 		if d.IsDir() || !strings.HasSuffix(path, ".sql") {
 			return nil
 		}

pkg/gosqlx/realworld_bench_test.go

@@ -0,0 +1,102 @@
+// Copyright 2026 GoSQLX Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gosqlx
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/ajitpratap0/GoSQLX/pkg/sql/keywords"
+)
+
+type realworldStmt struct {
+	dialect keywords.SQLDialect
+	sql     string
+}
+
+// loadRealworldParseable returns every statement of testdata/realworld that
+// currently parses in its dialect, so the benchmarks measure the user path on
+// production-shaped SQL rather than on synthetic one-liners.
+func loadRealworldParseable(b *testing.B) ([]realworldStmt, int64) {
+	b.Helper()
+	dirs, err := os.ReadDir(realworldRoot)
+	if err != nil {
+		b.Skipf("testdata/realworld not available: %v", err)
+	}
+	var stmts []realworldStmt
+	var bytes int64
+	for _, d := range dirs {
+		if !d.IsDir() {
+			continue
+		}
+		dialect := keywords.SQLDialect(d.Name())
+		files, _ := filepath.Glob(filepath.Join(realworldRoot, d.Name(), "*.sql"))
+		for _, file := range files {
+			data, err := os.ReadFile(file)
+			if err != nil {
+				b.Fatalf("read %s: %v", file, err)
+			}
+			for _, stmt := range SplitStatements(string(data), d.Name()) {
+				if onlyComments(stmt) {
+					continue
+				}
+				if _, err := ParseWithDialect(stmt, dialect); err != nil {
+					continue
+				}
+				stmts = append(stmts, realworldStmt{dialect, stmt})
+				bytes += int64(len(stmt))
+			}
+		}
+	}
+	if len(stmts) == 0 {
+		b.Skip("no parseable real-world statements")
+	}
+	return stmts, bytes
+}
+
+// BenchmarkRealWorldParse parses the whole parseable real-world corpus once
+// per iteration with the statement's own dialect.
+func BenchmarkRealWorldParse(b *testing.B) {
+	stmts, size := loadRealworldParseable(b)
+	b.SetBytes(size)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		for _, s := range stmts {
+			if _, err := ParseWithDialect(s.sql, s.dialect); err != nil {
+				b.Fatal(err)
+			}
+		}
+	}
+}
+
+// BenchmarkRealWorldParseFormat measures the format path users run in the
+// CLI and LSP: parse with dialect, then render.
+func BenchmarkRealWorldParseFormat(b *testing.B) {
+	stmts, size := loadRealworldParseable(b)
+	b.SetBytes(size)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		for _, s := range stmts {
+			tree, err := ParseWithDialect(s.sql, s.dialect)
+			if err != nil {
+				b.Fatal(err)
+			}
+			_ = FormatAST(tree)
+		}
+	}
+}