diff --git a/.github/lighthouse/README.md b/.github/lighthouse/README.md
index 3659c83b..5dc23f6b 100644
--- a/.github/lighthouse/README.md
+++ b/.github/lighthouse/README.md
@@ -13,6 +13,7 @@ Budgets are the measured value plus roughly 20-30% headroom for CI variance.
|---|---|---|---|---|---|
| `/` | 0.63, 0.80, 0.80 | 339 KB | 114 KB | 18 KB | 2.90 MB (2.4 MB is non-script/non-font assets) |
| `/compare/` | 0.87, 0.89, 0.89 | 329 KB | 114 KB | 18 KB | 0.49 MB |
+| `/playground/` (CI median, 2026-10-20, PR #825) | - | 420 KB | - | - | - |
The other audited routes (`/playground/`, `/docs/`, `/docs/getting-started/`, `/benchmarks/`, `/changelog/`) share the
same layout chunks; their per-route numbers are in the uploaded Lighthouse artifacts. The local run was cut short by
@@ -26,6 +27,11 @@ a full disk, so only `/` and `/compare/` were measured locally.
| accessibility, seo | error | 0.95 |
| best-practices | warn | 0.85 |
| resource-summary script / font / stylesheet / total | error | 420 KB / 150 KB / 25 KB / 3.6 MB |
+| resource-summary script, `/playground/` only | error | 480 KB |
+
+`/playground/` loads the CodeMirror editor and the GoSQLX highlighting bridge, so it gets its own script budget
+via `assertMatrix` (the measured 420 KB plus ~15% headroom). Every other route keeps the shared 420 KB budget, and
+the two matrix entries are otherwise identical: change an assertion in both when editing thresholds.
Re-measure and update this table whenever the budgets are changed.
diff --git a/.github/lighthouse/lighthouserc.json b/.github/lighthouse/lighthouserc.json
index e9e6386e..d8deea07 100644
--- a/.github/lighthouse/lighthouserc.json
+++ b/.github/lighthouse/lighthouserc.json
@@ -1,16 +1,34 @@
{
"ci": {
"assert": {
- "assertions": {
- "categories:performance": ["error", { "minScore": 0.75, "aggregationMethod": "median" }],
- "categories:accessibility": ["error", { "minScore": 0.95, "aggregationMethod": "median" }],
- "categories:seo": ["error", { "minScore": 0.95, "aggregationMethod": "median" }],
- "categories:best-practices": ["warn", { "minScore": 0.85, "aggregationMethod": "median" }],
- "resource-summary:script:size": ["error", { "maxNumericValue": 420000, "aggregationMethod": "median" }],
- "resource-summary:font:size": ["error", { "maxNumericValue": 150000, "aggregationMethod": "median" }],
- "resource-summary:stylesheet:size": ["error", { "maxNumericValue": 25000, "aggregationMethod": "median" }],
- "resource-summary:total:size": ["error", { "maxNumericValue": 3600000, "aggregationMethod": "median" }]
- }
+ "assertMatrix": [
+ {
+ "matchingUrlPattern": "^(?!.*/playground/).*$",
+ "assertions": {
+ "categories:performance": ["error", { "minScore": 0.75, "aggregationMethod": "median" }],
+ "categories:accessibility": ["error", { "minScore": 0.95, "aggregationMethod": "median" }],
+ "categories:seo": ["error", { "minScore": 0.95, "aggregationMethod": "median" }],
+ "categories:best-practices": ["warn", { "minScore": 0.85, "aggregationMethod": "median" }],
+ "resource-summary:script:size": ["error", { "maxNumericValue": 420000, "aggregationMethod": "median" }],
+ "resource-summary:font:size": ["error", { "maxNumericValue": 150000, "aggregationMethod": "median" }],
+ "resource-summary:stylesheet:size": ["error", { "maxNumericValue": 25000, "aggregationMethod": "median" }],
+ "resource-summary:total:size": ["error", { "maxNumericValue": 3600000, "aggregationMethod": "median" }]
+ }
+ },
+ {
+ "matchingUrlPattern": ".*/playground/$",
+ "assertions": {
+ "categories:performance": ["error", { "minScore": 0.75, "aggregationMethod": "median" }],
+ "categories:accessibility": ["error", { "minScore": 0.95, "aggregationMethod": "median" }],
+ "categories:seo": ["error", { "minScore": 0.95, "aggregationMethod": "median" }],
+ "categories:best-practices": ["warn", { "minScore": 0.85, "aggregationMethod": "median" }],
+ "resource-summary:script:size": ["error", { "maxNumericValue": 480000, "aggregationMethod": "median" }],
+ "resource-summary:font:size": ["error", { "maxNumericValue": 150000, "aggregationMethod": "median" }],
+ "resource-summary:stylesheet:size": ["error", { "maxNumericValue": 25000, "aggregationMethod": "median" }],
+ "resource-summary:total:size": ["error", { "maxNumericValue": 3600000, "aggregationMethod": "median" }]
+ }
+ }
+ ]
}
}
}
diff --git a/CHANGELOG.md b/CHANGELOG.md
index bebfe5cc..40250a08 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -210,8 +210,40 @@ acceptance, CLI refusals), so read **Behavior changes** before upgrading.
`OVER (w)`, and `WINDOW w AS (...)` no longer repeats its name inside the
parentheses. The non-standard `WINDOW w AS w2` is normalized to
`WINDOW w AS (w2)`. (#608, #629)
+- **LSP semantic token legend changed**: the non-standard `identifier` token
+ type is replaced by the standard `variable`, which now covers table, column
+ and alias names and quoted identifiers (double-quoted identifiers were
+ `string` before). Function names are `function` and type names are `type`.
+ Comments and placeholders (`$1`, `?`, `:name`, `@var`) are now emitted, as
+ `comment` and `parameter`. AND, OR and NOT stay `keyword`. Positions are
+ UTF-16 code units. Clients with custom rules keyed on the old `identifier`
+ type (for example Neovim `@lsp.type.identifier` highlight links) must switch
+ to `variable`.
### Added
+- **SQL syntax highlighting (`pkg/highlight`)**: `highlight.Spans(sql, opts...)`
+ classifies SQL into sorted, non-overlapping byte-offset spans tagged with one
+ of 13 stable categories (keyword, function, type, identifier,
+ quoted-identifier, string, number, constant, operator, punctuation, comment,
+ placeholder, error). It is dialect-aware via `WithDialect`, never fails
+ (input after the first lexical error becomes one `error` span), and detects
+ functions in call position and types in type position. Renderers: `ANSI` /
+ `WriteANSI` (256-color, true color or attributes only), `HTML` / `WriteHTML`
+ (classes or inline styles) and `CSS`, with `dark`, `light` and `mono` themes.
+ Spans marshal to JSON. Built on top of it:
+ - `gosqlx.Highlight`, `gosqlx.HighlightANSI` and `gosqlx.HighlightHTML`.
+ - `gosqlx highlight [file|SQL]` (alias `hl`, reads stdin) with
+ `--format ansi|html|json|css`, `--theme`, `--dialect`,
+ `--color auto|always|never` (honors TTY, `NO_COLOR` and `COLORTERM`) and
+ `--standalone` for a full HTML page.
+ - LSP `textDocument/semanticTokens/full` now uses `pkg/highlight` with the
+ standard token types (keyword, function, type, variable, string, number,
+ operator, comment, parameter), UTF-16 positions, per-line splitting of
+ multi-line tokens, and coverage of every dialect keyword and all comments.
+ - WASM `gosqlxHighlight(sql[, dialect])` returns spans with UTF-16 offsets;
+ the gosqlx.dev playground uses it for editor highlighting.
+
+ See [docs/HIGHLIGHTING.md](docs/HIGHLIGHTING.md).
- **`--dialect` flag for the `parse`, `format`, `analyze`, and `lint` CLI
commands**: dialect-specific SQL (e.g. MySQL `LIMIT offset, count`) can now be
parsed by these commands, matching the existing `validate --dialect`. Unknown
diff --git a/CLAUDE.md b/CLAUDE.md
index f2ebe604..0252732d 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -27,6 +27,7 @@ GoSQLX is a **production-ready**, **race-free**, high-performance SQL parsing SD
- **Metrics** (`pkg/metrics/`): Production performance monitoring
- **Security** (`pkg/sql/security/`): SQL injection detection with severity classification
- **Linter** (`pkg/linter/`): SQL linting engine with 30 built-in rules (L001-L030)
+- **Highlight** (`pkg/highlight/`): Dialect-aware SQL syntax highlighting (spans, ANSI/HTML/CSS renderers, themes)
- **LSP** (`pkg/lsp/`): Language Server Protocol for IDE integration
- **GoSQLX** (`pkg/gosqlx/`): High-level simple API (recommended for most users)
- **Compatibility** (`pkg/compatibility/`): API stability testing
@@ -67,6 +68,7 @@ fingerprint→ formatter, sql/ast, sql/parser, sql/tokenizer
security → sql/ast (scanner; tests also pull parser, tokenizer)
linter → sql/parser, sql/tokenizer
# rule sub-packages additionally import: linter, models, sql/ast
+highlight → models, errors, sql/keywords, sql/tokenizer
lsp → errors, models, gosqlx, sql/keywords, sql/parser, sql/tokenizer
cbinding → gosqlx, sql/ast (requires CGO; excluded from task test:race)
diff --git a/README.md b/README.md
index 2f13a5f0..1d22fb95 100644
--- a/README.md
+++ b/README.md
@@ -161,6 +161,7 @@ Formatting turns one-line SQL into a readable query:
- **[Parser](https://gosqlx.dev/docs/concepts/architecture/)** - zero-copy tokenizer, recursive descent parser and full AST generation.
- **[Analysis](https://gosqlx.dev/docs/reference/linting-rules/)** - SQL injection scanner, 30 lint rules (L001-L030), 20 optimizer rules and metadata extraction.
- **[Tooling](https://gosqlx.dev/docs/cli-guide/)** - AST-based formatter, query transforms API, VS Code extension and GitHub Action.
+- **[Syntax highlighting](docs/HIGHLIGHTING.md)** - dialect-aware highlighting for terminals (ANSI), HTML/CSS, JSON spans, LSP semantic tokens and WASM; `gosqlx highlight query.sql` or `gosqlx.HighlightANSI(sql)`.
- **[Multi-dialect](https://gosqlx.dev/docs/reference/sql-compatibility/)** - PostgreSQL, MySQL, MariaDB, SQL Server, Oracle, SQLite, Snowflake, ClickHouse.
- **[AI-ready](https://gosqlx.dev/docs/integrations/mcp/)** - MCP server with 7 tools, public remote endpoint, Streamable HTTP.
- **[Tested](https://gosqlx.dev/docs/reference/sql-compatibility/)** - race-detector clean in CI, ~80-85% SQL-99 (estimated).
diff --git a/cmd/gosqlx/cmd/highlight.go b/cmd/gosqlx/cmd/highlight.go
new file mode 100644
index 00000000..932dd29c
--- /dev/null
+++ b/cmd/gosqlx/cmd/highlight.go
@@ -0,0 +1,359 @@
+// Copyright 2026 GoSQLX Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "html"
+ "io"
+ "os"
+ "path/filepath"
+ "sort"
+ "strings"
+
+ "github.com/spf13/cobra"
+ "golang.org/x/term"
+
+ "github.com/ajitpratap0/GoSQLX/pkg/highlight"
+ "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords"
+)
+
+// Output formats accepted by the highlight command.
+const (
+ highlightFormatANSI = "ansi"
+ highlightFormatHTML = "html"
+ highlightFormatJSON = "json"
+ highlightFormatCSS = "css"
+)
+
+// Color modes accepted by the highlight command's --color flag.
+const (
+ highlightColorAuto = "auto"
+ highlightColorAlways = "always"
+ highlightColorNever = "never"
+)
+
+var (
+ highlightFormats = []string{highlightFormatANSI, highlightFormatHTML, highlightFormatJSON, highlightFormatCSS}
+ highlightColorModes = []string{highlightColorAuto, highlightColorAlways, highlightColorNever}
+)
+
+var (
+ highlightOutputFormat string
+ highlightTheme string
+ highlightDialect string
+ highlightColor string
+ highlightStandalone bool
+)
+
+// highlightCmd represents the highlight command
+var highlightCmd = &cobra.Command{
+ Use: "highlight [file|query]",
+ Aliases: []string{"hl"},
+ Short: "Syntax-highlight SQL for terminals, HTML or editors",
+ Long: `Syntax-highlight SQL as ANSI terminal colors, HTML, or JSON spans.
+
+Highlighting is lexical and never fails: invalid SQL is highlighted up to the
+first lexical error and the rest is marked as an error.
+
+Examples:
+ gosqlx highlight query.sql # Colored output in the terminal
+ gosqlx hl "SELECT * FROM users WHERE id = 1" # Highlight a query directly
+ gosqlx highlight --theme light query.sql # Theme for light terminals
+ gosqlx highlight --color always query.sql | less -R
+ gosqlx highlight -f html query.sql # HTML fragment (class-based)
+ gosqlx highlight -f html --standalone -o out.html query.sql
+ gosqlx highlight -f json query.sql # Spans for editors/tools
+ gosqlx highlight -f css --theme dark # Stylesheet for -f html output
+ gosqlx highlight --dialect mysql query.sql # Dialect-specific keywords
+
+Pipeline/Stdin Examples:
+ echo "SELECT 1" | gosqlx highlight # Highlight stdin (auto-detect)
+ gosqlx highlight - # Explicit stdin marker
+ gosqlx highlight < query.sql # Input redirection
+
+Formats:
+ ansi ANSI escape sequences (default). --color controls coloring: auto colors
+ only when stdout is a terminal and NO_COLOR is unset; never prints the
+ SQL unchanged. 24-bit color is used when COLORTERM is truecolor or 24bit.
+ html with
+ elements. --standalone emits a complete HTML document with an inline
+ stylesheet for --theme.
+ json Array of {start, end, category, text, line, column}. start/end are
+ 0-based byte offsets (end exclusive); line and column are 1-based, with
+ column counted in bytes.
+ css Stylesheet for --theme matching the html class names (no input read).
+
+Themes: dark (default), light, mono`,
+ Args: cobra.MaximumNArgs(1),
+ RunE: highlightRun,
+}
+
+// highlightConfig is the validated configuration of one highlight run.
+type highlightConfig struct {
+ format string
+ theme highlight.Theme
+ dialect string
+ color string
+ standalone bool
+}
+
+func highlightRun(cmd *cobra.Command, args []string) error {
+ cfg, err := newHighlightConfig(highlightOutputFormat, highlightTheme, highlightDialect, highlightColor, highlightStandalone)
+ if err != nil {
+ return err
+ }
+
+ // The stylesheet does not depend on the input.
+ if cfg.format == highlightFormatCSS {
+ return WriteOutput([]byte(highlight.CSS(cfg.theme, "")), outputFile, cmd.OutOrStdout())
+ }
+
+ input, err := readHighlightInput(args)
+ if err != nil {
+ return err
+ }
+
+ tty := outputFile == "" && isTerminalWriter(cmd.OutOrStdout())
+ out, err := renderHighlight(string(input.Content), input, cfg, tty, os.Getenv)
+ if err != nil {
+ return err
+ }
+ return WriteOutput(out, outputFile, cmd.OutOrStdout())
+}
+
+// newHighlightConfig validates the highlight flags.
+func newHighlightConfig(format, theme, dialect, color string, standalone bool) (highlightConfig, error) {
+ format = strings.ToLower(strings.TrimSpace(format))
+ if !containsString(highlightFormats, format) {
+ return highlightConfig{}, fmt.Errorf("invalid format %q; valid formats: %s", format, strings.Join(highlightFormats, ", "))
+ }
+ th, ok := highlight.ThemeByName(strings.TrimSpace(theme))
+ if !ok {
+ return highlightConfig{}, fmt.Errorf("unknown theme %q; valid themes: %s", theme, strings.Join(highlight.ThemeNames(), ", "))
+ }
+ if err := validateDialectName(dialect); err != nil {
+ return highlightConfig{}, err
+ }
+ color = strings.ToLower(strings.TrimSpace(color))
+ if !containsString(highlightColorModes, color) {
+ return highlightConfig{}, fmt.Errorf("invalid color mode %q; valid modes: %s", color, strings.Join(highlightColorModes, ", "))
+ }
+ if standalone && format != highlightFormatHTML {
+ return highlightConfig{}, fmt.Errorf("--standalone requires --format html")
+ }
+ return highlightConfig{format: format, theme: th, dialect: dialect, color: color, standalone: standalone}, nil
+}
+
+// readHighlightInput reads the SQL to highlight from stdin ("-" or piped with
+// no argument), a file, or an inline query, applying the shared size, path
+// and binary-content checks.
+func readHighlightInput(args []string) (*InputResult, error) {
+ if len(args) == 0 || args[0] == "-" {
+ if !ShouldReadFromStdin(args) {
+ return nil, fmt.Errorf("no input provided: specify file path, SQL query, or pipe via stdin")
+ }
+ content, err := ReadFromStdin()
+ if err != nil {
+ return nil, fmt.Errorf("failed to read from stdin: %w", err)
+ }
+ if err := ValidateStdinInput(content); err != nil {
+ return nil, fmt.Errorf("stdin validation failed: %w", err)
+ }
+ return &InputResult{Type: InputTypeSQL, Content: content, Source: "stdin"}, nil
+ }
+
+ input, err := GetInputSource(args[0])
+ if err != nil {
+ return nil, fmt.Errorf("failed to read input: %w", err)
+ }
+ return input, nil
+}
+
+// renderHighlight highlights sql and renders it in cfg.format. tty reports
+// whether the output is an interactive terminal; getenv supplies NO_COLOR and
+// COLORTERM. The result always ends with a newline.
+func renderHighlight(sql string, input *InputResult, cfg highlightConfig, tty bool, getenv func(string) string) ([]byte, error) {
+ var opts []highlight.Option
+ if cfg.dialect != "" {
+ opts = append(opts, highlight.WithDialect(keywords.SQLDialect(cfg.dialect)))
+ }
+ spans := highlight.Spans(sql, opts...)
+
+ var buf bytes.Buffer
+ switch cfg.format {
+ case highlightFormatANSI:
+ enabled, mode := resolveANSIColor(cfg.color, tty, getenv)
+ if !enabled {
+ buf.WriteString(sql)
+ break
+ }
+ if err := highlight.WriteANSI(&buf, sql, spans, highlight.ANSIOptions{Theme: cfg.theme, Mode: mode}); err != nil {
+ return nil, fmt.Errorf("failed to render ANSI output: %w", err)
+ }
+ case highlightFormatHTML:
+ if err := writeHighlightHTML(&buf, sql, spans, input, cfg); err != nil {
+ return nil, fmt.Errorf("failed to render HTML output: %w", err)
+ }
+ case highlightFormatJSON:
+ if err := writeHighlightJSON(&buf, sql, spans); err != nil {
+ return nil, fmt.Errorf("failed to render JSON output: %w", err)
+ }
+ default:
+ return nil, fmt.Errorf("invalid format %q", cfg.format)
+ }
+
+ if !bytes.HasSuffix(buf.Bytes(), []byte("\n")) {
+ buf.WriteByte('\n')
+ }
+ return buf.Bytes(), nil
+}
+
+// resolveANSIColor decides whether ANSI output is colored and with which
+// color encoding. "auto" colors only when tty is true and NO_COLOR is unset or
+// empty (https://no-color.org); "always" ignores both. 24-bit color is used
+// when COLORTERM is "truecolor" or "24bit".
+func resolveANSIColor(color string, tty bool, getenv func(string) string) (bool, highlight.ColorMode) {
+ switch color {
+ case highlightColorNever:
+ return false, highlight.NoColor
+ case highlightColorAuto:
+ if !tty || getenv("NO_COLOR") != "" {
+ return false, highlight.NoColor
+ }
+ }
+ switch strings.ToLower(getenv("COLORTERM")) {
+ case "truecolor", "24bit":
+ return true, highlight.TrueColor
+ }
+ return true, highlight.Color256
+}
+
+// standalonePalette is the page background and default foreground used by
+// --standalone for each built-in theme.
+var standalonePalette = map[string][2]string{
+ "dark": {"#1E1E1E", "#D4D4D4"},
+ "light": {"#FFFFFF", "#000000"},
+ "mono": {"#FFFFFF", "#000000"},
+}
+
+// writeHighlightHTML writes the class-based HTML fragment, or a complete
+// document with an inline stylesheet when cfg.standalone is set.
+func writeHighlightHTML(w *bytes.Buffer, sql string, spans []highlight.Span, input *InputResult, cfg highlightConfig) error {
+ htmlOpts := highlight.HTMLOptions{Wrap: true}
+ if !cfg.standalone {
+ return highlight.WriteHTML(w, sql, spans, htmlOpts)
+ }
+
+ colors, ok := standalonePalette[cfg.theme.Name]
+ if !ok {
+ colors = standalonePalette["dark"]
+ }
+ title := "SQL"
+ if input != nil && input.Type == InputTypeFile {
+ title = filepath.Base(input.Source)
+ }
+
+ w.WriteString("\n\n\n\n")
+ w.WriteString("\n")
+ w.WriteString("" + html.EscapeString(title) + "\n\n\n\n")
+ if err := highlight.WriteHTML(w, sql, spans, htmlOpts); err != nil {
+ return err
+ }
+ w.WriteString("\n\n\n")
+ return nil
+}
+
+// highlightJSONSpan is one span in --format json output. Line and Column are
+// 1-based; Column counts bytes from the start of the line.
+type highlightJSONSpan struct {
+ Start int `json:"start"`
+ End int `json:"end"`
+ Category highlight.Category `json:"category"`
+ Text string `json:"text"`
+ Line int `json:"line"`
+ Column int `json:"column"`
+}
+
+// writeHighlightJSON writes spans as a pretty-printed JSON array.
+func writeHighlightJSON(w io.Writer, sql string, spans []highlight.Span) error {
+ starts := []int{0}
+ for i := 0; i < len(sql); i++ {
+ if sql[i] == '\n' {
+ starts = append(starts, i+1)
+ }
+ }
+
+ out := make([]highlightJSONSpan, 0, len(spans))
+ for _, s := range spans {
+ // Index of the last line starting at or before s.Start.
+ line := sort.Search(len(starts), func(i int) bool { return starts[i] > s.Start }) - 1
+ if line < 0 {
+ line = 0
+ }
+ out = append(out, highlightJSONSpan{
+ Start: s.Start,
+ End: s.End,
+ Category: s.Category,
+ Text: s.Text(sql),
+ Line: line + 1,
+ Column: s.Start - starts[line] + 1,
+ })
+ }
+
+ enc := json.NewEncoder(w)
+ enc.SetIndent("", " ")
+ enc.SetEscapeHTML(false)
+ if err := enc.Encode(out); err != nil {
+ return fmt.Errorf("failed to encode spans: %w", err)
+ }
+ return nil
+}
+
+// isTerminalWriter reports whether w is a file attached to a terminal.
+func isTerminalWriter(w io.Writer) bool {
+ f, ok := w.(*os.File)
+ if !ok {
+ return false
+ }
+ return term.IsTerminal(int(f.Fd())) // #nosec G115
+}
+
+func containsString(list []string, s string) bool {
+ for _, v := range list {
+ if v == s {
+ return true
+ }
+ }
+ return false
+}
+
+func init() {
+ rootCmd.AddCommand(highlightCmd)
+
+ highlightCmd.Flags().StringVarP(&highlightOutputFormat, "format", "f", highlightFormatANSI, "output format: ansi, html, json, css")
+ highlightCmd.Flags().StringVar(&highlightTheme, "theme", "dark", "color theme: "+strings.Join(highlight.ThemeNames(), ", "))
+ highlightCmd.Flags().StringVar(&highlightDialect, "dialect", "", "SQL dialect: generic, postgresql, mysql, mariadb, sqlserver, oracle, sqlite, snowflake, bigquery, redshift, clickhouse (default: generic keyword set)")
+ highlightCmd.Flags().StringVar(&highlightColor, "color", highlightColorAuto, "ANSI coloring: auto, always, never (ansi format only)")
+ highlightCmd.Flags().BoolVar(&highlightStandalone, "standalone", false, "emit a complete HTML document with an inline stylesheet (html format only)")
+}
diff --git a/cmd/gosqlx/cmd/highlight_test.go b/cmd/gosqlx/cmd/highlight_test.go
new file mode 100644
index 00000000..a769b5ac
--- /dev/null
+++ b/cmd/gosqlx/cmd/highlight_test.go
@@ -0,0 +1,421 @@
+// Copyright 2026 GoSQLX Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+ "bytes"
+ "encoding/json"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/ajitpratap0/GoSQLX/pkg/highlight"
+)
+
+// resetHighlightFlags restores the highlight command's flag variables (and the
+// shared output flag) to their defaults, now and after the test.
+func resetHighlightFlags(t *testing.T) {
+ t.Helper()
+ reset := func() {
+ highlightOutputFormat = highlightFormatANSI
+ highlightTheme = "dark"
+ highlightDialect = ""
+ highlightColor = highlightColorAuto
+ highlightStandalone = false
+ outputFile = ""
+ }
+ reset()
+ t.Cleanup(reset)
+}
+
+// runHighlight executes `gosqlx ` with fresh highlight flags and
+// returns stdout, stderr and the command error.
+func runHighlight(t *testing.T, args ...string) (string, string, error) {
+ t.Helper()
+ resetHighlightFlags(t)
+ var out, errBuf bytes.Buffer
+ rootCmd.SetOut(&out)
+ rootCmd.SetErr(&errBuf)
+ rootCmd.SetArgs(args)
+ t.Cleanup(func() {
+ rootCmd.SetOut(nil)
+ rootCmd.SetErr(nil)
+ rootCmd.SetArgs(nil)
+ })
+ err := rootCmd.Execute()
+ return out.String(), errBuf.String(), err
+}
+
+// withStdin replaces os.Stdin with a pipe holding content for the test.
+func withStdin(t *testing.T, content string) {
+ t.Helper()
+ r, w, err := os.Pipe()
+ if err != nil {
+ t.Fatalf("os.Pipe: %v", err)
+ }
+ if _, err := w.WriteString(content); err != nil {
+ t.Fatalf("write stdin pipe: %v", err)
+ }
+ if err := w.Close(); err != nil {
+ t.Fatalf("close stdin pipe: %v", err)
+ }
+ old := os.Stdin
+ os.Stdin = r
+ t.Cleanup(func() {
+ os.Stdin = old
+ _ = r.Close()
+ })
+}
+
+func TestHighlightCmd_Formats(t *testing.T) {
+ const sql = "SELECT id FROM users WHERE name = 'a`,
+ `SELECT`,
+ `'a<b'`,
+ "
\n",
+ },
+ notContains: []string{"", "\x1b["},
+ },
+ {
+ name: "html standalone",
+ args: []string{"highlight", "--format", "html", "--standalone", "--theme", "light", sql},
+ contains: []string{
+ "",
+ ``,
+ "" + html
+func HighlightHTML(sql string, opts ...highlight.Option) string {
+ return highlight.HTML(sql, Highlight(sql, opts...), highlight.HTMLOptions{
+ Wrap: true,
+ })
+}
diff --git a/pkg/gosqlx/highlight_test.go b/pkg/gosqlx/highlight_test.go
new file mode 100644
index 00000000..6c9ee6b3
--- /dev/null
+++ b/pkg/gosqlx/highlight_test.go
@@ -0,0 +1,178 @@
+// Copyright 2026 GoSQLX Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gosqlx
+
+import (
+ "reflect"
+ "strings"
+ "sync"
+ "testing"
+
+ "github.com/ajitpratap0/GoSQLX/pkg/highlight"
+ "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords"
+)
+
+func TestHighlight_MatchesSpans(t *testing.T) {
+ tests := []struct {
+ name string
+ sql string
+ opts []highlight.Option
+ }{
+ {"simple select", "SELECT id, name FROM users WHERE active = TRUE", nil},
+ {"function and comment", "SELECT COUNT(*) FROM t -- total", nil},
+ {"mysql backticks", "SELECT `a` FROM t", []highlight.Option{highlight.WithDialect(keywords.DialectMySQL)}},
+ {"invalid sql", "SELECT 'unterminated", nil},
+ {"empty", "", nil},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got := Highlight(tt.sql, tt.opts...)
+ want := highlight.Spans(tt.sql, tt.opts...)
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("Highlight(%q) = %v, want %v", tt.sql, got, want)
+ }
+ })
+ }
+}
+
+func TestHighlight_Categories(t *testing.T) {
+ sql := "SELECT name, COUNT(*) FROM users WHERE id = 42 AND note = 'x' -- c"
+ want := map[string]highlight.Category{
+ "SELECT": highlight.Keyword,
+ "name": highlight.Identifier,
+ "COUNT": highlight.Function,
+ "42": highlight.Number,
+ "'x'": highlight.String,
+ "-- c": highlight.Comment,
+ }
+ got := map[string]highlight.Category{}
+ for _, s := range Highlight(sql) {
+ got[s.Text(sql)] = s.Category
+ }
+ for text, cat := range want {
+ if got[text] != cat {
+ t.Errorf("category of %q = %v, want %v", text, got[text], cat)
+ }
+ }
+}
+
+func TestHighlight_InvalidSQLMarksError(t *testing.T) {
+ sql := "SELECT 'unterminated"
+ spans := Highlight(sql)
+ if len(spans) == 0 {
+ t.Fatal("expected spans for invalid SQL")
+ }
+ last := spans[len(spans)-1]
+ if last.Category != highlight.Error || last.End != len(sql) {
+ t.Fatalf("last span = %+v, want Error span ending at %d", last, len(sql))
+ }
+ if spans[0].Category != highlight.Keyword {
+ t.Errorf("first span = %+v, want keyword", spans[0])
+ }
+}
+
+func TestHighlightANSI(t *testing.T) {
+ sql := "SELECT id FROM users"
+ got := HighlightANSI(sql)
+ want := highlight.ANSI(sql, highlight.Spans(sql), highlight.ANSIOptions{
+ Theme: highlight.DarkTheme(),
+ Mode: highlight.Color256,
+ })
+ if got != want {
+ t.Fatalf("HighlightANSI = %q, want %q", got, want)
+ }
+ if !strings.Contains(got, "\x1b[") || !strings.Contains(got, "\x1b[0m") {
+ t.Errorf("expected ANSI escape sequences, got %q", got)
+ }
+ if strings.Contains(got, "38;2;") {
+ t.Errorf("expected 256-color sequences, got true color: %q", got)
+ }
+ if stripANSI(got) != sql {
+ t.Errorf("stripped output = %q, want %q", stripANSI(got), sql)
+ }
+}
+
+func TestHighlightANSI_Empty(t *testing.T) {
+ if got := HighlightANSI(""); got != "" {
+ t.Fatalf("HighlightANSI(\"\") = %q, want empty", got)
+ }
+}
+
+func TestHighlightHTML(t *testing.T) {
+ sql := "SELECT a FROM t WHERE b < 'x&y'"
+ got := HighlightHTML(sql)
+ if !strings.HasPrefix(got, ``) || !strings.HasSuffix(got, "
") {
+ t.Fatalf("expected wrapper, got %q", got)
+ }
+ for _, sub := range []string{
+ `SELECT`,
+ `<`,
+ `'x&y'`,
+ } {
+ if !strings.Contains(got, sub) {
+ t.Errorf("HighlightHTML output missing %q:\n%s", sub, got)
+ }
+ }
+ if strings.Contains(got, "style=") {
+ t.Errorf("expected class-based output, got inline styles: %q", got)
+ }
+}
+
+func TestHighlightHTML_WithDialect(t *testing.T) {
+ sql := "SELECT `a` FROM t"
+ got := HighlightHTML(sql, highlight.WithDialect(keywords.DialectMySQL))
+ if !strings.Contains(got, ``+"`a`") {
+ t.Fatalf("expected quoted identifier span under mysql, got %q", got)
+ }
+}
+
+func TestHighlight_Concurrent(t *testing.T) {
+ sql := "SELECT id, name FROM users WHERE id = $1"
+ want := HighlightHTML(sql)
+ var wg sync.WaitGroup
+ errs := make(chan string, 32)
+ for i := 0; i < 32; i++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ if got := HighlightHTML(sql); got != want {
+ errs <- got
+ }
+ _ = HighlightANSI(sql)
+ }()
+ }
+ wg.Wait()
+ close(errs)
+ for got := range errs {
+ t.Errorf("concurrent HighlightHTML = %q, want %q", got, want)
+ }
+}
+
+// stripANSI removes SGR escape sequences ("\x1b[...m") from s.
+func stripANSI(s string) string {
+ var b strings.Builder
+ for i := 0; i < len(s); i++ {
+ if s[i] == 0x1b && i+1 < len(s) && s[i+1] == '[' {
+ j := i + 2
+ for j < len(s) && s[j] != 'm' {
+ j++
+ }
+ i = j
+ continue
+ }
+ b.WriteByte(s[i])
+ }
+ return b.String()
+}
diff --git a/pkg/highlight/category.go b/pkg/highlight/category.go
new file mode 100644
index 00000000..ffd46e5b
--- /dev/null
+++ b/pkg/highlight/category.go
@@ -0,0 +1,127 @@
+// Copyright 2026 GoSQLX Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package highlight
+
+import "fmt"
+
+// Category is the syntactic class assigned to a highlighted span of SQL.
+//
+// Category values and their String names are part of the public contract:
+// the names are used as JSON values, HTML class suffixes and theme keys, and
+// will not change. New categories may be appended in future releases.
+type Category uint8
+
+const (
+ // Plain is unclassified text. Spans never carry Plain; it is the implicit
+ // category of whitespace and any gap between spans.
+ Plain Category = iota
+ // Keyword is a SQL keyword such as SELECT, FROM, RETURNING or LATERAL.
+ Keyword
+ // Function is a function name in call position, such as COUNT or now.
+ Function
+ // DataType is a type name in type position, such as VARCHAR in a column
+ // definition or INT in x::int.
+ DataType
+ // Identifier is an unquoted table, column, alias or schema name.
+ Identifier
+ // QuotedIdentifier is a delimited identifier: "name", `name` or [name].
+ QuotedIdentifier
+ // String is a string literal, including prefixed (N'', E'', X'') and
+ // dollar-quoted forms.
+ String
+ // Number is a numeric literal.
+ Number
+ // Constant is a literal keyword value: TRUE, FALSE or NULL.
+ Constant
+ // Operator is an operator such as =, <>, ||, ->> or ::.
+ Operator
+ // Punctuation is a structural character: comma, period, semicolon,
+ // parenthesis, bracket or brace.
+ Punctuation
+ // Comment is a line (--) or block (/* */) comment.
+ Comment
+ // Placeholder is a bind parameter or variable: $1, ?, :name, @var, @@var.
+ Placeholder
+ // Error marks input the tokenizer rejected, such as an unterminated
+ // string or comment. It always extends to the end of the input.
+ Error
+
+ numCategories
+)
+
+var categoryNames = [numCategories]string{
+ Plain: "plain",
+ Keyword: "keyword",
+ Function: "function",
+ DataType: "type",
+ Identifier: "identifier",
+ QuotedIdentifier: "quoted-identifier",
+ String: "string",
+ Number: "number",
+ Constant: "constant",
+ Operator: "operator",
+ Punctuation: "punctuation",
+ Comment: "comment",
+ Placeholder: "placeholder",
+ Error: "error",
+}
+
+// String returns the stable lowercase name of the category, for example
+// "keyword" or "quoted-identifier".
+func (c Category) String() string {
+ if c < numCategories {
+ return categoryNames[c]
+ }
+ return fmt.Sprintf("category(%d)", uint8(c))
+}
+
+// MarshalText encodes the category as its stable name, so spans serialize
+// to JSON as {"category":"keyword"}.
+func (c Category) MarshalText() ([]byte, error) {
+ if c >= numCategories {
+ return nil, fmt.Errorf("highlight: invalid category %d", uint8(c))
+ }
+ return []byte(categoryNames[c]), nil
+}
+
+// UnmarshalText decodes a category from its stable name.
+func (c *Category) UnmarshalText(text []byte) error {
+ cat, ok := ParseCategory(string(text))
+ if !ok {
+ return fmt.Errorf("highlight: unknown category %q", text)
+ }
+ *c = cat
+ return nil
+}
+
+// ParseCategory returns the category with the given stable name.
+func ParseCategory(name string) (Category, bool) {
+ for i, n := range categoryNames {
+ if n == name {
+ return Category(i), true
+ }
+ }
+ return Plain, false
+}
+
+// Categories returns every category that can appear on a Span, in
+// declaration order. Plain is excluded. The result is a fresh slice.
+func Categories() []Category {
+ out := make([]Category, 0, numCategories-1)
+ for c := Keyword; c < numCategories; c++ {
+ out = append(out, c)
+ }
+ return out
+}
diff --git a/pkg/highlight/doc.go b/pkg/highlight/doc.go
new file mode 100644
index 00000000..e0b6ce30
--- /dev/null
+++ b/pkg/highlight/doc.go
@@ -0,0 +1,45 @@
+// Copyright 2026 GoSQLX Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package highlight provides dialect-aware SQL syntax highlighting built on
+// the GoSQLX tokenizer.
+//
+// Spans classifies SQL into byte-offset ranges tagged with a Category
+// (keyword, function, type, identifier, string, comment, ...). The
+// classification is lexical with light context: function names are detected
+// in call position, type names in type position (column definitions, x::int,
+// CAST(x AS DATE)), and prefixed literals such as N'...' or 0x1F are kept
+// whole. Spans never fails; invalid SQL is highlighted up to the first
+// lexical error and the remainder is marked Error, which makes it suitable
+// for editors where input is incomplete most of the time.
+//
+// Renderers turn spans into output:
+//
+// - ANSI / WriteANSI: terminal escape sequences (256-color, true color, or
+// attributes only).
+// - HTML / WriteHTML: escaped HTML with class="gsx-" spans, or
+// inline styles; CSS generates the matching stylesheet.
+//
+// Spans also marshal to JSON ({"start":0,"end":6,"category":"keyword"}) for
+// browser and editor integrations.
+//
+// Example:
+//
+// sql := "SELECT id, name FROM users WHERE active = TRUE"
+// spans := highlight.Spans(sql, highlight.WithDialect(keywords.DialectPostgreSQL))
+// fmt.Print(highlight.ANSI(sql, spans, highlight.ANSIOptions{}))
+//
+// html := highlight.HTML(sql, spans, highlight.HTMLOptions{Wrap: true})
+// css := highlight.CSS(highlight.DarkTheme(), "")
+package highlight
diff --git a/pkg/highlight/example_test.go b/pkg/highlight/example_test.go
new file mode 100644
index 00000000..e68d6769
--- /dev/null
+++ b/pkg/highlight/example_test.go
@@ -0,0 +1,57 @@
+// Copyright 2026 GoSQLX Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package highlight_test
+
+import (
+ "fmt"
+
+ "github.com/ajitpratap0/GoSQLX/pkg/highlight"
+ "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords"
+)
+
+func ExampleSpans() {
+ sql := "SELECT name FROM users WHERE id = $1 -- by id"
+ for _, s := range highlight.Spans(sql, highlight.WithDialect(keywords.DialectPostgreSQL)) {
+ fmt.Printf("%-11s %s\n", s.Category, s.Text(sql))
+ }
+ // Output:
+ // keyword SELECT
+ // identifier name
+ // keyword FROM
+ // identifier users
+ // keyword WHERE
+ // identifier id
+ // operator =
+ // placeholder $1
+ // comment -- by id
+}
+
+func ExampleHTML() {
+ sql := "SELECT 'a' AS x"
+ fmt.Println(highlight.HTML(sql, highlight.Spans(sql), highlight.HTMLOptions{}))
+ // Output:
+ // SELECT 'a' AS x
+}
+
+func ExampleANSI() {
+ sql := "SELECT 1"
+ out := highlight.ANSI(sql, highlight.Spans(sql), highlight.ANSIOptions{
+ Theme: highlight.MonoTheme(),
+ Mode: highlight.NoColor,
+ })
+ fmt.Printf("%q\n", out)
+ // Output:
+ // "\x1b[1mSELECT\x1b[0m 1"
+}
diff --git a/pkg/highlight/highlight.go b/pkg/highlight/highlight.go
new file mode 100644
index 00000000..683c11fd
--- /dev/null
+++ b/pkg/highlight/highlight.go
@@ -0,0 +1,809 @@
+// Copyright 2026 GoSQLX Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package highlight
+
+import (
+ goerrors "errors"
+ "strings"
+ "sync"
+
+ sqlerrors "github.com/ajitpratap0/GoSQLX/pkg/errors"
+ "github.com/ajitpratap0/GoSQLX/pkg/models"
+ "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords"
+ "github.com/ajitpratap0/GoSQLX/pkg/sql/tokenizer"
+)
+
+// Span is a classified byte range of the source SQL.
+//
+// Start is inclusive and End is exclusive, both 0-based byte offsets into the
+// string passed to Spans, so src[s.Start:s.End] is the span's text. Offsets
+// always fall on UTF-8 boundaries.
+type Span struct {
+ Start int `json:"start"`
+ End int `json:"end"`
+ Category Category `json:"category"`
+}
+
+// Text returns the source text covered by the span. It returns "" when the
+// span does not fit within src.
+func (s Span) Text(src string) string {
+ if s.Start < 0 || s.End > len(src) || s.Start > s.End {
+ return ""
+ }
+ return src[s.Start:s.End]
+}
+
+// Option configures Spans.
+type Option func(*config)
+
+type config struct {
+ dialect keywords.SQLDialect
+}
+
+// WithDialect selects the SQL dialect used for tokenization and keyword
+// recognition. Unknown or empty dialects fall back to the default behaviour.
+func WithDialect(d keywords.SQLDialect) Option {
+ return func(c *config) { c.dialect = d }
+}
+
+// maxRecoveries bounds how many times Spans re-tokenizes a shrinking prefix
+// after a lexical error before giving up and marking the remainder as Error.
+const maxRecoveries = 4
+
+// Spans classifies sql for syntax highlighting.
+//
+// The returned spans are sorted by Start, never overlap, and never carry
+// Plain; any byte not covered by a span (typically whitespace) is plain text.
+// Spans never fails: when the tokenizer rejects the input, everything before
+// the offending position is still classified and the rest of the input is
+// covered by a single Error span. Inputs the tokenizer refuses outright (for
+// example, beyond its size limit) yield no spans.
+//
+// Spans is safe for concurrent use.
+func Spans(sql string, opts ...Option) []Span {
+ var cfg config
+ for _, opt := range opts {
+ opt(&cfg)
+ }
+ if cfg.dialect != "" && !keywords.IsValidDialect(string(cfg.dialect)) {
+ cfg.dialect = ""
+ }
+ if sql == "" {
+ return nil
+ }
+
+ src := []byte(sql)
+ h := classifier{
+ src: sql,
+ kw: keywordsFor(cfg.dialect),
+ dialect: cfg.dialect,
+ }
+
+ h.indexLines()
+ limit := len(src)
+ errAt := -1
+ for attempt := 0; ; attempt++ {
+ spans, err := h.run(src[:limit], limit)
+ if err == nil {
+ if errAt >= 0 {
+ spans = append(spans, Span{Start: errAt, End: len(src), Category: Error})
+ }
+ return spans
+ }
+ off, ok := h.errorOffset(err, limit)
+ if !ok {
+ if errAt < 0 {
+ // The tokenizer refused the input outright (size or token
+ // limits): leave it unhighlighted rather than guess.
+ return nil
+ }
+ off = 0
+ }
+ if off <= 0 || attempt >= maxRecoveries {
+ // The prefix cannot be recovered; mark everything as Error.
+ return []Span{{Start: 0, End: len(src), Category: Error}}
+ }
+ // Re-tokenize only the text before the error.
+ errAt, limit = off, off
+ }
+}
+
+// lexeme is a token with resolved byte offsets.
+type lexeme struct {
+ start, end int
+ typ models.TokenType
+ value string
+}
+
+type classifier struct {
+ src string
+ lineStarts []int
+ kw *keywords.Keywords
+ dialect keywords.SQLDialect
+}
+
+// run tokenizes input (a prefix of h.src of length limit) and classifies it.
+func (h *classifier) run(input []byte, limit int) ([]Span, error) {
+ tkz := tokenizer.GetTokenizer()
+ defer tokenizer.PutTokenizer(tkz)
+ if h.dialect != "" {
+ tkz.SetDialect(h.dialect)
+ }
+ toks, err := tkz.Tokenize(input)
+ if err != nil {
+ return nil, err
+ }
+
+ comments := make([]Span, 0, len(tkz.Comments))
+ for _, c := range tkz.Comments {
+ start, ok1 := h.offset(c.Start, limit)
+ end, ok2 := h.offset(c.End, limit)
+ if !ok1 || !ok2 {
+ continue
+ }
+ for end > start && (h.src[end-1] == '\n' || h.src[end-1] == '\r') {
+ end--
+ }
+ if end > start {
+ comments = append(comments, Span{Start: start, End: end, Category: Comment})
+ }
+ }
+
+ lexemes := h.resolve(toks, comments, limit)
+ return h.classify(lexemes, comments), nil
+}
+
+// resolve computes byte ranges for tokens. Each start is derived from the
+// previous token's end by skipping whitespace and comments, so classification
+// depends only on token end locations and the captured comments.
+func (h *classifier) resolve(toks []models.TokenWithSpan, comments []Span, limit int) []lexeme {
+ out := make([]lexeme, 0, len(toks))
+ cursor, ci := 0, 0
+ for _, tok := range toks {
+ if tok.Token.Type == models.TokenTypeEOF {
+ continue
+ }
+ end, ok := h.offset(tok.End, limit)
+ if !ok || end <= cursor {
+ continue
+ }
+ start := cursor
+ for ci < len(comments) && comments[ci].Start < end {
+ if comments[ci].End > start {
+ start = comments[ci].End
+ }
+ ci++
+ }
+ for start < end && isSpace(h.src[start]) {
+ start++
+ }
+ if start >= end {
+ cursor = end
+ continue
+ }
+ out = append(out, lexeme{start: start, end: end, typ: tok.Token.Type, value: tok.Token.Value})
+ cursor = end
+ }
+ return out
+}
+
+// offset converts a tokenizer location to a byte offset. Tokenizer columns
+// are 1-based and count bytes, a tab included. Locations that do not fall on
+// their line (column 0, or past the line's end) are rejected.
+func (h *classifier) offset(loc models.Location, limit int) (int, bool) {
+ if loc.Line < 1 || loc.Line > len(h.lineStarts) || loc.Column < 1 {
+ return 0, false
+ }
+ off := h.lineStarts[loc.Line-1] + loc.Column - 1
+ if loc.Line < len(h.lineStarts) && off >= h.lineStarts[loc.Line] {
+ return 0, false
+ }
+ if off > limit {
+ return 0, false
+ }
+ return off, true
+}
+
+// errorOffset returns the byte offset where the lexical error err begins
+// within src[:limit]. It reports false for errors that are not lexical
+// (input or token limits, recovered panics).
+//
+// Tokenizer error locations are not always usable: some unterminated-literal
+// errors report column 0 or the end of input instead of the opening
+// delimiter. The location is therefore validated, and when it does not point
+// at a plausible opener the offset is recovered by scanning for the last
+// unclosed string, quoted identifier or comment.
+func (h *classifier) errorOffset(err error, limit int) (int, bool) {
+ var loc models.Location
+ var code sqlerrors.ErrorCode
+ var se *sqlerrors.Error
+ var te *models.TokenizerError
+ switch {
+ case goerrors.As(err, &se):
+ loc, code = se.Location, se.Code
+ case goerrors.As(err, &te):
+ loc = te.Location
+ default:
+ return 0, false
+ }
+ switch code {
+ case sqlerrors.ErrCodeInputTooLarge, sqlerrors.ErrCodeTokenLimitReached, sqlerrors.ErrCodeTokenizerPanic:
+ return 0, false
+ }
+
+ off, ok := h.offset(loc, limit)
+ ok = ok && off < limit
+ unterminated := code == sqlerrors.ErrCodeUnterminatedString ||
+ code == sqlerrors.ErrCodeUnterminatedBlockComment ||
+ strings.Contains(err.Error(), "unterminated")
+ if ok && (!unterminated || h.isOpener(off, limit)) {
+ return off, true
+ }
+ if u := h.lastUnclosed(limit); u >= 0 {
+ return u, true
+ }
+ if ok {
+ return off, true
+ }
+ return 0, true
+}
+
+// isOpener reports whether src[off:] starts a string, quoted identifier or
+// block comment (optionally behind a literal prefix such as N, E, X or U&).
+func (h *classifier) isOpener(off, limit int) bool {
+ s := h.src[off:limit]
+ if strings.HasPrefix(s, "/*") {
+ return true
+ }
+ for i := 0; i < len(s) && i < 3; i++ {
+ switch s[i] {
+ case '\'', '"', '`', '$', '[':
+ return true
+ }
+ if !isWordStart(s[i]) && s[i] != '&' {
+ return false
+ }
+ }
+ return false
+}
+
+// lastUnclosed scans src[:limit] and returns the offset of an opening quote,
+// dollar-quote tag or block-comment delimiter that is never closed, or -1.
+// It is only used to recover error positions, so it mirrors the tokenizer's
+// rules closely but not exhaustively.
+func (h *classifier) lastUnclosed(limit int) int {
+ s := h.src[:limit]
+ backslash := h.doubleQuotedIsString() // MySQL-family and BigQuery escape with '\'
+ for i := 0; i < len(s); {
+ c := s[i]
+ switch {
+ case c == '-' && strings.HasPrefix(s[i:], "--"):
+ j := strings.IndexByte(s[i:], '\n')
+ if j < 0 {
+ return -1
+ }
+ i += j + 1
+ case c == '/' && strings.HasPrefix(s[i:], "/*"):
+ j := strings.Index(s[i+2:], "*/")
+ if j < 0 {
+ return i
+ }
+ i += 2 + j + 2
+ case c == '\'' || c == '"' || c == '`' || (c == '[' && h.dialect == keywords.DialectSQLServer):
+ closer := c
+ if c == '[' {
+ closer = ']'
+ }
+ j := i + 1
+ for {
+ if j >= len(s) {
+ return i
+ }
+ switch {
+ case backslash && s[j] == '\\':
+ j += 2
+ continue
+ case s[j] == '\n' && c == '"' && !backslash:
+ return i // quoted identifiers cannot span lines
+ case s[j] == closer:
+ if j+1 < len(s) && s[j+1] == closer && closer != ']' {
+ j += 2 // doubled quote escape
+ continue
+ }
+ default:
+ j++
+ continue
+ }
+ break
+ }
+ i = j + 1
+ case c == '$':
+ j := i + 1
+ for j < len(s) && (isWordStart(s[j]) || s[j] >= '0' && s[j] <= '9') {
+ j++
+ }
+ if j < len(s) && s[j] == '$' && (j == i+1 || !(s[i+1] >= '0' && s[i+1] <= '9')) {
+ tag := s[i : j+1]
+ k := strings.Index(s[j+1:], tag)
+ if k < 0 {
+ return i
+ }
+ i = j + 1 + k + len(tag)
+ } else {
+ i = j
+ }
+ default:
+ i++
+ }
+ }
+ return -1
+}
+
+// classify assigns categories to lexemes and merges in comment spans.
+func (h *classifier) classify(ls []lexeme, comments []Span) []Span {
+ out := make([]Span, 0, len(ls)+len(comments)+1)
+ ci := 0
+ emit := func(s Span) {
+ for ci < len(comments) && comments[ci].Start < s.Start {
+ out = append(out, comments[ci])
+ ci++
+ }
+ out = append(out, s)
+ }
+
+ st := state{}
+ for i := 0; i < len(ls); i++ {
+ cat, consumed := h.category(ls, i, &st)
+ span := Span{Start: ls[i].start, End: ls[i+consumed].end, Category: cat}
+ st.observe(ls, i, cat)
+ i += consumed
+ if cat != Plain {
+ emit(span)
+ }
+ }
+ return append(out, comments[ci:]...)
+}
+
+// state carries left-to-right context used by the classification heuristics.
+type state struct {
+ parens []string // uppercased word that opened each enclosing paren ("" if none)
+ ddl bool // inside CREATE/ALTER/DECLARE until the next semicolon
+ prevCat Category // category of the previous lexeme
+}
+
+func (st *state) top() string {
+ if len(st.parens) == 0 {
+ return ""
+ }
+ return st.parens[len(st.parens)-1]
+}
+
+func (st *state) observe(ls []lexeme, i int, cat Category) {
+ l := ls[i]
+ switch l.typ {
+ case models.TokenTypeLParen:
+ opener := ""
+ if i > 0 {
+ opener = upper(ls[i-1].value)
+ }
+ st.parens = append(st.parens, opener)
+ case models.TokenTypeRParen:
+ if len(st.parens) > 0 {
+ st.parens = st.parens[:len(st.parens)-1]
+ }
+ case models.TokenTypeSemicolon:
+ st.ddl = false
+ st.parens = st.parens[:0]
+ }
+ if cat == Keyword {
+ switch upper(l.value) {
+ case "CREATE", "ALTER", "DECLARE":
+ st.ddl = true
+ }
+ }
+ st.prevCat = cat
+}
+
+// category classifies ls[i] and reports how many following lexemes it
+// absorbed (for prefixed strings, hex literals and named parameters).
+func (h *classifier) category(ls []lexeme, i int, st *state) (Category, int) {
+ l := ls[i]
+ var prev, next *lexeme
+ if i > 0 {
+ prev = &ls[i-1]
+ }
+ if i+1 < len(ls) {
+ next = &ls[i+1]
+ }
+ adjNext := next != nil && next.start == l.end
+
+ switch typ := l.typ; {
+ case typ == models.TokenTypeNumber:
+ // 0x1F and 0b101 tokenize as NUMBER "0" + IDENTIFIER "x1F".
+ if adjNext && l.value == "0" && isWordType(next.typ) && isRadixSuffix(next.value) {
+ return Number, 1
+ }
+ return Number, 0
+
+ case typ >= models.TokenRangeStringStart && typ < models.TokenRangeStringEnd:
+ if h.src[l.start] == '"' && !h.doubleQuotedIsString() {
+ return QuotedIdentifier, 0
+ }
+ return String, 0
+
+ case typ == models.TokenTypeTrue, typ == models.TokenTypeFalse, typ == models.TokenTypeNull:
+ if isQualified(prev, next, l) {
+ return Identifier, 0
+ }
+ return Constant, 0
+
+ case typ == models.TokenTypePlaceholder, typ == models.TokenTypeQuestion:
+ return Placeholder, 0
+
+ case typ == models.TokenTypeColon:
+ // :name / :1 named or positional parameter, but not a[1:2] slices or
+ // Snowflake col:path access.
+ if adjNext && (isWordType(next.typ) || next.typ == models.TokenTypeNumber) && !isOperand(prev) {
+ return Placeholder, 1
+ }
+ return Operator, 0
+
+ case typ == models.TokenTypeAtAt, typ == models.TokenTypeAtSign:
+ if adjNext && isWordType(next.typ) {
+ return Placeholder, 1
+ }
+ return Operator, 0
+
+ case isPunctuation(typ):
+ return Punctuation, 0
+
+ case typ.IsOperator():
+ return Operator, 0
+
+ case isWordType(typ):
+ return h.word(ls, i, prev, next, adjNext, st)
+
+ case typ.IsDataType():
+ return DataType, 0
+
+ case typ.IsKeyword():
+ if isQualified(prev, next, l) {
+ return Identifier, 0
+ }
+ w := upper(l.value)
+ if dataTypeNames[w] && h.typeContext(prev, st) {
+ return DataType, 0
+ }
+ if next != nil && next.typ == models.TokenTypeLParen {
+ if dataTypeNames[w] && st.ddl {
+ return DataType, 0
+ }
+ if typ.IsAggregateFunction() || functionKeywords[w] || (!h.kw.IsReserved(w) && !clauseKeywords[w]) {
+ return Function, 0
+ }
+ }
+ return Keyword, 0
+ }
+ // Token types outside the ranges above (e.g. ILIKE, dialect operators):
+ // word-like lexemes are keywords, anything else is an operator, so no
+ // significant input is ever left unclassified.
+ if isWordStart(h.src[l.start]) {
+ if isQualified(prev, next, l) {
+ return Identifier, 0
+ }
+ return Keyword, 0
+ }
+ if l.typ == models.TokenTypeEOF {
+ return Plain, 0
+ }
+ return Operator, 0
+}
+
+func isWordStart(c byte) bool {
+ return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c == '_'
+}
+
+// word classifies an IDENTIFIER/WORD lexeme.
+func (h *classifier) word(ls []lexeme, i int, prev, next *lexeme, adjNext bool, st *state) (Category, int) {
+ l := ls[i]
+ switch h.src[l.start] {
+ case '"', '`', '[':
+ return QuotedIdentifier, 0
+ case '@':
+ return Placeholder, 0
+ }
+
+ w := upper(l.value)
+ // N'..', E'..', X'..', B'..', U&'..', R'..' split into word + string.
+ if adjNext && isStringType(next.typ) && stringPrefixes[w] {
+ return String, 1
+ }
+ if isQualified(prev, next, l) {
+ if next != nil && next.typ == models.TokenTypeLParen {
+ return Function, 0 // schema.func(...)
+ }
+ return Identifier, 0
+ }
+ if dataTypeNames[w] && h.typeContext(prev, st) {
+ return DataType, 0
+ }
+ if next != nil && next.typ == models.TokenTypeLParen {
+ if dataTypeNames[w] && (st.ddl || dataTypeNames[st.top()]) {
+ return DataType, 0
+ }
+ if !blocksFunction(prev) && !clauseKeywords[w] {
+ return Function, 0
+ }
+ return Identifier, 0
+ }
+ if h.kw.IsKeyword(w) || extraKeywords[w] {
+ return Keyword, 0
+ }
+ return Identifier, 0
+}
+
+// typeContext reports whether the lexeme following prev is in type position.
+func (h *classifier) typeContext(prev *lexeme, st *state) bool {
+ if prev == nil {
+ return false
+ }
+ switch prev.typ {
+ case models.TokenTypeDoubleColon:
+ return true
+ case models.TokenTypeLParen, models.TokenTypeComma:
+ // Type arguments: Nullable(String), Map(String, UInt64).
+ return dataTypeNames[st.top()]
+ }
+ if st.prevCat == DataType {
+ // Multi-word types: DOUBLE PRECISION, CHARACTER VARYING,
+ // TIMESTAMP WITH TIME ZONE is left to the keyword path.
+ return true
+ }
+ pw := upper(prev.value)
+ switch pw {
+ case "AS":
+ switch st.top() {
+ case "CAST", "TRY_CAST", "SAFE_CAST", "CONVERT":
+ return true
+ }
+ return false
+ case "RETURNS", "TYPE":
+ return true
+ case "COLUMN", "ADD":
+ return false
+ }
+ if !st.ddl {
+ return false
+ }
+ // Column definitions: CREATE TABLE t (id INT, name TEXT),
+ // ALTER TABLE t ADD [COLUMN] c TEXT, DECLARE @x INT.
+ return st.prevCat == Identifier || st.prevCat == QuotedIdentifier || st.prevCat == Placeholder
+}
+
+func (h *classifier) doubleQuotedIsString() bool {
+ switch h.dialect {
+ case keywords.DialectMySQL, keywords.DialectMariaDB, keywords.DialectBigQuery:
+ return true
+ }
+ return false
+}
+
+func isWordType(t models.TokenType) bool {
+ return t == models.TokenTypeIdentifier || t == models.TokenTypeWord
+}
+
+func isStringType(t models.TokenType) bool {
+ return t >= models.TokenRangeStringStart && t < models.TokenRangeStringEnd
+}
+
+func isPunctuation(t models.TokenType) bool {
+ switch t {
+ case models.TokenTypeComma, models.TokenTypeLParen, models.TokenTypeRParen,
+ models.TokenTypePeriod, models.TokenTypeSemicolon,
+ models.TokenTypeLBracket, models.TokenTypeRBracket,
+ models.TokenTypeLBrace, models.TokenTypeRBrace:
+ return true
+ }
+ return false
+}
+
+// isOperand reports whether l ends a value, making a following ':' an
+// operator (slice, path access) rather than a parameter sigil.
+func isOperand(l *lexeme) bool {
+ if l == nil {
+ return false
+ }
+ switch l.typ {
+ case models.TokenTypeNumber, models.TokenTypeIdentifier, models.TokenTypeWord,
+ models.TokenTypeRParen, models.TokenTypeRBracket, models.TokenTypePlaceholder:
+ return true
+ }
+ return isStringType(l.typ)
+}
+
+// isQualified reports whether l is part of a dotted name (a.b.c).
+func isQualified(prev, next *lexeme, l lexeme) bool {
+ if prev != nil && prev.typ == models.TokenTypePeriod && prev.end == l.start {
+ return true
+ }
+ return next != nil && next.typ == models.TokenTypePeriod && next.start == l.end
+}
+
+// blocksFunction reports whether an identifier after prev followed by '('
+// is a relation or index name with a column list rather than a call:
+// INSERT INTO t (a), REFERENCES t (id), WITH cte (x) AS, CREATE INDEX i (c).
+func blocksFunction(prev *lexeme) bool {
+ if prev == nil {
+ return false
+ }
+ switch upper(prev.value) {
+ case "INTO", "TABLE", "REFERENCES", "VIEW", "EXISTS", "ON", "INDEX", "KEY",
+ "WITH", "RECURSIVE", "TRIGGER", "TYPE", "CONSTRAINT":
+ return true
+ }
+ return false
+}
+
+func isRadixSuffix(s string) bool {
+ if len(s) < 2 {
+ return false
+ }
+ switch s[0] {
+ case 'x', 'X':
+ for i := 1; i < len(s); i++ {
+ c := s[i]
+ if !(c >= '0' && c <= '9' || c >= 'a' && c <= 'f' || c >= 'A' && c <= 'F') {
+ return false
+ }
+ }
+ return true
+ case 'b', 'B':
+ for i := 1; i < len(s); i++ {
+ if s[i] != '0' && s[i] != '1' {
+ return false
+ }
+ }
+ return true
+ }
+ return false
+}
+
+func isSpace(c byte) bool {
+ return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v'
+}
+
+// upper returns s in upper case; ASCII-uppercase input is returned without
+// allocating.
+func upper(s string) string {
+ return strings.ToUpper(s)
+}
+
+// indexLines records the byte offset of each line start.
+func (h *classifier) indexLines() {
+ s := h.src
+ h.lineStarts = make([]int, 1, strings.Count(s, "\n")+1)
+ for i := 0; i < len(s); i++ {
+ if s[i] == '\n' {
+ h.lineStarts = append(h.lineStarts, i+1)
+ }
+ }
+}
+
+var (
+ kwMu sync.Mutex
+ kwCache = map[keywords.SQLDialect]*keywords.Keywords{}
+)
+
+// keywordsFor returns a shared, read-only keyword table for the dialect.
+// Construction is serialized because keywords.New resets package state.
+func keywordsFor(d keywords.SQLDialect) *keywords.Keywords {
+ if d == "" {
+ d = keywords.DialectGeneric
+ }
+ kwMu.Lock()
+ defer kwMu.Unlock()
+ if k, ok := kwCache[d]; ok {
+ return k
+ }
+ k := keywords.New(d, true)
+ kwCache[d] = k
+ return k
+}
+
+// stringPrefixes are identifier fragments that, when immediately followed by
+// a string literal, form a single prefixed literal.
+var stringPrefixes = map[string]bool{
+ "N": true, "E": true, "X": true, "B": true, "U&": true, "R": true, "U": true,
+ "BR": true, "RB": true,
+}
+
+// extraKeywords supplements the parser's deliberately conservative keyword
+// table with statement and clause words that are rarely used as bare
+// identifiers. Words that commonly name columns (COMMENT, OWNER, SOURCE, ...)
+// are intentionally absent.
+var extraKeywords = map[string]bool{
+ "DECLARE": true, "BEGIN": true, "COMMIT": true, "ROLLBACK": true, "TRANSACTION": true,
+ "SAVEPOINT": true, "RETURNS": true, "PROCEDURE": true, "FUNCTION": true, "TRIGGER": true,
+ "EXECUTE": true, "EXEC": true, "GRANT": true, "REVOKE": true, "VACUUM": true,
+ "COPY": true, "SHOW": true, "USE": true, "DESCRIBE": true, "UPSERT": true,
+ "CONFLICT": true, "DO": true, "NOTHING": true, "EXCLUDED": true, "LANGUAGE": true,
+ "ELSIF": true, "LOOP": true, "WHILE": true, "RETURN": true, "RAISE": true,
+ "SCHEMA": true, "DATABASE": true, "SEQUENCE": true, "TEMP": true, "UNLOGGED": true,
+ "MATERIALIZED": true, "ILIKE": true, "SIMILAR": true, "ESCAPE": true, "COLLATE": true,
+ "NULLS": true, "GENERATED": true, "ALWAYS": true, "AUTO_INCREMENT": true,
+ "UNSIGNED": true, "ZEROFILL": true, "CHARSET": true, "MODIFY": true, "AFTER": true,
+ "BEFORE": true, "INSTEAD": true, "EACH": true, "STATEMENT": true, "PRINT": true,
+}
+
+// clauseKeywords are keywords that are commonly followed by '(' without being
+// function calls: constraints, column lists and value lists.
+var clauseKeywords = map[string]bool{
+ "UNIQUE": true, "DEFAULT": true, "PRIMARY": true, "FOREIGN": true, "KEY": true,
+ "CHECK": true, "VALUES": true, "IN": true, "EXISTS": true, "OVER": true,
+ "FILTER": true, "WITHIN": true, "USING": true, "AS": true, "ON": true,
+ "PARTITION": true, "INCLUDE": true, "REFERENCES": true, "TABLE": true,
+ "RETURNS": true, "LATERAL": true,
+}
+
+// functionKeywords are reserved keywords that are functions when called.
+var functionKeywords = map[string]bool{
+ "CAST": true, "TRY_CAST": true, "SAFE_CAST": true, "CONVERT": true,
+ "COALESCE": true, "NULLIF": true, "EXTRACT": true, "SUBSTRING": true,
+ "TRIM": true, "REPLACE": true, "LEFT": true, "RIGHT": true, "POSITION": true,
+ "IF": true, "IFNULL": true, "ISNULL": true, "NVL": true, "GREATEST": true,
+ "LEAST": true, "OVERLAY": true, "FIRST": true, "LAST": true, "CHAR": true,
+ "CURRENT_DATE": true, "CURRENT_TIME": true, "CURRENT_TIMESTAMP": true,
+ "ROW": true, "ARRAY": true, "MOD": true, "UPPER": true, "LOWER": true,
+ "YEAR": true, "MONTH": true, "DAY": true, "HOUR": true, "MINUTE": true, "SECOND": true,
+}
+
+// dataTypeNames are type names recognised in type position across the
+// supported dialects. Membership alone never makes a word a DataType; the
+// surrounding context (column definition, ::, CAST ... AS) must agree.
+var dataTypeNames = map[string]bool{
+ // Integers and decimals
+ "INT": true, "INTEGER": true, "SMALLINT": true, "BIGINT": true, "TINYINT": true,
+ "MEDIUMINT": true, "INT2": true, "INT4": true, "INT8": true, "INT16": true,
+ "INT32": true, "INT64": true, "INT128": true, "INT256": true,
+ "UINT8": true, "UINT16": true, "UINT32": true, "UINT64": true, "UINT128": true, "UINT256": true,
+ "DECIMAL": true, "DEC": true, "NUMERIC": true, "NUMBER": true, "BIGNUMERIC": true,
+ "DECIMAL32": true, "DECIMAL64": true, "DECIMAL128": true, "DECIMAL256": true,
+ "MONEY": true, "SMALLMONEY": true, "SERIAL": true, "SMALLSERIAL": true, "BIGSERIAL": true,
+ // Floating point
+ "FLOAT": true, "FLOAT4": true, "FLOAT8": true, "FLOAT32": true, "FLOAT64": true,
+ "REAL": true, "DOUBLE": true, "PRECISION": true, "BINARY_FLOAT": true, "BINARY_DOUBLE": true,
+ // Boolean and bits
+ "BOOLEAN": true, "BOOL": true, "BIT": true, "VARBIT": true,
+ // Character
+ "CHAR": true, "CHARACTER": true, "VARYING": true, "VARCHAR": true, "VARCHAR2": true,
+ "NCHAR": true, "NVARCHAR": true, "NVARCHAR2": true, "TEXT": true, "TINYTEXT": true,
+ "MEDIUMTEXT": true, "LONGTEXT": true, "NTEXT": true, "CLOB": true, "NCLOB": true,
+ "STRING": true, "FIXEDSTRING": true, "CITEXT": true,
+ // Binary
+ "BLOB": true, "TINYBLOB": true, "MEDIUMBLOB": true, "LONGBLOB": true, "BYTEA": true,
+ "BINARY": true, "VARBINARY": true, "IMAGE": true, "BYTES": true, "RAW": true,
+ // Temporal
+ "DATE": true, "DATE32": true, "TIME": true, "TIMETZ": true, "TIMESTAMP": true,
+ "TIMESTAMPTZ": true, "TIMESTAMP_NTZ": true, "TIMESTAMP_LTZ": true, "TIMESTAMP_TZ": true,
+ "DATETIME": true, "DATETIME2": true, "DATETIME64": true, "DATETIMEOFFSET": true,
+ "SMALLDATETIME": true, "INTERVAL": true, "YEAR": true,
+ // Structured and other
+ "UUID": true, "UNIQUEIDENTIFIER": true, "JSON": true, "JSONB": true, "XML": true,
+ "VARIANT": true, "OBJECT": true, "ARRAY": true, "MAP": true, "TUPLE": true, "STRUCT": true,
+ "NULLABLE": true, "LOWCARDINALITY": true, "NESTED": true, "ENUM": true,
+ "GEOGRAPHY": true, "GEOMETRY": true, "INET": true, "CIDR": true, "MACADDR": true,
+ "TSVECTOR": true, "TSQUERY": true, "IPV4": true, "IPV6": true,
+ "ROWID": true, "UROWID": true, "ROWVERSION": true, "SQL_VARIANT": true, "HIERARCHYID": true,
+}
diff --git a/pkg/highlight/highlight_test.go b/pkg/highlight/highlight_test.go
new file mode 100644
index 00000000..b5309f29
--- /dev/null
+++ b/pkg/highlight/highlight_test.go
@@ -0,0 +1,453 @@
+// Copyright 2026 GoSQLX Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package highlight
+
+import (
+ "encoding/json"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync"
+ "testing"
+ "unicode/utf8"
+
+ "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords"
+)
+
+// render formats spans as "category:text" pairs for readable assertions.
+func render(src string, spans []Span) string {
+ parts := make([]string, 0, len(spans))
+ for _, s := range spans {
+ parts = append(parts, s.Category.String()+":"+s.Text(src))
+ }
+ return strings.Join(parts, " ")
+}
+
+// find returns the category of the first span whose text equals text.
+func find(t *testing.T, src string, spans []Span, text string) Category {
+ t.Helper()
+ for _, s := range spans {
+ if s.Text(src) == text {
+ return s.Category
+ }
+ }
+ t.Fatalf("no span with text %q in: %s", text, render(src, spans))
+ return Plain
+}
+
+func TestSpans_Classification(t *testing.T) {
+ tests := []struct {
+ name string
+ dialect keywords.SQLDialect
+ sql string
+ want map[string]Category
+ }{
+ {
+ name: "select basics",
+ sql: "SELECT u.id, COUNT(*) AS n FROM users u WHERE u.active = TRUE AND u.deleted_at IS NULL",
+ want: map[string]Category{
+ "SELECT": Keyword, "u": Identifier, ".": Punctuation, "id": Identifier,
+ "COUNT": Function, "*": Operator, "AS": Keyword, "users": Identifier,
+ "=": Operator, "TRUE": Constant, "NULL": Constant, ",": Punctuation,
+ },
+ },
+ {
+ name: "literals",
+ sql: "SELECT 'str', 1.5e3, 0x1F, E'esc', N'nat', X'FF', $$body$$ FROM t",
+ want: map[string]Category{
+ "'str'": String, "1.5e3": Number, "0x1F": Number, "E'esc'": String,
+ "N'nat'": String, "X'FF'": String, "$$body$$": String,
+ },
+ },
+ {
+ name: "placeholders",
+ sql: "SELECT * FROM t WHERE a = $1 AND b = ? AND c = :name AND d = @v",
+ want: map[string]Category{"$1": Placeholder, "?": Placeholder, ":name": Placeholder, "@v": Placeholder},
+ },
+ {
+ name: "comments",
+ sql: "-- lead\nSELECT 1 /* inline */ FROM t -- tail",
+ want: map[string]Category{"-- lead": Comment, "/* inline */": Comment, "-- tail": Comment, "SELECT": Keyword},
+ },
+ {
+ name: "non-reserved keywords the tokenizer leaves as identifiers",
+ sql: "INSERT INTO t (a) VALUES (1) RETURNING a",
+ want: map[string]Category{"RETURNING": Keyword, "INTO": Keyword, "VALUES": Keyword, "t": Identifier},
+ },
+ {
+ name: "column definitions",
+ sql: "CREATE TABLE t (id INT PRIMARY KEY, name VARCHAR(10) NOT NULL, d DOUBLE PRECISION, ts TIMESTAMP DEFAULT now())",
+ want: map[string]Category{
+ "INT": DataType, "VARCHAR": DataType, "DOUBLE": DataType, "PRECISION": DataType,
+ "TIMESTAMP": DataType, "now": Function, "PRIMARY": Keyword, "id": Identifier,
+ },
+ },
+ {
+ name: "type casts",
+ sql: "SELECT x::int, CAST(y AS DATE), date FROM t",
+ want: map[string]Category{"::": Operator, "int": DataType, "CAST": Function, "DATE": DataType, "date": Identifier},
+ },
+ {
+ name: "column lists are not calls",
+ sql: "INSERT INTO users (id) SELECT id FROM s; CREATE INDEX idx ON users (id)",
+ want: map[string]Category{"users": Identifier, "idx": Identifier},
+ },
+ {
+ name: "qualified names",
+ sql: "SELECT s.fn(x), t.null, t.order FROM s",
+ want: map[string]Category{"fn": Function, "null": Identifier},
+ },
+ {
+ name: "mysql backticks and double-quoted strings",
+ dialect: keywords.DialectMySQL,
+ sql: "SELECT `a b`, \"str\" FROM t",
+ want: map[string]Category{"`a b`": QuotedIdentifier, `"str"`: String},
+ },
+ {
+ name: "standard double-quoted identifiers",
+ sql: `SELECT "Col" FROM t`,
+ want: map[string]Category{`"Col"`: QuotedIdentifier},
+ },
+ {
+ name: "sql server",
+ dialect: keywords.DialectSQLServer,
+ sql: "DECLARE @x INT; SELECT [c d], @@VERSION FROM #tmp",
+ want: map[string]Category{
+ "DECLARE": Keyword, "@x": Placeholder, "INT": DataType, "[c d]": QuotedIdentifier,
+ "@@VERSION": Placeholder, "#tmp": Identifier,
+ },
+ },
+ {
+ name: "clickhouse nested types",
+ dialect: keywords.DialectClickHouse,
+ sql: "CREATE TABLE t (a Nullable(String), m Map(String, UInt64)) ENGINE = MergeTree()",
+ want: map[string]Category{
+ "Nullable": DataType, "String": DataType, "Map": DataType, "UInt64": DataType, "MergeTree": Function,
+ },
+ },
+ {
+ name: "json operators",
+ sql: "SELECT j->>'k', j->'a' FROM t",
+ want: map[string]Category{"->>": Operator, "->": Operator, "'k'": String},
+ },
+ {
+ name: "tabs",
+ sql: "\tSELECT\ta,\tb\tFROM\tt /* x */\tWHERE\tc = '\tv'",
+ want: map[string]Category{
+ "SELECT": Keyword, "a": Identifier, "b": Identifier, "FROM": Keyword, "t": Identifier,
+ "/* x */": Comment, "WHERE": Keyword, "'\tv'": String,
+ },
+ },
+ {
+ name: "tabs on later lines",
+ sql: "SELECT 1\n\t\tFROM\tt\n\tWHERE x = 1",
+ want: map[string]Category{"FROM": Keyword, "t": Identifier, "WHERE": Keyword, "x": Identifier},
+ },
+ {
+ name: "keywords before a parenthesis that are not calls",
+ sql: "CREATE TABLE t (id INT, UNIQUE (id), c INT DEFAULT (now()), CHECK (id > 0)); SELECT a IN (1) FROM t WHERE EXISTS (SELECT 1)",
+ want: map[string]Category{
+ "UNIQUE": Keyword, "DEFAULT": Keyword, "CHECK": Keyword, "IN": Keyword, "EXISTS": Keyword, "now": Function,
+ },
+ },
+ {
+ name: "token types outside the classified ranges",
+ sql: "SELECT a FROM t WHERE a ILIKE 'x' OR b NOT ILIKE 'y'",
+ want: map[string]Category{"ILIKE": Keyword},
+ },
+ {
+ name: "unicode",
+ sql: "SELECT é, 'ünï' FROM t",
+ want: map[string]Category{"é": Identifier, "'ünï'": String},
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ spans := Spans(tt.sql, WithDialect(tt.dialect))
+ assertInvariants(t, tt.sql, spans)
+ for text, want := range tt.want {
+ if got := find(t, tt.sql, spans, text); got != want {
+ t.Errorf("%q: got %s, want %s\nall: %s", text, got, want, render(tt.sql, spans))
+ }
+ }
+ })
+ }
+}
+
+func TestSpans_ErrorRecovery(t *testing.T) {
+ tests := []struct {
+ dialect keywords.SQLDialect
+ sql string
+ want string
+ }{
+ {"", "SELECT 'ok', 'unterminated", "keyword:SELECT string:'ok' punctuation:, error:'unterminated"},
+ {"", "SELECT 1 /* open", "keyword:SELECT number:1 error:/* open"},
+ {"", "'x", "error:'x"},
+ {"", "\tSELECT 1 /* x", "keyword:SELECT number:1 error:/* x"},
+ {"", "\t\tSELECT a, 'abc", "keyword:SELECT identifier:a punctuation:, error:'abc"},
+ // The tokenizer reports column 0 for a quoted identifier broken by a
+ // newline, and the end of input for unterminated dollar quotes.
+ {"", "SELECT 1,\n\"abc\nFROM t", "keyword:SELECT number:1 punctuation:, error:\"abc\nFROM t"},
+ {"", "\n\"\n", "error:\"\n"},
+ {"", "SELECT $$body", "keyword:SELECT error:$$body"},
+ {"", "SELECT $$a$$, $t$b$t$; SELECT $tag$body", "keyword:SELECT string:$$a$$ punctuation:, string:$t$b$t$ punctuation:; keyword:SELECT error:$tag$body"},
+ {keywords.DialectSQLServer, "SELECT [col", "keyword:SELECT error:[col"},
+ {keywords.DialectMySQL, `SELECT 'it\'s', 'open`, `keyword:SELECT string:'it\'s' punctuation:, error:'open`},
+ }
+ for _, tt := range tests {
+ t.Run(tt.sql, func(t *testing.T) {
+ spans := Spans(tt.sql, WithDialect(tt.dialect))
+ assertInvariants(t, tt.sql, spans)
+ if got := render(tt.sql, spans); got != tt.want {
+ t.Errorf("got %s\nwant %s", got, tt.want)
+ }
+ })
+ }
+}
+
+func TestSpans_EmptyAndWhitespace(t *testing.T) {
+ if got := Spans(""); got != nil {
+ t.Errorf("empty input: got %v, want nil", got)
+ }
+ if got := Spans(" \n\t "); len(got) != 0 {
+ t.Errorf("whitespace input: got %v, want no spans", got)
+ }
+}
+
+func TestSpans_UnknownDialectFallsBack(t *testing.T) {
+ sql := "SELECT 1"
+ got := render(sql, Spans(sql, WithDialect("no-such-dialect")))
+ if got != "keyword:SELECT number:1" {
+ t.Errorf("got %s", got)
+ }
+}
+
+func TestSpans_AllDialects(t *testing.T) {
+ sql := "WITH c AS (SELECT a, SUM(b) OVER (PARTITION BY a) FROM t /* x */ WHERE d > 1) SELECT * FROM c -- end"
+ for _, d := range keywords.AllDialects() {
+ t.Run(string(d), func(t *testing.T) {
+ spans := Spans(sql, WithDialect(d))
+ assertInvariants(t, sql, spans)
+ if find(t, sql, spans, "SELECT") != Keyword {
+ t.Errorf("SELECT not a keyword under %s", d)
+ }
+ if find(t, sql, spans, "/* x */") != Comment {
+ t.Errorf("block comment not a comment under %s", d)
+ }
+ })
+ }
+}
+
+// TestSpans_Corpus runs every SQL fixture in the repository through the
+// invariant checks, so tokens of new types can never silently go unhighlighted.
+func TestSpans_Corpus(t *testing.T) {
+ files, err := filepath.Glob("../../testdata/*/*.sql")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(files) == 0 {
+ t.Skip("no SQL fixtures found")
+ }
+ for _, f := range files {
+ data, err := os.ReadFile(f)
+ if err != nil {
+ t.Fatalf("read %s: %v", f, err)
+ }
+ dialect := keywords.SQLDialect(filepath.Base(filepath.Dir(f)))
+ t.Run(f, func(t *testing.T) {
+ sql := string(data)
+ assertInvariants(t, sql, Spans(sql, WithDialect(dialect)))
+ })
+ }
+}
+
+func TestSpans_JSON(t *testing.T) {
+ sql := "SELECT 1"
+ data, err := json.Marshal(Spans(sql))
+ if err != nil {
+ t.Fatal(err)
+ }
+ want := `[{"start":0,"end":6,"category":"keyword"},{"start":7,"end":8,"category":"number"}]`
+ if string(data) != want {
+ t.Errorf("got %s\nwant %s", data, want)
+ }
+ var back []Span
+ if err := json.Unmarshal(data, &back); err != nil {
+ t.Fatal(err)
+ }
+ if render(sql, back) != "keyword:SELECT number:1" {
+ t.Errorf("round trip: %s", render(sql, back))
+ }
+}
+
+func TestCategory_Names(t *testing.T) {
+ seen := map[string]bool{}
+ for _, c := range Categories() {
+ name := c.String()
+ if seen[name] {
+ t.Errorf("duplicate name %q", name)
+ }
+ seen[name] = true
+ back, ok := ParseCategory(name)
+ if !ok || back != c {
+ t.Errorf("ParseCategory(%q) = %v, %v", name, back, ok)
+ }
+ }
+ if _, err := Category(200).MarshalText(); err == nil {
+ t.Error("expected error marshaling invalid category")
+ }
+ var c Category
+ if err := c.UnmarshalText([]byte("bogus")); err == nil {
+ t.Error("expected error unmarshaling unknown category")
+ }
+ if Category(200).String() != "category(200)" {
+ t.Errorf("invalid category String: %s", Category(200))
+ }
+}
+
+func TestSpans_Concurrent(t *testing.T) {
+ sql := "SELECT a, COUNT(*) FROM t WHERE b = 'x' -- c\nGROUP BY a"
+ want := render(sql, Spans(sql))
+ dialects := keywords.AllDialects()
+ var wg sync.WaitGroup
+ for i := 0; i < 64; i++ {
+ wg.Add(1)
+ go func(i int) {
+ defer wg.Done()
+ _ = Spans(sql, WithDialect(dialects[i%len(dialects)]))
+ if got := render(sql, Spans(sql)); got != want {
+ t.Errorf("concurrent result differs: %s", got)
+ }
+ }(i)
+ }
+ wg.Wait()
+}
+
+// assertInvariants checks the contract every Spans result must satisfy.
+func assertInvariants(t *testing.T, src string, spans []Span) {
+ t.Helper()
+ prevEnd := 0
+ for i, s := range spans {
+ if s.Start < prevEnd || s.Start >= s.End || s.End > len(src) {
+ t.Fatalf("span %d %+v out of order or range (prevEnd=%d, len=%d)", i, s, prevEnd, len(src))
+ }
+ if s.Category == Plain || s.Category >= numCategories {
+ t.Fatalf("span %d has invalid category %d", i, s.Category)
+ }
+ if !utf8.RuneStart(src[s.Start]) || (s.End < len(src) && !utf8.RuneStart(src[s.End])) {
+ t.Fatalf("span %d %+v splits a UTF-8 sequence", i, s)
+ }
+ if s.Category == Error && s.End != len(src) {
+ t.Fatalf("error span %d does not extend to end of input", i)
+ }
+ prevEnd = s.End
+ }
+ // Every non-space byte outside spans is unexpected: the tokenizer accounts
+ // for all significant input.
+ cursor := 0
+ for _, s := range spans {
+ if gap := strings.TrimSpace(src[cursor:s.Start]); gap != "" {
+ t.Errorf("unclassified text %q before %+v", gap, s)
+ }
+ cursor = s.End
+ }
+ if gap := strings.TrimSpace(src[cursor:]); gap != "" && len(spans) > 0 {
+ t.Errorf("unclassified trailing text %q", gap)
+ }
+}
+
+func FuzzSpans(f *testing.F) {
+ for _, seed := range []string{
+ "SELECT * FROM t",
+ "SELECT 'unterminated",
+ "/* open",
+ "CREATE TABLE t (a INT, b VARCHAR(3))",
+ "SELECT $1, :a, @b, ?, x::int FROM t -- c",
+ "SELECT `q`, [b], \"c\", N'x', 0x1F",
+ "SELECT é FROM ünï",
+ "\tSELECT\t1,\t2 FROM t\t/* c */\tWHERE a = '\t'",
+ "\n\"\n",
+ "SELECT 1; SELECT $tag$body",
+ } {
+ f.Add(seed)
+ }
+ f.Fuzz(func(t *testing.T, sql string) {
+ if !utf8.ValidString(sql) {
+ return
+ }
+ spans := Spans(sql)
+ prevEnd := 0
+ for _, s := range spans {
+ if s.Start < prevEnd || s.Start >= s.End || s.End > len(sql) || s.Category == Plain {
+ t.Fatalf("invalid span %+v (prevEnd=%d len=%d)", s, prevEnd, len(sql))
+ }
+ prevEnd = s.End
+ }
+ // Token spans hug their text: only lexemes that can contain
+ // whitespace may start or end with it.
+ for _, s := range spans {
+ switch s.Category {
+ case String, Comment, QuotedIdentifier, Error:
+ continue
+ }
+ text := s.Text(sql)
+ if strings.TrimSpace(text) != text {
+ t.Fatalf("%s span %+v has surrounding whitespace: %q", s.Category, s, text)
+ }
+ }
+ // Clean input never leaves significant text unclassified.
+ if len(spans) == 0 || spans[len(spans)-1].Category != Error {
+ cursor := 0
+ for _, s := range spans {
+ if gap := strings.TrimSpace(sql[cursor:s.Start]); gap != "" {
+ t.Fatalf("unclassified text %q before %+v", gap, s)
+ }
+ cursor = s.End
+ }
+ if gap := strings.TrimSpace(sql[cursor:]); gap != "" {
+ t.Fatalf("unclassified trailing text %q (spans: %v)", gap, spans)
+ }
+ }
+ // Renderers must reproduce the source exactly once markup is removed.
+ // Source text is passed through verbatim, so input that already
+ // contains escape sequences cannot be told apart from added markup.
+ if strings.ContainsRune(sql, '\x1b') {
+ return
+ }
+ if got := stripANSI(ANSI(sql, spans, ANSIOptions{})); got != sql {
+ t.Fatalf("ANSI round trip mismatch:\n got %q\nwant %q", got, sql)
+ }
+ })
+}
+
+func BenchmarkSpans(b *testing.B) {
+ sql := strings.Repeat("SELECT u.id, u.name, COUNT(o.id) AS orders FROM users u "+
+ "LEFT JOIN orders o ON o.user_id = u.id WHERE u.created_at > $1 -- recent\n"+
+ "GROUP BY u.id, u.name HAVING COUNT(o.id) > 5 ORDER BY orders DESC LIMIT 10;\n", 10)
+ b.ReportAllocs()
+ b.SetBytes(int64(len(sql)))
+ for i := 0; i < b.N; i++ {
+ _ = Spans(sql)
+ }
+}
+
+func BenchmarkHTML(b *testing.B) {
+ sql := strings.Repeat("SELECT a, 'b' FROM t WHERE c = 1;\n", 50)
+ spans := Spans(sql)
+ b.ReportAllocs()
+ for i := 0; i < b.N; i++ {
+ _ = HTML(sql, spans, HTMLOptions{})
+ }
+}
diff --git a/pkg/highlight/render.go b/pkg/highlight/render.go
new file mode 100644
index 00000000..cb096de7
--- /dev/null
+++ b/pkg/highlight/render.go
@@ -0,0 +1,333 @@
+// Copyright 2026 GoSQLX Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package highlight
+
+import (
+ "fmt"
+ "io"
+ "strconv"
+ "strings"
+)
+
+// walk calls fn for every run of src in order: gaps between spans as Plain,
+// spans with their category. Spans that are out of range, empty, or overlap
+// an earlier span are skipped, so renderers never panic on bad input and
+// always reproduce src exactly.
+func walk(src string, spans []Span, fn func(text string, c Category) error) error {
+ cursor := 0
+ for _, s := range spans {
+ if s.Start < cursor || s.End > len(src) || s.Start >= s.End {
+ continue
+ }
+ if s.Start > cursor {
+ if err := fn(src[cursor:s.Start], Plain); err != nil {
+ return err
+ }
+ }
+ if err := fn(src[s.Start:s.End], s.Category); err != nil {
+ return err
+ }
+ cursor = s.End
+ }
+ if cursor < len(src) {
+ return fn(src[cursor:], Plain)
+ }
+ return nil
+}
+
+// ColorMode selects how ANSI output encodes colors.
+type ColorMode int
+
+const (
+ // Color256 maps theme colors to the xterm 256-color palette. It is the
+ // default and works in virtually every modern terminal.
+ Color256 ColorMode = iota
+ // TrueColor emits 24-bit colors. Use it when COLORTERM is "truecolor"
+ // or "24bit".
+ TrueColor
+ // NoColor emits only bold, italic and underline attributes.
+ NoColor
+)
+
+// ANSIOptions configures ANSI terminal rendering.
+type ANSIOptions struct {
+ // Theme supplies the styles. The zero Theme means DarkTheme.
+ Theme Theme
+ // Mode selects the color encoding. The zero value is Color256.
+ Mode ColorMode
+}
+
+const ansiReset = "\x1b[0m"
+
+// ANSI renders src with ANSI escape sequences for terminal display.
+//
+// Styles are reset before every newline and re-applied after it, so output
+// stays correct when piped through head, grep or a pager.
+func ANSI(src string, spans []Span, opts ANSIOptions) string {
+ var b strings.Builder
+ b.Grow(len(src) + len(spans)*12)
+ _ = WriteANSI(&b, src, spans, opts)
+ return b.String()
+}
+
+// WriteANSI is like ANSI but writes to w.
+func WriteANSI(w io.Writer, src string, spans []Span, opts ANSIOptions) error {
+ theme := opts.Theme
+ if theme.Styles == nil {
+ theme = DarkTheme()
+ }
+ var sgr [numCategories]string
+ for c := Keyword; c < numCategories; c++ {
+ sgr[c] = sgrFor(theme.Style(c), opts.Mode)
+ }
+ sw := stringWriter(w)
+ return walk(src, spans, func(text string, c Category) error {
+ seq := ""
+ if c < numCategories {
+ seq = sgr[c]
+ }
+ if seq == "" {
+ _, err := sw.WriteString(text)
+ return err
+ }
+ for {
+ line, rest, more := strings.Cut(text, "\n")
+ if line != "" {
+ if _, err := sw.WriteString(seq + line + ansiReset); err != nil {
+ return err
+ }
+ }
+ if !more {
+ return nil
+ }
+ if _, err := sw.WriteString("\n"); err != nil {
+ return err
+ }
+ text = rest
+ }
+ })
+}
+
+// sgrFor builds the "\x1b[...m" sequence for a style, or "" for no styling.
+func sgrFor(s Style, mode ColorMode) string {
+ var params []string
+ if s.Bold {
+ params = append(params, "1")
+ }
+ if s.Italic {
+ params = append(params, "3")
+ }
+ if s.Underline {
+ params = append(params, "4")
+ }
+ if r, g, b, ok := parseHex(s.Color); ok && mode != NoColor {
+ if mode == TrueColor {
+ params = append(params, fmt.Sprintf("38;2;%d;%d;%d", r, g, b))
+ } else {
+ params = append(params, "38;5;"+strconv.Itoa(xterm256(r, g, b)))
+ }
+ }
+ if len(params) == 0 {
+ return ""
+ }
+ return "\x1b[" + strings.Join(params, ";") + "m"
+}
+
+// xterm256 returns the closest xterm 256-color index (6x6x6 cube or
+// grayscale ramp) for an RGB color.
+func xterm256(r, g, b uint8) int {
+ levels := [6]int{0, 95, 135, 175, 215, 255}
+ nearest := func(v uint8) int {
+ best, bestD := 0, 1<<30
+ for i, l := range levels {
+ if d := abs(int(v) - l); d < bestD {
+ best, bestD = i, d
+ }
+ }
+ return best
+ }
+ ri, gi, bi := nearest(r), nearest(g), nearest(b)
+ cube := 16 + 36*ri + 6*gi + bi
+ cubeD := sq(int(r)-levels[ri]) + sq(int(g)-levels[gi]) + sq(int(b)-levels[bi])
+
+ avg := (int(r) + int(g) + int(b)) / 3
+ gi2 := (avg - 8) / 10
+ if gi2 < 0 {
+ gi2 = 0
+ } else if gi2 > 23 {
+ gi2 = 23
+ }
+ gray := 8 + gi2*10
+ grayD := sq(int(r)-gray) + sq(int(g)-gray) + sq(int(b)-gray)
+ if grayD < cubeD {
+ return 232 + gi2
+ }
+ return cube
+}
+
+func abs(v int) int {
+ if v < 0 {
+ return -v
+ }
+ return v
+}
+
+func sq(v int) int { return v * v }
+
+// DefaultClassPrefix is the CSS class prefix used by HTML and CSS when no
+// prefix is given: spans render as .
+const DefaultClassPrefix = "gsx-"
+
+// HTMLOptions configures HTML rendering.
+type HTMLOptions struct {
+ // ClassPrefix prefixes category class names. Empty means
+ // DefaultClassPrefix. Prefixes containing characters other than ASCII
+ // letters, digits, '-' and '_' are replaced by DefaultClassPrefix.
+ ClassPrefix string
+ // InlineStyles emits style="..." attributes from Theme instead of class
+ // names, for contexts without a stylesheet (e-mail, Markdown exports).
+ InlineStyles bool
+ // Theme supplies styles when InlineStyles is set. The zero Theme means
+ // DarkTheme.
+ Theme Theme
+ // Wrap encloses the output in .
+ Wrap bool
+}
+
+var htmlEscaper = strings.NewReplacer(
+ "&", "&", "<", "<", ">", ">", `"`, """, "'", "'",
+)
+
+// HTML renders src as HTML. All source text is escaped; only
+// elements (and the optional wrapper) are emitted.
+func HTML(src string, spans []Span, opts HTMLOptions) string {
+ var b strings.Builder
+ b.Grow(len(src) + len(spans)*32)
+ _ = WriteHTML(&b, src, spans, opts)
+ return b.String()
+}
+
+// WriteHTML is like HTML but writes to w.
+func WriteHTML(w io.Writer, src string, spans []Span, opts HTMLOptions) error {
+ prefix := sanitizePrefix(opts.ClassPrefix)
+ theme := opts.Theme
+ if theme.Styles == nil {
+ theme = DarkTheme()
+ }
+ var open [numCategories]string
+ for c := Keyword; c < numCategories; c++ {
+ if opts.InlineStyles {
+ if css := cssDecl(theme.Style(c)); css != "" {
+ open[c] = ``
+ }
+ } else {
+ open[c] = ``
+ }
+ }
+
+ sw := stringWriter(w)
+ if opts.Wrap {
+ if _, err := sw.WriteString(``); err != nil {
+ return err
+ }
+ }
+ err := walk(src, spans, func(text string, c Category) error {
+ tag := ""
+ if c < numCategories {
+ tag = open[c]
+ }
+ if tag != "" {
+ if _, err := sw.WriteString(tag); err != nil {
+ return err
+ }
+ }
+ if _, err := htmlEscaper.WriteString(w, text); err != nil {
+ return err
+ }
+ if tag != "" {
+ _, err := sw.WriteString("
")
+ return err
+ }
+ return nil
+ })
+ if err != nil {
+ return err
+ }
+ if opts.Wrap {
+ _, err = sw.WriteString("
")
+ }
+ return err
+}
+
+// CSS returns a stylesheet with one rule per category for the class names
+// emitted by HTML, e.g. ".gsx-keyword{color:#569CD6;font-weight:bold}".
+//
+// The zero Theme means DarkTheme, as for HTML and ANSI.
+func CSS(theme Theme, classPrefix string) string {
+ if theme.Styles == nil {
+ theme = DarkTheme()
+ }
+ prefix := sanitizePrefix(classPrefix)
+ var b strings.Builder
+ for c := Keyword; c < numCategories; c++ {
+ if decl := cssDecl(theme.Style(c)); decl != "" {
+ b.WriteString("." + prefix + c.String() + "{" + decl + "}\n")
+ }
+ }
+ return b.String()
+}
+
+func cssDecl(s Style) string {
+ var parts []string
+ if r, g, b, ok := parseHex(s.Color); ok {
+ parts = append(parts, fmt.Sprintf("color:#%02X%02X%02X", r, g, b))
+ }
+ if s.Bold {
+ parts = append(parts, "font-weight:bold")
+ }
+ if s.Italic {
+ parts = append(parts, "font-style:italic")
+ }
+ if s.Underline {
+ parts = append(parts, "text-decoration:underline")
+ }
+ return strings.Join(parts, ";")
+}
+
+func sanitizePrefix(p string) string {
+ if p == "" {
+ return DefaultClassPrefix
+ }
+ for i := 0; i < len(p); i++ {
+ c := p[i]
+ if !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '-' || c == '_') {
+ return DefaultClassPrefix
+ }
+ }
+ return p
+}
+
+func stringWriter(w io.Writer) io.StringWriter {
+ if sw, ok := w.(io.StringWriter); ok {
+ return sw
+ }
+ return stringWriterAdapter{w}
+}
+
+type stringWriterAdapter struct{ w io.Writer }
+
+func (a stringWriterAdapter) WriteString(s string) (int, error) {
+ return a.w.Write([]byte(s))
+}
diff --git a/pkg/highlight/render_test.go b/pkg/highlight/render_test.go
new file mode 100644
index 00000000..36f2d7fd
--- /dev/null
+++ b/pkg/highlight/render_test.go
@@ -0,0 +1,225 @@
+// Copyright 2026 GoSQLX Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package highlight
+
+import (
+ "bytes"
+ "errors"
+ "regexp"
+ "strings"
+ "testing"
+)
+
+var ansiRE = regexp.MustCompile("\x1b\\[[0-9;]*m")
+
+func stripANSI(s string) string { return ansiRE.ReplaceAllString(s, "") }
+
+func TestANSI(t *testing.T) {
+ sql := "SELECT 'a'"
+ spans := Spans(sql)
+
+ got := ANSI(sql, spans, ANSIOptions{Theme: MonoTheme(), Mode: NoColor})
+ if want := "\x1b[1mSELECT\x1b[0m 'a'"; got != want {
+ t.Errorf("mono: got %q, want %q", got, want)
+ }
+
+ got = ANSI(sql, spans, ANSIOptions{Mode: TrueColor})
+ if !strings.Contains(got, "\x1b[1;38;2;86;156;214mSELECT\x1b[0m") {
+ t.Errorf("truecolor keyword sequence missing: %q", got)
+ }
+
+ got = ANSI(sql, spans, ANSIOptions{})
+ if !strings.Contains(got, "38;5;") {
+ t.Errorf("256-color sequence missing: %q", got)
+ }
+ if stripANSI(got) != sql {
+ t.Errorf("stripped output %q != source", stripANSI(got))
+ }
+}
+
+func TestANSI_ResetsAtNewlines(t *testing.T) {
+ sql := "/* a\nb */ SELECT 1"
+ got := ANSI(sql, Spans(sql), ANSIOptions{Mode: TrueColor})
+ for i, line := range strings.Split(got, "\n") {
+ if strings.Count(line, "\x1b[") > 0 && !strings.HasSuffix(strings.TrimRight(line, " "), ansiReset) {
+ t.Errorf("line %d does not end with a reset: %q", i, line)
+ }
+ }
+ if stripANSI(got) != sql {
+ t.Errorf("stripped output %q != source", stripANSI(got))
+ }
+}
+
+func TestHTML(t *testing.T) {
+ sql := "SELECT '&\"' FROM t --