perf(tokenizer): zero-copy @param and @@var token values [W2-03]

CHANGELOG.md

@@ -49,6 +49,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - **Tokenizer**: `time.Now`/`time.Since` are now read only when metrics are enabled (previously twice per tokenization even when disabled). If metrics are toggled on mid-call the call is not recorded. No API or token-output change [W1-02].
 - **Tokenizer**: keyword and compound-keyword-start lookup now uses a single fused-hash open-addressed table (one FNV-1a hash computed while upper-casing, one probe) instead of two Go map lookups per identifier. The table is filled in a fixed order (hot keywords first, then sorted) so its probe layout is deterministic across processes. No behaviour change; token output is identical [W2-01].
 - **Tokenizer**: token Start/End locations are now computed from a cached current-line window (a range check and a subtraction) instead of two calls to the non-inlinable position lookup per token; out-of-window offsets take the existing lookup. Token positions are identical, including after a UTF-8 BOM and across pooled-tokenizer reuse (guarded by an oracle test). [W2-02]
+- **Tokenizer**: `@param` and SQL Server `@@var` token values are now slices of the input instead of `"@"+name` concatenations, removing one string allocation per parameter (SQL Server real-world allocs/op -90% to -100%, MariaDB/MySQL/Snowflake -50% to -67% on the large cells). Names that `readIdentifier` treats specially (`U&` escapes, a single letter before a quote, compound-keyword starts such as `@group by`, non-ASCII names) keep the previous path, so token types, values and spans are identical (pinned by a golden over 9 dialects). [W2-03]
 - **Pools**: the tokenizer, parser and AST pools now count Get/Put through precreated, cache-line padded atomic counters (`internal/poolstats`) instead of an RWMutex-guarded map keyed by pool name, removing a lock, map lookup and string hash from every pool operation. `metrics.GetPoolStats` and `metrics.ResetPoolStats` behave exactly as before; named custom pools keep the map path. [W1-01]
 - **Parser**: token preprocessing now takes a type-gated fast path. Only identifier, keyword, aggregate and compound tokens take the normalisation path; every other token is appended with a single copy. Output is identical (guarded by an old-versus-new differential test); no API or behaviour change.
 - **Parser**: the parser's current token is now a pointer into its token slice (with a per-parser zero EOF sentinel) instead of a 72-byte copy made on every `advance`, and `peekToken` returns a pointer, which removes most GC write-barrier work and struct copying from the parse loop. Parse results, errors and positions are byte-identical (behaviour goldens plus a recovery-path differential); no API change. A zero-value `&Parser{}` keeps working. [W3-02]

pkg/sql/tokenizer/at_param_test.go

@@ -0,0 +1,192 @@
+// Copyright 2026 GoSQLX Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tokenizer
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+	"unsafe"
+
+	tkerrors "github.com/ajitpratap0/GoSQLX/pkg/errors"
+	"github.com/ajitpratap0/GoSQLX/pkg/models"
+	"github.com/ajitpratap0/GoSQLX/pkg/sql/keywords"
+)
+
+// atParamInputs covers plain @param / @@var names and every edge case where
+// the identifier after '@' interacts with readIdentifier's special paths
+// (prefixed strings, U& escapes, compound keywords, non-ASCII folding).
+var atParamInputs = []string{
+	"SELECT @id",
+	"SELECT @id, @name FROM t WHERE a = @a1 AND b=@b_2",
+	"SELECT @@ROWCOUNT, @@version",
+	"SELECT @@ROWCOUNT",
+	"SELECT @@ x",
+	"SELECT @",
+	"SELECT @ id",
+	"SELECT @>",
+	"SELECT a @> b, a @@ b",
+	"SELECT @1",
+	"SELECT @_x$y#z",
+	"SELECT @x.y",
+	"SELECT @x::int",
+	"SELECT @é, @@ü, @日本",
+	"SELECT @ſ, @ı",
+	"SELECT @N'x'",
+	"SELECT @n'x', @X'ab', @B'1', @E'a\\nb', @q'[x]'",
+	"SELECT @a'x'",
+	"SELECT @U&'d\\0061t'",
+	"SELECT @u&\"d\"",
+	"SELECT @@U&'x'",
+	"SELECT @group by x",
+	"SELECT @GROUP   BY x",
+	"SELECT @group\nby x",
+	"SELECT @order by, @left join, @right outer, @inner join, @cross join",
+	"SELECT @natural join, @full outer join, @grouping sets, @outer apply",
+	"SELECT @rıght join x",
+	"SELECT @ınner join x",
+	"SELECT @groupıng sets",
+	"SELECT @@rıght join x",
+	"SELECT @select, @From, @Where",
+	"SELECT @group",
+	"SELECT @group, @order",
+	"SELECT @@group by",
+	"SELECT @@Left Outer Join",
+	"SELECT\n  @a,\n  @@b,\r\n  @c",
+	"SELECT '@id', \"@id\", -- @id\n 1 /* @id */",
+	"EXEC p @a = 1, @b = @c OUTPUT",
+	"SELECT @x ",
+	"SELECT @abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789",
+}
+
+var atParamDialects = []keywords.SQLDialect{
+	keywords.DialectGeneric, keywords.DialectSQLServer, keywords.DialectPostgreSQL,
+	keywords.DialectMySQL, keywords.DialectMariaDB, keywords.DialectOracle,
+	keywords.DialectSQLite, keywords.DialectSnowflake, keywords.DialectClickHouse,
+}
+
+func renderAtParam(dialect keywords.SQLDialect, sql string) string {
+	tkz, err := NewWithDialect(dialect)
+	if err != nil {
+		return "newerr: " + err.Error()
+	}
+	toks, err := tkz.Tokenize([]byte(sql))
+	var b strings.Builder
+	for _, tw := range toks {
+		fmt.Fprintf(&b, "%d %q q=%d %d:%d-%d:%d\n", tw.Token.Type, tw.Token.Value, tw.Token.Quote,
+			tw.Start.Line, tw.Start.Column, tw.End.Line, tw.End.Column)
+	}
+	if err != nil {
+		// Only code and position: the full rendering carries wording, context
+		// lines and help URLs that are not tokenization behaviour.
+		var te *tkerrors.Error
+		if errors.As(err, &te) {
+			fmt.Fprintf(&b, "ERR %s %d:%d\n", te.Code, te.Location.Line, te.Location.Column)
+		} else {
+			fmt.Fprintf(&b, "ERR other\n")
+		}
+	}
+	return b.String()
+}
+
+// TestAtParam_Golden pins the exact token stream (types, values, quote and
+// spans, plus any error) for @param / @@var inputs in every dialect. The
+// golden was generated before the zero-copy change, so any diff is a
+// behaviour change.
+func TestAtParam_Golden(t *testing.T) {
+	var out strings.Builder
+	for _, d := range atParamDialects {
+		for _, sql := range atParamInputs {
+			fmt.Fprintf(&out, "=== %s %q\n%s", d, sql, renderAtParam(d, sql))
+		}
+	}
+	const path = "testdata/at_param_golden.txt"
+	if os.Getenv("GOSQLX_UPDATE_GOLDEN") != "" {
+		if err := os.WriteFile(path, []byte(out.String()), 0o644); err != nil {
+			t.Fatal(err)
+		}
+		return
+	}
+	want, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := strings.ReplaceAll(out.String(), "\r\n", "\n"); got != strings.ReplaceAll(string(want), "\r\n", "\n") {
+		t.Fatalf("@param token stream differs from golden %s (regenerate only for an intentional change)", path)
+	}
+}
+
+// TestAtParam_ValueIsSourceSlice checks that plain @name / @@name values are
+// substrings of the TokenizeString input (same backing bytes, no copy).
+func TestAtParam_ValueIsSourceSlice(t *testing.T) {
+	tkz, err := NewWithDialect(keywords.DialectSQLServer)
+	if err != nil {
+		t.Fatal(err)
+	}
+	src := "SELECT @id, @@ROWCOUNT"
+	toks, err := tkz.TokenizeString(src)
+	if err != nil {
+		t.Fatal(err)
+	}
+	base := uintptr(unsafe.Pointer(unsafe.StringData(src)))
+	var got []string
+	for _, tw := range toks {
+		if tw.Token.Type != models.TokenTypePlaceholder && tw.Token.Type != models.TokenTypeIdentifier {
+			continue
+		}
+		v := tw.Token.Value
+		if !strings.HasPrefix(v, "@") {
+			continue
+		}
+		got = append(got, v)
+		p := uintptr(unsafe.Pointer(unsafe.StringData(v)))
+		if p < base || p+uintptr(len(v)) > base+uintptr(len(src)) {
+			t.Fatalf("value %q does not alias the input", v)
+		}
+	}
+	if strings.Join(got, ",") != "@id,@@ROWCOUNT" {
+		t.Fatalf("got %v", got)
+	}
+}
+
+// TestAtParam_NoPerTokenAlloc guards the zero-copy property: a statement
+// full of plain @params must not allocate a string per parameter.
+func TestAtParam_NoPerTokenAlloc(t *testing.T) {
+	var sb strings.Builder
+	sb.WriteString("SELECT ")
+	for i := 0; i < 64; i++ {
+		fmt.Fprintf(&sb, "@p%d, ", i)
+	}
+	sb.WriteString("@@ROWCOUNT")
+	sql := sb.String()
+	tkz, err := NewWithDialect(keywords.DialectSQLServer)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := tkz.TokenizeString(sql); err != nil {
+		t.Fatal(err)
+	}
+	allocs := testing.AllocsPerRun(20, func() {
+		if _, err := tkz.TokenizeString(sql); err != nil {
+			t.Fatal(err)
+		}
+	})
+	// Result slice growth only; the old code added >= 65 string allocations.
+	if allocs > 20 {
+		t.Fatalf("allocs/run = %v, want <= 20 (per-param concatenation regressed)", allocs)
+	}
+}