perf(parser): currentToken as a pointer into p.tokens with a per-parser EOF sentinel [W3-02]

CHANGELOG.md

@@ -51,6 +51,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - **Tokenizer**: token Start/End locations are now computed from a cached current-line window (a range check and a subtraction) instead of two calls to the non-inlinable position lookup per token; out-of-window offsets take the existing lookup. Token positions are identical, including after a UTF-8 BOM and across pooled-tokenizer reuse (guarded by an oracle test). [W2-02]
 - **Pools**: the tokenizer, parser and AST pools now count Get/Put through precreated, cache-line padded atomic counters (`internal/poolstats`) instead of an RWMutex-guarded map keyed by pool name, removing a lock, map lookup and string hash from every pool operation. `metrics.GetPoolStats` and `metrics.ResetPoolStats` behave exactly as before; named custom pools keep the map path. [W1-01]
 - **Parser**: token preprocessing now takes a type-gated fast path. Only identifier, keyword, aggregate and compound tokens take the normalisation path; every other token is appended with a single copy. Output is identical (guarded by an old-versus-new differential test); no API or behaviour change.
+- **Parser**: the parser's current token is now a pointer into its token slice (with a per-parser zero EOF sentinel) instead of a 72-byte copy made on every `advance`, and `peekToken` returns a pointer, which removes most GC write-barrier work and struct copying from the parse loop. Parse results, errors and positions are byte-identical (behaviour goldens plus a recovery-path differential); no API change. A zero-value `&Parser{}` keeps working. [W3-02]
 - **Fingerprint, formatter, transform**: `fingerprint.Normalize`/`Fingerprint`, `Formatter.Format` and `transform.ParseSQL`/`ParseSQLWithDialect` tokenize through the internal pooled token scratch instead of the public `TokenizeString`. `Normalize` allocates 42-70% fewer bytes/op and `transform.ParseSQL` 49-78% fewer, with 2-3 fewer allocs/op and no output change. [W5-01]
 - **Formatter**: compact formatting (`ast.CompactStyle()`, no clause newlines, no semicolon, no line width, no target dialect, preserve or upper keyword case) now renders SELECT, INSERT, UPDATE and DELETE through each statement's `SQL()` instead of the visitor renderer, and a single compact statement skips the parts slice and join. `Render/compact/*` is 27-49% faster with 1 alloc/op (was 4-42) and 60-82% fewer bytes/op; `Normalize` and `Fingerprint` are 6-13% faster with 15-39% fewer allocs. Output is byte-identical (differential test over every corpus statement in 9 dialects and all option combinations); lowercase keywords, target-dialect rewrites, set operations and ALTER TABLE keep the visitor path. [W5-02]
 

pkg/sql/parser/currenttoken_corpus_test.go

@@ -0,0 +1,73 @@
+// Copyright 2026 GoSQLX Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package parser_test
+
+import (
+	"testing"
+
+	"github.com/ajitpratap0/GoSQLX/internal/goldens"
+	"github.com/ajitpratap0/GoSQLX/pkg/models"
+	"github.com/ajitpratap0/GoSQLX/pkg/sql/ast"
+	"github.com/ajitpratap0/GoSQLX/pkg/sql/keywords"
+	"github.com/ajitpratap0/GoSQLX/pkg/sql/parser"
+	"github.com/ajitpratap0/GoSQLX/pkg/sql/tokenizer"
+)
+
+// TestCurrentTokenSentinelUnmodifiedOverCorpus drives all three token entry
+// points over the whole W0-02 benchmark corpus under every dialect and
+// requires the per-parser EOF sentinel to stay the zero token (PLAN W3-02).
+// This reaches dialect-specific parse paths where a future write through
+// p.currentToken would corrupt end-of-input checks. After each statement the
+// parser is reset and must hold no pointer into the token slice.
+func TestCurrentTokenSentinelUnmodifiedOverCorpus(t *testing.T) {
+	stmts := goldens.Corpus(t)
+	for _, dn := range goldens.Dialects() {
+		d := keywords.SQLDialect(dn)
+		tkz, err := tokenizer.NewWithDialect(d)
+		if err != nil {
+			t.Fatalf("%s: tokenizer: %v", dn, err)
+		}
+		p := parser.NewParser(parser.WithDialect(dn))
+		for _, st := range stmts {
+			toks, terr := tkz.Tokenize([]byte(st.SQL))
+			if terr != nil {
+				tkz.Reset()
+				continue
+			}
+			toks = append([]models.TokenWithSpan(nil), toks...)
+			tkz.Reset()
+
+			// Tokenizer output ends with an EOF token, so the parser almost
+			// never walks past len(p.tokens). Also feed the stream with that
+			// trailing EOF stripped so currentToken/peekToken reach the
+			// per-parser sentinel (&p.eof) on every dialect-specific path.
+			streams := [][]models.TokenWithSpan{toks}
+			if n := len(toks); n > 0 && toks[n-1].Token.Type == models.TokenTypeEOF {
+				streams = append(streams, toks[:n-1])
+			}
+			for _, ts := range streams {
+				if tree, err := p.ParseFromModelTokens(ts); err == nil {
+					ast.ReleaseAST(tree)
+				}
+				p.ParseWithRecoveryFromModelTokens(ts)
+				if tree, err := p.ParseContextFromModelTokens(t.Context(), ts); err == nil {
+					ast.ReleaseAST(tree)
+				}
+				if !parser.EOFSentinelClean(p) {
+					t.Fatalf("dialect %s, %s#%d: EOF sentinel modified", dn, st.File, st.Index)
+				}
+			}
+			p.Reset()
+			// Reset clears the dialect; re-apply so every statement runs under dn.
+			p.ApplyOptions(parser.WithDialect(dn))
+			if !parser.EOFSentinelClean(p) || !parser.CursorRepointed(p) {
+				t.Fatalf("dialect %s, %s#%d: Reset left a stale cursor or dirty sentinel", dn, st.File, st.Index)
+			}
+		}
+		p.Release()
+		if !parser.CursorRepointed(p) {
+			t.Fatalf("dialect %s: Release left a stale cursor", dn)
+		}
+	}
+}