diff --git a/CHANGELOG.md b/CHANGELOG.md index df4d1b49..4158fc93 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 are removed. ### Performance +- **Security**: `Scanner.ScanSQL` now skips the dollar-quote stripping lexer, the identifier-tautology regex and the ClickHouse table-function regex on inputs that provably cannot match them, using linear-time necessary-condition guards (`hasDollarOpener`, `hasEqualIdentPair`, `hasTableFnCandidate`). The regexes still confirm every finding, so results are byte-identical (guarded by a guards-on/off equivalence test over the real-world corpus and 200k random inputs); no API or behaviour change. [W6-01] - **Tokenizer**: `time.Now`/`time.Since` are now read only when metrics are enabled (previously twice per tokenization even when disabled). If metrics are toggled on mid-call the call is not recorded. No API or token-output change [W1-02]. - **Tokenizer**: keyword and compound-keyword-start lookup now uses a single fused-hash open-addressed table (one FNV-1a hash computed while upper-casing, one probe) instead of two Go map lookups per identifier. The table is filled in a fixed order (hot keywords first, then sorted) so its probe layout is deterministic across processes. No behaviour change; token output is identical [W2-01]. - **Pools**: the tokenizer, parser and AST pools now count Get/Put through precreated, cache-line padded atomic counters (`internal/poolstats`) instead of an RWMutex-guarded map keyed by pool name, removing a lock, map lookup and string hash from every pool operation. `metrics.GetPoolStats` and `metrics.ResetPoolStats` behave exactly as before; named custom pools keep the map path. [W1-01] diff --git a/pkg/sql/security/dollar_quote.go b/pkg/sql/security/dollar_quote.go index f52b2d3f..998b8390 100644 --- a/pkg/sql/security/dollar_quote.go +++ b/pkg/sql/security/dollar_quote.go @@ -23,7 +23,7 @@ import "strings" // is hidden. ScanSQL has no dialect: MySQL can use $x$ as an identifier, so a // MySQL-only payload of that shape is still stripped. func stripDollarQuotedStrings(sql string) string { - if strings.IndexByte(sql, '$') < 0 { + if strings.IndexByte(sql, '$') < 0 || !hasDollarOpener(sql) { return sql } // Cross product of escapes, nesting, bracket identifiers, hash comments and @@ -71,3 +71,20 @@ func resetDollarStates(states *[32]sqlLexState) { states[j] = sqlLexState{eOnly: j&16 != 0} } } + +// hasDollarOpener reports whether any '$' could open a dollar quote (the same +// opener condition the lexer loop below uses); when none can, the 32-state +// lexer can never strip anything, so it is skipped. +func hasDollarOpener(sql string) bool { + for i := strings.IndexByte(sql, '$'); i >= 0; { + if (i == 0 || !identifierByte(sql[i-1])) && dollarTagEnd(sql, i) != 0 { + return true + } + j := strings.IndexByte(sql[i+1:], '$') + if j < 0 { + return false + } + i += 1 + j + } + return false +} diff --git a/pkg/sql/security/fuzz_test.go b/pkg/sql/security/fuzz_test.go index 120b4fe1..ddd7cecc 100644 --- a/pkg/sql/security/fuzz_test.go +++ b/pkg/sql/security/fuzz_test.go @@ -90,6 +90,10 @@ func FuzzScanSQL(f *testing.F) { f.Add(s) } + for _, sql := range scanSQLGuardSeeds { + f.Add(sql) + } + f.Fuzz(func(t *testing.T, sql string) { scanner := NewScanner() _ = scanner.ScanSQL(sql) diff --git a/pkg/sql/security/guards.go b/pkg/sql/security/guards.go new file mode 100644 index 00000000..9544ec32 --- /dev/null +++ b/pkg/sql/security/guards.go @@ -0,0 +1,127 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package security + +import "strings" + +func withGuard(p compiledPattern, g func(string) bool) compiledPattern { p.guard = g; return p } + +func isASCII(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] >= 0x80 { + return false + } + } + return true +} + +func identRunByte(c byte) bool { + return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '_' || c == '.' +} + +// hasEqualIdentPair is a necessary condition for the identifier tautology +// capture pattern: around some '=', a suffix of the left identifier run equals +// (case-insensitively) a prefix of the right run. The runs include every byte +// accepted by [a-z_][a-z0-9_.]*; sqlWhitespace includes [\s\v]. Trying all +// suffix/prefix lengths up to 51 permits the regex's unanchored and partial +// matches: initTautologyCapturePatterns uses [a-z_][a-z0-9_.]{0,50}, so each +// ASCII capture has at most 51 bytes. A confirmed equality cannot require a +// longer comparison. Bounding k also keeps comparison work per '=' constant +// rather than quadratic in the lengths of unequal identifier runs. +// Call only for ASCII input: (?i) also folds U+017F (long s) to S and U+212A +// (Kelvin sign) to K, which these byte runs cannot recognize. +func hasEqualIdentPair(sql string) bool { + for off := 0; off < len(sql); { + r := strings.IndexByte(sql[off:], '=') + if r < 0 { + return false + } + eq := off + r + off = eq + 1 + l, rr := eq, eq+1 + for l > 0 && sqlWhitespace(sql[l-1]) { + l-- + } + for rr < len(sql) && sqlWhitespace(sql[rr]) { + rr++ + } + s, e := l, rr + for s > 0 && identRunByte(sql[s-1]) { + s-- + } + for e < len(sql) && identRunByte(sql[e]) { + e++ + } + left, right := sql[s:l], sql[rr:e] + for k := 1; k <= 51 && k <= len(left) && k <= len(right); k++ { + if strings.EqualFold(left[len(left)-k:], right[:k]) { + return true + } + } + } + return false +} + +var chNames = map[string]bool{} + +func init() { + for _, n := range strings.Split("url|urlcluster|file|filecluster|remote|remotesecure|s3|s3cluster|gcs|azureblobstorage|hdfs|hdfscluster|jdbc|odbc|mysql|postgresql|mongodb|redis|sqlite|executable", "|") { + chNames[n] = true + } +} + +func wordByte(c byte) bool { + return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '_' +} + +// hasTableFnCandidate: some '(' is preceded by optional whitespace, a listed +// table-function name as a whole word, and a whitespace byte. Any match of the +// FROM/JOIN regex must contain that suffix; the regex still confirms FROM/JOIN. +// sqlWhitespace includes exactly [\s\v], and the longest listed name fits buf. +// Call only for ASCII input: (?i) can match U+017F (long s) as S and U+212A +// (Kelvin sign) as K; ASCII name scanning would otherwise miss such matches. +func hasTableFnCandidate(sql string) bool { + var buf [16]byte + for off := 0; off < len(sql); { + r := strings.IndexByte(sql[off:], '(') + if r < 0 { + return false + } + p := off + r + off = p + 1 + e := p + for e > 0 && sqlWhitespace(sql[e-1]) { + e-- + } + s := e + for s > 0 && wordByte(sql[s-1]) { + s-- + } + if e-s < 2 || e-s > len(buf) || s == 0 || !sqlWhitespace(sql[s-1]) { + continue + } + for i := s; i < e; i++ { + c := sql[i] + if 'A' <= c && c <= 'Z' { + c += 32 + } + buf[i-s] = c + } + if chNames[string(buf[:e-s])] { + return true + } + } + return false +} diff --git a/pkg/sql/security/guards_boundary_test.go b/pkg/sql/security/guards_boundary_test.go new file mode 100644 index 00000000..7e1f2572 --- /dev/null +++ b/pkg/sql/security/guards_boundary_test.go @@ -0,0 +1,113 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package security + +import ( + "os" + "reflect" + "regexp" + "strings" + "testing" +) + +// TestHasEqualIdentPairCaptureBoundary pins the 51-byte capture bound of +// [a-z_][a-z0-9_.]{0,50}. A border-free identifier ("a" followed by 50 "b") +// equal on both sides of '=' can only be matched by the full 51-byte comparison. +func TestHasEqualIdentPairCaptureBoundary(t *testing.T) { + id51 := "a" + strings.Repeat("b", 50) + id52 := "a" + strings.Repeat("b", 51) + scanner := NewScanner() + cases := []struct { + name, sql string + want bool + }{ + {"51 plain", id51 + "=" + id51, true}, + {"51 spaced", id51 + " = " + id51, true}, + {"51 case-folded", strings.ToUpper(id51) + "=" + id51, true}, + {"51 left only prefix", "x " + id51 + "=" + id51 + " y", true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := hasEqualIdentPair(tc.sql); got != tc.want { + t.Fatalf("hasEqualIdentPair(%q) = %v, want %v", tc.sql, got, tc.want) + } + if r := scanner.ScanSQL(tc.sql); r.TotalCount == 0 { + t.Fatalf("ScanSQL(%q) reported no finding", tc.sql) + } + }) + } + // 52 bytes exceeds the regex capture bound; the guard must agree with the + // regex-only path whatever it decides. + for _, sql := range []string{id52 + "=" + id52, id52 + " = " + id52} { + got, want := scanner.ScanSQL(sql), scanSQLRegexOnly(scanner, sql) + if !reflect.DeepEqual(got, want) { + t.Fatalf("52-byte case %q: guarded %+v, regex-only %+v", sql, got, want) + } + } +} + +// TestTableFnGuardCoversEveryRegexName extracts the names from the ClickHouse +// table-function alternation in scanner.go and requires the guard's name set to +// match it exactly, and every name (mixed case, tab/VT whitespace) to be both a +// guard candidate and a ScanSQL finding. +func TestTableFnGuardCoversEveryRegexName(t *testing.T) { + src, err := os.ReadFile("scanner.go") + if err != nil { + t.Fatal(err) + } + m := regexp.MustCompile(`\(\?:FROM\|JOIN\)\[\\s\\v\]\+\(\?:([A-Za-z0-9_|]+)\)\[\\s\\v\]\*\\\(`).FindSubmatch(src) + if m == nil { + t.Fatal("ClickHouse table-function alternation not found in scanner.go") + } + names := strings.Split(string(m[1]), "|") + if len(names) != len(chNames) { + t.Fatalf("regex has %d names, guard has %d", len(names), len(chNames)) + } + scanner := NewScanner() + for _, name := range names { + if !chNames[strings.ToLower(name)] { + t.Errorf("guard is missing %q", name) + } + mixed := mixCase(name) + for _, sql := range []string{ + "SELECT * FROM " + mixed + " ('x')", + "SELECT * FROM t JOIN " + mixed + "\v('x')", + "select * from\t" + strings.ToUpper(name) + "(1)", + } { + if !hasTableFnCandidate(sql) { + t.Errorf("hasTableFnCandidate(%q) = false", sql) + } + found := false + for _, f := range scanner.ScanSQL(sql).Findings { + if f.Pattern == PatternOutOfBand { + found = true + } + } + if !found { + t.Errorf("ScanSQL(%q) did not report an out-of-band finding", sql) + } + } + } +} + +func mixCase(s string) string { + b := []byte(s) + for i := range b { + if i%2 == 1 && b[i] >= 'a' && b[i] <= 'z' { + b[i] -= 32 + } + } + return string(b) +} diff --git a/pkg/sql/security/prefilter.go b/pkg/sql/security/prefilter.go index 44d89b42..faf2899f 100644 --- a/pkg/sql/security/prefilter.go +++ b/pkg/sql/security/prefilter.go @@ -31,8 +31,9 @@ import ( // the regex cannot match and is skipped. A nil or empty lits means the regex always // runs. type compiledPattern struct { - re *regexp.Regexp - lits [][]byte + re *regexp.Regexp + lits [][]byte + guard func(string) bool // optional exact necessary condition (ASCII input only) } // compilePattern compiles expr and derives its literal prefilter. diff --git a/pkg/sql/security/scanner.go b/pkg/sql/security/scanner.go index 40eca789..bbd62590 100644 --- a/pkg/sql/security/scanner.go +++ b/pkg/sql/security/scanner.go @@ -289,7 +289,7 @@ func initCompiledPatterns() { compilePattern(`(?i)\bsp_oacreate\b`), // ClickHouse table functions that reach files, URLs, object // stores, other servers or external programs. - compilePattern(`(?i)\b(?:FROM|JOIN)[\s\v]+(?:url|urlCluster|file|fileCluster|remote|remoteSecure|s3|s3Cluster|gcs|azureBlobStorage|hdfs|hdfsCluster|jdbc|odbc|mysql|postgresql|mongodb|redis|sqlite|executable)[\s\v]*\(`), + withGuard(compilePattern(`(?i)\b(?:FROM|JOIN)[\s\v]+(?:url|urlCluster|file|fileCluster|remote|remoteSecure|s3|s3Cluster|gcs|azureBlobStorage|hdfs|hdfsCluster|jdbc|odbc|mysql|postgresql|mongodb|redis|sqlite|executable)[\s\v]*\(`), hasTableFnCandidate), }, severity: SeverityCritical, risk: "Out-of-band data exfiltration or command execution", @@ -1214,6 +1214,9 @@ func (s *Scanner) detectTautologyInSQL(views []string, folded []byte, result *Sc if patternIndex == 0 && folded != nil && !hasEqualNumericPair(input) { continue } + if patternIndex == 2 && folded != nil && isASCII(input) && !hasEqualIdentPair(input) { + continue + } for rest := input; rest != ""; { // FindString uses a stack capture buffer; SubmatchIndex allocates // an index slice for every unequal pair on large inputs. @@ -1337,6 +1340,9 @@ func (s *Scanner) detectRegexPatterns(views []string, folded []byte, patternType // no match, and a non-empty result is the same leftmost-first match that // FindStringSubmatch(sql)[0] would return. for _, sql := range views { + if p.guard != nil && folded != nil && isASCII(sql) && !p.guard(sql) { + continue + } if m := p.re.FindString(sql); m != "" { finding := Finding{ Severity: group.severity, diff --git a/pkg/sql/security/scansql_guards_test.go b/pkg/sql/security/scansql_guards_test.go new file mode 100644 index 00000000..6e02d75b --- /dev/null +++ b/pkg/sql/security/scansql_guards_test.go @@ -0,0 +1,228 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package security + +import ( + "io/fs" + "math/rand" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" +) + +// These are the verifier's mutation vocabulary, plus boundary and Unicode +// folding cases for the necessary-condition guards. Share them with fuzzing. +var scanSQLGuardSeeds = []string{ + " OR 1=1", " or a = A", " x=x ", "'a'='a'", " FROM url('http://x') ", + " JOIN s3 (", "from\tFile (", " $$ x $$ ", " $1 ", "$tag$ y $tag$", " WHERE id=id", + "ſ=s", "k = K", ";DROP TABLE x", " -- ", " UNION SELECT NULL,NULL ", "$", "$$", + "a.b=a.b", " from remote\v(", " FROM mysql (", "from redis(", " FROM s3Cluster (", + "FROM poſtgresql(", "JOIN ſqlite(", "FROM executable(K)", "s=ſ", "K=k", + "a$$ OR 1=1 $$", "$1 OR 1=1", "$tag$ OR 1=1 $tag$", "\xffa=a", +} + +func loadScanSQLFiles(t testing.TB, root string) []string { + t.Helper() + var inputs []string + err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() || !strings.HasSuffix(path, ".sql") { + return nil + } + data, err := os.ReadFile(path) // #nosec G304 -- repository test corpus + if err != nil { + return err + } + inputs = append(inputs, string(data)) + return nil + }) + if err != nil { + t.Fatalf("load SQL corpus %s: %v", root, err) + } + if len(inputs) == 0 { + t.Fatalf("no SQL files in %s", root) + } + return inputs +} + +func scanSQLStatements(files []string) []string { + var statements []string + for _, file := range files { + for _, statement := range strings.Split(file, ";\n") { + if statement = strings.TrimSpace(statement); statement != "" { + statements = append(statements, statement) + } + } + } + return statements +} + +// scanSQLRegexOnly keeps the pre-guard orchestration and dollar-quote handling +// independent of all three guards under test. A nil folded slice runs every +// regex, retaining the production regexes and finding construction verbatim. +func scanSQLRegexOnly(s *Scanner, sql string) *ScanResult { + result := &ScanResult{Findings: make([]Finding, 0)} + sql = stripDollarQuotedStringsReference(sql) + s.detectCommentPatterns(sql, nil, result) + views := append([]string{sql}, commentViews(sql)...) + s.detectRegexPatterns(views, nil, PatternTautology, result) + s.detectTautologyInSQL(views, nil, result) + for _, pattern := range []PatternType{PatternTimeBased, PatternOutOfBand, PatternDangerousFunc, PatternUnionInjection, PatternUnionGeneric, PatternStackedQuery} { + s.detectRegexPatterns(views, nil, pattern, result) + } + s.updateCounts(result) + return result +} + +// TestScanSQLGuardsEquivalence ports the verifier's seeded differential harness. +// Compare entire results (including every finding field and order), rather than +// only detection counts. Race/short runs retain every corpus and Unicode case, +// plus the first 20k random cases; ordinary runs cover the full 200k set. +func TestScanSQLGuardsEquivalence(t *testing.T) { + scanner := NewScanner() + check := func(sql string) { + t.Helper() + got, want := scanner.ScanSQL(sql), scanSQLRegexOnly(scanner, sql) + if !reflect.DeepEqual(got, want) { + t.Fatalf("ScanSQL guard mismatch for %q:\n got: %+v\nwant: %+v", sql, got, want) + } + } + rng := rand.New(rand.NewSource(7)) + mutate := func(sql string) string { + for n := 1 + rng.Intn(3); n > 0; n-- { + at := rng.Intn(len(sql) + 1) + sql = sql[:at] + scanSQLGuardSeeds[rng.Intn(len(scanSQLGuardSeeds))] + sql[at:] + } + return sql + } + root := filepath.Join("..", "..", "..", "testdata") + for _, name := range []string{"realworld", "corpus"} { + files := loadScanSQLFiles(t, filepath.Join(root, name)) + for _, file := range files { + check(file) + check(mutate(file)) + } + for i, statement := range scanSQLStatements(files) { + check(statement) + if i%3 == 0 { + for n := 0; n < 4; n++ { + check(mutate(statement)) + } + } + } + } + for _, sql := range scanSQLGuardSeeds { + check(sql) + } + alphabet := []string{"a", "B", "x", "1", "=", " ", "\t", "$", "(", ")", "'", "url", "FROM ", "s3", " JOIN ", "$$", "_", ".", "ſ", "K", "--", ";"} + randomCases := 200000 + if testing.Short() || raceEnabled { + randomCases = 20000 + } + t.Logf("checking %d seeded random/mutated inputs plus all corpus and seed cases", randomCases) + for i := 0; i < randomCases; i++ { + var sql strings.Builder + for n := rng.Intn(14); n > 0; n-- { + sql.WriteString(alphabet[rng.Intn(len(alphabet))]) + } + input := sql.String() + // Include both random inputs and mutations throughout the seeded sample. + if i%4 == 0 { + input = mutate(input) + } + check(input) + } +} + +// The unequal suffix forces every candidate comparison to fail at its last +// byte. Quadrupling input size must not quadruple both candidate count and +// comparison length. Minima reduce scheduler noise; the generous ratio permits +// linear scanning overhead without imposing an absolute timing threshold. +func TestHasEqualIdentPairLinear(t *testing.T) { + measure := func(size int) time.Duration { + t.Helper() + n := (size - 2) / 2 + sql := strings.Repeat("a", n) + "b=" + strings.Repeat("a", size-n-2) + var best time.Duration + for sample := 0; sample < 3; sample++ { + start := time.Now() + for repeat := 0; repeat < 4; repeat++ { + if hasEqualIdentPair(sql) { + t.Fatal("unequal identifier suffix unexpectedly passed guard") + } + } + elapsed := time.Since(start) + if sample == 0 || elapsed < best { + best = elapsed + } + } + return best + } + small, large := measure(16*1024), measure(64*1024) + t.Logf("16 KiB: %v; 64 KiB: %v; ratio %.2f", small, large, float64(large)/float64(small)) + if large > 10*small { + t.Fatalf("identifier guard scaling exceeds linear allowance: 16 KiB=%v, 64 KiB=%v", small, large) + } +} + +// stripDollarQuotedStringsReference is the pre-guard implementation. +func stripDollarQuotedStringsReference(sql string) string { + if strings.IndexByte(sql, '$') < 0 { + return sql + } + // Cross product of escapes, nesting, bracket identifiers, hash comments and + // PostgreSQL E-string-only escapes. + var states [32]sqlLexState + resetDollarStates(&states) + var out strings.Builder + start := 0 + for i := 0; i < len(sql); i++ { + code := true + for j := range states { + code = code && states[j].code() + } + if code && sql[i] == '$' && (i == 0 || !identifierByte(sql[i-1])) { + if end := dollarTagEnd(sql, i); end != 0 { + tag := sql[i:end] + closeAt := strings.Index(sql[end:], tag) + if closeAt < 0 { + break + } + if start == 0 { + out.Grow(len(sql)) + } + out.WriteString(sql[start:end]) + out.WriteString(tag) + i = end + closeAt + len(tag) - 1 + start = i + 1 + resetDollarStates(&states) + continue + } + } + for j := range states { + states[j].step(sql, i, j&1 != 0, j&2 != 0, j&8 != 0, j&4 != 0) + } + } + if start == 0 { + return sql + } + out.WriteString(sql[start:]) + return out.String() +}