From d55334e90966477ce5223199db376becd2336e0a Mon Sep 17 00:00:00 2001 From: Ajit Pratap Singh <18012955+ajitpratap0@users.noreply.github.com> Date: Thu, 29 Oct 2026 13:34:44 +0530 Subject: [PATCH 1/4] perf(transpiler): cached reflection plans and source prefilters for Transpile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Transpile's operator, || and string-literal passes each walked every exported field of every node by reflection. They now share per-type traversal plans (pkg/internal/reflplan, cached in a sync.Map) that skip fields whose static type cannot hold an interface, so strings, numbers, positions and quote metadata are never visited. The operator walk reuses pooled scratch (bounded, cleared before reuse), the per-dialect-pair rule lists are built once, and source-byte prefilters skip the || rules, string-literal re-rendering and data-type requoting when the SQL has no byte that could need them. Output and errors are byte-identical to before on 955,476 differential cases. BenchmarkTranspile: 5.18 µs / 52 allocs -> 2.68 µs / 43 allocs; wide OR/AND expressions -68%, set-operation LIMIT -57%. Tests: plan unit tests (cycles, maps, embedded, concurrency), a coverage guard against an independent reachability check, a plan-vs-naive walk over the real-world corpus, prefilter equivalence across all 81 dialect pairs, cache/pool hygiene, and a SelectOps benchmark. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DnbRnTFErhoMJdAjVEeCg2 --- CLAUDE.md | 2 +- pkg/internal/reflplan/reflplan.go | 125 +++++++ pkg/internal/reflplan/reflplan_test.go | 134 ++++++++ pkg/sql/ast/reflplan_coverage_test.go | 203 ++++++++++++ pkg/sql/ast/retarget_operators.go | 104 ++++-- pkg/sql/ast/retarget_operators_deep_test.go | 2 +- pkg/transpiler/dialect_rules.go | 50 ++- pkg/transpiler/prefilter_test.go | 341 ++++++++++++++++++++ pkg/transpiler/quoting.go | 6 +- pkg/transpiler/rules/pg_concat.go | 36 ++- pkg/transpiler/rules/pipes_or.go | 32 +- pkg/transpiler/string_literals.go | 44 ++- pkg/transpiler/transpiler.go | 51 ++- pkg/transpiler/transpiler_test.go | 36 +++ pkg/transpiler/walk_coverage_test.go | 206 ++++++++++++ 15 files changed, 1288 insertions(+), 84 deletions(-) create mode 100644 pkg/internal/reflplan/reflplan.go create mode 100644 pkg/internal/reflplan/reflplan_test.go create mode 100644 pkg/sql/ast/reflplan_coverage_test.go create mode 100644 pkg/transpiler/prefilter_test.go create mode 100644 pkg/transpiler/walk_coverage_test.go diff --git a/CLAUDE.md b/CLAUDE.md index 29e8b24c3..2e3500263 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -58,7 +58,7 @@ metrics → (no deps) keywords → (no deps) token → (no deps) tokenizer → models, errors, metrics, keywords -ast → models, metrics +ast → models, metrics, internal/reflplan parser → models, errors, keywords, token, tokenizer, ast # Higher-level / product packages diff --git a/pkg/internal/reflplan/reflplan.go b/pkg/internal/reflplan/reflplan.go new file mode 100644 index 000000000..f401b44ec --- /dev/null +++ b/pkg/internal/reflplan/reflplan.go @@ -0,0 +1,125 @@ +// Package reflplan precomputes, per reflect.Type, which parts of a value can +// hold an interface value. Reflective AST walkers use it to skip strings, +// numbers, positions and other leaf data without calling reflect.Type.Field +// on every visit. Plans are immutable once published and safe for concurrent +// use. Reach is the least fixpoint over the reachable type graph, seeded by +// interface kinds. Only exported struct fields participate; Fields contains +// exactly those that reach an interface, in declaration order. Pointer, slice, +// array (including zero-length arrays) and map plans always have Elem set. +// Plans are fully built before LoadOrStore publishes them and never mutated +// afterwards. Walkers prune only interface-free static types: every rewrite +// site is an interface slot, and BinaryExpression, UnaryExpression, FunctionCall, +// ParenthesizedExpression and LiteralValue all reach one. Reachable slots keep +// their original visit order. +package reflplan + +import ( + "reflect" + "sync" +) + +// Plan describes how to traverse values of one type. +type Plan struct { + Type reflect.Type + Kind reflect.Kind + // Reach reports whether a value of this type can (transitively, through + // exported fields, elements, pointers or interfaces) hold an interface. + Reach bool + // Fields lists the exported struct fields whose type Reaches, in + // declaration order. + Fields []Field + // Elem is the element plan of a pointer, slice, array or map. + Elem *Plan +} + +// Field is one exported struct field worth visiting. +type Field struct { + Index int + Plan *Plan +} + +var cache sync.Map // reflect.Type -> *Plan + +// Of returns the plan for t, building it (and the plans of every type +// reachable from t) on first use. +func Of(t reflect.Type) *Plan { + if p, ok := cache.Load(t); ok { + if plan, ok := p.(*Plan); ok { + return plan + } + } + return build(t) +} + +type rawField struct { + index int + t reflect.Type +} + +func build(root reflect.Type) *Plan { + local := map[reflect.Type]*Plan{} + raw := map[*Plan][]rawField{} + var order []*Plan + var get func(t reflect.Type) *Plan + get = func(t reflect.Type) *Plan { + if p, ok := local[t]; ok { + return p + } + if p, ok := cache.Load(t); ok { + if plan, ok := p.(*Plan); ok { + local[t] = plan + return plan + } + } + p := &Plan{Type: t, Kind: t.Kind(), Reach: t.Kind() == reflect.Interface} + local[t] = p + order = append(order, p) + switch t.Kind() { + case reflect.Pointer, reflect.Slice, reflect.Array, reflect.Map: + p.Elem = get(t.Elem()) + case reflect.Struct: + var fs []rawField + for i := 0; i < t.NumField(); i++ { + if sf := t.Field(i); sf.IsExported() { + fs = append(fs, rawField{i, sf.Type}) + get(sf.Type) + } + } + raw[p] = fs + } + return p + } + rootPlan := get(root) + // Least fixpoint of Reach over the (possibly cyclic) type graph. + for changed := true; changed; { + changed = false + for _, p := range order { + if p.Reach { + continue + } + switch p.Kind { + case reflect.Pointer, reflect.Slice, reflect.Array, reflect.Map: + p.Reach = p.Elem.Reach + case reflect.Struct: + for _, f := range raw[p] { + if local[f.t].Reach { + p.Reach = true + break + } + } + } + changed = changed || p.Reach + } + } + for _, p := range order { + for _, f := range raw[p] { + if fp := local[f.t]; fp.Reach { + p.Fields = append(p.Fields, Field{Index: f.index, Plan: fp}) + } + } + } + for _, p := range order { + cache.LoadOrStore(p.Type, p) + } + return rootPlan +} diff --git a/pkg/internal/reflplan/reflplan_test.go b/pkg/internal/reflplan/reflplan_test.go new file mode 100644 index 000000000..2f78b1467 --- /dev/null +++ b/pkg/internal/reflplan/reflplan_test.go @@ -0,0 +1,134 @@ +package reflplan + +import ( + "reflect" + "sync" + "testing" +) + +type leaf struct { + A string + B int + C struct{ X, Y int } +} +type selfLeaf struct { + Name string + Next *selfLeaf +} +type withIface struct { + Name string + E any + L leaf + S []leaf + M map[string]any + P *selfLeaf + hid any +} +type cycA struct { + B *cycB + N int +} +type cycB struct { + A []cycA + Val any +} + +type Embedded struct{ Value any } + +func TestPlanContainers(t *testing.T) { + for _, value := range []any{map[string]any{}, []any{}, [0]any{}, (*any)(nil)} { + typ := reflect.TypeOf(value) + p := Of(typ) + if !p.Reach || p.Elem == nil || !p.Elem.Reach || p.Elem.Type != typ.Elem() { + t.Errorf("%v: incomplete container plan: %+v", typ, p) + } + } + // Only map values carry rewrite slots; interface keys do not reach. + for _, value := range []any{map[any]int{}, []leaf{}, [0]leaf{}, (*selfLeaf)(nil)} { + typ := reflect.TypeOf(value) + p := Of(typ) + if p.Reach || p.Elem == nil || p.Elem.Reach || p.Elem.Type != typ.Elem() { + t.Errorf("%v: invalid leaf container plan: %+v", typ, p) + } + } + typ := reflect.TypeOf(struct { + Before int + Embedded + After string + }{}) + p := Of(typ) + if !p.Reach || !reflect.DeepEqual(indices(p), []int{1}) || !reflect.DeepEqual(indices(p.Fields[0].Plan), []int{0}) { + t.Fatalf("embedded plan = %+v", p) + } + if Of(reflect.TypeOf(struct{ hidden any }{})).Reach { + t.Fatal("unexported interface field must not reach") + } +} + +func indices(p *Plan) []int { + var out []int + for _, f := range p.Fields { + out = append(out, f.Index) + } + return out +} + +func TestPlans(t *testing.T) { + if Of(reflect.TypeOf(leaf{})).Reach || Of(reflect.TypeOf(selfLeaf{})).Reach { + t.Fatal("interface-free types must not reach") + } + p := Of(reflect.TypeOf(withIface{})) + if !p.Reach || !reflect.DeepEqual(indices(p), []int{1, 4}) { + t.Fatalf("withIface fields = %v", indices(p)) + } + a := Of(reflect.TypeOf(&cycA{})) + if !a.Reach || !a.Elem.Reach || !reflect.DeepEqual(indices(a.Elem), []int{0}) { + t.Fatalf("cycA = %+v", a.Elem) + } + b := a.Elem.Fields[0].Plan + if b.Elem == nil || !b.Reach || !reflect.DeepEqual(indices(b.Elem), []int{0, 1}) { + t.Fatalf("cycB = %+v", b) + } + back := b.Elem.Fields[0].Plan + if back.Elem == nil || back.Elem.Type != reflect.TypeOf(cycA{}) || !back.Reach { + t.Fatalf("cycle back edge = %+v", back) + } +} + +func TestConcurrentBuild(t *testing.T) { + // Unique types exercise cold-cache publication even after TestPlans. + type concurrentA struct { + Next *concurrentA + Value any + } + typ := reflect.TypeOf(struct{ A []map[int]*concurrentA }{}) + start := make(chan struct{}) + var wg sync.WaitGroup + for i := 0; i < 32; i++ { + wg.Add(1) + go func() { + defer wg.Done() + <-start + for range 32 { + p := Of(typ) + if !p.Reach || !reflect.DeepEqual(indices(p), []int{0}) { + t.Errorf("incomplete published plan: %+v", p) + return + } + elem := p.Fields[0].Plan + for _, kind := range []reflect.Kind{reflect.Slice, reflect.Map, reflect.Pointer} { + if elem.Kind != kind || elem.Elem == nil || !elem.Reach { + t.Errorf("incomplete container plan: %+v", elem) + return + } + elem = elem.Elem + } + if elem.Kind != reflect.Struct || !elem.Reach || !reflect.DeepEqual(indices(elem), []int{0, 1}) { + t.Errorf("incomplete cycle plan: %+v", elem) + } + } + }() + } + close(start) + wg.Wait() +} diff --git a/pkg/sql/ast/reflplan_coverage_test.go b/pkg/sql/ast/reflplan_coverage_test.go new file mode 100644 index 000000000..00abf955c --- /dev/null +++ b/pkg/sql/ast/reflplan_coverage_test.go @@ -0,0 +1,203 @@ +// Copyright 2026 GoSQLX Authors +// SPDX-License-Identifier: Apache-2.0 + +package ast + +import ( + "reflect" + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/internal/reflplan" +) + +// Each independent search has its own visited set, so a failed search through +// a cycle cannot hide a different path to an interface. Unlike reflplan this +// uses ordinary graph search, not propagation of Reach through a fixpoint. +func naiveTypeReachesInterface(typ reflect.Type, seen map[reflect.Type]bool) bool { + if seen[typ] { + return false + } + seen[typ] = true + switch typ.Kind() { + case reflect.Interface: + return true + case reflect.Pointer, reflect.Slice, reflect.Array, reflect.Map: + return naiveTypeReachesInterface(typ.Elem(), seen) + case reflect.Struct: + for i := range typ.NumField() { + f := typ.Field(i) + if f.IsExported() && naiveTypeReachesInterface(f.Type, seen) { + return true + } + } + } + return false +} + +func TestReflectionPlanASTCoverage(t *testing.T) { + visited := map[reflect.Type]bool{} + var check func(reflect.Type) + check = func(typ reflect.Type) { + if visited[typ] { + return + } + visited[typ] = true + p := reflplan.Of(typ) + wantReach := naiveTypeReachesInterface(typ, map[reflect.Type]bool{}) + if p.Type != typ || p.Kind != typ.Kind() || p.Reach != wantReach { + t.Errorf("%v: metadata/reach mismatch: %+v, want Reach=%v", typ, p, wantReach) + } + switch typ.Kind() { + case reflect.Pointer, reflect.Slice, reflect.Array, reflect.Map: + if p.Elem == nil || p.Elem.Type != typ.Elem() { + t.Errorf("%v: missing/wrong element plan", typ) + } + check(typ.Elem()) + case reflect.Struct: + var want []int + for i := range typ.NumField() { + f := typ.Field(i) + if f.IsExported() && naiveTypeReachesInterface(f.Type, map[reflect.Type]bool{}) { + want = append(want, i) + } + check(f.Type) + } + var got []int + for _, f := range p.Fields { + got = append(got, f.Index) + if f.Index < 0 || f.Index >= typ.NumField() { + t.Errorf("%v: invalid index %d", typ, f.Index) + continue + } + if f.Plan == nil || f.Plan.Type != typ.Field(f.Index).Type { + t.Errorf("%v field %d: wrong plan", typ, f.Index) + } + } + if !reflect.DeepEqual(got, want) { + t.Errorf("%v: planned fields %v, want %v", typ, got, want) + } + } + } + for _, c := range childrenCoverageCases() { + check(c.typ) + } + for _, value := range []any{BinaryExpression{}, UnaryExpression{}, FunctionCall{}, ParenthesizedExpression{}, LiteralValue{}} { + typ := reflect.TypeOf(value) + check(typ) + if !reflplan.Of(typ).Reach { + t.Errorf("matched rewrite type %v must reach an interface", typ) + } + } +} + +func TestRetargetScratchSharedPointersTwice(t *testing.T) { + abs := &UnaryExpression{Operator: PGAbs, Expr: &Identifier{Name: "a"}} + shared := &BinaryExpression{Left: abs, Operator: "+", Right: &LiteralValue{Type: "number", Value: 1}} + stmt := &SelectStatement{Columns: []Expression{shared, shared}} + for run := range 2 { + // Reuse the same addresses. Stale seen entries would skip the inner + // unary slot on the second call, even if outer slots were visited. + shared.Left = abs + if err := RetargetOperators(stmt, "postgresql", "mysql"); err != nil { + t.Fatal(err) + } + call, ok := shared.Left.(*FunctionCall) + if !ok || call.Name != "ABS" || len(call.Arguments) != 1 || call.Arguments[0] != abs.Expr { + t.Fatalf("run %d: shared child not rewritten: %#v", run, shared.Left) + } + if stmt.Columns[0] != shared || stmt.Columns[1] != shared { + t.Fatalf("run %d: shared tree identity changed", run) + } + } +} + +func TestRetargetScratchReset(t *testing.T) { + for _, tc := range []struct { + name string + seen, capacity int + wantKeep bool + }{ + {"empty", 0, 0, true}, + {"at_limit", 4096, 4096, true}, + {"seen_over_limit", 4097, 16, false}, + {"stack_over_limit", 1, 4097, false}, + } { + t.Run(tc.name, func(t *testing.T) { + sc := &retargetScratch{seen: make(map[uintptr]struct{}, tc.seen), stack: make([]operatorWalkFrame, tc.capacity)} + for i := range tc.seen { + sc.seen[uintptr(i+1)] = struct{}{} + } + for i := range sc.stack { + // Frames still on the stack (an early return on error) hold + // references; reset must drop every reference-bearing field. + v := reflect.ValueOf(&Identifier{Name: "retained"}) + sc.stack[i] = operatorWalkFrame{ + value: v, plan: reflplan.Of(v.Type()), slot: v, + mapValue: reflect.ValueOf(map[string]Expression{}), + mapKey: reflect.ValueOf("key"), + iter: reflect.ValueOf(map[string]Expression{}).MapRange(), + } + } + if keep := sc.reset(); keep != tc.wantKeep { + t.Fatalf("reset retention = %v, want %v", keep, tc.wantKeep) + } + if !tc.wantKeep { + return + } + if len(sc.seen) != 0 || len(sc.stack) != 0 || cap(sc.stack) != tc.capacity { + t.Fatal("reset did not clear state while preserving reusable capacity") + } + for i, frame := range sc.stack[:cap(sc.stack)] { + if !reflect.DeepEqual(frame, operatorWalkFrame{}) { + t.Fatalf("frame %d retains state after reset", i) + } + } + }) + } +} + +// TestRetargetWalkZeroesPoppedFrames pins the invariant reset relies on: +// after a successful walk no frame up to the stack's capacity still refers +// into the AST, so reset only has to clear the live length. +func TestRetargetWalkZeroesPoppedFrames(t *testing.T) { + stmt := &SelectStatement{Columns: []Expression{ + &BinaryExpression{Left: &Identifier{Name: "a"}, Operator: "^", Right: &Identifier{Name: "b"}}, + &FunctionCall{Name: "f", Arguments: []Expression{&Identifier{Name: "c"}}}, + }} + r := operatorRetargeter{from: "mysql", to: "mysql", seen: map[uintptr]struct{}{}} + stack := r.walk(reflect.ValueOf(stmt), false, make([]operatorWalkFrame, 0, 64)) + if r.err != nil { + t.Fatalf("walk: %v", r.err) + } + if len(stack) != 0 { + t.Fatalf("stack length after walk = %d, want 0", len(stack)) + } + for i, frame := range stack[:cap(stack)] { + if !reflect.DeepEqual(frame, operatorWalkFrame{}) { + t.Fatalf("frame %d still holds state after a successful walk", i) + } + } +} + +func TestRetargetScratchHugeThenSmall(t *testing.T) { + terms := 490000 + if testing.Short() || raceEnabled { + terms = 5000 // Still exceeds the 4096-entry pool cap. + } + var expr Expression = &LiteralValue{Type: "number", Value: 1} + for range terms { + expr = &BinaryExpression{Left: expr, Operator: "OR", Right: &LiteralValue{Type: "number", Value: 1}} + } + if err := RetargetOperators(&SelectStatement{Columns: []Expression{expr}}, "postgresql", "mysql"); err != nil { + t.Fatal(err) + } + // The small rewrite must not inherit seen entries or an old frame/error. + small := &SelectStatement{Columns: []Expression{&UnaryExpression{Operator: PGAbs, Expr: &Identifier{Name: "a"}}}} + if err := RetargetOperators(small, "postgresql", "mysql"); err != nil { + t.Fatal(err) + } + call, ok := small.Columns[0].(*FunctionCall) + if !ok || call.Name != "ABS" { + t.Fatalf("small rewrite after huge tree: %#v", small.Columns[0]) + } +} diff --git a/pkg/sql/ast/retarget_operators.go b/pkg/sql/ast/retarget_operators.go index 9b8fe5c99..f68f9e956 100644 --- a/pkg/sql/ast/retarget_operators.go +++ b/pkg/sql/ast/retarget_operators.go @@ -18,6 +18,9 @@ import ( "fmt" "reflect" "strings" + "sync" + + "github.com/ajitpratap0/GoSQLX/pkg/internal/reflplan" ) // RetargetOperators rewrites stmt in place for a target dialect. An empty @@ -47,11 +50,43 @@ func RetargetOperators(stmt Statement, from, to string) error { if (from != "" && from == to) || stmt == nil { return nil } - r := operatorRetargeter{from: normalDialect(from), to: normalDialect(to), seen: map[uintptr]struct{}{}} - r.walk(reflect.ValueOf(stmt), false) + sc, ok := retargetScratchPool.Get().(*retargetScratch) + if !ok || sc == nil { + sc = &retargetScratch{seen: make(map[uintptr]struct{}), stack: make([]operatorWalkFrame, 0, 16)} + } + defer func() { + if sc.reset() { + retargetScratchPool.Put(sc) + } + }() + r := operatorRetargeter{from: normalDialect(from), to: normalDialect(to), seen: sc.seen} + sc.stack = r.walk(reflect.ValueOf(stmt), false, sc.stack[:0]) return r.err } +type retargetScratch struct { + seen map[uintptr]struct{} + stack []operatorWalkFrame +} + +// reset drops AST references before reuse. Oversized scratch is discarded +// rather than retaining a huge chain in the pool. +func (sc *retargetScratch) reset() bool { + if len(sc.seen) > 4096 || cap(sc.stack) > 4096 { + return false + } + clear(sc.seen) + // walk zeroes each frame it pops, so only frames still on the stack (an + // early return on error) hold reflect.Values into the AST. + clear(sc.stack) + sc.stack = sc.stack[:0] + return true +} + +var retargetScratchPool = sync.Pool{New: func() any { + return &retargetScratch{seen: map[uintptr]struct{}{}, stack: make([]operatorWalkFrame, 0, 16)} +}} + // normalDialect maps the generic dialect to "" and lower-cases the name. func normalDialect(d string) string { s := strings.ToLower(d) @@ -86,6 +121,7 @@ const maxRetargetNesting = 512 type operatorWalkFrame struct { value reflect.Value + plan *reflplan.Plan slot reflect.Value inOperator bool nesting int @@ -98,10 +134,16 @@ type operatorWalkFrame struct { // walk visits exported fields in their original order using heap frames. Each // settable expression slot is rewritten after its children; map copies are -// written back only after their complete subtree has been rewritten. -func (r *operatorRetargeter) walk(v reflect.Value, inOperator bool) { - stack := make([]operatorWalkFrame, 1, 16) - stack[0] = operatorWalkFrame{value: v, inOperator: inOperator} +// written back only after their complete subtree has been rewritten. Fields, +// elements and map values whose static type cannot hold an interface (see +// reflplan) are skipped: they hold no expression slot to rewrite. Pruning +// interface-free pointers also removes them from address-only seen sets; this +// can only visit more, never fewer, so output differs from an unpruned walk +// only if a pruned value aliased the address of a rewrite-bearing node, which +// the parser does not produce. stack is reused scratch (nil is fine); the +// returned slice is the stack for the next call. +func (r *operatorRetargeter) walk(v reflect.Value, inOperator bool, stack []operatorWalkFrame) []operatorWalkFrame { + stack = append(stack, operatorWalkFrame{value: v, plan: reflplan.Of(v.Type()), inOperator: inOperator}) for len(stack) > 0 && r.err == nil { f := &stack[len(stack)-1] if !f.entered { @@ -118,58 +160,64 @@ func (r *operatorRetargeter) walk(v reflect.Value, inOperator bool) { break } r.seen[ptr] = struct{}{} + f.value = f.value.Elem() + f.plan = f.plan.Elem + } else { + f.value = f.value.Elem() + f.plan = reflplan.Of(f.value.Type()) } - f.value = f.value.Elem() } if f.value.Kind() == reflect.Struct && r.nestsOutput(f.value) { f.nesting++ if f.nesting > maxRetargetNesting { r.err = fmt.Errorf("expression nesting too deep to transpile") - return + return stack } } } var child reflect.Value + var childPlan *reflplan.Plan operands := false switch f.value.Kind() { case reflect.Struct: - t := f.value.Type() - operands = t == binaryExprType || t == unaryExprType - for f.index < f.value.NumField() { - i := f.index + if f.index < len(f.plan.Fields) { + t := f.plan.Type + operands = t == binaryExprType || t == unaryExprType + fp := f.plan.Fields[f.index] f.index++ - if t.Field(i).IsExported() { - child = f.value.Field(i) - break - } + child, childPlan = f.value.Field(fp.Index), fp.Plan } case reflect.Slice, reflect.Array: - if f.index < f.value.Len() { - child = f.value.Index(f.index) + if f.plan.Elem.Reach && f.index < f.value.Len() { + child, childPlan = f.value.Index(f.index), f.plan.Elem f.index++ } case reflect.Map: - if f.iter == nil { - f.iter = f.value.MapRange() - } - if f.iter.Next() { - child = reflect.New(f.value.Type().Elem()).Elem() - child.Set(f.iter.Value()) - frame := operatorWalkFrame{value: child, slot: child, nesting: f.nesting, mapValue: f.value, mapKey: f.iter.Key()} - stack = append(stack, frame) - continue + if f.plan.Elem.Reach { + if f.iter == nil { + f.iter = f.value.MapRange() + } + if f.iter.Next() { + child = reflect.New(f.value.Type().Elem()).Elem() + child.Set(f.iter.Value()) + frame := operatorWalkFrame{value: child, plan: f.plan.Elem, slot: child, nesting: f.nesting, mapValue: f.value, mapKey: f.iter.Key()} + stack = append(stack, frame) + continue + } } } if child.IsValid() { - stack = append(stack, operatorWalkFrame{value: child, slot: child, inOperator: operands, nesting: f.nesting}) + stack = append(stack, operatorWalkFrame{value: child, plan: childPlan, slot: child, inOperator: operands, nesting: f.nesting}) continue } r.slot(f.slot, f.inOperator) if r.err == nil && f.mapValue.IsValid() { f.mapValue.SetMapIndex(f.mapKey, f.slot) } + stack[len(stack)-1] = operatorWalkFrame{} stack = stack[:len(stack)-1] } + return stack } // nestsOutput identifies expression boundaries rendered recursively. Ordinary diff --git a/pkg/sql/ast/retarget_operators_deep_test.go b/pkg/sql/ast/retarget_operators_deep_test.go index 8b99d7a53..a5eeed453 100644 --- a/pkg/sql/ast/retarget_operators_deep_test.go +++ b/pkg/sql/ast/retarget_operators_deep_test.go @@ -52,7 +52,7 @@ func TestRetargetOperatorsPostOrderAndMap(t *testing.T) { power := &BinaryExpression{Left: &BinaryExpression{Left: a, Operator: "^", Right: b}, Operator: "^", Right: c} values := map[string]Expression{"concat": concat, "power": power} r := operatorRetargeter{from: "postgresql", to: "mysql", seen: map[uintptr]struct{}{}} - r.walk(reflect.ValueOf(values), false) + r.walk(reflect.ValueOf(values), false, nil) if r.err != nil { t.Fatal(r.err) } diff --git a/pkg/transpiler/dialect_rules.go b/pkg/transpiler/dialect_rules.go index 2fb7b47a4..36a22f407 100644 --- a/pkg/transpiler/dialect_rules.go +++ b/pkg/transpiler/dialect_rules.go @@ -15,6 +15,8 @@ package transpiler import ( + "sync" + "github.com/ajitpratap0/GoSQLX/pkg/sql/dialect" "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" "github.com/ajitpratap0/GoSQLX/pkg/transpiler/rules" @@ -53,12 +55,55 @@ func register(from, to keywords.SQLDialect, rs ...RewriteRule) { // Cross-dialect pairs include common row-limit rules; same-dialect pairs return nil. // Exported for testing. func RulesFor(from, to keywords.SQLDialect) []RewriteRule { + return buildRules(from, to, true) +} + +type rulesKey struct { + from, to keywords.SQLDialect + pipes bool +} + +// rulesCache is immutable after rulesCacheOnce publishes it. Only the nine +// supported dialects are cached; arbitrary caller strings cannot grow it. +var ( + rulesCacheOnce sync.Once + rulesCache map[rulesKey][]RewriteRule +) + +func rulesForSource(from, to keywords.SQLDialect, pipes bool) []RewriteRule { + rulesCacheOnce.Do(func() { + dialects := [...]keywords.SQLDialect{ + keywords.DialectGeneric, keywords.DialectMySQL, keywords.DialectPostgreSQL, + keywords.DialectSQLite, keywords.DialectSQLServer, keywords.DialectOracle, + keywords.DialectSnowflake, keywords.DialectClickHouse, keywords.DialectMariaDB, + } + rulesCache = make(map[rulesKey][]RewriteRule, len(dialects)*len(dialects)*2) + for _, f := range dialects { + for _, t := range dialects { + for _, p := range [...]bool{false, true} { + rulesCache[rulesKey{f, t, p}] = buildRules(f, t, p) + } + } + } + }) + if rs, ok := rulesCache[rulesKey{from, to, pipes}]; ok { + // Callers must never mutate the shared slice. rowLimitRule keeps all + // mutable traversal state per invocation, so its closure is safe to share. + return rs + } + return buildRules(from, to, pipes) +} + +func buildRules(from, to keywords.SQLDialect, pipes bool) []RewriteRule { if from == to { return nil } rs := ruleRegistry[dialectPair{from, to}] // Never append into the registry's backing array. rs = append(rs[:len(rs):len(rs)], rowLimitRule(to)) + if !pipes { + return rs + } if r := pipesRule(from, to); r != nil { return append(rs, r) } @@ -80,8 +125,3 @@ func pipesRule(from, to keywords.SQLDialect) RewriteRule { } return nil } - -// rulesFor is the internal version used by Transpile. -func rulesFor(from, to keywords.SQLDialect) []RewriteRule { - return RulesFor(from, to) -} diff --git a/pkg/transpiler/prefilter_test.go b/pkg/transpiler/prefilter_test.go new file mode 100644 index 000000000..92871426b --- /dev/null +++ b/pkg/transpiler/prefilter_test.go @@ -0,0 +1,341 @@ +// Copyright 2026 GoSQLX Authors +// SPDX-License-Identifier: Apache-2.0 + +package transpiler + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/models" + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" + "github.com/ajitpratap0/GoSQLX/pkg/sql/parser" + "github.com/ajitpratap0/GoSQLX/pkg/sql/tokenizer" +) + +func prefilterDialects() []keywords.SQLDialect { + return []keywords.SQLDialect{keywords.DialectGeneric, keywords.DialectPostgreSQL, + keywords.DialectMySQL, keywords.DialectMariaDB, keywords.DialectSQLServer, + keywords.DialectOracle, keywords.DialectSQLite, keywords.DialectSnowflake, + keywords.DialectClickHouse} +} + +// Compare the exact error text as well as output, including parse failures. +func checkPrefilterEquivalent(t *testing.T, sql string, from, to keywords.SQLDialect) { + t.Helper() + got, gotErr := transpile(sql, from, to, true) + want, wantErr := transpile(sql, from, to, false) + if got != want || fmt.Sprint(gotErr) != fmt.Sprint(wantErr) { + t.Fatalf("%s -> %s, SQL %q: filtered=(%q, %v), unfiltered=(%q, %v)", from, to, sql, got, gotErr, want, wantErr) + } +} + +func TestPrefilterEquivalence(t *testing.T) { + dialects := prefilterDialects() + if testing.Short() { + dialects = []keywords.SQLDialect{keywords.DialectPostgreSQL, keywords.DialectMySQL, keywords.DialectSQLite} + } + fixed := []string{ + "CREATE TABLE t (id INT AUTO_INCREMENT PRIMARY KEY, flag TINYINT(1))", + "SELECT a = 1 || b = 2 FROM t", "SELECT a || b FROM t", "SELECT a OR b FROM t", + "SELECT 'a\\nb'", "SELECT \"x\"", "SELECT $$a\\b$$", "SELECT $tag$a\\b$tag$", + "SELECT ‘a’", "SELECT «a»", "SELECT E'a\\nb'", "SELECT N'a'", "SELECT X'41'", "SELECT B'01'", + "SELECT q'[a]'", "SELECT _utf8mb4'a'", "SELECT 'a\x00b'", + `CREATE TABLE t (v "custom")`, "CREATE TABLE t (v `custom`)", "CREATE TABLE t (v [custom])", "CREATE TABLE t (v INT[])", + "SELECT 'a\x00b'; SELECT a ^ b", "SELECT a ^ b; SELECT 'a\x00b'", + "SELECT emp# FROM t", "SELECT 1 -- ' || [\n", "SELECT 1 /* $ || [ */", "SELECT 1; SELECT 'x'", "SELECT (((1)))", + } + for _, from := range dialects { + for _, to := range dialects { + t.Run(string(from)+"_"+string(to), func(t *testing.T) { + for _, sql := range fixed { + checkPrefilterEquivalent(t, sql, from, to) + } + }) + } + } + files := 0 + err := filepath.WalkDir("../../testdata/realworld", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || filepath.Ext(path) != ".sql" { + return nil + } + files++ + data, err := os.ReadFile(path) + if err != nil { + return err + } + // Deliberately split exactly on semicolons, including chunks inside quoted + // strings: malformed fragments also pin equality of parse errors. + statements := strings.Split(string(data), ";") + if testing.Short() && len(statements) > 2 { + statements = statements[:2] + } + for _, sql := range statements { + if strings.TrimSpace(sql) == "" { + continue + } + for _, from := range dialects { + for _, to := range dialects { + checkPrefilterEquivalent(t, sql, from, to) + } + } + } + return nil + }) + if err != nil { + t.Fatal(err) + } + if files == 0 { + t.Fatal("realworld corpus contained no SQL files") + } +} + +func TestScanSourceStringOpeners(t *testing.T) { + cases := []struct { + literal string + dialect keywords.SQLDialect + flags sourceFeatures + }{ + {"'abc'", keywords.DialectPostgreSQL, sourceFeatures{quote: true}}, + {`"abc"`, keywords.DialectMySQL, sourceFeatures{quote: true, typeDelim: true}}, + {"$$abc$$", keywords.DialectPostgreSQL, sourceFeatures{quote: true}}, + {"$tag$abc$tag$", keywords.DialectPostgreSQL, sourceFeatures{quote: true}}, + {"‘abc’", keywords.DialectPostgreSQL, sourceFeatures{nonASCII: true}}, + {"«abc»", keywords.DialectPostgreSQL, sourceFeatures{nonASCII: true}}, + {"E'abc'", keywords.DialectPostgreSQL, sourceFeatures{quote: true}}, + {"N'abc'", keywords.DialectSQLServer, sourceFeatures{quote: true}}, + {"X'41'", keywords.DialectMySQL, sourceFeatures{quote: true}}, + {"B'01'", keywords.DialectPostgreSQL, sourceFeatures{quote: true}}, + {"q'[abc]'", keywords.DialectOracle, sourceFeatures{quote: true, typeDelim: true}}, + {"_utf8mb4'abc'", keywords.DialectMySQL, sourceFeatures{quote: true}}, + } + for _, tc := range cases { + t.Run(tc.literal, func(t *testing.T) { + sql := "SELECT " + tc.literal + if got := scanSource(sql); got != tc.flags { + t.Fatalf("scanSource=%+v, want %+v", got, tc.flags) + } + tree, err := parser.ParseWithDialect(sql, tc.dialect) + if err != nil { + t.Fatal(err) + } + defer ast.ReleaseAST(tree) + found := false + ast.Inspect(tree, func(n ast.Node) bool { + if lit, ok := n.(*ast.LiteralValue); ok && lit.Type == "string" { + found = true + } + return true + }) + if !found { + // nextToken routes MySQL double quotes to readQuotedIdentifier. + // Pin that existing parser behavior as well as the conservative + // quote gate; changing its semantics is outside this optimization. + if tc.literal != `"abc"` { + t.Fatal("opener did not produce a string literal") + } + identifier := false + ast.Inspect(tree, func(n ast.Node) bool { + if _, ok := n.(*ast.Identifier); ok { + identifier = true + } + return true + }) + if !identifier { + t.Fatal("MySQL double quotes produced neither identifier nor literal") + } + } + flags := scanSource(sql) + if !flags.quote && !flags.nonASCII { + t.Fatal("string literal would skip retargeting") + } + }) + } +} + +func TestScanSourceTypeDelimiters(t *testing.T) { + for _, tc := range []struct { + typ string + dialect keywords.SQLDialect + }{ + {`"t"`, keywords.DialectPostgreSQL}, {"`t`", keywords.DialectMySQL}, {"[t]", keywords.DialectSQLServer}, {"INT[]", keywords.DialectPostgreSQL}, + } { + t.Run(tc.typ, func(t *testing.T) { + sql := "CREATE TABLE tbl (v " + tc.typ + ")" + if !scanSource(sql).typeDelim { + t.Fatal("quoted or array type would skip restyling") + } + tree, err := parser.ParseWithDialect(sql, tc.dialect) + if err != nil { + t.Fatal(err) + } + defer ast.ReleaseAST(tree) + }) + } +} + +func TestPrefilterPipesTokenAndAST(t *testing.T) { + tk := tokenizer.GetTokenizer() + defer tokenizer.PutTokenizer(tk) + for _, from := range prefilterDialects() { + tk.SetDialect(from) + for _, op := range []string{"||", "OR", "|"} { + sql := "SELECT a " + op + " b FROM t" + toks, err := tk.TokenizeString(sql) + if err != nil { + t.Fatal(err) + } + concatTokens := 0 + for _, tok := range toks { + if tok.Token.Type == models.TokenTypeStringConcat { + concatTokens++ + if tok.Token.Value != "||" { + t.Fatalf("concat token from %q", tok.Token.Value) + } + } + } + if (concatTokens == 1) != (op == "||") { + t.Fatalf("%s %q: %d StringConcat tokens", from, op, concatTokens) + } + tree, err := parser.ParseWithDialect(sql, from) + if err != nil { + t.Fatal(err) + } + matched := false + ast.Inspect(tree, func(n ast.Node) bool { + if bin, ok := n.(*ast.BinaryExpression); ok && (bin.PipesOr || bin.Operator == "||") { + matched = true + } + return true + }) + ast.ReleaseAST(tree) + if matched != (op == "||") { + t.Fatalf("%s %q: PipesOr/concat = %v", from, op, matched) + } + if matched && !scanSource(sql).pipe { + t.Fatal("pipe rewrite would be skipped") + } + } + } +} + +func TestPrefilterMustRunAndSkip(t *testing.T) { + cases := []struct { + sql string + from, to keywords.SQLDialect + contains, errorContains string + }{ + {"SELECT a = 1 || b = 2 FROM t", keywords.DialectMySQL, keywords.DialectPostgreSQL, " OR ", ""}, + {"SELECT a || b FROM t", keywords.DialectPostgreSQL, keywords.DialectMySQL, "CONCAT(", ""}, + {`SELECT 'a\0b'`, keywords.DialectMySQL, keywords.DialectSQLite, "", "control character U+0000"}, + {`CREATE TABLE t (v "custom")`, keywords.DialectPostgreSQL, keywords.DialectSQLServer, "[custom]", ""}, + } + for _, tc := range cases { + checkPrefilterEquivalent(t, tc.sql, tc.from, tc.to) + got, err := Transpile(tc.sql, tc.from, tc.to) + if tc.errorContains != "" { + if err == nil || !strings.Contains(err.Error(), tc.errorContains) { + t.Fatalf("%q: got %q, %v", tc.sql, got, err) + } + } else if err != nil || !strings.Contains(got, tc.contains) { + t.Fatalf("%q: got %q, %v; want %q", tc.sql, got, err, tc.contains) + } + } + for _, sql := range []string{`SELECT 'a\0b'; SELECT a ^ b`, `SELECT a ^ b; SELECT 'a\0b'`} { + checkPrefilterEquivalent(t, sql, keywords.DialectMySQL, keywords.DialectSQLite) + _, err := Transpile(sql, keywords.DialectMySQL, keywords.DialectSQLite) + if err == nil || !strings.Contains(err.Error(), "bitwise XOR operator") { + t.Fatalf("operator pass must fail before literal pass: %v", err) + } + } + const create = "CREATE TABLE t (id INT AUTO_INCREMENT PRIMARY KEY, flag TINYINT(1))" + if got := scanSource(create); got != (sourceFeatures{}) { + t.Fatalf("benchmark input should skip all three optional passes: %+v", got) + } + checkPrefilterEquivalent(t, create, keywords.DialectMySQL, keywords.DialectPostgreSQL) +} + +func TestRulesForFreshSliceAndBoundedCache(t *testing.T) { + from, to := keywords.DialectMySQL, keywords.DialectPostgreSQL + const sql = "SELECT a = 1 || b = 2 FROM t" + want, err := Transpile(sql, from, to) + if err != nil { + t.Fatal(err) + } + rs := RulesFor(from, to) + if len(rs) == 0 { + t.Fatal("expected registered rules") + } + rs[0] = func(ast.Statement) error { return fmt.Errorf("mutated caller slice") } + got, err := Transpile(sql, from, to) + if err != nil || got != want { + t.Fatalf("caller mutation affected Transpile: %q, %v", got, err) + } + if len(rulesCache) != 9*9*2 { + t.Fatalf("cache size = %d, want 162", len(rulesCache)) + } + for i := 0; i < 32; i++ { + unknown := keywords.SQLDialect(fmt.Sprintf("unknown-%d", i)) + for _, pipes := range []bool{false, true} { + for _, pair := range [][2]keywords.SQLDialect{{unknown, to}, {from, unknown}, {unknown, unknown}} { + cached := rulesForSource(pair[0], pair[1], pipes) + built := buildRules(pair[0], pair[1], pipes) + if len(cached) != len(built) { + t.Fatalf("unknown fallback length = %d, want %d", len(cached), len(built)) + } + // Execute both independently: row-limit closures must keep per-call state. + a, e1 := transpile("SELECT a FROM t LIMIT 2", pair[0], pair[1], true) + b, e2 := transpile("SELECT a FROM t LIMIT 2", pair[0], pair[1], false) + if a != b || fmt.Sprint(e1) != fmt.Sprint(e2) { + t.Fatalf("unknown fallback differs: %q %v vs %q %v", a, e1, b, e2) + } + } + } + } + if len(rulesCache) != 162 { + t.Fatalf("unknown dialects grew cache to %d", len(rulesCache)) + } + for _, from := range prefilterDialects() { + for _, to := range prefilterDialects() { + for _, pipes := range []bool{false, true} { + got, ok := rulesCache[rulesKey{from, to, pipes}] + if !ok { + t.Fatalf("missing cache pair %s/%s/%v", from, to, pipes) + } + want := buildRules(from, to, pipes) + if len(got) != len(want) { + t.Fatal("cached rule list differs") + } + for i := range got { + if reflect.ValueOf(got[i]).Pointer() != reflect.ValueOf(want[i]).Pointer() { + t.Fatal("cached rule order differs") + } + } + } + } + } +} + +func TestPrefilterParallelTranspile(t *testing.T) { + for _, from := range prefilterDialects() { + for _, to := range prefilterDialects() { + t.Run(string(from)+"_"+string(to), func(t *testing.T) { + t.Parallel() + for i := 0; i < 8; i++ { + for _, sql := range []string{"SELECT a || b FROM t LIMIT 2", "SELECT 'a\\b' FROM t", "CREATE TABLE t (a INT)"} { + checkPrefilterEquivalent(t, sql, from, to) + } + } + }) + } + } +} diff --git a/pkg/transpiler/quoting.go b/pkg/transpiler/quoting.go index 499da957e..e635b858c 100644 --- a/pkg/transpiler/quoting.go +++ b/pkg/transpiler/quoting.go @@ -38,10 +38,12 @@ func identifierDelimiter(d keywords.SQLDialect) rune { // the target dialect's (see identifierDelimiter), so a MySQL `my col` is not // carried into PostgreSQL output where backticks are invalid. Oracle bare // names with '#' (emp#) stay bare only when the target is Oracle too. -func requoteIdentifiers(stmt ast.Statement, from, to keywords.SQLDialect) { +func requoteIdentifiers(stmt ast.Statement, from, to keywords.SQLDialect, typeDelims bool) { if from == to { return } ast.SetNameQuoteStyle(stmt, identifierDelimiter(to), to == keywords.DialectOracle) - ast.RestyleDataTypes(stmt, identifierDelimiter(to)) + if typeDelims { + ast.RestyleDataTypes(stmt, identifierDelimiter(to)) + } } diff --git a/pkg/transpiler/rules/pg_concat.go b/pkg/transpiler/rules/pg_concat.go index 8f42f257c..aae51a282 100644 --- a/pkg/transpiler/rules/pg_concat.go +++ b/pkg/transpiler/rules/pg_concat.go @@ -17,6 +17,7 @@ package rules import ( "reflect" + "github.com/ajitpratap0/GoSQLX/pkg/internal/reflplan" "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" ) @@ -36,7 +37,8 @@ func PgConcatToMySQLConcat(stmt ast.Statement) error { if stmt == nil || !containsConcat(stmt) { return nil } - rewriteConcatIn(reflect.ValueOf(stmt), make(map[uintptr]struct{})) + v := reflect.ValueOf(stmt) + rewriteConcatIn(v, reflplan.Of(v.Type()), make(map[uintptr]struct{})) return nil } @@ -75,7 +77,11 @@ var funcCallType = reflect.TypeOf((*ast.FunctionCall)(nil)) // call. Expressions are stored in interface-typed fields, slices and value // structs (e.g. JoinClause), which ast.Walk cannot replace in place, hence // reflection. seen guards against revisiting shared pointers. -func rewriteConcatIn(v reflect.Value, seen map[uintptr]struct{}) { +// Pruned interface-free pointers no longer enter the address-only seen set. +// This can only visit more, never fewer; output could differ from an unpruned +// walk only if a pruned value aliased a rewrite node's address, which the +// parser does not produce. Reachable slot order is unchanged. +func rewriteConcatIn(v reflect.Value, p *reflplan.Plan, seen map[uintptr]struct{}) { switch v.Kind() { case reflect.Pointer: if v.IsNil() { @@ -86,29 +92,29 @@ func rewriteConcatIn(v reflect.Value, seen map[uintptr]struct{}) { return } seen[ptr] = struct{}{} - rewriteConcatIn(v.Elem(), seen) + rewriteConcatIn(v.Elem(), p.Elem, seen) case reflect.Interface: if !v.IsNil() { - rewriteConcatIn(v.Elem(), seen) + e := v.Elem() + rewriteConcatIn(e, reflplan.Of(e.Type()), seen) } case reflect.Struct: - t := v.Type() - for i := 0; i < v.NumField(); i++ { - if !t.Field(i).IsExported() { - continue - } - rewriteConcatSlot(v.Field(i), seen) + for _, f := range p.Fields { + rewriteConcatSlot(v.Field(f.Index), f.Plan, seen) } case reflect.Slice, reflect.Array: + if !p.Elem.Reach { + return + } for i := 0; i < v.Len(); i++ { - rewriteConcatSlot(v.Index(i), seen) + rewriteConcatSlot(v.Index(i), p.Elem, seen) } } } // rewriteConcatSlot replaces slot with CONCAT(...) when it is a settable // interface slot holding a concatenation; otherwise it descends into it. -func rewriteConcatSlot(slot reflect.Value, seen map[uintptr]struct{}) { +func rewriteConcatSlot(slot reflect.Value, p *reflplan.Plan, seen map[uintptr]struct{}) { if slot.Kind() == reflect.Interface && !slot.IsNil() && slot.CanSet() && funcCallType.AssignableTo(slot.Type()) { if e, ok := slot.Interface().(ast.Expression); ok { @@ -118,15 +124,17 @@ func rewriteConcatSlot(slot reflect.Value, seen map[uintptr]struct{}) { } } } - rewriteConcatIn(slot, seen) + rewriteConcatIn(slot, p, seen) } +var expressionPlan = reflplan.Of(reflect.TypeOf((*ast.Expression)(nil)).Elem()) + // buildConcatCall flattens a || chain into CONCAT(args...), rewriting any // concatenation nested inside the operands as well. func buildConcatCall(bin *ast.BinaryExpression, seen map[uintptr]struct{}) *ast.FunctionCall { args := flattenConcat(bin, nil) for i := range args { - rewriteConcatSlot(reflect.ValueOf(&args[i]).Elem(), seen) + rewriteConcatSlot(reflect.ValueOf(&args[i]).Elem(), expressionPlan, seen) } return &ast.FunctionCall{Name: "CONCAT", Arguments: args} } diff --git a/pkg/transpiler/rules/pipes_or.go b/pkg/transpiler/rules/pipes_or.go index 99635fb88..8c54e3826 100644 --- a/pkg/transpiler/rules/pipes_or.go +++ b/pkg/transpiler/rules/pipes_or.go @@ -17,6 +17,7 @@ package rules import ( "reflect" + "github.com/ajitpratap0/GoSQLX/pkg/internal/reflplan" "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" ) @@ -33,7 +34,8 @@ func PipesOrToKeywordOr(stmt ast.Statement) error { if stmt == nil { return nil } - clearPipesOr(reflect.ValueOf(stmt), make(map[uintptr]struct{})) + v := reflect.ValueOf(stmt) + clearPipesOr(v, reflplan.Of(v.Type()), make(map[uintptr]struct{})) return nil } @@ -42,8 +44,13 @@ var binaryExprType = reflect.TypeOf(ast.BinaryExpression{}) // clearPipesOr walks every exported field reachable from v and clears // PipesOr on each BinaryExpression. Reflection reaches expressions stored in // value structs and slices that ast.Walk does not visit; seen guards against -// revisiting shared pointers. -func clearPipesOr(v reflect.Value, seen map[uintptr]struct{}) { +// revisiting shared pointers. Parts whose type cannot hold an interface are +// skipped (reflplan): a BinaryExpression is only ever reached through one. +// Pruned interface-free pointers no longer enter the address-only seen set. +// This can only visit more, never fewer; output could differ from an unpruned +// walk only if a pruned value aliased a rewrite node's address, which the +// parser does not produce. Reachable slot order is unchanged. +func clearPipesOr(v reflect.Value, p *reflplan.Plan, seen map[uintptr]struct{}) { switch v.Kind() { case reflect.Pointer: if v.IsNil() { @@ -54,26 +61,27 @@ func clearPipesOr(v reflect.Value, seen map[uintptr]struct{}) { return } seen[ptr] = struct{}{} - clearPipesOr(v.Elem(), seen) + clearPipesOr(v.Elem(), p.Elem, seen) case reflect.Interface: if !v.IsNil() { - clearPipesOr(v.Elem(), seen) + e := v.Elem() + clearPipesOr(e, reflplan.Of(e.Type()), seen) } case reflect.Struct: - if v.Type() == binaryExprType && v.CanAddr() { + if p.Type == binaryExprType && v.CanAddr() { if bin, ok := v.Addr().Interface().(*ast.BinaryExpression); ok { bin.PipesOr = false } } - t := v.Type() - for i := 0; i < v.NumField(); i++ { - if t.Field(i).IsExported() { - clearPipesOr(v.Field(i), seen) - } + for _, f := range p.Fields { + clearPipesOr(v.Field(f.Index), f.Plan, seen) } case reflect.Slice, reflect.Array: + if !p.Elem.Reach { + return + } for i := 0; i < v.Len(); i++ { - clearPipesOr(v.Index(i), seen) + clearPipesOr(v.Index(i), p.Elem, seen) } } } diff --git a/pkg/transpiler/string_literals.go b/pkg/transpiler/string_literals.go index b634b1711..04741852e 100644 --- a/pkg/transpiler/string_literals.go +++ b/pkg/transpiler/string_literals.go @@ -19,6 +19,7 @@ import ( "reflect" "strings" + "github.com/ajitpratap0/GoSQLX/pkg/internal/reflplan" "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" "github.com/ajitpratap0/GoSQLX/pkg/sql/dialect" "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" @@ -131,34 +132,43 @@ type seenPtr struct { // keep the source escaping. seen guards against shared or cyclic pointers. // An explicit stack preserves pre-order without consuming the goroutine stack. func forEachLiteral(v reflect.Value, seen map[seenPtr]bool, f func(*ast.LiteralValue)) { - var small [64]reflect.Value - stack := append(small[:0], v) + type item struct { + v reflect.Value + p *reflplan.Plan + } + if !v.IsValid() { + return + } + var small [64]item + stack := append(small[:0], item{v, reflplan.Of(v.Type())}) for len(stack) > 0 { - v = stack[len(stack)-1] + it := stack[len(stack)-1] + v, p := it.v, it.p stack = stack[:len(stack)-1] switch v.Kind() { case reflect.Interface: if !v.IsNil() { - stack = append(stack, v.Elem()) + e := v.Elem() + stack = append(stack, item{e, reflplan.Of(e.Type())}) } case reflect.Pointer: if v.IsNil() { continue } - key := seenPtr{v.Type(), v.Pointer()} + key := seenPtr{p.Type, v.Pointer()} if seen[key] { continue } seen[key] = true - if v.Elem().Type() == literalType { + if p.Elem.Type == literalType { if lit, ok := v.Interface().(*ast.LiteralValue); ok { f(lit) } continue } - stack = append(stack, v.Elem()) + stack = append(stack, item{v.Elem(), p.Elem}) case reflect.Struct: - if v.Type() == literalType { + if p.Type == literalType { if v.CanAddr() { if lit, ok := v.Addr().Interface().(*ast.LiteralValue); ok { f(lit) @@ -166,21 +176,25 @@ func forEachLiteral(v reflect.Value, seen map[seenPtr]bool, f func(*ast.LiteralV } continue } - t := v.Type() - for i := v.NumField() - 1; i >= 0; i-- { - if t.Field(i).IsExported() { - stack = append(stack, v.Field(i)) - } + for i := len(p.Fields) - 1; i >= 0; i-- { + fp := p.Fields[i] + stack = append(stack, item{v.Field(fp.Index), fp.Plan}) } case reflect.Slice, reflect.Array: + if !p.Elem.Reach { + continue + } for i := v.Len() - 1; i >= 0; i-- { - stack = append(stack, v.Index(i)) + stack = append(stack, item{v.Index(i), p.Elem}) } case reflect.Map: + if !p.Elem.Reach { + continue + } iter := v.MapRange() start := len(stack) for iter.Next() { - stack = append(stack, iter.Value()) + stack = append(stack, item{iter.Value(), p.Elem}) } // Preserve this iterator's order with a LIFO stack. for i, j := start, len(stack)-1; i < j; i, j = i+1, j-1 { diff --git a/pkg/transpiler/transpiler.go b/pkg/transpiler/transpiler.go index 14d6245d0..ee1ce61fc 100644 --- a/pkg/transpiler/transpiler.go +++ b/pkg/transpiler/transpiler.go @@ -60,6 +60,10 @@ type RewriteRule func(stmt ast.Statement) error // object constants) are rewritten for the target or return an error; see // retargetOperators. func Transpile(sql string, from, to keywords.SQLDialect) (string, error) { + return transpile(sql, from, to, true) +} + +func transpile(sql string, from, to keywords.SQLDialect, prefilter bool) (string, error) { // Use ParseWithDialect so the tokenizer and parser understand dialect- // specific syntax (e.g. MySQL backtick identifiers, PG ILIKE, etc.). tree, err := parser.ParseWithDialect(sql, from) @@ -77,8 +81,13 @@ func Transpile(sql string, from, to keywords.SQLDialect) (string, error) { } } - // Apply rules for this dialect pair. - rules := rulesFor(from, to) + // Prefilters are superset tests on source bytes: false positives only cost + // a walk; false negatives would change output or error order. + src := scanSource(sql) + // The tokenizer emits TokenTypeStringConcat only for two adjacent |, and + // the parser's OR and concatenation parsing are the only producers of + // PipesOr and ||; no rewrite rule creates either. + rules := rulesForSource(from, to, !prefilter || src.pipe) for _, stmt := range tree.Statements { for _, rule := range rules { if err := rule(stmt); err != nil { @@ -88,14 +97,23 @@ func Transpile(sql string, from, to keywords.SQLDialect) (string, error) { if err := retargetOperators(stmt, from, to); err != nil { return "", fmt.Errorf("rewrite: %w", err) } - requoteIdentifiers(stmt, from, to) + // requoteDataType (ast/data_type_quotes.go) is the identity without + // ", ` or [. The parser inserts these only from quoted type tokens + // (typeNameTokenSQL) or [ tokens; rules only assign bare type names. + // SetNameQuoteStyle still runs: names can have content delimiters. + requoteIdentifiers(stmt, from, to, !prefilter || src.typeDelim || src.nonASCII) } // Re-render string literals for the target dialect (escaping and // introducers); see retargetStringLiterals. - for _, stmt := range tree.Statements { - if err := retargetStringLiterals(stmt, to); err != nil { - return "", fmt.Errorf("rewrite: %w", err) + // Type "string" is set only by parser/string_literal.go's + // newStringLiteral; all 11 callers consume string tokens. Their openers + // contain ', " or $, or a non-ASCII byte (Unicode quote delimiters). + if !prefilter || src.quote || src.nonASCII { + for _, stmt := range tree.Statements { + if err := retargetStringLiterals(stmt, to); err != nil { + return "", fmt.Errorf("rewrite: %w", err) + } } } @@ -110,3 +128,24 @@ func Transpile(sql string, from, to keywords.SQLDialect) (string, error) { } return out, nil } + +type sourceFeatures struct{ pipe, quote, nonASCII, typeDelim bool } + +func scanSource(sql string) sourceFeatures { + var f sourceFeatures + for i := 0; i < len(sql); i++ { + switch c := sql[i]; { + case c >= 0x80: + f.nonASCII = true + case c == '|': + f.pipe = true + case c == '\'' || c == '$': + f.quote = true + case c == '"': + f.quote, f.typeDelim = true, true + case c == '`' || c == '[': + f.typeDelim = true + } + } + return f +} diff --git a/pkg/transpiler/transpiler_test.go b/pkg/transpiler/transpiler_test.go index 7149dbbb1..8425d1e13 100644 --- a/pkg/transpiler/transpiler_test.go +++ b/pkg/transpiler/transpiler_test.go @@ -154,3 +154,39 @@ func TestTranspile_NestedParenthesizedWith(t *testing.T) { }) } } + +func BenchmarkTranspile_SelectOps(b *testing.B) { + const sql = "SELECT u.id, u.name || 'x', COUNT(*) AS c, a ^ b, CASE WHEN u.age > 18 AND u.flag = 1 THEN 'adult' ELSE 'minor' END FROM users u JOIN orders o ON o.uid = u.id AND o.total > 10 WHERE u.status IN ('a','b') OR (u.score * 2 + 1) >= 5 GROUP BY u.id, u.name HAVING COUNT(*) > 1 ORDER BY c DESC LIMIT 10" + for _, p := range []struct { + name string + from, to keywords.SQLDialect + }{ + {"mysql2pg", keywords.DialectMySQL, keywords.DialectPostgreSQL}, + {"pg2mysql", keywords.DialectPostgreSQL, keywords.DialectMySQL}, + } { + b.Run(p.name, func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if _, err := transpiler.Transpile(sql, p.from, p.to); err != nil { + b.Fatal(err) + } + } + }) + } +} + +// A large tree must not leave stale seen pointers or AST references in scratch +// retained for the next, small call. +func TestTranspileDeepChainFollowedBySmall(t *testing.T) { + if testing.Short() || raceEnabled { + t.Skip("near-ceiling chain skipped under -short and -race") + } + checkDeepChain(t, "OR", 490000, keywords.DialectPostgreSQL, keywords.DialectMySQL) + got, err := transpiler.Transpile("SELECT a || b FROM t", keywords.DialectPostgreSQL, keywords.DialectMySQL) + if err != nil { + t.Fatalf("small transpile after large tree: %v", err) + } + if want := "SELECT CONCAT(a, b) FROM t"; got != want { + t.Fatalf("small transpile after large tree = %q, want %q", got, want) + } +} diff --git a/pkg/transpiler/walk_coverage_test.go b/pkg/transpiler/walk_coverage_test.go new file mode 100644 index 000000000..b43bc9922 --- /dev/null +++ b/pkg/transpiler/walk_coverage_test.go @@ -0,0 +1,206 @@ +// Copyright 2026 GoSQLX Authors +// SPDX-License-Identifier: Apache-2.0 + +package transpiler_test + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/gosqlx" + "github.com/ajitpratap0/GoSQLX/pkg/internal/reflplan" + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" + "github.com/ajitpratap0/GoSQLX/pkg/sql/parser" +) + +type walkCoverageKey struct { + address uintptr + typ reflect.Type +} + +type walkCoverageSet map[walkCoverageKey]struct{} + +// recordWalkCoverage records interface slots (including scalar dynamic values) +// and the pointer rewrite sites that do not require an interface slot. Addressable +// interface slots identify scalar values; pointer values identify their pointees. +func recordWalkCoverage(v reflect.Value, found walkCoverageSet) { + if v.Kind() == reflect.Interface && !v.IsNil() { + dynamic := v.Elem() + var address uintptr + if dynamic.Kind() == reflect.Pointer { + address = dynamic.Pointer() + } else if v.CanAddr() { + address = v.Addr().Pointer() + } + found[walkCoverageKey{address, dynamic.Type()}] = struct{}{} + } + if v.Kind() == reflect.Pointer && !v.IsNil() && + (v.Type() == reflect.TypeFor[*ast.LiteralValue]() || v.Type() == reflect.TypeFor[*ast.BinaryExpression]()) { + found[walkCoverageKey{v.Pointer(), v.Type()}] = struct{}{} + } +} + +// naiveWalkCoverage deliberately examines every exported field and every +// container element without consulting reflplan or any static reachability test. +func naiveWalkCoverage(root reflect.Value) walkCoverageSet { + found := make(walkCoverageSet) + seen := make(map[walkCoverageKey]bool) + stack := []reflect.Value{root} + for len(stack) > 0 { + v := stack[len(stack)-1] + stack = stack[:len(stack)-1] + if !v.IsValid() { + continue + } + recordWalkCoverage(v, found) + switch v.Kind() { + case reflect.Pointer: + if v.IsNil() { + continue + } + key := walkCoverageKey{v.Pointer(), v.Type()} + if seen[key] { + continue + } + seen[key] = true + stack = append(stack, v.Elem()) + case reflect.Interface: + if !v.IsNil() { + stack = append(stack, v.Elem()) + } + case reflect.Struct: + for i := 0; i < v.NumField(); i++ { + if v.Type().Field(i).IsExported() { + stack = append(stack, v.Field(i)) + } + } + case reflect.Slice, reflect.Array: + for i := 0; i < v.Len(); i++ { + stack = append(stack, v.Index(i)) + } + case reflect.Map: + iter := v.MapRange() + for iter.Next() { + stack = append(stack, iter.Value()) + } + } + } + return found +} + +func plannedWalkCoverage(root reflect.Value) walkCoverageSet { + type frame struct { + v reflect.Value + p *reflplan.Plan + } + found := make(walkCoverageSet) + seen := make(map[walkCoverageKey]bool) + stack := []frame{{root, reflplan.Of(root.Type())}} + for len(stack) > 0 { + f := stack[len(stack)-1] + stack = stack[:len(stack)-1] + v, p := f.v, f.p + if !v.IsValid() { + continue + } + recordWalkCoverage(v, found) + switch v.Kind() { + case reflect.Pointer: + if v.IsNil() || !p.Elem.Reach { + continue + } + key := walkCoverageKey{v.Pointer(), v.Type()} + if seen[key] { + continue + } + seen[key] = true + stack = append(stack, frame{v.Elem(), p.Elem}) + case reflect.Interface: + if !v.IsNil() { + elem := v.Elem() + stack = append(stack, frame{elem, reflplan.Of(elem.Type())}) + } + case reflect.Struct: + for _, field := range p.Fields { + stack = append(stack, frame{v.Field(field.Index), field.Plan}) + } + case reflect.Slice, reflect.Array: + if p.Elem.Reach { + for i := 0; i < v.Len(); i++ { + stack = append(stack, frame{v.Index(i), p.Elem}) + } + } + case reflect.Map: + if p.Elem.Reach { + iter := v.MapRange() + for iter.Next() { + stack = append(stack, frame{iter.Value(), p.Elem}) + } + } + } + } + return found +} + +func TestReflectionPlanRealWorldWalkCoverage(t *testing.T) { + root := filepath.Join("..", "..", "testdata", "realworld") + parsed, skipped := 0, 0 + err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() || filepath.Ext(path) != ".sql" { + return nil + } + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + dialect := strings.Split(filepath.ToSlash(rel), "/")[0] + if !keywords.IsValidDialect(dialect) { + return fmt.Errorf("invalid corpus dialect %q", dialect) + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + for i, sql := range gosqlx.SplitStatements(string(data), dialect) { + tree, err := parser.ParseWithDialect(sql, keywords.SQLDialect(dialect)) + if err != nil { + // The corpus intentionally includes known parser failures. + skipped++ + continue + } + func() { + defer ast.ReleaseAST(tree) + for j, stmt := range tree.Statements { + parsed++ + value := reflect.ValueOf(stmt) + want, got := naiveWalkCoverage(value), plannedWalkCoverage(value) + if !reflect.DeepEqual(got, want) { + t.Errorf("%s#%d statement %d: planned coverage has %d sites, naive has %d", rel, i, j, len(got), len(want)) + for site := range want { + if _, ok := got[site]; !ok { + t.Errorf("missing rewrite site: %v at %#x", site.typ, site.address) + } + } + } + } + }() + } + return nil + }) + if err != nil { + t.Fatal(err) + } + if parsed == 0 { + t.Fatal("no corpus statements parsed") + } + t.Logf("compared %d parsed statements; skipped %d known/unparseable inputs", parsed, skipped) +} From bb13370dedf16c910af6c7e132d733c2570d55e9 Mon Sep 17 00:00:00 2001 From: Ajit Pratap Singh <18012955+ajitpratap0@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:03:59 +0530 Subject: [PATCH 2/4] test(reflplan): mark the intentional unexported test field for staticcheck Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DnbRnTFErhoMJdAjVEeCg2 --- pkg/internal/reflplan/reflplan_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/internal/reflplan/reflplan_test.go b/pkg/internal/reflplan/reflplan_test.go index 2f78b1467..f3a629ba1 100644 --- a/pkg/internal/reflplan/reflplan_test.go +++ b/pkg/internal/reflplan/reflplan_test.go @@ -22,7 +22,8 @@ type withIface struct { S []leaf M map[string]any P *selfLeaf - hid any + //lint:ignore U1000 an unexported interface field must be excluded from Fields + hid any } type cycA struct { B *cycB From d09d3160aa618819930a392d788640ce51848b6c Mon Sep 17 00:00:00 2001 From: Ajit Pratap Singh <18012955+ajitpratap0@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:14:54 +0530 Subject: [PATCH 3/4] fix(ast): do not pool retarget scratch after a panicking walk Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DnbRnTFErhoMJdAjVEeCg2 --- pkg/sql/ast/retarget_operators.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/sql/ast/retarget_operators.go b/pkg/sql/ast/retarget_operators.go index f68f9e956..c4f09ef29 100644 --- a/pkg/sql/ast/retarget_operators.go +++ b/pkg/sql/ast/retarget_operators.go @@ -54,13 +54,17 @@ func RetargetOperators(stmt Statement, from, to string) error { if !ok || sc == nil { sc = &retargetScratch{seen: make(map[uintptr]struct{}), stack: make([]operatorWalkFrame, 0, 16)} } + // A walk that panics leaves frames in the backing array that sc.stack no + // longer covers, so only scratch from a completed walk is reused. + completed := false defer func() { - if sc.reset() { + if completed && sc.reset() { retargetScratchPool.Put(sc) } }() r := operatorRetargeter{from: normalDialect(from), to: normalDialect(to), seen: sc.seen} sc.stack = r.walk(reflect.ValueOf(stmt), false, sc.stack[:0]) + completed = true return r.err } From 43820a53cf50246c716665b95eebaf363a3e1718 Mon Sep 17 00:00:00 2001 From: Ajit Pratap Singh <18012955+ajitpratap0@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:47:13 +0530 Subject: [PATCH 4/4] docs(changelog): note Transpile speed-up under Unreleased Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DnbRnTFErhoMJdAjVEeCg2 --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 835bc84cd..8d6160c75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Performance +- **`Transpile`** recovers most of the v1.17.0 correctness cost: the AST walks + follow cached per-type reflection plans that skip fields which cannot hold a + rewrite target, and cheap source checks skip the `||`, string-literal and + identifier-requoting passes when the SQL cannot need them. A short MySQL + `CREATE TABLE` to PostgreSQL goes from 5.2 µs to 2.7 µs (52 to 43 + allocations); wide `OR`/`AND` chains are about 68% faster and `||` chains + about 55% faster. Output and errors are unchanged on 955,476 differential + cases. (#889) + ## [1.17.0] - 2026-10-29 — Real-World Dialect Corpus, Dialect-Faithful Round Trips, Security Scanner & LSP Hardening Covers everything merged since v1.16.0, mostly the October 2026 dialect audit: