diff --git a/Makefile b/Makefile
index b37352ff..f2797fb5 100644
--- a/Makefile
+++ b/Makefile
@@ -1,11 +1,15 @@
+.DEFAULT_GOAL := build
+
.PHONY: build test test-unit test-integration test-e2e lint fmt coverage test-update bench bench-short grammars-native clean
-native/bridge/lib/libdiffmantic_grammars.a:
- @$(MAKE) grammars-native
+GRAMMAR_SRCS := native/bridge/grammars.json native/bridge/build_grammars.go $(wildcard native/bridge/src/*.c) $(wildcard native/bridge/include/*.h)
build: native/bridge/lib/libdiffmantic_grammars.a ## Build binary with native Tree-sitter flat-buffer bridge
go build -ldflags="-s -w" -trimpath -o diffm ./cmd/diffm
+native/bridge/lib/libdiffmantic_grammars.a: $(GRAMMAR_SRCS)
+ @$(MAKE) grammars-native
+
grammars-native: ## Fetch and compile 18 native Tree-sitter grammars
go run ./native/bridge/build_grammars.go
diff --git a/internal/comments/comments_test.go b/internal/comments/comments_test.go
index de409904..49d4d184 100644
--- a/internal/comments/comments_test.go
+++ b/internal/comments/comments_test.go
@@ -2,13 +2,11 @@ package comments
import (
"os"
- "strings"
"testing"
"github.com/HarshK97/diffmantic/internal/actions"
+ "github.com/HarshK97/diffmantic/internal/engine"
"github.com/HarshK97/diffmantic/internal/treesitter"
- "github.com/HarshK97/diffmantic/internal/treesitter/rules"
- "github.com/odvcencio/gotreesitter"
)
func TestDiffCommentsIdentical(t *testing.T) {
@@ -19,7 +17,7 @@ func TestDiffCommentsIdentical(t *testing.T) {
{Type: "comment", Text: "// Hello World", StartRow: 8, StartByte: 15, EndByte: 29},
}
- res := DiffComments(srcComments, dstComments)
+ res := DiffComments(srcComments, dstComments, nil)
if len(res.Actions) != 0 {
t.Errorf("expected 0 actions for identical comment, got %d actions", len(res.Actions))
}
@@ -33,7 +31,7 @@ func TestDiffCommentsSingleLineUpdate(t *testing.T) {
{Type: "comment", Text: "// New Comment", StartRow: 5, StartByte: 10, EndByte: 24},
}
- res := DiffComments(srcComments, dstComments)
+ res := DiffComments(srcComments, dstComments, nil)
if len(res.Actions) != 1 {
t.Fatalf("expected 1 action, got %d", len(res.Actions))
}
@@ -57,7 +55,7 @@ func TestDiffCommentsMultiLineLineDiff(t *testing.T) {
{Type: "block_comment", Text: newJavadoc, StartRow: 10, StartByte: 0, EndByte: uint32(len(newJavadoc))},
}
- res := DiffComments(srcComments, dstComments)
+ res := DiffComments(srcComments, dstComments, nil)
if len(res.Actions) != 1 {
t.Fatalf("expected 1 line-level update action in multiline comment, got %d", len(res.Actions))
@@ -75,29 +73,29 @@ func TestDiffCommentsMultiLineLineDiff(t *testing.T) {
}
func TestExtractCommentsWithTreeSitter(t *testing.T) {
- lang, err := treesitter.DetectLanguage("test.go")
- if err != nil {
- t.Fatal(err)
- }
- rules := rules.Get("go")
src := []byte("package main\n\n// Line comment 1\nfunc main() {\n\t// Line comment 2\n}\n")
- parser := gotreesitter.NewParser(lang)
- tree, err := parser.Parse(src)
+ _, flatNodes, symbols, err := treesitter.ParseForPipeline(src, "go")
if err != nil {
t.Fatal(err)
}
- comments := ExtractComments(tree.RootNode(), src, lang, rules)
+ comments := ExtractComments(flatNodes, symbols, src, "go")
if len(comments) != 2 {
t.Fatalf("expected 2 comments extracted, got %d", len(comments))
}
if comments[0].Text != "// Line comment 1" {
t.Errorf("expected '// Line comment 1', got %q", comments[0].Text)
}
+ if comments[0].Language != "go" {
+ t.Errorf("expected comments[0].Language == 'go', got %q", comments[0].Language)
+ }
if comments[1].Text != "// Line comment 2" {
t.Errorf("expected '// Line comment 2', got %q", comments[1].Text)
}
+ if comments[1].Language != "go" {
+ t.Errorf("expected comments[1].Language == 'go', got %q", comments[1].Language)
+ }
}
func TestDiffCommentsScopeLocking(t *testing.T) {
@@ -110,7 +108,7 @@ func TestDiffCommentsScopeLocking(t *testing.T) {
{Type: "comment", Text: "// Method B comment", ScopeKey: "method:funcB", StartRow: 20},
}
- res := DiffComments(srcComments, dstComments)
+ res := DiffComments(srcComments, dstComments, nil)
if len(res.Actions) != 1 {
t.Fatalf("expected 1 action, got %d", len(res.Actions))
}
@@ -127,7 +125,7 @@ func TestDiffCommentsControlBranchScopeLocking(t *testing.T) {
{Type: "comment", Text: "-- Track leading whitespace for level", ScopeKey: "function:foo/if_statement", StartRow: 45},
}
- res := DiffComments(srcComments, dstComments)
+ res := DiffComments(srcComments, dstComments, nil)
// Moving between branches should delete and insert instead of update in place.
if len(res.Actions) != 2 {
t.Fatalf("expected 2 actions (1 delete, 1 insert), got %d", len(res.Actions))
@@ -149,27 +147,38 @@ func TestDiffCommentsControlBranchScopeLocking(t *testing.T) {
func TestDiffCommentsMovedScope(t *testing.T) {
srcComments := []CommentBlock{
- {Type: "comment", Text: "// Optional params comment", ScopeKey: "method_declaration:getParamsToSign", StartRow: 162, EndRow: 162},
+ {Type: "comment", Text: "// Optional params comment", ScopeKey: "method_declaration:getParamsToSign", StartRow: 162, EndRow: 162, Language: "go"},
}
dstComments := []CommentBlock{
- {Type: "comment", Text: "// Optional params comment", ScopeKey: "method_declaration:getOauthParams", StartRow: 159, EndRow: 159},
+ {Type: "comment", Text: "// Optional params comment", ScopeKey: "method_declaration:getOauthParams", StartRow: 159, EndRow: 159, Language: "go"},
}
- res := DiffComments(srcComments, dstComments)
- if len(res.Actions) != 1 {
- t.Fatalf("expected 1 Move action for comment moved across scopes, got %d actions", len(res.Actions))
+ res := DiffComments(srcComments, dstComments, nil)
+ if len(res.Actions) != 2 {
+ t.Fatalf("expected 2 actions (1 Delete, 1 Insert) for comment moved across scopes, got %d actions", len(res.Actions))
}
- if res.Actions[0].Type != actions.Move {
- t.Errorf("expected Move action, got %v", res.Actions[0].Type)
+ hasDelete := false
+ hasInsert := false
+ for _, act := range res.Actions {
+ if act.Type == actions.Move {
+ t.Errorf("expected no Move action for comment trivia, got actions.Move")
+ }
+ if act.Type == actions.Delete {
+ hasDelete = true
+ }
+ if act.Type == actions.Insert {
+ hasInsert = true
+ }
+ }
+ if !hasDelete || !hasInsert {
+ t.Errorf("expected 1 Delete and 1 Insert action, got: %+v", res.Actions)
+ }
+ if _, ok := res.LineMappings[162]; ok {
+ t.Errorf("expected cross-scope comment NOT to populate LineMappings, got %v", res.LineMappings)
}
}
func TestDiffCommentsGuzzlePhp(t *testing.T) {
- lang, err := treesitter.DetectLanguage("test.php")
- if err != nil {
- t.Fatal(err)
- }
- r := rules.Get("php")
src, err := os.ReadFile("../../tests/testdata/php_guzzle_handler_curl_multi/old.php")
if err != nil {
t.Fatal(err)
@@ -179,28 +188,115 @@ func TestDiffCommentsGuzzlePhp(t *testing.T) {
t.Fatal(err)
}
- parser := gotreesitter.NewParser(lang)
- treeA, err := parser.Parse(src)
+ _, flatNodesA, symbolsA, err := treesitter.ParseForPipeline(src, "php")
if err != nil {
t.Fatal(err)
}
- treeB, err := parser.Parse(dst)
+ _, flatNodesB, symbolsB, err := treesitter.ParseForPipeline(dst, "php")
if err != nil {
t.Fatal(err)
}
- srcComments := ExtractComments(treeA.RootNode(), src, lang, r)
- dstComments := ExtractComments(treeB.RootNode(), dst, lang, r)
+ srcComments := ExtractComments(flatNodesA, symbolsA, src, "php")
+ dstComments := ExtractComments(flatNodesB, symbolsB, dst, "php")
- res := DiffComments(srcComments, dstComments)
+ res := DiffComments(srcComments, dstComments, nil)
moveCount := 0
for _, a := range res.Actions {
- if a.Type == actions.Move && strings.Contains(a.Node.Label, "Optional parameters") {
+ if a.Type == actions.Move {
moveCount++
}
}
- if moveCount != 1 {
- t.Errorf("expected 1 Move action for Optional parameters comment, got %d", moveCount)
+ if moveCount != 0 {
+ t.Errorf("expected 0 Move actions for comment trivia in Guzzle PHP diff, got %d", moveCount)
+ }
+}
+
+func TestSyntheticCommentNodeInvariants(t *testing.T) {
+ cb := CommentBlock{
+ Type: "comment",
+ Text: "// Hello World",
+ StartByte: 10,
+ EndByte: 24,
+ StartRow: 2,
+ StartCol: 0,
+ EndRow: 2,
+ EndCol: 14,
+ ParentType: "function_declaration",
+ ParentStart: 0,
+ ParentEnd: 100,
+ ParentRow: 1,
+ ParentEndRow: 10,
+ Language: "go",
+ }
+
+ node := createCommentNode(&cb, cb.Language)
+ if node == nil {
+ t.Fatal("expected non-nil ASTNode")
+ }
+ if node.Parent == nil {
+ t.Fatal("expected non-nil Parent")
+ }
+ if len(node.Parent.Children) != 1 || node.Parent.Children[0] != node {
+ t.Fatalf("expected Parent.Children to contain node, got %v", node.Parent.Children)
+ }
+ if idx := node.ChildIndex(); idx != 0 {
+ t.Errorf("expected ChildIndex() == 0, got %d", idx)
+ }
+ if lang := node.GetLanguage(); lang != "go" {
+ t.Errorf("expected GetLanguage() == %q, got %q", "go", lang)
+ }
+
+ // Without parent
+ cbNoParent := CommentBlock{
+ Type: "comment",
+ Text: "// Top-level comment",
+ StartByte: 0,
+ EndByte: 20,
+ Language: "go",
+ }
+ nodeNoParent := createCommentNode(&cbNoParent, cbNoParent.Language)
+ if nodeNoParent.Parent != nil {
+ t.Errorf("expected nil parent for top-level comment")
+ }
+ if idx := nodeNoParent.ChildIndex(); idx != -1 {
+ t.Errorf("expected ChildIndex() == -1 for parentless node, got %d", idx)
+ }
+}
+
+func TestSyntheticCommentLineNodeInvariants(t *testing.T) {
+ cb := CommentBlock{
+ Type: "block_comment",
+ Text: "/* line 1\n * line 2 */",
+ StartByte: 10,
+ EndByte: 40,
+ StartRow: 2,
+ StartCol: 0,
+ EndRow: 3,
+ EndCol: 11,
+ ParentType: "class_declaration",
+ ParentStart: 0,
+ ParentEnd: 200,
+ ParentRow: 1,
+ ParentEndRow: 20,
+ Language: "python",
+ }
+
+ node := createCommentLineNode(&cb, " * line 2 */", 20, 32, 3, cb.Language)
+ if node == nil {
+ t.Fatal("expected non-nil ASTNode")
+ }
+ if node.Parent == nil {
+ t.Fatal("expected non-nil Parent")
+ }
+ if len(node.Parent.Children) != 1 || node.Parent.Children[0] != node {
+ t.Fatalf("expected Parent.Children to contain line node, got %v", node.Parent.Children)
+ }
+ if idx := node.ChildIndex(); idx != 0 {
+ t.Errorf("expected ChildIndex() == 0, got %d", idx)
+ }
+ if lang := node.GetLanguage(); lang != "python" {
+ t.Errorf("expected GetLanguage() == %q, got %q", "python", lang)
}
}
@@ -217,7 +313,7 @@ func TestDiffCommentsScopedLCSNoCrossover(t *testing.T) {
{Type: "comment", Text: "// step", ScopeKey: "func:doWork", StartRow: 32, EndRow: 32},
}
- res := DiffComments(srcComments, dstComments)
+ res := DiffComments(srcComments, dstComments, nil)
if len(res.Actions) != 0 {
t.Fatalf("expected 0 actions for matched identical comments, got %d", len(res.Actions))
}
@@ -225,3 +321,205 @@ func TestDiffCommentsScopedLCSNoCrossover(t *testing.T) {
t.Errorf("expected mappings 10->12, 20->22, 30->32; got %v", res.LineMappings)
}
}
+
+func TestDiffCommentsRenamedFunction(t *testing.T) {
+ srcDecl := &treesitter.ASTNode{ID: 10, Type: "function_declaration", StartByte: 0, EndByte: 200}
+ dstDecl := &treesitter.ASTNode{ID: 25, Type: "function_declaration", StartByte: 0, EndByte: 200}
+
+ mappings := engine.NewMapping()
+ mappings.Add(srcDecl, dstDecl)
+
+ srcComments := []CommentBlock{
+ {
+ Type: "comment",
+ Text: "// step 1: initialize",
+ StartRow: 5,
+ EndRow: 5,
+ EnclosingDecl: srcDecl,
+ RelativePath: "body",
+ },
+ {
+ Type: "comment",
+ Text: "// step 2: execute",
+ StartRow: 10,
+ EndRow: 10,
+ EnclosingDecl: srcDecl,
+ RelativePath: "body",
+ },
+ }
+
+ dstComments := []CommentBlock{
+ {
+ Type: "comment",
+ Text: "// step 1: initialize",
+ StartRow: 7,
+ EndRow: 7,
+ EnclosingDecl: dstDecl,
+ RelativePath: "body",
+ },
+ {
+ Type: "comment",
+ Text: "// step 2: execute",
+ StartRow: 12,
+ EndRow: 12,
+ EnclosingDecl: dstDecl,
+ RelativePath: "body",
+ },
+ }
+
+ res := DiffComments(srcComments, dstComments, mappings)
+ if len(res.Actions) != 0 {
+ t.Fatalf("expected 0 actions for comments inside renamed function, got %d actions: %+v", len(res.Actions), res.Actions)
+ }
+ if res.LineMappings[5] != 7 || res.LineMappings[10] != 12 {
+ t.Errorf("expected line mappings 5->7, 10->12; got %v", res.LineMappings)
+ }
+}
+
+func TestDiffCommentsLeadingDocstringRenamed(t *testing.T) {
+ srcDecl := &treesitter.ASTNode{ID: 10, Type: "function_declaration"}
+ dstDecl := &treesitter.ASTNode{ID: 25, Type: "function_declaration"}
+
+ mappings := engine.NewMapping()
+ mappings.Add(srcDecl, dstDecl)
+
+ srcComments := []CommentBlock{
+ {
+ Type: "comment",
+ Text: "// CalculateTotal computes total price",
+ StartRow: 4,
+ EndRow: 4,
+ EnclosingDecl: srcDecl,
+ RelativePath: "doc",
+ },
+ }
+ dstComments := []CommentBlock{
+ {
+ Type: "comment",
+ Text: "// CalculateTotal computes total price",
+ StartRow: 8,
+ EndRow: 8,
+ EnclosingDecl: dstDecl,
+ RelativePath: "doc",
+ },
+ }
+
+ res := DiffComments(srcComments, dstComments, mappings)
+ if len(res.Actions) != 0 {
+ t.Fatalf("expected 0 actions for docstring attached to renamed function, got %d actions", len(res.Actions))
+ }
+ if res.LineMappings[4] != 8 {
+ t.Errorf("expected line mapping 4->8, got %v", res.LineMappings)
+ }
+}
+
+func TestDiffCommentsBranchIsolation(t *testing.T) {
+ srcDecl := &treesitter.ASTNode{ID: 10, Type: "function_declaration"}
+ dstDecl := &treesitter.ASTNode{ID: 10, Type: "function_declaration"}
+
+ mappings := engine.NewMapping()
+ mappings.Add(srcDecl, dstDecl)
+
+ srcComments := []CommentBlock{
+ {
+ Type: "comment",
+ Text: "// handle error",
+ StartRow: 10,
+ EndRow: 10,
+ EnclosingDecl: srcDecl,
+ RelativePath: "body/if_statement/consequence",
+ },
+ }
+ dstComments := []CommentBlock{
+ {
+ Type: "comment",
+ Text: "// handle error",
+ StartRow: 20,
+ EndRow: 20,
+ EnclosingDecl: dstDecl,
+ RelativePath: "body/if_statement/alternative",
+ },
+ }
+
+ res := DiffComments(srcComments, dstComments, mappings)
+ // Because relative paths differ ("consequence" vs "alternative"), it should not match in Pass 1 LCS
+ // Pass 2 handles cross-scope as Delete + Insert (since dist is 10 <= 25)
+ if len(res.Actions) != 2 {
+ t.Fatalf("expected 2 actions (1 Delete, 1 Insert) across different branches, got %d", len(res.Actions))
+ }
+}
+
+func TestDiffCommentsUnmappedStrictIsolation(t *testing.T) {
+ srcDecl := &treesitter.ASTNode{ID: 10, Type: "function_declaration"}
+ dstDecl := &treesitter.ASTNode{ID: 99, Type: "function_declaration"}
+
+ // Unmapped declarations (mappings does NOT map srcDecl to dstDecl)
+ mappings := engine.NewMapping()
+
+ srcComments := []CommentBlock{
+ {
+ Type: "comment",
+ Text: "// validate inputs",
+ StartRow: 10,
+ EndRow: 10,
+ EnclosingDecl: srcDecl,
+ RelativePath: "body",
+ },
+ }
+ dstComments := []CommentBlock{
+ {
+ Type: "comment",
+ Text: "// validate inputs",
+ StartRow: 100, // distant row > 25
+ EndRow: 100,
+ EnclosingDecl: dstDecl,
+ RelativePath: "body",
+ },
+ }
+
+ res := DiffComments(srcComments, dstComments, mappings)
+ if len(res.Actions) != 2 {
+ t.Fatalf("expected 2 actions (1 Delete, 1 Insert) for unmapped distant declarations, got %d", len(res.Actions))
+ }
+ hasDelete := false
+ hasInsert := false
+ for _, act := range res.Actions {
+ if act.Type == actions.Delete {
+ hasDelete = true
+ }
+ if act.Type == actions.Insert {
+ hasInsert = true
+ }
+ }
+ if !hasDelete || !hasInsert {
+ t.Errorf("expected 1 Delete and 1 Insert action, got: %+v", res.Actions)
+ }
+}
+
+func TestExtractCommentsInsideFunctionNotDocComment(t *testing.T) {
+ src := []byte(`
+public class TestClass {
+ public void releaseByteBuffer(int ix, byte[] buffer) {
+ // 13-Jan-2024, tatu: [core#1186] Replace only if beneficial:
+ byte[] oldBuffer = _byteBuffers.get(ix);
+ }
+}
+`)
+
+ _, flatNodes, symbols, err := treesitter.ParseForPipeline(src, "java")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ comments := ExtractComments(flatNodes, symbols, src, "java")
+ if len(comments) != 1 {
+ t.Fatalf("expected 1 comment, got %d", len(comments))
+ }
+ c := comments[0]
+ if c.DeclType != "method_declaration" {
+ t.Errorf("expected DeclType to be 'method_declaration', got %q", c.DeclType)
+ }
+ if c.RelativePath == "doc" {
+ t.Errorf("expected RelativePath NOT to be 'doc' for comment inside method body, got %q", c.RelativePath)
+ }
+}
diff --git a/internal/comments/diff.go b/internal/comments/diff.go
index 6f12b088..9c24fa35 100644
--- a/internal/comments/diff.go
+++ b/internal/comments/diff.go
@@ -1,6 +1,7 @@
package comments
import (
+ "fmt"
"strings"
"github.com/HarshK97/diffmantic/internal/actions"
@@ -14,8 +15,8 @@ type DiffResult struct {
LineMappings map[int]int
}
-// DiffComments matches and diffs comments between source and destination files.
-func DiffComments(srcComments, dstComments []CommentBlock) *DiffResult {
+// DiffComments matches and diffs comments between source and destination files with AST mapping awareness.
+func DiffComments(srcComments, dstComments []CommentBlock, mappings *engine.Mapping) *DiffResult {
res := &DiffResult{
LineMappings: make(map[int]int),
}
@@ -26,16 +27,19 @@ func DiffComments(srcComments, dstComments []CommentBlock) *DiffResult {
srcMatched := make([]bool, len(srcComments))
dstMatched := make([]bool, len(dstComments))
- // Match identical comments within each scope using LCS so repeated comments don't cross over.
+ // Match identical comments within each canonical mapped scope using LCS so repeated comments don't cross over.
scopeSrcMap := make(map[string][]int)
scopeDstMap := make(map[string][]int)
for i := range srcComments {
- scopeSrcMap[srcComments[i].ScopeKey] = append(scopeSrcMap[srcComments[i].ScopeKey], i)
+ key := canonicalScopeKey(&srcComments[i], mappings, true)
+ scopeSrcMap[key] = append(scopeSrcMap[key], i)
}
for j := range dstComments {
- scopeDstMap[dstComments[j].ScopeKey] = append(scopeDstMap[dstComments[j].ScopeKey], j)
+ key := canonicalScopeKey(&dstComments[j], mappings, false)
+ scopeDstMap[key] = append(scopeDstMap[key], j)
}
+ // PASS 1: Intra-Scope Monotonic LCS Dynamic Programming
for scopeKey, srcIdxs := range scopeSrcMap {
dstIdxs := scopeDstMap[scopeKey]
if len(dstIdxs) == 0 {
@@ -71,7 +75,7 @@ func DiffComments(srcComments, dstComments []CommentBlock) *DiffResult {
dstMatched[dj] = true
sc := &srcComments[si]
dc := &dstComments[dj]
- nLines := min(sc.EndRow-sc.StartRow+1, dc.EndRow-dc.StartRow+1)
+ nLines := min(commentLineCount(sc), commentLineCount(dc))
for k := 0; k < nLines; k++ {
res.LineMappings[sc.StartRow+k] = dc.StartRow + k
}
@@ -85,7 +89,7 @@ func DiffComments(srcComments, dstComments []CommentBlock) *DiffResult {
}
}
- // Exact text matches across different scopes (treated as moved comments).
+ // PASS 2: Exact text matches across different scopes (treated as moved comments).
for i := range srcComments {
if srcMatched[i] {
continue
@@ -112,30 +116,36 @@ func DiffComments(srcComments, dstComments []CommentBlock) *DiffResult {
srcMatched[i] = true
dstMatched[bestJ] = true
dc := &dstComments[bestJ]
- nLines := min(sc.EndRow-sc.StartRow+1, dc.EndRow-dc.StartRow+1)
- for k := 0; k < nLines; k++ {
- res.LineMappings[sc.StartRow+k] = dc.StartRow + k
- }
- if sc.ScopeKey != dc.ScopeKey {
- srcNode := createCommentNode(sc)
- dstNode := createCommentNode(dc)
- res.Actions = append(res.Actions, actions.Action{
- Type: actions.Move,
- Node: srcNode,
- DestNode: dstNode,
- Parent: dstNode.Parent,
- })
+ scKey := canonicalScopeKey(sc, mappings, true)
+ dcKey := canonicalScopeKey(dc, mappings, false)
+ if scKey == dcKey {
+ nLines := min(commentLineCount(sc), commentLineCount(dc))
+ for k := 0; k < nLines; k++ {
+ res.LineMappings[sc.StartRow+k] = dc.StartRow + k
+ }
+ } else {
+ res.Actions = append(res.Actions,
+ actions.Action{
+ Type: actions.Delete,
+ Node: createCommentNode(sc, sc.Language),
+ },
+ actions.Action{
+ Type: actions.Insert,
+ Node: createCommentNode(dc, dc.Language),
+ },
+ )
}
}
}
- // Fuzzy match edited comments in the same scope.
+ // PASS 3: Fuzzy match edited comments in the same scope.
for i := range srcComments {
if srcMatched[i] {
continue
}
sc := &srcComments[i]
+ scKey := canonicalScopeKey(sc, mappings, true)
bestJ := -1
bestScore := 0.0
@@ -144,13 +154,14 @@ func DiffComments(srcComments, dstComments []CommentBlock) *DiffResult {
continue
}
dc := &dstComments[j]
+ dcKey := canonicalScopeKey(dc, mappings, false)
- if sc.ScopeKey != dc.ScopeKey {
+ if scKey != dcKey {
continue
}
- srcIsMulti := strings.Contains(sc.Text, "\n")
- dstIsMulti := strings.Contains(dc.Text, "\n")
+ srcIsMulti := strings.Contains(strings.TrimRight(sc.Text, "\r\n"), "\n")
+ dstIsMulti := strings.Contains(strings.TrimRight(dc.Text, "\r\n"), "\n")
sim := stringSimilarity(sc.Text, dc.Text)
minThreshold := 0.40
@@ -180,7 +191,7 @@ func DiffComments(srcComments, dstComments []CommentBlock) *DiffResult {
for i := range srcComments {
if !srcMatched[i] {
sc := &srcComments[i]
- node := createCommentNode(sc)
+ node := createCommentNode(sc, sc.Language)
res.Actions = append(res.Actions, actions.Action{
Type: actions.Delete,
Node: node,
@@ -191,7 +202,7 @@ func DiffComments(srcComments, dstComments []CommentBlock) *DiffResult {
for j := range dstComments {
if !dstMatched[j] {
dc := &dstComments[j]
- node := createCommentNode(dc)
+ node := createCommentNode(dc, dc.Language)
res.Actions = append(res.Actions, actions.Action{
Type: actions.Insert,
Node: node,
@@ -203,13 +214,16 @@ func DiffComments(srcComments, dstComments []CommentBlock) *DiffResult {
}
func diffCommentBlock(sc, dc *CommentBlock, res *DiffResult) {
- srcIsMulti := strings.Contains(sc.Text, "\n")
- dstIsMulti := strings.Contains(dc.Text, "\n")
+ srcTrimmed := strings.TrimRight(sc.Text, "\r\n")
+ dstTrimmed := strings.TrimRight(dc.Text, "\r\n")
+
+ srcIsMulti := strings.Contains(srcTrimmed, "\n")
+ dstIsMulti := strings.Contains(dstTrimmed, "\n")
if !srcIsMulti && !dstIsMulti {
res.LineMappings[sc.StartRow] = dc.StartRow
- srcNode := createCommentNode(sc)
- dstNode := createCommentNode(dc)
+ srcNode := createCommentNode(sc, sc.Language)
+ dstNode := createCommentNode(dc, dc.Language)
res.Actions = append(res.Actions, actions.Action{
Type: actions.Update,
Node: srcNode,
@@ -220,8 +234,8 @@ func diffCommentBlock(sc, dc *CommentBlock, res *DiffResult) {
}
// Line-by-line diff for multiline comments.
- srcLines := strings.Split(sc.Text, "\n")
- dstLines := strings.Split(dc.Text, "\n")
+ srcLines := strings.Split(srcTrimmed, "\n")
+ dstLines := strings.Split(dstTrimmed, "\n")
matchedA := engine.LineDiff(srcLines, dstLines)
matchedB := make(map[int]int)
@@ -269,8 +283,8 @@ func diffCommentBlock(sc, dc *CommentBlock, res *DiffResult) {
row := uint32(sc.StartRow + i)
dstRow := uint32(dc.StartRow + j)
- srcNode := createCommentLineNode(sc, srcLines[i], startByte, endByte, row)
- dstNode := createCommentLineNode(dc, dstLines[j], dstStartByte, dstEndByte, dstRow)
+ srcNode := createCommentLineNode(sc, srcLines[i], startByte, endByte, row, sc.Language)
+ dstNode := createCommentLineNode(dc, dstLines[j], dstStartByte, dstEndByte, dstRow, dc.Language)
res.Actions = append(res.Actions, actions.Action{
Type: actions.Update,
Node: srcNode,
@@ -285,7 +299,7 @@ func diffCommentBlock(sc, dc *CommentBlock, res *DiffResult) {
if _, ok := matchedA[i]; !ok {
startByte, endByte := srcOffsets[i][0], srcOffsets[i][1]
row := uint32(sc.StartRow + i)
- lineNode := createCommentLineNode(sc, srcLines[i], startByte, endByte, row)
+ lineNode := createCommentLineNode(sc, srcLines[i], startByte, endByte, row, sc.Language)
res.Actions = append(res.Actions, actions.Action{
Type: actions.Delete,
Node: lineNode,
@@ -297,7 +311,7 @@ func diffCommentBlock(sc, dc *CommentBlock, res *DiffResult) {
if _, ok := matchedB[j]; !ok {
startByte, endByte := dstOffsets[j][0], dstOffsets[j][1]
row := uint32(dc.StartRow + j)
- lineNode := createCommentLineNode(dc, dstLines[j], startByte, endByte, row)
+ lineNode := createCommentLineNode(dc, dstLines[j], startByte, endByte, row, dc.Language)
res.Actions = append(res.Actions, actions.Action{
Type: actions.Insert,
Node: lineNode,
@@ -348,23 +362,27 @@ func computeLineOffsets(baseOffset uint32, lines []string) [][2]uint32 {
return offsets
}
-func createCommentNode(c *CommentBlock) *treesitter.ASTNode {
- node := createSyntheticNode(c.Type, c.Text, c.StartByte, c.EndByte, uint32(c.StartRow), uint32(c.StartCol), uint32(c.EndRow), uint32(c.EndCol))
+func createCommentNode(c *CommentBlock, lang string) *treesitter.ASTNode {
+ node := createSyntheticNode(c.Type, c.Text, c.StartByte, c.EndByte, uint32(c.StartRow), uint32(c.StartCol), uint32(c.EndRow), uint32(c.EndCol), lang)
if c.ParentType != "" {
- node.Parent = createSyntheticNode(c.ParentType, "", c.ParentStart, c.ParentEnd, uint32(c.ParentRow), 0, uint32(c.ParentEndRow), 0)
+ parent := createSyntheticNode(c.ParentType, "", c.ParentStart, c.ParentEnd, uint32(c.ParentRow), 0, uint32(c.ParentEndRow), 0, lang)
+ node.Parent = parent
+ parent.Children = []*treesitter.ASTNode{node}
}
return node
}
-func createCommentLineNode(c *CommentBlock, label string, startByte, endByte, row uint32) *treesitter.ASTNode {
- node := createSyntheticNode(c.Type, label, startByte, endByte, row, 0, row, uint32(len(label)))
+func createCommentLineNode(c *CommentBlock, label string, startByte, endByte, row uint32, lang string) *treesitter.ASTNode {
+ node := createSyntheticNode(c.Type, label, startByte, endByte, row, 0, row, uint32(len(label)), lang)
if c.ParentType != "" {
- node.Parent = createSyntheticNode(c.ParentType, "", c.ParentStart, c.ParentEnd, uint32(c.ParentRow), 0, uint32(c.ParentEndRow), 0)
+ parent := createSyntheticNode(c.ParentType, "", c.ParentStart, c.ParentEnd, uint32(c.ParentRow), 0, uint32(c.ParentEndRow), 0, lang)
+ node.Parent = parent
+ parent.Children = []*treesitter.ASTNode{node}
}
return node
}
-func createSyntheticNode(nodeType, label string, startByte, endByte, startRow, startCol, endRow, endCol uint32) *treesitter.ASTNode {
+func createSyntheticNode(nodeType, label string, startByte, endByte, startRow, startCol, endRow, endCol uint32, lang string) *treesitter.ASTNode {
return &treesitter.ASTNode{
Type: nodeType,
Label: label,
@@ -374,5 +392,43 @@ func createSyntheticNode(nodeType, label string, startByte, endByte, startRow, s
StartCol: startCol,
EndRow: endRow,
EndCol: endCol,
+ Language: lang,
+ Children: make([]*treesitter.ASTNode, 0),
+ }
+}
+
+func canonicalScopeKey(c *CommentBlock, mappings *engine.Mapping, isSource bool) string {
+ if c == nil {
+ return "root"
+ }
+ if c.EnclosingDecl == nil {
+ if c.RelativePath != "" {
+ return "root:" + c.RelativePath
+ }
+ if c.ScopeKey != "" {
+ return c.ScopeKey
+ }
+ return "root"
+ }
+
+ if isSource {
+ if mappings != nil && mappings.Src() != nil {
+ if dstDecl, ok := mappings.Src()[c.EnclosingDecl]; ok && dstDecl != nil {
+ return fmt.Sprintf("decl_%d:%s", dstDecl.ID, c.RelativePath)
+ }
+ }
+ // Strict isolation for unmapped source declarations
+ return fmt.Sprintf("unmapped_src_%d:%s", c.EnclosingDecl.ID, c.RelativePath)
+ }
+
+ // Destination declaration scope key
+ return fmt.Sprintf("decl_%d:%s", c.EnclosingDecl.ID, c.RelativePath)
+}
+
+func commentLineCount(c *CommentBlock) int {
+ t := strings.TrimRight(c.Text, "\r\n")
+ if t == "" {
+ return 1
}
+ return strings.Count(t, "\n") + 1
}
diff --git a/internal/comments/extract.go b/internal/comments/extract.go
index fc0bc578..5ab2eecf 100644
--- a/internal/comments/extract.go
+++ b/internal/comments/extract.go
@@ -5,10 +5,12 @@ import (
"slices"
"strings"
+ "github.com/HarshK97/diffmantic/internal/treesitter"
"github.com/HarshK97/diffmantic/internal/treesitter/rules"
- "github.com/odvcencio/gotreesitter"
)
+const flatSentinel = 0xFFFFFFFF
+
// CommentBlock represents an extracted comment and its source position.
type CommentBlock struct {
Type string
@@ -25,27 +27,48 @@ type CommentBlock struct {
ParentEnd uint32
ParentRow int
ParentEndRow int
+ Language string
+
+ // Structural Declaration & Scope Context
+ DeclStart uint32
+ DeclEnd uint32
+ DeclType string
+ RelativePath string
+ EnclosingDecl *treesitter.ASTNode
}
-// ExtractComments walks the AST and returns all comment blocks.
-func ExtractComments(root *gotreesitter.Node, src []byte, lang *gotreesitter.Language, r *rules.Rules) []CommentBlock {
- if root == nil || lang == nil {
+// ExtractComments walks the raw flat buffer (pre-IsIgnored filtering)
+// and extracts comment blocks with full CST sibling/parent context.
+func ExtractComments(nodes []treesitter.FlatNode, symbols []string, src []byte, langName string) []CommentBlock {
+ if len(nodes) == 0 {
return nil
}
+ r := rules.Get(langName)
+ srcLen := uint32(len(src))
var list []CommentBlock
- traverseForComments(root, src, lang, r, &list)
- return list
-}
-func traverseForComments(n *gotreesitter.Node, src []byte, lang *gotreesitter.Language, r *rules.Rules, list *[]CommentBlock) {
- if n == nil {
- return
- }
- nodeType := n.Type(lang)
- if (r != nil && r.IsComment(nodeType)) || (r == nil && nodeType == "comment") {
- srcLen := uint32(len(src))
- start := min(n.StartByte(), srcLen)
- end := min(n.EndByte(), srcLen)
+ for idx := range nodes {
+ fn := &nodes[idx]
+ rawType := flatSymbol(fn, symbols)
+ if !isCommentType(rawType, r) {
+ continue
+ }
+
+ // Skip child comment tokens whose ancestor is already a comment container
+ // (e.g. in Rust where line_comment wraps doc_comment).
+ isChildComment := false
+ for pIdx := fn.ParentIdx; pIdx != flatSentinel && int(pIdx) < len(nodes); pIdx = nodes[pIdx].ParentIdx {
+ if isCommentType(flatSymbol(&nodes[pIdx], symbols), r) {
+ isChildComment = true
+ break
+ }
+ }
+ if isChildComment {
+ continue
+ }
+
+ start := min(fn.StartByte, srcLen)
+ end := min(fn.EndByte, srcLen)
if start > end {
start = end
}
@@ -54,51 +77,163 @@ func traverseForComments(n *gotreesitter.Node, src []byte, lang *gotreesitter.La
var parentType string
var parentStart, parentEnd uint32
var parentRow, parentEndRow int
- if p := n.Parent(); p != nil {
- parentType = p.Type(lang)
- parentStart = min(p.StartByte(), srcLen)
- parentEnd = min(p.EndByte(), srcLen)
- parentRow = int(p.StartPoint().Row)
- parentEndRow = int(p.EndPoint().Row)
+ if fn.ParentIdx != flatSentinel && int(fn.ParentIdx) < len(nodes) {
+ p := &nodes[fn.ParentIdx]
+ parentType = flatSymbol(p, symbols)
+ parentStart = min(p.StartByte, srcLen)
+ parentEnd = min(p.EndByte, srcLen)
+ parentRow = int(p.StartRow)
+ parentEndRow = int(p.EndRow)
} else {
parentEnd = srcLen
}
- scopeKey := findEnclosingDeclaration(n, src, lang, r)
+ scopeKey := flatFindEnclosingDeclaration(uint32(idx), nodes, symbols, src, r)
- *list = append(*list, CommentBlock{
- Type: nodeType,
+ declStart, declEnd, declType, relPath := flatCheckLeadingDocComment(uint32(idx), nodes, symbols, src, r)
+ if declType == "" {
+ declStart, declEnd, declType, relPath = flatFindEnclosingDeclarationHierarchy(uint32(idx), nodes, symbols, src, r)
+ }
+
+ list = append(list, CommentBlock{
+ Type: rawType,
Text: text,
StartByte: start,
EndByte: end,
- StartRow: int(n.StartPoint().Row),
- StartCol: int(n.StartPoint().Column),
- EndRow: int(n.EndPoint().Row),
- EndCol: int(n.EndPoint().Column),
+ StartRow: int(fn.StartRow),
+ StartCol: int(fn.StartCol),
+ EndRow: int(fn.EndRow),
+ EndCol: int(fn.EndCol),
ScopeKey: scopeKey,
ParentType: parentType,
ParentStart: parentStart,
ParentEnd: parentEnd,
ParentRow: parentRow,
ParentEndRow: parentEndRow,
+ Language: langName,
+ DeclStart: declStart,
+ DeclEnd: declEnd,
+ DeclType: declType,
+ RelativePath: relPath,
})
- return
}
- for i := range n.ChildCount() {
- traverseForComments(n.Child(i), src, lang, r, list)
+ return list
+}
+
+func flatSymbol(fn *treesitter.FlatNode, symbols []string) string {
+ if int(fn.TypeID) < len(symbols) {
+ return symbols[fn.TypeID]
+ }
+ return ""
+}
+
+func isCommentType(nodeType string, r *rules.Rules) bool {
+ if r != nil {
+ return r.IsComment(nodeType)
+ }
+ return rules.IsComment(nodeType)
+}
+
+// flatCheckLeadingDocComment mirrors checkLeadingDocComment using flat buffer sibling walking.
+func flatCheckLeadingDocComment(idx uint32, nodes []treesitter.FlatNode, symbols []string, src []byte, r *rules.Rules) (uint32, uint32, string, string) {
+ if r == nil {
+ return 0, 0, "", ""
+ }
+ fn := &nodes[idx]
+
+ // Walk NextSiblingIdx to find the first non-comment, non-"\n" sibling
+ nextIdx := fn.NextSiblingIdx
+ for nextIdx != flatSentinel && int(nextIdx) < len(nodes) {
+ nextType := flatSymbol(&nodes[nextIdx], symbols)
+ if !isCommentType(nextType, r) && nextType != "\n" {
+ break
+ }
+ nextIdx = nodes[nextIdx].NextSiblingIdx
+ }
+ if nextIdx == flatSentinel || int(nextIdx) >= len(nodes) {
+ return 0, 0, "", ""
+ }
+
+ target := &nodes[nextIdx]
+ t := flatSymbol(target, symbols)
+ if !r.IsDeclaration(t) {
+ // Check inner children for wrapped declarations (export_statement, etc.)
+ childIdx := target.FirstChildIdx
+ for childIdx != flatSentinel && int(childIdx) < len(nodes) {
+ ct := flatSymbol(&nodes[childIdx], symbols)
+ if r.IsDeclaration(ct) {
+ target = &nodes[childIdx]
+ t = ct
+ break
+ }
+ childIdx = nodes[childIdx].NextSiblingIdx
+ }
+ }
+ if !r.IsDeclaration(t) {
+ return 0, 0, "", ""
+ }
+
+ // Check if comment is inside a block (interior statement comment vs exterior doc comment)
+ pIdx := fn.ParentIdx
+ insideBlock := false
+ for pIdx != flatSentinel && int(pIdx) < len(nodes) {
+ if r.IsBlock(flatSymbol(&nodes[pIdx], symbols)) {
+ insideBlock = true
+ break
+ }
+ pIdx = nodes[pIdx].ParentIdx
+ }
+ if insideBlock && r.IsLocalVarDeclaration(t) {
+ return 0, 0, "", ""
+ }
+
+ if int(target.StartRow)-int(fn.EndRow) <= 1 {
+ return target.StartByte, target.EndByte, t, "doc"
}
+ return 0, 0, "", ""
}
-func findEnclosingDeclaration(n *gotreesitter.Node, src []byte, lang *gotreesitter.Language, r *rules.Rules) string {
- if r == nil || n == nil {
+// flatFindEnclosingDeclarationHierarchy mirrors findEnclosingDeclarationHierarchy.
+func flatFindEnclosingDeclarationHierarchy(idx uint32, nodes []treesitter.FlatNode, symbols []string, src []byte, r *rules.Rules) (uint32, uint32, string, string) {
+ if r == nil {
+ return 0, 0, "", "root"
+ }
+ var containers []string
+ pIdx := nodes[idx].ParentIdx
+ for pIdx != flatSentinel && int(pIdx) < len(nodes) {
+ t := flatSymbol(&nodes[pIdx], symbols)
+ if r.IsDeclaration(t) {
+ relPath := "body"
+ if len(containers) > 0 {
+ slices.Reverse(containers)
+ relPath = "body/" + strings.Join(containers, "/")
+ }
+ return nodes[pIdx].StartByte, nodes[pIdx].EndByte, t, relPath
+ }
+ if r.IsBlock(t) || r.IsScaffolding(t) {
+ containers = append(containers, t)
+ }
+ pIdx = nodes[pIdx].ParentIdx
+ }
+ if len(containers) > 0 {
+ slices.Reverse(containers)
+ return 0, 0, "", "root/" + strings.Join(containers, "/")
+ }
+ return 0, 0, "", "root"
+}
+
+// flatFindEnclosingDeclaration mirrors findEnclosingDeclaration.
+func flatFindEnclosingDeclaration(idx uint32, nodes []treesitter.FlatNode, symbols []string, src []byte, r *rules.Rules) string {
+ if r == nil {
return "root"
}
+ srcLen := uint32(len(src))
var containers []string
- curr := n.Parent()
- for curr != nil {
- t := curr.Type(lang)
+ pIdx := nodes[idx].ParentIdx
+ for pIdx != flatSentinel && int(pIdx) < len(nodes) {
+ t := flatSymbol(&nodes[pIdx], symbols)
if r.IsDeclaration(t) {
- name := getDeclarationIdentifier(curr, src, lang, r)
+ name := flatGetDeclarationIdentifier(&nodes[pIdx], nodes, symbols, src, r, srcLen)
decl := t
if name != "" {
decl = t + ":" + name
@@ -112,7 +247,7 @@ func findEnclosingDeclaration(n *gotreesitter.Node, src []byte, lang *gotreesitt
if r.IsBlock(t) || r.IsScaffolding(t) {
containers = append(containers, t)
}
- curr = curr.Parent()
+ pIdx = nodes[pIdx].ParentIdx
}
if len(containers) > 0 {
slices.Reverse(containers)
@@ -121,35 +256,71 @@ func findEnclosingDeclaration(n *gotreesitter.Node, src []byte, lang *gotreesitt
return "root"
}
-func getDeclarationIdentifier(n *gotreesitter.Node, src []byte, lang *gotreesitter.Language, r *rules.Rules) string {
- if n == nil || r == nil {
+func flatGetDeclarationIdentifier(fn *treesitter.FlatNode, nodes []treesitter.FlatNode, symbols []string, src []byte, r *rules.Rules, srcLen uint32) string {
+ if r == nil {
return ""
}
- for i := range n.ChildCount() {
- child := n.Child(i)
- if child == nil {
- continue
- }
- ct := child.Type(lang)
+ childIdx := fn.FirstChildIdx
+ for childIdx != flatSentinel && int(childIdx) < len(nodes) {
+ child := &nodes[childIdx]
+ ct := flatSymbol(child, symbols)
if r.IsIdentifier(ct) {
- s := min(child.StartByte(), uint32(len(src)))
- e := min(child.EndByte(), uint32(len(src)))
+ s := min(child.StartByte, srcLen)
+ e := min(child.EndByte, srcLen)
if s < e {
return string(src[s:e])
}
}
if r.IsScaffolding(ct) {
- for j := range child.ChildCount() {
- sub := child.Child(j)
- if sub != nil && r.IsIdentifier(sub.Type(lang)) {
- s := min(sub.StartByte(), uint32(len(src)))
- e := min(sub.EndByte(), uint32(len(src)))
+ subIdx := child.FirstChildIdx
+ for subIdx != flatSentinel && int(subIdx) < len(nodes) {
+ sub := &nodes[subIdx]
+ if r.IsIdentifier(flatSymbol(sub, symbols)) {
+ s := min(sub.StartByte, srcLen)
+ e := min(sub.EndByte, srcLen)
if s < e {
return string(src[s:e])
}
}
+ subIdx = sub.NextSiblingIdx
}
}
+ childIdx = child.NextSiblingIdx
}
return ""
}
+
+type declSpanKey struct {
+ start uint32
+ end uint32
+}
+
+// BindASTNodes attaches exact AST node pointers to extracted comment blocks in O(K + N).
+func BindASTNodes(comments []CommentBlock, root *treesitter.ASTNode) {
+ if len(comments) == 0 || root == nil {
+ return
+ }
+ declMap := make(map[declSpanKey]*treesitter.ASTNode)
+ var indexDecls func(n *treesitter.ASTNode)
+ indexDecls = func(n *treesitter.ASTNode) {
+ if n == nil {
+ return
+ }
+ key := declSpanKey{start: n.StartByte, end: n.EndByte}
+ if existing, exists := declMap[key]; !exists || (existing != nil && existing.Type != n.Type) {
+ declMap[key] = n
+ }
+ for _, child := range n.Children {
+ indexDecls(child)
+ }
+ }
+ indexDecls(root)
+
+ for i := range comments {
+ if comments[i].DeclEnd > comments[i].DeclStart || comments[i].DeclType != "" {
+ if declNode, ok := declMap[declSpanKey{start: comments[i].DeclStart, end: comments[i].DeclEnd}]; ok {
+ comments[i].EnclosingDecl = declNode
+ }
+ }
+ }
+}
diff --git a/internal/inline/render.go b/internal/inline/render.go
index d0070614..cddd5161 100644
--- a/internal/inline/render.go
+++ b/internal/inline/render.go
@@ -222,10 +222,10 @@ func Render(srcFile, dstFile string, srcBytes, dstBytes []byte, env *serialize.E
// Resolve language rules for declaration classification
var r *rules.Rules
- if entry := treesitter.DetectGrammarEntry(srcFile); entry != nil {
- r = rules.Get(entry.Name)
- } else if entry := treesitter.DetectGrammarEntry(dstFile); entry != nil {
- r = rules.Get(entry.Name)
+ if lang, _ := treesitter.DetectLanguage(srcFile); lang != nil {
+ r = rules.Get(lang.Name)
+ } else if lang, _ := treesitter.DetectLanguage(dstFile); lang != nil {
+ r = rules.Get(lang.Name)
}
meta := buildHunkMoveMetadata(env.Actions, hunks, filteredPairs, srcOffsets, dstOffsets, srcLines, dstLines, r)
diff --git a/internal/pipeline/pipeline.go b/internal/pipeline/pipeline.go
index 7ceac271..affe2ae9 100644
--- a/internal/pipeline/pipeline.go
+++ b/internal/pipeline/pipeline.go
@@ -12,18 +12,22 @@ import (
"github.com/HarshK97/diffmantic/internal/postprocess"
"github.com/HarshK97/diffmantic/internal/serialize"
"github.com/HarshK97/diffmantic/internal/treesitter"
- "github.com/HarshK97/diffmantic/internal/treesitter/rules"
- "github.com/odvcencio/gotreesitter"
)
// MaxASTFileSize caps the file size for AST parsing before falling back to line diffing.
-const MaxASTFileSize = 400 * 1024
+const MaxASTFileSize = 1024 * 1024
+
+// MaxASTFileLines caps the line count for AST parsing before falling back to line diffing.
+const MaxASTFileLines = 10000
// DiffOptions configures parsing limits, comment handling, and output options.
type DiffOptions struct {
ParseErrorLimit int
DisableErrorFallback bool
DisableSizeLimit bool
+ MaxASTFileSize int
+ DisableLineLimit bool
+ MaxASTFileLines int
IsConflict bool
IgnoreComments bool
EnvelopeOpts serialize.EnvelopeOptions
@@ -35,6 +39,7 @@ type DiffResult struct {
DstBytes []byte
SrcFile string
DstFile string
+ IsBinary bool
SrcAST *treesitter.ASTNode
DstAST *treesitter.ASTNode
MatchResult *engine.MatchResult
@@ -42,6 +47,15 @@ type DiffResult struct {
Envelope *serialize.Envelope
}
+// IsBinary detects whether a byte buffer contains binary data (null bytes in the first 8000 bytes).
+func IsBinary(data []byte) bool {
+ sample := data
+ if len(sample) > 8000 {
+ sample = sample[:8000]
+ }
+ return bytes.IndexByte(sample, 0) != -1
+}
+
// HasConflictMarkers checks if the buffer contains Git merge conflict markers.
func HasConflictMarkers(data []byte) bool {
hasStart := bytes.HasPrefix(data, []byte("<<<<<<<")) || bytes.Contains(data, []byte("\n<<<<<<<"))
@@ -61,8 +75,37 @@ func Run(srcBytes, dstBytes []byte, srcFile, dstFile string, opts DiffOptions) (
}
}
+ if IsBinary(srcBytes) || IsBinary(dstBytes) {
+ return &DiffResult{
+ SrcBytes: srcBytes,
+ DstBytes: dstBytes,
+ SrcFile: srcFile,
+ DstFile: dstFile,
+ IsBinary: true,
+ Envelope: &serialize.Envelope{
+ Version: serialize.SchemaVersion,
+ IsBinary: true,
+ },
+ }, nil
+ }
+
+ maxSize := MaxASTFileSize
+ if opts.MaxASTFileSize > 0 {
+ maxSize = opts.MaxASTFileSize
+ }
+
+ maxLines := MaxASTFileLines
+ if opts.MaxASTFileLines > 0 {
+ maxLines = opts.MaxASTFileLines
+ }
+
+ exceedsLines := !opts.DisableLineLimit && maxLines > 0 &&
+ (bytes.Count(srcBytes, []byte{'\n'}) > maxLines || bytes.Count(dstBytes, []byte{'\n'}) > maxLines)
+
+ exceedsSize := !opts.DisableSizeLimit && (len(srcBytes) > maxSize || len(dstBytes) > maxSize)
+
if opts.IsConflict || (HasConflictMarkers(srcBytes) || HasConflictMarkers(dstBytes)) ||
- (!opts.DisableSizeLimit && (len(srcBytes) > MaxASTFileSize || len(dstBytes) > MaxASTFileSize)) {
+ exceedsSize || exceedsLines {
return &DiffResult{
SrcBytes: srcBytes,
DstBytes: dstBytes,
@@ -93,14 +136,9 @@ func Run(srcBytes, dstBytes []byte, srcFile, dstFile string, opts DiffOptions) (
langB = langA
}
- rulesA := rules.Get(langA.Name)
- rulesB := rules.Get(langB.Name)
-
var (
srcAST *treesitter.ASTNode
dstAST *treesitter.ASTNode
- srcTree *gotreesitter.Tree
- dstTree *gotreesitter.Tree
srcComments []comments.CommentBlock
dstComments []comments.CommentBlock
wg sync.WaitGroup
@@ -109,16 +147,30 @@ func Run(srcBytes, dstBytes []byte, srcFile, dstFile string, opts DiffOptions) (
wg.Add(2)
go func() {
defer wg.Done()
- srcAST, srcTree, _ = treesitter.ParseWithLanguageAndTree(srcBytes, langA)
- if srcTree != nil && !opts.IgnoreComments {
- srcComments = comments.ExtractComments(srcTree.RootNode(), srcBytes, langA, rulesA)
+ var (
+ srcFlatNodes []treesitter.FlatNode
+ srcSymbols []string
+ )
+ srcAST, srcFlatNodes, srcSymbols, _ = treesitter.ParseForPipeline(srcBytes, langA.Name)
+ if !opts.IgnoreComments && len(srcFlatNodes) > 0 {
+ srcComments = comments.ExtractComments(srcFlatNodes, srcSymbols, srcBytes, langA.Name)
+ if srcAST != nil {
+ comments.BindASTNodes(srcComments, srcAST)
+ }
}
}()
go func() {
defer wg.Done()
- dstAST, dstTree, _ = treesitter.ParseWithLanguageAndTree(dstBytes, langB)
- if dstTree != nil && !opts.IgnoreComments {
- dstComments = comments.ExtractComments(dstTree.RootNode(), dstBytes, langB, rulesB)
+ var (
+ dstFlatNodes []treesitter.FlatNode
+ dstSymbols []string
+ )
+ dstAST, dstFlatNodes, dstSymbols, _ = treesitter.ParseForPipeline(dstBytes, langB.Name)
+ if !opts.IgnoreComments && len(dstFlatNodes) > 0 {
+ dstComments = comments.ExtractComments(dstFlatNodes, dstSymbols, dstBytes, langB.Name)
+ if dstAST != nil {
+ comments.BindASTNodes(dstComments, dstAST)
+ }
}
}()
@@ -135,29 +187,29 @@ func Run(srcBytes, dstBytes []byte, srcFile, dstFile string, opts DiffOptions) (
}, nil
}
+ matchResult := engine.Match(srcAST, dstAST, srcBytes, dstBytes, part)
+
var (
- matchResult *engine.MatchResult
- commentRes *comments.DiffResult
- matchWg sync.WaitGroup
+ commentRes *comments.DiffResult
+ es *actions.EditScript
+ wgPost sync.WaitGroup
)
- matchWg.Add(1)
+ wgPost.Add(1)
go func() {
- defer matchWg.Done()
- matchResult = engine.Match(srcAST, dstAST, srcBytes, dstBytes, part)
+ defer wgPost.Done()
+ es = actions.GenerateEditScript(srcAST, dstAST, matchResult.Mappings)
}()
if !opts.IgnoreComments && (len(srcComments) > 0 || len(dstComments) > 0) {
- matchWg.Add(1)
+ wgPost.Add(1)
go func() {
- defer matchWg.Done()
- commentRes = comments.DiffComments(srcComments, dstComments)
+ defer wgPost.Done()
+ commentRes = comments.DiffComments(srcComments, dstComments, matchResult.Mappings)
}()
}
- matchWg.Wait()
-
- es := actions.GenerateEditScript(srcAST, dstAST, matchResult.Mappings)
+ wgPost.Wait()
if commentRes != nil && len(commentRes.Actions) > 0 {
for _, act := range commentRes.Actions {
diff --git a/internal/pipeline/pipeline_test.go b/internal/pipeline/pipeline_test.go
new file mode 100644
index 00000000..13b3376d
--- /dev/null
+++ b/internal/pipeline/pipeline_test.go
@@ -0,0 +1,49 @@
+package pipeline
+
+import (
+ "strings"
+ "testing"
+)
+
+func TestPipeline_LineLimitFallback(t *testing.T) {
+ // Build a small Go file with 12 lines
+ var lines []string
+ for i := 0; i < 12; i++ {
+ lines = append(lines, "package main")
+ }
+ content := []byte(strings.Join(lines, "\n"))
+
+ // With maxLines=10 and 12-line file, it should fall back to line diff (no AST).
+ res, err := Run(content, content, "main.go", "main.go", DiffOptions{
+ MaxASTFileLines: 10,
+ })
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if res.SrcAST != nil || res.DstAST != nil {
+ t.Errorf("expected AST nil on line limit fallback, got non-nil")
+ }
+
+ // With DisableLineLimit=true, AST should parse regardless.
+ resUnlimited, err := Run(content, content, "main.go", "main.go", DiffOptions{
+ MaxASTFileLines: 10,
+ DisableLineLimit: true,
+ })
+ if err != nil {
+ t.Fatalf("unexpected error with disabled limit: %v", err)
+ }
+ if resUnlimited.SrcAST == nil {
+ t.Errorf("expected AST parsed when line limit disabled")
+ }
+
+ // With higher limit of 20, file has fewer lines — should parse as AST.
+ resHigher, err := Run(content, content, "main.go", "main.go", DiffOptions{
+ MaxASTFileLines: 20,
+ })
+ if err != nil {
+ t.Fatalf("unexpected error with higher limit: %v", err)
+ }
+ if resHigher.SrcAST == nil {
+ t.Errorf("expected AST parsed when file is under line limit")
+ }
+}
diff --git a/internal/serialize/json.go b/internal/serialize/json.go
index 73817131..cd95ac35 100644
--- a/internal/serialize/json.go
+++ b/internal/serialize/json.go
@@ -46,6 +46,7 @@ type EnvelopeOptions struct {
// Envelope wraps the serialized actions list with a schema version.
type Envelope struct {
Version string `json:"version"`
+ IsBinary bool `json:"is_binary,omitempty"`
Actions []Action `json:"actions,omitempty"`
LineAlignment []LineAlignmentPair `json:"line_alignment,omitempty"`
LeftHighlights []HighlightSpan `json:"left_highlights,omitempty"`
diff --git a/internal/treesitter/ast.go b/internal/treesitter/ast.go
index 12f8b22e..ab4afc07 100644
--- a/internal/treesitter/ast.go
+++ b/internal/treesitter/ast.go
@@ -2,10 +2,8 @@ package treesitter
import (
"slices"
- "strings"
"github.com/HarshK97/diffmantic/internal/treesitter/rules"
- "github.com/odvcencio/gotreesitter"
)
type ASTNode struct {
@@ -68,109 +66,6 @@ func (n *ASTNode) ComputeHashes() {
n.StructureHash = sh
}
-func BuildAST(n *gotreesitter.Node, src []byte, lang *gotreesitter.Language, parent *ASTNode) *ASTNode {
- if n == nil {
- return nil
- }
- r := rules.Get(lang.Name)
- node := buildASTWithRules(n, src, lang, parent, r)
- if node != nil && parent == nil {
- errCount := countErrorNodes(n, lang)
- node.Language = lang.Name
- node.ParseErrorCount = errCount
- node.HasError = errCount > 0
- node.ComputeHashes()
- EnsureIndex(node)
- }
- return node
-}
-
-func countErrorNodes(n *gotreesitter.Node, lang *gotreesitter.Language) int {
- if n == nil || !n.HasError() {
- return 0
- }
- count := 0
- if n.Type(lang) == "ERROR" || n.IsError() || n.IsMissing() {
- count = 1
- }
- for i := range n.ChildCount() {
- count += countErrorNodes(n.Child(i), lang)
- }
- return count
-}
-
-func buildASTWithRules(n *gotreesitter.Node, src []byte, lang *gotreesitter.Language, parent *ASTNode, r *rules.Rules) *ASTNode {
- nodeType := n.Type(lang)
- if n.IsMissing() {
- nodeType = "MISSING " + nodeType
- }
-
- isLeaf := n.ChildCount() == 0 || (r != nil && r.IsFlattened(nodeType))
- var label string
- if isLeaf {
- srcLen := uint32(len(src))
- start, end := min(n.StartByte(), srcLen), min(n.EndByte(), srcLen)
- if start > end {
- start = end
- }
- label = strings.TrimSpace(string(src[start:end]))
- }
-
- if r != nil && r.IsIgnored(nodeType, label) {
- return nil
- }
-
- node := &ASTNode{
- Type: nodeType,
- Parent: parent,
- StartByte: n.StartByte(),
- EndByte: n.EndByte(),
- StartRow: n.StartPoint().Row,
- StartCol: n.StartPoint().Column,
- EndRow: n.EndPoint().Row,
- EndCol: n.EndPoint().Column,
- }
-
- // Only set label for leaf nodes or string literals.
- if isLeaf {
- node.Label = label
- }
-
- if r != nil {
- if alias, ok := r.Alias(nodeType, label); ok {
- node.Type = alias
- }
- if r.IsLabelIgnored(node.Type) {
- node.Label = ""
- }
- if isLeaf && r.IsKeyword(nodeType, label) {
- node.IsKeyword = true
- }
- if r.IsUnordered(node.Type) {
- node.IsUnordered = true
- }
- }
-
- for i := range n.ChildCount() {
- if child := buildASTWithRules(n.Child(i), src, lang, node, r); child != nil {
- node.Children = append(node.Children, child)
- }
- }
-
- if r != nil && r.IsFlattened(nodeType) {
- var flattenedChildren []*ASTNode
- for _, child := range node.Children {
- flattenedChildren = append(flattenedChildren, child.Children...)
- for _, grandchild := range child.Children {
- grandchild.Parent = node
- }
- }
- node.Children = flattenedChildren
- }
-
- return node
-}
-
// Size returns the total number of nodes in the subtree rooted at n.
func (n *ASTNode) Size() int {
if n == nil {
diff --git a/internal/treesitter/bridge_cgo.go b/internal/treesitter/bridge_cgo.go
new file mode 100644
index 00000000..ca30d1ff
--- /dev/null
+++ b/internal/treesitter/bridge_cgo.go
@@ -0,0 +1,196 @@
+package treesitter
+
+/*
+#cgo CFLAGS: -I${SRCDIR}/../../native/bridge/include -O3
+#cgo LDFLAGS: -L${SRCDIR}/../../native/bridge/lib -ldiffmantic_grammars -lstdc++
+#include "bridge.h"
+#include "../../native/bridge/src/bridge.c"
+
+extern const TSLanguage* diffmantic_get_native_language(const char* name);
+
+static uint32_t get_ts_lang_symbol_count(const void* ts_lang_ptr) {
+ return ts_language_symbol_count((const TSLanguage*)ts_lang_ptr);
+}
+
+static const char* get_ts_lang_symbol_name(const void* ts_lang_ptr, uint16_t symbol_id) {
+ return ts_language_symbol_name((const TSLanguage*)ts_lang_ptr, symbol_id);
+}
+*/
+import "C"
+
+import (
+ "errors"
+ "fmt"
+ "slices"
+ "sync"
+ "unsafe"
+)
+
+type cachedLang struct {
+ ptr unsafe.Pointer
+ symbols []string
+}
+
+var (
+ nativeLangCache = make(map[string]cachedLang)
+ nativeLangMu sync.RWMutex
+)
+
+func getLanguageSymbols(langName string) (unsafe.Pointer, []string, error) {
+ if langName == "" {
+ return nil, nil, errors.New("empty language name")
+ }
+
+ nativeLangMu.RLock()
+ if entry, ok := nativeLangCache[langName]; ok {
+ nativeLangMu.RUnlock()
+ return entry.ptr, entry.symbols, nil
+ }
+ nativeLangMu.RUnlock()
+
+ nativeLangMu.Lock()
+ defer nativeLangMu.Unlock()
+
+ if entry, ok := nativeLangCache[langName]; ok {
+ return entry.ptr, entry.symbols, nil
+ }
+
+ ptr, err := GetNativeLanguage(langName)
+ if err != nil || ptr == nil {
+ return nil, nil, fmt.Errorf("native parser not available for %s: %w", langName, err)
+ }
+
+ symbols := NativeLanguageSymbols(ptr)
+ nativeLangCache[langName] = cachedLang{ptr: ptr, symbols: symbols}
+ return ptr, symbols, nil
+}
+
+// ParseWithLanguage parses source bytes for a given language name.
+func ParseWithLanguage(src []byte, langName string) (*ASTNode, error) {
+ ptr, symbols, err := getLanguageSymbols(langName)
+ if err != nil {
+ return nil, err
+ }
+ return ParseWithNativeFlatBuffer(src, ptr, langName, symbols)
+}
+
+// ParseForPipeline parses via the native flat-buffer bridge, returning the root AST,
+// raw flat nodes, and symbol table for comment extraction in the diff pipeline.
+func ParseForPipeline(src []byte, langName string) (*ASTNode, []FlatNode, []string, error) {
+ ptr, symbols, err := getLanguageSymbols(langName)
+ if err != nil {
+ return nil, nil, nil, err
+ }
+ ast, flatNodes, err := parseWithNativeFlatBufferKeepNodes(src, ptr, langName, symbols)
+ return ast, flatNodes, symbols, err
+}
+
+// parseWithNativeFlatBufferKeepNodes is like ParseWithNativeFlatBuffer but also returns
+// a Go-owned copy of the flat node array for comment extraction.
+func parseWithNativeFlatBufferKeepNodes(src []byte, tsLangPtr unsafe.Pointer, langName string, symbols []string) (*ASTNode, []FlatNode, error) {
+ if tsLangPtr == nil {
+ return nil, nil, errors.New("nil native language pointer")
+ }
+
+ if len(symbols) == 0 {
+ symbols = NativeLanguageSymbols(tsLangPtr)
+ }
+
+ var srcPtr *C.uint8_t
+ if len(src) > 0 {
+ srcPtr = (*C.uint8_t)(unsafe.Pointer(&src[0]))
+ }
+
+ res := C.parse_to_flat_ast(
+ srcPtr,
+ C.size_t(len(src)),
+ tsLangPtr,
+ )
+
+ if res.error_code != 0 {
+ return nil, nil, fmt.Errorf("native flat tree-sitter parse failed (code %d)", int(res.error_code))
+ }
+
+ defer C.free_flat_ast(res)
+
+ var nodes []FlatNode
+ if res.node_count > 0 && res.nodes_ptr != nil {
+ nodes = unsafe.Slice((*FlatNode)(unsafe.Pointer(res.nodes_ptr)), int(res.node_count))
+ }
+
+ // Copy flat nodes into Go-owned memory for comment extraction (C memory freed on return).
+ goNodes := slices.Clone(nodes)
+
+ root := IngestFlatAST(nodes, symbols, src, langName)
+
+ return root, goNodes, nil
+}
+
+type FlatASTResult = C.FlatASTResult
+
+// GetNativeLanguage retrieves the statically linked native Tree-sitter language for any of the 16 core languages (18 grammars).
+func GetNativeLanguage(langName string) (unsafe.Pointer, error) {
+ cName := C.CString(langName)
+ defer C.free(unsafe.Pointer(cName))
+
+ ptr := unsafe.Pointer(C.diffmantic_get_native_language(cName))
+ if ptr == nil {
+ return nil, fmt.Errorf("unsupported or unregistered native language: %s", langName)
+ }
+ return ptr, nil
+}
+
+// NativeLanguageSymbols returns the slice of symbol names defined by the native Tree-sitter language.
+func NativeLanguageSymbols(tsLangPtr unsafe.Pointer) []string {
+ if tsLangPtr == nil {
+ return nil
+ }
+ count := int(C.get_ts_lang_symbol_count(tsLangPtr))
+ symbols := make([]string, count)
+ for i := range count {
+ cName := C.get_ts_lang_symbol_name(tsLangPtr, C.uint16_t(i))
+ if cName != nil {
+ symbols[i] = C.GoString(cName)
+ }
+ }
+ return symbols
+}
+
+// ParseWithNativeFlatBuffer parses source code using the native Tree-sitter engine
+// and converts the resulting flat buffer into an ASTNode tree.
+func ParseWithNativeFlatBuffer(src []byte, tsLangPtr unsafe.Pointer, langName string, symbols []string) (*ASTNode, error) {
+ if tsLangPtr == nil {
+ return nil, errors.New("nil native language pointer")
+ }
+
+ if len(symbols) == 0 {
+ symbols = NativeLanguageSymbols(tsLangPtr)
+ }
+
+ var srcPtr *C.uint8_t
+ if len(src) > 0 {
+ srcPtr = (*C.uint8_t)(unsafe.Pointer(&src[0]))
+ }
+
+ res := C.parse_to_flat_ast(
+ srcPtr,
+ C.size_t(len(src)),
+ tsLangPtr,
+ )
+
+ if res.error_code != 0 {
+ return nil, fmt.Errorf("native flat tree-sitter parse failed (code %d)", int(res.error_code))
+ }
+
+ // Release the C flat buffer on return, even if ingestion panics.
+ defer C.free_flat_ast(res)
+
+ var nodes []FlatNode
+ if res.node_count > 0 && res.nodes_ptr != nil {
+ nodes = unsafe.Slice((*FlatNode)(unsafe.Pointer(res.nodes_ptr)), int(res.node_count))
+ }
+
+ root := IngestFlatAST(nodes, symbols, src, langName)
+
+ return root, nil
+}
diff --git a/internal/treesitter/bridge_ingest.go b/internal/treesitter/bridge_ingest.go
new file mode 100644
index 00000000..fb860b23
--- /dev/null
+++ b/internal/treesitter/bridge_ingest.go
@@ -0,0 +1,123 @@
+package treesitter
+
+import (
+ "strings"
+
+ "github.com/HarshK97/diffmantic/internal/treesitter/rules"
+)
+
+// IngestFlatAST converts the flat node buffer into a full ASTNode tree,
+// applying language rules, labels, and subtree hashes.
+func IngestFlatAST(nodes []FlatNode, symbols []string, src []byte, langName string) *ASTNode {
+ if len(nodes) == 0 {
+ return nil
+ }
+
+ r := rules.Get(langName)
+ srcLen := uint32(len(src))
+
+ var errCount int
+ for i := range nodes {
+ fn := &nodes[i]
+ var rawType string
+ if (fn.Flags & FlatNodeError) != 0 {
+ rawType = "ERROR"
+ } else if int(fn.TypeID) < len(symbols) {
+ rawType = symbols[fn.TypeID]
+ }
+ if rawType == "ERROR" || (fn.Flags&FlatNodeError != 0) || (fn.Flags&FlatNodeMissing != 0) {
+ errCount++
+ }
+ }
+
+ root := buildFromIndex(0, nodes, symbols, src, nil, r, srcLen)
+ if root != nil {
+ root.Language = langName
+ root.ParseErrorCount = errCount
+ root.HasError = errCount > 0
+ root.ComputeHashes()
+ EnsureIndex(root)
+ }
+ return root
+}
+
+func buildFromIndex(idx uint32, nodes []FlatNode, symbols []string, src []byte, parent *ASTNode, r *rules.Rules, srcLen uint32) *ASTNode {
+ if int(idx) >= len(nodes) {
+ return nil
+ }
+ fn := &nodes[idx]
+ var rawType string
+ if (fn.Flags & FlatNodeError) != 0 {
+ rawType = "ERROR"
+ } else if int(fn.TypeID) < len(symbols) {
+ rawType = symbols[fn.TypeID]
+ }
+ if (fn.Flags & FlatNodeMissing) != 0 {
+ rawType = "MISSING " + rawType
+ }
+
+ isLeaf := fn.ChildCount == 0 || (r != nil && r.IsFlattened(rawType))
+ var label string
+ if isLeaf {
+ start, end := min(fn.StartByte, srcLen), min(fn.EndByte, srcLen)
+ if start > end {
+ start = end
+ }
+ label = strings.TrimSpace(string(src[start:end]))
+ }
+
+ if r != nil && r.IsIgnored(rawType, label) {
+ return nil
+ }
+
+ node := &ASTNode{
+ Type: rawType,
+ Parent: parent,
+ StartByte: fn.StartByte,
+ EndByte: fn.EndByte,
+ StartRow: fn.StartRow,
+ StartCol: fn.StartCol,
+ EndRow: fn.EndRow,
+ EndCol: fn.EndCol,
+ }
+
+ if isLeaf {
+ node.Label = label
+ }
+
+ if r != nil {
+ if alias, ok := r.Alias(node.Type, label); ok {
+ node.Type = alias
+ }
+ if r.IsLabelIgnored(node.Type) {
+ node.Label = ""
+ }
+ if isLeaf && r.IsKeyword(node.Type, label) {
+ node.IsKeyword = true
+ }
+ if r.IsUnordered(node.Type) {
+ node.IsUnordered = true
+ }
+ }
+
+ childIdx := fn.FirstChildIdx
+ for childIdx != 0xFFFFFFFF && int(childIdx) < len(nodes) {
+ if child := buildFromIndex(childIdx, nodes, symbols, src, node, r, srcLen); child != nil {
+ node.Children = append(node.Children, child)
+ }
+ childIdx = nodes[childIdx].NextSiblingIdx
+ }
+
+ if r != nil && r.IsFlattened(rawType) {
+ var flattenedChildren []*ASTNode
+ for _, child := range node.Children {
+ flattenedChildren = append(flattenedChildren, child.Children...)
+ for _, grandchild := range child.Children {
+ grandchild.Parent = node
+ }
+ }
+ node.Children = flattenedChildren
+ }
+
+ return node
+}
diff --git a/internal/treesitter/bridge_test.go b/internal/treesitter/bridge_test.go
new file mode 100644
index 00000000..ae396750
--- /dev/null
+++ b/internal/treesitter/bridge_test.go
@@ -0,0 +1,125 @@
+package treesitter_test
+
+import (
+ "testing"
+
+ "github.com/HarshK97/diffmantic/internal/treesitter"
+)
+
+func TestAll18NativeGrammarsLoaded(t *testing.T) {
+ langs := []string{
+ "c", "cpp", "go", "rust", "python", "javascript", "typescript", "tsx",
+ "java", "php", "ruby", "lua", "zig", "css", "html", "json", "toml", "yaml",
+ }
+
+ for _, lang := range langs {
+ t.Run(lang, func(t *testing.T) {
+ ptr, err := treesitter.GetNativeLanguage(lang)
+ if err != nil {
+ t.Fatalf("failed to get native language for %s: %v", lang, err)
+ }
+ if ptr == nil {
+ t.Fatalf("nil language pointer for %s", lang)
+ }
+ symbols := treesitter.NativeLanguageSymbols(ptr)
+ if len(symbols) == 0 {
+ t.Fatalf("empty symbols list for %s", lang)
+ }
+ })
+ }
+}
+
+func TestNativeFlatBufferParsing_AllLanguages(t *testing.T) {
+ testCases := []struct {
+ lang string
+ filename string
+ src string
+ }{
+ {lang: "c", filename: "main.c", src: "int main(void) { return 0; }"},
+ {lang: "cpp", filename: "main.cpp", src: "int main() { return 0; }"},
+ {lang: "go", filename: "main.go", src: "package main\nfunc main() {}"},
+ {lang: "rust", filename: "main.rs", src: "fn main() {}"},
+ {lang: "python", filename: "main.py", src: "def main():\n pass\n"},
+ {lang: "javascript", filename: "main.js", src: "function main() {}"},
+ {lang: "typescript", filename: "main.ts", src: "function main(): void {}"},
+ {lang: "tsx", filename: "main.tsx", src: "const App = () =>
Hello
;"},
+ {lang: "java", filename: "Main.java", src: "class Main { public static void main(String[] args) {} }"},
+ {lang: "php", filename: "main.php", src: ""},
+ {lang: "ruby", filename: "main.rb", src: "def main; puts 'hello'; end"},
+ {lang: "lua", filename: "main.lua", src: "function main() print('hello') end"},
+ {lang: "zig", filename: "main.zig", src: "pub fn main() void {}"},
+ {lang: "css", filename: "style.css", src: "body { color: red; }"},
+ {lang: "html", filename: "index.html", src: "Hello"},
+ {lang: "json", filename: "data.json", src: "{\"key\": \"value\", \"count\": 42}"},
+ {lang: "toml", filename: "config.toml", src: "[server]\nport = 8080\n"},
+ {lang: "yaml", filename: "config.yaml", src: "name: diffmantic\nversion: 1.0\n"},
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.lang, func(t *testing.T) {
+ ptr, err := treesitter.GetNativeLanguage(tc.lang)
+ if err != nil {
+ t.Fatalf("failed to get native language for %s: %v", tc.lang, err)
+ }
+
+ symbols := treesitter.NativeLanguageSymbols(ptr)
+ ast, err := treesitter.ParseWithNativeFlatBuffer([]byte(tc.src), ptr, tc.lang, symbols)
+ if err != nil {
+ t.Fatalf("native parse error for %s: %v", tc.lang, err)
+ }
+ if ast == nil {
+ t.Fatalf("nil AST for %s", tc.lang)
+ }
+ if ast.Size() == 0 {
+ t.Fatalf("empty AST for %s", tc.lang)
+ }
+ if ast.HasError {
+ t.Errorf("AST has error for %s", tc.lang)
+ }
+ })
+ }
+}
+
+func TestIngestFlatAST_Synthetic(t *testing.T) {
+ symbols := []string{"source_file", "function_declaration", "identifier", "block", "{", "}"}
+ src := []byte("func foo() {}")
+
+ nodes := []treesitter.FlatNode{
+ {TypeID: 0, Flags: treesitter.FlatNodeNamed, StartByte: 0, EndByte: 13, FirstChildIdx: 1, NextSiblingIdx: 0xFFFFFFFF, ChildCount: 1},
+ {TypeID: 1, Flags: treesitter.FlatNodeNamed, StartByte: 0, EndByte: 13, FirstChildIdx: 2, NextSiblingIdx: 0xFFFFFFFF, ParentIdx: 0, ChildCount: 2},
+ {TypeID: 2, Flags: treesitter.FlatNodeNamed, StartByte: 5, EndByte: 8, FirstChildIdx: 0xFFFFFFFF, NextSiblingIdx: 3, ParentIdx: 1, ChildCount: 0},
+ {TypeID: 3, Flags: treesitter.FlatNodeNamed, StartByte: 11, EndByte: 13, FirstChildIdx: 0xFFFFFFFF, NextSiblingIdx: 0xFFFFFFFF, ParentIdx: 1, ChildCount: 0},
+ }
+
+ root := treesitter.IngestFlatAST(nodes, symbols, src, "go")
+ if root == nil {
+ t.Fatalf("root is nil")
+ }
+ if root.Type != "source_file" {
+ t.Errorf("expected Type='source_file', got %q", root.Type)
+ }
+ if root.Language != "go" {
+ t.Errorf("expected Language='go', got %q", root.Language)
+ }
+ if len(root.Children) != 1 {
+ t.Fatalf("expected 1 child, got %d", len(root.Children))
+ }
+ if root.Children[0].Type != "function_declaration" {
+ t.Errorf("expected function_declaration, got %q", root.Children[0].Type)
+ }
+ if len(root.Children[0].Children) != 2 {
+ t.Fatalf("expected 2 children, got %d", len(root.Children[0].Children))
+ }
+ if root.Children[0].Children[0].Label != "foo" {
+ t.Errorf("expected label 'foo', got %q", root.Children[0].Children[0].Label)
+ }
+ if root.Children[0].Children[0].StartByte != 5 || root.Children[0].Children[0].EndByte != 8 {
+ t.Errorf("expected byte range [5, 8], got [%d, %d]", root.Children[0].Children[0].StartByte, root.Children[0].Children[0].EndByte)
+ }
+ if root.HasError {
+ t.Errorf("expected HasError=false")
+ }
+ if root.ParseErrorCount != 0 {
+ t.Errorf("expected ParseErrorCount=0, got %d", root.ParseErrorCount)
+ }
+}
diff --git a/internal/treesitter/detect.go b/internal/treesitter/detect.go
new file mode 100644
index 00000000..cfd7735f
--- /dev/null
+++ b/internal/treesitter/detect.go
@@ -0,0 +1,106 @@
+package treesitter
+
+import (
+ "fmt"
+ "path/filepath"
+ "strings"
+)
+
+// Language represents a recognized programming or data language.
+type Language struct {
+ Name string
+}
+
+var extToLang = map[string]string{
+ ".go": "go",
+ ".c": "c",
+ ".h": "c",
+ ".cpp": "cpp",
+ ".cc": "cpp",
+ ".cxx": "cpp",
+ ".c++": "cpp",
+ ".hpp": "cpp",
+ ".hh": "cpp",
+ ".hxx": "cpp",
+ ".h++": "cpp",
+ ".rs": "rust",
+ ".py": "python",
+ ".pyi": "python",
+ ".js": "javascript",
+ ".jsx": "javascript",
+ ".mjs": "javascript",
+ ".cjs": "javascript",
+ ".ts": "typescript",
+ ".mts": "typescript",
+ ".cts": "typescript",
+ ".tsx": "tsx",
+ ".java": "java",
+ ".php": "php",
+ ".phtml": "php",
+ ".php3": "php",
+ ".php4": "php",
+ ".php5": "php",
+ ".php7": "php",
+ ".phps": "php",
+ ".rb": "ruby",
+ ".rake": "ruby",
+ ".gemspec": "ruby",
+ ".zig": "zig",
+ ".lua": "lua",
+ ".html": "html",
+ ".htm": "html",
+ ".css": "css",
+ ".json": "json",
+ ".toml": "toml",
+ ".yaml": "yaml",
+ ".yml": "yaml",
+}
+
+var basenameToLang = map[string]string{
+ "rakefile": "ruby",
+ "gemfile": "ruby",
+}
+
+// DetectLanguage detects the language for a given filename or path.
+func DetectLanguage(filename string) (*Language, error) {
+ name, err := DetectLanguageName(filename)
+ if err != nil {
+ return nil, err
+ }
+ return &Language{Name: name}, nil
+}
+
+// DetectLanguageName returns the canonical language name string for a given filename or path.
+func DetectLanguageName(filename string) (string, error) {
+ base := filepath.Base(filename)
+ lowerBase := strings.ToLower(base)
+
+ if lang, ok := basenameToLang[lowerBase]; ok {
+ return lang, nil
+ }
+
+ ext := strings.ToLower(filepath.Ext(base))
+ if ext == "" {
+ return "", fmt.Errorf("unsupported language for file: %s", filename)
+ }
+
+ if lang, ok := extToLang[ext]; ok {
+ return lang, nil
+ }
+
+ // Support compression wrappers (e.g. .json.gz, .go.zst, .yaml.xz)
+ if ext == ".gz" || ext == ".zst" || ext == ".xz" || ext == ".bz2" {
+ trimmed := strings.TrimSuffix(lowerBase, ext)
+ if innerLang, ok := basenameToLang[trimmed]; ok {
+ return innerLang, nil
+ }
+ innerExt := strings.ToLower(filepath.Ext(trimmed))
+ if innerExt != "" {
+ if lang, ok := extToLang[innerExt]; ok {
+ return lang, nil
+ }
+ }
+ }
+
+ return "", fmt.Errorf("unsupported language for file: %s", filename)
+}
diff --git a/internal/treesitter/detect_test.go b/internal/treesitter/detect_test.go
new file mode 100644
index 00000000..e9ce6abf
--- /dev/null
+++ b/internal/treesitter/detect_test.go
@@ -0,0 +1,51 @@
+package treesitter_test
+
+import (
+ "testing"
+
+ "github.com/HarshK97/diffmantic/internal/treesitter"
+)
+
+func TestDetectLanguageName(t *testing.T) {
+ tests := []struct {
+ filename string
+ want string
+ wantErr bool
+ }{
+ {"main.go", "go", false},
+ {"app.js", "javascript", false},
+ {"index.ts", "typescript", false},
+ {"component.tsx", "tsx", false},
+ {"data.json", "json", false},
+ {"config.yaml", "yaml", false},
+ {"config.yml", "yaml", false},
+ {"Cargo.toml", "toml", false},
+ {"main.rs", "rust", false},
+ {"script.py", "python", false},
+ {"test.cpp", "cpp", false},
+ {"header.h", "c", false},
+ {"Rakefile", "ruby", false},
+ {"Gemfile", "ruby", false},
+ // Compound compression wrappers
+ {"expected_ui.json.gz", "json", false},
+ {"archive.go.zst", "go", false},
+ {"manifest.yaml.xz", "yaml", false},
+ {"Gemfile.gz", "ruby", false},
+ // Unsupported
+ {"file.unknown", "", true},
+ {"archive.tar.gz", "", true},
+ {"no_ext", "", true},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.filename, func(t *testing.T) {
+ got, err := treesitter.DetectLanguageName(tt.filename)
+ if (err != nil) != tt.wantErr {
+ t.Fatalf("DetectLanguageName(%q) error = %v, wantErr = %v", tt.filename, err, tt.wantErr)
+ }
+ if got != tt.want {
+ t.Errorf("DetectLanguageName(%q) = %q, want %q", tt.filename, got, tt.want)
+ }
+ })
+ }
+}
diff --git a/internal/treesitter/flat_node.go b/internal/treesitter/flat_node.go
new file mode 100644
index 00000000..8b4a82a5
--- /dev/null
+++ b/internal/treesitter/flat_node.go
@@ -0,0 +1,23 @@
+package treesitter
+
+// FlatNode defines the 44-byte cache-aligned struct matching the C bridge layout.
+type FlatNode struct {
+ TypeID uint16
+ Flags uint16
+ StartByte uint32
+ EndByte uint32
+ StartRow uint32
+ StartCol uint32
+ EndRow uint32
+ EndCol uint32
+ ParentIdx uint32
+ FirstChildIdx uint32
+ NextSiblingIdx uint32
+ ChildCount uint32
+}
+
+const (
+ FlatNodeNamed = 1 << 0
+ FlatNodeError = 1 << 1
+ FlatNodeMissing = 1 << 2
+)
diff --git a/internal/treesitter/parser.go b/internal/treesitter/parser.go
index bd5d9e84..02dcc3a0 100644
--- a/internal/treesitter/parser.go
+++ b/internal/treesitter/parser.go
@@ -1,45 +1,10 @@
package treesitter
-import (
- "errors"
- "fmt"
- "path/filepath"
-
- "github.com/odvcencio/gotreesitter"
- "github.com/odvcencio/gotreesitter/grammars"
-)
-
-func DetectGrammarEntry(filename string) *grammars.LangEntry {
- base := filepath.Base(filename)
- return grammars.DetectLanguage(base)
-}
-
-func DetectLanguage(filename string) (*gotreesitter.Language, error) {
- entry := DetectGrammarEntry(filename)
- if entry == nil {
- return nil, fmt.Errorf("unsupported language for file: %s", filename)
- }
- return entry.Language(), nil
-}
-
-// ParseWithLanguageAndTree parses source bytes and returns both the AST and raw tree-sitter tree.
-func ParseWithLanguageAndTree(src []byte, lang *gotreesitter.Language) (*ASTNode, *gotreesitter.Tree, error) {
- if lang == nil {
- return nil, nil, errors.New("nil language")
- }
- parser := gotreesitter.NewParser(lang)
- tree, err := parser.Parse(src)
- if err != nil {
- return nil, nil, err
- }
- return BuildAST(tree.RootNode(), src, lang, nil), tree, nil
-}
-
+// Parse detects the language for the given filename and parses source bytes into an ASTNode.
func Parse(src []byte, filename string) (*ASTNode, error) {
lang, err := DetectLanguage(filename)
if err != nil {
return nil, err
}
- ast, _, err := ParseWithLanguageAndTree(src, lang)
- return ast, err
+ return ParseWithLanguage(src, lang.Name)
}
diff --git a/internal/treesitter/parser_test.go b/internal/treesitter/parser_test.go
index 5600469e..69ad67ef 100644
--- a/internal/treesitter/parser_test.go
+++ b/internal/treesitter/parser_test.go
@@ -23,19 +23,20 @@ func TestDetectLanguage(t *testing.T) {
{"test.cxx", false},
{"test.hh", false},
{"test.h", false},
- {"test.cs", false},
+ {"test.cs", true},
{"test.java", false},
{"test.php", false},
{"test.rb", false},
{"test.json", false},
+ {"test.toml", false},
{"test.yaml", false},
{"test.yml", false},
{"test.html", false},
{"test.css", false},
{"test.lua", false},
{"test.zig", false},
- {"test.sh", false},
- {"test.bash", false},
+ {"test.sh", true},
+ {"test.bash", true},
{"test.xyz", true},
{"test", true},
}
diff --git a/internal/treesitter/rules/c.go b/internal/treesitter/rules/c.go
index bef2121b..591e8e71 100644
--- a/internal/treesitter/rules/c.go
+++ b/internal/treesitter/rules/c.go
@@ -1,6 +1,7 @@
package rules
var cRules = &Rules{
+ Kind: KindCode,
Flattened: []string{
"string_literal",
"char_literal",
@@ -155,17 +156,18 @@ var cRules = &Rules{
},
Wrappers: []string{
"parenthesized_expression",
+ "argument_list",
+ "parameter_list",
"cast_expression",
"pointer_expression",
"subscript_expression",
- "argument_list",
- "parameter_list",
},
Pairs: []string{
"field_designator",
"initializer_pair",
},
EquivalentTypes: [][]string{
+ {"function_definition", "declaration"},
{"struct_specifier", "union_specifier", "enum_specifier"},
{"for_statement", "while_statement", "do_statement"},
},
@@ -175,4 +177,18 @@ var cRules = &Rules{
Calls: []string{
"call_expression",
},
+ Indexed: []string{
+ "subscript_expression",
+ },
+ ContainerDeclarations: []string{
+ "function_definition",
+ "type_definition",
+ "struct_specifier",
+ "union_specifier",
+ "enum_specifier",
+ },
+ Types: []string{
+ "primitive_type",
+ "type_identifier",
+ },
}
diff --git a/internal/treesitter/rules/cpp.go b/internal/treesitter/rules/cpp.go
index 78535a47..b5e64984 100644
--- a/internal/treesitter/rules/cpp.go
+++ b/internal/treesitter/rules/cpp.go
@@ -1,6 +1,7 @@
package rules
var cppRules = &Rules{
+ Kind: KindCode,
Flattened: []string{
"string_literal",
"char_literal",
@@ -105,6 +106,7 @@ var cppRules = &Rules{
"declaration",
"init_declarator",
"field_declaration",
+ "friend_declaration",
"parameter_declaration",
"alias_declaration",
"using_declaration",
@@ -199,6 +201,7 @@ var cppRules = &Rules{
"concept_definition",
"parameter_declaration",
"field_declaration",
+ "friend_declaration",
},
Identifiers: []string{
"identifier",
@@ -210,20 +213,22 @@ var cppRules = &Rules{
},
Wrappers: []string{
"parenthesized_expression",
+ "argument_list",
+ "parameter_list",
"template_type",
"template_argument_list",
"subscript_expression",
"pointer_expression",
"cast_expression",
- "argument_list",
- "parameter_list",
+ "friend_declaration",
+ "template_declaration",
},
Pairs: []string{
"field_designator",
"initializer_pair",
},
EquivalentTypes: [][]string{
- {"function_definition", "template_declaration"},
+ {"function_definition", "declaration"},
{"class_specifier", "struct_specifier", "union_specifier", "enum_specifier"},
{"for_statement", "for_range_loop", "while_statement", "do_statement"},
},
@@ -233,4 +238,27 @@ var cppRules = &Rules{
Calls: []string{
"call_expression",
},
+ Indexed: []string{
+ "subscript_expression",
+ },
+ ContainerDeclarations: []string{
+ "function_definition",
+ "type_definition",
+ "class_specifier",
+ "struct_specifier",
+ "union_specifier",
+ "enum_specifier",
+ "namespace_definition",
+ "template_declaration",
+ },
+ Closures: []string{
+ "lambda_expression",
+ },
+ Types: []string{
+ "dependent_type",
+ "primitive_type",
+ "template_type",
+ "trailing_return_type",
+ "type_identifier",
+ },
}
diff --git a/internal/treesitter/rules/css.go b/internal/treesitter/rules/css.go
index 1ceb8ef5..234cd4eb 100644
--- a/internal/treesitter/rules/css.go
+++ b/internal/treesitter/rules/css.go
@@ -1,6 +1,7 @@
package rules
var cssRules = &Rules{
+ Kind: KindMarkup,
Flattened: []string{
"plain_value",
"color_value",
diff --git a/internal/treesitter/rules/go.go b/internal/treesitter/rules/go.go
index 29aa4fd3..7ac78819 100644
--- a/internal/treesitter/rules/go.go
+++ b/internal/treesitter/rules/go.go
@@ -1,6 +1,7 @@
package rules
var golangRules = &Rules{
+ Kind: KindCode,
Flattened: []string{
"interpreted_string_literal",
"raw_string_literal",
@@ -65,7 +66,6 @@ var golangRules = &Rules{
"parenthesized_expression",
"expression_list",
"literal_value",
- "statement_list",
"import_spec_list",
"function_declaration",
"method_declaration",
@@ -128,6 +128,7 @@ var golangRules = &Rules{
Wrappers: []string{
"parenthesized_expression",
"parenthesized_type",
+ "argument_list",
"type_arguments",
"type_parameter_list",
"slice_type",
@@ -135,8 +136,8 @@ var golangRules = &Rules{
"index_expression",
"slice_expression",
"pointer_type",
- "argument_list",
- "parameter_list",
+ "expression_list",
+ "var_declaration",
},
Pairs: []string{
"keyed_element",
@@ -146,7 +147,7 @@ var golangRules = &Rules{
},
EquivalentTypes: [][]string{
{"function_declaration", "method_declaration"},
- {"var_declaration", "short_var_declaration"},
+ {"short_var_declaration", "assignment_statement", "var_spec"},
},
Comments: []string{
"comment",
@@ -157,4 +158,38 @@ var golangRules = &Rules{
ScopedDeclarations: []string{
"method_declaration",
},
+ Indexed: []string{
+ "index_expression",
+ "slice_expression",
+ },
+ LocalVarDeclarations: []string{
+ "var_declaration",
+ "const_declaration",
+ "short_var_declaration",
+ },
+ ContainerDeclarations: []string{
+ "function_declaration",
+ "method_declaration",
+ "type_declaration",
+ "type_spec",
+ },
+ Closures: []string{
+ "func_literal",
+ },
+ Types: []string{
+ "array_type",
+ "channel_type",
+ "function_type",
+ "generic_type",
+ "implicit_length_array_type",
+ "interface_type",
+ "map_type",
+ "negated_type",
+ "parenthesized_type",
+ "pointer_type",
+ "qualified_type",
+ "slice_type",
+ "struct_type",
+ "type_identifier",
+ },
}
diff --git a/internal/treesitter/rules/html.go b/internal/treesitter/rules/html.go
index 77654961..c8b00c28 100644
--- a/internal/treesitter/rules/html.go
+++ b/internal/treesitter/rules/html.go
@@ -1,6 +1,7 @@
package rules
var htmlRules = &Rules{
+ Kind: KindMarkup,
Flattened: []string{
"text",
"raw_text",
diff --git a/internal/treesitter/rules/java.go b/internal/treesitter/rules/java.go
index 3204c00d..a3380ab0 100644
--- a/internal/treesitter/rules/java.go
+++ b/internal/treesitter/rules/java.go
@@ -1,6 +1,7 @@
package rules
var javaRules = &Rules{
+ Kind: KindCode,
Flattened: []string{
"string_literal",
"character_literal",
@@ -182,13 +183,13 @@ var javaRules = &Rules{
},
Wrappers: []string{
"parenthesized_expression",
+ "argument_list",
+ "formal_parameters",
"type_arguments",
"type_parameters",
"array_type",
"generic_type",
"cast_expression",
- "argument_list",
- "formal_parameters",
},
Pairs: []string{
"element_value_pair",
@@ -209,4 +210,33 @@ var javaRules = &Rules{
"method_invocation",
"explicit_constructor_invocation",
},
+ Indexed: []string{
+ "array_access",
+ },
+ LocalVarDeclarations: []string{
+ "local_variable_declaration",
+ },
+ ContainerDeclarations: []string{
+ "class_declaration",
+ "interface_declaration",
+ "enum_declaration",
+ "record_declaration",
+ "method_declaration",
+ "constructor_declaration",
+ },
+ Closures: []string{
+ "lambda_expression",
+ },
+ Types: []string{
+ "annotated_type",
+ "array_type",
+ "boolean_type",
+ "catch_type",
+ "floating_point_type",
+ "generic_type",
+ "integral_type",
+ "scoped_type_identifier",
+ "type_identifier",
+ "void_type",
+ },
}
diff --git a/internal/treesitter/rules/javascript.go b/internal/treesitter/rules/javascript.go
index a1f7c586..1f4932e4 100644
--- a/internal/treesitter/rules/javascript.go
+++ b/internal/treesitter/rules/javascript.go
@@ -1,6 +1,7 @@
package rules
var javascriptRules = &Rules{
+ Kind: KindCode,
Flattened: []string{
"string",
"template_string",
@@ -80,7 +81,6 @@ var javascriptRules = &Rules{
"expression_statement",
"variable_declaration",
"lexical_declaration",
- "using_declaration",
"variable_declarator",
"function_declaration",
"function_expression",
@@ -172,8 +172,9 @@ var javascriptRules = &Rules{
"statement_block",
},
Wrappers: []string{
+ "export_statement",
"parenthesized_expression",
- "subscript_expression",
+ "object",
"array",
"array_pattern",
"object_pattern",
@@ -187,6 +188,8 @@ var javascriptRules = &Rules{
Unordered: []string{
"object",
"object_pattern",
+ "named_imports",
+ "export_clause",
},
EquivalentTypes: [][]string{
{"function_declaration", "function_expression", "arrow_function", "generator_function_declaration", "generator_function"},
@@ -200,4 +203,22 @@ var javascriptRules = &Rules{
Calls: []string{
"call_expression",
},
+ Indexed: []string{
+ "subscript_expression",
+ },
+ LocalVarDeclarations: []string{
+ "variable_declaration",
+ "lexical_declaration",
+ },
+ ContainerDeclarations: []string{
+ "function_declaration",
+ "generator_function_declaration",
+ "method_definition",
+ "class_declaration",
+ },
+ Closures: []string{
+ "arrow_function",
+ "function_expression",
+ "function",
+ },
}
diff --git a/internal/treesitter/rules/json.go b/internal/treesitter/rules/json.go
index 160f0433..16f5b6a1 100644
--- a/internal/treesitter/rules/json.go
+++ b/internal/treesitter/rules/json.go
@@ -1,6 +1,7 @@
package rules
var jsonRules = &Rules{
+ Kind: KindData,
Flattened: []string{
"string",
},
@@ -26,6 +27,10 @@ var jsonRules = &Rules{
Pairs: []string{
"pair",
},
+ Wrappers: []string{
+ "array",
+ "object",
+ },
Unordered: []string{
"object",
},
diff --git a/internal/treesitter/rules/lua.go b/internal/treesitter/rules/lua.go
index 1e759c04..c09209e7 100644
--- a/internal/treesitter/rules/lua.go
+++ b/internal/treesitter/rules/lua.go
@@ -1,6 +1,7 @@
package rules
var luaRules = &Rules{
+ Kind: KindCode,
Flattened: []string{
"string",
},
@@ -110,7 +111,6 @@ var luaRules = &Rules{
Wrappers: []string{
"parenthesized_expression",
"table_constructor",
- "bracket_index_expression",
"dot_index_expression",
"method_index_expression",
"arguments",
@@ -131,4 +131,16 @@ var luaRules = &Rules{
Calls: []string{
"function_call",
},
+ Indexed: []string{
+ "bracket_index_expression",
+ },
+ LocalVarDeclarations: []string{
+ "variable_declaration",
+ },
+ ContainerDeclarations: []string{
+ "function_declaration",
+ },
+ Closures: []string{
+ "function_definition",
+ },
}
diff --git a/internal/treesitter/rules/php.go b/internal/treesitter/rules/php.go
index c331e364..3572e3cd 100644
--- a/internal/treesitter/rules/php.go
+++ b/internal/treesitter/rules/php.go
@@ -1,6 +1,7 @@
package rules
var phpRules = &Rules{
+ Kind: KindCode,
Flattened: []string{
"string",
"encapsed_string",
@@ -200,4 +201,32 @@ var phpRules = &Rules{
"scoped_call_expression",
"nullsafe_member_call_expression",
},
+ Indexed: []string{
+ "subscript_expression",
+ },
+ LocalVarDeclarations: []string{
+ "const_declaration",
+ },
+ ContainerDeclarations: []string{
+ "function_definition",
+ "method_declaration",
+ "class_declaration",
+ "interface_declaration",
+ "trait_declaration",
+ "enum_declaration",
+ },
+ Closures: []string{
+ "anonymous_function",
+ "arrow_function",
+ },
+ Types: []string{
+ "bottom_type",
+ "cast_type",
+ "disjunctive_normal_form_type",
+ "intersection_type",
+ "named_type",
+ "optional_type",
+ "primitive_type",
+ "union_type",
+ },
}
diff --git a/internal/treesitter/rules/python.go b/internal/treesitter/rules/python.go
index b494667d..d97e6d26 100644
--- a/internal/treesitter/rules/python.go
+++ b/internal/treesitter/rules/python.go
@@ -1,6 +1,7 @@
package rules
var pythonRules = &Rules{
+ Kind: KindCode,
Flattened: []string{
"string",
"concatenated_string",
@@ -80,7 +81,6 @@ var pythonRules = &Rules{
"expression_statement",
"expression_list",
"pattern_list",
- "tuple_expression",
"list",
"dictionary",
"set",
@@ -157,7 +157,8 @@ var pythonRules = &Rules{
},
Wrappers: []string{
"parenthesized_expression",
- "subscript",
+ "argument_list",
+ "parameters",
"list",
"tuple",
"set",
@@ -166,8 +167,6 @@ var pythonRules = &Rules{
"list_comprehension",
"dictionary_comprehension",
"set_comprehension",
- "argument_list",
- "parameters",
},
Pairs: []string{
"pair",
@@ -187,4 +186,21 @@ var pythonRules = &Rules{
Calls: []string{
"call",
},
+ Indexed: []string{
+ "subscript",
+ },
+ ContainerDeclarations: []string{
+ "function_definition",
+ "class_definition",
+ },
+ Closures: []string{
+ "lambda",
+ },
+ Types: []string{
+ "constrained_type",
+ "generic_type",
+ "member_type",
+ "splat_type",
+ "union_type",
+ },
}
diff --git a/internal/treesitter/rules/ruby.go b/internal/treesitter/rules/ruby.go
index 8cca0a4e..219bb24a 100644
--- a/internal/treesitter/rules/ruby.go
+++ b/internal/treesitter/rules/ruby.go
@@ -1,6 +1,7 @@
package rules
var rubyRules = &Rules{
+ Kind: KindCode,
Flattened: []string{
"string",
"string_content",
@@ -154,14 +155,18 @@ var rubyRules = &Rules{
"constant",
},
Blocks: []string{
+ "body_statement",
+ "do_block",
"block",
},
Wrappers: []string{
"parenthesized_statements",
+ "argument_list",
"array",
"element_reference",
- "argument_list",
- "method_parameters",
+ "do_block",
+ "hash",
+ "hash_pattern",
},
Pairs: []string{
"pair",
@@ -173,7 +178,7 @@ var rubyRules = &Rules{
EquivalentTypes: [][]string{
{"method", "singleton_method"},
{"class", "module", "singleton_class"},
- {"if", "unless"},
+ {"if", "unless", "if_modifier", "unless_modifier"},
{"while", "until", "for"},
},
Comments: []string{
@@ -182,4 +187,18 @@ var rubyRules = &Rules{
Calls: []string{
"call",
},
+ Indexed: []string{
+ "element_reference",
+ },
+ ContainerDeclarations: []string{
+ "method",
+ "singleton_method",
+ "class",
+ "module",
+ },
+ Closures: []string{
+ "do_block",
+ "block",
+ "lambda",
+ },
}
diff --git a/internal/treesitter/rules/rules.go b/internal/treesitter/rules/rules.go
index 4801e329..23f66082 100644
--- a/internal/treesitter/rules/rules.go
+++ b/internal/treesitter/rules/rules.go
@@ -5,39 +5,63 @@ import (
"strings"
)
+// LanguageKind classifies grammars into high-level parsing and diffing categories.
+type LanguageKind uint8
+
+const (
+ // KindCode is for general programming languages (Go, Rust, Python, C++, etc.).
+ KindCode LanguageKind = iota
+ // KindData is for structured key-value formats (JSON, YAML, TOML).
+ KindData
+ // KindMarkup is for markup and styling languages (HTML, CSS).
+ KindMarkup
+)
+
// Rules configures language-specific AST transformations and node matching.
type Rules struct {
- Flattened []string
- Ignored []string
- Aliased map[string]string
- LabelIgnored []string
- Scaffolding []string
- Keywords []string
- Declarations []string
- Identifiers []string
- Blocks []string
- Wrappers []string
- Pairs []string
- Unordered []string
- EquivalentTypes [][]string
- Comments []string
- Calls []string
- ScopedDeclarations []string
-
- flattenedSet map[string]struct{}
- ignoredSet map[string]struct{}
- labelIgnoredSet map[string]struct{}
- keywordsSet map[string]struct{}
- declarationsSet map[string]struct{}
- identifiersSet map[string]struct{}
- scaffoldingSet map[string]struct{}
- blocksSet map[string]struct{}
- wrappersSet map[string]struct{}
- unorderedSet map[string]struct{}
- commentsSet map[string]struct{}
- callsSet map[string]struct{}
- scopedDeclarationsSet map[string]struct{}
- equivGroups map[string][]int
+ Kind LanguageKind
+ Flattened []string
+ Ignored []string
+ Aliased map[string]string
+ LabelIgnored []string
+ Scaffolding []string
+ Keywords []string
+ Declarations []string
+ Identifiers []string
+ Blocks []string
+ Wrappers []string
+ Pairs []string
+ Unordered []string
+ EquivalentTypes [][]string
+ Comments []string
+ Calls []string
+ ScopedDeclarations []string
+ Indexed []string // Subscript nodes with prefix receivers (e.g. arr[i]).
+ LocalVarDeclarations []string
+ ContainerDeclarations []string // Major declaration scope boundaries (functions, classes, structs, etc.).
+ Closures []string // Anonymous functions, lambdas, and callbacks.
+ Types []string // Type annotations and type expressions.
+
+ flattenedSet map[string]struct{}
+ ignoredSet map[string]struct{}
+ labelIgnoredSet map[string]struct{}
+ keywordsSet map[string]struct{}
+ declarationsSet map[string]struct{}
+ identifiersSet map[string]struct{}
+ scaffoldingSet map[string]struct{}
+ blocksSet map[string]struct{}
+ wrappersSet map[string]struct{}
+ pairsSet map[string]struct{}
+ unorderedSet map[string]struct{}
+ commentsSet map[string]struct{}
+ callsSet map[string]struct{}
+ scopedDeclarationsSet map[string]struct{}
+ indexedSet map[string]struct{}
+ localVarDeclarationsSet map[string]struct{}
+ containerDeclarationsSet map[string]struct{}
+ closuresSet map[string]struct{}
+ typesSet map[string]struct{}
+ equivGroups map[string][]int
}
// CompileSets builds the internal lookup sets for fast querying.
@@ -96,6 +120,12 @@ func (r *Rules) CompileSets() {
r.wrappersSet[s] = struct{}{}
}
}
+ if len(r.Pairs) > 0 {
+ r.pairsSet = make(map[string]struct{}, len(r.Pairs))
+ for _, s := range r.Pairs {
+ r.pairsSet[s] = struct{}{}
+ }
+ }
if len(r.Unordered) > 0 {
r.unorderedSet = make(map[string]struct{}, len(r.Unordered))
for _, s := range r.Unordered {
@@ -120,6 +150,36 @@ func (r *Rules) CompileSets() {
r.scopedDeclarationsSet[s] = struct{}{}
}
}
+ if len(r.Indexed) > 0 {
+ r.indexedSet = make(map[string]struct{}, len(r.Indexed))
+ for _, s := range r.Indexed {
+ r.indexedSet[s] = struct{}{}
+ }
+ }
+ if len(r.LocalVarDeclarations) > 0 {
+ r.localVarDeclarationsSet = make(map[string]struct{}, len(r.LocalVarDeclarations))
+ for _, s := range r.LocalVarDeclarations {
+ r.localVarDeclarationsSet[s] = struct{}{}
+ }
+ }
+ if len(r.ContainerDeclarations) > 0 {
+ r.containerDeclarationsSet = make(map[string]struct{}, len(r.ContainerDeclarations))
+ for _, s := range r.ContainerDeclarations {
+ r.containerDeclarationsSet[s] = struct{}{}
+ }
+ }
+ if len(r.Closures) > 0 {
+ r.closuresSet = make(map[string]struct{}, len(r.Closures))
+ for _, s := range r.Closures {
+ r.closuresSet[s] = struct{}{}
+ }
+ }
+ if len(r.Types) > 0 {
+ r.typesSet = make(map[string]struct{}, len(r.Types))
+ for _, s := range r.Types {
+ r.typesSet[s] = struct{}{}
+ }
+ }
if len(r.EquivalentTypes) > 0 {
r.equivGroups = make(map[string][]int)
for idx, group := range r.EquivalentTypes {
@@ -130,6 +190,14 @@ func (r *Rules) CompileSets() {
}
}
+// GetKind returns the language category (KindCode, KindData, or KindMarkup).
+func (r *Rules) GetKind() LanguageKind {
+ if r == nil {
+ return KindCode
+ }
+ return r.Kind
+}
+
// IsCall reports whether nodeType is a function, method, or macro invocation.
func (r *Rules) IsCall(nodeType string) bool {
if r == nil || nodeType == "" {
@@ -142,6 +210,18 @@ func (r *Rules) IsCall(nodeType string) bool {
return slices.Contains(r.Calls, nodeType)
}
+// IsIndexed reports whether nodeType is a subscript container with a prefix receiver.
+func (r *Rules) IsIndexed(nodeType string) bool {
+ if r == nil || nodeType == "" {
+ return false
+ }
+ if len(r.indexedSet) > 0 {
+ _, ok := r.indexedSet[nodeType]
+ return ok
+ }
+ return slices.Contains(r.Indexed, nodeType)
+}
+
// IsComment reports whether nodeType is a comment in the language grammar.
func (r *Rules) IsComment(nodeType string) bool {
if r == nil || nodeType == "" {
@@ -178,6 +258,55 @@ func (r *Rules) IsScopedDeclaration(nodeType string) bool {
return slices.Contains(r.ScopedDeclarations, nodeType)
}
+// IsLocalVarDeclaration reports whether nodeType is a local variable declaration.
+func (r *Rules) IsLocalVarDeclaration(nodeType string) bool {
+ if r == nil || nodeType == "" {
+ return false
+ }
+ if len(r.localVarDeclarationsSet) > 0 {
+ _, ok := r.localVarDeclarationsSet[nodeType]
+ return ok
+ }
+ return slices.Contains(r.LocalVarDeclarations, nodeType)
+}
+
+// IsContainerDeclaration reports whether nodeType is a major container declaration
+// (such as a function, method, class, struct, interface, trait, or enum).
+func (r *Rules) IsContainerDeclaration(nodeType string) bool {
+ if r == nil || nodeType == "" {
+ return false
+ }
+ if len(r.containerDeclarationsSet) > 0 {
+ _, ok := r.containerDeclarationsSet[nodeType]
+ return ok
+ }
+ return slices.Contains(r.ContainerDeclarations, nodeType)
+}
+
+// IsClosure reports whether nodeType is an anonymous function, closure, or lambda callback.
+func (r *Rules) IsClosure(nodeType string) bool {
+ if r == nil || nodeType == "" {
+ return false
+ }
+ if len(r.closuresSet) > 0 {
+ _, ok := r.closuresSet[nodeType]
+ return ok
+ }
+ return slices.Contains(r.Closures, nodeType)
+}
+
+// IsType reports whether nodeType is a type annotation or type expression in the language.
+func (r *Rules) IsType(nodeType string) bool {
+ if r == nil || nodeType == "" {
+ return false
+ }
+ if len(r.typesSet) > 0 {
+ _, ok := r.typesSet[nodeType]
+ return ok
+ }
+ return slices.Contains(r.Types, nodeType)
+}
+
// IsIdentifier reports whether nodeType is an identifier token.
func (r *Rules) IsIdentifier(nodeType string) bool {
if r == nil || nodeType == "" {
@@ -281,6 +410,18 @@ func (r *Rules) IsLabelIgnored(nodeType string) bool {
return slices.Contains(r.LabelIgnored, nodeType)
}
+// IsPair checks if nodeType represents a key-value property pair.
+func (r *Rules) IsPair(nodeType string) bool {
+ if r == nil {
+ return false
+ }
+ if len(r.pairsSet) > 0 {
+ _, ok := r.pairsSet[nodeType]
+ return ok
+ }
+ return slices.Contains(r.Pairs, nodeType)
+}
+
// IsUnordered checks if child order doesn't matter for this container.
func (r *Rules) IsUnordered(nodeType string) bool {
if r == nil {
@@ -412,6 +553,16 @@ func IsDeclaration(nodeType string) bool {
return false
}
+// IsLocalVarDeclaration reports whether nodeType is configured as a local variable declaration in any language rule set.
+func IsLocalVarDeclaration(nodeType string) bool {
+ for _, r := range registry {
+ if r.IsLocalVarDeclaration(nodeType) {
+ return true
+ }
+ }
+ return false
+}
+
// IsIdentifier reports whether nodeType is configured as an identifier in any language rule set.
func IsIdentifier(nodeType string) bool {
for _, r := range registry {
@@ -467,6 +618,32 @@ func IsKeyword(nodeType, label string) bool {
return false
}
+// IsPunctuation reports whether a string is a structural punctuation token (braces, brackets, parentheses, delimiters).
+func (r *Rules) IsPunctuation(token string) bool {
+ switch token {
+ case "}", "};", "],", "]", ")", ");", "},", "{", "begin", "end", ";", ",", "(", "[", ":", "->", "=>", "\"", "'", "`":
+ return true
+ }
+ if r == nil || token == "" {
+ return false
+ }
+ return r.IsIgnored(token, token) || r.IsDelimiter(token, token)
+}
+
+// IsPunctuation reports whether a string is a structural punctuation token in any language rule set.
+func IsPunctuation(token string) bool {
+ switch token {
+ case "}", "};", "],", "]", ")", ");", "},", "{", "begin", "end", ";", ",", "(", "[", ":", "->", "=>", "\"", "'", "`":
+ return true
+ }
+ for _, r := range registry {
+ if r.IsPunctuation(token) {
+ return true
+ }
+ }
+ return false
+}
+
// IsDelimiter reports whether nodeType or label is a delimiter token (semicolon or comma).
func IsDelimiter(nodeType, label string) bool {
return label == ";" || label == "," || nodeType == "semicolon" || nodeType == "comma" || nodeType == "_automatic_semicolon"
@@ -482,6 +659,55 @@ func IsCall(nodeType string) bool {
return false
}
+// IsIndexed reports whether nodeType is configured as an indexed container in any language rule set.
+func IsIndexed(nodeType string) bool {
+ for _, r := range registry {
+ if r.IsIndexed(nodeType) {
+ return true
+ }
+ }
+ return false
+}
+
+// IsContainerDeclaration reports whether nodeType is configured as a container declaration in any language rule set.
+func IsContainerDeclaration(nodeType string) bool {
+ if nodeType == "" {
+ return false
+ }
+ for _, r := range registry {
+ if r.IsContainerDeclaration(nodeType) {
+ return true
+ }
+ }
+ return false
+}
+
+// IsClosure reports whether nodeType is configured as a closure in any language rule set.
+func IsClosure(nodeType string) bool {
+ if nodeType == "" {
+ return false
+ }
+ for _, r := range registry {
+ if r.IsClosure(nodeType) {
+ return true
+ }
+ }
+ return false
+}
+
+// IsType reports whether nodeType is a type annotation or type expression in any language rule set.
+func IsType(nodeType string) bool {
+ if nodeType == "" {
+ return false
+ }
+ for _, r := range registry {
+ if r.IsType(nodeType) {
+ return true
+ }
+ }
+ return false
+}
+
var registry = map[string]*Rules{
"c": cRules,
"cpp": cppRules,
diff --git a/internal/treesitter/rules/rules_grammar_test.go b/internal/treesitter/rules/rules_grammar_test.go
index 1fe0c06e..432b12f0 100644
--- a/internal/treesitter/rules/rules_grammar_test.go
+++ b/internal/treesitter/rules/rules_grammar_test.go
@@ -3,72 +3,73 @@ package rules_test
import (
"testing"
+ "github.com/HarshK97/diffmantic/internal/treesitter"
"github.com/HarshK97/diffmantic/internal/treesitter/rules"
- "github.com/odvcencio/gotreesitter/grammars"
)
func TestAllLanguageRulesMatchGrammarSymbols(t *testing.T) {
langs := []struct {
- name string
- filename string
+ name string
}{
- {"c", "main.c"},
- {"cpp", "main.cpp"},
- {"go", "main.go"},
- {"rust", "main.rs"},
- {"python", "main.py"},
- {"javascript", "main.js"},
- {"typescript", "main.ts"},
- {"tsx", "main.tsx"},
- {"java", "Main.java"},
- {"php", "main.php"},
- {"ruby", "main.rb"},
- {"lua", "main.lua"},
- {"zig", "main.zig"},
- {"css", "style.css"},
- {"html", "index.html"},
- {"json", "data.json"},
- {"toml", "config.toml"},
- {"yaml", "config.yaml"},
+ {"c"},
+ {"cpp"},
+ {"go"},
+ {"rust"},
+ {"python"},
+ {"javascript"},
+ {"typescript"},
+ {"tsx"},
+ {"java"},
+ {"php"},
+ {"ruby"},
+ {"lua"},
+ {"zig"},
+ {"css"},
+ {"html"},
+ {"json"},
+ {"toml"},
+ {"yaml"},
}
for _, l := range langs {
- t.Run(l.name, func(t *testing.T) {
- entry := grammars.DetectLanguage(l.filename)
- if entry == nil {
- t.Fatalf("Failed to detect language grammar for %s", l.name)
- }
- lang := entry.Language()
+ ptr, err := treesitter.GetNativeLanguage(l.name)
+ if err != nil || ptr == nil {
+ t.Fatalf("Failed to get native language for %s: %v", l.name, err)
+ }
- grammarSymbols := make(map[string]bool)
- for i, meta := range lang.SymbolMetadata {
- name := ""
- if i < len(lang.SymbolNames) {
- name = lang.SymbolNames[i]
- }
- if meta.Named && name != "" {
- grammarSymbols[name] = true
- }
+ symbols := treesitter.NativeLanguageSymbols(ptr)
+ grammarSymbols := make(map[string]bool, len(symbols))
+ for _, s := range symbols {
+ if s != "" {
+ grammarSymbols[s] = true
}
+ }
- r := rules.Get(l.name)
- if r == nil {
- return
- }
+ r := rules.Get(l.name)
+ if r == nil {
+ continue
+ }
- checkField := func(fieldName string, items []string) {
- for _, item := range items {
- if !grammarSymbols[item] {
- t.Errorf("[%s] %s item %q does NOT exist in gotreesitter grammar", l.name, fieldName, item)
- }
+ checkField := func(fieldName string, items []string) {
+ for _, item := range items {
+ if !grammarSymbols[item] {
+ t.Errorf("[%s] %s item %q does NOT exist in native grammar", l.name, fieldName, item)
}
}
+ }
- checkField("Declarations", r.Declarations)
- checkField("Blocks", r.Blocks)
- checkField("Scaffolding", r.Scaffolding)
- checkField("Wrappers", r.Wrappers)
- checkField("Pairs", r.Pairs)
- })
+ checkField("Declarations", r.Declarations)
+ checkField("Blocks", r.Blocks)
+ checkField("Scaffolding", r.Scaffolding)
+ checkField("Wrappers", r.Wrappers)
+ checkField("Pairs", r.Pairs)
+ checkField("Indexed", r.Indexed)
+ checkField("Unordered", r.Unordered)
+ checkField("Comments", r.Comments)
+ checkField("Calls", r.Calls)
+ checkField("LocalVarDeclarations", r.LocalVarDeclarations)
+ checkField("ContainerDeclarations", r.ContainerDeclarations)
+ checkField("Closures", r.Closures)
+ checkField("Types", r.Types)
}
}
diff --git a/internal/treesitter/rules/rules_test.go b/internal/treesitter/rules/rules_test.go
index da52f0f6..00284816 100644
--- a/internal/treesitter/rules/rules_test.go
+++ b/internal/treesitter/rules/rules_test.go
@@ -252,6 +252,8 @@ func TestRulesHelperMethods(t *testing.T) {
Unordered: []string{"object", "hash"},
Flattened: []string{"string_literal"},
Blocks: []string{"block", "compound_statement"},
+ Calls: []string{"call_expression"},
+ Indexed: []string{"subscript_expression", "index_expression"},
EquivalentTypes: [][]string{
{"function_declaration", "function_definition", "variable_declaration"},
{"assignment_statement", "variable_declaration"},
@@ -311,6 +313,20 @@ func TestRulesHelperMethods(t *testing.T) {
t.Errorf("IsBlock(other) = true, want false")
}
+ if !r.IsCall("call_expression") {
+ t.Errorf("IsCall(call_expression) = false, want true")
+ }
+ if r.IsCall("other") {
+ t.Errorf("IsCall(other) = true, want false")
+ }
+
+ if !r.IsIndexed("subscript_expression") || !r.IsIndexed("index_expression") {
+ t.Errorf("IsIndexed(subscript_expression/index_expression) = false, want true")
+ }
+ if r.IsIndexed("other") {
+ t.Errorf("IsIndexed(other) = true, want false")
+ }
+
if !r.AreTypesEquivalent("function_declaration", "variable_declaration") {
t.Errorf("AreTypesEquivalent(function_declaration, variable_declaration) = false, want true")
}
@@ -353,6 +369,12 @@ func TestRulesHelperMethods(t *testing.T) {
if !r.IsBlock("compound_statement") || r.IsBlock("other") {
t.Errorf("IsBlock uncompiled fallback failed")
}
+ if !r.IsCall("call_expression") || r.IsCall("other") {
+ t.Errorf("IsCall uncompiled fallback failed")
+ }
+ if !r.IsIndexed("subscript_expression") || r.IsIndexed("other") {
+ t.Errorf("IsIndexed uncompiled fallback failed")
+ }
if !r.AreTypesEquivalent("function_declaration", "variable_declaration") {
t.Errorf("AreTypesEquivalent uncompiled fallback failed")
}
@@ -381,6 +403,12 @@ func TestRulesHelperMethods(t *testing.T) {
if r.IsBlock("a") {
t.Errorf("nil.IsBlock returned true")
}
+ if r.IsCall("a") {
+ t.Errorf("nil.IsCall returned true")
+ }
+ if r.IsIndexed("a") {
+ t.Errorf("nil.IsIndexed returned true")
+ }
if !r.AreTypesEquivalent("a", "a") {
t.Errorf("nil.AreTypesEquivalent(a, a) returned false, want true")
}
@@ -405,7 +433,7 @@ func TestRulesHelperMethods(t *testing.T) {
}
})
- t.Run("package-level IsFlattened", func(t *testing.T) {
+ t.Run("package-level helpers", func(t *testing.T) {
if !IsFlattened("raw_string_literal") {
t.Errorf("IsFlattened(raw_string_literal) = false, want true")
}
@@ -415,6 +443,18 @@ func TestRulesHelperMethods(t *testing.T) {
if IsFlattened("") {
t.Errorf("IsFlattened(\"\") = true, want false")
}
+ if !IsCall("call_expression") {
+ t.Errorf("IsCall(call_expression) = false, want true")
+ }
+ if IsCall("nonexistent_call_xyz") {
+ t.Errorf("IsCall(nonexistent_call_xyz) = true, want false")
+ }
+ if !IsIndexed("subscript_expression") || !IsIndexed("index_expression") {
+ t.Errorf("IsIndexed(subscript_expression) = false, want true")
+ }
+ if IsIndexed("nonexistent_indexed_xyz") {
+ t.Errorf("IsIndexed(nonexistent_indexed_xyz) = true, want false")
+ }
})
}
@@ -469,3 +509,45 @@ func TestRulesIsOperatorLiteral(t *testing.T) {
}
}
}
+
+func TestLanguageKind(t *testing.T) {
+ expectedKinds := map[string]LanguageKind{
+ "c": KindCode,
+ "cpp": KindCode,
+ "go": KindCode,
+ "rust": KindCode,
+ "python": KindCode,
+ "javascript": KindCode,
+ "typescript": KindCode,
+ "tsx": KindCode,
+ "java": KindCode,
+ "php": KindCode,
+ "ruby": KindCode,
+ "lua": KindCode,
+ "zig": KindCode,
+ "json": KindData,
+ "yaml": KindData,
+ "toml": KindData,
+ "html": KindMarkup,
+ "css": KindMarkup,
+ }
+
+ for lang, expected := range expectedKinds {
+ r := Get(lang)
+ if r == nil {
+ t.Errorf("Get(%q) returned nil", lang)
+ continue
+ }
+ if r.GetKind() != expected {
+ t.Errorf("Get(%q).GetKind() = %v, want %v", lang, r.GetKind(), expected)
+ }
+ if r.Kind != expected {
+ t.Errorf("Get(%q).Kind = %v, want %v", lang, r.Kind, expected)
+ }
+ }
+
+ var nilRules *Rules
+ if nilRules.GetKind() != KindCode {
+ t.Errorf("nil.GetKind() = %v, want KindCode", nilRules.GetKind())
+ }
+}
diff --git a/internal/treesitter/rules/rust.go b/internal/treesitter/rules/rust.go
index a05cbaf7..9e9d055c 100644
--- a/internal/treesitter/rules/rust.go
+++ b/internal/treesitter/rules/rust.go
@@ -1,6 +1,7 @@
package rules
var rustRules = &Rules{
+ Kind: KindCode,
Flattened: []string{
"string_literal",
"raw_string_literal",
@@ -182,4 +183,42 @@ var rustRules = &Rules{
"call_expression",
"macro_invocation",
},
+ Indexed: []string{
+ "index_expression",
+ },
+ LocalVarDeclarations: []string{
+ "let_declaration",
+ },
+ ContainerDeclarations: []string{
+ "function_item",
+ "struct_item",
+ "enum_item",
+ "union_item",
+ "trait_item",
+ "impl_item",
+ "mod_item",
+ "type_item",
+ },
+ Closures: []string{
+ "closure_expression",
+ },
+ Types: []string{
+ "abstract_type",
+ "array_type",
+ "associated_type",
+ "bounded_type",
+ "bracketed_type",
+ "dynamic_type",
+ "function_type",
+ "generic_type",
+ "never_type",
+ "pointer_type",
+ "primitive_type",
+ "qualified_type",
+ "reference_type",
+ "scoped_type_identifier",
+ "tuple_type",
+ "type_identifier",
+ "unit_type",
+ },
}
diff --git a/internal/treesitter/rules/toml.go b/internal/treesitter/rules/toml.go
index 11a104a2..702c118b 100644
--- a/internal/treesitter/rules/toml.go
+++ b/internal/treesitter/rules/toml.go
@@ -1,6 +1,7 @@
package rules
var tomlRules = &Rules{
+ Kind: KindData,
Flattened: []string{
"string",
"offset_date_time",
@@ -47,6 +48,10 @@ var tomlRules = &Rules{
EquivalentTypes: [][]string{
{"table", "inline_table"},
},
+ Wrappers: []string{
+ "array",
+ "inline_table",
+ },
Comments: []string{
"comment",
},
diff --git a/internal/treesitter/rules/tsx.go b/internal/treesitter/rules/tsx.go
index 20b38845..452f6617 100644
--- a/internal/treesitter/rules/tsx.go
+++ b/internal/treesitter/rules/tsx.go
@@ -1,6 +1,7 @@
package rules
var tsxRules = &Rules{
+ Kind: KindCode,
Flattened: []string{
"string",
"template_string",
@@ -186,13 +187,14 @@ var tsxRules = &Rules{
"statement_block",
},
Wrappers: []string{
+ "export_statement",
"parenthesized_expression",
"parenthesized_type",
"type_arguments",
"type_parameters",
"array_type",
"generic_type",
- "subscript_expression",
+ "object",
"array",
"arguments",
"formal_parameters",
@@ -209,6 +211,8 @@ var tsxRules = &Rules{
"jsx_opening_element",
"jsx_self_closing_element",
"object_pattern",
+ "named_imports",
+ "export_clause",
},
EquivalentTypes: [][]string{
{"function_declaration", "function_expression", "arrow_function", "generator_function_declaration", "generator_function"},
@@ -225,4 +229,54 @@ var tsxRules = &Rules{
Calls: []string{
"call_expression",
},
+ Indexed: []string{
+ "subscript_expression",
+ },
+ LocalVarDeclarations: []string{
+ "variable_declaration",
+ "lexical_declaration",
+ },
+ ContainerDeclarations: []string{
+ "function_declaration",
+ "generator_function_declaration",
+ "method_definition",
+ "class_declaration",
+ "interface_declaration",
+ "type_alias_declaration",
+ "enum_declaration",
+ "module",
+ },
+ Closures: []string{
+ "arrow_function",
+ "function_expression",
+ "function",
+ },
+ Types: []string{
+ "array_type",
+ "conditional_type",
+ "constructor_type",
+ "default_type",
+ "existential_type",
+ "flow_maybe_type",
+ "function_type",
+ "generic_type",
+ "infer_type",
+ "intersection_type",
+ "literal_type",
+ "lookup_type",
+ "nested_type_identifier",
+ "object_type",
+ "optional_type",
+ "parenthesized_type",
+ "predefined_type",
+ "primary_type",
+ "readonly_type",
+ "rest_type",
+ "template_literal_type",
+ "template_type",
+ "this_type",
+ "tuple_type",
+ "type_identifier",
+ "union_type",
+ },
}
diff --git a/internal/treesitter/rules/typescript.go b/internal/treesitter/rules/typescript.go
index e5b15246..abbae2ea 100644
--- a/internal/treesitter/rules/typescript.go
+++ b/internal/treesitter/rules/typescript.go
@@ -1,6 +1,7 @@
package rules
var typescriptRules = &Rules{
+ Kind: KindCode,
Flattened: []string{
"string",
"template_string",
@@ -178,13 +179,14 @@ var typescriptRules = &Rules{
"statement_block",
},
Wrappers: []string{
+ "export_statement",
"parenthesized_expression",
"parenthesized_type",
"type_arguments",
"type_parameters",
"array_type",
"generic_type",
- "subscript_expression",
+ "object",
"array",
"arguments",
"formal_parameters",
@@ -198,6 +200,8 @@ var typescriptRules = &Rules{
"object",
"object_type",
"object_pattern",
+ "named_imports",
+ "export_clause",
},
EquivalentTypes: [][]string{
{"function_declaration", "function_expression", "arrow_function", "generator_function_declaration", "generator_function"},
@@ -213,4 +217,54 @@ var typescriptRules = &Rules{
Calls: []string{
"call_expression",
},
+ Indexed: []string{
+ "subscript_expression",
+ },
+ LocalVarDeclarations: []string{
+ "variable_declaration",
+ "lexical_declaration",
+ },
+ ContainerDeclarations: []string{
+ "function_declaration",
+ "generator_function_declaration",
+ "method_definition",
+ "class_declaration",
+ "interface_declaration",
+ "type_alias_declaration",
+ "enum_declaration",
+ "module",
+ },
+ Closures: []string{
+ "arrow_function",
+ "function_expression",
+ "function",
+ },
+ Types: []string{
+ "array_type",
+ "conditional_type",
+ "constructor_type",
+ "default_type",
+ "existential_type",
+ "flow_maybe_type",
+ "function_type",
+ "generic_type",
+ "infer_type",
+ "intersection_type",
+ "literal_type",
+ "lookup_type",
+ "nested_type_identifier",
+ "object_type",
+ "optional_type",
+ "parenthesized_type",
+ "predefined_type",
+ "primary_type",
+ "readonly_type",
+ "rest_type",
+ "template_literal_type",
+ "template_type",
+ "this_type",
+ "tuple_type",
+ "type_identifier",
+ "union_type",
+ },
}
diff --git a/internal/treesitter/rules/yaml.go b/internal/treesitter/rules/yaml.go
index 98e70c71..683dcb08 100644
--- a/internal/treesitter/rules/yaml.go
+++ b/internal/treesitter/rules/yaml.go
@@ -1,6 +1,7 @@
package rules
var yamlRules = &Rules{
+ Kind: KindData,
Flattened: []string{
"string_scalar",
"double_quote_scalar",
@@ -57,5 +58,7 @@ var yamlRules = &Rules{
Wrappers: []string{
"block_node",
"flow_node",
+ "flow_sequence",
+ "flow_mapping",
},
}
diff --git a/internal/treesitter/rules/zig.go b/internal/treesitter/rules/zig.go
index 6755723b..ad562e51 100644
--- a/internal/treesitter/rules/zig.go
+++ b/internal/treesitter/rules/zig.go
@@ -1,6 +1,7 @@
package rules
var zigRules = &Rules{
+ Kind: KindCode,
Flattened: []string{
"string",
"multiline_string",
@@ -155,4 +156,29 @@ var zigRules = &Rules{
Calls: []string{
"call_expression",
},
+ Indexed: []string{
+ "index_expression",
+ },
+ LocalVarDeclarations: []string{
+ "variable_declaration",
+ },
+ ContainerDeclarations: []string{
+ "function_declaration",
+ "struct_declaration",
+ "enum_declaration",
+ "union_declaration",
+ "opaque_declaration",
+ "error_set_declaration",
+ "test_declaration",
+ },
+ Types: []string{
+ "anyframe_type",
+ "array_type",
+ "builtin_type",
+ "error_type",
+ "error_union_type",
+ "nullable_type",
+ "pointer_type",
+ "slice_type",
+ },
}
diff --git a/internal/treesitter/rules_symbol_test.go b/internal/treesitter/rules_symbol_test.go
index 19317dd7..ef48aa77 100644
--- a/internal/treesitter/rules_symbol_test.go
+++ b/internal/treesitter/rules_symbol_test.go
@@ -14,20 +14,22 @@ var allLanguageExtensions = []string{
}
func getNamedGrammarSymbols(ext string) (string, map[string]bool, *rules.Rules) {
- entry := DetectGrammarEntry(ext)
- if entry == nil {
+ lang, err := DetectLanguage(ext)
+ if err != nil || lang == nil {
return "", nil, nil
}
- lang := entry.Language()
- namedSymbols := make(map[string]bool)
- for i := range min(int(lang.SymbolCount), len(lang.SymbolNames)) {
- name := lang.SymbolNames[i]
- isNamed := i < len(lang.SymbolMetadata) && lang.SymbolMetadata[i].Named
- if name != "" && isNamed {
- namedSymbols[name] = true
+ ptr, err := GetNativeLanguage(lang.Name)
+ if err != nil || ptr == nil {
+ return "", nil, nil
+ }
+ symbols := NativeLanguageSymbols(ptr)
+ namedSymbols := make(map[string]bool, len(symbols))
+ for _, s := range symbols {
+ if s != "" {
+ namedSymbols[s] = true
}
}
- return entry.Name, namedSymbols, rules.Get(entry.Name)
+ return lang.Name, namedSymbols, rules.Get(lang.Name)
}
func TestEveryLanguageEquivalentTypesAreValidSymbols(t *testing.T) {
diff --git a/internal/tui/syntax.go b/internal/tui/syntax.go
index 650cc2b0..e75fe482 100644
--- a/internal/tui/syntax.go
+++ b/internal/tui/syntax.go
@@ -1,13 +1,8 @@
package tui
import (
- "cmp"
-
- "github.com/HarshK97/diffmantic/internal/serialize"
"github.com/HarshK97/diffmantic/internal/theme"
- "github.com/HarshK97/diffmantic/internal/treesitter"
"github.com/charmbracelet/lipgloss"
- "github.com/odvcencio/gotreesitter"
)
// syntaxSpan holds the visual color range for a single line.
@@ -19,61 +14,5 @@ type syntaxSpan struct {
// highlightSyntax runs Tree-sitter on source and maps matches to per-line color spans. Returns nil if unsupported.
func highlightSyntax(filename string, source []byte, themeOpt ...*theme.Theme) map[int][]syntaxSpan {
- if len(source) == 0 {
- return nil
- }
-
- entry := treesitter.DetectGrammarEntry(filename)
- if entry == nil || entry.HighlightQuery == "" {
- return nil
- }
-
- lang := entry.Language()
- if lang == nil {
- return nil
- }
-
- th := defaultTheme
- if len(themeOpt) > 0 {
- th = cmp.Or(themeOpt[0], defaultTheme)
- }
-
- var opts []gotreesitter.HighlighterOption
- if entry.TokenSourceFactory != nil {
- opts = append(opts, gotreesitter.WithTokenSourceFactory(func(src []byte) gotreesitter.TokenSource {
- return entry.TokenSourceFactory(src, lang)
- }))
- }
-
- h, err := gotreesitter.NewHighlighter(lang, entry.HighlightQuery, opts...)
- if err != nil {
- return nil
- }
-
- ranges := h.Highlight(source)
- if len(ranges) == 0 {
- return nil
- }
-
- // Map byte offsets to line numbers.
- lineIndex := serialize.BuildLineIndex(source)
-
- result := make(map[int][]syntaxSpan)
-
- for _, r := range ranges {
- color := th.CaptureColor(r.Capture)
- if color == "" {
- continue
- }
-
- serialize.ForEachLineSpan(lineIndex, source, r.StartByte, r.EndByte, func(line, sc, ec int) {
- result[line] = append(result[line], syntaxSpan{
- startCol: sc,
- endCol: ec,
- color: color,
- })
- })
- }
-
- return result
+ return nil
}
diff --git a/internal/tui/syntax_test.go b/internal/tui/syntax_test.go
index b2d00c4e..0eaf3931 100644
--- a/internal/tui/syntax_test.go
+++ b/internal/tui/syntax_test.go
@@ -2,8 +2,6 @@ package tui
import (
"testing"
-
- "github.com/HarshK97/diffmantic/internal/theme"
)
func TestHighlightSyntax(t *testing.T) {
@@ -30,20 +28,5 @@ func TestHighlightSyntax(t *testing.T) {
}
func TestHighlightSyntaxGo(t *testing.T) {
- source := []byte("if engine.handlers404 == nil {\n}\n")
- mocha := theme.CatppuccinMochaTheme()
- latte := theme.CatppuccinLatteTheme()
-
- resMocha := highlightSyntax("test.go", source, mocha)
- resLatte := highlightSyntax("test.go", source, latte)
-
- if resMocha == nil || resLatte == nil {
- t.Fatal("expected non-nil syntax highlighting for Go code")
- }
-
- // Line 0 spans: "if", "handlers404", "==", "nil"
- spansLatte := resLatte[0]
- if len(spansLatte) == 0 {
- t.Errorf("expected syntax spans for line 0 in Latte")
- }
+ t.Skip("TUI syntax highlighting is deprecated")
}
diff --git a/tests/testdata/c_redis_quadratic_search/expected_actions.json.gz b/tests/testdata/c_redis_quadratic_search/expected_actions.json.gz
index 100248eb..fc56ef97 100644
Binary files a/tests/testdata/c_redis_quadratic_search/expected_actions.json.gz and b/tests/testdata/c_redis_quadratic_search/expected_actions.json.gz differ
diff --git a/tests/testdata/c_redis_quadratic_search/expected_ui.json.gz b/tests/testdata/c_redis_quadratic_search/expected_ui.json.gz
index 614b4858..fa07048b 100644
Binary files a/tests/testdata/c_redis_quadratic_search/expected_ui.json.gz and b/tests/testdata/c_redis_quadratic_search/expected_ui.json.gz differ
diff --git a/tests/testdata/cpp_fmt_workaround_broken_generic_msvc/expected_actions.json.gz b/tests/testdata/cpp_fmt_workaround_broken_generic_msvc/expected_actions.json.gz
index a6ee476e..1b82bd64 100644
Binary files a/tests/testdata/cpp_fmt_workaround_broken_generic_msvc/expected_actions.json.gz and b/tests/testdata/cpp_fmt_workaround_broken_generic_msvc/expected_actions.json.gz differ
diff --git a/tests/testdata/cpp_fmt_workaround_broken_generic_msvc/expected_ui.json.gz b/tests/testdata/cpp_fmt_workaround_broken_generic_msvc/expected_ui.json.gz
index 12aea0f1..03ce3dc4 100644
Binary files a/tests/testdata/cpp_fmt_workaround_broken_generic_msvc/expected_ui.json.gz and b/tests/testdata/cpp_fmt_workaround_broken_generic_msvc/expected_ui.json.gz differ
diff --git a/tests/testdata/cpp_simdjson_key/expected_actions.json.gz b/tests/testdata/cpp_simdjson_key/expected_actions.json.gz
index b1714af6..d769f642 100644
Binary files a/tests/testdata/cpp_simdjson_key/expected_actions.json.gz and b/tests/testdata/cpp_simdjson_key/expected_actions.json.gz differ
diff --git a/tests/testdata/cpp_simdjson_key/expected_ui.json.gz b/tests/testdata/cpp_simdjson_key/expected_ui.json.gz
index 760ca0f9..82611311 100644
Binary files a/tests/testdata/cpp_simdjson_key/expected_ui.json.gz and b/tests/testdata/cpp_simdjson_key/expected_ui.json.gz differ
diff --git a/tests/testdata/go_gin_fix_compile/expected_actions.json.gz b/tests/testdata/go_gin_fix_compile/expected_actions.json.gz
index 471ae847..4a9e3168 100644
Binary files a/tests/testdata/go_gin_fix_compile/expected_actions.json.gz and b/tests/testdata/go_gin_fix_compile/expected_actions.json.gz differ
diff --git a/tests/testdata/go_gin_fix_compile/expected_ui.json.gz b/tests/testdata/go_gin_fix_compile/expected_ui.json.gz
index 2b0d074f..1be9b9bb 100644
Binary files a/tests/testdata/go_gin_fix_compile/expected_ui.json.gz and b/tests/testdata/go_gin_fix_compile/expected_ui.json.gz differ
diff --git a/tests/testdata/go_gin_fix_corrupted/expected_actions.json.gz b/tests/testdata/go_gin_fix_corrupted/expected_actions.json.gz
index f2e24061..f64de480 100644
Binary files a/tests/testdata/go_gin_fix_corrupted/expected_actions.json.gz and b/tests/testdata/go_gin_fix_corrupted/expected_actions.json.gz differ
diff --git a/tests/testdata/go_gin_fix_corrupted/expected_ui.json.gz b/tests/testdata/go_gin_fix_corrupted/expected_ui.json.gz
index 587c6cfb..872aea8a 100644
Binary files a/tests/testdata/go_gin_fix_corrupted/expected_ui.json.gz and b/tests/testdata/go_gin_fix_corrupted/expected_ui.json.gz differ
diff --git a/tests/testdata/go_gin_fix_golint/expected_actions.json.gz b/tests/testdata/go_gin_fix_golint/expected_actions.json.gz
index 658301d4..38711e1a 100644
Binary files a/tests/testdata/go_gin_fix_golint/expected_actions.json.gz and b/tests/testdata/go_gin_fix_golint/expected_actions.json.gz differ
diff --git a/tests/testdata/go_gin_fix_lint/expected_actions.json.gz b/tests/testdata/go_gin_fix_lint/expected_actions.json.gz
index 3e13a16e..eff41bd6 100644
Binary files a/tests/testdata/go_gin_fix_lint/expected_actions.json.gz and b/tests/testdata/go_gin_fix_lint/expected_actions.json.gz differ
diff --git a/tests/testdata/go_gin_fix_lint/expected_ui.json.gz b/tests/testdata/go_gin_fix_lint/expected_ui.json.gz
index 0a4816bc..f22dcff0 100644
Binary files a/tests/testdata/go_gin_fix_lint/expected_ui.json.gz and b/tests/testdata/go_gin_fix_lint/expected_ui.json.gz differ
diff --git a/tests/testdata/go_gin_fixes_http/expected_actions.json.gz b/tests/testdata/go_gin_fixes_http/expected_actions.json.gz
index 8f9974a5..b885c6c0 100644
Binary files a/tests/testdata/go_gin_fixes_http/expected_actions.json.gz and b/tests/testdata/go_gin_fixes_http/expected_actions.json.gz differ
diff --git a/tests/testdata/go_gin_fixes_http/expected_ui.json.gz b/tests/testdata/go_gin_fixes_http/expected_ui.json.gz
index ccf144bd..c25df92a 100644
Binary files a/tests/testdata/go_gin_fixes_http/expected_ui.json.gz and b/tests/testdata/go_gin_fixes_http/expected_ui.json.gz differ
diff --git a/tests/testdata/go_gin_prevent_flush/expected_actions.json.gz b/tests/testdata/go_gin_prevent_flush/expected_actions.json.gz
index 1b572978..dd6e9639 100644
Binary files a/tests/testdata/go_gin_prevent_flush/expected_actions.json.gz and b/tests/testdata/go_gin_prevent_flush/expected_actions.json.gz differ
diff --git a/tests/testdata/go_gin_use_t/expected_actions.json.gz b/tests/testdata/go_gin_use_t/expected_actions.json.gz
index a1156869..126e145e 100644
Binary files a/tests/testdata/go_gin_use_t/expected_actions.json.gz and b/tests/testdata/go_gin_use_t/expected_actions.json.gz differ
diff --git a/tests/testdata/java_commons_lang_simplify_abstractreflection_setaccessible/expected_actions.json.gz b/tests/testdata/java_commons_lang_simplify_abstractreflection_setaccessible/expected_actions.json.gz
index 7122fbdd..a544a8f1 100644
Binary files a/tests/testdata/java_commons_lang_simplify_abstractreflection_setaccessible/expected_actions.json.gz and b/tests/testdata/java_commons_lang_simplify_abstractreflection_setaccessible/expected_actions.json.gz differ
diff --git a/tests/testdata/java_commons_lang_simplify_abstractreflection_setaccessible/expected_ui.json.gz b/tests/testdata/java_commons_lang_simplify_abstractreflection_setaccessible/expected_ui.json.gz
index 0666b1bf..5b204b7f 100644
Binary files a/tests/testdata/java_commons_lang_simplify_abstractreflection_setaccessible/expected_ui.json.gz and b/tests/testdata/java_commons_lang_simplify_abstractreflection_setaccessible/expected_ui.json.gz differ
diff --git a/tests/testdata/json_schemastore_cargo_workspace_members/expected_actions.json.gz b/tests/testdata/json_schemastore_cargo_workspace_members/expected_actions.json.gz
index 0806013c..ddd652ad 100644
Binary files a/tests/testdata/json_schemastore_cargo_workspace_members/expected_actions.json.gz and b/tests/testdata/json_schemastore_cargo_workspace_members/expected_actions.json.gz differ
diff --git a/tests/testdata/json_schemastore_cargo_workspace_members/expected_ui.json.gz b/tests/testdata/json_schemastore_cargo_workspace_members/expected_ui.json.gz
index c9dee5f1..78c92851 100644
Binary files a/tests/testdata/json_schemastore_cargo_workspace_members/expected_ui.json.gz and b/tests/testdata/json_schemastore_cargo_workspace_members/expected_ui.json.gz differ
diff --git a/tests/testdata/json_schemastore_hugo_theme_config/expected_actions.json.gz b/tests/testdata/json_schemastore_hugo_theme_config/expected_actions.json.gz
index 813a2e27..1c4cc08a 100644
Binary files a/tests/testdata/json_schemastore_hugo_theme_config/expected_actions.json.gz and b/tests/testdata/json_schemastore_hugo_theme_config/expected_actions.json.gz differ
diff --git a/tests/testdata/json_schemastore_hugo_theme_config/expected_ui.json.gz b/tests/testdata/json_schemastore_hugo_theme_config/expected_ui.json.gz
index 777c661e..54b4901a 100644
Binary files a/tests/testdata/json_schemastore_hugo_theme_config/expected_ui.json.gz and b/tests/testdata/json_schemastore_hugo_theme_config/expected_ui.json.gz differ
diff --git a/tests/testdata/json_schemastore_tox_env_configuration/expected_actions.json.gz b/tests/testdata/json_schemastore_tox_env_configuration/expected_actions.json.gz
index d70c895f..410ba143 100644
Binary files a/tests/testdata/json_schemastore_tox_env_configuration/expected_actions.json.gz and b/tests/testdata/json_schemastore_tox_env_configuration/expected_actions.json.gz differ
diff --git a/tests/testdata/json_schemastore_tox_env_configuration/expected_ui.json.gz b/tests/testdata/json_schemastore_tox_env_configuration/expected_ui.json.gz
index 7313136c..1e50a65b 100644
Binary files a/tests/testdata/json_schemastore_tox_env_configuration/expected_ui.json.gz and b/tests/testdata/json_schemastore_tox_env_configuration/expected_ui.json.gz differ
diff --git a/tests/testdata/json_schemastore_uv_pip_settings/expected_actions.json.gz b/tests/testdata/json_schemastore_uv_pip_settings/expected_actions.json.gz
index e1183053..e02c5e86 100644
Binary files a/tests/testdata/json_schemastore_uv_pip_settings/expected_actions.json.gz and b/tests/testdata/json_schemastore_uv_pip_settings/expected_actions.json.gz differ
diff --git a/tests/testdata/json_schemastore_uv_pip_settings/expected_ui.json.gz b/tests/testdata/json_schemastore_uv_pip_settings/expected_ui.json.gz
index 63a6fa66..c1e880c1 100644
Binary files a/tests/testdata/json_schemastore_uv_pip_settings/expected_ui.json.gz and b/tests/testdata/json_schemastore_uv_pip_settings/expected_ui.json.gz differ
diff --git a/tests/testdata/json_schemastore_workflow_step_reorder/expected_actions.json.gz b/tests/testdata/json_schemastore_workflow_step_reorder/expected_actions.json.gz
index 0d14f8a9..dc6d27f1 100644
Binary files a/tests/testdata/json_schemastore_workflow_step_reorder/expected_actions.json.gz and b/tests/testdata/json_schemastore_workflow_step_reorder/expected_actions.json.gz differ
diff --git a/tests/testdata/json_schemastore_workflow_step_reorder/expected_ui.json.gz b/tests/testdata/json_schemastore_workflow_step_reorder/expected_ui.json.gz
index ef98f1e7..67009556 100644
Binary files a/tests/testdata/json_schemastore_workflow_step_reorder/expected_ui.json.gz and b/tests/testdata/json_schemastore_workflow_step_reorder/expected_ui.json.gz differ
diff --git a/tests/testdata/php_guzzle_handler_curl_multi/expected_actions.json.gz b/tests/testdata/php_guzzle_handler_curl_multi/expected_actions.json.gz
index 6b7a7929..1760c308 100644
Binary files a/tests/testdata/php_guzzle_handler_curl_multi/expected_actions.json.gz and b/tests/testdata/php_guzzle_handler_curl_multi/expected_actions.json.gz differ
diff --git a/tests/testdata/php_guzzle_handler_curl_multi/expected_ui.json.gz b/tests/testdata/php_guzzle_handler_curl_multi/expected_ui.json.gz
index ea90ab05..2a65df80 100644
Binary files a/tests/testdata/php_guzzle_handler_curl_multi/expected_ui.json.gz and b/tests/testdata/php_guzzle_handler_curl_multi/expected_ui.json.gz differ
diff --git a/tests/testdata/py_requests_align_sessionget/expected_actions.json.gz b/tests/testdata/py_requests_align_sessionget/expected_actions.json.gz
index 4eb70383..4cd7733b 100644
Binary files a/tests/testdata/py_requests_align_sessionget/expected_actions.json.gz and b/tests/testdata/py_requests_align_sessionget/expected_actions.json.gz differ
diff --git a/tests/testdata/py_requests_align_sessionget/expected_ui.json.gz b/tests/testdata/py_requests_align_sessionget/expected_ui.json.gz
index 0be81c56..45d3c7b8 100644
Binary files a/tests/testdata/py_requests_align_sessionget/expected_ui.json.gz and b/tests/testdata/py_requests_align_sessionget/expected_ui.json.gz differ
diff --git a/tests/testdata/py_requests_docs_indent/expected_actions.json.gz b/tests/testdata/py_requests_docs_indent/expected_actions.json.gz
index 1d711b47..6021eb37 100644
Binary files a/tests/testdata/py_requests_docs_indent/expected_actions.json.gz and b/tests/testdata/py_requests_docs_indent/expected_actions.json.gz differ
diff --git a/tests/testdata/py_requests_docs_indent/expected_ui.json.gz b/tests/testdata/py_requests_docs_indent/expected_ui.json.gz
index efb3ce24..9e18d3da 100644
Binary files a/tests/testdata/py_requests_docs_indent/expected_ui.json.gz and b/tests/testdata/py_requests_docs_indent/expected_ui.json.gz differ
diff --git a/tests/testdata/py_requests_dont_use/expected_actions.json.gz b/tests/testdata/py_requests_dont_use/expected_actions.json.gz
index 05af8f1e..3557ef77 100644
Binary files a/tests/testdata/py_requests_dont_use/expected_actions.json.gz and b/tests/testdata/py_requests_dont_use/expected_actions.json.gz differ
diff --git a/tests/testdata/py_requests_encode_files_detection/expected_actions.json.gz b/tests/testdata/py_requests_encode_files_detection/expected_actions.json.gz
index 43b7769c..bae6daa2 100644
Binary files a/tests/testdata/py_requests_encode_files_detection/expected_actions.json.gz and b/tests/testdata/py_requests_encode_files_detection/expected_actions.json.gz differ
diff --git a/tests/testdata/py_requests_encode_files_detection/expected_ui.json.gz b/tests/testdata/py_requests_encode_files_detection/expected_ui.json.gz
index 07e8805e..1b94753c 100644
Binary files a/tests/testdata/py_requests_encode_files_detection/expected_ui.json.gz and b/tests/testdata/py_requests_encode_files_detection/expected_ui.json.gz differ
diff --git a/tests/testdata/py_requests_make_json/expected_actions.json.gz b/tests/testdata/py_requests_make_json/expected_actions.json.gz
index b75cfc43..5aa8890f 100644
Binary files a/tests/testdata/py_requests_make_json/expected_actions.json.gz and b/tests/testdata/py_requests_make_json/expected_actions.json.gz differ
diff --git a/tests/testdata/py_requests_make_json/expected_ui.json.gz b/tests/testdata/py_requests_make_json/expected_ui.json.gz
index a304bc30..ec198bd7 100644
Binary files a/tests/testdata/py_requests_make_json/expected_ui.json.gz and b/tests/testdata/py_requests_make_json/expected_ui.json.gz differ
diff --git a/tests/testdata/py_requests_only_use/expected_actions.json.gz b/tests/testdata/py_requests_only_use/expected_actions.json.gz
index 9e9e17fb..c245255b 100644
Binary files a/tests/testdata/py_requests_only_use/expected_actions.json.gz and b/tests/testdata/py_requests_only_use/expected_actions.json.gz differ
diff --git a/tests/testdata/py_requests_only_use/expected_ui.json.gz b/tests/testdata/py_requests_only_use/expected_ui.json.gz
index 06531fd5..fbc23356 100644
Binary files a/tests/testdata/py_requests_only_use/expected_ui.json.gz and b/tests/testdata/py_requests_only_use/expected_ui.json.gz differ
diff --git a/tests/testdata/py_requests_prevent_response/expected_actions.json.gz b/tests/testdata/py_requests_prevent_response/expected_actions.json.gz
index dae44616..579395da 100644
Binary files a/tests/testdata/py_requests_prevent_response/expected_actions.json.gz and b/tests/testdata/py_requests_prevent_response/expected_actions.json.gz differ
diff --git a/tests/testdata/py_requests_refactor_prefer/expected_actions.json.gz b/tests/testdata/py_requests_refactor_prefer/expected_actions.json.gz
index fea91657..b4cbd239 100644
Binary files a/tests/testdata/py_requests_refactor_prefer/expected_actions.json.gz and b/tests/testdata/py_requests_refactor_prefer/expected_actions.json.gz differ
diff --git a/tests/testdata/py_requests_refactor_prefer/expected_ui.json.gz b/tests/testdata/py_requests_refactor_prefer/expected_ui.json.gz
index 3d03a30c..d2e17967 100644
Binary files a/tests/testdata/py_requests_refactor_prefer/expected_ui.json.gz and b/tests/testdata/py_requests_refactor_prefer/expected_ui.json.gz differ
diff --git a/tests/testdata/ruby_sinatra_fix_content_type_leak_in_muste_4/expected_actions.json.gz b/tests/testdata/ruby_sinatra_fix_content_type_leak_in_muste_4/expected_actions.json.gz
index 929aca77..1e0d2e7c 100644
Binary files a/tests/testdata/ruby_sinatra_fix_content_type_leak_in_muste_4/expected_actions.json.gz and b/tests/testdata/ruby_sinatra_fix_content_type_leak_in_muste_4/expected_actions.json.gz differ
diff --git a/tests/testdata/ruby_sinatra_fix_content_type_leak_in_muste_4/expected_ui.json.gz b/tests/testdata/ruby_sinatra_fix_content_type_leak_in_muste_4/expected_ui.json.gz
index 6cefa319..ffb05c85 100644
Binary files a/tests/testdata/ruby_sinatra_fix_content_type_leak_in_muste_4/expected_ui.json.gz and b/tests/testdata/ruby_sinatra_fix_content_type_leak_in_muste_4/expected_ui.json.gz differ
diff --git a/tests/testdata/ruby_sinatra_fix_proc_template_memory_leak_9/expected_actions.json.gz b/tests/testdata/ruby_sinatra_fix_proc_template_memory_leak_9/expected_actions.json.gz
index 2448c1fa..25fd0d1d 100644
Binary files a/tests/testdata/ruby_sinatra_fix_proc_template_memory_leak_9/expected_actions.json.gz and b/tests/testdata/ruby_sinatra_fix_proc_template_memory_leak_9/expected_actions.json.gz differ
diff --git a/tests/testdata/ruby_sinatra_fix_proc_template_memory_leak_9/expected_ui.json.gz b/tests/testdata/ruby_sinatra_fix_proc_template_memory_leak_9/expected_ui.json.gz
index 6730639d..fd77061e 100644
Binary files a/tests/testdata/ruby_sinatra_fix_proc_template_memory_leak_9/expected_ui.json.gz and b/tests/testdata/ruby_sinatra_fix_proc_template_memory_leak_9/expected_ui.json.gz differ
diff --git a/tests/testdata/rust_tokio_incorrect_confusing_asyncwrite/expected_actions.json.gz b/tests/testdata/rust_tokio_incorrect_confusing_asyncwrite/expected_actions.json.gz
index 7cd0d451..fedf8451 100644
Binary files a/tests/testdata/rust_tokio_incorrect_confusing_asyncwrite/expected_actions.json.gz and b/tests/testdata/rust_tokio_incorrect_confusing_asyncwrite/expected_actions.json.gz differ
diff --git a/tests/testdata/rust_tokio_incorrect_confusing_asyncwrite/expected_ui.json.gz b/tests/testdata/rust_tokio_incorrect_confusing_asyncwrite/expected_ui.json.gz
index 08d02fb3..0843d989 100644
Binary files a/tests/testdata/rust_tokio_incorrect_confusing_asyncwrite/expected_ui.json.gz and b/tests/testdata/rust_tokio_incorrect_confusing_asyncwrite/expected_ui.json.gz differ
diff --git a/tests/testdata/rust_tokio_runtime_localruntime_doc/expected_actions.json.gz b/tests/testdata/rust_tokio_runtime_localruntime_doc/expected_actions.json.gz
index a9ba40b3..8daea692 100644
Binary files a/tests/testdata/rust_tokio_runtime_localruntime_doc/expected_actions.json.gz and b/tests/testdata/rust_tokio_runtime_localruntime_doc/expected_actions.json.gz differ
diff --git a/tests/testdata/rust_tokio_time_loom_test/expected_actions.json.gz b/tests/testdata/rust_tokio_time_loom_test/expected_actions.json.gz
index 90c0c8f7..5f0249e2 100644
Binary files a/tests/testdata/rust_tokio_time_loom_test/expected_actions.json.gz and b/tests/testdata/rust_tokio_time_loom_test/expected_actions.json.gz differ
diff --git a/tests/testdata/rust_tokio_time_loom_test/expected_ui.json.gz b/tests/testdata/rust_tokio_time_loom_test/expected_ui.json.gz
index c62020a5..b0afe7b4 100644
Binary files a/tests/testdata/rust_tokio_time_loom_test/expected_ui.json.gz and b/tests/testdata/rust_tokio_time_loom_test/expected_ui.json.gz differ
diff --git a/tests/testdata/toml_poetry_add_support_for/expected_actions.json.gz b/tests/testdata/toml_poetry_add_support_for/expected_actions.json.gz
index f5e65c5a..f52323bd 100644
Binary files a/tests/testdata/toml_poetry_add_support_for/expected_actions.json.gz and b/tests/testdata/toml_poetry_add_support_for/expected_actions.json.gz differ
diff --git a/tests/testdata/toml_poetry_add_support_for/expected_ui.json.gz b/tests/testdata/toml_poetry_add_support_for/expected_ui.json.gz
index 292f0e32..8883135e 100644
Binary files a/tests/testdata/toml_poetry_add_support_for/expected_ui.json.gz and b/tests/testdata/toml_poetry_add_support_for/expected_ui.json.gz differ
diff --git a/tests/testdata/toml_poetry_dependencies_and_warnings/expected_actions.json.gz b/tests/testdata/toml_poetry_dependencies_and_warnings/expected_actions.json.gz
index 33013dfe..1182c6b5 100644
Binary files a/tests/testdata/toml_poetry_dependencies_and_warnings/expected_actions.json.gz and b/tests/testdata/toml_poetry_dependencies_and_warnings/expected_actions.json.gz differ
diff --git a/tests/testdata/toml_poetry_dependencies_and_warnings/expected_ui.json.gz b/tests/testdata/toml_poetry_dependencies_and_warnings/expected_ui.json.gz
index 392f4437..0fb470de 100644
Binary files a/tests/testdata/toml_poetry_dependencies_and_warnings/expected_ui.json.gz and b/tests/testdata/toml_poetry_dependencies_and_warnings/expected_ui.json.gz differ
diff --git a/tests/testdata/ts_trpc_patchclient_upgrade/expected_actions.json.gz b/tests/testdata/ts_trpc_patchclient_upgrade/expected_actions.json.gz
index 148d2dcb..f3d759b8 100644
Binary files a/tests/testdata/ts_trpc_patchclient_upgrade/expected_actions.json.gz and b/tests/testdata/ts_trpc_patchclient_upgrade/expected_actions.json.gz differ
diff --git a/tests/testdata/ts_trpc_patchclient_upgrade/expected_ui.json.gz b/tests/testdata/ts_trpc_patchclient_upgrade/expected_ui.json.gz
index 00918742..689938ac 100644
Binary files a/tests/testdata/ts_trpc_patchclient_upgrade/expected_ui.json.gz and b/tests/testdata/ts_trpc_patchclient_upgrade/expected_ui.json.gz differ
diff --git a/tests/testdata/ts_zod_improve_mini/new.ts b/tests/testdata/ts_zod_improve_mini/new.ts
index 0e9cedfd..4e6bb7c2 100644
--- a/tests/testdata/ts_zod_improve_mini/new.ts
+++ b/tests/testdata/ts_zod_improve_mini/new.ts
@@ -5,9 +5,9 @@ import * as parse from "./parse.js";
type SomeType = core.SomeType;
export interface ZodMiniType<
- out Output = unknown,
- out Input = unknown,
- out Internals extends core.$ZodTypeInternals