diff --git a/Makefile b/Makefile index b37352ff..f2797fb5 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,15 @@ +.DEFAULT_GOAL := build + .PHONY: build test test-unit test-integration test-e2e lint fmt coverage test-update bench bench-short grammars-native clean -native/bridge/lib/libdiffmantic_grammars.a: - @$(MAKE) grammars-native +GRAMMAR_SRCS := native/bridge/grammars.json native/bridge/build_grammars.go $(wildcard native/bridge/src/*.c) $(wildcard native/bridge/include/*.h) build: native/bridge/lib/libdiffmantic_grammars.a ## Build binary with native Tree-sitter flat-buffer bridge go build -ldflags="-s -w" -trimpath -o diffm ./cmd/diffm +native/bridge/lib/libdiffmantic_grammars.a: $(GRAMMAR_SRCS) + @$(MAKE) grammars-native + grammars-native: ## Fetch and compile 18 native Tree-sitter grammars go run ./native/bridge/build_grammars.go diff --git a/internal/comments/comments_test.go b/internal/comments/comments_test.go index de409904..49d4d184 100644 --- a/internal/comments/comments_test.go +++ b/internal/comments/comments_test.go @@ -2,13 +2,11 @@ package comments import ( "os" - "strings" "testing" "github.com/HarshK97/diffmantic/internal/actions" + "github.com/HarshK97/diffmantic/internal/engine" "github.com/HarshK97/diffmantic/internal/treesitter" - "github.com/HarshK97/diffmantic/internal/treesitter/rules" - "github.com/odvcencio/gotreesitter" ) func TestDiffCommentsIdentical(t *testing.T) { @@ -19,7 +17,7 @@ func TestDiffCommentsIdentical(t *testing.T) { {Type: "comment", Text: "// Hello World", StartRow: 8, StartByte: 15, EndByte: 29}, } - res := DiffComments(srcComments, dstComments) + res := DiffComments(srcComments, dstComments, nil) if len(res.Actions) != 0 { t.Errorf("expected 0 actions for identical comment, got %d actions", len(res.Actions)) } @@ -33,7 +31,7 @@ func TestDiffCommentsSingleLineUpdate(t *testing.T) { {Type: "comment", Text: "// New Comment", StartRow: 5, StartByte: 10, EndByte: 24}, } - res := DiffComments(srcComments, dstComments) + res := DiffComments(srcComments, dstComments, nil) if len(res.Actions) != 1 { t.Fatalf("expected 1 action, got %d", len(res.Actions)) } @@ -57,7 +55,7 @@ func TestDiffCommentsMultiLineLineDiff(t *testing.T) { {Type: "block_comment", Text: newJavadoc, StartRow: 10, StartByte: 0, EndByte: uint32(len(newJavadoc))}, } - res := DiffComments(srcComments, dstComments) + res := DiffComments(srcComments, dstComments, nil) if len(res.Actions) != 1 { t.Fatalf("expected 1 line-level update action in multiline comment, got %d", len(res.Actions)) @@ -75,29 +73,29 @@ func TestDiffCommentsMultiLineLineDiff(t *testing.T) { } func TestExtractCommentsWithTreeSitter(t *testing.T) { - lang, err := treesitter.DetectLanguage("test.go") - if err != nil { - t.Fatal(err) - } - rules := rules.Get("go") src := []byte("package main\n\n// Line comment 1\nfunc main() {\n\t// Line comment 2\n}\n") - parser := gotreesitter.NewParser(lang) - tree, err := parser.Parse(src) + _, flatNodes, symbols, err := treesitter.ParseForPipeline(src, "go") if err != nil { t.Fatal(err) } - comments := ExtractComments(tree.RootNode(), src, lang, rules) + comments := ExtractComments(flatNodes, symbols, src, "go") if len(comments) != 2 { t.Fatalf("expected 2 comments extracted, got %d", len(comments)) } if comments[0].Text != "// Line comment 1" { t.Errorf("expected '// Line comment 1', got %q", comments[0].Text) } + if comments[0].Language != "go" { + t.Errorf("expected comments[0].Language == 'go', got %q", comments[0].Language) + } if comments[1].Text != "// Line comment 2" { t.Errorf("expected '// Line comment 2', got %q", comments[1].Text) } + if comments[1].Language != "go" { + t.Errorf("expected comments[1].Language == 'go', got %q", comments[1].Language) + } } func TestDiffCommentsScopeLocking(t *testing.T) { @@ -110,7 +108,7 @@ func TestDiffCommentsScopeLocking(t *testing.T) { {Type: "comment", Text: "// Method B comment", ScopeKey: "method:funcB", StartRow: 20}, } - res := DiffComments(srcComments, dstComments) + res := DiffComments(srcComments, dstComments, nil) if len(res.Actions) != 1 { t.Fatalf("expected 1 action, got %d", len(res.Actions)) } @@ -127,7 +125,7 @@ func TestDiffCommentsControlBranchScopeLocking(t *testing.T) { {Type: "comment", Text: "-- Track leading whitespace for level", ScopeKey: "function:foo/if_statement", StartRow: 45}, } - res := DiffComments(srcComments, dstComments) + res := DiffComments(srcComments, dstComments, nil) // Moving between branches should delete and insert instead of update in place. if len(res.Actions) != 2 { t.Fatalf("expected 2 actions (1 delete, 1 insert), got %d", len(res.Actions)) @@ -149,27 +147,38 @@ func TestDiffCommentsControlBranchScopeLocking(t *testing.T) { func TestDiffCommentsMovedScope(t *testing.T) { srcComments := []CommentBlock{ - {Type: "comment", Text: "// Optional params comment", ScopeKey: "method_declaration:getParamsToSign", StartRow: 162, EndRow: 162}, + {Type: "comment", Text: "// Optional params comment", ScopeKey: "method_declaration:getParamsToSign", StartRow: 162, EndRow: 162, Language: "go"}, } dstComments := []CommentBlock{ - {Type: "comment", Text: "// Optional params comment", ScopeKey: "method_declaration:getOauthParams", StartRow: 159, EndRow: 159}, + {Type: "comment", Text: "// Optional params comment", ScopeKey: "method_declaration:getOauthParams", StartRow: 159, EndRow: 159, Language: "go"}, } - res := DiffComments(srcComments, dstComments) - if len(res.Actions) != 1 { - t.Fatalf("expected 1 Move action for comment moved across scopes, got %d actions", len(res.Actions)) + res := DiffComments(srcComments, dstComments, nil) + if len(res.Actions) != 2 { + t.Fatalf("expected 2 actions (1 Delete, 1 Insert) for comment moved across scopes, got %d actions", len(res.Actions)) } - if res.Actions[0].Type != actions.Move { - t.Errorf("expected Move action, got %v", res.Actions[0].Type) + hasDelete := false + hasInsert := false + for _, act := range res.Actions { + if act.Type == actions.Move { + t.Errorf("expected no Move action for comment trivia, got actions.Move") + } + if act.Type == actions.Delete { + hasDelete = true + } + if act.Type == actions.Insert { + hasInsert = true + } + } + if !hasDelete || !hasInsert { + t.Errorf("expected 1 Delete and 1 Insert action, got: %+v", res.Actions) + } + if _, ok := res.LineMappings[162]; ok { + t.Errorf("expected cross-scope comment NOT to populate LineMappings, got %v", res.LineMappings) } } func TestDiffCommentsGuzzlePhp(t *testing.T) { - lang, err := treesitter.DetectLanguage("test.php") - if err != nil { - t.Fatal(err) - } - r := rules.Get("php") src, err := os.ReadFile("../../tests/testdata/php_guzzle_handler_curl_multi/old.php") if err != nil { t.Fatal(err) @@ -179,28 +188,115 @@ func TestDiffCommentsGuzzlePhp(t *testing.T) { t.Fatal(err) } - parser := gotreesitter.NewParser(lang) - treeA, err := parser.Parse(src) + _, flatNodesA, symbolsA, err := treesitter.ParseForPipeline(src, "php") if err != nil { t.Fatal(err) } - treeB, err := parser.Parse(dst) + _, flatNodesB, symbolsB, err := treesitter.ParseForPipeline(dst, "php") if err != nil { t.Fatal(err) } - srcComments := ExtractComments(treeA.RootNode(), src, lang, r) - dstComments := ExtractComments(treeB.RootNode(), dst, lang, r) + srcComments := ExtractComments(flatNodesA, symbolsA, src, "php") + dstComments := ExtractComments(flatNodesB, symbolsB, dst, "php") - res := DiffComments(srcComments, dstComments) + res := DiffComments(srcComments, dstComments, nil) moveCount := 0 for _, a := range res.Actions { - if a.Type == actions.Move && strings.Contains(a.Node.Label, "Optional parameters") { + if a.Type == actions.Move { moveCount++ } } - if moveCount != 1 { - t.Errorf("expected 1 Move action for Optional parameters comment, got %d", moveCount) + if moveCount != 0 { + t.Errorf("expected 0 Move actions for comment trivia in Guzzle PHP diff, got %d", moveCount) + } +} + +func TestSyntheticCommentNodeInvariants(t *testing.T) { + cb := CommentBlock{ + Type: "comment", + Text: "// Hello World", + StartByte: 10, + EndByte: 24, + StartRow: 2, + StartCol: 0, + EndRow: 2, + EndCol: 14, + ParentType: "function_declaration", + ParentStart: 0, + ParentEnd: 100, + ParentRow: 1, + ParentEndRow: 10, + Language: "go", + } + + node := createCommentNode(&cb, cb.Language) + if node == nil { + t.Fatal("expected non-nil ASTNode") + } + if node.Parent == nil { + t.Fatal("expected non-nil Parent") + } + if len(node.Parent.Children) != 1 || node.Parent.Children[0] != node { + t.Fatalf("expected Parent.Children to contain node, got %v", node.Parent.Children) + } + if idx := node.ChildIndex(); idx != 0 { + t.Errorf("expected ChildIndex() == 0, got %d", idx) + } + if lang := node.GetLanguage(); lang != "go" { + t.Errorf("expected GetLanguage() == %q, got %q", "go", lang) + } + + // Without parent + cbNoParent := CommentBlock{ + Type: "comment", + Text: "// Top-level comment", + StartByte: 0, + EndByte: 20, + Language: "go", + } + nodeNoParent := createCommentNode(&cbNoParent, cbNoParent.Language) + if nodeNoParent.Parent != nil { + t.Errorf("expected nil parent for top-level comment") + } + if idx := nodeNoParent.ChildIndex(); idx != -1 { + t.Errorf("expected ChildIndex() == -1 for parentless node, got %d", idx) + } +} + +func TestSyntheticCommentLineNodeInvariants(t *testing.T) { + cb := CommentBlock{ + Type: "block_comment", + Text: "/* line 1\n * line 2 */", + StartByte: 10, + EndByte: 40, + StartRow: 2, + StartCol: 0, + EndRow: 3, + EndCol: 11, + ParentType: "class_declaration", + ParentStart: 0, + ParentEnd: 200, + ParentRow: 1, + ParentEndRow: 20, + Language: "python", + } + + node := createCommentLineNode(&cb, " * line 2 */", 20, 32, 3, cb.Language) + if node == nil { + t.Fatal("expected non-nil ASTNode") + } + if node.Parent == nil { + t.Fatal("expected non-nil Parent") + } + if len(node.Parent.Children) != 1 || node.Parent.Children[0] != node { + t.Fatalf("expected Parent.Children to contain line node, got %v", node.Parent.Children) + } + if idx := node.ChildIndex(); idx != 0 { + t.Errorf("expected ChildIndex() == 0, got %d", idx) + } + if lang := node.GetLanguage(); lang != "python" { + t.Errorf("expected GetLanguage() == %q, got %q", "python", lang) } } @@ -217,7 +313,7 @@ func TestDiffCommentsScopedLCSNoCrossover(t *testing.T) { {Type: "comment", Text: "// step", ScopeKey: "func:doWork", StartRow: 32, EndRow: 32}, } - res := DiffComments(srcComments, dstComments) + res := DiffComments(srcComments, dstComments, nil) if len(res.Actions) != 0 { t.Fatalf("expected 0 actions for matched identical comments, got %d", len(res.Actions)) } @@ -225,3 +321,205 @@ func TestDiffCommentsScopedLCSNoCrossover(t *testing.T) { t.Errorf("expected mappings 10->12, 20->22, 30->32; got %v", res.LineMappings) } } + +func TestDiffCommentsRenamedFunction(t *testing.T) { + srcDecl := &treesitter.ASTNode{ID: 10, Type: "function_declaration", StartByte: 0, EndByte: 200} + dstDecl := &treesitter.ASTNode{ID: 25, Type: "function_declaration", StartByte: 0, EndByte: 200} + + mappings := engine.NewMapping() + mappings.Add(srcDecl, dstDecl) + + srcComments := []CommentBlock{ + { + Type: "comment", + Text: "// step 1: initialize", + StartRow: 5, + EndRow: 5, + EnclosingDecl: srcDecl, + RelativePath: "body", + }, + { + Type: "comment", + Text: "// step 2: execute", + StartRow: 10, + EndRow: 10, + EnclosingDecl: srcDecl, + RelativePath: "body", + }, + } + + dstComments := []CommentBlock{ + { + Type: "comment", + Text: "// step 1: initialize", + StartRow: 7, + EndRow: 7, + EnclosingDecl: dstDecl, + RelativePath: "body", + }, + { + Type: "comment", + Text: "// step 2: execute", + StartRow: 12, + EndRow: 12, + EnclosingDecl: dstDecl, + RelativePath: "body", + }, + } + + res := DiffComments(srcComments, dstComments, mappings) + if len(res.Actions) != 0 { + t.Fatalf("expected 0 actions for comments inside renamed function, got %d actions: %+v", len(res.Actions), res.Actions) + } + if res.LineMappings[5] != 7 || res.LineMappings[10] != 12 { + t.Errorf("expected line mappings 5->7, 10->12; got %v", res.LineMappings) + } +} + +func TestDiffCommentsLeadingDocstringRenamed(t *testing.T) { + srcDecl := &treesitter.ASTNode{ID: 10, Type: "function_declaration"} + dstDecl := &treesitter.ASTNode{ID: 25, Type: "function_declaration"} + + mappings := engine.NewMapping() + mappings.Add(srcDecl, dstDecl) + + srcComments := []CommentBlock{ + { + Type: "comment", + Text: "// CalculateTotal computes total price", + StartRow: 4, + EndRow: 4, + EnclosingDecl: srcDecl, + RelativePath: "doc", + }, + } + dstComments := []CommentBlock{ + { + Type: "comment", + Text: "// CalculateTotal computes total price", + StartRow: 8, + EndRow: 8, + EnclosingDecl: dstDecl, + RelativePath: "doc", + }, + } + + res := DiffComments(srcComments, dstComments, mappings) + if len(res.Actions) != 0 { + t.Fatalf("expected 0 actions for docstring attached to renamed function, got %d actions", len(res.Actions)) + } + if res.LineMappings[4] != 8 { + t.Errorf("expected line mapping 4->8, got %v", res.LineMappings) + } +} + +func TestDiffCommentsBranchIsolation(t *testing.T) { + srcDecl := &treesitter.ASTNode{ID: 10, Type: "function_declaration"} + dstDecl := &treesitter.ASTNode{ID: 10, Type: "function_declaration"} + + mappings := engine.NewMapping() + mappings.Add(srcDecl, dstDecl) + + srcComments := []CommentBlock{ + { + Type: "comment", + Text: "// handle error", + StartRow: 10, + EndRow: 10, + EnclosingDecl: srcDecl, + RelativePath: "body/if_statement/consequence", + }, + } + dstComments := []CommentBlock{ + { + Type: "comment", + Text: "// handle error", + StartRow: 20, + EndRow: 20, + EnclosingDecl: dstDecl, + RelativePath: "body/if_statement/alternative", + }, + } + + res := DiffComments(srcComments, dstComments, mappings) + // Because relative paths differ ("consequence" vs "alternative"), it should not match in Pass 1 LCS + // Pass 2 handles cross-scope as Delete + Insert (since dist is 10 <= 25) + if len(res.Actions) != 2 { + t.Fatalf("expected 2 actions (1 Delete, 1 Insert) across different branches, got %d", len(res.Actions)) + } +} + +func TestDiffCommentsUnmappedStrictIsolation(t *testing.T) { + srcDecl := &treesitter.ASTNode{ID: 10, Type: "function_declaration"} + dstDecl := &treesitter.ASTNode{ID: 99, Type: "function_declaration"} + + // Unmapped declarations (mappings does NOT map srcDecl to dstDecl) + mappings := engine.NewMapping() + + srcComments := []CommentBlock{ + { + Type: "comment", + Text: "// validate inputs", + StartRow: 10, + EndRow: 10, + EnclosingDecl: srcDecl, + RelativePath: "body", + }, + } + dstComments := []CommentBlock{ + { + Type: "comment", + Text: "// validate inputs", + StartRow: 100, // distant row > 25 + EndRow: 100, + EnclosingDecl: dstDecl, + RelativePath: "body", + }, + } + + res := DiffComments(srcComments, dstComments, mappings) + if len(res.Actions) != 2 { + t.Fatalf("expected 2 actions (1 Delete, 1 Insert) for unmapped distant declarations, got %d", len(res.Actions)) + } + hasDelete := false + hasInsert := false + for _, act := range res.Actions { + if act.Type == actions.Delete { + hasDelete = true + } + if act.Type == actions.Insert { + hasInsert = true + } + } + if !hasDelete || !hasInsert { + t.Errorf("expected 1 Delete and 1 Insert action, got: %+v", res.Actions) + } +} + +func TestExtractCommentsInsideFunctionNotDocComment(t *testing.T) { + src := []byte(` +public class TestClass { + public void releaseByteBuffer(int ix, byte[] buffer) { + // 13-Jan-2024, tatu: [core#1186] Replace only if beneficial: + byte[] oldBuffer = _byteBuffers.get(ix); + } +} +`) + + _, flatNodes, symbols, err := treesitter.ParseForPipeline(src, "java") + if err != nil { + t.Fatal(err) + } + + comments := ExtractComments(flatNodes, symbols, src, "java") + if len(comments) != 1 { + t.Fatalf("expected 1 comment, got %d", len(comments)) + } + c := comments[0] + if c.DeclType != "method_declaration" { + t.Errorf("expected DeclType to be 'method_declaration', got %q", c.DeclType) + } + if c.RelativePath == "doc" { + t.Errorf("expected RelativePath NOT to be 'doc' for comment inside method body, got %q", c.RelativePath) + } +} diff --git a/internal/comments/diff.go b/internal/comments/diff.go index 6f12b088..9c24fa35 100644 --- a/internal/comments/diff.go +++ b/internal/comments/diff.go @@ -1,6 +1,7 @@ package comments import ( + "fmt" "strings" "github.com/HarshK97/diffmantic/internal/actions" @@ -14,8 +15,8 @@ type DiffResult struct { LineMappings map[int]int } -// DiffComments matches and diffs comments between source and destination files. -func DiffComments(srcComments, dstComments []CommentBlock) *DiffResult { +// DiffComments matches and diffs comments between source and destination files with AST mapping awareness. +func DiffComments(srcComments, dstComments []CommentBlock, mappings *engine.Mapping) *DiffResult { res := &DiffResult{ LineMappings: make(map[int]int), } @@ -26,16 +27,19 @@ func DiffComments(srcComments, dstComments []CommentBlock) *DiffResult { srcMatched := make([]bool, len(srcComments)) dstMatched := make([]bool, len(dstComments)) - // Match identical comments within each scope using LCS so repeated comments don't cross over. + // Match identical comments within each canonical mapped scope using LCS so repeated comments don't cross over. scopeSrcMap := make(map[string][]int) scopeDstMap := make(map[string][]int) for i := range srcComments { - scopeSrcMap[srcComments[i].ScopeKey] = append(scopeSrcMap[srcComments[i].ScopeKey], i) + key := canonicalScopeKey(&srcComments[i], mappings, true) + scopeSrcMap[key] = append(scopeSrcMap[key], i) } for j := range dstComments { - scopeDstMap[dstComments[j].ScopeKey] = append(scopeDstMap[dstComments[j].ScopeKey], j) + key := canonicalScopeKey(&dstComments[j], mappings, false) + scopeDstMap[key] = append(scopeDstMap[key], j) } + // PASS 1: Intra-Scope Monotonic LCS Dynamic Programming for scopeKey, srcIdxs := range scopeSrcMap { dstIdxs := scopeDstMap[scopeKey] if len(dstIdxs) == 0 { @@ -71,7 +75,7 @@ func DiffComments(srcComments, dstComments []CommentBlock) *DiffResult { dstMatched[dj] = true sc := &srcComments[si] dc := &dstComments[dj] - nLines := min(sc.EndRow-sc.StartRow+1, dc.EndRow-dc.StartRow+1) + nLines := min(commentLineCount(sc), commentLineCount(dc)) for k := 0; k < nLines; k++ { res.LineMappings[sc.StartRow+k] = dc.StartRow + k } @@ -85,7 +89,7 @@ func DiffComments(srcComments, dstComments []CommentBlock) *DiffResult { } } - // Exact text matches across different scopes (treated as moved comments). + // PASS 2: Exact text matches across different scopes (treated as moved comments). for i := range srcComments { if srcMatched[i] { continue @@ -112,30 +116,36 @@ func DiffComments(srcComments, dstComments []CommentBlock) *DiffResult { srcMatched[i] = true dstMatched[bestJ] = true dc := &dstComments[bestJ] - nLines := min(sc.EndRow-sc.StartRow+1, dc.EndRow-dc.StartRow+1) - for k := 0; k < nLines; k++ { - res.LineMappings[sc.StartRow+k] = dc.StartRow + k - } - if sc.ScopeKey != dc.ScopeKey { - srcNode := createCommentNode(sc) - dstNode := createCommentNode(dc) - res.Actions = append(res.Actions, actions.Action{ - Type: actions.Move, - Node: srcNode, - DestNode: dstNode, - Parent: dstNode.Parent, - }) + scKey := canonicalScopeKey(sc, mappings, true) + dcKey := canonicalScopeKey(dc, mappings, false) + if scKey == dcKey { + nLines := min(commentLineCount(sc), commentLineCount(dc)) + for k := 0; k < nLines; k++ { + res.LineMappings[sc.StartRow+k] = dc.StartRow + k + } + } else { + res.Actions = append(res.Actions, + actions.Action{ + Type: actions.Delete, + Node: createCommentNode(sc, sc.Language), + }, + actions.Action{ + Type: actions.Insert, + Node: createCommentNode(dc, dc.Language), + }, + ) } } } - // Fuzzy match edited comments in the same scope. + // PASS 3: Fuzzy match edited comments in the same scope. for i := range srcComments { if srcMatched[i] { continue } sc := &srcComments[i] + scKey := canonicalScopeKey(sc, mappings, true) bestJ := -1 bestScore := 0.0 @@ -144,13 +154,14 @@ func DiffComments(srcComments, dstComments []CommentBlock) *DiffResult { continue } dc := &dstComments[j] + dcKey := canonicalScopeKey(dc, mappings, false) - if sc.ScopeKey != dc.ScopeKey { + if scKey != dcKey { continue } - srcIsMulti := strings.Contains(sc.Text, "\n") - dstIsMulti := strings.Contains(dc.Text, "\n") + srcIsMulti := strings.Contains(strings.TrimRight(sc.Text, "\r\n"), "\n") + dstIsMulti := strings.Contains(strings.TrimRight(dc.Text, "\r\n"), "\n") sim := stringSimilarity(sc.Text, dc.Text) minThreshold := 0.40 @@ -180,7 +191,7 @@ func DiffComments(srcComments, dstComments []CommentBlock) *DiffResult { for i := range srcComments { if !srcMatched[i] { sc := &srcComments[i] - node := createCommentNode(sc) + node := createCommentNode(sc, sc.Language) res.Actions = append(res.Actions, actions.Action{ Type: actions.Delete, Node: node, @@ -191,7 +202,7 @@ func DiffComments(srcComments, dstComments []CommentBlock) *DiffResult { for j := range dstComments { if !dstMatched[j] { dc := &dstComments[j] - node := createCommentNode(dc) + node := createCommentNode(dc, dc.Language) res.Actions = append(res.Actions, actions.Action{ Type: actions.Insert, Node: node, @@ -203,13 +214,16 @@ func DiffComments(srcComments, dstComments []CommentBlock) *DiffResult { } func diffCommentBlock(sc, dc *CommentBlock, res *DiffResult) { - srcIsMulti := strings.Contains(sc.Text, "\n") - dstIsMulti := strings.Contains(dc.Text, "\n") + srcTrimmed := strings.TrimRight(sc.Text, "\r\n") + dstTrimmed := strings.TrimRight(dc.Text, "\r\n") + + srcIsMulti := strings.Contains(srcTrimmed, "\n") + dstIsMulti := strings.Contains(dstTrimmed, "\n") if !srcIsMulti && !dstIsMulti { res.LineMappings[sc.StartRow] = dc.StartRow - srcNode := createCommentNode(sc) - dstNode := createCommentNode(dc) + srcNode := createCommentNode(sc, sc.Language) + dstNode := createCommentNode(dc, dc.Language) res.Actions = append(res.Actions, actions.Action{ Type: actions.Update, Node: srcNode, @@ -220,8 +234,8 @@ func diffCommentBlock(sc, dc *CommentBlock, res *DiffResult) { } // Line-by-line diff for multiline comments. - srcLines := strings.Split(sc.Text, "\n") - dstLines := strings.Split(dc.Text, "\n") + srcLines := strings.Split(srcTrimmed, "\n") + dstLines := strings.Split(dstTrimmed, "\n") matchedA := engine.LineDiff(srcLines, dstLines) matchedB := make(map[int]int) @@ -269,8 +283,8 @@ func diffCommentBlock(sc, dc *CommentBlock, res *DiffResult) { row := uint32(sc.StartRow + i) dstRow := uint32(dc.StartRow + j) - srcNode := createCommentLineNode(sc, srcLines[i], startByte, endByte, row) - dstNode := createCommentLineNode(dc, dstLines[j], dstStartByte, dstEndByte, dstRow) + srcNode := createCommentLineNode(sc, srcLines[i], startByte, endByte, row, sc.Language) + dstNode := createCommentLineNode(dc, dstLines[j], dstStartByte, dstEndByte, dstRow, dc.Language) res.Actions = append(res.Actions, actions.Action{ Type: actions.Update, Node: srcNode, @@ -285,7 +299,7 @@ func diffCommentBlock(sc, dc *CommentBlock, res *DiffResult) { if _, ok := matchedA[i]; !ok { startByte, endByte := srcOffsets[i][0], srcOffsets[i][1] row := uint32(sc.StartRow + i) - lineNode := createCommentLineNode(sc, srcLines[i], startByte, endByte, row) + lineNode := createCommentLineNode(sc, srcLines[i], startByte, endByte, row, sc.Language) res.Actions = append(res.Actions, actions.Action{ Type: actions.Delete, Node: lineNode, @@ -297,7 +311,7 @@ func diffCommentBlock(sc, dc *CommentBlock, res *DiffResult) { if _, ok := matchedB[j]; !ok { startByte, endByte := dstOffsets[j][0], dstOffsets[j][1] row := uint32(dc.StartRow + j) - lineNode := createCommentLineNode(dc, dstLines[j], startByte, endByte, row) + lineNode := createCommentLineNode(dc, dstLines[j], startByte, endByte, row, dc.Language) res.Actions = append(res.Actions, actions.Action{ Type: actions.Insert, Node: lineNode, @@ -348,23 +362,27 @@ func computeLineOffsets(baseOffset uint32, lines []string) [][2]uint32 { return offsets } -func createCommentNode(c *CommentBlock) *treesitter.ASTNode { - node := createSyntheticNode(c.Type, c.Text, c.StartByte, c.EndByte, uint32(c.StartRow), uint32(c.StartCol), uint32(c.EndRow), uint32(c.EndCol)) +func createCommentNode(c *CommentBlock, lang string) *treesitter.ASTNode { + node := createSyntheticNode(c.Type, c.Text, c.StartByte, c.EndByte, uint32(c.StartRow), uint32(c.StartCol), uint32(c.EndRow), uint32(c.EndCol), lang) if c.ParentType != "" { - node.Parent = createSyntheticNode(c.ParentType, "", c.ParentStart, c.ParentEnd, uint32(c.ParentRow), 0, uint32(c.ParentEndRow), 0) + parent := createSyntheticNode(c.ParentType, "", c.ParentStart, c.ParentEnd, uint32(c.ParentRow), 0, uint32(c.ParentEndRow), 0, lang) + node.Parent = parent + parent.Children = []*treesitter.ASTNode{node} } return node } -func createCommentLineNode(c *CommentBlock, label string, startByte, endByte, row uint32) *treesitter.ASTNode { - node := createSyntheticNode(c.Type, label, startByte, endByte, row, 0, row, uint32(len(label))) +func createCommentLineNode(c *CommentBlock, label string, startByte, endByte, row uint32, lang string) *treesitter.ASTNode { + node := createSyntheticNode(c.Type, label, startByte, endByte, row, 0, row, uint32(len(label)), lang) if c.ParentType != "" { - node.Parent = createSyntheticNode(c.ParentType, "", c.ParentStart, c.ParentEnd, uint32(c.ParentRow), 0, uint32(c.ParentEndRow), 0) + parent := createSyntheticNode(c.ParentType, "", c.ParentStart, c.ParentEnd, uint32(c.ParentRow), 0, uint32(c.ParentEndRow), 0, lang) + node.Parent = parent + parent.Children = []*treesitter.ASTNode{node} } return node } -func createSyntheticNode(nodeType, label string, startByte, endByte, startRow, startCol, endRow, endCol uint32) *treesitter.ASTNode { +func createSyntheticNode(nodeType, label string, startByte, endByte, startRow, startCol, endRow, endCol uint32, lang string) *treesitter.ASTNode { return &treesitter.ASTNode{ Type: nodeType, Label: label, @@ -374,5 +392,43 @@ func createSyntheticNode(nodeType, label string, startByte, endByte, startRow, s StartCol: startCol, EndRow: endRow, EndCol: endCol, + Language: lang, + Children: make([]*treesitter.ASTNode, 0), + } +} + +func canonicalScopeKey(c *CommentBlock, mappings *engine.Mapping, isSource bool) string { + if c == nil { + return "root" + } + if c.EnclosingDecl == nil { + if c.RelativePath != "" { + return "root:" + c.RelativePath + } + if c.ScopeKey != "" { + return c.ScopeKey + } + return "root" + } + + if isSource { + if mappings != nil && mappings.Src() != nil { + if dstDecl, ok := mappings.Src()[c.EnclosingDecl]; ok && dstDecl != nil { + return fmt.Sprintf("decl_%d:%s", dstDecl.ID, c.RelativePath) + } + } + // Strict isolation for unmapped source declarations + return fmt.Sprintf("unmapped_src_%d:%s", c.EnclosingDecl.ID, c.RelativePath) + } + + // Destination declaration scope key + return fmt.Sprintf("decl_%d:%s", c.EnclosingDecl.ID, c.RelativePath) +} + +func commentLineCount(c *CommentBlock) int { + t := strings.TrimRight(c.Text, "\r\n") + if t == "" { + return 1 } + return strings.Count(t, "\n") + 1 } diff --git a/internal/comments/extract.go b/internal/comments/extract.go index fc0bc578..5ab2eecf 100644 --- a/internal/comments/extract.go +++ b/internal/comments/extract.go @@ -5,10 +5,12 @@ import ( "slices" "strings" + "github.com/HarshK97/diffmantic/internal/treesitter" "github.com/HarshK97/diffmantic/internal/treesitter/rules" - "github.com/odvcencio/gotreesitter" ) +const flatSentinel = 0xFFFFFFFF + // CommentBlock represents an extracted comment and its source position. type CommentBlock struct { Type string @@ -25,27 +27,48 @@ type CommentBlock struct { ParentEnd uint32 ParentRow int ParentEndRow int + Language string + + // Structural Declaration & Scope Context + DeclStart uint32 + DeclEnd uint32 + DeclType string + RelativePath string + EnclosingDecl *treesitter.ASTNode } -// ExtractComments walks the AST and returns all comment blocks. -func ExtractComments(root *gotreesitter.Node, src []byte, lang *gotreesitter.Language, r *rules.Rules) []CommentBlock { - if root == nil || lang == nil { +// ExtractComments walks the raw flat buffer (pre-IsIgnored filtering) +// and extracts comment blocks with full CST sibling/parent context. +func ExtractComments(nodes []treesitter.FlatNode, symbols []string, src []byte, langName string) []CommentBlock { + if len(nodes) == 0 { return nil } + r := rules.Get(langName) + srcLen := uint32(len(src)) var list []CommentBlock - traverseForComments(root, src, lang, r, &list) - return list -} -func traverseForComments(n *gotreesitter.Node, src []byte, lang *gotreesitter.Language, r *rules.Rules, list *[]CommentBlock) { - if n == nil { - return - } - nodeType := n.Type(lang) - if (r != nil && r.IsComment(nodeType)) || (r == nil && nodeType == "comment") { - srcLen := uint32(len(src)) - start := min(n.StartByte(), srcLen) - end := min(n.EndByte(), srcLen) + for idx := range nodes { + fn := &nodes[idx] + rawType := flatSymbol(fn, symbols) + if !isCommentType(rawType, r) { + continue + } + + // Skip child comment tokens whose ancestor is already a comment container + // (e.g. in Rust where line_comment wraps doc_comment). + isChildComment := false + for pIdx := fn.ParentIdx; pIdx != flatSentinel && int(pIdx) < len(nodes); pIdx = nodes[pIdx].ParentIdx { + if isCommentType(flatSymbol(&nodes[pIdx], symbols), r) { + isChildComment = true + break + } + } + if isChildComment { + continue + } + + start := min(fn.StartByte, srcLen) + end := min(fn.EndByte, srcLen) if start > end { start = end } @@ -54,51 +77,163 @@ func traverseForComments(n *gotreesitter.Node, src []byte, lang *gotreesitter.La var parentType string var parentStart, parentEnd uint32 var parentRow, parentEndRow int - if p := n.Parent(); p != nil { - parentType = p.Type(lang) - parentStart = min(p.StartByte(), srcLen) - parentEnd = min(p.EndByte(), srcLen) - parentRow = int(p.StartPoint().Row) - parentEndRow = int(p.EndPoint().Row) + if fn.ParentIdx != flatSentinel && int(fn.ParentIdx) < len(nodes) { + p := &nodes[fn.ParentIdx] + parentType = flatSymbol(p, symbols) + parentStart = min(p.StartByte, srcLen) + parentEnd = min(p.EndByte, srcLen) + parentRow = int(p.StartRow) + parentEndRow = int(p.EndRow) } else { parentEnd = srcLen } - scopeKey := findEnclosingDeclaration(n, src, lang, r) + scopeKey := flatFindEnclosingDeclaration(uint32(idx), nodes, symbols, src, r) - *list = append(*list, CommentBlock{ - Type: nodeType, + declStart, declEnd, declType, relPath := flatCheckLeadingDocComment(uint32(idx), nodes, symbols, src, r) + if declType == "" { + declStart, declEnd, declType, relPath = flatFindEnclosingDeclarationHierarchy(uint32(idx), nodes, symbols, src, r) + } + + list = append(list, CommentBlock{ + Type: rawType, Text: text, StartByte: start, EndByte: end, - StartRow: int(n.StartPoint().Row), - StartCol: int(n.StartPoint().Column), - EndRow: int(n.EndPoint().Row), - EndCol: int(n.EndPoint().Column), + StartRow: int(fn.StartRow), + StartCol: int(fn.StartCol), + EndRow: int(fn.EndRow), + EndCol: int(fn.EndCol), ScopeKey: scopeKey, ParentType: parentType, ParentStart: parentStart, ParentEnd: parentEnd, ParentRow: parentRow, ParentEndRow: parentEndRow, + Language: langName, + DeclStart: declStart, + DeclEnd: declEnd, + DeclType: declType, + RelativePath: relPath, }) - return } - for i := range n.ChildCount() { - traverseForComments(n.Child(i), src, lang, r, list) + return list +} + +func flatSymbol(fn *treesitter.FlatNode, symbols []string) string { + if int(fn.TypeID) < len(symbols) { + return symbols[fn.TypeID] + } + return "" +} + +func isCommentType(nodeType string, r *rules.Rules) bool { + if r != nil { + return r.IsComment(nodeType) + } + return rules.IsComment(nodeType) +} + +// flatCheckLeadingDocComment mirrors checkLeadingDocComment using flat buffer sibling walking. +func flatCheckLeadingDocComment(idx uint32, nodes []treesitter.FlatNode, symbols []string, src []byte, r *rules.Rules) (uint32, uint32, string, string) { + if r == nil { + return 0, 0, "", "" + } + fn := &nodes[idx] + + // Walk NextSiblingIdx to find the first non-comment, non-"\n" sibling + nextIdx := fn.NextSiblingIdx + for nextIdx != flatSentinel && int(nextIdx) < len(nodes) { + nextType := flatSymbol(&nodes[nextIdx], symbols) + if !isCommentType(nextType, r) && nextType != "\n" { + break + } + nextIdx = nodes[nextIdx].NextSiblingIdx + } + if nextIdx == flatSentinel || int(nextIdx) >= len(nodes) { + return 0, 0, "", "" + } + + target := &nodes[nextIdx] + t := flatSymbol(target, symbols) + if !r.IsDeclaration(t) { + // Check inner children for wrapped declarations (export_statement, etc.) + childIdx := target.FirstChildIdx + for childIdx != flatSentinel && int(childIdx) < len(nodes) { + ct := flatSymbol(&nodes[childIdx], symbols) + if r.IsDeclaration(ct) { + target = &nodes[childIdx] + t = ct + break + } + childIdx = nodes[childIdx].NextSiblingIdx + } + } + if !r.IsDeclaration(t) { + return 0, 0, "", "" + } + + // Check if comment is inside a block (interior statement comment vs exterior doc comment) + pIdx := fn.ParentIdx + insideBlock := false + for pIdx != flatSentinel && int(pIdx) < len(nodes) { + if r.IsBlock(flatSymbol(&nodes[pIdx], symbols)) { + insideBlock = true + break + } + pIdx = nodes[pIdx].ParentIdx + } + if insideBlock && r.IsLocalVarDeclaration(t) { + return 0, 0, "", "" + } + + if int(target.StartRow)-int(fn.EndRow) <= 1 { + return target.StartByte, target.EndByte, t, "doc" } + return 0, 0, "", "" } -func findEnclosingDeclaration(n *gotreesitter.Node, src []byte, lang *gotreesitter.Language, r *rules.Rules) string { - if r == nil || n == nil { +// flatFindEnclosingDeclarationHierarchy mirrors findEnclosingDeclarationHierarchy. +func flatFindEnclosingDeclarationHierarchy(idx uint32, nodes []treesitter.FlatNode, symbols []string, src []byte, r *rules.Rules) (uint32, uint32, string, string) { + if r == nil { + return 0, 0, "", "root" + } + var containers []string + pIdx := nodes[idx].ParentIdx + for pIdx != flatSentinel && int(pIdx) < len(nodes) { + t := flatSymbol(&nodes[pIdx], symbols) + if r.IsDeclaration(t) { + relPath := "body" + if len(containers) > 0 { + slices.Reverse(containers) + relPath = "body/" + strings.Join(containers, "/") + } + return nodes[pIdx].StartByte, nodes[pIdx].EndByte, t, relPath + } + if r.IsBlock(t) || r.IsScaffolding(t) { + containers = append(containers, t) + } + pIdx = nodes[pIdx].ParentIdx + } + if len(containers) > 0 { + slices.Reverse(containers) + return 0, 0, "", "root/" + strings.Join(containers, "/") + } + return 0, 0, "", "root" +} + +// flatFindEnclosingDeclaration mirrors findEnclosingDeclaration. +func flatFindEnclosingDeclaration(idx uint32, nodes []treesitter.FlatNode, symbols []string, src []byte, r *rules.Rules) string { + if r == nil { return "root" } + srcLen := uint32(len(src)) var containers []string - curr := n.Parent() - for curr != nil { - t := curr.Type(lang) + pIdx := nodes[idx].ParentIdx + for pIdx != flatSentinel && int(pIdx) < len(nodes) { + t := flatSymbol(&nodes[pIdx], symbols) if r.IsDeclaration(t) { - name := getDeclarationIdentifier(curr, src, lang, r) + name := flatGetDeclarationIdentifier(&nodes[pIdx], nodes, symbols, src, r, srcLen) decl := t if name != "" { decl = t + ":" + name @@ -112,7 +247,7 @@ func findEnclosingDeclaration(n *gotreesitter.Node, src []byte, lang *gotreesitt if r.IsBlock(t) || r.IsScaffolding(t) { containers = append(containers, t) } - curr = curr.Parent() + pIdx = nodes[pIdx].ParentIdx } if len(containers) > 0 { slices.Reverse(containers) @@ -121,35 +256,71 @@ func findEnclosingDeclaration(n *gotreesitter.Node, src []byte, lang *gotreesitt return "root" } -func getDeclarationIdentifier(n *gotreesitter.Node, src []byte, lang *gotreesitter.Language, r *rules.Rules) string { - if n == nil || r == nil { +func flatGetDeclarationIdentifier(fn *treesitter.FlatNode, nodes []treesitter.FlatNode, symbols []string, src []byte, r *rules.Rules, srcLen uint32) string { + if r == nil { return "" } - for i := range n.ChildCount() { - child := n.Child(i) - if child == nil { - continue - } - ct := child.Type(lang) + childIdx := fn.FirstChildIdx + for childIdx != flatSentinel && int(childIdx) < len(nodes) { + child := &nodes[childIdx] + ct := flatSymbol(child, symbols) if r.IsIdentifier(ct) { - s := min(child.StartByte(), uint32(len(src))) - e := min(child.EndByte(), uint32(len(src))) + s := min(child.StartByte, srcLen) + e := min(child.EndByte, srcLen) if s < e { return string(src[s:e]) } } if r.IsScaffolding(ct) { - for j := range child.ChildCount() { - sub := child.Child(j) - if sub != nil && r.IsIdentifier(sub.Type(lang)) { - s := min(sub.StartByte(), uint32(len(src))) - e := min(sub.EndByte(), uint32(len(src))) + subIdx := child.FirstChildIdx + for subIdx != flatSentinel && int(subIdx) < len(nodes) { + sub := &nodes[subIdx] + if r.IsIdentifier(flatSymbol(sub, symbols)) { + s := min(sub.StartByte, srcLen) + e := min(sub.EndByte, srcLen) if s < e { return string(src[s:e]) } } + subIdx = sub.NextSiblingIdx } } + childIdx = child.NextSiblingIdx } return "" } + +type declSpanKey struct { + start uint32 + end uint32 +} + +// BindASTNodes attaches exact AST node pointers to extracted comment blocks in O(K + N). +func BindASTNodes(comments []CommentBlock, root *treesitter.ASTNode) { + if len(comments) == 0 || root == nil { + return + } + declMap := make(map[declSpanKey]*treesitter.ASTNode) + var indexDecls func(n *treesitter.ASTNode) + indexDecls = func(n *treesitter.ASTNode) { + if n == nil { + return + } + key := declSpanKey{start: n.StartByte, end: n.EndByte} + if existing, exists := declMap[key]; !exists || (existing != nil && existing.Type != n.Type) { + declMap[key] = n + } + for _, child := range n.Children { + indexDecls(child) + } + } + indexDecls(root) + + for i := range comments { + if comments[i].DeclEnd > comments[i].DeclStart || comments[i].DeclType != "" { + if declNode, ok := declMap[declSpanKey{start: comments[i].DeclStart, end: comments[i].DeclEnd}]; ok { + comments[i].EnclosingDecl = declNode + } + } + } +} diff --git a/internal/inline/render.go b/internal/inline/render.go index d0070614..cddd5161 100644 --- a/internal/inline/render.go +++ b/internal/inline/render.go @@ -222,10 +222,10 @@ func Render(srcFile, dstFile string, srcBytes, dstBytes []byte, env *serialize.E // Resolve language rules for declaration classification var r *rules.Rules - if entry := treesitter.DetectGrammarEntry(srcFile); entry != nil { - r = rules.Get(entry.Name) - } else if entry := treesitter.DetectGrammarEntry(dstFile); entry != nil { - r = rules.Get(entry.Name) + if lang, _ := treesitter.DetectLanguage(srcFile); lang != nil { + r = rules.Get(lang.Name) + } else if lang, _ := treesitter.DetectLanguage(dstFile); lang != nil { + r = rules.Get(lang.Name) } meta := buildHunkMoveMetadata(env.Actions, hunks, filteredPairs, srcOffsets, dstOffsets, srcLines, dstLines, r) diff --git a/internal/pipeline/pipeline.go b/internal/pipeline/pipeline.go index 7ceac271..affe2ae9 100644 --- a/internal/pipeline/pipeline.go +++ b/internal/pipeline/pipeline.go @@ -12,18 +12,22 @@ import ( "github.com/HarshK97/diffmantic/internal/postprocess" "github.com/HarshK97/diffmantic/internal/serialize" "github.com/HarshK97/diffmantic/internal/treesitter" - "github.com/HarshK97/diffmantic/internal/treesitter/rules" - "github.com/odvcencio/gotreesitter" ) // MaxASTFileSize caps the file size for AST parsing before falling back to line diffing. -const MaxASTFileSize = 400 * 1024 +const MaxASTFileSize = 1024 * 1024 + +// MaxASTFileLines caps the line count for AST parsing before falling back to line diffing. +const MaxASTFileLines = 10000 // DiffOptions configures parsing limits, comment handling, and output options. type DiffOptions struct { ParseErrorLimit int DisableErrorFallback bool DisableSizeLimit bool + MaxASTFileSize int + DisableLineLimit bool + MaxASTFileLines int IsConflict bool IgnoreComments bool EnvelopeOpts serialize.EnvelopeOptions @@ -35,6 +39,7 @@ type DiffResult struct { DstBytes []byte SrcFile string DstFile string + IsBinary bool SrcAST *treesitter.ASTNode DstAST *treesitter.ASTNode MatchResult *engine.MatchResult @@ -42,6 +47,15 @@ type DiffResult struct { Envelope *serialize.Envelope } +// IsBinary detects whether a byte buffer contains binary data (null bytes in the first 8000 bytes). +func IsBinary(data []byte) bool { + sample := data + if len(sample) > 8000 { + sample = sample[:8000] + } + return bytes.IndexByte(sample, 0) != -1 +} + // HasConflictMarkers checks if the buffer contains Git merge conflict markers. func HasConflictMarkers(data []byte) bool { hasStart := bytes.HasPrefix(data, []byte("<<<<<<<")) || bytes.Contains(data, []byte("\n<<<<<<<")) @@ -61,8 +75,37 @@ func Run(srcBytes, dstBytes []byte, srcFile, dstFile string, opts DiffOptions) ( } } + if IsBinary(srcBytes) || IsBinary(dstBytes) { + return &DiffResult{ + SrcBytes: srcBytes, + DstBytes: dstBytes, + SrcFile: srcFile, + DstFile: dstFile, + IsBinary: true, + Envelope: &serialize.Envelope{ + Version: serialize.SchemaVersion, + IsBinary: true, + }, + }, nil + } + + maxSize := MaxASTFileSize + if opts.MaxASTFileSize > 0 { + maxSize = opts.MaxASTFileSize + } + + maxLines := MaxASTFileLines + if opts.MaxASTFileLines > 0 { + maxLines = opts.MaxASTFileLines + } + + exceedsLines := !opts.DisableLineLimit && maxLines > 0 && + (bytes.Count(srcBytes, []byte{'\n'}) > maxLines || bytes.Count(dstBytes, []byte{'\n'}) > maxLines) + + exceedsSize := !opts.DisableSizeLimit && (len(srcBytes) > maxSize || len(dstBytes) > maxSize) + if opts.IsConflict || (HasConflictMarkers(srcBytes) || HasConflictMarkers(dstBytes)) || - (!opts.DisableSizeLimit && (len(srcBytes) > MaxASTFileSize || len(dstBytes) > MaxASTFileSize)) { + exceedsSize || exceedsLines { return &DiffResult{ SrcBytes: srcBytes, DstBytes: dstBytes, @@ -93,14 +136,9 @@ func Run(srcBytes, dstBytes []byte, srcFile, dstFile string, opts DiffOptions) ( langB = langA } - rulesA := rules.Get(langA.Name) - rulesB := rules.Get(langB.Name) - var ( srcAST *treesitter.ASTNode dstAST *treesitter.ASTNode - srcTree *gotreesitter.Tree - dstTree *gotreesitter.Tree srcComments []comments.CommentBlock dstComments []comments.CommentBlock wg sync.WaitGroup @@ -109,16 +147,30 @@ func Run(srcBytes, dstBytes []byte, srcFile, dstFile string, opts DiffOptions) ( wg.Add(2) go func() { defer wg.Done() - srcAST, srcTree, _ = treesitter.ParseWithLanguageAndTree(srcBytes, langA) - if srcTree != nil && !opts.IgnoreComments { - srcComments = comments.ExtractComments(srcTree.RootNode(), srcBytes, langA, rulesA) + var ( + srcFlatNodes []treesitter.FlatNode + srcSymbols []string + ) + srcAST, srcFlatNodes, srcSymbols, _ = treesitter.ParseForPipeline(srcBytes, langA.Name) + if !opts.IgnoreComments && len(srcFlatNodes) > 0 { + srcComments = comments.ExtractComments(srcFlatNodes, srcSymbols, srcBytes, langA.Name) + if srcAST != nil { + comments.BindASTNodes(srcComments, srcAST) + } } }() go func() { defer wg.Done() - dstAST, dstTree, _ = treesitter.ParseWithLanguageAndTree(dstBytes, langB) - if dstTree != nil && !opts.IgnoreComments { - dstComments = comments.ExtractComments(dstTree.RootNode(), dstBytes, langB, rulesB) + var ( + dstFlatNodes []treesitter.FlatNode + dstSymbols []string + ) + dstAST, dstFlatNodes, dstSymbols, _ = treesitter.ParseForPipeline(dstBytes, langB.Name) + if !opts.IgnoreComments && len(dstFlatNodes) > 0 { + dstComments = comments.ExtractComments(dstFlatNodes, dstSymbols, dstBytes, langB.Name) + if dstAST != nil { + comments.BindASTNodes(dstComments, dstAST) + } } }() @@ -135,29 +187,29 @@ func Run(srcBytes, dstBytes []byte, srcFile, dstFile string, opts DiffOptions) ( }, nil } + matchResult := engine.Match(srcAST, dstAST, srcBytes, dstBytes, part) + var ( - matchResult *engine.MatchResult - commentRes *comments.DiffResult - matchWg sync.WaitGroup + commentRes *comments.DiffResult + es *actions.EditScript + wgPost sync.WaitGroup ) - matchWg.Add(1) + wgPost.Add(1) go func() { - defer matchWg.Done() - matchResult = engine.Match(srcAST, dstAST, srcBytes, dstBytes, part) + defer wgPost.Done() + es = actions.GenerateEditScript(srcAST, dstAST, matchResult.Mappings) }() if !opts.IgnoreComments && (len(srcComments) > 0 || len(dstComments) > 0) { - matchWg.Add(1) + wgPost.Add(1) go func() { - defer matchWg.Done() - commentRes = comments.DiffComments(srcComments, dstComments) + defer wgPost.Done() + commentRes = comments.DiffComments(srcComments, dstComments, matchResult.Mappings) }() } - matchWg.Wait() - - es := actions.GenerateEditScript(srcAST, dstAST, matchResult.Mappings) + wgPost.Wait() if commentRes != nil && len(commentRes.Actions) > 0 { for _, act := range commentRes.Actions { diff --git a/internal/pipeline/pipeline_test.go b/internal/pipeline/pipeline_test.go new file mode 100644 index 00000000..13b3376d --- /dev/null +++ b/internal/pipeline/pipeline_test.go @@ -0,0 +1,49 @@ +package pipeline + +import ( + "strings" + "testing" +) + +func TestPipeline_LineLimitFallback(t *testing.T) { + // Build a small Go file with 12 lines + var lines []string + for i := 0; i < 12; i++ { + lines = append(lines, "package main") + } + content := []byte(strings.Join(lines, "\n")) + + // With maxLines=10 and 12-line file, it should fall back to line diff (no AST). + res, err := Run(content, content, "main.go", "main.go", DiffOptions{ + MaxASTFileLines: 10, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if res.SrcAST != nil || res.DstAST != nil { + t.Errorf("expected AST nil on line limit fallback, got non-nil") + } + + // With DisableLineLimit=true, AST should parse regardless. + resUnlimited, err := Run(content, content, "main.go", "main.go", DiffOptions{ + MaxASTFileLines: 10, + DisableLineLimit: true, + }) + if err != nil { + t.Fatalf("unexpected error with disabled limit: %v", err) + } + if resUnlimited.SrcAST == nil { + t.Errorf("expected AST parsed when line limit disabled") + } + + // With higher limit of 20, file has fewer lines — should parse as AST. + resHigher, err := Run(content, content, "main.go", "main.go", DiffOptions{ + MaxASTFileLines: 20, + }) + if err != nil { + t.Fatalf("unexpected error with higher limit: %v", err) + } + if resHigher.SrcAST == nil { + t.Errorf("expected AST parsed when file is under line limit") + } +} diff --git a/internal/serialize/json.go b/internal/serialize/json.go index 73817131..cd95ac35 100644 --- a/internal/serialize/json.go +++ b/internal/serialize/json.go @@ -46,6 +46,7 @@ type EnvelopeOptions struct { // Envelope wraps the serialized actions list with a schema version. type Envelope struct { Version string `json:"version"` + IsBinary bool `json:"is_binary,omitempty"` Actions []Action `json:"actions,omitempty"` LineAlignment []LineAlignmentPair `json:"line_alignment,omitempty"` LeftHighlights []HighlightSpan `json:"left_highlights,omitempty"` diff --git a/internal/treesitter/ast.go b/internal/treesitter/ast.go index 12f8b22e..ab4afc07 100644 --- a/internal/treesitter/ast.go +++ b/internal/treesitter/ast.go @@ -2,10 +2,8 @@ package treesitter import ( "slices" - "strings" "github.com/HarshK97/diffmantic/internal/treesitter/rules" - "github.com/odvcencio/gotreesitter" ) type ASTNode struct { @@ -68,109 +66,6 @@ func (n *ASTNode) ComputeHashes() { n.StructureHash = sh } -func BuildAST(n *gotreesitter.Node, src []byte, lang *gotreesitter.Language, parent *ASTNode) *ASTNode { - if n == nil { - return nil - } - r := rules.Get(lang.Name) - node := buildASTWithRules(n, src, lang, parent, r) - if node != nil && parent == nil { - errCount := countErrorNodes(n, lang) - node.Language = lang.Name - node.ParseErrorCount = errCount - node.HasError = errCount > 0 - node.ComputeHashes() - EnsureIndex(node) - } - return node -} - -func countErrorNodes(n *gotreesitter.Node, lang *gotreesitter.Language) int { - if n == nil || !n.HasError() { - return 0 - } - count := 0 - if n.Type(lang) == "ERROR" || n.IsError() || n.IsMissing() { - count = 1 - } - for i := range n.ChildCount() { - count += countErrorNodes(n.Child(i), lang) - } - return count -} - -func buildASTWithRules(n *gotreesitter.Node, src []byte, lang *gotreesitter.Language, parent *ASTNode, r *rules.Rules) *ASTNode { - nodeType := n.Type(lang) - if n.IsMissing() { - nodeType = "MISSING " + nodeType - } - - isLeaf := n.ChildCount() == 0 || (r != nil && r.IsFlattened(nodeType)) - var label string - if isLeaf { - srcLen := uint32(len(src)) - start, end := min(n.StartByte(), srcLen), min(n.EndByte(), srcLen) - if start > end { - start = end - } - label = strings.TrimSpace(string(src[start:end])) - } - - if r != nil && r.IsIgnored(nodeType, label) { - return nil - } - - node := &ASTNode{ - Type: nodeType, - Parent: parent, - StartByte: n.StartByte(), - EndByte: n.EndByte(), - StartRow: n.StartPoint().Row, - StartCol: n.StartPoint().Column, - EndRow: n.EndPoint().Row, - EndCol: n.EndPoint().Column, - } - - // Only set label for leaf nodes or string literals. - if isLeaf { - node.Label = label - } - - if r != nil { - if alias, ok := r.Alias(nodeType, label); ok { - node.Type = alias - } - if r.IsLabelIgnored(node.Type) { - node.Label = "" - } - if isLeaf && r.IsKeyword(nodeType, label) { - node.IsKeyword = true - } - if r.IsUnordered(node.Type) { - node.IsUnordered = true - } - } - - for i := range n.ChildCount() { - if child := buildASTWithRules(n.Child(i), src, lang, node, r); child != nil { - node.Children = append(node.Children, child) - } - } - - if r != nil && r.IsFlattened(nodeType) { - var flattenedChildren []*ASTNode - for _, child := range node.Children { - flattenedChildren = append(flattenedChildren, child.Children...) - for _, grandchild := range child.Children { - grandchild.Parent = node - } - } - node.Children = flattenedChildren - } - - return node -} - // Size returns the total number of nodes in the subtree rooted at n. func (n *ASTNode) Size() int { if n == nil { diff --git a/internal/treesitter/bridge_cgo.go b/internal/treesitter/bridge_cgo.go new file mode 100644 index 00000000..ca30d1ff --- /dev/null +++ b/internal/treesitter/bridge_cgo.go @@ -0,0 +1,196 @@ +package treesitter + +/* +#cgo CFLAGS: -I${SRCDIR}/../../native/bridge/include -O3 +#cgo LDFLAGS: -L${SRCDIR}/../../native/bridge/lib -ldiffmantic_grammars -lstdc++ +#include "bridge.h" +#include "../../native/bridge/src/bridge.c" + +extern const TSLanguage* diffmantic_get_native_language(const char* name); + +static uint32_t get_ts_lang_symbol_count(const void* ts_lang_ptr) { + return ts_language_symbol_count((const TSLanguage*)ts_lang_ptr); +} + +static const char* get_ts_lang_symbol_name(const void* ts_lang_ptr, uint16_t symbol_id) { + return ts_language_symbol_name((const TSLanguage*)ts_lang_ptr, symbol_id); +} +*/ +import "C" + +import ( + "errors" + "fmt" + "slices" + "sync" + "unsafe" +) + +type cachedLang struct { + ptr unsafe.Pointer + symbols []string +} + +var ( + nativeLangCache = make(map[string]cachedLang) + nativeLangMu sync.RWMutex +) + +func getLanguageSymbols(langName string) (unsafe.Pointer, []string, error) { + if langName == "" { + return nil, nil, errors.New("empty language name") + } + + nativeLangMu.RLock() + if entry, ok := nativeLangCache[langName]; ok { + nativeLangMu.RUnlock() + return entry.ptr, entry.symbols, nil + } + nativeLangMu.RUnlock() + + nativeLangMu.Lock() + defer nativeLangMu.Unlock() + + if entry, ok := nativeLangCache[langName]; ok { + return entry.ptr, entry.symbols, nil + } + + ptr, err := GetNativeLanguage(langName) + if err != nil || ptr == nil { + return nil, nil, fmt.Errorf("native parser not available for %s: %w", langName, err) + } + + symbols := NativeLanguageSymbols(ptr) + nativeLangCache[langName] = cachedLang{ptr: ptr, symbols: symbols} + return ptr, symbols, nil +} + +// ParseWithLanguage parses source bytes for a given language name. +func ParseWithLanguage(src []byte, langName string) (*ASTNode, error) { + ptr, symbols, err := getLanguageSymbols(langName) + if err != nil { + return nil, err + } + return ParseWithNativeFlatBuffer(src, ptr, langName, symbols) +} + +// ParseForPipeline parses via the native flat-buffer bridge, returning the root AST, +// raw flat nodes, and symbol table for comment extraction in the diff pipeline. +func ParseForPipeline(src []byte, langName string) (*ASTNode, []FlatNode, []string, error) { + ptr, symbols, err := getLanguageSymbols(langName) + if err != nil { + return nil, nil, nil, err + } + ast, flatNodes, err := parseWithNativeFlatBufferKeepNodes(src, ptr, langName, symbols) + return ast, flatNodes, symbols, err +} + +// parseWithNativeFlatBufferKeepNodes is like ParseWithNativeFlatBuffer but also returns +// a Go-owned copy of the flat node array for comment extraction. +func parseWithNativeFlatBufferKeepNodes(src []byte, tsLangPtr unsafe.Pointer, langName string, symbols []string) (*ASTNode, []FlatNode, error) { + if tsLangPtr == nil { + return nil, nil, errors.New("nil native language pointer") + } + + if len(symbols) == 0 { + symbols = NativeLanguageSymbols(tsLangPtr) + } + + var srcPtr *C.uint8_t + if len(src) > 0 { + srcPtr = (*C.uint8_t)(unsafe.Pointer(&src[0])) + } + + res := C.parse_to_flat_ast( + srcPtr, + C.size_t(len(src)), + tsLangPtr, + ) + + if res.error_code != 0 { + return nil, nil, fmt.Errorf("native flat tree-sitter parse failed (code %d)", int(res.error_code)) + } + + defer C.free_flat_ast(res) + + var nodes []FlatNode + if res.node_count > 0 && res.nodes_ptr != nil { + nodes = unsafe.Slice((*FlatNode)(unsafe.Pointer(res.nodes_ptr)), int(res.node_count)) + } + + // Copy flat nodes into Go-owned memory for comment extraction (C memory freed on return). + goNodes := slices.Clone(nodes) + + root := IngestFlatAST(nodes, symbols, src, langName) + + return root, goNodes, nil +} + +type FlatASTResult = C.FlatASTResult + +// GetNativeLanguage retrieves the statically linked native Tree-sitter language for any of the 16 core languages (18 grammars). +func GetNativeLanguage(langName string) (unsafe.Pointer, error) { + cName := C.CString(langName) + defer C.free(unsafe.Pointer(cName)) + + ptr := unsafe.Pointer(C.diffmantic_get_native_language(cName)) + if ptr == nil { + return nil, fmt.Errorf("unsupported or unregistered native language: %s", langName) + } + return ptr, nil +} + +// NativeLanguageSymbols returns the slice of symbol names defined by the native Tree-sitter language. +func NativeLanguageSymbols(tsLangPtr unsafe.Pointer) []string { + if tsLangPtr == nil { + return nil + } + count := int(C.get_ts_lang_symbol_count(tsLangPtr)) + symbols := make([]string, count) + for i := range count { + cName := C.get_ts_lang_symbol_name(tsLangPtr, C.uint16_t(i)) + if cName != nil { + symbols[i] = C.GoString(cName) + } + } + return symbols +} + +// ParseWithNativeFlatBuffer parses source code using the native Tree-sitter engine +// and converts the resulting flat buffer into an ASTNode tree. +func ParseWithNativeFlatBuffer(src []byte, tsLangPtr unsafe.Pointer, langName string, symbols []string) (*ASTNode, error) { + if tsLangPtr == nil { + return nil, errors.New("nil native language pointer") + } + + if len(symbols) == 0 { + symbols = NativeLanguageSymbols(tsLangPtr) + } + + var srcPtr *C.uint8_t + if len(src) > 0 { + srcPtr = (*C.uint8_t)(unsafe.Pointer(&src[0])) + } + + res := C.parse_to_flat_ast( + srcPtr, + C.size_t(len(src)), + tsLangPtr, + ) + + if res.error_code != 0 { + return nil, fmt.Errorf("native flat tree-sitter parse failed (code %d)", int(res.error_code)) + } + + // Release the C flat buffer on return, even if ingestion panics. + defer C.free_flat_ast(res) + + var nodes []FlatNode + if res.node_count > 0 && res.nodes_ptr != nil { + nodes = unsafe.Slice((*FlatNode)(unsafe.Pointer(res.nodes_ptr)), int(res.node_count)) + } + + root := IngestFlatAST(nodes, symbols, src, langName) + + return root, nil +} diff --git a/internal/treesitter/bridge_ingest.go b/internal/treesitter/bridge_ingest.go new file mode 100644 index 00000000..fb860b23 --- /dev/null +++ b/internal/treesitter/bridge_ingest.go @@ -0,0 +1,123 @@ +package treesitter + +import ( + "strings" + + "github.com/HarshK97/diffmantic/internal/treesitter/rules" +) + +// IngestFlatAST converts the flat node buffer into a full ASTNode tree, +// applying language rules, labels, and subtree hashes. +func IngestFlatAST(nodes []FlatNode, symbols []string, src []byte, langName string) *ASTNode { + if len(nodes) == 0 { + return nil + } + + r := rules.Get(langName) + srcLen := uint32(len(src)) + + var errCount int + for i := range nodes { + fn := &nodes[i] + var rawType string + if (fn.Flags & FlatNodeError) != 0 { + rawType = "ERROR" + } else if int(fn.TypeID) < len(symbols) { + rawType = symbols[fn.TypeID] + } + if rawType == "ERROR" || (fn.Flags&FlatNodeError != 0) || (fn.Flags&FlatNodeMissing != 0) { + errCount++ + } + } + + root := buildFromIndex(0, nodes, symbols, src, nil, r, srcLen) + if root != nil { + root.Language = langName + root.ParseErrorCount = errCount + root.HasError = errCount > 0 + root.ComputeHashes() + EnsureIndex(root) + } + return root +} + +func buildFromIndex(idx uint32, nodes []FlatNode, symbols []string, src []byte, parent *ASTNode, r *rules.Rules, srcLen uint32) *ASTNode { + if int(idx) >= len(nodes) { + return nil + } + fn := &nodes[idx] + var rawType string + if (fn.Flags & FlatNodeError) != 0 { + rawType = "ERROR" + } else if int(fn.TypeID) < len(symbols) { + rawType = symbols[fn.TypeID] + } + if (fn.Flags & FlatNodeMissing) != 0 { + rawType = "MISSING " + rawType + } + + isLeaf := fn.ChildCount == 0 || (r != nil && r.IsFlattened(rawType)) + var label string + if isLeaf { + start, end := min(fn.StartByte, srcLen), min(fn.EndByte, srcLen) + if start > end { + start = end + } + label = strings.TrimSpace(string(src[start:end])) + } + + if r != nil && r.IsIgnored(rawType, label) { + return nil + } + + node := &ASTNode{ + Type: rawType, + Parent: parent, + StartByte: fn.StartByte, + EndByte: fn.EndByte, + StartRow: fn.StartRow, + StartCol: fn.StartCol, + EndRow: fn.EndRow, + EndCol: fn.EndCol, + } + + if isLeaf { + node.Label = label + } + + if r != nil { + if alias, ok := r.Alias(node.Type, label); ok { + node.Type = alias + } + if r.IsLabelIgnored(node.Type) { + node.Label = "" + } + if isLeaf && r.IsKeyword(node.Type, label) { + node.IsKeyword = true + } + if r.IsUnordered(node.Type) { + node.IsUnordered = true + } + } + + childIdx := fn.FirstChildIdx + for childIdx != 0xFFFFFFFF && int(childIdx) < len(nodes) { + if child := buildFromIndex(childIdx, nodes, symbols, src, node, r, srcLen); child != nil { + node.Children = append(node.Children, child) + } + childIdx = nodes[childIdx].NextSiblingIdx + } + + if r != nil && r.IsFlattened(rawType) { + var flattenedChildren []*ASTNode + for _, child := range node.Children { + flattenedChildren = append(flattenedChildren, child.Children...) + for _, grandchild := range child.Children { + grandchild.Parent = node + } + } + node.Children = flattenedChildren + } + + return node +} diff --git a/internal/treesitter/bridge_test.go b/internal/treesitter/bridge_test.go new file mode 100644 index 00000000..ae396750 --- /dev/null +++ b/internal/treesitter/bridge_test.go @@ -0,0 +1,125 @@ +package treesitter_test + +import ( + "testing" + + "github.com/HarshK97/diffmantic/internal/treesitter" +) + +func TestAll18NativeGrammarsLoaded(t *testing.T) { + langs := []string{ + "c", "cpp", "go", "rust", "python", "javascript", "typescript", "tsx", + "java", "php", "ruby", "lua", "zig", "css", "html", "json", "toml", "yaml", + } + + for _, lang := range langs { + t.Run(lang, func(t *testing.T) { + ptr, err := treesitter.GetNativeLanguage(lang) + if err != nil { + t.Fatalf("failed to get native language for %s: %v", lang, err) + } + if ptr == nil { + t.Fatalf("nil language pointer for %s", lang) + } + symbols := treesitter.NativeLanguageSymbols(ptr) + if len(symbols) == 0 { + t.Fatalf("empty symbols list for %s", lang) + } + }) + } +} + +func TestNativeFlatBufferParsing_AllLanguages(t *testing.T) { + testCases := []struct { + lang string + filename string + src string + }{ + {lang: "c", filename: "main.c", src: "int main(void) { return 0; }"}, + {lang: "cpp", filename: "main.cpp", src: "int main() { return 0; }"}, + {lang: "go", filename: "main.go", src: "package main\nfunc main() {}"}, + {lang: "rust", filename: "main.rs", src: "fn main() {}"}, + {lang: "python", filename: "main.py", src: "def main():\n pass\n"}, + {lang: "javascript", filename: "main.js", src: "function main() {}"}, + {lang: "typescript", filename: "main.ts", src: "function main(): void {}"}, + {lang: "tsx", filename: "main.tsx", src: "const App = () =>
Hello
;"}, + {lang: "java", filename: "Main.java", src: "class Main { public static void main(String[] args) {} }"}, + {lang: "php", filename: "main.php", src: ""}, + {lang: "ruby", filename: "main.rb", src: "def main; puts 'hello'; end"}, + {lang: "lua", filename: "main.lua", src: "function main() print('hello') end"}, + {lang: "zig", filename: "main.zig", src: "pub fn main() void {}"}, + {lang: "css", filename: "style.css", src: "body { color: red; }"}, + {lang: "html", filename: "index.html", src: "Hello"}, + {lang: "json", filename: "data.json", src: "{\"key\": \"value\", \"count\": 42}"}, + {lang: "toml", filename: "config.toml", src: "[server]\nport = 8080\n"}, + {lang: "yaml", filename: "config.yaml", src: "name: diffmantic\nversion: 1.0\n"}, + } + + for _, tc := range testCases { + t.Run(tc.lang, func(t *testing.T) { + ptr, err := treesitter.GetNativeLanguage(tc.lang) + if err != nil { + t.Fatalf("failed to get native language for %s: %v", tc.lang, err) + } + + symbols := treesitter.NativeLanguageSymbols(ptr) + ast, err := treesitter.ParseWithNativeFlatBuffer([]byte(tc.src), ptr, tc.lang, symbols) + if err != nil { + t.Fatalf("native parse error for %s: %v", tc.lang, err) + } + if ast == nil { + t.Fatalf("nil AST for %s", tc.lang) + } + if ast.Size() == 0 { + t.Fatalf("empty AST for %s", tc.lang) + } + if ast.HasError { + t.Errorf("AST has error for %s", tc.lang) + } + }) + } +} + +func TestIngestFlatAST_Synthetic(t *testing.T) { + symbols := []string{"source_file", "function_declaration", "identifier", "block", "{", "}"} + src := []byte("func foo() {}") + + nodes := []treesitter.FlatNode{ + {TypeID: 0, Flags: treesitter.FlatNodeNamed, StartByte: 0, EndByte: 13, FirstChildIdx: 1, NextSiblingIdx: 0xFFFFFFFF, ChildCount: 1}, + {TypeID: 1, Flags: treesitter.FlatNodeNamed, StartByte: 0, EndByte: 13, FirstChildIdx: 2, NextSiblingIdx: 0xFFFFFFFF, ParentIdx: 0, ChildCount: 2}, + {TypeID: 2, Flags: treesitter.FlatNodeNamed, StartByte: 5, EndByte: 8, FirstChildIdx: 0xFFFFFFFF, NextSiblingIdx: 3, ParentIdx: 1, ChildCount: 0}, + {TypeID: 3, Flags: treesitter.FlatNodeNamed, StartByte: 11, EndByte: 13, FirstChildIdx: 0xFFFFFFFF, NextSiblingIdx: 0xFFFFFFFF, ParentIdx: 1, ChildCount: 0}, + } + + root := treesitter.IngestFlatAST(nodes, symbols, src, "go") + if root == nil { + t.Fatalf("root is nil") + } + if root.Type != "source_file" { + t.Errorf("expected Type='source_file', got %q", root.Type) + } + if root.Language != "go" { + t.Errorf("expected Language='go', got %q", root.Language) + } + if len(root.Children) != 1 { + t.Fatalf("expected 1 child, got %d", len(root.Children)) + } + if root.Children[0].Type != "function_declaration" { + t.Errorf("expected function_declaration, got %q", root.Children[0].Type) + } + if len(root.Children[0].Children) != 2 { + t.Fatalf("expected 2 children, got %d", len(root.Children[0].Children)) + } + if root.Children[0].Children[0].Label != "foo" { + t.Errorf("expected label 'foo', got %q", root.Children[0].Children[0].Label) + } + if root.Children[0].Children[0].StartByte != 5 || root.Children[0].Children[0].EndByte != 8 { + t.Errorf("expected byte range [5, 8], got [%d, %d]", root.Children[0].Children[0].StartByte, root.Children[0].Children[0].EndByte) + } + if root.HasError { + t.Errorf("expected HasError=false") + } + if root.ParseErrorCount != 0 { + t.Errorf("expected ParseErrorCount=0, got %d", root.ParseErrorCount) + } +} diff --git a/internal/treesitter/detect.go b/internal/treesitter/detect.go new file mode 100644 index 00000000..cfd7735f --- /dev/null +++ b/internal/treesitter/detect.go @@ -0,0 +1,106 @@ +package treesitter + +import ( + "fmt" + "path/filepath" + "strings" +) + +// Language represents a recognized programming or data language. +type Language struct { + Name string +} + +var extToLang = map[string]string{ + ".go": "go", + ".c": "c", + ".h": "c", + ".cpp": "cpp", + ".cc": "cpp", + ".cxx": "cpp", + ".c++": "cpp", + ".hpp": "cpp", + ".hh": "cpp", + ".hxx": "cpp", + ".h++": "cpp", + ".rs": "rust", + ".py": "python", + ".pyi": "python", + ".js": "javascript", + ".jsx": "javascript", + ".mjs": "javascript", + ".cjs": "javascript", + ".ts": "typescript", + ".mts": "typescript", + ".cts": "typescript", + ".tsx": "tsx", + ".java": "java", + ".php": "php", + ".phtml": "php", + ".php3": "php", + ".php4": "php", + ".php5": "php", + ".php7": "php", + ".phps": "php", + ".rb": "ruby", + ".rake": "ruby", + ".gemspec": "ruby", + ".zig": "zig", + ".lua": "lua", + ".html": "html", + ".htm": "html", + ".css": "css", + ".json": "json", + ".toml": "toml", + ".yaml": "yaml", + ".yml": "yaml", +} + +var basenameToLang = map[string]string{ + "rakefile": "ruby", + "gemfile": "ruby", +} + +// DetectLanguage detects the language for a given filename or path. +func DetectLanguage(filename string) (*Language, error) { + name, err := DetectLanguageName(filename) + if err != nil { + return nil, err + } + return &Language{Name: name}, nil +} + +// DetectLanguageName returns the canonical language name string for a given filename or path. +func DetectLanguageName(filename string) (string, error) { + base := filepath.Base(filename) + lowerBase := strings.ToLower(base) + + if lang, ok := basenameToLang[lowerBase]; ok { + return lang, nil + } + + ext := strings.ToLower(filepath.Ext(base)) + if ext == "" { + return "", fmt.Errorf("unsupported language for file: %s", filename) + } + + if lang, ok := extToLang[ext]; ok { + return lang, nil + } + + // Support compression wrappers (e.g. .json.gz, .go.zst, .yaml.xz) + if ext == ".gz" || ext == ".zst" || ext == ".xz" || ext == ".bz2" { + trimmed := strings.TrimSuffix(lowerBase, ext) + if innerLang, ok := basenameToLang[trimmed]; ok { + return innerLang, nil + } + innerExt := strings.ToLower(filepath.Ext(trimmed)) + if innerExt != "" { + if lang, ok := extToLang[innerExt]; ok { + return lang, nil + } + } + } + + return "", fmt.Errorf("unsupported language for file: %s", filename) +} diff --git a/internal/treesitter/detect_test.go b/internal/treesitter/detect_test.go new file mode 100644 index 00000000..e9ce6abf --- /dev/null +++ b/internal/treesitter/detect_test.go @@ -0,0 +1,51 @@ +package treesitter_test + +import ( + "testing" + + "github.com/HarshK97/diffmantic/internal/treesitter" +) + +func TestDetectLanguageName(t *testing.T) { + tests := []struct { + filename string + want string + wantErr bool + }{ + {"main.go", "go", false}, + {"app.js", "javascript", false}, + {"index.ts", "typescript", false}, + {"component.tsx", "tsx", false}, + {"data.json", "json", false}, + {"config.yaml", "yaml", false}, + {"config.yml", "yaml", false}, + {"Cargo.toml", "toml", false}, + {"main.rs", "rust", false}, + {"script.py", "python", false}, + {"test.cpp", "cpp", false}, + {"header.h", "c", false}, + {"Rakefile", "ruby", false}, + {"Gemfile", "ruby", false}, + // Compound compression wrappers + {"expected_ui.json.gz", "json", false}, + {"archive.go.zst", "go", false}, + {"manifest.yaml.xz", "yaml", false}, + {"Gemfile.gz", "ruby", false}, + // Unsupported + {"file.unknown", "", true}, + {"archive.tar.gz", "", true}, + {"no_ext", "", true}, + } + + for _, tt := range tests { + t.Run(tt.filename, func(t *testing.T) { + got, err := treesitter.DetectLanguageName(tt.filename) + if (err != nil) != tt.wantErr { + t.Fatalf("DetectLanguageName(%q) error = %v, wantErr = %v", tt.filename, err, tt.wantErr) + } + if got != tt.want { + t.Errorf("DetectLanguageName(%q) = %q, want %q", tt.filename, got, tt.want) + } + }) + } +} diff --git a/internal/treesitter/flat_node.go b/internal/treesitter/flat_node.go new file mode 100644 index 00000000..8b4a82a5 --- /dev/null +++ b/internal/treesitter/flat_node.go @@ -0,0 +1,23 @@ +package treesitter + +// FlatNode defines the 44-byte cache-aligned struct matching the C bridge layout. +type FlatNode struct { + TypeID uint16 + Flags uint16 + StartByte uint32 + EndByte uint32 + StartRow uint32 + StartCol uint32 + EndRow uint32 + EndCol uint32 + ParentIdx uint32 + FirstChildIdx uint32 + NextSiblingIdx uint32 + ChildCount uint32 +} + +const ( + FlatNodeNamed = 1 << 0 + FlatNodeError = 1 << 1 + FlatNodeMissing = 1 << 2 +) diff --git a/internal/treesitter/parser.go b/internal/treesitter/parser.go index bd5d9e84..02dcc3a0 100644 --- a/internal/treesitter/parser.go +++ b/internal/treesitter/parser.go @@ -1,45 +1,10 @@ package treesitter -import ( - "errors" - "fmt" - "path/filepath" - - "github.com/odvcencio/gotreesitter" - "github.com/odvcencio/gotreesitter/grammars" -) - -func DetectGrammarEntry(filename string) *grammars.LangEntry { - base := filepath.Base(filename) - return grammars.DetectLanguage(base) -} - -func DetectLanguage(filename string) (*gotreesitter.Language, error) { - entry := DetectGrammarEntry(filename) - if entry == nil { - return nil, fmt.Errorf("unsupported language for file: %s", filename) - } - return entry.Language(), nil -} - -// ParseWithLanguageAndTree parses source bytes and returns both the AST and raw tree-sitter tree. -func ParseWithLanguageAndTree(src []byte, lang *gotreesitter.Language) (*ASTNode, *gotreesitter.Tree, error) { - if lang == nil { - return nil, nil, errors.New("nil language") - } - parser := gotreesitter.NewParser(lang) - tree, err := parser.Parse(src) - if err != nil { - return nil, nil, err - } - return BuildAST(tree.RootNode(), src, lang, nil), tree, nil -} - +// Parse detects the language for the given filename and parses source bytes into an ASTNode. func Parse(src []byte, filename string) (*ASTNode, error) { lang, err := DetectLanguage(filename) if err != nil { return nil, err } - ast, _, err := ParseWithLanguageAndTree(src, lang) - return ast, err + return ParseWithLanguage(src, lang.Name) } diff --git a/internal/treesitter/parser_test.go b/internal/treesitter/parser_test.go index 5600469e..69ad67ef 100644 --- a/internal/treesitter/parser_test.go +++ b/internal/treesitter/parser_test.go @@ -23,19 +23,20 @@ func TestDetectLanguage(t *testing.T) { {"test.cxx", false}, {"test.hh", false}, {"test.h", false}, - {"test.cs", false}, + {"test.cs", true}, {"test.java", false}, {"test.php", false}, {"test.rb", false}, {"test.json", false}, + {"test.toml", false}, {"test.yaml", false}, {"test.yml", false}, {"test.html", false}, {"test.css", false}, {"test.lua", false}, {"test.zig", false}, - {"test.sh", false}, - {"test.bash", false}, + {"test.sh", true}, + {"test.bash", true}, {"test.xyz", true}, {"test", true}, } diff --git a/internal/treesitter/rules/c.go b/internal/treesitter/rules/c.go index bef2121b..591e8e71 100644 --- a/internal/treesitter/rules/c.go +++ b/internal/treesitter/rules/c.go @@ -1,6 +1,7 @@ package rules var cRules = &Rules{ + Kind: KindCode, Flattened: []string{ "string_literal", "char_literal", @@ -155,17 +156,18 @@ var cRules = &Rules{ }, Wrappers: []string{ "parenthesized_expression", + "argument_list", + "parameter_list", "cast_expression", "pointer_expression", "subscript_expression", - "argument_list", - "parameter_list", }, Pairs: []string{ "field_designator", "initializer_pair", }, EquivalentTypes: [][]string{ + {"function_definition", "declaration"}, {"struct_specifier", "union_specifier", "enum_specifier"}, {"for_statement", "while_statement", "do_statement"}, }, @@ -175,4 +177,18 @@ var cRules = &Rules{ Calls: []string{ "call_expression", }, + Indexed: []string{ + "subscript_expression", + }, + ContainerDeclarations: []string{ + "function_definition", + "type_definition", + "struct_specifier", + "union_specifier", + "enum_specifier", + }, + Types: []string{ + "primitive_type", + "type_identifier", + }, } diff --git a/internal/treesitter/rules/cpp.go b/internal/treesitter/rules/cpp.go index 78535a47..b5e64984 100644 --- a/internal/treesitter/rules/cpp.go +++ b/internal/treesitter/rules/cpp.go @@ -1,6 +1,7 @@ package rules var cppRules = &Rules{ + Kind: KindCode, Flattened: []string{ "string_literal", "char_literal", @@ -105,6 +106,7 @@ var cppRules = &Rules{ "declaration", "init_declarator", "field_declaration", + "friend_declaration", "parameter_declaration", "alias_declaration", "using_declaration", @@ -199,6 +201,7 @@ var cppRules = &Rules{ "concept_definition", "parameter_declaration", "field_declaration", + "friend_declaration", }, Identifiers: []string{ "identifier", @@ -210,20 +213,22 @@ var cppRules = &Rules{ }, Wrappers: []string{ "parenthesized_expression", + "argument_list", + "parameter_list", "template_type", "template_argument_list", "subscript_expression", "pointer_expression", "cast_expression", - "argument_list", - "parameter_list", + "friend_declaration", + "template_declaration", }, Pairs: []string{ "field_designator", "initializer_pair", }, EquivalentTypes: [][]string{ - {"function_definition", "template_declaration"}, + {"function_definition", "declaration"}, {"class_specifier", "struct_specifier", "union_specifier", "enum_specifier"}, {"for_statement", "for_range_loop", "while_statement", "do_statement"}, }, @@ -233,4 +238,27 @@ var cppRules = &Rules{ Calls: []string{ "call_expression", }, + Indexed: []string{ + "subscript_expression", + }, + ContainerDeclarations: []string{ + "function_definition", + "type_definition", + "class_specifier", + "struct_specifier", + "union_specifier", + "enum_specifier", + "namespace_definition", + "template_declaration", + }, + Closures: []string{ + "lambda_expression", + }, + Types: []string{ + "dependent_type", + "primitive_type", + "template_type", + "trailing_return_type", + "type_identifier", + }, } diff --git a/internal/treesitter/rules/css.go b/internal/treesitter/rules/css.go index 1ceb8ef5..234cd4eb 100644 --- a/internal/treesitter/rules/css.go +++ b/internal/treesitter/rules/css.go @@ -1,6 +1,7 @@ package rules var cssRules = &Rules{ + Kind: KindMarkup, Flattened: []string{ "plain_value", "color_value", diff --git a/internal/treesitter/rules/go.go b/internal/treesitter/rules/go.go index 29aa4fd3..7ac78819 100644 --- a/internal/treesitter/rules/go.go +++ b/internal/treesitter/rules/go.go @@ -1,6 +1,7 @@ package rules var golangRules = &Rules{ + Kind: KindCode, Flattened: []string{ "interpreted_string_literal", "raw_string_literal", @@ -65,7 +66,6 @@ var golangRules = &Rules{ "parenthesized_expression", "expression_list", "literal_value", - "statement_list", "import_spec_list", "function_declaration", "method_declaration", @@ -128,6 +128,7 @@ var golangRules = &Rules{ Wrappers: []string{ "parenthesized_expression", "parenthesized_type", + "argument_list", "type_arguments", "type_parameter_list", "slice_type", @@ -135,8 +136,8 @@ var golangRules = &Rules{ "index_expression", "slice_expression", "pointer_type", - "argument_list", - "parameter_list", + "expression_list", + "var_declaration", }, Pairs: []string{ "keyed_element", @@ -146,7 +147,7 @@ var golangRules = &Rules{ }, EquivalentTypes: [][]string{ {"function_declaration", "method_declaration"}, - {"var_declaration", "short_var_declaration"}, + {"short_var_declaration", "assignment_statement", "var_spec"}, }, Comments: []string{ "comment", @@ -157,4 +158,38 @@ var golangRules = &Rules{ ScopedDeclarations: []string{ "method_declaration", }, + Indexed: []string{ + "index_expression", + "slice_expression", + }, + LocalVarDeclarations: []string{ + "var_declaration", + "const_declaration", + "short_var_declaration", + }, + ContainerDeclarations: []string{ + "function_declaration", + "method_declaration", + "type_declaration", + "type_spec", + }, + Closures: []string{ + "func_literal", + }, + Types: []string{ + "array_type", + "channel_type", + "function_type", + "generic_type", + "implicit_length_array_type", + "interface_type", + "map_type", + "negated_type", + "parenthesized_type", + "pointer_type", + "qualified_type", + "slice_type", + "struct_type", + "type_identifier", + }, } diff --git a/internal/treesitter/rules/html.go b/internal/treesitter/rules/html.go index 77654961..c8b00c28 100644 --- a/internal/treesitter/rules/html.go +++ b/internal/treesitter/rules/html.go @@ -1,6 +1,7 @@ package rules var htmlRules = &Rules{ + Kind: KindMarkup, Flattened: []string{ "text", "raw_text", diff --git a/internal/treesitter/rules/java.go b/internal/treesitter/rules/java.go index 3204c00d..a3380ab0 100644 --- a/internal/treesitter/rules/java.go +++ b/internal/treesitter/rules/java.go @@ -1,6 +1,7 @@ package rules var javaRules = &Rules{ + Kind: KindCode, Flattened: []string{ "string_literal", "character_literal", @@ -182,13 +183,13 @@ var javaRules = &Rules{ }, Wrappers: []string{ "parenthesized_expression", + "argument_list", + "formal_parameters", "type_arguments", "type_parameters", "array_type", "generic_type", "cast_expression", - "argument_list", - "formal_parameters", }, Pairs: []string{ "element_value_pair", @@ -209,4 +210,33 @@ var javaRules = &Rules{ "method_invocation", "explicit_constructor_invocation", }, + Indexed: []string{ + "array_access", + }, + LocalVarDeclarations: []string{ + "local_variable_declaration", + }, + ContainerDeclarations: []string{ + "class_declaration", + "interface_declaration", + "enum_declaration", + "record_declaration", + "method_declaration", + "constructor_declaration", + }, + Closures: []string{ + "lambda_expression", + }, + Types: []string{ + "annotated_type", + "array_type", + "boolean_type", + "catch_type", + "floating_point_type", + "generic_type", + "integral_type", + "scoped_type_identifier", + "type_identifier", + "void_type", + }, } diff --git a/internal/treesitter/rules/javascript.go b/internal/treesitter/rules/javascript.go index a1f7c586..1f4932e4 100644 --- a/internal/treesitter/rules/javascript.go +++ b/internal/treesitter/rules/javascript.go @@ -1,6 +1,7 @@ package rules var javascriptRules = &Rules{ + Kind: KindCode, Flattened: []string{ "string", "template_string", @@ -80,7 +81,6 @@ var javascriptRules = &Rules{ "expression_statement", "variable_declaration", "lexical_declaration", - "using_declaration", "variable_declarator", "function_declaration", "function_expression", @@ -172,8 +172,9 @@ var javascriptRules = &Rules{ "statement_block", }, Wrappers: []string{ + "export_statement", "parenthesized_expression", - "subscript_expression", + "object", "array", "array_pattern", "object_pattern", @@ -187,6 +188,8 @@ var javascriptRules = &Rules{ Unordered: []string{ "object", "object_pattern", + "named_imports", + "export_clause", }, EquivalentTypes: [][]string{ {"function_declaration", "function_expression", "arrow_function", "generator_function_declaration", "generator_function"}, @@ -200,4 +203,22 @@ var javascriptRules = &Rules{ Calls: []string{ "call_expression", }, + Indexed: []string{ + "subscript_expression", + }, + LocalVarDeclarations: []string{ + "variable_declaration", + "lexical_declaration", + }, + ContainerDeclarations: []string{ + "function_declaration", + "generator_function_declaration", + "method_definition", + "class_declaration", + }, + Closures: []string{ + "arrow_function", + "function_expression", + "function", + }, } diff --git a/internal/treesitter/rules/json.go b/internal/treesitter/rules/json.go index 160f0433..16f5b6a1 100644 --- a/internal/treesitter/rules/json.go +++ b/internal/treesitter/rules/json.go @@ -1,6 +1,7 @@ package rules var jsonRules = &Rules{ + Kind: KindData, Flattened: []string{ "string", }, @@ -26,6 +27,10 @@ var jsonRules = &Rules{ Pairs: []string{ "pair", }, + Wrappers: []string{ + "array", + "object", + }, Unordered: []string{ "object", }, diff --git a/internal/treesitter/rules/lua.go b/internal/treesitter/rules/lua.go index 1e759c04..c09209e7 100644 --- a/internal/treesitter/rules/lua.go +++ b/internal/treesitter/rules/lua.go @@ -1,6 +1,7 @@ package rules var luaRules = &Rules{ + Kind: KindCode, Flattened: []string{ "string", }, @@ -110,7 +111,6 @@ var luaRules = &Rules{ Wrappers: []string{ "parenthesized_expression", "table_constructor", - "bracket_index_expression", "dot_index_expression", "method_index_expression", "arguments", @@ -131,4 +131,16 @@ var luaRules = &Rules{ Calls: []string{ "function_call", }, + Indexed: []string{ + "bracket_index_expression", + }, + LocalVarDeclarations: []string{ + "variable_declaration", + }, + ContainerDeclarations: []string{ + "function_declaration", + }, + Closures: []string{ + "function_definition", + }, } diff --git a/internal/treesitter/rules/php.go b/internal/treesitter/rules/php.go index c331e364..3572e3cd 100644 --- a/internal/treesitter/rules/php.go +++ b/internal/treesitter/rules/php.go @@ -1,6 +1,7 @@ package rules var phpRules = &Rules{ + Kind: KindCode, Flattened: []string{ "string", "encapsed_string", @@ -200,4 +201,32 @@ var phpRules = &Rules{ "scoped_call_expression", "nullsafe_member_call_expression", }, + Indexed: []string{ + "subscript_expression", + }, + LocalVarDeclarations: []string{ + "const_declaration", + }, + ContainerDeclarations: []string{ + "function_definition", + "method_declaration", + "class_declaration", + "interface_declaration", + "trait_declaration", + "enum_declaration", + }, + Closures: []string{ + "anonymous_function", + "arrow_function", + }, + Types: []string{ + "bottom_type", + "cast_type", + "disjunctive_normal_form_type", + "intersection_type", + "named_type", + "optional_type", + "primitive_type", + "union_type", + }, } diff --git a/internal/treesitter/rules/python.go b/internal/treesitter/rules/python.go index b494667d..d97e6d26 100644 --- a/internal/treesitter/rules/python.go +++ b/internal/treesitter/rules/python.go @@ -1,6 +1,7 @@ package rules var pythonRules = &Rules{ + Kind: KindCode, Flattened: []string{ "string", "concatenated_string", @@ -80,7 +81,6 @@ var pythonRules = &Rules{ "expression_statement", "expression_list", "pattern_list", - "tuple_expression", "list", "dictionary", "set", @@ -157,7 +157,8 @@ var pythonRules = &Rules{ }, Wrappers: []string{ "parenthesized_expression", - "subscript", + "argument_list", + "parameters", "list", "tuple", "set", @@ -166,8 +167,6 @@ var pythonRules = &Rules{ "list_comprehension", "dictionary_comprehension", "set_comprehension", - "argument_list", - "parameters", }, Pairs: []string{ "pair", @@ -187,4 +186,21 @@ var pythonRules = &Rules{ Calls: []string{ "call", }, + Indexed: []string{ + "subscript", + }, + ContainerDeclarations: []string{ + "function_definition", + "class_definition", + }, + Closures: []string{ + "lambda", + }, + Types: []string{ + "constrained_type", + "generic_type", + "member_type", + "splat_type", + "union_type", + }, } diff --git a/internal/treesitter/rules/ruby.go b/internal/treesitter/rules/ruby.go index 8cca0a4e..219bb24a 100644 --- a/internal/treesitter/rules/ruby.go +++ b/internal/treesitter/rules/ruby.go @@ -1,6 +1,7 @@ package rules var rubyRules = &Rules{ + Kind: KindCode, Flattened: []string{ "string", "string_content", @@ -154,14 +155,18 @@ var rubyRules = &Rules{ "constant", }, Blocks: []string{ + "body_statement", + "do_block", "block", }, Wrappers: []string{ "parenthesized_statements", + "argument_list", "array", "element_reference", - "argument_list", - "method_parameters", + "do_block", + "hash", + "hash_pattern", }, Pairs: []string{ "pair", @@ -173,7 +178,7 @@ var rubyRules = &Rules{ EquivalentTypes: [][]string{ {"method", "singleton_method"}, {"class", "module", "singleton_class"}, - {"if", "unless"}, + {"if", "unless", "if_modifier", "unless_modifier"}, {"while", "until", "for"}, }, Comments: []string{ @@ -182,4 +187,18 @@ var rubyRules = &Rules{ Calls: []string{ "call", }, + Indexed: []string{ + "element_reference", + }, + ContainerDeclarations: []string{ + "method", + "singleton_method", + "class", + "module", + }, + Closures: []string{ + "do_block", + "block", + "lambda", + }, } diff --git a/internal/treesitter/rules/rules.go b/internal/treesitter/rules/rules.go index 4801e329..23f66082 100644 --- a/internal/treesitter/rules/rules.go +++ b/internal/treesitter/rules/rules.go @@ -5,39 +5,63 @@ import ( "strings" ) +// LanguageKind classifies grammars into high-level parsing and diffing categories. +type LanguageKind uint8 + +const ( + // KindCode is for general programming languages (Go, Rust, Python, C++, etc.). + KindCode LanguageKind = iota + // KindData is for structured key-value formats (JSON, YAML, TOML). + KindData + // KindMarkup is for markup and styling languages (HTML, CSS). + KindMarkup +) + // Rules configures language-specific AST transformations and node matching. type Rules struct { - Flattened []string - Ignored []string - Aliased map[string]string - LabelIgnored []string - Scaffolding []string - Keywords []string - Declarations []string - Identifiers []string - Blocks []string - Wrappers []string - Pairs []string - Unordered []string - EquivalentTypes [][]string - Comments []string - Calls []string - ScopedDeclarations []string - - flattenedSet map[string]struct{} - ignoredSet map[string]struct{} - labelIgnoredSet map[string]struct{} - keywordsSet map[string]struct{} - declarationsSet map[string]struct{} - identifiersSet map[string]struct{} - scaffoldingSet map[string]struct{} - blocksSet map[string]struct{} - wrappersSet map[string]struct{} - unorderedSet map[string]struct{} - commentsSet map[string]struct{} - callsSet map[string]struct{} - scopedDeclarationsSet map[string]struct{} - equivGroups map[string][]int + Kind LanguageKind + Flattened []string + Ignored []string + Aliased map[string]string + LabelIgnored []string + Scaffolding []string + Keywords []string + Declarations []string + Identifiers []string + Blocks []string + Wrappers []string + Pairs []string + Unordered []string + EquivalentTypes [][]string + Comments []string + Calls []string + ScopedDeclarations []string + Indexed []string // Subscript nodes with prefix receivers (e.g. arr[i]). + LocalVarDeclarations []string + ContainerDeclarations []string // Major declaration scope boundaries (functions, classes, structs, etc.). + Closures []string // Anonymous functions, lambdas, and callbacks. + Types []string // Type annotations and type expressions. + + flattenedSet map[string]struct{} + ignoredSet map[string]struct{} + labelIgnoredSet map[string]struct{} + keywordsSet map[string]struct{} + declarationsSet map[string]struct{} + identifiersSet map[string]struct{} + scaffoldingSet map[string]struct{} + blocksSet map[string]struct{} + wrappersSet map[string]struct{} + pairsSet map[string]struct{} + unorderedSet map[string]struct{} + commentsSet map[string]struct{} + callsSet map[string]struct{} + scopedDeclarationsSet map[string]struct{} + indexedSet map[string]struct{} + localVarDeclarationsSet map[string]struct{} + containerDeclarationsSet map[string]struct{} + closuresSet map[string]struct{} + typesSet map[string]struct{} + equivGroups map[string][]int } // CompileSets builds the internal lookup sets for fast querying. @@ -96,6 +120,12 @@ func (r *Rules) CompileSets() { r.wrappersSet[s] = struct{}{} } } + if len(r.Pairs) > 0 { + r.pairsSet = make(map[string]struct{}, len(r.Pairs)) + for _, s := range r.Pairs { + r.pairsSet[s] = struct{}{} + } + } if len(r.Unordered) > 0 { r.unorderedSet = make(map[string]struct{}, len(r.Unordered)) for _, s := range r.Unordered { @@ -120,6 +150,36 @@ func (r *Rules) CompileSets() { r.scopedDeclarationsSet[s] = struct{}{} } } + if len(r.Indexed) > 0 { + r.indexedSet = make(map[string]struct{}, len(r.Indexed)) + for _, s := range r.Indexed { + r.indexedSet[s] = struct{}{} + } + } + if len(r.LocalVarDeclarations) > 0 { + r.localVarDeclarationsSet = make(map[string]struct{}, len(r.LocalVarDeclarations)) + for _, s := range r.LocalVarDeclarations { + r.localVarDeclarationsSet[s] = struct{}{} + } + } + if len(r.ContainerDeclarations) > 0 { + r.containerDeclarationsSet = make(map[string]struct{}, len(r.ContainerDeclarations)) + for _, s := range r.ContainerDeclarations { + r.containerDeclarationsSet[s] = struct{}{} + } + } + if len(r.Closures) > 0 { + r.closuresSet = make(map[string]struct{}, len(r.Closures)) + for _, s := range r.Closures { + r.closuresSet[s] = struct{}{} + } + } + if len(r.Types) > 0 { + r.typesSet = make(map[string]struct{}, len(r.Types)) + for _, s := range r.Types { + r.typesSet[s] = struct{}{} + } + } if len(r.EquivalentTypes) > 0 { r.equivGroups = make(map[string][]int) for idx, group := range r.EquivalentTypes { @@ -130,6 +190,14 @@ func (r *Rules) CompileSets() { } } +// GetKind returns the language category (KindCode, KindData, or KindMarkup). +func (r *Rules) GetKind() LanguageKind { + if r == nil { + return KindCode + } + return r.Kind +} + // IsCall reports whether nodeType is a function, method, or macro invocation. func (r *Rules) IsCall(nodeType string) bool { if r == nil || nodeType == "" { @@ -142,6 +210,18 @@ func (r *Rules) IsCall(nodeType string) bool { return slices.Contains(r.Calls, nodeType) } +// IsIndexed reports whether nodeType is a subscript container with a prefix receiver. +func (r *Rules) IsIndexed(nodeType string) bool { + if r == nil || nodeType == "" { + return false + } + if len(r.indexedSet) > 0 { + _, ok := r.indexedSet[nodeType] + return ok + } + return slices.Contains(r.Indexed, nodeType) +} + // IsComment reports whether nodeType is a comment in the language grammar. func (r *Rules) IsComment(nodeType string) bool { if r == nil || nodeType == "" { @@ -178,6 +258,55 @@ func (r *Rules) IsScopedDeclaration(nodeType string) bool { return slices.Contains(r.ScopedDeclarations, nodeType) } +// IsLocalVarDeclaration reports whether nodeType is a local variable declaration. +func (r *Rules) IsLocalVarDeclaration(nodeType string) bool { + if r == nil || nodeType == "" { + return false + } + if len(r.localVarDeclarationsSet) > 0 { + _, ok := r.localVarDeclarationsSet[nodeType] + return ok + } + return slices.Contains(r.LocalVarDeclarations, nodeType) +} + +// IsContainerDeclaration reports whether nodeType is a major container declaration +// (such as a function, method, class, struct, interface, trait, or enum). +func (r *Rules) IsContainerDeclaration(nodeType string) bool { + if r == nil || nodeType == "" { + return false + } + if len(r.containerDeclarationsSet) > 0 { + _, ok := r.containerDeclarationsSet[nodeType] + return ok + } + return slices.Contains(r.ContainerDeclarations, nodeType) +} + +// IsClosure reports whether nodeType is an anonymous function, closure, or lambda callback. +func (r *Rules) IsClosure(nodeType string) bool { + if r == nil || nodeType == "" { + return false + } + if len(r.closuresSet) > 0 { + _, ok := r.closuresSet[nodeType] + return ok + } + return slices.Contains(r.Closures, nodeType) +} + +// IsType reports whether nodeType is a type annotation or type expression in the language. +func (r *Rules) IsType(nodeType string) bool { + if r == nil || nodeType == "" { + return false + } + if len(r.typesSet) > 0 { + _, ok := r.typesSet[nodeType] + return ok + } + return slices.Contains(r.Types, nodeType) +} + // IsIdentifier reports whether nodeType is an identifier token. func (r *Rules) IsIdentifier(nodeType string) bool { if r == nil || nodeType == "" { @@ -281,6 +410,18 @@ func (r *Rules) IsLabelIgnored(nodeType string) bool { return slices.Contains(r.LabelIgnored, nodeType) } +// IsPair checks if nodeType represents a key-value property pair. +func (r *Rules) IsPair(nodeType string) bool { + if r == nil { + return false + } + if len(r.pairsSet) > 0 { + _, ok := r.pairsSet[nodeType] + return ok + } + return slices.Contains(r.Pairs, nodeType) +} + // IsUnordered checks if child order doesn't matter for this container. func (r *Rules) IsUnordered(nodeType string) bool { if r == nil { @@ -412,6 +553,16 @@ func IsDeclaration(nodeType string) bool { return false } +// IsLocalVarDeclaration reports whether nodeType is configured as a local variable declaration in any language rule set. +func IsLocalVarDeclaration(nodeType string) bool { + for _, r := range registry { + if r.IsLocalVarDeclaration(nodeType) { + return true + } + } + return false +} + // IsIdentifier reports whether nodeType is configured as an identifier in any language rule set. func IsIdentifier(nodeType string) bool { for _, r := range registry { @@ -467,6 +618,32 @@ func IsKeyword(nodeType, label string) bool { return false } +// IsPunctuation reports whether a string is a structural punctuation token (braces, brackets, parentheses, delimiters). +func (r *Rules) IsPunctuation(token string) bool { + switch token { + case "}", "};", "],", "]", ")", ");", "},", "{", "begin", "end", ";", ",", "(", "[", ":", "->", "=>", "\"", "'", "`": + return true + } + if r == nil || token == "" { + return false + } + return r.IsIgnored(token, token) || r.IsDelimiter(token, token) +} + +// IsPunctuation reports whether a string is a structural punctuation token in any language rule set. +func IsPunctuation(token string) bool { + switch token { + case "}", "};", "],", "]", ")", ");", "},", "{", "begin", "end", ";", ",", "(", "[", ":", "->", "=>", "\"", "'", "`": + return true + } + for _, r := range registry { + if r.IsPunctuation(token) { + return true + } + } + return false +} + // IsDelimiter reports whether nodeType or label is a delimiter token (semicolon or comma). func IsDelimiter(nodeType, label string) bool { return label == ";" || label == "," || nodeType == "semicolon" || nodeType == "comma" || nodeType == "_automatic_semicolon" @@ -482,6 +659,55 @@ func IsCall(nodeType string) bool { return false } +// IsIndexed reports whether nodeType is configured as an indexed container in any language rule set. +func IsIndexed(nodeType string) bool { + for _, r := range registry { + if r.IsIndexed(nodeType) { + return true + } + } + return false +} + +// IsContainerDeclaration reports whether nodeType is configured as a container declaration in any language rule set. +func IsContainerDeclaration(nodeType string) bool { + if nodeType == "" { + return false + } + for _, r := range registry { + if r.IsContainerDeclaration(nodeType) { + return true + } + } + return false +} + +// IsClosure reports whether nodeType is configured as a closure in any language rule set. +func IsClosure(nodeType string) bool { + if nodeType == "" { + return false + } + for _, r := range registry { + if r.IsClosure(nodeType) { + return true + } + } + return false +} + +// IsType reports whether nodeType is a type annotation or type expression in any language rule set. +func IsType(nodeType string) bool { + if nodeType == "" { + return false + } + for _, r := range registry { + if r.IsType(nodeType) { + return true + } + } + return false +} + var registry = map[string]*Rules{ "c": cRules, "cpp": cppRules, diff --git a/internal/treesitter/rules/rules_grammar_test.go b/internal/treesitter/rules/rules_grammar_test.go index 1fe0c06e..432b12f0 100644 --- a/internal/treesitter/rules/rules_grammar_test.go +++ b/internal/treesitter/rules/rules_grammar_test.go @@ -3,72 +3,73 @@ package rules_test import ( "testing" + "github.com/HarshK97/diffmantic/internal/treesitter" "github.com/HarshK97/diffmantic/internal/treesitter/rules" - "github.com/odvcencio/gotreesitter/grammars" ) func TestAllLanguageRulesMatchGrammarSymbols(t *testing.T) { langs := []struct { - name string - filename string + name string }{ - {"c", "main.c"}, - {"cpp", "main.cpp"}, - {"go", "main.go"}, - {"rust", "main.rs"}, - {"python", "main.py"}, - {"javascript", "main.js"}, - {"typescript", "main.ts"}, - {"tsx", "main.tsx"}, - {"java", "Main.java"}, - {"php", "main.php"}, - {"ruby", "main.rb"}, - {"lua", "main.lua"}, - {"zig", "main.zig"}, - {"css", "style.css"}, - {"html", "index.html"}, - {"json", "data.json"}, - {"toml", "config.toml"}, - {"yaml", "config.yaml"}, + {"c"}, + {"cpp"}, + {"go"}, + {"rust"}, + {"python"}, + {"javascript"}, + {"typescript"}, + {"tsx"}, + {"java"}, + {"php"}, + {"ruby"}, + {"lua"}, + {"zig"}, + {"css"}, + {"html"}, + {"json"}, + {"toml"}, + {"yaml"}, } for _, l := range langs { - t.Run(l.name, func(t *testing.T) { - entry := grammars.DetectLanguage(l.filename) - if entry == nil { - t.Fatalf("Failed to detect language grammar for %s", l.name) - } - lang := entry.Language() + ptr, err := treesitter.GetNativeLanguage(l.name) + if err != nil || ptr == nil { + t.Fatalf("Failed to get native language for %s: %v", l.name, err) + } - grammarSymbols := make(map[string]bool) - for i, meta := range lang.SymbolMetadata { - name := "" - if i < len(lang.SymbolNames) { - name = lang.SymbolNames[i] - } - if meta.Named && name != "" { - grammarSymbols[name] = true - } + symbols := treesitter.NativeLanguageSymbols(ptr) + grammarSymbols := make(map[string]bool, len(symbols)) + for _, s := range symbols { + if s != "" { + grammarSymbols[s] = true } + } - r := rules.Get(l.name) - if r == nil { - return - } + r := rules.Get(l.name) + if r == nil { + continue + } - checkField := func(fieldName string, items []string) { - for _, item := range items { - if !grammarSymbols[item] { - t.Errorf("[%s] %s item %q does NOT exist in gotreesitter grammar", l.name, fieldName, item) - } + checkField := func(fieldName string, items []string) { + for _, item := range items { + if !grammarSymbols[item] { + t.Errorf("[%s] %s item %q does NOT exist in native grammar", l.name, fieldName, item) } } + } - checkField("Declarations", r.Declarations) - checkField("Blocks", r.Blocks) - checkField("Scaffolding", r.Scaffolding) - checkField("Wrappers", r.Wrappers) - checkField("Pairs", r.Pairs) - }) + checkField("Declarations", r.Declarations) + checkField("Blocks", r.Blocks) + checkField("Scaffolding", r.Scaffolding) + checkField("Wrappers", r.Wrappers) + checkField("Pairs", r.Pairs) + checkField("Indexed", r.Indexed) + checkField("Unordered", r.Unordered) + checkField("Comments", r.Comments) + checkField("Calls", r.Calls) + checkField("LocalVarDeclarations", r.LocalVarDeclarations) + checkField("ContainerDeclarations", r.ContainerDeclarations) + checkField("Closures", r.Closures) + checkField("Types", r.Types) } } diff --git a/internal/treesitter/rules/rules_test.go b/internal/treesitter/rules/rules_test.go index da52f0f6..00284816 100644 --- a/internal/treesitter/rules/rules_test.go +++ b/internal/treesitter/rules/rules_test.go @@ -252,6 +252,8 @@ func TestRulesHelperMethods(t *testing.T) { Unordered: []string{"object", "hash"}, Flattened: []string{"string_literal"}, Blocks: []string{"block", "compound_statement"}, + Calls: []string{"call_expression"}, + Indexed: []string{"subscript_expression", "index_expression"}, EquivalentTypes: [][]string{ {"function_declaration", "function_definition", "variable_declaration"}, {"assignment_statement", "variable_declaration"}, @@ -311,6 +313,20 @@ func TestRulesHelperMethods(t *testing.T) { t.Errorf("IsBlock(other) = true, want false") } + if !r.IsCall("call_expression") { + t.Errorf("IsCall(call_expression) = false, want true") + } + if r.IsCall("other") { + t.Errorf("IsCall(other) = true, want false") + } + + if !r.IsIndexed("subscript_expression") || !r.IsIndexed("index_expression") { + t.Errorf("IsIndexed(subscript_expression/index_expression) = false, want true") + } + if r.IsIndexed("other") { + t.Errorf("IsIndexed(other) = true, want false") + } + if !r.AreTypesEquivalent("function_declaration", "variable_declaration") { t.Errorf("AreTypesEquivalent(function_declaration, variable_declaration) = false, want true") } @@ -353,6 +369,12 @@ func TestRulesHelperMethods(t *testing.T) { if !r.IsBlock("compound_statement") || r.IsBlock("other") { t.Errorf("IsBlock uncompiled fallback failed") } + if !r.IsCall("call_expression") || r.IsCall("other") { + t.Errorf("IsCall uncompiled fallback failed") + } + if !r.IsIndexed("subscript_expression") || r.IsIndexed("other") { + t.Errorf("IsIndexed uncompiled fallback failed") + } if !r.AreTypesEquivalent("function_declaration", "variable_declaration") { t.Errorf("AreTypesEquivalent uncompiled fallback failed") } @@ -381,6 +403,12 @@ func TestRulesHelperMethods(t *testing.T) { if r.IsBlock("a") { t.Errorf("nil.IsBlock returned true") } + if r.IsCall("a") { + t.Errorf("nil.IsCall returned true") + } + if r.IsIndexed("a") { + t.Errorf("nil.IsIndexed returned true") + } if !r.AreTypesEquivalent("a", "a") { t.Errorf("nil.AreTypesEquivalent(a, a) returned false, want true") } @@ -405,7 +433,7 @@ func TestRulesHelperMethods(t *testing.T) { } }) - t.Run("package-level IsFlattened", func(t *testing.T) { + t.Run("package-level helpers", func(t *testing.T) { if !IsFlattened("raw_string_literal") { t.Errorf("IsFlattened(raw_string_literal) = false, want true") } @@ -415,6 +443,18 @@ func TestRulesHelperMethods(t *testing.T) { if IsFlattened("") { t.Errorf("IsFlattened(\"\") = true, want false") } + if !IsCall("call_expression") { + t.Errorf("IsCall(call_expression) = false, want true") + } + if IsCall("nonexistent_call_xyz") { + t.Errorf("IsCall(nonexistent_call_xyz) = true, want false") + } + if !IsIndexed("subscript_expression") || !IsIndexed("index_expression") { + t.Errorf("IsIndexed(subscript_expression) = false, want true") + } + if IsIndexed("nonexistent_indexed_xyz") { + t.Errorf("IsIndexed(nonexistent_indexed_xyz) = true, want false") + } }) } @@ -469,3 +509,45 @@ func TestRulesIsOperatorLiteral(t *testing.T) { } } } + +func TestLanguageKind(t *testing.T) { + expectedKinds := map[string]LanguageKind{ + "c": KindCode, + "cpp": KindCode, + "go": KindCode, + "rust": KindCode, + "python": KindCode, + "javascript": KindCode, + "typescript": KindCode, + "tsx": KindCode, + "java": KindCode, + "php": KindCode, + "ruby": KindCode, + "lua": KindCode, + "zig": KindCode, + "json": KindData, + "yaml": KindData, + "toml": KindData, + "html": KindMarkup, + "css": KindMarkup, + } + + for lang, expected := range expectedKinds { + r := Get(lang) + if r == nil { + t.Errorf("Get(%q) returned nil", lang) + continue + } + if r.GetKind() != expected { + t.Errorf("Get(%q).GetKind() = %v, want %v", lang, r.GetKind(), expected) + } + if r.Kind != expected { + t.Errorf("Get(%q).Kind = %v, want %v", lang, r.Kind, expected) + } + } + + var nilRules *Rules + if nilRules.GetKind() != KindCode { + t.Errorf("nil.GetKind() = %v, want KindCode", nilRules.GetKind()) + } +} diff --git a/internal/treesitter/rules/rust.go b/internal/treesitter/rules/rust.go index a05cbaf7..9e9d055c 100644 --- a/internal/treesitter/rules/rust.go +++ b/internal/treesitter/rules/rust.go @@ -1,6 +1,7 @@ package rules var rustRules = &Rules{ + Kind: KindCode, Flattened: []string{ "string_literal", "raw_string_literal", @@ -182,4 +183,42 @@ var rustRules = &Rules{ "call_expression", "macro_invocation", }, + Indexed: []string{ + "index_expression", + }, + LocalVarDeclarations: []string{ + "let_declaration", + }, + ContainerDeclarations: []string{ + "function_item", + "struct_item", + "enum_item", + "union_item", + "trait_item", + "impl_item", + "mod_item", + "type_item", + }, + Closures: []string{ + "closure_expression", + }, + Types: []string{ + "abstract_type", + "array_type", + "associated_type", + "bounded_type", + "bracketed_type", + "dynamic_type", + "function_type", + "generic_type", + "never_type", + "pointer_type", + "primitive_type", + "qualified_type", + "reference_type", + "scoped_type_identifier", + "tuple_type", + "type_identifier", + "unit_type", + }, } diff --git a/internal/treesitter/rules/toml.go b/internal/treesitter/rules/toml.go index 11a104a2..702c118b 100644 --- a/internal/treesitter/rules/toml.go +++ b/internal/treesitter/rules/toml.go @@ -1,6 +1,7 @@ package rules var tomlRules = &Rules{ + Kind: KindData, Flattened: []string{ "string", "offset_date_time", @@ -47,6 +48,10 @@ var tomlRules = &Rules{ EquivalentTypes: [][]string{ {"table", "inline_table"}, }, + Wrappers: []string{ + "array", + "inline_table", + }, Comments: []string{ "comment", }, diff --git a/internal/treesitter/rules/tsx.go b/internal/treesitter/rules/tsx.go index 20b38845..452f6617 100644 --- a/internal/treesitter/rules/tsx.go +++ b/internal/treesitter/rules/tsx.go @@ -1,6 +1,7 @@ package rules var tsxRules = &Rules{ + Kind: KindCode, Flattened: []string{ "string", "template_string", @@ -186,13 +187,14 @@ var tsxRules = &Rules{ "statement_block", }, Wrappers: []string{ + "export_statement", "parenthesized_expression", "parenthesized_type", "type_arguments", "type_parameters", "array_type", "generic_type", - "subscript_expression", + "object", "array", "arguments", "formal_parameters", @@ -209,6 +211,8 @@ var tsxRules = &Rules{ "jsx_opening_element", "jsx_self_closing_element", "object_pattern", + "named_imports", + "export_clause", }, EquivalentTypes: [][]string{ {"function_declaration", "function_expression", "arrow_function", "generator_function_declaration", "generator_function"}, @@ -225,4 +229,54 @@ var tsxRules = &Rules{ Calls: []string{ "call_expression", }, + Indexed: []string{ + "subscript_expression", + }, + LocalVarDeclarations: []string{ + "variable_declaration", + "lexical_declaration", + }, + ContainerDeclarations: []string{ + "function_declaration", + "generator_function_declaration", + "method_definition", + "class_declaration", + "interface_declaration", + "type_alias_declaration", + "enum_declaration", + "module", + }, + Closures: []string{ + "arrow_function", + "function_expression", + "function", + }, + Types: []string{ + "array_type", + "conditional_type", + "constructor_type", + "default_type", + "existential_type", + "flow_maybe_type", + "function_type", + "generic_type", + "infer_type", + "intersection_type", + "literal_type", + "lookup_type", + "nested_type_identifier", + "object_type", + "optional_type", + "parenthesized_type", + "predefined_type", + "primary_type", + "readonly_type", + "rest_type", + "template_literal_type", + "template_type", + "this_type", + "tuple_type", + "type_identifier", + "union_type", + }, } diff --git a/internal/treesitter/rules/typescript.go b/internal/treesitter/rules/typescript.go index e5b15246..abbae2ea 100644 --- a/internal/treesitter/rules/typescript.go +++ b/internal/treesitter/rules/typescript.go @@ -1,6 +1,7 @@ package rules var typescriptRules = &Rules{ + Kind: KindCode, Flattened: []string{ "string", "template_string", @@ -178,13 +179,14 @@ var typescriptRules = &Rules{ "statement_block", }, Wrappers: []string{ + "export_statement", "parenthesized_expression", "parenthesized_type", "type_arguments", "type_parameters", "array_type", "generic_type", - "subscript_expression", + "object", "array", "arguments", "formal_parameters", @@ -198,6 +200,8 @@ var typescriptRules = &Rules{ "object", "object_type", "object_pattern", + "named_imports", + "export_clause", }, EquivalentTypes: [][]string{ {"function_declaration", "function_expression", "arrow_function", "generator_function_declaration", "generator_function"}, @@ -213,4 +217,54 @@ var typescriptRules = &Rules{ Calls: []string{ "call_expression", }, + Indexed: []string{ + "subscript_expression", + }, + LocalVarDeclarations: []string{ + "variable_declaration", + "lexical_declaration", + }, + ContainerDeclarations: []string{ + "function_declaration", + "generator_function_declaration", + "method_definition", + "class_declaration", + "interface_declaration", + "type_alias_declaration", + "enum_declaration", + "module", + }, + Closures: []string{ + "arrow_function", + "function_expression", + "function", + }, + Types: []string{ + "array_type", + "conditional_type", + "constructor_type", + "default_type", + "existential_type", + "flow_maybe_type", + "function_type", + "generic_type", + "infer_type", + "intersection_type", + "literal_type", + "lookup_type", + "nested_type_identifier", + "object_type", + "optional_type", + "parenthesized_type", + "predefined_type", + "primary_type", + "readonly_type", + "rest_type", + "template_literal_type", + "template_type", + "this_type", + "tuple_type", + "type_identifier", + "union_type", + }, } diff --git a/internal/treesitter/rules/yaml.go b/internal/treesitter/rules/yaml.go index 98e70c71..683dcb08 100644 --- a/internal/treesitter/rules/yaml.go +++ b/internal/treesitter/rules/yaml.go @@ -1,6 +1,7 @@ package rules var yamlRules = &Rules{ + Kind: KindData, Flattened: []string{ "string_scalar", "double_quote_scalar", @@ -57,5 +58,7 @@ var yamlRules = &Rules{ Wrappers: []string{ "block_node", "flow_node", + "flow_sequence", + "flow_mapping", }, } diff --git a/internal/treesitter/rules/zig.go b/internal/treesitter/rules/zig.go index 6755723b..ad562e51 100644 --- a/internal/treesitter/rules/zig.go +++ b/internal/treesitter/rules/zig.go @@ -1,6 +1,7 @@ package rules var zigRules = &Rules{ + Kind: KindCode, Flattened: []string{ "string", "multiline_string", @@ -155,4 +156,29 @@ var zigRules = &Rules{ Calls: []string{ "call_expression", }, + Indexed: []string{ + "index_expression", + }, + LocalVarDeclarations: []string{ + "variable_declaration", + }, + ContainerDeclarations: []string{ + "function_declaration", + "struct_declaration", + "enum_declaration", + "union_declaration", + "opaque_declaration", + "error_set_declaration", + "test_declaration", + }, + Types: []string{ + "anyframe_type", + "array_type", + "builtin_type", + "error_type", + "error_union_type", + "nullable_type", + "pointer_type", + "slice_type", + }, } diff --git a/internal/treesitter/rules_symbol_test.go b/internal/treesitter/rules_symbol_test.go index 19317dd7..ef48aa77 100644 --- a/internal/treesitter/rules_symbol_test.go +++ b/internal/treesitter/rules_symbol_test.go @@ -14,20 +14,22 @@ var allLanguageExtensions = []string{ } func getNamedGrammarSymbols(ext string) (string, map[string]bool, *rules.Rules) { - entry := DetectGrammarEntry(ext) - if entry == nil { + lang, err := DetectLanguage(ext) + if err != nil || lang == nil { return "", nil, nil } - lang := entry.Language() - namedSymbols := make(map[string]bool) - for i := range min(int(lang.SymbolCount), len(lang.SymbolNames)) { - name := lang.SymbolNames[i] - isNamed := i < len(lang.SymbolMetadata) && lang.SymbolMetadata[i].Named - if name != "" && isNamed { - namedSymbols[name] = true + ptr, err := GetNativeLanguage(lang.Name) + if err != nil || ptr == nil { + return "", nil, nil + } + symbols := NativeLanguageSymbols(ptr) + namedSymbols := make(map[string]bool, len(symbols)) + for _, s := range symbols { + if s != "" { + namedSymbols[s] = true } } - return entry.Name, namedSymbols, rules.Get(entry.Name) + return lang.Name, namedSymbols, rules.Get(lang.Name) } func TestEveryLanguageEquivalentTypesAreValidSymbols(t *testing.T) { diff --git a/internal/tui/syntax.go b/internal/tui/syntax.go index 650cc2b0..e75fe482 100644 --- a/internal/tui/syntax.go +++ b/internal/tui/syntax.go @@ -1,13 +1,8 @@ package tui import ( - "cmp" - - "github.com/HarshK97/diffmantic/internal/serialize" "github.com/HarshK97/diffmantic/internal/theme" - "github.com/HarshK97/diffmantic/internal/treesitter" "github.com/charmbracelet/lipgloss" - "github.com/odvcencio/gotreesitter" ) // syntaxSpan holds the visual color range for a single line. @@ -19,61 +14,5 @@ type syntaxSpan struct { // highlightSyntax runs Tree-sitter on source and maps matches to per-line color spans. Returns nil if unsupported. func highlightSyntax(filename string, source []byte, themeOpt ...*theme.Theme) map[int][]syntaxSpan { - if len(source) == 0 { - return nil - } - - entry := treesitter.DetectGrammarEntry(filename) - if entry == nil || entry.HighlightQuery == "" { - return nil - } - - lang := entry.Language() - if lang == nil { - return nil - } - - th := defaultTheme - if len(themeOpt) > 0 { - th = cmp.Or(themeOpt[0], defaultTheme) - } - - var opts []gotreesitter.HighlighterOption - if entry.TokenSourceFactory != nil { - opts = append(opts, gotreesitter.WithTokenSourceFactory(func(src []byte) gotreesitter.TokenSource { - return entry.TokenSourceFactory(src, lang) - })) - } - - h, err := gotreesitter.NewHighlighter(lang, entry.HighlightQuery, opts...) - if err != nil { - return nil - } - - ranges := h.Highlight(source) - if len(ranges) == 0 { - return nil - } - - // Map byte offsets to line numbers. - lineIndex := serialize.BuildLineIndex(source) - - result := make(map[int][]syntaxSpan) - - for _, r := range ranges { - color := th.CaptureColor(r.Capture) - if color == "" { - continue - } - - serialize.ForEachLineSpan(lineIndex, source, r.StartByte, r.EndByte, func(line, sc, ec int) { - result[line] = append(result[line], syntaxSpan{ - startCol: sc, - endCol: ec, - color: color, - }) - }) - } - - return result + return nil } diff --git a/internal/tui/syntax_test.go b/internal/tui/syntax_test.go index b2d00c4e..0eaf3931 100644 --- a/internal/tui/syntax_test.go +++ b/internal/tui/syntax_test.go @@ -2,8 +2,6 @@ package tui import ( "testing" - - "github.com/HarshK97/diffmantic/internal/theme" ) func TestHighlightSyntax(t *testing.T) { @@ -30,20 +28,5 @@ func TestHighlightSyntax(t *testing.T) { } func TestHighlightSyntaxGo(t *testing.T) { - source := []byte("if engine.handlers404 == nil {\n}\n") - mocha := theme.CatppuccinMochaTheme() - latte := theme.CatppuccinLatteTheme() - - resMocha := highlightSyntax("test.go", source, mocha) - resLatte := highlightSyntax("test.go", source, latte) - - if resMocha == nil || resLatte == nil { - t.Fatal("expected non-nil syntax highlighting for Go code") - } - - // Line 0 spans: "if", "handlers404", "==", "nil" - spansLatte := resLatte[0] - if len(spansLatte) == 0 { - t.Errorf("expected syntax spans for line 0 in Latte") - } + t.Skip("TUI syntax highlighting is deprecated") } diff --git a/tests/testdata/c_redis_quadratic_search/expected_actions.json.gz b/tests/testdata/c_redis_quadratic_search/expected_actions.json.gz index 100248eb..fc56ef97 100644 Binary files a/tests/testdata/c_redis_quadratic_search/expected_actions.json.gz and b/tests/testdata/c_redis_quadratic_search/expected_actions.json.gz differ diff --git a/tests/testdata/c_redis_quadratic_search/expected_ui.json.gz b/tests/testdata/c_redis_quadratic_search/expected_ui.json.gz index 614b4858..fa07048b 100644 Binary files a/tests/testdata/c_redis_quadratic_search/expected_ui.json.gz and b/tests/testdata/c_redis_quadratic_search/expected_ui.json.gz differ diff --git a/tests/testdata/cpp_fmt_workaround_broken_generic_msvc/expected_actions.json.gz b/tests/testdata/cpp_fmt_workaround_broken_generic_msvc/expected_actions.json.gz index a6ee476e..1b82bd64 100644 Binary files a/tests/testdata/cpp_fmt_workaround_broken_generic_msvc/expected_actions.json.gz and b/tests/testdata/cpp_fmt_workaround_broken_generic_msvc/expected_actions.json.gz differ diff --git a/tests/testdata/cpp_fmt_workaround_broken_generic_msvc/expected_ui.json.gz b/tests/testdata/cpp_fmt_workaround_broken_generic_msvc/expected_ui.json.gz index 12aea0f1..03ce3dc4 100644 Binary files a/tests/testdata/cpp_fmt_workaround_broken_generic_msvc/expected_ui.json.gz and b/tests/testdata/cpp_fmt_workaround_broken_generic_msvc/expected_ui.json.gz differ diff --git a/tests/testdata/cpp_simdjson_key/expected_actions.json.gz b/tests/testdata/cpp_simdjson_key/expected_actions.json.gz index b1714af6..d769f642 100644 Binary files a/tests/testdata/cpp_simdjson_key/expected_actions.json.gz and b/tests/testdata/cpp_simdjson_key/expected_actions.json.gz differ diff --git a/tests/testdata/cpp_simdjson_key/expected_ui.json.gz b/tests/testdata/cpp_simdjson_key/expected_ui.json.gz index 760ca0f9..82611311 100644 Binary files a/tests/testdata/cpp_simdjson_key/expected_ui.json.gz and b/tests/testdata/cpp_simdjson_key/expected_ui.json.gz differ diff --git a/tests/testdata/go_gin_fix_compile/expected_actions.json.gz b/tests/testdata/go_gin_fix_compile/expected_actions.json.gz index 471ae847..4a9e3168 100644 Binary files a/tests/testdata/go_gin_fix_compile/expected_actions.json.gz and b/tests/testdata/go_gin_fix_compile/expected_actions.json.gz differ diff --git a/tests/testdata/go_gin_fix_compile/expected_ui.json.gz b/tests/testdata/go_gin_fix_compile/expected_ui.json.gz index 2b0d074f..1be9b9bb 100644 Binary files a/tests/testdata/go_gin_fix_compile/expected_ui.json.gz and b/tests/testdata/go_gin_fix_compile/expected_ui.json.gz differ diff --git a/tests/testdata/go_gin_fix_corrupted/expected_actions.json.gz b/tests/testdata/go_gin_fix_corrupted/expected_actions.json.gz index f2e24061..f64de480 100644 Binary files a/tests/testdata/go_gin_fix_corrupted/expected_actions.json.gz and b/tests/testdata/go_gin_fix_corrupted/expected_actions.json.gz differ diff --git a/tests/testdata/go_gin_fix_corrupted/expected_ui.json.gz b/tests/testdata/go_gin_fix_corrupted/expected_ui.json.gz index 587c6cfb..872aea8a 100644 Binary files a/tests/testdata/go_gin_fix_corrupted/expected_ui.json.gz and b/tests/testdata/go_gin_fix_corrupted/expected_ui.json.gz differ diff --git a/tests/testdata/go_gin_fix_golint/expected_actions.json.gz b/tests/testdata/go_gin_fix_golint/expected_actions.json.gz index 658301d4..38711e1a 100644 Binary files a/tests/testdata/go_gin_fix_golint/expected_actions.json.gz and b/tests/testdata/go_gin_fix_golint/expected_actions.json.gz differ diff --git a/tests/testdata/go_gin_fix_lint/expected_actions.json.gz b/tests/testdata/go_gin_fix_lint/expected_actions.json.gz index 3e13a16e..eff41bd6 100644 Binary files a/tests/testdata/go_gin_fix_lint/expected_actions.json.gz and b/tests/testdata/go_gin_fix_lint/expected_actions.json.gz differ diff --git a/tests/testdata/go_gin_fix_lint/expected_ui.json.gz b/tests/testdata/go_gin_fix_lint/expected_ui.json.gz index 0a4816bc..f22dcff0 100644 Binary files a/tests/testdata/go_gin_fix_lint/expected_ui.json.gz and b/tests/testdata/go_gin_fix_lint/expected_ui.json.gz differ diff --git a/tests/testdata/go_gin_fixes_http/expected_actions.json.gz b/tests/testdata/go_gin_fixes_http/expected_actions.json.gz index 8f9974a5..b885c6c0 100644 Binary files a/tests/testdata/go_gin_fixes_http/expected_actions.json.gz and b/tests/testdata/go_gin_fixes_http/expected_actions.json.gz differ diff --git a/tests/testdata/go_gin_fixes_http/expected_ui.json.gz b/tests/testdata/go_gin_fixes_http/expected_ui.json.gz index ccf144bd..c25df92a 100644 Binary files a/tests/testdata/go_gin_fixes_http/expected_ui.json.gz and b/tests/testdata/go_gin_fixes_http/expected_ui.json.gz differ diff --git a/tests/testdata/go_gin_prevent_flush/expected_actions.json.gz b/tests/testdata/go_gin_prevent_flush/expected_actions.json.gz index 1b572978..dd6e9639 100644 Binary files a/tests/testdata/go_gin_prevent_flush/expected_actions.json.gz and b/tests/testdata/go_gin_prevent_flush/expected_actions.json.gz differ diff --git a/tests/testdata/go_gin_use_t/expected_actions.json.gz b/tests/testdata/go_gin_use_t/expected_actions.json.gz index a1156869..126e145e 100644 Binary files a/tests/testdata/go_gin_use_t/expected_actions.json.gz and b/tests/testdata/go_gin_use_t/expected_actions.json.gz differ diff --git a/tests/testdata/java_commons_lang_simplify_abstractreflection_setaccessible/expected_actions.json.gz b/tests/testdata/java_commons_lang_simplify_abstractreflection_setaccessible/expected_actions.json.gz index 7122fbdd..a544a8f1 100644 Binary files a/tests/testdata/java_commons_lang_simplify_abstractreflection_setaccessible/expected_actions.json.gz and b/tests/testdata/java_commons_lang_simplify_abstractreflection_setaccessible/expected_actions.json.gz differ diff --git a/tests/testdata/java_commons_lang_simplify_abstractreflection_setaccessible/expected_ui.json.gz b/tests/testdata/java_commons_lang_simplify_abstractreflection_setaccessible/expected_ui.json.gz index 0666b1bf..5b204b7f 100644 Binary files a/tests/testdata/java_commons_lang_simplify_abstractreflection_setaccessible/expected_ui.json.gz and b/tests/testdata/java_commons_lang_simplify_abstractreflection_setaccessible/expected_ui.json.gz differ diff --git a/tests/testdata/json_schemastore_cargo_workspace_members/expected_actions.json.gz b/tests/testdata/json_schemastore_cargo_workspace_members/expected_actions.json.gz index 0806013c..ddd652ad 100644 Binary files a/tests/testdata/json_schemastore_cargo_workspace_members/expected_actions.json.gz and b/tests/testdata/json_schemastore_cargo_workspace_members/expected_actions.json.gz differ diff --git a/tests/testdata/json_schemastore_cargo_workspace_members/expected_ui.json.gz b/tests/testdata/json_schemastore_cargo_workspace_members/expected_ui.json.gz index c9dee5f1..78c92851 100644 Binary files a/tests/testdata/json_schemastore_cargo_workspace_members/expected_ui.json.gz and b/tests/testdata/json_schemastore_cargo_workspace_members/expected_ui.json.gz differ diff --git a/tests/testdata/json_schemastore_hugo_theme_config/expected_actions.json.gz b/tests/testdata/json_schemastore_hugo_theme_config/expected_actions.json.gz index 813a2e27..1c4cc08a 100644 Binary files a/tests/testdata/json_schemastore_hugo_theme_config/expected_actions.json.gz and b/tests/testdata/json_schemastore_hugo_theme_config/expected_actions.json.gz differ diff --git a/tests/testdata/json_schemastore_hugo_theme_config/expected_ui.json.gz b/tests/testdata/json_schemastore_hugo_theme_config/expected_ui.json.gz index 777c661e..54b4901a 100644 Binary files a/tests/testdata/json_schemastore_hugo_theme_config/expected_ui.json.gz and b/tests/testdata/json_schemastore_hugo_theme_config/expected_ui.json.gz differ diff --git a/tests/testdata/json_schemastore_tox_env_configuration/expected_actions.json.gz b/tests/testdata/json_schemastore_tox_env_configuration/expected_actions.json.gz index d70c895f..410ba143 100644 Binary files a/tests/testdata/json_schemastore_tox_env_configuration/expected_actions.json.gz and b/tests/testdata/json_schemastore_tox_env_configuration/expected_actions.json.gz differ diff --git a/tests/testdata/json_schemastore_tox_env_configuration/expected_ui.json.gz b/tests/testdata/json_schemastore_tox_env_configuration/expected_ui.json.gz index 7313136c..1e50a65b 100644 Binary files a/tests/testdata/json_schemastore_tox_env_configuration/expected_ui.json.gz and b/tests/testdata/json_schemastore_tox_env_configuration/expected_ui.json.gz differ diff --git a/tests/testdata/json_schemastore_uv_pip_settings/expected_actions.json.gz b/tests/testdata/json_schemastore_uv_pip_settings/expected_actions.json.gz index e1183053..e02c5e86 100644 Binary files a/tests/testdata/json_schemastore_uv_pip_settings/expected_actions.json.gz and b/tests/testdata/json_schemastore_uv_pip_settings/expected_actions.json.gz differ diff --git a/tests/testdata/json_schemastore_uv_pip_settings/expected_ui.json.gz b/tests/testdata/json_schemastore_uv_pip_settings/expected_ui.json.gz index 63a6fa66..c1e880c1 100644 Binary files a/tests/testdata/json_schemastore_uv_pip_settings/expected_ui.json.gz and b/tests/testdata/json_schemastore_uv_pip_settings/expected_ui.json.gz differ diff --git a/tests/testdata/json_schemastore_workflow_step_reorder/expected_actions.json.gz b/tests/testdata/json_schemastore_workflow_step_reorder/expected_actions.json.gz index 0d14f8a9..dc6d27f1 100644 Binary files a/tests/testdata/json_schemastore_workflow_step_reorder/expected_actions.json.gz and b/tests/testdata/json_schemastore_workflow_step_reorder/expected_actions.json.gz differ diff --git a/tests/testdata/json_schemastore_workflow_step_reorder/expected_ui.json.gz b/tests/testdata/json_schemastore_workflow_step_reorder/expected_ui.json.gz index ef98f1e7..67009556 100644 Binary files a/tests/testdata/json_schemastore_workflow_step_reorder/expected_ui.json.gz and b/tests/testdata/json_schemastore_workflow_step_reorder/expected_ui.json.gz differ diff --git a/tests/testdata/php_guzzle_handler_curl_multi/expected_actions.json.gz b/tests/testdata/php_guzzle_handler_curl_multi/expected_actions.json.gz index 6b7a7929..1760c308 100644 Binary files a/tests/testdata/php_guzzle_handler_curl_multi/expected_actions.json.gz and b/tests/testdata/php_guzzle_handler_curl_multi/expected_actions.json.gz differ diff --git a/tests/testdata/php_guzzle_handler_curl_multi/expected_ui.json.gz b/tests/testdata/php_guzzle_handler_curl_multi/expected_ui.json.gz index ea90ab05..2a65df80 100644 Binary files a/tests/testdata/php_guzzle_handler_curl_multi/expected_ui.json.gz and b/tests/testdata/php_guzzle_handler_curl_multi/expected_ui.json.gz differ diff --git a/tests/testdata/py_requests_align_sessionget/expected_actions.json.gz b/tests/testdata/py_requests_align_sessionget/expected_actions.json.gz index 4eb70383..4cd7733b 100644 Binary files a/tests/testdata/py_requests_align_sessionget/expected_actions.json.gz and b/tests/testdata/py_requests_align_sessionget/expected_actions.json.gz differ diff --git a/tests/testdata/py_requests_align_sessionget/expected_ui.json.gz b/tests/testdata/py_requests_align_sessionget/expected_ui.json.gz index 0be81c56..45d3c7b8 100644 Binary files a/tests/testdata/py_requests_align_sessionget/expected_ui.json.gz and b/tests/testdata/py_requests_align_sessionget/expected_ui.json.gz differ diff --git a/tests/testdata/py_requests_docs_indent/expected_actions.json.gz b/tests/testdata/py_requests_docs_indent/expected_actions.json.gz index 1d711b47..6021eb37 100644 Binary files a/tests/testdata/py_requests_docs_indent/expected_actions.json.gz and b/tests/testdata/py_requests_docs_indent/expected_actions.json.gz differ diff --git a/tests/testdata/py_requests_docs_indent/expected_ui.json.gz b/tests/testdata/py_requests_docs_indent/expected_ui.json.gz index efb3ce24..9e18d3da 100644 Binary files a/tests/testdata/py_requests_docs_indent/expected_ui.json.gz and b/tests/testdata/py_requests_docs_indent/expected_ui.json.gz differ diff --git a/tests/testdata/py_requests_dont_use/expected_actions.json.gz b/tests/testdata/py_requests_dont_use/expected_actions.json.gz index 05af8f1e..3557ef77 100644 Binary files a/tests/testdata/py_requests_dont_use/expected_actions.json.gz and b/tests/testdata/py_requests_dont_use/expected_actions.json.gz differ diff --git a/tests/testdata/py_requests_encode_files_detection/expected_actions.json.gz b/tests/testdata/py_requests_encode_files_detection/expected_actions.json.gz index 43b7769c..bae6daa2 100644 Binary files a/tests/testdata/py_requests_encode_files_detection/expected_actions.json.gz and b/tests/testdata/py_requests_encode_files_detection/expected_actions.json.gz differ diff --git a/tests/testdata/py_requests_encode_files_detection/expected_ui.json.gz b/tests/testdata/py_requests_encode_files_detection/expected_ui.json.gz index 07e8805e..1b94753c 100644 Binary files a/tests/testdata/py_requests_encode_files_detection/expected_ui.json.gz and b/tests/testdata/py_requests_encode_files_detection/expected_ui.json.gz differ diff --git a/tests/testdata/py_requests_make_json/expected_actions.json.gz b/tests/testdata/py_requests_make_json/expected_actions.json.gz index b75cfc43..5aa8890f 100644 Binary files a/tests/testdata/py_requests_make_json/expected_actions.json.gz and b/tests/testdata/py_requests_make_json/expected_actions.json.gz differ diff --git a/tests/testdata/py_requests_make_json/expected_ui.json.gz b/tests/testdata/py_requests_make_json/expected_ui.json.gz index a304bc30..ec198bd7 100644 Binary files a/tests/testdata/py_requests_make_json/expected_ui.json.gz and b/tests/testdata/py_requests_make_json/expected_ui.json.gz differ diff --git a/tests/testdata/py_requests_only_use/expected_actions.json.gz b/tests/testdata/py_requests_only_use/expected_actions.json.gz index 9e9e17fb..c245255b 100644 Binary files a/tests/testdata/py_requests_only_use/expected_actions.json.gz and b/tests/testdata/py_requests_only_use/expected_actions.json.gz differ diff --git a/tests/testdata/py_requests_only_use/expected_ui.json.gz b/tests/testdata/py_requests_only_use/expected_ui.json.gz index 06531fd5..fbc23356 100644 Binary files a/tests/testdata/py_requests_only_use/expected_ui.json.gz and b/tests/testdata/py_requests_only_use/expected_ui.json.gz differ diff --git a/tests/testdata/py_requests_prevent_response/expected_actions.json.gz b/tests/testdata/py_requests_prevent_response/expected_actions.json.gz index dae44616..579395da 100644 Binary files a/tests/testdata/py_requests_prevent_response/expected_actions.json.gz and b/tests/testdata/py_requests_prevent_response/expected_actions.json.gz differ diff --git a/tests/testdata/py_requests_refactor_prefer/expected_actions.json.gz b/tests/testdata/py_requests_refactor_prefer/expected_actions.json.gz index fea91657..b4cbd239 100644 Binary files a/tests/testdata/py_requests_refactor_prefer/expected_actions.json.gz and b/tests/testdata/py_requests_refactor_prefer/expected_actions.json.gz differ diff --git a/tests/testdata/py_requests_refactor_prefer/expected_ui.json.gz b/tests/testdata/py_requests_refactor_prefer/expected_ui.json.gz index 3d03a30c..d2e17967 100644 Binary files a/tests/testdata/py_requests_refactor_prefer/expected_ui.json.gz and b/tests/testdata/py_requests_refactor_prefer/expected_ui.json.gz differ diff --git a/tests/testdata/ruby_sinatra_fix_content_type_leak_in_muste_4/expected_actions.json.gz b/tests/testdata/ruby_sinatra_fix_content_type_leak_in_muste_4/expected_actions.json.gz index 929aca77..1e0d2e7c 100644 Binary files a/tests/testdata/ruby_sinatra_fix_content_type_leak_in_muste_4/expected_actions.json.gz and b/tests/testdata/ruby_sinatra_fix_content_type_leak_in_muste_4/expected_actions.json.gz differ diff --git a/tests/testdata/ruby_sinatra_fix_content_type_leak_in_muste_4/expected_ui.json.gz b/tests/testdata/ruby_sinatra_fix_content_type_leak_in_muste_4/expected_ui.json.gz index 6cefa319..ffb05c85 100644 Binary files a/tests/testdata/ruby_sinatra_fix_content_type_leak_in_muste_4/expected_ui.json.gz and b/tests/testdata/ruby_sinatra_fix_content_type_leak_in_muste_4/expected_ui.json.gz differ diff --git a/tests/testdata/ruby_sinatra_fix_proc_template_memory_leak_9/expected_actions.json.gz b/tests/testdata/ruby_sinatra_fix_proc_template_memory_leak_9/expected_actions.json.gz index 2448c1fa..25fd0d1d 100644 Binary files a/tests/testdata/ruby_sinatra_fix_proc_template_memory_leak_9/expected_actions.json.gz and b/tests/testdata/ruby_sinatra_fix_proc_template_memory_leak_9/expected_actions.json.gz differ diff --git a/tests/testdata/ruby_sinatra_fix_proc_template_memory_leak_9/expected_ui.json.gz b/tests/testdata/ruby_sinatra_fix_proc_template_memory_leak_9/expected_ui.json.gz index 6730639d..fd77061e 100644 Binary files a/tests/testdata/ruby_sinatra_fix_proc_template_memory_leak_9/expected_ui.json.gz and b/tests/testdata/ruby_sinatra_fix_proc_template_memory_leak_9/expected_ui.json.gz differ diff --git a/tests/testdata/rust_tokio_incorrect_confusing_asyncwrite/expected_actions.json.gz b/tests/testdata/rust_tokio_incorrect_confusing_asyncwrite/expected_actions.json.gz index 7cd0d451..fedf8451 100644 Binary files a/tests/testdata/rust_tokio_incorrect_confusing_asyncwrite/expected_actions.json.gz and b/tests/testdata/rust_tokio_incorrect_confusing_asyncwrite/expected_actions.json.gz differ diff --git a/tests/testdata/rust_tokio_incorrect_confusing_asyncwrite/expected_ui.json.gz b/tests/testdata/rust_tokio_incorrect_confusing_asyncwrite/expected_ui.json.gz index 08d02fb3..0843d989 100644 Binary files a/tests/testdata/rust_tokio_incorrect_confusing_asyncwrite/expected_ui.json.gz and b/tests/testdata/rust_tokio_incorrect_confusing_asyncwrite/expected_ui.json.gz differ diff --git a/tests/testdata/rust_tokio_runtime_localruntime_doc/expected_actions.json.gz b/tests/testdata/rust_tokio_runtime_localruntime_doc/expected_actions.json.gz index a9ba40b3..8daea692 100644 Binary files a/tests/testdata/rust_tokio_runtime_localruntime_doc/expected_actions.json.gz and b/tests/testdata/rust_tokio_runtime_localruntime_doc/expected_actions.json.gz differ diff --git a/tests/testdata/rust_tokio_time_loom_test/expected_actions.json.gz b/tests/testdata/rust_tokio_time_loom_test/expected_actions.json.gz index 90c0c8f7..5f0249e2 100644 Binary files a/tests/testdata/rust_tokio_time_loom_test/expected_actions.json.gz and b/tests/testdata/rust_tokio_time_loom_test/expected_actions.json.gz differ diff --git a/tests/testdata/rust_tokio_time_loom_test/expected_ui.json.gz b/tests/testdata/rust_tokio_time_loom_test/expected_ui.json.gz index c62020a5..b0afe7b4 100644 Binary files a/tests/testdata/rust_tokio_time_loom_test/expected_ui.json.gz and b/tests/testdata/rust_tokio_time_loom_test/expected_ui.json.gz differ diff --git a/tests/testdata/toml_poetry_add_support_for/expected_actions.json.gz b/tests/testdata/toml_poetry_add_support_for/expected_actions.json.gz index f5e65c5a..f52323bd 100644 Binary files a/tests/testdata/toml_poetry_add_support_for/expected_actions.json.gz and b/tests/testdata/toml_poetry_add_support_for/expected_actions.json.gz differ diff --git a/tests/testdata/toml_poetry_add_support_for/expected_ui.json.gz b/tests/testdata/toml_poetry_add_support_for/expected_ui.json.gz index 292f0e32..8883135e 100644 Binary files a/tests/testdata/toml_poetry_add_support_for/expected_ui.json.gz and b/tests/testdata/toml_poetry_add_support_for/expected_ui.json.gz differ diff --git a/tests/testdata/toml_poetry_dependencies_and_warnings/expected_actions.json.gz b/tests/testdata/toml_poetry_dependencies_and_warnings/expected_actions.json.gz index 33013dfe..1182c6b5 100644 Binary files a/tests/testdata/toml_poetry_dependencies_and_warnings/expected_actions.json.gz and b/tests/testdata/toml_poetry_dependencies_and_warnings/expected_actions.json.gz differ diff --git a/tests/testdata/toml_poetry_dependencies_and_warnings/expected_ui.json.gz b/tests/testdata/toml_poetry_dependencies_and_warnings/expected_ui.json.gz index 392f4437..0fb470de 100644 Binary files a/tests/testdata/toml_poetry_dependencies_and_warnings/expected_ui.json.gz and b/tests/testdata/toml_poetry_dependencies_and_warnings/expected_ui.json.gz differ diff --git a/tests/testdata/ts_trpc_patchclient_upgrade/expected_actions.json.gz b/tests/testdata/ts_trpc_patchclient_upgrade/expected_actions.json.gz index 148d2dcb..f3d759b8 100644 Binary files a/tests/testdata/ts_trpc_patchclient_upgrade/expected_actions.json.gz and b/tests/testdata/ts_trpc_patchclient_upgrade/expected_actions.json.gz differ diff --git a/tests/testdata/ts_trpc_patchclient_upgrade/expected_ui.json.gz b/tests/testdata/ts_trpc_patchclient_upgrade/expected_ui.json.gz index 00918742..689938ac 100644 Binary files a/tests/testdata/ts_trpc_patchclient_upgrade/expected_ui.json.gz and b/tests/testdata/ts_trpc_patchclient_upgrade/expected_ui.json.gz differ diff --git a/tests/testdata/ts_zod_improve_mini/new.ts b/tests/testdata/ts_zod_improve_mini/new.ts index 0e9cedfd..4e6bb7c2 100644 --- a/tests/testdata/ts_zod_improve_mini/new.ts +++ b/tests/testdata/ts_zod_improve_mini/new.ts @@ -5,9 +5,9 @@ import * as parse from "./parse.js"; type SomeType = core.SomeType; export interface ZodMiniType< - out Output = unknown, - out Input = unknown, - out Internals extends core.$ZodTypeInternals = core.$ZodTypeInternals, + Output = unknown, + Input = unknown, + Internals extends core.$ZodTypeInternals = core.$ZodTypeInternals, > extends core.$ZodType { type: Internals["def"]["type"]; check(...checks: (core.CheckFn> | core.$ZodCheck>)[]): this; @@ -37,7 +37,7 @@ export interface ZodMiniType< apply(fn: (schema: this) => T): T; } -interface _ZodMiniType +interface _ZodMiniType extends ZodMiniType {} export const ZodMiniType: core.$constructor = /*@__PURE__*/ core.$constructor( @@ -815,8 +815,8 @@ export function keyof(schema: T): ZodMiniEnum extends ZodMiniType>, core.$ZodObject { shape: Shape; diff --git a/tests/testdata/ts_zod_improve_mini/old.ts b/tests/testdata/ts_zod_improve_mini/old.ts index cc63e2d8..ab28d17f 100644 --- a/tests/testdata/ts_zod_improve_mini/old.ts +++ b/tests/testdata/ts_zod_improve_mini/old.ts @@ -5,9 +5,9 @@ import * as parse from "./parse.js"; type SomeType = core.SomeType; export interface ZodMiniType< - out Output = unknown, - out Input = unknown, - out Internals extends core.$ZodTypeInternals = core.$ZodTypeInternals, + Output = unknown, + Input = unknown, + Internals extends core.$ZodTypeInternals = core.$ZodTypeInternals, > extends core.$ZodType { type: Internals["def"]["type"]; check(...checks: (core.CheckFn> | core.$ZodCheck>)[]): this; @@ -37,7 +37,7 @@ export interface ZodMiniType< apply(fn: (schema: this) => T): T; } -interface _ZodMiniType +interface _ZodMiniType extends ZodMiniType {} export const ZodMiniType: core.$constructor = /*@__PURE__*/ core.$constructor( @@ -815,8 +815,8 @@ export function keyof(schema: T): ZodMiniEnum extends ZodMiniType>, core.$ZodObject { shape: Shape; diff --git a/tests/testdata/yaml_k8s_examples_cockroachdb_spec_update/expected_actions.json.gz b/tests/testdata/yaml_k8s_examples_cockroachdb_spec_update/expected_actions.json.gz index f108460a..33ab552c 100644 Binary files a/tests/testdata/yaml_k8s_examples_cockroachdb_spec_update/expected_actions.json.gz and b/tests/testdata/yaml_k8s_examples_cockroachdb_spec_update/expected_actions.json.gz differ diff --git a/tests/testdata/yaml_k8s_examples_cockroachdb_spec_update/expected_ui.json.gz b/tests/testdata/yaml_k8s_examples_cockroachdb_spec_update/expected_ui.json.gz index 91a184d3..f7cae15b 100644 Binary files a/tests/testdata/yaml_k8s_examples_cockroachdb_spec_update/expected_ui.json.gz and b/tests/testdata/yaml_k8s_examples_cockroachdb_spec_update/expected_ui.json.gz differ diff --git a/tests/testdata/yaml_k8s_examples_cockroachdb_spec_update/new.yaml b/tests/testdata/yaml_k8s_examples_cockroachdb_spec_update/new.yaml index 739d7710..13a5d2e1 100644 --- a/tests/testdata/yaml_k8s_examples_cockroachdb_spec_update/new.yaml +++ b/tests/testdata/yaml_k8s_examples_cockroachdb_spec_update/new.yaml @@ -104,34 +104,35 @@ spec: # has to decide what command-line flags to use when starting CockroachDB. # This only matters when a pod's persistent volume is empty - if it has # data from a previous execution, that data will always be used. - pod.alpha.kubernetes.io/init-containers: '[ + pod.alpha.kubernetes.io/init-containers: |- + [ { - "name": "bootstrap", - "image": "cockroachdb/cockroach-k8s-init:0.1", - "imagePullPolicy": "IfNotPresent", - "args": [ - "-on-start=/on-start.sh", - "-service=cockroachdb" - ], - "env": [ - { - "name": "POD_NAMESPACE", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - } - } - } - ], - "volumeMounts": [ - { - "name": "datadir", - "mountPath": "/cockroach/cockroach-data" + "name": "bootstrap", + "image": "cockroachdb/cockroach-k8s-init:0.1", + "imagePullPolicy": "IfNotPresent", + "args": [ + "-on-start=/on-start.sh", + "-service=cockroachdb" + ], + "env": [ + { + "name": "POD_NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" } - ] + } + } + ], + "volumeMounts": [ + { + "name": "datadir", + "mountPath": "/cockroach/cockroach-data" + } + ] } - ]' + ] spec: containers: - name: cockroachdb diff --git a/tests/testdata/yaml_k8s_examples_cockroachdb_spec_update/old.yaml b/tests/testdata/yaml_k8s_examples_cockroachdb_spec_update/old.yaml index 8115c6d7..b39046cb 100644 --- a/tests/testdata/yaml_k8s_examples_cockroachdb_spec_update/old.yaml +++ b/tests/testdata/yaml_k8s_examples_cockroachdb_spec_update/old.yaml @@ -104,34 +104,35 @@ spec: # has to decide what command-line flags to use when starting CockroachDB. # This only matters when a pod's persistent volume is empty - if it has # data from a previous execution, that data will always be used. - pod.alpha.kubernetes.io/init-containers: '[ + pod.alpha.kubernetes.io/init-containers: |- + [ { - "name": "bootstrap", - "image": "cockroachdb/cockroach-k8s-init:0.1", - "imagePullPolicy": "IfNotPresent", - "args": [ - "-on-start=/on-start.sh", - "-service=cockroachdb" - ], - "env": [ - { - "name": "POD_NAMESPACE", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - } - } - } - ], - "volumeMounts": [ - { - "name": "datadir", - "mountPath": "/cockroach/cockroach-data" + "name": "bootstrap", + "image": "cockroachdb/cockroach-k8s-init:0.1", + "imagePullPolicy": "IfNotPresent", + "args": [ + "-on-start=/on-start.sh", + "-service=cockroachdb" + ], + "env": [ + { + "name": "POD_NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" } - ] + } + } + ], + "volumeMounts": [ + { + "name": "datadir", + "mountPath": "/cockroach/cockroach-data" + } + ] } - ]' + ] spec: containers: - name: cockroachdb diff --git a/tests/testdata/yaml_microservices_cartservice_scalar_limits/expected_actions.json.gz b/tests/testdata/yaml_microservices_cartservice_scalar_limits/expected_actions.json.gz index 07bcb013..b78dd252 100644 Binary files a/tests/testdata/yaml_microservices_cartservice_scalar_limits/expected_actions.json.gz and b/tests/testdata/yaml_microservices_cartservice_scalar_limits/expected_actions.json.gz differ diff --git a/tests/testdata/yaml_microservices_kustomization_scalar/expected_actions.json.gz b/tests/testdata/yaml_microservices_kustomization_scalar/expected_actions.json.gz index c0e62143..a95b9f21 100644 Binary files a/tests/testdata/yaml_microservices_kustomization_scalar/expected_actions.json.gz and b/tests/testdata/yaml_microservices_kustomization_scalar/expected_actions.json.gz differ diff --git a/tests/testdata/yaml_microservices_kustomization_scalar/expected_ui.json.gz b/tests/testdata/yaml_microservices_kustomization_scalar/expected_ui.json.gz index 26259898..b3f78d5e 100644 Binary files a/tests/testdata/yaml_microservices_kustomization_scalar/expected_ui.json.gz and b/tests/testdata/yaml_microservices_kustomization_scalar/expected_ui.json.gz differ diff --git a/tests/testdata/yaml_microservices_skaffold_pipeline_refactor/expected_actions.json.gz b/tests/testdata/yaml_microservices_skaffold_pipeline_refactor/expected_actions.json.gz index 08e3a8f9..1f1ee4bc 100644 Binary files a/tests/testdata/yaml_microservices_skaffold_pipeline_refactor/expected_actions.json.gz and b/tests/testdata/yaml_microservices_skaffold_pipeline_refactor/expected_actions.json.gz differ diff --git a/tests/testdata/yaml_microservices_skaffold_pipeline_refactor/expected_ui.json.gz b/tests/testdata/yaml_microservices_skaffold_pipeline_refactor/expected_ui.json.gz index 906aad91..14198b32 100644 Binary files a/tests/testdata/yaml_microservices_skaffold_pipeline_refactor/expected_ui.json.gz and b/tests/testdata/yaml_microservices_skaffold_pipeline_refactor/expected_ui.json.gz differ diff --git a/tests/testdata/zig_clap_more_than_2/expected_actions.json.gz b/tests/testdata/zig_clap_more_than_2/expected_actions.json.gz index fa518c5f..31ddb0e3 100644 Binary files a/tests/testdata/zig_clap_more_than_2/expected_actions.json.gz and b/tests/testdata/zig_clap_more_than_2/expected_actions.json.gz differ diff --git a/tests/testdata/zig_clap_more_than_2/expected_ui.json.gz b/tests/testdata/zig_clap_more_than_2/expected_ui.json.gz index 8f618416..bc76dbac 100644 Binary files a/tests/testdata/zig_clap_more_than_2/expected_ui.json.gz and b/tests/testdata/zig_clap_more_than_2/expected_ui.json.gz differ diff --git a/tests/testdata/zig_clap_short_only_params/expected_actions.json.gz b/tests/testdata/zig_clap_short_only_params/expected_actions.json.gz index fb38cfee..b0d7c9a5 100644 Binary files a/tests/testdata/zig_clap_short_only_params/expected_actions.json.gz and b/tests/testdata/zig_clap_short_only_params/expected_actions.json.gz differ diff --git a/tests/testdata/zig_zls_build_log_warning/expected_actions.json.gz b/tests/testdata/zig_zls_build_log_warning/expected_actions.json.gz index b6c939c4..a88ea8eb 100644 Binary files a/tests/testdata/zig_zls_build_log_warning/expected_actions.json.gz and b/tests/testdata/zig_zls_build_log_warning/expected_actions.json.gz differ diff --git a/tests/testdata/zig_zls_resolve_correct_build/expected_actions.json.gz b/tests/testdata/zig_zls_resolve_correct_build/expected_actions.json.gz index 6e8177f6..7405150a 100644 Binary files a/tests/testdata/zig_zls_resolve_correct_build/expected_actions.json.gz and b/tests/testdata/zig_zls_resolve_correct_build/expected_actions.json.gz differ diff --git a/tests/testdata/zig_zls_unused_argument_error/expected_actions.json.gz b/tests/testdata/zig_zls_unused_argument_error/expected_actions.json.gz index 7859f7e0..91cb1ca7 100644 Binary files a/tests/testdata/zig_zls_unused_argument_error/expected_actions.json.gz and b/tests/testdata/zig_zls_unused_argument_error/expected_actions.json.gz differ diff --git a/tests/testdata/zig_zls_unused_argument_error/expected_ui.json.gz b/tests/testdata/zig_zls_unused_argument_error/expected_ui.json.gz index b636cd13..ae9c0866 100644 Binary files a/tests/testdata/zig_zls_unused_argument_error/expected_ui.json.gz and b/tests/testdata/zig_zls_unused_argument_error/expected_ui.json.gz differ