From e912a8a50ea6e551f7975f1d05648fdff0d1dc41 Mon Sep 17 00:00:00 2001 From: Bryan Woodruff Date: Wed, 14 Oct 2026 13:12:12 +0100 Subject: [PATCH 1/3] fix(kg): skip binary files sharing a source extension before tree-sitter parse MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A NUL-byte head check in processWithTreeSitter skips binary content (e.g. HLS .ts video segments that collide with the TypeScript extension) before the parser runs, avoiding the 10s parse deadline tree-sitter otherwise burns per file — and the accompanying warning — on every re-index. The files were already skipped (no entities), just slowly and noisily. Uses git's binary heuristic (NUL in the first 8000 bytes); valid source text never contains NUL. Co-Authored-By: Claude --- .../internal/knowledge/indexer_treesitter.go | 9 +++ .../knowledge/indexer_treesitter_test.go | 56 +++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/src/kg/internal/knowledge/indexer_treesitter.go b/src/kg/internal/knowledge/indexer_treesitter.go index 7f5a334..377a935 100644 --- a/src/kg/internal/knowledge/indexer_treesitter.go +++ b/src/kg/internal/knowledge/indexer_treesitter.go @@ -1,6 +1,7 @@ package knowledge import ( + "bytes" "context" "fmt" "os" @@ -578,6 +579,14 @@ func (idx *Indexer) processWithTreeSitter( return fmt.Errorf("read %s: %w", absPath, err) } + // Tree-sitter spends the full parse timeout chewing on binary content that + // happens to share a source extension (e.g. HLS .ts video segments vs. + // TypeScript). A NUL byte in the head is a reliable binary signal — valid + // source text never contains one — so skip rather than burn the deadline. + if bytes.IndexByte(src[:min(len(src), 8000)], 0) >= 0 { + return nil + } + parser := sitter.NewParser() parser.SetLanguage(cfg.Language) diff --git a/src/kg/internal/knowledge/indexer_treesitter_test.go b/src/kg/internal/knowledge/indexer_treesitter_test.go index 777e121..bde8aeb 100644 --- a/src/kg/internal/knowledge/indexer_treesitter_test.go +++ b/src/kg/internal/knowledge/indexer_treesitter_test.go @@ -755,6 +755,62 @@ export function createShape(): Shape { } } +// TestIndexer_BinaryTSSegment verifies that a binary file sharing a source +// extension (an HLS .ts MPEG transport-stream segment, not TypeScript) is +// skipped cleanly: the NUL-byte guard in processWithTreeSitter returns before +// the parser runs, so no file entity is written and the indexer does not burn +// the 10s parse deadline on content tree-sitter cannot parse. +// +// Verified pre/post behavior (guard removed vs. present): +// - Without the guard, the blob is fed to the TypeScript parser, which always +// writes the "file:clip.ts" entity before walking — so the no-entity +// assertion below FAILS. (In production the real symptom is the 10s parse +// timeout + warning that HLS .ts segments trigger; this minimal synthetic +// blob parses fast, so the clean, deterministic signal here is the written +// file entity rather than a stall.) +// - With the guard, processWithTreeSitter returns nil before the entity write, +// so no entity is created, no error is counted, and the test PASSES. +func TestIndexer_BinaryTSSegment(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + tmpDir := t.TempDir() + dbPath := filepath.Join(tmpDir, "test.db") + srcDir := filepath.Join(tmpDir, "src") + if err := os.MkdirAll(srcDir, 0755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + + // Simulate an MPEG-TS segment: 188-byte packets each starting with the + // 0x47 sync byte and containing NUL padding bytes. The NUL in the head is + // the binary signal the guard keys off. + var seg []byte + for p := 0; p < 4; p++ { + seg = append(seg, 0x47, 0x40, 0x00, 0x10) // TS packet header incl. a NUL + for i := 0; i < 184; i++ { + seg = append(seg, 0x00) // payload padding (NUL bytes) + } + } + if err := os.WriteFile(filepath.Join(srcDir, "clip.ts"), seg, 0644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + store, stats := runIndexer(t, srcDir, dbPath) + + // The binary segment must be skipped before any file entity is written. + // This is the assertion that fails without the guard. + if entityExistsByName(t, store, "clip.ts", EntityTypeFile) { + t.Error("expected no 'file:clip.ts' entity for a skipped binary segment") + } + + // A clean skip (return nil) must not be counted as an indexing error — in + // production the pre-guard path instead surfaced a parse-timeout error here. + if stats.Errors != 0 { + t.Errorf("expected 0 indexing errors for a cleanly skipped binary segment, got %d", stats.Errors) + } +} + // TestIndexer_ScalaCalls verifies CALLS edges are attributed to the enclosing // function, resolve within a file, and include calls nested inside lambdas. func TestIndexer_ScalaCalls(t *testing.T) { From e8877d834aa22ded4e962b154f5699f5226d489e Mon Sep 17 00:00:00 2001 From: Bryan Woodruff Date: Wed, 14 Oct 2026 13:49:39 +0100 Subject: [PATCH 2/3] refactor(kg): make binary-skip observable, name the sniff constant, log in CHANGELOG Address PR #18 review nits: add IndexStats.BinarySkipped (incremented in the NUL-byte guard, surfaced in the index summary when >0) so a legitimate but UTF-16/32-encoded source file wrongly skipped isn't invisible; extract const binarySniffBytes = 8000 documenting git parity; document the UTF-16/32 tradeoff at the guard; and record the fix under CHANGELOG [Unreleased] Fixed. Co-Authored-By: Claude --- CHANGELOG.md | 4 ++++ src/kg/index.go | 3 +++ src/kg/internal/knowledge/indexer.go | 4 ++++ src/kg/internal/knowledge/indexer_treesitter.go | 14 ++++++++++++-- .../internal/knowledge/indexer_treesitter_test.go | 11 +++++++++++ 5 files changed, 34 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32e0756..14c25b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,10 @@ everything below it is unreleased. - Observation-age statistics over a graph with no timestamped rows no longer fail the whole `kg health` run; a null registry entry no longer panics the hub at startup. (#4) - Legacy PDF cleanup matches extensions case-insensitively. +- Binary files that share a source extension (e.g. HLS `.ts` video segments masquerading + as TypeScript) are skipped by a NUL-byte sniff before the tree-sitter parse, instead of + burning the full 10s parse deadline on content the parser cannot handle. Skips are + counted in the index summary. (#18) ### Changed diff --git a/src/kg/index.go b/src/kg/index.go index 281a290..bad8d17 100644 --- a/src/kg/index.go +++ b/src/kg/index.go @@ -168,6 +168,9 @@ func indexScopeDB(root, aiDir, scopeName string, missingPKHint *bool) error { fmt.Printf(" Files scanned: %d\n", stats.FilesScanned) fmt.Printf(" Entities created: %d\n", stats.EntitiesCreated) fmt.Printf(" Relations created: %d\n", stats.RelationsCreated) + if stats.BinarySkipped > 0 { + fmt.Printf(" Binary files skipped: %d\n", stats.BinarySkipped) + } fmt.Printf(" Duration: %.3fs\n", codesDur.Seconds()) // Second pass: index agent execution logs from .ai/tasks/ diff --git a/src/kg/internal/knowledge/indexer.go b/src/kg/internal/knowledge/indexer.go index 1483775..58992bc 100644 --- a/src/kg/internal/knowledge/indexer.go +++ b/src/kg/internal/knowledge/indexer.go @@ -49,6 +49,10 @@ type IndexStats struct { EntitiesCreated int RelationsCreated int Errors int + // BinarySkipped counts files skipped as binary by the NUL-byte sniff in + // processWithTreeSitter (e.g. HLS .ts video segments sharing the TypeScript + // extension). These are not errors; see the guard for the UTF-16/32 caveat. + BinarySkipped int } // entityRecord holds entity data before batch insert diff --git a/src/kg/internal/knowledge/indexer_treesitter.go b/src/kg/internal/knowledge/indexer_treesitter.go index 377a935..467419a 100644 --- a/src/kg/internal/knowledge/indexer_treesitter.go +++ b/src/kg/internal/knowledge/indexer_treesitter.go @@ -563,6 +563,11 @@ func init() { langRegistry[".bash"] = bashCfg } +// binarySniffBytes is how many leading bytes the NUL-byte binary guard +// inspects — matching the 8000-byte buffer git fills for its own binary +// detection, so our "valid source has no NUL in the head" claim has git parity. +const binarySniffBytes = 8000 + // processWithTreeSitter parses the given file using tree-sitter and emits // entities/relations into the provided CSV writers. func (idx *Indexer) processWithTreeSitter( @@ -582,8 +587,13 @@ func (idx *Indexer) processWithTreeSitter( // Tree-sitter spends the full parse timeout chewing on binary content that // happens to share a source extension (e.g. HLS .ts video segments vs. // TypeScript). A NUL byte in the head is a reliable binary signal — valid - // source text never contains one — so skip rather than burn the deadline. - if bytes.IndexByte(src[:min(len(src), 8000)], 0) >= 0 { + // UTF-8 source text never contains one — so skip rather than burn the + // deadline. Known tradeoff: a legitimate UTF-16/UTF-32 source file (NUL + // bytes interleaved with ASCII) would also be skipped; git's own binary + // heuristic sniffs for NUL the same way and shares this limitation. The + // skip is counted in stats.BinarySkipped so it isn't silently invisible. + if bytes.IndexByte(src[:min(len(src), binarySniffBytes)], 0) >= 0 { + stats.BinarySkipped++ return nil } diff --git a/src/kg/internal/knowledge/indexer_treesitter_test.go b/src/kg/internal/knowledge/indexer_treesitter_test.go index bde8aeb..b66997d 100644 --- a/src/kg/internal/knowledge/indexer_treesitter_test.go +++ b/src/kg/internal/knowledge/indexer_treesitter_test.go @@ -134,6 +134,11 @@ type ExportedType struct { t.Errorf("expected at least 1 entity, got %d", stats.EntitiesCreated) } + // A normal UTF-8 source file must not trip the binary guard. + if stats.BinarySkipped != 0 { + t.Errorf("expected BinarySkipped == 0 for a normal source file, got %d", stats.BinarySkipped) + } + // Exported function must be extracted. if !entityExistsByName(t, store, "ExportedFunc", "function") { t.Error("expected entity 'ExportedFunc' (function) to be created") @@ -809,6 +814,12 @@ func TestIndexer_BinaryTSSegment(t *testing.T) { if stats.Errors != 0 { t.Errorf("expected 0 indexing errors for a cleanly skipped binary segment, got %d", stats.Errors) } + + // The skip must be observable in stats so this failure mode isn't invisible + // if the NUL sniff ever fires on a real repo (see IndexStats.BinarySkipped). + if stats.BinarySkipped != 1 { + t.Errorf("expected BinarySkipped == 1 for the clip.ts segment, got %d", stats.BinarySkipped) + } } // TestIndexer_ScalaCalls verifies CALLS edges are attributed to the enclosing From bf26a8a30ed945a6f39d103b22d8643c6f3fa8ce Mon Sep 17 00:00:00 2001 From: Bryan Woodruff Date: Wed, 14 Oct 2026 14:24:58 +0100 Subject: [PATCH 3/3] fix(kg): surface BinarySkipped in the index_project MCP summary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses PR #18 Major: the BinarySkipped counter was only shown on the CLI index path, leaving the MCP index_project tool — the second Index() caller — as invisible as before, defeating the counter's purpose. Mirror the CLI's conditional line in the MCP summary (gated on >0). Audit confirms these are the only two user-facing Index() summary sites. Adds MCP tests covering present-when ->0 and absent-when-zero. Co-Authored-By: Claude --- .../knowledge/mcp_binary_summary_test.go | 84 +++++++++++++++++++ src/kg/internal/knowledge/mcp_server.go | 3 + 2 files changed, 87 insertions(+) create mode 100644 src/kg/internal/knowledge/mcp_binary_summary_test.go diff --git a/src/kg/internal/knowledge/mcp_binary_summary_test.go b/src/kg/internal/knowledge/mcp_binary_summary_test.go new file mode 100644 index 0000000..bc08b74 --- /dev/null +++ b/src/kg/internal/knowledge/mcp_binary_summary_test.go @@ -0,0 +1,84 @@ +package knowledge + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// The whole point of IndexStats.BinarySkipped is observability: a legitimate but +// binary-looking source file that gets skipped must not be invisible. The CLI +// path surfaces it (index.go); index_project is the only other caller that +// builds a user-facing summary, so it must surface it too. These tests pin the +// MCP summary line to the counter in both directions. +func TestMCPIndexProjectSurfacesBinarySkipped(t *testing.T) { + root := t.TempDir() + aiDir := filepath.Join(root, ".ai") + if err := os.MkdirAll(aiDir, 0755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + // A normal source file so indexing has real work to do... + if err := os.WriteFile(filepath.Join(root, "main.go"), + []byte("package main\n\nfunc Run() {}\n"), 0644); err != nil { + t.Fatalf("WriteFile source: %v", err) + } + // ...plus a file sharing a source extension whose head holds a NUL byte, + // which the tree-sitter path skips as binary and counts in BinarySkipped. + if err := os.WriteFile(filepath.Join(root, "blob.go"), + []byte("package x\x00\x00binary"), 0644); err != nil { + t.Fatalf("WriteFile binary: %v", err) + } + + dbPath := filepath.Join(aiDir, "knowledge.db") + store, err := OpenStore(dbPath) + if err != nil { + t.Fatalf("OpenStore: %v", err) + } + store.Close() + + _, handlers := buildTools(aiDir, nil, "testproj", root, PersonalConfig{}) + res, err := mcpCall(t, handlers, "index_project", map[string]interface{}{}) + if err != nil { + t.Fatalf("index_project: %v", err) + } + msg, ok := res.(string) + if !ok { + t.Fatalf("index_project returned %T, want string", res) + } + if !strings.Contains(msg, "Binary files skipped: 1") { + t.Errorf("summary missing binary-skip line; got:\n%s", msg) + } +} + +func TestMCPIndexProjectOmitsBinarySkippedWhenZero(t *testing.T) { + root := t.TempDir() + aiDir := filepath.Join(root, ".ai") + if err := os.MkdirAll(aiDir, 0755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + if err := os.WriteFile(filepath.Join(root, "main.go"), + []byte("package main\n\nfunc Run() {}\n"), 0644); err != nil { + t.Fatalf("WriteFile source: %v", err) + } + + dbPath := filepath.Join(aiDir, "knowledge.db") + store, err := OpenStore(dbPath) + if err != nil { + t.Fatalf("OpenStore: %v", err) + } + store.Close() + + _, handlers := buildTools(aiDir, nil, "testproj", root, PersonalConfig{}) + res, err := mcpCall(t, handlers, "index_project", map[string]interface{}{}) + if err != nil { + t.Fatalf("index_project: %v", err) + } + msg, ok := res.(string) + if !ok { + t.Fatalf("index_project returned %T, want string", res) + } + if strings.Contains(msg, "Binary files skipped") { + t.Errorf("summary surfaced a binary-skip line with no binaries; got:\n%s", msg) + } +} diff --git a/src/kg/internal/knowledge/mcp_server.go b/src/kg/internal/knowledge/mcp_server.go index 647cf61..45f1fbc 100644 --- a/src/kg/internal/knowledge/mcp_server.go +++ b/src/kg/internal/knowledge/mcp_server.go @@ -277,6 +277,9 @@ func buildTools(aiDir string, scopeConfig *ScopeConfig, projectID, projectRoot s msg := fmt.Sprintf("Indexed %d files, created %d entities and %d relations in project '%s'", stats.FilesScanned, stats.EntitiesCreated, stats.RelationsCreated, projectID) + if stats.BinarySkipped > 0 { + msg += fmt.Sprintf("\nBinary files skipped: %d", stats.BinarySkipped) + } switch { case replayErr != nil: // Surfaced rather than swallowed: the hand-written knowledge is