Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,10 @@ everything below it is unreleased.
- Observation-age statistics over a graph with no timestamped rows no longer fail the
whole `kg health` run; a null registry entry no longer panics the hub at startup. (#4)
- Legacy PDF cleanup matches extensions case-insensitively.
- Binary files that share a source extension (e.g. HLS `.ts` video segments masquerading
as TypeScript) are skipped by a NUL-byte sniff before the tree-sitter parse, instead of
burning the full 10s parse deadline on content the parser cannot handle. Skips are
counted in the index summary. (#18)

### Changed

Expand Down
3 changes: 3 additions & 0 deletions src/kg/index.go
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,9 @@ func indexScopeDB(root, aiDir, scopeName string, missingPKHint *bool) error {
fmt.Printf(" Files scanned: %d\n", stats.FilesScanned)
fmt.Printf(" Entities created: %d\n", stats.EntitiesCreated)
fmt.Printf(" Relations created: %d\n", stats.RelationsCreated)
if stats.BinarySkipped > 0 {
fmt.Printf(" Binary files skipped: %d\n", stats.BinarySkipped)
}
fmt.Printf(" Duration: %.3fs\n", codesDur.Seconds())

// Second pass: index agent execution logs from .ai/tasks/
Expand Down
4 changes: 4 additions & 0 deletions src/kg/internal/knowledge/indexer.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,10 @@ type IndexStats struct {
EntitiesCreated int
RelationsCreated int
Errors int
// BinarySkipped counts files skipped as binary by the NUL-byte sniff in
// processWithTreeSitter (e.g. HLS .ts video segments sharing the TypeScript
// extension). These are not errors; see the guard for the UTF-16/32 caveat.
BinarySkipped int
}

// entityRecord holds entity data before batch insert
Expand Down
19 changes: 19 additions & 0 deletions src/kg/internal/knowledge/indexer_treesitter.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package knowledge

import (
"bytes"
"context"
"fmt"
"os"
Expand Down Expand Up @@ -562,6 +563,11 @@ func init() {
langRegistry[".bash"] = bashCfg
}

// binarySniffBytes is how many leading bytes the NUL-byte binary guard
// inspects — matching the 8000-byte buffer git fills for its own binary
// detection, so our "valid source has no NUL in the head" claim has git parity.
const binarySniffBytes = 8000

// processWithTreeSitter parses the given file using tree-sitter and emits
// entities/relations into the provided CSV writers.
func (idx *Indexer) processWithTreeSitter(
Expand All @@ -578,6 +584,19 @@ func (idx *Indexer) processWithTreeSitter(
return fmt.Errorf("read %s: %w", absPath, err)
}

// Tree-sitter spends the full parse timeout chewing on binary content that
// happens to share a source extension (e.g. HLS .ts video segments vs.
// TypeScript). A NUL byte in the head is a reliable binary signal — valid
// UTF-8 source text never contains one — so skip rather than burn the
// deadline. Known tradeoff: a legitimate UTF-16/UTF-32 source file (NUL
// bytes interleaved with ASCII) would also be skipped; git's own binary
// heuristic sniffs for NUL the same way and shares this limitation. The
// skip is counted in stats.BinarySkipped so it isn't silently invisible.
if bytes.IndexByte(src[:min(len(src), binarySniffBytes)], 0) >= 0 {
stats.BinarySkipped++
return nil
}

parser := sitter.NewParser()
parser.SetLanguage(cfg.Language)

Expand Down
67 changes: 67 additions & 0 deletions src/kg/internal/knowledge/indexer_treesitter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,11 @@ type ExportedType struct {
t.Errorf("expected at least 1 entity, got %d", stats.EntitiesCreated)
}

// A normal UTF-8 source file must not trip the binary guard.
if stats.BinarySkipped != 0 {
t.Errorf("expected BinarySkipped == 0 for a normal source file, got %d", stats.BinarySkipped)
}

// Exported function must be extracted.
if !entityExistsByName(t, store, "ExportedFunc", "function") {
t.Error("expected entity 'ExportedFunc' (function) to be created")
Expand Down Expand Up @@ -755,6 +760,68 @@ export function createShape(): Shape {
}
}

// TestIndexer_BinaryTSSegment verifies that a binary file sharing a source
// extension (an HLS .ts MPEG transport-stream segment, not TypeScript) is
// skipped cleanly: the NUL-byte guard in processWithTreeSitter returns before
// the parser runs, so no file entity is written and the indexer does not burn
// the 10s parse deadline on content tree-sitter cannot parse.
//
// Verified pre/post behavior (guard removed vs. present):
// - Without the guard, the blob is fed to the TypeScript parser, which always
// writes the "file:clip.ts" entity before walking — so the no-entity
// assertion below FAILS. (In production the real symptom is the 10s parse
// timeout + warning that HLS .ts segments trigger; this minimal synthetic
// blob parses fast, so the clean, deterministic signal here is the written
// file entity rather than a stall.)
// - With the guard, processWithTreeSitter returns nil before the entity write,
// so no entity is created, no error is counted, and the test PASSES.
func TestIndexer_BinaryTSSegment(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}

tmpDir := t.TempDir()
dbPath := filepath.Join(tmpDir, "test.db")
srcDir := filepath.Join(tmpDir, "src")
if err := os.MkdirAll(srcDir, 0755); err != nil {
t.Fatalf("MkdirAll: %v", err)
}

// Simulate an MPEG-TS segment: 188-byte packets each starting with the
// 0x47 sync byte and containing NUL padding bytes. The NUL in the head is
// the binary signal the guard keys off.
var seg []byte
for p := 0; p < 4; p++ {
seg = append(seg, 0x47, 0x40, 0x00, 0x10) // TS packet header incl. a NUL
for i := 0; i < 184; i++ {
seg = append(seg, 0x00) // payload padding (NUL bytes)
}
}
if err := os.WriteFile(filepath.Join(srcDir, "clip.ts"), seg, 0644); err != nil {
t.Fatalf("WriteFile: %v", err)
}

store, stats := runIndexer(t, srcDir, dbPath)

// The binary segment must be skipped before any file entity is written.
// This is the assertion that fails without the guard.
if entityExistsByName(t, store, "clip.ts", EntityTypeFile) {
t.Error("expected no 'file:clip.ts' entity for a skipped binary segment")
}

// A clean skip (return nil) must not be counted as an indexing error — in
// production the pre-guard path instead surfaced a parse-timeout error here.
if stats.Errors != 0 {
t.Errorf("expected 0 indexing errors for a cleanly skipped binary segment, got %d", stats.Errors)
}

// The skip must be observable in stats so this failure mode isn't invisible
// if the NUL sniff ever fires on a real repo (see IndexStats.BinarySkipped).
if stats.BinarySkipped != 1 {
t.Errorf("expected BinarySkipped == 1 for the clip.ts segment, got %d", stats.BinarySkipped)
}
}

// TestIndexer_ScalaCalls verifies CALLS edges are attributed to the enclosing
// function, resolve within a file, and include calls nested inside lambdas.
func TestIndexer_ScalaCalls(t *testing.T) {
Expand Down
84 changes: 84 additions & 0 deletions src/kg/internal/knowledge/mcp_binary_summary_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
package knowledge

import (
"os"
"path/filepath"
"strings"
"testing"
)

// The whole point of IndexStats.BinarySkipped is observability: a legitimate but
// binary-looking source file that gets skipped must not be invisible. The CLI
// path surfaces it (index.go); index_project is the only other caller that
// builds a user-facing summary, so it must surface it too. These tests pin the
// MCP summary line to the counter in both directions.
func TestMCPIndexProjectSurfacesBinarySkipped(t *testing.T) {
root := t.TempDir()
aiDir := filepath.Join(root, ".ai")
if err := os.MkdirAll(aiDir, 0755); err != nil {
t.Fatalf("MkdirAll: %v", err)
}
// A normal source file so indexing has real work to do...
if err := os.WriteFile(filepath.Join(root, "main.go"),
[]byte("package main\n\nfunc Run() {}\n"), 0644); err != nil {
t.Fatalf("WriteFile source: %v", err)
}
// ...plus a file sharing a source extension whose head holds a NUL byte,
// which the tree-sitter path skips as binary and counts in BinarySkipped.
if err := os.WriteFile(filepath.Join(root, "blob.go"),
[]byte("package x\x00\x00binary"), 0644); err != nil {
t.Fatalf("WriteFile binary: %v", err)
}

dbPath := filepath.Join(aiDir, "knowledge.db")
store, err := OpenStore(dbPath)
if err != nil {
t.Fatalf("OpenStore: %v", err)
}
store.Close()

_, handlers := buildTools(aiDir, nil, "testproj", root, PersonalConfig{})
res, err := mcpCall(t, handlers, "index_project", map[string]interface{}{})
if err != nil {
t.Fatalf("index_project: %v", err)
}
msg, ok := res.(string)
if !ok {
t.Fatalf("index_project returned %T, want string", res)
}
if !strings.Contains(msg, "Binary files skipped: 1") {
t.Errorf("summary missing binary-skip line; got:\n%s", msg)
}
}

func TestMCPIndexProjectOmitsBinarySkippedWhenZero(t *testing.T) {
root := t.TempDir()
aiDir := filepath.Join(root, ".ai")
if err := os.MkdirAll(aiDir, 0755); err != nil {
t.Fatalf("MkdirAll: %v", err)
}
if err := os.WriteFile(filepath.Join(root, "main.go"),
[]byte("package main\n\nfunc Run() {}\n"), 0644); err != nil {
t.Fatalf("WriteFile source: %v", err)
}

dbPath := filepath.Join(aiDir, "knowledge.db")
store, err := OpenStore(dbPath)
if err != nil {
t.Fatalf("OpenStore: %v", err)
}
store.Close()

_, handlers := buildTools(aiDir, nil, "testproj", root, PersonalConfig{})
res, err := mcpCall(t, handlers, "index_project", map[string]interface{}{})
if err != nil {
t.Fatalf("index_project: %v", err)
}
msg, ok := res.(string)
if !ok {
t.Fatalf("index_project returned %T, want string", res)
}
if strings.Contains(msg, "Binary files skipped") {
t.Errorf("summary surfaced a binary-skip line with no binaries; got:\n%s", msg)
}
}
3 changes: 3 additions & 0 deletions src/kg/internal/knowledge/mcp_server.go
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,9 @@ func buildTools(aiDir string, scopeConfig *ScopeConfig, projectID, projectRoot s

msg := fmt.Sprintf("Indexed %d files, created %d entities and %d relations in project '%s'",
stats.FilesScanned, stats.EntitiesCreated, stats.RelationsCreated, projectID)
if stats.BinarySkipped > 0 {
msg += fmt.Sprintf("\nBinary files skipped: %d", stats.BinarySkipped)
}
switch {
case replayErr != nil:
// Surfaced rather than swallowed: the hand-written knowledge is
Expand Down
Loading