diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3018b3a --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.tmp/ diff --git a/README.md b/README.md index d95308e..b711ac2 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ All CRUD methods are available on both `*Graph` and `*Tx`. ### Query builder -The query builder compiles to SQL. Simple matches use JOINs; multi-hop traversals use recursive CTEs. Cycles are handled safely via `UNION` deduplication with a configurable depth cap (max 10 hops). +The query builder compiles to SQL. Each traversal step compiles to a CTE of node IDs; multi-hop steps use recursive CTEs. Cycles are handled safely via `UNION` deduplication with a configurable depth cap (max 10 hops). ```go // Match by label. @@ -119,13 +119,13 @@ g.Match("Person").Where("name", "=", "Alice").Run(ctx) // Filter on JSON properties. g.Match("Person").WhereJSON("age", ">", 30).Run(ctx) -// Single-hop traversal (compiles to JOIN). +// Single-hop traversal. g.Match("Person"). Where("name", "=", "Alice"). Related("KNOWS", 1, 1). Run(ctx) -// Multi-hop traversal (compiles to recursive CTE). +// Multi-hop traversal (recursive CTE). g.Match("Person"). Where("name", "=", "Alice"). Related("KNOWS", 1, 3). @@ -145,11 +145,27 @@ g.Match("Person"). RelatedDir("KNOWS", graph.Incoming, 1, 1). // who knows Bob? Run(ctx) +// Expand each reached node once. Faster on dense graphs with cycles, +// slower on chains and sparse graphs; see Query.BreadthFirst. +g.Match("Person"). + Where("name", "=", "Alice"). + Related("KNOWS", 1, 6). + BreadthFirst(). + Run(ctx) + +// Project columns and property paths; unreturned fields stay zero. +g.Match("Person"). + Related("KNOWS", 1, 3). + Return("name", "age"). + Run(ctx) + // Count and pagination. count, _ := g.Match("Person").Count(ctx) results, _ := g.Match("Person").Limit(10).Offset(20).Run(ctx) ``` +`Where` and `WhereRel` accept the node columns `id`, `name`, `created_at`, `updated_at`, and `properties`. Filter properties with `WhereJSON` and `WhereRelJSON`. + ### Results ```go @@ -180,42 +196,42 @@ count := res.Len() go test -bench=. -benchmem ./... ``` -Results on Apple M3 Max: +Results on Apple M3 Max, in-memory database unless noted: ``` -BenchmarkTraversalChain/hops=1 37976 31510 ns/op 2707 B/op 61 allocs/op -BenchmarkTraversalChain/hops=2 15195 79016 ns/op 4170 B/op 70 allocs/op -BenchmarkTraversalChain/hops=3 12860 93882 ns/op 4917 B/op 86 allocs/op -BenchmarkTraversalChain/hops=5 10000 115139 ns/op 6408 B/op 117 allocs/op -BenchmarkTraversalChain/hops=10 6820 175823 ns/op 10102 B/op 194 allocs/op - -BenchmarkTraversalFanout/depth=2 18614 64336 ns/op 11518 B/op 217 allocs/op (13 nodes) -BenchmarkTraversalFanout/depth=4 926 1307569 ns/op 92186 B/op 1840 allocs/op (121 nodes) -BenchmarkTraversalFanout/depth=6 12 93185375 ns/op 828125 B/op 16426 allocs/op (1093 nodes) - -BenchmarkTraversalDense/hops=1 3205 378662 ns/op 5734 B/op 129 allocs/op (500 nodes, 5 edges/node) -BenchmarkTraversalDense/hops=2 220 5481183 ns/op 25130 B/op 524 allocs/op -BenchmarkTraversalDense/hops=3 45 25483407 ns/op 104028 B/op 2218 allocs/op -BenchmarkTraversalDense/hops=5 5 209771308 ns/op 366190 B/op 7815 allocs/op -BenchmarkTraversalDense/hops=10 2 931694438 ns/op 375852 B/op 8013 allocs/op - -BenchmarkCreateNode 78876 15556 ns/op 1817 B/op 38 allocs/op -BenchmarkCreateEdge 83337 15518 ns/op 818 B/op 19 allocs/op -BenchmarkBulkInsertTx/batch=10 6548 204779 ns/op 14577 B/op 382 allocs/op -BenchmarkBulkInsertTx/batch=100 715 1800590 ns/op 141614 B/op 3801 allocs/op -BenchmarkBulkInsertTx/batch=1000 70 16154103 ns/op 1437031 B/op 39484 allocs/op - -BenchmarkMatchSimple/nodes=100 51266 23259 ns/op 1525 B/op 29 allocs/op -BenchmarkMatchSimple/nodes=1000 7406 164950 ns/op 1523 B/op 29 allocs/op -BenchmarkMatchSimple/nodes=10000 786 1526852 ns/op 1526 B/op 29 allocs/op +BenchmarkTraversalChain/hops=1 7.6µs 2220 B/op 33 allocs/op (100-node chain) +BenchmarkTraversalChain/hops=10 31.4µs 8889 B/op 173 allocs/op + +BenchmarkTraversalFanout/depth=2 33.1µs 10417 B/op 201 allocs/op (13 nodes) +BenchmarkTraversalFanout/depth=4 199.0µs 90999 B/op 1824 allocs/op (121 nodes) +BenchmarkTraversalFanout/depth=6 1.92ms 826277 B/op 16408 allocs/op (1093 nodes) + +BenchmarkTraversalDense/hops=1 16.2µs 5263 B/op 101 allocs/op (500 nodes, 5 edges/node) +BenchmarkTraversalDense/hops=3 279.0µs 102656 B/op 2196 allocs/op +BenchmarkTraversalDense/hops=10 9.72ms 373273 B/op 7985 allocs/op + +BenchmarkTraversalLarge/out/hops=1 13.0µs 3738 B/op 68 allocs/op (10k nodes, 3 edges/node) +BenchmarkTraversalLarge/out/hops=3 98.8µs 30919 B/op 648 allocs/op +BenchmarkTraversalLarge/out/hops=6 2.55ms 757988 B/op 16267 allocs/op +BenchmarkTraversalLarge/both/hops=1 23.6µs 6007 B/op 118 allocs/op +BenchmarkTraversalLarge/both/hops=3 484.2µs 164892 B/op 3498 allocs/op + +BenchmarkMatchSimple/nodes=100 4.1µs 1619 B/op 22 allocs/op +BenchmarkMatchSimple/nodes=10000 4.4µs 1619 B/op 22 allocs/op + +BenchmarkCreateNode 11.6µs 1679 B/op 36 allocs/op +BenchmarkCreateNodeFile 67.5µs 1686 B/op 35 allocs/op (on disk, WAL) +BenchmarkCreateEdge 7.2µs 714 B/op 17 allocs/op +BenchmarkBulkInsertTx/batch=1000 6.47ms 835897 B/op 29476 allocs/op ``` Key takeaways: -- **Chain traversal** scales linearly with hop depth (~31us at 1 hop to ~176us at 10 hops on a 100-node chain). -- **Fanout traversal** scales with the number of nodes reached; a depth-6 tree with fanout 3 (1093 nodes) takes ~93ms. -- **Dense graph traversal** is the most expensive — on a 500-node graph with 5 edges/node, the reachable set explodes quickly. The `UNION` deduplication in the CTE prevents infinite loops but can't prevent visiting the full reachable set. -- **Simple match** scales linearly with table size (SQLite index scan). -- **Bulk insert** throughput is ~16ms for 1000 nodes+edges in a single transaction (~62k inserts/sec). +- **Traversal cost tracks the reached set**, not the graph size. Each hop is a covering-index lookup, so one hop costs about the same on a 100-node chain and a 10k-node graph. +- **Dense graph traversal** is the most expensive. `UNION` deduplicates on (node, depth), so a node reachable at several depths is expanded once per depth. +- **Result decoding** is a large share of big traversals: each node's JSON properties are unmarshaled into a map. `Return` skips it; returning only `name` from a 6-hop traversal with 10-key properties cuts time from 6.6ms to 1.8ms. + +[`bench/compare`](bench/compare) benchmarks against [GraphQLite](https://github.com/colliery-io/graphqlite) on 10k-node random and 100k-node power-law graphs. +- **Name matches** use an index and do not grow with label size. ## License diff --git a/bench/compare/README.md b/bench/compare/README.md new file mode 100644 index 0000000..93eddeb --- /dev/null +++ b/bench/compare/README.md @@ -0,0 +1,35 @@ +# Comparison benchmarks + +Benchmarks go-sqlite-graph against [GraphQLite](https://github.com/colliery-io/graphqlite), a SQLite extension with Cypher support. This is a separate module because it needs CGo (`mattn/go-sqlite3`) to load the extension. + +Each graph is loaded into three engines, and the harness fails if they disagree on any result size: + +| Engine | What runs | +|---|---| +| `ours` | The go-sqlite-graph API on modernc SQLite (pure Go) | +| `ours-sql-on-c` | The SQL go-sqlite-graph generates, run on C SQLite against the same database file | +| `graphqlite` | Variable-length Cypher through `cypher_rows`, with data loaded the way GraphQLite's bulk insert API loads it | + +`ours-sql-on-c` separates the library's schema and query design from the cost of the pure-Go SQLite engine. + +Graphs: + +- `random-10k`: 10k nodes, 3 uniform random outgoing edges per node. +- `powerlaw-100k`: 100k nodes, 500k edges with Zipf-distributed targets. Node 0 is a hub with ~10k in-edges. + +Every query starts from a node matched by label and name, and returns the distinct names of reached nodes. + +## Running + +Get the GraphQLite extension. The PyPI wheel ships a prebuilt library: + +``` +pip download graphqlite --no-deps --only-binary=:all: -d /tmp/gql +unzip -o /tmp/gql/*.whl -d /tmp/gql +``` + +Then run: + +``` +GRAPHQLITE_EXT=/tmp/gql/graphqlite/graphqlite.dylib go test -bench . -benchmem +``` diff --git a/bench/compare/compare_test.go b/bench/compare/compare_test.go new file mode 100644 index 0000000..07cf6fa --- /dev/null +++ b/bench/compare/compare_test.go @@ -0,0 +1,373 @@ +// Package compare benchmarks go-sqlite-graph against GraphQLite and against +// go-sqlite-graph's own SQL run on C SQLite. It is a separate module so its +// CGo dependency stays out of the library. +package compare + +import ( + "context" + "database/sql" + "fmt" + "math/rand" + "os" + "path/filepath" + "sync" + "testing" + "time" + + graph "github.com/justintout/go-sqlite-graph" + "github.com/mattn/go-sqlite3" +) + +// graphSpec describes a generated graph. Every node has label Node and name +// "n"; every edge has type LINK. +type graphSpec struct { + name string + nodes int + edges int + // zipf draws edge targets from a Zipf distribution, giving a few hubs + // with large in-degree. Otherwise each node gets edges/nodes uniform + // random targets. + zipf bool +} + +var ( + random10k = graphSpec{name: "random-10k", nodes: 10_000, edges: 30_000} + powerLaw100 = graphSpec{name: "powerlaw-100k", nodes: 100_000, edges: 500_000, zipf: true} +) + +func (s graphSpec) generate() [][2]int { + rng := rand.New(rand.NewSource(42)) + out := make([][2]int, 0, s.edges) + if s.zipf { + // s=1.01, v=5 gives node 0 roughly 10k in-edges on 100k/500k. + z := rand.NewZipf(rng, 1.01, 5, uint64(s.nodes-1)) + for range s.edges { + out = append(out, [2]int{rng.Intn(s.nodes), int(z.Uint64())}) + } + return out + } + per := s.edges / s.nodes + for i := range s.nodes { + for range per { + out = append(out, [2]int{i, rng.Intn(s.nodes)}) + } + } + return out +} + +type query struct { + name string + start int + in bool + maxHops int +} + +func (s graphSpec) queries() []query { + typical := s.nodes / 2 + qs := []query{ + {"out/hops=1", typical, false, 1}, + {"out/hops=2", typical, false, 2}, + {"out/hops=3", typical, false, 3}, + {"out/hops=1..5", typical, false, 5}, + } + if s.zipf { + qs = append(qs, query{"hub-in/hops=1", 0, true, 1}, query{"hub-in/hops=2", 0, true, 2}) + } + return qs +} + +// engine runs a traversal and returns the number of distinct nodes reached. +type engine interface { + run(q query) (int, error) +} + +type built struct { + engines map[string]engine +} + +var ( + dir string + cacheMu sync.Mutex + cache = map[string]*built{} + extPath = os.Getenv("GRAPHQLITE_EXT") + register sync.Once +) + +func TestMain(m *testing.M) { + var err error + dir, err = os.MkdirTemp("", "graph-compare-") + if err != nil { + panic(err) + } + code := m.Run() + os.RemoveAll(dir) + os.Exit(code) +} + +// load builds spec in every engine once per process and checks that all +// engines agree on each query's result size. +func load(b *testing.B, spec graphSpec) *built { + b.Helper() + cacheMu.Lock() + defer cacheMu.Unlock() + if g, ok := cache[spec.name]; ok { + return g + } + if extPath == "" { + b.Skip("set GRAPHQLITE_EXT to the GraphQLite extension path") + } + register.Do(func() { + sql.Register("sqlite3_graphqlite", &sqlite3.SQLiteDriver{Extensions: []string{extPath}}) + }) + + edges := spec.generate() + ours, err := loadOurs(b, spec, edges) + if err != nil { + b.Fatal(err) + } + cSQL, err := openCSQL(spec) + if err != nil { + b.Fatal(err) + } + gql, err := loadGraphQLite(b, spec, edges) + if err != nil { + b.Fatal(err) + } + g := &built{engines: map[string]engine{"ours": ours, "ours-sql-on-c": cSQL, "graphqlite": gql}} + + for _, q := range spec.queries() { + counts := map[string]int{} + for name, e := range g.engines { + n, err := e.run(q) + if err != nil { + b.Fatalf("%s %s: %v", name, q.name, err) + } + counts[name] = n + } + b.Logf("%s %s: reached %v", spec.name, q.name, counts) + if counts["ours"] != counts["graphqlite"] || counts["ours"] != counts["ours-sql-on-c"] { + b.Fatalf("%s %s: engines disagree: %v", spec.name, q.name, counts) + } + } + cache[spec.name] = g + return g +} + +func benchSpec(b *testing.B, spec graphSpec) { + g := load(b, spec) + for _, q := range spec.queries() { + for _, name := range []string{"ours", "ours-sql-on-c", "graphqlite"} { + e := g.engines[name] + b.Run(q.name+"/"+name, func(b *testing.B) { + for range b.N { + if _, err := e.run(q); err != nil { + b.Fatal(err) + } + } + }) + } + } +} + +func BenchmarkRandom10k(b *testing.B) { benchSpec(b, random10k) } +func BenchmarkPowerLaw100k(b *testing.B) { benchSpec(b, powerLaw100) } + +// ours uses the go-sqlite-graph API. +type ours struct{ g *graph.Graph } + +func loadOurs(b *testing.B, spec graphSpec, edges [][2]int) (*ours, error) { + ctx := context.Background() + g, err := graph.Open("file:"+filepath.Join(dir, spec.name+"-ours.db"), &graph.Options{PoolSize: 2}) + if err != nil { + return nil, err + } + start := time.Now() + tx, err := g.BeginTx(ctx) + if err != nil { + return nil, err + } + defer tx.Rollback() + ids := make([]int64, spec.nodes) + for i := range spec.nodes { + n := &graph.Node{Name: fmt.Sprintf("n%d", i), Labels: []string{"Node"}} + if err := tx.CreateNode(ctx, n); err != nil { + return nil, err + } + ids[i] = n.ID + } + for _, e := range edges { + if err := tx.CreateEdge(ctx, &graph.Edge{SourceID: ids[e[0]], TargetID: ids[e[1]], Type: "LINK"}); err != nil { + return nil, err + } + } + if err := tx.Commit(); err != nil { + return nil, err + } + b.Logf("%s ours: loaded in %v", spec.name, time.Since(start)) + return &ours{g: g}, nil +} + +func (o *ours) run(q query) (int, error) { + dir := graph.Outgoing + if q.in { + dir = graph.Incoming + } + res, err := o.g.Match("Node"). + Where("name", "=", fmt.Sprintf("n%d", q.start)). + RelatedDir("LINK", dir, 1, q.maxHops). + Return("name"). + Run(context.Background()) + if err != nil { + return 0, err + } + return res.Len(), nil +} + +// cSQL runs the SQL go-sqlite-graph generates against the same database file +// through C SQLite, isolating the SQLite engine from the library's design. +type cSQL struct { + db *sql.DB + stmts map[query]*sql.Stmt +} + +func openCSQL(spec graphSpec) (*cSQL, error) { + db, err := sql.Open("sqlite3", "file:"+filepath.Join(dir, spec.name+"-ours.db")+"?mode=ro") + if err != nil { + return nil, err + } + db.SetMaxOpenConns(1) + return &cSQL{db: db, stmts: map[query]*sql.Stmt{}}, nil +} + +// oursSQL mirrors go-sqlite-graph's compiled SQL for +// Match("Node").Where("name", "=", ?).RelatedDir("LINK", dir, 1, maxHops).Return("name"). +func oursSQL(q query) string { + near, far := "source_id", "target_id" + if q.in { + near, far = far, near + } + start := "WITH RECURSIVE s0(node_id) AS (SELECT n.id FROM nodes n JOIN node_labels nl ON nl.node_id = n.id WHERE +nl.label = 'Node' AND n.name = ?)" + var step string + if q.maxHops == 1 { + step = fmt.Sprintf(", s1(node_id) AS (SELECT e.%s FROM s0 p CROSS JOIN edges e ON e.%s = p.node_id AND e.type = 'LINK')", far, near) + } else { + step = fmt.Sprintf(", w1(node_id, depth) AS (SELECT node_id, 0 FROM s0 UNION SELECT e.%s, p.depth + 1 FROM w1 p CROSS JOIN edges e ON e.%s = p.node_id AND e.type = 'LINK' WHERE p.depth < %d), s1(node_id) AS (SELECT node_id FROM w1 WHERE depth >= 1)", far, near, q.maxHops) + } + return start + step + " SELECT n.id, n.name FROM (SELECT DISTINCT node_id FROM s1) s CROSS JOIN nodes n ON n.id = s.node_id" +} + +func (c *cSQL) run(q query) (int, error) { + stmt, ok := c.stmts[q] + if !ok { + var err error + if stmt, err = c.db.Prepare(oursSQL(q)); err != nil { + return 0, err + } + c.stmts[q] = stmt + } + return countRows(stmt.Query(fmt.Sprintf("n%d", q.start))) +} + +// graphQLite runs Cypher through the GraphQLite extension. +type graphQLite struct { + db *sql.DB + stmt *sql.Stmt +} + +func loadGraphQLite(b *testing.B, spec graphSpec, edges [][2]int) (*graphQLite, error) { + db, err := sql.Open("sqlite3_graphqlite", "file:"+filepath.Join(dir, spec.name+"-graphqlite.db")+"?_journal_mode=WAL") + if err != nil { + return nil, err + } + db.SetMaxOpenConns(1) + + // Mirrors GraphQLite's Python insert_nodes_bulk/insert_edges_bulk, its + // fastest documented load path. + start := time.Now() + tx, err := db.Begin() + if err != nil { + return nil, err + } + defer tx.Rollback() + res, err := tx.Exec("INSERT INTO property_keys (key) VALUES ('name')") + if err != nil { + return nil, err + } + nameKey, _ := res.LastInsertId() + insNode, _ := tx.Prepare("INSERT INTO nodes DEFAULT VALUES") + insLabel, _ := tx.Prepare("INSERT INTO node_labels (node_id, label) VALUES (?, 'Node')") + insName, _ := tx.Prepare("INSERT INTO node_props_text (node_id, key_id, value) VALUES (?, ?, ?)") + insEdge, _ := tx.Prepare("INSERT INTO edges (source_id, target_id, type) VALUES (?, ?, 'LINK')") + ids := make([]int64, spec.nodes) + for i := range spec.nodes { + r, err := insNode.Exec() + if err != nil { + return nil, err + } + ids[i], _ = r.LastInsertId() + if _, err := insLabel.Exec(ids[i]); err != nil { + return nil, err + } + if _, err := insName.Exec(ids[i], nameKey, fmt.Sprintf("n%d", i)); err != nil { + return nil, err + } + } + for _, e := range edges { + if _, err := insEdge.Exec(ids[e[0]], ids[e[1]]); err != nil { + return nil, err + } + } + if err := tx.Commit(); err != nil { + return nil, err + } + b.Logf("%s graphqlite: loaded in %v", spec.name, time.Since(start)) + + stmt, err := db.Prepare("SELECT c0 FROM cypher_rows(?, ?)") + if err != nil { + return nil, err + } + return &graphQLite{db: db, stmt: stmt}, nil +} + +func cypherFor(q query) string { + rel := "[:LINK]" + if q.maxHops > 1 { + rel = fmt.Sprintf("[:LINK*1..%d]", q.maxHops) + } + if q.in { + return "MATCH (a:Node {name: $name})<-" + rel + "-(b) RETURN DISTINCT b.name" + } + return "MATCH (a:Node {name: $name})-" + rel + "->(b) RETURN DISTINCT b.name" +} + +func (g *graphQLite) run(q query) (int, error) { + params := fmt.Sprintf(`{"name": "n%d"}`, q.start) + n, err := countRows(g.stmt.Query(cypherFor(q), params)) + if err != nil { + return 0, fmt.Errorf("cypher %q: %w", cypherFor(q), err) + } + return n, nil +} + +func countRows(rows *sql.Rows, err error) (int, error) { + if err != nil { + return 0, err + } + defer rows.Close() + cols, err := rows.Columns() + if err != nil { + return 0, err + } + dest := make([]any, len(cols)) + for i := range dest { + dest[i] = new(any) + } + n := 0 + for rows.Next() { + if err := rows.Scan(dest...); err != nil { + return 0, err + } + n++ + } + return n, rows.Err() +} diff --git a/bench/compare/go.mod b/bench/compare/go.mod new file mode 100644 index 0000000..9283360 --- /dev/null +++ b/bench/compare/go.mod @@ -0,0 +1,25 @@ +module github.com/justintout/go-sqlite-graph/bench/compare + +go 1.23.0 + +replace github.com/justintout/go-sqlite-graph => ../.. + +require ( + github.com/justintout/go-sqlite-graph v0.0.0-20260714170249-4e85730c39e9 + github.com/mattn/go-sqlite3 v1.14.52 +) + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 // indirect + golang.org/x/sys v0.33.0 // indirect + modernc.org/libc v1.65.7 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect + modernc.org/sqlite v1.37.1 // indirect + zombiezen.com/go/sqlite v1.4.2 // indirect +) diff --git a/bench/compare/go.sum b/bench/compare/go.sum new file mode 100644 index 0000000..14ac7c6 --- /dev/null +++ b/bench/compare/go.sum @@ -0,0 +1,53 @@ +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U= +github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 h1:R84qjqJb5nVJMxqWYb3np9L5ZsaDtB+a39EqjV0JSUM= +golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0/go.mod h1:S9Xr4PYopiDyqSyp5NjCrhFrqg6A5zA2E/iPHPhqnS8= +golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= +golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= +golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc= +golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI= +modernc.org/cc/v4 v4.26.1 h1:+X5NtzVBn0KgsBCBe+xkDC7twLb/jNVj9FPgiwSQO3s= +modernc.org/cc/v4 v4.26.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= +modernc.org/ccgo/v4 v4.28.0 h1:rjznn6WWehKq7dG4JtLRKxb52Ecv8OUGah8+Z/SfpNU= +modernc.org/ccgo/v4 v4.28.0/go.mod h1:JygV3+9AV6SmPhDasu4JgquwU81XAKLd3OKTUDNOiKE= +modernc.org/fileutil v1.3.1 h1:8vq5fe7jdtEvoCf3Zf9Nm0Q05sH6kGx0Op2CPx1wTC8= +modernc.org/fileutil v1.3.1/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/libc v1.65.7 h1:Ia9Z4yzZtWNtUIuiPuQ7Qf7kxYrxP1/jeHZzG8bFu00= +modernc.org/libc v1.65.7/go.mod h1:011EQibzzio/VX3ygj1qGFt5kMjP0lHb0qCW5/D/pQU= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= +modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.37.1 h1:EgHJK/FPoqC+q2YBXg7fUmES37pCHFc97sI7zSayBEs= +modernc.org/sqlite v1.37.1/go.mod h1:XwdRtsE1MpiBcL54+MbKcaDvcuej+IYSMfLN6gSKV8g= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +zombiezen.com/go/sqlite v1.4.2 h1:KZXLrBuJ7tKNEm+VJcApLMeQbhmAUOKA5VWS93DfFRo= +zombiezen.com/go/sqlite v1.4.2/go.mod h1:5Kd4taTAD4MkBzT25mQ9uaAlLjyR0rFhsR6iINO70jc= diff --git a/bench_test.go b/bench_test.go index fc199ef..494ac97 100644 --- a/bench_test.go +++ b/bench_test.go @@ -339,3 +339,191 @@ func BenchmarkMatchSimple(b *testing.B) { }) } } + +// buildRandomGraphTx creates n "Node" nodes, each with `degree` random outgoing +// LINK edges, inside a single transaction. Node i gets props(i) as properties. +// Returns node IDs. +func buildRandomGraphTx(b *testing.B, g *Graph, n, degree int, props func(i int) map[string]any) []int64 { + b.Helper() + ctx := context.Background() + rng := rand.New(rand.NewSource(42)) + + tx, err := g.BeginTx(ctx) + if err != nil { + b.Fatal(err) + } + defer tx.Rollback() + + ids := make([]int64, n) + for i := range n { + node := &Node{Name: fmt.Sprintf("n%d", i), Labels: []string{"Node"}, Properties: props(i)} + if err := tx.CreateNode(ctx, node); err != nil { + b.Fatal(err) + } + ids[i] = node.ID + } + for i := range n { + for range degree { + if err := tx.CreateEdge(ctx, &Edge{SourceID: ids[i], TargetID: ids[rng.Intn(n)], Type: "LINK"}); err != nil { + b.Fatal(err) + } + } + } + if err := tx.Commit(); err != nil { + b.Fatal(err) + } + return ids +} + +// BenchmarkTraversalLarge measures traversals on a 10k-node random graph with +// 3 outgoing edges per node, where per-step cost should not grow with edge count. +func BenchmarkTraversalLarge(b *testing.B) { + g := openBenchGraph(b) + buildRandomGraphTx(b, g, 10000, 3, func(i int) map[string]any { return map[string]any{"idx": i} }) + ctx := context.Background() + + cases := []struct { + name string + dir Direction + min, max int + }{ + {"out/hops=1", Outgoing, 1, 1}, + {"out/hops=2", Outgoing, 1, 2}, + {"out/hops=3", Outgoing, 1, 3}, + {"out/hops=4", Outgoing, 1, 4}, + {"out/hops=6", Outgoing, 1, 6}, + {"out/hops=3-6", Outgoing, 3, 6}, + {"both/hops=1", Both, 1, 1}, + {"both/hops=3", Both, 1, 3}, + {"both/hops=4", Both, 1, 4}, + } + for _, c := range cases { + b.Run(c.name, func(b *testing.B) { + for range b.N { + res, err := g.Match("Node"). + Where("name", "=", "n0"). + RelatedDir("LINK", c.dir, c.min, c.max). + Run(ctx) + if err != nil { + b.Fatal(err) + } + if res.Len() == 0 { + b.Fatal("expected results") + } + } + }) + } +} + +// BenchmarkCreateNodeFile measures autocommit node creation against an on-disk database. +func BenchmarkCreateNodeFile(b *testing.B) { + g, err := Open("file:"+b.TempDir()+"/bench.db", &Options{PoolSize: 2}) + if err != nil { + b.Fatal(err) + } + b.Cleanup(func() { g.Close() }) + ctx := context.Background() + + b.ResetTimer() + for i := range b.N { + n := &Node{Name: fmt.Sprintf("n%d", i), Labels: []string{"Bench"}, Properties: map[string]any{"i": i}} + if err := g.CreateNode(ctx, n); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkReturn measures projecting columns and property paths with Return +// against returning whole nodes, on a 6-hop traversal reaching ~700 nodes that +// each carry a 10-key property document. +func BenchmarkReturn(b *testing.B) { + g := openBenchGraph(b) + buildRandomGraphTx(b, g, 10000, 3, func(i int) map[string]any { + return map[string]any{ + "idx": i, "email": fmt.Sprintf("user%d@example.com", i), "active": i%2 == 0, + "score": float64(i) / 7, "city": "Springfield", "country": "US", + "tags": []string{"a", "b", "c"}, "bio": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", + "address": map[string]any{"street": "742 Evergreen Terrace", "zip": "49007"}, "version": 3, + } + }) + ctx := context.Background() + + cases := []struct { + name string + cols []string + }{ + {"all", nil}, + {"id", []string{"id"}}, + {"name", []string{"name"}}, + {"name+2props", []string{"name", "email", "score"}}, + } + for _, c := range cases { + b.Run(c.name, func(b *testing.B) { + for range b.N { + q := g.Match("Node").Where("name", "=", "n0").Related("LINK", 1, 6) + if c.cols != nil { + q = q.Return(c.cols...) + } + res, err := q.Run(ctx) + if err != nil { + b.Fatal(err) + } + if res.Len() == 0 { + b.Fatal("expected results") + } + } + }) + } +} + +// BenchmarkBreadthFirst compares the default walk with BreadthFirst on graph +// shapes where each wins: chains and trees never revisit nodes, while dense +// graphs with cycles revisit them at many depths. +func BenchmarkBreadthFirst(b *testing.B) { + ctx := context.Background() + chain := openBenchGraph(b) + buildChainGraph(b, chain, 100) + tree := openBenchGraph(b) + buildFanoutGraph(b, tree, 6, 3) + dense := openBenchGraph(b) + buildDenseGraph(b, dense, 500, 5) + large := openBenchGraph(b) + buildRandomGraphTx(b, large, 10000, 3, func(i int) map[string]any { return nil }) + + cases := []struct { + name string + query func() *Query + }{ + {"chain/hops=10", func() *Query { return chain.Match("Node").Where("name", "=", "n0").Related("LINK", 1, 10) }}, + {"tree/depth=6", func() *Query { return tree.Match("Root").Related("CHILD", 1, 6) }}, + {"dense/hops=5", func() *Query { return dense.Match("Node").Where("name", "=", "n0").Related("LINK", 1, 5) }}, + {"dense/hops=10", func() *Query { return dense.Match("Node").Where("name", "=", "n0").Related("LINK", 1, 10) }}, + {"large/out/hops=6", func() *Query { return large.Match("Node").Where("name", "=", "n0").Related("LINK", 1, 6) }}, + {"large/both/hops=4", func() *Query { + return large.Match("Node").Where("name", "=", "n0").RelatedDir("LINK", Both, 1, 4) + }}, + } + for _, c := range cases { + for _, bfs := range []bool{false, true} { + name := c.name + "/default" + if bfs { + name = c.name + "/breadth-first" + } + b.Run(name, func(b *testing.B) { + for range b.N { + q := c.query() + if bfs { + q = q.BreadthFirst() + } + res, err := q.Run(ctx) + if err != nil { + b.Fatal(err) + } + if res.Len() == 0 { + b.Fatal("expected results") + } + } + }) + } + } +} diff --git a/edge.go b/edge.go index a5f8796..d263466 100644 --- a/edge.go +++ b/edge.go @@ -66,31 +66,16 @@ func createEdgeInternal(conn *sqlite.Conn, e *Edge) error { return fmt.Errorf("graph: marshal properties: %w", err) } + now := timestamp() err = sqlitex.Execute(conn, - "INSERT INTO edges (source_id, target_id, type, name, properties) VALUES (?, ?, ?, ?, ?);", - &sqlitex.ExecOptions{Args: []any{e.SourceID, e.TargetID, e.Type, e.Name, props}}, + "INSERT INTO edges (source_id, target_id, type, name, properties, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?);", + &sqlitex.ExecOptions{Args: []any{e.SourceID, e.TargetID, e.Type, e.Name, props, now, now}}, ) if err != nil { return fmt.Errorf("graph: insert edge: %w", err) } - e.ID = conn.LastInsertRowID() - - // Read back timestamps - err = sqlitex.Execute(conn, - "SELECT created_at, updated_at FROM edges WHERE id = ?;", - &sqlitex.ExecOptions{ - Args: []any{e.ID}, - ResultFunc: func(stmt *sqlite.Stmt) error { - e.CreatedAt = stmt.ColumnText(0) - e.UpdatedAt = stmt.ColumnText(1) - return nil - }, - }, - ) - if err != nil { - return fmt.Errorf("graph: read timestamps: %w", err) - } + e.CreatedAt, e.UpdatedAt = now, now return nil } @@ -133,9 +118,10 @@ func updateEdgeInternal(conn *sqlite.Conn, e *Edge) error { return fmt.Errorf("graph: marshal properties: %w", err) } + now := timestamp() err = sqlitex.Execute(conn, - "UPDATE edges SET type = ?, name = ?, properties = ?, updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') WHERE id = ?;", - &sqlitex.ExecOptions{Args: []any{e.Type, e.Name, props, e.ID}}, + "UPDATE edges SET type = ?, name = ?, properties = ?, updated_at = ? WHERE id = ?;", + &sqlitex.ExecOptions{Args: []any{e.Type, e.Name, props, now, e.ID}}, ) if err != nil { return fmt.Errorf("graph: update edge: %w", err) @@ -143,21 +129,7 @@ func updateEdgeInternal(conn *sqlite.Conn, e *Edge) error { if conn.Changes() == 0 { return fmt.Errorf("graph: edge %d not found", e.ID) } - - // Read back updated_at - err = sqlitex.Execute(conn, - "SELECT updated_at FROM edges WHERE id = ?;", - &sqlitex.ExecOptions{ - Args: []any{e.ID}, - ResultFunc: func(stmt *sqlite.Stmt) error { - e.UpdatedAt = stmt.ColumnText(0) - return nil - }, - }, - ) - if err != nil { - return fmt.Errorf("graph: read updated_at: %w", err) - } + e.UpdatedAt = now return nil } diff --git a/graph.go b/graph.go index bb2ccdb..326c08a 100644 --- a/graph.go +++ b/graph.go @@ -25,6 +25,10 @@ type Graph struct { uri string } +// openFlags enables WAL so pooled readers do not block on, or get blocked by, +// the single writer, and commits append to the log instead of rewriting pages. +const openFlags = sqlite.OpenReadWrite | sqlite.OpenCreate | sqlite.OpenURI | sqlite.OpenWAL + func prepareConn(conn *sqlite.Conn) error { return sqlitex.ExecuteTransient(conn, "PRAGMA foreign_keys = ON;", nil) } @@ -50,7 +54,7 @@ func Open(uri string, opts *Options) (*Graph, error) { if autoMigrate { pool := sqlitemigration.NewPool(uri, graphSchema(), sqlitemigration.Options{ PoolSize: poolSize, - Flags: sqlite.OpenReadWrite | sqlite.OpenCreate | sqlite.OpenURI, + Flags: openFlags, PrepareConn: func(conn *sqlite.Conn) error { return prepareConn(conn) }, @@ -59,7 +63,7 @@ func Open(uri string, opts *Options) (*Graph, error) { } else { pool, err := sqlitex.NewPool(uri, sqlitex.PoolOptions{ PoolSize: poolSize, - Flags: sqlite.OpenReadWrite | sqlite.OpenCreate | sqlite.OpenURI, + Flags: openFlags, PrepareConn: func(conn *sqlite.Conn) error { return prepareConn(conn) }, diff --git a/integration_test.go b/integration_test.go index 834b2de..7a254fa 100644 --- a/integration_test.go +++ b/integration_test.go @@ -2,6 +2,9 @@ package graph import ( "context" + "fmt" + "maps" + "math/rand" "testing" ) @@ -214,6 +217,122 @@ func TestWhereRelFilter(t *testing.T) { } } +func TestWhereRelIntermediateStep(t *testing.T) { + g := openTestGraph(t) + ctx := context.Background() + buildSocialGraph(t, g) + + // Alice knows Bob and Charlie within 2 hops. Filtering to Bob reaches + // Globex; filtering to Charlie, who works nowhere, reaches nothing. + for _, bfs := range []bool{false, true} { + for filter, want := range map[string]int{"Bob": 1, "Charlie": 0} { + q := g.Match("Person").Where("name", "=", "Alice").Related("KNOWS", 1, 2) + if bfs { + q = q.BreadthFirst() + } + res, err := q.WhereRel("name", "=", filter).Related("WORKS_AT", 1, 1).Run(ctx) + if err != nil { + t.Fatal(err) + } + if res.Len() != want { + t.Errorf("breadthFirst=%v filter=%s: got %v, want %d results", bfs, filter, nodeNames(res.Nodes()), want) + } + } + } +} + +func TestBothDirection(t *testing.T) { + g := openTestGraph(t) + ctx := context.Background() + buildSocialGraph(t, g) + + for _, maxHops := range []int{1, 2} { + res, err := g.Match("Person"). + Where("name", "=", "Charlie"). + RelatedDir("KNOWS", Both, 1, maxHops). + Run(ctx) + if err != nil { + t.Fatal(err) + } + names := nodeNames(res.Nodes()) + want := []string{"Bob", "Diana", "Eve"} + if maxHops == 2 { + // Alice via Bob; Charlie via any neighbor and back. + want = append(want, "Alice", "Charlie") + } + if len(names) != len(want) || !containsAll(names, want...) { + t.Errorf("maxHops=%d: got %v, want %v", maxHops, names, want) + } + } +} + +func TestFilterInjection(t *testing.T) { + g := openTestGraph(t) + ctx := context.Background() + buildSocialGraph(t, g) + + evil := "name = name OR 1=1; DROP TABLE nodes; --" + if _, err := g.Match("Person").Where(evil, "=", "x").Run(ctx); err == nil { + t.Error("Where accepted an arbitrary column name") + } + if _, err := g.Match("Person").Related("KNOWS", 1, 1).WhereRel(evil, "=", "x").Run(ctx); err == nil { + t.Error("WhereRel accepted an arbitrary column name") + } + + // A hostile path is bound as a JSON path: it matches nothing and leaves the data intact. + path := "age' IS NOT NULL OR 1=1 OR '" + for _, q := range []*Query{ + g.Match("Person").WhereJSON(path, "=", 30), + g.Match("Person").Related("KNOWS", 1, 1).WhereRelJSON(path, "=", 30), + g.Match("Person").Return(path), + } { + res, err := q.Run(ctx) + if err == nil && q.returnCols == nil && res.Len() != 0 { + t.Errorf("hostile path matched %v", nodeNames(res.Nodes())) + } + } + if n, err := g.Match("Person").Count(ctx); err != nil || n != 5 { + t.Errorf("count = %d, %v; want 5", n, err) + } +} + +func TestReturn(t *testing.T) { + g := openTestGraph(t) + ctx := context.Background() + alice, _, _, _, _, _, _ := buildSocialGraph(t, g) + + // Traversal from Alice via WORKS_AT reaches Acme, which has industry but no age. + for _, q := range []*Query{ + g.Match("Person").Where("name", "=", "Alice").Return("name", "age", "missing"), + g.Match("Person").Where("name", "=", "Alice").Related("KNOWS", 1, 1).RelatedDir("KNOWS", Incoming, 1, 1).Return("id", "name", "age"), + } { + res, err := q.Run(ctx) + if err != nil { + t.Fatal(err) + } + if res.Len() != 1 { + t.Fatalf("got %d results, want 1", res.Len()) + } + n := res.Nodes()[0] + if n.ID != alice.ID || n.Name != "Alice" || n.CreatedAt != "" { + t.Errorf("got %+v, want ID %d, name Alice, no created_at", n, alice.ID) + } + if len(n.Properties) != 1 || n.Properties["age"] != float64(30) { + t.Errorf("properties = %v, want map[age:30]", n.Properties) + } + } + + res, err := g.Match("Company").Return("properties", "industry").Run(ctx) + if err != nil { + t.Fatal(err) + } + for _, n := range res.Nodes() { + if n.Name != "" || n.Properties["industry"] == nil { + t.Errorf("got %+v, want only properties", n) + } + } +} + func TestCycleHandling(t *testing.T) { g := openTestGraph(t) ctx := context.Background() @@ -271,6 +390,14 @@ func TestLimit(t *testing.T) { if res.Len() != 2 { t.Errorf("got %d results, want 2", res.Len()) } + + res, err = g.Match("Person").Offset(3).Run(ctx) + if err != nil { + t.Fatal(err) + } + if res.Len() != 2 { + t.Errorf("offset only: got %d results, want 2", res.Len()) + } } func TestIncomingDirection(t *testing.T) { @@ -349,3 +476,100 @@ func containsAny(haystack []string, needles ...string) bool { } return false } + +// TestTraversalMatchesReference checks every traversal compilation path +// against walks computed in Go on a random graph with cycles and self-loops. +func TestTraversalMatchesReference(t *testing.T) { + g := openTestGraph(t) + ctx := context.Background() + rng := rand.New(rand.NewSource(7)) + + const n = 60 + ids := make([]int64, n) + index := map[int64]int{} + for i := range n { + node := &Node{Name: fmt.Sprintf("v%d", i), Labels: []string{"V"}} + if err := g.CreateNode(ctx, node); err != nil { + t.Fatal(err) + } + ids[i] = node.ID + index[node.ID] = i + } + out := make([][]int, n) + in := make([][]int, n) + for range 2 * n { + s, d := rng.Intn(n), rng.Intn(n) + if err := g.CreateEdge(ctx, &Edge{SourceID: ids[s], TargetID: ids[d], Type: "E"}); err != nil { + t.Fatal(err) + } + out[s] = append(out[s], d) + in[d] = append(in[d], s) + } + + reference := func(start int, dir Direction, minHops, maxHops int) map[string]bool { + frontier := map[int]bool{start: true} + reached := map[string]bool{} + for depth := 1; depth <= maxHops; depth++ { + next := map[int]bool{} + for v := range frontier { + if dir != Incoming { + for _, u := range out[v] { + next[u] = true + } + } + if dir != Outgoing { + for _, u := range in[v] { + next[u] = true + } + } + } + if depth >= minHops { + for v := range next { + reached[fmt.Sprintf("v%d", v)] = true + } + } + frontier = next + } + return reached + } + + for _, dir := range []Direction{Outgoing, Incoming, Both} { + for _, minHops := range []int{1, 2} { + for maxHops := minHops; maxHops <= 6; maxHops++ { + for _, start := range []int{0, 17, 42} { + for _, bfs := range []bool{false, true} { + if bfs && minHops != 1 { + continue + } + q := g.Match("V"). + Where("name", "=", fmt.Sprintf("v%d", start)). + RelatedDir("E", dir, minHops, maxHops) + if bfs { + q = q.BreadthFirst() + } + res, err := q.Return("name").Run(ctx) + if err != nil { + t.Fatal(err) + } + want := reference(start, dir, minHops, maxHops) + got := map[string]bool{} + for _, node := range res.Nodes() { + got[node.Name] = true + } + if res.Len() != len(got) || !maps.Equal(got, want) { + t.Errorf("dir=%d hops=%d..%d start=v%d breadthFirst=%v: got %d nodes (%d rows), want %d", + dir, minHops, maxHops, start, bfs, len(got), res.Len(), len(want)) + } + } + } + } + } + } +} + +func TestBreadthFirstRequiresMinHopsOne(t *testing.T) { + g := openTestGraph(t) + if _, err := g.Match("V").Related("E", 2, 4).BreadthFirst().Run(context.Background()); err == nil { + t.Error("BreadthFirst accepted minHops of 2") + } +} diff --git a/node.go b/node.go index 6ffcb4a..be1dc8c 100644 --- a/node.go +++ b/node.go @@ -3,6 +3,7 @@ package graph import ( "context" "fmt" + "time" "zombiezen.com/go/sqlite" "zombiezen.com/go/sqlite/sqlitex" @@ -78,23 +79,45 @@ func (g *Graph) RemoveLabels(ctx context.Context, nodeID int64, labels ...string return removeLabelsInternal(conn, nodeID, labels) } +// timestamp returns the current time in the format of the schema's +// strftime('%Y-%m-%dT%H:%M:%fZ', 'now') column defaults. Writes bind it +// instead of using RETURNING: SQLite buffers RETURNING rows in a temp table, +// which spills to a temp file under a large transaction and made a 500k-edge +// load 3x slower. +func timestamp() string { + return time.Now().UTC().Format("2006-01-02T15:04:05.000Z") +} + +// atomic makes a multi-statement write all-or-nothing when conn has no open +// transaction; defer its result with the named error. Inside a transaction it +// does nothing: the caller's transaction provides rollback, and a savepoint +// there makes SQLite copy each touched page to the statement sub-journal, +// which made bulk node inserts 5x slower. +func atomic(conn *sqlite.Conn) func(*error) { + if conn.AutocommitEnabled() { + return sqlitex.Save(conn) + } + return func(*error) {} +} + func createNodeInternal(conn *sqlite.Conn, n *Node) (err error) { - defer sqlitex.Save(conn)(&err) + defer atomic(conn)(&err) props, err := MarshalProperties(n.Properties) if err != nil { return fmt.Errorf("graph: marshal properties: %w", err) } + now := timestamp() err = sqlitex.Execute(conn, - "INSERT INTO nodes (name, properties) VALUES (?, ?);", - &sqlitex.ExecOptions{Args: []any{n.Name, props}}, + "INSERT INTO nodes (name, properties, created_at, updated_at) VALUES (?, ?, ?, ?);", + &sqlitex.ExecOptions{Args: []any{n.Name, props, now, now}}, ) if err != nil { return fmt.Errorf("graph: insert node: %w", err) } - n.ID = conn.LastInsertRowID() + n.CreatedAt, n.UpdatedAt = now, now for _, label := range n.Labels { err = sqlitex.Execute(conn, @@ -106,22 +129,6 @@ func createNodeInternal(conn *sqlite.Conn, n *Node) (err error) { } } - // Read back timestamps - err = sqlitex.Execute(conn, - "SELECT created_at, updated_at FROM nodes WHERE id = ?;", - &sqlitex.ExecOptions{ - Args: []any{n.ID}, - ResultFunc: func(stmt *sqlite.Stmt) error { - n.CreatedAt = stmt.ColumnText(0) - n.UpdatedAt = stmt.ColumnText(1) - return nil - }, - }, - ) - if err != nil { - return fmt.Errorf("graph: read timestamps: %w", err) - } - return nil } @@ -169,16 +176,17 @@ func getNodeInternal(conn *sqlite.Conn, id int64) (*Node, error) { } func updateNodeInternal(conn *sqlite.Conn, n *Node) (err error) { - defer sqlitex.Save(conn)(&err) + defer atomic(conn)(&err) props, err := MarshalProperties(n.Properties) if err != nil { return fmt.Errorf("graph: marshal properties: %w", err) } + now := timestamp() err = sqlitex.Execute(conn, - "UPDATE nodes SET name = ?, properties = ?, updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') WHERE id = ?;", - &sqlitex.ExecOptions{Args: []any{n.Name, props, n.ID}}, + "UPDATE nodes SET name = ?, properties = ?, updated_at = ? WHERE id = ?;", + &sqlitex.ExecOptions{Args: []any{n.Name, props, now, n.ID}}, ) if err != nil { return fmt.Errorf("graph: update node: %w", err) @@ -186,6 +194,7 @@ func updateNodeInternal(conn *sqlite.Conn, n *Node) (err error) { if conn.Changes() == 0 { return fmt.Errorf("graph: node %d not found", n.ID) } + n.UpdatedAt = now // Sync labels: delete all, re-insert err = sqlitex.Execute(conn, @@ -206,21 +215,6 @@ func updateNodeInternal(conn *sqlite.Conn, n *Node) (err error) { } } - // Read back updated_at - err = sqlitex.Execute(conn, - "SELECT updated_at FROM nodes WHERE id = ?;", - &sqlitex.ExecOptions{ - Args: []any{n.ID}, - ResultFunc: func(stmt *sqlite.Stmt) error { - n.UpdatedAt = stmt.ColumnText(0) - return nil - }, - }, - ) - if err != nil { - return fmt.Errorf("graph: read updated_at: %w", err) - } - return nil } @@ -239,7 +233,7 @@ func deleteNodeInternal(conn *sqlite.Conn, id int64) error { } func addLabelsInternal(conn *sqlite.Conn, nodeID int64, labels []string) (err error) { - defer sqlitex.Save(conn)(&err) + defer atomic(conn)(&err) for _, label := range labels { err = sqlitex.Execute(conn, "INSERT OR IGNORE INTO node_labels (node_id, label) VALUES (?, ?);", @@ -253,7 +247,7 @@ func addLabelsInternal(conn *sqlite.Conn, nodeID int64, labels []string) (err er } func removeLabelsInternal(conn *sqlite.Conn, nodeID int64, labels []string) (err error) { - defer sqlitex.Save(conn)(&err) + defer atomic(conn)(&err) for _, label := range labels { err = sqlitex.Execute(conn, "DELETE FROM node_labels WHERE node_id = ? AND label = ?;", diff --git a/query.go b/query.go index cb7cb84..687e09b 100644 --- a/query.go +++ b/query.go @@ -36,6 +36,19 @@ func validateOp(op string) error { return nil } +// filterCols is the whitelist of node columns Where and WhereRel accept. +// Column names are written into the SQL text, so nothing else may pass. +var filterCols = map[string]bool{ + "id": true, "name": true, "created_at": true, "updated_at": true, "properties": true, +} + +func validateWhere(field, op string) error { + if !filterCols[field] { + return fmt.Errorf("graph: invalid column %q; use WhereJSON for properties", field) + } + return validateOp(op) +} + type whereClause struct { field string op string @@ -49,6 +62,8 @@ type relStep struct { minHops int maxHops int wheres []whereClause + // breadthFirst compiles the step with writeBFS. + breadthFirst bool } // Query is a fluent builder for graph traversal queries. @@ -74,7 +89,7 @@ func (g *Graph) Match(label string) *Query { // Where adds a column-level filter on the starting node set. func (q *Query) Where(field, op string, value any) *Query { - if err := validateOp(op); err != nil { + if err := validateWhere(field, op); err != nil { q.err = err return q } @@ -120,9 +135,34 @@ func (q *Query) RelatedDir(edgeType string, dir Direction, minHops, maxHops int) return q } +// BreadthFirst makes the most recent Related() step expand each reached node +// once. By default a multi-hop step deduplicates on (node, depth), so a node +// reachable at several depths is expanded at each of them. Results are the +// same either way. +// +// Breadth-first pays a fixed cost per hop level, so it helps only when the +// walk revisits many nodes, as on densely connected graphs with cycles. On a +// 500-node graph with 5 edges per node, a 10-hop step drops from 9.5ms to +// 2.0ms; on a 100-node chain it rises from 40µs to 243µs. Measure before +// choosing it. It requires minHops == 1: nodes reachable only at an exact +// larger depth are discarded by a visited set. +func (q *Query) BreadthFirst() *Query { + if len(q.rels) == 0 { + q.err = fmt.Errorf("graph: BreadthFirst called without a preceding Related()") + return q + } + r := &q.rels[len(q.rels)-1] + if r.minHops != 1 { + q.err = fmt.Errorf("graph: BreadthFirst requires minHops of 1, got %d", r.minHops) + return q + } + r.breadthFirst = r.maxHops > 1 + return q +} + // WhereRel adds a column filter on nodes reached in the most recent Related() step. func (q *Query) WhereRel(field, op string, value any) *Query { - if err := validateOp(op); err != nil { + if err := validateWhere(field, op); err != nil { q.err = err return q } @@ -151,9 +191,18 @@ func (q *Query) WhereRelJSON(path, op string, value any) *Query { } // Return specifies which columns/properties to project in results. -// Known columns (id, name, created_at, updated_at) map directly. -// Other names are treated as JSON property paths. +// Known columns (name, created_at, updated_at, properties) map to Node fields. +// Other names are treated as JSON property paths and set in Node.Properties +// under the name as given; paths missing from a node are omitted. Node.ID is +// always set. Fields not returned are left zero, and Properties is nil unless +// properties or a path is returned. Without Return, all columns are returned. func (q *Query) Return(cols ...string) *Query { + for _, c := range cols { + if c == "" { + q.err = fmt.Errorf("graph: Return called with an empty column name") + return q + } + } q.returnCols = cols return q } @@ -176,15 +225,11 @@ func (q *Query) Run(ctx context.Context) (*Result, error) { return nil, q.err } - compiled, err := q.compile() - if err != nil { - return nil, err - } + compiled := q.compile(false) - var conn *sqlite.Conn - if q.conn != nil { - conn = q.conn - } else { + conn := q.conn + if conn == nil { + var err error conn, err = q.g.conn(ctx) if err != nil { return nil, err @@ -201,15 +246,11 @@ func (q *Query) Count(ctx context.Context) (int64, error) { return 0, q.err } - compiled, err := q.compileCount() - if err != nil { - return 0, err - } + compiled := q.compile(true) - var conn *sqlite.Conn - if q.conn != nil { - conn = q.conn - } else { + conn := q.conn + if conn == nil { + var err error conn, err = q.g.conn(ctx) if err != nil { return 0, err diff --git a/query_compile.go b/query_compile.go index c5ea6be..0dc6825 100644 --- a/query_compile.go +++ b/query_compile.go @@ -1,7 +1,10 @@ package graph import ( + "encoding/json" "fmt" + "slices" + "strconv" "strings" "zombiezen.com/go/sqlite" @@ -11,347 +14,287 @@ import ( type compiledQuery struct { sql string args []any + // cols lists the selected columns after n.id, as Return names. + cols []string } -// Known node columns that map directly (not JSON). -var knownNodeCols = map[string]bool{ - "id": true, "name": true, "created_at": true, "updated_at": true, "properties": true, +// sqlBuilder accumulates SQL text and its bound arguments. It writes plain +// strings rather than using fmt, which allocated for every formatted value and +// made compiling a large share of a small query's cost. +type sqlBuilder struct { + strings.Builder + args []any } -func (q *Query) compile() (*compiledQuery, error) { - c := &compiledQuery{} - - if len(q.rels) == 0 { - return q.compileSimpleMatch(c) +func (b *sqlBuilder) w(parts ...string) { + for _, p := range parts { + b.WriteString(p) } - return q.compileTraversal(c) } -func (q *Query) compileCount() (*compiledQuery, error) { +var allCols = []string{"name", "created_at", "updated_at", "properties"} + +var knownNodeCols = map[string]bool{"name": true, "created_at": true, "updated_at": true, "properties": true} + +// compile builds the SQL for q. With count set, it selects COUNT(*) and +// ignores Limit and Offset. +// +// Traversals compile to a chain of CTEs, one per step, each yielding a set of +// node IDs. Nodes are read only for the final set. Selecting full rows through +// the joins would force SQLite to DISTINCT over the JSON properties text and to +// scan the nodes table to probe the reached set. The set is joined as a +// DISTINCT subquery rather than probed with IN, because IN builds a Bloom +// filter whose allocation costs an mmap syscall per query under modernc. +// +// Joins from a step's set use CROSS JOIN, which SQLite never reorders. The +// planner has no row estimates for CTEs and otherwise flattens the chain and +// scans a whole edge index, probing the previous step's set per edge. +// +// LIMIT and OFFSET are bound parameters because zombiezen caches prepared +// statements per connection by SQL text; interpolated values would grow that +// cache without bound when paginating. The clause is omitted when unused: +// a bound LIMIT makes SQLite re-prepare the statement on every execution. +func (q *Query) compile(count bool) *compiledQuery { c := &compiledQuery{} + b := &sqlBuilder{args: make([]any, 0, 8)} + b.Grow(512) if len(q.rels) == 0 { - return q.compileSimpleMatchCount(c) - } - return q.compileTraversalCount(c) -} - -// compileSimpleMatch: no Related() steps, just a node label + where filter. -func (q *Query) compileSimpleMatch(c *compiledQuery) (*compiledQuery, error) { - var sb strings.Builder - sb.WriteString("SELECT n.id, n.name, n.created_at, n.updated_at, n.properties") - sb.WriteString(" FROM nodes n") - sb.WriteString(" JOIN node_labels nl ON nl.node_id = n.id") - sb.WriteString(" WHERE nl.label = ?") - c.args = append(c.args, q.matchLabel) - - for _, w := range q.wheres { - clause, arg := buildWhereExpr("n", w) - sb.WriteString(" AND ") - sb.WriteString(clause) - c.args = append(c.args, arg) - } - - if q.limitVal > 0 { - sb.WriteString(fmt.Sprintf(" LIMIT %d", q.limitVal)) - } - if q.offsetVal > 0 { - sb.WriteString(fmt.Sprintf(" OFFSET %d", q.offsetVal)) - } - - c.sql = sb.String() - return c, nil -} + c.writeSelect(b, q, count) + writeStartFrom(b, q) + } else { + b.w("WITH RECURSIVE s0(node_id) AS (SELECT n.id") + writeStartFrom(b, q) + b.w(")") -func (q *Query) compileSimpleMatchCount(c *compiledQuery) (*compiledQuery, error) { - var sb strings.Builder - sb.WriteString("SELECT COUNT(*)") - sb.WriteString(" FROM nodes n") - sb.WriteString(" JOIN node_labels nl ON nl.node_id = n.id") - sb.WriteString(" WHERE nl.label = ?") - c.args = append(c.args, q.matchLabel) + for i, r := range q.rels { + writeStep(b, i+1, r) + } - for _, w := range q.wheres { - clause, arg := buildWhereExpr("n", w) - sb.WriteString(" AND ") - sb.WriteString(clause) - c.args = append(c.args, arg) + b.w(" ") + c.writeSelect(b, q, count) + b.w(" FROM (SELECT DISTINCT node_id FROM s", strconv.Itoa(len(q.rels)), ") s CROSS JOIN nodes n ON n.id = s.node_id") } - c.sql = sb.String() - return c, nil -} - -// compileTraversal handles queries with Related() steps. -func (q *Query) compileTraversal(c *compiledQuery) (*compiledQuery, error) { - needsCTE := false - for _, r := range q.rels { - if r.maxHops > 1 { - needsCTE = true - break + if !count && (q.limitVal > 0 || q.offsetVal > 0) { + b.w(" LIMIT ? OFFSET ?") + limit := q.limitVal + if limit <= 0 { + limit = -1 } + b.args = append(b.args, limit, max(q.offsetVal, 0)) } - if needsCTE { - return q.compileWithCTEs(c) - } - return q.compileSingleHopJoins(c) + c.sql = b.String() + c.args = b.args + return c } -func (q *Query) compileTraversalCount(c *compiledQuery) (*compiledQuery, error) { - // Build the same traversal but wrap it in COUNT - inner, err := q.compileTraversal(c) - if err != nil { - return nil, err +// writeSelect appends the SELECT list: COUNT(*), or n.id followed by the +// columns named by Return. Property paths extract only their JSON value, so +// rows skip decoding the whole properties document. +func (c *compiledQuery) writeSelect(b *sqlBuilder, q *Query, count bool) { + if count { + b.w("SELECT COUNT(*)") + return } - // Replace the SELECT columns with COUNT - // We wrap the whole thing as a subquery - inner.sql = fmt.Sprintf("SELECT COUNT(*) FROM (%s)", inner.sql) - return inner, nil -} - -// compileSingleHopJoins: all Related() steps are exactly 1 hop, use simple JOINs. -func (q *Query) compileSingleHopJoins(c *compiledQuery) (*compiledQuery, error) { - var sb strings.Builder - - // Final node alias - lastAlias := fmt.Sprintf("n%d", len(q.rels)) - - sb.WriteString(fmt.Sprintf("SELECT DISTINCT %s.id, %s.name, %s.created_at, %s.updated_at, %s.properties", - lastAlias, lastAlias, lastAlias, lastAlias, lastAlias)) - sb.WriteString(" FROM nodes n0") - sb.WriteString(" JOIN node_labels nl0 ON nl0.node_id = n0.id") - - for i, r := range q.rels { - eAlias := fmt.Sprintf("e%d", i) - nAlias := fmt.Sprintf("n%d", i+1) - prevAlias := fmt.Sprintf("n%d", i) - - switch r.direction { - case Outgoing: - sb.WriteString(fmt.Sprintf(" JOIN edges %s ON %s.source_id = %s.id AND %s.type = ?", - eAlias, eAlias, prevAlias, eAlias)) - sb.WriteString(fmt.Sprintf(" JOIN nodes %s ON %s.id = %s.target_id", - nAlias, nAlias, eAlias)) - case Incoming: - sb.WriteString(fmt.Sprintf(" JOIN edges %s ON %s.target_id = %s.id AND %s.type = ?", - eAlias, eAlias, prevAlias, eAlias)) - sb.WriteString(fmt.Sprintf(" JOIN nodes %s ON %s.id = %s.source_id", - nAlias, nAlias, eAlias)) - case Both: - sb.WriteString(fmt.Sprintf(" JOIN edges %s ON (%s.source_id = %s.id OR %s.target_id = %s.id) AND %s.type = ?", - eAlias, eAlias, prevAlias, eAlias, prevAlias, eAlias)) - sb.WriteString(fmt.Sprintf(" JOIN nodes %s ON %s.id = CASE WHEN %s.source_id = %s.id THEN %s.target_id ELSE %s.source_id END", - nAlias, nAlias, eAlias, prevAlias, eAlias, eAlias)) + c.cols = allCols + if len(q.returnCols) > 0 { + c.cols = nil + all := slices.Contains(q.returnCols, "properties") + for _, col := range q.returnCols { + // Paths are redundant when the whole document is returned. + if col == "id" || (all && !knownNodeCols[col]) || slices.Contains(c.cols, col) { + continue + } + c.cols = append(c.cols, col) } - c.args = append(c.args, r.edgeType) } - sb.WriteString(" WHERE nl0.label = ?") - c.args = append(c.args, q.matchLabel) - - // Append starting node wheres - for _, w := range q.wheres { - clause, arg := buildWhereExpr("n0", w) - sb.WriteString(" AND ") - sb.WriteString(clause) - c.args = append(c.args, arg) - } - - // Append per-step node wheres - for i, r := range q.rels { - nAlias := fmt.Sprintf("n%d", i+1) - for _, w := range r.wheres { - clause, arg := buildWhereExpr(nAlias, w) - sb.WriteString(" AND ") - sb.WriteString(clause) - c.args = append(c.args, arg) + b.w("SELECT n.id") + for _, col := range c.cols { + if knownNodeCols[col] { + b.w(", n.", col) + continue } + b.w(", n.properties -> ?") + b.args = append(b.args, "$."+col) } - - if q.limitVal > 0 { - sb.WriteString(fmt.Sprintf(" LIMIT %d", q.limitVal)) - } - if q.offsetVal > 0 { - sb.WriteString(fmt.Sprintf(" OFFSET %d", q.offsetVal)) - } - - c.sql = sb.String() - return c, nil } -// compileWithCTEs: at least one Related() step has maxHops > 1. -func (q *Query) compileWithCTEs(c *compiledQuery) (*compiledQuery, error) { - var sb strings.Builder - - sb.WriteString("WITH RECURSIVE ") - - for i, r := range q.rels { - if i > 0 { - sb.WriteString(", ") - } - - stepName := fmt.Sprintf("step%d", i) - - if r.maxHops == 1 { - // Non-recursive CTE for single-hop steps - q.writeSingleHopCTE(&sb, c, i, stepName, r) - } else { - // Recursive CTE for multi-hop steps - q.writeRecursiveCTE(&sb, c, i, stepName, r) +// writeStartFrom appends the FROM and WHERE clauses selecting the start nodes n. +func writeStartFrom(b *sqlBuilder, q *Query) { + // Without ANALYZE statistics the planner rates the label and name indexes + // equally and drives from the label, scanning every node carrying it. A + // name equality is almost always more selective, so the unary + removes + // the label index from consideration and the label is checked by primary + // key per matched node. + label := "nl.label" + for _, w := range q.wheres { + if !w.isJSON && w.field == "name" && (w.op == "=" || w.op == "IS") { + label = "+nl.label" + break } } + b.w(" FROM nodes n JOIN node_labels nl ON nl.node_id = n.id WHERE ", label, " = ?") + b.args = append(b.args, q.matchLabel) + writeWheres(b, "n", q.wheres) +} - // Final SELECT - lastStep := fmt.Sprintf("step%d", len(q.rels)-1) - lastRel := q.rels[len(q.rels)-1] - - sb.WriteString(" SELECT DISTINCT n.id, n.name, n.created_at, n.updated_at, n.properties") - sb.WriteString(fmt.Sprintf(" FROM %s s", lastStep)) - sb.WriteString(" JOIN nodes n ON n.id = s.node_id") - - if lastRel.maxHops > 1 { - sb.WriteString(fmt.Sprintf(" WHERE s.depth >= %d AND s.depth <= %d", lastRel.minHops, lastRel.maxHops)) +// writeStep appends CTE s(node_id): the nodes reached from s by r. +func writeStep(b *sqlBuilder, i int, r relStep) { + step := strconv.Itoa(i) + prev := "s" + strconv.Itoa(i-1) + reached := "s" + step + if len(r.wheres) > 0 { + reached = "u" + step } - // Apply wheres on the final step's reached nodes - for _, w := range lastRel.wheres { - clause, arg := buildWhereExpr("n", w) - sb.WriteString(" AND ") - sb.WriteString(clause) - c.args = append(c.args, arg) + switch { + case r.maxHops == 1: + b.w(", ", reached, "(node_id) AS (") + writeHops(b, prev, r, hopAll, "") + b.w(")") + case r.breadthFirst: + writeBFS(b, step, prev, reached, r) + default: + // UNION deduplicates on (node_id, depth), which bounds the work per + // depth level by the node count and stops cycles. + walk := "w" + step + b.w(", ", walk, "(node_id, depth) AS (SELECT node_id, 0 FROM ", prev, " UNION ") + writeHops(b, walk, r, hopRecursive, "p.depth < "+strconv.Itoa(r.maxHops)) + b.w("), ", reached, "(node_id) AS (SELECT node_id FROM ", walk, " WHERE depth >= ", strconv.Itoa(r.minHops), ")") } - if q.limitVal > 0 { - sb.WriteString(fmt.Sprintf(" LIMIT %d", q.limitVal)) + if len(r.wheres) > 0 { + b.w(", s", step, "(node_id) AS (SELECT n.id FROM (SELECT DISTINCT node_id FROM ", reached, ") x CROSS JOIN nodes n ON n.id = x.node_id") + writeWheres(b, "n", r.wheres) + b.w(")") } - if q.offsetVal > 0 { - sb.WriteString(fmt.Sprintf(" OFFSET %d", q.offsetVal)) - } - - c.sql = sb.String() - return c, nil } -func (q *Query) writeSingleHopCTE(sb *strings.Builder, c *compiledQuery, stepIdx int, stepName string, r relStep) { - sb.WriteString(fmt.Sprintf("%s(node_id) AS (", stepName)) - - if stepIdx == 0 { - // First step: source is the MATCH node - switch r.direction { - case Outgoing: - sb.WriteString("SELECT e.target_id FROM nodes n") - sb.WriteString(" JOIN node_labels nl ON nl.node_id = n.id") - sb.WriteString(" JOIN edges e ON e.source_id = n.id AND e.type = ?") - case Incoming: - sb.WriteString("SELECT e.source_id FROM nodes n") - sb.WriteString(" JOIN node_labels nl ON nl.node_id = n.id") - sb.WriteString(" JOIN edges e ON e.target_id = n.id AND e.type = ?") - case Both: - sb.WriteString("SELECT CASE WHEN e.source_id = n.id THEN e.target_id ELSE e.source_id END FROM nodes n") - sb.WriteString(" JOIN node_labels nl ON nl.node_id = n.id") - sb.WriteString(" JOIN edges e ON (e.source_id = n.id OR e.target_id = n.id) AND e.type = ?") - } - sb.WriteString(" WHERE nl.label = ?") - c.args = append(c.args, r.edgeType, q.matchLabel) - - for _, w := range q.wheres { - clause, arg := buildWhereExpr("n", w) - sb.WriteString(" AND ") - sb.WriteString(clause) - c.args = append(c.args, arg) - } - } else { - // Subsequent step: source is the previous step's result - prevStep := fmt.Sprintf("step%d", stepIdx-1) - prevRel := q.rels[stepIdx-1] - - switch r.direction { - case Outgoing: - sb.WriteString(fmt.Sprintf("SELECT e.target_id FROM %s p", prevStep)) - sb.WriteString(" JOIN edges e ON e.source_id = p.node_id AND e.type = ?") - case Incoming: - sb.WriteString(fmt.Sprintf("SELECT e.source_id FROM %s p", prevStep)) - sb.WriteString(" JOIN edges e ON e.target_id = p.node_id AND e.type = ?") - case Both: - sb.WriteString(fmt.Sprintf("SELECT CASE WHEN e.source_id = p.node_id THEN e.target_id ELSE e.source_id END FROM %s p", prevStep)) - sb.WriteString(" JOIN edges e ON (e.source_id = p.node_id OR e.target_id = p.node_id) AND e.type = ?") - } - c.args = append(c.args, r.edgeType) - - if prevRel.maxHops > 1 { - sb.WriteString(fmt.Sprintf(" WHERE p.depth >= %d AND p.depth <= %d", prevRel.minHops, prevRel.maxHops)) +// writeBFS appends CTEs reaching nodes within 1..maxHops of prev, expanding +// each node once. The recursive CTE deduplicates on (node, depth), so on +// graphs with cycles it re-expands a node at every depth it is reachable. +// Here frontier f holds nodes first reached at depth k: the expansion of +// f minus all earlier frontiers. A start node is not excluded, so it is +// reached again through a cycle, matching the recursive form. +// +// Each level costs a fixed ~20µs to build its temporary tables, so this only +// wins when the walk revisits many nodes; see Query.BreadthFirst. +func writeBFS(b *sqlBuilder, step, prev, reached string, r relStep) { + front := func(k int) string { return "f" + step + "_" + strconv.Itoa(k) } + for k := 1; k <= r.maxHops; k++ { + from := prev + exclude := "" + if k > 1 { + from = front(k - 1) + var ex strings.Builder + for j := 1; j < k; j++ { + if j > 1 { + ex.WriteString(" UNION ALL ") + } + ex.WriteString("SELECT node_id FROM " + front(j)) + } + exclude = ex.String() } + // MATERIALIZED keeps each frontier from being re-evaluated by every + // later level that excludes it. + b.w(", ", front(k), "(node_id) AS MATERIALIZED (") + writeHops(b, from, r, hopNew, exclude) + b.w(")") } - - sb.WriteString(")") -} - -func (q *Query) writeRecursiveCTE(sb *strings.Builder, c *compiledQuery, stepIdx int, stepName string, r relStep) { - sb.WriteString(fmt.Sprintf("%s(node_id, depth) AS (", stepName)) - - // Base case - if stepIdx == 0 { - // First step: base is the MATCH nodes at depth 0 - sb.WriteString("SELECT n.id, 0 FROM nodes n") - sb.WriteString(" JOIN node_labels nl ON nl.node_id = n.id") - sb.WriteString(" WHERE nl.label = ?") - c.args = append(c.args, q.matchLabel) - - for _, w := range q.wheres { - clause, arg := buildWhereExpr("n", w) - sb.WriteString(" AND ") - sb.WriteString(clause) - c.args = append(c.args, arg) - } - } else { - // Subsequent step: base is previous step's results - prevStep := fmt.Sprintf("step%d", stepIdx-1) - prevRel := q.rels[stepIdx-1] - - sb.WriteString(fmt.Sprintf("SELECT p.node_id, 0 FROM %s p", prevStep)) - if prevRel.maxHops > 1 { - sb.WriteString(fmt.Sprintf(" WHERE p.depth >= %d AND p.depth <= %d", prevRel.minHops, prevRel.maxHops)) + b.w(", ", reached, "(node_id) AS (") + for k := 1; k <= r.maxHops; k++ { + if k > 1 { + b.w(" UNION ALL ") } + b.w("SELECT node_id FROM ", front(k)) } + b.w(")") +} - sb.WriteString(" UNION ") +// hopMode selects how writeHops shapes its SELECT. +type hopMode int + +const ( + // hopAll selects every reached node, duplicates included. + hopAll hopMode = iota + // hopRecursive is the recursive member of a walk CTE: it carries + // depth + 1 and filters on the depth condition. + hopRecursive + // hopNew selects distinct reached nodes not in the exclusion subquery. + hopNew +) - // Recursive case +// writeHops appends a SELECT of the nodes one hop from table from along r. +// cond is the depth condition for hopRecursive, or the exclusion subquery for +// hopNew (empty to exclude nothing). +// Both directions compile to two SELECTs, one per endpoint column, so each can +// use its covering index; an OR in the join condition defeats index use. +func writeHops(b *sqlBuilder, from string, r relStep, mode hopMode, cond string) { + type side struct{ near, far string } + var sides []side switch r.direction { case Outgoing: - sb.WriteString(fmt.Sprintf("SELECT e.target_id, t.depth + 1 FROM %s t", stepName)) - sb.WriteString(" JOIN edges e ON e.source_id = t.node_id") + sides = []side{{"source_id", "target_id"}} case Incoming: - sb.WriteString(fmt.Sprintf("SELECT e.source_id, t.depth + 1 FROM %s t", stepName)) - sb.WriteString(" JOIN edges e ON e.target_id = t.node_id") + sides = []side{{"target_id", "source_id"}} case Both: - sb.WriteString(fmt.Sprintf("SELECT CASE WHEN e.source_id = t.node_id THEN e.target_id ELSE e.source_id END, t.depth + 1 FROM %s t", stepName)) - sb.WriteString(" JOIN edges e ON (e.source_id = t.node_id OR e.target_id = t.node_id)") + sides = []side{{"source_id", "target_id"}, {"target_id", "source_id"}} } - sb.WriteString(fmt.Sprintf(" WHERE e.type = ? AND t.depth < %d", r.maxHops)) - c.args = append(c.args, r.edgeType) - sb.WriteString(")") + for j, s := range sides { + if j > 0 { + if mode == hopAll { + b.w(" UNION ALL ") + } else { + b.w(" UNION ") + } + } + b.w("SELECT ") + if mode == hopNew && len(sides) == 1 { + b.w("DISTINCT ") + } + b.w("e.", s.far) + if mode == hopRecursive { + b.w(", p.depth + 1") + } + b.w(" FROM ", from, " p CROSS JOIN edges e ON e.", s.near, " = p.node_id AND e.type = ?") + switch { + case mode == hopRecursive: + b.w(" WHERE ", cond) + case mode == hopNew && cond != "": + b.w(" WHERE e.", s.far, " NOT IN (", cond, ")") + } + b.args = append(b.args, r.edgeType) + } } -// buildWhereExpr builds a single WHERE expression clause and returns it with the bound arg. -func buildWhereExpr(tableAlias string, w whereClause) (string, any) { - if w.isJSON { - expr := jsonExtractExpr(tableAlias, w.field, w.value) - return fmt.Sprintf("%s %s ?", expr, w.op), w.value +func writeWheres(b *sqlBuilder, tableAlias string, wheres []whereClause) { + for _, w := range wheres { + b.w(" AND ") + if w.isJSON { + // The path is bound, never interpolated, so any key is safe. + b.w(jsonExtractExpr(tableAlias, w.value), " ", w.op, " ?") + b.args = append(b.args, "$."+w.field, w.value) + continue + } + // w.field is checked against filterCols and w.op against validOps. + b.w(tableAlias, ".", w.field, " ", w.op, " ?") + b.args = append(b.args, w.value) } - return fmt.Sprintf("%s.%s %s ?", tableAlias, w.field, w.op), w.value } -// jsonExtractExpr returns the SQLite expression to extract and optionally cast a JSON property. -func jsonExtractExpr(tableAlias, path string, value any) string { - extract := fmt.Sprintf("%s.properties->>'$.%s'", tableAlias, path) +// jsonExtractExpr returns the SQLite expression that extracts the property at +// a bound path parameter, cast to match the Go type of value. +func jsonExtractExpr(tableAlias string, value any) string { + extract := tableAlias + ".properties ->> ?" switch value.(type) { case int, int8, int16, int32, int64: - return fmt.Sprintf("CAST(%s AS INTEGER)", extract) + return "CAST(" + extract + " AS INTEGER)" case float32, float64: - return fmt.Sprintf("CAST(%s AS REAL)", extract) + return "CAST(" + extract + " AS REAL)" default: return extract } @@ -363,16 +306,35 @@ func (c *compiledQuery) execute(conn *sqlite.Conn) (*Result, error) { err := sqlitex.Execute(conn, c.sql, &sqlitex.ExecOptions{ Args: c.args, ResultFunc: func(stmt *sqlite.Stmt) error { - n := &Node{ - ID: stmt.ColumnInt64(0), - Name: stmt.ColumnText(1), - CreatedAt: stmt.ColumnText(2), - UpdatedAt: stmt.ColumnText(3), - } - var err error - n.Properties, err = UnmarshalProperties(stmt.ColumnText(4)) - if err != nil { - return err + n := &Node{ID: stmt.ColumnInt64(0)} + for i, col := range c.cols { + i++ + switch col { + case "name": + n.Name = stmt.ColumnText(i) + case "created_at": + n.CreatedAt = stmt.ColumnText(i) + case "updated_at": + n.UpdatedAt = stmt.ColumnText(i) + case "properties": + props, err := UnmarshalProperties(stmt.ColumnText(i)) + if err != nil { + return err + } + n.Properties = props + default: + if n.Properties == nil { + n.Properties = map[string]any{} + } + if stmt.ColumnType(i) == sqlite.TypeNull { + continue + } + var v any + if err := json.Unmarshal([]byte(stmt.ColumnText(i)), &v); err != nil { + return fmt.Errorf("graph: decode property %q: %w", col, err) + } + n.Properties[col] = v + } } res.rows = append(res.rows, ResultRow{Node: n}) return nil diff --git a/schema.go b/schema.go index c4985a8..19617eb 100644 --- a/schema.go +++ b/schema.go @@ -37,8 +37,38 @@ CREATE INDEX idx_edges_source_type ON edges(source_id, type); CREATE INDEX idx_edges_target_type ON edges(target_id, type); ` +// migration2 reworks indexes for traversal and write cost. +// +// A single-column index on edges(type) is a trap: without ANALYZE statistics +// the planner prefers it for "source_id = ? AND type = ?" joins, scanning every +// edge of that type per expanded node. The composite indexes append the far +// endpoint so traversal steps never touch the edges table. Prefix-redundant +// indexes are dropped to cut write cost, and node_labels becomes WITHOUT ROWID +// so its primary key is the table instead of a second b-tree. +const migration2 = ` +DROP INDEX idx_edges_source; +DROP INDEX idx_edges_target; +DROP INDEX idx_edges_type; +DROP INDEX idx_edges_source_type; +DROP INDEX idx_edges_target_type; +CREATE INDEX idx_edges_source_type_target ON edges(source_id, type, target_id); +CREATE INDEX idx_edges_target_type_source ON edges(target_id, type, source_id); + +CREATE TABLE node_labels_new ( + node_id INTEGER NOT NULL REFERENCES nodes(id) ON DELETE CASCADE, + label TEXT NOT NULL, + PRIMARY KEY (node_id, label) +) WITHOUT ROWID; +INSERT INTO node_labels_new (node_id, label) SELECT node_id, label FROM node_labels; +DROP TABLE node_labels; +ALTER TABLE node_labels_new RENAME TO node_labels; +CREATE INDEX idx_node_labels_label ON node_labels(label); + +CREATE INDEX idx_nodes_name ON nodes(name); +` + func graphSchema() sqlitemigration.Schema { return sqlitemigration.Schema{ - Migrations: []string{migration1}, + Migrations: []string{migration1, migration2}, } } diff --git a/tx.go b/tx.go index ce70be7..5146840 100644 --- a/tx.go +++ b/tx.go @@ -9,6 +9,9 @@ import ( ) // Tx represents an explicit graph transaction. +// +// A method that returns an error may have applied part of its writes, such as +// a node without its labels. Roll back the transaction after any error. type Tx struct { g *Graph conn *sqlite.Conn