From 5d5a8d28c5e9d046d9e332c7f6c4fa1ecd8368ca Mon Sep 17 00:00:00 2001 From: Justin Tout <3639630+justintout@users.noreply.github.com> Date: Mon, 14 Sep 2026 10:22:24 -0400 Subject: [PATCH 1/9] Revise README for accuracy and add missing API details Document PoolSize, WAL, the viz subpackage, UpdateNode label replacement, partial writes inside a Tx, query result contents, supported operators, and BreadthFirst trade-offs. Remove claims that the hop cap is configurable and that the module has one dependency. Add Return and BreadthFirst benchmark results. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YB9owLYxU6NsNuVk2NVc7C --- README.md | 133 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 85 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index b725345..bffd433 100644 --- a/README.md +++ b/README.md @@ -2,15 +2,15 @@ [![test](https://github.com/justintout/go-sqlite-graph/actions/workflows/test.yml/badge.svg)](https://github.com/justintout/go-sqlite-graph/actions/workflows/test.yml) -Pure Go labeled property graph database backed by SQLite. No CGo required. +A labeled property graph database for Go, stored in SQLite. It uses the pure-Go `modernc.org/sqlite` driver, so it needs no CGo. -Store nodes and edges with typed columns and arbitrary JSON properties, then traverse relationships using a fluent Go query builder that compiles to recursive CTEs for multi-hop path queries. +Nodes and edges have fixed columns plus a JSON properties document. A query builder compiles traversals to SQL, using recursive CTEs for multi-hop paths. ```go g, _ := graph.Open("file:my.db", nil) defer g.Close() -// Find friends of friends within 3 hops +// Find people Alice knows within 3 hops. results, _ := g.Match("Person"). Where("name", "=", "Alice"). Related("KNOWS", 1, 3). @@ -27,56 +27,59 @@ for _, n := range results.Nodes() { go get github.com/justintout/go-sqlite-graph ``` -Requires Go 1.23+. The only dependency is `zombiezen.com/go/sqlite`. +Requires Go 1.23 or later. The `graph` package depends only on `zombiezen.com/go/sqlite`. The [`viz`](viz) subpackage renders nodes and edges as interactive HTML and adds `go-echarts`. ## Schema -The library manages three tables automatically on `Open()`: +`Open` creates and migrates three tables: -| Table | Purpose | +| Table | Columns | |---|---| | `nodes` | `id`, `name`, `created_at`, `updated_at`, `properties` (JSON) | -| `node_labels` | `node_id`, `label` — multiple labels per node | +| `node_labels` | `node_id`, `label`; a node can have many labels | | `edges` | `id`, `source_id`, `target_id`, `type`, `name`, `created_at`, `updated_at`, `properties` (JSON) | -Schema is versioned with `sqlitemigration` and applied automatically by default. Pass `AutoMigrate: false` in options and call `g.Migrate(ctx)` manually for more control. +`sqlitemigration` versions the schema. Deleting a node deletes its labels and edges through foreign key cascades. Timestamps are UTC strings in `2006-01-02T15:04:05.000Z` format. ## Go API ### Opening a graph ```go -// Auto-migrate schema (default). +// Migrate the schema on first use (default). g, err := graph.Open("file:my.db", nil) -// Manual migration. +// Migrate explicitly. f := false -g, err := graph.Open("file:my.db", &graph.Options{AutoMigrate: &f}) -g.Migrate(ctx) +g, err := graph.Open("file:my.db", &graph.Options{AutoMigrate: &f, PoolSize: 4}) +err = g.Migrate(ctx) ``` +`Open` takes a SQLite URI. Every connection opens in WAL mode with foreign keys on. `PoolSize` defaults to 10. + ### Nodes ```go -// Create a node with multiple labels and JSON properties. alice := &graph.Node{ Name: "Alice", Labels: []string{"Person", "Engineer"}, Properties: map[string]any{"age": 30, "city": "NYC"}, } -g.CreateNode(ctx, alice) // alice.ID is set after create +g.CreateNode(ctx, alice) // sets alice.ID, CreatedAt, and UpdatedAt -// Read, update, delete. node, err := g.GetNode(ctx, alice.ID) + alice.Name = "Alice Smith" g.UpdateNode(ctx, alice) -g.DeleteNode(ctx, alice.ID) // cascades to labels and edges -// Manage labels independently. +g.DeleteNode(ctx, alice.ID) + g.AddLabels(ctx, alice.ID, "Manager") g.RemoveLabels(ctx, alice.ID, "Engineer") ``` +`UpdateNode` replaces the name, labels, and properties with the values on the struct. A `Node` with nil `Labels` loses all its labels. Read the node with `GetNode` before changing one field. + ### Edges ```go @@ -89,51 +92,59 @@ edge := &graph.Edge{ g.CreateEdge(ctx, edge) e, err := g.GetEdge(ctx, edge.ID) -g.UpdateEdge(ctx, edge) +g.UpdateEdge(ctx, edge) // replaces type, name, and properties g.DeleteEdge(ctx, edge.ID) ``` +Edges are directed. `UpdateEdge` does not change the endpoints. + +Get, update, and delete return an error when the ID does not exist. + ### Transactions ```go tx, err := g.BeginTx(ctx) -defer tx.Rollback() // no-op if already committed +defer tx.Rollback() // does nothing after Commit tx.CreateNode(ctx, node) tx.CreateEdge(ctx, edge) +tx.Match("Person").Count(ctx) err = tx.Commit() ``` -All CRUD methods are available on both `*Graph` and `*Tx`. +`*Tx` has every CRUD method on `*Graph`, and `Match`. `BeginTx` takes the write lock immediately, so only one transaction writes at a time. Outside a transaction, each write is atomic. Inside one, a method that returns an error can leave part of its writes behind, such as a node without its labels. Roll back after any error. + +Load large batches in one transaction. On an in-memory database, 1,000 nodes and 999 edges take about 19ms as separate writes and 6.5ms in one transaction. The gap is larger on disk, where each write outside a transaction is its own commit. -### Query builder +### Queries -The query builder compiles to SQL. Each traversal step compiles to a CTE of node IDs; multi-hop steps use recursive CTEs. Cycles are handled safely via `UNION` deduplication with a configurable depth cap (max 10 hops). +`Match` selects nodes by label. Each `Related` step replaces the current node set with the nodes it reaches. The result holds the distinct nodes of the last step, in no guaranteed order. ```go // Match by label. g.Match("Person").Run(ctx) -// Filter on typed columns. +// Filter on node columns. g.Match("Person").Where("name", "=", "Alice").Run(ctx) -// Filter on JSON properties. +// Filter on JSON properties. Nested paths use dots. g.Match("Person").WhereJSON("age", ">", 30).Run(ctx) +g.Match("Person").WhereJSON("address.city", "=", "NYC").Run(ctx) -// Single-hop traversal. +// One hop. g.Match("Person"). Where("name", "=", "Alice"). Related("KNOWS", 1, 1). Run(ctx) -// Multi-hop traversal (recursive CTE). +// One to three hops. g.Match("Person"). Where("name", "=", "Alice"). Related("KNOWS", 1, 3). Run(ctx) -// Chained traversals. +// Chained steps. WhereRel filters the nodes the last step reached. g.Match("Person"). Where("name", "=", "Alice"). Related("KNOWS", 1, 2). @@ -141,64 +152,80 @@ g.Match("Person"). WhereRel("name", "=", "Acme"). Run(ctx) -// Direction control. +// Direction: Outgoing (the Related default), Incoming, or Both. g.Match("Person"). Where("name", "=", "Bob"). RelatedDir("KNOWS", graph.Incoming, 1, 1). // who knows Bob? Run(ctx) -// Expand each reached node once. Faster on dense graphs with cycles, -// slower on chains and sparse graphs; see Query.BreadthFirst. +// Expand each reached node once. See "Breadth-first steps" below. g.Match("Person"). Where("name", "=", "Alice"). Related("KNOWS", 1, 6). BreadthFirst(). Run(ctx) -// Project columns and property paths; unreturned fields stay zero. +// Return only some columns and property paths. g.Match("Person"). Related("KNOWS", 1, 3). Return("name", "age"). Run(ctx) -// Count and pagination. +// Count and paginate. count, _ := g.Match("Person").Count(ctx) results, _ := g.Match("Person").Limit(10).Offset(20).Run(ctx) ``` -`Where` and `WhereRel` accept the node columns `id`, `name`, `created_at`, `updated_at`, and `properties`. Filter properties with `WhereJSON` and `WhereRelJSON`. +`Where` and `WhereRel` accept the columns `id`, `name`, `created_at`, `updated_at`, and `properties`. `WhereJSON` and `WhereRelJSON` accept any property path. When the value is a Go integer or float, the property is cast to `INTEGER` or `REAL` before comparison. + +The operators are `=`, `!=`, `<>`, `<`, `<=`, `>`, `>=`, `LIKE`, `NOT LIKE`, `IS`, and `IS NOT`. + +A step takes between 1 and `graph.MaxHops` (10) hops. A start node is in the result only if the step reaches it, for example through a cycle. Cycles do not loop forever: a walk expands each node at most once per depth. + +Query results populate the node columns and properties. They do not populate `Labels`, and they do not return edges. `Return` sets `ID` plus the named fields. It sets a property path as a key in `Properties` and leaves every other field zero. Returning fewer fields skips JSON decoding, which is a large share of the cost of big traversals. + +`Count` ignores `Limit` and `Offset`. + +Builder errors, such as an unknown column or a hop count out of range, are returned by `Run` or `Count`. + +#### Breadth-first steps + +By default, a multi-hop step expands a node once for every depth at which it is reachable. On a dense graph with cycles, most nodes are reachable at many depths. `BreadthFirst` makes the last step expand each node once. The results are the same. + +`BreadthFirst` adds a fixed cost per hop level, so it is slower when the walk rarely revisits nodes. Measure your graph before using it. It requires a minimum of 1 hop. ### Results ```go res, _ := g.Match("Person").WhereJSON("age", ">", 25).Run(ctx) -// Iterate. for res.Next() { row := res.Row() fmt.Println(row.Node.Name, row.Node.Properties) } -// Or get all at once. nodes := res.Nodes() count := res.Len() ``` +`Run` reads every row before it returns. + ## Design -- **Recursive CTEs** for variable-length path queries, with `UNION` to prevent cycles and a hard cap of 10 hops. -- **Labeled property graph**: nodes support multiple labels (stored in a separate table), edges have a single directed type. -- **Typed columns + JSON overflow**: `id`, `name`, and timestamps are real columns; arbitrary properties live in a JSON column queryable via SQLite's `->>'$.path'` operator. -- **Connection pooling** via `sqlitex.Pool` / `sqlitemigration.Pool` for concurrent access. -- **Single package**: everything lives in `package graph` at the module root. +- Multi-hop steps compile to recursive CTEs. `UNION` deduplicates on (node, depth), which stops cycles and bounds the work at each depth. +- Each step yields a set of node IDs. The query reads node rows only for the final set. +- Edge indexes cover `(source_id, type, target_id)` and `(target_id, type, source_id)`, so a hop reads only the index. +- Nodes can have many labels, stored in `node_labels`. Each edge has one type. +- `id`, `name`, and the timestamps are columns. Other properties are a JSON column, queried with SQLite's `->>` operator. +- Connections come from a `sqlitex.Pool` or `sqlitemigration.Pool`. ## Benchmarks ``` -go test -bench=. -benchmem ./... +go test -bench=. -benchmem . ``` -Results on Apple M3 Max, in-memory database unless noted: +Results on an Apple M3 Max, in-memory database unless noted: ``` BenchmarkTraversalChain/hops=1 7.6µs 2220 B/op 33 allocs/op (100-node chain) @@ -218,6 +245,17 @@ BenchmarkTraversalLarge/out/hops=6 2.55ms 757988 B/op 16267 allocs/op BenchmarkTraversalLarge/both/hops=1 23.6µs 6007 B/op 118 allocs/op BenchmarkTraversalLarge/both/hops=3 484.2µs 164892 B/op 3498 allocs/op +BenchmarkReturn/all 6.48ms 2936649 B/op 62979 allocs/op (6 hops over 10k nodes, 10 properties each) +BenchmarkReturn/name 1.77ms 122085 B/op 2059 allocs/op +BenchmarkReturn/name+2props 4.21ms 952593 B/op 16273 allocs/op + +BenchmarkBreadthFirst/chain/hops=10/default 41.1µs 9049 B/op 174 allocs/op +BenchmarkBreadthFirst/chain/hops=10/breadth-first 250.2µs 25037 B/op 224 allocs/op +BenchmarkBreadthFirst/dense/hops=10/default 9.47ms 373405 B/op 7985 allocs/op +BenchmarkBreadthFirst/dense/hops=10/breadth-first 2.07ms 389410 B/op 8035 allocs/op +BenchmarkBreadthFirst/large/both/hops=4/default 2.28ms 258041 B/op 6800 allocs/op +BenchmarkBreadthFirst/large/both/hops=4/breadth-first 1.76ms 262176 B/op 6816 allocs/op + BenchmarkMatchSimple/nodes=100 4.1µs 1619 B/op 22 allocs/op BenchmarkMatchSimple/nodes=10000 4.4µs 1619 B/op 22 allocs/op @@ -227,13 +265,12 @@ BenchmarkCreateEdge 7.2µs 714 B/op 17 allocs/op BenchmarkBulkInsertTx/batch=1000 6.47ms 835897 B/op 29476 allocs/op ``` -Key takeaways: -- **Traversal cost tracks the reached set**, not the graph size. Each hop is a covering-index lookup, so one hop costs about the same on a 100-node chain and a 10k-node graph. -- **Dense graph traversal** is the most expensive. `UNION` deduplicates on (node, depth), so a node reachable at several depths is expanded once per depth. -- **Result decoding** is a large share of big traversals: each node's JSON properties are unmarshaled into a map. `Return` skips it; returning only `name` from a 6-hop traversal with 10-key properties cuts time from 6.6ms to 1.8ms. +- Traversal cost follows the number of nodes reached, not the graph size. One hop costs about the same on a 100-node chain and on a 10k-node graph. +- Dense graphs cost the most, because the default walk expands a node at every depth it is reachable. `BreadthFirst` cuts 10 hops on the dense graph from 9.5ms to 2.1ms, and raises 10 hops on a chain from 41µs to 250µs. +- Decoding each node's JSON properties is a large share of a big traversal. Returning only `name` cuts a 6-hop traversal from 6.5ms to 1.8ms. +- A match by name uses an index, so its cost does not grow with the number of nodes. -[`bench/compare`](bench/compare) benchmarks against [GraphQLite](https://github.com/colliery-io/graphqlite) on 10k-node random and 100k-node power-law graphs. -- **Name matches** use an index and do not grow with label size. +[`bench/compare`](bench/compare) compares this library with [GraphQLite](https://github.com/colliery-io/graphqlite) on a 10k-node random graph and a 100k-node power-law graph. ## License From 8b5fa45254a96b1ab27c278d1731758a7558b312 Mon Sep 17 00:00:00 2001 From: Justin Tout <3639630+justintout@users.noreply.github.com> Date: Mon, 14 Sep 2026 10:33:55 -0400 Subject: [PATCH 2/9] Support IN and NOT IN filters with slice values Both operators were allowed but compiled to "x IN ?", which SQLite rejects. They now take a slice, bound as one JSON array and read with json_each, so the SQL text does not change with the list length and the prepared statement cache stays bounded. A property filter casts by the slice element type. A non-slice value is a builder error. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YB9owLYxU6NsNuVk2NVc7C --- integration_test.go | 34 ++++++++++++++++++ query.go | 87 ++++++++++++++++++++++++++------------------- query_compile.go | 21 ++++++++--- 3 files changed, 101 insertions(+), 41 deletions(-) diff --git a/integration_test.go b/integration_test.go index 7a254fa..2b306f6 100644 --- a/integration_test.go +++ b/integration_test.go @@ -110,6 +110,40 @@ func TestMatchWhereJSON(t *testing.T) { } } +func TestWhereIn(t *testing.T) { + g := openTestGraph(t) + ctx := context.Background() + buildSocialGraph(t, g) + + cases := []struct { + name string + q *Query + want []string + }{ + {"column", g.Match("Person").Where("name", "in", []string{"Alice", "Eve", "Nobody"}), []string{"Alice", "Eve"}}, + {"json ints", g.Match("Person").WhereJSON("age", "IN", []int{25, 40}), []string{"Bob", "Eve"}}, + {"not in", g.Match("Person").WhereJSON("age", "NOT IN", []any{25, 30, 35}), []string{"Diana", "Eve"}}, + {"rel", g.Match("Person").Where("name", "=", "Alice").Related("KNOWS", 1, 3).WhereRel("name", "IN", []string{"Bob", "Diana"}), []string{"Bob", "Diana"}}, + {"empty", g.Match("Person").Where("name", "IN", []string{}), nil}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + res, err := c.q.Run(ctx) + if err != nil { + t.Fatal(err) + } + names := nodeNames(res.Nodes()) + if len(names) != len(c.want) || !containsAll(names, c.want...) { + t.Errorf("got %v, want %v", names, c.want) + } + }) + } + + if _, err := g.Match("Person").Where("name", "IN", "Alice").Run(ctx); err == nil { + t.Error("IN accepted a non-slice value") + } +} + func TestSingleHopRelated(t *testing.T) { g := openTestGraph(t) ctx := context.Background() diff --git a/query.go b/query.go index 687e09b..fa7d1de 100644 --- a/query.go +++ b/query.go @@ -2,7 +2,9 @@ package graph import ( "context" + "encoding/json" "fmt" + "reflect" "strings" "zombiezen.com/go/sqlite" @@ -29,31 +31,47 @@ var validOps = map[string]bool{ "IS": true, "IS NOT": true, } -func validateOp(op string) error { - if !validOps[strings.ToUpper(op)] { - return fmt.Errorf("graph: invalid operator %q", op) - } - return nil -} - // filterCols is the whitelist of node columns Where and WhereRel accept. // Column names are written into the SQL text, so nothing else may pass. var filterCols = map[string]bool{ "id": true, "name": true, "created_at": true, "updated_at": true, "properties": true, } -func validateWhere(field, op string) error { - if !filterCols[field] { - return fmt.Errorf("graph: invalid column %q; use WhereJSON for properties", field) - } - return validateOp(op) -} - type whereClause struct { field string op string value any isJSON bool + // elem is a zero value of the slice element type for IN and NOT IN, whose + // value is the slice encoded as a JSON array. It picks the property cast. + elem any +} + +// newWhere validates a filter on a column, or on a property path if isJSON. +func newWhere(field, op string, value any, isJSON bool) (whereClause, error) { + if !isJSON && !filterCols[field] { + return whereClause{}, fmt.Errorf("graph: invalid column %q; use WhereJSON for properties", field) + } + w := whereClause{field: field, op: strings.ToUpper(op), value: value, isJSON: isJSON} + if !validOps[w.op] { + return whereClause{}, fmt.Errorf("graph: invalid operator %q", op) + } + if w.op != "IN" && w.op != "NOT IN" { + return w, nil + } + // The list is bound as one JSON array so the SQL text, and so the + // connection's prepared statement cache, does not vary with its length. + v := reflect.ValueOf(value) + if v.Kind() != reflect.Slice || v.Type().Elem().Kind() == reflect.Uint8 { + return whereClause{}, fmt.Errorf("graph: %s requires a slice value, got %T", w.op, value) + } + b, err := json.Marshal(value) + if err != nil { + return whereClause{}, fmt.Errorf("graph: encode %s list: %w", w.op, err) + } + w.value = string(b) + w.elem = reflect.Zero(v.Type().Elem()).Interface() + return w, nil } type relStep struct { @@ -88,22 +106,24 @@ func (g *Graph) Match(label string) *Query { } // Where adds a column-level filter on the starting node set. +// IN and NOT IN take a slice value. func (q *Query) Where(field, op string, value any) *Query { - if err := validateWhere(field, op); err != nil { - q.err = err - return q - } - q.wheres = append(q.wheres, whereClause{field: field, op: strings.ToUpper(op), value: value}) - return q + return q.where(field, op, value, false) } // WhereJSON adds a JSON property filter on the starting node set. +// IN and NOT IN take a slice value. func (q *Query) WhereJSON(path, op string, value any) *Query { - if err := validateOp(op); err != nil { + return q.where(path, op, value, true) +} + +func (q *Query) where(field, op string, value any, isJSON bool) *Query { + w, err := newWhere(field, op, value, isJSON) + if err != nil { q.err = err return q } - q.wheres = append(q.wheres, whereClause{field: path, op: strings.ToUpper(op), value: value, isJSON: true}) + q.wheres = append(q.wheres, w) return q } @@ -162,31 +182,26 @@ func (q *Query) BreadthFirst() *Query { // WhereRel adds a column filter on nodes reached in the most recent Related() step. func (q *Query) WhereRel(field, op string, value any) *Query { - if err := validateWhere(field, op); err != nil { - q.err = err - return q - } - if len(q.rels) == 0 { - q.err = fmt.Errorf("graph: WhereRel called without a preceding Related()") - return q - } - idx := len(q.rels) - 1 - q.rels[idx].wheres = append(q.rels[idx].wheres, whereClause{field: field, op: strings.ToUpper(op), value: value}) - return q + return q.whereRel(field, op, value, false) } // WhereRelJSON adds a JSON property filter on nodes reached in the most recent Related() step. func (q *Query) WhereRelJSON(path, op string, value any) *Query { - if err := validateOp(op); err != nil { + return q.whereRel(path, op, value, true) +} + +func (q *Query) whereRel(field, op string, value any, isJSON bool) *Query { + w, err := newWhere(field, op, value, isJSON) + if err != nil { q.err = err return q } if len(q.rels) == 0 { - q.err = fmt.Errorf("graph: WhereRelJSON called without a preceding Related()") + q.err = fmt.Errorf("graph: WhereRel called without a preceding Related()") return q } idx := len(q.rels) - 1 - q.rels[idx].wheres = append(q.rels[idx].wheres, whereClause{field: path, op: strings.ToUpper(op), value: value, isJSON: true}) + q.rels[idx].wheres = append(q.rels[idx].wheres, w) return q } diff --git a/query_compile.go b/query_compile.go index 0dc6825..eca29b1 100644 --- a/query_compile.go +++ b/query_compile.go @@ -274,14 +274,25 @@ func writeHops(b *sqlBuilder, from string, r relStep, mode hopMode, cond string) func writeWheres(b *sqlBuilder, tableAlias string, wheres []whereClause) { for _, w := range wheres { b.w(" AND ") + list := w.op == "IN" || w.op == "NOT IN" if w.isJSON { // The path is bound, never interpolated, so any key is safe. - b.w(jsonExtractExpr(tableAlias, w.value), " ", w.op, " ?") - b.args = append(b.args, "$."+w.field, w.value) - continue + sample := w.value + if list { + sample = w.elem + } + b.w(jsonExtractExpr(tableAlias, sample)) + b.args = append(b.args, "$."+w.field) + } else { + // w.field is checked against filterCols. + b.w(tableAlias, ".", w.field) + } + // w.op is checked against validOps. + if list { + b.w(" ", w.op, " (SELECT value FROM json_each(?))") + } else { + b.w(" ", w.op, " ?") } - // w.field is checked against filterCols and w.op against validOps. - b.w(tableAlias, ".", w.field, " ", w.op, " ?") b.args = append(b.args, w.value) } } From 3560706a11e0a330de64cdb9e0ae475fb5f3f33e Mon Sep 17 00:00:00 2001 From: Justin Tout <3639630+justintout@users.noreply.github.com> Date: Mon, 14 Sep 2026 10:33:55 -0400 Subject: [PATCH 3/9] Move benchmark results to BENCHMARKS.md and document IN Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YB9owLYxU6NsNuVk2NVc7C --- BENCHMARKS.md | 52 ++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 55 +++------------------------------------------------ 2 files changed, 55 insertions(+), 52 deletions(-) create mode 100644 BENCHMARKS.md diff --git a/BENCHMARKS.md b/BENCHMARKS.md new file mode 100644 index 0000000..147f464 --- /dev/null +++ b/BENCHMARKS.md @@ -0,0 +1,52 @@ +# Benchmarks + +``` +go test -bench=. -benchmem . +``` + +Results on an Apple M3 Max, in-memory database unless noted: + +``` +BenchmarkTraversalChain/hops=1 7.6µs 2220 B/op 33 allocs/op (100-node chain) +BenchmarkTraversalChain/hops=10 31.4µs 8889 B/op 173 allocs/op + +BenchmarkTraversalFanout/depth=2 33.1µs 10417 B/op 201 allocs/op (13 nodes) +BenchmarkTraversalFanout/depth=4 199.0µs 90999 B/op 1824 allocs/op (121 nodes) +BenchmarkTraversalFanout/depth=6 1.92ms 826277 B/op 16408 allocs/op (1093 nodes) + +BenchmarkTraversalDense/hops=1 16.2µs 5263 B/op 101 allocs/op (500 nodes, 5 edges/node) +BenchmarkTraversalDense/hops=3 279.0µs 102656 B/op 2196 allocs/op +BenchmarkTraversalDense/hops=10 9.72ms 373273 B/op 7985 allocs/op + +BenchmarkTraversalLarge/out/hops=1 13.0µs 3738 B/op 68 allocs/op (10k nodes, 3 edges/node) +BenchmarkTraversalLarge/out/hops=3 98.8µs 30919 B/op 648 allocs/op +BenchmarkTraversalLarge/out/hops=6 2.55ms 757988 B/op 16267 allocs/op +BenchmarkTraversalLarge/both/hops=1 23.6µs 6007 B/op 118 allocs/op +BenchmarkTraversalLarge/both/hops=3 484.2µs 164892 B/op 3498 allocs/op + +BenchmarkReturn/all 6.48ms 2936649 B/op 62979 allocs/op (6 hops over 10k nodes, 10 properties each) +BenchmarkReturn/name 1.77ms 122085 B/op 2059 allocs/op +BenchmarkReturn/name+2props 4.21ms 952593 B/op 16273 allocs/op + +BenchmarkBreadthFirst/chain/hops=10/default 41.1µs 9049 B/op 174 allocs/op +BenchmarkBreadthFirst/chain/hops=10/breadth-first 250.2µs 25037 B/op 224 allocs/op +BenchmarkBreadthFirst/dense/hops=10/default 9.47ms 373405 B/op 7985 allocs/op +BenchmarkBreadthFirst/dense/hops=10/breadth-first 2.07ms 389410 B/op 8035 allocs/op +BenchmarkBreadthFirst/large/both/hops=4/default 2.28ms 258041 B/op 6800 allocs/op +BenchmarkBreadthFirst/large/both/hops=4/breadth-first 1.76ms 262176 B/op 6816 allocs/op + +BenchmarkMatchSimple/nodes=100 4.1µs 1619 B/op 22 allocs/op +BenchmarkMatchSimple/nodes=10000 4.4µs 1619 B/op 22 allocs/op + +BenchmarkCreateNode 11.6µs 1679 B/op 36 allocs/op +BenchmarkCreateNodeFile 67.5µs 1686 B/op 35 allocs/op (on disk, WAL) +BenchmarkCreateEdge 7.2µs 714 B/op 17 allocs/op +BenchmarkBulkInsertTx/batch=1000 6.47ms 835897 B/op 29476 allocs/op +``` + +- Traversal cost follows the number of nodes reached, not the graph size. One hop costs about the same on a 100-node chain and on a 10k-node graph. +- Dense graphs cost the most, because the default walk expands a node at every depth it is reachable. `BreadthFirst` cuts 10 hops on the dense graph from 9.5ms to 2.1ms, and raises 10 hops on a chain from 41µs to 250µs. +- Decoding each node's JSON properties is a large share of a big traversal. Returning only `name` cuts a 6-hop traversal from 6.5ms to 1.8ms. +- A match by name uses an index, so its cost does not grow with the number of nodes. + +[`bench/compare`](bench/compare) compares this library with [GraphQLite](https://github.com/colliery-io/graphqlite) on a 10k-node random graph and a 100k-node power-law graph. diff --git a/README.md b/README.md index bffd433..9facdb1 100644 --- a/README.md +++ b/README.md @@ -176,9 +176,9 @@ count, _ := g.Match("Person").Count(ctx) results, _ := g.Match("Person").Limit(10).Offset(20).Run(ctx) ``` -`Where` and `WhereRel` accept the columns `id`, `name`, `created_at`, `updated_at`, and `properties`. `WhereJSON` and `WhereRelJSON` accept any property path. When the value is a Go integer or float, the property is cast to `INTEGER` or `REAL` before comparison. +`Where` and `WhereRel` accept the columns `id`, `name`, `created_at`, `updated_at`, and `properties`. `WhereJSON` and `WhereRelJSON` accept any property path. When the value, or the element type of an `IN` slice, is a Go integer or float, the property is cast to `INTEGER` or `REAL` before comparison. -The operators are `=`, `!=`, `<>`, `<`, `<=`, `>`, `>=`, `LIKE`, `NOT LIKE`, `IS`, and `IS NOT`. +The operators are `=`, `!=`, `<>`, `<`, `<=`, `>`, `>=`, `LIKE`, `NOT LIKE`, `IN`, `NOT IN`, `IS`, and `IS NOT`. `IN` and `NOT IN` take a slice, such as `[]string{"Alice", "Bob"}`. A step takes between 1 and `graph.MaxHops` (10) hops. A start node is in the result only if the step reaches it, for example through a cycle. Cycles do not loop forever: a walk expands each node at most once per depth. @@ -221,56 +221,7 @@ count := res.Len() ## Benchmarks -``` -go test -bench=. -benchmem . -``` - -Results on an Apple M3 Max, in-memory database unless noted: - -``` -BenchmarkTraversalChain/hops=1 7.6µs 2220 B/op 33 allocs/op (100-node chain) -BenchmarkTraversalChain/hops=10 31.4µs 8889 B/op 173 allocs/op - -BenchmarkTraversalFanout/depth=2 33.1µs 10417 B/op 201 allocs/op (13 nodes) -BenchmarkTraversalFanout/depth=4 199.0µs 90999 B/op 1824 allocs/op (121 nodes) -BenchmarkTraversalFanout/depth=6 1.92ms 826277 B/op 16408 allocs/op (1093 nodes) - -BenchmarkTraversalDense/hops=1 16.2µs 5263 B/op 101 allocs/op (500 nodes, 5 edges/node) -BenchmarkTraversalDense/hops=3 279.0µs 102656 B/op 2196 allocs/op -BenchmarkTraversalDense/hops=10 9.72ms 373273 B/op 7985 allocs/op - -BenchmarkTraversalLarge/out/hops=1 13.0µs 3738 B/op 68 allocs/op (10k nodes, 3 edges/node) -BenchmarkTraversalLarge/out/hops=3 98.8µs 30919 B/op 648 allocs/op -BenchmarkTraversalLarge/out/hops=6 2.55ms 757988 B/op 16267 allocs/op -BenchmarkTraversalLarge/both/hops=1 23.6µs 6007 B/op 118 allocs/op -BenchmarkTraversalLarge/both/hops=3 484.2µs 164892 B/op 3498 allocs/op - -BenchmarkReturn/all 6.48ms 2936649 B/op 62979 allocs/op (6 hops over 10k nodes, 10 properties each) -BenchmarkReturn/name 1.77ms 122085 B/op 2059 allocs/op -BenchmarkReturn/name+2props 4.21ms 952593 B/op 16273 allocs/op - -BenchmarkBreadthFirst/chain/hops=10/default 41.1µs 9049 B/op 174 allocs/op -BenchmarkBreadthFirst/chain/hops=10/breadth-first 250.2µs 25037 B/op 224 allocs/op -BenchmarkBreadthFirst/dense/hops=10/default 9.47ms 373405 B/op 7985 allocs/op -BenchmarkBreadthFirst/dense/hops=10/breadth-first 2.07ms 389410 B/op 8035 allocs/op -BenchmarkBreadthFirst/large/both/hops=4/default 2.28ms 258041 B/op 6800 allocs/op -BenchmarkBreadthFirst/large/both/hops=4/breadth-first 1.76ms 262176 B/op 6816 allocs/op - -BenchmarkMatchSimple/nodes=100 4.1µs 1619 B/op 22 allocs/op -BenchmarkMatchSimple/nodes=10000 4.4µs 1619 B/op 22 allocs/op - -BenchmarkCreateNode 11.6µs 1679 B/op 36 allocs/op -BenchmarkCreateNodeFile 67.5µs 1686 B/op 35 allocs/op (on disk, WAL) -BenchmarkCreateEdge 7.2µs 714 B/op 17 allocs/op -BenchmarkBulkInsertTx/batch=1000 6.47ms 835897 B/op 29476 allocs/op -``` - -- Traversal cost follows the number of nodes reached, not the graph size. One hop costs about the same on a 100-node chain and on a 10k-node graph. -- Dense graphs cost the most, because the default walk expands a node at every depth it is reachable. `BreadthFirst` cuts 10 hops on the dense graph from 9.5ms to 2.1ms, and raises 10 hops on a chain from 41µs to 250µs. -- Decoding each node's JSON properties is a large share of a big traversal. Returning only `name` cuts a 6-hop traversal from 6.5ms to 1.8ms. -- A match by name uses an index, so its cost does not grow with the number of nodes. - -[`bench/compare`](bench/compare) compares this library with [GraphQLite](https://github.com/colliery-io/graphqlite) on a 10k-node random graph and a 100k-node power-law graph. +Traversal cost follows the number of nodes reached, not the graph size. See [BENCHMARKS.md](BENCHMARKS.md) for results and a comparison with GraphQLite. ## License From 55fc9c0aca54e07f57bc591ed004d8b809c403fd Mon Sep 17 00:00:00 2001 From: Justin Tout <3639630+justintout@users.noreply.github.com> Date: Mon, 14 Sep 2026 11:05:22 -0400 Subject: [PATCH 4/9] Check gofmt and vet the compare module in CI, add package doc comment The test workflow now fails on unformatted files and vets bench/compare, and uses current checkout and setup-go actions. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YB9owLYxU6NsNuVk2NVc7C --- .github/workflows/test.yml | 15 ++++++++++++--- graph.go | 4 ++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d9bc7f0..2ff2b1f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,10 +9,19 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 with: go-version-file: go.mod check-latest: true - - run: go vet ./... + + - name: gofmt + run: | + unformatted=$(gofmt -l .) + if [ -n "$unformatted" ]; then + echo "$unformatted" + exit 1 + fi + + - run: go vet ./... ./bench/compare/... - run: go test -race ./... diff --git a/graph.go b/graph.go index 326c08a..b4fc669 100644 --- a/graph.go +++ b/graph.go @@ -1,3 +1,7 @@ +// Package graph is a labeled property graph database stored in SQLite. +// +// Nodes carry labels, a name, and JSON properties. Edges are directed and +// typed. Match starts a query builder that compiles traversals to SQL. package graph import ( From ed99ed22683c829eb2a94f16ef61200f45a86194 Mon Sep 17 00:00:00 2001 From: Justin Tout <3639630+justintout@users.noreply.github.com> Date: Mon, 14 Sep 2026 11:05:22 -0400 Subject: [PATCH 5/9] Identify viz nodes by ID so repeated names stay distinct Echarts identifies graph nodes by id, then name. Without an id it kept only the first node with a given name and attached the edges of the others to it. Nodes now carry their graph ID, and edges link by it. Also gofmt convert.go. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YB9owLYxU6NsNuVk2NVc7C --- viz/convert.go | 57 ++++++++++++++++++++++++++------------------- viz/convert_test.go | 19 ++------------- viz/example_test.go | 2 +- viz/export_test.go | 1 - viz/viz.go | 4 +++- viz/viz_test.go | 17 ++++++++++++++ 6 files changed, 56 insertions(+), 44 deletions(-) diff --git a/viz/convert.go b/viz/convert.go index 87dcccf..0924f2c 100644 --- a/viz/convert.go +++ b/viz/convert.go @@ -1,6 +1,8 @@ package viz import ( + "strconv" + graph "github.com/justintout/go-sqlite-graph" "github.com/go-echarts/go-echarts/v2/opts" @@ -32,18 +34,33 @@ func buildCategories(nodes []*graph.Node, p *palette) ([]opts.GraphCategory, map return cats, catIndex } -// convertNodes converts graph nodes to go-echarts GraphNode values. -func convertNodes(nodes []*graph.Node, catIndex map[string]int) []opts.GraphNode { - result := make([]opts.GraphNode, len(nodes)) +// graphNode adds the id echarts identifies a node by. Without an id, echarts +// identifies nodes by name: it drops every node after the first with a given +// name and attaches their edges to that first node. +type graphNode struct { + opts.GraphNode + ID string `json:"id"` +} + +func nodeID(id int64) string { + return strconv.FormatInt(id, 10) +} + +// convertNodes converts graph nodes to echarts graph nodes. +func convertNodes(nodes []*graph.Node, catIndex map[string]int) []graphNode { + result := make([]graphNode, len(nodes)) for i, n := range nodes { label := "(unlabeled)" if len(n.Labels) > 0 { label = n.Labels[0] } - result[i] = opts.GraphNode{ - Name: n.Name, - Category: catIndex[label], - SymbolSize: 30, + result[i] = graphNode{ + GraphNode: opts.GraphNode{ + Name: n.Name, + Category: catIndex[label], + SymbolSize: 30, + }, + ID: nodeID(n.ID), } } return result @@ -52,32 +69,24 @@ func convertNodes(nodes []*graph.Node, catIndex map[string]int) []opts.GraphNode // convertEdges converts graph edges to go-echarts GraphLink values. // Edges referencing nodes not present in the node list are silently skipped. func convertEdges(edges []*graph.Edge, nodes []*graph.Node) []opts.GraphLink { - nameIdx := nodeNameIndex(nodes) + present := make(map[int64]bool, len(nodes)) + for _, n := range nodes { + present[n.ID] = true + } var links []opts.GraphLink for _, e := range edges { - srcName, srcOK := nameIdx[e.SourceID] - tgtName, tgtOK := nameIdx[e.TargetID] - if !srcOK || !tgtOK { + if !present[e.SourceID] || !present[e.TargetID] { continue } links = append(links, opts.GraphLink{ - Source: srcName, - Target: tgtName, + Source: nodeID(e.SourceID), + Target: nodeID(e.TargetID), Label: &opts.EdgeLabel{ - Show: opts.Bool(true), - Position: "middle", + Show: opts.Bool(true), + Position: "middle", Formatter: e.Type, }, }) } return links } - -// nodeNameIndex builds a map from node ID to node Name. -func nodeNameIndex(nodes []*graph.Node) map[int64]string { - m := make(map[int64]string, len(nodes)) - for _, n := range nodes { - m[n.ID] = n.Name - } - return m -} diff --git a/viz/convert_test.go b/viz/convert_test.go index 9a47cad..b30196f 100644 --- a/viz/convert_test.go +++ b/viz/convert_test.go @@ -85,11 +85,8 @@ func TestConvertEdges(t *testing.T) { if len(links) != 2 { t.Fatalf("expected 2 links, got %d", len(links)) } - if links[0].Source != "Alice" { - t.Errorf("expected source Alice, got %v", links[0].Source) - } - if links[0].Target != "Bob" { - t.Errorf("expected target Bob, got %v", links[0].Target) + if links[0].Source != "1" || links[0].Target != "2" { + t.Errorf("got link %v -> %v, want 1 -> 2", links[0].Source, links[0].Target) } } @@ -105,15 +102,3 @@ func TestConvertEdgesOrphanSkipped(t *testing.T) { t.Fatalf("expected 1 link (orphan skipped), got %d", len(links)) } } - -func TestNodeNameIndex(t *testing.T) { - nodes := testNodes() - idx := viz.NodeNameIndex(nodes) - - if idx[1] != "Alice" { - t.Errorf("expected Alice for ID 1, got %s", idx[1]) - } - if idx[3] != "Acme" { - t.Errorf("expected Acme for ID 3, got %s", idx[3]) - } -} diff --git a/viz/example_test.go b/viz/example_test.go index 980642d..13aec08 100644 --- a/viz/example_test.go +++ b/viz/example_test.go @@ -42,5 +42,5 @@ func Example() { fi, _ := f.Stat() fmt.Printf("rendered %d bytes of HTML\n", fi.Size()) // Output: - // rendered 1616 bytes of HTML + // rendered 1625 bytes of HTML } diff --git a/viz/export_test.go b/viz/export_test.go index 60d9e38..979f309 100644 --- a/viz/export_test.go +++ b/viz/export_test.go @@ -10,7 +10,6 @@ var ( BuildCategories = buildCategories ConvertNodes = convertNodes ConvertEdges = convertEdges - NodeNameIndex = nodeNameIndex NewPalette = newPalette ) diff --git a/viz/viz.go b/viz/viz.go index 60b921e..33ad22a 100644 --- a/viz/viz.go +++ b/viz/viz.go @@ -114,9 +114,11 @@ func (c *Chart) Render(w io.Writer) error { } } - g.AddSeries("graph", gNodes, gLinks, + g.AddSeries("graph", nil, gLinks, charts.WithGraphChartOpts(graphOpts), ) + // AddSeries accepts only opts.GraphNode, which has no id field. + g.MultiSeries[0].Data = gNodes return g.Render(w) } diff --git a/viz/viz_test.go b/viz/viz_test.go index bfc53fc..2bfa8d2 100644 --- a/viz/viz_test.go +++ b/viz/viz_test.go @@ -5,6 +5,7 @@ import ( "strings" "testing" + graph "github.com/justintout/go-sqlite-graph" "github.com/justintout/go-sqlite-graph/viz" ) @@ -132,3 +133,19 @@ func TestRenderContainsEdgeTypes(t *testing.T) { } } } + +func TestRenderDuplicateNames(t *testing.T) { + nodes := []*graph.Node{{ID: 1, Name: "Sam"}, {ID: 2, Name: "Sam"}} + edges := []*graph.Edge{{SourceID: 1, TargetID: 2, Type: "KNOWS"}} + + var buf bytes.Buffer + if err := viz.New(nodes, edges).Render(&buf); err != nil { + t.Fatalf("render error: %v", err) + } + html := buf.String() + for _, want := range []string{`"id":"1"`, `"id":"2"`, `"source":"1"`, `"target":"2"`} { + if !strings.Contains(html, want) { + t.Errorf("expected HTML to contain %s", want) + } + } +} From 7235b86c22e6c096735b62e96d10d065fe73e6fb Mon Sep 17 00:00:00 2001 From: Justin Tout <3639630+justintout@users.noreply.github.com> Date: Mon, 14 Sep 2026 11:05:22 -0400 Subject: [PATCH 6/9] Replace bench/compare replace directive with a go.work workspace Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YB9owLYxU6NsNuVk2NVc7C --- bench/compare/README.md | 2 +- bench/compare/go.mod | 2 -- go.work | 6 ++++++ go.work.sum | 1 + 4 files changed, 8 insertions(+), 3 deletions(-) create mode 100644 go.work create mode 100644 go.work.sum diff --git a/bench/compare/README.md b/bench/compare/README.md index 93eddeb..a52fb6d 100644 --- a/bench/compare/README.md +++ b/bench/compare/README.md @@ -1,6 +1,6 @@ # Comparison benchmarks -Benchmarks go-sqlite-graph against [GraphQLite](https://github.com/colliery-io/graphqlite), a SQLite extension with Cypher support. This is a separate module because it needs CGo (`mattn/go-sqlite3`) to load the extension. +Benchmarks go-sqlite-graph against [GraphQLite](https://github.com/colliery-io/graphqlite), a SQLite extension with Cypher support. This is a separate module because it needs CGo (`mattn/go-sqlite3`) to load the extension. The `go.work` file at the repository root builds it against the local copy of go-sqlite-graph. Each graph is loaded into three engines, and the harness fails if they disagree on any result size: diff --git a/bench/compare/go.mod b/bench/compare/go.mod index 9283360..1e34519 100644 --- a/bench/compare/go.mod +++ b/bench/compare/go.mod @@ -2,8 +2,6 @@ module github.com/justintout/go-sqlite-graph/bench/compare go 1.23.0 -replace github.com/justintout/go-sqlite-graph => ../.. - require ( github.com/justintout/go-sqlite-graph v0.0.0-20260714170249-4e85730c39e9 github.com/mattn/go-sqlite3 v1.14.52 diff --git a/go.work b/go.work new file mode 100644 index 0000000..216d262 --- /dev/null +++ b/go.work @@ -0,0 +1,6 @@ +go 1.23.0 + +use ( + . + ./bench/compare +) diff --git a/go.work.sum b/go.work.sum new file mode 100644 index 0000000..0fe4f24 --- /dev/null +++ b/go.work.sum @@ -0,0 +1 @@ +github.com/justintout/go-sqlite-graph v0.0.0-20260714170249-4e85730c39e9/go.mod h1:nd9w5KeKprKHOzYPVQuhDgkiahi+tRqmrIeknOouVDw= From 0ea45368df5792b9cbfd3545f3d59c682e557602 Mon Sep 17 00:00:00 2001 From: Justin Tout <3639630+justintout@users.noreply.github.com> Date: Mon, 14 Sep 2026 11:29:15 -0400 Subject: [PATCH 7/9] Return node labels from queries with Return("labels") Query results never set Node.Labels, so viz.NewFromResult put every node in one color category. Labels are opt-in: reading them runs a subquery and decodes a JSON array per node, which made default traversals 30-50% slower. The subquery reads node_labels in primary key order; aggregate ORDER BY built a temp b-tree per node and was slower still. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YB9owLYxU6NsNuVk2NVc7C --- README.md | 4 ++-- integration_test.go | 17 +++++++++++++++++ query.go | 5 +++-- query_compile.go | 18 +++++++++++++++++- viz/README.md | 3 ++- 5 files changed, 41 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 9facdb1..f350210 100644 --- a/README.md +++ b/README.md @@ -168,7 +168,7 @@ g.Match("Person"). // Return only some columns and property paths. g.Match("Person"). Related("KNOWS", 1, 3). - Return("name", "age"). + Return("name", "labels", "age"). Run(ctx) // Count and paginate. @@ -182,7 +182,7 @@ The operators are `=`, `!=`, `<>`, `<`, `<=`, `>`, `>=`, `LIKE`, `NOT LIKE`, `IN A step takes between 1 and `graph.MaxHops` (10) hops. A start node is in the result only if the step reaches it, for example through a cycle. Cycles do not loop forever: a walk expands each node at most once per depth. -Query results populate the node columns and properties. They do not populate `Labels`, and they do not return edges. `Return` sets `ID` plus the named fields. It sets a property path as a key in `Properties` and leaves every other field zero. Returning fewer fields skips JSON decoding, which is a large share of the cost of big traversals. +By default, query results populate every `Node` field except `Labels`, because reading labels makes queries about 40% slower. Select them with `Return`. Query results do not return edges. `Return` sets `ID` plus the named fields: `name`, `labels`, `created_at`, `updated_at`, `properties`, or a property path. A property whose name matches one of those columns can only be filtered, not returned. It sets a property path as a key in `Properties` and leaves every other field zero. Returning fewer fields skips JSON decoding, which is a large share of the cost of big traversals. `Count` ignores `Limit` and `Offset`. diff --git a/integration_test.go b/integration_test.go index 2b306f6..5060f37 100644 --- a/integration_test.go +++ b/integration_test.go @@ -5,6 +5,7 @@ import ( "fmt" "maps" "math/rand" + "slices" "testing" ) @@ -356,6 +357,22 @@ func TestReturn(t *testing.T) { } } + if err := g.AddLabels(ctx, alice.ID, "Engineer"); err != nil { + t.Fatal(err) + } + for _, q := range []*Query{ + g.Match("Person").Where("name", "=", "Alice").Return("labels"), + g.Match("Company").RelatedDir("WORKS_AT", Incoming, 1, 1).WhereRel("name", "=", "Alice").Return("labels"), + } { + res, err := q.Run(ctx) + if err != nil { + t.Fatal(err) + } + if got := res.Nodes()[0].Labels; !slices.Equal(got, []string{"Engineer", "Person"}) { + t.Errorf("labels = %v, want [Engineer Person]", got) + } + } + res, err := g.Match("Company").Return("properties", "industry").Run(ctx) if err != nil { t.Fatal(err) diff --git a/query.go b/query.go index fa7d1de..a6cdf42 100644 --- a/query.go +++ b/query.go @@ -206,11 +206,12 @@ func (q *Query) whereRel(field, op string, value any, isJSON bool) *Query { } // Return specifies which columns/properties to project in results. -// Known columns (name, created_at, updated_at, properties) map to Node fields. +// Known columns (name, labels, created_at, updated_at, properties) map to Node fields. // Other names are treated as JSON property paths and set in Node.Properties // under the name as given; paths missing from a node are omitted. Node.ID is // always set. Fields not returned are left zero, and Properties is nil unless -// properties or a path is returned. Without Return, all columns are returned. +// properties or a path is returned. Without Return, every field except Labels +// is returned. func (q *Query) Return(cols ...string) *Query { for _, c := range cols { if c == "" { diff --git a/query_compile.go b/query_compile.go index eca29b1..3166c5d 100644 --- a/query_compile.go +++ b/query_compile.go @@ -32,9 +32,15 @@ func (b *sqlBuilder) w(parts ...string) { } } +// allCols is the default projection. It omits labels: reading them runs a +// subquery and decodes a JSON array per node, which made traversals ~40% +// slower. Return("labels") selects them. var allCols = []string{"name", "created_at", "updated_at", "properties"} -var knownNodeCols = map[string]bool{"name": true, "created_at": true, "updated_at": true, "properties": true} +var knownNodeCols = map[string]bool{"name": true, "labels": true, "created_at": true, "updated_at": true, "properties": true} + +// labelsExpr selects a node's labels as a JSON array, sorted like GetNode. +const labelsExpr = "(SELECT json_group_array(label) FROM (SELECT label FROM node_labels WHERE node_id = n.id ORDER BY label))" // compile builds the SQL for q. With count set, it selects COUNT(*) and // ignores Limit and Offset. @@ -113,6 +119,10 @@ func (c *compiledQuery) writeSelect(b *sqlBuilder, q *Query, count bool) { b.w("SELECT n.id") for _, col := range c.cols { + if col == "labels" { + b.w(", ", labelsExpr) + continue + } if knownNodeCols[col] { b.w(", n.", col) continue @@ -323,6 +333,12 @@ func (c *compiledQuery) execute(conn *sqlite.Conn) (*Result, error) { switch col { case "name": n.Name = stmt.ColumnText(i) + case "labels": + if text := stmt.ColumnText(i); text != "[]" { + if err := json.Unmarshal([]byte(text), &n.Labels); err != nil { + return fmt.Errorf("graph: decode labels: %w", err) + } + } case "created_at": n.CreatedAt = stmt.ColumnText(i) case "updated_at": diff --git a/viz/README.md b/viz/README.md index cd16ded..00e93ae 100644 --- a/viz/README.md +++ b/viz/README.md @@ -51,6 +51,7 @@ c.Render(f) result, _ := g.Match("Person"). Where("name", "=", "Alice"). Related("KNOWS", 1, 3). + Return("name", "labels"). Run(ctx) c := viz.NewFromResult(result, edges) @@ -77,7 +78,7 @@ http.ListenAndServe(":8080", nil) ## How it works -- Each node's first label determines its color category. Nodes without labels are grouped as "(unlabeled)". +- Each node's first label determines its color category. Nodes without labels are grouped as "(unlabeled)". Queries return labels only when asked, so pass `Return("name", "labels")` to a query whose result you render. - Colors cycle through the palette when there are more categories than colors. - Edge `Type` fields are displayed as edge labels. - Edges referencing nodes not in the provided slice are silently skipped. From a9163252d3a18d5f1f939cf4c3df020b29f554a4 Mon Sep 17 00:00:00 2001 From: Justin Tout <3639630+justintout@users.noreply.github.com> Date: Mon, 14 Sep 2026 11:42:51 -0400 Subject: [PATCH 8/9] Add EdgesBetween and Query.WithLabels EdgesBetween returns the edges whose endpoints are both in a set of node IDs, optionally filtered by type. It is the only way to read edges for a query result without raw SQL. The ID and type lists bind as JSON arrays, and a unary + keeps the planner seeking by source instead of once per source and target pair. WithLabels adds labels to whatever fields a query returns, where Return("labels") replaces them. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YB9owLYxU6NsNuVk2NVc7C --- README.md | 8 +++- edge.go | 94 ++++++++++++++++++++++++++++++++++++++------- edge_test.go | 50 ++++++++++++++++++++++++ integration_test.go | 5 +++ query.go | 8 ++++ query_compile.go | 6 ++- tx.go | 5 +++ 7 files changed, 160 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index f350210..8648cb8 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,12 @@ g.DeleteEdge(ctx, edge.ID) Edges are directed. `UpdateEdge` does not change the endpoints. +`EdgesBetween` returns the edges whose endpoints are both in a set of node IDs, such as the nodes of a query result. Pass edge types to return only those. + +```go +edges, err := g.EdgesBetween(ctx, []int64{alice.ID, bob.ID}, "KNOWS") +``` + Get, update, and delete return an error when the ID does not exist. ### Transactions @@ -182,7 +188,7 @@ The operators are `=`, `!=`, `<>`, `<`, `<=`, `>`, `>=`, `LIKE`, `NOT LIKE`, `IN A step takes between 1 and `graph.MaxHops` (10) hops. A start node is in the result only if the step reaches it, for example through a cycle. Cycles do not loop forever: a walk expands each node at most once per depth. -By default, query results populate every `Node` field except `Labels`, because reading labels makes queries about 40% slower. Select them with `Return`. Query results do not return edges. `Return` sets `ID` plus the named fields: `name`, `labels`, `created_at`, `updated_at`, `properties`, or a property path. A property whose name matches one of those columns can only be filtered, not returned. It sets a property path as a key in `Properties` and leaves every other field zero. Returning fewer fields skips JSON decoding, which is a large share of the cost of big traversals. +By default, query results populate every `Node` field except `Labels`, because reading labels makes queries about 40% slower. `WithLabels` adds them to the returned fields, and `Return` can name `labels`. Query results do not include edges; read them with `EdgesBetween`. `Return` sets `ID` plus the named fields: `name`, `labels`, `created_at`, `updated_at`, `properties`, or a property path. A property whose name matches one of those columns can only be filtered, not returned. It sets a property path as a key in `Properties` and leaves every other field zero. Returning fewer fields skips JSON decoding, which is a large share of the cost of big traversals. `Count` ignores `Limit` and `Offset`. diff --git a/edge.go b/edge.go index d263466..7786676 100644 --- a/edge.go +++ b/edge.go @@ -2,6 +2,7 @@ package graph import ( "context" + "encoding/json" "fmt" "zombiezen.com/go/sqlite" @@ -60,6 +61,18 @@ func (g *Graph) DeleteEdge(ctx context.Context, id int64) error { return deleteEdgeInternal(conn, id) } +// EdgesBetween returns the edges whose source and target are both in nodeIDs, +// in no guaranteed order. With types given, it returns only edges of those +// types. +func (g *Graph) EdgesBetween(ctx context.Context, nodeIDs []int64, types ...string) ([]*Edge, error) { + conn, err := g.conn(ctx) + if err != nil { + return nil, err + } + defer g.put(conn) + return edgesBetweenInternal(conn, nodeIDs, types) +} + func createEdgeInternal(conn *sqlite.Conn, e *Edge) error { props, err := MarshalProperties(e.Properties) if err != nil { @@ -80,24 +93,33 @@ func createEdgeInternal(conn *sqlite.Conn, e *Edge) error { return nil } -func getEdgeInternal(conn *sqlite.Conn, id int64) (*Edge, error) { - e := &Edge{ID: id} - found := false +const edgeCols = "e.id, e.source_id, e.target_id, e.type, e.name, e.created_at, e.updated_at, e.properties" + +// scanEdge reads a row selected with edgeCols. +func scanEdge(stmt *sqlite.Stmt) (*Edge, error) { + e := &Edge{ + ID: stmt.ColumnInt64(0), + SourceID: stmt.ColumnInt64(1), + TargetID: stmt.ColumnInt64(2), + Type: stmt.ColumnText(3), + Name: stmt.ColumnText(4), + CreatedAt: stmt.ColumnText(5), + UpdatedAt: stmt.ColumnText(6), + } + var err error + e.Properties, err = UnmarshalProperties(stmt.ColumnText(7)) + return e, err +} +func getEdgeInternal(conn *sqlite.Conn, id int64) (*Edge, error) { + var e *Edge err := sqlitex.Execute(conn, - "SELECT source_id, target_id, type, name, created_at, updated_at, properties FROM edges WHERE id = ?;", + "SELECT "+edgeCols+" FROM edges e WHERE e.id = ?;", &sqlitex.ExecOptions{ Args: []any{id}, ResultFunc: func(stmt *sqlite.Stmt) error { - found = true - e.SourceID = stmt.ColumnInt64(0) - e.TargetID = stmt.ColumnInt64(1) - e.Type = stmt.ColumnText(2) - e.Name = stmt.ColumnText(3) - e.CreatedAt = stmt.ColumnText(4) - e.UpdatedAt = stmt.ColumnText(5) var err error - e.Properties, err = UnmarshalProperties(stmt.ColumnText(6)) + e, err = scanEdge(stmt) return err }, }, @@ -105,13 +127,57 @@ func getEdgeInternal(conn *sqlite.Conn, id int64) (*Edge, error) { if err != nil { return nil, fmt.Errorf("graph: get edge: %w", err) } - if !found { + if e == nil { return nil, fmt.Errorf("graph: edge %d not found", id) } - return e, nil } +// edgesBetweenInternal binds the ID and type lists as JSON arrays, so the SQL +// text does not vary with their lengths. Each node's outgoing edges are read +// from the (source_id, type, target_id) index, and the target is checked +// against the set. The unary + on target_id stops the planner from seeking +// the target index once per (source, target) pair, which is quadratic in the +// number of nodes. +func edgesBetweenInternal(conn *sqlite.Conn, nodeIDs []int64, types []string) ([]*Edge, error) { + if len(nodeIDs) == 0 { + return nil, nil + } + ids, err := json.Marshal(nodeIDs) + if err != nil { + return nil, fmt.Errorf("graph: encode node IDs: %w", err) + } + query := "SELECT " + edgeCols + " FROM (SELECT DISTINCT value AS id FROM json_each(?)) s" + + " CROSS JOIN edges e ON e.source_id = s.id" + + " WHERE +e.target_id IN (SELECT value FROM json_each(?))" + args := []any{string(ids), string(ids)} + if len(types) > 0 { + t, err := json.Marshal(types) + if err != nil { + return nil, fmt.Errorf("graph: encode edge types: %w", err) + } + query += " AND e.type IN (SELECT value FROM json_each(?))" + args = append(args, string(t)) + } + + var edges []*Edge + err = sqlitex.Execute(conn, query+";", &sqlitex.ExecOptions{ + Args: args, + ResultFunc: func(stmt *sqlite.Stmt) error { + e, err := scanEdge(stmt) + if err != nil { + return err + } + edges = append(edges, e) + return nil + }, + }) + if err != nil { + return nil, fmt.Errorf("graph: edges between: %w", err) + } + return edges, nil +} + func updateEdgeInternal(conn *sqlite.Conn, e *Edge) error { props, err := MarshalProperties(e.Properties) if err != nil { diff --git a/edge_test.go b/edge_test.go index 4e03bed..96ad1fe 100644 --- a/edge_test.go +++ b/edge_test.go @@ -64,3 +64,53 @@ func TestEdgeCRUD(t *testing.T) { t.Error("expected error getting deleted edge") } } + +func TestEdgesBetween(t *testing.T) { + g := openTestGraph(t) + ctx := context.Background() + + var n [3]*Node + for i := range n { + n[i] = &Node{Name: "n"} + if err := g.CreateNode(ctx, n[i]); err != nil { + t.Fatal(err) + } + } + for _, e := range []*Edge{ + {SourceID: n[0].ID, TargetID: n[1].ID, Type: "KNOWS"}, + {SourceID: n[1].ID, TargetID: n[0].ID, Type: "LIKES"}, + {SourceID: n[0].ID, TargetID: n[0].ID, Type: "KNOWS"}, + {SourceID: n[1].ID, TargetID: n[2].ID, Type: "KNOWS"}, + } { + if err := g.CreateEdge(ctx, e); err != nil { + t.Fatal(err) + } + } + + ids := []int64{n[0].ID, n[1].ID, n[0].ID} + for _, c := range []struct { + types []string + want int + }{ + {nil, 3}, + {[]string{"KNOWS"}, 2}, + {[]string{"LIKES", "MISSING"}, 1}, + } { + edges, err := g.EdgesBetween(ctx, ids, c.types...) + if err != nil { + t.Fatal(err) + } + if len(edges) != c.want { + t.Errorf("types %v: got %d edges, want %d", c.types, len(edges), c.want) + } + for _, e := range edges { + if e.TargetID == n[2].ID || e.Properties == nil { + t.Errorf("types %v: got %+v", c.types, e) + } + } + } + + if edges, err := g.EdgesBetween(ctx, nil); err != nil || edges != nil { + t.Errorf("no IDs: got %v, %v", edges, err) + } +} diff --git a/integration_test.go b/integration_test.go index 5060f37..dfe061f 100644 --- a/integration_test.go +++ b/integration_test.go @@ -362,6 +362,8 @@ func TestReturn(t *testing.T) { } for _, q := range []*Query{ g.Match("Person").Where("name", "=", "Alice").Return("labels"), + g.Match("Person").Where("name", "=", "Alice").WithLabels(), + g.Match("Person").Where("name", "=", "Alice").Return("name").WithLabels(), g.Match("Company").RelatedDir("WORKS_AT", Incoming, 1, 1).WhereRel("name", "=", "Alice").Return("labels"), } { res, err := q.Run(ctx) @@ -371,6 +373,9 @@ func TestReturn(t *testing.T) { if got := res.Nodes()[0].Labels; !slices.Equal(got, []string{"Engineer", "Person"}) { t.Errorf("labels = %v, want [Engineer Person]", got) } + if q.withLabels && res.Nodes()[0].Name != "Alice" { + t.Errorf("WithLabels dropped name: %+v", res.Nodes()[0]) + } } res, err := g.Match("Company").Return("properties", "industry").Run(ctx) diff --git a/query.go b/query.go index a6cdf42..28a2d98 100644 --- a/query.go +++ b/query.go @@ -92,6 +92,7 @@ type Query struct { wheres []whereClause rels []relStep returnCols []string + withLabels bool limitVal int offsetVal int err error // captures builder errors @@ -223,6 +224,13 @@ func (q *Query) Return(cols ...string) *Query { return q } +// WithLabels adds Node.Labels to the fields the query returns, whether those +// are the defaults or the columns named by Return. +func (q *Query) WithLabels() *Query { + q.withLabels = true + return q +} + // Limit sets a maximum number of results. func (q *Query) Limit(n int) *Query { q.limitVal = n diff --git a/query_compile.go b/query_compile.go index 3166c5d..27e491c 100644 --- a/query_compile.go +++ b/query_compile.go @@ -34,7 +34,7 @@ func (b *sqlBuilder) w(parts ...string) { // allCols is the default projection. It omits labels: reading them runs a // subquery and decodes a JSON array per node, which made traversals ~40% -// slower. Return("labels") selects them. +// slower. Return("labels") and WithLabels select them. var allCols = []string{"name", "created_at", "updated_at", "properties"} var knownNodeCols = map[string]bool{"name": true, "labels": true, "created_at": true, "updated_at": true, "properties": true} @@ -116,6 +116,10 @@ func (c *compiledQuery) writeSelect(b *sqlBuilder, q *Query, count bool) { c.cols = append(c.cols, col) } } + if q.withLabels && !slices.Contains(c.cols, "labels") { + // Clip so the append copies instead of writing into allCols. + c.cols = append(slices.Clip(c.cols), "labels") + } b.w("SELECT n.id") for _, col := range c.cols { diff --git a/tx.go b/tx.go index 5146840..a2ca387 100644 --- a/tx.go +++ b/tx.go @@ -112,6 +112,11 @@ func (tx *Tx) DeleteEdge(ctx context.Context, id int64) error { return deleteEdgeInternal(tx.conn, id) } +// EdgesBetween returns the edges between nodeIDs within this transaction. +func (tx *Tx) EdgesBetween(ctx context.Context, nodeIDs []int64, types ...string) ([]*Edge, error) { + return edgesBetweenInternal(tx.conn, nodeIDs, types) +} + // Match starts a query within this transaction. func (tx *Tx) Match(label string) *Query { return &Query{ From f5aff4fa479fd3b51c202093eb9dbf6e5842f89a Mon Sep 17 00:00:00 2001 From: Justin Tout <3639630+justintout@users.noreply.github.com> Date: Mon, 14 Sep 2026 11:42:51 -0400 Subject: [PATCH 9/9] Replace viz.NewFromResult with FromQuery NewFromResult needed labels the query had not selected and edges the caller had no API to read. FromQuery runs the query with labels and reads the edges between its nodes. New remains for choosing nodes and edges directly. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YB9owLYxU6NsNuVk2NVc7C --- viz/README.md | 28 ++++++++++++++++++++------- viz/viz.go | 29 +++++++++++++++++++++++++--- viz/viz_test.go | 50 +++++++++++++++++++++++++++++++++++++++---------- 3 files changed, 87 insertions(+), 20 deletions(-) diff --git a/viz/README.md b/viz/README.md index 00e93ae..f5bc902 100644 --- a/viz/README.md +++ b/viz/README.md @@ -45,19 +45,33 @@ defer f.Close() c.Render(f) ``` -### From a query result +### From a query ```go -result, _ := g.Match("Person"). +q := g.Match("Person"). Where("name", "=", "Alice"). - Related("KNOWS", 1, 3). - Return("name", "labels"). - Run(ctx) + Related("KNOWS", 1, 3) -c := viz.NewFromResult(result, edges) +c, err := viz.FromQuery(ctx, g, q, viz.WithTitle("Alice's network")) c.Render(os.Stdout) ``` +`FromQuery` runs the query with labels selected, then reads every edge between the returned nodes. Pass a `*graph.Tx` instead of the `*graph.Graph` when the query came from that transaction. + +To choose the edges, build the chart with `New`: + +```go +res, _ := q.WithLabels().Run(ctx) +nodes := res.Nodes() +ids := make([]int64, len(nodes)) +for i, n := range nodes { + ids[i] = n.ID +} +edges, _ := g.EdgesBetween(ctx, ids, "KNOWS") + +c := viz.New(nodes, edges) +``` + ### HTTP handler ```go @@ -78,7 +92,7 @@ http.ListenAndServe(":8080", nil) ## How it works -- Each node's first label determines its color category. Nodes without labels are grouped as "(unlabeled)". Queries return labels only when asked, so pass `Return("name", "labels")` to a query whose result you render. +- Each node's first label determines its color category. Nodes without labels are grouped as "(unlabeled)". Queries return labels only when asked: `FromQuery` asks for them, and with `New` use `WithLabels` on the query. - Colors cycle through the palette when there are more categories than colors. - Edge `Type` fields are displayed as edge labels. - Edges referencing nodes not in the provided slice are silently skipped. diff --git a/viz/viz.go b/viz/viz.go index 33ad22a..701ed5d 100644 --- a/viz/viz.go +++ b/viz/viz.go @@ -2,6 +2,7 @@ package viz import ( + "context" "io" "net/http" @@ -69,9 +70,31 @@ func New(nodes []*graph.Node, edges []*graph.Edge, options ...Option) *Chart { return c } -// NewFromResult creates a Chart from a query Result and edges. -func NewFromResult(result *graph.Result, edges []*graph.Edge, options ...Option) *Chart { - return New(result.Nodes(), edges, options...) +// EdgeReader reads the edges between a set of nodes. *graph.Graph and +// *graph.Tx implement it. +type EdgeReader interface { + EdgesBetween(ctx context.Context, nodeIDs []int64, types ...string) ([]*graph.Edge, error) +} + +// FromQuery runs q with labels selected and creates a Chart of its nodes and +// every edge between them, read from db. When q was started from a Tx, pass +// that Tx as db so the edges come from the same transaction. Use New to +// choose the nodes or edges yourself. +func FromQuery(ctx context.Context, db EdgeReader, q *graph.Query, options ...Option) (*Chart, error) { + res, err := q.WithLabels().Run(ctx) + if err != nil { + return nil, err + } + nodes := res.Nodes() + ids := make([]int64, len(nodes)) + for i, n := range nodes { + ids[i] = n.ID + } + edges, err := db.EdgesBetween(ctx, ids) + if err != nil { + return nil, err + } + return New(nodes, edges, options...), nil } // Render writes the chart as a self-contained HTML page to w. diff --git a/viz/viz_test.go b/viz/viz_test.go index 2bfa8d2..f7325a8 100644 --- a/viz/viz_test.go +++ b/viz/viz_test.go @@ -2,6 +2,7 @@ package viz_test import ( "bytes" + "context" "strings" "testing" @@ -88,22 +89,51 @@ func TestRenderContainsNodeNames(t *testing.T) { } } -func TestNewFromResult(t *testing.T) { - // Result is constructed with Nodes() returning []*Node. - // Since Result has unexported fields, we test via New directly - // with the same node data. NewFromResult calls result.Nodes() - // which returns the same type. - nodes := testNodes() - edges := testEdges() +func TestFromQuery(t *testing.T) { + ctx := context.Background() + g, err := graph.Open("file:TestFromQuery?mode=memory&cache=shared", nil) + if err != nil { + t.Fatal(err) + } + defer g.Close() + + alice := &graph.Node{Name: "Alice", Labels: []string{"Person"}} + bob := &graph.Node{Name: "Bob", Labels: []string{"Person"}} + acme := &graph.Node{Name: "Acme", Labels: []string{"Company"}} + for _, n := range []*graph.Node{alice, bob, acme} { + if err := g.CreateNode(ctx, n); err != nil { + t.Fatal(err) + } + } + for _, e := range []*graph.Edge{ + {SourceID: alice.ID, TargetID: bob.ID, Type: "KNOWS"}, + {SourceID: bob.ID, TargetID: acme.ID, Type: "WORKS_AT"}, + } { + if err := g.CreateEdge(ctx, e); err != nil { + t.Fatal(err) + } + } - c := viz.New(nodes, edges, viz.WithTitle("From Result")) + // From Acme, WORKS_AT in both directions reaches Bob, then Acme again. + // The result excludes Alice, so the KNOWS edge must not be drawn. + q := g.Match("Company").RelatedDir("WORKS_AT", graph.Both, 1, 2).Return("name") + c, err := viz.FromQuery(ctx, g, q, viz.WithTitle("From Query")) + if err != nil { + t.Fatal(err) + } var buf bytes.Buffer if err := c.Render(&buf); err != nil { t.Fatalf("render error: %v", err) } - if !strings.Contains(buf.String(), "From Result") { - t.Error("expected title in output") + html := buf.String() + for _, want := range []string{"From Query", `"name":"Company"`, `"name":"Person"`, "WORKS_AT"} { + if !strings.Contains(html, want) { + t.Errorf("expected HTML to contain %s", want) + } + } + if strings.Contains(html, "KNOWS") { + t.Error("rendered an edge to a node outside the result") } }