contextdb v2: a search index for vector columns

.github/workflows/check.yml

@@ -20,8 +20,8 @@ env:
   # The size of the shipped code, rounded up to 500 lines, and the number of tests, rounded up
   # to 10. A change that crosses either number edits it here, in the same change, where its
   # reader sees that the code grew or shrank. Neither ever passes 150,000 lines or 1,000 tests.
-  SHIPPED_LINES: 11000
-  TESTS: 180
+  SHIPPED_LINES: 15500
+  TESTS: 260
 
 jobs:
   lint:

v2/README.md

@@ -64,3 +64,10 @@ UPDATE_EXPECTED=1 cargo test --manifest-path v2/Cargo.toml -p contextdb --test s
 
 `v2/numbers.sh` prints the numbers in the library's README for the commit it is run at. It only
 measures: it never passes or fails, and the check does not run it.
+
+The lines about vector search are taken by the programs in `contextdb/examples/`, the generated
+vectors always and the groups of lines about real search passages only on data given in the
+environment variables named at the head of the script (without them each of those lines prints
+`unavailable`). Every time of a search is taken by one thread, in a read session under the memory
+limit that a line of the output states; `NUMBERS_LARGE=1` adds five million generated vectors and
+takes the better part of an hour. Run it on a machine that is doing nothing else.

v2/contextdb-cli/src/main.rs

@@ -67,13 +67,14 @@ MEMORY:
   `none` and 9437184 bytes used, the 1M and the 8M page cache.
 
 JSON (--json), one document a line:
-  a SELECT or SHOW MEMORY_LIMIT   {"result":{"columns":[...],"rows":[{...}]}}
+  a SELECT or any SHOW statement  {"result":{"columns":[...],"rows":[{...}]}}
                                   a vector is an array of numbers, each at the full
                                   precision of the stored 32-bit value, so a number
                                   may print with more digits than were typed
   any other statement             {"rows_affected":N}
   .tables                         {"tables":{"items":[...]}}
   .schema <table>                 {"schema":{"table":...,"ddl":...}}
+  .explain <sql>                  {"explain":{"lines":[...]}}
   .help                           {"help":[...]}
   a refusal                       {"error":{"class":...,"message":...,"detail":{"kind":...}}}
                                   on the error stream; "detail" is left out when the

v2/contextdb-cli/src/output.rs

@@ -40,6 +40,32 @@ impl Output {
         }
     }
 
+    /// The rows of an inspection statement: one `name: value` to a line, with an empty line
+    /// between rows.
+    pub fn fields(self, result: &QueryResult) {
+        if self.json {
+            return self.result(result);
+        }
+        for (at, row) in result.rows().iter().enumerate() {
+            if at > 0 {
+                println!();
+            }
+            for (name, value) in result.columns().iter().zip(row) {
+                println!("{name}: {}", text(value));
+            }
+        }
+    }
+
+    /// What `.explain` answers: its lines.
+    pub fn explain(self, lines: &str) {
+        if self.json {
+            let lines: Vec<&str> = lines.lines().collect();
+            println!("{}", json!({ "explain": { "lines": lines } }));
+        } else {
+            println!("{lines}");
+        }
+    }
+
     /// A statement that returns no rows succeeded and changed this many.
     pub fn rows_affected(self, rows: u64) {
         if self.json {

v2/contextdb-cli/src/shell.rs

@@ -18,11 +18,13 @@ pub enum Store {
     Write(Database),
 }
 
-const HELP: [&str; 6] = [
+const HELP: [&str; 8] = [
     "<statement>;        Run a SQL statement. It may span lines and ends at its `;`.",
     "BEGIN; ... COMMIT;  Run statements as one transaction; ROLLBACK; undoes them, except SET MEMORY_LIMIT. Needs --write.",
     ".tables   \\dt       List the tables.",
     ".schema <table>   \\d <table>   Show a table's declaration as SQL.",
+    ".explain <sql>      Show how a similarity search would be answered, without running it.",
+    "SHOW VECTOR_INDEXES; SHOW VECTOR_PARTITIONS;   Describe the VECTOR columns and their partitions.",
     ".help     \\?        Show this list.",
     ".quit     \\q        End the session.",
 ];
@@ -151,6 +153,15 @@ impl<'a> Shell<'a> {
                 Store::Write(db) => db.declaration(table),
             }
             .map(|declaration| self.output.schema(table, &declaration)),
+            (Some(".explain"), Some(_), _) => {
+                let sql = line[".explain".len()..].trim();
+                let sql = sql.strip_suffix(';').unwrap_or(sql);
+                match self.store {
+                    Store::Read(reader) => reader.explain(sql),
+                    Store::Write(db) => db.explain(sql),
+                }
+                .map(|lines| self.output.explain(&lines))
+            }
             (Some(".help" | "\\?"), None, _) => {
                 self.output.help(&HELP);
                 Ok(())
@@ -186,7 +197,17 @@ impl<'a> Shell<'a> {
                 Ok(())
             })
         } else {
-            self.execute(sql).map(|result| self.output.result(&result))
+            // The rows of the two inspection statements are too wide for a table.
+            let mut words = sql.split_whitespace().map(str::to_ascii_uppercase);
+            let inspection = words.next().as_deref() == Some("SHOW")
+                && words.next().is_some_and(|word| word.starts_with("VECTOR_"));
+            self.execute(sql).map(|result| {
+                if inspection {
+                    self.output.fields(&result);
+                } else {
+                    self.output.result(&result);
+                }
+            })
         };
         if let Err(error) = ran {
             self.refused = true;