contextdb v2: vector columns, exact search by distance, compressed large values

.github/workflows/check.yml

@@ -17,6 +17,11 @@ concurrency:
 
 env:
   CARGO_TERM_COLOR: always
+  # The size of the shipped code, rounded up to 500 lines, and the number of tests, rounded up
+  # to 10. A change that crosses either number edits it here, in the same change, where its
+  # reader sees that the code grew or shrank. Neither ever passes 150,000 lines or 1,000 tests.
+  SHIPPED_LINES: 11000
+  TESTS: 180
 
 jobs:
   lint:
@@ -26,8 +31,21 @@ jobs:
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
       - run: cargo fmt --manifest-path v2/Cargo.toml --all --check
       - run: cargo clippy --manifest-path v2/Cargo.toml --workspace --all-targets --locked -- -D warnings
-      - name: Shipped lines stay under 150,000
-        run: test "$(find v2/contextdb/src v2/contextdb-cli/src \( -name '*.rs' -o -name '*.pest' \) ! -name tests.rs -exec cat {} + | wc -l)" -le 150000
+      - name: Shipped lines are at most the stated number and within 500 of it
+        run: |
+          lines=$(find v2/contextdb/src v2/contextdb-cli/src \( -name '*.rs' -o -name '*.pest' \) ! -name tests.rs -exec cat {} + | wc -l)
+          echo "$lines shipped lines; the stated number is $SHIPPED_LINES"
+          test "$SHIPPED_LINES" -le 150000
+          test "$lines" -le "$SHIPPED_LINES"
+          test "$lines" -gt "$((SHIPPED_LINES - 500))"
+      - name: No shipped file is longer than 1,500 lines
+        run: test -z "$(find v2/contextdb/src v2/contextdb-cli/src -name '*.rs' ! -name tests.rs -exec wc -l {} + | awk '$2 != "total" && $1 > 1500')"
+      - name: No test-only name in shipped code
+        run: test -z "$(grep -rnE --include='*.rs' --exclude=tests.rs '_for_test|test[-_]seam|feature *= *"test' v2/contextdb/src v2/contextdb-cli/src)"
+      - name: No ignored test
+        run: test -z "$(grep -rn --include='*.rs' '#\[ignore' v2/contextdb v2/contextdb-cli)"
+      - name: Only the four allow attributes that v2/AGENTS.md names, and no expect attribute
+        run: test "$(grep -rnE --include='*.rs' '#!?\[ *(allow|expect) *\(' v2/contextdb v2/contextdb-cli | wc -l)" -eq 4
 
   test:
     strategy:
@@ -45,9 +63,15 @@ jobs:
       - run: cargo test --manifest-path v2/Cargo.toml --workspace --locked --doc
       - name: The storage library's own tests
         run: cargo test --manifest-path v2/redb/Cargo.toml --locked --lib --tests
-      # A test is one the runner lists, or one script file.
-      - name: Tests stay under 1,000
-        run: test "$(( $(cargo nextest list --manifest-path v2/Cargo.toml --workspace --locked -T oneline | wc -l) + $(find v2/contextdb/tests/sql -name '*.sql' | wc -l) ))" -le 1000
+      # A test is one the runner lists, or one script file. Counted on Linux, where every test runs.
+      - name: Tests are at most the stated number and within 10 of it
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          tests=$(( $(cargo nextest list --manifest-path v2/Cargo.toml --workspace --locked -T oneline | wc -l) + $(find v2/contextdb/tests/sql -name '*.sql' | wc -l) ))
+          echo "$tests tests; the stated number is $TESTS"
+          test "$TESTS" -le 1000
+          test "$tests" -le "$TESTS"
+          test "$tests" -gt "$((TESTS - 10))"
 
   minimum-rust:
     runs-on: ubuntu-latest

v2/README.md

@@ -27,8 +27,12 @@ cargo +1.89 check --manifest-path v2/Cargo.toml --workspace --locked
 ```
 
 The last line checks the oldest Rust the packages declare (`rustup toolchain install 1.89`
-first). The workflow also fails when the library and the program together pass 150,000 lines or
-the tests pass 1,000; the two lines that count them are in the workflow file.
+first). The workflow also states the size of the library and the program, rounded up to 500
+lines, and the number of tests, rounded up to 10, and fails when either is passed or is no
+longer within that distance: a change that grows or shrinks the code past a boundary edits
+the number in the workflow file, in the same change. It also refuses a shipped file longer
+than 1,500 lines, a test-only name in shipped code, an ignored test and an `allow` attribute
+beyond the four `AGENTS.md` names. `AGENTS.md` says how a change is made so that these hold.
 
 ## One test
 

v2/contextdb-cli/src/main.rs

@@ -51,16 +51,26 @@ EXIT STATUS:
   2  the command line was refused; nothing was opened
 
 MEMORY:
-  --memory-limit is this session's ceiling, in bytes or with K, M or G. Without it a
-  --write session has the limit the store remembers and a read session has none.
-  The store keeps 8M for its page cache and 1M is set aside before that, so the
-  smallest limit that opens a store is 9M.
+  --memory-limit is this session's ceiling, in bytes or with K, M or G; the store does
+  not remember it. Without it a --write session has the limit the store remembers and
+  a read session has none.
+  A session sets aside 1M and its page cache, sized when it starts: a quarter of its
+  limit and never less than 8M, which is also its size with no limit. So the smallest
+  limit that opens a store is 9M.
   `SET MEMORY_LIMIT 512M;` (or NONE) declares the limit the store remembers; it needs
   --write, and inside a transaction it still takes effect at once and ROLLBACK does not
-  undo it. `SHOW MEMORY_LIMIT;` reports this session's limit and what is in use.
+  undo it. The smallest limit a session takes is 1M plus its page cache plus what it
+  holds, which is 9M in a session started with no limit; a limit below it is refused.
+  To go below the page cache of a session started under a larger limit, start with the
+  smaller one, `--memory-limit 64M`, and then run `SET MEMORY_LIMIT 64M;`.
+  `SHOW MEMORY_LIMIT;` reports this session's limit and what is in use: with no limit,
+  `none` and 9437184 bytes used, the 1M and the 8M page cache.
 
 JSON (--json), one document a line:
   a SELECT or SHOW MEMORY_LIMIT   {"result":{"columns":[...],"rows":[{...}]}}
+                                  a vector is an array of numbers, each at the full
+                                  precision of the stored 32-bit value, so a number
+                                  may print with more digits than were typed
   any other statement             {"rows_affected":N}
   .tables                         {"tables":{"items":[...]}}
   .schema <table>                 {"schema":{"table":...,"ddl":...}}