diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml new file mode 100644 index 000000000..194bf3575 --- /dev/null +++ b/.github/workflows/check.yml @@ -0,0 +1,58 @@ +name: Check + +# The check of the packages under v2/. Every step is one cargo command that runs as written from +# the repository root. +on: + push: + branches: [dev, main] + pull_request: + branches: [dev, main] + +permissions: + contents: read + +concurrency: + group: check-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true + +env: + CARGO_TERM_COLOR: always + +jobs: + lint: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - run: cargo fmt --manifest-path v2/Cargo.toml --all --check + - run: cargo clippy --manifest-path v2/Cargo.toml --workspace --all-targets --locked -- -D warnings + - name: Shipped lines stay under 150,000 + run: test "$(find v2/contextdb/src v2/contextdb-cli/src \( -name '*.rs' -o -name '*.pest' \) ! -name tests.rs -exec cat {} + | wc -l)" -le 150000 + + test: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: taiki-e/install-action@cfdb7ed4ccc7c7dce37759eae89dc5674b469b39 # v2 + with: + tool: cargo-nextest@0.9.138 + - run: cargo nextest run --manifest-path v2/Cargo.toml --workspace --locked --no-tests=pass + - run: cargo test --manifest-path v2/Cargo.toml --workspace --locked --doc + - name: The storage library's own tests + run: cargo test --manifest-path v2/redb/Cargo.toml --locked --lib --tests + # A test is one the runner lists, or one script file. + - name: Tests stay under 1,000 + run: test "$(( $(cargo nextest list --manifest-path v2/Cargo.toml --workspace --locked -T oneline | wc -l) + $(find v2/contextdb/tests/sql -name '*.sql' | wc -l) ))" -le 1000 + + minimum-rust: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - run: rustup toolchain install 1.89 --profile minimal + - run: cargo +1.89 check --manifest-path v2/Cargo.toml --workspace --locked diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d337dade7..b7ee728d1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,8 +3,10 @@ name: CI on: push: branches: [dev, main] + paths-ignore: ['v2/**', '.github/workflows/check.yml'] pull_request: branches: [dev, main] + paths-ignore: ['v2/**', '.github/workflows/check.yml'] workflow_dispatch: permissions: diff --git a/v2/.config/nextest.toml b/v2/.config/nextest.toml new file mode 100644 index 000000000..ceaec6f4c --- /dev/null +++ b/v2/.config/nextest.toml @@ -0,0 +1,9 @@ +# A test that runs past its limit is stopped and fails by name. Nothing is retried. +[profile.default] +retries = 0 +slow-timeout = { period = "30s", terminate-after = 2 } + +# The million-row load is the one test that needs longer. +[[profile.default.overrides]] +filter = 'test(a_million_rows_load_and_scan_under_64_mib)' +slow-timeout = { period = "120s", terminate-after = 5 } diff --git a/v2/.gitignore b/v2/.gitignore new file mode 100644 index 000000000..ea8c4bf7f --- /dev/null +++ b/v2/.gitignore @@ -0,0 +1 @@ +/target diff --git a/v2/AGENTS.md b/v2/AGENTS.md new file mode 100644 index 000000000..b76f75fdd --- /dev/null +++ b/v2/AGENTS.md @@ -0,0 +1,112 @@ +# contextdb 2: working in `v2/` + +This workspace is contextdb 2: an embedded SQL database kept in one folder, the `contextdb` +program over it, and the storage library both are built on. Repository-wide policy (commit +messages, licence, contribution rules) is in the root [`AGENTS.md`](../AGENTS.md) and is not +repeated here; every command is in [`README.md`](README.md). + +## Packages and parts + +| Package | What it is | +|---|---| +| `contextdb/` | The library. Its public calls are the re-exports in `contextdb/src/lib.rs`; `contextdb/README.md` is also its documentation page, and the program in it is compiled as a documentation test. | +| `contextdb-cli/` | The `contextdb` program, a SQL shell. Every statement and command goes through the library's public calls; it never opens a store's files itself. | +| `redb/` | redb 4.1.0 with the changes listed at the top of `redb/README.md`. It keeps its own manifest and lock file and is not a workspace member. | + +The library is ten parts, one folder each under `contextdb/src/`: + +| Part | Does | Never | +|---|---|---| +| `types` | `Value`, the one `Error`, ids, table descriptions | holds behaviour: reads no store, checks no rule, decides no answer | +| `system` | the time of day, through `Clock`; the one place in the library that reads a clock | decides anything | +| `memory` | the budget and the reservations taken from it | allocates for another part: it counts what they are about to take and says no first | +| `sql` | grammar, parser, planner: text to a statement, a statement to a plan | touches storage or knows what a table holds | +| `store` | the folder and data file, how each record is keyed, views of one moment, the reader files (one in `readers/` per reader that is inside a statement, naming the commit it reads) and the locks | interprets SQL or a table's rules | +| `schema` | table declarations: checking, storing, printing them back | changes a row | +| `query` | answers a SELECT from one table; holds the one expression evaluator; counts the rows it reads | changes anything | +| `write` | applies INSERT, UPDATE and DELETE to a change set; checks types, NOT NULL, the primary key and UNIQUE rules; keeps index entries and row counts | returns a query's result to a caller | +| `database` | the handle a store is opened for writing with, its transactions, the one commit function | has a second way to commit | +| `read` | a session that reads a store without holding it for writing, from any process | changes anything a writer stored | + +## Where the parts meet + +A part reaches another only through what that part makes visible: the `pub(crate)` items of its +`mod.rs`, and for `sql` its `ast` and `plan` modules. The types handed across: + +| Type | Defined in | Who takes it | +|---|---|---| +| `Plan` (with `Effect`, `Expr`, `Statement`) | `sql` | `schema`, `write` and `query` each take a planned statement | +| `Read` (implemented by `View`, the store as of one commit, and by `ChangeSet`) | `store` | `query` and `write` read through it | +| `ChangeSet`, everything one transaction means to change, held in memory over a `View` | `store` | `schema` and `write` stage into it; `database` hands it to `Store::commit` | +| `Store`, `Key`, `Row`, `Cell`, `TableId`, `IndexId` | `store` | `database` and `read` open a `Store`; the rest name rows and indexes | +| `Budget`, `Reservation` | `memory` | everything whose memory grows with data holds a reservation | +| `Catalog`, `Table` | `schema` | `query`, `write`, `database` and `read` look tables up in it | +| `eval`, `matching` | `query` | `write` finds the rows of an UPDATE or DELETE with `query::matching` and works out values with `query::eval` | +| `QueryResult` | `query` | `database` and `read` build it and hand it to the caller | + +## Invariants and the tests that guard them + +Every path in this guide is written from `v2/`. + +| Invariant | Test | +|---|---| +| A commit is all or nothing under a kill at any storage call | `contextdb/tests/crash/main.rs`: `a_kill_at_any_storage_call_of_a_commit_leaves_all_or_nothing`, `a_kill_inside_a_commit_of_many_rows_keeps_rows_index_and_count_together` | +| What was committed is there after reopen | `contextdb/tests/store.rs`: `what_was_committed_is_there_after_close_and_reopen` | +| A commit that reached the disk is reported as committed | `contextdb/tests/crash/main.rs`: `a_commit_on_disk_is_reported_and_served_though_the_read_after_it_fails` | +| A reader in another process reads while the writer commits, and keeps answering when the writer is killed and restarted | `contextdb/tests/readers.rs`: `a_reader_in_another_process_reads_while_the_writer_commits`, `a_reader_keeps_answering_when_the_writer_is_killed_and_follows_the_restarted_writer` | +| A reader that reads the file header inside a commit gets the last whole commit | `redb/tests/contextdb_changes.rs`: `a_reader_that_catches_the_header_inside_a_commit_reads_the_last_whole_commit` | +| One writer; the second is refused at once | `contextdb/tests/store.rs`: `a_second_writer_is_refused_at_once_naming_the_holder_and_the_next_open_needs_no_retry` | +| The memory limit holds | `contextdb/tests/memory_ceiling.rs`: `a_million_rows_load_and_scan_under_64_mib` | +| A refused statement changes nothing | `contextdb/tests/memory.rs`: `a_statement_that_would_cross_the_limit_is_refused_by_name_and_committed_rows_stay`; `contextdb/tests/transactions.rs`: `a_refused_statement_changes_nothing_and_leaves_the_transaction_usable` | +| A damaged page is reported as damage | `contextdb/tests/crash/main.rs`: `a_damaged_page_is_refused_as_damage_by_name`; `redb/tests/contextdb_changes.rs`: `a_page_with_a_wrong_checksum_is_refused_when_read` | +| Counted work: a count reads no rows, a key lookup reads one | `contextdb/tests/counted_work.rs`: `count_of_a_whole_table_reads_no_rows`, `a_lookup_by_primary_key_reads_one_row` | + +## Where a change lives + +| Change | Folder | Test it needs | +|---|---|---| +| A new SQL statement or clause | `contextdb/src/sql/` (`grammar.pest`, `parser/`, the `Statement` type in `ast.rs`, `plan.rs`, `planner.rs`; take its first words out of the grammar's list of what a later version reads), then the part that runs it: `schema`, `write` or `query` | a script under `contextdb/tests/sql/` | +| A new refusal (error kind) | `Error` in `contextdb/src/types/mod.rs`; how the program prints it in `contextdb-cli/src/output.rs` | `contextdb/tests/refusals.rs` for its fields and that nothing changed; `contextdb/tests/sql/refusals.sql` for its name | +| What a statement returns | `contextdb/src/query/` for a SELECT, `contextdb/src/write/` for a row count | the expected output of the scripts under `contextdb/tests/sql/`, rewritten and the diff read | +| How a store is laid out on disk | `contextdb/src/store/` (keys, records, the data file, the folder's files) | `contextdb/tests/values.rs` and `contextdb/tests/store.rs` for reopen, `contextdb/tests/crash/` for a kill | +| The storage library | `redb/src/`; `redb/README.md` lists every change against upstream, so add yours there. The storage library decides which pages may be reused; the `store` part only tells it the oldest commit a reader still holds, and keeps the reader files and the lock | `redb/tests/contextdb_changes.rs` | +| A new shell option or dot command | `contextdb-cli/src/main.rs` for an option, `contextdb-cli/src/shell.rs` for a command | `contextdb-cli/tests/shell.rs` | +| A new public call | the part that owns it, re-exported in `contextdb/src/lib.rs`, described in `contextdb/README.md` | a Rust test under `contextdb/tests/` | + +## How tests are written here + +- A test goes through a public call of the library or through the built program, never an inner item. +- SQL behaviour is a script under `contextdb/tests/sql/` with its expected output; tests that kill a process and reopen the store live in `contextdb/tests/crash/`. +- A unit test of one pure function may sit beside it in a file named `tests.rs`. +- A test asserts states, values, error kinds and counts, never a duration, and waits only on something a process announces. +- Its data is fixed, and each test has its own store folder (`support::folder` in `contextdb/tests/support/mod.rs`). +- Nothing is retried (`.config/nextest.toml` sets `retries = 0`) and no test is ignored. + +Two lint settings hold this up: + +- `clippy.toml` refuses `std::thread::sleep`, starting a thread, `Instant::now`, `SystemTime::now` and `uuid::Uuid::new_v4`: the engine does not sleep, start a thread or draw a random number, and reads the time only through `Clock`. +- `Cargo.toml` denies `clippy::iter_over_hash_type` for both members: a hash map is walked in a different order in every process, so nothing is built by walking one. + +## Do not add + +- A second way to do one thing: there is one commit function, one expression evaluator, one `Error`. +- A test-only item in shipped code: no test-only name, feature or `pub` door; tests use what a user has. +- A check another part already makes: `write` checks a table's rules, `memory` says no, `store` only keeps. +- A lint switched off. Outside `redb/` there are four `allow` attributes and they stay the only ones: the machine clock in `contextdb/src/system/mod.rs` (the one place the time of day is read); the timer in `contextdb/examples/measure.rs` (measuring elapsed time is what that program is for); and two in `contextdb/tests/support/mod.rs`, one on its one thread (it moves a child's output lines to a channel so that a wait can give up) and `dead_code` for the whole file (each test program uses only some of its helpers). + +## Fast commands + +Every command is run from the repository's root folder, the one that contains `v2/`. The whole check, then one Rust test, then one SQL script: + +```sh +cargo fmt --manifest-path v2/Cargo.toml --all --check +cargo clippy --manifest-path v2/Cargo.toml --workspace --all-targets --locked -- -D warnings +cargo nextest run --manifest-path v2/Cargo.toml --workspace --locked +cargo test --manifest-path v2/Cargo.toml --workspace --locked --doc +cargo test --manifest-path v2/redb/Cargo.toml --locked --lib --tests +cargo +1.89 check --manifest-path v2/Cargo.toml --workspace --locked + +cargo nextest run --manifest-path v2/Cargo.toml -p contextdb a_fresh_store_is_one_folder + +SQL_SCRIPT=keys.sql cargo test --manifest-path v2/Cargo.toml -p contextdb --test sql +``` diff --git a/v2/Cargo.lock b/v2/Cargo.lock new file mode 100644 index 000000000..94b653711 --- /dev/null +++ b/v2/Cargo.lock @@ -0,0 +1,682 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "ar_archive_writer" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73cd58deff2140a0a8eae87e417bd01db68a33e148aa93d1e8cd837e55e312b6" +dependencies = [ + "object", +] + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "cc" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ff2a8126fff0f8c839080d619675a585e61720bb12d2baceae45769fd66f155" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "clipboard-win" +version = "5.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "052af7878466feccda12ba7a6a67d019d712f296809559b47c12c8cc196d3a9a" +dependencies = [ + "error-code", +] + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "contextdb" +version = "2.0.0" +dependencies = [ + "contextdb-redb", + "pest", + "pest_derive", + "serde_json", + "uuid", +] + +[[package]] +name = "contextdb-cli" +version = "2.0.0" +dependencies = [ + "clap", + "contextdb", + "rustyline", + "serde_json", +] + +[[package]] +name = "contextdb-redb" +version = "4.1.0" +dependencies = [ + "libc", +] + +[[package]] +name = "error-code" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5343afd4a8365a643ac588dab4cf234a190c7f6c88c9f6dd6ffe00837661b7" + +[[package]] +name = "find-msvc-tools" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4" + +[[package]] +name = "js-sys" +version = "0.3.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dc09d2154098bbf4ef54eab6b117a3c24cbbffe94046b16ee6d6d4510584d3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "log" +version = "0.4.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d15f86d12da44cd8d4ec3c73984537eb7f4cadcb9776a575a0a8a05ca11005c" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "nix" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "object" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "pest" +version = "2.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ec53fd1d256900a9a771bf2720d42cef6f8bc91b4e25065fbd291e3e7222576" +dependencies = [ + "memchr", + "pest_stack_guard", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adcf794fd301ddc20818a6b4de85ed507f4b45300245cec948982e69e12e3bd7" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3073afae5db334c25dea9fc3e230a4f191e920c1df080cb9c0b37c9ce1dbb41" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pest_meta" +version = "2.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bc7f614b3742541642af1685faaa910bf6333e3cd3729c2cccd118146a57327" +dependencies = [ + "pest", +] + +[[package]] +name = "pest_stack_guard" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ac6154fd4418b8f0b82532c59c26f303f9ca59a64c883e8ce3971fd82733439" +dependencies = [ + "psm", + "stacker", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "psm" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd034599e63b970727f70d79e02d62390a4a84f7c6b827c27c46d5ac3fa622" +dependencies = [ + "ar_archive_writer", + "cc", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "rustyline" +version = "17.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e902948a25149d50edc1a8e0141aad50f54e22ba83ff988cf8f7c9ef07f50564" +dependencies = [ + "bitflags", + "cfg-if", + "clipboard-win", + "libc", + "log", + "memchr", + "nix", + "unicode-segmentation", + "unicode-width", + "utf8parse", + "windows-sys 0.60.2", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "stacker" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.61.2", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "3.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae9ccbea3a7b0b09443036490c8f34a8b1b760fd345e5c0dae0aac718faea79" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca718471876992059f251e33aafc334346ada08042b7661dd1fe313a9bced557" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bed99139f7dac9b5cfe5f17be4d66dc7079a34dee1d74bcc77217119240e267c" +dependencies = [ + "getrandom", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0181c14ddd6c7b8b0b0132a5f1ea2f32be0d893cd3ea97b6e3b28886b8a3d1c" +dependencies = [ + "cfg-if", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e931a02a39abdbf66966fc35902b5ed58344750a70885bed4ebb3d8eaf28ed9" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ca4b54280e455bd831e33a6b8e734baf432fe63fbc35c42ac14ce67a043a53a" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b1f9c3798725016c6ee5f2f57869ec0831e8bbc0ab0310a86e787e97f41389e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/v2/Cargo.toml b/v2/Cargo.toml new file mode 100644 index 000000000..727d0de02 --- /dev/null +++ b/v2/Cargo.toml @@ -0,0 +1,12 @@ +# The library, the program and the storage library of contextdb 2. They have their own lock file, +# and nothing outside this folder depends on them. Every command that checks or tests them is run +# from the repository root with `--manifest-path v2/Cargo.toml`; see README.md beside this file. +[workspace] +resolver = "3" +members = ["contextdb", "contextdb-cli"] +# The storage library keeps the manifest and the lock file it was published with. +exclude = ["redb"] + +# A hash map is walked in a different order in every process, so nothing is built by walking one. +[workspace.lints.clippy] +iter_over_hash_type = "deny" diff --git a/v2/README.md b/v2/README.md new file mode 100644 index 000000000..d0029a4f2 --- /dev/null +++ b/v2/README.md @@ -0,0 +1,62 @@ +# contextdb 2 + +This folder is contextdb 2: the library (`contextdb/`), the `contextdb` program +(`contextdb-cli/`) and the storage library both are built on (`redb/`, which is redb 4.1.0 with +a few changes listed in its own README). It is a Cargo workspace of its own, with its own lock +file. The packages under `crates/` are contextdb 1; nothing there depends on anything here, and +nothing here depends on anything there. What the library does is in +[`contextdb/README.md`](contextdb/README.md). + +Every command below is run from the repository root. Pull requests go to the `dev` branch, where +`.github/workflows/check.yml` runs the same commands (the test run with `--no-tests=pass` +added): the three test commands on Linux and macOS, and the format check, the lint and the +oldest-Rust check on Linux only. + +## The whole check + +Tests run under [cargo-nextest](https://nexte.st), which stops a test that hangs and names it; +install it once with `cargo install cargo-nextest --version 0.9.138 --locked`. + +```sh +cargo fmt --manifest-path v2/Cargo.toml --all --check +cargo clippy --manifest-path v2/Cargo.toml --workspace --all-targets --locked -- -D warnings +cargo nextest run --manifest-path v2/Cargo.toml --workspace --locked +cargo test --manifest-path v2/Cargo.toml --workspace --locked --doc +cargo test --manifest-path v2/redb/Cargo.toml --locked --lib --tests +cargo +1.89 check --manifest-path v2/Cargo.toml --workspace --locked +``` + +The last line checks the oldest Rust the packages declare (`rustup toolchain install 1.89` +first). The workflow also fails when the library and the program together pass 150,000 lines or +the tests pass 1,000; the two lines that count them are in the workflow file. + +## One test + +```sh +cargo nextest run --manifest-path v2/Cargo.toml -p contextdb a_fresh_store_is_one_folder +``` + +The Rust tests of the library are in `contextdb/tests/`, the ones that kill a process in +`contextdb/tests/crash/`, and the program's in `contextdb-cli/tests/`. A test uses only what a +user of the library or the program has. + +## One SQL script + +A script in `contextdb/tests/sql/` is statements with the output expected of each; the format is +described at the top of `contextdb/tests/sql.rs`. One test runs them all. To run one file: + +```sh +SQL_SCRIPT=keys.sql cargo test --manifest-path v2/Cargo.toml -p contextdb --test sql +``` + +To rewrite the expected output of the scripts from what the engine now answers, then read the +diff before committing it: + +```sh +UPDATE_EXPECTED=1 cargo test --manifest-path v2/Cargo.toml -p contextdb --test sql +``` + +## Measurements + +`v2/numbers.sh` prints the numbers in the library's README for the commit it is run at. It only +measures: it never passes or fails, and the check does not run it. diff --git a/v2/clippy.toml b/v2/clippy.toml new file mode 100644 index 000000000..a43b201b2 --- /dev/null +++ b/v2/clippy.toml @@ -0,0 +1,10 @@ +# The one lint list of this workspace: calls the library never makes, and the clock read that +# only its `system` module makes. +disallowed-methods = [ + { path = "std::thread::sleep", reason = "wait on something a process announces; nothing in the engine sleeps" }, + { path = "std::thread::spawn", reason = "the engine starts no thread" }, + { path = "std::thread::Builder::spawn", reason = "the engine starts no thread" }, + { path = "std::time::Instant::now", reason = "read the time through contextdb::Clock" }, + { path = "std::time::SystemTime::now", reason = "read the time through contextdb::Clock" }, + { path = "uuid::Uuid::new_v4", reason = "the engine draws no random number; a test uses fixed ids" }, +] diff --git a/v2/contextdb-cli/Cargo.toml b/v2/contextdb-cli/Cargo.toml new file mode 100644 index 000000000..929a62571 --- /dev/null +++ b/v2/contextdb-cli/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "contextdb-cli" +version = "2.0.0" +edition = "2024" +rust-version = "1.89" +license = "Apache-2.0" +description = "The contextdb program: a SQL shell for a contextdb store" +repository = "https://github.com/context-graph-ai/contextdb" + +[[bin]] +name = "contextdb" +path = "src/main.rs" + +[dependencies] +clap = { version = "4", features = ["derive"] } +contextdb = { version = "2.0.0", path = "../contextdb" } +# Line editing and recall at a terminal, without its history-file and key-binding features. +# Version 19 needs a newer compiler than this workspace builds with. +rustyline = { version = "17", default-features = false } + +[dev-dependencies] +serde_json = "1" + +[lints] +workspace = true diff --git a/v2/contextdb-cli/src/input.rs b/v2/contextdb-cli/src/input.rs new file mode 100644 index 000000000..c21a2a28c --- /dev/null +++ b/v2/contextdb-cli/src/input.rs @@ -0,0 +1,51 @@ +//! Finds where a SQL statement ends in the text typed or piped so far. + +/// What the text gathered so far holds. +#[derive(Debug, PartialEq, Eq)] +pub enum Scan { + /// A whole statement: the text up to this byte offset, which is just past its `;`. + Ends(usize), + /// The start of a statement that the next line continues. + Open, + /// Nothing but spaces and closed comments: no statement has begun. + Blank, +} + +/// Looks for the first statement in `text`, as `psql` and `sqlite3` do: a statement ends at the +/// first `;` that is not inside a quoted string, a quoted name, a `--` comment or a `/* */` +/// comment, however many lines it spans. +/// +/// A `/*` that is not closed yet is [`Scan::Open`]: the next line continues the comment. +pub fn scan(text: &str) -> Scan { + let mut chars = text.char_indices().peekable(); + let mut blank = true; + while let Some((at, c)) = chars.next() { + let next = chars.peek().map(|(_, next)| *next); + match c { + ';' => return Scan::Ends(at + 1), + '-' if next == Some('-') => { + chars.by_ref().find(|(_, c)| *c == '\n'); + } + '/' if next == Some('*') => { + chars.next(); + let mut star = false; + let closed = chars.by_ref().any(|(_, c)| { + let closes = star && c == '/'; + star = c == '*'; + closes + }); + if !closed { + return Scan::Open; + } + } + '\'' | '"' => { + blank = false; + // A quote written twice inside its own quotes closes and opens again, which + // reads the same as staying inside. + chars.by_ref().find(|(_, close)| *close == c); + } + _ => blank &= c.is_whitespace(), + } + } + if blank { Scan::Blank } else { Scan::Open } +} diff --git a/v2/contextdb-cli/src/main.rs b/v2/contextdb-cli/src/main.rs new file mode 100644 index 000000000..82603005b --- /dev/null +++ b/v2/contextdb-cli/src/main.rs @@ -0,0 +1,198 @@ +//! The contextdb program: a SQL shell for a store. `server` and the other names of commands +//! that arrive with later versions are reserved and refused; they are not commands it has. +//! +//! It never does what the library cannot: every statement and every command goes through the +//! public API of `contextdb`, and nothing here opens the store's files itself. + +mod input; +mod output; +mod shell; +#[cfg(test)] +mod tests; + +use std::path::PathBuf; +use std::process::ExitCode; + +use clap::Parser; +use contextdb::{Database, DatabaseOpenOptions, ReadSession, ReadSessionOptions}; + +use output::Output; +use shell::Store; + +/// The exit status of a run in which the store could not be opened or closed, or a piped +/// statement or shell command was refused. +const EXIT_FAILED: u8 = 1; +/// The exit status of a command line that asks for something the program cannot do. Nothing +/// was opened. The argument parser exits with the same status for a line it cannot read. +const EXIT_USAGE: u8 = 2; + +/// Commands that arrive with later versions. The word is theirs already, so a folder with one +/// of these names is opened by writing it as a path. +const LATER_COMMANDS: [&str; 6] = [ + "server", "snapshot", "inspect", "diagnose", "reset", "purge", +]; + +/// What `--help` says after the options. +const AFTER_HELP: &str = r#"INPUT: + Statements are read from standard input, each ending at its `;`. On a terminal the + shell prompts and edits lines; piped input gets no prompt. + A refused statement in piped input is reported on the error stream, the statements + after it still run, and the run exits 1. + With --write the store is held for writing until the input ends: other processes can + read it meanwhile, and a second --write is refused until then. + A transaction that BEGIN opened and no COMMIT ended before the input ended is rolled + back. + +EXIT STATUS: + 0 every piped statement and command ran; a session at a terminal ends with 0 + whatever was refused in it + 1 a piped statement or command was refused, or the store could not be opened or + closed (a session at a terminal too) + 2 the command line was refused; nothing was opened + +MEMORY: + --memory-limit is this session's ceiling, in bytes or with K, M or G. Without it a + --write session has the limit the store remembers and a read session has none. + The store keeps 8M for its page cache and 1M is set aside before that, so the + smallest limit that opens a store is 9M. + `SET MEMORY_LIMIT 512M;` (or NONE) declares the limit the store remembers; it needs + --write, and inside a transaction it still takes effect at once and ROLLBACK does not + undo it. `SHOW MEMORY_LIMIT;` reports this session's limit and what is in use. + +JSON (--json), one document a line: + a SELECT or SHOW MEMORY_LIMIT {"result":{"columns":[...],"rows":[{...}]}} + any other statement {"rows_affected":N} + .tables {"tables":{"items":[...]}} + .schema {"schema":{"table":...,"ddl":...}} + .help {"help":[...]} + a refusal {"error":{"class":...,"message":...,"detail":{"kind":...}}} + on the error stream; "detail" is left out when the + refusal is the shell's own (an unknown command, a + reserved name) + A command line the program cannot read is refused in plain text. + +RESERVED NAMES: + server, snapshot, inspect, diagnose, reset and purge are kept for commands of later + versions. A folder with one of these names is written as a path: ./server + +THE STORE: + The folder holds `data`, `lock` and `readers`; together they are the store. The + README of the contextdb library says what each is. + +EXAMPLES: + contextdb ./notes + Read a store that is there, also while another process writes it. Never creates a + store and never changes its data. + + contextdb ./notes --write + Open the store for writing, creating it if it is missing. + + echo "SELECT * FROM notes LIMIT 5;" | contextdb ./notes --json + Pipe statements in and read one JSON document per statement. + + In the shell: .help + List the shell's commands."#; + +/// A SQL shell for a contextdb store. +#[derive(Parser)] +#[command( + name = "contextdb", + version, + override_usage = "contextdb [OPTIONS] ", + after_help = AFTER_HELP +)] +struct Args { + /// The store's folder. A folder named like a command is written as a path: ./server + path: PathBuf, + + /// Open the store for writing, creating it if it is missing. Without this the shell + /// only reads. + #[arg(long)] + write: bool, + + /// Machine output: one JSON document per statement or command on the output, and every + /// refusal as a JSON document on the error stream. + #[arg(long)] + json: bool, + + /// The most memory the session may use for data, in bytes or with K, M or G (64M, 4G). + #[arg(long, value_name = "SIZE", value_parser = parse_size)] + memory_limit: Option, +} + +fn main() -> ExitCode { + let args = Args::parse(); + let output = Output { json: args.json }; + + if let Some(command) = LATER_COMMANDS + .iter() + .find(|command| args.path.as_os_str() == **command) + { + output.refusal( + "usage", + &format!( + "`{command}` is a command, and this version does not have it yet; to open a \ + folder with that name write ./{command}" + ), + ); + return ExitCode::from(EXIT_USAGE); + } + + let store = match open(&args) { + Ok(store) => store, + Err(error) => { + output.error(&error); + return ExitCode::from(EXIT_FAILED); + } + }; + let mut succeeded = shell::run(&store, output); + if let Store::Write(db) = &store + && let Err(error) = db.close() + { + output.error(&error); + succeeded = false; + } + if succeeded { + ExitCode::SUCCESS + } else { + ExitCode::from(EXIT_FAILED) + } +} + +/// Opens the store the way the command line asked: for writing, or for reading only. +fn open(args: &Args) -> contextdb::Result { + if args.write { + let mut options = DatabaseOpenOptions::default(); + options.memory_limit = args.memory_limit; + Database::open_with_options(&args.path, options).map(Store::Write) + } else { + let mut options = ReadSessionOptions::default(); + options.memory_limit = args.memory_limit; + ReadSession::open_with_options(&args.path, options).map(Store::Read) + } +} + +/// Reads a size in bytes: digits, then nothing, K, M or G as powers of 1024. +fn parse_size(text: &str) -> Result { + let digits_end = text + .find(|c: char| !c.is_ascii_digit()) + .unwrap_or(text.len()); + let (digits, suffix) = text.split_at(digits_end); + let number: usize = digits.parse().map_err(|_| { + format!("'{text}' is not a size: write digits, then K, M or G, or nothing for bytes") + })?; + let unit: usize = match suffix.to_ascii_uppercase().as_str() { + "" => 1, + "K" => 1 << 10, + "M" => 1 << 20, + "G" => 1 << 30, + other => { + return Err(format!( + "unknown size suffix '{other}': use K, M or G, or no suffix for bytes" + )); + } + }; + number + .checked_mul(unit) + .ok_or_else(|| format!("'{text}' is too large")) +} diff --git a/v2/contextdb-cli/src/output.rs b/v2/contextdb-cli/src/output.rs new file mode 100644 index 000000000..35870b3c4 --- /dev/null +++ b/v2/contextdb-cli/src/output.rs @@ -0,0 +1,260 @@ +//! Everything the shell prints: results on the output, refusals on the error stream, each as +//! text for a person or, under `--json`, as one JSON document a line. + +use contextdb::serde_json::{Map, Value as Json, json}; +use contextdb::{Error, QueryResult, Value}; + +/// How this run prints. +#[derive(Clone, Copy)] +pub struct Output { + pub json: bool, +} + +impl Output { + /// The result of one statement: the rows of a SELECT, or how many rows a write changed. + pub fn result(self, result: &QueryResult) { + if result.columns().is_empty() { + self.rows_affected(result.rows_affected()); + } else if self.json { + // A row is written key by key so that its keys come in the order of `columns`. + let mut rows = String::new(); + for row in result.rows() { + if !rows.is_empty() { + rows.push(','); + } + rows.push('{'); + for (at, (column, value)) in result.columns().iter().zip(row).enumerate() { + if at > 0 { + rows.push(','); + } + rows.push_str(&format!("{}:{}", json!(column), cell(value))); + } + rows.push('}'); + } + println!( + "{{\"result\":{{\"columns\":{},\"rows\":[{rows}]}}}}", + json!(result.columns()) + ); + } else { + print!("{}", table(result.columns(), result.rows())); + } + } + + /// A statement that returns no rows succeeded and changed this many. + pub fn rows_affected(self, rows: u64) { + if self.json { + println!("{}", json!({ "rows_affected": rows })); + } else { + println!("ok (rows_affected={rows})"); + } + } + + /// What `.tables` answers: the table names, one a line. + pub fn tables(self, names: &[String]) { + if self.json { + println!("{}", json!({ "tables": { "items": names } })); + } else { + for name in names { + println!("{name}"); + } + } + } + + /// What `.schema` answers: the table's declaration as SQL. + pub fn schema(self, table: &str, declaration: &str) { + if self.json { + println!( + "{}", + json!({ "schema": { "table": table, "ddl": declaration } }) + ); + } else { + println!("{}", declaration.trim_end()); + } + } + + /// What `.help` answers. + pub fn help(self, lines: &[&str]) { + if self.json { + println!("{}", json!({ "help": lines })); + } else { + for line in lines { + println!("{line}"); + } + } + } + + /// A refusal from the library. A program reads `class` and `detail.kind`; the message is + /// for a person. + pub fn error(self, error: &Error) { + let class = match error { + Error::StoreMissing { .. } + | Error::DatabaseLocked { .. } + | Error::StoreCorrupted { .. } + | Error::StoreIo { .. } + | Error::MemoryBudgetExceeded { .. } => "io", + _ => "sql", + }; + let mut detail = Map::new(); + detail.insert("kind".to_owned(), json!(kind(error))); + if let Error::MemoryBudgetExceeded { + requested_bytes, + available_bytes, + budget_limit_bytes, + .. + } = error + { + detail.insert("requested_bytes".to_owned(), json!(requested_bytes)); + detail.insert("available_bytes".to_owned(), json!(available_bytes)); + detail.insert("budget_limit_bytes".to_owned(), json!(budget_limit_bytes)); + } + // Where an option of this program is the way on, the sentence is the program's own: + // the library's does not know the options. + let message = match error { + Error::StoreMissing { path } => format!( + "there is no store at this path: {}; --write creates it", + path.display() + ), + Error::DatabaseLocked { holder_pid, path } => format!( + "process {holder_pid} holds the store at {} for writing; wait for it to \ + finish, or open without --write to read", + path.display() + ), + Error::ReadOnlySession => { + "this statement writes and this session only reads; rerun with --write".to_owned() + } + Error::MemoryBudgetExceeded { .. } => format!( + "{}; raise it with --memory-limit", + error.to_string().trim_end_matches('.') + ), + other => other.to_string(), + }; + self.refused(class, &message, Some(detail)); + } + + /// A refusal of the shell's own: `class` is `usage` for a command line or a shell command + /// it cannot act on, `sql` for a statement. + pub fn refusal(self, class: &str, message: &str) { + self.refused(class, message, None); + } + + /// A line that starts like a shell command and is not one. For a person it is a plain line + /// with no `Error:` before it. + pub fn unknown_command(self, line: &str) { + let message = format!("Unknown command: {line}. Type \\? for help."); + if self.json { + self.refused("usage", &message, None); + } else { + eprintln!("{message}"); + } + } + + fn refused(self, class: &str, message: &str, detail: Option>) { + if self.json { + let mut error = Map::new(); + error.insert("class".to_owned(), json!(class)); + error.insert("message".to_owned(), json!(message)); + if let Some(detail) = detail { + error.insert("detail".to_owned(), Json::Object(detail)); + } + eprintln!("{}", json!({ "error": error })); + } else { + eprintln!("Error: {message}"); + } + } +} + +/// The word a program reads for a refusal's kind: the three the earlier program published +/// under another word keep it, and every other kind is its own name in lower snake case. +fn kind(error: &Error) -> String { + match error { + Error::StoreMissing { .. } => "store_not_found".to_owned(), + Error::DatabaseLocked { .. } => "held_by_writer".to_owned(), + Error::ReadOnlySession => "write_requires_flag".to_owned(), + other => { + let mut word = String::new(); + for c in format!("{other:?}") + .chars() + .take_while(char::is_ascii_alphanumeric) + { + if c.is_ascii_uppercase() && !word.is_empty() { + word.push('_'); + } + word.push(c.to_ascii_lowercase()); + } + word + } + } +} + +/// One value as JSON. +fn cell(value: &Value) -> Json { + match value { + Value::Null => Json::Null, + Value::Bool(value) => json!(value), + Value::Int64(value) | Value::Timestamp(value) => json!(value), + Value::Float64(value) => json!(value), + Value::Text(value) => json!(value), + Value::Uuid(value) => json!(value.to_string()), + Value::Json(value) => value.clone(), + Value::Vector(value) => json!(value), + Value::TxId(value) => json!(value.0), + } +} + +/// One value as a person reads it in a table. +fn text(value: &Value) -> String { + match value { + Value::Null => "NULL".to_owned(), + Value::Bool(value) => value.to_string(), + Value::Int64(value) | Value::Timestamp(value) => value.to_string(), + Value::Float64(value) => value.to_string(), + Value::Text(value) => value.clone(), + Value::Uuid(value) => value.to_string(), + Value::Json(value) => value.to_string(), + Value::Vector(value) => format!("{value:?}"), + Value::TxId(value) => value.0.to_string(), + } +} + +/// The rows as a bordered table with a `(N rows)` line under it, `(1 row)` for one. With no rows only that line +/// is printed. +pub fn table(columns: &[String], rows: &[Vec]) -> String { + let footer = match rows.len() { + 1 => "(1 row)\n".to_owned(), + count => format!("({count} rows)\n"), + }; + if rows.is_empty() { + return footer; + } + let rows: Vec> = rows + .iter() + .map(|row| row.iter().map(text).collect()) + .collect(); + let mut widths: Vec = columns.iter().map(|name| name.chars().count()).collect(); + for row in &rows { + for (width, cell) in widths.iter_mut().zip(row) { + *width = (*width).max(cell.chars().count()); + } + } + + let mut border = String::new(); + for width in &widths { + border.push('+'); + border.push_str(&"-".repeat(width + 2)); + } + border.push_str("+\n"); + let line = |cells: &[String]| { + let mut line = String::from("|"); + for (cell, width) in cells.iter().zip(&widths) { + line.push_str(&format!(" {cell:; Run a SQL statement. It may span lines and ends at its `;`.", + "BEGIN; ... COMMIT; Run statements as one transaction; ROLLBACK; undoes them, except SET MEMORY_LIMIT. Needs --write.", + ".tables \\dt List the tables.", + ".schema
\\d
Show a table's declaration as SQL.", + ".help \\? Show this list.", + ".quit \\q End the session.", +]; + +struct Shell<'a> { + store: &'a Store, + output: Output, + /// The transaction `BEGIN` opened, until `COMMIT` or `ROLLBACK`. + transaction: Option>, + /// Lines of a statement whose `;` has not come yet. + pending: String, + /// Whether anything was refused. + refused: bool, +} + +/// Runs the shell over the standard input until it ends or `.quit`. +/// +/// Returns whether the run succeeded. A piped run succeeds only when every statement and every +/// shell command did; a session at a terminal showed each refusal as it happened and always +/// succeeds. +pub fn run(store: &Store, output: Output) -> bool { + let mut shell = Shell { + store, + output, + transaction: None, + pending: String::new(), + refused: false, + }; + if std::io::stdin().is_terminal() { + shell.at_a_terminal(); + return true; + } + for line in std::io::stdin().lock().lines() { + let Ok(line) = line else { break }; + if shell.line(&line).is_break() { + return !shell.refused; + } + } + // Piped input ends its last statement without a `;`. + let rest = std::mem::take(&mut shell.pending); + shell.statement(&rest); + !shell.refused +} + +impl<'a> Shell<'a> { + /// Reads lines a person types, with line editing and recall of the session's earlier + /// lines. Nothing of the session is kept after it ends. + fn at_a_terminal(&mut self) { + let (banner, prompt) = match self.store { + Store::Read(_) => ( + "read-only session — pass --write to mutate", + "contextdb(ro)> ", + ), + Store::Write(_) => ("write session — mutations are enabled", "contextdb> "), + }; + let mut editor = match DefaultEditor::new() { + Ok(editor) => editor, + Err(error) => { + self.output + .refusal("io", &format!("the terminal cannot be read: {error}")); + return; + } + }; + eprintln!("{banner}"); + loop { + let open = !self.pending.is_empty(); + match editor.readline(if open { " ...> " } else { prompt }) { + Ok(line) => { + if line.trim().is_empty() && !open { + continue; + } + // A line that cannot be kept for recall is still run. + let _ = editor.add_history_entry(line.trim()); + if self.line(&line).is_break() { + return; + } + } + // Ctrl-C drops what was typed and the session goes on. + Err(ReadlineError::Interrupted) => self.pending.clear(), + // Ctrl-D, or a terminal that went away. + Err(_) => break, + } + } + if !self.pending.is_empty() { + eprintln!("Discarding incomplete statement (no closing `;`)."); + } + } + + /// Takes one line of input. Breaks when the line ends the session. + fn line(&mut self, line: &str) -> ControlFlow<()> { + // A shell command is one line, and only where no statement is open: inside a statement + // a line that starts with a dot is that statement's text. + let command = line.trim(); + if self.pending.is_empty() && command.starts_with(['.', '\\']) { + return self.command(command); + } + self.pending.push_str(line); + self.pending.push('\n'); + loop { + match scan(&self.pending) { + Scan::Ends(end) => { + let statement: String = self.pending.drain(..end).collect(); + self.statement(&statement); + } + Scan::Open => break, + Scan::Blank => { + self.pending.clear(); + break; + } + } + } + ControlFlow::Continue(()) + } + + /// Runs one shell command. + fn command(&mut self, line: &str) -> ControlFlow<()> { + let mut words = line.split_whitespace(); + let answered = match (words.next(), words.next(), words.next()) { + (Some(".tables" | "\\dt"), None, _) => match self.store { + Store::Read(reader) => reader.table_names(), + Store::Write(db) => Ok(db.table_names()), + } + .map(|names| self.output.tables(&names)), + (Some(".schema" | "\\d"), Some(table), None) => match self.store { + Store::Read(reader) => reader.declaration(table), + Store::Write(db) => db.declaration(table), + } + .map(|declaration| self.output.schema(table, &declaration)), + (Some(".help" | "\\?"), None, _) => { + self.output.help(&HELP); + Ok(()) + } + (Some(".quit" | ".exit" | "\\q"), None, _) => return ControlFlow::Break(()), + _ => { + self.refused = true; + self.output.unknown_command(line); + Ok(()) + } + }; + if let Err(error) = answered { + self.refused = true; + self.output.error(&error); + } + ControlFlow::Continue(()) + } + + /// Runs one statement and prints its result or its refusal. + fn statement(&mut self, text: &str) { + let sql = text.trim(); + let sql = sql.strip_suffix(';').unwrap_or(sql).trim_end(); + if sql.is_empty() { + return; + } + let ran = if sql.eq_ignore_ascii_case("BEGIN") { + self.begin() + } else if sql.eq_ignore_ascii_case("COMMIT") { + self.end(Transaction::commit) + } else if sql.eq_ignore_ascii_case("ROLLBACK") { + self.end(|transaction| { + transaction.rollback(); + Ok(()) + }) + } else { + self.execute(sql).map(|result| self.output.result(&result)) + }; + if let Err(error) = ran { + self.refused = true; + self.output.error(&error); + } + } + + fn execute(&self, sql: &str) -> Result { + let params = HashMap::new(); + match (&self.transaction, self.store) { + (Some(transaction), _) => transaction.execute(sql, ¶ms), + (None, Store::Write(db)) => db.execute(sql, ¶ms), + (None, Store::Read(reader)) => reader.execute(sql, ¶ms), + } + } + + /// `BEGIN`. With a transaction open already it changes nothing and answers the same. + fn begin(&mut self) -> Result<()> { + let Store::Write(db) = self.store else { + return Err(Error::ReadOnlySession); + }; + if self.transaction.is_none() { + self.transaction = Some(db.transaction()?); + } + self.output.rows_affected(0); + Ok(()) + } + + /// `COMMIT` or `ROLLBACK`. With no transaction open it changes nothing and answers the + /// same. + fn end(&mut self, how: impl FnOnce(Transaction<'a>) -> Result<()>) -> Result<()> { + if let Store::Read(_) = self.store { + return Err(Error::ReadOnlySession); + } + if let Some(transaction) = self.transaction.take() { + how(transaction)?; + } + self.output.rows_affected(0); + Ok(()) + } +} diff --git a/v2/contextdb-cli/src/tests.rs b/v2/contextdb-cli/src/tests.rs new file mode 100644 index 000000000..3a54262b8 --- /dev/null +++ b/v2/contextdb-cli/src/tests.rs @@ -0,0 +1,37 @@ +use contextdb::Value; + +use crate::input::{Scan, scan}; +use crate::output::table; + +#[test] +fn a_statement_ends_at_the_first_semicolon_outside_quotes_and_comments() { + assert_eq!(scan("SELECT 1; SELECT 2;"), Scan::Ends(9)); + assert_eq!(scan("SELECT ';' -- ;\n, \"a;b\" /* ; */ ;"), Scan::Ends(33)); + assert_eq!(scan("SELECT 'it''s; here'"), Scan::Open); + assert_eq!(scan("SELECT 1\n"), Scan::Open); + assert_eq!(scan("/* not closed ;"), Scan::Open); + assert_eq!(scan(" -- done\n/* and */\n"), Scan::Blank); + assert_eq!(scan(""), Scan::Blank); +} + +#[test] +fn a_table_is_as_wide_as_its_widest_cell() { + let columns = ["id".to_owned(), "body".to_owned()]; + let rows = [ + vec![Value::Int64(1), Value::Text("première".to_owned())], + vec![Value::Int64(20), Value::Null], + ]; + assert_eq!( + table(&columns, &rows), + "\ ++----+----------+ +| id | body | ++----+----------+ +| 1 | première | +| 20 | NULL | ++----+----------+ +(2 rows) +" + ); + assert_eq!(table(&columns, &[]), "(0 rows)\n"); +} diff --git a/v2/contextdb-cli/tests/shell.rs b/v2/contextdb-cli/tests/shell.rs new file mode 100644 index 000000000..6e9b49d14 --- /dev/null +++ b/v2/contextdb-cli/tests/shell.rs @@ -0,0 +1,732 @@ +//! Runs the built `contextdb` program the way a script does: statements piped to its standard +//! input, and its output, its error stream and its exit status read back. +//! +//! Each test has its own folder under the build's temporary folder, left behind for inspection. + +use std::io::{ErrorKind, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +use contextdb::{Database, params}; +use serde_json::{Value as Json, json}; + +/// What one run of the program gave back. +struct Run { + stdout: String, + stderr: String, + code: Option, +} + +impl Run { + /// The lines of the error stream that report a refused statement to a person. + fn errors(&self) -> Vec<&str> { + self.stderr + .lines() + .filter(|line| line.starts_with("Error:")) + .collect() + } +} + +/// An empty folder of this test's own. +fn folder(test: &str) -> PathBuf { + let folder = Path::new(env!("CARGO_TARGET_TMPDIR")).join(format!("shell-{test}")); + if folder.exists() { + std::fs::remove_dir_all(&folder).unwrap(); + } + std::fs::create_dir_all(&folder).unwrap(); + folder +} + +/// Runs `contextdb` in `dir` with these arguments and this text piped to its standard input. +/// +/// The input is written before anything is read back. Every input here is far smaller than a +/// pipe holds, so the write cannot wait on the program. +fn shell(dir: &Path, args: &[&str], input: &str) -> Run { + let mut child = Command::new(env!("CARGO_BIN_EXE_contextdb")) + .args(args) + .current_dir(dir) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + // A program that refuses its command line exits without reading its input. + match child.stdin.take().unwrap().write_all(input.as_bytes()) { + Err(error) if error.kind() != ErrorKind::BrokenPipe => panic!("{error}"), + _ => {} + } + let output = child.wait_with_output().unwrap(); + let run = Run { + stdout: String::from_utf8(output.stdout).unwrap(), + stderr: String::from_utf8(output.stderr).unwrap(), + code: output.status.code(), + }; + assert!( + !run.stderr.contains("panicked at"), + "the program panicked:\n{}", + run.stderr + ); + run +} + +/// A path as a command-line argument. +fn arg(path: &Path) -> &str { + path.to_str().unwrap() +} + +/// Every line of a stream as a JSON document. +fn documents(stream: &str) -> Vec { + stream + .lines() + .map(|line| { + serde_json::from_str(line).unwrap_or_else(|_| panic!("not a JSON document: {line}")) + }) + .collect() +} + +/// The names in a folder, sorted. +fn listing(folder: &Path) -> Vec { + let mut names: Vec = std::fs::read_dir(folder) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + names.sort(); + names +} + +const CREATE_NOTES: &str = "CREATE TABLE notes (id INTEGER PRIMARY KEY, body TEXT);\n"; +const SELECT_NOTES: &str = "SELECT id, body FROM notes ORDER BY id;\n"; + +const ONE_NOTE: &str = "\ ++----+-------+ +| id | body | ++----+-------+ +| 1 | first | ++----+-------+ +(1 row) +"; + +const TWO_NOTES: &str = "\ ++----+--------+ +| id | body | ++----+--------+ +| 1 | first | +| 2 | second | ++----+--------+ +(2 rows) +"; + +#[test] +fn reading_a_missing_store_is_refused_and_creates_nothing() { + let dir = folder("reading-a-missing-store"); + let store = dir.join("store"); + + let run = shell(&dir, &[arg(&store)], SELECT_NOTES); + assert_eq!(run.code, Some(1), "{}", run.stderr); + assert_eq!(run.stdout, ""); + assert!( + run.stderr.contains("there is no store at this path") + && run.stderr.contains("--write creates it"), + "{}", + run.stderr + ); + assert_eq!( + listing(&dir), + Vec::::new(), + "a refused read creates nothing" + ); + + // The same refusal for a program that reads it: one document on the error stream. + let run = shell(&dir, &[arg(&store), "--json"], SELECT_NOTES); + assert_eq!(run.code, Some(1), "{}", run.stderr); + assert_eq!(run.stdout, ""); + let refusals = documents(&run.stderr); + assert_eq!(refusals.len(), 1, "{}", run.stderr); + assert_eq!(refusals[0]["error"]["detail"]["kind"], "store_not_found"); + assert_eq!( + listing(&dir), + Vec::::new(), + "a refused read creates nothing" + ); +} + +#[test] +fn write_creates_a_missing_store() { + let dir = folder("write-creates-a-missing-store"); + let store = dir.join("store"); + + let insert = "INSERT INTO notes (id, body) VALUES (1, 'first');\n"; + let run = shell( + &dir, + &[arg(&store), "--write"], + &format!("{CREATE_NOTES}{insert}"), + ); + assert_eq!(run.code, Some(0), "{}", run.stderr); + assert_eq!(run.errors(), Vec::<&str>::new()); + assert!( + store.is_dir(), + "the store is one folder at the path that was given" + ); + assert_eq!( + listing(&dir), + ["store"], + "nothing is written beside the store's folder" + ); + + // It was a whole read-write session: the next shell reads what it wrote. + let run = shell(&dir, &[arg(&store)], SELECT_NOTES); + assert_eq!(run.code, Some(0), "{}", run.stderr); + assert_eq!(run.stdout, ONE_NOTE); + + // On a store that is there, --write opens it and keeps what it holds. + let insert = "INSERT INTO notes (id, body) VALUES (2, 'second');\n"; + let run = shell( + &dir, + &[arg(&store), "--write"], + &format!("{insert}{SELECT_NOTES}"), + ); + assert_eq!(run.code, Some(0), "{}", run.stderr); + assert!(run.stdout.contains(TWO_NOTES), "{}", run.stdout); +} + +#[test] +fn a_removed_option_is_refused_by_name_with_exit_2_before_the_store_opens() { + let dir = folder("a-removed-option-is-refused"); + let store = dir.join("store"); + let removed = [ + ["--read-owner-connect-ms", "1000"], + ["--read-owner-routing-retry-ms", "1000"], + ["--read-owner-response-ms", "11000"], + ["--owner-read-concurrency", "4"], + ["--owner-read-request-ms", "10000"], + ["--owner-read-result-rows", "500"], + ]; + for [option, value] in removed { + // With --write a store would be created here if the program got as far as opening one. + for extra in [&[][..], &["--json"][..]] { + let mut args = vec![arg(&store), "--write", option, value]; + args.extend_from_slice(extra); + let run = shell(&dir, &args, CREATE_NOTES); + assert_eq!(run.code, Some(2), "{option}: {}", run.stderr); + assert!( + run.stderr.contains(option), + "{option} is not named in: {}", + run.stderr + ); + assert_eq!(run.stdout, "", "{option}"); + // The command line is refused in the argument parser's own words, with or + // without --json. + assert!( + !run.stderr.trim_start().starts_with('{'), + "{option}: {}", + run.stderr + ); + assert_eq!( + listing(&dir), + Vec::::new(), + "{option}: the store was opened" + ); + } + } + + // An option the program never had, and a command line with no store on it, are refused the + // same way: exit 2 in the argument parser's words, with nothing opened or created. + let unknown = shell( + &dir, + &[arg(&store), "--write", "--frobnicate"], + CREATE_NOTES, + ); + assert_eq!(unknown.code, Some(2), "{}", unknown.stderr); + assert!( + unknown.stderr.contains("--frobnicate"), + "{}", + unknown.stderr + ); + assert_eq!(unknown.stdout, ""); + for no_path in [&[][..], &["--write"][..], &["--write", "--json"][..]] { + let run = shell(&dir, no_path, CREATE_NOTES); + assert_eq!(run.code, Some(2), "{no_path:?}: {}", run.stderr); + assert_eq!(run.stdout, "", "{no_path:?}"); + assert!( + !run.stderr.trim_start().starts_with('{'), + "{no_path:?}: {}", + run.stderr + ); + } + assert_eq!( + listing(&dir), + Vec::::new(), + "a refused command line created something" + ); +} + +#[test] +fn a_folder_named_like_a_command_opens_when_written_as_a_path() { + let dir = folder("a-folder-named-like-a-command"); + let made = shell( + &dir, + &["./server", "--write"], + "CREATE TABLE notes (id INTEGER PRIMARY KEY);\nINSERT INTO notes (id) VALUES (7);\n", + ); + assert_eq!(made.code, Some(0), "{}", made.stderr); + assert!(dir.join("server").is_dir()); + + // Written as a path, the folder is the store. + let as_path = shell(&dir, &["./server"], "SELECT id FROM notes;\n"); + assert_eq!(as_path.code, Some(0), "{}", as_path.stderr); + assert_eq!( + as_path.stdout, + "+----+\n| id |\n+----+\n| 7 |\n+----+\n(1 row)\n" + ); + + // Written bare, the word is the command, which this version refuses by name; the store + // beside it is not opened as a shell. + let as_command = shell(&dir, &["server"], "SELECT id FROM notes;\n"); + assert_ne!(as_command.code, Some(0)); + assert!( + as_command.stderr.contains("server"), + "{}", + as_command.stderr + ); + assert!(!as_command.stdout.contains("| 7"), "{}", as_command.stdout); +} + +#[test] +fn piped_statements_all_run_and_a_refused_one_fails_the_run() { + let dir = folder("piped-statements-all-run"); + let store = dir.join("store"); + let input = "\ +CREATE TABLE notes (id INTEGER PRIMARY KEY, body TEXT); +.tables +INSERT INTO notes (id, body) VALUES (1, 'first'); +INSERT INTO notes (id, body) VALUES (1, 'again'); +SELECT id, + body + FROM notes + ORDER BY id; +SELECT nope FROM notes; +INSERT INTO notes (id, body) VALUES (2, 'second'); +SELECT COUNT(*) AS n FROM notes; +"; + let run = shell(&dir, &[arg(&store), "--write"], input); + + // A statement may span lines, a command is one line, and every result is on the output. + let first = run + .stdout + .find(ONE_NOTE) + .expect("the statement written over four lines ran"); + assert!( + run.stdout.lines().any(|line| line == "notes"), + "{}", + run.stdout + ); + + // Each refused statement is reported on the error stream, never as a result and never as + // a success: three statements changed something and three say so. + assert_eq!(run.errors().len(), 2, "{}", run.stderr); + assert!(!run.stdout.contains("Error"), "{}", run.stdout); + let ok: Vec<&str> = run + .stdout + .lines() + .filter(|line| line.starts_with("ok (")) + .collect(); + assert_eq!( + ok, + [ + "ok (rows_affected=0)", + "ok (rows_affected=1)", + "ok (rows_affected=1)" + ] + ); + + // The statements after a refused one still ran. + let count = "+---+\n| n |\n+---+\n| 2 |\n+---+\n(1 row)\n"; + let last = run + .stdout + .find(count) + .expect("the statements after a refused one ran"); + assert!(first < last); + + // No prompt and no banner on either stream, and the run failed because a statement did. + for stream in [&run.stdout, &run.stderr] { + assert!( + !stream.contains("contextdb>") && !stream.contains("contextdb(ro)>"), + "{stream}" + ); + assert!(!stream.contains("session"), "{stream}"); + } + assert_eq!(run.code, Some(1)); + + // A run in which every statement succeeded exits 0 and prints its results and nothing else. + let run = shell(&dir, &[arg(&store)], SELECT_NOTES); + assert_eq!(run.code, Some(0), "{}", run.stderr); + assert_eq!(run.stdout, TWO_NOTES); +} + +#[test] +fn json_prints_one_document_per_statement_with_columns_in_declared_order() { + let dir = folder("json-prints-one-document-per-statement"); + let store = dir.join("store"); + let input = "\ +CREATE TABLE notes (id INTEGER PRIMARY KEY, zeta TEXT, alpha TEXT); +INSERT INTO notes (id, zeta, alpha) VALUES (1, 'z1', 'a1'), (2, 'z2', NULL); +SELECT zeta, id, alpha FROM notes ORDER BY id; +SELECT * FROM notes WHERE id = 404; +SELECT nope FROM notes; +.tables +"; + let run = shell(&dir, &[arg(&store), "--write", "--json"], input); + + // One document per statement that gave a result, in the order they were given. `columns` + // is the order the statement or the table declared, which the keys of a row cannot carry. + let results = documents(&run.stdout); + assert_eq!(results.len(), 5, "{}", run.stdout); + assert_eq!(results[0], json!({"rows_affected": 0})); + assert_eq!(results[1], json!({"rows_affected": 2})); + assert_eq!( + results[2], + json!({"result": { + "columns": ["zeta", "id", "alpha"], + "rows": [ + {"zeta": "z1", "id": 1, "alpha": "a1"}, + {"zeta": "z2", "id": 2, "alpha": null}, + ], + }}) + ); + assert_eq!( + results[3], + json!({"result": {"columns": ["id", "zeta", "alpha"], "rows": []}}) + ); + assert_eq!(results[4]["tables"]["items"], json!(["notes"])); + + // The keys of a row come in the order of `columns` too, so a consumer that reads the row + // as written sees zeta, id, alpha and not the alphabet. + let line = run.stdout.lines().nth(2).unwrap(); + let rows = &line[line.find("\"rows\"").expect("the document has rows")..]; + let at = |key: &str| { + rows.find(&format!("\"{key}\"")) + .expect("the row has the key") + }; + assert!(at("zeta") < at("id") && at("id") < at("alpha"), "{rows}"); + + // The refused statement is a document too, on the error stream. + let refusals: Vec = documents(&run.stderr) + .into_iter() + .filter(|document| document.get("error").is_some()) + .collect(); + assert_eq!(refusals.len(), 1, "{}", run.stderr); + assert_eq!(refusals[0]["error"]["class"], "sql"); + assert!(refusals[0]["error"]["message"].is_string()); + assert_eq!(run.code, Some(1)); +} + +#[test] +fn a_read_shell_refuses_a_write_by_name() { + let dir = folder("a-read-shell-refuses-a-write"); + let store = dir.join("store"); + let insert = "INSERT INTO notes (id, body) VALUES (1, 'first');\n"; + let made = shell( + &dir, + &[arg(&store), "--write"], + &format!("{CREATE_NOTES}{insert}"), + ); + assert_eq!(made.code, Some(0), "{}", made.stderr); + + let writes = "\ +INSERT INTO notes (id, body) VALUES (2, 'second'); +UPDATE notes SET body = 'changed'; +DELETE FROM notes; +CREATE TABLE other (id INTEGER PRIMARY KEY); +BEGIN; +"; + // Each of the five is refused, telling the person what to add, and the read after them + // answers with the row as it was. + let run = shell(&dir, &[arg(&store)], &format!("{writes}{SELECT_NOTES}")); + assert_eq!(run.code, Some(1)); + assert_eq!(run.errors().len(), 5, "{}", run.stderr); + assert!( + run.errors().iter().all(|line| line.contains("--write")), + "{}", + run.stderr + ); + assert!(run.stdout.contains(ONE_NOTE), "{}", run.stdout); + assert!(!run.stdout.contains("ok ("), "{}", run.stdout); + + // A program reads the refusal's kind. + let run = shell(&dir, &[arg(&store), "--json"], writes); + assert_eq!(run.code, Some(1)); + let refusals: Vec = documents(&run.stderr) + .into_iter() + .filter(|document| document.get("error").is_some()) + .collect(); + assert_eq!(refusals.len(), 5, "{}", run.stderr); + for refusal in &refusals { + assert_eq!( + refusal["error"]["detail"]["kind"], "write_requires_flag", + "{refusal}" + ); + } + + // Nothing was written: one row, one table. + let run = shell( + &dir, + &[arg(&store), "--json"], + &format!("{SELECT_NOTES}.tables\n"), + ); + assert_eq!(run.code, Some(0), "{}", run.stderr); + let results = documents(&run.stdout); + assert_eq!( + results[0]["result"]["rows"], + json!([{"id": 1, "body": "first"}]) + ); + assert_eq!(results[1]["tables"]["items"], json!(["notes"])); +} + +#[test] +fn a_read_shell_answers_while_a_writer_holds_the_store() { + let dir = folder("a-read-shell-answers-while-a-writer-holds"); + let store = dir.join("store"); + + // This test is the writer, and holds the store until the end. + let writer = Database::open(&store).unwrap(); + writer + .execute( + "CREATE TABLE notes (id INTEGER PRIMARY KEY, body TEXT)", + ¶ms! {}, + ) + .unwrap(); + writer + .execute( + "INSERT INTO notes (id, body) VALUES (1, 'first')", + ¶ms! {}, + ) + .unwrap(); + + // The store really is held: a second writer is refused, naming this process. + let second = shell(&dir, &[arg(&store), "--write"], SELECT_NOTES); + assert_eq!(second.code, Some(1), "{}", second.stderr); + assert!( + second.stderr.contains(&std::process::id().to_string()), + "{}", + second.stderr + ); + + // A read shell needs nothing from the writer and answers. + let run = shell(&dir, &[arg(&store)], SELECT_NOTES); + assert_eq!(run.code, Some(0), "{}", run.stderr); + assert_eq!(run.stdout, ONE_NOTE); + + // The writer was not held up by it, and the next read shell reads its next commit. + writer + .execute( + "INSERT INTO notes (id, body) VALUES (2, 'second')", + ¶ms! {}, + ) + .unwrap(); + let run = shell(&dir, &[arg(&store)], SELECT_NOTES); + assert_eq!(run.code, Some(0), "{}", run.stderr); + assert_eq!(run.stdout, TWO_NOTES); + + writer.close().unwrap(); +} + +#[test] +fn begin_commit_and_rollback_run_a_transaction() { + let dir = folder("begin-commit-and-rollback"); + let store = dir.join("store"); + let input = "\ +CREATE TABLE notes (id INTEGER PRIMARY KEY, body TEXT); +BEGIN; +INSERT INTO notes (id, body) VALUES (1, 'first'); +INSERT INTO notes (id, body) VALUES (2, 'second'); +COMMIT; +BEGIN; +INSERT INTO notes (id, body) VALUES (3, 'third'); +DELETE FROM notes WHERE id = 1; +SELECT id FROM notes ORDER BY id; +ROLLBACK; +SELECT id, body FROM notes ORDER BY id; +"; + let run = shell(&dir, &[arg(&store), "--write"], input); + assert_eq!(run.code, Some(0), "{}", run.stderr); + assert_eq!(run.errors(), Vec::<&str>::new()); + + // Inside the second transaction its own changes are read; after ROLLBACK they are gone and + // what the first one committed is there. + let inside = "+----+\n| id |\n+----+\n| 2 |\n| 3 |\n+----+\n(2 rows)\n"; + let seen_inside = run + .stdout + .find(inside) + .expect("a transaction reads its own changes"); + let seen_after = run + .stdout + .find(TWO_NOTES) + .expect("ROLLBACK left what COMMIT stored"); + assert!(seen_inside < seen_after); + + // The committed transaction reached the store, and the rolled-back one never did. + let run = shell(&dir, &[arg(&store)], SELECT_NOTES); + assert_eq!(run.code, Some(0), "{}", run.stderr); + assert_eq!(run.stdout, TWO_NOTES); +} + +#[test] +fn tables_and_schema_print_what_was_declared() { + let dir = folder("tables-and-schema"); + let store = dir.join("store"); + // What `.schema notes` prints: the declaration as SQL. A column with no rule prints its name + // and type alone, and nothing the author did not write is printed: no index behind the key + // or the unique rules, no table option. + let notes = "\ +CREATE TABLE notes ( + id INTEGER PRIMARY KEY, + title TEXT NOT NULL, + body TEXT, + stars INTEGER DEFAULT 3, + slug TEXT UNIQUE, + UNIQUE (title, stars) +); +"; + let authors = "CREATE TABLE authors (id UUID PRIMARY KEY, name TEXT);\n"; + let made = shell( + &dir, + &[arg(&store), "--write"], + &format!("{notes}{authors}"), + ); + assert_eq!(made.code, Some(0), "{}", made.stderr); + + // `.tables` lists the names in order, one a line. + let tables = shell(&dir, &[arg(&store)], ".tables\n"); + assert_eq!(tables.code, Some(0), "{}", tables.stderr); + let names: Vec<&str> = tables + .stdout + .lines() + .filter(|line| ["authors", "notes"].contains(line)) + .collect(); + assert_eq!(names, ["authors", "notes"], "{}", tables.stdout); + + let schema = shell(&dir, &[arg(&store)], ".schema notes\n"); + assert_eq!(schema.code, Some(0), "{}", schema.stderr); + assert_eq!(schema.stdout, notes); + + // Each command has a short spelling that prints the same. + assert_eq!(shell(&dir, &[arg(&store)], "\\dt\n").stdout, tables.stdout); + assert_eq!( + shell(&dir, &[arg(&store)], "\\d notes\n").stdout, + schema.stdout + ); + + // What `.schema` printed declares the same table again in another store. + let copy = dir.join("copy"); + let made = shell(&dir, &[arg(©), "--write"], &schema.stdout); + assert_eq!(made.code, Some(0), "{}", made.stderr); + assert_eq!(shell(&dir, &[arg(©)], ".schema notes\n").stdout, notes); +} + +#[test] +fn the_memory_limit_option_is_the_sessions_ceiling() { + let dir = folder("the-memory-limit-option"); + let store = dir.join("store"); + + // The option is the ceiling the session reports, in bytes, with K, M and G as powers of + // 1024; without it there is none. + let limit = |args: &[&str]| { + let run = shell(&dir, args, "SHOW MEMORY_LIMIT;\n"); + assert_eq!(run.code, Some(0), "{}", run.stderr); + documents(&run.stdout)[0]["result"]["rows"][0]["limit"].clone() + }; + assert_eq!(limit(&[arg(&store), "--write", "--json"]), json!("none")); + for (size, bytes) in [ + ("65536K", 67_108_864_u64), + ("64M", 67_108_864), + ("1G", 1_073_741_824), + ] { + let args = [arg(&store), "--write", "--json", "--memory-limit", size]; + assert_eq!(limit(&args), json!(bytes), "--memory-limit {size}"); + } + + // A read shell answers the same statement without --write, with all five columns and with + // the ceiling of its own session. + let columns = [ + "limit", + "used", + "available", + "startup_ceiling", + "ceiling_refresh_failures", + ]; + let run = shell(&dir, &[arg(&store), "--json"], "SHOW MEMORY_LIMIT;\n"); + assert_eq!(run.code, Some(0), "{}", run.stderr); + assert_eq!( + documents(&run.stdout)[0]["result"]["columns"], + json!(columns) + ); + let args = [arg(&store), "--json", "--memory-limit", "64M"]; + assert_eq!( + limit(&args), + json!(67_108_864_u64), + "a read shell's --memory-limit" + ); + + // A size that cannot be read is a mistake in the command line: exit 2, nothing opened. + let unread = dir.join("unread"); + let run = shell( + &dir, + &[arg(&unread), "--write", "--memory-limit", "12Q"], + CREATE_NOTES, + ); + assert_eq!(run.code, Some(2), "{}", run.stderr); + assert!(!unread.exists()); + + // 64 rows of one mebibyte each, written with no ceiling. + let big = dir.join("big"); + let writer = Database::open(&big).unwrap(); + writer + .execute( + "CREATE TABLE blobs (id INTEGER PRIMARY KEY, body TEXT)", + ¶ms! {}, + ) + .unwrap(); + for id in 1..=64_i64 { + let body = format!("{id:08}").repeat(131_072); + writer + .execute( + "INSERT INTO blobs (id, body) VALUES ($id, $body)", + ¶ms! { "id" => id, "body" => body }, + ) + .unwrap(); + } + writer.close().unwrap(); + + // Under a 16 MiB ceiling, reading or writing, the statement that would bring all 64 MiB + // into memory is refused by name with the numbers of the refusal, and gives no result; the + // session goes on, and a statement that fits is answered. + let statements = "SELECT id, body FROM blobs ORDER BY id;\nSELECT COUNT(*) AS n FROM blobs;\n"; + for mode in [&[][..], &["--write"][..]] { + let mut args = vec![arg(&big), "--json", "--memory-limit", "16M"]; + args.extend_from_slice(mode); + let run = shell(&dir, &args, statements); + assert_eq!(run.code, Some(1), "{mode:?}: {}", run.stderr); + let refusals: Vec = documents(&run.stderr) + .into_iter() + .filter(|document| document.get("error").is_some()) + .collect(); + assert_eq!(refusals.len(), 1, "{mode:?}: {}", run.stderr); + let error = &refusals[0]["error"]; + assert_eq!(error["class"], "io", "{mode:?}: {error}"); + assert_eq!( + error["detail"]["kind"], "memory_budget_exceeded", + "{mode:?}: {error}" + ); + for number in ["requested_bytes", "available_bytes", "budget_limit_bytes"] { + assert!( + error["detail"][number].is_u64(), + "{mode:?}: no {number} in {error}" + ); + } + assert_eq!( + documents(&run.stdout), + [json!({"result": {"columns": ["n"], "rows": [{"n": 64}]}})], + "{mode:?}" + ); + } +} diff --git a/v2/contextdb/Cargo.toml b/v2/contextdb/Cargo.toml new file mode 100644 index 000000000..1002a1d08 --- /dev/null +++ b/v2/contextdb/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "contextdb" +version = "2.0.0" +edition = "2024" +rust-version = "1.89" +license = "Apache-2.0" +description = "An embedded SQL database in one folder: typed tables, transactions, one writer with any number of readers in other processes, and a memory limit that holds" +repository = "https://github.com/context-graph-ai/contextdb" +readme = "README.md" + +[dependencies] +pest = "2.8.6" +pest_derive = "2.8.6" +redb = { package = "contextdb-redb", version = "4.1.0", path = "../redb" } +# Numbers in a stored JSON value are read back exactly as they were written. +serde_json = { version = "1", features = ["float_roundtrip"] } +uuid = { version = "1", features = ["v4"] } + +[lints] +workspace = true diff --git a/v2/contextdb/README.md b/v2/contextdb/README.md new file mode 100644 index 000000000..0d75e9cbf --- /dev/null +++ b/v2/contextdb/README.md @@ -0,0 +1,251 @@ +# contextdb + +An embedded SQL database: tables in one folder, inside your own process. + +This is the first version of contextdb 2. What it has: + +- tables with typed columns, a primary key, `NOT NULL`, `UNIQUE` and `DEFAULT`, and indexes; +- `INSERT`, `UPDATE`, `DELETE` and `SELECT` over one table; +- transactions; +- one process writing a store while any number of other processes read it; +- a memory limit that holds. + +What it does not have yet: vector search, relationships between rows, joins, grouping, and sync +between machines. A statement that uses one of them is refused and changes nothing. + +It is not on crates.io yet. Clone and use it from that checkout: + +```toml +[dependencies] +contextdb = { path = "/v2/contextdb" } +``` + +```rust,no_run +use contextdb::{params, Database, Uuid, Value}; + +fn main() -> contextdb::Result<()> { + let db = Database::open("./notes")?; // a folder, created if it is missing + db.execute( + "CREATE TABLE notes (id UUID PRIMARY KEY, body TEXT NOT NULL, made TIMESTAMP DEFAULT NOW())", + ¶ms! {}, + )?; + db.execute( + "INSERT INTO notes (id, body) VALUES ($id, $body)", + ¶ms! { "id" => Uuid::new_v4(), "body" => "buy milk" }, + )?; + + let tx = db.transaction()?; // rolls back if dropped + tx.execute("UPDATE notes SET body = 'buy oat milk' WHERE body = 'buy milk'", ¶ms! {})?; + tx.commit()?; + + let found = db.execute("SELECT body, made FROM notes ORDER BY made LIMIT 10", ¶ms! {})?; + for row in found.rows() { + if let Value::Text(body) = &row[0] { + println!("{body}"); + } + } + Ok(()) +} +``` + +It prints `buy oat milk`. This page is also the crate's documentation, and the documentation +tests compile this program as it is written here. Rust 1.89 or newer; Linux and macOS. Licence: +Apache-2.0. It is developed in the open at by the context-graph-ai organisation. + +## Statements + +```sql +CREATE TABLE [IF NOT EXISTS] name ( + column type [PRIMARY KEY] [NOT NULL] [UNIQUE] [DEFAULT value], + ..., + [UNIQUE (column, ...)] +); +CREATE INDEX name ON table (column [ASC | DESC], ...); +INSERT INTO table (column, ...) VALUES (value, ...), (value, ...); +UPDATE table SET column = value, ... [WHERE condition]; +DELETE FROM table [WHERE condition]; +SELECT column, ... | * FROM table [WHERE condition] [ORDER BY column [ASC | DESC], ...] [LIMIT n]; +SET MEMORY_LIMIT 512M; SET MEMORY_LIMIT NONE; SHOW MEMORY_LIMIT; +``` + +- Every table has exactly one `PRIMARY KEY` column. A default is a literal or `NOW()`. +- A condition or a value is built from column names, literals, `$name` values, + `= != <> < <= > >=`, `AND`, `OR`, `NOT`, `IS NULL`, `IS NOT NULL`, `+ - * /`, + parentheses and `NOW()`. A comparison with NULL is not true, as in standard SQL. +- `SELECT` also takes `COUNT(*)`, `COUNT(column)`, `SUM`, `MIN` and `MAX` over the rows that + match, and `AS` to name a column of the result. +- `ORDER BY` puts NULL last ascending and first descending. +- `--` and `/* */` comments are read past. + +Not in this version, and refused: `JOIN`, `GROUP BY`, `DISTINCT`, `IN`, `LIKE`, `BETWEEN`, +`OFFSET`, `UNION`, subqueries, table aliases, `ALTER`, `DROP`, a primary key over several +columns, and similarity search (`<=>`). Most are refused by name; this is the sentence the +refusal prints: + +```text +plan error: LIKE is not supported in this version of the library +``` + +## Column types + +| Type | Holds | In Rust | +|---|---|---| +| `INTEGER`, `INT` | a 64-bit signed whole number | `Value::Int64` | +| `REAL`, `FLOAT` | a 64-bit floating-point number | `Value::Float64` | +| `TEXT` | text | `Value::Text` | +| `BOOLEAN`, `BOOL` | `TRUE` or `FALSE` | `Value::Bool` | +| `UUID` | a UUID, written as quoted text | `Value::Uuid` | +| `TIMESTAMP` | milliseconds since the first moment of 1970, UTC; written as ISO 8601 text (`'2026-01-01T00:00:01Z'`) or taken from `NOW()` | `Value::Timestamp` | +| `JSON` | a JSON value, written as quoted text; the value is kept, not the text's spacing | `Value::Json` | +| `VECTOR(n)` | a vector of exactly `n` numbers, written `[0.1, 0.2, 0.3]`; stored and returned, not searchable in this version | `Value::Vector` | + +A value must be of its column's type: text into an `INTEGER` column is refused, and so is NULL +into a `NOT NULL` column, on `INSERT` and on `UPDATE` alike. A whole number is accepted into a +`REAL` column. + +## Binding values + +`$name` in a statement is the entry `"name"` of the map passed with it. `params!` writes the +map: `params! { "id" => Uuid::new_v4(), "body" => "buy milk" }`, or `params! {}` for none. It +takes `bool`, `i64`, `f64`, `&str`, `String`, `Uuid`, a `serde_json::Value` and a `Vec`; +any other value is written as a `Value`, for example `Value::Timestamp(1_767_225_601_000)` or +`Value::Null`. `contextdb::Uuid` and `contextdb::serde_json` are the types the library uses, so +`Uuid::new_v4()` and JSON values need no dependency of your own. + +## Refusals + +Every call that can fail returns `contextdb::Result` with one `contextdb::Error`. A program +matches on the kind (`Error::UniqueViolation { table, column }`, `Error::ColumnNotNullable`, +`Error::TableNotFound`, `Error::DatabaseLocked`, `Error::MemoryBudgetExceeded`, and the rest +listed on the `Error` type); a person reads the sentence it prints: + +```text +unique constraint violation: notes (id) +column notes.body is NOT NULL +column not found: notes.nothing +parse error: syntax error at line 1, column 1: unexpected 'SELEC' +``` + +A refused statement changes nothing. Inside a transaction it leaves the transaction usable. + +## Transactions + +`db.execute` is a transaction of one statement: when it returns, the change is on disk. +`db.transaction()` begins a longer one. Its statements see what it has written and nobody else +does until `commit` returns; dropping it, or `rollback`, discards it. A transaction either +declares tables and indexes or writes rows, not both. When two open transactions insert the same +key, or change the same row, the first to commit wins and the other's `commit` is refused. +`SET MEMORY_LIMIT` is not part of a transaction: run inside one, it takes effect and the store +remembers it at once, and a rollback does not undo it. + +## One writer, many readers + +One process at a time holds a store for writing. A second `Database::open` on the same folder is +refused at once, naming the holder: + +```text +another session holds this store for writing: process 1983518 holds ./notes; open a read session to read it +``` + +Any number of processes read meanwhile with `ReadSession::open`, which never creates anything. +Each statement of a reader sees the newest commit. A reader never makes the writer fail, and +holds it up only for the instant a statement takes to note which commit it reads. A reader keeps +answering when the writer stops, is killed or starts again. + +## The memory limit + +`db.set_memory_limit(Some(bytes))` or `SET MEMORY_LIMIT 512M` declares how much memory the +engine may use for data in the writing process, and the store remembers it until it is changed +or set to `NONE`. A statement that would pass the limit is refused and committed rows stay as +they were: + +```text +memory limit reached during transaction: 30000210 bytes were needed and 7340032 bytes of the 16777216-byte limit were free. select or write less at once, or raise the limit with SET MEMORY_LIMIT or the memory_limit open option +``` + +With nothing declared there is no ceiling. A reader has a limit of its own, given when it opens, +and none unless one is given. Every open sets 9 MiB of the limit aside, 8 MiB for the page cache +and 1 MiB for small fixed things, so 9 MiB is the smallest limit that opens a store and a +statement needs room above it. + +`SHOW MEMORY_LIMIT` returns one row about the session that asks, a writer's or a reader's: +`limit` (its limit in bytes, or `none`), `used` (bytes in use now, the 9 MiB included), +`available` (what is left under the limit), `startup_ceiling` (the limit the session was opened +under) and `ceiling_refresh_failures` (always 0 in this version). + +## The store on disk + +```text +notes/ + data everything committed + lock names the process that holds the store for writing, or held it last + readers/ one small file per reader, while a statement of its runs +``` + +Nothing is written anywhere else. While no process has the folder open for writing it is a plain +folder: copy it, move it or delete it with ordinary tools. Do not copy it while a writer is +running; this version has no command that copies a store beside a live writer. + +**When a process is killed.** Nothing committed is lost: a commit that had returned is there, +and one that was in flight is whole or absent. A killed writer leaves its `lock` file and a +reader killed inside a statement leaves its file in `readers/`; the next open for writing clears +both, and nobody cleans anything by hand. Readers open and answer at once. The next open for writing after a kill +first reads every page the store uses, which takes time that grows with the store, measured on +the machine named below with the file out of the operating system's cache: + +| Store size | Open for writing after a kill | Open after a clean close | +|---|---|---| +| 257 MiB | 72 ms | 0.9 ms | +| 2 GiB | about 0.5 s | 1.2 ms | +| 8 GiB | about 3.4 s | 1.7 ms | + +## The `contextdb` program + +The `contextdb-cli` package beside this one builds `contextdb`, a SQL shell for a store. +Install it from the checkout, then open a store with it: + +```text +cargo install --path /v2/contextdb-cli --locked +contextdb ./notes --write +``` + +```text +contextdb ./notes read the store, also while another process writes it +contextdb ./notes --write read and write it, creating it if it is missing +``` + +A statement ends at `;`. `BEGIN;`, `COMMIT;` and `ROLLBACK;` run a transaction, which needs +`--write`; `.tables` lists +the tables, `.schema
` prints a table's declaration and `.help` lists the commands. With +statements piped in there is no prompt, and a refusal does not stop what comes after it. A +refused statement or command is one line on the error stream, `Error: ` and then the refusal's +sentence, or the program's own sentence where one of its options is the way on; a line that +starts like a command and is not one is answered with `Unknown command:`. The rest still run. A +piped run exits 1 if any statement or command was refused, an unknown command and `.schema` of +a table that is not there included, and 0 otherwise. However the input arrives, the exit status is 1 when the +store could not be opened or closed, and 2 when the command line itself is refused. `--json` +prints one JSON document per statement or command and each refusal as a JSON document on the +error stream. `contextdb --help` has the +options, the exit statuses and the JSON forms. + +## Numbers + +Measured at one commit by `v2/numbers.sh`, which prints them again on any machine: a release +build on a 16-core AMD Ryzen 9 9955HX with 64 GB of memory and an NVMe drive, running Linux and +nothing else. + +| What | Measured | +|---|---| +| 20,000 rows of 1.6 KB, one commit per row: time to insert | 17.2 s | +| That store on disk | 60.3 MiB | +| Open that store | 0.631 ms | +| `SELECT COUNT(*)` over it | 0.026 ms, 0 rows read | +| A million rows under a 64 MiB limit: time to load | 11.9 s | +| Scan the million rows | 246 ms | +| Most memory the engine had allocated during that load and scan | 13.4 MiB | +| Most memory the process held, as the operating system reports it | 18.0 MiB | +| First build of a program that uses the library | 5 s, 46 packages, a 2.7 MiB program | +| From a fresh clone to a passing `cargo test` | 2.1 min | +| Lines of Rust and grammar in the library and the program | 7,369 | +| Tests | 98 | +| Lines that differ from redb 4.1.0 in the storage library under the engine, which is redb 4.1.0 with the changes listed in `v2/redb/README.md` | 577 | diff --git a/v2/contextdb/examples/measure.rs b/v2/contextdb/examples/measure.rs new file mode 100644 index 000000000..c19a110bb --- /dev/null +++ b/v2/contextdb/examples/measure.rs @@ -0,0 +1,203 @@ +//! Prints the numbers this library is held to, one per line as `name value unit`. +//! +//! `measure readme ` runs the README's first program at 20,000 rows of 1.6 KB, one +//! commit per row, and prints the store's size on disk, how long opening it takes and how long +//! `COUNT(*)` takes. `measure million ` loads a million small rows under a 64 MiB limit +//! in commits of 10,000, scans them, and prints the most memory the process had allocated beside +//! the most the operating system saw it hold. The folder must be missing or empty. +//! +//! This program measures. It never passes or fails, no test reads what it prints, and +//! `numbers.sh` is what runs it. + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::fmt::Display; +use std::path::Path; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{Duration, Instant}; + +use contextdb::{Database, DatabaseOpenOptions, Result, Uuid, params}; + +/// The system allocator, counting the bytes handed out and not yet given back. +struct Counting; + +static IN_USE: AtomicUsize = AtomicUsize::new(0); +static PEAK: AtomicUsize = AtomicUsize::new(0); + +// SAFETY: every request goes to the system allocator unchanged; this only counts its bytes. +unsafe impl GlobalAlloc for Counting { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let in_use = IN_USE.fetch_add(layout.size(), Ordering::Relaxed) + layout.size(); + PEAK.fetch_max(in_use, Ordering::Relaxed); + unsafe { System.alloc(layout) } + } + + unsafe fn dealloc(&self, pointer: *mut u8, layout: Layout) { + IN_USE.fetch_sub(layout.size(), Ordering::Relaxed); + unsafe { System.dealloc(pointer, layout) } + } +} + +#[global_allocator] +static ALLOCATOR: Counting = Counting; + +/// Runs `work` and says how long it took. Apart from the library's `system` part, this is the +/// only place in the workspace that reads a clock. +#[allow( + clippy::disallowed_methods, + reason = "measuring elapsed time is what this program is for" +)] +fn timed(work: impl FnOnce() -> T) -> (T, Duration) { + let start = Instant::now(); + let result = work(); + (result, start.elapsed()) +} + +fn line(name: &str, value: impl Display, unit: &str) { + println!("{name} {value} {unit}"); +} + +fn milliseconds(duration: Duration) -> String { + format!("{:.3}", duration.as_secs_f64() * 1000.0) +} + +fn mebibytes(bytes: u64) -> String { + format!("{:.1}", bytes as f64 / 1_048_576.0) +} + +/// The bytes of every file under a folder. +fn bytes_on_disk(folder: &Path) -> std::io::Result { + let mut bytes = 0; + for entry in std::fs::read_dir(folder)? { + let entry = entry?; + let meta = entry.metadata()?; + bytes += if meta.is_dir() { + bytes_on_disk(&entry.path())? + } else { + meta.len() + }; + } + Ok(bytes) +} + +/// The most memory the operating system has seen this process hold, where it says so. +fn peak_resident_bytes() -> Option { + let status = std::fs::read_to_string("/proc/self/status").ok()?; + let line = status.lines().find(|line| line.starts_with("VmHWM:"))?; + let kibibytes: u64 = line.split_whitespace().nth(1)?.parse().ok()?; + Some(kibibytes * 1024) +} + +/// The README's first program, at 20,000 rows of 1.6 KB. +fn readme(folder: &Path) -> Result<()> { + const ROWS: u128 = 20_000; + let db = Database::open(folder)?; + db.execute( + "CREATE TABLE notes (id UUID PRIMARY KEY, body TEXT)", + ¶ms! {}, + )?; + let (inserted, insert) = timed(|| -> Result<()> { + for n in 1..=ROWS { + let body = format!("note {n:08} ").repeat(115)[..1600].to_owned(); + db.execute( + "INSERT INTO notes (id, body) VALUES ($id, $body)", + ¶ms! { "id" => Uuid::from_u128(n), "body" => body }, + )?; + } + Ok(()) + }); + inserted?; + db.close()?; + line("readme_rows", ROWS, "rows"); + line("readme_insert", format!("{:.1}", insert.as_secs_f64()), "s"); + let on_disk = bytes_on_disk(folder).expect("the store's folder can be listed"); + line("readme_store_size", mebibytes(on_disk), "MiB"); + + let (db, open) = timed(|| Database::open(folder)); + let db = db?; + line("readme_open", milliseconds(open), "ms"); + let (count, counting) = timed(|| db.execute("SELECT COUNT(*) FROM notes", ¶ms! {})); + let count = count?; + line("readme_count_star", milliseconds(counting), "ms"); + line( + "readme_count_star_rows_examined", + count.rows_examined(), + "rows", + ); + db.close() +} + +/// A million small rows loaded and scanned under a 64 MiB limit. +fn million(folder: &Path) -> Result<()> { + const ROWS: i64 = 1_000_000; + const PER_COMMIT: i64 = 10_000; + let mut options = DatabaseOpenOptions::default(); + options.memory_limit = Some(64 << 20); + let db = Database::open_with_options(folder, options)?; + db.execute( + "CREATE TABLE small (id INTEGER PRIMARY KEY, n INTEGER, word TEXT)", + ¶ms! {}, + )?; + let (loaded, load) = timed(|| -> Result<()> { + for first in (1..=ROWS).step_by(PER_COMMIT as usize) { + let tx = db.transaction()?; + for id in first..first + PER_COMMIT { + tx.execute( + "INSERT INTO small (id, n, word) VALUES ($id, $n, $word)", + ¶ms! { "id" => id, "n" => id % 1000, "word" => format!("w{id:07}") }, + )?; + } + tx.commit()?; + } + Ok(()) + }); + loaded?; + let (sum, scan) = timed(|| db.execute("SELECT SUM(n) FROM small", ¶ms! {})); + let sum = sum?; + line( + "million_rows_load", + format!("{:.1}", load.as_secs_f64()), + "s", + ); + line("million_rows_scan", milliseconds(scan), "ms"); + line( + "million_rows_scan_rows_examined", + sum.rows_examined(), + "rows", + ); + line( + "million_rows_peak_allocated", + mebibytes(PEAK.load(Ordering::Relaxed) as u64), + "MiB", + ); + if let Some(bytes) = peak_resident_bytes() { + line("million_rows_peak_resident", mebibytes(bytes), "MiB"); + } + db.close() +} + +fn main() -> Result<()> { + let arguments: Vec = std::env::args().skip(1).collect(); + let [what, folder] = arguments.as_slice() else { + eprintln!("usage: measure readme|million "); + std::process::exit(2); + }; + let folder = Path::new(folder); + if folder + .read_dir() + .is_ok_and(|mut entries| entries.next().is_some()) + { + eprintln!( + "{} is not empty; give a folder that is missing or empty", + folder.display() + ); + std::process::exit(2); + } + match what.as_str() { + "readme" => readme(folder), + "million" => million(folder), + _ => { + eprintln!("usage: measure readme|million "); + std::process::exit(2); + } + } +} diff --git a/v2/contextdb/src/database/mod.rs b/v2/contextdb/src/database/mod.rs new file mode 100644 index 000000000..f9d1e0e13 --- /dev/null +++ b/v2/contextdb/src/database/mod.rs @@ -0,0 +1,551 @@ +//! database: the handle a store is opened for writing with, its transactions, and the one +//! commit function. +//! +//! It never has a second way to commit: a statement run on its own, a transaction and a change +//! of the memory limit each build a change set and hand it to the same function. + +#[cfg(test)] +mod tests; + +use std::cell::RefCell; +use std::collections::HashMap; +use std::path::Path; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; + +use crate::memory::Budget; +use crate::query::{self, QueryResult}; +use crate::schema::{self, Catalog}; +use crate::sql::{self, Effect, Plan}; +use crate::store::{ChangeSet, Read, Store, View, WrapStorage}; +use crate::system::{self, Clock}; +use crate::types::{Error, Result, TxId, Value}; +use crate::write; + +/// What a host may say when it opens a store for writing. +/// +/// Start from `DatabaseOpenOptions::default()` and assign the fields that matter. The struct may +/// gain fields in a later version, so it cannot be written out field by field outside this +/// crate. +/// +/// ``` +/// let mut options = contextdb::DatabaseOpenOptions::default(); +/// options.memory_limit = Some(64 << 20); // 64 MiB +/// ``` +#[derive(Clone, Default)] +#[non_exhaustive] +pub struct DatabaseOpenOptions { + /// The most memory, in bytes, that this open may hold for its page cache, its transactions + /// and its results. + /// + /// `None`, the default, takes the limit the store remembers from the last + /// `SET MEMORY_LIMIT` or [`Database::set_memory_limit`], and means no limit when none was + /// ever declared. A limit given here holds until the store is closed and leaves what the + /// store remembers as it is. + /// + /// Every open sets 9 MiB aside out of the limit, 8 MiB of it for the store's page cache. A + /// smaller limit is refused with [`crate::Error::MemoryBudgetExceeded`] and opens no store. + /// + /// A [`crate::ReadSession`] never takes the limit the store remembers: `None` in its own + /// options means no limit at all, so a reader that needs a ceiling is given one there. + pub memory_limit: Option, + /// Where the time comes from for `NOW()` and `DEFAULT NOW()`. `None`, the default, is the + /// machine's clock. + pub clock: Option>, + /// A wrapper around the store's data file: the store opens the file, hands it to this, and + /// reads, writes and syncs it only through what this returns. `None`, the default, uses the + /// file as it is. + pub storage: Option, +} + +/// One store folder, held for writing. +/// +/// One handle holds a store for writing at a time. A second [`Database::open`] of the same +/// folder is refused with [`crate::Error::DatabaseLocked`], from another process and from this +/// one alike. Any number of [`crate::ReadSession`]s, in this process or others, read the store +/// meanwhile. +/// +/// A handle can be shared between threads, and its calls block until they are done. +/// +/// Dropping the handle lets go of the store as [`Database::close`] does. Every commit that +/// returned is already on disk, so neither writes anything. +pub struct Database { + /// The open store, `None` once [`Database::close`] has let go of it. + open: Mutex>>, +} + +/// One store held for writing. A call holds it for as long as the call runs, so the folder is +/// let go when the handle has let go and the last running call has returned. +struct Open { + budget: Budget, + clock: Arc, + /// The limit this open started under, as `SHOW MEMORY_LIMIT` reports it. + started_under: Option, + /// The commit lock: held from before a commit looks at the newest moment until it has + /// replaced it. + committing: Mutex<()>, + /// What every statement and every transaction begins from. Held to copy it, to replace it + /// and to read it again after a read that failed, never while a commit goes to the disk. + newest: Mutex, + /// The last field, so that the view above is dropped before the folder is let go. + store: Store, +} + +/// The newest commit. +struct Newest { + commit: TxId, + /// The tables declared as of it. + catalog: Arc, + /// A view of it. `None` when the read that follows a commit failed: the next call that + /// needs a view reads again, so that nothing is answered from before that commit. + view: Option, +} + +/// What one transaction has staged. +struct Staged { + changes: ChangeSet, + /// The tables as the transaction sees them: those of the moment it began at and those it + /// has declared since. + catalog: Arc, + /// The declarations it ran, in order, for a commit that has to run them again. + declared: Vec, +} + +fn planned(sql: &str) -> Result { + sql::plan(&sql::parse(sql)?) +} + +/// Takes a lock. None of this part's locks is held across a step that can leave what the lock +/// guards half changed, so a holder that panicked left it whole. +fn locked(lock: &Mutex) -> MutexGuard<'_, T> { + lock.lock().unwrap_or_else(PoisonError::into_inner) +} + +impl Open { + /// A view of the newest commit and the tables declared as of it. + fn newest(&self) -> Result<(View, Arc)> { + let mut newest = locked(&self.newest); + if let Some(view) = &newest.view { + return Ok((view.clone(), newest.catalog.clone())); + } + // The tables are read from the view itself, so the two are of one commit even when + // another commit lands in this instant. + let view = self.store.view()?; + let catalog = Arc::new(Catalog::load(&view)?); + *newest = Newest { + commit: view.commit(), + catalog: catalog.clone(), + view: Some(view.clone()), + }; + Ok((view, catalog)) + } + + /// An empty transaction over the newest commit. + fn begin(&self) -> Result { + let (view, catalog) = self.newest()?; + Ok(Staged { + changes: ChangeSet::new(view, &self.budget)?, + catalog, + declared: Vec::new(), + }) + } + + /// Runs one planned statement inside a transaction. A refusal may leave part of the + /// statement staged: the caller rewinds the change set or drops it. + fn run( + &self, + plan: &Plan, + params: &HashMap, + staged: &mut Staged, + ) -> Result { + // One time for the whole statement. + let now = system::now_millis(&*self.clock); + match plan.effect() { + Effect::Declares => { + if staged.declared.is_empty() && !staged.changes.is_empty() { + return Err(Error::MixedSchemaAndRowWrite); + } + let catalog = self.declare(plan, &staged.catalog, &mut staged.changes)?; + staged.catalog = Arc::new(catalog); + staged.declared.push(plan.clone()); + Ok(QueryResult::affected(0)) + } + Effect::WritesRows => { + if !staged.declared.is_empty() { + return Err(Error::MixedSchemaAndRowWrite); + } + let (catalog, changes) = (&staged.catalog, &mut staged.changes); + write::apply(plan, params, catalog, changes, &self.budget, now) + .map(QueryResult::affected) + } + Effect::Reads => { + if let Plan::ShowMemoryLimit = plan { + return Ok(query::memory_limit_shown(&self.budget, self.started_under)); + } + let (catalog, changes) = (&staged.catalog, &staged.changes); + query::run(plan, params, catalog, changes, &self.budget, now) + } + // A setting is not part of the transaction: it takes hold at once, in a commit of + // its own, and a rollback does not undo it. + Effect::Sets(limit) => self + .set_memory_limit(limit) + .map(|()| QueryResult::affected(0)), + } + } + + /// Stages one declaration and, for a new index, its entries for the rows already there. + fn declare(&self, plan: &Plan, catalog: &Catalog, changes: &mut ChangeSet) -> Result { + let declared = schema::declare(plan, catalog, changes)?; + if let Plan::CreateIndex(create) = plan + && let Some(table) = declared.table(&create.table) + && let Some(&index) = table.index_ids.last() + { + write::fill_index(table, index, changes, &self.budget)?; + } + Ok(declared) + } + + /// The one commit function: everything that changes the store is written here, one commit + /// at a time. `_held` is the commit lock. + /// + /// A transaction that began before the newest commit was overtaken. One that declared runs + /// its declarations again over the newest moment, because a table's and an index's number + /// are counted on from the catalog and a new index is filled from the rows that are there; + /// a refusal of that second run is the commit's refusal. One that wrote rows is laid over + /// the newest moment and `write` checks its rows against it. + fn commit(&self, _held: &MutexGuard<'_, ()>, staged: Staged) -> Result<()> { + let Staged { + mut changes, + mut catalog, + declared, + } = staged; + let (view, tables) = self.newest()?; + if changes.commit() != view.commit() { + if !declared.is_empty() { + changes = ChangeSet::new(view, &self.budget)?; + catalog = tables; + for plan in &declared { + catalog = Arc::new(self.declare(plan, &catalog, &mut changes)?); + } + } else if !changes.is_empty() { + let before = changes.rebase(view); + // The rows were staged with the index entries of the tables as they were + // declared then; an index declared since would be left without them. + if before.schema_commit()? != changes.base().schema_commit()? { + return Err(Error::SchemaInvalid { + reason: "a table or an index was declared while this transaction was \ + open, so nothing of it was committed; run it again" + .to_owned(), + }); + } + write::check_at_commit(&changes, &catalog, &before)?; + } + } + if changes.is_empty() { + return Ok(()); + } + let commit = self.store.commit(changes)?; + // The commit is on disk whatever the read after it says. When that read fails no view + // is kept and the next call reads again, so none answers from before this commit. + let view = self.store.view().ok(); + *locked(&self.newest) = Newest { + commit, + catalog, + view, + }; + Ok(()) + } + + /// The one path of `SET MEMORY_LIMIT` and [`Database::set_memory_limit`]: the limit takes + /// hold in this open and is committed for the store to remember, or neither happens. + fn set_memory_limit(&self, limit: Option) -> Result<()> { + let held = locked(&self.committing); + let mut staged = self.begin()?; + staged.changes.put_memory_limit(limit)?; + let before = self.budget.limit(); + // Refused here, before anything is written, when the limit is below what is in use. + self.budget.set_limit(limit)?; + let committed = self.commit(&held, staged); + if committed.is_err() { + // The store did not take the limit, so this open goes back to the one it had. That + // is refused only when another thread has, in this instant, taken more than the old + // limit allows; the new limit then stands until the store is closed. + let _ = self.budget.set_limit(before); + } + committed + } +} + +impl Database { + /// Opens the store in this folder for writing. A folder that is missing or empty becomes a + /// new store, so this never returns [`crate::Error::StoreMissing`]. + /// + /// After a clean close this takes a few milliseconds whatever the store's size. After a + /// writer that was killed, the open first reads every page the store has in use. + /// + /// # Errors + /// + /// [`crate::Error::DatabaseLocked`], at once and without waiting, when another handle holds + /// the store for writing, in another process or in this one. [`crate::Error::StoreIo`] when + /// the folder or a file in it cannot be created, read or written, and + /// [`crate::Error::StoreCorrupted`] when stored data fails its check. + /// [`crate::Error::MemoryBudgetExceeded`] when the memory limit is too small to open a + /// store under. + pub fn open(path: impl AsRef) -> Result { + Database::open_with_options(path, DatabaseOpenOptions::default()) + } + + /// [`Database::open`] with options. + pub fn open_with_options( + path: impl AsRef, + options: DatabaseOpenOptions, + ) -> Result { + let budget = Budget::new(options.memory_limit.map(|limit| limit as u64))?; + let store = Store::open_for_writing(path.as_ref(), &budget, options.storage.as_ref())?; + let view = store.view()?; + // A limit the options name is this open's ceiling. With none named, the ceiling is the + // limit the store remembers. + if options.memory_limit.is_none() { + budget.set_limit(view.memory_limit()?)?; + } + let newest = Newest { + commit: view.commit(), + catalog: Arc::new(Catalog::load(&view)?), + view: Some(view), + }; + let open = Open { + started_under: budget.limit(), + budget, + clock: options.clock.unwrap_or_else(system::machine_clock), + committing: Mutex::new(()), + newest: Mutex::new(newest), + store, + }; + Ok(Database { + open: Mutex::new(Some(Arc::new(open))), + }) + } + + /// The open store, or the refusal of a handle that was closed. + fn opened(&self) -> Result> { + locked(&self.open).clone().ok_or(Error::DatabaseClosed) + } + + /// The open store, for the four calls that return a plain value and so have no way to + /// refuse. On a closed handle they panic with the refusal's sentence. + fn must_be_open(&self) -> Arc { + self.opened().unwrap_or_else(|closed| panic!("{closed}")) + } + + /// Runs one SQL statement in a transaction of its own and returns its result. + /// + /// A statement that changes something is committed before this returns: on `Ok` the change + /// is on disk, on `Err` nothing of it is. + /// + /// Values are bound by name. `$name` in the statement takes the value kept under the key + /// `name`, written without the `$`, in `params`; [`crate::params!`] writes the map. There + /// is no positional form. A `$name` with no key in the map is refused with + /// [`crate::Error::PlanError`], and a key the statement does not use is ignored. + /// + /// This may be called while a [`Transaction`] of the same handle is open, on the same + /// thread too. The statement commits on its own, the open transaction does not see it, and + /// the transaction's commit is then checked against it as [`Transaction::commit`] says. + /// + /// # Errors + /// + /// Whatever refuses the statement, each by its own kind of [`crate::Error`]; + /// [`crate::Error::StoreIo`] or [`crate::Error::StoreCorrupted`] when the store cannot be + /// read or written; [`crate::Error::DatabaseClosed`] on a handle that was closed. + pub fn execute(&self, sql: &str, params: &HashMap) -> Result { + let plan = planned(sql)?; + let open = self.opened()?; + if matches!(plan.effect(), Effect::Declares | Effect::WritesRows) { + // Taken before the statement reads anything, so no other commit can overtake it. + let held = locked(&open.committing); + let mut staged = open.begin()?; + let result = open.run(&plan, params, &mut staged)?; + open.commit(&held, staged)?; + Ok(result) + } else { + // A statement that stages nothing has nothing to commit. + open.run(&plan, params, &mut open.begin()?) + } + } + + /// Begins a transaction over the newest commit. Nothing it does is visible to anyone else + /// until it commits. Several transactions of one handle may be open at once. + /// + /// # Errors + /// + /// [`crate::Error::DatabaseClosed`] on a handle that was closed, and + /// [`crate::Error::StoreIo`] or [`crate::Error::StoreCorrupted`] when the store cannot be + /// read. + pub fn transaction(&self) -> Result> { + Ok(Transaction { + staged: RefCell::new(self.opened()?.begin()?), + db: self, + }) + } + + /// Declares the memory limit in bytes, or no limit for `None`. + /// + /// The limit takes hold at once and is committed, so the store remembers it for every + /// later open whose options name no limit, exactly as `SET MEMORY_LIMIT` does. + /// + /// # Errors + /// + /// [`crate::Error::MemoryBudgetExceeded`] when the limit is below what is in use now, which + /// is never less than the 9 MiB every open sets aside. [`crate::Error::StoreIo`] when the + /// commit cannot be written. In both cases the limit stays what it was. + /// [`crate::Error::DatabaseClosed`] on a handle that was closed. + pub fn set_memory_limit(&self, limit: Option) -> Result<()> { + self.opened()? + .set_memory_limit(limit.map(|limit| limit as u64)) + } + + /// The memory limit in force, in bytes; `None` when there is none. + /// + /// # Panics + /// + /// On a handle that was closed, with the message "database handle is closed". + pub fn memory_limit(&self) -> Option { + let limit = self.must_be_open().budget.limit(); + limit.map(|limit| limit as usize) + } + + /// The bytes counted against the memory limit at this moment: what every open sets aside, + /// and what open transactions and results not yet dropped hold. `SHOW MEMORY_LIMIT` reports + /// the same number as `used`. + /// + /// # Panics + /// + /// On a handle that was closed, with the message "database handle is closed". + pub fn memory_used(&self) -> usize { + self.must_be_open().budget.used() as usize + } + + /// The names of the store's tables as of the newest commit, in name order. A new store has + /// none. + /// + /// # Panics + /// + /// On a handle that was closed, with the message "database handle is closed". + pub fn table_names(&self) -> Vec { + let open = self.must_be_open(); + let newest = locked(&open.newest); + let tables = newest.catalog.tables().into_iter(); + tables.map(|table| table.name.clone()).collect() + } + + /// A table's declaration as SQL text, as of the newest commit: the `CREATE TABLE` statement + /// that would declare the table as it stands, with every clause left at its default not + /// written, then one `CREATE INDEX` statement for each index that was made with one. Each + /// statement ends with `;` and a new line. + /// + /// # Errors + /// + /// [`crate::Error::TableNotFound`] when the store has no table of this name, and + /// [`crate::Error::DatabaseClosed`] on a handle that was closed. The answer is not read + /// from the disk, so no other error is returned. + pub fn declaration(&self, table: &str) -> Result { + let open = self.opened()?; + let newest = locked(&open.newest); + let found = newest.catalog.table(table).map(schema::render); + found.ok_or_else(|| Error::TableNotFound(table.to_owned())) + } + + /// The position of the newest commit on disk: 0 for a store nothing was ever committed to, + /// and one more for every commit since. A transaction that commits nothing does not move + /// it, and no position is ever reported for work that did not commit. + /// + /// The position is for comparing two moments of one store. No call of this library takes + /// one. + /// + /// # Panics + /// + /// On a handle that was closed, with the message "database handle is closed". + pub fn committed_watermark(&self) -> TxId { + locked(&self.must_be_open().newest).commit + } + + /// Lets go of the store, so that the next writer can open it. Every commit that returned is + /// already on disk, and closing writes nothing. While a call is still running on another + /// thread, the store is let go when that call returns. Closing a handle that was closed + /// changes nothing. + /// + /// Every later call on this handle, and on a transaction that was begun from it, returns + /// [`crate::Error::DatabaseClosed`]. Four calls return a plain value and have no way to say + /// so; on a closed handle they panic: [`Database::memory_limit`], + /// [`Database::memory_used`], [`Database::table_names`] and + /// [`Database::committed_watermark`]. + pub fn close(&self) -> Result<()> { + drop(locked(&self.open).take()); + Ok(()) + } +} + +/// A transaction: statements that commit together or not at all. +/// +/// It reads the store as of the commit it began at, together with what it has written itself; +/// nobody else sees its changes until [`Transaction::commit`] returns. Dropping it without +/// committing rolls it back. +/// +/// A transaction can be moved to another thread and cannot be shared between two: it is `Send` +/// and not `Sync`. +pub struct Transaction<'db> { + staged: RefCell, + db: &'db Database, +} + +impl Transaction<'_> { + /// Runs one SQL statement inside the transaction. Values are bound by name, as + /// [`Database::execute`] binds them. + /// + /// A refused statement changes nothing and leaves the transaction usable. A transaction + /// either declares tables and indexes or writes rows: the other kind of statement is + /// refused with [`crate::Error::MixedSchemaAndRowWrite`]. + /// + /// `SET MEMORY_LIMIT` is not part of the transaction. It takes hold at once, and a rollback + /// does not undo it. + pub fn execute(&self, sql: &str, params: &HashMap) -> Result { + let plan = planned(sql)?; + let open = self.db.opened()?; + let staged = &mut *self.staged.borrow_mut(); + let mark = staged.changes.mark(); + let result = open.run(&plan, params, staged); + if result.is_err() { + staged.changes.rewind(mark); + } + result + } + + /// Commits. On `Ok(())` everything the transaction did is on disk. On `Err` none of it is, + /// and the transaction is gone: to try again, begin a new one and run its statements again. + /// + /// # Errors + /// + /// A transaction that was open while another commit landed is checked against that commit: + /// + /// - [`crate::Error::UniqueViolation`] when the other commit took a key, or a value of a + /// unique column, that this transaction inserted; + /// - [`crate::Error::ConditionalUpdateConflict`] when the other commit changed or deleted a + /// row that this transaction changed or deleted; + /// - [`crate::Error::SchemaInvalid`] when a table or an index was declared meanwhile and + /// this transaction wrote rows; + /// - for a transaction that declared tables or indexes, whatever would refuse its + /// statements if they were run now: for one, [`crate::Error::SchemaInvalid`] when a table + /// name it declared has been taken. + /// + /// A statement run with [`Database::execute`] is never overtaken and meets none of these. + /// + /// Also [`crate::Error::StoreIo`] when the commit cannot be written, and + /// [`crate::Error::DatabaseClosed`] when the handle was closed. + pub fn commit(self) -> Result<()> { + let open = self.db.opened()?; + let held = locked(&open.committing); + open.commit(&held, self.staged.into_inner()) + } + + /// Rolls back now. What the transaction staged is held in memory only, so this drops it, + /// writes nothing and cannot fail. Dropping the transaction does the same. + pub fn rollback(self) {} +} diff --git a/v2/contextdb/src/database/tests.rs b/v2/contextdb/src/database/tests.rs new file mode 100644 index 000000000..7913820de --- /dev/null +++ b/v2/contextdb/src/database/tests.rs @@ -0,0 +1,119 @@ +//! What the commit function does for a transaction that another commit overtook, and what a +//! closed handle answers. + +use std::collections::HashMap; + +use super::{Database, Transaction}; +use crate::types::{Error, Value}; + +/// Runs a test on a fresh store in a folder of its own, which is removed when the test passes. +fn with_store(name: &str, test: impl FnOnce(&Database)) { + let folder = std::env::temp_dir().join(format!("contextdb-{name}-{}", std::process::id())); + let db = Database::open(&folder).expect("open a fresh store"); + for sql in [ + "CREATE TABLE t (id INTEGER PRIMARY KEY, tag INTEGER)", + "INSERT INTO t (id, tag) VALUES (1, 7)", + ] { + db.execute(sql, &HashMap::new()).expect(sql); + } + test(&db); + drop(db); + std::fs::remove_dir_all(&folder).expect("remove the test's folder"); +} + +fn rows(db: &Database, sql: &str) -> Vec> { + db.execute(sql, &HashMap::new()).expect(sql).rows().to_vec() +} + +fn ints(numbers: [i64; N]) -> Vec> { + numbers.map(|number| vec![Value::Int64(number)]).to_vec() +} + +#[test] +fn transactions_that_each_declare_a_table_commit_with_a_number_each() { + with_store("declare-side-by-side", |db| { + let declaring = [ + "CREATE TABLE a (id INTEGER PRIMARY KEY)", + "CREATE TABLE b (id INTEGER PRIMARY KEY)", + "CREATE TABLE a (id INTEGER PRIMARY KEY, more TEXT)", + ]; + // All three begin from the same tables. + let [first, second, third] = declaring.map(|sql| { + let tx = db.transaction().expect("begin"); + tx.execute(sql, &HashMap::new()).expect(sql); + tx + }); + first.commit().expect("the first declaration commits"); + second.commit().expect("a table of another name commits"); + // The name was free when the third declared it and is taken now. + assert!(matches!(third.commit(), Err(Error::SchemaInvalid { .. }))); + assert_eq!(db.table_names(), ["a", "b", "t"]); + + // A row of one table is no row of the other. + db.execute("INSERT INTO a (id) VALUES (1)", &HashMap::new()) + .expect("insert into a"); + assert_eq!(rows(db, "SELECT COUNT(*) FROM a"), ints([1])); + assert_eq!(rows(db, "SELECT COUNT(*) FROM b"), ints([0])); + assert!(rows(db, "SELECT id FROM b").is_empty()); + }); +} + +#[test] +fn a_new_index_has_the_rows_committed_while_its_transaction_was_open() { + with_store("index-overtaken", |db| { + let tx = db.transaction().expect("begin"); + tx.execute("CREATE INDEX t_tag ON t (tag)", &HashMap::new()) + .expect("declare the index"); + db.execute("INSERT INTO t (id, tag) VALUES (2, 7)", &HashMap::new()) + .expect("insert while the index is being declared"); + tx.commit().expect("commit the index"); + assert_eq!( + rows(db, "SELECT id FROM t WHERE tag = 7 ORDER BY id"), + ints([1, 2]) + ); + }); +} + +#[test] +fn rows_staged_before_an_index_was_declared_are_not_committed_without_its_entries() { + with_store("rows-overtaken", |db| { + let tx = db.transaction().expect("begin"); + tx.execute("INSERT INTO t (id, tag) VALUES (2, 7)", &HashMap::new()) + .expect("insert in the transaction"); + db.execute("CREATE INDEX t_tag ON t (tag)", &HashMap::new()) + .expect("declare an index beside the transaction"); + assert!(matches!(tx.commit(), Err(Error::SchemaInvalid { .. }))); + assert_eq!(rows(db, "SELECT id FROM t WHERE tag = 7"), ints([1])); + }); +} + +#[test] +fn a_closed_handle_refuses_by_name() { + with_store("closed-handle", |db| { + let tx = db.transaction().expect("begin"); + db.close().expect("close"); + db.close().expect("close a second time"); + let select = "SELECT id FROM t"; + assert!(matches!( + db.execute(select, &HashMap::new()), + Err(Error::DatabaseClosed) + )); + assert!(matches!( + tx.execute(select, &HashMap::new()), + Err(Error::DatabaseClosed) + )); + assert!(matches!(tx.commit(), Err(Error::DatabaseClosed))); + assert!(matches!(db.transaction(), Err(Error::DatabaseClosed))); + }); +} + +fn shared() {} +fn sent() {} + +/// No test may start a thread, so what "calls may be made from many threads" rests on is +/// checked here by the compiler. +#[test] +fn a_handle_is_shared_between_threads_and_a_transaction_moves_to_one() { + shared::(); + sent::>(); +} diff --git a/v2/contextdb/src/lib.rs b/v2/contextdb/src/lib.rs new file mode 100644 index 000000000..77df6f5ac --- /dev/null +++ b/v2/contextdb/src/lib.rs @@ -0,0 +1,37 @@ +#![doc = include_str!("../README.md")] + +mod database; +mod memory; +mod query; +mod read; +mod schema; +mod sql; +mod store; +mod system; +mod types; +mod write; + +pub use database::{Database, DatabaseOpenOptions, Transaction}; +pub use query::QueryResult; +pub use read::{ReadSession, ReadSessionOptions}; +pub use store::{StorageBackend, WrapStorage}; +pub use system::Clock; +pub use types::{ColumnType, Error, Result, TxId, Uuid, Value}; + +/// The JSON package whose `Value` a [`Value::Json`] holds, so a host needs no copy of its own. +pub use serde_json; + +/// Writes the map of named values that [`Database::execute`] binds into a statement. +/// +/// `params! {}` is the empty map. Each value is anything a [`Value`] can be made from. +#[macro_export] +macro_rules! params { + () => { + ::std::collections::HashMap::<::std::string::String, $crate::Value>::new() + }; + ($($name:expr => $value:expr),+ $(,)?) => {{ + let mut map = ::std::collections::HashMap::<::std::string::String, $crate::Value>::new(); + $(map.insert(::std::string::String::from($name), $crate::Value::from($value));)+ + map + }}; +} diff --git a/v2/contextdb/src/memory/mod.rs b/v2/contextdb/src/memory/mod.rs new file mode 100644 index 000000000..719d1c545 --- /dev/null +++ b/v2/contextdb/src/memory/mod.rs @@ -0,0 +1,206 @@ +//! memory: the budget and the reservations taken from it. +//! +//! It never allocates on another part's behalf: it counts what the other parts say they are +//! about to take, and says no before they take it. + +use std::sync::atomic::{AtomicU64, Ordering::SeqCst}; +use std::sync::{Arc, Mutex, PoisonError}; + +use crate::types::{Error, Result}; + +#[cfg(test)] +mod tests; + +/// Bytes every budget sets aside for the small fixed things nobody reserves one by one: a +/// parsed statement and its plan, the table descriptions, a row on its way through. What keeps +/// the figure honest is the test that counts every byte the engine really takes +/// (`tests/memory_ceiling.rs`). +const ALLOWANCE: u64 = 1 << 20; + +/// How "no ceiling" is kept: a limit that no count of bytes can pass. +const NO_CEILING: u64 = u64::MAX; + +/// The declared memory limit of one open store, and how much of it is spoken for. +/// +/// Every figure is in bytes. One budget belongs to one writer or one reading session, and +/// cloning gives another handle to the same budget. The default is no limit: a writer has the +/// limit its store remembers, if one was ever declared, and a reader has one only if its open +/// names one. With no limit there is no ceiling, and what is reserved is still counted. +/// +/// Three kinds of memory are counted against it: small fixed things, covered by an allowance +/// of 1 MiB set aside in [`Budget::new`]; the page cache of 8 MiB, reserved once when the store +/// is opened; and everything that grows with data, which must hold a [`Reservation`]. A limit +/// below those 9 MiB therefore cannot open any store: the open is refused with +/// `MemoryBudgetExceeded`. +#[derive(Clone)] +pub(crate) struct Budget { + counts: Arc, +} + +/// What the handles of one budget share. +/// +/// Every read and write of the two numbers is sequentially consistent, because the limit and +/// the count are judged together: a reservation counts its bytes and then looks at the limit +/// again, a change of the limit publishes it and then looks at the count again, and so one of +/// the two always sees the other. +struct Counts { + /// The limit in bytes, `NO_CEILING` when none is declared. + limit: AtomicU64, + /// Bytes spoken for, to the byte. + used: AtomicU64, + /// Held while the limit changes, so that one change is settled before the next begins. + /// Asking for memory and giving it back never take it. + changing_limit: Mutex<()>, +} + +impl Budget { + /// A budget with this limit in bytes, or with no ceiling for `None`. + /// + /// Refuses with `MemoryBudgetExceeded` a limit too small to hold the fixed allowance. + pub(crate) fn new(limit: Option) -> Result { + let budget = Budget { + counts: Arc::new(Counts { + limit: AtomicU64::new(limit.unwrap_or(NO_CEILING)), + used: AtomicU64::new(0), + changing_limit: Mutex::new(()), + }), + }; + budget.take(ALLOWANCE, "open")?; + Ok(budget) + } + + /// Asks for `bytes` before they are allocated. + /// + /// Refuses with `MemoryBudgetExceeded`, naming `operation`, when the bytes would pass the + /// limit; nothing is counted on refusal. Never waits and never allocates: it costs a few + /// atomic operations, so it is cheap enough to call per row but is meant to be called per + /// batch. + pub(crate) fn reserve(&self, bytes: u64, operation: &'static str) -> Result { + self.take(bytes, operation)?; + Ok(Reservation { + budget: self.clone(), + bytes, + operation, + }) + } + + /// The declared limit in bytes, `None` when there is no ceiling. + pub(crate) fn limit(&self) -> Option { + let limit = self.counts.limit.load(SeqCst); + (limit != NO_CEILING).then_some(limit) + } + + /// Bytes spoken for now: the cache, the allowance and every live reservation. + pub(crate) fn used(&self) -> u64 { + self.counts.used.load(SeqCst) + } + + /// Changes the limit. Refuses a limit below what is already spoken for. + /// + /// Once this returns `Ok`, no more than the new limit is spoken for, for as long as that + /// limit stands. A reservation asked for at the very moment a lower limit is being set may + /// be refused although it would have fit. + pub(crate) fn set_limit(&self, limit: Option) -> Result<()> { + let counts = &*self.counts; + let new = limit.unwrap_or(NO_CEILING); + // The lock guards no data, so a holder that panicked left nothing half done. + let _one_change_at_a_time = counts + .changing_limit + .lock() + .unwrap_or_else(PoisonError::into_inner); + let old = counts.limit.load(SeqCst); + let mut used = counts.used.load(SeqCst); + if used <= new { + counts.limit.store(new, SeqCst); + // Bytes granted against the old limit since the look above are counted by now; + // bytes counted from here on are judged against the new limit. + used = counts.used.load(SeqCst); + if used <= new { + return Ok(()); + } + counts.limit.store(old, SeqCst); + } + Err(refused("set_limit", used, new, new)) + } + + /// Counts `bytes` as spoken for, or refuses and counts nothing. + /// + /// With no ceiling the only request refused is one that would carry the count past what a + /// `u64` holds, which no machine has. + fn take(&self, bytes: u64, operation: &'static str) -> Result<()> { + let counts = &*self.counts; + let mut used = counts.used.load(SeqCst); + loop { + let limit = counts.limit.load(SeqCst); + let available = limit.saturating_sub(used); + if bytes > available { + return Err(refused(operation, bytes, available, limit)); + } + match counts + .used + .compare_exchange_weak(used, used + bytes, SeqCst, SeqCst) + { + Ok(_) if counts.limit.load(SeqCst) == limit => return Ok(()), + // The limit changed between the look and the count: uncount, and ask again. + Ok(_) => used = counts.used.fetch_sub(bytes, SeqCst) - bytes, + Err(now) => used = now, + } + } + } + + fn give_back(&self, bytes: u64) { + self.counts.used.fetch_sub(bytes, SeqCst); + } +} + +/// The refusal of this part. Its labels are fixed words, so making it allocates nothing. +fn refused(operation: &'static str, requested: u64, available: u64, limit: u64) -> Error { + Error::MemoryBudgetExceeded { + subsystem: "memory", + operation, + requested_bytes: requested, + available_bytes: available, + budget_limit_bytes: limit, + hint: "select or write less at once, or raise the limit with SET MEMORY_LIMIT or the \ + memory_limit open option", + } +} + +/// Bytes a caller was granted and has not given back. +/// +/// The holder of memory that grows with data (a result, a sort, a transaction's changes) keeps +/// one of these beside the container and grows it before the container grows. Dropping it +/// gives everything back. +pub(crate) struct Reservation { + budget: Budget, + bytes: u64, + /// What the holder is doing, as a refusal to grow names it. + operation: &'static str, +} + +impl Reservation { + /// Asks for `bytes` more. On refusal the reservation keeps what it had. + pub(crate) fn grow(&mut self, bytes: u64) -> Result<()> { + self.budget.take(bytes, self.operation)?; + self.bytes += bytes; + Ok(()) + } + + /// Gives `bytes` back early, or everything held if that is less. + pub(crate) fn shrink(&mut self, bytes: u64) { + let bytes = bytes.min(self.bytes); + self.budget.give_back(bytes); + self.bytes -= bytes; + } + + /// Bytes held now. + pub(crate) fn bytes(&self) -> u64 { + self.bytes + } +} + +impl Drop for Reservation { + fn drop(&mut self) { + self.budget.give_back(self.bytes); + } +} diff --git a/v2/contextdb/src/memory/tests.rs b/v2/contextdb/src/memory/tests.rs new file mode 100644 index 000000000..03ea151e3 --- /dev/null +++ b/v2/contextdb/src/memory/tests.rs @@ -0,0 +1,85 @@ +use super::{ALLOWANCE, Budget}; +use crate::types::Error; + +/// A budget with `room` bytes left after its allowance, and what it counts as used when new. +fn budget_with_room(room: u64) -> (Budget, u64) { + let limit = ALLOWANCE + room; + let budget = Budget::new(Some(limit)).expect("a limit that holds the allowance"); + assert_eq!(budget.limit(), Some(limit)); + let baseline = budget.used(); + (budget, baseline) +} + +#[test] +fn bytes_are_counted_while_held_and_given_back_when_released() { + let (budget, baseline) = budget_with_room(100); + let mut held = budget.reserve(60, "sort").expect("60 of 100"); + held.grow(30).expect("30 of the 40 left"); + assert_eq!((held.bytes(), budget.used()), (90, baseline + 90)); + held.shrink(50); + assert_eq!((held.bytes(), budget.used()), (40, baseline + 40)); + // Another handle is the same budget. + let other = budget.clone().reserve(60, "result").expect("60 of 60"); + assert_eq!(budget.used(), baseline + 100); + drop(held); + drop(other); + assert_eq!(budget.used(), baseline); +} + +#[test] +fn a_refusal_names_what_was_asked_and_what_was_left_and_counts_nothing() { + let (budget, baseline) = budget_with_room(100); + let mut held = budget.reserve(60, "sort").expect("60 of 100"); + for refusal in [budget.reserve(41, "sort").err(), held.grow(41).err()] { + match refusal { + Some(Error::MemoryBudgetExceeded { + operation: "sort", + requested_bytes: 41, + available_bytes: 40, + budget_limit_bytes, + .. + }) => assert_eq!(budget_limit_bytes, ALLOWANCE + 100), + other => panic!("41 bytes of 40 gave {other:?}"), + } + } + assert_eq!((held.bytes(), budget.used()), (60, baseline + 60)); + held.grow(40).expect("exactly what was left"); +} + +#[test] +fn with_no_limit_every_reservation_succeeds_and_is_counted() { + let budget = Budget::new(None).expect("no ceiling"); + assert_eq!(budget.limit(), None); + let baseline = budget.used(); + let held = budget.reserve(1 << 50, "result").expect("no ceiling"); + assert_eq!(budget.used(), baseline + (1 << 50)); + drop(held); + assert_eq!(budget.used(), baseline); +} + +#[test] +fn a_limit_below_what_is_in_use_is_refused_and_the_old_one_stays() { + assert!(Budget::new(Some(ALLOWANCE - 1)).is_err()); + let (budget, baseline) = budget_with_room(100); + let held = budget.reserve(60, "result").expect("60 of 100"); + match budget.set_limit(Some(baseline + 59)) { + Err(Error::MemoryBudgetExceeded { + requested_bytes, + available_bytes, + .. + }) => assert_eq!( + (requested_bytes, available_bytes), + (baseline + 60, baseline + 59) + ), + other => panic!("a limit one byte below what is in use gave {other:?}"), + } + assert_eq!(budget.limit(), Some(ALLOWANCE + 100)); + budget + .set_limit(Some(baseline + 60)) + .expect("exactly what is in use"); + assert!(budget.reserve(1, "result").is_err()); + budget.set_limit(None).expect("no ceiling"); + assert_eq!(budget.limit(), None); + drop(held); + assert_eq!(budget.used(), baseline); +} diff --git a/v2/contextdb/src/query/access.rs b/v2/contextdb/src/query/access.rs new file mode 100644 index 000000000..bfe6428cd --- /dev/null +++ b/v2/contextdb/src/query/access.rs @@ -0,0 +1,387 @@ +//! Index choice: which rows of a table a statement has to read, and reading them. + +use std::collections::HashMap; +use std::ops::Bound; + +use super::expression::{column_index, eval}; +use super::{heap_bytes, row_overhead}; +use crate::memory::Reservation; +use crate::schema::Table; +use crate::sql::Expr; +use crate::sql::ast::{BinOp, SortDirection as Asked}; +use crate::sql::plan::SortKey; +use crate::store::{Cell, IndexId, Key, Read, Row}; +use crate::types::{ColumnType, Result, SortDirection, Value}; + +/// How the rows of one table are reached. +pub(super) enum Path { + /// The one row with this primary key. + Key(Key), + /// The table between two keys, in key order. + Rows(Bound, Bound), + /// The rows the entries of an index between two entries point at, in entry order. + Index(IndexId, Bound, Bound), +} + +/// A part of a filter that every kept row must pass, and that compares a column with a value +/// known before any row is read. +struct Term { + column: usize, + op: BinOp, + value: Value, +} + +/// Something that keeps the rows of a table in an order: the table itself, under its primary +/// key, or one of its indexes. +struct Ordered { + /// `None` for the table itself. + index: Option, + /// The columns it is ordered by, each with its direction. + columns: Vec<(usize, SortDirection)>, +} + +impl Ordered { + /// The key these leading values make: the bytes every key with these values begins with. + /// A row key is all ascending, which the store is told with no directions. + fn key(&self, values: &[Value]) -> Result { + let directions: Vec = match self.index { + Some(_) => self.columns[..values.len()].iter().map(|c| c.1).collect(), + None => Vec::new(), + }; + Key::from_values(values, &directions) + } +} + +/// The bound that ends a walk after the last key that begins with this one. +fn until_after(key: &Key) -> Bound { + key.prefix_end().map_or(Bound::Unbounded, Bound::Excluded) +} + +/// The bound that starts a walk after the last key that begins with this one. +fn starting_after(key: Key) -> Bound { + let after = key.prefix_end(); + after.map_or(Bound::Excluded(key), Bound::Included) +} + +/// The value as a key of a column of this type holds it, when there is one such value: only +/// then is looking the value up the same as comparing every row with it. +fn as_key(column: ColumnType, value: Value) -> Option { + match (column, value) { + (ColumnType::Integer, value @ Value::Int64(_)) + | (ColumnType::Real, value @ Value::Float64(_)) + | (ColumnType::Text, value @ Value::Text(_)) + | (ColumnType::Boolean, value @ Value::Bool(_)) + | (ColumnType::Uuid, value @ Value::Uuid(_)) + | (ColumnType::Timestamp, value @ Value::Timestamp(_)) => Some(value), + (ColumnType::Real, Value::Int64(value)) => Some(Value::Float64(value as f64)), + (ColumnType::Uuid, Value::Text(value)) => value.parse().ok().map(Value::Uuid), + _ => None, + } +} + +/// One table as one statement reads it. +pub(super) struct Source<'a> { + table: &'a Table, + filter: Option<&'a Expr>, + params: &'a HashMap, + now: i64, + read: &'a dyn Read, + /// The columns the filter reads: taken from every row that is read. + filtered: Vec, + /// The other columns the statement reads: taken only from a row that passed the filter. + rest: Vec, +} + +impl<'a> Source<'a> { + /// `filtered` marks the columns the filter reads and `wanted` every column the statement + /// reads after it. A column in neither is never taken from a row, so a large value in it + /// is never loaded. + pub(super) fn new( + table: &'a Table, + filter: Option<&'a Expr>, + params: &'a HashMap, + now: i64, + read: &'a dyn Read, + filtered: Vec, + wanted: &[bool], + ) -> Source<'a> { + let rest = wanted.iter().zip(&filtered).map(|(w, f)| *w && !*f); + Source { + table, + filter, + params, + now, + read, + rest: rest.collect(), + filtered, + } + } + + /// Picks the path that reads the fewest rows for the filter, and says whether its rows + /// arrive in the order asked for, so that they need no sort. + /// + /// The table and each index are tried in turn. One is better the more of its leading + /// columns the filter gives a value for, then if the filter gives a range of its next + /// column, then if it keeps the order asked for; the table itself wins a tie. The filter is + /// still applied to every row read, so a path only has to reach every row that can pass. + /// + /// Rows are kept under their primary key, so the key is answered from the table itself. Its + /// own index, `__pk_`, is declared and has no entries, and is never walked. + pub(super) fn choose(&self, order: &[SortKey]) -> Result<(Path, bool)> { + let mut terms = Vec::new(); + if let Some(filter) = self.filter { + self.collect(filter, &mut terms)?; + } + let columns = &self.table.meta.columns; + let column_named = |name: &String| columns.iter().position(|column| column.name == *name); + let key = self.key_columns(); + // The order asked for, when it is plain columns. + let asked: Option> = order + .iter() + .map(|key| { + let Expr::Column(column) = &key.expr else { + return None; + }; + let direction = match key.direction { + Asked::Asc => SortDirection::Asc, + Asked::Desc => SortDirection::Desc, + }; + Some((column_index(self.table, column).ok()?, direction)) + }) + .collect(); + + let the_table = Ordered { + index: None, + columns: key.iter().map(|at| (*at, SortDirection::Asc)).collect(), + }; + let indexes = self.table.meta.indexes.iter().zip(&self.table.index_ids); + let indexes = indexes + .filter(|(index, _)| !index.name.starts_with("__pk_")) + .map(|(index, id)| Ordered { + index: Some(*id), + columns: index + .columns + .iter() + .filter_map(|(name, direction)| Some((column_named(name)?, *direction))) + .collect(), + }); + + // Until something better is found: the whole table, in no order that was asked for. + let mut best = ( + (0, false, false), + Path::Rows(Bound::Unbounded, Bound::Unbounded), + ); + for ordered in std::iter::once(the_table).chain(indexes) { + let given: Vec = ordered + .columns + .iter() + .map_while(|(column, _)| { + let equal = |term: &&Term| term.column == *column && term.op == BinOp::Eq; + Some(terms.iter().find(equal)?.value.clone()) + }) + .collect(); + let next = &ordered.columns[given.len()..]; + if ordered.index.is_none() && next.is_empty() { + return Ok((Path::Key(ordered.key(&given)?), true)); + } + let (mut from, mut to) = (Bound::Unbounded, Bound::Unbounded); + if !given.is_empty() { + let all = ordered.key(&given)?; + to = until_after(&all); + from = Bound::Included(all); + } + let mut ranged = false; + if let Some((column, direction)) = next.first() { + let with = |value: &Value| { + let mut values = given.clone(); + values.push(value.clone()); + ordered.key(&values) + }; + let ranges = terms.iter().filter(|term| { + let range = matches!(term.op, BinOp::Gt | BinOp::Gte | BinOp::Lt | BinOp::Lte); + term.column == *column && range + }); + for term in ranges { + if !ranged { + // No NULL is in a range. A key keeps NULL after every value going up + // and before every value going down, and the walk stops short of it. + let null = with(&Value::Null)?; + match direction { + SortDirection::Asc => to = Bound::Excluded(null), + SortDirection::Desc => from = starting_after(null), + } + ranged = true; + } + let key = with(&term.value)?; + // A descending column keeps its greater values first. When more columns + // follow, many keys begin with this one, and "after it" is after them all. + match (term.op, direction) { + (BinOp::Gt, SortDirection::Asc) | (BinOp::Lt, SortDirection::Desc) => { + from = starting_after(key); + } + (BinOp::Gte, SortDirection::Asc) | (BinOp::Lte, SortDirection::Desc) => { + from = Bound::Included(key); + } + (BinOp::Lt, SortDirection::Asc) | (BinOp::Gt, SortDirection::Desc) => { + to = Bound::Excluded(key); + } + // `<=` going up and `>=` going down. + _ => to = until_after(&key), + } + } + } + // A key keeps its columns as ORDER BY sorts them, NULL included, so what follows + // the columns the filter fixed is in order if it is what was asked for. + let in_order = asked.as_ref().is_some_and(|asked| next.starts_with(asked)); + let score = (given.len(), ranged, in_order); + if score > best.0 { + let path = match ordered.index { + Some(index) => Path::Index(index, from, to), + None => Path::Rows(from, to), + }; + best = (score, path); + } + } + let ((_, _, in_order), path) = best; + Ok((path, in_order)) + } + + /// Where the columns of the primary key sit in a row. + pub(super) fn key_columns(&self) -> Vec { + let columns = &self.table.meta.columns; + (0..columns.len()) + .filter(|at| columns[*at].primary_key) + .collect() + } + + /// Gathers the terms of a filter: the comparisons joined by AND at its top. + fn collect(&self, filter: &Expr, terms: &mut Vec) -> Result<()> { + let Expr::BinaryOp { left, op, right } = filter else { + return Ok(()); + }; + if *op == BinOp::And { + self.collect(left, terms)?; + return self.collect(right, terms); + } + let (column, value, op) = match (&**left, &**right) { + (Expr::Column(column), value @ (Expr::Literal(_) | Expr::Parameter(_))) => { + (column, value, *op) + } + // `5 < id` is `id > 5`. + (value @ (Expr::Literal(_) | Expr::Parameter(_)), Expr::Column(column)) => { + let turned = match op { + BinOp::Lt => BinOp::Gt, + BinOp::Lte => BinOp::Gte, + BinOp::Gt => BinOp::Lt, + BinOp::Gte => BinOp::Lte, + other => *other, + }; + (column, value, turned) + } + _ => return Ok(()), + }; + let column = column_index(self.table, column)?; + let value = eval(value, self.table, &[], self.params, self.now)?; + if let Some(value) = as_key(self.table.meta.columns[column].column_type, value) { + terms.push(Term { column, op, value }); + } + Ok(()) + } + + /// Reads the rows the path reaches and hands `each` every one that passes the filter: its + /// key, its values, and the bytes of `reservation` it holds. Stops when `each` answers + /// false. Returns how many stored rows were read. + pub(super) fn walk( + &self, + path: &Path, + reservation: &mut Reservation, + mut each: impl FnMut(Key, Vec, u64, &mut Reservation) -> Result, + ) -> Result { + let mut examined = 0; + let mut offer = |key: Key, row: Row| { + examined += 1; + let before = reservation.bytes(); + match self.values(row, reservation)? { + Some(values) => { + let bytes = reservation.bytes() - before; + each(key, values, bytes, reservation) + } + None => { + reservation.shrink(reservation.bytes() - before); + Ok(true) + } + } + }; + let table = self.table.id; + match path { + Path::Key(key) => { + if let Some(row) = self.read.row(table, key)? { + offer(key.clone(), row)?; + } + } + Path::Rows(from, to) => { + for item in self.read.rows(table, from.as_ref(), to.as_ref())? { + let (key, row) = item?; + if !offer(key, row)? { + break; + } + } + } + Path::Index(index, from, to) => { + for entry in self.read.index(*index, from.as_ref(), to.as_ref())? { + let (_, key) = entry?; + if let Some(row) = self.read.row(table, &key)? + && !offer(key, row)? + { + break; + } + } + } + } + Ok(examined) + } + + /// The values of a stored row if it passes the filter, charged to `reservation`. A column + /// the statement does not read is left as NULL. + fn values(&self, row: Row, reservation: &mut Reservation) -> Result>> { + let mut cells = row.cells; + reservation.grow(row_overhead(cells.len()))?; + let mut values = vec![Value::Null; cells.len()]; + self.take(&mut cells, &mut values, &self.filtered, reservation)?; + if let Some(filter) = self.filter + && eval(filter, self.table, &values, self.params, self.now)? != Value::Bool(true) + { + return Ok(None); + } + self.take(&mut cells, &mut values, &self.rest, reservation)?; + Ok(Some(values)) + } + + /// Moves the marked cells into `values`, loading a value kept beside its row. The memory + /// of the values is asked for before any is moved or loaded. + fn take( + &self, + cells: &mut [Cell], + values: &mut [Value], + marked: &[bool], + reservation: &mut Reservation, + ) -> Result<()> { + let here = cells.iter().zip(marked).map(|(cell, marked)| match cell { + Cell::Here(here) if *marked => heap_bytes(here), + _ => 0, + }); + reservation.grow(here.sum())?; + for ((cell, value), _) in cells + .iter_mut() + .zip(values) + .zip(marked) + .filter(|(_, marked)| **marked) + { + *value = match std::mem::replace(cell, Cell::Here(Value::Null)) { + Cell::Here(here) => here, + Cell::Beside(beside) => self.read.load(&beside, reservation)?, + }; + } + Ok(()) + } +} diff --git a/v2/contextdb/src/query/expression.rs b/v2/contextdb/src/query/expression.rs new file mode 100644 index 000000000..c0a95719a --- /dev/null +++ b/v2/contextdb/src/query/expression.rs @@ -0,0 +1,318 @@ +//! The expression evaluator, and how two values compare and sort. + +use std::cmp::Ordering; +use std::collections::HashMap; + +use super::heap_bytes; +use crate::schema::Table; +use crate::sql::Expr; +use crate::sql::ast::{BinOp, ColumnRef, Literal, UnaryOp}; +use crate::types::{Error, Result, Uuid, Value}; + +/// Works out one expression against one row. The only expression evaluator in the library. +/// +/// `row` holds one value per column of `table`, in declared order, and `now` is the statement's +/// one time in milliseconds, which `NOW()` answers with. A comparison with NULL, and one between +/// values of types that do not compare, is unknown: it gives NULL, and a filter keeps a row +/// only for TRUE. +pub(crate) fn eval( + expr: &Expr, + table: &Table, + row: &[Value], + params: &HashMap, + now: i64, +) -> Result { + let value = |expr: &Expr| eval(expr, table, row, params, now); + Ok(match expr { + Expr::Column(column) => { + let at = column_index(table, column)?; + row.get(at).cloned().ok_or_else(|| missing(table, column))? + } + Expr::Literal(literal) => match literal { + Literal::Null => Value::Null, + Literal::Bool(value) => Value::Bool(*value), + Literal::Integer(value) => Value::Int64(*value), + Literal::Real(value) => Value::Float64(*value), + Literal::Text(value) => Value::Text(value.clone()), + Literal::Vector(value) => Value::Vector(value.clone()), + }, + Expr::Parameter(name) => params.get(name).cloned().ok_or_else(|| unbound(name))?, + Expr::BinaryOp { left, op, right } => { + let holds: fn(Ordering) -> bool = match op { + BinOp::And | BinOp::Or => { + // The value that decides the answer alone: FALSE for AND, TRUE for OR. + let settles = *op == BinOp::Or; + let left = truth(&value(left)?); + if left == Some(settles) { + return Ok(Value::Bool(settles)); + } + return Ok(match (left, truth(&value(right)?)) { + (_, Some(right)) if right == settles => Value::Bool(settles), + (Some(_), Some(_)) => Value::Bool(!settles), + _ => Value::Null, + }); + } + BinOp::Eq => Ordering::is_eq, + BinOp::Neq => Ordering::is_ne, + BinOp::Lt => Ordering::is_lt, + BinOp::Lte => Ordering::is_le, + BinOp::Gt => Ordering::is_gt, + BinOp::Gte => Ordering::is_ge, + }; + compare(&value(left)?, &value(right)?) + .map_or(Value::Null, |order| Value::Bool(holds(order))) + } + Expr::UnaryOp { op, operand } => match (op, value(operand)?) { + (UnaryOp::Not, value) => truth(&value).map_or(Value::Null, |truth| Value::Bool(!truth)), + (UnaryOp::Neg, Value::Null) => Value::Null, + (UnaryOp::Neg, Value::Int64(value)) => { + Value::Int64(value.checked_neg().ok_or_else(out_of_range)?) + } + (UnaryOp::Neg, Value::Float64(value)) => Value::Float64(-value), + (UnaryOp::Neg, _) => { + return Err(Error::PlanError( + "cannot negate a value that is not a number".to_owned(), + )); + } + }, + Expr::IsNull { expr, negated } => Value::Bool((value(expr)? == Value::Null) != *negated), + Expr::FunctionCall { name, args } => match (scalar(name), args.as_slice()) { + (Some(Scalar::Arithmetic(op)), [left, right]) => { + arithmetic(op, &value(left)?, &value(right)?)? + } + (Some(Scalar::Now), []) => Value::Timestamp(now), + (Some(Scalar::Coalesce), args) => { + for arg in args { + let first = value(arg)?; + if first != Value::Null { + return Ok(first); + } + } + Value::Null + } + _ => return Err(refused(name)), + }, + }) +} + +/// The most memory the value of an expression can take outside its own cell. It is known +/// before the expression is worked out, so the memory is asked for before the value is made. +pub(super) fn most_bytes( + expr: &Expr, + table: &Table, + row: &[Value], + params: &HashMap, +) -> u64 { + match expr { + Expr::Column(column) => { + let at = column_index(table, column).ok(); + at.and_then(|at| row.get(at)).map_or(0, heap_bytes) + } + Expr::Literal(Literal::Text(text)) => text.len() as u64, + Expr::Literal(Literal::Vector(vector)) => size_of_val(vector.as_slice()) as u64, + Expr::Parameter(name) => params.get(name).map_or(0, heap_bytes), + Expr::FunctionCall { name, args } if matches!(scalar(name), Some(Scalar::Coalesce)) => { + let each = args.iter().map(|arg| most_bytes(arg, table, row, params)); + each.max().unwrap_or(0) + } + // Everything else gives a number, a truth, a time or NULL. + _ => 0, + } +} + +/// Marks in `read` each column the expression reads. +/// +/// Refuses, before any row is read, what [`eval`] would refuse at the first row: a column the +/// table does not have, a `$name` nothing is bound to, and a form this version does not work +/// out. A statement is therefore refused or not whatever rows its table holds. +pub(super) fn reads( + expr: &Expr, + table: &Table, + params: &HashMap, + read: &mut [bool], +) -> Result<()> { + match expr { + Expr::Column(column) => read[column_index(table, column)?] = true, + Expr::Literal(_) => {} + Expr::Parameter(name) => { + if !params.contains_key(name) { + return Err(unbound(name)); + } + } + Expr::BinaryOp { left, right, .. } => { + reads(left, table, params, read)?; + reads(right, table, params, read)?; + } + Expr::UnaryOp { operand, .. } | Expr::IsNull { expr: operand, .. } => { + reads(operand, table, params, read)?; + } + Expr::FunctionCall { name, args } => { + if scalar(name).is_none() { + return Err(refused(name)); + } + for arg in args { + reads(arg, table, params, read)?; + } + } + } + Ok(()) +} + +/// Where a column sits in a row of its table. A name the table does not have, and a name +/// qualified with anything but the table's own name, is `ColumnNotFound`. +pub(super) fn column_index(table: &Table, column: &ColumnRef) -> Result { + let in_scope = column.table.as_ref().is_none_or(|name| *name == table.name); + table + .meta + .columns + .iter() + .position(|declared| in_scope && declared.name == column.column) + .ok_or_else(|| missing(table, column)) +} + +fn missing(table: &Table, column: &ColumnRef) -> Error { + Error::ColumnNotFound { + table: table.name.clone(), + column: column.column.clone(), + } +} + +fn unbound(name: &str) -> Error { + Error::PlanError(format!("no value is bound to ${name}")) +} + +fn out_of_range() -> Error { + Error::PlanError("integer out of range".to_owned()) +} + +/// The refusal of something this version of the library does not do, by its name. +pub(super) fn unsupported(what: &str) -> Error { + Error::PlanError(format!( + "{what} is not supported in this version of the library" + )) +} + +/// The refusal of a call [`eval`] does not work out. +fn refused(name: &str) -> Error { + unsupported(&format!("the function {name}")) +} + +/// A function that gives one value for one row. +#[derive(Clone, Copy)] +pub(super) enum Scalar { + Arithmetic(Arithmetic), + Now, + /// The first of its values that is not NULL. + Coalesce, +} + +/// The four operators as the parser names them: `a + b` arrives as the call `__add(a, b)`. +#[derive(Clone, Copy)] +pub(super) enum Arithmetic { + Add, + Sub, + Mul, + Div, +} + +/// The scalar function of this name, if it is one. +pub(super) fn scalar(name: &str) -> Option { + [ + ("__add", Scalar::Arithmetic(Arithmetic::Add)), + ("__sub", Scalar::Arithmetic(Arithmetic::Sub)), + ("__mul", Scalar::Arithmetic(Arithmetic::Mul)), + ("__div", Scalar::Arithmetic(Arithmetic::Div)), + ("now", Scalar::Now), + ("coalesce", Scalar::Coalesce), + ] + .into_iter() + .find(|(known, _)| name.eq_ignore_ascii_case(known)) + .map(|(_, scalar)| scalar) +} + +/// Arithmetic as standard SQL does it: NULL in gives NULL out, two whole numbers give a whole +/// number or are refused when it would not fit, and a REAL on either side gives a REAL. +/// Dividing by zero is refused. +pub(super) fn arithmetic(op: Arithmetic, left: &Value, right: &Value) -> Result { + let by_zero = || Error::PlanError("division by zero".to_owned()); + Ok(match (left, right) { + (Value::Null, _) | (_, Value::Null) => Value::Null, + (Value::Int64(left), Value::Int64(right)) => Value::Int64( + match op { + Arithmetic::Add => left.checked_add(*right), + Arithmetic::Sub => left.checked_sub(*right), + Arithmetic::Mul => left.checked_mul(*right), + Arithmetic::Div if *right == 0 => return Err(by_zero()), + Arithmetic::Div => left.checked_div(*right), + } + .ok_or_else(out_of_range)?, + ), + _ => { + let (Some(left), Some(right)) = (real(left), real(right)) else { + return Err(Error::PlanError( + "arithmetic needs numbers on both sides".to_owned(), + )); + }; + Value::Float64(match op { + Arithmetic::Add => left + right, + Arithmetic::Sub => left - right, + Arithmetic::Mul => left * right, + Arithmetic::Div if right == 0.0 => return Err(by_zero()), + Arithmetic::Div => left / right, + }) + } + }) +} + +fn real(value: &Value) -> Option { + match value { + Value::Int64(value) => Some(*value as f64), + Value::Float64(value) => Some(*value), + _ => None, + } +} + +/// TRUE or FALSE, or `None` for NULL and for anything that is not a boolean. +fn truth(value: &Value) -> Option { + match value { + Value::Bool(value) => Some(*value), + _ => None, + } +} + +/// How two values compare, or `None` when either is NULL or their types do not compare. +/// +/// A REAL has one rule for `=`, ORDER BY and the bytes of a key: the two zeros are one value, +/// and every NaN is one value, equal to itself and above positive infinity. A whole number +/// compares with a REAL as a REAL, and a UUID with the text that spells one. +pub(super) fn compare(left: &Value, right: &Value) -> Option { + let reals = |left: f64, right: f64| { + let unordered = || left.is_nan().cmp(&right.is_nan()); + left.partial_cmp(&right).unwrap_or_else(unordered) + }; + Some(match (left, right) { + (Value::Int64(left), Value::Int64(right)) => left.cmp(right), + (Value::Float64(left), Value::Float64(right)) => reals(*left, *right), + (Value::Int64(left), Value::Float64(right)) => reals(*left as f64, *right), + (Value::Float64(left), Value::Int64(right)) => reals(*left, *right as f64), + (Value::Text(left), Value::Text(right)) => left.cmp(right), + (Value::Bool(left), Value::Bool(right)) => left.cmp(right), + (Value::Timestamp(left), Value::Timestamp(right)) => left.cmp(right), + (Value::TxId(left), Value::TxId(right)) => left.cmp(right), + (Value::Uuid(left), Value::Uuid(right)) => left.cmp(right), + (Value::Uuid(left), Value::Text(right)) => left.cmp(&right.parse().ok()?), + (Value::Text(left), Value::Uuid(right)) => left.parse::().ok()?.cmp(right), + _ => return None, + }) +} + +/// How two values sort going up: as they compare, with NULL after every value. Going down is +/// the reverse, which puts NULL first. +pub(super) fn sort_order(left: &Value, right: &Value) -> Ordering { + match (left, right) { + (Value::Null, Value::Null) => Ordering::Equal, + (Value::Null, _) => Ordering::Greater, + (_, Value::Null) => Ordering::Less, + _ => compare(left, right).unwrap_or(Ordering::Equal), + } +} diff --git a/v2/contextdb/src/query/mod.rs b/v2/contextdb/src/query/mod.rs new file mode 100644 index 000000000..414a7587a --- /dev/null +++ b/v2/contextdb/src/query/mod.rs @@ -0,0 +1,538 @@ +//! query: answers a SELECT from one table: which rows to read and through which index, the +//! filter, the select list, COUNT, SUM, MIN and MAX, ORDER BY and LIMIT. It holds the one +//! expression evaluator, and counts exactly the rows it reads. +//! +//! It never changes anything. + +mod access; +mod expression; +#[cfg(test)] +mod tests; + +use std::cmp::Ordering; +use std::collections::HashMap; +use std::fmt; + +pub(crate) use expression::eval; + +use crate::memory::{Budget, Reservation}; +use crate::schema::{Catalog, Table}; +use crate::sql::ast::SortDirection; +use crate::sql::plan::{ProjectColumn, SortKey}; +use crate::sql::{Expr, Plan}; +use crate::store::{Key, Read}; +use crate::types::{ColumnType, Error, Result, Value}; +use access::Source; +use expression::{ + Arithmetic, arithmetic, column_index, compare, most_bytes, reads, sort_order, unsupported, +}; + +/// What a statement returned: the column names and rows of a SELECT, or the number of rows a +/// write affected. Its rows stay counted against the memory limit until it is dropped. +pub struct QueryResult { + columns: Vec, + rows: Vec>, + rows_affected: u64, + rows_examined: u64, + /// Never read: held so that the memory the rows take stays counted against the limit, and + /// dropping it with the result gives that memory back. + _reservation: Option, +} + +impl fmt::Debug for QueryResult { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("QueryResult") + .field("columns", &self.columns) + .field("rows", &self.rows) + .field("rows_affected", &self.rows_affected) + .field("rows_examined", &self.rows_examined) + .finish_non_exhaustive() + } +} + +impl QueryResult { + /// The column names of a SELECT, in order; empty for a write. + pub fn columns(&self) -> &[String] { + &self.columns + } + + /// The rows of a SELECT, each a list of values in column order; empty for a write. + pub fn rows(&self) -> &[Vec] { + &self.rows + } + + /// How many rows a write changed; zero for a SELECT. + pub fn rows_affected(&self) -> u64 { + self.rows_affected + } + + /// How many stored rows the statement read to produce this result. An exact count, the same + /// on every run over the same store: a whole-table `COUNT(*)` reads none, a lookup by key + /// reads one. Speed is guarded in tests by this number, never by a time. + pub fn rows_examined(&self) -> u64 { + self.rows_examined + } + + /// The result of a write that changed `rows` rows. + pub(crate) fn affected(rows: u64) -> QueryResult { + QueryResult { + rows_affected: rows, + ..QueryResult::from_rows(Vec::new(), Vec::new()) + } + } + + /// The result of a statement its caller answers itself, such as `SHOW MEMORY_LIMIT`: these + /// columns and rows, with no row affected and none examined. + pub(crate) fn from_rows(columns: Vec, rows: Vec>) -> QueryResult { + QueryResult { + columns, + rows, + rows_affected: 0, + rows_examined: 0, + _reservation: None, + } + } +} + +/// What `SHOW MEMORY_LIMIT` answers for one budget, a writer's or a reading session's. +/// +/// One row: `limit`, `used` and `available` in bytes; `startup_ceiling`, the limit the writer or +/// the session was opened under; and `ceiling_refresh_failures`, which is always 0 because this +/// engine reads no ceiling from the operating system. No limit is the text `none`, and a number +/// of bytes larger than an INTEGER column holds is shown as the largest it holds. +pub(crate) fn memory_limit_shown(budget: &Budget, opened_under: Option) -> QueryResult { + let bytes = |bytes: Option| match bytes { + Some(bytes) => Value::Int64(i64::try_from(bytes).unwrap_or(i64::MAX)), + None => Value::Text("none".to_owned()), + }; + let (limit, used) = (budget.limit(), budget.used()); + let columns = [ + "limit", + "used", + "available", + "startup_ceiling", + "ceiling_refresh_failures", + ]; + let row = vec![ + bytes(limit), + bytes(Some(used)), + bytes(limit.map(|limit| limit.saturating_sub(used))), + bytes(opened_under), + Value::Int64(0), + ]; + QueryResult::from_rows(columns.map(str::to_owned).to_vec(), vec![row]) +} + +/// Answers one planned SELECT from one reader. +/// +/// Picks an index when the plan's filter can use one and scans otherwise; `COUNT(*)` of a whole +/// table is answered from the kept count. Loads a value kept beside its row only when a selected +/// or filtered column needs it. The result, and any sort or grouping on the way to it, grows +/// through reservations on `budget` and refuses with `MemoryBudgetExceeded` before it grows. +pub(crate) fn run( + plan: &Plan, + params: &HashMap, + catalog: &Catalog, + read: &dyn Read, + budget: &Budget, + now: i64, +) -> Result { + let select = Select::of(plan)?; + let table = catalog + .table(select.table) + .ok_or_else(|| Error::TableNotFound(select.table.to_owned()))?; + + // Everything the statement names is looked up before a row is read. + let width = table.meta.columns.len(); + let (mut filtered, mut wanted) = (vec![false; width], vec![false; width]); + if let Some(filter) = select.filter { + reads(filter, table, params, &mut filtered)?; + } + for key in select.order { + reads(&key.expr, table, params, &mut wanted)?; + } + let mut aggregates = Vec::new(); + let columns = match select.columns { + None => { + wanted.fill(true); + let declared = table.meta.columns.iter(); + declared.map(|column| column.name.clone()).collect() + } + Some(columns) => { + for column in columns { + let Some(aggregate) = Aggregate::of(&column.expr)? else { + reads(&column.expr, table, params, &mut wanted)?; + continue; + }; + if let Some(of) = aggregate.of { + reads(of, table, params, &mut wanted)?; + aggregate.check(table, of)?; + } + aggregates.push(aggregate); + } + if !aggregates.is_empty() && aggregates.len() < columns.len() { + return Err(unsupported("a column beside an aggregate (GROUP BY)")); + } + columns + .iter() + .map(|column| match (&column.alias, &column.expr) { + (Some(alias), _) => alias.clone(), + (None, Expr::Column(column)) => column.column.clone(), + (None, Expr::FunctionCall { name, .. }) if !aggregates.is_empty() => { + name.clone() + } + (None, _) => "expr".to_owned(), + }) + .collect() + } + }; + + let source = Source::new(table, select.filter, params, now, read, filtered, &wanted); + let limit = select.limit.map_or(usize::MAX, |limit| { + usize::try_from(limit).unwrap_or(usize::MAX) + }); + let mut reservation = budget.reserve(0, "query")?; + let mut rows = Vec::new(); + let mut examined = 0; + + if !aggregates.is_empty() { + if select.filter.is_none() && aggregates.iter().all(|aggregate| aggregate.of.is_none()) { + let count = read.count(table.id)? as i64; + for aggregate in &mut aggregates { + aggregate.seen = count; + } + } else { + let (path, _) = source.choose(&[])?; + examined = source.walk(&path, &mut reservation, |_, row, bytes, reservation| { + for aggregate in &mut aggregates { + aggregate.add(table, &row, params, now, reservation)?; + } + reservation.shrink(bytes); + Ok(true) + })?; + } + if limit > 0 { + // What a least or greatest value takes is already held. + reservation.grow(row_overhead(aggregates.len()))?; + rows.push(aggregates.into_iter().map(Aggregate::answer).collect()); + } + } else if limit > 0 { + // The select list worked out for one row. The memory of its values is asked for before + // they are made, and the row's bytes are given back once they are. + let output = + |row: Vec, bytes: u64, reservation: &mut Reservation| -> Result> { + let Some(columns) = select.columns else { + return Ok(row); + }; + let most = columns + .iter() + .map(|column| most_bytes(&column.expr, table, &row, params)); + reservation.grow(row_overhead(columns.len()) + most.sum::())?; + let values = columns + .iter() + .map(|column| eval(&column.expr, table, &row, params, now)) + .collect::>>()?; + reservation.shrink(bytes); + Ok(values) + }; + let (path, in_order) = source.choose(select.order)?; + if in_order { + examined = source.walk(&path, &mut reservation, |_, row, bytes, reservation| { + rows.push(output(row, bytes, reservation)?); + Ok(rows.len() < limit) + })?; + } else { + let mut sorting: Vec = Vec::new(); + let mut key_bytes = 0; + examined = source.walk(&path, &mut reservation, |_, row, bytes, reservation| { + let order = select.order; + let most = order + .iter() + .map(|key| most_bytes(&key.expr, table, &row, params)); + let held = row_overhead(order.len()) + most.sum::(); + reservation.grow(held + size_of::() as u64)?; + key_bytes += held + size_of::() as u64; + let keys = order + .iter() + .map(|key| eval(&key.expr, table, &row, params, now)) + .collect::>>()?; + sorting.push(Sorting { keys, row, bytes }); + Ok(true) + })?; + sorting.sort_by(|a, b| { + let keys = a.keys.iter().zip(&b.keys).zip(select.order); + keys.map(|((a, b), key)| match key.direction { + SortDirection::Desc => sort_order(a, b).reverse(), + _ => sort_order(a, b), + }) + .find(|order| order.is_ne()) + .unwrap_or(Ordering::Equal) + }); + for cut in sorting.drain(limit.min(sorting.len())..) { + reservation.shrink(cut.bytes); + } + for kept in sorting { + rows.push(output(kept.row, kept.bytes, &mut reservation)?); + } + reservation.shrink(key_bytes); + } + } + + Ok(QueryResult { + columns, + rows, + rows_affected: 0, + rows_examined: examined, + _reservation: Some(reservation), + }) +} + +/// The rows [`matching`] found, each with its key. +pub(crate) type Matched = Vec<(Key, Vec)>; + +/// The rows of one table that pass a filter, with their keys: what an UPDATE or DELETE acts on. +/// Uses the same index choice and the same evaluator as [`run`]. The rows and their keys stay +/// counted against the memory limit for as long as the caller holds the reservation that comes +/// with them. +pub(crate) fn matching( + table: &Table, + filter: Option<&Expr>, + params: &HashMap, + read: &dyn Read, + budget: &Budget, + now: i64, +) -> Result<(Matched, Reservation)> { + let width = table.meta.columns.len(); + let mut filtered = vec![false; width]; + if let Some(filter) = filter { + reads(filter, table, params, &mut filtered)?; + } + let source = Source::new( + table, + filter, + params, + now, + read, + filtered, + &vec![true; width], + ); + let (path, _) = source.choose(&[])?; + let mut reservation = budget.reserve(0, "query")?; + let mut found = Vec::new(); + let key_columns = source.key_columns(); + source.walk(&path, &mut reservation, |key, row, _, reservation| { + // A key is its row's key values once more, and is asked for as what they take. + let values = key_columns.iter().map(|at| heap_bytes(&row[*at])); + reservation.grow(row_overhead(key_columns.len()) + values.sum::())?; + found.push((key, row)); + Ok(true) + })?; + Ok((found, reservation)) +} + +/// One planned SELECT, as the planner nests it: LIMIT over the select list over ORDER BY over +/// one table with its filter. +struct Select<'a> { + table: &'a str, + filter: Option<&'a Expr>, + order: &'a [SortKey], + /// `None` for `SELECT *`. + columns: Option<&'a [ProjectColumn]>, + limit: Option, +} + +impl<'a> Select<'a> { + /// Reads the shape out of a plan, refusing by name a plan this version does not answer. + fn of(mut plan: &'a Plan) -> Result> { + let (mut limit, mut columns) = (None, None); + let mut order: &[SortKey] = &[]; + if let Plan::Limit { input, count } = plan { + limit = Some(*count); + plan = &**input; + } + if let Plan::Project { + input, + columns: list, + } = plan + { + columns = Some(list.as_slice()); + plan = &**input; + } + if let Plan::Sort { input, keys } = plan { + order = keys; + plan = &**input; + } + let Plan::Scan { table, filter } = plan else { + return Err(unsupported("this statement")); + }; + let (table, filter) = (table.as_str(), filter.as_ref()); + Ok(Select { + table, + filter, + order, + columns, + limit, + }) + } +} + +/// A row waiting to be sorted: the values it sorts by, and the bytes the row holds of the +/// statement's reservation. +struct Sorting { + keys: Vec, + row: Vec, + bytes: u64, +} + +/// One aggregate of a select list, and its answer over the rows seen so far. +struct Aggregate<'a> { + function: Function, + /// What is counted, added or compared; `None` for `COUNT(*)`. + of: Option<&'a Expr>, + /// How many rows took part: every row for `COUNT(*)`, otherwise those where `of` is not NULL. + seen: i64, + /// The sum, the least or the greatest so far; NULL until a row takes part. + value: Value, + /// The bytes of the statement's reservation that `value` holds. + held: u64, +} + +#[derive(Clone, Copy, PartialEq)] +enum Function { + Count, + Sum, + Min, + Max, +} + +impl<'a> Aggregate<'a> { + /// The aggregate an item of a select list is, if it is one. + fn of(expr: &'a Expr) -> Result>> { + let Expr::FunctionCall { name, args } = expr else { + return Ok(None); + }; + let functions = [ + ("COUNT", Function::Count), + ("SUM", Function::Sum), + ("MIN", Function::Min), + ("MAX", Function::Max), + ]; + let Some((_, function)) = functions + .into_iter() + .find(|(known, _)| name.eq_ignore_ascii_case(known)) + else { + return Ok(None); + }; + let [of] = args.as_slice() else { + return Err(Error::PlanError(format!("{name} takes one argument"))); + }; + let every_row = function == Function::Count + && matches!(of, Expr::Column(column) if column.column == "*"); + Ok(Some(Aggregate { + function, + of: (!every_row).then_some(of), + seen: 0, + value: Value::Null, + held: 0, + })) + } + + /// Refuses a SUM over a column that does not hold numbers, whatever rows the table has. + fn check(&self, table: &Table, of: &Expr) -> Result<()> { + let Expr::Column(column) = of else { + return Ok(()); + }; + let column = &table.meta.columns[column_index(table, column)?]; + if self.function == Function::Sum + && !matches!(column.column_type, ColumnType::Integer | ColumnType::Real) + { + return Err(Error::PlanError(format!( + "SUM needs numbers, and the column {} holds none", + column.name + ))); + } + Ok(()) + } + + /// Takes one row into the answer. A NULL takes no part in any aggregate but `COUNT(*)`. + /// The memory of the value is asked for before it is made, and given back unless the value + /// is kept as the least or the greatest so far. + fn add( + &mut self, + table: &Table, + row: &[Value], + params: &HashMap, + now: i64, + reservation: &mut Reservation, + ) -> Result<()> { + if let Some(of) = self.of { + let most = most_bytes(of, table, row, params); + reservation.grow(most)?; + let value = eval(of, table, row, params, now)?; + if value == Value::Null { + reservation.shrink(most); + return Ok(()); + } + let first = self.value == Value::Null; + let better = match self.function { + Function::Count => None, + Function::Sum => { + let so_far = if first { &Value::Int64(0) } else { &self.value }; + self.value = arithmetic(Arithmetic::Add, so_far, &value)?; + None + } + Function::Min => Some(Ordering::Less), + Function::Max => Some(Ordering::Greater), + }; + if better.is_some_and(|better| first || compare(&value, &self.value) == Some(better)) { + reservation.shrink(self.held); + (self.value, self.held) = (value, most); + } else { + reservation.shrink(most); + } + } + self.seen += 1; + Ok(()) + } + + /// The answer over the rows taken: a count of nothing is zero, and every other aggregate of + /// nothing is NULL. + fn answer(self) -> Value { + match self.function { + Function::Count => Value::Int64(self.seen), + _ => self.value, + } + } +} + +/// The memory a row of this many values takes before any text, JSON or vector in it. +fn row_overhead(values: usize) -> u64 { + (size_of::>() + values * size_of::()) as u64 +} + +/// The memory a value takes outside its own cell. +fn heap_bytes(value: &Value) -> u64 { + fn json(value: &serde_json::Value) -> u64 { + let cell = size_of::(); + match value { + serde_json::Value::String(text) => text.len() as u64, + serde_json::Value::Array(items) => { + let items = items.iter(); + items.map(|item| cell as u64 + json(item)).sum() + } + serde_json::Value::Object(members) => { + let members = members.iter(); + members + .map(|(name, member)| (name.len() + cell) as u64 + json(member)) + .sum() + } + _ => 0, + } + } + match value { + Value::Text(text) => text.len() as u64, + Value::Vector(vector) => size_of_val(vector.as_slice()) as u64, + Value::Json(value) => json(value), + _ => 0, + } +} diff --git a/v2/contextdb/src/query/tests.rs b/v2/contextdb/src/query/tests.rs new file mode 100644 index 000000000..19290c4c8 --- /dev/null +++ b/v2/contextdb/src/query/tests.rs @@ -0,0 +1,131 @@ +use std::cmp::Ordering; +use std::collections::HashMap; + +use super::expression::{Arithmetic, arithmetic, compare, eval, sort_order}; +use crate::schema::Table; +use crate::sql::Expr; +use crate::sql::ast::{BinOp, Literal, UnaryOp}; +use crate::store::TableId; +use crate::types::{Error, TableMeta, Uuid, Value}; + +/// An expression that reads no column, worked out. +fn worked_out(expr: &Expr) -> Value { + let table = Table { + id: TableId(1), + name: "t".to_owned(), + meta: TableMeta { + columns: Vec::new(), + unique_constraints: Vec::new(), + indexes: Vec::new(), + }, + index_ids: Vec::new(), + }; + eval(expr, &table, &[], &HashMap::new(), 0).expect("an expression of literals") +} + +fn joined(left: Literal, op: BinOp, right: Literal) -> Expr { + Expr::BinaryOp { + left: Box::new(Expr::Literal(left)), + op, + right: Box::new(Expr::Literal(right)), + } +} + +#[test] +fn null_is_unknown_unless_the_other_side_settles_the_answer() { + let (null, yes, no) = (Literal::Null, Literal::Bool(true), Literal::Bool(false)); + let and = |left: &Literal, right: &Literal| { + worked_out(&joined(left.clone(), BinOp::And, right.clone())) + }; + let or = |left: &Literal, right: &Literal| { + worked_out(&joined(left.clone(), BinOp::Or, right.clone())) + }; + assert_eq!(and(&null, &no), Value::Bool(false)); + assert_eq!(and(&no, &null), Value::Bool(false)); + assert_eq!(and(&null, &yes), Value::Null); + assert_eq!(and(&yes, &yes), Value::Bool(true)); + assert_eq!(or(&null, &yes), Value::Bool(true)); + assert_eq!(or(&yes, &null), Value::Bool(true)); + assert_eq!(or(&null, &no), Value::Null); + assert_eq!(or(&no, &no), Value::Bool(false)); + + let not_null = Expr::UnaryOp { + op: UnaryOp::Not, + operand: Box::new(Expr::Literal(Literal::Null)), + }; + assert_eq!(worked_out(¬_null), Value::Null); + let null_equals_null = joined(Literal::Null, BinOp::Eq, Literal::Null); + assert_eq!(worked_out(&null_equals_null), Value::Null); +} + +#[test] +fn arithmetic_keeps_whole_numbers_whole_and_refuses_what_does_not_fit() { + let (int, real) = (Value::Int64, Value::Float64); + let ok = |op, left: &Value, right: &Value| arithmetic(op, left, right).expect("arithmetic"); + assert_eq!(ok(Arithmetic::Div, &int(7), &int(2)), int(3)); + assert_eq!(ok(Arithmetic::Mul, &int(7), &real(0.5)), real(3.5)); + assert_eq!(ok(Arithmetic::Sub, &real(1.5), &int(1)), real(0.5)); + assert_eq!(ok(Arithmetic::Add, &Value::Null, &int(1)), Value::Null); + for (op, left, right) in [ + (Arithmetic::Add, int(i64::MAX), int(1)), + (Arithmetic::Mul, int(i64::MIN), int(-1)), + (Arithmetic::Div, int(7), int(0)), + (Arithmetic::Div, real(7.0), real(0.0)), + (Arithmetic::Add, int(7), Value::Text("x".to_owned())), + ] { + let refused = arithmetic(op, &left, &right); + assert!( + matches!(refused, Err(Error::PlanError(_))), + "{left:?} and {right:?} gave {refused:?}" + ); + } +} + +#[test] +fn values_sort_with_nan_above_every_number_and_null_after_everything() { + let mut values = [ + Value::Null, + Value::Float64(f64::NAN), + Value::Float64(f64::MAX), + Value::Int64(3), + Value::Float64(-1.5), + ]; + values.sort_by(sort_order); + assert_eq!( + values[..3], + [ + Value::Float64(-1.5), + Value::Int64(3), + Value::Float64(f64::MAX) + ] + ); + assert!(matches!(values[3], Value::Float64(nan) if nan.is_nan())); + assert_eq!(values[4], Value::Null); +} + +#[test] +fn a_uuid_compares_with_the_text_that_spells_it() { + let uuid = Value::Uuid(Uuid::from_u128(0x550e_8400_e29b_41d4_a716_4466_5544_0000)); + let spelled = |text: &str| Value::Text(text.to_owned()); + assert_eq!( + compare(&uuid, &spelled("550E8400-E29B-41D4-A716-446655440000")), + Some(Ordering::Equal) + ); + assert_eq!(compare(&spelled("not a uuid"), &uuid), None); + assert_eq!(compare(&uuid, &Value::Int64(1)), None); +} + +#[test] +fn a_real_compares_as_a_key_holds_it() { + let real = Value::Float64; + assert_eq!(compare(&real(-0.0), &real(0.0)), Some(Ordering::Equal)); + assert_eq!( + compare(&real(f64::NAN), &real(-f64::NAN)), + Some(Ordering::Equal) + ); + assert_eq!( + compare(&real(-f64::NAN), &real(f64::INFINITY)), + Some(Ordering::Greater) + ); + assert_eq!(compare(&Value::Int64(2), &real(2.5)), Some(Ordering::Less)); +} diff --git a/v2/contextdb/src/read/mod.rs b/v2/contextdb/src/read/mod.rs new file mode 100644 index 000000000..0cc5d7f38 --- /dev/null +++ b/v2/contextdb/src/read/mod.rs @@ -0,0 +1,185 @@ +//! read: reading a store without holding it for writing, from the writer's process or any +//! other. A session opens the folder itself, and each statement reads the newest commit under +//! the session's own memory limit. +//! +//! It never writes to the store itself. + +use std::collections::HashMap; +use std::path::Path; +use std::sync::{Arc, Mutex, PoisonError}; + +use crate::memory::Budget; +use crate::query::{self, QueryResult}; +use crate::schema::{self, Catalog}; +use crate::sql::{self, Effect, Plan}; +use crate::store::{Store, View, WrapStorage}; +use crate::system::{self, Clock}; +use crate::types::{Error, Result, TxId, Value}; + +/// What a reader may say when it opens a store. +/// +/// Start from `ReadSessionOptions::default()` and assign the fields to change. The struct is +/// `#[non_exhaustive]`, so outside this crate it cannot be written out field by field. +#[derive(Clone, Default)] +#[non_exhaustive] +pub struct ReadSessionOptions { + /// The most memory, in bytes, the session may use for its page cache and for the results + /// it has returned and that are still alive. `None`, the default, is no ceiling: a reader + /// does not take the limit the store remembers for its writer, so a reader that needs a + /// cap sets it here. Every open sets aside 8 MiB for the page cache and 1 MiB for the + /// small fixed needs of a statement, so a limit under 9 MiB is refused at open with + /// [`crate::Error::MemoryBudgetExceeded`]. + pub memory_limit: Option, + /// Where the session gets the time that `NOW()` in a statement answers with. `None`, the + /// default, is the machine's clock. + pub clock: Option>, + /// A wrapper around the data file the session has opened; the session then reads the file + /// only through what the wrapper returns. `None`, the default, reads the file as it is. + pub storage: Option, +} + +/// A store opened for reading. +/// +/// A session is opened by the folder's path, with [`ReadSession::open`]; no call on +/// [`crate::Database`] returns one. It reads the folder itself and needs nothing from the +/// writer's process: it opens whether or not a writer is running, never makes a writer fail, +/// holds one up only for the instant a call takes to note which commit it reads, and keeps +/// answering when the writer stops, is killed or starts again. It changes nothing a writer +/// stored. While a call runs it keeps one small file under `readers/` in the +/// folder, which is how a writer knows the pages being read are still needed. +/// +/// Each call sees the newest commit as of the moment it starts and keeps that one moment until +/// it returns. The next call starts from the newest commit again: no call holds one commit +/// across two statements. +/// +/// Which to read with: the process that holds the store for writing reads through its +/// [`crate::Database`], where a transaction's statements also see what the transaction has +/// written. Every other process reads through a `ReadSession`, and so does code that must not +/// be able to write: a session refuses a statement that would change something with +/// [`crate::Error::ReadOnlySession`]. +/// +/// Calls block and may be made from many threads. +pub struct ReadSession { + store: Store, + budget: Budget, + clock: Arc, + /// The tables as last read, with the schema commit they were read at. + catalog: Mutex)>>, +} + +impl ReadSession { + /// Opens the store in this folder for reading, with no memory limit and the machine's + /// clock. Never creates anything. + /// + /// # Errors + /// + /// - [`crate::Error::StoreMissing`] when there is no store at the path. + /// - [`crate::Error::StoreIo`] when the folder or its data file cannot be opened or read. + /// - [`crate::Error::StoreCorrupted`] when the data file fails its check. + pub fn open(path: impl AsRef) -> Result { + ReadSession::open_with_options(path, ReadSessionOptions::default()) + } + + /// [`ReadSession::open`] with options. + /// + /// # Errors + /// + /// Those of [`ReadSession::open`], and [`crate::Error::MemoryBudgetExceeded`] when + /// [`ReadSessionOptions::memory_limit`] is under the 9 MiB every open sets aside. + pub fn open_with_options( + path: impl AsRef, + options: ReadSessionOptions, + ) -> Result { + let budget = Budget::new(options.memory_limit.map(|bytes| bytes as u64))?; + let store = Store::open_for_reading(path.as_ref(), &budget, options.storage.as_ref())?; + Ok(ReadSession { + store, + budget, + clock: options.clock.unwrap_or_else(system::machine_clock), + catalog: Mutex::new(None), + }) + } + + /// Runs one statement that reads, `SELECT` or `SHOW MEMORY_LIMIT`, and returns its result. + /// + /// `params` gives a value for each `$name` in the statement; [`crate::params!`] writes the + /// map. + /// + /// # Errors + /// + /// - [`crate::Error::ReadOnlySession`] when the statement would declare a table or an + /// index, write rows or change the memory limit. Nothing ran; run it on a + /// [`crate::Database`]. + /// - [`crate::Error::ParseError`] or [`crate::Error::PlanError`] when the text is not a + /// statement this version runs, in the words a [`crate::Database`] refuses it with. + /// - [`crate::Error::TableNotFound`] when the newest commit has no table of that name. + /// - [`crate::Error::MemoryBudgetExceeded`] when the result would pass the session's + /// memory limit. The session stays usable. + /// - [`crate::Error::StoreIo`] when the folder or its data file cannot be read, and + /// [`crate::Error::StoreCorrupted`] when what was read fails its check. + pub fn execute(&self, sql: &str, params: &HashMap) -> Result { + let plan = sql::plan(&sql::parse(sql)?)?; + if plan.effect() != Effect::Reads { + return Err(Error::ReadOnlySession); + } + if let Plan::ShowMemoryLimit = plan { + // Nothing changes a session's limit, so the one it has is the one it opened under. + let opened_under = self.budget.limit(); + return Ok(query::memory_limit_shown(&self.budget, opened_under)); + } + let now = system::now_millis(&*self.clock); + let (view, catalog) = self.newest()?; + query::run(&plan, params, &catalog, &view, &self.budget, now) + } + + /// The names of the store's tables as of the newest commit, in name order. A store with no + /// table gives an empty list. + /// + /// # Errors + /// + /// This call reads the folder, so unlike [`crate::Database::table_names`] it can fail: + /// [`crate::Error::StoreIo`] when the folder or its data file cannot be read, and + /// [`crate::Error::StoreCorrupted`] when what was read fails its check. + pub fn table_names(&self) -> Result> { + let (_, catalog) = self.newest()?; + let tables = catalog.tables().into_iter(); + Ok(tables.map(|table| table.name.clone()).collect()) + } + + /// A table's declaration as SQL text, as of the newest commit: its `CREATE TABLE` statement, + /// then one `CREATE INDEX` statement for each index made with `CREATE INDEX`, each ended + /// by `;` and a new line. A clause left at its default is not written. It is the text + /// [`crate::Database::declaration`] gives, and running it on an empty store declares the + /// same table. + /// + /// # Errors + /// + /// - [`crate::Error::TableNotFound`] when the newest commit has no table of this name. + /// - [`crate::Error::StoreIo`] when the folder or its data file cannot be read, and + /// [`crate::Error::StoreCorrupted`] when what was read fails its check. + pub fn declaration(&self, table: &str) -> Result { + let (_, catalog) = self.newest()?; + match catalog.table(table) { + Some(table) => Ok(schema::render(table)), + None => Err(Error::TableNotFound(table.to_owned())), + } + } + + /// The newest commit and the tables declared at it. The view holds that commit's pages for + /// as long as the caller keeps it; the tables are read again only when a declaration has + /// been committed since they were last read. + fn newest(&self) -> Result<(View, Arc)> { + let view = self.store.view()?; + let declared_at = view.schema_commit()?; + // The lock guards a finished catalog or none, so a holder that panicked left it whole. + let mut kept = self.catalog.lock().unwrap_or_else(PoisonError::into_inner); + if let Some((commit, catalog)) = &*kept + && *commit == declared_at + { + return Ok((view, catalog.clone())); + } + let catalog = Arc::new(Catalog::load(&view)?); + *kept = Some((declared_at, catalog.clone())); + Ok((view, catalog)) + } +} diff --git a/v2/contextdb/src/schema/check.rs b/v2/contextdb/src/schema/check.rs new file mode 100644 index 000000000..4fff4efd9 --- /dev/null +++ b/v2/contextdb/src/schema/check.rs @@ -0,0 +1,220 @@ +//! Checking a declaration: what CREATE TABLE and CREATE INDEX may say, and the table each gives. + +use super::{Catalog, RESERVED_PREFIXES, Table}; +use crate::sql::ast::{self, DataType, Expr, Literal, UnaryOp}; +use crate::sql::plan::{CreateIndexPlan, CreateTablePlan}; +use crate::store::{IndexId, TableId}; +use crate::types::{ + ColumnDef, ColumnDefault, ColumnType, Error, IndexDecl, Result, SortDirection, TableMeta, Value, +}; + +/// The refusal of a statement or clause that arrives with a later version. +pub(super) fn not_supported(clause: &str) -> Error { + Error::PlanError(format!( + "{clause} is not supported in this version of the library" + )) +} + +fn invalid(reason: String) -> Error { + Error::SchemaInvalid { reason } +} + +/// The table a CREATE TABLE declares, numbered after the tables the catalog holds. +pub(super) fn new_table(create: &CreateTablePlan, catalog: &Catalog) -> Result
{ + let table = &create.name; + let mut columns: Vec = Vec::with_capacity(create.columns.len()); + for declared in &create.columns { + if columns.iter().any(|earlier| earlier.name == declared.name) { + return Err(invalid(format!( + "table '{table}' declares the column '{}' twice", + declared.name + ))); + } + columns.push(column(table, declared)?); + } + match columns.iter().filter(|column| column.primary_key).count() { + 0 => return Err(not_supported("a table without a PRIMARY KEY")), + 1 => {} + _ => { + return Err(invalid(format!( + "table '{table}' declares more than one PRIMARY KEY" + ))); + } + } + + let mut meta = TableMeta { + columns, + unique_constraints: create.unique_constraints.clone(), + indexes: Vec::new(), + }; + // The index behind the key, then behind each UNIQUE column, then behind each UNIQUE rule. + // The key's index already keeps the key column unique. + let one = |column: &ColumnDef| vec![column.name.clone()]; + let key = meta.columns.iter().filter(|column| column.primary_key); + let unique = meta.columns.iter(); + let unique = unique.filter(|column| column.unique && !column.primary_key); + let rules = create.unique_constraints.iter().cloned(); + let behind = key + .map(|column| ("__pk_", one(column))) + .chain(unique.map(|column| ("__unique_", one(column)))) + .chain(rules.map(|rule| ("__unique_", rule))); + for (prefix, rule) in behind { + // Two rules can spell one name (`a_b`, and `a` with `b`); the later one takes a number. + let plain = format!("{prefix}{}", rule.join("_")); + let mut name = plain.clone(); + let mut copy = 1; + while meta.indexes.iter().any(|other| other.name == name) { + copy += 1; + name = format!("{plain}_{copy}"); + } + let columns = rule.into_iter().map(|column| (column, SortDirection::Asc)); + let index = IndexDecl { + name, + columns: columns.collect(), + }; + check_columns(table, &meta, &index, true)?; + meta.indexes.push(index); + } + + let id = catalog.tables.values().map(|other| other.id.0 + 1).max(); + let index_ids = (next_index_id(catalog)..) + .take(meta.indexes.len()) + .map(IndexId) + .collect(); + Ok(Table { + id: TableId(id.unwrap_or(0)), + name: table.clone(), + meta, + index_ids, + }) +} + +/// The table a CREATE INDEX names, with the new index as its last. +pub(super) fn with_index(create: &CreateIndexPlan, catalog: &Catalog) -> Result
{ + let reserved = RESERVED_PREFIXES + .into_iter() + .find(|prefix| create.name.starts_with(*prefix)); + if let Some(prefix) = reserved { + return Err(Error::ReservedIndexName { + table: create.table.clone(), + name: create.name.clone(), + prefix: prefix.to_owned(), + }); + } + let mut table = catalog + .table(&create.table) + .ok_or_else(|| Error::TableNotFound(create.table.clone()))? + .clone(); + let index = IndexDecl { + name: create.name.clone(), + columns: create.columns.clone(), + }; + check_columns(&table.name, &table.meta, &index, false)?; + if table + .meta + .indexes + .iter() + .any(|other| other.name == index.name) + { + return Err(Error::DuplicateIndex { + table: table.name, + index: index.name, + }); + } + table.meta.indexes.push(index); + table.index_ids.push(IndexId(next_index_id(catalog))); + Ok(table) +} + +/// One past the highest index number in use. Nothing in this version removes an index or a +/// table, so counting on from the highest never gives a number twice. +fn next_index_id(catalog: &Catalog) -> u32 { + let ids = catalog.tables.values().flat_map(|table| &table.index_ids); + ids.map(|id| id.0 + 1).max().unwrap_or(0) +} + +/// Refuses an index over a column the table does not have or whose type it cannot be over. +/// `rule` says the index is the one behind a key or a UNIQUE rule. +fn check_columns(table: &str, meta: &TableMeta, index: &IndexDecl, rule: bool) -> Result<()> { + for (name, _) in &index.columns { + let column = meta.columns.iter().find(|column| column.name == *name); + let column = column.ok_or_else(|| Error::ColumnNotFound { + table: table.to_owned(), + column: name.clone(), + })?; + // JSON and VECTOR have no order to keep an index by. A REAL has one, and is still no + // identity for a key or a UNIQUE rule: it has two zeros and a NaN. + let refused = match column.column_type { + ColumnType::Json | ColumnType::Vector(_) => true, + ColumnType::Real => rule, + _ => false, + }; + if refused { + return Err(Error::ColumnNotIndexable { + table: table.to_owned(), + column: name.clone(), + column_type: column.column_type, + }); + } + } + Ok(()) +} + +/// One column as the catalog keeps it. +fn column(table: &str, declared: &ast::ColumnDef) -> Result { + let column_type = match declared.data_type { + DataType::Integer => ColumnType::Integer, + DataType::Real => ColumnType::Real, + DataType::Text => ColumnType::Text, + DataType::Boolean => ColumnType::Boolean, + DataType::Uuid => ColumnType::Uuid, + DataType::Timestamp => ColumnType::Timestamp, + DataType::Json => ColumnType::Json, + DataType::Vector(size) => ColumnType::Vector(size as usize), + }; + let default = match &declared.default { + Some(expr) => Some(default(expr).ok_or_else(|| { + invalid(format!( + "the DEFAULT of '{table}.{}' must be NULL, TRUE, FALSE, a number, quoted text \ + or NOW()", + declared.name + )) + })?), + None => None, + }; + Ok(ColumnDef { + name: declared.name.clone(), + column_type, + nullable: declared.nullable, + primary_key: declared.primary_key, + unique: declared.unique, + default, + }) +} + +/// A DEFAULT as the catalog keeps it, in the type the author wrote. `None` for anything but +/// NULL, TRUE, FALSE, a number, quoted text or `NOW()`. +pub(super) fn default(expr: &Expr) -> Option { + let value = match expr { + Expr::FunctionCall { name, args } + if name.eq_ignore_ascii_case("NOW") && args.is_empty() => + { + return Some(ColumnDefault::Now); + } + Expr::Literal(Literal::Null) => Value::Null, + Expr::Literal(Literal::Bool(flag)) => Value::Bool(*flag), + Expr::Literal(Literal::Integer(whole)) => Value::Int64(*whole), + Expr::Literal(Literal::Real(real)) => Value::Float64(*real), + Expr::Literal(Literal::Text(words)) => Value::Text(words.clone()), + Expr::UnaryOp { + op: UnaryOp::Neg, + operand, + } => match **operand { + Expr::Literal(Literal::Integer(whole)) => Value::Int64(-whole), + Expr::Literal(Literal::Real(real)) => Value::Float64(-real), + _ => return None, + }, + _ => return None, + }; + Some(ColumnDefault::Literal(value)) +} diff --git a/v2/contextdb/src/schema/mod.rs b/v2/contextdb/src/schema/mod.rs new file mode 100644 index 000000000..0faace21c --- /dev/null +++ b/v2/contextdb/src/schema/mod.rs @@ -0,0 +1,101 @@ +//! schema: table declarations: checking them, storing them, printing them back. +//! +//! It never changes a row. +//! +//! A table's declaration is kept as one schema record under the table's name: its number, its +//! columns and rules as they were declared, and its indexes with their numbers. +//! +//! A table's indexes come in one order: the index behind the primary key (`__pk_`), +//! the index behind each UNIQUE column in column order (`__unique_`; the key column has +//! its key's index and no other), the index behind each UNIQUE rule of the table in declared +//! order (`__unique__...`), then the indexes of CREATE INDEX in the order they +//! were made. + +mod check; +mod record; +mod render; +#[cfg(test)] +mod tests; + +use std::collections::BTreeMap; + +use crate::sql::Plan; +use crate::store::{ChangeSet, IndexId, TableId, View}; +use crate::types::{Error, Result, TableMeta}; + +/// What the name of an index the engine makes for itself begins with. An index of CREATE INDEX +/// may not take one of these. +const RESERVED_PREFIXES: [&str; 3] = ["__pk_", "__unique_", "__fk_"]; + +/// One declared table as the other parts use it. +#[derive(Debug, Clone)] +pub(crate) struct Table { + pub id: TableId, + pub name: String, + pub meta: TableMeta, + /// One id per entry of `meta.indexes`, in the same order. + pub index_ids: Vec, +} + +/// Every table declared in a store at one moment. +pub(crate) struct Catalog { + tables: BTreeMap, +} + +impl Catalog { + /// Reads the declarations the reader shows. Costs one small record per table; `database` + /// keeps the result and loads again only when [`View::schema_commit`] has moved. + pub(crate) fn load(view: &View) -> Result { + let mut tables = BTreeMap::new(); + for bytes in view.schema_records()? { + let table = record::decode(&bytes); + tables.insert(table.name.clone(), table); + } + Ok(Catalog { tables }) + } + + /// A table by name, as SQL names it. + pub(crate) fn table(&self, name: &str) -> Option<&Table> { + self.tables.get(name) + } + + /// Every table, in name order. + pub(crate) fn tables(&self) -> Vec<&Table> { + self.tables.values().collect() + } +} + +/// Applies one declaration (CREATE TABLE, CREATE INDEX, ALTER TABLE, DROP) to a change set. +/// +/// Checks the declaration against the catalog, stages its schema record and returns the +/// catalog as it will be once the change set commits. Stages no row and no index entry; after +/// a CREATE INDEX on a table that has rows, `database` asks `write` to fill the index. +pub(crate) fn declare(plan: &Plan, catalog: &Catalog, changes: &mut ChangeSet) -> Result { + let mut tables = catalog.tables.clone(); + let table = match plan { + Plan::CreateTable(create) if !tables.contains_key(&create.name) => { + check::new_table(create, catalog)? + } + // A taken name is settled before the rest of the declaration is looked at. + Plan::CreateTable(create) if create.if_not_exists => return Ok(Catalog { tables }), + Plan::CreateTable(create) => { + return Err(Error::SchemaInvalid { + reason: format!( + "table '{}' already exists: CREATE TABLE never redefines a table, so nothing \ + was changed; CREATE TABLE IF NOT EXISTS leaves an existing table as it is", + create.name + ), + }); + } + Plan::CreateIndex(create) => check::with_index(create, catalog)?, + _ => return Err(check::not_supported("this declaration")), + }; + changes.put_schema(&table.name, record::encode(&table))?; + tables.insert(table.name.clone(), table); + Ok(Catalog { tables }) +} + +/// The declaration as `.schema` prints it: text that, run again, declares the same table. +pub(crate) fn render(table: &Table) -> String { + render::declaration(table) +} diff --git a/v2/contextdb/src/schema/record.rs b/v2/contextdb/src/schema/record.rs new file mode 100644 index 000000000..0650e5cd1 --- /dev/null +++ b/v2/contextdb/src/schema/record.rs @@ -0,0 +1,193 @@ +//! Storing a declaration: the bytes of a table's schema record, and the table read back from +//! them. +//! +//! Every number is four bytes, low byte first. A text is its length in bytes and then its +//! bytes. A list is its length and then its items. +//! +//! Reading trusts the bytes: a record is only ever what `encode` wrote, and the store checks +//! every page it reads. Where a byte says which of several kinds follows, the last kind is read +//! for any byte the earlier ones do not claim. + +use super::Table; +use crate::store::{IndexId, TableId}; +use crate::types::{ + ColumnDef, ColumnDefault, ColumnType, IndexDecl, SortDirection, TableMeta, Value, +}; + +/// The schema record of a table. +pub(super) fn encode(table: &Table) -> Vec { + let meta = &table.meta; + let mut out = Vec::new(); + number(&mut out, table.id.0 as usize); + text(&mut out, &table.name); + number(&mut out, meta.columns.len()); + for column in &meta.columns { + text(&mut out, &column.name); + let (kind, size) = match column.column_type { + ColumnType::Integer => (0, 0), + ColumnType::Real => (1, 0), + ColumnType::Text => (2, 0), + ColumnType::Boolean => (3, 0), + ColumnType::Json => (4, 0), + ColumnType::Uuid => (5, 0), + ColumnType::Timestamp => (6, 0), + ColumnType::Vector(size) => (8, size), + }; + out.push(kind); + number(&mut out, size); + out.push(u8::from(column.nullable)); + out.push(u8::from(column.primary_key)); + out.push(u8::from(column.unique)); + match &column.default { + None => out.push(0), + Some(ColumnDefault::Now) => out.push(1), + Some(ColumnDefault::Literal(Value::Null)) => out.push(2), + Some(ColumnDefault::Literal(Value::Bool(flag))) => out.extend([3, u8::from(*flag)]), + Some(ColumnDefault::Literal(Value::Int64(whole))) => { + out.push(4); + out.extend(whole.to_le_bytes()); + } + Some(ColumnDefault::Literal(Value::Float64(real))) => { + out.push(5); + out.extend(real.to_le_bytes()); + } + Some(ColumnDefault::Literal(Value::Text(words))) => { + out.push(6); + text(&mut out, words); + } + Some(ColumnDefault::Literal(other)) => { + unreachable!("a default is declared as a literal, never as {other:?}") + } + } + } + number(&mut out, meta.unique_constraints.len()); + for rule in &meta.unique_constraints { + number(&mut out, rule.len()); + for column in rule { + text(&mut out, column); + } + } + number(&mut out, meta.indexes.len()); + for (index, id) in meta.indexes.iter().zip(&table.index_ids) { + number(&mut out, id.0 as usize); + text(&mut out, &index.name); + number(&mut out, index.columns.len()); + for (column, direction) in &index.columns { + text(&mut out, column); + out.push(u8::from(*direction == SortDirection::Desc)); + } + } + out +} + +/// The table a schema record was written from. +pub(super) fn decode(record: &[u8]) -> Table { + let mut read = Reader(record); + let id = TableId(read.number() as u32); + let name = read.text(); + let columns = read.list(|read| { + let name = read.text(); + let kind = read.byte(); + let size = read.number(); + ColumnDef { + name, + column_type: match kind { + 0 => ColumnType::Integer, + 1 => ColumnType::Real, + 2 => ColumnType::Text, + 3 => ColumnType::Boolean, + 4 => ColumnType::Json, + 5 => ColumnType::Uuid, + 6 => ColumnType::Timestamp, + _ => ColumnType::Vector(size), + }, + nullable: read.flag(), + primary_key: read.flag(), + unique: read.flag(), + default: match read.byte() { + 0 => None, + 1 => Some(ColumnDefault::Now), + kind => Some(ColumnDefault::Literal(match kind { + 2 => Value::Null, + 3 => Value::Bool(read.flag()), + 4 => Value::Int64(i64::from_le_bytes(read.array())), + 5 => Value::Float64(f64::from_le_bytes(read.array())), + _ => Value::Text(read.text()), + })), + }, + } + }); + let unique_constraints = read.list(|read| read.list(Reader::text)); + let mut index_ids = Vec::new(); + let indexes = read.list(|read| { + index_ids.push(IndexId(read.number() as u32)); + let name = read.text(); + let columns = read.list(|read| { + let column = read.text(); + let direction = if read.flag() { + SortDirection::Desc + } else { + SortDirection::Asc + }; + (column, direction) + }); + IndexDecl { name, columns } + }); + Table { + id, + name, + meta: TableMeta { + columns, + unique_constraints, + indexes, + }, + index_ids, + } +} + +fn number(out: &mut Vec, number: usize) { + out.extend((number as u32).to_le_bytes()); +} + +fn text(out: &mut Vec, text: &str) { + number(out, text.len()); + out.extend(text.as_bytes()); +} + +/// What is left of a record being read. +struct Reader<'a>(&'a [u8]); + +impl<'a> Reader<'a> { + fn bytes(&mut self, count: usize) -> &'a [u8] { + let (taken, rest) = self.0.split_at(count); + self.0 = rest; + taken + } + + fn array(&mut self) -> [u8; N] { + let mut array = [0; N]; + array.copy_from_slice(self.bytes(N)); + array + } + + fn byte(&mut self) -> u8 { + self.bytes(1)[0] + } + + fn flag(&mut self) -> bool { + self.byte() != 0 + } + + fn number(&mut self) -> usize { + u32::from_le_bytes(self.array()) as usize + } + + fn text(&mut self) -> String { + let count = self.number(); + String::from_utf8_lossy(self.bytes(count)).into_owned() + } + + fn list(&mut self, mut item: impl FnMut(&mut Self) -> T) -> Vec { + (0..self.number()).map(|_| item(self)).collect() + } +} diff --git a/v2/contextdb/src/schema/render.rs b/v2/contextdb/src/schema/render.rs new file mode 100644 index 000000000..0e0ed6bd7 --- /dev/null +++ b/v2/contextdb/src/schema/render.rs @@ -0,0 +1,100 @@ +//! Printing a declaration back as SQL. + +use super::{RESERVED_PREFIXES, Table}; +use crate::types::{ColumnDef, ColumnDefault, ColumnType, SortDirection, Value}; + +/// A table's declaration as SQL: the CREATE TABLE, then one CREATE INDEX for each index the +/// author made, every statement ended by `;` and a new line. A clause left at its default is +/// not printed, and neither are the indexes the engine keeps behind a key or a UNIQUE rule. +pub(super) fn declaration(table: &Table) -> String { + let meta = &table.meta; + let mut lines: Vec = meta.columns.iter().map(column).collect(); + for rule in &meta.unique_constraints { + let columns: Vec = rule.iter().map(|column| name(column)).collect(); + lines.push(format!("UNIQUE ({})", columns.join(", "))); + } + let table = name(&table.name); + let mut text = format!("CREATE TABLE {table} (\n {}\n);\n", lines.join(",\n ")); + for index in &meta.indexes { + if RESERVED_PREFIXES + .iter() + .any(|prefix| index.name.starts_with(*prefix)) + { + continue; + } + let columns: Vec = index + .columns + .iter() + .map(|(column, direction)| match direction { + SortDirection::Asc => name(column), + SortDirection::Desc => format!("{} DESC", name(column)), + }) + .collect(); + text.push_str(&format!( + "CREATE INDEX {} ON {table} ({});\n", + name(&index.name), + columns.join(", ") + )); + } + text +} + +/// One column's line: its name, its type, and the clauses it was given. +fn column(column: &ColumnDef) -> String { + let mut text = name(&column.name); + text.push(' '); + match column.column_type { + ColumnType::Integer => text.push_str("INTEGER"), + ColumnType::Real => text.push_str("REAL"), + ColumnType::Text => text.push_str("TEXT"), + ColumnType::Boolean => text.push_str("BOOLEAN"), + ColumnType::Json => text.push_str("JSON"), + ColumnType::Uuid => text.push_str("UUID"), + ColumnType::Timestamp => text.push_str("TIMESTAMP"), + ColumnType::Vector(size) => text.push_str(&format!("VECTOR({size})")), + } + if column.primary_key { + text.push_str(" PRIMARY KEY"); + } + if !column.nullable { + text.push_str(" NOT NULL"); + } + match &column.default { + Some(ColumnDefault::Literal(value)) => { + text.push_str(" DEFAULT "); + text.push_str(&literal(value)); + } + Some(ColumnDefault::Now) => text.push_str(" DEFAULT NOW()"), + None => {} + } + if column.unique { + text.push_str(" UNIQUE"); + } + text +} + +/// A default's value as the author wrote it. +fn literal(value: &Value) -> String { + match value { + Value::Null => "NULL".to_owned(), + Value::Bool(true) => "TRUE".to_owned(), + Value::Bool(false) => "FALSE".to_owned(), + Value::Int64(whole) => whole.to_string(), + // A whole REAL keeps its decimal point, so it reads back as a REAL. + Value::Float64(real) if real.fract() == 0.0 => format!("{real:.1}"), + Value::Float64(real) => real.to_string(), + Value::Text(words) => format!("'{}'", words.replace('\'', "''")), + other => unreachable!("a default is declared as a literal, never as {other:?}"), + } +} + +/// A name as SQL reads it back: bare when it is one plain word, in double quotes otherwise. +fn name(name: &str) -> String { + let plain = name.starts_with(|first: char| first.is_ascii_alphabetic() || first == '_') + && name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_'); + if plain { + name.to_owned() + } else { + format!("\"{}\"", name.replace('"', "\"\"")) + } +} diff --git a/v2/contextdb/src/schema/tests.rs b/v2/contextdb/src/schema/tests.rs new file mode 100644 index 000000000..b648333bd --- /dev/null +++ b/v2/contextdb/src/schema/tests.rs @@ -0,0 +1,305 @@ +use std::collections::BTreeMap; + +use super::{Catalog, Table, check, record, render}; +use crate::sql::ast::{self, DataType, Expr, Literal, UnaryOp}; +use crate::sql::plan::{CreateIndexPlan, CreateTablePlan}; +use crate::store::{IndexId, TableId}; +use crate::types::{ + ColumnDef, ColumnDefault, ColumnType, Error, IndexDecl, SortDirection, TableMeta, Value, +}; + +fn column(name: &str, column_type: ColumnType) -> ColumnDef { + ColumnDef { + name: name.to_owned(), + column_type, + nullable: true, + primary_key: false, + unique: false, + default: None, + } +} + +fn index(name: &str, columns: &[(&str, SortDirection)]) -> IndexDecl { + IndexDecl { + name: name.to_owned(), + columns: columns + .iter() + .map(|(column, direction)| ((*column).to_owned(), *direction)) + .collect(), + } +} + +/// A table with every kind of clause this version keeps, as CREATE TABLE and two CREATE INDEX +/// statements leave it. +fn notes() -> Table { + use SortDirection::{Asc, Desc}; + let literal = |value| Some(ColumnDefault::Literal(value)); + Table { + id: TableId(4), + name: "notes".to_owned(), + meta: TableMeta { + columns: vec![ + ColumnDef { + primary_key: true, + ..column("id", ColumnType::Integer) + }, + ColumnDef { + nullable: false, + default: literal(Value::Text("it's".to_owned())), + ..column("title", ColumnType::Text) + }, + ColumnDef { + default: literal(Value::Null), + ..column("body", ColumnType::Text) + }, + ColumnDef { + default: literal(Value::Int64(3)), + ..column("stars", ColumnType::Integer) + }, + ColumnDef { + unique: true, + ..column("slug", ColumnType::Text) + }, + ColumnDef { + nullable: false, + default: Some(ColumnDefault::Now), + ..column("made at", ColumnType::Timestamp) + }, + ColumnDef { + default: literal(Value::Float64(-2.0)), + ..column("score", ColumnType::Real) + }, + ColumnDef { + default: literal(Value::Bool(true)), + ..column("kept", ColumnType::Boolean) + }, + column("embedding", ColumnType::Vector(384)), + ], + unique_constraints: vec![vec!["title".to_owned(), "stars".to_owned()]], + indexes: vec![ + index("__pk_id", &[("id", Asc)]), + index("__unique_slug", &[("slug", Asc)]), + index("__unique_title_stars", &[("title", Asc), ("stars", Asc)]), + index("by_stars", &[("stars", Desc), ("made at", Asc)]), + index("by_body", &[("body", Asc)]), + ], + }, + index_ids: [7, 8, 9, 12, 15].map(IndexId).to_vec(), + } +} + +#[test] +fn a_record_reads_back_as_the_table_it_was_written_from() { + let table = notes(); + let back = record::decode(&record::encode(&table)); + assert_eq!(back.id, table.id); + assert_eq!(back.name, table.name); + assert_eq!(back.meta, table.meta); + assert_eq!(back.index_ids, table.index_ids); +} + +#[test] +fn render_prints_what_the_author_declared_and_nothing_else() { + assert_eq!( + render(¬es()), + "\ +CREATE TABLE notes ( + id INTEGER PRIMARY KEY, + title TEXT NOT NULL DEFAULT 'it''s', + body TEXT DEFAULT NULL, + stars INTEGER DEFAULT 3, + slug TEXT UNIQUE, + \"made at\" TIMESTAMP NOT NULL DEFAULT NOW(), + score REAL DEFAULT -2.0, + kept BOOLEAN DEFAULT TRUE, + embedding VECTOR(384), + UNIQUE (title, stars) +); +CREATE INDEX by_stars ON notes (stars DESC, \"made at\"); +CREATE INDEX by_body ON notes (body); +" + ); +} + +#[test] +fn a_default_is_kept_in_the_type_the_author_wrote() { + let literal = Expr::Literal; + let minus = |literal| Expr::UnaryOp { + op: UnaryOp::Neg, + operand: Box::new(Expr::Literal(literal)), + }; + let now = Expr::FunctionCall { + name: "now".to_owned(), + args: Vec::new(), + }; + for (expr, value) in [ + (literal(Literal::Null), Value::Null), + (literal(Literal::Bool(true)), Value::Bool(true)), + (literal(Literal::Integer(3)), Value::Int64(3)), + (literal(Literal::Real(0.0)), Value::Float64(0.0)), + (literal(Literal::Text("it's".to_owned())), "it's".into()), + (minus(Literal::Integer(1)), Value::Int64(-1)), + (minus(Literal::Real(2.5)), Value::Float64(-2.5)), + ] { + assert_eq!(check::default(&expr), Some(ColumnDefault::Literal(value))); + } + assert_eq!(check::default(&now), Some(ColumnDefault::Now)); + // Anything that is not a literal or NOW() is no default this version keeps. + assert_eq!(check::default(&Expr::Parameter("p".to_owned())), None); + assert_eq!(check::default(&minus(Literal::Text("x".to_owned()))), None); +} + +/// A column as the parser hands over one that wrote no clause. +fn declared(name: &str, data_type: DataType) -> ast::ColumnDef { + ast::ColumnDef { + name: name.to_owned(), + data_type, + nullable: true, + primary_key: false, + unique: false, + default: None, + } +} + +/// A CREATE TABLE as the planner hands over one that wrote no table option. +fn create(name: &str, columns: Vec, unique: &[&[&str]]) -> CreateTablePlan { + CreateTablePlan { + name: name.to_owned(), + columns, + unique_constraints: unique + .iter() + .map(|rule| rule.iter().map(|column| (*column).to_owned()).collect()) + .collect(), + if_not_exists: false, + } +} + +#[test] +fn a_declared_table_gets_its_numbers_and_the_index_behind_each_rule() { + let key = |name: &str| ast::ColumnDef { + primary_key: true, + ..declared(name, DataType::Integer) + }; + let people = create( + "people", + vec![ + ast::ColumnDef { + nullable: false, + unique: true, + ..key("id") + }, + ast::ColumnDef { + unique: true, + ..declared("email", DataType::Text) + }, + ast::ColumnDef { + default: Some(Expr::Literal(Literal::Real(0.0))), + ..declared("score", DataType::Real) + }, + declared("born", DataType::Integer), + declared("doc", DataType::Json), + ], + &[&["email", "born"]], + ); + let mut catalog = Catalog { + tables: BTreeMap::new(), + }; + let table = check::new_table(&people, &catalog).expect("a plain table"); + let names = |table: &Table| -> Vec { + let indexes = table.meta.indexes.iter(); + indexes.map(|index| index.name.clone()).collect() + }; + assert_eq!(table.id, TableId(0)); + // The key column keeps the UNIQUE its author wrote and gets no second index for it. + assert_eq!( + names(&table), + ["__pk_id", "__unique_email", "__unique_email_born"] + ); + assert_eq!(table.index_ids, [0, 1, 2].map(IndexId)); + assert!(render(&table).starts_with( + "CREATE TABLE people (\n id INTEGER PRIMARY KEY NOT NULL UNIQUE,\n email TEXT UNIQUE,\n \ + score REAL DEFAULT 0.0,\n" + )); + catalog.tables.insert(table.name.clone(), table); + + // The next table and the next index count on from the ones the catalog holds. + let tags = create("tags", vec![key("id")], &[]); + let tags = check::new_table(&tags, &catalog).expect("a second table"); + assert_eq!( + (tags.id, &tags.index_ids[..]), + (TableId(1), &[IndexId(3)][..]) + ); + catalog.tables.insert(tags.name.clone(), tags); + let index = |name: &str, column: &str| CreateIndexPlan { + name: name.to_owned(), + table: "people".to_owned(), + columns: vec![(column.to_owned(), SortDirection::Desc)], + }; + let indexed = check::with_index(&index("by_born", "born"), &catalog).expect("an index"); + assert_eq!(names(&indexed)[3], "by_born"); + assert_eq!(indexed.index_ids, [0, 1, 2, 4].map(IndexId)); + catalog.tables.insert(indexed.name.clone(), indexed); + + // What is refused, and as what. + assert!(matches!( + check::with_index(&index("__unique_born", "born"), &catalog), + Err(Error::ReservedIndexName { prefix, .. }) if prefix == "__unique_" + )); + assert!(matches!( + check::with_index(&index("by_age", "age"), &catalog), + Err(Error::ColumnNotFound { table, column }) if table == "people" && column == "age" + )); + assert!(matches!( + check::with_index(&index("by_doc", "doc"), &catalog), + Err(Error::ColumnNotIndexable { column, column_type: ColumnType::Json, .. }) + if column == "doc" + )); + // A REAL may be indexed, and may not be a key or under a UNIQUE rule. + check::with_index(&index("by_score", "score"), &catalog).expect("an index over a REAL"); + for unique in [&[][..], &[&["id", "ratio"][..]][..]] { + let ratio = ast::ColumnDef { + unique: unique.is_empty(), + ..declared("ratio", DataType::Real) + }; + assert!(matches!( + check::new_table(&create("ratios", vec![key("id"), ratio], unique), &catalog), + Err(Error::ColumnNotIndexable { column, column_type: ColumnType::Real, .. }) + if column == "ratio" + )); + } + assert!(matches!( + check::with_index(&index("by_born", "email"), &catalog), + Err(Error::DuplicateIndex { table, index }) if table == "people" && index == "by_born" + )); + // Rules whose columns spell one name stay separate rules, each with its own index. + let pairs = create( + "pairs", + vec![ + key("id"), + declared("a", DataType::Integer), + declared("b", DataType::Integer), + declared("a_b", DataType::Integer), + ], + &[&["a_b"], &["a", "b"], &["a", "b"]], + ); + let pairs = check::new_table(&pairs, &catalog).expect("three rules"); + assert_eq!( + names(&pairs), + [ + "__pk_id", + "__unique_a_b", + "__unique_a_b_2", + "__unique_a_b_3" + ] + ); + assert_eq!(pairs.meta.indexes[1].columns.len(), 1); + assert_eq!(pairs.meta.indexes[2].columns.len(), 2); + let not_supported = |plan: &CreateTablePlan, clause: &str| { + matches!( + check::new_table(plan, &catalog), + Err(Error::PlanError(text)) if text.starts_with(clause) && text.contains("not supported") + ) + }; + let keyless = create("keyless", vec![declared("word", DataType::Text)], &[]); + assert!(not_supported(&keyless, "a table without a PRIMARY KEY")); +} diff --git a/v2/contextdb/src/sql/ast.rs b/v2/contextdb/src/sql/ast.rs new file mode 100644 index 000000000..fbc96b757 --- /dev/null +++ b/v2/contextdb/src/sql/ast.rs @@ -0,0 +1,162 @@ +//! A statement as it was written: what `parse` returns and `plan` reads. + +pub use crate::types::SortDirection; + +#[derive(Debug, Clone)] +pub enum Statement { + CreateTable(CreateTable), + CreateIndex(CreateIndex), + Insert(Insert), + Delete(Delete), + Update(Update), + Select(Select), + SetMemoryLimit(SetMemoryLimitValue), + ShowMemoryLimit, +} + +#[derive(Debug, Clone)] +pub enum SetMemoryLimitValue { + Bytes(usize), + None, +} + +#[derive(Debug, Clone)] +pub struct Select { + /// `SELECT *` is one column named `*`. + pub columns: Vec, + pub table: String, + pub where_clause: Option, + pub order_by: Vec, + pub limit: Option, +} + +#[derive(Debug, Clone)] +pub struct SelectColumn { + pub expr: Expr, + pub alias: Option, +} + +#[derive(Debug, Clone)] +pub struct OrderByItem { + pub expr: Expr, + pub direction: SortDirection, +} + +#[derive(Debug, Clone)] +pub enum Expr { + Column(ColumnRef), + Literal(Literal), + Parameter(String), + BinaryOp { + left: Box, + op: BinOp, + right: Box, + }, + UnaryOp { + op: UnaryOp, + operand: Box, + }, + /// A call by name. `a + b` arrives as the call `__add(a, b)`, and likewise `__sub`, `__mul` + /// and `__div`. The `*` of `COUNT(*)` is a column named `*`. + FunctionCall { + name: String, + args: Vec, + }, + IsNull { + expr: Box, + negated: bool, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ColumnRef { + pub table: Option, + pub column: String, +} + +#[derive(Debug, Clone)] +pub enum Literal { + Null, + Bool(bool), + Integer(i64), + Real(f64), + Text(String), + Vector(Vec), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BinOp { + Eq, + Neq, + Lt, + Lte, + Gt, + Gte, + And, + Or, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum UnaryOp { + Not, + Neg, +} + +#[derive(Debug, Clone)] +pub struct Insert { + pub table: String, + /// Empty when the statement names no columns. + pub columns: Vec, + pub values: Vec>, +} + +#[derive(Debug, Clone)] +pub struct Delete { + pub table: String, + pub where_clause: Option, +} + +#[derive(Debug, Clone)] +pub struct Update { + pub table: String, + pub assignments: Vec<(String, Expr)>, + pub where_clause: Option, +} + +#[derive(Debug, Clone)] +pub struct CreateTable { + pub name: String, + pub columns: Vec, + /// The columns of each table-level `UNIQUE (...)`, in the order written. + pub unique_constraints: Vec>, + pub if_not_exists: bool, +} + +#[derive(Debug, Clone)] +pub struct ColumnDef { + pub name: String, + pub data_type: DataType, + pub nullable: bool, + pub primary_key: bool, + pub unique: bool, + pub default: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DataType { + Uuid, + Text, + Integer, + Real, + Boolean, + Timestamp, + Json, + Vector(u32), +} + +#[derive(Debug, Clone)] +pub struct CreateIndex { + pub name: String, + pub table: String, + pub columns: Vec<(String, SortDirection)>, +} diff --git a/v2/contextdb/src/sql/grammar.pest b/v2/contextdb/src/sql/grammar.pest new file mode 100644 index 000000000..909af2aef --- /dev/null +++ b/v2/contextdb/src/sql/grammar.pest @@ -0,0 +1,157 @@ +WHITESPACE = _{ SPACE | line_comment | block_comment } +SPACE = _{ " " | "\t" | "\r" | "\n" } +line_comment = _{ "--" ~ (!"\n" ~ ANY)* } +block_comment = _{ "/*" ~ (!"*/" ~ ANY)* ~ "*/" } + +statement = { SOI ~ stmt ~ ";"? ~ EOI } +stmt = _{ + create_table_stmt + | create_index_stmt + | insert_stmt + | delete_stmt + | update_stmt + | set_memory_limit + | show_memory_limit + | select_stmt + | later +} + +// A statement or clause that a later version of the library reads. It is recognised by its +// first words alone and the rest of the text is left unread, so that the statement is refused by +// that name and not as a syntax error. `later` stands only where nothing this version reads can +// begin with one of these words: after a whole statement or clause, and after an expression, +// where IN, LIKE, BETWEEN and the distance operator `<=>` would follow. DISTINCT and GRAPH_TABLE +// stand where a name could, so they are looked for first. +later = { later_name ~ ANY* } +later_name = @{ + "<=>" + | ( + ^"ALTER" | ^"DROP" | ^"WITH" | ^"BEGIN" | ^"COMMIT" | ^"ROLLBACK" | ^"PURGE" | ^"DISCARD" + | ^"DECLARE" | ^"DISTINCT" | ^"GRAPH_TABLE" + | ((^"LEFT" | ^"INNER" | ^"CROSS") ~ SPACE+)? ~ ^"JOIN" + | ^"GROUP" ~ SPACE+ ~ ^"BY" | ^"HAVING" | ^"UNION" | ^"OFFSET" + | ^"USE" ~ SPACE+ ~ (^"VECTOR" | ^"RANK") + | ^"ON" ~ SPACE+ ~ ^"CONFLICT" + | (^"NOT" ~ SPACE+)? ~ (^"IN" | ^"LIKE" | ^"BETWEEN") + | ^"FOREIGN" ~ SPACE+ ~ ^"KEY" | ^"REFERENCES" | ^"EXPIRES" | ^"IMMUTABLE" + | ^"RANK_POLICY" | ^"CONTEXT_ID" | ^"SCOPE_LABEL_READ" | ^"SCOPE_LABEL" | ^"ACL" + | ^"PARTITION_KEY" | ^"MAX_PARTITIONS" | ^"SEARCH_MODE" | ^"AUTO_INDEX_AT" | ^"SEARCH" + | ^"IVF" | ^"CONSOLIDATION" + | ^"RETAIN" | ^"HISTORY" | ^"SYNC" | ^"STATE_MACHINE" | ^"STATE" ~ SPACE+ ~ ^"MACHINE" + | ^"DAG" | ^"PROPAGATE" | ^"DELIVERY" ~ SPACE+ ~ ^"MANIFEST" + | ^"EDGE" ~ SPACE+ ~ ^"DISCARD" | ^"MAINTAIN" ~ SPACE+ ~ ^"COUNT" + ) ~ !ident_char +} + +set_memory_limit = { ^"SET" ~ ^"MEMORY_LIMIT" ~ "="? ~ ("'" ~ memory_limit_value ~ "'" | memory_limit_value) } +show_memory_limit = { ^"SHOW" ~ ^"MEMORY_LIMIT" } +memory_limit_value = { ^"none" | size_with_unit } +size_with_unit = @{ ASCII_DIGIT+ ~ ("G" | "M" | "K") } + +select_stmt = { + ^"SELECT" + ~ ( + &^"DISTINCT" ~ later + | select_list ~ ^"FROM" + ~ ( + &(^"GRAPH_TABLE" ~ "(") ~ later + | identifier ~ where_clause? ~ order_by_clause? ~ limit_clause? ~ later? + ) + ) +} +select_list = { star | select_item ~ ("," ~ select_item)* } +star = { "*" } +select_item = { expr ~ (^"AS" ~ identifier)? } +where_clause = { ^"WHERE" ~ expr } +order_by_clause = { ^"ORDER" ~ ^"BY" ~ order_item ~ ("," ~ order_item)* } +order_item = { expr ~ sort_dir? } +sort_dir = { ^"ASC" | ^"DESC" } +limit_clause = { ^"LIMIT" ~ integer } + +create_table_stmt = { + ^"CREATE" ~ ^"TABLE" ~ if_not_exists? ~ identifier + ~ "(" ~ table_element ~ ("," ~ table_element)* + // A later clause may follow a column, stand as an element of its own, or follow the list. + ~ (")" ~ later? | ","? ~ later) +} +if_not_exists = { ^"IF" ~ ^"NOT" ~ ^"EXISTS" } +table_element = _{ column_def | unique_table_constraint } +unique_table_constraint = { ^"UNIQUE" ~ "(" ~ identifier ~ ("," ~ identifier)* ~ ")" } +column_def = { identifier ~ data_type ~ column_constraint* } +data_type = { + vector_type + | ^"UUID" | ^"TEXT" | ^"INTEGER" | ^"INT" | ^"REAL" | ^"FLOAT" | ^"BOOLEAN" | ^"BOOL" + | ^"TIMESTAMP" | ^"JSON" +} +vector_type = { ^"VECTOR" ~ "(" ~ integer ~ ")" } +column_constraint = _{ not_null | nullable_marker | primary_key | unique | default_clause } +not_null = { ^"NOT" ~ ^"NULL" } +nullable_marker = { ^"NULL" } +primary_key = { ^"PRIMARY" ~ ^"KEY" } +unique = { ^"UNIQUE" } +default_clause = { ^"DEFAULT" ~ expr } + +create_index_stmt = { + ^"CREATE" ~ ^"INDEX" ~ identifier ~ ^"ON" ~ identifier + ~ "(" ~ indexed_column ~ ("," ~ indexed_column)* ~ ")" +} +indexed_column = { identifier ~ sort_dir? } + +insert_stmt = { + ^"INSERT" ~ ^"INTO" ~ identifier + ~ ("(" ~ identifier ~ ("," ~ identifier)* ~ ")")? + ~ ^"VALUES" ~ values_row ~ ("," ~ values_row)* + ~ later? +} +values_row = { "(" ~ expr ~ ("," ~ expr)* ~ ")" } +assignment = { identifier ~ "=" ~ expr } + +delete_stmt = { ^"DELETE" ~ ^"FROM" ~ identifier ~ where_clause? } +update_stmt = { ^"UPDATE" ~ identifier ~ ^"SET" ~ assignment ~ ("," ~ assignment)* ~ where_clause? } + +// One level of rule here is one level of the parser's recursion for every parenthesis, so the +// boolean operators share one flat rule and the arithmetic operators another, and the builder +// applies their precedence: NOT, then AND, then OR; `*` and `/` before `+` and `-`. +expr = { not_op* ~ additive_expr ~ comparison_suffix? ~ (bool_op ~ not_op* ~ additive_expr ~ comparison_suffix?)* } +bool_op = _{ and_op | or_op } +or_op = @{ ^"OR" ~ !ident_char } +and_op = @{ ^"AND" ~ !ident_char } +not_op = @{ ^"NOT" ~ !ident_char } + +comparison_suffix = _{ is_null_suffix | cmp_suffix | later } +cmp_suffix = { cmp_op ~ additive_expr } +cmp_op = { "=" | "!=" | "<>" | "<=" | ">=" | "<" | ">" } +is_null_suffix = { ^"IS" ~ not_op? ~ ^"NULL" } + +additive_expr = { unary_minus* ~ primary_expr ~ (math_op ~ unary_minus* ~ primary_expr)* } +math_op = { "+" | "-" | "*" | "/" } +unary_minus = { "-" } + +primary_expr = _{ + function_call + | parameter + | null_lit + | bool_lit + | float + | integer + | string + | vector_lit + | column_ref + | "(" ~ expr ~ ")" +} +vector_lit = { "[" ~ vector_component ~ ("," ~ vector_component)* ~ "]" } +vector_component = @{ "-"? ~ ASCII_DIGIT+ ~ ("." ~ ASCII_DIGIT+)? } +function_call = { identifier ~ "(" ~ (star | (expr ~ ("," ~ expr)*))? ~ ")" } +column_ref = { identifier ~ ("." ~ identifier)? } + +parameter = @{ "$" ~ ident } +null_lit = { ^"NULL" } +bool_lit = { ^"TRUE" | ^"FALSE" } +float = @{ ASCII_DIGIT+ ~ "." ~ ASCII_DIGIT+ } +integer = @{ ASCII_DIGIT+ } +string = @{ "'" ~ ("''" | (!"'" ~ ANY))* ~ "'" } + +ident_char = _{ ASCII_ALPHANUMERIC | "_" } +ident = @{ (ASCII_ALPHA | "_") ~ ident_char* } +quoted_ident = @{ "\"" ~ ("\"\"" | (!"\"" ~ ANY))* ~ "\"" } +identifier = { quoted_ident | ident } diff --git a/v2/contextdb/src/sql/mod.rs b/v2/contextdb/src/sql/mod.rs new file mode 100644 index 000000000..91a5f3c7c --- /dev/null +++ b/v2/contextdb/src/sql/mod.rs @@ -0,0 +1,29 @@ +//! sql text: grammar, parser, planner. +//! +//! It never touches storage: it turns text into a statement and a statement into a plan, and +//! knows nothing of what any table holds. +//! +//! The fourth thing the parts meet at is defined here: [`Plan`], the planned statement that +//! `schema`, `write` and `query` each take. + +pub(crate) mod ast; +mod parser; +pub(crate) mod plan; +mod planner; + +pub(crate) use ast::{Expr, Statement}; +pub(crate) use plan::{Effect, PhysicalPlan as Plan}; + +use crate::types::Result; + +/// Reads one SQL statement. Refuses with `ParseError`, or with `PlanError` when the text begins +/// a statement or a clause that a later version of the library reads. Costs time in proportion +/// to the text. +pub(crate) fn parse(text: &str) -> Result { + parser::parse(text) +} + +/// Plans a parsed statement. Refuses with `PlanError`. Looks at no table and no data. +pub(crate) fn plan(statement: &Statement) -> Result { + planner::plan(statement) +} diff --git a/v2/contextdb/src/sql/parser/expr.rs b/v2/contextdb/src/sql/parser/expr.rs new file mode 100644 index 000000000..15249888c --- /dev/null +++ b/v2/contextdb/src/sql/parser/expr.rs @@ -0,0 +1,243 @@ +//! Expressions. The grammar reads the boolean operators as one flat run and the arithmetic +//! operators as another; the functions here apply their precedence. +//! +//! `expr`, `comparison_expr`, `additive_expr` and `primary_expr` call each other once for every +//! level of nesting, to the limit, on the caller's stack. So each is kept small: what is not on +//! that path is in a function of its own, and a list is gathered with a loop. + +use super::{MAX_EXPR_NESTING, Pair, Rule, depth_error, has, ident, number, string}; +use crate::sql::ast::*; +use crate::types::Result; + +pub(super) fn expr(pair: Pair<'_>) -> Result { + // `[NOT] comparison {AND | OR [NOT] comparison}`. AND binds tighter than OR: `all` gathers + // the comparisons joined by AND, `any` the runs between ORs. + let mut any = Vec::new(); + let mut all = Vec::new(); + let mut nots = 0; + let mut parts = pair.into_inner().peekable(); + while let Some(part) = parts.next() { + match part.as_rule() { + Rule::not_op => nots += 1, + Rule::and_op => {} + Rule::or_op => any.push(balanced(std::mem::take(&mut all), BinOp::And)), + _ => { + let suffix = + parts.next_if(|next| !matches!(next.as_rule(), Rule::and_op | Rule::or_op)); + let comparison = comparison_expr(part, suffix)?; + all.push( + (0..std::mem::take(&mut nots)).fold(comparison, |operand, _| not(operand)), + ); + } + } + } + any.push(balanced(all, BinOp::And)); + let built = balanced(any, BinOp::Or); + let depth = depth_of(&built); + if depth > MAX_EXPR_NESTING { + return Err(depth_error(depth)); + } + Ok(built) +} + +/// Every expression directly inside `pair`, in order. +pub(super) fn exprs(pair: Pair<'_>) -> Result> { + let mut exprs = Vec::new(); + for part in pair.into_inner() { + if part.as_rule() == Rule::expr { + exprs.push(expr(part)?); + } + } + Ok(exprs) +} + +/// How many expressions deep the tree goes below its root. +fn depth_of(expr: &Expr) -> usize { + let below = match expr { + Expr::BinaryOp { left, right, .. } => depth_of(left).max(depth_of(right)), + Expr::UnaryOp { operand, .. } => depth_of(operand), + Expr::FunctionCall { args, .. } => args.iter().map(depth_of).max().unwrap_or(0), + Expr::IsNull { expr, .. } => depth_of(expr), + Expr::Column(_) | Expr::Literal(_) | Expr::Parameter(_) => return 0, + }; + below + 1 +} + +/// Joins the operands of a run of ANDs or ORs into a tree as shallow as it can be, keeping +/// their order, so a long run does not count against the nesting limit. +fn balanced(operands: Vec, op: BinOp) -> Expr { + fn build(operands: &mut std::vec::IntoIter, len: usize, op: BinOp) -> Expr { + if len == 1 { + return operands.next().expect("an operand"); + } + let left = build(operands, len / 2, op); + let right = build(operands, len - len / 2, op); + Expr::BinaryOp { + left: Box::new(left), + op, + right: Box::new(right), + } + } + let len = operands.len(); + build(&mut operands.into_iter(), len, op) +} + +fn not(operand: Expr) -> Expr { + Expr::UnaryOp { + op: UnaryOp::Not, + operand: Box::new(operand), + } +} + +/// One comparison: its left side and what the grammar read after it, if anything. +fn comparison_expr(left: Pair<'_>, suffix: Option>) -> Result { + let left = Box::new(additive_expr(left)?); + let Some(suffix) = suffix else { + return Ok(*left); + }; + if suffix.as_rule() == Rule::is_null_suffix { + return Ok(Expr::IsNull { + expr: left, + negated: has(&suffix, Rule::not_op), + }); + } + let mut parts = suffix.into_inner(); + let op = match parts.next().expect("an operator").as_str() { + "=" => BinOp::Eq, + "!=" | "<>" => BinOp::Neq, + "<" => BinOp::Lt, + "<=" => BinOp::Lte, + ">" => BinOp::Gt, + _ => BinOp::Gte, + }; + Ok(Expr::BinaryOp { + left, + op, + right: Box::new(additive_expr(parts.next().expect("an operand"))?), + }) +} + +/// `left right` as a call of `__add`, `__sub`, `__mul` or `__div`. A chain of operators +/// builds a tree one deeper per operator, so its depth is checked as it grows. +fn arithmetic(op: &str, left: Expr, right: Expr) -> Result { + let depth = depth_of(&left).max(depth_of(&right)) + 1; + if depth > MAX_EXPR_NESTING { + return Err(depth_error(depth)); + } + let name = match op { + "+" => "__add", + "-" => "__sub", + "*" => "__mul", + _ => "__div", + }; + Ok(Expr::FunctionCall { + name: name.to_owned(), + args: vec![left, right], + }) +} + +/// `[-] primary {+ | - | * | / [-] primary}`, read left to right with `*` and `/` binding +/// tighter. +fn additive_expr(pair: Pair<'_>) -> Result { + // `sum` is everything before the last `+` or `-`, with that operator; `product` is the run + // of `*` and `/` being read, with the operator that waits for its right side. + let mut sum: Option<(Expr, &str)> = None; + let mut product: Option<(Expr, &str)> = None; + let mut minuses = 0; + for part in pair.into_inner() { + match part.as_rule() { + Rule::unary_minus => minuses += 1, + Rule::math_op => { + let (operand, _) = product.take().expect("an operand before an operator"); + let op = part.as_str(); + if op == "*" || op == "/" { + product = Some((operand, op)); + } else { + let total = match sum.take() { + Some((left, plus_or_minus)) => arithmetic(plus_or_minus, left, operand)?, + None => operand, + }; + sum = Some((total, op)); + } + } + _ => { + let mut operand = primary_expr(part)?; + for _ in 0..std::mem::take(&mut minuses) { + operand = Expr::UnaryOp { + op: UnaryOp::Neg, + operand: Box::new(operand), + }; + } + product = Some(match product.take() { + Some((left, times_or_over)) => (arithmetic(times_or_over, left, operand)?, ""), + None => (operand, ""), + }); + } + } + } + let (operand, _) = product.expect("an operand"); + match sum { + Some((left, plus_or_minus)) => arithmetic(plus_or_minus, left, operand), + None => Ok(operand), + } +} + +/// The `*` of `SELECT *` and of `COUNT(*)`: a column of that name. +pub(super) fn star() -> Expr { + Expr::Column(ColumnRef { + table: None, + column: "*".to_owned(), + }) +} + +fn primary_expr(primary: Pair<'_>) -> Result { + match primary.as_rule() { + // A parenthesised expression. + Rule::expr => expr(primary), + Rule::function_call => function_call(primary), + Rule::column_ref => Ok(column_ref(primary)), + _ => literal(&primary), + } +} + +fn literal(pair: &Pair<'_>) -> Result { + Ok(match pair.as_rule() { + Rule::parameter => Expr::Parameter(pair.as_str()[1..].to_owned()), + Rule::null_lit => Expr::Literal(Literal::Null), + Rule::bool_lit => Expr::Literal(Literal::Bool(pair.as_str().eq_ignore_ascii_case("true"))), + Rule::float => Expr::Literal(Literal::Real(number(pair, "the number")?)), + Rule::integer => Expr::Literal(Literal::Integer(number(pair, "the whole number")?)), + Rule::string => Expr::Literal(Literal::Text(string(pair))), + // The one literal left: a vector. + _ => { + let components = pair + .clone() + .into_inner() + .map(|component| number(&component, "the vector component")) + .collect::>()?; + Expr::Literal(Literal::Vector(components)) + } + }) +} + +fn column_ref(pair: Pair<'_>) -> Expr { + let mut names: Vec = pair.into_inner().map(|name| ident(&name)).collect(); + let column = names.pop().expect("a column name"); + Expr::Column(ColumnRef { + table: names.pop(), + column, + }) +} + +fn function_call(pair: Pair<'_>) -> Result { + let mut parts = pair.into_inner(); + let name = ident(&parts.next().expect("a function name")); + let mut args = Vec::new(); + for arg in parts { + args.push(match arg.as_rule() { + Rule::star => star(), + _ => expr(arg)?, + }); + } + Ok(Expr::FunctionCall { name, args }) +} diff --git a/v2/contextdb/src/sql/parser/mod.rs b/v2/contextdb/src/sql/parser/mod.rs new file mode 100644 index 000000000..3a06ff0a3 --- /dev/null +++ b/v2/contextdb/src/sql/parser/mod.rs @@ -0,0 +1,463 @@ +//! Reads SQL text into a [`Statement`]. The grammar is `grammar.pest`; the functions here and in +//! the files beside this one turn what the grammar matched into the statement types. +//! +//! A builder trusts the grammar for the shape of what it is handed (which parts are present and +//! in what order) and checks only what a grammar cannot say: a number's range, a clause written +//! twice, two clauses that contradict each other. + +mod expr; + +use std::str::FromStr; + +use pest::Parser; +use pest::error::{InputLocation, LineColLocation}; + +use super::ast::*; +use crate::types::{Error, Result}; + +#[derive(pest_derive::Parser)] +#[grammar = "sql/grammar.pest"] +struct SqlParser; + +type Pair<'a> = pest::iterators::Pair<'a, Rule>; + +/// The deepest an expression may nest, in the text and in the tree built from it. It bounds the +/// parser's own recursion and the recursion of everything that later walks an expression. +const MAX_EXPR_NESTING: usize = 128; + +pub(super) fn parse(text: &str) -> Result { + check_source_nesting(text)?; + let read = + SqlParser::parse(Rule::statement, text).map_err(|error| syntax_error(text, &error))?; + if let Some(later) = read + .clone() + .flatten() + .find(|part| part.as_rule() == Rule::later_name) + { + let words: Vec<&str> = later.as_str().split_whitespace().collect(); + return Err(Error::PlanError(format!( + "{} is not supported in this version of the library", + words.join(" ").to_ascii_uppercase() + ))); + } + let pair = read + .flatten() + .nth(1) + .expect("the grammar's statement rule holds one statement"); + match pair.as_rule() { + Rule::create_table_stmt => create_table(pair).map(Statement::CreateTable), + Rule::create_index_stmt => { + let [name, table] = two_idents(pair.clone()); + Ok(Statement::CreateIndex(CreateIndex { + name, + table, + columns: pair + .into_inner() + .filter(|part| part.as_rule() == Rule::indexed_column) + .map(|column| { + let mut parts = column.into_inner(); + let name = ident(&parts.next().expect("a column")); + (name, direction(parts.next())) + }) + .collect(), + })) + } + Rule::insert_stmt => insert(pair).map(Statement::Insert), + Rule::delete_stmt => Ok(Statement::Delete(Delete { + table: first_ident(pair.clone()), + where_clause: where_clause(pair)?, + })), + Rule::update_stmt => Ok(Statement::Update(Update { + table: first_ident(pair.clone()), + assignments: assignments(pair.clone())?, + where_clause: where_clause(pair)?, + })), + Rule::select_stmt => select(pair).map(Statement::Select), + Rule::set_memory_limit => memory_limit(pair).map(Statement::SetMemoryLimit), + // The one statement left: SHOW MEMORY_LIMIT. + _ => Ok(Statement::ShowMemoryLimit), + } +} + +/// The refusal for text the grammar does not read: where it stopped and the word it stopped at. +fn syntax_error(text: &str, error: &pest::error::Error) -> Error { + let (LineColLocation::Pos((line, column)) | LineColLocation::Span((line, column), _)) = + error.line_col; + let (InputLocation::Pos(at) | InputLocation::Span((at, _))) = error.location; + let word: String = text[at..] + .trim_start() + .chars() + .take_while(|ch| !ch.is_whitespace()) + .take(64) + .collect(); + if word.is_empty() { + refuse(format!( + "syntax error at line {line}, column {column}: the statement ends too soon" + )) + } else { + refuse(format!( + "syntax error at line {line}, column {column}: unexpected '{word}'" + )) + } +} + +fn depth_error(depth: usize) -> Error { + refuse(format!( + "expression nesting of {depth} is deeper than the limit of {MAX_EXPR_NESTING}" + )) +} + +/// The words and marks of a statement, without its comments. A quoted text or name comes as its +/// opening quote alone, so nothing inside quotes is ever taken for a keyword or a parenthesis. +struct Tokens<'a> { + text: &'a str, + at: usize, +} + +impl<'a> Iterator for Tokens<'a> { + type Item = &'a str; + + fn next(&mut self) -> Option<&'a str> { + loop { + let rest = &self.text[self.at..]; + let first = rest.chars().next()?; + let len = if first.is_whitespace() { + first.len_utf8() + } else if rest.starts_with("--") { + rest.find('\n').unwrap_or(rest.len()) + } else if rest.starts_with("/*") { + rest.find("*/").map_or(rest.len(), |end| end + 2) + } else { + break; + }; + self.at += len; + } + let rest = &self.text[self.at..]; + let first = rest.chars().next()?; + let (token, consumed) = if first == '\'' || first == '"' { + // A doubled quote inside is the quote itself, so the text ends at the first quote + // that is not followed by another. + let mut end = 1; + loop { + match rest[end..].find(first) { + None => break (&rest[..1], rest.len()), + Some(quote) if rest[end + quote + 1..].starts_with(first) => end += quote + 2, + Some(quote) => break (&rest[..1], end + quote + 1), + } + } + } else if first.is_ascii_alphanumeric() || first == '_' { + let end = rest + .find(|ch: char| !ch.is_ascii_alphanumeric() && ch != '_') + .unwrap_or(rest.len()); + (&rest[..end], end) + } else { + (&rest[..first.len_utf8()], first.len_utf8()) + }; + self.at += consumed; + Some(token) + } +} + +/// Refuses text whose expressions nest deeper than the limit, before the grammar recurses into +/// it. A parenthesis, a NOT and a unary minus each count one level. +fn check_source_nesting(text: &str) -> Result<()> { + // The depth outside each open parenthesis, innermost last. + let mut frames = vec![0usize]; + // NOT and unary minus read since the last operand. + let mut prefixes = 0; + // Whether a minus read now would be unary. + let mut operand_next = true; + for token in (Tokens { text, at: 0 }) { + match token { + "(" => { + frames.push(frames[frames.len() - 1] + prefixes + 1); + prefixes = 0; + operand_next = true; + } + ")" => { + if frames.len() > 1 { + frames.pop(); + } + prefixes = 0; + operand_next = false; + } + "-" if operand_next => prefixes += 1, + _ if token.eq_ignore_ascii_case("NOT") => { + prefixes += 1; + operand_next = true; + } + _ => { + prefixes = 0; + operand_next = matches!(token, "=" | "<" | ">" | "+" | "-" | "*" | "/" | ",") + || ["AND", "OR", "WHERE", "SELECT", "BY"] + .iter() + .any(|word| token.eq_ignore_ascii_case(word)); + } + } + let depth = frames[frames.len() - 1] + prefixes; + if depth > MAX_EXPR_NESTING { + return Err(depth_error(depth)); + } + } + Ok(()) +} + +fn refuse(message: impl Into) -> Error { + Error::ParseError(message.into()) +} + +/// A name as written, or what is between the double quotes of a quoted one. +fn ident(pair: &Pair<'_>) -> String { + let raw = pair.as_str(); + match raw.strip_prefix('"').and_then(|raw| raw.strip_suffix('"')) { + Some(quoted) => quoted.replace("\"\"", "\""), + None => raw.to_owned(), + } +} + +/// What is between the single quotes of a quoted text. +fn string(pair: &Pair<'_>) -> String { + let raw = pair.as_str(); + raw[1..raw.len() - 1].replace("''", "'") +} + +/// Every name directly inside `pair`, in order. +fn idents(pair: Pair<'_>) -> Vec { + pair.into_inner() + .filter(|part| part.as_rule() == Rule::identifier) + .map(|part| ident(&part)) + .collect() +} + +fn first_ident(pair: Pair<'_>) -> String { + idents(pair).swap_remove(0) +} + +fn two_idents(pair: Pair<'_>) -> [String; 2] { + let mut names = idents(pair).into_iter(); + [(); 2].map(|()| names.next().expect("the grammar reads two names here")) +} + +/// The first part of `pair` that the grammar read with `rule`. +fn child<'a>(pair: Pair<'a>, rule: Rule) -> Option> { + pair.into_inner().find(|part| part.as_rule() == rule) +} + +fn has(pair: &Pair<'_>, rule: Rule) -> bool { + child(pair.clone(), rule).is_some() +} + +/// A number the grammar read as digits, refused when it does not fit the type it is for. +fn number(pair: &Pair<'_>, what: &str) -> Result { + pair.as_str() + .parse() + .map_err(|_| refuse(format!("{what} {} is out of range", pair.as_str()))) +} + +fn insert(pair: Pair<'_>) -> Result { + let mut values = Vec::new(); + for row in pair.clone().into_inner() { + if row.as_rule() == Rule::values_row { + values.push(expr::exprs(row)?); + } + } + let mut names = idents(pair); + let columns = names.split_off(1); + Ok(Insert { + table: names.swap_remove(0), + columns, + values, + }) +} + +/// Every `column = value` directly inside `pair`. +fn assignments(pair: Pair<'_>) -> Result> { + pair.into_inner() + .filter(|part| part.as_rule() == Rule::assignment) + .map(|assignment| { + let mut parts = assignment.into_inner(); + let column = ident(&parts.next().expect("a column")); + Ok((column, expr::expr(parts.next().expect("a value"))?)) + }) + .collect() +} + +fn where_clause(pair: Pair<'_>) -> Result> { + child(pair, Rule::where_clause) + .map(|clause| expr::expr(clause.into_inner().next().expect("a condition"))) + .transpose() +} + +/// ASC when nothing is written. +fn direction(written: Option>) -> SortDirection { + match written { + Some(written) if written.as_str().eq_ignore_ascii_case("DESC") => SortDirection::Desc, + _ => SortDirection::Asc, + } +} + +fn select(pair: Pair<'_>) -> Result