diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 6e3a9949..48377c44 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -7,6 +7,10 @@ on: branches: [dev, main] pull_request: branches: [dev, main] + # Once a night, and by hand, the same check runs with the slow work added. + schedule: + - cron: '30 3 * * *' + workflow_dispatch: permissions: contents: read @@ -20,8 +24,9 @@ env: # The size of the shipped code, rounded up to 500 lines, and the number of tests, rounded up # to 10. A change that crosses either number edits it here, in the same change, where its # reader sees that the code grew or shrank. Neither ever passes 150,000 lines or 1,000 tests. - SHIPPED_LINES: 15500 - TESTS: 260 + SHIPPED_LINES: 21000 + TESTS: 380 + TEST_LINES: 38000 jobs: lint: @@ -60,6 +65,9 @@ jobs: with: tool: cargo-nextest@0.9.138 - run: cargo nextest run --manifest-path v2/Cargo.toml --workspace --locked --no-tests=pass + - name: Slow work, in the nightly run only + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + run: cargo nextest run --manifest-path v2/Cargo.toml --workspace --locked --profile nightly - run: cargo test --manifest-path v2/Cargo.toml --workspace --locked --doc - name: The storage library's own tests run: cargo test --manifest-path v2/redb/Cargo.toml --locked --lib --tests @@ -67,11 +75,17 @@ jobs: - name: Tests are at most the stated number and within 10 of it if: matrix.os == 'ubuntu-latest' run: | - tests=$(( $(cargo nextest list --manifest-path v2/Cargo.toml --workspace --locked -T oneline | wc -l) + $(find v2/contextdb/tests/sql -name '*.sql' | wc -l) )) + tests=$(( $(cargo nextest list --manifest-path v2/Cargo.toml --workspace --locked --ignore-default-filter -T oneline | wc -l) + $(find v2/contextdb/tests/sql -name '*.sql' | wc -l) )) echo "$tests tests; the stated number is $TESTS" test "$TESTS" -le 1000 test "$tests" -le "$TESTS" test "$tests" -gt "$((TESTS - 10))" + - name: Test lines are at most the stated number + if: matrix.os == 'ubuntu-latest' + run: | + lines=$(find v2/*/tests v2/*/src -type f \( -path '*/tests/*' -o -name tests.rs \) -exec cat {} + | wc -l) + echo "$lines test lines; the stated number is $TEST_LINES" + test "$lines" -le "$TEST_LINES" minimum-rust: runs-on: ubuntu-latest diff --git a/v2/.config/nextest.toml b/v2/.config/nextest.toml index ceaec6f4..f82401e3 100644 --- a/v2/.config/nextest.toml +++ b/v2/.config/nextest.toml @@ -1,9 +1,14 @@ # A test that runs past its limit is stopped and fails by name. Nothing is retried. +# The pull-request check (the default profile) runs every test except the million-row load, +# which is slow work and runs every night in the `nightly` profile. [profile.default] retries = 0 slow-timeout = { period = "30s", terminate-after = 2 } +default-filter = 'not test(=a_million_rows_load_and_scan_under_64_mib)' -# The million-row load is the one test that needs longer. -[[profile.default.overrides]] -filter = 'test(a_million_rows_load_and_scan_under_64_mib)' -slow-timeout = { period = "120s", terminate-after = 5 } +# The nightly profile runs only the million-row load. Its limit is a hang detector, well clear +# of the slowest hosted run seen (428 seconds). +[profile.nightly] +retries = 0 +default-filter = 'test(=a_million_rows_load_and_scan_under_64_mib)' +slow-timeout = { period = "300s", terminate-after = 6 } diff --git a/v2/AGENTS.md b/v2/AGENTS.md index 2664c8a5..74eeaa5d 100644 --- a/v2/AGENTS.md +++ b/v2/AGENTS.md @@ -17,15 +17,15 @@ The library is twelve parts, one folder each under `contextdb/src/`: | Part | Does | Never | |---|---|---| -| `types` | `Value`, the one `Error`, ids, table descriptions | holds behaviour: reads no store, checks no rule, decides no answer | +| `types` | `Value`, the one `Error`, ids, table descriptions | holds behaviour: reads no store, checks no rule of a table, decides no answer; the one refusal it makes is `typed`, which turns away a value that is not of its column's type | | `system` | the time of day, through `Clock`; the one place in the library that reads a clock | decides anything | | `memory` | the budget and the reservations taken from it | allocates for another part: it counts what they are about to take and says no first | -| `sql` | grammar, parser, planner: text to a statement, a statement to a plan | touches storage or knows what a table holds | +| `sql` | grammar, parser, planner: text to a statement, a statement to a plan; reads every table option and column constraint (including `STATE MACHINE`, `PROPAGATE`, `CONTEXT_ID`, `SCOPE_LABEL`, `ACL` and the `ALTER TABLE` forms); `sql::formula` reads a rank formula, the one reading of it | touches storage or knows what a table holds | | `store` | the folder and data file, how each record is keyed, a row's large text, JSON and vectors kept beside the row in slices (text and JSON compressed there when that saves an eighth, `store/record.rs`), views of one moment, the reader files (one in `readers/` per reader that is inside a statement, naming the commit it reads) and the locks | interprets SQL or a table's rules | -| `schema` | table declarations: checking, storing, printing them back | changes a row | -| `query` | answers a SELECT from one table; holds the one expression evaluator; counts the rows it reads and a search's distance evaluations; answers `SHOW VECTOR_INDEXES`, `SHOW VECTOR_PARTITIONS` and `explain` from the vector part's small records | changes anything | -| `write` | applies INSERT, UPDATE and DELETE to a change set; checks types, NOT NULL, the primary key and UNIQUE rules; keeps index entries and row counts | returns a query's result to a caller | -| `database` | the handle a store is opened for writing with, its transactions, the one commit function; `run_maintenance_cycle` (the sealing commit, then one step for every partition with work, each worked out with no lock held and committed through the one commit function); `explain` | has a second way to commit | +| `schema` | table declarations: checking, storing, printing them back; the `Rules` of a table (references, immutable columns, propagation, the Context, label and access columns, counts) worked out once from its declaration, and `ALTER TABLE` and `DROP` (`schema/alter.rs`) with the refusals of what a change would leave dangling | changes a row | +| `query` | answers a SELECT from one table; holds the one expression evaluator and the one filter of who may see a row (`Reader`, `Visible`, in `query/visible.rs` with its tests in `query/visible/`); a narrowed `Reader` is handed to every read and every search; counts the rows it reads and a search's distance evaluations; answers `SHOW VECTOR_INDEXES`, `SHOW VECTOR_PARTITIONS` and `explain` (which takes the `Reader`) from the vector part's small records | changes anything | +| `write` | applies INSERT, UPDATE and DELETE to a change set; checks types, NOT NULL, the primary key (also the hidden key of a table with none) and UNIQUE rules, `IMMUTABLE`, the state machine, references both ways, a `DAG`, and the rows a `PROPAGATE` moves in the same statement; asks `Visible` whether the handle may write the row; keeps index entries and row counts; checks at commit what only the whole transaction answers (`write/commit.rs`) | returns a query's result to a caller | +| `database` | the handle a store is opened for writing with, the handles narrowed from it (`scoped_with_constraints`), the refusal of a statement by the kind of handle (`Database::execute` refuses what `Plan::admin_only` names when the `Reader` is narrowed; the Rust call `set_memory_limit` makes the same refusal; `run_maintenance_cycle` does not look at the narrowing), its transactions and their numbers, row guards, the one commit function (where two transactions' conflicts are decided: first commit wins, no waiting); `table_meta` and `declaration`; `run_maintenance_cycle` (the sealing commit, then one step for every partition with work, each worked out with no lock held and committed through the one commit function); `explain` | has a second way to commit | | `read` | a session that reads a store without holding it for writing, from any process; its `explain` | changes anything a writer stored | | `vector` | the blocks of a VECTOR column (a one-byte copy of every vector), the lists of vectors not yet in a block, the column's partitions and their count, the grouping of a partition (its centres, a one-bit sketch of every slot, the partition's mean), search over them (every slot, or the sketches of the nearest groups and of the tail, then the slots those keep), building a grouping in steps and putting it in force, sealing, and what inspection reads | decides which rows a reader may see: `query` tells it, row by row, and a row is in an answer only with the distance `query` worked out from the row's stored vector; reads a clock, takes the commit lock, or starts a thread | | `upkeep` | the sealing of one cycle, at most 4,096 waiting vectors moved into blocks in one commit (the 4,096 is `SEAL_ROWS` in `upkeep/mod.rs`), and the report of a cycle; `database` drives the grouping steps that follow it | starts a thread, reads a clock, or runs unless called | @@ -58,14 +58,16 @@ A part reaches another only through what that part makes visible: the `pub(crate | `ChangeSet`, everything one transaction means to change, held in memory over a `View` | `store` | `schema` and `write` stage into it; `database` hands it to `Store::commit` | | `Store`, `Key`, `Row`, `Cell`, `TableId`, `IndexId` | `store` | `database` and `read` open a `Store`; the rest name rows and indexes | | `Budget`, `Reservation` | `memory` | everything whose memory grows with data holds a reservation | -| `Catalog`, `Table` | `schema` | `query`, `write`, `database` and `read` look tables up in it | -| `eval`, `matching` | `query` | `write` finds the rows of an UPDATE or DELETE with `query::matching` and works out values with `query::eval` | +| `Catalog`, `Table`, `Rules` | `schema` | `query`, `write`, `database` and `read` look tables up in it; a table's references, immutable columns, propagation edges and its Context, label and access columns are fields of `Table::rules`, the other options are in `Table::meta` | +| `Reader`, `Visible` | `query` | `database` and `read` make a `Reader` for a handle (what it is narrowed to); `write` and `query` ask `Visible` before any row is shown or staged | +| `eval`, `rows_for_write` | `query` | `write` finds the rows of an UPDATE or DELETE with `query::rows_for_write`, a batch at a time, and works out values with `query::eval` | +| `Asked`, `EvalContext`, `reads`; `Visible::new`, `Visible::reads`, `Visible::may_write`, `Reader::admin` | `query` | `write` is also handed these to evaluate and to ask who may write; from `store` it uses `Beside`, `Entry`, `EntryShape`, `KEY_LIMIT`, `RowGuard`, `View` and `CountId`, and from `schema` the fields of `Table` | | `QueryResult` | `query` | `database` and `read` build it and hand it to the caller | | `PhysicalPlan::VectorSearch`; `Expr::Distance`, `Expr::Row`, `Expr::InList` | `sql` | `query` runs the search, puts its distance wherever the select list writes `<=>`, and evaluates `IN` lists and row values | | `VectorColumn` (a VECTOR column's number, partition key, limit, mode, quantization, `AUTO_INDEX_AT` and consolidation, with the numbers in force), from `Table::vectors`, `Table::vector` and `Catalog::vector` | `schema` | `query`, `write` and `vector` | | `staged`, `vectors` and `vector_number` on `Read`; `put_vector`, `delete_vector` and `put_vector_number` on `ChangeSet`; `VectorRecord`, `VectorNumber`, `VectorId` | `store` | `vector` reads and stages its blocks, waiting entries, partition entries and counts through them; `store` keeps them and never interprets one | | `search`, `plan`, `Nearest`, `Route`, `distance` | `vector` | `query` calls `search` with the partitions the filter selects, the mode in force, the column's declaration and whether the statement has a filter on rows, and answers it, row by row, with `distance` of the row's stored vector, or with nothing for a row that is gone, filtered out or has no vector; `search` answers nothing at all only when INDEXED is refused (a selected partition is not usable, or a filter on rows leaves the index nothing to choose among), which `query` turns into the refusal. `plan` settles what a search does with each partition, from small records alone, and `search` runs exactly that plan, so `explain` cannot disagree with it | -| `Tally`, the partitions a transaction has started and emptied | `vector` | `database` holds one beside each change set and forgets a refused statement's part of it; `write` notes every row it stages and settles the tally as a statement's last act, which is where a statement over `MAX_PARTITIONS` is refused | +| `Tally`, the partitions a transaction has started and emptied | `vector` | `database` holds one beside each change set and forgets a refused statement's part of it; `write` notes every row it stages, including the rows a propagation moves, and settles the tally once per statement after following, which is where a statement over `MAX_PARTITIONS` is refused; `vector::tally::tuples` decides which rows have an entry (none for a row in an `EXCLUDE VECTOR` state, with a NULL vector, or missing) | | `stage`, `seal` | `vector` | `database` calls `stage` inside the one commit function, after the commit's other rules have passed; `upkeep` calls `seal` | | `waiting`, `prepare`, `stage_step`, `stage_refusal`, `remaining` (and the values they return) | `vector` | `database` lists the partitions with work, has a step worked out with no lock held, and stages it through the one commit function; a step the memory limit refuses is recorded, not returned | | `partitions`, `Inspected`, `State` | `vector` | `query` answers the two SHOW statements and `explain` with them | @@ -112,6 +114,17 @@ Every path in this guide is written from `v2/`. | A text or JSON value reads back byte for byte whether or not it is stored compressed; one that does not shrink by an eighth takes the room it takes uncompressed, and text that repeats takes less than its own bytes | `contextdb/tests/large_values.rs`: `large_text_and_json_come_back_byte_for_byte`, `a_value_that_does_not_shrink_is_kept_as_it_is`, `a_store_of_repeating_text_is_smaller_than_the_text` | | No value is produced past the memory limit: a compressed value whose real size does not fit is refused before its bytes exist, and compression never refuses a commit | `contextdb/tests/large_values.rs`: `a_compressed_value_too_large_for_the_limit_is_refused`, `a_commit_with_no_room_to_compress_is_written_plain`; `contextdb/tests/memory_ceiling.rs`: `a_refused_compressed_value_is_never_produced` | | A compressed value whose stored bytes do not give the length it records is reported as store damage | `contextdb/tests/large_values.rs`: `a_damaged_length_of_a_compressed_value_is_reported_as_damage` | +| A table's rules hold for every write: `IMMUTABLE` rows and columns are never changed, a state moves only along its declared arrows, a row is never left under a parent that is gone, a `DAG` never closes a cycle | `contextdb/tests/rules.rs`: `an_immutable_refusal_names_the_table_or_the_column_and_changes_nothing`, `a_state_refusal_names_the_table_the_column_and_both_states`, `a_reference_is_checked_when_its_transaction_commits_and_a_refused_commit_leaves_nothing`, `a_parent_is_not_removed_or_given_another_key_under_a_row_that_names_it`, `a_cycle_is_refused_by_the_statement_and_by_the_commit_that_would_close_it`, `a_table_another_table_references_is_not_dropped_until_the_reference_is_gone` | +| Rows follow a change of another row inside its statement, are seen and undone with its transaction, are bounded by `MAX DEPTH`, and a follower the handle may not write refuses the statement | `contextdb/tests/propagation.rs`: `rows_follow_inside_the_statement_and_are_seen_and_undone_with_its_transaction`, `an_aborting_propagation_names_its_table_and_undoes_its_statement_and_no_other`, `max_depth_reaches_rows_at_most_that_many_edges_from_the_changed_row`, `a_follower_the_handle_may_not_write_refuses_the_statement_by_kind_and_nothing_changes` | +| Of two transactions that take one key or change one row, exactly one commits; a transaction either declares or writes | `contextdb/tests/transactions.rs`: `of_two_transactions_inserting_one_key_exactly_one_commits`, `of_two_transactions_changing_one_row_the_first_commits_and_the_second_is_refused`, `a_transaction_that_declares_a_table_may_not_also_write_rows`, `a_transaction_sees_its_own_writes_and_a_dropped_or_rolled_back_one_leaves_nothing` | +| A `TXID` column holds its transaction's number, and a commit is never numbered below it | `contextdb/tests/transaction_numbers.rs`: `a_txid_column_holds_its_transactions_number_and_a_commit_is_never_numbered_below_it`, `a_stamp_above_the_transactions_own_number_and_a_stamp_written_as_a_number_are_refused` | +| A row guard is true only for a row that is visible and holds every value, and a commit is refused when a guarded value changed meanwhile | `contextdb/tests/row_guards.rs`: `a_row_guard_answers_true_only_for_a_row_that_is_there_and_holds_every_value`, `a_row_guard_answers_false_for_a_row_hidden_from_its_handle`, `a_commit_is_refused_when_a_guarded_value_was_changed_meanwhile_and_writes_nothing`, `a_guarded_row_is_not_a_changed_row` | +| A narrowed handle sees no row outside its Contexts, labels and grants: a list leaves a hidden row out, a row named by key is refused by the rule that hid it, a refusal carries no value or number of the row, an update or delete changes no hidden row | `contextdb/tests/visibility.rs`: `a_hidden_row_named_whole_is_refused_and_a_list_read_leaves_it_out`, `a_row_of_another_context_named_by_key_is_a_context_refusal_and_a_list_read_leaves_it_out`, `a_row_under_another_label_named_by_key_is_a_label_refusal_and_a_list_read_leaves_it_out`, `a_row_with_no_grant_named_by_key_is_an_access_refusal_and_a_list_read_leaves_it_out`, `an_update_or_a_delete_changes_no_row_the_handle_may_not_see`, `no_refusal_holds_a_value_or_a_number_of_a_row_the_handle_may_not_see`; `contextdb/tests/visibility_grants.rs`: `a_grant_row_the_reader_may_not_see_gives_no_access`; `contextdb/tests/table_source.rs`: `a_narrowed_reader_naming_a_hidden_row_by_its_key_is_refused_by_the_rule_that_hid_it` | +| A narrowed handle writes no row outside what it may see, is refused every declaration and setting, and a handle made from it sees no more | `contextdb/tests/visibility.rs`: `a_new_or_changed_row_outside_the_handles_contexts_is_refused_and_nothing_is_written`, `a_narrowed_handle_is_refused_every_declaration_and_setting_and_changes_nothing`, `a_handle_made_from_a_narrowed_handle_sees_no_more_than_it`, `a_narrowed_handle_with_no_principal_that_holds_grants_is_refused_an_access_controlled_table`, `a_simple_label_is_checked_when_a_row_is_written_and_hides_no_row`, `a_label_with_a_read_side_hides_rows_and_is_checked_when_a_row_is_written` | +| A search and a count on a narrowed handle count and return only rows it may see | `contextdb/tests/visibility.rs`: `a_search_returns_the_nearest_rows_the_handle_may_see_and_no_hidden_row`; `contextdb/tests/kept_counts_narrowed.rs`: `a_handle_that_may_not_see_every_row_counts_the_rows_it_sees_by_reading_them`; `contextdb/tests/kept_counts.rs`: `a_kept_count_is_the_number_of_rows_after_every_kind_of_write_and_reads_no_row` | +| `ALTER TABLE` leaves every other column's values where they were, an added column reads its default from older rows, a dropped column's bytes leave the store, a refused change names what depends and changes nothing, a table another table references is not dropped | `contextdb/tests/alter.rs`: `dropping_a_column_never_shifts_what_the_other_columns_hold`, `an_added_column_reads_its_default_from_a_row_written_before_it`, `a_dropped_columns_bytes_leave_the_store`, `each_dependent_refusal_names_what_depends_and_changes_nothing`, `an_alter_or_a_drop_in_a_transaction_takes_hold_when_it_commits`, `an_altered_table_is_printed_as_what_it_now_declares` | +| A declaration is printed back as text that declares the same table again; a refused declaration says why and declares nothing | `contextdb/tests/declarations.rs`: `every_clause_a_table_may_declare_is_printed_back_and_declares_the_same_table_again`, `a_refused_declaration_says_why_and_declares_nothing`, `a_rank_policy_finds_the_column_it_joins_by_one_rule_or_is_refused`; `contextdb/tests/table_meta.rs`: `table_meta_gives_each_tables_rules` | +| A key too long to store is refused by name, and a second `CREATE TABLE` of a name changes nothing | `contextdb/tests/long_keys.rs`: `a_key_of_2704_bytes_is_stored_and_one_of_2705_is_refused_by_name`; `contextdb/tests/refusals.rs`: `a_second_create_table_names_the_table_and_changes_nothing`, `a_unique_refusal_names_the_table_and_every_column_of_the_rule` | ## Where a change lives @@ -120,6 +133,10 @@ Every path in this guide is written from `v2/`. | A new SQL statement or clause | `contextdb/src/sql/` (`grammar.pest`, `parser/`, the `Statement` type in `ast.rs`, `plan.rs`, `planner.rs`; take its first words out of the grammar's list of what a later version reads), then the part that runs it: `schema`, `write` or `query` | a script under `contextdb/tests/sql/` | | A new vector clause | `contextdb/src/sql/` reads it, as any clause; `contextdb/src/schema/` checks, stores and prints back one that is declared with the column; `contextdb/src/query/search.rs` settles what a search is asked (mode, vector, partitions, filter); `contextdb/src/vector/` only when blocks, partitions or the search itself change | `contextdb/tests/vectors_statements.rs` for what is read and refused, `contextdb/tests/vectors.rs` for the rows a search returns | | When or how a partition is grouped (the size at which a column is grouped, the share of change that groups it again, the steps or the numbers a build measures) | `contextdb/src/vector/build.rs` (`due`, `usable`, the steps), `contextdb/src/vector/pack.rs` (packing and measuring), `contextdb/src/vector/train.rs` (centres); the numbers in force from `VectorColumn` in `contextdb/src/schema/`. A new record for a grouping is a `VectorRecord` kind in `contextdb/src/store/`, which keeps it and never reads it. A change that moves what a build measures changes how a search uses it in `contextdb/src/vector/search.rs`, in the same change | `contextdb/tests/vectors_upkeep.rs` for what a build does and when, `contextdb/tests/vectors_index.rs` for what a search returns and does, `contextdb/tests/crash/main.rs` for a kill inside a build; the cycle counts and sizes the tests record change only with the design they state | +| A new table option or column constraint (a declaration) | `contextdb/src/sql/` reads it; `contextdb/src/schema/` checks it (`check.rs`), works it into the table's `Rules` or `meta` once (`rules.rs`) and prints it back (`render.rs`); an option nothing acts on yet stays in `meta` | `contextdb/tests/declarations.rs` for the round trip and the refusals, a script under `contextdb/tests/sql/` | +| A write rule (what an INSERT, UPDATE or DELETE must obey) | `contextdb/src/write/`; a rule only the whole transaction answers is checked in `write/commit.rs`, at commit; a rule about a key, a `UNIQUE` value or a cycle is checked in both places (`write/mod.rs` and `write/commit.rs`, the second for a transaction another commit overtook), so a new rule of that kind is added in both; the refusal is a kind of `Error` | `contextdb/tests/rules.rs` (or `propagation.rs`), and `contextdb/tests/transactions.rs` when the rule is judged at commit | +| A visibility rule (who may see or write a row) | `Visible` in `contextdb/src/query/visible.rs`, the one filter, asked by `query` and `write`; the handle's narrowing is made in `contextdb/src/database/`; a rule answered anywhere else is a second path | `contextdb/tests/visibility.rs`, and a test of each refusal's sentence carrying no value of a hidden row | +| A change to `ALTER TABLE` | `contextdb/src/schema/alter.rs`; what dangling it would leave is refused there, by name | `contextdb/tests/alter.rs`, `contextdb/tests/sql/alter_table.sql` | | A new refusal (error kind) | `Error` in `contextdb/src/types/mod.rs`; how the program prints it in `contextdb-cli/src/output.rs` | `contextdb/tests/refusals.rs` for its fields and that nothing changed; `contextdb/tests/sql/refusals.sql` for its name | | What a statement returns | `contextdb/src/query/` for a SELECT, `contextdb/src/write/` for a row count | the expected output of the scripts under `contextdb/tests/sql/`, rewritten and the diff read | | How a store is laid out on disk | `contextdb/src/store/` (keys, records, the data file, the folder's files) | `contextdb/tests/values.rs` and `contextdb/tests/store.rs` for reopen, `contextdb/tests/crash/` for a kill | @@ -135,6 +152,7 @@ Every path in this guide is written from `v2/`. - A test asserts states, values, error kinds and counts, never a duration, and waits only on something a process announces. - Its data is fixed, and each test has its own store folder (`support::folder` in `contextdb/tests/support/mod.rs`). - Nothing is retried (`.config/nextest.toml` sets `retries = 0`) and no test is ignored. +- The check holds every test but slow work. Slow work is listed in the `nightly` profile of `.config/nextest.toml` and runs once a night in the check's scheduled run. A test that nears its limit on a hosted runner does less work or, when the size is the promise, moves there. Two lint settings hold this up: diff --git a/v2/contextdb-cli/src/output.rs b/v2/contextdb-cli/src/output.rs index 31b202f3..8635c326 100644 --- a/v2/contextdb-cli/src/output.rs +++ b/v2/contextdb-cli/src/output.rs @@ -1,5 +1,8 @@ //! Everything the shell prints: results on the output, refusals on the error stream, each as -//! text for a person or, under `--json`, as one JSON document a line. +//! text for a person or, under `--json`, as one JSON document a line. A result that cannot be +//! written is handed back to the shell as the failure it is. + +use std::io::{Result, Write, stdout}; use contextdb::serde_json::{Map, Value as Json, json}; use contextdb::{Error, QueryResult, Value}; @@ -12,9 +15,9 @@ pub struct Output { impl Output { /// The result of one statement: the rows of a SELECT, or how many rows a write changed. - pub fn result(self, result: &QueryResult) { + pub fn result(self, result: &QueryResult) -> Result<()> { if result.columns().is_empty() { - self.rows_affected(result.rows_affected()); + self.rows_affected(result.rows_affected()) } else if self.json { // A row is written key by key so that its keys come in the order of `columns`. let mut rows = String::new(); @@ -31,81 +34,81 @@ impl Output { } rows.push('}'); } - println!( + writeln!( + stdout(), "{{\"result\":{{\"columns\":{},\"rows\":[{rows}]}}}}", json!(result.columns()) - ); + ) } else { - print!("{}", table(result.columns(), result.rows())); + write!(stdout(), "{}", table(result.columns(), result.rows())) } } /// The rows of an inspection statement: one `name: value` to a line, with an empty line /// between rows. - pub fn fields(self, result: &QueryResult) { + pub fn fields(self, result: &QueryResult) -> Result<()> { if self.json { return self.result(result); } for (at, row) in result.rows().iter().enumerate() { if at > 0 { - println!(); + writeln!(stdout())?; } for (name, value) in result.columns().iter().zip(row) { - println!("{name}: {}", text(value)); + writeln!(stdout(), "{name}: {}", text(value))?; } } + Ok(()) } /// What `.explain` answers: its lines. - pub fn explain(self, lines: &str) { + pub fn explain(self, lines: &str) -> Result<()> { if self.json { let lines: Vec<&str> = lines.lines().collect(); - println!("{}", json!({ "explain": { "lines": lines } })); + writeln!(stdout(), "{}", json!({ "explain": { "lines": lines } })) } else { - println!("{lines}"); + writeln!(stdout(), "{lines}") } } /// A statement that returns no rows succeeded and changed this many. - pub fn rows_affected(self, rows: u64) { + pub fn rows_affected(self, rows: u64) -> Result<()> { if self.json { - println!("{}", json!({ "rows_affected": rows })); + writeln!(stdout(), "{}", json!({ "rows_affected": rows })) } else { - println!("ok (rows_affected={rows})"); + writeln!(stdout(), "ok (rows_affected={rows})") } } /// What `.tables` answers: the table names, one a line. - pub fn tables(self, names: &[String]) { + pub fn tables(self, names: &[String]) -> Result<()> { if self.json { - println!("{}", json!({ "tables": { "items": names } })); + writeln!(stdout(), "{}", json!({ "tables": { "items": names } })) } else { - for name in names { - println!("{name}"); - } + names + .iter() + .try_for_each(|name| writeln!(stdout(), "{name}")) } } /// What `.schema` answers: the table's declaration as SQL. - pub fn schema(self, table: &str, declaration: &str) { + pub fn schema(self, table: &str, declaration: &str) -> Result<()> { if self.json { - println!( - "{}", - json!({ "schema": { "table": table, "ddl": declaration } }) - ); + let schema = json!({ "schema": { "table": table, "ddl": declaration } }); + writeln!(stdout(), "{schema}") } else { - println!("{}", declaration.trim_end()); + writeln!(stdout(), "{}", declaration.trim_end()) } } /// What `.help` answers. - pub fn help(self, lines: &[&str]) { + pub fn help(self, lines: &[&str]) -> Result<()> { if self.json { - println!("{}", json!({ "help": lines })); + writeln!(stdout(), "{}", json!({ "help": lines })) } else { - for line in lines { - println!("{line}"); - } + lines + .iter() + .try_for_each(|line| writeln!(stdout(), "{line}")) } } @@ -157,16 +160,24 @@ impl Output { self.refused(class, &message, Some(detail)); } + /// A warning: the statement still answers and the run goes on, as in Postgres. + pub fn warning(self, message: &str) { + eprintln!("warning: {message}"); + } + /// A refusal of the shell's own: `class` is `usage` for a command line or a shell command /// it cannot act on, `sql` for a statement. pub fn refusal(self, class: &str, message: &str) { self.refused(class, message, None); } - /// A line that starts like a shell command and is not one. For a person it is a plain line - /// with no `Error:` before it. - pub fn unknown_command(self, line: &str) { - let message = format!("Unknown command: {line}. Type \\? for help."); + /// A line that starts like a shell command and is not one, or a command used wrongly, with + /// its `usage`. For a person it is a plain line with no `Error:` before it. + pub fn unknown_command(self, line: &str, usage: Option<&str>) { + let message = match usage { + Some(usage) => format!("Usage: {usage}"), + None => format!("Unknown command: {line}. Type \\? for help."), + }; if self.json { self.refused("usage", &message, None); } else { diff --git a/v2/contextdb-cli/src/shell.rs b/v2/contextdb-cli/src/shell.rs index ca7dca57..46c7f384 100644 --- a/v2/contextdb-cli/src/shell.rs +++ b/v2/contextdb-cli/src/shell.rs @@ -2,7 +2,7 @@ //! prints what it gave. use std::collections::HashMap; -use std::io::{BufRead, IsTerminal}; +use std::io::{self, BufRead, IsTerminal}; use std::ops::ControlFlow; use contextdb::{Database, Error, QueryResult, ReadSession, Result, Transaction}; @@ -57,15 +57,24 @@ pub fn run(store: &Store, output: Output) -> bool { shell.at_a_terminal(); return true; } - for line in std::io::stdin().lock().lines() { - let Ok(line) = line else { break }; + for (at, line) in std::io::stdin().lock().lines().enumerate() { + // What follows a line that cannot be read may rest on it: nothing after it runs, and + // a transaction left open is not committed. + let Ok(line) = line else { + let message = format!( + "line {} of the input is not text, so the run ends there", + at + 1 + ); + shell.output.refusal("io", &message); + return false; + }; if shell.line(&line).is_break() { return !shell.refused; } } // Piped input ends its last statement without a `;`. let rest = std::mem::take(&mut shell.pending); - shell.statement(&rest); + let _ = shell.statement(&rest); !shell.refused } @@ -127,7 +136,9 @@ impl<'a> Shell<'a> { match scan(&self.pending) { Scan::Ends(end) => { let statement: String = self.pending.drain(..end).collect(); - self.statement(&statement); + if self.statement(&statement).is_break() { + return ControlFlow::Break(()); + } } Scan::Open => break, Scan::Blank => { @@ -162,30 +173,28 @@ impl<'a> Shell<'a> { } .map(|lines| self.output.explain(&lines)) } - (Some(".help" | "\\?"), None, _) => { - self.output.help(&HELP); - Ok(()) - } + (Some(".help" | "\\?"), None, _) => Ok(self.output.help(&HELP)), (Some(".quit" | ".exit" | "\\q"), None, _) => return ControlFlow::Break(()), _ => { self.refused = true; - self.output.unknown_command(line); - Ok(()) + let first = line.split_whitespace().next().unwrap_or_default(); + let usage = HELP + .into_iter() + .find(|h| h.split_whitespace().any(|w| w == first)); + self.output.unknown_command(line, usage); + Ok(Ok(())) } }; - if let Err(error) = answered { - self.refused = true; - self.output.error(&error); - } - ControlFlow::Continue(()) + self.answered(answered) } - /// Runs one statement and prints its result or its refusal. - fn statement(&mut self, text: &str) { + /// Runs one statement and prints its result or its refusal. Breaks when the result + /// cannot be written. + fn statement(&mut self, text: &str) -> ControlFlow<()> { let sql = text.trim(); let sql = sql.strip_suffix(';').unwrap_or(sql).trim_end(); if sql.is_empty() { - return; + return ControlFlow::Continue(()); } let ran = if sql.eq_ignore_ascii_case("BEGIN") { self.begin() @@ -201,18 +210,33 @@ impl<'a> Shell<'a> { let mut words = sql.split_whitespace().map(str::to_ascii_uppercase); let inspection = words.next().as_deref() == Some("SHOW") && words.next().is_some_and(|word| word.starts_with("VECTOR_")); - self.execute(sql).map(|result| { - if inspection { - self.output.fields(&result); - } else { - self.output.result(&result); - } + self.execute(sql).map(|result| match inspection { + true => self.output.fields(&result), + false => self.output.result(&result), }) }; - if let Err(error) = ran { - self.refused = true; - self.output.error(&error); + self.answered(ran) + } + + /// Prints a refusal, or ends the session when the answer could not be written: as `.quit` + /// does when nobody reads the output any more, and as a refusal for any other failure. + fn answered(&mut self, answered: Result>) -> ControlFlow<()> { + match answered { + Ok(Ok(())) => {} + Err(error) => { + self.refused = true; + self.output.error(&error); + } + Ok(Err(error)) => { + if error.kind() != io::ErrorKind::BrokenPipe { + self.refused = true; + let message = format!("the output cannot be written: {error}"); + self.output.refusal("io", &message); + } + return ControlFlow::Break(()); + } } + ControlFlow::Continue(()) } fn execute(&self, sql: &str) -> Result { @@ -224,28 +248,30 @@ impl<'a> Shell<'a> { } } - /// `BEGIN`. With a transaction open already it changes nothing and answers the same. - fn begin(&mut self) -> Result<()> { + /// `BEGIN`. With a transaction open already it changes nothing, warns, and answers the same. + fn begin(&mut self) -> Result> { let Store::Write(db) = self.store else { return Err(Error::ReadOnlySession); }; - if self.transaction.is_none() { - self.transaction = Some(db.transaction()?); + match self.transaction { + None => self.transaction = Some(db.transaction()?), + Some(_) => self + .output + .warning("there is already a transaction in progress"), } - self.output.rows_affected(0); - Ok(()) + Ok(self.output.rows_affected(0)) } - /// `COMMIT` or `ROLLBACK`. With no transaction open it changes nothing and answers the - /// same. - fn end(&mut self, how: impl FnOnce(Transaction<'a>) -> Result<()>) -> Result<()> { + /// `COMMIT` or `ROLLBACK`. With no transaction open it changes nothing, warns, and answers + /// the same. + fn end(&mut self, how: impl FnOnce(Transaction<'a>) -> Result<()>) -> Result> { if let Store::Read(_) = self.store { return Err(Error::ReadOnlySession); } - if let Some(transaction) = self.transaction.take() { - how(transaction)?; + match self.transaction.take() { + Some(transaction) => how(transaction)?, + None => self.output.warning("there is no transaction in progress"), } - self.output.rows_affected(0); - Ok(()) + Ok(self.output.rows_affected(0)) } } diff --git a/v2/contextdb-cli/tests/shell.rs b/v2/contextdb-cli/tests/shell.rs index 4829643b..36999879 100644 --- a/v2/contextdb-cli/tests/shell.rs +++ b/v2/contextdb-cli/tests/shell.rs @@ -41,7 +41,12 @@ fn folder(test: &str) -> PathBuf { /// /// The input is written before anything is read back. Every input here is far smaller than a /// pipe holds, so the write cannot wait on the program. -fn shell(dir: &Path, args: &[&str], input: &str) -> Run { +fn shell(dir: &Path, args: &[&str], input: impl AsRef<[u8]>) -> Run { + shell_to(dir, args, input, false) +} + +/// [`shell`], with the program's output closed before it writes when `closed` is set. +fn shell_to(dir: &Path, args: &[&str], input: impl AsRef<[u8]>, closed: bool) -> Run { let mut child = Command::new(env!("CARGO_BIN_EXE_contextdb")) .args(args) .current_dir(dir) @@ -50,8 +55,11 @@ fn shell(dir: &Path, args: &[&str], input: &str) -> Run { .stderr(Stdio::piped()) .spawn() .unwrap(); + if closed { + drop(child.stdout.take()); + } // A program that refuses its command line exits without reading its input. - match child.stdin.take().unwrap().write_all(input.as_bytes()) { + match child.stdin.take().unwrap().write_all(input.as_ref()) { Err(error) if error.kind() != ErrorKind::BrokenPipe => panic!("{error}"), _ => {} } @@ -179,18 +187,28 @@ fn reading_a_missing_store_is_refused_and_creates_nothing() { "a refused read creates nothing" ); - // The same refusal for a program that reads it: one document on the error stream. - let run = shell(&dir, &[arg(&store), "--json"], SELECT_NOTES); - assert_eq!(run.code, Some(1), "{}", run.stderr); - assert_eq!(run.stdout, ""); - let refusals = documents(&run.stderr); - assert_eq!(refusals.len(), 1, "{}", run.stderr); - assert_eq!(refusals[0]["error"]["detail"]["kind"], "store_not_found"); - assert_eq!( - listing(&dir), - Vec::::new(), - "a refused read creates nothing" - ); + // The same refusal for a program that reads it: one document on the error stream, class io. + // A path whose `data` is a folder cannot be read as a store either: kind store_io. + let broken = dir.join("broken"); + std::fs::create_dir_all(broken.join("data")).unwrap(); + for (path, kind) in [(&store, "store_not_found"), (&broken, "store_io")] { + let run = shell(&dir, &[arg(path), "--json"], SELECT_NOTES); + assert_eq!(run.code, Some(1), "{}", run.stderr); + assert_eq!(run.stdout, ""); + let refusals = documents(&run.stderr); + assert_eq!(refusals.len(), 1, "{}", run.stderr); + assert_eq!( + refusals[0]["error"]["detail"]["kind"], kind, + "{}", + run.stderr + ); + assert_eq!( + refusals[0]["error"]["class"], "io", + "a store that cannot be read is a failure of the storage, not of the statement: {}", + run.stderr + ); + } + assert_eq!(listing(&dir), ["broken"], "a refused read creates nothing"); } #[test] @@ -202,7 +220,7 @@ fn write_creates_a_missing_store() { let run = shell( &dir, &[arg(&store), "--write"], - &format!("{CREATE_NOTES}{insert}"), + format!("{CREATE_NOTES}{insert}"), ); assert_eq!(run.code, Some(0), "{}", run.stderr); assert_eq!(run.errors(), Vec::<&str>::new()); @@ -226,7 +244,7 @@ fn write_creates_a_missing_store() { let run = shell( &dir, &[arg(&store), "--write"], - &format!("{insert}{SELECT_NOTES}"), + format!("{insert}{SELECT_NOTES}"), ); assert_eq!(run.code, Some(0), "{}", run.stderr); assert!(run.stdout.contains(TWO_NOTES), "{}", run.stdout); @@ -404,6 +422,61 @@ SELECT COUNT(*) AS n FROM notes; let run = shell(&dir, &[arg(&store)], SELECT_NOTES); assert_eq!(run.code, Some(0), "{}", run.stderr); assert_eq!(run.stdout, TWO_NOTES); + + // Output that nobody reads any more ends the session quietly: no crash report, and the + // exit status 0, as no statement was refused. + let run = shell_to( + &dir, + &[arg(&store)], + "SELECT id FROM notes ORDER BY id;\nSELECT id FROM notes ORDER BY id;\n", + true, + ); + assert_eq!( + run.code, + Some(0), + "a closed output must end the run quietly with status 0: {}", + run.stderr + ); + + // A line that is not valid text ENDS the run there: the refusal names the line, the status + // is 1, and no statement after it runs, inside a transaction or not; one left open by the + // end is not committed. + for (script, line, kept) in [ + ( + "INSERT INTO notes (id, body) VALUES (10, 'before');\n", + 2, + 3, + ), + ( + "BEGIN;\nINSERT INTO notes (id, body) VALUES (12, 'inside');\n", + 3, + 3, + ), + ] { + let mut input = script.as_bytes().to_vec(); + input.extend_from_slice(b"\xff\xfe\n"); + input.extend_from_slice(b"INSERT INTO notes (id, body) VALUES (11, 'after');\nCOMMIT;\n"); + let run = shell(&dir, &[arg(&store), "--write"], input); + assert_eq!( + run.code, + Some(1), + "an undecodable line must fail the run: {}", + run.stderr + ); + assert!( + run.errors() + .iter() + .any(|error| error.contains(&format!("line {line}"))), + "the refusal must name the line that cannot be decoded, line {line}: {}", + run.stderr + ); + let run = shell(&dir, &[arg(&store)], "SELECT COUNT(*) AS n FROM notes;\n"); + assert!( + run.stdout.contains(&format!("| {kept} |")), + "only the statements before the undecodable line stand ({kept} notes): {}", + run.stdout + ); + } } #[test] @@ -471,7 +544,7 @@ fn a_read_shell_refuses_a_write_by_name() { let made = shell( &dir, &[arg(&store), "--write"], - &format!("{CREATE_NOTES}{insert}"), + format!("{CREATE_NOTES}{insert}"), ); assert_eq!(made.code, Some(0), "{}", made.stderr); @@ -484,7 +557,7 @@ BEGIN; "; // Each of the five is refused, telling the person what to add, and the read after them // answers with the row as it was. - let run = shell(&dir, &[arg(&store)], &format!("{writes}{SELECT_NOTES}")); + let run = shell(&dir, &[arg(&store)], format!("{writes}{SELECT_NOTES}")); assert_eq!(run.code, Some(1)); assert_eq!(run.errors().len(), 5, "{}", run.stderr); assert!( @@ -514,7 +587,7 @@ BEGIN; let run = shell( &dir, &[arg(&store), "--json"], - &format!("{SELECT_NOTES}.tables\n"), + format!("{SELECT_NOTES}.tables\n"), ); assert_eq!(run.code, Some(0), "{}", run.stderr); let results = documents(&run.stdout); @@ -611,6 +684,30 @@ SELECT id, body FROM notes ORDER BY id; let run = shell(&dir, &[arg(&store)], SELECT_NOTES); assert_eq!(run.code, Some(0), "{}", run.stderr); assert_eq!(run.stdout, TWO_NOTES); + + // As in Postgres, BEGIN inside a transaction and COMMIT or ROLLBACK outside one each warn on + // the error stream and the run still ends well, so a script that lost its BEGIN is told. + let run = shell( + &dir, + &[arg(&store), "--write"], + "BEGIN;\nBEGIN;\nCOMMIT;\nCOMMIT;\nROLLBACK;\n", + ); + assert_eq!(run.code, Some(0), "{}", run.stderr); + let warnings: Vec<&str> = run + .stderr + .lines() + .filter(|line| line.starts_with("warning:")) + .collect(); + assert_eq!( + warnings, + [ + "warning: there is already a transaction in progress", + "warning: there is no transaction in progress", + "warning: there is no transaction in progress", + ], + "{}", + run.stderr + ); } #[test] @@ -631,11 +728,7 @@ CREATE TABLE notes ( ); "; let authors = "CREATE TABLE authors (id UUID PRIMARY KEY, name TEXT);\n"; - let made = shell( - &dir, - &[arg(&store), "--write"], - &format!("{notes}{authors}"), - ); + let made = shell(&dir, &[arg(&store), "--write"], format!("{notes}{authors}")); assert_eq!(made.code, Some(0), "{}", made.stderr); // `.tables` lists the names in order, one a line. @@ -664,6 +757,22 @@ CREATE TABLE notes ( let made = shell(&dir, &[arg(©), "--write"], &schema.stdout); assert_eq!(made.code, Some(0), "{}", made.stderr); assert_eq!(shell(&dir, &[arg(©)], ".schema notes\n").stdout, notes); + + // A command used wrongly prints how it is used, never that it is unknown. + for (input, command) in [(".schema\n", ".schema"), (".tables extra\n", ".tables")] { + let run = shell(&dir, &[arg(&store)], input); + let usage = run.stderr.lines().find(|line| line.starts_with("Usage:")); + assert!( + usage.is_some_and(|line| line.contains(command)), + "{command} used wrongly must print a Usage: line that names it:\n{}", + run.stderr + ); + assert!( + !run.stderr.contains("Unknown command"), + "{command} is a command; its wrong use was called unknown:\n{}", + run.stderr + ); + } } #[test] @@ -812,7 +921,7 @@ VALUES (1, 'a', [1.0, 0.0, 0.0]), (2, 'a', [0.0, 1.0, 0.0]), (3, 'b', [-1.0, 0.0 let run = shell( &dir, &[arg(&store), "--write"], - &format!("{declare}{search}{rows}"), + format!("{declare}{search}{rows}"), ); assert_eq!(run.code, Some(0), "{}", run.stderr); assert_eq!(run.errors(), Vec::<&str>::new()); @@ -839,7 +948,7 @@ VALUES (1, 'a', [1.0, 0.0, 0.0]), (2, 'a', [0.0, 1.0, 0.0]), (3, 'b', [-1.0, 0.0 assert_eq!(printed[1][5], ["5", "[0.1, 0.2, 0.3]"], "{}", run.stdout); // For a program the distance is a number, a vector an array of numbers and no vector null. - let run = shell(&dir, &[arg(&store), "--json"], &format!("{search}{rows}")); + let run = shell(&dir, &[arg(&store), "--json"], format!("{search}{rows}")); assert_eq!(run.code, Some(0), "{}", run.stderr); let results = documents(&run.stdout); assert_eq!(results.len(), 2, "{}", run.stdout); @@ -935,6 +1044,32 @@ fn an_indexed_search_runs_through_the_program() { "{mode:?}" ); } + + // Two vectors outside any grouping under AUTO_INDEX_AT 1: the index is behind its writes, + // and INDEXED is refused by name rather than answered as an exact scan. + let behind = dir.join("behind"); + let made = shell( + &dir, + &[arg(&behind), "--write"], + "CREATE TABLE behind (id INTEGER PRIMARY KEY, v VECTOR(3) AUTO_INDEX_AT 1);\n\ + INSERT INTO behind (id, v) VALUES (1, [1.0, 0.0, 0.0]), (2, [0.0, 1.0, 0.0]);\n", + ); + assert_eq!(made.code, Some(0), "{}", made.stderr); + let run = shell( + &dir, + &[arg(&behind), "--json"], + search("INDEXED") + .replace("passages", "behind") + .replace("embedding", "v"), + ); + assert_eq!(run.code, Some(1), "{}", run.stderr); + let refusals = documents(&run.stderr); + assert_eq!(refusals.len(), 1, "{}", run.stderr); + assert_eq!( + refusals[0]["error"]["detail"]["kind"], "vector_indexed_route_unavailable", + "{}", + run.stderr + ); } /// What the two tests below search and inspect: a column with a partition for each `kind`, two @@ -1008,7 +1143,7 @@ fn vector_inspection_prints_one_field_to_a_line() { let dir = folder("vector-inspection-prints"); let store = dir.join("store"); let more = "CREATE TABLE other (id INTEGER PRIMARY KEY, w VECTOR(2));\n"; - let made = shell(&dir, &[arg(&store), "--write"], &format!("{KINDS}{more}")); + let made = shell(&dir, &[arg(&store), "--write"], format!("{KINDS}{more}")); assert_eq!(made.code, Some(0), "{}", made.stderr); // Each statement with the fields it prints and, for each row, the values of three of them. @@ -1124,7 +1259,7 @@ fn explain_prints_how_each_partition_is_searched() { // Any other statement is refused in one line, and the session goes on to the next. let other = ".explain SELECT id FROM notes ORDER BY id\n"; let next = "SELECT id FROM notes WHERE id = 5;\n"; - let run = shell(&dir, &[arg(&store)], &format!("{other}{next}")); + let run = shell(&dir, &[arg(&store)], format!("{other}{next}")); assert_eq!(run.code, Some(1), "{}", run.stderr); let refusals: Vec<&str> = run.stderr.lines().collect(); assert_eq!(refusals.len(), 1, "{}", run.stderr); diff --git a/v2/contextdb/README.md b/v2/contextdb/README.md index b209699d..bcad7987 100644 --- a/v2/contextdb/README.md +++ b/v2/contextdb/README.md @@ -8,18 +8,27 @@ commit together, and one `SELECT` filters rows and orders them by distance. This is the first version of contextdb 2. What it has: -- tables with typed columns, a primary key, `NOT NULL`, `UNIQUE` and `DEFAULT`, and indexes; -- `INSERT`, `UPDATE`, `DELETE` and `SELECT` over one table; +- tables with typed columns, a primary key of one column, of several or of none, `NOT NULL`, + `UNIQUE`, `DEFAULT`, references between tables, and indexes; +- rules a table declares and every write obeys: rows that are never changed, the states a column + may move through, edges that may not close a cycle, rows that follow a change of another row; +- who may see and write a row: Contexts, scope labels and principals; +- `ALTER TABLE`, `DROP TABLE` and `DROP INDEX`; +- `INSERT` (also `ON CONFLICT ... DO UPDATE`), `UPDATE`, `DELETE` and `SELECT` over one table; - vector search: the rows nearest a vector by cosine distance, each with its distance, over vectors that a column may keep in partitions; exact on request, and through an index that `run_maintenance_cycle` builds and brings up to date once a column holds many vectors; -- transactions; +- transactions, whose conflicts are decided when they commit; - one process writing a store while any number of other processes read it; - a memory limit that holds. -What it does not have yet: relationships between rows, joins, grouping, and sync between -machines. A statement that uses one of them is -refused and changes nothing. +What it does not have yet: joins, grouping, graph patterns, ranked search (`USE RANK`) and sync +between machines; the statements that use them are listed under Statements, and each is refused +and changes nothing. Some clauses are accepted, kept and printed back by `.schema`, and nothing +acts on them yet: `RETAIN` (no row expires), `HISTORY`, `SYNC`, `SYNC CONFLICT`, +`DELIVERY MANIFEST`, `EDGE DISCARD` and `RANK_POLICY` (which only keeps `ALTER` from dropping or +renaming a column it reads); so is an index on a JSON field, `CREATE INDEX name ON table ((column -> 'key'))`: +writes to its table are accepted and pass over it, and no query reads it. `table_meta` reports `RETAIN`, `HISTORY`, `SYNC` and `SYNC CONFLICT`. It is not on crates.io yet. Clone and use it from that checkout: @@ -65,28 +74,65 @@ fn main() -> contextdb::Result<()> { } ``` -It prints `buy oat milk: 0.006` and then `call the plumber: 1.000`. After writing vectors, a host -calls `db.run_maintenance_cycle()?` again and again, as the paragraph on upkeep under Vector search says, because -this version does not do it by itself; a search before that still finds every vector, and reads -more to do it. This page is also the crate's -documentation, and the documentation tests compile this program as it is written here. Rust 1.89 or newer; Linux and macOS. Licence: -Apache-2.0. It is developed in the open at by the context-graph-ai organisation. +It prints `buy oat milk: 0.006` and then `call the plumber: 1.000`. Call `db.close()?` when you +are done, as the later programs do: it lets the store go at once, and every later call on the +handle returns `DatabaseClosed`. After writing vectors, a host calls `db.run_maintenance_cycle()?` +again and again (see Upkeep under Vector search), because this version does not do it by itself; +a search before that still finds every vector, and reads more to do it. This page is also the +crate's documentation, and the documentation tests compile this program as it is written here. +Rust 1.89 or newer; Linux and macOS. Licence: Apache-2.0. It is developed in the open at + by the context-graph-ai organisation. ## Statements ```sql CREATE TABLE [IF NOT EXISTS] name ( - column type [PRIMARY KEY] [NOT NULL] [UNIQUE] [DEFAULT value], + column type [PRIMARY KEY] [NOT NULL] [UNIQUE] [DEFAULT value] [IMMUTABLE] + [REFERENCES table (column) + [ON STATE state PROPAGATE SET state [MAX DEPTH n] [ABORT ON FAILURE]]] + [CONTEXT_ID] + [SCOPE_LABEL ('label', ...) | SCOPE_LABEL_READ ('label', ...) WRITE ('label', ...)] + [ACL REFERENCES table (column)] + [RANK_POLICY (JOIN table ON column, FORMULA 'formula', SORT_KEY key)], column VECTOR(n) [WITH (quantization = 'F32' | 'SQ8')] -- F32 when left out [PARTITION_KEY (column, ...) [MAX_PARTITIONS m]] -- m is 256 when left out [SEARCH_MODE AUTO | EXACT | INDEXED] [AUTO_INDEX_AT a] -- a is 100000 when left out [CONSOLIDATION NONE | CONSOLIDATION (CHANGE_PERCENT = c, TOMBSTONE_PERCENT = t)], ..., - [UNIQUE (column, ...)] -); + [PRIMARY KEY (column, ...)], + [UNIQUE (column, ...)], + [FOREIGN KEY (column, ...) REFERENCES table (column, ...)] +) + [IMMUTABLE] + [STATE MACHINE (column: state -> [state, ...], state -> state, ...)] + [DAG ('edge_type', ...)] + [PROPAGATE ON EDGE edge_type INCOMING | OUTGOING | BOTH STATE state SET state + [MAX DEPTH n] [ABORT ON FAILURE]] + [PROPAGATE ON STATE state EXCLUDE VECTOR] + [RETAIN n SECONDS | MINUTES | HOURS | DAYS] + [HISTORY ALL | CURRENT ONLY] + [SYNC OFF | PUSH ONLY | PULL ONLY | TWO WAY] + [SYNC CONFLICT KEEP FIRST | KEEP LATEST] + [MAINTAIN COUNT (column, ...) [, (column, ...)] | ALL PREFIXES | NONE] + [DELIVERY MANIFEST OVER table, ...] + [EDGE DISCARD NEVER | AFTER OUTCOME | ALWAYS]; CREATE INDEX name ON table (column [ASC | DESC], ...); -INSERT INTO table (column, ...) VALUES (value, ...), (value, ...); +CREATE INDEX name ON table ((column -> 'key')); +ALTER TABLE table ADD [COLUMN] column type ...; +ALTER TABLE table DROP [COLUMN] column [RESTRICT | CASCADE]; +ALTER TABLE table RENAME COLUMN column TO column; +ALTER TABLE table SET RETAIN ... | SET HISTORY ... | SET SYNC ... | SET SYNC CONFLICT ... + | SET MAINTAIN COUNT ...; +ALTER TABLE table DROP RETAIN; +ALTER TABLE table ALTER COLUMN column SET MAX_PARTITIONS m | SET SEARCH_MODE mode + | SET AUTO_INDEX_AT a | SET AUTO_INDEX_AT DEFAULT + | SET CONSOLIDATION NONE | SET CONSOLIDATION (CHANGE_PERCENT = c, TOMBSTONE_PERCENT = t) + | SET CONSOLIDATION DEFAULT; +DROP TABLE [IF EXISTS] table; +DROP INDEX [IF EXISTS] name ON table; +INSERT INTO table (column, ...) VALUES (value, ...), (value, ...) + [ON CONFLICT (column, ...) DO UPDATE SET column = value, ...]; UPDATE table SET column = value, ... [WHERE condition]; DELETE FROM table [WHERE condition]; SELECT column, ... | * FROM table [WHERE condition] [ORDER BY column [ASC | DESC], ...] [LIMIT n]; @@ -97,7 +143,8 @@ SHOW VECTOR_INDEXES; SHOW VECTOR_PARTITIONS [FOR table.column] [LIMIT count [OFFSET start]]; ``` -- Every table has exactly one `PRIMARY KEY` column. A default is a literal or `NOW()`. +- A default is a literal or `NOW()`. A table, column or index name written without quotes is + folded to lower case, as in Postgres; a state or an edge type is a value and is kept as written. - A condition or a value is built from column names, literals, `$name` values, `= != <> < <= > >=`, `AND`, `OR`, `NOT`, `IS NULL`, `IS NOT NULL`, `+ - * /`, parentheses and `NOW()`. A comparison with NULL is not true, as in standard SQL. @@ -107,14 +154,18 @@ SHOW VECTOR_PARTITIONS [FOR table.column] [LIMIT count [OFFSET start]]; - `SELECT` also takes `COUNT(*)`, `COUNT(column)`, `SUM`, `MIN` and `MAX` over the rows that match, and `AS` to name a column of the result. - `ORDER BY` puts NULL last ascending and first descending. -- The second `SELECT` is a vector search; [Vector search](#vector-search) below says what it - returns. The clauses of a `VECTOR` column are described under Vector search (quantization, modes, upkeep), - and the two `SHOW` statements there too, under inspection. +- The second `SELECT` is a vector search, and the clauses of a `VECTOR` column and the two + `SHOW` statements are described under [Vector search](#vector-search). - `--` and `/* */` comments are read past. +- The library has one way to run a transaction, `Database::transaction`; its `execute` refuses + `BEGIN`, `COMMIT` and `ROLLBACK` by name, which are words of the `contextdb` program. -Not in this version, and refused: `JOIN`, `GROUP BY`, `DISTINCT`, `LIKE`, `BETWEEN`, -`OFFSET`, `UNION`, subqueries, table aliases, `ALTER`, `DROP`, and a primary key over several -columns; and the vector clauses `IVF`, `SEARCH (...)` and `WITH (quantization = 'SQ4')`. Most are refused by name; this is the sentence the refusal prints: +Not in this version, and refused: `JOIN`, `GROUP BY`, `HAVING`, `DISTINCT`, `LIKE`, `BETWEEN`, +`OFFSET`, `UNION`, `WITH`, subqueries, `INSERT ... SELECT`, graph patterns, `USE RANK`, +`DROP TRIGGER`, `CREATE PROCEDURE` and `CREATE FUNCTION`, table aliases, `CREATE UNIQUE INDEX` +(write `UNIQUE` on the column or in the table); and the vector clauses `IVF (...)`, `SEARCH (...)` and +`WITH (quantization = 'SQ4')`, which no version reads; the refusal of each says what to write instead. +Most are refused by name; this is the sentence the refusal prints: ```text plan error: LIKE is not supported in this version of the library @@ -130,6 +181,7 @@ plan error: LIKE is not supported in this version of the library | `BOOLEAN`, `BOOL` | `TRUE` or `FALSE` | `Value::Bool` | | `UUID` | a UUID, written as quoted text | `Value::Uuid` | | `TIMESTAMP` | milliseconds since the first moment of 1970, UTC; written as ISO 8601 text (`'2026-01-01T00:00:01Z'`) or taken from `NOW()` | `Value::Timestamp` | +| `TXID` | the number of a transaction (see Transactions); written only as a bound `Value::TxId` | `Value::TxId` | | `JSON` | a JSON value, written as quoted text; the value is kept, not the text's spacing | `Value::Json` | | `VECTOR(n)` | a vector of exactly `n` finite numbers, each kept as an `f32` (a column declared `SQ8` keeps a rounded vector, see the paragraph on quantization under Vector search), written `[0.1, 0.2, 0.3]`; searched with `<=>` | `Value::Vector` | @@ -137,6 +189,460 @@ A value must be of its column's type: text into an `INTEGER` column is refused, into a `NOT NULL` column, on `INSERT` and on `UPDATE` alike. A whole number is accepted into a `REAL` column. +## Declaring a table and its rules + +A table declares what its rows must obey, and every write is checked against it, whoever makes +the write. The shell prints each statement's answer; these statements declare two tables, one +following the other: + +```sql +CREATE TABLE docs ( + id INTEGER PRIMARY KEY, + title TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'live' +) STATE MACHINE (status: live -> [stale, gone], stale -> [live, gone]); + +CREATE TABLE chunks ( + id INTEGER PRIMARY KEY, + doc_id INTEGER REFERENCES docs(id) ON STATE stale PROPAGATE SET stale, + status TEXT NOT NULL DEFAULT 'live' +) STATE MACHINE (status: live -> [stale], stale -> [live]); +``` + +- **Keys.** `PRIMARY KEY` is written on one column or as a table element over several, and + `UNIQUE` the same. A table may declare no primary key: it keeps its rows under a hidden key, + made of the number of the transaction that wrote the row and a count within it, which is not a + column and which no statement can name. A key, a `UNIQUE` rule or an index has a limit on the + columns it may name, and a row key a limit on its length; a declaration or a write past + either is refused (`SchemaInvalid`, `KeyTooLong`). An index entry is never refused for a long + value: it keeps the first 256 bytes of a long text and a digest of the whole, and a read checks the + row, so no answer changes. +- **References.** `REFERENCES parent(column)` on a column, or `FOREIGN KEY (a, b) REFERENCES + parent(x, y)` over several. The parent's columns must be its primary key or a `UNIQUE` rule, in + that order, and of the child's types. A reference with a NULL in it is not checked. + There is no `ON DELETE`, `ON UPDATE` or `CHECK`; what a change in a parent does to its children + is `PROPAGATE`. +- **`IMMUTABLE`** on a table: rows are inserted once and never updated or deleted. On a column: + it is set when the row is inserted and never changed. +- **`STATE MACHINE (column: from -> [to, ...])`**: an `UPDATE` may move the column only along a + listed arrow. A state with no list is a state a row cannot leave. A new row may start in any + state. +- **`DAG ('edge_type', ...)`** is for a table of edges, one with `source_id UUID`, + `target_id UUID` and `edge_type TEXT`: an edge of a listed type that would close a cycle is + refused, by the statement that writes it and again at commit. +- **`PROPAGATE`**: when a row enters a state, the rows that reference it (`ON STATE s PROPAGATE + SET t` on the reference), or the rows at the other end of its edges (`PROPAGATE ON EDGE`), are + moved to another state in the same statement, along their own state machines. A table that + declares `PROPAGATE ON EDGE` also needs `id UUID`. `MAX DEPTH n` bounds how many steps a change is + followed from the row the statement changed (edges for `PROPAGATE ON EDGE`, references for `ON + STATE`), 10 when not written; `ABORT ON FAILURE` refuses the whole statement when a row cannot move, where otherwise that row + stays. +- **`PROPAGATE ON STATE s EXCLUDE VECTOR`** keeps the vector of a row in state `s` out of every + search and every count of `SHOW VECTOR_*`. It is refused on a table with no state machine, and + naming a state the machine does not have. + +Run in the shell, the rules answer as they are broken (`chunks` row 12 names a document that is +not there; `docs` row 1 is moved to `stale`, and its chunks follow): + +```text +contextdb> INSERT INTO docs (id, title) VALUES (1, 'plan'); +ok (rows_affected=1) +contextdb> INSERT INTO chunks (id, doc_id) VALUES (10, 1), (11, 1); +ok (rows_affected=2) +contextdb> INSERT INTO chunks (id, doc_id) VALUES (12, 99); +Error: foreign key violation: chunks (doc_id) = (99) has no row in docs +contextdb> UPDATE docs SET status = 'stale' WHERE id = 1; +ok (rows_affected=1) +contextdb> SELECT id, status FROM chunks ORDER BY id; ++----+--------+ +| id | status | ++----+--------+ +| 10 | stale | +| 11 | stale | ++----+--------+ +(2 rows) +contextdb> UPDATE docs SET status = 'gone' WHERE id = 1; +ok (rows_affected=1) +contextdb> UPDATE docs SET status = 'live' WHERE id = 1; +Error: invalid state transition of docs.status: gone -> live +contextdb> DELETE FROM docs WHERE id = 1; +Error: foreign key violation: docs is still referenced by chunks (doc_id) = (1) +``` + +**If you know Postgres.** A reference is checked when the transaction commits, not when the +statement ends, as if every one were `DEFERRABLE INITIALLY DEFERRED`; in the shell, where a +statement is its own transaction, the two come to the same. A primary key is judged when the +statement ends, as the SQL standard says: an `UPDATE` that swaps two keys or shifts a run of +them succeeds, where Postgres decides by the order it visits the rows unless the key is +declared `DEFERRABLE`; two rows left under one key are still refused, by that key. An index +name that begins `__pk_`, +`__unique_`, `__fk_` or `__graph_` is the engine's, and is refused for an index of yours +(`ReservedIndexName`). + +## Writing rows + +`INSERT`, `UPDATE` and `DELETE` check, before they change anything: the type of each value, `NOT +NULL`, the key, every `UNIQUE` rule, and the rules of the table above. A statement that breaks one +is refused, names the table, the column or rule and the value, and changes nothing. In a transaction it +leaves the transaction usable. + +```text +contextdb> INSERT INTO people (id, name) VALUES (2, 'ada'); +Error: unique constraint violation: people (name) = (ada) +contextdb> INSERT INTO people (id, name, born) VALUES (3, 'bob', 'x'); +Error: column type mismatch: people.born expected INTEGER, got Text +contextdb> INSERT INTO people (id) VALUES (4); +Error: column people.name is NOT NULL +contextdb> INSERT INTO people (id, name, name) VALUES (5, 'c', 'd'); +Error: plan error: name is named twice as a column people is written in +``` + +`INSERT ... ON CONFLICT (columns) DO UPDATE SET ...` takes the columns of the key or of a +`UNIQUE` rule, and updates the row that is in the way; it is subject to every rule an `UPDATE` is +(`IMMUTABLE` and a state machine included). `DO NOTHING`, `EXCLUDED` and `RETURNING` are not +there. + +A refusal that depends only on the declaration (the table is not there, the table is +`IMMUTABLE`) comes before one that depends on the rows. Each refusal is its own kind of +`contextdb::Error`, listed on the type. + +**If you know Postgres.** A table with no primary key keeps its rows under the hidden key, so +the same row written twice is two rows, and a refusal about a row cannot name a key. + +## Transactions + +`db.execute` is a transaction of one statement: when it returns, the change is on disk. +`db.transaction()` begins a longer one. Its statements see what it has written and nobody else +does until `commit` returns; dropping it, or `rollback`, discards it. A statement of it is +refused at once for a type, `NOT NULL`, a key too long, a taken key or `UNIQUE` value, +`IMMUTABLE`, the state machine, a cycle and a partition limit, and the transaction goes on; a +reference, a guarded row and a clash with another commit are refused by `commit`, which +writes nothing and ends the transaction. + +Two open transactions that take the same key, the same `UNIQUE` value, or change the same row do +not wait for each other: the first to commit wins, and the other's `commit` is refused. The +refusal is the kind the clash is: `UniqueViolation` for a key or a `UNIQUE` value, +`ConditionalUpdateConflict` for a row changed meanwhile. This program runs the clashes: + +```rust,no_run +use contextdb::{params, Database, Value}; + +fn main() -> contextdb::Result<()> { + let db = Database::open("./accounts")?; + db.execute( + "CREATE TABLE accounts (id INTEGER PRIMARY KEY, owner TEXT NOT NULL UNIQUE)", + ¶ms! {}, + )?; + + // Two transactions begin together and both take key 1. + let first = db.transaction()?; + let second = db.transaction()?; + first.execute("INSERT INTO accounts VALUES (1, 'ana')", ¶ms! {})?; + second.execute("INSERT INTO accounts VALUES (1, 'bo')", ¶ms! {})?; + first.commit()?; + match second.commit() { + Ok(_) => println!("second committed"), + Err(error) => println!("second: {error}"), + } + + // One that takes a key and gives it up again, while another handle takes it. + let tx = db.transaction()?; + tx.execute("INSERT INTO accounts VALUES (2, 'cy')", ¶ms! {})?; + tx.execute("DELETE FROM accounts WHERE id = 2", ¶ms! {})?; + db.execute("INSERT INTO accounts VALUES (2, 'cy')", ¶ms! {})?; + match tx.commit() { + Ok(_) => println!("third committed"), + Err(error) => println!("third: {error}"), + } + + // A transaction that declares a table writes no rows. + let tx = db.transaction()?; + tx.execute("CREATE TABLE extra (id INTEGER PRIMARY KEY)", ¶ms! {})?; + match tx.execute("INSERT INTO accounts VALUES (3, 'di')", ¶ms! {}) { + Ok(_) => println!("wrote"), + Err(error) => println!("declaring: {error}"), + } + tx.rollback(); + + // A guard: this commit stands only while account 1 is still owned by ana. + db.execute("CREATE TABLE stamped (id INTEGER PRIMARY KEY, at TXID)", ¶ms! {})?; + let tx = db.transaction()?; + let held = tx.guard_row_conditions( + "accounts", + "id", + &Value::Int64(1), + &[("owner".to_owned(), Value::Text("ana".to_owned()))], + )?; + // A TXID column takes the transaction's own number. + tx.execute( + "INSERT INTO stamped VALUES (1, $at)", + ¶ms! { "at" => Value::TxId(tx.id()) }, + )?; + println!("guard holds now: {held}"); + db.execute("UPDATE accounts SET owner = 'ana2' WHERE id = 1", ¶ms! {})?; + match tx.commit() { + Ok(_) => println!("fourth committed"), + Err(error) => println!("fourth: {error}"), + } + + let found = db.execute("SELECT id, owner FROM accounts ORDER BY id", ¶ms! {})?; + println!("{} rows", found.rows().len()); + db.close() +} +``` + +It prints (`./accounts` must not exist): + +```text +second: unique constraint violation: accounts (id) = (1) +third: conditional update conflict: 1 staged update(s) no longer matched at commit +declaring: a transaction either declares tables or writes rows: commit this one and run the statement in the next +guard holds now: true +fourth: conditional update conflict: 1 staged update(s) no longer matched at commit +2 rows +``` + +- A transaction either declares (`CREATE`, `ALTER`, `DROP`) or writes rows, not both: the + statement that would mix them is refused (`MixedSchemaAndRowWrite`), in either order, and the + transaction goes on. Its declarations commit together and take hold when it commits. +- `Transaction::id()` is the transaction's number, taken when it begins and above every commit + there is. A `TXID` column holds such a number, bound as `Value::TxId`; SQL has no `TXID` + literal, and a number above the transaction's own is refused (`TxIdOutOfRange`). The commit + carries the transaction's number, or the next after the newest commit when another commit + overtook it. +- `Transaction::guard_row_conditions(table, key_column, &key, &[(column, value)])` answers + whether the row `key_column = key` is visible to the handle and holds every value now, and if + so makes the commit stand only while it still does. A guard writes nothing. +- `SET MEMORY_LIMIT` is not part of a transaction: run inside one, it takes effect and the store + remembers it at once, and a rollback does not undo it. + +**If you know Postgres.** The second of two transactions that clash does not wait for a lock; it +is refused when it commits, and then it has lost everything it wrote, also when it inserted a key +and deleted it again while another commit took the key. A transaction that declares a table +cannot also write rows. + +## Who may see and write a row + +A handle made with `Database::scoped_with_constraints(contexts, scope_labels, principal)` (or +`scoped_with_contexts(contexts)`) shares the open store and sees, and may write, only the rows +those three allow; `None` declares nothing for that rule; labels are built with +`ScopeLabel::new("a")`. A handle with no narrowing (`Database::open`, or +`scoped_with_constraints(None, None, None)`) sees every row and is the only one that may run +`CREATE`, `ALTER`, `DROP`, `SET MEMORY_LIMIT` and the two `SHOW VECTOR_*` statements; +`set_memory_limit` is refused as `SET MEMORY_LIMIT` is; `run_maintenance_cycle` does not look at +the narrowing. A handle made from a +narrowed handle keeps the Contexts and labels both name and the principal the first had, and a +transaction begun on a narrowed handle is narrowed the same way. + +A table opts in by declaring a column: + +- `CONTEXT_ID` on a `UUID` column: a row belongs to the Context it holds, and a handle narrowed + to Contexts reads and writes only rows of those. +- `SCOPE_LABEL ('a', 'b')` on a `TEXT` column: a row written must carry one of the listed labels + and a label the handle holds; it hides no row. `SCOPE_LABEL_READ ('a') WRITE ('a', 'b')` hides + rows as well: a handle that holds labels reads the rows whose label is on the read list and + one it holds, and a row with no label is hidden from it; it writes under the write list. +- `ACL REFERENCES grants(acl_id)` on a `UUID` column: a row is shown to a principal that holds a + grant for its `acl_id`. A grant is a row of the table `grants`, with `principal_kind` (`Agent` + or `Human`), `principal_id` and `acl_id`; the grant row has to be visible to the handle too. A + narrowed handle that names no principal, or `Principal::System`, is refused such a table + (`PrincipalRequired`). + +What a narrowed handle gets: + +- A list read (a scan, a range, a count) leaves hidden rows out, and says nothing of them. +- A `SELECT` that names one hidden row whole, by its primary key or a `UNIQUE` rule, is refused + by the rule that hid it: `ContextScopeViolation`, `ScopeLabelViolation`, `AclDenied`, or + `PrincipalRequired` for a table that wants a principal and got none. That tells "not yours" + from "not there", which is an empty answer; the sentence never carries a value or a number of + the hidden row. +- An `UPDATE` or `DELETE` changes no row the handle cannot see, and says it changed none; an + `INSERT` or `UPDATE` that would put a row outside the handle's Contexts, labels or grants is + refused by the same kinds, and so is a statement whose `PROPAGATE` would move a row the handle + may not write. +- The statements named above as the unnarrowed handle's reach every Context at once and are + refused with `AdminRequired`, naming the statement. + `explain` prints the declaration lines of a search and no record of a partition. + +```rust,no_run +use std::collections::BTreeSet; + +use contextdb::{params, ContextId, Database, Principal, Uuid}; + +fn main() -> contextdb::Result<()> { + let db = Database::open("./reports")?; + db.execute( + "CREATE TABLE grants (id INTEGER PRIMARY KEY, principal_kind TEXT, principal_id TEXT, + acl_id UUID)", + ¶ms! {}, + )?; + db.execute( + "CREATE TABLE reports (id INTEGER PRIMARY KEY, context_id UUID CONTEXT_ID, + acl_id UUID ACL REFERENCES grants(acl_id), body TEXT)", + ¶ms! {}, + )?; + let (mine, theirs, key) = (Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4()); + db.execute( + "INSERT INTO grants VALUES (1, 'Agent', 'reader', $key)", + ¶ms! { "key" => key }, + )?; + for (id, context) in [(1_i64, mine), (2, theirs)] { + db.execute( + "INSERT INTO reports VALUES ($id, $context, $key, 'text')", + ¶ms! { "id" => id, "context" => context, "key" => key }, + )?; + } + + let contexts = BTreeSet::from([ContextId::new(mine)]); + let agent = db.scoped_with_constraints( + Some(contexts), + None, + Some(Principal::Agent("reader".to_owned())), + ); + let list = agent.execute("SELECT id FROM reports", ¶ms! {})?; + println!("a list read: {} row", list.rows().len()); + for sql in ["SELECT body FROM reports WHERE id = 2", "DELETE FROM reports WHERE id = 2"] { + match agent.execute(sql, ¶ms! {}) { + Ok(result) => println!("{} rows changed", result.rows_affected()), + Err(error) => println!("{error}"), + } + } + if let Err(error) = agent.execute( + "INSERT INTO reports VALUES (3, $other, $key, 'text')", + ¶ms! { "other" => theirs, "key" => key }, + ) { + println!("{error}"); + } + if let Err(error) = agent.execute("DROP TABLE reports", ¶ms! {}) { + println!("{error}"); + } + let nobody = db.scoped_with_contexts(BTreeSet::from([ContextId::new(mine)])); + if let Err(error) = nobody.execute("SELECT id FROM reports", ¶ms! {}) { + println!("{error}"); + } + db.close() +} +``` + +It prints (`./reports` must not exist): + +```text +a list read: 1 row +context scope violation: this row of reports is outside the Contexts this handle was given +0 rows changed +context scope violation: this row of reports is outside the Contexts this handle was given +DROP TABLE reaches every Context, label and principal: run it through the handle that declared no narrowing +table reports is access-controlled: read it through a handle that names a principal +``` + +**If you know Postgres.** This is not row-level security, which leaves a hidden row out of every +read, a `SELECT` by key included. + +## Changing a table + +```text +contextdb> ALTER TABLE docs ADD COLUMN owner TEXT DEFAULT 'nobody'; +ok (rows_affected=0) +contextdb> SELECT id, owner FROM docs; ++----+--------+ +| id | owner | ++----+--------+ +| 1 | nobody | ++----+--------+ +(1 row) +contextdb> ALTER TABLE docs RENAME COLUMN title TO heading; +ok (rows_affected=0) +contextdb> ALTER TABLE docs RENAME COLUMN id TO ident; +Error: schema invalid: column 'docs.id' is in the PRIMARY KEY, which stays as the table declared it, so it is neither dropped nor renamed +contextdb> CREATE INDEX docs_by_heading ON docs (heading); +ok (rows_affected=0) +contextdb> ALTER TABLE docs DROP COLUMN heading; +Error: column docs.heading is used by the index docs_by_heading; drop the index first, or write CASCADE +contextdb> ALTER TABLE docs DROP COLUMN heading CASCADE; ++-----------------+ +| dropped_indexes | ++-----------------+ +| docs_by_heading | ++-----------------+ +(1 row) +contextdb> DROP TABLE docs; +Error: schema invalid: table 'docs' is referenced by table 'chunks', so it was not dropped; drop that table first +``` + +- `ADD COLUMN` writes no row again: a row written before it reads the column's `DEFAULT` (NULL + without one). A `NOT NULL` column added to a table that has rows needs a `DEFAULT`. +- `DROP COLUMN` is `RESTRICT` unless `CASCADE` is written: a column an index names is refused, + and `CASCADE` drops those indexes and reports them. The column's bytes leave the store at the + commit. +- A column is neither dropped nor renamed while the table is declared by it: a column of the + primary key, an `IMMUTABLE` column, a vector column's `PARTITION_KEY`, the three columns of a + table of edges that has a `DAG`, or a column a rank policy reads. `CASCADE` does not lift that. +- `RENAME COLUMN` carries the column's indexes and its vectors with it. +- `SET RETAIN`, `SET HISTORY`, `SET SYNC`, `SET SYNC CONFLICT` and `SET MAINTAIN COUNT` replace + that clause; `DROP RETAIN` takes `RETAIN` off. No statement removes a `HISTORY` clause: `SET + HISTORY ALL` is the way back. A new set of counts is installed whole or not at all. +- `ALTER COLUMN v SET MAX_PARTITIONS m`, `SET SEARCH_MODE mode`, `SET AUTO_INDEX_AT a` and + `SET CONSOLIDATION ...` change a `VECTOR` column; `DEFAULT` after `AUTO_INDEX_AT` or + `CONSOLIDATION` puts back the default. +- `DROP TABLE` refuses a table another table references, and `DROP INDEX name ON table` names + the table, because two tables may use one index name. `IF EXISTS` makes a missing table or + index a no-op. + +**If you know Postgres.** A key column can be neither renamed nor dropped, and `DROP COLUMN` +refuses where Postgres drops an index silently. + +## Reading a table's rules back + +`Database::declaration(table)` returns the declaration as text, the same text `.schema` prints, and +declaring that text again declares the same table. `Database::table_meta(table)` returns it as a +value: `columns` (each with its name, type, `nullable`, `primary_key`, `unique` and default), `unique_constraints`, +`indexes`, `immutable`, `state_machine`, `dag`, `retain`, `history`, `sync`, `conflict` and +`kept_counts`. The type is `#[non_exhaustive]`. It carries only those; references, `PROPAGATE`, +`EXCLUDE VECTOR`, rank policies, `VECTOR` column clauses, `DELIVERY MANIFEST`, `EDGE DISCARD` and +the rules of who may see a row are in the text `declaration` returns. + +```rust,no_run +use contextdb::{params, Database}; + +fn main() -> contextdb::Result<()> { + let db = Database::open("./docs")?; + db.execute( + "CREATE TABLE docs (a INTEGER, b TEXT, PRIMARY KEY (a, b), note TEXT) + RETAIN 30 DAYS SYNC OFF", + ¶ms! {}, + )?; + let meta = db.table_meta("docs")?; + for column in &meta.columns { + println!("{}: nullable {}", column.name, column.nullable); + } + println!("retain: {:?}", meta.retain); + println!("sync: {:?}", meta.sync); + println!("state machine: {:?}", meta.state_machine); + println!("{}", db.declaration("docs")?); + db.close() +} +``` + +It prints (`./docs` must not exist): + +```text +a: nullable false +b: nullable false +note: nullable true +retain: Some(Retain { count: 30, unit: Days, sync_safe: false }) +sync: Some(Off) +state machine: None +CREATE TABLE docs ( + a INTEGER, + b TEXT, + note TEXT, + PRIMARY KEY (a, b) +) RETAIN 30 DAYS SYNC OFF; +``` + +`table_meta` reports every column of the primary key as not nullable, however the key was +written. + ## Vector search A `VECTOR(n)` column is searched by ordering its rows by their distance from a vector and @@ -388,7 +894,7 @@ one a `SELECT` returns and every search, `EXACT` included, ranks by (`[0.1, 0.2, read back as `[0.0992126, 0.2007874, 0.3]`); and on those stores, searched exactly, the ten nearest rows of an `SQ8` column differ from the full-precision column's on 12 to 21 questions in 100, by whether the search covers one, several or all of its partitions. It is an option a table chooses and never a default: a -column keeps `F32` unless it says otherwise. `SQ4` is refused by name. +column keeps `F32` unless it says otherwise. `SQ4` is refused by name, and the refusal says to write `'SQ8'` or leave the clause out. **Inspection.** `SHOW VECTOR_INDEXES` returns one row for each `VECTOR` column of the store, in the order of table name and column, and `SHOW VECTOR_PARTITIONS` one row for each partition @@ -442,24 +948,7 @@ probes: 8 recovery_action: none ``` -`.explain ` in the shell prints what `Database::explain` returns; `.explain SELECT id FROM notes -ORDER BY embedding <=> $q LIMIT 10` prints: - -```text -requested_mode: AUTO -resolved_mode: AUTO -effective_auto_index_at: 1000 -aggregate_allowed_vectors: 2000 -partition_key_columns: [] -partitions: 1 -partition: {} - route: indexed - lists: 45 - probes: 8 - live_rows: 2000 - outside_grouping: 0 -refusal: none -``` +`.explain ` in the shell prints what `Database::explain` returns, the text shown above. Inspection tells the groups a partition has (`lists`) and the most groups a search scanned while the index was measured (`probes`); it does not print the measured distance or width. A @@ -483,29 +972,17 @@ any other value is written as a `Value`, for example `Value::Timestamp(1_767_225 Every call that can fail returns `contextdb::Result` with one `contextdb::Error`, except `Database::memory_limit`, `memory_used`, `table_names` and `committed_watermark`, which return a plain value and panic on a handle that was closed. A program -matches on the kind (`Error::UniqueViolation { table, column }`, `Error::ColumnNotNullable`, +matches on the kind (`Error::UniqueViolation { table, column, key }`, `Error::ColumnNotNullable`, `Error::TableNotFound`, `Error::DatabaseLocked`, `Error::MemoryBudgetExceeded`, and the rest listed on the `Error` type); a person reads the sentence it prints: ```text -unique constraint violation: notes (id) +unique constraint violation: notes (id) = (5b1f0a9e-3c47-4d2a-9e51-8f0c6a2d7b14) column notes.body is NOT NULL column not found: notes.nothing parse error: syntax error at line 1, column 1: unexpected 'SELEC' ``` -A refused statement changes nothing. Inside a transaction it leaves the transaction usable. - -## Transactions - -`db.execute` is a transaction of one statement: when it returns, the change is on disk. -`db.transaction()` begins a longer one. Its statements see what it has written and nobody else -does until `commit` returns; dropping it, or `rollback`, discards it. A transaction either -declares tables and indexes or writes rows, not both. When two open transactions insert the same -key, or change the same row, the first to commit wins and the other's `commit` is refused. -`SET MEMORY_LIMIT` is not part of a transaction: run inside one, it takes effect and the store -remembers it at once, and a rollback does not undo it. - ## One writer, many readers One process at a time holds a store for writing. A second `Database::open` on the same folder is @@ -623,7 +1100,8 @@ contextdb ./notes --write read and write it, creating it if it is missing ``` A statement ends at `;`. `BEGIN;`, `COMMIT;` and `ROLLBACK;` run a transaction, which needs -`--write`; `.tables` lists +`--write`; a `BEGIN` inside a transaction, and a `COMMIT` or `ROLLBACK` outside one, change nothing, +answer `ok` and print a `warning:` line on the error stream; `.tables` lists the tables, `.schema ` prints a table's declaration, `.explain ` prints how a similarity search would be answered without running it (anything else it refuses like a statement of a later version), and `.help` lists the commands. With @@ -632,7 +1110,10 @@ refused statement or command is one line on the error stream, `Error: ` and then sentence, or the program's own sentence where one of its options is the way on; a line that starts like a command and is not one is answered with `Unknown command:`. The rest still run. A piped run exits 1 if any statement or command was refused, an unknown command and `.schema` of -a table that is not there included, and 0 otherwise. However the input arrives, the exit status is 1 when the +a table that is not there included, and 0 otherwise. A piped run that meets a line that is not +text ends there with `Error: line 4 of the input is not text, so the run ends there` (the line's +number), exit status 1, and a transaction it had open is not committed. An output pipe closed +by the reader is not an error: the program stops and exits 0. However the input arrives, the exit status is 1 when the store could not be opened or closed, and 2 when the command line itself is refused. `--json` prints one JSON document per statement or command and each refusal as a JSON document on the error stream. A vector in a result is a JSON array of numbers, each written at the full @@ -653,20 +1134,21 @@ how many of the 100 questions got all ten. | What | Measured | |---|---| -| 20,000 rows of 1.6 KB, one commit per row: time to insert | 11.1 s | +| 20,000 rows of 1.6 KB, one commit per row: time to insert | 11.3 s | | That store on disk (its rows repeat their text, which compresses far better than real text) | 2.4 MiB | -| Open that store | 0.561 ms | -| `SELECT COUNT(*)` over it | 0.030 ms, 0 rows read | -| A million rows under a 64 MiB limit: time to load | 10.5 s | -| Scan the million rows | 236.582 ms | +| Open that store | 0.558 ms | +| `SELECT COUNT(*)` over it | 0.032 ms, 0 rows read | +| A million rows under a 64 MiB limit: time to load | 10.7 s | +| Scan the million rows | 255.040 ms | | Most memory the engine had allocated during that load and scan | 16.4 MiB | -| Most memory the process held, as the operating system reports it | 21.5 MiB | -| First build of a program that uses the library | 8 s, 47 packages, a 3.2 MiB program | -| From a fresh clone to a passing `cargo test` | 3.2 min | -| Ten nearest of 20,000 vectors, sealed, through SQL, exact: 95th percentile | 1.548 ms | +| Most memory the process held, as the operating system reports it | 22.3 MiB | +| First build of a program that uses the library | 11 s, 47 packages, a 4.1 MiB program | +| From a fresh clone to a passing `cargo test` | 4.0 min | +| Ten nearest of 20,000 vectors, sealed, through SQL, exact: 95th percentile | 1.505 ms | | That vector store on disk, and how many times the bytes it holds | 44.3 MiB, 1.50 times | -| Lines of Rust and grammar in the library and the program | 15,440 | -| Tests | 253 | +| Lines of Rust and grammar in the library and the program | 20,742 | +| Tests | 371 | +| 173 session files of real coding sessions (150,756 rows): the megabytes put in, and that store on disk | 344.9 MiB in, 227.9 MiB on disk, 0.66 times | | Lines that differ from redb 4.1.0 in the storage library under the engine, which is redb 4.1.0 with the changes listed in `v2/redb/README.md` | 626 | Searches for the ten nearest, indexed beside exact from the same run. Where a store is below @@ -674,26 +1156,26 @@ Searches for the ten nearest, indexed beside exact from the same run. Where a st | Search | Indexed | Exact | Recall, questions with all ten | |---|---|---|---| -| 10,000 generated vectors, one partition, 100 groups (`AUTO_INDEX_AT 1000`) | 0.541 ms | 1.185 ms | 0.9960, 96 of 100 | -| 100,008 real passages, eight partitions, 884 groups: one partition | 0.682 ms | 1.411 ms | 1.0000, 100 of 100 | -| the same: several partitions | 1.537 ms | 3.841 ms | 0.9990, 99 of 100 | -| the same: all eight | 4.052 ms | 10.880 ms | 0.9990, 99 of 100 | -| the same: all eight, under 64 MiB | 11.383 ms | not taken | 0.9990, 99 of 100 | -| the same: all eight, a filter that passes 992 of the 100,008 rows | 32.411 ms | 38.453 ms | 1.0000, 100 of 100 | -| the same, 9,996 more passages written after the build and sealed | 3.749 ms | 10.781 ms | 1.0000, 100 of 100 | -| 1,000,000 generated vectors of 768 numbers around 300 hidden centres, one partition, 1,000 groups | 1.109 ms | 180.929 ms | 1.0000, 100 of 100 | -| 1,000,000 generated camera sightings, one partition, 1,000 groups | 30.348 ms | 175.409 ms | 1.0000, 100 of 100 | -| 11,588 real passages, eight partitions, below the size: one / several / all | 0.303 / 0.724 / 1.440 ms | 0.302 / 0.724 / 1.440 ms | 1.0000, 100 of 100 each | - -Getting a store to those groupings takes `run_maintenance_cycle` calls: 34 cycles and 1.8 s for -the 10,000, 56 cycles and 26.9 s for the 100,008 passages, 1,150 cycles and 362.8 s for the +| 10,000 generated vectors, one partition, 100 groups (`AUTO_INDEX_AT 1000`) | 0.544 ms | 1.194 ms | 0.9960, 96 of 100 | +| 100,008 real passages, eight partitions, 884 groups: one partition | 0.696 ms | 1.407 ms | 1.0000, 100 of 100 | +| the same: several partitions | 1.568 ms | 3.829 ms | 0.9990, 99 of 100 | +| the same: all eight | 4.115 ms | 10.904 ms | 0.9990, 99 of 100 | +| the same: all eight, under 64 MiB | 10.975 ms | not taken | 0.9990, 99 of 100 | +| the same: all eight, a filter that passes 992 of the 100,008 rows | 32.761 ms | 38.290 ms | 1.0000, 100 of 100 | +| the same, 9,996 more passages written after the build and sealed | 3.797 ms | 10.874 ms | 1.0000, 100 of 100 | +| 1,000,000 generated vectors of 768 numbers around 300 hidden centres, one partition, 1,000 groups | 1.122 ms | 180.378 ms | 1.0000, 100 of 100 | +| 1,000,000 generated camera sightings, one partition, 1,000 groups | 33.734 ms | 183.724 ms | 1.0000, 100 of 100 | +| 11,588 real passages, eight partitions, below the size: one / several / all | 0.309 / 0.727 / 1.447 ms | 0.305 / 0.727 / 1.447 ms | 1.0000, 100 of 100 each | + +Getting a store to those groupings takes `run_maintenance_cycle` calls: 34 cycles and 1.5 s for +the 10,000, 56 cycles and 19.3 s for the 100,008 passages, 1,150 cycles and 365.8 s for the million. The index takes 129, 125 and 101 bytes a vector there. Right after a build a store's file is 1.20, 1.13 and 1.31 times the size it had sealed, which is more than the index holds, because the pages the build freed stay in the file in this version; a store that is never grouped stays at 1.00 times (the 11,588 passages). With `WITH (quantization = 'SQ8')` the same two stores of real passages take 0.45 times the room -(a passage 1,968 bytes in place of 4,411 in the 11,588, and 1,992 in place of 4,946 in the +(a passage 1,968 bytes in place of 4,411 in the 11,588, and 1,989 in place of 4,946 in the 100,008). Searched exactly, the same ten rows come back as from the full-precision store on 88, 84 and 79 questions of 100 in the 11,588 (one, several, all partitions) and on 86, 83 and 85 in the 100,008. diff --git a/v2/contextdb/examples/queries.rs b/v2/contextdb/examples/queries.rs new file mode 100644 index 00000000..20f6afb8 --- /dev/null +++ b/v2/contextdb/examples/queries.rs @@ -0,0 +1,830 @@ +//! Measures Context Graph's own declarations and statements, printing one number per line as +//! `name value unit`. Every store is declared with the tables and indexes of +//! `tests/fixtures/consumer_schema.sql`, the declarations Context Graph runs, and nothing else. +//! +//! - `queries declarations `: every table and index text of the fixture, each second +//! text in a store of its own after the tables written before its first; how many are +//! accepted, and of the tables, how many print back unchanged: the printed text, declared +//! again in an empty store, prints the same text. (That it says every clause its author wrote +//! and no other is `tests/declarations.rs`.) +//! - `queries declared `: the dump `passages.rs` reads, +//! loaded into `event_search_passages` with the `contexts` and `event_search_generations` +//! rows it refers to, 500 passages to a commit, then every maintenance cycle. A column the +//! dump does not hold is written short: the connector's name, a session named by the pair, +//! the place in the dump as ordinal and sequences, an empty text, the id's digits as the +//! content's digest. Then Context Graph's find, as its branch writes it, in a read session +//! under 1 GiB (`declared_find__...`, five rounds after 5 untimed searches, and its +//! ten beside the recorded ones); a count by an index range and by a kept count; three +//! flips of `is_current` over the passages of the dump's pair of index 1, with the bytes the +//! storage door wrote; and the DELETE of that pair's passages under 64 MiB. +//! - `queries anchors `: every file of the folder read as a session +//! whose lines are JSON, as Context Graph's connector reads one: each tool call of a message +//! is an event, with a `command` anchor for its `input.command`, a `file` anchor for its file +//! path and a `working_directory` anchor for its line's `cwd`, all in one Context, 500 events +//! and their anchors to a commit. The size beside the bytes put in, the index entries as the +//! store holds them, and the exact-match read for 100 commands, 50 of them longer than an +//! entry holds whole, five rounds. It prints nothing of a session but counts, sizes and +//! times. +//! - `queries dag `: 300,000 generated edges of `edges` (`DAG('CITES')`) over 100,000 +//! nodes in groups of 100, each from a node drawn at random to a later node of its group +//! (fixed seed), 500 to a commit; each INSERT timed. Synthetic. +//! - `queries grants `: 100,000 rows in ten Contexts, each under one of 20,000 access +//! ids, of which an agent holds every twentieth: a Context's rows are under 2,000 ids, and the +//! agent may see half its rows. The rows of one Context are read through a handle narrowed to +//! that Context and that agent, and through the handle narrowed to nothing, five rounds of +//! twenty reads each. Synthetic. +//! +//! "Held" is the bytes of the values put in: a UUID 16, a number or TXID 8, a flag 1, a text or +//! JSON its bytes, a vector four a number. The folder must be missing or empty. This program +//! measures. It never passes or fails, no test reads what it prints, and `numbers.sh` is what +//! runs it. + +// Public, so that what this program does not use of what the three programs share is not +// read as unused. +pub mod support; + +use std::collections::BTreeSet; +use std::fs::File; +use std::io::{BufRead, BufReader, Read}; +use std::os::unix::fs::FileExt; +use std::path::Path; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + +use contextdb::serde_json::{self, Value as Json}; +use contextdb::{ + ContextId, Database, DatabaseOpenOptions, Principal, StorageBackend, Uuid, Value, params, +}; +use support::{ + AUTO, Asked, Fallible, READER_LIMIT, Values, ask, bytes_on_disk, line, mebibytes, milliseconds, + must_be_empty, percentile, ratio, seconds, session, setting, timed, times, +}; + +#[global_allocator] +static ALLOCATOR: support::Counting = support::Counting; + +const SCHEMA: &str = include_str!("../tests/fixtures/consumer_schema.sql"); +const PER_COMMIT: usize = 500; +const ROUNDS: usize = 5; +const DIMENSIONS: usize = 768; + +/// The statements of a text: each ends with `;` at the end of a line; `--` lines are comments. +fn statements(text: &str) -> Vec { + let (mut found, mut statement) = (Vec::new(), String::new()); + for line in text.lines().filter(|line| !line.starts_with("--")) { + statement.push_str(line); + statement.push('\n'); + if let Some(whole) = statement.trim().strip_suffix(';') { + found.push(whole.to_owned()); + statement.clear(); + } + } + found +} + +/// The table a `CREATE TABLE t` or a `CREATE INDEX i ON t` is about. +fn table_of(sql: &str) -> String { + let rest = sql.strip_prefix("CREATE TABLE ").unwrap_or_else(|| { + let (_, on) = sql.split_once(" ON ").unwrap_or_default(); + on + }); + rest.chars() + .take_while(|c| c.is_ascii_alphanumeric() || *c == '_') + .collect() +} + +/// The fixture's statements a single store takes, and its second texts, which follow the line +/// that begins `-- Second texts`. +fn schema() -> (Vec, Vec) { + let (first, second) = SCHEMA + .split_once("\n-- Second texts") + .unwrap_or((SCHEMA, "")); + let (_, second) = second.split_once('\n').unwrap_or_default(); + (statements(first), statements(second)) +} + +/// A new store with every table and index Context Graph declares in one store. +fn declared_store(folder: &Path, options: DatabaseOpenOptions) -> Fallible { + let db = Database::open_with_options(folder, options)?; + for sql in schema().0 { + db.execute(&sql, ¶ms! {})?; + } + Ok(db) +} + +/// Declares the statements in a new store: whether each was accepted, and for each table in +/// the order declared whether its printed text, declared in another new store, prints the +/// same. +fn declare(folder: &Path, texts: &[String]) -> Fallible<(Vec, Vec)> { + let db = Database::open(folder.join("written"))?; + let accepted: Vec = texts + .iter() + .map(|sql| db.execute(sql, ¶ms! {}).is_ok()) + .collect(); + let again = Database::open(folder.join("printed"))?; + let mut same = Vec::new(); + for sql in texts.iter().filter(|sql| sql.starts_with("CREATE TABLE")) { + let Ok(printed) = db.declaration(&table_of(sql)) else { + same.push(false); + continue; + }; + let declared = statements(&printed) + .iter() + .all(|sql| again.execute(sql, ¶ms! {}).is_ok()); + same.push(declared && again.declaration(&table_of(sql)).ok() == Some(printed)); + } + Ok((accepted, same)) +} + +fn declarations(folder: &Path) -> Fallible<()> { + let (first, second) = schema(); + let (mut accepted, mut same) = declare(&folder.join("first"), &first)?; + for (n, text) in second.iter().enumerate() { + let table = table_of(text); + let before = first + .iter() + .take_while(|sql| table_of(sql) != table || !sql.starts_with("CREATE TABLE")); + let texts: Vec = before + .filter(|sql| sql.starts_with("CREATE TABLE")) + .cloned() + .chain([text.clone()]) + .collect(); + let (taken, printed) = declare(&folder.join(format!("second-{n}")), &texts)?; + accepted.push(taken[taken.len() - 1]); + same.push(printed[printed.len() - 1]); + } + let count = |of: &[bool]| { + ( + of.iter().filter(|ok| **ok).count(), + format!("of_{}", of.len()), + ) + }; + let (n, of) = count(&accepted); + line("declarations_accepted", n, &of); + let (n, of) = count(&same); + line("declarations_print_back_unchanged", n, &of); + Ok(()) +} + +/// The bytes of the values put in, as the head of this file counts them. +fn held(values: &Values) -> u64 { + let size = |value: &Value| match value { + Value::Null => 0, + Value::Bool(_) => 1, + Value::Uuid(_) => 16, + Value::Text(text) => text.len(), + Value::Json(json) => json.to_string().len(), + Value::Vector(numbers) => 4 * numbers.len(), + _ => 8, + }; + values.values().map(|value| size(value) as u64).sum() +} + +/// Inserts a row of these values into a table. +fn insert(tx: &contextdb::Transaction<'_>, table: &str, values: &Values) -> Fallible { + let names: Vec<&str> = values.keys().map(String::as_str).collect(); + let places: Vec = names.iter().map(|name| format!("${name}")).collect(); + let sql = format!( + "INSERT INTO {table} ({}) VALUES ({})", + names.join(", "), + places.join(", ") + ); + tx.execute(&sql, values)?; + Ok(held(values)) +} + +/// A read statement asked of every value in `rounds` rounds after 5 untimed: the times of each +/// round as `ask` gives them, and from the first round the rows each examined. +fn reads( + run: &dyn Fn(&Values) -> contextdb::Result, + asked: &[Values], +) -> Fallible { + for values in asked.iter().take(support::UNTIMED) { + run(values)?; + } + let mut out = Asked::default(); + for round in 0..ROUNDS { + let mut spent = Vec::new(); + for values in asked { + let (found, time) = timed(|| run(values)); + let found = found?; + spent.push(time); + if round == 0 { + out.examined.push(found.rows_examined()); + out.evaluations.push(found.rows().len() as u64); + } + } + out.p50.push(percentile(&spent, 0.5)); + out.p95.push(percentile(&spent, 0.95)); + } + Ok(out) +} + +/// Prints the times of `asked` and the rows it examined at the middle and the 95th percentile. +fn read_lines(name: &str, asked: &Asked) { + times(name, asked); + line( + &format!("{name}_rows_examined_p50"), + percentile(&asked.examined, 0.5), + "rows", + ); + line( + &format!("{name}_rows_examined_p95"), + percentile(&asked.examined, 0.95), + "rows", + ); +} + +// --------------------------------------------------------------------------------------------- +// The passages, declared as Context Graph declares them + +/// A dump as `passages.rs` reads it: each Context, each pair's Context and generation, the +/// questions, and each passage's id, pair and flags. +struct Dump { + contexts: Vec, + pairs: Vec<(usize, Uuid)>, + questions: Vec>, + passages: Vec<(Uuid, usize, u32)>, +} + +fn floats(bytes: &[u8]) -> Vec { + let (fours, _) = bytes.as_chunks::<4>(); + fours.iter().map(|four| f32::from_le_bytes(*four)).collect() +} + +fn read_dump(folder: &Path) -> Fallible { + let text = |name: &str| std::fs::read_to_string(folder.join(name)); + let field = |line: &str, at: usize| line.split(' ').nth(at).unwrap_or_default().to_owned(); + let contexts = text("contexts.txt")? + .lines() + .map(|l| field(l, 1).parse()) + .collect::>()?; + let mut pairs = Vec::new(); + for l in text("pairs.txt")?.lines() { + pairs.push((field(l, 1).parse()?, field(l, 2).parse()?)); + } + let questions = floats(&std::fs::read(folder.join("queries.f32"))?); + let mut passages = Vec::new(); + for record in std::fs::read(folder.join("rows.bin"))?.chunks_exact(36) { + let number = |at: usize| u32::from_le_bytes(std::array::from_fn(|b| record[at + b])); + passages.push(( + Uuid::from_slice(&record[8..24])?, + number(28) as usize, + number(32), + )); + } + let questions = questions + .chunks_exact(DIMENSIONS) + .map(<[f32]>::to_vec) + .collect(); + Ok(Dump { + contexts, + pairs, + questions, + passages, + }) +} + +/// Of the recorded answers in a file, how many questions have the same ten ids as `asked`. +fn same_ten(asked: &Asked, recorded: &Path) -> Fallible { + let mut same = 0; + for text in std::fs::read_to_string(recorded)?.lines() { + let recorded: Json = serde_json::from_str(text)?; + let (Some(question), Some(ids)) = (recorded["q"].as_u64(), recorded["ids"].as_array()) + else { + continue; // the summary line + }; + let ids: BTreeSet = ids + .iter() + .filter_map(|id| id.as_str()?.parse::().ok()) + .map(|id| id.as_u128()) + .collect(); + let ours = asked + .answers + .get(question as usize) + .ok_or("a question out of range")?; + same += usize::from(ours.iter().map(|row| row.0).collect::>() == ids); + } + Ok(same) +} + +/// The longest keys a closed store holds, in bytes: an index entry (its row key left out), a +/// row key, a kept count's group and a partition's key tuple, each without the bytes that name +/// its table, index, count or column; and how many index entries hold a cut value. Read from +/// the data file with the storage library, as `contextdb/src/store/disk.rs` lays it out: an +/// index entry's record is its length in four bytes, and for an entry that holds a cut value +/// four more; a partition's record is kept under the byte 1, its column and its key tuple. +struct Keys { + entry: u32, + cut: u64, + row: usize, + count: usize, + tuple: usize, +} + +fn stored_keys(store: &Path) -> Fallible { + use redb::{ReadableDatabase, ReadableTable, TableDefinition}; + let data = redb::ReadOnlyDatabase::open(store.join("data"))?; + let read = data.begin_read()?; + let table = |name: &str| read.open_table(TableDefinition::<&[u8], &[u8]>::new(name)); + let mut keys = Keys { + entry: 0, + cut: 0, + row: 0, + count: 0, + tuple: 0, + }; + for record in table("index")?.iter()? { + let record = record?.1.value().to_vec(); + keys.entry = keys + .entry + .max(u32::from_le_bytes(std::array::from_fn(|b| record[b]))); + keys.cut += u64::from(record.len() == 8); + } + let longest = |name: &str, named_by: usize| -> Fallible { + let table = table(name)?; + let lengths = table + .iter()? + .map(|record| record.map(|(key, _)| key.value().len())); + let longest = lengths.collect::, _>>()?.into_iter().max(); + Ok(longest.unwrap_or(named_by) - named_by) + }; + keys.row = longest("rows", 4)?; + keys.count = longest("counts", 4)?; + for record in table("vectors")?.range::<&[u8]>(&[1_u8][..]..&[2_u8][..])? { + keys.tuple = keys.tuple.max(record?.0.value().len() - 5); + } + Ok(keys) +} + +/// The storage door of a data file that counts the bytes written through it. +#[derive(Debug)] +struct Written(File, Arc); + +impl StorageBackend for Written { + fn len(&self) -> std::io::Result { + Ok(self.0.metadata()?.len()) + } + fn read(&self, offset: u64, out: &mut [u8]) -> std::io::Result<()> { + self.0.read_exact_at(out, offset) + } + fn set_len(&self, len: u64) -> std::io::Result<()> { + self.0.set_len(len) + } + fn sync_data(&self) -> std::io::Result<()> { + self.0.sync_data() + } + fn write(&self, offset: u64, data: &[u8]) -> std::io::Result<()> { + self.1.fetch_add(data.len() as u64, Ordering::Relaxed); + self.0.write_all_at(data, offset) + } +} + +fn declared(dump_folder: &Path, answers: &Path, folder: &Path) -> Fallible<()> { + let dump = read_dump(dump_folder)?; + let store = folder.join("declared"); + let db = declared_store(&store, DatabaseOpenOptions::default())?; + let prefix = format!("declared_{}_", dump.passages.len()); + let name = |what: &str| format!("{prefix}{what}"); + let (held_bytes, load) = timed(|| -> Fallible { + let tx = db.transaction()?; + let space = params! { "embedding_space_id" => "space", "dimension" => DIMENSIONS as i64, "metric" => "cosine", "created_at" => Value::TxId(tx.id()) }; + let mut held_bytes = insert(&tx, "embedding_spaces", &space)?; + for (n, context) in dump.contexts.iter().enumerate() { + let row = params! { "id" => *context, "name" => format!("context {n}"), "created_at" => Value::TxId(tx.id()) }; + held_bytes += insert(&tx, "contexts", &row)?; + } + for (context, generation) in &dump.pairs { + let mut row = params! { + "id" => *generation, "context_id" => dump.contexts[*context], + "interpretation_snapshot" => Json::Object(Default::default()), "interpretation_snapshot_sha256" => "", + "event_count" => 0_i64, "ordered_source_set_sha256" => "", "renderer_id" => "r", + "renderer_version" => "1", "renderer_profile" => Json::Object(Default::default()), + "embedding_space_id" => "space", "encoding_profile" => "e", "state" => "active", + "last_attempt_token" => Uuid::nil(), + }; + row.insert("started_at".to_owned(), Value::TxId(tx.id())); + held_bytes += insert(&tx, "event_search_generations", &row)?; + } + tx.commit()?; + let mut vectors = BufReader::new(File::open(dump_folder.join("vectors.f32"))?); + let mut vector = vec![0u8; 4 * DIMENSIONS]; + let mut tx = db.transaction()?; + for (at, (id, pair, flags)) in dump.passages.iter().enumerate() { + vectors.read_exact(&mut vector)?; + let (context, generation) = dump.pairs[*pair]; + let mut row = params! { + "id" => *id, "context_id" => dump.contexts[context], "generation_id" => generation, + "connector_id" => "claude-code", "session_ref" => format!("pair {pair}"), + "source_instance_ref" => format!("pair {pair}"), "passage_ordinal" => at as i64, + "first_event_sequence" => at as i64, "last_event_sequence" => at as i64, + "rendered_text" => "", "content_sha256" => id.simple().to_string(), + "dimension" => DIMENSIONS as i64, "is_current" => flags & 1 != 0, + "is_searchable" => flags & 2 != 0, + }; + row.insert("created_at".to_owned(), Value::TxId(tx.id())); + let vector = if flags & 4 != 0 { + Value::Vector(floats(&vector)) + } else { + Value::Null + }; + row.insert("vector_text".to_owned(), vector); + held_bytes += insert(&tx, "event_search_passages", &row)?; + if (at + 1) % PER_COMMIT == 0 { + std::mem::replace(&mut tx, db.transaction()?).commit()?; + } + } + tx.commit()?; + support::upkeep(&db)?; + Ok(held_bytes) + }); + let held_bytes = held_bytes?; + db.close()?; + let bytes = bytes_on_disk(&store)?; + line(&name("load"), seconds(load), "s"); + line(&name("store_size"), mebibytes(bytes), "MiB"); + line( + &name("bytes_per_passage"), + bytes / dump.passages.len() as u64, + "bytes", + ); + line(&name("size_over_held"), ratio(bytes, held_bytes), "x"); + line( + "key_longest_partition_tuple", + stored_keys(&store)?.tuple, + "bytes", + ); + + // The find, in three scopes, under AUTO as Context Graph sends it. + let reader = session(&store, READER_LIMIT)?; + let every: Vec = (0..dump.contexts.len()).collect(); + for (scope, contexts) in [ + ("one", &every[..1]), + ("several", &every[..3]), + ("all", &every[..]), + ] { + let wanted: BTreeSet = contexts.iter().copied().collect(); + let mut values = params! {}; + let (mut context_names, mut generation_names) = (Vec::new(), Vec::new()); + for (n, context) in contexts.iter().enumerate() { + values.insert(format!("c{n}"), Value::Uuid(dump.contexts[*context])); + context_names.push(format!("$c{n}")); + } + for (n, (_, generation)) in dump + .pairs + .iter() + .filter(|(context, _)| wanted.contains(context)) + .enumerate() + { + values.insert(format!("g{n}"), Value::Uuid(*generation)); + generation_names.push(format!("$g{n}")); + } + let sql = format!( + "SELECT id, vector_text <=> $q AS distance FROM event_search_passages \ + WHERE context_id IN ({}) AND generation_id IN ({}) AND is_current = TRUE \ + AND is_searchable = TRUE ORDER BY distance", + context_names.join(", "), + generation_names.join(", ") + ); + let find = |tail: &str, question: &[f32]| { + let mut values = values.clone(); + values.insert("q".to_owned(), Value::Vector(question.to_vec())); + (format!("{sql}{tail}"), values) + }; + let asked = ask(&reader, &find, &[AUTO], &dump.questions, ROUNDS)?.remove(0); + let find_name = format!("declared_find_{scope}"); + setting(&find_name, READER_LIMIT); + read_lines(&find_name, &asked); + let same = same_ten(&asked, &answers.join(format!("engine-{scope}.jsonl")))?; + line(&format!("declared_same_ten_{scope}"), same, "of_100"); + } + + // Counts of one pair's current passages: by the index range, then by a kept count. + let (context, generation) = dump.pairs[1]; + let pair = params! { "c" => dump.contexts[context], "g" => generation }; + let count = "SELECT COUNT(*) FROM event_search_passages WHERE context_id = $c AND generation_id = $g AND is_current = TRUE"; + let asked = reads( + &|values| reader.execute(count, values), + &vec![pair.clone(); 100], + )?; + read_lines("declared_count_context", &asked); + drop(reader); + let db = Database::open(&store)?; + db.execute("ALTER TABLE event_search_passages SET MAINTAIN COUNT (context_id, generation_id, is_current)", ¶ms! {})?; + let asked = reads( + &|values| db.execute(count, values), + &vec![pair.clone(); 100], + )?; + read_lines("kept_count_context", &asked); + db.close()?; + + // Three flips of the pair's current flag, through a door that counts the bytes written. + let before = bytes_on_disk(&store)?; + let written = Arc::new(AtomicU64::new(0)); + let counter = written.clone(); + let mut options = DatabaseOpenOptions::default(); + options.storage = Some(Arc::new(move |file| { + Ok(Box::new(Written(file, counter.clone())) as Box) + })); + let db = Database::open_with_options(&store, options)?; + let flip = "UPDATE event_search_passages SET is_current = $to WHERE context_id = $c AND generation_id = $g"; + let (mut spent, mut changed) = (Vec::new(), 0); + written.store(0, Ordering::Relaxed); + for to in [false, true, false] { + let mut values = pair.clone(); + values.insert("to".to_owned(), Value::Bool(to)); + let (found, time) = timed(|| db.execute(flip, &values)); + changed = found?.rows_affected(); + spent.push(time); + } + let flips = written.load(Ordering::Relaxed); + db.close()?; + let flip_name = format!("flip_current_{changed}"); + line(&flip_name, milliseconds(percentile(&spent, 0.5)), "ms"); + line( + &format!("{flip_name}_bytes_written"), + mebibytes(flips / 3), + "MiB", + ); + line( + &format!("{flip_name}_store_size_before"), + mebibytes(before), + "MiB", + ); + line( + &format!("{flip_name}_store_size_after_3_flips"), + mebibytes(bytes_on_disk(&store)?), + "MiB", + ); + + // The pair's passages deleted under 64 MiB. + let mut options = DatabaseOpenOptions::default(); + options.memory_limit = Some(64 << 20); + let db = Database::open_with_options(&store, options)?; + support::take_peak_allocated(); + let deleted = db + .execute( + "DELETE FROM event_search_passages WHERE context_id = $c AND generation_id = $g", + &pair, + )? + .rows_affected(); + line( + &format!("delete_generation_{deleted}_peak_allocated"), + mebibytes(support::take_peak_allocated()), + "MiB", + ); + db.close()?; + Ok(()) +} + +// --------------------------------------------------------------------------------------------- +// The anchors of the sealed sessions + +/// An anchor as the connector makes one: its key and its value. +fn anchors_of(line: &Json) -> Vec> { + let cwd = line["cwd"].as_str(); + let blocks = line["message"]["content"].as_array().into_iter().flatten(); + let calls = blocks.filter(|block| block["type"] == "tool_use"); + calls + .map(|call| { + let input = &call["input"]; + let file = ["file_path", "path", "notebook_path"] + .iter() + .find_map(|field| input[*field].as_str()); + [ + ("command", input["command"].as_str()), + ("file", file), + ("working_directory", cwd), + ] + .into_iter() + .filter_map(|(key, value)| Some((key, value?.to_owned()))) + .collect() + }) + .collect() +} + +fn anchors(corpus: &Path, folder: &Path) -> Fallible<()> { + let store = folder.join("anchors"); + let db = declared_store(&store, DatabaseOpenOptions::default())?; + let context = Uuid::from_u128(1); + let mut names: Vec<_> = std::fs::read_dir(corpus)?.collect::, _>>()?; + names.sort_by_key(|entry| entry.file_name()); + let (mut events, mut rows, mut commands) = (0_u128, 0_u64, Vec::new()); + let (held_bytes, load) = timed(|| -> Fallible { + let mut tx = db.transaction()?; + let row = + params! { "id" => context, "name" => "sessions", "created_at" => Value::TxId(tx.id()) }; + let mut held_bytes = insert(&tx, "contexts", &row)?; + for entry in &names { + let session = entry.file_name().to_string_lossy().into_owned(); + let mut seq = 0_i64; + for text in BufReader::new(File::open(entry.path())?).lines() { + let Ok(json) = serde_json::from_str::(&text?) else { + continue; + }; + for anchors in anchors_of(&json) { + events += 1; + seq += 1; + let event = Uuid::from_u128(1 << 64 | events); + let mut row = params! { + "id" => event, "context_id" => context, "connector_id" => "claude-code", + "adapter_version" => "1", "anchor_declaration_version" => "anc1", + "source_instance_ref" => session.as_str(), "session_ref" => session.as_str(), + "source_event_key" => seq.to_string(), "source_position" => Json::Object(Default::default()), + "seq" => seq, "content_hash" => "", "anchor_count" => anchors.len() as i64, + "anchor_set_hash" => "", "author" => "assistant", "kind" => "tool_call", + "render_role" => "tool", "coverage" => "full", + }; + held_bytes += insert(&tx, "events", &row)?; + for (key, value) in anchors { + rows += 1; + let identity = serde_json::to_string(&value)?; + row = params! { + "id" => Uuid::from_u128(2 << 64 | u128::from(rows)), "context_id" => context, + "event_id" => event, "correlation_scope_hash" => session.as_str(), + "declaration_version" => "anc1", "connector_id" => "claude-code", + "source_instance_ref" => session.as_str(), "session_ref" => session.as_str(), + "render_role" => "tool", "key" => key, "value" => value.as_str(), + "identity_value" => identity, "original_value" => Json::String(value.clone()), + }; + held_bytes += insert(&tx, "event_anchors", &row)?; + if key == "command" { + commands.push(value); + } + } + if events % PER_COMMIT as u128 == 0 { + std::mem::replace(&mut tx, db.transaction()?).commit()?; + } + } + } + } + tx.commit()?; + Ok(held_bytes) + }); + let held_bytes = held_bytes?; + db.close()?; + let bytes = bytes_on_disk(&store)?; + let name = |what: &str| format!("anchors_{}_{what}", commands.len()); + line(&name("load"), seconds(load), "s"); + line(&name("rows"), rows, "rows"); + line(&name("events"), events, "rows"); + line(&name("store_size"), mebibytes(bytes), "MiB"); + line(&name("size_over_held"), ratio(bytes, held_bytes), "x"); + let keys = stored_keys(&store)?; + line("anchor_entry_longest", keys.entry, "bytes"); + line("anchor_entries_cut", keys.cut, "entries"); + line("key_longest_row_key", keys.row, "bytes"); + line("key_longest_count_key", keys.count, "bytes"); + + // 100 commands, evenly spread through the distinct ones: 50 longer than the 876 bytes the + // exact index keeps whole, and 50 not. + let distinct: BTreeSet<&String> = commands.iter().collect(); + let (long, short): (Vec<&String>, Vec<&String>) = + distinct.into_iter().partition(|c| c.len() > 876); + let spread = |of: &[&String]| { + (0..50) + .map(|n| of[n * of.len() / 50].clone()) + .collect::>() + }; + let asked: Vec = [spread(&long), spread(&short)] + .concat() + .into_iter() + .map(|value| params! { "c" => context, "v" => value }) + .collect(); + let reader = session(&store, READER_LIMIT)?; + let exact = + "SELECT id FROM event_anchors WHERE context_id = $c AND key = 'command' AND value = $v"; + let asked = reads(&|values| reader.execute(exact, values), &asked)?; + setting("anchor_exact", READER_LIMIT); + times("anchor_exact", &asked); + line( + "anchor_exact_rows_examined_p95", + percentile(&asked.examined, 0.95), + "rows", + ); + line( + "anchor_exact_rows_returned_p95", + percentile(&asked.evaluations, 0.95), + "rows", + ); + Ok(()) +} + +// --------------------------------------------------------------------------------------------- +// Synthetic: edges of a DAG, and a read narrowed by a grant + +fn dag(folder: &Path) -> Fallible<()> { + let db = declared_store(&folder.join("dag"), DatabaseOpenOptions::default())?; + let tx = db.transaction()?; + let context = Uuid::from_u128(1); + insert( + &tx, + "contexts", + ¶ms! { "id" => context, "name" => "graph", "created_at" => Value::TxId(tx.id()) }, + )?; + tx.commit()?; + let mut seed = 0x9E37_79B9_7F4A_7C15_u64; + let mut next = |below: u64| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed % below + }; + let (mut spent, mut tx) = (Vec::with_capacity(300_000), db.transaction()?); + // Edges in the order drawn, each once: a node cites a later node of its group of 100. + let (mut edges, mut drawn) = (Vec::with_capacity(300_000), BTreeSet::new()); + while edges.len() < 300_000 { + let source = next(100_000); + let room = 99 - source % 100; + let edge = (source, source + 1 + next(room.max(1))); + if room > 0 && drawn.insert(edge) { + edges.push(edge); + } + } + for (n, (source, target)) in edges.iter().enumerate() { + let mut row = params! { + "id" => Uuid::from_u128(n as u128 + 1), "source_id" => Uuid::from_u128(u128::from(*source) + (1 << 64)), + "target_id" => Uuid::from_u128(u128::from(*target) + (1 << 64)), "edge_type" => "CITES", "context_id" => context, + }; + row.insert("valid_from".to_owned(), Value::TxId(tx.id())); + let (done, time) = timed(|| insert(&tx, "edges", &row)); + done?; + spent.push(time); + if (n + 1) % PER_COMMIT == 0 { + std::mem::replace(&mut tx, db.transaction()?).commit()?; + } + } + tx.commit()?; + line( + "dag_insert_p95", + milliseconds(percentile(&spent, 0.95)), + "ms", + ); + Ok(()) +} + +fn grants(folder: &Path) -> Fallible<()> { + let db = Database::open(folder.join("grants"))?; + for sql in [ + "CREATE TABLE grants (id INTEGER PRIMARY KEY, principal_kind TEXT, principal_id TEXT, acl_id UUID)", + "CREATE INDEX grants_acl ON grants (acl_id)", + "CREATE TABLE guarded (id INTEGER PRIMARY KEY, context_id UUID CONTEXT_ID, acl_id UUID ACL REFERENCES grants(acl_id), body TEXT)", + "CREATE INDEX guarded_context ON guarded (context_id)", + ] { + db.execute(sql, ¶ms! {})?; + } + let mut tx = db.transaction()?; + for n in 0..100_000_i64 { + let row = params! { "id" => n, "context_id" => Uuid::from_u128((n % 10) as u128 + 1), "acl_id" => Uuid::from_u128((n % 20_000) as u128 + 1), "body" => "kept" }; + insert(&tx, "guarded", &row)?; + if n < 20_000 && n % 20 == 0 { + let grant = params! { "id" => n, "principal_kind" => "Agent", "principal_id" => "a", "acl_id" => Uuid::from_u128(n as u128 + 1) }; + insert(&tx, "grants", &grant)?; + } + if (n + 1) % PER_COMMIT as i64 == 0 { + std::mem::replace(&mut tx, db.transaction()?).commit()?; + } + } + tx.commit()?; + let context = Uuid::from_u128(1); + let narrowed = db.scoped_with_constraints( + Some(BTreeSet::from([ContextId::new(context)])), + None, + Some(Principal::Agent("a".to_owned())), + ); + let read = "SELECT id, body FROM guarded WHERE context_id = $c"; + let asked = vec![params! { "c" => context }; 20]; + for (name, handle) in [ + ("grants_read_narrowed", &narrowed), + ("grants_read_whole", &db), + ] { + let asked = reads(&|values| handle.execute(read, values), &asked)?; + read_lines(name, &asked); + line( + &format!("{name}_rows_returned"), + asked.evaluations[0], + "rows", + ); + } + Ok(()) +} + +fn main() -> Fallible<()> { + let arguments: Vec = std::env::args().skip(1).collect(); + let arguments: Vec<&str> = arguments.iter().map(String::as_str).collect(); + let folder = Path::new(arguments.last().copied().unwrap_or_default()); + must_be_empty(folder); + match arguments.as_slice() { + ["declarations", _] => declarations(folder), + ["declared", dump, answers, _] => declared(Path::new(dump), Path::new(answers), folder), + ["anchors", corpus, _] => anchors(Path::new(corpus), folder), + ["dag", _] => dag(folder), + ["grants", _] => grants(folder), + _ => { + eprintln!( + "usage: queries declarations|dag|grants \n queries declared \n queries anchors " + ); + std::process::exit(2); + } + } +} diff --git a/v2/contextdb/src/database/mod.rs b/v2/contextdb/src/database/mod.rs index b21f800e..1c929647 100644 --- a/v2/contextdb/src/database/mod.rs +++ b/v2/contextdb/src/database/mod.rs @@ -5,6 +5,7 @@ //! the memory limit and every step of a maintenance cycle each build a change set and hand it //! to the same function. +mod scoped; #[cfg(test)] mod tests; @@ -16,10 +17,11 @@ use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; use crate::memory::Budget; use crate::query::{self, QueryResult}; use crate::schema::{self, Catalog}; -use crate::sql::{self, Effect, Plan}; -use crate::store::{ChangeSet, Read, Store, View, WrapStorage}; +use crate::sql::ast::{BinOp, ColumnRef}; +use crate::sql::{self, Effect, Expr, Plan}; +use crate::store::{Cell, ChangeSet, Read, Store, View, WrapStorage}; use crate::system::{self, Clock}; -use crate::types::{Error, Result, TxId, Value}; +use crate::types::{Error, Result, TableMeta, TxId, Value}; use crate::upkeep::{self, MaintenanceReport}; use crate::vector::{self, Tally}; use crate::write; @@ -77,6 +79,8 @@ pub struct DatabaseOpenOptions { pub struct Database { /// The open store, `None` once [`Database::close`] has let go of it. open: Mutex>>, + /// Who asks through this handle: every statement of it reads and writes as this reader. + reader: query::Reader, } /// One store held for writing. A call holds it for as long as the call runs, so the folder is @@ -92,6 +96,8 @@ struct Open { /// What every statement and every transaction begins from. Held to copy it, to replace it /// and to read it again after a read that failed, never while a commit goes to the disk. newest: Mutex, + /// The highest number a transaction was given or a commit carries. + numbered: Mutex, /// The last field, so that the view above is dropped before the folder is let go. store: Store, } @@ -112,17 +118,26 @@ struct Staged { /// The tables as the transaction sees them: those of the moment it began at and those it /// has declared since. catalog: Arc, - /// The declarations it ran, in order, for a commit that has to run them again. - declared: Vec, + /// The declarations it ran, in order, each with the time it read, for a commit that has + /// to run them again. + declared: Vec<(Plan, i64)>, /// The vector partitions its statements have made and emptied, for the limit a statement /// is refused by. The commit function does not read it: it counts again. partitions: Tally, + /// The transaction's number; zero for one that runs no statement of a host. + number: TxId, } fn planned(sql: &str) -> Result { sql::plan(&sql::parse(sql)?) } +/// Refuses `statement`, by its name, to a handle that declared a narrowing. +fn admin(reader: &query::Reader, statement: Option<&str>) -> Result<()> { + let statement = statement.filter(|_| !reader.is_admin()).map(str::to_owned); + statement.map_or(Ok(()), |statement| Err(Error::AdminRequired { statement })) +} + /// Takes a lock. None of this part's locks is held across a step that can leave what the lock /// guards half changed, so a holder that panicked left it whole. fn locked(lock: &Mutex) -> MutexGuard<'_, T> { @@ -156,9 +171,17 @@ impl Open { catalog, declared: Vec::new(), partitions: Tally::new(&self.budget)?, + number: TxId(0), }) } + /// The next transaction's number: one past every number given out and every commit. + fn number(&self) -> TxId { + let mut last = locked(&self.numbered); + *last += 1; + TxId(*last) + } + /// Runs one planned statement inside a transaction. A refusal may leave part of the /// statement staged: the caller rewinds the change set or drops it. fn run( @@ -166,18 +189,26 @@ impl Open { plan: &Plan, params: &HashMap, staged: &mut Staged, + reader: &query::Reader, ) -> Result { // One time for the whole statement. let now = system::now_millis(&*self.clock); + admin(reader, plan.admin_only())?; match plan.effect() { Effect::Declares => { if staged.declared.is_empty() && !staged.changes.is_empty() { return Err(Error::MixedSchemaAndRowWrite); } - let catalog = self.declare(plan, &staged.catalog, &mut staged.changes)?; + let (catalog, dropped) = + self.declare(plan, &staged.catalog, &mut staged.changes, now)?; staged.catalog = Arc::new(catalog); - staged.declared.push(plan.clone()); - Ok(QueryResult::affected(0)) + staged.declared.push((plan.clone(), now)); + if dropped.is_empty() { + return Ok(QueryResult::affected(0)); + } + let dropped = dropped.into_iter().map(|index| vec![Value::Text(index)]); + let columns = vec!["dropped_indexes".to_owned()]; + Ok(QueryResult::from_rows(columns, dropped.collect())) } Effect::WritesRows => { if !staged.declared.is_empty() { @@ -185,23 +216,22 @@ impl Open { } let (catalog, changes) = (&staged.catalog, &mut staged.changes); let partitions = &mut staged.partitions; - write::apply( - plan, + let given = write::Given { params, catalog, - changes, - partitions, - &self.budget, + budget: &self.budget, + reader, now, - ) - .map(QueryResult::affected) + number: staged.number, + }; + write::apply(plan, &given, changes, partitions).map(QueryResult::affected) } Effect::Reads => { if let Plan::ShowMemoryLimit = plan { return Ok(query::memory_limit_shown(&self.budget, self.started_under)); } let (catalog, changes) = (&staged.catalog, &staged.changes); - query::run(plan, params, catalog, changes, &self.budget, now) + query::run(plan, params, catalog, changes, reader, &self.budget, now) } // A setting is not part of the transaction: it takes hold at once, in a commit of // its own, and a rollback does not undo it. @@ -211,16 +241,32 @@ impl Open { } } - /// Stages one declaration and, for a new index, its entries for the rows already there. - fn declare(&self, plan: &Plan, catalog: &Catalog, changes: &mut ChangeSet) -> Result { - let declared = schema::declare(plan, catalog, changes)?; + /// Stages one declaration, with the entries of each index and the records of each kept + /// count it makes for the rows already there. Returns the tables as they will be, and the + /// names of the indexes a `DROP COLUMN ... CASCADE` dropped. + fn declare( + &self, + plan: &Plan, + catalog: &Catalog, + changes: &mut ChangeSet, + now: i64, + ) -> Result<(Catalog, Vec)> { + let declared = schema::declare(plan, catalog, changes, now)?; + let catalog = declared.catalog; + write::fill_kept_count(&declared.counts, &catalog, changes, &self.budget)?; + let mut made = declared.indexes; if let Plan::CreateIndex(create) = plan - && let Some(table) = declared.table(&create.table) - && let Some(&index) = table.index_ids.last() + && let Some(table) = catalog.table(&create.table) { - write::fill_index(table, index, changes, &self.budget)?; + made.extend(table.index_ids.last()); } - Ok(declared) + for index in made { + let of = |table: &&schema::Table| table.index_ids.contains(&index); + if let Some(table) = catalog.tables().into_iter().find(of) { + write::fill_index(table, index, &catalog, changes, &self.budget)?; + } + } + Ok((catalog, declared.dropped)) } /// The one commit function: everything that changes the store is written here, one commit @@ -230,42 +276,56 @@ impl Open { /// its declarations again over the newest moment, because a table's and an index's number /// are counted on from the catalog and a new index is filled from the rows that are there; /// a refusal of that second run is the commit's refusal. One that wrote rows is laid over - /// the newest moment and `write` checks its rows against it. Then the vector part stages - /// what the rows mean for it, so a row and its place in a search commit together. + /// the newest moment and `write` checks its rows against it. Every row guard it took is + /// checked there too, whatever else it staged. Then the vector part stages + /// what the rows mean for it, so a row and its place in a search commit together. The + /// commit carries the transaction's number, or the next after the newest commit when that + /// is higher. fn commit(&self, _held: &MutexGuard<'_, ()>, staged: Staged) -> Result<()> { let Staged { mut changes, mut catalog, declared, partitions: _, + number, } = staged; let (view, tables) = self.newest()?; + let mut overtaken = None; if changes.commit() != view.commit() { + // What the declarations staged is taken back, to be staged again below. A rewind + // leaves the row guards. if !declared.is_empty() { - changes = ChangeSet::new(view, &self.budget)?; - catalog = tables; - for plan in &declared { - catalog = Arc::new(self.declare(plan, &catalog, &mut changes)?); - } - } else if !changes.is_empty() { - let before = changes.rebase(view); - // The rows were staged with the index entries of the tables as they were - // declared then; an index declared since would be left without them. - if before.schema_commit()? != changes.base().schema_commit()? { - return Err(Error::SchemaInvalid { - reason: "a table or an index was declared while this transaction was \ - open, so nothing of it was committed; run it again" - .to_owned(), - }); - } - write::check_at_commit(&changes, &catalog, &before)?; + changes.rewind(0); + } + let before = changes.rebase(view); + // The rows were staged with the index entries of the tables as they were declared + // then, and a guard holds the cells of its row as they lay then; an index or a + // column declared since would leave either wrong. + if !(changes.is_empty() && changes.guards().is_empty()) + && before.schema_commit()? != changes.base().schema_commit()? + { + return Err(Error::SchemaInvalid { + reason: "a table or an index was declared while this transaction was \ + open, so nothing of it was committed; run it again" + .to_owned(), + }); + } + catalog = tables; + for (plan, now) in &declared { + catalog = Arc::new(self.declare(plan, &catalog, &mut changes, *now)?.0); } + overtaken = Some(before); } + write::check_at_commit(&changes, &catalog, overtaken.as_ref(), &self.budget)?; if changes.is_empty() { return Ok(()); } vector::stage(&mut changes, &catalog, &self.budget)?; + changes.set_number(number); let commit = self.store.commit(changes)?; + let mut last = locked(&self.numbered); + *last = (*last).max(commit.0); + drop(last); // The commit is on disk whatever the read after it says. When that read fails no view // is kept and the next call reads again, so none answers from before this commit. let view = self.store.view().ok(); @@ -338,6 +398,7 @@ impl Database { view: Some(view), }; let open = Open { + numbered: Mutex::new(newest.commit.0), started_under: budget.limit(), budget, clock: options.clock.unwrap_or_else(system::machine_clock), @@ -347,6 +408,7 @@ impl Database { }; Ok(Database { open: Mutex::new(Some(Arc::new(open))), + reader: query::Reader::admin(), }) } @@ -361,6 +423,23 @@ impl Database { self.opened().unwrap_or_else(|closed| panic!("{closed}")) } + /// A table as it was declared, as of the newest commit: its columns, its UNIQUE rules, its + /// indexes and the options `TableMeta` has fields for. A primary key column is not nullable, however the key was written. + /// + /// # Errors + /// + /// [`crate::Error::TableNotFound`] when the store has no table of this name, and + /// [`crate::Error::DatabaseClosed`] on a handle that was closed. + pub fn table_meta(&self, name: &str) -> Result { + let open = self.opened()?; + let newest = locked(&open.newest); + let found = newest.catalog.table(name).map(|table| table.meta.clone()); + let mut meta = found.ok_or_else(|| Error::TableNotFound(name.to_owned()))?; + let key = meta.columns.iter_mut().filter(|column| column.primary_key); + key.for_each(|column| column.nullable = false); + Ok(meta) + } + /// Runs one SQL statement in a transaction of its own and returns its result. /// /// A statement that changes something is committed before this returns: on `Ok` the change @@ -387,12 +466,13 @@ impl Database { // Taken before the statement reads anything, so no other commit can overtake it. let held = locked(&open.committing); let mut staged = open.begin()?; - let result = open.run(&plan, params, &mut staged)?; + staged.number = open.number(); + let result = open.run(&plan, params, &mut staged, &self.reader)?; open.commit(&held, staged)?; Ok(result) } else { // A statement that stages nothing has nothing to commit. - open.run(&plan, params, &mut open.begin()?) + open.run(&plan, params, &mut open.begin()?, &self.reader) } } @@ -405,8 +485,11 @@ impl Database { /// [`crate::Error::StoreIo`] or [`crate::Error::StoreCorrupted`] when the store cannot be /// read. pub fn transaction(&self) -> Result> { + let open = self.opened()?; + let mut staged = open.begin()?; + staged.number = open.number(); Ok(Transaction { - staged: RefCell::new(self.opened()?.begin()?), + staged: RefCell::new(staged), db: self, }) } @@ -427,7 +510,9 @@ impl Database { /// opened. [`crate::Error::StoreIo`] when the commit cannot be written. In both cases the /// limit stays what it was. /// [`crate::Error::DatabaseClosed`] on a handle that was closed. + /// [`crate::Error::AdminRequired`] on a narrowed handle, as `SET MEMORY_LIMIT` is. pub fn set_memory_limit(&self, limit: Option) -> Result<()> { + admin(&self.reader, Some("SET MEMORY_LIMIT"))?; self.opened()? .set_memory_limit(limit.map(|limit| limit as u64)) } @@ -468,7 +553,7 @@ impl Database { /// A table's declaration as SQL text, as of the newest commit: the `CREATE TABLE` statement /// that would declare the table as it stands, with every clause left at its default not - /// written, then one `CREATE INDEX` statement for each index that was made with one. Each + /// written (except `MAX_PARTITIONS`, which is printed for every column with a `PARTITION_KEY`), then one `CREATE INDEX` statement for each index that was made with one. Each /// statement ends with `;` and a new line. /// /// # Errors @@ -483,9 +568,10 @@ impl Database { found.ok_or_else(|| Error::TableNotFound(table.to_owned())) } - /// The position of the newest commit on disk: 0 for a store nothing was ever committed to, - /// and one more for every commit since. A transaction that commits nothing does not move - /// it, and no position is ever reported for work that did not commit. + /// The number of the newest commit on disk: 0 for a store nothing was ever committed to, + /// and higher with every commit since, by one or by more: a commit carries its + /// transaction's number ([`Transaction::id`]). A transaction that commits nothing does not + /// move it, and no number is ever reported for work that did not commit. /// /// The position is for comparing two moments of one store. No call of this library takes /// one. @@ -513,7 +599,7 @@ impl Database { let open = self.opened()?; let (view, catalog) = open.newest()?; let now = system::now_millis(&*open.clock); - query::explain(&plan, &catalog, &view, &open.budget, now) + query::explain(&plan, &catalog, &view, &self.reader, &open.budget, now) } /// Runs one cycle of upkeep and returns what it did. @@ -637,7 +723,7 @@ impl Transaction<'_> { let open = self.db.opened()?; let staged = &mut *self.staged.borrow_mut(); let mark = staged.changes.mark(); - let result = open.run(&plan, params, staged); + let result = open.run(&plan, params, staged, &self.db.reader); if result.is_err() { staged.changes.rewind(mark); staged.partitions.forget(); @@ -655,7 +741,8 @@ impl Transaction<'_> { /// - [`crate::Error::UniqueViolation`] when the other commit took a key, or a value of a /// unique column, that this transaction inserted; /// - [`crate::Error::ConditionalUpdateConflict`] when the other commit changed or deleted a - /// row that this transaction changed or deleted; + /// row that this transaction changed or deleted, or a row it guarded no longer holds the + /// guarded values ([`Transaction::guard_row_conditions`]); /// - [`crate::Error::SchemaInvalid`] when a table or an index was declared meanwhile and /// this transaction wrote rows; /// - for a transaction that declared tables or indexes, whatever would refuse its @@ -672,6 +759,80 @@ impl Transaction<'_> { open.commit(&held, self.staged.into_inner()) } + /// The transaction's number, taken when it began: above every commit there was then and + /// above the number of every transaction begun before it. It is the highest value a `TXID` + /// column takes from this transaction; a higher one is refused with + /// [`crate::Error::TxIdOutOfRange`]. The commit carries this number, or the next after the + /// newest commit when another commit has taken this number or a higher one meanwhile. + pub fn id(&self) -> TxId { + self.staged.borrow().number + } + + /// Answers whether the row that `key_column = key_value` names is shown to the handle and + /// holds every `(column, value)` of `predicates` now, and if so asks the commit to stand + /// only if the row still holds them then: otherwise the commit is refused with + /// [`crate::Error::ConditionalUpdateConflict`]. Of several rows with that value it is the + /// first one found. The guard writes nothing, so a guarded row is not a changed row. + /// + /// # Errors + /// + /// [`crate::Error::TableNotFound`] and [`crate::Error::ColumnNotFound`] for a name that is + /// not declared, and whatever refuses a read of the table through this handle. + pub fn guard_row_conditions( + &self, + table: &str, + key_column: &str, + key_value: &Value, + predicates: &[(String, Value)], + ) -> Result { + let open = self.db.opened()?; + let staged = &mut *self.staged.borrow_mut(); + let catalog = staged.catalog.clone(); + let missing = || Error::TableNotFound(table.to_owned()); + let found = catalog.table(table).ok_or_else(missing)?; + let columns = &found.meta.columns; + let mut needed = vec![false; columns.len()]; + let mut held = Vec::new(); + for (name, value) in predicates { + let at = columns.iter().position(|column| column.name == *name); + let at = at.ok_or_else(|| Error::ColumnNotFound { + table: table.to_owned(), + column: name.clone(), + })?; + needed[at] = true; + held.push((at, value.clone())); + } + let filter = Expr::BinaryOp { + left: Box::new(Expr::Column(ColumnRef { + table: None, + column: key_column.to_owned(), + })), + op: BinOp::Eq, + right: Box::new(Expr::Parameter("key".to_owned())), + }; + let asked = query::Asked { + table: found, + filter: Some(&filter), + assigned: &[], + needed: &needed, + reader: &self.db.reader, + params: &HashMap::from([("key".to_owned(), key_value.clone())]), + now: 0, + }; + let rows = query::rows_for_write(&asked, None, &catalog, &staged.changes, &open.budget)?; + let Some((key, cells)) = rows.and_then(|(rows, _)| rows.into_iter().next()) else { + return Ok(false); + }; + if held + .iter() + .any(|(at, value)| !matches!(&cells[*at], Cell::Here(holds) if holds == value)) + { + return Ok(false); + } + staged.changes.guard_row(found.id, key, held)?; + Ok(true) + } + /// Rolls back now. What the transaction staged is held in memory only, so this drops it, /// writes nothing and cannot fail. Dropping the transaction does the same. pub fn rollback(self) {} diff --git a/v2/contextdb/src/database/scoped.rs b/v2/contextdb/src/database/scoped.rs new file mode 100644 index 00000000..7cc43533 --- /dev/null +++ b/v2/contextdb/src/database/scoped.rs @@ -0,0 +1,50 @@ +//! Handles narrowed to the rows one reader may see and write. + +use std::collections::BTreeSet; +use std::sync::Mutex; + +use super::{Database, locked}; +use crate::query::Reader; +use crate::types::{ContextId, Principal, ScopeLabel}; + +impl Database { + /// A handle over the same open store that is shown, and may write, only the rows these + /// Contexts, scope labels and principal may. `None` declares nothing for that rule. + /// + /// The new handle never has more than this one: of Contexts and of scope labels it keeps + /// those that both name, and a principal this handle has is kept whatever is asked. It + /// holds the store open like the handle it was made from, so the store is let go when each + /// of them is closed or dropped. Made from a closed handle, it is closed. + pub fn scoped_with_constraints( + &self, + contexts: Option>, + scope_labels: Option>, + principal: Option, + ) -> Database { + let held = &self.reader; + Database { + open: Mutex::new(locked(&self.open).clone()), + reader: Reader { + contexts: both(&held.contexts, contexts), + scope_labels: both(&held.scope_labels, scope_labels), + principal: held.principal.clone().or(principal), + }, + } + } + + /// [`Database::scoped_with_constraints`] with these Contexts and nothing else declared. + pub fn scoped_with_contexts(&self, contexts: BTreeSet) -> Database { + self.scoped_with_constraints(Some(contexts), None, None) + } +} + +/// What a handle that holds `held` is narrowed to when `asked` is asked of it. +fn both( + held: &Option>, + asked: Option>, +) -> Option> { + match (held, asked) { + (Some(held), Some(asked)) => Some(held.intersection(&asked).cloned().collect()), + (held, asked) => asked.or_else(|| held.clone()), + } +} diff --git a/v2/contextdb/src/lib.rs b/v2/contextdb/src/lib.rs index 0492c21a..609a246c 100644 --- a/v2/contextdb/src/lib.rs +++ b/v2/contextdb/src/lib.rs @@ -19,8 +19,8 @@ pub use read::{ReadSession, ReadSessionOptions}; pub use store::{StorageBackend, WrapStorage}; pub use system::Clock; pub use types::{ - ColumnType, Error, Result, TxId, Uuid, Value, VectorIndexRef, VectorPartitionDeclarationIssue, - VectorPolicyDeclarationIssue, + ColumnType, ContextId, Error, Principal, Result, ScopeLabel, TableMeta, TxId, Uuid, Value, + VectorIndexRef, VectorPartitionDeclarationIssue, VectorPolicyDeclarationIssue, }; pub use upkeep::{MaintenanceReport, VectorMaintenanceReport}; diff --git a/v2/contextdb/src/query/access.rs b/v2/contextdb/src/query/access.rs deleted file mode 100644 index 9410e380..00000000 --- a/v2/contextdb/src/query/access.rs +++ /dev/null @@ -1,456 +0,0 @@ -//! Index choice: which rows of a table a statement has to read, reading them, and working out -//! the select list for a row that was read. - -use std::collections::HashMap; -use std::ops::Bound; - -use super::expression::{column_index, eval, eval_found, most_bytes, reads}; -use super::{heap_bytes, row_overhead}; -use crate::memory::Reservation; -use crate::schema::Table; -use crate::sql::Expr; -use crate::sql::ast::{BinOp, SortDirection as Asked}; -use crate::sql::plan::{ProjectColumn, SortKey}; -use crate::store::{Cell, IndexId, Key, Read, Row}; -use crate::types::{ColumnType, Result, SortDirection, Value}; - -/// How the rows of one table are reached. -pub(super) enum Path { - /// The one row with this primary key. - Key(Key), - /// The table between two keys, in key order. - Rows(Bound, Bound), - /// The rows the entries of an index between two entries point at, in entry order. - Index(IndexId, Bound, Bound), -} - -/// A part of a filter that every kept row must pass, and that compares a column with a value -/// known before any row is read. -struct Term { - column: usize, - op: BinOp, - value: Value, -} - -/// Something that keeps the rows of a table in an order: the table itself, under its primary -/// key, or one of its indexes. -struct Ordered { - /// `None` for the table itself. - index: Option, - /// The columns it is ordered by, each with its direction. - columns: Vec<(usize, SortDirection)>, -} - -impl Ordered { - /// The key these leading values make: the bytes every key with these values begins with. - /// A row key is all ascending, which the store is told with no directions. - fn key(&self, values: &[Value]) -> Result { - let directions: Vec = match self.index { - Some(_) => self.columns[..values.len()].iter().map(|c| c.1).collect(), - None => Vec::new(), - }; - Key::from_values(values, &directions) - } -} - -/// The bound that ends a walk after the last key that begins with this one. -fn until_after(key: &Key) -> Bound { - key.prefix_end().map_or(Bound::Unbounded, Bound::Excluded) -} - -/// The bound that starts a walk after the last key that begins with this one. -fn starting_after(key: Key) -> Bound { - let after = key.prefix_end(); - after.map_or(Bound::Excluded(key), Bound::Included) -} - -/// The value as a key of a column of this type holds it, when there is one such value: only -/// then is looking the value up the same as comparing every row with it. -pub(super) fn as_key(column: ColumnType, value: Value) -> Option { - match (column, value) { - (ColumnType::Integer, value @ Value::Int64(_)) - | (ColumnType::Real, value @ Value::Float64(_)) - | (ColumnType::Text, value @ Value::Text(_)) - | (ColumnType::Boolean, value @ Value::Bool(_)) - | (ColumnType::Uuid, value @ Value::Uuid(_)) - | (ColumnType::Timestamp, value @ Value::Timestamp(_)) => Some(value), - (ColumnType::Real, Value::Int64(value)) => Some(Value::Float64(value as f64)), - (ColumnType::Uuid, Value::Text(value)) => value.parse().ok().map(Value::Uuid), - _ => None, - } -} - -/// One table as one statement reads it. -pub(super) struct Source<'a> { - table: &'a Table, - filter: Option<&'a Expr>, - params: &'a HashMap, - now: i64, - read: &'a dyn Read, - /// The columns the filter reads: taken from every row that is read. - filtered: Vec, - /// The other columns the statement reads: taken only from a row that passed the filter. - rest: Vec, -} - -impl<'a> Source<'a> { - /// `filtered` marks the columns the filter reads and `wanted` every column the statement - /// reads after it. A column in neither is never taken from a row, so a large value in it - /// is never loaded. - pub(super) fn new( - table: &'a Table, - filter: Option<&'a Expr>, - params: &'a HashMap, - now: i64, - read: &'a dyn Read, - filtered: Vec, - wanted: &[bool], - ) -> Source<'a> { - let rest = wanted.iter().zip(&filtered).map(|(w, f)| *w && !*f); - Source { - table, - filter, - params, - now, - read, - rest: rest.collect(), - filtered, - } - } - - /// Picks the path that reads the fewest rows for the filter, and says whether its rows - /// arrive in the order asked for, so that they need no sort. - /// - /// The table and each index are tried in turn. One is better the more of its leading - /// columns the filter gives a value for, then if the filter gives a range of its next - /// column, then if it keeps the order asked for; the table itself wins a tie. The filter is - /// still applied to every row read, so a path only has to reach every row that can pass. - /// - /// Rows are kept under their primary key, so the key is answered from the table itself. Its - /// own index, `__pk_`, is declared and has no entries, and is never walked. - pub(super) fn choose(&self, order: &[SortKey]) -> Result<(Path, bool)> { - let mut terms = Vec::new(); - if let Some(filter) = self.filter { - self.collect(filter, &mut terms)?; - } - let columns = &self.table.meta.columns; - let column_named = |name: &String| columns.iter().position(|column| column.name == *name); - let key = self.key_columns(); - // The order asked for, when it is plain columns. - let asked: Option> = order - .iter() - .map(|key| { - let Expr::Column(column) = &key.expr else { - return None; - }; - let direction = match key.direction { - Asked::Asc => SortDirection::Asc, - Asked::Desc => SortDirection::Desc, - }; - Some((column_index(self.table, column).ok()?, direction)) - }) - .collect(); - - let the_table = Ordered { - index: None, - columns: key.iter().map(|at| (*at, SortDirection::Asc)).collect(), - }; - let indexes = self.table.meta.indexes.iter().zip(&self.table.index_ids); - let indexes = indexes - .filter(|(index, _)| !index.name.starts_with("__pk_")) - .map(|(index, id)| Ordered { - index: Some(*id), - columns: index - .columns - .iter() - .filter_map(|(name, direction)| Some((column_named(name)?, *direction))) - .collect(), - }); - - // Until something better is found: the whole table, in no order that was asked for. - let mut best = ( - (0, false, false), - Path::Rows(Bound::Unbounded, Bound::Unbounded), - ); - for ordered in std::iter::once(the_table).chain(indexes) { - let given: Vec = ordered - .columns - .iter() - .map_while(|(column, _)| { - let equal = |term: &&Term| term.column == *column && term.op == BinOp::Eq; - Some(terms.iter().find(equal)?.value.clone()) - }) - .collect(); - let next = &ordered.columns[given.len()..]; - if ordered.index.is_none() && next.is_empty() { - return Ok((Path::Key(ordered.key(&given)?), true)); - } - let (mut from, mut to) = (Bound::Unbounded, Bound::Unbounded); - if !given.is_empty() { - let all = ordered.key(&given)?; - to = until_after(&all); - from = Bound::Included(all); - } - let mut ranged = false; - if let Some((column, direction)) = next.first() { - let with = |value: &Value| { - let mut values = given.clone(); - values.push(value.clone()); - ordered.key(&values) - }; - let ranges = terms.iter().filter(|term| { - let range = matches!(term.op, BinOp::Gt | BinOp::Gte | BinOp::Lt | BinOp::Lte); - term.column == *column && range - }); - for term in ranges { - if !ranged { - // No NULL is in a range. A key keeps NULL after every value going up - // and before every value going down, and the walk stops short of it. - let null = with(&Value::Null)?; - match direction { - SortDirection::Asc => to = Bound::Excluded(null), - SortDirection::Desc => from = starting_after(null), - } - ranged = true; - } - let key = with(&term.value)?; - // A descending column keeps its greater values first. When more columns - // follow, many keys begin with this one, and "after it" is after them all. - match (term.op, direction) { - (BinOp::Gt, SortDirection::Asc) | (BinOp::Lt, SortDirection::Desc) => { - from = starting_after(key); - } - (BinOp::Gte, SortDirection::Asc) | (BinOp::Lte, SortDirection::Desc) => { - from = Bound::Included(key); - } - (BinOp::Lt, SortDirection::Asc) | (BinOp::Gt, SortDirection::Desc) => { - to = Bound::Excluded(key); - } - // `<=` going up and `>=` going down. - _ => to = until_after(&key), - } - } - } - // A key keeps its columns as ORDER BY sorts them, NULL included, so what follows - // the columns the filter fixed is in order if it is what was asked for. - let in_order = asked.as_ref().is_some_and(|asked| next.starts_with(asked)); - let score = (given.len(), ranged, in_order); - if score > best.0 { - let path = match ordered.index { - Some(index) => Path::Index(index, from, to), - None => Path::Rows(from, to), - }; - best = (score, path); - } - } - let ((_, _, in_order), path) = best; - Ok((path, in_order)) - } - - /// Where the columns of the primary key sit in a row. - pub(super) fn key_columns(&self) -> Vec { - let columns = &self.table.meta.columns; - (0..columns.len()) - .filter(|at| columns[*at].primary_key) - .collect() - } - - /// Gathers the terms of a filter: the comparisons joined by AND at its top. - fn collect(&self, filter: &Expr, terms: &mut Vec) -> Result<()> { - let Expr::BinaryOp { left, op, right } = filter else { - return Ok(()); - }; - if *op == BinOp::And { - self.collect(left, terms)?; - return self.collect(right, terms); - } - let (column, value, op) = match (&**left, &**right) { - (Expr::Column(column), value @ (Expr::Literal(_) | Expr::Parameter(_))) => { - (column, value, *op) - } - // `5 < id` is `id > 5`. - (value @ (Expr::Literal(_) | Expr::Parameter(_)), Expr::Column(column)) => { - let turned = match op { - BinOp::Lt => BinOp::Gt, - BinOp::Lte => BinOp::Gte, - BinOp::Gt => BinOp::Lt, - BinOp::Gte => BinOp::Lte, - other => *other, - }; - (column, value, turned) - } - _ => return Ok(()), - }; - let column = column_index(self.table, column)?; - let value = eval(value, self.table, &[], self.params, self.now)?; - if let Some(value) = as_key(self.table.meta.columns[column].column_type, value) { - terms.push(Term { column, op, value }); - } - Ok(()) - } - - /// Reads the rows the path reaches and hands `each` every one that passes the filter: its - /// key, its values, and the bytes of `reservation` it holds. Stops when `each` answers - /// false. Returns how many stored rows were read. - pub(super) fn walk( - &self, - path: &Path, - reservation: &mut Reservation, - mut each: impl FnMut(Key, Vec, u64, &mut Reservation) -> Result, - ) -> Result { - let mut examined = 0; - let mut offer = |key: Key, row: Row| { - examined += 1; - match self.values(row, reservation)? { - Some((values, bytes)) => each(key, values, bytes, reservation), - None => Ok(true), - } - }; - let table = self.table.id; - match path { - Path::Key(key) => { - if let Some(row) = self.read.row(table, key)? { - offer(key.clone(), row)?; - } - } - Path::Rows(from, to) => { - for item in self.read.rows(table, from.as_ref(), to.as_ref())? { - let (key, row) = item?; - if !offer(key, row)? { - break; - } - } - } - Path::Index(index, from, to) => { - for entry in self.read.index(*index, from.as_ref(), to.as_ref())? { - let (_, key) = entry?; - if let Some(row) = self.read.row(table, &key)? - && !offer(key, row)? - { - break; - } - } - } - } - Ok(examined) - } - - /// The row with this key if it is there and passes the filter, as [`Source::walk`] hands a - /// row over: its values, and the bytes of `reservation` it holds. - pub(super) fn row( - &self, - key: &Key, - reservation: &mut Reservation, - ) -> Result, u64)>> { - match self.read.row(self.table.id, key)? { - Some(row) => self.values(row, reservation), - None => Ok(None), - } - } - - /// For each item of a select list, the column whose value it moves out of a row: the column - /// it names, when no other item of the list reads that column. Any other item copies what - /// it reads. Empty for `SELECT *`, which hands the row over as it is. - pub(super) fn moved(&self, columns: Option<&[ProjectColumn]>) -> Result>> { - let columns = columns.unwrap_or_default(); - let mut readers = vec![0; self.table.meta.columns.len()]; - for column in columns { - let mut read = vec![false; readers.len()]; - reads(&column.expr, self.table, self.params, &mut read)?; - for (readers, read) in readers.iter_mut().zip(read) { - *readers += usize::from(read); - } - } - let alone = |column: &ProjectColumn| match &column.expr { - Expr::Column(named) => column_index(self.table, named).ok(), - _ => None, - }; - let moved = columns.iter().map(alone); - Ok(moved.map(|at| at.filter(|at| readers[*at] == 1)).collect()) - } - - /// The select list worked out for one row this source handed over, or the row itself for - /// `SELECT *`. `moved` is [`Source::moved`] of the list, and `distance` is the row's distance - /// when a similarity search found it. The memory of the values is asked for before they - /// are made, and the row's bytes are given back once they are. A value that is moved is - /// held once: what it takes passes from the row to the result and is not asked for again. - pub(super) fn output( - &self, - columns: Option<&[ProjectColumn]>, - moved: &[Option], - mut row: Vec, - bytes: u64, - distance: Option, - reservation: &mut Reservation, - ) -> Result> { - let Some(columns) = columns else { - return Ok(row); - }; - let most = columns - .iter() - .map(|column| most_bytes(&column.expr, self.table, &row, self.params)); - let passed = moved.iter().flatten().map(|at| heap_bytes(&row[*at])); - let passed = passed.sum::().min(bytes); - reservation.grow(row_overhead(columns.len()) + most.sum::() - passed)?; - let values = columns - .iter() - .zip(moved) - .map(|(column, moved)| match moved { - Some(at) => Ok(std::mem::replace(&mut row[*at], Value::Null)), - None => { - let (params, now) = (self.params, self.now); - eval_found(&column.expr, self.table, &row, distance, params, now) - } - }) - .collect::>>()?; - reservation.shrink(bytes - passed); - Ok(values) - } - - /// The values of a stored row if it passes the filter, and the bytes of `reservation` they - /// hold; a row that does not pass holds none. A column the statement does not read is left - /// as NULL. - fn values(&self, row: Row, reservation: &mut Reservation) -> Result, u64)>> { - let before = reservation.bytes(); - let mut cells = row.cells; - reservation.grow(row_overhead(cells.len()))?; - let mut values = vec![Value::Null; cells.len()]; - self.take(&mut cells, &mut values, &self.filtered, reservation)?; - if let Some(filter) = self.filter - && eval(filter, self.table, &values, self.params, self.now)? != Value::Bool(true) - { - reservation.shrink(reservation.bytes() - before); - return Ok(None); - } - self.take(&mut cells, &mut values, &self.rest, reservation)?; - Ok(Some((values, reservation.bytes() - before))) - } - - /// Moves the marked cells into `values`, loading a value kept beside its row. The memory - /// of the values is asked for before any is moved or loaded. - fn take( - &self, - cells: &mut [Cell], - values: &mut [Value], - marked: &[bool], - reservation: &mut Reservation, - ) -> Result<()> { - let here = cells.iter().zip(marked).map(|(cell, marked)| match cell { - Cell::Here(here) if *marked => heap_bytes(here), - _ => 0, - }); - reservation.grow(here.sum())?; - for ((cell, value), _) in cells - .iter_mut() - .zip(values) - .zip(marked) - .filter(|(_, marked)| **marked) - { - *value = match std::mem::replace(cell, Cell::Here(Value::Null)) { - Cell::Here(here) => here, - Cell::Beside(beside) => self.read.load(&beside, reservation)?, - }; - } - Ok(()) - } -} diff --git a/v2/contextdb/src/query/bind.rs b/v2/contextdb/src/query/bind.rs new file mode 100644 index 00000000..0f83e463 --- /dev/null +++ b/v2/contextdb/src/query/bind.rs @@ -0,0 +1,125 @@ +//! Binding: every name of an expression resolved and every type known before a row is read, so +//! that whether a statement is refused never depends on the rows. + +use std::collections::HashMap; + +use super::expression::{Scalar, Scope, refused, scalar, unbound}; +use crate::sql::Expr; +use crate::sql::ast::{BinOp, Literal, UnaryOp}; +use crate::types::{ColumnType, Error, Result, Value}; + +/// Refuses what no row could answer: a column no table in reach has, a `$name` nothing is bound +/// to, a function this version does not work out, and a comparison between two types that +/// neither compare nor convert. +pub(crate) fn bind(expr: &Expr, scope: &Scope, params: &HashMap) -> Result<()> { + kind(expr, scope, params).map(|_| ()) +} + +/// The type of an expression's value, or `None` where it is not known before a row is read: +/// NULL, a row value, and arithmetic over either. +fn kind(expr: &Expr, scope: &Scope, params: &HashMap) -> Result> { + let kind = |expr: &Expr| kind(expr, scope, params); + // The types of one side of IN: those of a row value, or the one type of anything else. + let side = |expr: &Expr| -> Result> { + match expr { + Expr::Row(items) => items.iter().map(kind).collect(), + one => Ok(vec![kind(one)?]), + } + }; + Ok(match expr { + Expr::Column(column) => Some(scope.find(column)?.1), + Expr::Literal(literal) => match literal { + Literal::Null => None, + Literal::Bool(_) => Some(ColumnType::Boolean), + Literal::Integer(_) => Some(ColumnType::Integer), + Literal::Real(_) => Some(ColumnType::Real), + Literal::Text(_) => Some(ColumnType::Text), + Literal::Vector(vector) => Some(ColumnType::Vector(vector.len())), + }, + Expr::Parameter(name) => match params.get(name).ok_or_else(|| unbound(name))? { + Value::Null => None, + Value::Bool(_) => Some(ColumnType::Boolean), + Value::Int64(_) => Some(ColumnType::Integer), + Value::Float64(_) => Some(ColumnType::Real), + Value::Text(_) => Some(ColumnType::Text), + Value::Uuid(_) => Some(ColumnType::Uuid), + Value::Timestamp(_) => Some(ColumnType::Timestamp), + Value::Json(_) => Some(ColumnType::Json), + Value::Vector(vector) => Some(ColumnType::Vector(vector.len())), + Value::TxId(_) => Some(ColumnType::TxId), + }, + Expr::BinaryOp { left, op, right } => { + let (left, right) = (kind(left)?, kind(right)?); + if !matches!(op, BinOp::And | BinOp::Or) { + comparable(left, right)?; + } + Some(ColumnType::Boolean) + } + Expr::UnaryOp { op, operand } => match (op, kind(operand)?) { + (UnaryOp::Not, _) => Some(ColumnType::Boolean), + (UnaryOp::Neg, operand) => operand, + }, + Expr::IsNull { expr, .. } => { + kind(expr)?; + Some(ColumnType::Boolean) + } + Expr::FunctionCall { name, args } => { + let scalar = scalar(name).ok_or_else(|| refused(name))?; + let args = args.iter().map(kind).collect::>>()?; + match scalar { + Scalar::Arithmetic(_) if args.contains(&Some(ColumnType::Real)) => { + Some(ColumnType::Real) + } + Scalar::Arithmetic(_) => { + let whole = args.iter().all(|arg| *arg == Some(ColumnType::Integer)); + whole.then_some(ColumnType::Integer) + } + Scalar::Now => Some(ColumnType::Timestamp), + Scalar::Coalesce => args.into_iter().flatten().next(), + } + } + Expr::InList { expr, list, .. } => { + let left = side(expr)?; + for item in list { + for (left, item) in left.iter().zip(side(item)?) { + comparable(*left, item)?; + } + } + Some(ColumnType::Boolean) + } + Expr::Row(items) => { + for item in items { + kind(item)?; + } + None + } + Expr::Distance { left, right } => { + kind(left)?; + kind(right)?; + Some(ColumnType::Real) + } + }) +} + +/// Whether values of the two types compare: the same type, or one that converts to the other +/// as a write converts it (quoted text to a UUID, TIMESTAMP, JSON or VECTOR, an INTEGER to a +/// REAL). A type that is not known compares with any. +fn comparable(left: Option, right: Option) -> Result<()> { + use ColumnType::{Integer, Json, Real, Text, Timestamp, Uuid, Vector}; + let (Some(left), Some(right)) = (left, right) else { + return Ok(()); + }; + let converts = |from, to| { + matches!( + (from, to), + (Integer, Real) | (Text, Uuid | Timestamp | Json | Vector(_)) + ) + }; + let same = std::mem::discriminant(&left) == std::mem::discriminant(&right); + if same || converts(left, right) || converts(right, left) { + return Ok(()); + } + Err(Error::PlanError(format!( + "cannot compare {left} with {right}" + ))) +} diff --git a/v2/contextdb/src/query/counts.rs b/v2/contextdb/src/query/counts.rs new file mode 100644 index 00000000..590e20ab --- /dev/null +++ b/v2/contextdb/src/query/counts.rs @@ -0,0 +1,103 @@ +//! A `COUNT(*)` answered with no row read. + +use std::ops::Bound; + +use super::expression::EvalContext; +use super::table::collect; +use super::visible::Reader; +use crate::schema::Table; +use crate::sql::Expr; +use crate::sql::ast::BinOp; +use crate::store::{Key, Read}; +use crate::types::{Result, Value}; + +/// The number of rows of `table` that pass `filter`, when it is known with no row read: the +/// table's own count with no filter; one kept number when the filter gives one value for each +/// column of a kept count; otherwise the number of entries of the one index range the filter +/// gives, when it gives one value for each of an index's leading columns and nothing else. +/// +/// `None` when the rows have to be read: for a reader that is not shown every row, for a +/// filter that says more, for a table the transaction has staged a row of, and for a value too +/// long for its index entry, which is there as a digest. +pub(crate) fn count_without_rows( + table: &Table, + filter: Option<&Expr>, + reader: &Reader, + ctx: &EvalContext, + read: &dyn Read, +) -> Result> { + if !reader.is_admin() { + return Ok(None); + } + let Some(filter) = filter else { + return read.count(table.id).map(Some); + }; + let mut terms = Vec::new(); + if !collect(table, filter, ctx, &mut terms)? { + return Ok(None); + } + // Each column the filter names is given one value, once. + let mut pins: Vec<(usize, &Value)> = Vec::new(); + for term in &terms { + let [value] = term.values.as_slice() else { + return Ok(None); + }; + if term.op != BinOp::Eq || pins.iter().any(|pin| pin.0 == term.column) { + return Ok(None); + } + pins.push((term.column, value)); + } + let mut staged = false; + read.staged(table.id, &mut |_| { + staged = true; + Ok(false) + })?; + if staged { + return Ok(None); + } + // The values given for these columns, when every one of them is given. + let given = |columns: &mut dyn Iterator| -> Option> { + let value = |column| { + pins.iter() + .find(|pin| pin.0 == column) + .map(|pin| pin.1.clone()) + }; + columns.map(value).collect() + }; + + for count in &table.rules.counts { + let values = given(&mut count.columns.iter().map(|column| column.0)); + if let Some(values) = values.filter(|values| values.len() == pins.len()) { + let directions: Vec<_> = count.columns.iter().map(|column| column.1).collect(); + let key = Key::from_values(&values, &directions)?; + let mut kept = read.kept(count.id, Bound::Included(&key), Bound::Included(&key))?; + // A group with no row has no record. + return Ok(Some(kept.next().transpose()?.map_or(0, |(_, rows)| rows))); + } + } + let columns = &table.meta.columns; + for (at, index) in table.meta.indexes.iter().enumerate() { + let leading = index.columns.iter().take(pins.len()); + let mut leading = + leading.filter_map(|(name, _)| columns.iter().position(|c| c.name == *name)); + // The key's own index has no entries, and an index over a JSON path none of a column. + let holds = !index.name.starts_with("__pk_") && index.json_path.is_empty(); + if let Some(values) = + given(&mut leading).filter(|values| holds && values.len() == pins.len()) + { + let entry = table.entry_shape(at).entry(&values)?; + if entry.cut.is_some() { + return Ok(None); + } + let to = entry.key.prefix_end(); + let to = to.as_ref().map_or(Bound::Unbounded, Bound::Excluded); + let mut entries = 0; + for found in read.index(table.index_ids[at], Bound::Included(&entry.key), to)? { + found?; + entries += 1; + } + return Ok(Some(entries)); + } + } + Ok(None) +} diff --git a/v2/contextdb/src/query/expression.rs b/v2/contextdb/src/query/expression.rs index a22d7dff..66e985ef 100644 --- a/v2/contextdb/src/query/expression.rs +++ b/v2/contextdb/src/query/expression.rs @@ -3,46 +3,108 @@ use std::cmp::Ordering; use std::collections::HashMap; -use super::heap_bytes; +use super::bind::bind; use crate::schema::Table; use crate::sql::Expr; use crate::sql::ast::{BinOp, ColumnRef, Literal, UnaryOp}; -use crate::types::{Error, Result, Uuid, Value}; +use crate::types::{ + ColumnDef, ColumnType, Error, Result, Uuid, Value, heap_bytes, timestamp_millis, +}; + +#[cfg(test)] +mod tests; + +/// The columns an expression has in reach: for each table of the statement, in the order their +/// values stand in a row, the name that qualifies its columns and the columns themselves. +#[derive(Default)] +pub(crate) struct Scope<'a> { + relations: Vec<(&'a str, &'a [ColumnDef])>, +} + +impl<'a> Scope<'a> { + /// The scope of one table's columns. + pub(crate) fn of(qualifier: &'a str, columns: &'a [ColumnDef]) -> Self { + Scope { + relations: vec![(qualifier, columns)], + } + } + + /// Where a column sits in a row, and its type. A name no table in reach has is + /// `ColumnNotFound`, and so is a name qualified with anything but one of the tables: that + /// refusal names the qualifier as it was written. + pub(super) fn find(&self, column: &ColumnRef) -> Result<(usize, ColumnType)> { + let mut before = 0; + for (qualifier, columns) in &self.relations { + let named = |declared: &ColumnDef| declared.name == column.column; + if column + .table + .as_deref() + .is_none_or(|table| table == *qualifier) + && let Some(at) = columns.iter().position(named) + { + return Ok((before + at, columns[at].column_type)); + } + before += columns.len(); + } + Err(self.missing(column)) + } + + /// The refusal of a column that is not in reach, under the qualifier it was written with or + /// else the first table. + fn missing(&self, column: &ColumnRef) -> Error { + let first = self.relations.first().map(|(qualifier, _)| *qualifier); + Error::ColumnNotFound { + table: column + .table + .as_deref() + .or(first) + .unwrap_or_default() + .to_owned(), + column: column.column.clone(), + } + } +} + +/// What an expression is worked out with beside its row: the values the statement bound, the +/// statement's one time in milliseconds, which `NOW()` answers with, and the row's distance +/// when a similarity search found it. +pub(crate) struct EvalContext<'a> { + params: &'a HashMap, + now: i64, + distance: Option, +} + +impl<'a> EvalContext<'a> { + pub(crate) fn new(params: &'a HashMap, now: i64) -> Self { + EvalContext { + params, + now, + distance: None, + } + } + + /// The context for a row a similarity search found at this distance: every `<=>` in an + /// expression is that number as it is. + pub(super) fn found(self, distance: Option) -> Self { + EvalContext { distance, ..self } + } +} /// Works out one expression against one row. The only expression evaluator in the library. /// -/// `row` holds one value per column of `table`, in declared order, and `now` is the statement's -/// one time in milliseconds, which `NOW()` answers with. A comparison with NULL, and one between -/// values of types that do not compare, is unknown: it gives NULL, and a filter keeps a row +/// `row` holds one value per column of `scope`, in its order. A comparison with NULL, and one +/// between values that do not compare, is unknown: it gives NULL, and a filter keeps a row /// only for TRUE. /// -/// A distance, `column <=> vector`, has no value here: only a similarity search gives a row one -/// ([`eval_found`]), so it is refused by name. -pub(crate) fn eval( - expr: &Expr, - table: &Table, - row: &[Value], - params: &HashMap, - now: i64, -) -> Result { - eval_found(expr, table, row, None, params, now) -} - -/// [`eval`] for a row a similarity search found. `distance` is the distance the search worked -/// out for the row, and every `<=>` in the expression is that number as it is. -pub(super) fn eval_found( - expr: &Expr, - table: &Table, - row: &[Value], - distance: Option, - params: &HashMap, - now: i64, -) -> Result { - let value = |expr: &Expr| eval_found(expr, table, row, distance, params, now); +/// A distance, `column <=> vector`, has no value of its own here: only a similarity search +/// gives a row one, through the context, so without one it is refused by name. +pub(crate) fn eval(expr: &Expr, scope: &Scope, row: &[Value], ctx: &EvalContext) -> Result { + let value = |expr: &Expr| eval(expr, scope, row, ctx); Ok(match expr { Expr::Column(column) => { - let at = column_index(table, column)?; - row.get(at).cloned().ok_or_else(|| missing(table, column))? + let (at, _) = scope.find(column)?; + let value = row.get(at).cloned(); + value.ok_or_else(|| scope.missing(column))? } Expr::Literal(literal) => match literal { Literal::Null => Value::Null, @@ -52,7 +114,7 @@ pub(super) fn eval_found( Literal::Text(value) => Value::Text(value.clone()), Literal::Vector(value) => Value::Vector(value.clone()), }, - Expr::Parameter(name) => params.get(name).cloned().ok_or_else(|| unbound(name))?, + Expr::Parameter(name) => ctx.params.get(name).cloned().ok_or_else(|| unbound(name))?, Expr::BinaryOp { left, op, right } => { let holds: fn(Ordering) -> bool = match op { BinOp::And | BinOp::Or => { @@ -96,7 +158,7 @@ pub(super) fn eval_found( (Some(Scalar::Arithmetic(op)), [left, right]) => { arithmetic(op, &value(left)?, &value(right)?)? } - (Some(Scalar::Now), []) => Value::Timestamp(now), + (Some(Scalar::Now), []) => Value::Timestamp(ctx.now), (Some(Scalar::Coalesce), args) => { for arg in args { let first = value(arg)?; @@ -139,7 +201,7 @@ pub(super) fn eval_found( "a row value, (a, b), stands only on either side of IN".to_owned(), )); } - Expr::Distance { .. } => Value::Float64(distance.ok_or_else(|| { + Expr::Distance { .. } => Value::Float64(ctx.distance.ok_or_else(|| { Error::PlanError( "<=> has a value only for the rows of a similarity search, SELECT ... ORDER BY \ column <=> vector LIMIT k, and in that statement's select list" @@ -176,20 +238,20 @@ fn rows_equal(left: &[Value], right: &[Value]) -> Option { /// before the expression is worked out, so the memory is asked for before the value is made. pub(super) fn most_bytes( expr: &Expr, - table: &Table, + scope: &Scope, row: &[Value], params: &HashMap, ) -> u64 { match expr { Expr::Column(column) => { - let at = column_index(table, column).ok(); - at.and_then(|at| row.get(at)).map_or(0, heap_bytes) + let at = scope.find(column).ok(); + at.and_then(|(at, _)| row.get(at)).map_or(0, heap_bytes) } Expr::Literal(Literal::Text(text)) => text.len() as u64, Expr::Literal(Literal::Vector(vector)) => size_of_val(vector.as_slice()) as u64, Expr::Parameter(name) => params.get(name).map_or(0, heap_bytes), Expr::FunctionCall { name, args } if matches!(scalar(name), Some(Scalar::Coalesce)) => { - let each = args.iter().map(|arg| most_bytes(arg, table, row, params)); + let each = args.iter().map(|arg| most_bytes(arg, scope, row, params)); each.max().unwrap_or(0) } // Everything else gives a number, a truth, a time or NULL. @@ -197,82 +259,50 @@ pub(super) fn most_bytes( } } -/// Marks in `read` each column the expression reads. -/// -/// Refuses, before any row is read, what [`eval`] would refuse at the first row: a column the -/// table does not have, a `$name` nothing is bound to, and a function this version does not -/// work out. A statement is therefore refused or not whatever rows its table holds. A distance -/// and a row value are walked and not judged: the planner refuses the ones a SELECT cannot -/// hold, and [`eval`] any other when it meets it. -pub(super) fn reads( +/// Binds the expression against the table, then marks in `read` each column it reads. The +/// column on the left of a distance is not marked: the search gives a row's distance. +pub(crate) fn reads( expr: &Expr, table: &Table, params: &HashMap, read: &mut [bool], ) -> Result<()> { + let scope = table.scope(); + bind(expr, &scope, params)?; + mark(expr, &scope, read); + Ok(()) +} + +fn mark(expr: &Expr, scope: &Scope, read: &mut [bool]) { + let mut all = |items: &[Expr]| items.iter().for_each(|item| mark(item, scope, read)); match expr { - Expr::Column(column) => read[column_index(table, column)?] = true, - Expr::Literal(_) => {} - Expr::Parameter(name) => { - if !params.contains_key(name) { - return Err(unbound(name)); + Expr::Column(column) => { + if let Ok((at, _)) = scope.find(column) { + read[at] = true; } } + Expr::Literal(_) | Expr::Parameter(_) => {} Expr::BinaryOp { left, right, .. } => { - reads(left, table, params, read)?; - reads(right, table, params, read)?; - } - Expr::UnaryOp { operand, .. } | Expr::IsNull { expr: operand, .. } => { - reads(operand, table, params, read)?; - } - Expr::FunctionCall { name, args } => { - if scalar(name).is_none() { - return Err(refused(name)); - } - for arg in args { - reads(arg, table, params, read)?; - } - } - // The search gives a row's distance: the column on the left is looked up, not read. - Expr::Distance { left, right } => { - reads(left, table, params, &mut read.to_vec())?; - reads(right, table, params, read)?; - } - Expr::Row(items) => { - for item in items { - reads(item, table, params, read)?; - } + mark(left, scope, read); + mark(right, scope, read); } + Expr::UnaryOp { operand, .. } + | Expr::IsNull { expr: operand, .. } + | Expr::Distance { right: operand, .. } => mark(operand, scope, read), + Expr::FunctionCall { args: items, .. } | Expr::Row(items) => all(items), Expr::InList { expr, list, .. } => { - reads(expr, table, params, read)?; - for item in list { - reads(item, table, params, read)?; - } + all(list); + mark(expr, scope, read); } } - Ok(()) } -/// Where a column sits in a row of its table. A name the table does not have, and a name -/// qualified with anything but the table's own name, is `ColumnNotFound`. +/// Where a column sits in a row of its table. pub(super) fn column_index(table: &Table, column: &ColumnRef) -> Result { - let in_scope = column.table.as_ref().is_none_or(|name| *name == table.name); - table - .meta - .columns - .iter() - .position(|declared| in_scope && declared.name == column.column) - .ok_or_else(|| missing(table, column)) -} - -fn missing(table: &Table, column: &ColumnRef) -> Error { - Error::ColumnNotFound { - table: table.name.clone(), - column: column.column.clone(), - } + Ok(table.scope().find(column)?.0) } -fn unbound(name: &str) -> Error { +pub(super) fn unbound(name: &str) -> Error { Error::PlanError(format!("no value is bound to ${name}")) } @@ -288,7 +318,7 @@ pub(super) fn unsupported(what: &str) -> Error { } /// The refusal of a call [`eval`] does not work out. -fn refused(name: &str) -> Error { +pub(super) fn refused(name: &str) -> Error { unsupported(&format!("the function {name}")) } @@ -379,7 +409,7 @@ fn truth(value: &Value) -> Option { /// /// A REAL has one rule for `=`, ORDER BY and the bytes of a key: the two zeros are one value, /// and every NaN is one value, equal to itself and above positive infinity. A whole number -/// compares with a REAL as a REAL, and a UUID with the text that spells one. +/// compares with a REAL as a REAL, and a UUID or a TIMESTAMP with the text that spells one. pub(super) fn compare(left: &Value, right: &Value) -> Option { let reals = |left: f64, right: f64| { let unordered = || left.is_nan().cmp(&right.is_nan()); @@ -397,6 +427,8 @@ pub(super) fn compare(left: &Value, right: &Value) -> Option { (Value::Uuid(left), Value::Uuid(right)) => left.cmp(right), (Value::Uuid(left), Value::Text(right)) => left.cmp(&right.parse().ok()?), (Value::Text(left), Value::Uuid(right)) => left.parse::().ok()?.cmp(right), + (Value::Timestamp(left), Value::Text(right)) => left.cmp(×tamp_millis(right)?), + (Value::Text(left), Value::Timestamp(right)) => timestamp_millis(left)?.cmp(right), _ => return None, }) } diff --git a/v2/contextdb/src/query/expression/tests.rs b/v2/contextdb/src/query/expression/tests.rs new file mode 100644 index 00000000..74cd7f2d --- /dev/null +++ b/v2/contextdb/src/query/expression/tests.rs @@ -0,0 +1,51 @@ +//! The evaluator itself, over expressions that read no column. + +use std::collections::HashMap; + +use super::eval; +use crate::query::{EvalContext, Scope}; +use crate::sql::Expr; +use crate::sql::ast::{BinOp, Literal, UnaryOp}; +use crate::types::Value; + +/// An expression that reads no column, worked out. +fn worked_out(expr: &Expr) -> Value { + let bound = HashMap::new(); + eval(expr, &Scope::default(), &[], &EvalContext::new(&bound, 0)) + .expect("an expression of literals") +} + +fn joined(left: Literal, op: BinOp, right: Literal) -> Expr { + Expr::BinaryOp { + left: Box::new(Expr::Literal(left)), + op, + right: Box::new(Expr::Literal(right)), + } +} + +#[test] +fn null_is_unknown_unless_the_other_side_settles_the_answer() { + let (null, yes, no) = (Literal::Null, Literal::Bool(true), Literal::Bool(false)); + let and = |left: &Literal, right: &Literal| { + worked_out(&joined(left.clone(), BinOp::And, right.clone())) + }; + let or = |left: &Literal, right: &Literal| { + worked_out(&joined(left.clone(), BinOp::Or, right.clone())) + }; + assert_eq!(and(&null, &no), Value::Bool(false)); + assert_eq!(and(&no, &null), Value::Bool(false)); + assert_eq!(and(&null, &yes), Value::Null); + assert_eq!(and(&yes, &yes), Value::Bool(true)); + assert_eq!(or(&null, &yes), Value::Bool(true)); + assert_eq!(or(&yes, &null), Value::Bool(true)); + assert_eq!(or(&null, &no), Value::Null); + assert_eq!(or(&no, &no), Value::Bool(false)); + + let not_null = Expr::UnaryOp { + op: UnaryOp::Not, + operand: Box::new(Expr::Literal(Literal::Null)), + }; + assert_eq!(worked_out(¬_null), Value::Null); + let null_equals_null = joined(Literal::Null, BinOp::Eq, Literal::Null); + assert_eq!(worked_out(&null_equals_null), Value::Null); +} diff --git a/v2/contextdb/src/query/for_write.rs b/v2/contextdb/src/query/for_write.rs new file mode 100644 index 00000000..ce265893 --- /dev/null +++ b/v2/contextdb/src/query/for_write.rs @@ -0,0 +1,86 @@ +//! The rows an UPDATE or DELETE acts on, handed over a batch at a time. + +use std::collections::HashMap; + +use super::expression::reads; +use super::table::Source; +use super::visible::{Reader, Visible}; +use crate::memory::{Budget, Reservation}; +use crate::schema::{Catalog, Table}; +use crate::sql::Expr; +use crate::store::{Cell, Key, Read}; +use crate::types::heap_bytes; +use crate::types::{Result, Value}; + +/// What an UPDATE or DELETE asks of its table, the same for every batch. +pub(crate) struct Asked<'a> { + pub table: &'a Table, + pub filter: Option<&'a Expr>, + /// The columns the statement assigns. No index over one of them is walked, so a row the + /// statement has changed is never met again. + pub assigned: &'a [usize], + /// The columns the caller reads. A large value in any other column is handed over as the + /// cell it is stored as and never loaded. + pub needed: &'a [bool], + pub reader: &'a Reader, + pub params: &'a HashMap, + pub now: i64, +} + +/// The most rows one batch holds. +const BATCH: usize = 1024; + +/// Rows with their keys, each row one cell for each column of its table. +pub(crate) type Batch = Vec<(Key, Vec)>; + +/// The next rows an UPDATE or DELETE acts on: those after the row with the key `after` in the +/// order the table source walks, which the statement cannot disturb. `None` when there are no +/// more. +/// +/// The rows are read through the one table source, with the index choice and the evaluator of a +/// SELECT, so a row the reader may not see is never handed over: named by its key, it is no +/// row. The rows stay counted against the memory limit for as long as the caller holds the +/// reservation that comes with them. +/// +/// Read through a change set, a batch may be staged before the next is asked for. A walk by +/// an index then passes the entries before its place again, reading no row for them. +pub(crate) fn rows_for_write( + asked: &Asked, + after: Option<&Key>, + catalog: &Catalog, + read: &dyn Read, + budget: &Budget, +) -> Result> { + let Asked { table, filter, .. } = *asked; + let mut filtered = vec![false; table.meta.columns.len()]; + if let Some(filter) = filter { + reads(filter, table, asked.params, &mut filtered)?; + } + // What the reader's filter holds is given back when the batch is found. + let mut seeing = budget.reserve(0, "query")?; + let visible = Visible::new(table, catalog, asked.reader, &mut seeing)?; + let source = Source::new(table, filter, asked.params, asked.now, read, visible) + .reading(filtered, asked.needed); + let mut path = source.choose(&[], asked.assigned)?; + path.one = false; + let mut reservation = budget.reserve(0, "query")?; + let mut batch = Vec::new(); + let mut keep = |(key, values, left, _): (Key, Vec, Vec, u64), + reservation: &mut Reservation| { + // Beside the values that were asked for: the key, and each small value that was not. + let small = left.iter().map(|cell| match cell { + Cell::Here(value) => heap_bytes(value), + Cell::Beside(_) => 0, + }); + reservation.grow(key.as_bytes().len() as u64 + small.sum::())?; + // A cell that was taken is left as NULL, and its value is the cell. + let cells = values.into_iter().zip(left).map(|pair| match pair { + (value, Cell::Here(Value::Null)) => Cell::Here(value), + (_, left) => left, + }); + batch.push((key, cells.collect())); + Ok(batch.len() < BATCH) + }; + source.cells(&path, after, &mut reservation, &mut keep)?; + Ok((!batch.is_empty()).then_some((batch, reservation))) +} diff --git a/v2/contextdb/src/query/inspect.rs b/v2/contextdb/src/query/inspect.rs index 5f12b0bb..ba6e5eb7 100644 --- a/v2/contextdb/src/query/inspect.rs +++ b/v2/contextdb/src/query/inspect.rs @@ -7,8 +7,8 @@ use std::collections::HashMap; use serde_json::json; -use super::QueryResult; use super::search::Scope; +use super::{QueryResult, Reader}; use crate::memory::Budget; use crate::schema::{Catalog, Consolidation, Quantization, Table, VectorColumn}; use crate::sql::{Plan, not_supported}; @@ -311,6 +311,7 @@ pub(crate) fn explain( plan: &Plan, catalog: &Catalog, read: &dyn Read, + reader: &Reader, budget: &Budget, now: i64, ) -> Result { @@ -349,6 +350,16 @@ pub(crate) fn explain( now, unbound: Cell::default(), }; + // A narrowed handle is told what the statement and the declaration say: no partition + // record is read for it, as one names its Context's values and counts its vectors. + if !reader.is_admin() { + let lines = [ + format!("requested_mode: {}", mode_name(mode)), + format!("effective_auto_index_at: {}", vector.index_at()), + format!("partition_key_columns: {}", key_columns(vector)), + ]; + return Ok(lines.join("\n")); + } let mut keys = scope.keys(filter.as_ref())?; let every = scope.unbound.take(); if every.is_some() { diff --git a/v2/contextdb/src/query/mod.rs b/v2/contextdb/src/query/mod.rs index c05f6001..9f929f4c 100644 --- a/v2/contextdb/src/query/mod.rs +++ b/v2/contextdb/src/query/mod.rs @@ -5,32 +5,37 @@ //! //! It never changes anything. -mod access; +mod bind; +mod counts; mod expression; +mod for_write; mod inspect; mod search; +mod table; #[cfg(test)] mod tests; +mod visible; use std::cmp::Ordering; use std::collections::HashMap; use std::fmt; -pub(crate) use expression::eval; +pub(crate) use expression::{EvalContext, Scope, eval, reads}; +pub(crate) use for_write::{Asked, rows_for_write}; pub(crate) use inspect::explain; -pub(crate) use search::vector_from_text; +pub(crate) use visible::{Reader, Visible}; use crate::memory::{Budget, Reservation}; use crate::schema::{Catalog, Table}; use crate::sql::ast::SortDirection; use crate::sql::plan::{ProjectColumn, SortKey}; use crate::sql::{Expr, Plan}; -use crate::store::{Key, Read}; +use crate::store::Read; use crate::types::{ColumnType, Error, Result, Value}; -use access::Source; use expression::{ - Arithmetic, arithmetic, column_index, compare, most_bytes, reads, sort_order, unsupported, + Arithmetic, arithmetic, column_index, compare, most_bytes, sort_order, unsupported, }; +use table::Source; /// What a statement returned: the column names and rows of a SELECT, or the number of rows a /// write affected. Its rows stay counted against the memory limit until it is dropped. @@ -143,12 +148,14 @@ pub(crate) fn memory_limit_shown(budget: &Budget, opened_under: Option) -> QueryResult::from_rows(columns.map(str::to_owned).to_vec(), vec![row]) } -/// Answers one planned SELECT from one reader. +/// Answers one planned SELECT from one store, for one reader. /// -/// Picks an index when the plan's filter can use one and scans otherwise; `COUNT(*)` of a whole -/// table is answered from the kept count. Loads a value kept beside its row only when a selected -/// or filtered column needs it. The result, and any sort or grouping on the way to it, grows -/// through reservations on `budget` and refuses with `MemoryBudgetExceeded` before it grows. +/// Every row comes from the table source, which picks an index when the plan's filter can use +/// one and scans otherwise, and hands on only the rows `reader` may see. A `COUNT(*)` is +/// answered with no row read when a kept number or the entries of one index range give it. +/// Loads a value kept beside its row only when a selected or filtered column needs it. The +/// result, and any sort or grouping on the way to it, grows through reservations on `budget` +/// and refuses with `MemoryBudgetExceeded` before it grows. /// /// A similarity search, `ORDER BY column <=> vector LIMIT k`, is answered by the vector part /// from the rows this part lets it see, and returns the nearest rows first. @@ -157,10 +164,11 @@ pub(crate) fn run( params: &HashMap, catalog: &Catalog, read: &dyn Read, + reader: &Reader, budget: &Budget, now: i64, ) -> Result { - if let Some(found) = search::run(plan, params, catalog, read, budget, now)? { + if let Some(found) = search::run(plan, params, catalog, read, reader, budget, now)? { return Ok(found); } if let Some(shown) = inspect::show(plan, catalog, read, budget)? { @@ -202,25 +210,36 @@ pub(crate) fn run( } let columns = names(table, select.columns, !aggregates.is_empty()); - let source = Source::new(table, select.filter, params, now, read, filtered, &wanted); + // What the reader's filter holds is given back with the statement. + let mut seeing = budget.reserve(0, "query")?; + let visible = Visible::new(table, catalog, reader, &mut seeing)?; + let source = + Source::new(table, select.filter, params, now, read, visible).reading(filtered, &wanted); let limit = select.limit.map_or(usize::MAX, |limit| { usize::try_from(limit).unwrap_or(usize::MAX) }); let mut reservation = budget.reserve(0, "query")?; let mut rows = Vec::new(); let mut examined = 0; + let scope = table.scope(); if !aggregates.is_empty() { - if select.filter.is_none() && aggregates.iter().all(|aggregate| aggregate.of.is_none()) { - let count = read.count(table.id)? as i64; + let counted = match aggregates.iter().all(|aggregate| aggregate.of.is_none()) { + true => { + let ctx = EvalContext::new(params, now); + counts::count_without_rows(table, select.filter, reader, &ctx, read)? + } + false => None, + }; + if let Some(count) = counted { for aggregate in &mut aggregates { - aggregate.seen = count; + aggregate.seen = count as i64; } } else { - let (path, _) = source.choose(&[])?; + let path = source.choose(&[], &[])?; examined = source.walk(&path, &mut reservation, |_, row, bytes, reservation| { for aggregate in &mut aggregates { - aggregate.add(table, &row, params, now, reservation)?; + aggregate.add(&scope, &row, params, now, reservation)?; } reservation.shrink(bytes); Ok(true) @@ -237,8 +256,8 @@ pub(crate) fn run( let output = |row: Vec, bytes: u64, reservation: &mut Reservation| { source.output(select.columns, &moved, row, bytes, None, reservation) }; - let (path, in_order) = source.choose(select.order)?; - if in_order { + let path = source.choose(select.order, &[])?; + if path.in_order { examined = source.walk(&path, &mut reservation, |_, row, bytes, reservation| { rows.push(output(row, bytes, reservation)?); Ok(rows.len() < limit) @@ -250,13 +269,13 @@ pub(crate) fn run( let order = select.order; let most = order .iter() - .map(|key| most_bytes(&key.expr, table, &row, params)); + .map(|key| most_bytes(&key.expr, &scope, &row, params)); let held = row_overhead(order.len()) + most.sum::(); reservation.grow(held + size_of::() as u64)?; key_bytes += held + size_of::() as u64; let keys = order .iter() - .map(|key| eval(&key.expr, table, &row, params, now)) + .map(|key| eval(&key.expr, &scope, &row, &EvalContext::new(params, now))) .collect::>>()?; sorting.push(Sorting { keys, row, bytes }); Ok(true) @@ -290,49 +309,6 @@ pub(crate) fn run( }) } -/// The rows [`matching`] found, each with its key. -pub(crate) type Matched = Vec<(Key, Vec)>; - -/// The rows of one table that pass a filter, with their keys: what an UPDATE or DELETE acts on. -/// Uses the same index choice and the same evaluator as [`run`]. The rows and their keys stay -/// counted against the memory limit for as long as the caller holds the reservation that comes -/// with them. -pub(crate) fn matching( - table: &Table, - filter: Option<&Expr>, - params: &HashMap, - read: &dyn Read, - budget: &Budget, - now: i64, -) -> Result<(Matched, Reservation)> { - let width = table.meta.columns.len(); - let mut filtered = vec![false; width]; - if let Some(filter) = filter { - reads(filter, table, params, &mut filtered)?; - } - let source = Source::new( - table, - filter, - params, - now, - read, - filtered, - &vec![true; width], - ); - let (path, _) = source.choose(&[])?; - let mut reservation = budget.reserve(0, "query")?; - let mut found = Vec::new(); - let key_columns = source.key_columns(); - source.walk(&path, &mut reservation, |key, row, _, reservation| { - // A key is its row's key values once more, and is asked for as what they take. - let values = key_columns.iter().map(|at| heap_bytes(&row[*at])); - reservation.grow(row_overhead(key_columns.len()) + values.sum::())?; - found.push((key, row)); - Ok(true) - })?; - Ok((found, reservation)) -} - /// What a result calls its columns: the table's own names for `SELECT *` (`None`), otherwise /// each item's `AS` name, a bare column's name, an aggregate's name, and `expr` for any other /// expression. @@ -481,16 +457,16 @@ impl<'a> Aggregate<'a> { /// is kept as the least or the greatest so far. fn add( &mut self, - table: &Table, + scope: &Scope, row: &[Value], params: &HashMap, now: i64, reservation: &mut Reservation, ) -> Result<()> { if let Some(of) = self.of { - let most = most_bytes(of, table, row, params); + let most = most_bytes(of, scope, row, params); reservation.grow(most)?; - let value = eval(of, table, row, params, now)?; + let value = eval(of, scope, row, &EvalContext::new(params, now))?; if value == Value::Null { reservation.shrink(most); return Ok(()); @@ -531,30 +507,3 @@ impl<'a> Aggregate<'a> { fn row_overhead(values: usize) -> u64 { (size_of::>() + values * size_of::()) as u64 } - -/// The memory a value takes outside its own cell. -fn heap_bytes(value: &Value) -> u64 { - fn json(value: &serde_json::Value) -> u64 { - let cell = size_of::(); - match value { - serde_json::Value::String(text) => text.len() as u64, - serde_json::Value::Array(items) => { - let items = items.iter(); - items.map(|item| cell as u64 + json(item)).sum() - } - serde_json::Value::Object(members) => { - let members = members.iter(); - members - .map(|(name, member)| (name.len() + cell) as u64 + json(member)) - .sum() - } - _ => 0, - } - } - match value { - Value::Text(text) => text.len() as u64, - Value::Vector(vector) => size_of_val(vector.as_slice()) as u64, - Value::Json(value) => json(value), - _ => 0, - } -} diff --git a/v2/contextdb/src/query/search.rs b/v2/contextdb/src/query/search.rs index 5a9be8f8..486c5981 100644 --- a/v2/contextdb/src/query/search.rs +++ b/v2/contextdb/src/query/search.rs @@ -6,15 +6,16 @@ use std::cell::Cell; use std::collections::HashMap; -use super::access::{Source, as_key}; -use super::expression::{column_index, eval, reads}; -use super::{QueryResult, names}; +use super::expression::{EvalContext, column_index, eval, reads}; +use super::table::{Source, as_key}; +use super::visible::Visible; +use super::{QueryResult, Reader, names}; use crate::memory::Budget; use crate::schema::{Catalog, Table}; use crate::sql::ast::{BinOp, ColumnRef}; use crate::sql::{Expr, Plan}; use crate::store::{Key, Read}; -use crate::types::{ColumnType, Error, Result, Value, VectorIndexRef}; +use crate::types::{ColumnType, Error, Result, Value, VectorIndexRef, vector_from_text}; use crate::vector::{self, Nearest}; /// The most key tuples a filter may name; past that the search reads every partition. @@ -35,6 +36,7 @@ pub(super) fn run( params: &HashMap, catalog: &Catalog, read: &dyn Read, + reader: &Reader, budget: &Budget, now: i64, ) -> Result> { @@ -69,7 +71,7 @@ pub(super) fn run( let Some(vector) = table.vector(column) else { return Err(Error::UnknownVectorIndex { index }); }; - let near = match eval(query, table, &[], params, now)? { + let near = match eval(query, &table.scope(), &[], &EvalContext::new(params, now))? { Value::Vector(vector) => Some(vector), Value::Text(text) => vector_from_text(&text), _ => None, @@ -129,8 +131,15 @@ pub(super) fn run( // the answer for what the select list reads, so a large value is loaded only then. let mut its_vector = vec![false; width]; its_vector[at] = true; - let asked = Source::new(table, filter, params, now, read, filtered, &its_vector); - let returned = Source::new(table, None, params, now, read, vec![false; width], &wanted); + // Each is read through the reader's filter, and what a filter holds is given back with + // the statement. + let (mut seeing, mut showing) = (budget.reserve(0, "query")?, budget.reserve(0, "query")?); + let visible = Visible::new(table, catalog, reader, &mut seeing)?; + let asked = + Source::new(table, filter, params, now, read, visible).reading(filtered, &its_vector); + let visible = Visible::new(table, catalog, reader, &mut showing)?; + let returned = + Source::new(table, None, params, now, read, visible).reading(vec![false; width], &wanted); // What the vector part holds is given back once the rows are built; the rows stay counted // with the result. let mut searching = budget.reserve(0, "query")?; @@ -351,7 +360,8 @@ impl Scope<'_> { self.unbound.set(Some(name.clone())); } value => { - let value = eval(value, self.table, &[], self.params, self.now)?; + let ctx = EvalContext::new(self.params, self.now); + let value = eval(value, &self.table.scope(), &[], &ctx)?; tuple[place] = as_key(self.table.meta.columns[at].column_type, value); } } @@ -367,13 +377,3 @@ fn sides(expr: &Expr) -> &[Expr] { one => std::slice::from_ref(one), } } - -/// Reads the text spelling of a vector, `[0.1, 0.2, 0.3]`: the one reader of it, for a value -/// written into a VECTOR column and for the vector of a search. -pub(crate) fn vector_from_text(text: &str) -> Option> { - let inside = text.trim().strip_prefix('[')?.strip_suffix(']')?; - inside - .split(',') - .map(|number| number.trim().parse().ok()) - .collect() -} diff --git a/v2/contextdb/src/query/table.rs b/v2/contextdb/src/query/table.rs new file mode 100644 index 00000000..facb137e --- /dev/null +++ b/v2/contextdb/src/query/table.rs @@ -0,0 +1,771 @@ +//! The one source of a table's rows: which rows a statement has to read, reading them through +//! the reader's filter and the statement's own, and working out the select list for a row that +//! was read. + +use std::cell::RefCell; +use std::cmp::Ordering; +use std::collections::HashMap; +use std::ops::Bound; + +use super::expression::{EvalContext, Scope, column_index, eval, most_bytes, reads, sort_order}; +use super::row_overhead; +use super::visible::Visible; +use crate::memory::Reservation; +use crate::schema::Table; +use crate::sql::Expr; +use crate::sql::ast::{BinOp, SortDirection as Asked}; +use crate::sql::plan::{ProjectColumn, SortKey}; +use crate::store::{Cell, Entry, Key, Read, Row}; +use crate::types::{ColumnType, Result, SortDirection, Value, heap_bytes}; + +/// The keys or entries between two bounds. +type Range = (Bound, Bound); + +/// Where the rows of one table are reached. +enum Reach { + /// The rows with these primary keys, in key order. + Keys(Vec), + /// The table between each pair of keys, in key order. + Rows(Vec), + /// The rows that the entries of the index at this place among the table's indexes point + /// at, between each pair of entries, in entry order. + Index(usize, Vec), +} + +/// How the rows of one table are reached for one statement. +pub(super) struct Path { + reach: Reach, + /// True when the statement names one row whole, by its key or by every column of a UNIQUE + /// rule: a reader that row is hidden from is refused by the rule that hid it. + pub(super) one: bool, + /// True when the rows arrive in the order asked for, so that they need no sort. + pub(super) in_order: bool, + /// The columns of that order. A run of entries that hold cut values is in the order of + /// their digests, and is put in the order of the values themselves before it is handed on. + order: Vec<(usize, SortDirection)>, +} + +/// A part of a filter that every kept row must pass, and that compares a column with values +/// known before any row is read: one value, or for `=` the values of an `IN` list. +pub(super) struct Term { + pub(super) column: usize, + pub(super) op: BinOp, + pub(super) values: Vec, + /// True for `column = value`, which names one value. + one: bool, +} + +/// Something that keeps the rows of a table in an order: the table itself, under its primary +/// key, or one of its indexes. +struct Ordered { + /// Where the index is among the table's indexes; `None` for the table itself. + index: Option, + /// True when no two rows have the same values in all of its columns. + unique: bool, + /// The columns it is ordered by, each with its direction. + columns: Vec<(usize, SortDirection)>, +} + +impl Ordered { + /// What these leading values make: the bytes every key or entry with these values begins + /// with. A row key is whole and all ascending; an entry is made as its index makes it. + fn key(&self, table: &Table, values: &[Value]) -> Result { + match self.index { + Some(at) => table.entry_shape(at).entry(values), + None => Ok(Entry { + key: Key::from_values(values, &[])?, + cut: None, + }), + } + } + + /// What the filter leaves of the keys that begin with `given`: what `given` makes, the + /// range, and whether the filter gives a range of the next column. + fn range( + &self, + table: &Table, + given: &[Value], + terms: &[Term], + ) -> Result<(Entry, Range, bool)> { + let all = self.key(table, given)?; + let (mut from, mut to) = (Bound::Unbounded, Bound::Unbounded); + if !given.is_empty() { + to = until_after(&all.key); + from = Bound::Included(all.key.clone()); + } + let mut ranged = false; + // Behind a cut value nothing says where the next column's values lie. + let next = self.columns.get(given.len()).filter(|_| all.cut.is_none()); + if let Some((column, direction)) = next { + // The key of a bound, and whether the bound is a cut value. A cut bound is cut back + // to its run, the entries that share its kept bytes, and the filter leaves out the + // rows of the run that are outside the range. + let with = |value: &Value| -> Result<(Key, bool)> { + let entry = self.key(table, &[given, std::slice::from_ref(value)].concat())?; + Ok(match entry.cut { + Some(cut) => (Key::from_bytes(&entry.key.as_bytes()[..cut + 2]), true), + None => (entry.key, false), + }) + }; + let ranges = terms.iter().filter(|term| { + let range = matches!(term.op, BinOp::Gt | BinOp::Gte | BinOp::Lt | BinOp::Lte); + term.column == *column && range + }); + for term in ranges { + if !ranged { + // No NULL is in a range. A key keeps NULL after every value going up + // and before every value going down, and the walk stops short of it. + let null = with(&Value::Null)?.0; + match direction { + SortDirection::Asc => to = Bound::Excluded(null), + SortDirection::Desc => from = starting_after(null), + } + ranged = true; + } + let (key, cut) = with(&term.values[0])?; + // A descending column keeps its greater values first. When more columns + // follow, many keys begin with this one, and "after it" is after them all. + match (term.op, direction, cut) { + (BinOp::Gt, SortDirection::Asc, false) + | (BinOp::Lt, SortDirection::Desc, false) => from = starting_after(key), + (BinOp::Gt | BinOp::Gte, SortDirection::Asc, _) + | (BinOp::Lt | BinOp::Lte, SortDirection::Desc, _) => { + from = Bound::Included(key); + } + (BinOp::Lt, SortDirection::Asc, false) + | (BinOp::Gt, SortDirection::Desc, false) => to = Bound::Excluded(key), + // `<=` going up and `>=` going down, and the far end of a run. + _ => to = until_after(&key), + } + } + } + Ok((all, (from, to), ranged)) + } +} + +/// The bound that ends a walk after the last key that begins with this one. +fn until_after(key: &Key) -> Bound { + key.prefix_end().map_or(Bound::Unbounded, Bound::Excluded) +} + +/// The bound that starts a walk after the last key that begins with this one. +fn starting_after(key: Key) -> Bound { + let after = key.prefix_end(); + after.map_or(Bound::Excluded(key), Bound::Included) +} + +/// The value as a key of a column of this type holds it, when there is one such value: only +/// then is looking the value up the same as comparing every row with it. +pub(super) fn as_key(column: ColumnType, value: Value) -> Option { + match (column, value) { + (ColumnType::Integer, value @ Value::Int64(_)) + | (ColumnType::Real, value @ Value::Float64(_)) + | (ColumnType::Text, value @ Value::Text(_)) + | (ColumnType::Boolean, value @ Value::Bool(_)) + | (ColumnType::Uuid, value @ Value::Uuid(_)) + | (ColumnType::Timestamp, value @ Value::Timestamp(_)) + | (ColumnType::TxId, value @ Value::TxId(_)) => Some(value), + (ColumnType::Real, Value::Int64(value)) => Some(Value::Float64(value as f64)), + (ColumnType::Uuid, Value::Text(value)) => value.parse().ok().map(Value::Uuid), + _ => None, + } +} + +/// Gathers the terms of a filter: the comparisons, `IS NULL` and `IN` lists joined by AND at +/// its top. Answers whether the filter is nothing but the terms gathered. +pub(super) fn collect( + table: &Table, + filter: &Expr, + ctx: &EvalContext, + terms: &mut Vec, +) -> Result { + let known = |value: &Expr| matches!(value, Expr::Literal(_) | Expr::Parameter(_)); + let (column, op, values, one) = match filter { + Expr::BinaryOp { left, op, right } if *op == BinOp::And => { + let left = collect(table, left, ctx, terms)?; + return Ok(collect(table, right, ctx, terms)? && left); + } + Expr::BinaryOp { left, op, right } => match (&**left, &**right, op) { + (_, _, BinOp::Neq | BinOp::Or) => return Ok(false), + (Expr::Column(column), value, op) if known(value) => (column, *op, vec![value], true), + // `5 < id` is `id > 5`. + (value, Expr::Column(column), op) if known(value) => { + let turned = match op { + BinOp::Lt => BinOp::Gt, + BinOp::Lte => BinOp::Gte, + BinOp::Gt => BinOp::Lt, + BinOp::Gte => BinOp::Lte, + other => *other, + }; + (column, turned, vec![value], true) + } + _ => return Ok(false), + }, + Expr::IsNull { expr, negated } => match &**expr { + Expr::Column(column) if !negated => { + terms.push(Term { + column: column_index(table, column)?, + op: BinOp::Eq, + values: vec![Value::Null], + one: false, + }); + return Ok(true); + } + _ => return Ok(false), + }, + Expr::InList { + expr, + list, + negated, + } => match &**expr { + Expr::Column(column) if !negated && list.iter().all(known) => { + (column, BinOp::Eq, list.iter().collect(), false) + } + _ => return Ok(false), + }, + _ => return Ok(false), + }; + let column = column_index(table, column)?; + let mut keys = Vec::new(); + for value in values { + match eval(value, &table.scope(), &[], ctx)? { + // A NULL in a list equals nothing. + Value::Null if !one => {} + value => match as_key(table.meta.columns[column].column_type, value) { + Some(value) => keys.push(value), + None => return Ok(false), + }, + } + } + let values = keys; + terms.push(Term { + column, + op, + values, + one, + }); + Ok(true) +} + +/// A row a walk holds back: its key, its values, the cells not taken from it, and the bytes +/// its values hold. +type Held = (Key, Vec, Vec, u64); + +/// The bytes a held row takes beside its values. +fn held_bytes(key: &Key) -> u64 { + (size_of::() + key.as_bytes().len()) as u64 +} + +/// One table as one statement reads it. Every row a statement reads or changes comes from +/// here, so every one has passed the reader's filter. +pub(super) struct Source<'a> { + table: &'a Table, + scope: Scope<'a>, + filter: Option<&'a Expr>, + params: &'a HashMap, + now: i64, + read: &'a dyn Read, + /// The reader's filter, which remembers the grants it has looked up; `None` when it reads + /// no column of a row, and so hides none. + visible: Option>>, + /// The columns the reader's filter reads: taken from every row that is read. + seen: Vec, + /// The other columns the statement's filter reads: taken from every row the reader may see. + filtered: Vec, + /// The other columns the statement reads: taken only from a row that passed both filters. + rest: Vec, +} + +impl<'a> Source<'a> { + /// The source of `table` for a statement with this filter, read through `visible`. + pub(super) fn new( + table: &'a Table, + filter: Option<&'a Expr>, + params: &'a HashMap, + now: i64, + read: &'a dyn Read, + visible: Visible<'a>, + ) -> Source<'a> { + let width = table.meta.columns.len(); + let mut seen = vec![false; width]; + visible.reads(&mut seen); + Source { + table, + scope: table.scope(), + filter, + params, + now, + read, + visible: seen.contains(&true).then(|| RefCell::new(visible)), + seen, + filtered: vec![false; width], + rest: vec![false; width], + } + } + + /// `filtered` marks the columns the filter reads and `wanted` every column the statement + /// reads after it. A column in neither is never taken from a row, so a large value in it + /// is never loaded. + pub(super) fn reading(mut self, filtered: Vec, wanted: &[bool]) -> Source<'a> { + let columns = 0..filtered.len(); + let rest = columns.map(|at| wanted[at] && !filtered[at] && !self.seen[at]); + self.rest = rest.collect(); + let filtered = filtered.iter().zip(&self.seen); + self.filtered = filtered + .map(|(filtered, seen)| *filtered && !*seen) + .collect(); + self + } + + /// Picks the path that reads the fewest rows for the filter, and says whether its rows + /// arrive in the order asked for, so that they need no sort. + /// + /// The table and each index are tried in turn. One is better the more of its leading + /// columns the filter gives a value for (`=`, `IS NULL`, and one `IN` list), then if the + /// filter gives a range of its next column, then if it keeps the order asked for; the table + /// itself wins a tie. The filter is still applied to every row read, so a path only has to + /// reach every row that can pass. An index over a column of `assigned` is not tried: the + /// statement moves its entries. + /// + /// Rows are kept under their primary key, so the key is answered from the table itself. Its + /// own index, `__pk_`, is declared and has no entries, and is never walked. + pub(super) fn choose(&self, order: &[SortKey], assigned: &[usize]) -> Result { + let mut terms = Vec::new(); + if let Some(filter) = self.filter { + let ctx = EvalContext::new(self.params, self.now); + collect(self.table, filter, &ctx, &mut terms)?; + } + let columns = &self.table.meta.columns; + let column_named = |name: &String| columns.iter().position(|column| column.name == *name); + // The order asked for, when it is plain columns. + let asked: Option> = order + .iter() + .map(|key| { + let Expr::Column(column) = &key.expr else { + return None; + }; + let direction = match key.direction { + Asked::Asc => SortDirection::Asc, + Asked::Desc => SortDirection::Desc, + }; + Some((column_index(self.table, column).ok()?, direction)) + }) + .collect(); + + let the_table = Ordered { + index: None, + unique: true, + columns: (self.table.key.iter()) + .map(|at| (*at, SortDirection::Asc)) + .collect(), + }; + let indexes = self.table.meta.indexes.iter().enumerate(); + let indexes = indexes + .filter(|(_, index)| !index.name.starts_with("__pk_") && index.json_path.is_empty()) + .map(|(at, index)| Ordered { + index: Some(at), + unique: index.name.starts_with("__unique_"), + columns: index + .columns + .iter() + .filter_map(|(name, direction)| Some((column_named(name)?, *direction))) + .collect(), + }) + .filter(|index| { + !index + .columns + .iter() + .any(|column| assigned.contains(&column.0)) + }); + + // Until something better is found: the whole table, in no order that was asked for. + let mut best = ( + (0, false, false), + Path { + reach: Reach::Rows(vec![(Bound::Unbounded, Bound::Unbounded)]), + one: false, + in_order: false, + order: Vec::new(), + }, + ); + // A table that declares no key keeps its rows under numbers no statement names. + let the_table = Some(the_table).filter(|_| !self.table.key.is_empty()); + for ordered in the_table.into_iter().chain(indexes) { + // The leading columns the filter gives values for. One list at most is taken, so + // the ranges read are no more than the values the statement wrote. + let mut listed = false; + let pins: Vec<&Term> = (ordered.columns.iter()) + .map_while(|(column, _)| { + let equal = |term: &&Term| term.column == *column && term.op == BinOp::Eq; + let term = terms.iter().find(equal)?; + let list = term.values.len() != 1; + let taken = !(list && listed); + listed |= list; + taken.then_some(term) + }) + .collect(); + let mut tuples = vec![Vec::new()]; + for pin in &pins { + let mut longer = Vec::new(); + for tuple in &tuples { + for value in &pin.values { + longer.push([tuple.as_slice(), std::slice::from_ref(value)].concat()); + } + } + tuples = longer; + } + let mut ranges = Vec::new(); + let (mut ranged, mut cut) = (false, false); + for tuple in &tuples { + let (entry, range, is_ranged) = ordered.range(self.table, tuple, &terms)?; + cut |= entry.cut.is_some(); + ranged = is_ranged; + ranges.push((entry.key, range)); + } + // In key order, and a value written twice read once. + ranges.sort_by(|a, b| a.0.cmp(&b.0)); + ranges.dedup_by(|a, b| a.0 == b.0); + + let next = &ordered.columns[pins.len()..]; + let one = ordered.unique && next.is_empty() && pins.iter().all(|pin| pin.one); + let whole_key = ordered.index.is_none() && next.is_empty(); + // A key keeps its columns as ORDER BY sorts them, NULL included, so what follows + // the columns the filter fixed is in order if it is what was asked for. Behind a + // list each value starts the order again, and behind a cut value entries follow + // their digests. + let single = ranges.len() <= 1 && !cut; + let asked = asked.as_ref().filter(|asked| { + asked.is_empty() || single && (whole_key || next.starts_with(asked)) + }); + let in_order = asked.is_some() || single && whole_key; + let order = asked.cloned().unwrap_or_default(); + let (keys, ranges): (Vec, Vec) = ranges.into_iter().unzip(); + let reach = match ordered.index { + Some(index) => Reach::Index(index, ranges), + None if whole_key => Reach::Keys(keys), + None => Reach::Rows(ranges), + }; + let path = Path { + reach, + one, + in_order, + order, + }; + if whole_key { + return Ok(path); + } + let score = (pins.len(), ranged, in_order); + if score > best.0 { + best = (score, path); + } + } + Ok(best.1) + } + + /// Reads the rows the path reaches and hands `each` every one the reader may see and the + /// filter passes: its key, its values, and the bytes of `reservation` it holds. Stops when + /// `each` answers false. Returns how many stored rows were read. + pub(super) fn walk( + &self, + path: &Path, + reservation: &mut Reservation, + mut each: impl FnMut(Key, Vec, u64, &mut Reservation) -> Result, + ) -> Result { + let mut each = |(key, values, _, bytes): Held, reservation: &mut Reservation| { + each(key, values, bytes, reservation) + }; + self.cells(path, None, reservation, &mut each) + } + + /// [`Source::walk`] from the row after the one with the key `after`, handing over with + /// each row the cells that were not taken from it, in column order. + pub(super) fn cells( + &self, + path: &Path, + after: Option<&Key>, + reservation: &mut Reservation, + each: &mut dyn FnMut(Held, &mut Reservation) -> Result, + ) -> Result { + let table = self.table.id; + let mut examined = 0; + match &path.reach { + Reach::Keys(keys) => { + let keys = keys + .iter() + .filter(|key| after.is_none_or(|after| *key > after)); + for key in keys { + let Some(row) = self.read.row(table, key)? else { + continue; + }; + examined += 1; + if let Some((values, cells, bytes)) = self.pass(row, path.one, reservation)? + && !each((key.clone(), values, cells, bytes), reservation)? + { + break; + } + } + } + Reach::Rows(ranges) => { + 'ranges: for (from, to) in ranges { + // A bound stands for every key that begins with it, so the walk resumes + // after `after` only where that is later than where the range starts. + let resumed = after.filter(|after| match from { + Bound::Included(from) => *after >= from, + Bound::Excluded(from) => *after > from && !after.starts_with(from), + Bound::Unbounded => true, + }); + let from = resumed.map_or(from.as_ref(), Bound::Excluded); + for item in self.read.rows(table, from, to.as_ref())? { + let (key, row) = item?; + examined += 1; + if let Some((values, cells, bytes)) = + self.pass(row, path.one, reservation)? + && !each((key, values, cells, bytes), reservation)? + { + break 'ranges; + } + } + } + } + Reach::Index(at, ranges) => { + let index = self.table.index_ids[*at]; + // An index is resumed by passing the entries up to the one of the row resumed + // after. When that row is gone, so is every row matched before it. + let mut passing = match after { + Some(after) => self.read.row(table, after)?.is_some(), + None => false, + }; + // The run of cut entries held back, and the bytes its entries share. + let mut run: Vec = Vec::new(); + let mut shared = Vec::new(); + 'ranges: for (from, to) in ranges { + for entry in self.read.index(index, from.as_ref(), to.as_ref())? { + let (entry, key, cut) = entry?; + if passing { + passing = Some(&key) != after; + continue; + } + // Cut entries with the same kept bytes are a run; only an order that + // is used asks for the values of one. + let cut = cut.filter(|_| !path.order.is_empty()); + let begins = match cut.map(|cut| entry.as_bytes().get(..cut + 2)) { + Some(None) => { + return Err(self + .read + .damaged("an index entry ends inside its cut")); + } + Some(begins) => begins, + None => None, + }; + if !run.is_empty() + && begins != Some(shared.as_slice()) + && !self.hand_on(&mut run, &path.order, reservation, each)? + { + break 'ranges; + } + let Some(row) = self.read.row(table, &key)? else { + continue; + }; + examined += 1; + let Some((values, cells, bytes)) = self.pass(row, path.one, reservation)? + else { + continue; + }; + match begins { + Some(begins) => { + reservation.grow(held_bytes(&key))?; + shared = begins.to_vec(); + run.push((key, values, cells, bytes)); + } + None => { + if !each((key, values, cells, bytes), reservation)? { + break 'ranges; + } + } + } + } + } + self.hand_on(&mut run, &path.order, reservation, each)?; + } + } + Ok(examined) + } + + /// Hands on a run of rows in the order of their values in the columns asked for, each in + /// its direction, and otherwise as their entries came. Answers false when `each` did. + fn hand_on( + &self, + run: &mut Vec, + order: &[(usize, SortDirection)], + reservation: &mut Reservation, + each: &mut dyn FnMut(Held, &mut Reservation) -> Result, + ) -> Result { + run.sort_by(|a, b| { + let mut columns = order.iter().map(|(column, direction)| { + let order = sort_order(&a.1[*column], &b.1[*column]); + match direction { + SortDirection::Asc => order, + SortDirection::Desc => order.reverse(), + } + }); + columns + .find(|order| order.is_ne()) + .unwrap_or(Ordering::Equal) + }); + let mut more = true; + for held in run.drain(..) { + reservation.shrink(held_bytes(&held.0)); + match more { + true => more = each(held, reservation)?, + false => reservation.shrink(held.3), + } + } + Ok(more) + } + + /// The row with this key if it is there, the reader may see it and it passes the filter, + /// as [`Source::walk`] hands a row over: its values, and the bytes of `reservation` it + /// holds. + pub(super) fn row( + &self, + key: &Key, + reservation: &mut Reservation, + ) -> Result, u64)>> { + let Some(row) = self.read.row(self.table.id, key)? else { + return Ok(None); + }; + let passed = self.pass(row, false, reservation)?; + Ok(passed.map(|(values, _, bytes)| (values, bytes))) + } + + /// For each item of a select list, the column whose value it moves out of a row: the column + /// it names, when no other item of the list reads that column. Any other item copies what + /// it reads. Empty for `SELECT *`, which hands the row over as it is. + pub(super) fn moved(&self, columns: Option<&[ProjectColumn]>) -> Result>> { + let columns = columns.unwrap_or_default(); + let mut readers = vec![0; self.table.meta.columns.len()]; + for column in columns { + let mut read = vec![false; readers.len()]; + reads(&column.expr, self.table, self.params, &mut read)?; + for (readers, read) in readers.iter_mut().zip(read) { + *readers += usize::from(read); + } + } + let alone = |column: &ProjectColumn| match &column.expr { + Expr::Column(named) => column_index(self.table, named).ok(), + _ => None, + }; + let moved = columns.iter().map(alone); + Ok(moved.map(|at| at.filter(|at| readers[*at] == 1)).collect()) + } + + /// The select list worked out for one row this source handed over, or the row itself for + /// `SELECT *`. `moved` is [`Source::moved`] of the list, and `distance` is the row's distance + /// when a similarity search found it. The memory of the values is asked for before they + /// are made, and the row's bytes are given back once they are. A value that is moved is + /// held once: what it takes passes from the row to the result and is not asked for again. + pub(super) fn output( + &self, + columns: Option<&[ProjectColumn]>, + moved: &[Option], + mut row: Vec, + bytes: u64, + distance: Option, + reservation: &mut Reservation, + ) -> Result> { + let Some(columns) = columns else { + return Ok(row); + }; + let most = columns + .iter() + .map(|column| most_bytes(&column.expr, &self.scope, &row, self.params)); + let passed = moved.iter().flatten().map(|at| heap_bytes(&row[*at])); + let passed = passed.sum::().min(bytes); + reservation.grow(row_overhead(columns.len()) + most.sum::() - passed)?; + let values = columns + .iter() + .zip(moved) + .map(|(column, moved)| match moved { + Some(at) => Ok(std::mem::replace(&mut row[*at], Value::Null)), + None => { + let ctx = EvalContext::new(self.params, self.now).found(distance); + eval(&column.expr, &self.scope, &row, &ctx) + } + }) + .collect::>>()?; + reservation.shrink(bytes - passed); + Ok(values) + } + + /// A stored row the reader may see and the filter passes: its values, the cells not taken + /// from it, both in column order, and the bytes of `reservation` the values hold. A row + /// that does not pass holds none. A column the statement does not read is left as NULL. + /// + /// The reader's filter is asked first, and nothing else of a row it hides is looked at. + /// With `refuse`, such a row is the refusal of the rule that hid it. + fn pass( + &self, + row: Row, + refuse: bool, + reservation: &mut Reservation, + ) -> Result> { + let before = reservation.bytes(); + let missing = &self.table.missing; + reservation.grow(row_overhead(missing.len()))?; + // A column added since the row was written has no cell and reads as the table says. + let mut cells = row.cells; + cells.extend(missing.iter().skip(cells.len()).cloned().map(Cell::Here)); + let mut values = vec![Value::Null; cells.len()]; + let mut hidden = None; + if let Some(visible) = &self.visible { + self.take(&mut cells, &mut values, &self.seen, reservation)?; + hidden = visible.borrow_mut().hidden(&values, self.read)?; + } + let mut passes = hidden.is_none(); + if passes { + self.take(&mut cells, &mut values, &self.filtered, reservation)?; + if let Some(filter) = self.filter { + let ctx = EvalContext::new(self.params, self.now); + passes = eval(filter, &self.scope, &values, &ctx)? == Value::Bool(true); + } + } + if !passes { + reservation.shrink(reservation.bytes() - before); + return hidden.filter(|_| refuse).map_or(Ok(None), Err); + } + self.take(&mut cells, &mut values, &self.rest, reservation)?; + Ok(Some((values, cells, reservation.bytes() - before))) + } + + /// Moves the marked cells into `values`, loading a value kept beside its row. The memory + /// of the values is asked for before any is moved or loaded. + fn take( + &self, + cells: &mut [Cell], + values: &mut [Value], + marked: &[bool], + reservation: &mut Reservation, + ) -> Result<()> { + let here = cells.iter().zip(marked).map(|(cell, marked)| match cell { + Cell::Here(here) if *marked => heap_bytes(here), + _ => 0, + }); + reservation.grow(here.sum())?; + for ((cell, value), _) in cells + .iter_mut() + .zip(values) + .zip(marked) + .filter(|(_, marked)| **marked) + { + *value = match std::mem::replace(cell, Cell::Here(Value::Null)) { + Cell::Here(here) => here, + Cell::Beside(beside) => self.read.load(&beside, reservation)?, + }; + } + Ok(()) + } +} + +/// What a row that passed hands over: its values, the cells not taken from it, and the bytes +/// its values hold. +type Passed = (Vec, Vec, u64); diff --git a/v2/contextdb/src/query/tests.rs b/v2/contextdb/src/query/tests.rs index 41ff41fd..307b922e 100644 --- a/v2/contextdb/src/query/tests.rs +++ b/v2/contextdb/src/query/tests.rs @@ -1,63 +1,9 @@ -use std::cmp::Ordering; -use std::collections::HashMap; - -use super::expression::{Arithmetic, arithmetic, compare, eval, sort_order}; -use crate::schema::Table; -use crate::sql::Expr; -use crate::sql::ast::{BinOp, Literal, UnaryOp}; -use crate::store::TableId; -use crate::types::{Error, TableMeta, Uuid, Value}; - -/// An expression that reads no column, worked out. -fn worked_out(expr: &Expr) -> Value { - let table = Table { - id: TableId(1), - name: "t".to_owned(), - meta: TableMeta { - columns: Vec::new(), - unique_constraints: Vec::new(), - indexes: Vec::new(), - }, - index_ids: Vec::new(), - vectors: Vec::new(), - }; - eval(expr, &table, &[], &HashMap::new(), 0).expect("an expression of literals") -} - -fn joined(left: Literal, op: BinOp, right: Literal) -> Expr { - Expr::BinaryOp { - left: Box::new(Expr::Literal(left)), - op, - right: Box::new(Expr::Literal(right)), - } -} +//! How two values compare and sort, and arithmetic: the evaluator's pure functions. -#[test] -fn null_is_unknown_unless_the_other_side_settles_the_answer() { - let (null, yes, no) = (Literal::Null, Literal::Bool(true), Literal::Bool(false)); - let and = |left: &Literal, right: &Literal| { - worked_out(&joined(left.clone(), BinOp::And, right.clone())) - }; - let or = |left: &Literal, right: &Literal| { - worked_out(&joined(left.clone(), BinOp::Or, right.clone())) - }; - assert_eq!(and(&null, &no), Value::Bool(false)); - assert_eq!(and(&no, &null), Value::Bool(false)); - assert_eq!(and(&null, &yes), Value::Null); - assert_eq!(and(&yes, &yes), Value::Bool(true)); - assert_eq!(or(&null, &yes), Value::Bool(true)); - assert_eq!(or(&yes, &null), Value::Bool(true)); - assert_eq!(or(&null, &no), Value::Null); - assert_eq!(or(&no, &no), Value::Bool(false)); +use std::cmp::Ordering; - let not_null = Expr::UnaryOp { - op: UnaryOp::Not, - operand: Box::new(Expr::Literal(Literal::Null)), - }; - assert_eq!(worked_out(¬_null), Value::Null); - let null_equals_null = joined(Literal::Null, BinOp::Eq, Literal::Null); - assert_eq!(worked_out(&null_equals_null), Value::Null); -} +use super::expression::{Arithmetic, arithmetic, compare, sort_order}; +use crate::types::{Error, Uuid, Value}; #[test] fn arithmetic_keeps_whole_numbers_whole_and_refuses_what_does_not_fit() { diff --git a/v2/contextdb/src/query/visible.rs b/v2/contextdb/src/query/visible.rs new file mode 100644 index 00000000..2b175147 --- /dev/null +++ b/v2/contextdb/src/query/visible.rs @@ -0,0 +1,263 @@ +//! Who is asking, and the one filter that says which rows of a table that reader may see and +//! write. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use super::expression::reads; +use super::table::Source; +use crate::memory::Reservation; +use crate::schema::{Catalog, Table}; +use crate::sql::Expr; +use crate::sql::ast::{BinOp, ColumnRef, Literal}; +use crate::store::Read; +use crate::types::{ContextId, Error, Principal, Result, ScopeLabel, Uuid, Value}; + +/// Who is asking. Each rule a reader declares narrows what it is shown; `None` declares +/// nothing for that rule. +pub(crate) struct Reader { + pub contexts: Option>, + pub scope_labels: Option>, + pub principal: Option, +} + +impl Reader { + /// The reader that declares nothing and sees everything. + pub(crate) fn admin() -> Reader { + Reader { + contexts: None, + scope_labels: None, + principal: None, + } + } + + /// True for a reader that declares nothing. + pub(crate) fn is_admin(&self) -> bool { + self.contexts.is_none() && self.scope_labels.is_none() && self.principal.is_none() + } +} + +/// How many access ids a filter remembers the grants of. +const REMEMBERED: usize = 1024; + +/// The bytes a filter is counted for what it remembers, asked for whole before its first row. +const ROOM: u64 = 64 * 1024; + +/// The filter of one table for one reader, made for one statement. +pub(crate) struct Visible<'a> { + table: &'a Table, + reader: &'a Reader, + /// The table's access column and the grants its rows are asked for; `None` when no row is + /// asked for a grant. The grants are `None` when the catalog has no table of grants with + /// the columns a grant is read from: then nobody holds one. + acl: Option<(usize, Option>)>, +} + +/// The grants of one principal: where they are found, and the ids already looked up. +struct Grants<'a> { + /// The table of grants. + table: &'a Table, + /// What a row of it passes when it grants the access id bound to `$id` to the principal, + /// the values bound in it, and the columns it reads. + filter: Expr, + params: HashMap, + filtered: Vec, + /// What a grant kept beside its row is loaded under. + reservation: &'a mut Reservation, + /// Whether the principal holds a grant, for at most the `REMEMBERED` ids looked up last. + known: BTreeMap, +} + +impl<'a> Visible<'a> { + /// The filter of `table` for `reader`. Refuses, before any row, a reader the table's rules + /// cannot answer. + pub(crate) fn new( + table: &'a Table, + catalog: &'a Catalog, + reader: &'a Reader, + reservation: &'a mut Reservation, + ) -> Result> { + let mut acl = None; + if let Some((column, from, access)) = &table.rules.acl + && !reader.is_admin() + { + let (kind, name) = match &reader.principal { + Some(Principal::Agent(name)) => ("Agent", name.as_str()), + Some(Principal::Human(name)) => ("Human", name.as_str()), + Some(Principal::System) | None => { + return Err(Error::PrincipalRequired { + table: table.name.clone(), + }); + } + }; + // A grant row holds the access id, and the principal's kind and name as text. + let column_is = |name: &str, value: Expr| Expr::BinaryOp { + left: Box::new(Expr::Column(ColumnRef { + table: None, + column: name.to_owned(), + })), + op: BinOp::Eq, + right: Box::new(value), + }; + let both = |left: Expr, right: Expr| Expr::BinaryOp { + left: Box::new(left), + op: BinOp::And, + right: Box::new(right), + }; + let parameter = |name: &str| Expr::Parameter(name.to_owned()); + let of_kind = column_is( + "principal_kind", + Expr::Literal(Literal::Text(kind.to_owned())), + ); + let of_name = column_is("principal_id", parameter("name")); + let filter = both(column_is(access, parameter("id")), both(of_kind, of_name)); + let params = HashMap::from([ + ("id".to_owned(), Value::Uuid(Uuid::nil())), + ("name".to_owned(), Value::Text(name.to_owned())), + ]); + // A table of grants that is missing, or that no grant can be read from, grants + // nobody. + let grants = catalog.table(from).and_then(|table| { + let mut filtered = vec![false; table.meta.columns.len()]; + reads(&filter, table, ¶ms, &mut filtered).ok()?; + Some((table, filtered)) + }); + let grants = match grants { + Some((table, filtered)) => { + reservation.grow(ROOM)?; + Some(Grants { + table, + filter, + params, + filtered, + reservation, + known: BTreeMap::new(), + }) + } + None => None, + }; + acl = Some((*column, grants)); + } + Ok(Visible { table, reader, acl }) + } + + /// Marks the columns of a row that the filter reads. + pub(crate) fn reads(&self, read: &mut [bool]) { + let rules = &self.table.rules; + let context = rules + .context_column + .filter(|_| self.reader.contexts.is_some()); + let label = rules.scope_labels.as_ref().map(|(column, _)| *column); + let label = label.filter(|_| self.reader.scope_labels.is_some()); + let access = self.acl.as_ref().map(|(column, _)| *column); + for column in [context, label, access].into_iter().flatten() { + read[column] = true; + } + } + + /// `None` for a row the reader may see, and otherwise the refusal of the rule that hides + /// it. + pub(crate) fn hidden(&mut self, row: &[Value], read: &dyn Read) -> Result> { + self.refusal(row, read, false) + } + + /// Refuses a new or changed row the reader may not write. + pub(crate) fn may_write(&mut self, row: &[Value], read: &dyn Read) -> Result<()> { + self.refusal(row, read, true)?.map_or(Ok(()), Err) + } + + /// The refusal of the first rule that keeps the reader from `row`, whose values are in + /// column order: its Context, then its label, then its grant. + fn refusal(&mut self, row: &[Value], read: &dyn Read, write: bool) -> Result> { + let (table, reader) = (self.table, self.reader); + if let Some(refusal) = outside(table, reader, write, |column| row.get(column)) { + return Ok(Some(refusal)); + } + if let Some((column, grants)) = &mut self.acl { + let held = match (row.get(*column), grants) { + (Some(Value::Uuid(id)), Some(grants)) => grants.holds(*id, reader, read)?, + _ => false, + }; + if !held { + let table = table.name.clone(); + return Ok(Some(Error::AclDenied { table })); + } + } + Ok(None) + } +} + +/// The refusal of the Context rule or the label rule of `table`, in that order, when one keeps +/// `reader` from a row whose values `value` gives by column. A row with no Context is in no +/// reader's Contexts. A row is under a label the table lists, for the reading or the writing +/// that is asked, and the reader holds; a table with one list for both hides no row. +fn outside<'v>( + table: &Table, + reader: &Reader, + write: bool, + value: impl Fn(usize) -> Option<&'v Value>, +) -> Option { + let table_name = || table.name.clone(); + if let (Some(contexts), Some(column)) = (&reader.contexts, table.rules.context_column) + && !matches!(value(column), Some(Value::Uuid(id)) if contexts.contains(&ContextId(*id))) + { + return Some(Error::ContextScopeViolation { + table: table_name(), + }); + } + if let (Some(held), Some((column, labels))) = (&reader.scope_labels, &table.rules.scope_labels) + { + let listed = match (&labels.write, write) { + (Some(written), true) => written, + (Some(_), false) | (None, true) => &labels.read, + (None, false) => return None, + }; + if !matches!(value(*column), Some(Value::Text(label)) + if listed.contains(label) && held.iter().any(|held| held.0 == *label)) + { + return Some(Error::ScopeLabelViolation { + table: table_name(), + }); + } + } + None +} + +impl Grants<'_> { + /// Whether the principal holds a grant for this access id. + fn holds(&mut self, id: Uuid, reader: &Reader, read: &dyn Read) -> Result { + if let Some(known) = self.known.get(&id) { + return Ok(*known); + } + let found = self.find(id, reader, read)?; + if self.known.len() == REMEMBERED { + self.known.clear(); + } + self.known.insert(id, found); + Ok(found) + } + + /// Looks the id up in the table of grants, as any statement reads that table: through the + /// index its filter can use, and row by row when there is none. A grant counts when it + /// names the principal and the reader's Contexts and labels let it see the row of the + /// grant; the access rule of the table of grants itself is not asked. + fn find(&mut self, id: Uuid, reader: &Reader, read: &dyn Read) -> Result { + self.params.insert("id".to_owned(), Value::Uuid(id)); + let seen = Visible { + table: self.table, + reader, + acl: None, + }; + let source = Source::new(self.table, Some(&self.filter), &self.params, 0, read, seen) + .reading(self.filtered.clone(), &self.filtered); + let mut path = source.choose(&[], &[])?; + // A grant row the reader may not see is no grant, however the lookup names it. + path.one = false; + let mut found = false; + source.walk(&path, self.reservation, |_, _, bytes, reservation| { + reservation.shrink(bytes); + found = true; + Ok(false) + })?; + Ok(found) + } +} diff --git a/v2/contextdb/src/read/mod.rs b/v2/contextdb/src/read/mod.rs index 1cce806d..8d4462cd 100644 --- a/v2/contextdb/src/read/mod.rs +++ b/v2/contextdb/src/read/mod.rs @@ -130,7 +130,8 @@ impl ReadSession { } let now = system::now_millis(&*self.clock); let (view, catalog) = self.newest()?; - query::run(&plan, params, &catalog, &view, &self.budget, now) + let reader = query::Reader::admin(); + query::run(&plan, params, &catalog, &view, &reader, &self.budget, now) } /// What a statement would do, without running it: as [`crate::Database::explain`], for @@ -143,7 +144,8 @@ impl ReadSession { let plan = sql::plan(&sql::parse(sql)?)?; let now = system::now_millis(&*self.clock); let (view, catalog) = self.newest()?; - query::explain(&plan, &catalog, &view, &self.budget, now) + let admin = query::Reader::admin(); + query::explain(&plan, &catalog, &view, &admin, &self.budget, now) } /// The names of the store's tables as of the newest commit, in name order. A store with no diff --git a/v2/contextdb/src/schema/alter.rs b/v2/contextdb/src/schema/alter.rs new file mode 100644 index 00000000..a62264c9 --- /dev/null +++ b/v2/contextdb/src/schema/alter.rs @@ -0,0 +1,503 @@ +//! Changing and dropping what was declared: ALTER TABLE, DROP TABLE and DROP INDEX. +//! +//! An altered table is declared again from its own printed declaration with the change made +//! to it, so that an ALTER is refused whatever a CREATE TABLE is refused. The table keeps its +//! number, and every index, kept count and VECTOR column that is still declared keeps its own. +//! What is no longer declared is staged to leave the store in the same commit. + +use super::rules::{self, Count, RankPolicy}; +use super::{Catalog, RESERVED_PREFIXES, Table, VectorColumn, check, record, render}; +use crate::sql::ast::{AlterAction, Statement, VectorClauses}; +use crate::sql::plan::CreateTablePlan; +use crate::sql::{self, Plan}; +use crate::store::{ChangeSet, CountId, Dropped, IndexId, Read, VectorNumber}; +use crate::types::{ColumnDefault, Error, IndexDecl, KeptCounts, Result, Value}; + +/// What a declaration leaves: the catalog as it will be once the change set commits, and what +/// `database` still has to do and to answer. +pub(crate) struct Declared { + pub catalog: Catalog, + /// The indexes the declaration made on its table, which hold no entry yet. + pub indexes: Vec, + /// The kept counts the declaration made on its table, which hold no record yet. + pub counts: Vec, + /// The names of the indexes a `DROP COLUMN ... CASCADE` dropped, in the order they were + /// made. + pub dropped: Vec, +} + +impl From for Declared { + fn from(catalog: Catalog) -> Declared { + Declared { + catalog, + indexes: Vec::new(), + counts: Vec::new(), + dropped: Vec::new(), + } + } +} + +/// Applies an ALTER TABLE, a DROP TABLE or a DROP INDEX: checks it, stages the table's schema +/// record and the removal of everything the table no longer declares. +pub(super) fn declare( + plan: &Plan, + catalog: &Catalog, + changes: &mut ChangeSet, + now: i64, +) -> Result { + let (name, if_exists) = match plan { + Plan::AlterTable(alter) => (&alter.table, false), + Plan::DropTable(drop) => (&drop.name, drop.if_exists), + Plan::DropIndex(drop) => (&drop.table, false), + other => unreachable!("{other:?} alters and drops nothing"), + }; + let (tables, left) = (catalog.tables.clone(), catalog.left.clone()); + let mut declared = Declared::from(Catalog { tables, left }); + let Some(old) = catalog.table(name) else { + return match if_exists { + true => Ok(declared), + false => Err(Error::TableNotFound(name.clone())), + }; + }; + declared.catalog.left.push(old.clone()); + let table = match plan { + Plan::AlterTable(alter) => { + let action = &alter.action; + Some(altered(old, action, catalog, changes, now, &mut declared)?) + } + Plan::DropIndex(drop) => { + let made = |index: &&IndexDecl| index.name == drop.name && !engine(index); + let Some(index) = old.meta.indexes.iter().find(made) else { + return match drop.if_exists { + true => Ok(declared), + false => Err(Error::IndexNotFound { + table: name.clone(), + index: drop.name.clone(), + }), + }; + }; + ranked(catalog, name, |_, policy| { + let leads = index.columns.first(); + let joins = policy.policy.table == *name + && leads.is_some_and(|(column, _)| *column == policy.policy.column); + joins.then(|| policy.policy.column.clone()) + })?; + let without = |_: &mut CreateTablePlan, made: &mut Vec<(IndexDecl, IndexId)>| { + made.retain(|(index, _)| index.name != drop.name); + }; + Some(again(old, catalog, without)?) + } + _ => { + let mut others = catalog.tables.values().filter(|other| other.name != *name); + let refers = |other: &&Table| other.rules.references.iter().any(|r| r.parent == *name); + if let Some(child) = others.find(refers) { + return Err(check::invalid(format!( + "table '{name}' is referenced by table '{}', so it was not dropped; drop \ + that table first", + child.name + ))); + } + ranked(catalog, name, |reader, policy| { + let joins = reader.name != *name && policy.policy.table == *name; + joins.then(|| policy.policy.column.clone()) + })?; + None + } + }; + + // Whatever the table no longer declares leaves the store with this commit. + let new = table.as_ref(); + for id in &old.index_ids { + if !new.is_some_and(|table| table.index_ids.contains(id)) { + changes.stage_drop(Dropped::Index(*id))?; + } + } + let counts = |table: &Table| -> Vec { + table.rules.counts.iter().map(|count| count.id).collect() + }; + let kept = new.map(counts).unwrap_or_default(); + for id in counts(old) { + if !kept.contains(&id) { + changes.stage_drop(Dropped::Kept(id))?; + } + } + for vector in &old.vectors { + if !new.is_some_and(|table| table.vectors.iter().any(|kept| kept.id == vector.id)) { + changes.stage_drop(Dropped::Vectors(vector.id))?; + } + } + match table { + Some(table) => { + let made = table.index_ids.iter().copied(); + declared.indexes = made.filter(|id| !old.index_ids.contains(id)).collect(); + let before = counts(old); + declared.counts = kept; + declared.counts.retain(|id| !before.contains(id)); + changes.put_schema(&table.name, record::encode(&table))?; + let tables = &mut declared.catalog.tables; + tables.insert(table.name.clone(), table.shaped()); + } + None => { + changes.stage_drop(Dropped::Rows(old.id))?; + changes.delete_schema(name)?; + declared.catalog.tables.remove(name); + } + } + Ok(declared) +} + +/// The table after one ALTER TABLE action. Stages what the action does beyond the table's own +/// schema record: the cell of a dropped column, and the record of another table whose +/// reference follows a renamed column. +fn altered( + old: &Table, + action: &AlterAction, + catalog: &Catalog, + changes: &mut ChangeSet, + now: i64, + declared: &mut Declared, +) -> Result
{ + let name = &old.name; + let position = |column: &str| rules::position(name, &old.meta, column); + // The VECTOR column an `ALTER COLUMN ... SET` names. + let vector = |column: &str| -> Result<&VectorColumn> { + position(column)?; + old.vector(column).ok_or_else(|| { + check::invalid(format!( + "column '{name}.{column}' is not a VECTOR column, and this clause is a VECTOR \ + column's" + )) + }) + }; + // The table declared again with one of its options changed. + let option = |set: &dyn Fn(&mut CreateTablePlan)| again(old, catalog, |create, _| set(create)); + // The table declared again with a clause of one of its VECTOR columns changed. + let clause = |column: &str, set: &dyn Fn(&mut VectorClauses)| { + vector(column)?; + option(&|create| { + let columns = create.columns.iter_mut().filter(|c| c.name == column); + columns.for_each(|c| set(&mut c.vector)); + }) + }; + match action { + AlterAction::AddColumn(column) => { + let refused = if column.primary_key { + "PRIMARY KEY: a table's key is declared with the table" + } else if !column.nullable && column.default.is_none() && changes.count(old.id)? > 0 { + "NOT NULL with no DEFAULT on a table that has rows, which would have nothing to \ + read in it: give it a DEFAULT" + } else { + "" + }; + if !refused.is_empty() { + return Err(check::invalid(format!( + "table '{name}': the added column '{}' declares {refused}", + column.name + ))); + } + let add = |create: &mut CreateTablePlan, _: &mut Vec<(IndexDecl, IndexId)>| { + create.columns.push((**column).clone()); + }; + let mut table = again(old, catalog, add)?; + // What a row written before the column reads in it, worked out once, here. + let added = table.meta.columns.last(); + let default = added.and_then(|c| Some((c, c.default.clone()?))); + if let (Some(missing), Some((added, default))) = (table.missing.last_mut(), default) { + *missing = match default { + ColumnDefault::Literal(value) => crate::types::typed(name, added, value)?, + ColumnDefault::Now => Value::Timestamp(now), + }; + } + Ok(table) + } + AlterAction::DropColumn { column, cascade } => { + let at = position(column)?; + held(old, column, at, catalog)?; + // The column a rule of the table is declared by stays while the rule does. + let lists = match &old.meta.kept_counts { + Some(KeptCounts::Lists(lists)) => &lists[..], + _ => &[], + }; + let machine = old.meta.state_machine.iter(); + let machine = machine.map(|m| (&m.column, "STATE MACHINE")); + let counts = lists.iter().flatten().map(|(c, _)| (c, "MAINTAIN COUNT")); + if let Some((_, rule)) = machine.chain(counts).find(|(c, _)| *c == column) { + return Err(check::invalid(format!( + "column '{name}.{column}' is named by the table's {rule}, so it was not dropped" + ))); + } + let parents = catalog.tables.values().flat_map(|t| &t.rules.references); + let mut parents = parents.filter(|reference| reference.parent == *name); + if parents.any(|reference| reference.parent_columns.contains(column)) { + return Err(check::invalid(format!( + "column '{name}.{column}' is referenced by another table, so it was not \ + dropped; drop the column or the table that references it first" + ))); + } + // An index over the column stops the drop, unless the engine made it for this + // column alone or for the three columns of a table of edges: those go with it. + let names = |index: &IndexDecl| index.columns.iter().any(|(c, _)| c == column); + let stops = |index: &&IndexDecl| { + let own = engine(index) && index.columns.len() == 1; + names(index) && !own && !index.name.starts_with("__graph_") + }; + let mut stopping = old.meta.indexes.iter().filter(stops); + if *cascade != Some(true) + && let Some(index) = stopping.next() + { + return Err(Error::ColumnInIndex { + table: name.clone(), + column: column.clone(), + index: index.name.clone(), + }); + } + declared.dropped = stopping.map(|index| index.name.clone()).collect(); + let table = again(old, catalog, |create, made| { + create.columns.retain(|c| c.name != *column); + create.unique_constraints.retain(|c| !c.contains(column)); + create.foreign_keys.retain(|(c, _)| !c.contains(column)); + made.retain(|(index, _)| !names(index)); + })?; + // A row's cells are in the order of the table's columns. + changes.stage_drop(Dropped::Cell(old.id, at))?; + Ok(table) + } + AlterAction::RenameColumn { from, to } => { + let at = position(from)?; + held(old, from, at, catalog)?; + if old.meta.columns.iter().any(|column| column.name == *to) { + return Err(check::invalid(format!( + "table '{name}' already has a column '{to}'; choose another name" + ))); + } + let rename = |column: &mut String| { + if *column == *from { + column.clone_from(to); + } + }; + let mut table = old.clone(); + let meta = &mut table.meta; + rename(&mut meta.columns[at].name); + meta.unique_constraints + .iter_mut() + .flatten() + .for_each(rename); + for index in &mut meta.indexes { + index + .columns + .iter_mut() + .for_each(|(column, _)| rename(column)); + // The index behind a rule is named for the rule's columns. + let behind = RESERVED_PREFIXES[..3].iter(); + if let Some(prefix) = behind.into_iter().find(|p| index.name.starts_with(**p)) { + let columns: Vec<&str> = index.columns.iter().map(|(c, _)| &**c).collect(); + index.name = format!("{prefix}{}", columns.join("_")); + } + } + if let Some(machine) = &mut meta.state_machine { + rename(&mut machine.column); + } + if let Some(KeptCounts::Lists(lists)) = &mut meta.kept_counts { + let listed = lists.iter_mut().flatten(); + listed.for_each(|(column, _)| rename(column)); + } + table.vectors.iter_mut().for_each(|v| rename(&mut v.column)); + // A reference to the column follows it, from this table and from any other. + let follow = |child: &mut Table| { + let mut followed = false; + let to_here = child.rules.references.iter_mut(); + let to_here = to_here.filter(|reference| reference.parent == *name); + for column in to_here.flat_map(|reference| &mut reference.parent_columns) { + followed |= *column == *from; + rename(column); + } + followed + }; + follow(&mut table); + for other in declared.catalog.tables.values_mut() { + if other.id != old.id && follow(other) { + changes.put_schema(&other.name, record::encode(other))?; + } + } + again(&table, catalog, |_, _| {}) + } + AlterAction::SetRetain(retain) => { + option(&|create| create.options.retain = Some(retain.clone())) + } + AlterAction::DropRetain => option(&|create| create.options.retain = None), + AlterAction::SetHistory(history) => { + option(&|create| create.options.history = Some(*history)) + } + AlterAction::SetSync(sync) => option(&|create| create.options.sync = Some(*sync)), + AlterAction::SetConflict(conflict) => { + option(&|create| create.options.conflict = Some(*conflict)) + } + AlterAction::SetKeptCounts(kept) => { + option(&|create| create.options.kept_counts.clone_from(kept)) + } + AlterAction::SetMaxPartitions { + column, + max_partitions, + } => { + let vector = vector(column)?; + let table = clause(column, &|v| v.max_partitions = Some(max_partitions.clone()))?; + // The limit is never set below the partitions the column has. + let has = changes.vector_number(VectorNumber::Partitions(vector.id))?; + match table.vector(column).and_then(VectorColumn::limit) { + Some(limit) if has > u64::from(limit) => Err(Error::VectorPartitionLimitExceeded { + index: old.vector_ref(vector), + max_partitions: limit, + }), + _ => Ok(table), + } + } + AlterAction::SetSearchMode { + column, + search_mode, + } => clause(column, &|v| v.search_mode = Some(*search_mode)), + AlterAction::SetAutoIndexAt(c, to) => clause(c, &|v| v.auto_index_at.clone_from(to)), + AlterAction::SetConsolidation(c, to) => clause(c, &|v| v.consolidation.clone_from(to)), + } +} + +/// Whether the engine made the index for itself. +fn engine(index: &IndexDecl) -> bool { + let mut reserved = RESERVED_PREFIXES.iter(); + reserved.any(|prefix| index.name.starts_with(prefix)) +} + +/// The table `old` declared again after `edit` has changed its CREATE TABLE and the indexes of +/// CREATE INDEX it has, each with its number. Whatever the new table still declares keeps the +/// number it had, and every column still there keeps what an older row reads in it. +fn again( + old: &Table, + catalog: &Catalog, + edit: impl FnOnce(&mut CreateTablePlan, &mut Vec<(IndexDecl, IndexId)>), +) -> Result
{ + // A kept count may be led by an index of CREATE INDEX, so the counts are settled after + // those indexes are back. + let mut bare = old.clone(); + bare.meta.kept_counts = None; + bare.meta.indexes.retain(engine); + let text = render::declaration(&bare); + let text = text.trim_end().trim_end_matches(';'); + let Statement::CreateTable(mut create) = sql::parse(text)? else { + unreachable!("a table is printed back as a CREATE TABLE"); + }; + let indexes = old + .meta + .indexes + .iter() + .cloned() + .zip(old.index_ids.iter().copied()); + let mut made: Vec<(IndexDecl, IndexId)> = indexes.filter(|(index, _)| !engine(index)).collect(); + create.options.kept_counts.clone_from(&old.meta.kept_counts); + edit(&mut create, &mut made); + let kept_counts = create.options.kept_counts.take(); + + let mut table = check::new_table(&create, catalog)?; + table.id = old.id; + for (index, id) in table.meta.indexes.iter().zip(&mut table.index_ids) { + if let Some(at) = old + .meta + .indexes + .iter() + .position(|had| had.name == index.name) + { + *id = old.index_ids[at]; + } + } + for (index, id) in made { + table.meta.indexes.push(index); + table.index_ids.push(id); + } + for vector in &mut table.vectors { + if let Some(had) = old.vector(&vector.column) { + vector.id = had.id; + } + } + let meta = &table.meta; + let place = |column: &str| meta.columns.iter().position(|c| c.name == column); + let was = |column: &str| old.meta.columns.iter().position(|c| c.name == column); + let missing = meta.columns.iter().map(|column| match was(&column.name) { + Some(at) => old.missing[at].clone(), + None => Value::Null, + }); + table.missing = missing.collect(); + // A count over columns that are all still there is the count it was. + let had = old.rules.counts.iter().filter_map(|count| { + let columns = count + .columns + .iter() + .map(|(at, direction)| Some((place(&old.meta.columns[*at].name)?, *direction))); + Some(Count { + id: count.id, + columns: columns.collect::>()?, + }) + }); + let had: Vec = had.collect(); + table.meta.kept_counts = kept_counts; + table.rules.counts = rules::counts(&table.name, &table.meta, &had, catalog)?; + Ok(table) +} + +/// Refuses to drop or rename a column that something of its table is declared by: a VECTOR +/// column's PARTITION_KEY, IMMUTABLE, the primary key, the shape of a DAG table, a rank policy. +fn held(old: &Table, column: &str, at: usize, catalog: &Catalog) -> Result<()> { + let (table, column) = (old.name.clone(), column.to_owned()); + let in_key = |vector: &&VectorColumn| vector.partition_key.contains(&column); + if let Some(vector) = old.vectors.iter().find(in_key) { + let index = vector.column.clone(); + return Err(Error::VectorPartitionKeyInUse { + table, + column, + index, + }); + } + if old.rules.immutable_columns.contains(&at) { + return Err(Error::ImmutableColumn { table, column }); + } + let edge = ["source_id", "target_id", "edge_type"].contains(&&*column); + let refused = if old.key.contains(&at) { + "is in the PRIMARY KEY, which stays as the table declared it" + } else if edge && !old.meta.dag.is_empty() { + "is one of the three columns that make this DAG table a table of edges" + } else { + "" + }; + if !refused.is_empty() { + return Err(check::invalid(format!( + "column '{table}.{column}' {refused}, so it is neither dropped nor renamed" + ))); + } + ranked(catalog, &table, |reader, policy| { + let read = rules::formula_reads(&policy.policy.formula).unwrap_or_default(); + let read = read.contains(&column); + let own = reader.id == old.id + && (read || reader.meta.columns[policy.anchor].name == column) + && policy.column != at; + let joined = policy.policy.table == table && (read || policy.policy.column == column); + (own || joined).then(|| column.clone()) + }) +} + +/// Refuses a drop or an ALTER of something of `table` that a rank policy reads: `reads` says, +/// for each policy of each table, the column of `table` it reads that the statement touches. +fn ranked( + catalog: &Catalog, + table: &str, + reads: impl Fn(&Table, &RankPolicy) -> Option, +) -> Result<()> { + for reader in catalog.tables.values() { + for policy in &reader.rules.rank_policies { + if let Some(column) = reads(reader, policy) { + return Err(Error::DropBlockedByRankPolicy { + table: table.to_owned(), + column, + policy: policy.policy.sort_key.clone(), + }); + } + } + } + Ok(()) +} diff --git a/v2/contextdb/src/schema/check.rs b/v2/contextdb/src/schema/check.rs index 29b79e1d..b2e2506e 100644 --- a/v2/contextdb/src/schema/check.rs +++ b/v2/contextdb/src/schema/check.rs @@ -1,8 +1,8 @@ //! Checking a declaration: what CREATE TABLE and CREATE INDEX may say, and the table each gives. -use super::{Catalog, Consolidation, Quantization, RESERVED_PREFIXES, Table, VectorColumn}; -use crate::sql::ast::{self, ConsolidationClause, DataType, Expr, Literal}; -use crate::sql::not_supported; +use super::{Catalog, Consolidation, Quantization, RESERVED_PREFIXES, Table, VectorColumn, rules}; +use crate::sql::ast::{self, ConsolidationClause, Expr, Literal}; +use crate::sql::not_carried; use crate::sql::plan::{CreateIndexPlan, CreateTablePlan}; use crate::store::{IndexId, TableId, VectorId}; use crate::types::{ @@ -11,7 +11,7 @@ use crate::types::{ VectorPolicyDeclarationIssue as PolicyIssue, }; -fn invalid(reason: String) -> Error { +pub(super) fn invalid(reason: String) -> Error { Error::SchemaInvalid { reason } } @@ -28,62 +28,69 @@ pub(super) fn new_table(create: &CreateTablePlan, catalog: &Catalog) -> Result return Err(not_supported("a table without a PRIMARY KEY")), - 1 => {} - _ => { - return Err(invalid(format!( - "table '{table}' declares more than one PRIMARY KEY" - ))); - } + // One key to a table: written with one column, or as an element of the table. + let key = columns.iter().filter(|column| column.primary_key); + let mut key: Vec = key.map(|column| column.name.clone()).collect(); + if key.len() + usize::from(!create.primary_key.is_empty()) > 1 { + return Err(invalid(format!( + "table '{table}' declares more than one PRIMARY KEY: write one, over one column or \ + several" + ))); + } + key.extend(create.primary_key.iter().cloned()); + for column in columns + .iter_mut() + .filter(|column| key.contains(&column.name)) + { + column.primary_key = true; } + let options = &create.options; let mut meta = TableMeta { columns, unique_constraints: create.unique_constraints.clone(), indexes: Vec::new(), + immutable: options.immutable, + state_machine: options.state_machine.clone(), + dag: options.dag.clone(), + retain: options.retain.clone(), + history: options.history, + sync: options.sync, + conflict: options.conflict, + kept_counts: options.kept_counts.clone(), }; // The index behind the key, then behind each UNIQUE column, then behind each UNIQUE rule. // The key's index already keeps the key column unique. let one = |column: &ColumnDef| vec![column.name.clone()]; - let key = meta.columns.iter().filter(|column| column.primary_key); let unique = meta.columns.iter(); let unique = unique.filter(|column| column.unique && !column.primary_key); let rules = create.unique_constraints.iter().cloned(); - let behind = key - .map(|column| ("__pk_", one(column))) + let behind: Vec<(&str, Vec)> = (!key.is_empty()) + .then(|| ("__pk_", key.clone())) + .into_iter() .chain(unique.map(|column| ("__unique_", one(column)))) - .chain(rules.map(|rule| ("__unique_", rule))); + .chain(rules.map(|rule| ("__unique_", rule))) + .collect(); for (prefix, rule) in behind { - // Two rules can spell one name (`a_b`, and `a` with `b`); the later one takes a number. - let plain = format!("{prefix}{}", rule.join("_")); - let mut name = plain.clone(); - let mut copy = 1; - while meta.indexes.iter().any(|other| other.name == name) { - copy += 1; - name = format!("{plain}_{copy}"); - } - let columns = rule.into_iter().map(|column| (column, SortDirection::Asc)); - let index = IndexDecl { - name, - columns: columns.collect(), - }; - check_columns(table, &meta, &index, true)?; - meta.indexes.push(index); + let name = format!("{prefix}{}", rule.join("_")); + engine_index(table, &mut meta, name, rule, true)?; } + let rules = rules::of(create, &mut meta, catalog)?; + let position = |name: &String| meta.columns.iter().position(|column| column.name == *name); + let key: Vec = key.iter().filter_map(position).collect(); // Each VECTOR column is numbered on from the highest number in the catalog, as indexes are. - let numbered = catalog.tables.values().flat_map(|other| &other.vectors); + let numbered = catalog.numbered().flat_map(|other| &other.vectors); let next_vector = numbered.map(|vector| vector.id.0 + 1).max().unwrap_or(0); let mut vectors = Vec::new(); for declared in &create.columns { - if matches!(declared.data_type, DataType::Vector(_)) { + if matches!(declared.data_type, ColumnType::Vector(_)) { let id = VectorId(next_vector + vectors.len() as u32); vectors.push(vector_column(table, declared, &meta.columns, id)?); } } - let id = catalog.tables.values().map(|other| other.id.0 + 1).max(); + let id = catalog.numbered().map(|other| other.id.0 + 1).max(); let index_ids = (next_index_id(catalog)..) .take(meta.indexes.len()) .map(IndexId) @@ -91,12 +98,45 @@ pub(super) fn new_table(create: &CreateTablePlan, catalog: &Catalog) -> Result, + rule: bool, +) -> Result<()> { + let mut name = plain.clone(); + let mut copy = 1; + while meta.indexes.iter().any(|other| other.name == name) { + copy += 1; + name = format!("{plain}_{copy}"); + } + let columns = columns + .into_iter() + .map(|column| (column, SortDirection::Asc)); + let index = IndexDecl { + name, + columns: columns.collect(), + json_path: Vec::new(), + }; + check_columns(table, meta, &index, rule)?; + meta.indexes.push(index); + Ok(()) +} + /// What the table keeps about the VECTOR column `declared`, or the refusal of a clause of it /// that cannot stand. `columns` is every column of the table. fn vector_column( @@ -113,7 +153,7 @@ fn vector_column( index: index.clone(), issue, }; - if declared.data_type == DataType::Vector(0) { + if declared.data_type == ColumnType::Vector(0) { return Err(policy(PolicyIssue::DimensionsNotPositive)); } let max_partitions = partitions(declared, columns).map_err(|issue| { @@ -126,7 +166,11 @@ fn vector_column( let quantization = match written.map(str::to_ascii_uppercase).as_deref() { None | Some("F32") => Quantization::F32, Some("SQ8") => Quantization::Sq8, - Some("SQ4") => return Err(not_supported("WITH (quantization = 'SQ4')")), + Some("SQ4") => { + let instead = "write 'SQ8' to store one byte a component, or leave the clause out to \ + store each vector as given"; + return Err(not_carried("WITH (quantization = 'SQ4')", instead)); + } Some(_) => { let member = "quantization"; return Err(policy(PolicyIssue::InvalidMember { member })); @@ -229,6 +273,7 @@ pub(super) fn with_index(create: &CreateIndexPlan, catalog: &Catalog) -> Result< let index = IndexDecl { name: create.name.clone(), columns: create.columns.clone(), + json_path: create.json_path.clone(), }; check_columns(&table.name, &table.meta, &index, false)?; if table @@ -244,31 +289,42 @@ pub(super) fn with_index(create: &CreateIndexPlan, catalog: &Catalog) -> Result< } table.meta.indexes.push(index); table.index_ids.push(IndexId(next_index_id(catalog))); + // A table that keeps a count for every beginning of every index keeps the new index's. + table.rules.counts = rules::counts(&table.name, &table.meta, &table.rules.counts, catalog)?; Ok(table) } -/// One past the highest index number in use. Nothing in this version removes an index or a -/// table, so counting on from the highest never gives a number twice. +/// One past the highest index number in use. fn next_index_id(catalog: &Catalog) -> u32 { - let ids = catalog.tables.values().flat_map(|table| &table.index_ids); + let ids = catalog.numbered().flat_map(|table| &table.index_ids); ids.map(|id| id.0 + 1).max().unwrap_or(0) } /// Refuses an index over a column the table does not have or whose type it cannot be over. /// `rule` says the index is the one behind a key or a UNIQUE rule. fn check_columns(table: &str, meta: &TableMeta, index: &IndexDecl, rule: bool) -> Result<()> { + if index.columns.len() > 32 { + return Err(invalid(format!( + "table '{table}': '{}' names {} columns, and a key, a UNIQUE rule or an index names \ + at most 32; name fewer", + index.name, + index.columns.len() + ))); + } for (name, _) in &index.columns { let column = meta.columns.iter().find(|column| column.name == *name); let column = column.ok_or_else(|| Error::ColumnNotFound { table: table.to_owned(), column: name.clone(), })?; - // JSON and VECTOR have no order to keep an index by. A REAL has one, and is still no - // identity for a key or a UNIQUE rule: it has two zeros and a NaN. + // JSON and VECTOR have no order to keep an index by; a path into a JSON column has, + // and only a JSON column has a path. A REAL has an order, and is still no identity for + // a key or a UNIQUE rule: it has two zeros and a NaN. let refused = match column.column_type { - ColumnType::Json | ColumnType::Vector(_) => true, + ColumnType::Json => index.json_path.is_empty(), + ColumnType::Vector(_) => true, ColumnType::Real => rule, - _ => false, + _ => !index.json_path.is_empty(), }; if refused { return Err(Error::ColumnNotIndexable { @@ -283,16 +339,6 @@ fn check_columns(table: &str, meta: &TableMeta, index: &IndexDecl, rule: bool) - /// One column as the catalog keeps it. fn column(table: &str, declared: &ast::ColumnDef) -> Result { - let column_type = match declared.data_type { - DataType::Integer => ColumnType::Integer, - DataType::Real => ColumnType::Real, - DataType::Text => ColumnType::Text, - DataType::Boolean => ColumnType::Boolean, - DataType::Uuid => ColumnType::Uuid, - DataType::Timestamp => ColumnType::Timestamp, - DataType::Json => ColumnType::Json, - DataType::Vector(size) => ColumnType::Vector(size as usize), - }; let default = match &declared.default { Some(expr) => Some(default(expr).ok_or_else(|| { invalid(format!( @@ -303,14 +349,19 @@ fn column(table: &str, declared: &ast::ColumnDef) -> Result { })?), None => None, }; - Ok(ColumnDef { + let column = ColumnDef { name: declared.name.clone(), - column_type, + column_type: declared.data_type, nullable: declared.nullable, primary_key: declared.primary_key, unique: declared.unique, default, - }) + }; + // A default that a row could not hold in the column is refused here, not at the first row. + if let Some(ColumnDefault::Literal(value)) = &column.default { + crate::types::typed(table, &column, value.clone())?; + } + Ok(column) } /// A DEFAULT as the catalog keeps it, in the type the author wrote. `None` for anything but diff --git a/v2/contextdb/src/schema/mod.rs b/v2/contextdb/src/schema/mod.rs index 312632f1..9be7b95a 100644 --- a/v2/contextdb/src/schema/mod.rs +++ b/v2/contextdb/src/schema/mod.rs @@ -9,27 +9,38 @@ //! A table's indexes come in one order: the index behind the primary key (`__pk_`), //! the index behind each UNIQUE column in column order (`__unique_`; the key column has //! its key's index and no other), the index behind each UNIQUE rule of the table in declared -//! order (`__unique__...`), then the indexes of CREATE INDEX in the order they -//! were made. +//! order (`__unique__...`), the index that serves the child side of each +//! reference no earlier index leads with (`__fk_...`), the two indexes of a table of +//! edges (`__graph_edge_...`), then the indexes of CREATE INDEX in the order they were made. +mod alter; mod check; mod record; mod render; +mod rules; #[cfg(test)] mod tests; +use std::borrow::Cow; use std::collections::BTreeMap; +use std::sync::Arc; +pub(crate) use self::alter::Declared; +pub(crate) use self::rules::{Reference, Rules}; +use crate::query::Scope; use crate::sql::Plan; -use crate::store::{ChangeSet, IndexId, Read, TableId, VectorId, View}; -use crate::types::{ColumnType, Error, Result, TableMeta, VectorIndexRef, VectorSearchMode}; +use crate::store::{Cell, ChangeSet, EntryShape, IndexId, Read, Row, TableId, VectorId, View}; +use crate::types::{ + ColumnType, Error, IndexDecl, Result, TableMeta, Value, VectorIndexRef, VectorSearchMode, +}; /// What the name of an index the engine makes for itself begins with. An index of CREATE INDEX /// may not take one of these. -const RESERVED_PREFIXES: [&str; 3] = ["__pk_", "__unique_", "__fk_"]; +const RESERVED_PREFIXES: [&str; 4] = ["__pk_", "__unique_", "__fk_", "__graph_"]; -/// One declared table as the other parts use it. -#[derive(Debug, Clone)] +/// One declared table as the other parts use it. The rules a host reads are in `meta`; what +/// `schema` works out from the declaration beyond them is in `rules`. +#[derive(Clone)] pub(crate) struct Table { pub id: TableId, pub name: String, @@ -38,6 +49,15 @@ pub(crate) struct Table { pub index_ids: Vec, /// One entry per VECTOR column, in column order. pub vectors: Vec, + /// The primary key's columns in declared order; empty for a table that declares no key. + pub key: Vec, + /// The key was written as an element of the table, `PRIMARY KEY (...)`, not on its column. + pub key_is_element: bool, + /// For each column, what a row with no cell for it reads as. + pub missing: Vec, + pub rules: Rules, + /// How the entries of each index are made, in the order of `meta.indexes`. + shapes: Arc<[EntryShape]>, } /// How many partitions a VECTOR column with a PARTITION_KEY may have when its declaration @@ -141,6 +161,45 @@ impl VectorColumn { } impl Table { + /// The table's columns as an expression reaches them: under the table's name, in row order. + pub(crate) fn scope(&self) -> Scope<'_> { + Scope::of(&self.name, &self.meta.columns) + } + + /// The cell a stored row holds for a column. A row written before the column was added has + /// none, and reads as the table says such a row does. + pub(crate) fn cell<'a>(&self, row: &'a Row, column: usize) -> Cow<'a, Cell> { + match row.cells.get(column) { + Some(cell) => Cow::Borrowed(cell), + None => Cow::Owned(Cell::Here(self.missing[column].clone())), + } + } + + /// How the entries of the index at this place in `meta.indexes` are made. + pub(crate) fn entry_shape(&self, index: usize) -> &EntryShape { + &self.shapes[index] + } + + /// The table with the shape of each of its indexes, made once here: every table of a + /// catalog has been through this. + fn shaped(mut self) -> Table { + let shape = |index: &IndexDecl| { + let columns = index.columns.iter(); + let directions: Vec<_> = columns.clone().map(|(_, direction)| *direction).collect(); + let of = |name: &String| self.meta.columns.iter().find(|c| c.name == *name); + let types: Vec = match index.json_path.is_empty() { + true => columns + .filter_map(|(name, _)| Some(of(name)?.column_type)) + .collect(), + // An index over a JSON path is one TEXT column to its entries. + false => vec![ColumnType::Text], + }; + EntryShape::new(&types, &directions) + }; + self.shapes = self.meta.indexes.iter().map(shape).collect(); + self + } + /// What the table keeps about the VECTOR column of this name, `None` when it has no such /// column or the column is of another type. pub(crate) fn vector(&self, column: &str) -> Option<&VectorColumn> { @@ -169,6 +228,10 @@ impl Table { /// Every table declared in a store at one moment. pub(crate) struct Catalog { tables: BTreeMap, + /// The tables a transaction has altered or dropped, as they were before. What they held a + /// number for may still be in the store under it until the commit, so no declaration of + /// the same transaction is given one of their numbers. + left: Vec
, } impl Catalog { @@ -177,7 +240,7 @@ impl Catalog { pub(crate) fn load(view: &View) -> Result { let mut tables = BTreeMap::new(); for bytes in view.schema_records()? { - let table = record::decode(&bytes); + let table = record::decode(&bytes, view)?; // No declaration names, as a VECTOR column, a column that is not one of its table. let is_vector = |vector: &VectorColumn| { let column = table.meta.columns.iter().find(|c| c.name == vector.column); @@ -186,9 +249,10 @@ impl Catalog { if !table.vectors.iter().all(is_vector) { return Err(view.damaged("a schema record's VECTOR column is not one")); } - tables.insert(table.name.clone(), table); + tables.insert(table.name.clone(), table.shaped()); } - Ok(Catalog { tables }) + let left = Vec::new(); + Ok(Catalog { tables, left }) } /// A table by name, as SQL names it. @@ -196,6 +260,13 @@ impl Catalog { self.tables.get(name) } + /// Every table that holds numbers: those declared, and those a transaction has altered or + /// dropped, as they were. A new table, index, kept count or VECTOR column is numbered one + /// past the highest of these. + fn numbered(&self) -> impl Iterator { + self.tables.values().chain(&self.left) + } + /// Every table, in name order. pub(crate) fn tables(&self) -> Vec<&Table> { self.tables.values().collect() @@ -214,15 +285,24 @@ impl Catalog { /// /// Checks the declaration against the catalog, stages its schema record and returns the /// catalog as it will be once the change set commits. Stages no row and no index entry; after -/// a CREATE INDEX on a table that has rows, `database` asks `write` to fill the index. -pub(crate) fn declare(plan: &Plan, catalog: &Catalog, changes: &mut ChangeSet) -> Result { - let mut tables = catalog.tables.clone(); +/// a CREATE INDEX on a table that has rows, `database` asks `write` to fill the index. `now` +/// is the time of the statement, which a row written before an added column reads where the +/// column's default is `NOW()`. +pub(crate) fn declare( + plan: &Plan, + catalog: &Catalog, + changes: &mut ChangeSet, + now: i64, +) -> Result { + let (mut tables, left) = (catalog.tables.clone(), catalog.left.clone()); let table = match plan { Plan::CreateTable(create) if !tables.contains_key(&create.name) => { check::new_table(create, catalog)? } // A taken name is settled before the rest of the declaration is looked at. - Plan::CreateTable(create) if create.if_not_exists => return Ok(Catalog { tables }), + Plan::CreateTable(create) if create.if_not_exists => { + return Ok(Catalog { tables, left }.into()); + } Plan::CreateTable(create) => { return Err(Error::SchemaInvalid { reason: format!( @@ -233,11 +313,18 @@ pub(crate) fn declare(plan: &Plan, catalog: &Catalog, changes: &mut ChangeSet) - }); } Plan::CreateIndex(create) => check::with_index(create, catalog)?, - _ => return Err(crate::sql::not_supported("this declaration")), + _ => return alter::declare(plan, catalog, changes, now), }; + // The counts a CREATE INDEX made on its table hold no record yet. + let old = catalog.table(&table.name); + let before = |id| old.is_some_and(|old| old.rules.counts.iter().any(|count| count.id == id)); + let made = table.rules.counts.iter().map(|count| count.id); + let counts = made.filter(|id| !before(*id)).collect(); changes.put_schema(&table.name, record::encode(&table))?; - tables.insert(table.name.clone(), table); - Ok(Catalog { tables }) + tables.insert(table.name.clone(), table.shaped()); + let mut declared = Declared::from(Catalog { tables, left }); + declared.counts = counts; + Ok(declared) } /// The declaration as `.schema` prints it: text that, run again, declares the same table. diff --git a/v2/contextdb/src/schema/record.rs b/v2/contextdb/src/schema/record.rs index 4a868acc..b1280967 100644 --- a/v2/contextdb/src/schema/record.rs +++ b/v2/contextdb/src/schema/record.rs @@ -1,248 +1,374 @@ //! Storing a declaration: the bytes of a table's schema record, and the table read back from //! them. //! -//! Every number is four bytes, low byte first. A text is its length in bytes and then its -//! bytes. A list is its length and then its items. +//! A record begins with one byte that says which version of the record follows, and then holds +//! the table field by field. A number is its bytes, low byte first; the number of a column and +//! a length are four bytes. A text is its length in bytes and then its bytes. A list is its +//! length and then its items. A value that may be absent is one byte, 0 or 1, and then the +//! value when it is there. One of several kinds is one byte that says which. //! -//! Reading trusts the bytes: a record is only ever what `encode` wrote, and the store checks -//! every page it reads. Where a byte says which of several kinds follows, the last kind is read -//! for any byte the earlier ones do not claim. +//! Reading trusts no byte: a length past the record's end, a byte that names no kind, a column +//! number that is no column and bytes left over are all damage. -use super::{Consolidation, Quantization, Table, VectorColumn}; -use crate::store::{IndexId, TableId, VectorId}; +use super::rules::{Count, RankPolicy, Reference}; +use super::{Consolidation, Quantization, Rules, Table, VectorColumn}; +use crate::sql::ast; +use crate::store::{CountId, IndexId, Read, TableId, VectorId, View}; use crate::types::{ - ColumnDef, ColumnDefault, ColumnType, IndexDecl, SortDirection, TableMeta, Value, + ColumnDef, ColumnDefault, ColumnType, ConflictPolicy, History, IndexDecl, KeptCounts, Result, + Retain, RetainUnit, SortDirection, StateMachine, SyncDirection, TableMeta, TxId, Uuid, Value, VectorSearchMode, }; +/// The version of the record this code writes, and the only one it reads. +const VERSION: u8 = 1; + /// The schema record of a table. pub(super) fn encode(table: &Table) -> Vec { - let meta = &table.meta; - let mut out = Vec::new(); - number(&mut out, table.id.0 as usize); - text(&mut out, &table.name); - number(&mut out, meta.columns.len()); - for column in &meta.columns { - text(&mut out, &column.name); - let (kind, size) = match column.column_type { - ColumnType::Integer => (0, 0), - ColumnType::Real => (1, 0), - ColumnType::Text => (2, 0), - ColumnType::Boolean => (3, 0), - ColumnType::Json => (4, 0), - ColumnType::Uuid => (5, 0), - ColumnType::Timestamp => (6, 0), - ColumnType::Vector(size) => (8, size), - }; - out.push(kind); - number(&mut out, size); - out.push(u8::from(column.nullable)); - out.push(u8::from(column.primary_key)); - out.push(u8::from(column.unique)); - match &column.default { - None => out.push(0), - Some(ColumnDefault::Now) => out.push(1), - Some(ColumnDefault::Literal(Value::Null)) => out.push(2), - Some(ColumnDefault::Literal(Value::Bool(flag))) => out.extend([3, u8::from(*flag)]), - Some(ColumnDefault::Literal(Value::Int64(whole))) => { - out.push(4); - out.extend(whole.to_le_bytes()); - } - Some(ColumnDefault::Literal(Value::Float64(real))) => { - out.push(5); - out.extend(real.to_le_bytes()); - } - Some(ColumnDefault::Literal(Value::Text(words))) => { - out.push(6); - text(&mut out, words); + let mut out = vec![VERSION]; + table.put(&mut out); + out +} + +/// The table a schema record was written from; damage when the bytes are not a record. +pub(super) fn decode(record: &[u8], view: &View) -> Result
{ + let mut read = Reader(record); + let table = match u8::get(&mut read) { + Some(VERSION) => Table::get(&mut read), + _ => None, + }; + table + .filter(|table| read.0.is_empty() && agrees(table)) + .ok_or_else(|| view.damaged("a schema record does not read as a table")) +} + +/// Whether every column and index a table's rules name by number is one of the table's. +fn agrees(table: &Table) -> bool { + let (meta, rules) = (&table.meta, &table.rules); + let references = rules.references.iter(); + let counts = rules.counts.iter(); + let policies = rules.rank_policies.iter(); + let mut columns = (table.key.iter()) + .chain(&rules.immutable_columns) + .chain(references.clone().flat_map(|reference| &reference.columns)) + .chain(counts.flat_map(|count| count.columns.iter().map(|(column, _)| column))) + .chain(&rules.context_column) + .chain(rules.scope_labels.iter().map(|(column, _)| column)) + .chain(rules.acl.iter().map(|(column, ..)| column)) + .chain(policies.flat_map(|policy| [&policy.column, &policy.anchor])); + columns.all(|column| *column < meta.columns.len()) + && table.missing.len() == meta.columns.len() + && table.index_ids.len() == meta.indexes.len() + && { references }.all(|reference| reference.index < meta.indexes.len()) +} + +/// What is left of a record being read. +struct Reader<'a>(&'a [u8]); + +impl<'a> Reader<'a> { + fn bytes(&mut self, count: usize) -> Option<&'a [u8]> { + let (taken, rest) = self.0.split_at_checked(count)?; + self.0 = rest; + Some(taken) + } +} + +/// A value as a schema record holds it. +trait Stored: Sized { + fn put(&self, out: &mut Vec); + + /// The value at the front of what is left; `None` when the bytes there are not one. + fn get(read: &mut Reader) -> Option; +} + +macro_rules! numbers { + ($($number:ty),*) => {$( + impl Stored for $number { + fn put(&self, out: &mut Vec) { + out.extend(self.to_le_bytes()); } - Some(ColumnDefault::Literal(other)) => { - unreachable!("a default is declared as a literal, never as {other:?}") + + fn get(read: &mut Reader) -> Option { + let bytes = read.bytes(size_of::())?; + Some(Self::from_le_bytes(bytes.try_into().ok()?)) } } + )*}; +} +numbers!(u8, u32, u64, u128, i64, f32, f64); + +impl Stored for usize { + fn put(&self, out: &mut Vec) { + (*self as u32).put(out); } - number(&mut out, meta.unique_constraints.len()); - for rule in &meta.unique_constraints { - number(&mut out, rule.len()); - for column in rule { - text(&mut out, column); - } + + fn get(read: &mut Reader) -> Option { + u32::get(read).map(|number| number as usize) + } +} + +impl Stored for bool { + fn put(&self, out: &mut Vec) { + out.push(u8::from(*self)); } - number(&mut out, meta.indexes.len()); - for (index, id) in meta.indexes.iter().zip(&table.index_ids) { - number(&mut out, id.0 as usize); - text(&mut out, &index.name); - number(&mut out, index.columns.len()); - for (column, direction) in &index.columns { - text(&mut out, column); - out.push(u8::from(*direction == SortDirection::Desc)); + + fn get(read: &mut Reader) -> Option { + match u8::get(read)? { + 0 => Some(false), + 1 => Some(true), + _ => None, } } - number(&mut out, table.vectors.len()); - for vector in &table.vectors { - number(&mut out, vector.id.0 as usize); - text(&mut out, &vector.column); - number(&mut out, vector.partition_key.len()); - for column in &vector.partition_key { - text(&mut out, column); +} + +impl Stored for String { + fn put(&self, out: &mut Vec) { + self.len().put(out); + out.extend(self.as_bytes()); + } + + fn get(read: &mut Reader) -> Option { + let count = usize::get(read)?; + String::from_utf8(read.bytes(count)?.to_vec()).ok() + } +} + +impl Stored for Vec { + fn put(&self, out: &mut Vec) { + self.len().put(out); + self.iter().for_each(|item| item.put(out)); + } + + fn get(read: &mut Reader) -> Option { + // Every item takes a byte at least, so a length past the record's end stops there. + (0..usize::get(read)?).map(|_| T::get(read)).collect() + } +} + +impl Stored for Option { + fn put(&self, out: &mut Vec) { + self.is_some().put(out); + self.iter().for_each(|value| value.put(out)); + } + + fn get(read: &mut Reader) -> Option { + match bool::get(read)? { + true => T::get(read).map(Some), + false => Some(None), } - // A declared MAX_PARTITIONS is never zero, so zero says that none was declared. - number(&mut out, vector.max_partitions.unwrap_or(0) as usize); - out.push(match vector.search_mode { - None => 0, - Some(VectorSearchMode::Auto) => 1, - Some(VectorSearchMode::Exact) => 2, - Some(VectorSearchMode::Indexed) => 3, - }); - // A declared AUTO_INDEX_AT and a declared percentage are never zero either. - out.extend(vector.auto_index_at.unwrap_or(0).to_le_bytes()); - out.extend(match vector.consolidation { - None => [0, 0, 0], - Some(Consolidation::None) => [1, 0, 0], - Some(Consolidation::Percent { change, tombstone }) => { - [2, change.unwrap_or(0), tombstone.unwrap_or(0)] - } - }); - out.push(u8::from(vector.quantization == Quantization::Sq8)); } - out } -/// The table a schema record was written from. -pub(super) fn decode(record: &[u8]) -> Table { - let mut read = Reader(record); - let id = TableId(read.number() as u32); - let name = read.text(); - let columns = read.list(|read| { - let name = read.text(); - let kind = read.byte(); - let size = read.number(); - ColumnDef { - name, - column_type: match kind { - 0 => ColumnType::Integer, - 1 => ColumnType::Real, - 2 => ColumnType::Text, - 3 => ColumnType::Boolean, - 4 => ColumnType::Json, - 5 => ColumnType::Uuid, - 6 => ColumnType::Timestamp, - _ => ColumnType::Vector(size), - }, - nullable: read.flag(), - primary_key: read.flag(), - unique: read.flag(), - default: match read.byte() { - 0 => None, - 1 => Some(ColumnDefault::Now), - kind => Some(ColumnDefault::Literal(match kind { - 2 => Value::Null, - 3 => Value::Bool(read.flag()), - 4 => Value::Int64(i64::from_le_bytes(read.array())), - 5 => Value::Float64(f64::from_le_bytes(read.array())), - _ => Value::Text(read.text()), - })), - }, +macro_rules! tuples { + ($(($($item:ident $at:tt),*))*) => {$( + impl<$($item: Stored),*> Stored for ($($item,)*) { + fn put(&self, out: &mut Vec) { + $(self.$at.put(out);)* + } + + fn get(read: &mut Reader) -> Option { + Some(($($item::get(read)?,)*)) + } } - }); - let unique_constraints = read.list(|read| read.list(Reader::text)); - let mut index_ids = Vec::new(); - let indexes = read.list(|read| { - index_ids.push(IndexId(read.number() as u32)); - let name = read.text(); - let columns = read.list(|read| { - let column = read.text(); - let direction = if read.flag() { - SortDirection::Desc - } else { - SortDirection::Asc - }; - (column, direction) - }); - IndexDecl { name, columns } - }); - let vectors = read.list(|read| VectorColumn { - id: VectorId(read.number() as u32), - column: read.text(), - partition_key: read.list(Reader::text), - max_partitions: Some(read.number() as u32).filter(|limit| *limit != 0), - search_mode: match read.byte() { - 0 => None, - 1 => Some(VectorSearchMode::Auto), - 2 => Some(VectorSearchMode::Exact), - _ => Some(VectorSearchMode::Indexed), - }, - auto_index_at: Some(u64::from_le_bytes(read.array())).filter(|at| *at != 0), - consolidation: match read.array::<3>() { - [0, ..] => None, - [1, ..] => Some(Consolidation::None), - [_, change, tombstone] => Some(Consolidation::Percent { - change: Some(change).filter(|percent| *percent != 0), - tombstone: Some(tombstone).filter(|percent| *percent != 0), - }), - }, - quantization: if read.flag() { - Quantization::Sq8 - } else { - Quantization::F32 - }, - }); - Table { - id, - name, - meta: TableMeta { - columns, - unique_constraints, - indexes, - }, - index_ids, - vectors, + )*}; +} +tuples!((A 0, B 1) (A 0, B 1, C 2)); + +/// A type whose record is its fields in this order. A field after `;` is not stored: it is +/// worked out from the others when the table is loaded. +macro_rules! fields { + ($($name:ty { $($field:tt),* $(; $made:ident)? })*) => {$( + impl Stored for $name { + fn put(&self, out: &mut Vec) { + $(self.$field.put(out);)* + } + + fn get(read: &mut Reader) -> Option { + Some(Self { + $($field: Stored::get(read)?,)* + $($made: Default::default(),)? + }) + } + } + )*}; +} +fields! { + TableId { 0 } + IndexId { 0 } + VectorId { 0 } + CountId { 0 } + Table { id, name, meta, index_ids, vectors, key, key_is_element, missing, rules; shapes } + TableMeta { + columns, unique_constraints, indexes, immutable, state_machine, dag, retain, history, + sync, conflict, kept_counts + } + ColumnDef { name, column_type, nullable, primary_key, unique, default } + IndexDecl { name, columns, json_path } + VectorColumn { + id, column, partition_key, max_partitions, search_mode, auto_index_at, consolidation, + quantization } + StateMachine { column, transitions } + Retain { count, unit, sync_safe } + Rules { + immutable_columns, references, propagate_edges, exclude_vector_states, + delivery_manifest, edge_discard, counts, context_column, scope_labels, acl, + rank_policies + } + Reference { columns, parent, parent_columns, on_state, index } + Count { id, columns } + RankPolicy { column, anchor, policy } + ast::Propagation { on_state, set, max_depth, abort_on_failure } + ast::EdgePropagation { edge_type, direction, rule } + ast::ScopeLabels { read, write } + ast::RankPolicy { table, column, formula, sort_key } } -fn number(out: &mut Vec, number: usize) { - out.extend((number as u32).to_le_bytes()); +/// A type that is one of these kinds and nothing more: one byte, the kind's place here. +macro_rules! kinds { + ($($name:ty { $($kind:ident),* })*) => {$( + impl Stored for $name { + fn put(&self, out: &mut Vec) { + out.push(*self as u8); + } + + fn get(read: &mut Reader) -> Option { + let kind = u8::get(read)?; + [$(Self::$kind),*].into_iter().find(|known| *known as u8 == kind) + } + } + )*}; +} +kinds! { + SortDirection { Asc, Desc } + VectorSearchMode { Auto, Exact, Indexed } + Quantization { F32, Sq8 } + RetainUnit { Seconds, Minutes, Hours, Days } + History { All, CurrentOnly } + SyncDirection { Off, PushOnly, PullOnly, TwoWay } + ConflictPolicy { KeepFirst, KeepLatest } + ast::EdgeDiscard { Never, AfterOutcome, Always } } -fn text(out: &mut Vec, text: &str) { - number(out, text.len()); - out.extend(text.as_bytes()); +impl Stored for ColumnType { + fn put(&self, out: &mut Vec) { + let kind: (u8, usize) = match *self { + ColumnType::Integer => (0, 0), + ColumnType::Real => (1, 0), + ColumnType::Text => (2, 0), + ColumnType::Boolean => (3, 0), + ColumnType::Json => (4, 0), + ColumnType::Uuid => (5, 0), + ColumnType::Timestamp => (6, 0), + ColumnType::TxId => (7, 0), + ColumnType::Vector(size) => (8, size), + }; + kind.put(out); + } + + fn get(read: &mut Reader) -> Option { + let (kind, size) = <(u8, usize)>::get(read)?; + Some(match kind { + 0 => ColumnType::Integer, + 1 => ColumnType::Real, + 2 => ColumnType::Text, + 3 => ColumnType::Boolean, + 4 => ColumnType::Json, + 5 => ColumnType::Uuid, + 6 => ColumnType::Timestamp, + 7 => ColumnType::TxId, + 8 => ColumnType::Vector(size), + _ => return None, + }) + } } -/// What is left of a record being read. -struct Reader<'a>(&'a [u8]); +impl Stored for ColumnDefault { + fn put(&self, out: &mut Vec) { + match self { + ColumnDefault::Now => out.push(0), + ColumnDefault::Literal(value) => (1u8, value.clone()).put(out), + } + } -impl<'a> Reader<'a> { - fn bytes(&mut self, count: usize) -> &'a [u8] { - let (taken, rest) = self.0.split_at(count); - self.0 = rest; - taken + fn get(read: &mut Reader) -> Option { + match u8::get(read)? { + 0 => Some(ColumnDefault::Now), + 1 => Value::get(read).map(ColumnDefault::Literal), + _ => None, + } } +} - fn array(&mut self) -> [u8; N] { - let mut array = [0; N]; - array.copy_from_slice(self.bytes(N)); - array +/// A default's value, and what a row with no cell for a column reads as. +impl Stored for Value { + fn put(&self, out: &mut Vec) { + match self { + Value::Null => out.push(0), + Value::Bool(flag) => (1u8, *flag).put(out), + Value::Int64(whole) => (2u8, *whole).put(out), + Value::Float64(real) => (3u8, *real).put(out), + Value::Text(words) => (4u8, words.clone()).put(out), + Value::Uuid(uuid) => (5u8, uuid.as_u128()).put(out), + Value::Timestamp(millis) => (6u8, *millis).put(out), + Value::Json(json) => (7u8, json.to_string()).put(out), + Value::Vector(vector) => (8u8, vector.clone()).put(out), + Value::TxId(tx) => (9u8, tx.0).put(out), + } } - fn byte(&mut self) -> u8 { - self.bytes(1)[0] + fn get(read: &mut Reader) -> Option { + Some(match u8::get(read)? { + 0 => Value::Null, + 1 => Value::Bool(Stored::get(read)?), + 2 => Value::Int64(Stored::get(read)?), + 3 => Value::Float64(Stored::get(read)?), + 4 => Value::Text(Stored::get(read)?), + 5 => Value::Uuid(Uuid::from_u128(Stored::get(read)?)), + 6 => Value::Timestamp(Stored::get(read)?), + 7 => Value::Json(serde_json::from_str(&String::get(read)?).ok()?), + 8 => Value::Vector(Stored::get(read)?), + 9 => Value::TxId(TxId(Stored::get(read)?)), + _ => return None, + }) } +} - fn flag(&mut self) -> bool { - self.byte() != 0 +impl Stored for Consolidation { + fn put(&self, out: &mut Vec) { + match *self { + Consolidation::None => out.push(0), + Consolidation::Percent { change, tombstone } => (1u8, change, tombstone).put(out), + } } - fn number(&mut self) -> usize { - u32::from_le_bytes(self.array()) as usize + fn get(read: &mut Reader) -> Option { + match u8::get(read)? { + 0 => Some(Consolidation::None), + 1 => Some(Consolidation::Percent { + change: Stored::get(read)?, + tombstone: Stored::get(read)?, + }), + _ => None, + } } +} - fn text(&mut self) -> String { - let count = self.number(); - String::from_utf8_lossy(self.bytes(count)).into_owned() +impl Stored for KeptCounts { + fn put(&self, out: &mut Vec) { + match self { + KeptCounts::None => out.push(0), + KeptCounts::AllPrefixes => out.push(1), + KeptCounts::Lists(lists) => (2u8, lists.clone()).put(out), + } } - fn list(&mut self, mut item: impl FnMut(&mut Self) -> T) -> Vec { - (0..self.number()).map(|_| item(self)).collect() + fn get(read: &mut Reader) -> Option { + match u8::get(read)? { + 0 => Some(KeptCounts::None), + 1 => Some(KeptCounts::AllPrefixes), + 2 => Stored::get(read).map(KeptCounts::Lists), + _ => None, + } } } diff --git a/v2/contextdb/src/schema/render.rs b/v2/contextdb/src/schema/render.rs index b4113a05..8ecf53ff 100644 --- a/v2/contextdb/src/schema/render.rs +++ b/v2/contextdb/src/schema/render.rs @@ -1,24 +1,125 @@ //! Printing a declaration back as SQL. use super::{Consolidation, Quantization, RESERVED_PREFIXES, Table}; -use crate::types::{ColumnDef, ColumnDefault, ColumnType, SortDirection, Value, VectorSearchMode}; +use crate::sql::ast::{EdgeDiscard, Propagation}; +use crate::types::{ + ColumnDefault, ConflictPolicy, History, KeptCounts, RetainUnit, SortDirection, SyncDirection, + Value, VectorSearchMode, +}; /// A table's declaration as SQL: the CREATE TABLE, then one CREATE INDEX for each index the -/// author made, every statement ended by `;` and a new line. A clause left at its default is -/// not printed, and neither are the indexes the engine keeps behind a key or a UNIQUE rule. +/// author made, every statement ended by `;` and a new line. The clauses come in one order +/// whatever order they were written in. A clause left at its default is not printed, except +/// `MAX_PARTITIONS` of a column with a `PARTITION_KEY`, and neither are the indexes the engine makes for itself. pub(super) fn declaration(table: &Table) -> String { - let meta = &table.meta; - let mut lines: Vec = meta - .columns - .iter() - .map(|declared| column(table, declared)) + let (meta, rules) = (&table.meta, &table.rules); + let column_name = |at: &usize| name(&meta.columns[*at].name); + let mut lines: Vec = (0..meta.columns.len()) + .map(|at| column(table, at)) .collect(); + if table.key_is_element { + lines.push(format!("PRIMARY KEY ({})", list(&table.key, column_name))); + } for rule in &meta.unique_constraints { - let columns: Vec = rule.iter().map(|column| name(column)).collect(); - lines.push(format!("UNIQUE ({})", columns.join(", "))); + lines.push(format!("UNIQUE ({})", list(rule, |column| name(column)))); + } + for reference in rules.references.iter().filter(|r| r.columns.len() > 1) { + let columns = list(&reference.columns, column_name); + lines.push(format!("FOREIGN KEY ({columns}) {}", parent(reference))); + } + + let mut options: Vec = Vec::new(); + if meta.immutable { + options.push("IMMUTABLE".to_owned()); + } + if let Some(machine) = &meta.state_machine { + let transition = |(from, to): &(String, Vec)| { + format!( + "{} -> [{}]", + as_written(from), + list(to, |state| as_written(state)) + ) + }; + let transitions = list(&machine.transitions, transition); + options.push(format!( + "STATE MACHINE ({}: {transitions})", + name(&machine.column) + )); + } + if !meta.dag.is_empty() { + options.push(format!("DAG({})", list(&meta.dag, |edge| quoted(edge)))); + } + for edge in &rules.propagate_edges { + let (edge_type, rule) = (as_written(&edge.edge_type), &edge.rule); + options.push(format!( + "PROPAGATE ON EDGE {edge_type} {} STATE {} SET {}{}", + edge.direction, + as_written(&rule.on_state), + as_written(&rule.set), + limits(rule) + )); + } + for state in &rules.exclude_vector_states { + options.push(format!( + "PROPAGATE ON STATE {} EXCLUDE VECTOR", + as_written(state) + )); + } + if let Some(retain) = &meta.retain { + let unit = match retain.unit { + RetainUnit::Seconds => "SECONDS", + RetainUnit::Minutes => "MINUTES", + RetainUnit::Hours => "HOURS", + RetainUnit::Days => "DAYS", + }; + let safe = if retain.sync_safe { " SYNC SAFE" } else { "" }; + options.push(format!("RETAIN {} {unit}{safe}", retain.count)); + } + let words = [ + meta.history.map(|history| match history { + History::All => "HISTORY ALL", + History::CurrentOnly => "HISTORY CURRENT ONLY", + }), + meta.sync.map(|sync| match sync { + SyncDirection::Off => "SYNC OFF", + SyncDirection::PushOnly => "SYNC PUSH ONLY", + SyncDirection::PullOnly => "SYNC PULL ONLY", + SyncDirection::TwoWay => "SYNC TWO WAY", + }), + meta.conflict.map(|conflict| match conflict { + ConflictPolicy::KeepFirst => "SYNC CONFLICT KEEP FIRST", + ConflictPolicy::KeepLatest => "SYNC CONFLICT KEEP LATEST", + }), + ]; + options.extend(words.into_iter().flatten().map(str::to_owned)); + if !rules.delivery_manifest.is_empty() { + let over = list(&rules.delivery_manifest, |table| name(table)); + options.push(format!("DELIVERY MANIFEST OVER {over}")); } + options.extend(rules.edge_discard.map(|discard| { + let when = match discard { + EdgeDiscard::Never => "NEVER", + EdgeDiscard::AfterOutcome => "AFTER OUTCOME", + EdgeDiscard::Always => "ALWAYS", + }; + format!("EDGE DISCARD {when}") + })); + options.extend(meta.kept_counts.as_ref().map(|counts| { + let counts = match counts { + KeptCounts::None => "NONE".to_owned(), + KeptCounts::AllPrefixes => "ALL PREFIXES".to_owned(), + KeptCounts::Lists(lists) => list(lists, |columns| format!("({})", sorted(columns))), + }; + format!("MAINTAIN COUNT {counts}") + })); + let table = name(&table.name); - let mut text = format!("CREATE TABLE {table} (\n {}\n);\n", lines.join(",\n ")); + let mut text = format!("CREATE TABLE {table} (\n {}\n)", lines.join(",\n ")); + for option in options { + text.push(' '); + text.push_str(&option); + } + text.push_str(";\n"); for index in &meta.indexes { if RESERVED_PREFIXES .iter() @@ -26,37 +127,24 @@ pub(super) fn declaration(table: &Table) -> String { { continue; } - let columns: Vec = index - .columns - .iter() - .map(|(column, direction)| match direction { - SortDirection::Asc => name(column), - SortDirection::Desc => format!("{} DESC", name(column)), - }) - .collect(); - text.push_str(&format!( - "CREATE INDEX {} ON {table} ({});\n", - name(&index.name), - columns.join(", ") - )); + let mut columns = sorted(&index.columns); + if !index.json_path.is_empty() { + let path = index.json_path.iter().map(|key| quoted(key)); + let path: Vec = path.collect(); + columns = format!("({columns} -> {})", path.join(" -> ")); + } + let index = name(&index.name); + text.push_str(&format!("CREATE INDEX {index} ON {table} ({columns});\n")); } text } /// One column's line: its name, its type, and the clauses it was given. -fn column(table: &Table, column: &ColumnDef) -> String { +fn column(table: &Table, at: usize) -> String { + let (column, rules) = (&table.meta.columns[at], &table.rules); let mut text = name(&column.name); text.push(' '); - match column.column_type { - ColumnType::Integer => text.push_str("INTEGER"), - ColumnType::Real => text.push_str("REAL"), - ColumnType::Text => text.push_str("TEXT"), - ColumnType::Boolean => text.push_str("BOOLEAN"), - ColumnType::Json => text.push_str("JSON"), - ColumnType::Uuid => text.push_str("UUID"), - ColumnType::Timestamp => text.push_str("TIMESTAMP"), - ColumnType::Vector(size) => text.push_str(&format!("VECTOR({size})")), - } + text.push_str(&column.column_type.to_string()); if let Some(vector) = table.vector(&column.name) { if vector.quantization == Quantization::Sq8 { text.push_str(" WITH (quantization = 'SQ8')"); @@ -65,11 +153,8 @@ fn column(table: &Table, column: &ColumnDef) -> String { // PARTITION_KEY, whether or not it was written. The clauses after it are printed only // when they were written. if let Some(limit) = vector.limit() { - let key: Vec = vector.partition_key.iter().map(|key| name(key)).collect(); - text.push_str(&format!( - " PARTITION_KEY ({}) MAX_PARTITIONS {limit}", - key.join(", ") - )); + let key = list(&vector.partition_key, |key| name(key)); + text.push_str(&format!(" PARTITION_KEY ({key}) MAX_PARTITIONS {limit}")); } text.push_str(match vector.search_mode { None => "", @@ -93,7 +178,37 @@ fn column(table: &Table, column: &ColumnDef) -> String { } } } - if column.primary_key { + for ranked in rules.rank_policies.iter().filter(|p| p.column == at) { + let policy = &ranked.policy; + text.push_str(&format!( + " RANK_POLICY (JOIN {} ON {}, FORMULA {}, SORT_KEY {})", + name(&policy.table), + name(&policy.column), + quoted(&policy.formula), + name(&policy.sort_key) + )); + } + if let Some((_, labels)) = rules.scope_labels.as_ref().filter(|(c, _)| *c == at) { + let read = list(&labels.read, |label| quoted(label)); + text.push_str(&match &labels.write { + Some(write) => { + let write = list(write, |label| quoted(label)); + format!(" SCOPE_LABEL_READ ({read}) WRITE ({write})") + } + None => format!(" SCOPE_LABEL ({read})"), + }); + } + if let Some((_, grants, grant)) = rules.acl.as_ref().filter(|(c, ..)| *c == at) { + text.push_str(&format!( + " ACL REFERENCES {}({})", + name(grants), + name(grant) + )); + } + if rules.context_column == Some(at) { + text.push_str(" CONTEXT_ID"); + } + if column.primary_key && !table.key_is_element { text.push_str(" PRIMARY KEY"); } if !column.nullable { @@ -110,9 +225,56 @@ fn column(table: &Table, column: &ColumnDef) -> String { if column.unique { text.push_str(" UNIQUE"); } + if rules.immutable_columns.contains(&at) { + text.push_str(" IMMUTABLE"); + } + for reference in rules.references.iter().filter(|r| r.columns == [at]) { + text.push(' '); + text.push_str(&parent(reference)); + } text } +/// What a reference says after its own columns: the parent, its columns, and the rule. +fn parent(reference: &super::rules::Reference) -> String { + let columns = list(&reference.parent_columns, |column| name(column)); + let mut text = format!("REFERENCES {}({columns})", name(&reference.parent)); + if let Some(rule) = &reference.on_state { + text.push_str(&format!( + " ON STATE {} PROPAGATE SET {}{}", + as_written(&rule.on_state), + as_written(&rule.set), + limits(rule) + )); + } + text +} + +/// The end of a propagation rule: how far it goes and whether a failure stops it, as written. +fn limits(rule: &Propagation) -> String { + let depth = rule.max_depth.map(|depth| format!(" MAX DEPTH {depth}")); + let abort = if rule.abort_on_failure { + " ABORT ON FAILURE" + } else { + "" + }; + depth.unwrap_or_default() + abort +} + +/// Columns with the direction of each, as an index or a kept count lists them. +fn sorted(columns: &[(String, SortDirection)]) -> String { + list(columns, |(column, direction)| match direction { + SortDirection::Asc => name(column), + SortDirection::Desc => format!("{} DESC", name(column)), + }) +} + +/// The items, each as `show` prints it, with a comma between two. +fn list(items: &[T], show: impl Fn(&T) -> String) -> String { + let items: Vec = items.iter().map(show).collect(); + items.join(", ") +} + /// A default's value as the author wrote it. fn literal(value: &Value) -> String { match value { @@ -123,18 +285,35 @@ fn literal(value: &Value) -> String { // A whole REAL keeps its decimal point, so it reads back as a REAL. Value::Float64(real) if real.fract() == 0.0 => format!("{real:.1}"), Value::Float64(real) => real.to_string(), - Value::Text(words) => format!("'{}'", words.replace('\'', "''")), + Value::Text(words) => quoted(words), other => unreachable!("a default is declared as a literal, never as {other:?}"), } } -/// A name as SQL reads it back: bare when it is one plain word, in double quotes otherwise. +/// A text between single quotes. +fn quoted(text: &str) -> String { + format!("'{}'", text.replace('\'', "''")) +} + +/// The name of a table, a column or an index as SQL reads it back: bare when it is one plain +/// word in lower case, which is how a name without quotes is stored, in double quotes otherwise. fn name(name: &str) -> String { - let plain = name.starts_with(|first: char| first.is_ascii_alphabetic() || first == '_') - && name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_'); - if plain { - name.to_owned() + word(name, !name.contains(|c: char| c.is_ascii_uppercase())) +} + +/// A state or an edge type, which keeps its letters with or without quotes: bare when it is +/// one plain word. +fn as_written(text: &str) -> String { + word(text, true) +} + +/// `text` bare when `bare` allows it and it is one plain word, in double quotes otherwise. +fn word(text: &str, bare: bool) -> String { + let plain = text.starts_with(|first: char| first.is_ascii_alphabetic() || first == '_') + && text.chars().all(|c| c.is_ascii_alphanumeric() || c == '_'); + if bare && plain { + text.to_owned() } else { - format!("\"{}\"", name.replace('"', "\"\"")) + format!("\"{}\"", text.replace('"', "\"\"")) } } diff --git a/v2/contextdb/src/schema/rules.rs b/v2/contextdb/src/schema/rules.rs new file mode 100644 index 00000000..bbbce118 --- /dev/null +++ b/v2/contextdb/src/schema/rules.rs @@ -0,0 +1,439 @@ +//! The rules of a declared table beyond its columns, its key and its UNIQUE rules: what a +//! CREATE TABLE may say of them, the combinations that cannot stand, and what the table keeps. + +use super::Catalog; +use super::check::{engine_index, invalid}; +use crate::sql::ast; +use crate::sql::plan::CreateTablePlan; +use crate::store::CountId; +use crate::types::{ + ColumnType, ConflictPolicy, Error, History, KeptCounts, Result, SortDirection, SyncDirection, + TableMeta, VectorIndexRef, +}; + +/// What a table keeps of its declaration beside `TableMeta`. A column is its place in +/// `meta.columns`; a state, an edge type and a label keep the letters they were written in. +#[derive(Debug, Clone, Default)] +pub(crate) struct Rules { + /// The columns declared IMMUTABLE. + pub immutable_columns: Vec, + pub references: Vec, + pub propagate_edges: Vec, + /// The states of `PROPAGATE ON STATE ... EXCLUDE VECTOR`. + pub exclude_vector_states: Vec, + pub delivery_manifest: Vec, + pub edge_discard: Option, + /// The counts the table keeps: the lists of MAINTAIN COUNT, or every beginning of every + /// index under ALL PREFIXES. + pub counts: Vec, + /// The column declared CONTEXT_ID. + pub context_column: Option, + /// The column that holds a row's labels, and the labels a reader and a writer need. + pub scope_labels: Option<(usize, ast::ScopeLabels)>, + /// The column declared ACL, then the table and the column of grants it references. + pub acl: Option<(usize, String, String)>, + pub rank_policies: Vec, +} + +/// One reference of a table to a parent's key or UNIQUE rule. +#[derive(Debug, Clone)] +pub(crate) struct Reference { + pub columns: Vec, + pub parent: String, + pub parent_columns: Vec, + pub on_state: Option, + /// The index of the table that leads with `columns`, by its place in `meta.indexes`. + pub index: usize, +} + +/// One kept count: its number in the store, and its columns with the direction of each. +#[derive(Debug, Clone)] +pub(crate) struct Count { + pub id: CountId, + pub columns: Vec<(usize, SortDirection)>, +} + +/// The RANK_POLICY of a VECTOR column. +#[derive(Debug, Clone)] +pub(crate) struct RankPolicy { + /// The VECTOR column. + pub column: usize, + /// The column of this table whose value finds the joined row. + pub anchor: usize, + /// The joined table and column, the formula as it was written, and the sort key. + pub policy: ast::RankPolicy, +} + +pub(super) fn position(table: &str, meta: &TableMeta, name: &str) -> Result { + let found = meta.columns.iter().position(|column| column.name == name); + found.ok_or_else(|| Error::ColumnNotFound { + table: table.to_owned(), + column: name.to_owned(), + }) +} + +/// Whether the index is one behind a key or a UNIQUE rule. +fn is_rule(index: &crate::types::IndexDecl) -> bool { + index.name.starts_with("__pk_") || index.name.starts_with("__unique_") +} + +/// Checks every rule a CREATE TABLE declares and gives what the table keeps of them. Adds to +/// `meta` the indexes the rules need: one for the child side of a reference that no earlier +/// index leads with, and the two of a table of edges. +pub(super) fn of( + create: &CreateTablePlan, + meta: &mut TableMeta, + catalog: &Catalog, +) -> Result { + let table = &create.name; + let options = &create.options; + let delivers = !matches!( + meta.sync, + Some(SyncDirection::Off | SyncDirection::PullOnly) + ); + let current = meta.history == Some(History::CurrentOnly); + let refused = if meta.immutable && current { + "IMMUTABLE with HISTORY CURRENT ONLY: an IMMUTABLE table has no older version to drop, \ + so leave one of the two out" + } else if meta.immutable && meta.retain.is_some() { + "IMMUTABLE with RETAIN: an IMMUTABLE table has no row to expire, so leave one of the two \ + out" + } else if current && delivers && meta.conflict != Some(ConflictPolicy::KeepLatest) { + "HISTORY CURRENT ONLY on a table that sends its rows out and keeps the first write: \ + another machine would take a newest value as the first, so write SYNC CONFLICT KEEP \ + LATEST or SYNC OFF" + } else if meta.retain.as_ref().is_some_and(|retain| retain.sync_safe) && !delivers { + "RETAIN ... SYNC SAFE waits for a delivery that SYNC OFF and SYNC PULL ONLY never make: \ + leave SYNC SAFE out, or choose a direction that delivers" + } else { + "" + }; + if !refused.is_empty() { + return Err(invalid(format!("table '{table}' declares {refused}"))); + } + if let Some(machine) = &meta.state_machine { + position(table, meta, &machine.column)?; + } + let machine = meta.state_machine.iter().flat_map(|m| &m.transitions); + let states = machine.flat_map(|(from, to)| std::iter::once(from).chain(to)); + let mut excluded = options.exclude_vector_states.iter(); + if let Some(state) = excluded.find(|s| !states.clone().any(|k| k == *s)) { + return Err(invalid(format!( + "table '{table}' declares EXCLUDE VECTOR for '{state}', no state of its STATE MACHINE" + ))); + } + + let mut rules = Rules { + propagate_edges: options.propagate_edges.clone(), + exclude_vector_states: options.exclude_vector_states.clone(), + delivery_manifest: options.delivery_manifest.clone(), + edge_discard: options.edge_discard, + ..Rules::default() + }; + let of_columns = create.columns.iter(); + let of_columns = + of_columns.filter_map(|c| Some((vec![c.name.clone()], c.references.as_ref()?))); + let of_table = create.foreign_keys.iter(); + for (columns, written) in of_columns.chain(of_table.map(|(columns, to)| (columns.clone(), to))) + { + // A table that references itself is checked against the declaration being made. + let parent = match catalog.table(&written.table) { + _ if written.table == *table => &*meta, + Some(parent) => &parent.meta, + None => return Err(Error::TableNotFound(written.table.clone())), + }; + let covers = |index: &crate::types::IndexDecl| { + is_rule(index) + && index.columns.len() == columns.len() + && index + .columns + .iter() + .map(|(name, _)| name) + .eq(&written.columns) + }; + if !parent.indexes.iter().any(covers) { + return Err(invalid(format!( + "table '{table}': REFERENCES {}({}) names columns that no PRIMARY KEY or \ + UNIQUE rule covers in that order; reference a key, or declare the UNIQUE rule on the parent \ + first", + written.table, + written.columns.join(", ") + ))); + } + // A key matches only a value of its own type. + for (column, to) in columns.iter().zip(&written.columns) { + let at = position(table, meta, column)?; + let to = parent.columns.iter().find(|parent| parent.name == *to); + if to.is_some_and(|to| to.column_type != meta.columns[at].column_type) { + return Err(invalid(format!( + "table '{table}': '{column}' REFERENCES {}, a column of another type", + written.table + ))); + } + } + let leads = |index: &crate::types::IndexDecl| { + index.columns.len() >= columns.len() + && index + .columns + .iter() + .zip(&columns) + .all(|((name, _), column)| name == column) + }; + let index = match meta.indexes.iter().position(leads) { + Some(index) => index, + None => { + let name = format!("__fk_{}", columns.join("_")); + engine_index(table, meta, name, columns.clone(), false)?; + meta.indexes.len() - 1 + } + }; + rules.references.push(Reference { + columns: columns + .iter() + .map(|c| position(table, meta, c)) + .collect::>()?, + parent: written.table.clone(), + parent_columns: written.columns.clone(), + on_state: written.on_state.clone(), + index, + }); + } + + // A table of edges has exactly these three columns, and two indexes the engine makes. + let has = |name: &str, column_type: ColumnType| { + let mut columns = meta.columns.iter(); + columns.any(|column| column.name == name && column.column_type == column_type) + }; + if !rules.propagate_edges.is_empty() && !has("id", ColumnType::Uuid) { + return Err(invalid(format!( + "table '{table}' declares PROPAGATE ON EDGE, which finds a row's edges by its id: \ + declare id UUID" + ))); + } + if has("source_id", ColumnType::Uuid) + && has("target_id", ColumnType::Uuid) + && has("edge_type", ColumnType::Text) + { + for (name, columns) in [ + ( + "source_target_type", + ["source_id", "target_id", "edge_type"], + ), + ( + "target_source_type", + ["target_id", "source_id", "edge_type"], + ), + ] { + let columns = columns.map(str::to_owned).to_vec(); + engine_index(table, meta, format!("__graph_edge_{name}"), columns, false)?; + } + } else if !meta.dag.is_empty() { + return Err(invalid(format!( + "table '{table}' declares DAG, which is for a table of edges: declare source_id UUID, \ + target_id UUID and edge_type TEXT, or leave DAG out" + ))); + } + rules.counts = counts(table, meta, &[], catalog)?; + + for (at, column) in create.columns.iter().enumerate() { + if column.immutable { + rules.immutable_columns.push(at); + } + if column.context_id { + rules.context_column = Some(at); + } + if let Some(labels) = &column.scope_labels { + rules.scope_labels = Some((at, labels.clone())); + } + if let Some((grants, grant)) = &column.acl { + rules.acl = Some((at, grants.clone(), grant.clone())); + } + if let Some(policy) = &column.rank_policy { + let anchor = rank_policy(table, &column.name, policy, meta, catalog)?; + rules.rank_policies.push(RankPolicy { + column: at, + anchor, + policy: policy.clone(), + }); + } + } + Ok(rules) +} + +/// The counts a table keeps under its MAINTAIN COUNT, each with its number: the number it has +/// in `kept` when it is kept already, and the next free number of the catalog otherwise. +pub(super) fn counts( + table: &str, + meta: &TableMeta, + kept: &[Count], + catalog: &Catalog, +) -> Result> { + let indexes = || { + meta.indexes + .iter() + .filter(|index| index.json_path.is_empty()) + }; + // A list that holds every column of a key or a UNIQUE rule counts one row at most. + let bounded = |list: &[(String, SortDirection)]| { + let listed = |(column, _): &(String, SortDirection)| list.iter().any(|(c, _)| c == column); + indexes().any(|index| is_rule(index) && index.columns.iter().all(listed)) + }; + let lists = match &meta.kept_counts { + None | Some(KeptCounts::None) => return Ok(Vec::new()), + Some(KeptCounts::Lists(lists)) => lists.clone(), + Some(KeptCounts::AllPrefixes) => { + let mut all: Vec> = Vec::new(); + for index in indexes() { + for length in 1..=index.columns.len() { + let list = &index.columns[..length]; + if !bounded(list) && !all.iter().any(|other| other == list) { + all.push(list.to_vec()); + } + } + } + all + } + }; + let numbered = catalog.numbered().flat_map(|other| &other.rules.counts); + let mut next = numbered + .chain(kept) + .map(|count| count.id.0 + 1) + .max() + .unwrap_or(0); + let mut found = Vec::new(); + for (at, list) in lists.iter().enumerate() { + let names: Vec<&str> = list.iter().map(|(column, _)| column.as_str()).collect(); + let unled; + let refused = if lists[..at].contains(list) { + "is written twice; write it once" + } else if (1..names.len()).any(|at| names[..at].contains(&names[at])) { + "names a column twice; name it once" + } else if !indexes().any(|index| index.columns.starts_with(list)) { + let looked: Vec<&str> = indexes().map(|index| index.name.as_str()).collect(); + unled = format!( + "is led by none of the table's indexes ({}), so nothing could read the count; \ + declare a UNIQUE rule that begins with these columns, or leave the list out", + looked.join(", ") + ); + &unled + } else if bounded(list) { + "holds every column of a key or a UNIQUE rule, which already says the count is at \ + most one; leave the list out" + } else { + "" + }; + let columns = list.iter().map(|(column, direction)| { + position(table, meta, column).map(|column| (column, *direction)) + }); + let columns: Vec<(usize, SortDirection)> = columns.collect::>()?; + if !refused.is_empty() { + return Err(invalid(format!( + "table '{table}': the MAINTAIN COUNT list ({}) {refused}", + names.join(", ") + ))); + } + let id = kept + .iter() + .find(|count| count.columns == columns) + .map(|count| count.id); + let id = id.unwrap_or_else(|| { + next += 1; + CountId(next - 1) + }); + found.push(Count { id, columns }); + } + Ok(found) +} + +/// Checks the RANK_POLICY of the VECTOR column `vector` and gives the column of the table that +/// finds the joined row: when the join is by the joined table's key, the one column named for +/// that table (`result_id` or `results_id` for `results`); else the column of the join column's +/// name; else the table's own key. +fn rank_policy( + table: &str, + vector: &str, + policy: &ast::RankPolicy, + meta: &TableMeta, + catalog: &Catalog, +) -> Result { + let refuse = |reason: String| Error::RankPolicyInvalid { + index: VectorIndexRef { + table: table.to_owned(), + column: vector.to_owned(), + }, + reason, + }; + let (joined, on) = (&policy.table, &policy.column); + let other = catalog.table(joined).map(|other| &other.meta); + let other = other.ok_or_else(|| refuse(format!("it joins the unknown table '{joined}'")))?; + let column = other.columns.iter().find(|column| column.name == *on); + let column = column.ok_or_else(|| refuse(format!("'{joined}' has no column '{on}'")))?; + let leads = |index: &crate::types::IndexDecl| { + index.json_path.is_empty() && index.columns.first().is_some_and(|(name, _)| name == on) + }; + if !other.indexes.iter().any(leads) { + return Err(refuse(format!( + "no index of '{joined}' leads with '{on}'; create one first" + ))); + } + let has = |name: &String| meta.columns.iter().any(|column| column.name == *name); + let mut found: Vec = Vec::new(); + if column.primary_key { + let singular = joined.strip_suffix('s').unwrap_or(joined); + found.extend( + [format!("{singular}_id"), format!("{joined}_id")] + .into_iter() + .filter(has), + ); + found.dedup(); + } + if found.is_empty() { + found.extend(Some(on.clone()).filter(has)); + } + if found.is_empty() { + let key = meta.columns.iter().filter(|column| column.primary_key); + found.extend(key.map(|column| column.name.clone())); + } + let [anchor] = &found[..] else { + return Err(refuse(format!( + "'{table}' needs one column to find the row of '{joined}' by, and has {}: {}", + found.len(), + found.join(", ") + ))); + }; + + let read = formula_reads(&policy.formula) + .map_err(|error| refuse(format!("its FORMULA is not a formula: {error}")))?; + for name in read.iter().filter(|name| *name != "vector_score") { + let named = meta.columns.iter().chain(&other.columns); + let mut named = named.filter(|column| column.name == *name); + let (Some(column), None) = (named.next(), named.next()) else { + return Err(refuse(format!( + "its FORMULA reads '{name}', which is not a column of exactly one of '{table}' \ + and '{joined}'" + ))); + }; + use ColumnType::{Boolean, Integer, Real}; + if !matches!(column.column_type, Integer | Real | Boolean) { + return Err(refuse(format!( + "its FORMULA reads '{name}', which is no numeric column" + ))); + } + } + position(table, meta, anchor) +} + +/// The names a rank formula reads as `{name}`, `vector_score` among them, or the parser's +/// refusal of a text that is not one. +pub(super) fn formula_reads(formula: &str) -> Result> { + fn reads(expr: &ast::Expr, names: &mut Vec) { + match expr { + ast::Expr::Column(column) => names.push(column.column.clone()), + ast::Expr::FunctionCall { args, .. } => args.iter().for_each(|arg| reads(arg, names)), + _ => {} + } + } + let mut names = Vec::new(); + reads(&crate::sql::formula(formula)?, &mut names); + Ok(names) +} diff --git a/v2/contextdb/src/schema/tests.rs b/v2/contextdb/src/schema/tests.rs index 2f98af46..f2f87c44 100644 --- a/v2/contextdb/src/schema/tests.rs +++ b/v2/contextdb/src/schema/tests.rs @@ -1,330 +1,213 @@ -use std::collections::BTreeMap; +//! What `schema` makes of a schema record that disagrees with itself, and of the numbers a +//! vector column's index clauses give: no declaration writes such a record, so these tests write +//! one through the store itself. What a declaration is stored as and printed back as is proved +//! through the public calls in `tests/declarations.rs`. +//! +//! Every table is declared through the public calls, in a store of its own, and then read as +//! `schema` reads it: `Catalog::load` over the newest commit. -use super::{Catalog, Table, check, record, render}; -use crate::sql::ast::{self, DataType, Expr, Literal}; -use crate::sql::plan::{CreateIndexPlan, CreateTablePlan}; -use crate::store::{IndexId, TableId}; -use crate::types::{ - ColumnDef, ColumnDefault, ColumnType, Error, IndexDecl, SortDirection, TableMeta, Value, -}; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; -fn column(name: &str, column_type: ColumnType) -> ColumnDef { - ColumnDef { - name: name.to_owned(), - column_type, - nullable: true, - primary_key: false, - unique: false, - default: None, +use super::{Catalog, Table, render}; +use crate::Database; +use crate::memory::Budget; +use crate::store::{ChangeSet, Store}; +use crate::types::{Error, Result}; + +/// An empty folder for one test's store. +fn folder(test: &str) -> PathBuf { + let folder = + std::env::temp_dir().join(format!("contextdb-schema-{test}-{}", std::process::id())); + if folder.exists() { + std::fs::remove_dir_all(&folder).expect("clear the folder an earlier run left"); } + folder } -fn index(name: &str, columns: &[(&str, SortDirection)]) -> IndexDecl { - IndexDecl { - name: name.to_owned(), - columns: columns - .iter() - .map(|(column, direction)| ((*column).to_owned(), *direction)) - .collect(), +/// Runs statements on the store in a folder, each of which must be accepted. +fn declare(folder: &Path, statements: &[&str]) { + let db = Database::open(folder).expect("open the store"); + for sql in statements { + if let Err(error) = db.execute(sql, &HashMap::new()) { + panic!("{sql}\nwas refused: {error:?}"); + } } + db.close().expect("close"); } -/// A table with every kind of clause this version keeps, as CREATE TABLE and two CREATE INDEX -/// statements leave it. -fn notes() -> Table { - use SortDirection::{Asc, Desc}; - let literal = |value| Some(ColumnDefault::Literal(value)); - Table { - id: TableId(4), - name: "notes".to_owned(), - meta: TableMeta { - columns: vec![ - ColumnDef { - primary_key: true, - ..column("id", ColumnType::Integer) - }, - ColumnDef { - nullable: false, - default: literal(Value::Text("it's".to_owned())), - ..column("title", ColumnType::Text) - }, - ColumnDef { - default: literal(Value::Null), - ..column("body", ColumnType::Text) - }, - ColumnDef { - default: literal(Value::Int64(3)), - ..column("stars", ColumnType::Integer) - }, - ColumnDef { - unique: true, - ..column("slug", ColumnType::Text) - }, - ColumnDef { - nullable: false, - default: Some(ColumnDefault::Now), - ..column("made at", ColumnType::Timestamp) - }, - ColumnDef { - default: literal(Value::Float64(-2.0)), - ..column("score", ColumnType::Real) - }, - ColumnDef { - default: literal(Value::Bool(true)), - ..column("kept", ColumnType::Boolean) - }, - column("embedding", ColumnType::Vector(384)), - ], - unique_constraints: vec![vec!["title".to_owned(), "stars".to_owned()]], - indexes: vec![ - index("__pk_id", &[("id", Asc)]), - index("__unique_slug", &[("slug", Asc)]), - index("__unique_title_stars", &[("title", Asc), ("stars", Asc)]), - index("by_stars", &[("stars", Desc), ("made at", Asc)]), - index("by_body", &[("body", Asc)]), - ], - }, - index_ids: [7, 8, 9, 12, 15].map(IndexId).to_vec(), - vectors: Vec::new(), - } +/// The tables of the store in a folder, as `schema` reads them from its newest commit. +fn load(folder: &Path) -> Result { + let budget = Budget::new(None).expect("a budget"); + let store = Store::open_for_writing(folder, &budget, None).expect("open the store"); + let view = store.view().expect("the newest commit"); + Catalog::load(&view) } -#[test] -fn a_record_reads_back_as_the_table_it_was_written_from() { - let table = notes(); - let back = record::decode(&record::encode(&table)); - assert_eq!(back.id, table.id); - assert_eq!(back.name, table.name); - assert_eq!(back.meta, table.meta); - assert_eq!(back.index_ids, table.index_ids); +/// One table of the store in a folder. +fn table(folder: &Path, name: &str) -> Table { + let catalog = load(folder).expect("the tables"); + let table = catalog.table(name); + table.unwrap_or_else(|| panic!("no table {name}")).clone() } -#[test] -fn render_prints_what_the_author_declared_and_nothing_else() { - assert_eq!( - render(¬es()), - "\ -CREATE TABLE notes ( - id INTEGER PRIMARY KEY, - title TEXT NOT NULL DEFAULT 'it''s', - body TEXT DEFAULT NULL, - stars INTEGER DEFAULT 3, - slug TEXT UNIQUE, - \"made at\" TIMESTAMP NOT NULL DEFAULT NOW(), - score REAL DEFAULT -2.0, - kept BOOLEAN DEFAULT TRUE, - embedding VECTOR(384), - UNIQUE (title, stars) -); -CREATE INDEX by_stars ON notes (stars DESC, \"made at\"); -CREATE INDEX by_body ON notes (body); -" +/// A store with two tables, the second of which declares rules of many kinds, so that its +/// schema record holds most of the parts a record may have. Gives back the store and that +/// table's record, as it is stored. +fn store_with_every_rule(test: &str) -> (Rules, Vec) { + let folder = folder(test); + declare( + &folder, + &[ + "CREATE TABLE a_parents (id UUID PRIMARY KEY, context_id UUID NOT NULL, \ + UNIQUE (context_id, id))", + "CREATE TABLE z_rules (id UUID PRIMARY KEY, context_id UUID CONTEXT_ID NOT NULL, \ + parent_id UUID REFERENCES a_parents(id) ON STATE archived PROPAGATE SET stale \ + MAX DEPTH 2 ABORT ON FAILURE, \ + kind TEXT NOT NULL IMMUTABLE, scope TEXT SCOPE_LABEL_READ ('r', 'w') WRITE ('w'), \ + status TEXT NOT NULL DEFAULT 'open', made_at TXID, doc JSON DEFAULT '{}', \ + confidence REAL, \ + v VECTOR(4) PARTITION_KEY (context_id) MAX_PARTITIONS 8 SEARCH_MODE EXACT, \ + UNIQUE (context_id, kind), \ + FOREIGN KEY (context_id, parent_id) REFERENCES a_parents(context_id, id)) \ + STATE MACHINE (status: open -> [stale, closed], stale -> [closed]) \ + PROPAGATE ON EDGE CITES INCOMING STATE stale SET stale MAX DEPTH 3 \ + PROPAGATE ON STATE closed EXCLUDE VECTOR \ + RETAIN 48 HOURS SYNC SAFE HISTORY ALL SYNC PUSH ONLY SYNC CONFLICT KEEP FIRST \ + MAINTAIN COUNT (context_id)", + "CREATE INDEX z_by_kind ON z_rules (kind DESC, status)", + "CREATE INDEX z_by_owner ON z_rules ((doc -> 'owner'))", + ], ); + let budget = Budget::new(None).expect("a budget"); + let store = Store::open_for_writing(&folder, &budget, None).expect("open the store"); + let view = store.view().expect("the newest commit"); + // The records come in the order of their tables' names. + let mut records = view.schema_records().expect("the schema records"); + assert_eq!(records.len(), 2, "two tables have two schema records"); + let record = records.pop().expect("the record of the second table"); + assert!(record.len() > 200, "a record of {} bytes", record.len()); + ( + Rules { + folder, + budget, + store, + }, + record, + ) } -/// The DEFAULT of a column as the parser hands it over, for this text after the word. -fn written_default(text: &str) -> Expr { - let statement = format!("CREATE TABLE t (c INTEGER DEFAULT {text})"); - let parsed = match crate::sql::parse(&statement) { - Ok(ast::Statement::CreateTable(create)) => create, - other => panic!("{statement}\nwas read as {other:?}"), - }; - let [column] = parsed.columns.as_slice() else { - panic!( - "{statement}\nwas read with {} columns", - parsed.columns.len() - ); - }; - column - .default - .clone() - .unwrap_or_else(|| panic!("{statement}\nwas read with no default")) +/// The store of [`store_with_every_rule`], held open by the store part alone. +struct Rules { + folder: PathBuf, + budget: Budget, + store: Store, } -#[test] -fn a_default_is_kept_in_the_type_the_author_wrote() { - for (text, value) in [ - ("NULL", Value::Null), - ("TRUE", Value::Bool(true)), - ("3", Value::Int64(3)), - ("0.0", Value::Float64(0.0)), - ("'it''s'", "it's".into()), - ("-1", Value::Int64(-1)), - ("-2.5", Value::Float64(-2.5)), - ] { - assert_eq!( - check::default(&written_default(text)), - Some(ColumnDefault::Literal(value)), - "DEFAULT {text}" - ); +impl Rules { + /// What `schema` reads once the store alone has written `record` as the schema record of + /// the table `z_rules`. + fn load_with(&self, record: Vec) -> Result { + let view = self.store.view().expect("the newest commit"); + let mut changes = ChangeSet::new(view, &self.budget).expect("a change set"); + changes.put_schema("z_rules", record).expect("stage"); + self.store.commit(changes).expect("commit the record"); + Catalog::load(&self.store.view().expect("the newest commit")) } - assert_eq!( - check::default(&written_default("NOW()")), - Some(ColumnDefault::Now) - ); - // Anything that is not a literal or NOW() is no default this version keeps. - assert_eq!(check::default(&written_default("$p")), None); - assert_eq!(check::default(&written_default("-'x'")), None); -} -/// A column as the parser hands over one that wrote no clause. -fn declared(name: &str, data_type: DataType) -> ast::ColumnDef { - ast::ColumnDef { - name: name.to_owned(), - data_type, - nullable: true, - primary_key: false, - unique: false, - default: None, - vector: ast::VectorClauses::default(), + /// Lets go of the store and removes its folder. + fn remove(self) { + let Rules { folder, store, .. } = self; + drop(store); + std::fs::remove_dir_all(&folder).expect("remove the test's folder"); } } -/// A CREATE TABLE as the planner hands over one that wrote no table option. -fn create(name: &str, columns: Vec, unique: &[&[&str]]) -> CreateTablePlan { - CreateTablePlan { - name: name.to_owned(), - columns, - unique_constraints: unique - .iter() - .map(|rule| rule.iter().map(|column| (*column).to_owned()).collect()) - .collect(), - if_not_exists: false, +/// Asserts that reading the tables told the store's damage, by name and naming the folder. +fn assert_damage(read: Result, folder: &Path, what: &str) { + match read { + Err(Error::StoreCorrupted { path, reason }) => { + assert_eq!(path, folder, "{what}"); + assert!(!reason.is_empty(), "{what}"); + } + Err(other) => panic!("{what} was read as {other:?}"), + Ok(_) => panic!("{what} was read as a table"), } } #[test] -fn a_declared_table_gets_its_numbers_and_the_index_behind_each_rule() { - let key = |name: &str| ast::ColumnDef { - primary_key: true, - ..declared(name, DataType::Integer) - }; - let people = create( - "people", - vec![ - ast::ColumnDef { - nullable: false, - unique: true, - ..key("id") - }, - ast::ColumnDef { - unique: true, - ..declared("email", DataType::Text) - }, - ast::ColumnDef { - default: Some(Expr::Literal(Literal::Real(0.0))), - ..declared("score", DataType::Real) - }, - declared("born", DataType::Integer), - declared("doc", DataType::Json), - ], - &[&["email", "born"]], - ); - let mut catalog = Catalog { - tables: BTreeMap::new(), - }; - let table = check::new_table(&people, &catalog).expect("a plain table"); - let names = |table: &Table| -> Vec { - let indexes = table.meta.indexes.iter(); - indexes.map(|index| index.name.clone()).collect() - }; - assert_eq!(table.id, TableId(0)); - // The key column keeps the UNIQUE its author wrote and gets no second index for it. - assert_eq!( - names(&table), - ["__pk_id", "__unique_email", "__unique_email_born"] - ); - assert_eq!(table.index_ids, [0, 1, 2].map(IndexId)); - assert!(render(&table).starts_with( - "CREATE TABLE people (\n id INTEGER PRIMARY KEY NOT NULL UNIQUE,\n email TEXT UNIQUE,\n \ - score REAL DEFAULT 0.0,\n" - )); - catalog.tables.insert(table.name.clone(), table); +fn a_schema_record_cut_short_anywhere_is_damage() { + let (rules, record) = store_with_every_rule("record-cut-short"); + // Every length from nothing to one byte short of the whole: each ends inside a number, a + // text or a list that says it is longer than what is left. + for length in 0..record.len() { + let what = format!( + "a schema record cut to {length} of its {} bytes", + record.len() + ); + assert_damage( + rules.load_with(record[..length].to_vec()), + &rules.folder, + &what, + ); + } + rules.remove(); +} - // The next table and the next index count on from the ones the catalog holds. - let tags = create("tags", vec![key("id")], &[]); - let tags = check::new_table(&tags, &catalog).expect("a second table"); - assert_eq!( - (tags.id, &tags.index_ids[..]), - (TableId(1), &[IndexId(3)][..]) - ); - catalog.tables.insert(tags.name.clone(), tags); - let index = |name: &str, column: &str| CreateIndexPlan { - name: name.to_owned(), - table: "people".to_owned(), - columns: vec![(column.to_owned(), SortDirection::Desc)], - }; - let indexed = check::with_index(&index("by_born", "born"), &catalog).expect("an index"); - assert_eq!(names(&indexed)[3], "by_born"); - assert_eq!(indexed.index_ids, [0, 1, 2, 4].map(IndexId)); - catalog.tables.insert(indexed.name.clone(), indexed); +#[test] +fn a_schema_record_of_an_unknown_version_is_damage() { + let (rules, record) = store_with_every_rule("record-unknown-version"); + // A record begins with one byte that says which version of the record follows. + for version in [record[0].wrapping_add(1), 0xFF] { + let mut changed = record.clone(); + changed[0] = version; + let what = format!("a schema record that begins with the version byte {version}"); + assert_damage(rules.load_with(changed), &rules.folder, &what); + } + rules.remove(); +} - // What is refused, and as what. - assert!(matches!( - check::with_index(&index("__unique_born", "born"), &catalog), - Err(Error::ReservedIndexName { prefix, .. }) if prefix == "__unique_" - )); - assert!(matches!( - check::with_index(&index("by_age", "age"), &catalog), - Err(Error::ColumnNotFound { table, column }) if table == "people" && column == "age" - )); - assert!(matches!( - check::with_index(&index("by_doc", "doc"), &catalog), - Err(Error::ColumnNotIndexable { column, column_type: ColumnType::Json, .. }) - if column == "doc" - )); - // A REAL may be indexed, and may not be a key or under a UNIQUE rule. - check::with_index(&index("by_score", "score"), &catalog).expect("an index over a REAL"); - for unique in [&[][..], &[&["id", "ratio"][..]][..]] { - let ratio = ast::ColumnDef { - unique: unique.is_empty(), - ..declared("ratio", DataType::Real) - }; - assert!(matches!( - check::new_table(&create("ratios", vec![key("id"), ratio], unique), &catalog), - Err(Error::ColumnNotIndexable { column, column_type: ColumnType::Real, .. }) - if column == "ratio" - )); +#[test] +fn a_schema_record_with_one_byte_changed_is_a_whole_table_or_damage_and_never_a_panic() { + let (rules, record) = store_with_every_rule("record-byte-changed"); + // Each byte in turn becomes 255: as part of a length it points past the record's end, as + // the byte that says which kind follows it names no kind, and as the number of a column + // it names no column of the table. Inside a name or a text it may leave a record that + // still agrees with itself, and that record is read as tables that can be printed. + let (mut damaged, mut read) = (0, 0); + for position in 0..record.len() { + if record[position] == 0xFF { + continue; + } + let mut changed = record.clone(); + changed[position] = 0xFF; + match rules.load_with(changed) { + Err(Error::StoreCorrupted { path, .. }) => { + assert_eq!(path, rules.folder, "byte {position}"); + damaged += 1; + } + Err(other) => panic!("with byte {position} changed the record was read as {other:?}"), + Ok(catalog) => { + for table in catalog.tables() { + assert!( + render(table).starts_with("CREATE TABLE "), + "byte {position}" + ); + } + read += 1; + } + } } - assert!(matches!( - check::with_index(&index("by_born", "email"), &catalog), - Err(Error::DuplicateIndex { table, index }) if table == "people" && index == "by_born" - )); - // Rules whose columns spell one name stay separate rules, each with its own index. - let pairs = create( - "pairs", - vec![ - key("id"), - declared("a", DataType::Integer), - declared("b", DataType::Integer), - declared("a_b", DataType::Integer), - ], - &[&["a_b"], &["a", "b"], &["a", "b"]], + // A record holds lengths, kinds and column numbers, so some of the changes are damage. + assert!( + damaged > 0, + "of the changed records none was damage and {read} were read as tables" ); - let pairs = check::new_table(&pairs, &catalog).expect("three rules"); - assert_eq!( - names(&pairs), - [ - "__pk_id", - "__unique_a_b", - "__unique_a_b_2", - "__unique_a_b_3" - ] - ); - assert_eq!(pairs.meta.indexes[1].columns.len(), 1); - assert_eq!(pairs.meta.indexes[2].columns.len(), 2); - let not_supported = |plan: &CreateTablePlan, clause: &str| { - matches!( - check::new_table(plan, &catalog), - Err(Error::PlanError(text)) if text.starts_with(clause) && text.contains("not supported") - ) - }; - let keyless = create("keyless", vec![declared("word", DataType::Text)], &[]); - assert!(not_supported(&keyless, "a table without a PRIMARY KEY")); + rules.remove(); } #[test] -fn the_index_clauses_of_a_vector_column_are_stored_and_give_the_numbers_in_force() { +fn the_index_clauses_of_a_vector_column_give_the_numbers_in_force() { use super::{Consolidation, Quantization, VectorColumn}; use crate::store::VectorId; let column = |id, auto_index_at, consolidation, quantization| VectorColumn { @@ -338,7 +221,7 @@ fn the_index_clauses_of_a_vector_column_are_stored_and_give_the_numbers_in_force quantization, }; let percent = |change, tombstone| Some(Consolidation::Percent { change, tombstone }); - let vectors = vec![ + let vectors = [ column(0, None, None, Quantization::F32), column(1, Some(1000), Some(Consolidation::None), Quantization::Sq8), column( @@ -372,10 +255,63 @@ fn the_index_clauses_of_a_vector_column_are_stored_and_give_the_numbers_in_force (50, 1, Some((1, 2))), ] ); - // What was written is what the record gives back: a clause that was not stays unwritten. - let table = Table { - vectors: vectors.clone(), - ..notes() - }; - assert_eq!(record::decode(&record::encode(&table)).vectors, vectors); +} + +#[test] +fn dropping_a_vector_column_leaves_no_record_of_any_kind_for_it() { + use crate::store::VectorRecord::*; + use crate::store::{Key, Read}; + let folder = folder("drop-vector-records"); + declare( + &folder, + &["CREATE TABLE t (id INTEGER PRIMARY KEY, v VECTOR(4) AUTO_INDEX_AT 1000)"], + ); + let id = table(&folder, "t").vectors[0].id; + // One record of every kind the vector part keeps for the column, as sealing and building + // an index leave them; each is both staged and the place the check scans from. + let (tuple, row) = (Key::from_bytes(&[]), Key::from_bytes(&[1])); + let records = [ + Partition(id, &tuple), + Pending(id, 1, &row), + Block(id, 1, 0), + Grouping(id, 2), + Centres(id, 2, 0), + Build(id, 1), + Assigned(id, 2, 0), + Sketch(id, 1, 0), + Mean(id, 1), + Left(id, 1, &row), + ]; + let budget = Budget::new(None).expect("a budget"); + { + let store = Store::open_for_writing(&folder, &budget, None).expect("open the store"); + let mut changes = ChangeSet::new(store.view().expect("a view"), &budget).expect("begin"); + for record in records { + changes.put_vector(record, vec![0; 8]).expect("stage"); + } + store.commit(changes).expect("commit the records"); + } + declare(&folder, &["ALTER TABLE t DROP COLUMN v"]); + let store = Store::open_for_writing(&folder, &budget, None).expect("open the store"); + let view = store.view().expect("the newest commit"); + let mut left = Vec::new(); + for start in records { + let kind = format!("{start:?}"); + let mine = format!("{}({id:?},", &kind[..kind.find('(').expect("a kind")]); + view.vectors(start, &mut |record, _| { + let name = format!("{record:?}"); + let ours = name.starts_with(&mine); + if ours { + left.push(name); + } + Ok(ours) + }) + .expect("scan"); + } + assert!( + left.is_empty(), + "a dropped column keeps nothing in the vector part, yet these records remain: {left:?}" + ); + drop((view, store)); + std::fs::remove_dir_all(&folder).expect("remove the test's folder"); } diff --git a/v2/contextdb/src/sql/ast.rs b/v2/contextdb/src/sql/ast.rs index 31ad3d63..6ae0bcda 100644 --- a/v2/contextdb/src/sql/ast.rs +++ b/v2/contextdb/src/sql/ast.rs @@ -1,11 +1,17 @@ //! A statement as it was written: what `parse` returns and `plan` reads. -pub use crate::types::{SortDirection, VectorSearchMode}; +pub use crate::types::{ + ColumnType as DataType, ConflictPolicy, History, KeptCounts, Retain, RetainUnit, SortDirection, + StateMachine, SyncDirection, VectorSearchMode, +}; #[derive(Debug, Clone)] pub enum Statement { CreateTable(CreateTable), CreateIndex(CreateIndex), + AlterTable(AlterTable), + DropTable(DropTable), + DropIndex(DropIndex), Insert(Insert), Delete(Delete), Update(Update), @@ -140,6 +146,14 @@ pub struct Insert { /// Empty when the statement names no columns. pub columns: Vec, pub values: Vec>, + pub on_conflict: Option, +} + +/// `ON CONFLICT (columns) DO UPDATE SET column = value, ...`. +#[derive(Debug, Clone)] +pub struct OnConflict { + pub columns: Vec, + pub assignments: Vec<(String, Expr)>, } #[derive(Debug, Clone)] @@ -161,7 +175,67 @@ pub struct CreateTable { pub columns: Vec, /// The columns of each table-level `UNIQUE (...)`, in the order written. pub unique_constraints: Vec>, + /// The columns of a table-level `PRIMARY KEY (...)`; empty when it was not written. + pub primary_key: Vec, + /// Each `FOREIGN KEY (...) REFERENCES ...`: its own columns, then what they refer to. + pub foreign_keys: Vec<(Vec, Reference)>, pub if_not_exists: bool, + pub options: TableOptions, +} + +/// The options written after a table's columns. Each is empty, false or `None` when it was +/// not written. +#[derive(Debug, Clone, Default)] +pub struct TableOptions { + pub immutable: bool, + pub state_machine: Option, + /// The edge types of `DAG(...)`. + pub dag: Vec, + pub propagate_edges: Vec, + /// The state of each `PROPAGATE ON STATE s EXCLUDE VECTOR`. + pub exclude_vector_states: Vec, + pub retain: Option, + pub history: Option, + pub sync: Option, + pub conflict: Option, + /// The tables of `DELIVERY MANIFEST OVER ...`. + pub delivery_manifest: Vec, + pub edge_discard: Option, + pub kept_counts: Option, +} + +/// `REFERENCES table (column, ...)`, with its `ON STATE` rule when one was written. +#[derive(Debug, Clone)] +pub struct Reference { + pub table: String, + pub columns: Vec, + pub on_state: Option, +} + +/// `ON STATE on_state PROPAGATE SET set [MAX DEPTH n] [ABORT ON FAILURE]`, and the same four +/// things of a `PROPAGATE ON EDGE`. A state keeps the letters it was written in. +#[derive(Debug, Clone)] +pub struct Propagation { + pub on_state: String, + pub set: String, + pub max_depth: Option, + pub abort_on_failure: bool, +} + +/// `PROPAGATE ON EDGE edge_type direction STATE ... SET ...`. The edge type keeps the letters +/// it was written in; the direction is INCOMING, OUTGOING or BOTH, in capitals. +#[derive(Debug, Clone)] +pub struct EdgePropagation { + pub edge_type: String, + pub direction: String, + pub rule: Propagation, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EdgeDiscard { + Never, + AfterOutcome, + Always, } #[derive(Debug, Clone)] @@ -174,6 +248,30 @@ pub struct ColumnDef { pub default: Option, /// The clauses after `VECTOR(n)`. Left as it starts on a column of any other type. pub vector: VectorClauses, + pub references: Option, + pub immutable: bool, + pub context_id: bool, + pub scope_labels: Option, + /// The table and column of `ACL REFERENCES table (column)`. + pub acl: Option<(String, String)>, + pub rank_policy: Option, +} + +/// `SCOPE_LABEL (...)`, whose one list is for reading and writing alike and leaves `write` +/// empty, or `SCOPE_LABEL_READ (...) WRITE (...)`. +#[derive(Debug, Clone)] +pub struct ScopeLabels { + pub read: Vec, + pub write: Option>, +} + +/// `RANK_POLICY (JOIN table ON column, FORMULA '...', SORT_KEY key)`. +#[derive(Debug, Clone)] +pub struct RankPolicy { + pub table: String, + pub column: String, + pub formula: String, + pub sort_key: String, } /// The clauses a VECTOR column may carry, as they were written. @@ -208,21 +306,68 @@ pub enum ConsolidationClause { }, } -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum DataType { - Uuid, - Text, - Integer, - Real, - Boolean, - Timestamp, - Json, - Vector(u32), -} - #[derive(Debug, Clone)] pub struct CreateIndex { pub name: String, pub table: String, pub columns: Vec<(String, SortDirection)>, + /// The keys of an index over a path into a JSON column, `((column -> 'key' -> ...))`, whose + /// one column is that column; empty for an index over columns. + pub json_path: Vec, +} + +#[derive(Debug, Clone)] +pub struct AlterTable { + pub table: String, + pub action: AlterAction, +} + +/// The one thing an ALTER TABLE changes, as it was written. +#[derive(Debug, Clone)] +pub enum AlterAction { + AddColumn(Box), + DropColumn { + column: String, + /// `Some(true)` for CASCADE, `Some(false)` for RESTRICT, `None` when neither was + /// written, which means RESTRICT. + cascade: Option, + }, + RenameColumn { + from: String, + to: String, + }, + SetRetain(Retain), + DropRetain, + SetHistory(History), + SetSync(SyncDirection), + SetConflict(ConflictPolicy), + /// `SET MAINTAIN COUNT ...`; `None` is the word DEFAULT. + SetKeptCounts(Option), + /// `ALTER COLUMN column SET MAX_PARTITIONS n`, the number kept as written. + SetMaxPartitions { + column: String, + max_partitions: String, + }, + /// `ALTER COLUMN column SET SEARCH_MODE mode`. + SetSearchMode { + column: String, + search_mode: VectorSearchMode, + }, + /// `ALTER COLUMN column SET AUTO_INDEX_AT n`, the number kept as written; `None` is DEFAULT. + SetAutoIndexAt(String, Option), + /// `ALTER COLUMN column SET CONSOLIDATION ...`; `None` is DEFAULT. + SetConsolidation(String, Option), +} + +#[derive(Debug, Clone)] +pub struct DropTable { + pub name: String, + pub if_exists: bool, +} + +#[derive(Debug, Clone)] +pub struct DropIndex { + pub name: String, + pub table: String, + pub if_exists: bool, } diff --git a/v2/contextdb/src/sql/grammar.pest b/v2/contextdb/src/sql/grammar.pest index c50013f8..c84de52f 100644 --- a/v2/contextdb/src/sql/grammar.pest +++ b/v2/contextdb/src/sql/grammar.pest @@ -5,8 +5,12 @@ block_comment = _{ "/*" ~ (!"*/" ~ ANY)* ~ "*/" } statement = { SOI ~ stmt ~ ";"? ~ EOI } stmt = _{ - create_table_stmt + stored_procedure + | create_table_stmt | create_index_stmt + | alter_table_stmt + | drop_table_stmt + | drop_index_stmt | insert_stmt | delete_stmt | update_stmt @@ -18,28 +22,22 @@ stmt = _{ | later } -// A statement or clause that a later version of the library reads. It is recognised by its -// first words alone and the rest of the text is left unread, so that the statement is refused by -// that name and not as a syntax error. `later` stands only where nothing this version reads can -// begin with one of these words: after a whole statement or clause, and after an expression, -// where LIKE and BETWEEN would follow. DISTINCT and GRAPH_TABLE stand where a name could, so -// they are looked for first. +// A statement or clause that a later version of the library reads, or that no version reads +// (IVF and SEARCH). It is recognised by its first words alone and the rest of the text is left +// unread, so that the statement is refused by that name and not as a syntax error. `later` +// stands only where nothing this version reads can begin with one of these words: after a whole +// statement or clause, and after an expression, where LIKE and BETWEEN would follow. DISTINCT and +// GRAPH_TABLE stand where a name could, so they are looked for first. later = { later_name ~ ANY* } later_name = @{ ( - ^"ALTER" | ^"DROP" | ^"WITH" | ^"BEGIN" | ^"COMMIT" | ^"ROLLBACK" | ^"PURGE" | ^"DISCARD" + ^"DROP" ~ SPACE+ ~ (^"TRIGGER" | ^"ROUTE" | ^"SCHEDULE") | ^"WITH" | ^"BEGIN" | ^"COMMIT" | ^"ROLLBACK" | ^"PURGE" | ^"DISCARD" | ^"DECLARE" | ^"DISTINCT" | ^"GRAPH_TABLE" | ((^"LEFT" | ^"INNER" | ^"CROSS") ~ SPACE+)? ~ ^"JOIN" | ^"GROUP" ~ SPACE+ ~ ^"BY" | ^"HAVING" | ^"UNION" | ^"OFFSET" | ^"USE" ~ SPACE+ ~ ^"RANK" - | ^"ON" ~ SPACE+ ~ ^"CONFLICT" | (^"NOT" ~ SPACE+)? ~ (^"LIKE" | ^"BETWEEN") - | ^"FOREIGN" ~ SPACE+ ~ ^"KEY" | ^"REFERENCES" | ^"EXPIRES" | ^"IMMUTABLE" - | ^"RANK_POLICY" | ^"CONTEXT_ID" | ^"SCOPE_LABEL_READ" | ^"SCOPE_LABEL" | ^"ACL" - | ^"SEARCH" | ^"IVF" - | ^"RETAIN" | ^"HISTORY" | ^"SYNC" | ^"STATE_MACHINE" | ^"STATE" ~ SPACE+ ~ ^"MACHINE" - | ^"DAG" | ^"PROPAGATE" | ^"DELIVERY" ~ SPACE+ ~ ^"MANIFEST" - | ^"EDGE" ~ SPACE+ ~ ^"DISCARD" | ^"MAINTAIN" ~ SPACE+ ~ ^"COUNT" + | ^"EXPIRES" | ^"SEARCH" | ^"IVF" ) ~ !ident_char } // Two forms of a later version that begin with words this version reads, so each is known by @@ -87,16 +85,22 @@ create_table_stmt = { ^"CREATE" ~ ^"TABLE" ~ if_not_exists? ~ identifier ~ "(" ~ table_element ~ ("," ~ table_element)* // A later clause may follow a column, stand as an element of its own, or follow the list. - ~ (")" ~ later? | ","? ~ later) + ~ (")" ~ table_option* ~ later? | ","? ~ later) } +// The program stored in a database that no version reads: it is refused by its own kind. +stored_procedure = { ^"CREATE" ~ stored_word ~ ANY* } +stored_word = @{ (^"PROCEDURE" | ^"FUNCTION") ~ !ident_char } if_not_exists = { ^"IF" ~ ^"NOT" ~ ^"EXISTS" } -table_element = _{ column_def | unique_table_constraint } +table_element = _{ column_def | unique_table_constraint | primary_key_columns | foreign_key } unique_table_constraint = { ^"UNIQUE" ~ "(" ~ identifier ~ ("," ~ identifier)* ~ ")" } +primary_key_columns = { ^"PRIMARY" ~ ^"KEY" ~ columns } +foreign_key = { ^"FOREIGN" ~ ^"KEY" ~ columns ~ ^"REFERENCES" ~ identifier ~ columns } +columns = { "(" ~ identifier ~ ("," ~ identifier)* ~ ")" } column_def = { identifier ~ data_type ~ column_constraint* } data_type = { vector_type | ^"UUID" | ^"TEXT" | ^"INTEGER" | ^"INT" | ^"REAL" | ^"FLOAT" | ^"BOOLEAN" | ^"BOOL" - | ^"TIMESTAMP" | ^"JSON" + | ^"TIMESTAMP" | ^"TXID" | ^"JSON" } vector_type = { ^"VECTOR" ~ "(" ~ integer ~ ")" @@ -122,24 +126,117 @@ change_percent = { ^"CHANGE_PERCENT" ~ "=" ~ signed } tombstone_percent = { ^"TOMBSTONE_PERCENT" ~ "=" ~ signed } // Read with its sign, so that `schema` refuses zero and a negative number by their kind. signed = @{ "-"? ~ ASCII_DIGIT+ } -column_constraint = _{ not_null | nullable_marker | primary_key | unique | default_clause } +column_constraint = _{ + not_null | nullable_marker | primary_key | unique | default_clause + | references | immutable | context_id | scope_labels | acl | rank_policy +} not_null = { ^"NOT" ~ ^"NULL" } nullable_marker = { ^"NULL" } primary_key = { ^"PRIMARY" ~ ^"KEY" } unique = { ^"UNIQUE" } default_clause = { ^"DEFAULT" ~ expr } +references = { ^"REFERENCES" ~ identifier ~ "(" ~ identifier ~ ")" ~ propagation? } +propagation = { + ^"ON" ~ ^"STATE" ~ as_written ~ ^"PROPAGATE" ~ ^"SET" ~ as_written ~ max_depth? ~ abort_on_failure? +} +max_depth = { ^"MAX" ~ ^"DEPTH" ~ integer } +abort_on_failure = { ^"ABORT" ~ ^"ON" ~ ^"FAILURE" } +// A state or an edge type, which is a value that rows hold, and the name of a function: none +// of them is folded to lower case as the name of a table, a column or an index is. +as_written = { quoted_ident | ident } +immutable = { ^"IMMUTABLE" } +context_id = { ^"CONTEXT_ID" } +scope_labels = { ^"SCOPE_LABEL_READ" ~ labels ~ ^"WRITE" ~ labels | ^"SCOPE_LABEL" ~ labels } +labels = { "(" ~ string ~ ("," ~ string)* ~ ")" } +acl = { ^"ACL" ~ ^"REFERENCES" ~ identifier ~ "(" ~ identifier ~ ")" } +rank_policy = { + ^"RANK_POLICY" ~ "(" ~ ^"JOIN" ~ identifier ~ ^"ON" ~ identifier + ~ "," ~ ^"FORMULA" ~ string ~ "," ~ ^"SORT_KEY" ~ identifier ~ ")" +} + +table_option = _{ + immutable | state_machine | dag | propagate_edge | exclude_vector | retain | history + | sync_conflict | sync | maintain_count | delivery_manifest | edge_discard +} +state_machine = { + (^"STATE_MACHINE" | ^"STATE" ~ ^"MACHINE") + ~ "(" ~ identifier ~ ":" ~ transition ~ ("," ~ transition)* ~ ")" +} +transition = { as_written ~ "->" ~ ("[" ~ as_written ~ ("," ~ as_written)* ~ "]" | as_written) } +dag = { ^"DAG" ~ "(" ~ string ~ ("," ~ string)* ~ ")" } +propagate_edge = { + ^"PROPAGATE" ~ ^"ON" ~ ^"EDGE" ~ as_written ~ edge_direction + ~ ^"STATE" ~ as_written ~ ^"SET" ~ as_written ~ max_depth? ~ abort_on_failure? +} +edge_direction = { ^"INCOMING" | ^"OUTGOING" | ^"BOTH" } +exclude_vector = { ^"PROPAGATE" ~ ^"ON" ~ ^"STATE" ~ as_written ~ ^"EXCLUDE" ~ ^"VECTOR" } +retain = { ^"RETAIN" ~ integer ~ retain_unit ~ sync_safe? } +retain_unit = { ^"SECONDS" | ^"MINUTES" | ^"HOURS" | ^"DAYS" } +// RETAIN names no direction. One written here is read so that the refusal can name the clause +// to write instead. +sync_safe = { ^"SYNC" ~ ^"SAFE" ~ nested_direction? } +nested_direction = { ^"PUSH" ~ ^"ONLY" | ^"TWO" ~ ^"WAY" } +history = { ^"HISTORY" ~ history_kept } +history_kept = { ^"ALL" | ^"CURRENT" ~ ^"ONLY" } +sync = { ^"SYNC" ~ sync_direction } +sync_direction = { ^"OFF" | ^"PUSH" ~ ^"ONLY" | ^"PULL" ~ ^"ONLY" | ^"TWO" ~ ^"WAY" } +sync_conflict = { ^"SYNC" ~ ^"CONFLICT" ~ ^"KEEP" ~ conflict_kept } +conflict_kept = { ^"FIRST" | ^"LATEST" } +// DEFAULT takes a declared list away, so a new table is refused for it by the builder. +maintain_count = { ^"MAINTAIN" ~ ^"COUNT" ~ (count_word | count_list ~ ("," ~ count_list)*) } +count_word = { ^"ALL" ~ ^"PREFIXES" | ^"NONE" | ^"DEFAULT" } +count_list = { "(" ~ indexed_column ~ ("," ~ indexed_column)* ~ ")" } +delivery_manifest = { ^"DELIVERY" ~ ^"MANIFEST" ~ ^"OVER" ~ identifier ~ ("," ~ identifier)* } +edge_discard = { ^"EDGE" ~ ^"DISCARD" ~ discard_when } +discard_when = { ^"NEVER" | ^"AFTER" ~ ^"OUTCOME" | ^"ALWAYS" } create_index_stmt = { ^"CREATE" ~ ^"INDEX" ~ identifier ~ ^"ON" ~ identifier - ~ "(" ~ indexed_column ~ ("," ~ indexed_column)* ~ ")" + ~ "(" ~ (json_path | indexed_column ~ ("," ~ indexed_column)*) ~ ")" } +// A path into a JSON column, written as an expression in its own parentheses. +json_path = { "(" ~ identifier ~ ("->" ~ string)+ ~ ")" } indexed_column = { identifier ~ sort_dir? } +alter_table_stmt = { ^"ALTER" ~ ^"TABLE" ~ identifier ~ alter_action } +alter_action = _{ + add_column | drop_retain | drop_column | rename_column | alter_column + | ^"SET" ~ (retain | history | sync_conflict | sync | maintain_count) +} +// The clauses of a later version may follow the column, as in CREATE TABLE. +add_column = { ^"ADD" ~ column_word? ~ column_def ~ later? } +drop_retain = { ^"DROP" ~ retain_word } +drop_column = { ^"DROP" ~ column_word? ~ identifier ~ drop_behaviour? } +drop_behaviour = { ^"RESTRICT" | ^"CASCADE" } +rename_column = { ^"RENAME" ~ column_word ~ identifier ~ ^"TO" ~ identifier } +// The clauses of a VECTOR column that may change, DEFAULT taking one back to what a column that +// does not write it has; the SET of any other is a later version's. +alter_column = { + ^"ALTER" ~ column_word ~ identifier ~ ^"SET" + ~ ( + ^"MAX_PARTITIONS" ~ signed | ^"SEARCH_MODE" ~ search_mode | auto_index_at | consolidation + | auto_index_default | consolidation_default | later + ) +} +auto_index_default = { ^"AUTO_INDEX_AT" ~ ^"DEFAULT" } +consolidation_default = { ^"CONSOLIDATION" ~ ^"DEFAULT" } +// Whole words, so that a column whose name begins with one is still a column. +column_word = @{ ^"COLUMN" ~ !ident_char } +retain_word = @{ ^"RETAIN" ~ !ident_char } + +drop_table_stmt = { ^"DROP" ~ ^"TABLE" ~ if_exists? ~ identifier } +drop_index_stmt = { ^"DROP" ~ ^"INDEX" ~ if_exists? ~ identifier ~ on_table } +on_table = { ^"ON" ~ identifier } +if_exists = { ^"IF" ~ ^"EXISTS" } + insert_stmt = { ^"INSERT" ~ ^"INTO" ~ identifier ~ ("(" ~ identifier ~ ("," ~ identifier)* ~ ")")? ~ ^"VALUES" ~ values_row ~ ("," ~ values_row)* - ~ later? + ~ on_conflict? ~ later? +} +on_conflict = { + ^"ON" ~ ^"CONFLICT" ~ columns ~ ^"DO" ~ ^"UPDATE" ~ ^"SET" ~ assignment ~ ("," ~ assignment)* } values_row = { "(" ~ expr ~ ("," ~ expr)* ~ ")" } assignment = { identifier ~ "=" ~ expr } @@ -187,9 +284,18 @@ primary_expr = _{ parenthesized = { "(" ~ expr ~ ("," ~ expr)* ~ ")" } vector_lit = { "[" ~ vector_component ~ ("," ~ vector_component)* ~ "]" } vector_component = @{ "-"? ~ ASCII_DIGIT+ ~ ("." ~ ASCII_DIGIT+)? } -function_call = { identifier ~ "(" ~ (star | (expr ~ ("," ~ expr)*))? ~ ")" } +function_call = { as_written ~ "(" ~ (star | (expr ~ ("," ~ expr)*))? ~ ")" } column_ref = { identifier ~ ("." ~ identifier)? } +// A rank formula, which no statement reads: numbers and `{column}`s joined by `+` and `*`, in +// parentheses or in `coalesce(, )`. +formula = { SOI ~ formula_sum ~ EOI } +formula_sum = { formula_operand ~ (formula_op ~ formula_operand)* } +formula_op = { "+" | "*" } +formula_operand = _{ coalesce | float | integer | braced | "(" ~ formula_sum ~ ")" } +coalesce = { ^"coalesce" ~ "(" ~ formula_sum ~ "," ~ (float | integer) ~ ")" } +braced = { "{" ~ identifier ~ "}" } + parameter = @{ "$" ~ ident } null_lit = { ^"NULL" } bool_lit = { ^"TRUE" | ^"FALSE" } diff --git a/v2/contextdb/src/sql/mod.rs b/v2/contextdb/src/sql/mod.rs index efd9c3cc..8a6f2ca6 100644 --- a/v2/contextdb/src/sql/mod.rs +++ b/v2/contextdb/src/sql/mod.rs @@ -23,6 +23,13 @@ pub(crate) fn parse(text: &str) -> Result { parser::parse(text) } +/// Reads the formula of a rank policy: numbers and `{column}`s joined by `+` and `*`, in +/// parentheses or in `coalesce(, )`, as the tree any expression is. Refuses +/// with `ParseError`. +pub(crate) fn formula(text: &str) -> Result { + parser::formula(text) +} + /// Plans a parsed statement. Refuses with `PlanError`. Looks at no table and no data. pub(crate) fn plan(statement: &Statement) -> Result { planner::plan(statement) @@ -35,3 +42,9 @@ pub(crate) fn not_supported(what: &str) -> Error { "{what} is not supported in this version of the library" )) } + +/// The refusal of a form of the earlier engine that no version of the library reads, by its +/// name, with what to write in its place. +pub(crate) fn not_carried(what: &str, instead: &str) -> Error { + Error::PlanError(format!("{what} is not part of this library: {instead}")) +} diff --git a/v2/contextdb/src/sql/parser/alter.rs b/v2/contextdb/src/sql/parser/alter.rs new file mode 100644 index 00000000..74400f19 --- /dev/null +++ b/v2/contextdb/src/sql/parser/alter.rs @@ -0,0 +1,73 @@ +//! ALTER TABLE: one action, read into what it changes. + +use super::ddl::{column_def, consolidation, table_option}; +use super::{Pair, Rule, child, first_ident, ident, search_mode, signed, two_idents}; +use crate::sql::ast::*; +use crate::types::Result; + +pub(super) fn alter_table(pair: Pair<'_>) -> Result { + let mut parts = pair.into_inner(); + let table = ident(&parts.next().expect("a table")); + let part = parts.next().expect("an action"); + let action = match part.as_rule() { + Rule::add_column => { + let column = child(part, Rule::column_def).expect("a column"); + AlterAction::AddColumn(Box::new(column_def(column)?)) + } + Rule::drop_column => AlterAction::DropColumn { + column: first_ident(part.clone()), + cascade: child(part, Rule::drop_behaviour) + .map(|written| written.as_str().eq_ignore_ascii_case("CASCADE")), + }, + Rule::rename_column => { + let [from, to] = two_idents(part); + AlterAction::RenameColumn { from, to } + } + Rule::drop_retain => AlterAction::DropRetain, + Rule::alter_column => { + let column = first_ident(part.clone()); + let set = part.into_inner().nth(2).expect("a clause"); + match set.as_rule() { + Rule::signed => AlterAction::SetMaxPartitions { + column, + max_partitions: set.as_str().to_owned(), + }, + Rule::auto_index_at => AlterAction::SetAutoIndexAt(column, signed(set)), + Rule::auto_index_default => AlterAction::SetAutoIndexAt(column, None), + Rule::consolidation => { + AlterAction::SetConsolidation(column, Some(consolidation(set))) + } + Rule::consolidation_default => AlterAction::SetConsolidation(column, None), + // The one clause left: SEARCH_MODE. + _ => AlterAction::SetSearchMode { + column, + search_mode: search_mode(&set), + }, + } + } + // DEFAULT takes the declared lists away, which only an ALTER may ask. + Rule::maintain_count + if child(part.clone(), Rule::count_word) + .is_some_and(|word| word.as_str().eq_ignore_ascii_case("DEFAULT")) => + { + AlterAction::SetKeptCounts(None) + } + // SET with one option of a table, read as CREATE TABLE reads it. + _ => { + let mut options = TableOptions::default(); + table_option(part, &mut options)?; + if let Some(retain) = options.retain { + AlterAction::SetRetain(retain) + } else if let Some(history) = options.history { + AlterAction::SetHistory(history) + } else if let Some(sync) = options.sync { + AlterAction::SetSync(sync) + } else if let Some(conflict) = options.conflict { + AlterAction::SetConflict(conflict) + } else { + AlterAction::SetKeptCounts(options.kept_counts) + } + } + }; + Ok(AlterTable { table, action }) +} diff --git a/v2/contextdb/src/sql/parser/ddl.rs b/v2/contextdb/src/sql/parser/ddl.rs new file mode 100644 index 00000000..fd35ad37 --- /dev/null +++ b/v2/contextdb/src/sql/parser/ddl.rs @@ -0,0 +1,407 @@ +//! The declarations: CREATE TABLE with every clause of a column and of a table, and CREATE INDEX. + +use super::{ + Pair, Rule, child, direction, expr, first_ident, has, ident, idents, number, refuse, + search_mode, signed, string, two_idents, +}; +use crate::sql::ast::*; +use crate::types::Result; + +/// A clause written twice is refused. +fn once(written: bool, clause: &str) -> Result<()> { + match written { + true => Err(refuse(format!("duplicate {clause} clause"))), + false => Ok(()), + } +} + +/// A list that names a column twice is refused. +fn distinct(what: &str, columns: &[String]) -> Result<()> { + let twice = (columns.iter().enumerate()) + .find_map(|(at, column)| columns[..at].contains(column).then_some(column)); + match twice { + Some(twice) => Err(refuse(format!("{what} names '{twice}' twice"))), + None => Ok(()), + } +} + +/// The first part of what the grammar read as `pair`. +fn first(pair: Pair<'_>) -> Pair<'_> { + pair.into_inner() + .next() + .expect("the grammar reads a part here") +} + +/// The first word of a clause, in capitals. +fn keyword(pair: &Pair<'_>) -> String { + let letters = pair.as_str().chars().take_while(char::is_ascii_alphabetic); + letters.map(|letter| letter.to_ascii_uppercase()).collect() +} + +/// Every quoted text directly inside `pair`, in order. +fn strings(pair: Pair<'_>) -> Vec { + let texts = pair + .into_inner() + .filter(|part| part.as_rule() == Rule::string); + texts.map(|text| string(&text)).collect() +} + +/// Every state and edge type directly inside `pair`, in order. +fn values(pair: Pair<'_>) -> Vec { + let values = pair + .into_inner() + .filter(|part| part.as_rule() == Rule::as_written); + values.map(|value| ident(&value)).collect() +} + +fn indexed_column(pair: Pair<'_>) -> (String, SortDirection) { + let mut parts = pair.into_inner(); + let name = ident(&parts.next().expect("a column")); + (name, direction(parts.next())) +} + +/// The rule of an `ON STATE` and of a `PROPAGATE ON EDGE`: the last two values written are the +/// state that starts it and the state it sets. +fn propagation(pair: Pair<'_>) -> Result { + let mut states = values(pair.clone()); + let set = states.pop().expect("a state to set"); + let on_state = states.pop().expect("a state to start from"); + Ok(Propagation { + on_state, + set, + max_depth: child(pair.clone(), Rule::max_depth) + .map(|depth| number(&first(depth), "MAX DEPTH")) + .transpose()?, + abort_on_failure: has(&pair, Rule::abort_on_failure), + }) +} + +fn kept_counts(pair: Pair<'_>) -> Result { + let mut lists = Vec::new(); + for part in pair.into_inner() { + if part.as_rule() == Rule::count_word { + return match keyword(&part).as_str() { + "ALL" => Ok(KeptCounts::AllPrefixes), + "NONE" => Ok(KeptCounts::None), + _ => Err(refuse( + "MAINTAIN COUNT DEFAULT takes a declared list away, and a new table has none", + )), + }; + } + // A list or a column written twice is read as written; the table's check refuses it. + lists.push(part.into_inner().map(indexed_column).collect()); + } + Ok(KeptCounts::Lists(lists)) +} + +pub(super) fn create_table(pair: Pair<'_>) -> Result { + let mut table = CreateTable { + name: first_ident(pair.clone()), + columns: Vec::new(), + unique_constraints: Vec::new(), + primary_key: Vec::new(), + foreign_keys: Vec::new(), + if_not_exists: has(&pair, Rule::if_not_exists), + options: TableOptions::default(), + }; + for part in pair.into_inner() { + match part.as_rule() { + Rule::column_def => table.columns.push(column_def(part)?), + Rule::unique_table_constraint => { + let columns = idents(part); + distinct("UNIQUE", &columns)?; + table.unique_constraints.push(columns); + } + Rule::primary_key_columns => { + once(!table.primary_key.is_empty(), "PRIMARY KEY")?; + table.primary_key = idents(first(part)); + distinct("PRIMARY KEY", &table.primary_key)?; + } + Rule::foreign_key => { + let parent = first_ident(part.clone()); + let mut lists = part + .into_inner() + .filter(|list| list.as_rule() == Rule::columns); + let [own, columns] = [(); 2].map(|()| idents(lists.next().expect("a list"))); + distinct("FOREIGN KEY", &own)?; + if own.len() != columns.len() { + return Err(refuse(format!( + "FOREIGN KEY names {} columns and REFERENCES {}", + own.len(), + columns.len() + ))); + } + let reference = Reference { + table: parent, + columns, + on_state: None, + }; + table.foreign_keys.push((own, reference)); + } + // IF NOT EXISTS and the table's name are read above; the rest is an option. + _ => table_option(part, &mut table.options)?, + } + } + let unique = table.unique_constraints.iter(); + let foreign = table.foreign_keys.iter(); + let lists = (unique.map(|columns| ("UNIQUE", columns))) + .chain([("PRIMARY KEY", &table.primary_key)]) + .chain(foreign.map(|(own, _)| ("FOREIGN KEY", own))); + for (what, columns) in lists { + let declared = |name: &&String| table.columns.iter().any(|column| column.name == **name); + if let Some(unknown) = columns.iter().find(|name| !declared(name)) { + return Err(refuse(format!( + "{what} constraint references unknown column '{unknown}'" + ))); + } + } + Ok(table) +} + +/// Reads one option of a table into `options`, refusing one written twice. +pub(super) fn table_option(part: Pair<'_>, options: &mut TableOptions) -> Result<()> { + match part.as_rule() { + Rule::immutable => { + once(options.immutable, "IMMUTABLE")?; + options.immutable = true; + } + Rule::state_machine => { + once(options.state_machine.is_some(), "STATE MACHINE")?; + let transitions = part.clone().into_inner().skip(1).map(|transition| { + let mut states = values(transition); + (states.remove(0), states) + }); + options.state_machine = Some(StateMachine { + transitions: transitions.collect(), + column: first_ident(part), + }); + } + Rule::dag => { + once(!options.dag.is_empty(), "DAG")?; + options.dag = strings(part); + } + Rule::propagate_edge => options.propagate_edges.push(EdgePropagation { + edge_type: values(part.clone()).swap_remove(0), + direction: keyword(&child(part.clone(), Rule::edge_direction).expect("a direction")), + rule: propagation(part)?, + }), + Rule::exclude_vector => options.exclude_vector_states.extend(values(part)), + Rule::retain => { + once(options.retain.is_some(), "RETAIN")?; + let mut parts = part.into_inner(); + let count = number(&parts.next().expect("a count"), "RETAIN")?; + let unit = match keyword(&parts.next().expect("a unit")).as_str() { + "SECONDS" => RetainUnit::Seconds, + "MINUTES" => RetainUnit::Minutes, + "HOURS" => RetainUnit::Hours, + _ => RetainUnit::Days, + }; + let sync_safe = parts.next(); + if let Some(direction) = sync_safe.clone().and_then(|safe| safe.into_inner().next()) { + let clause = match keyword(&direction).as_str() { + "PUSH" => "SYNC PUSH ONLY", + _ => "SYNC TWO WAY", + }; + return Err(refuse(format!( + "RETAIN names no direction: write {clause} as a clause of its own" + ))); + } + options.retain = Some(Retain { + count, + unit, + sync_safe: sync_safe.is_some(), + }); + } + Rule::history => { + once(options.history.is_some(), "HISTORY")?; + options.history = Some(match keyword(&first(part)).as_str() { + "ALL" => History::All, + _ => History::CurrentOnly, + }); + } + Rule::sync => { + once(options.sync.is_some(), "SYNC")?; + options.sync = Some(match keyword(&first(part)).as_str() { + "OFF" => SyncDirection::Off, + "PUSH" => SyncDirection::PushOnly, + "PULL" => SyncDirection::PullOnly, + _ => SyncDirection::TwoWay, + }); + } + Rule::sync_conflict => { + once(options.conflict.is_some(), "SYNC CONFLICT")?; + options.conflict = Some(match keyword(&first(part)).as_str() { + "FIRST" => ConflictPolicy::KeepFirst, + _ => ConflictPolicy::KeepLatest, + }); + } + Rule::maintain_count => { + once(options.kept_counts.is_some(), "MAINTAIN COUNT")?; + options.kept_counts = Some(kept_counts(part)?); + } + Rule::delivery_manifest => { + once(!options.delivery_manifest.is_empty(), "DELIVERY MANIFEST")?; + options.delivery_manifest = idents(part); + } + Rule::edge_discard => { + once(options.edge_discard.is_some(), "EDGE DISCARD")?; + options.edge_discard = Some(match keyword(&first(part)).as_str() { + "NEVER" => EdgeDiscard::Never, + "AFTER" => EdgeDiscard::AfterOutcome, + _ => EdgeDiscard::Always, + }); + } + _ => {} + } + Ok(()) +} + +/// `CONSOLIDATION NONE` or `CONSOLIDATION (...)`, which is NONE when it has no member. +pub(super) fn consolidation(clause: Pair<'_>) -> ConsolidationClause { + let member = |rule| child(clause.clone(), rule).and_then(signed); + let [change, tombstone] = [Rule::change_percent, Rule::tombstone_percent].map(member); + match (change, tombstone) { + (None, None) => ConsolidationClause::None, + (change, tombstone) => ConsolidationClause::Percent { change, tombstone }, + } +} + +pub(super) fn column_def(pair: Pair<'_>) -> Result { + let mut parts = pair.into_inner(); + let name = ident(&parts.next().expect("a column name")); + let type_pair = parts.next().expect("a column type"); + let type_name = type_pair.as_str().to_ascii_uppercase(); + let mut vector = VectorClauses::default(); + let data_type = match (type_pair.into_inner().next(), type_name.as_str()) { + (Some(declared), _) => { + let mut parts = declared.into_inner(); + let dimension = parts.next().expect("a dimension"); + for clause in parts { + match clause.as_rule() { + Rule::quantization => { + let text = clause.into_inner().next().expect("a quoted text"); + vector.quantization = Some(string(&text)); + } + Rule::partition_key => vector.partition_key = idents(clause), + Rule::max_partitions => vector.max_partitions = signed(clause), + Rule::search_mode => vector.search_mode = Some(search_mode(&clause)), + Rule::auto_index_at => vector.auto_index_at = signed(clause), + // The one clause left: CONSOLIDATION. + _ => vector.consolidation = Some(consolidation(clause)), + } + } + DataType::Vector(number::(&dimension, "the VECTOR dimension")? as usize) + } + (None, "UUID") => DataType::Uuid, + (None, "TEXT") => DataType::Text, + (None, "INTEGER" | "INT") => DataType::Integer, + (None, "REAL" | "FLOAT") => DataType::Real, + (None, "BOOLEAN" | "BOOL") => DataType::Boolean, + (None, "TIMESTAMP") => DataType::Timestamp, + (None, "TXID") => DataType::TxId, + (None, _) => DataType::Json, + }; + let mut column = ColumnDef { + name, + data_type, + nullable: true, + primary_key: false, + unique: false, + default: None, + vector, + references: None, + immutable: false, + context_id: false, + scope_labels: None, + acl: None, + rank_policy: None, + }; + for constraint in parts { + match constraint.as_rule() { + Rule::not_null => { + once(!column.nullable, "NOT NULL")?; + column.nullable = false; + } + Rule::nullable_marker => {} + Rule::primary_key => { + once(column.primary_key, "PRIMARY KEY")?; + column.primary_key = true; + } + Rule::unique => { + once(column.unique, "UNIQUE")?; + column.unique = true; + } + Rule::references => { + once(column.references.is_some(), "REFERENCES")?; + let [table, parent] = two_idents(constraint.clone()); + column.references = Some(Reference { + table, + columns: vec![parent], + on_state: child(constraint, Rule::propagation) + .map(propagation) + .transpose()?, + }); + } + Rule::immutable => { + once(column.immutable, "IMMUTABLE")?; + column.immutable = true; + } + Rule::context_id => { + once(column.context_id, "CONTEXT_ID")?; + column.context_id = true; + } + Rule::scope_labels => { + once(column.scope_labels.is_some(), "SCOPE_LABEL")?; + let mut lists = constraint.into_inner().map(strings); + column.scope_labels = Some(ScopeLabels { + read: lists.next().expect("a list of labels"), + write: lists.next(), + }); + } + Rule::acl => { + once(column.acl.is_some(), "ACL")?; + let [table, granted] = two_idents(constraint); + column.acl = Some((table, granted)); + } + Rule::rank_policy => { + once(column.rank_policy.is_some(), "RANK_POLICY")?; + let mut names = idents(constraint.clone()).into_iter(); + let mut name = || names.next().expect("the grammar reads three names here"); + column.rank_policy = Some(RankPolicy { + table: name(), + column: name(), + sort_key: name(), + formula: strings(constraint).swap_remove(0), + }); + } + // The one constraint left: DEFAULT. + _ => { + once(column.default.is_some(), "DEFAULT")?; + column.default = Some(expr::expr(first(constraint))?); + } + } + } + Ok(column) +} + +pub(super) fn create_index(pair: Pair<'_>) -> CreateIndex { + let [name, table] = two_idents(pair.clone()); + let mut index = CreateIndex { + name, + table, + columns: Vec::new(), + json_path: Vec::new(), + }; + for part in pair.into_inner() { + match part.as_rule() { + Rule::indexed_column => index.columns.push(indexed_column(part)), + Rule::json_path => { + index.json_path = strings(part.clone()); + index.columns.push((first_ident(part), SortDirection::Asc)); + } + // The index's name and its table's, read above. + _ => {} + } + } + index +} diff --git a/v2/contextdb/src/sql/parser/expr.rs b/v2/contextdb/src/sql/parser/expr.rs index be63b271..4927518f 100644 --- a/v2/contextdb/src/sql/parser/expr.rs +++ b/v2/contextdb/src/sql/parser/expr.rs @@ -163,8 +163,8 @@ fn arithmetic(op: &str, left: Expr, right: Expr) -> Result { } /// `[-] primary {+ | - | * | / [-] primary}`, read left to right with `*` and `/` binding -/// tighter. -fn additive_expr(pair: Pair<'_>) -> Result { +/// tighter; a rank formula's sum the same way. +pub(super) fn additive_expr(pair: Pair<'_>) -> Result { // `sum` is everything before the last `+` or `-`, with that operator; `product` is the run // of `*` and `/` being read, with the operator that waits for its right side. let mut sum: Option<(Expr, &str)> = None; @@ -173,7 +173,7 @@ fn additive_expr(pair: Pair<'_>) -> Result { for part in pair.into_inner() { match part.as_rule() { Rule::unary_minus => minuses += 1, - Rule::math_op => { + Rule::math_op | Rule::formula_op => { let (operand, _) = product.take().expect("an operand before an operator"); let op = part.as_str(); if op == "*" || op == "/" { @@ -237,7 +237,14 @@ fn primary_expr(primary: Pair<'_>) -> Result { }) } Rule::function_call => function_call(primary), - Rule::column_ref => Ok(column_ref(primary)), + Rule::column_ref | Rule::braced => Ok(column_ref(primary)), + // The parts of a rank formula. + Rule::formula_sum => additive_expr(primary), + Rule::coalesce => { + let (name, parts) = ("coalesce".to_owned(), primary.into_inner()); + let args = parts.map(primary_expr).collect::>()?; + Ok(Expr::FunctionCall { name, args }) + } _ => literal(&primary), } } diff --git a/v2/contextdb/src/sql/parser/mod.rs b/v2/contextdb/src/sql/parser/mod.rs index b64b1911..4fbaec77 100644 --- a/v2/contextdb/src/sql/parser/mod.rs +++ b/v2/contextdb/src/sql/parser/mod.rs @@ -5,6 +5,8 @@ //! in what order) and checks only what a grammar cannot say: a number's range, a clause written //! twice, two clauses that contradict each other. +mod alter; +mod ddl; mod expr; use std::str::FromStr; @@ -13,7 +15,7 @@ use pest::Parser; use pest::error::{InputLocation, LineColLocation}; use super::ast::*; -use super::not_supported; +use super::{not_carried, not_supported}; use crate::types::{Error, Result}; #[derive(pest_derive::Parser)] @@ -40,6 +42,12 @@ pub(super) fn parse(text: &str) -> Result { Rule::vector_options => "USE VECTOR with options".to_owned(), _ => continue, }; + if matches!(&*later, "IVF" | "SEARCH") { + let instead = "the index sizes its own groups and measures how far a search scans; \ + for the true nearest rows write USE VECTOR EXACT or SEARCH_MODE EXACT, \ + and to move the size from which AUTO uses the index write AUTO_INDEX_AT"; + return Err(not_carried(&format!("{later} (...)"), instead)); + } return Err(not_supported(&later)); } let pair = read @@ -47,21 +55,20 @@ pub(super) fn parse(text: &str) -> Result { .nth(1) .expect("the grammar's statement rule holds one statement"); match pair.as_rule() { - Rule::create_table_stmt => create_table(pair).map(Statement::CreateTable), - Rule::create_index_stmt => { - let [name, table] = two_idents(pair.clone()); - Ok(Statement::CreateIndex(CreateIndex { - name, - table, - columns: pair - .into_inner() - .filter(|part| part.as_rule() == Rule::indexed_column) - .map(|column| { - let mut parts = column.into_inner(); - let name = ident(&parts.next().expect("a column")); - (name, direction(parts.next())) - }) - .collect(), + Rule::stored_procedure => Err(Error::StoredProcNotSupported), + Rule::create_table_stmt => ddl::create_table(pair).map(Statement::CreateTable), + Rule::create_index_stmt => Ok(Statement::CreateIndex(ddl::create_index(pair))), + Rule::alter_table_stmt => alter::alter_table(pair).map(Statement::AlterTable), + Rule::drop_table_stmt => Ok(Statement::DropTable(DropTable { + if_exists: has(&pair, Rule::if_exists), + name: first_ident(pair), + })), + Rule::drop_index_stmt => { + let on_table = child(pair.clone(), Rule::on_table).expect("a table"); + Ok(Statement::DropIndex(DropIndex { + if_exists: has(&pair, Rule::if_exists), + name: first_ident(pair), + table: first_ident(on_table), })) } Rule::insert_stmt => insert(pair).map(Statement::Insert), @@ -218,16 +225,25 @@ fn check_source_nesting(text: &str) -> Result<()> { Ok(()) } +/// Reads a rank formula by the grammar's formula rules, into the tree any expression is. +pub(super) fn formula(text: &str) -> Result { + check_source_nesting(text)?; + let read = SqlParser::parse(Rule::formula, text).map_err(|error| syntax_error(text, &error))?; + expr::additive_expr(read.flatten().nth(1).expect("a formula is one sum")) +} + fn refuse(message: impl Into) -> Error { Error::ParseError(message.into()) } -/// A name as written, or what is between the double quotes of a quoted one. +/// A name in lower case, or what is between the double quotes of a quoted one. What the +/// grammar reads `as_written` keeps its letters without quotes too. fn ident(pair: &Pair<'_>) -> String { let raw = pair.as_str(); match raw.strip_prefix('"').and_then(|raw| raw.strip_suffix('"')) { Some(quoted) => quoted.replace("\"\"", "\""), - None => raw.to_owned(), + None if pair.as_rule() == Rule::as_written => raw.to_owned(), + None => raw.to_ascii_lowercase(), } } @@ -277,9 +293,18 @@ fn insert(pair: Pair<'_>) -> Result { values.push(expr::exprs(row)?); } } + let on_conflict = child(pair.clone(), Rule::on_conflict) + .map(|conflict| { + Ok(OnConflict { + columns: idents(child(conflict.clone(), Rule::columns).expect("a list")), + assignments: assignments(conflict)?, + }) + }) + .transpose()?; let mut names = idents(pair); let columns = names.split_off(1); Ok(Insert { + on_conflict, table: names.swap_remove(0), columns, values, @@ -398,130 +423,5 @@ fn memory_limit(pair: Pair<'_>) -> Result { .ok_or_else(|| refuse("the size is out of range")) } -fn create_table(pair: Pair<'_>) -> Result { - let mut table = CreateTable { - name: first_ident(pair.clone()), - columns: Vec::new(), - unique_constraints: Vec::new(), - if_not_exists: has(&pair, Rule::if_not_exists), - }; - for part in pair.into_inner() { - match part.as_rule() { - Rule::column_def => table.columns.push(column_def(part)?), - Rule::unique_table_constraint => { - let columns = idents(part); - if let Some(twice) = columns - .iter() - .enumerate() - .find_map(|(at, column)| columns[..at].contains(column).then_some(column)) - { - return Err(refuse(format!("UNIQUE names '{twice}' twice"))); - } - table.unique_constraints.push(columns); - } - // IF NOT EXISTS and the table's name, read above. - _ => {} - } - } - let declared = |name: &String| table.columns.iter().any(|column| column.name == *name); - if let Some(unknown) = table - .unique_constraints - .iter() - .flatten() - .find(|column| !declared(column)) - { - return Err(refuse(format!( - "UNIQUE constraint references unknown column '{unknown}'" - ))); - } - Ok(table) -} - -fn column_def(pair: Pair<'_>) -> Result { - let mut parts = pair.into_inner(); - let name = ident(&parts.next().expect("a column name")); - let type_pair = parts.next().expect("a column type"); - let type_name = type_pair.as_str().to_ascii_uppercase(); - let mut vector = VectorClauses::default(); - let data_type = match (type_pair.into_inner().next(), type_name.as_str()) { - (Some(declared), _) => { - let mut parts = declared.into_inner(); - let dimension = parts.next().expect("a dimension"); - for clause in parts { - match clause.as_rule() { - Rule::quantization => { - let text = clause.into_inner().next().expect("a quoted text"); - vector.quantization = Some(string(&text)); - } - Rule::partition_key => vector.partition_key = idents(clause), - Rule::max_partitions => vector.max_partitions = signed(clause), - Rule::search_mode => vector.search_mode = Some(search_mode(&clause)), - Rule::auto_index_at => vector.auto_index_at = signed(clause), - // The one clause left: CONSOLIDATION, which is NONE when it has no member. - _ => { - let member = |rule| child(clause.clone(), rule).and_then(signed); - let members = ( - member(Rule::change_percent), - member(Rule::tombstone_percent), - ); - vector.consolidation = Some(match members { - (None, None) => ConsolidationClause::None, - (change, tombstone) => { - ConsolidationClause::Percent { change, tombstone } - } - }); - } - } - } - DataType::Vector(number(&dimension, "the VECTOR dimension")?) - } - (None, "UUID") => DataType::Uuid, - (None, "TEXT") => DataType::Text, - (None, "INTEGER" | "INT") => DataType::Integer, - (None, "REAL" | "FLOAT") => DataType::Real, - (None, "BOOLEAN" | "BOOL") => DataType::Boolean, - (None, "TIMESTAMP") => DataType::Timestamp, - (None, _) => DataType::Json, - }; - let mut column = ColumnDef { - name, - data_type, - nullable: true, - primary_key: false, - unique: false, - default: None, - vector, - }; - // A constraint written twice is refused. - let once = |written: bool, clause: &str| match written { - true => Err(refuse(format!("duplicate {clause} clause"))), - false => Ok(()), - }; - for constraint in parts { - match constraint.as_rule() { - Rule::not_null => { - once(!column.nullable, "NOT NULL")?; - column.nullable = false; - } - Rule::nullable_marker => {} - Rule::primary_key => { - once(column.primary_key, "PRIMARY KEY")?; - column.primary_key = true; - } - Rule::unique => { - once(column.unique, "UNIQUE")?; - column.unique = true; - } - // The one constraint left: DEFAULT. - _ => { - once(column.default.is_some(), "DEFAULT")?; - let value = constraint.into_inner().next().expect("a value"); - column.default = Some(expr::expr(value)?); - } - } - } - Ok(column) -} - #[cfg(test)] mod tests; diff --git a/v2/contextdb/src/sql/parser/tests.rs b/v2/contextdb/src/sql/parser/tests.rs index df385258..1db73655 100644 --- a/v2/contextdb/src/sql/parser/tests.rs +++ b/v2/contextdb/src/sql/parser/tests.rs @@ -199,9 +199,8 @@ fn comments_are_skipped_and_kept_inside_quoted_text() { #[test] fn what_a_later_version_reads_is_refused_by_its_name() { for (sql, name) in [ - ("ALTER TABLE t ADD COLUMN c TEXT", "ALTER"), - ("drop table t", "DROP"), ("BEGIN", "BEGIN"), + ("purge FROM t WHERE id = 1", "PURGE"), ( "WITH recent AS (SELECT a FROM t) SELECT a FROM recent", "WITH", @@ -223,24 +222,8 @@ fn what_a_later_version_reads_is_refused_by_its_name() { ("SELECT id FROM t WHERE a LIKE 'x%'", "LIKE"), ("UPDATE t SET a = 1 WHERE b BETWEEN 1 AND 2", "BETWEEN"), ( - "INSERT INTO t (a) VALUES (1) ON CONFLICT (a) DO UPDATE SET a = 2", - "ON CONFLICT", - ), - ( - "CREATE TABLE t (id UUID PRIMARY KEY) RETAIN 7 DAYS", - "RETAIN", - ), - ( - "CREATE TABLE t (id UUID PRIMARY KEY) SYNC PUSH ONLY", - "SYNC", - ), - ( - "CREATE TABLE t (id UUID PRIMARY KEY, o UUID REFERENCES p(id))", - "REFERENCES", - ), - ( - "CREATE TABLE t (a INT, b INT, FOREIGN KEY (a, b) REFERENCES o (a, b))", - "FOREIGN KEY", + "CREATE TABLE t (id UUID PRIMARY KEY, until TIMESTAMP EXPIRES)", + "EXPIRES", ), ] { match parse(sql) { @@ -556,6 +539,39 @@ fn the_clauses_of_a_vector_column_and_in_lists_are_read() { )); } +#[test] +fn ten_thousand_operands_of_and_and_or_are_read_in_the_order_written() { + /// The numbers the comparisons of a tree of ANDs and ORs hold, from left to right. + fn numbers(expr: &Expr, found: &mut Vec) { + match expr { + Expr::BinaryOp { + left, + op: BinOp::And | BinOp::Or, + right, + } => { + numbers(left, found); + numbers(right, found); + } + Expr::BinaryOp { right, .. } => match &**right { + Expr::Literal(Literal::Integer(number)) => found.push(*number), + other => panic!("a comparison with {other:?}"), + }, + other => panic!("an operand that is no comparison: {other:?}"), + } + } + for operator in ["AND", "OR"] { + let operands: Vec = (0..10_000).map(|n| format!("a = {n}")).collect(); + let sql = format!( + "SELECT id FROM t WHERE {}", + operands.join(&format!(" {operator} ")) + ); + let filter = select(&sql).where_clause.expect("a WHERE"); + let mut found = Vec::new(); + numbers(&filter, &mut found); + assert!(found.iter().copied().eq(0..10_000), "{operator}"); + } +} + #[test] fn the_index_clauses_of_a_vector_column_are_read_as_written() { let Statement::CreateTable(table) = parse( diff --git a/v2/contextdb/src/sql/plan.rs b/v2/contextdb/src/sql/plan.rs index b1bf826b..f92ee02c 100644 --- a/v2/contextdb/src/sql/plan.rs +++ b/v2/contextdb/src/sql/plan.rs @@ -1,11 +1,21 @@ //! The planned statement: what `schema`, `write` and `query` each take. -use super::ast::{ColumnDef, Expr, SetMemoryLimitValue, SortDirection, VectorSearchMode}; +/// A declaration or a write is planned as it was written: `schema` and `write` judge every +/// clause of it. +pub use super::ast::{ + AlterTable as AlterTablePlan, CreateIndex as CreateIndexPlan, CreateTable as CreateTablePlan, + Delete as DeletePlan, DropIndex as DropIndexPlan, DropTable as DropTablePlan, + Insert as InsertPlan, Update as UpdatePlan, +}; +use super::ast::{Expr, SetMemoryLimitValue, SortDirection, VectorSearchMode}; #[derive(Debug, Clone)] pub enum PhysicalPlan { CreateTable(CreateTablePlan), CreateIndex(CreateIndexPlan), + AlterTable(AlterTablePlan), + DropTable(DropTablePlan), + DropIndex(DropIndexPlan), Insert(InsertPlan), Delete(DeletePlan), Update(UpdatePlan), @@ -60,7 +70,7 @@ pub enum PhysicalPlan { /// What a planned statement does to the store. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum Effect { - /// Declares a table or an index. + /// Declares, changes or drops a table or an index. Declares, /// Adds, changes or removes rows. WritesRows, @@ -76,7 +86,11 @@ impl PhysicalPlan { /// one place every part asks. pub(crate) fn effect(&self) -> Effect { match self { - PhysicalPlan::CreateTable(_) | PhysicalPlan::CreateIndex(_) => Effect::Declares, + PhysicalPlan::CreateTable(_) + | PhysicalPlan::CreateIndex(_) + | PhysicalPlan::AlterTable(_) + | PhysicalPlan::DropTable(_) + | PhysicalPlan::DropIndex(_) => Effect::Declares, PhysicalPlan::Insert(_) | PhysicalPlan::Delete(_) | PhysicalPlan::Update(_) => { Effect::WritesRows } @@ -94,42 +108,22 @@ impl PhysicalPlan { | PhysicalPlan::ShowVectorPartitions { .. } => Effect::Reads, } } -} - -#[derive(Debug, Clone)] -pub struct CreateTablePlan { - pub name: String, - pub columns: Vec, - pub unique_constraints: Vec>, - /// `IF NOT EXISTS` was written: a table of this name that is already there is left as it is. - pub if_not_exists: bool, -} -#[derive(Debug, Clone)] -pub struct CreateIndexPlan { - pub name: String, - pub table: String, - pub columns: Vec<(String, SortDirection)>, -} - -#[derive(Debug, Clone)] -pub struct InsertPlan { - pub table: String, - pub columns: Vec, - pub values: Vec>, -} - -#[derive(Debug, Clone)] -pub struct DeletePlan { - pub table: String, - pub where_clause: Option, -} - -#[derive(Debug, Clone)] -pub struct UpdatePlan { - pub table: String, - pub assignments: Vec<(String, Expr)>, - pub where_clause: Option, + /// The statement's name when only a handle that declared no narrowing may run it: what it + /// declares, sets or reads holds for every Context, label and principal at once. + pub(crate) fn admin_only(&self) -> Option<&'static str> { + Some(match self { + PhysicalPlan::CreateTable(_) => "CREATE TABLE", + PhysicalPlan::CreateIndex(_) => "CREATE INDEX", + PhysicalPlan::AlterTable(_) => "ALTER TABLE", + PhysicalPlan::DropTable(_) => "DROP TABLE", + PhysicalPlan::DropIndex(_) => "DROP INDEX", + PhysicalPlan::SetMemoryLimit(_) => "SET MEMORY_LIMIT", + PhysicalPlan::ShowVectorIndexes => "SHOW VECTOR_INDEXES", + PhysicalPlan::ShowVectorPartitions { .. } => "SHOW VECTOR_PARTITIONS", + _ => return None, + }) + } } #[derive(Debug, Clone)] diff --git a/v2/contextdb/src/sql/planner.rs b/v2/contextdb/src/sql/planner.rs index a561de2c..b6ba85c3 100644 --- a/v2/contextdb/src/sql/planner.rs +++ b/v2/contextdb/src/sql/planner.rs @@ -15,41 +15,27 @@ const SORTABLE_FUNCTIONS: [&str; 6] = ["__add", "__sub", "__mul", "__div", "coal pub(super) fn plan(statement: &Statement) -> Result { Ok(match statement { - Statement::CreateTable(table) => PhysicalPlan::CreateTable(CreateTablePlan { - name: table.name.clone(), - columns: table.columns.clone(), - unique_constraints: table.unique_constraints.clone(), - if_not_exists: table.if_not_exists, - }), - Statement::CreateIndex(index) => PhysicalPlan::CreateIndex(CreateIndexPlan { - name: index.name.clone(), - table: index.table.clone(), - columns: index.columns.clone(), - }), + Statement::CreateTable(table) => PhysicalPlan::CreateTable(table.clone()), + Statement::CreateIndex(index) => PhysicalPlan::CreateIndex(index.clone()), + Statement::AlterTable(alter) => PhysicalPlan::AlterTable(alter.clone()), + Statement::DropTable(drop) => PhysicalPlan::DropTable(drop.clone()), + Statement::DropIndex(drop) => PhysicalPlan::DropIndex(drop.clone()), Statement::Insert(insert) => { insert.values.iter().flatten().try_for_each(rows_stand)?; - PhysicalPlan::Insert(InsertPlan { - table: insert.table.clone(), - columns: insert.columns.clone(), - values: insert.values.clone(), - }) + let conflict = insert.on_conflict.iter(); + let mut set = conflict.flat_map(|conflict| conflict.assignments.iter()); + set.try_for_each(|(_, value)| in_a_write(value))?; + PhysicalPlan::Insert(insert.clone()) } Statement::Delete(delete) => { delete.where_clause.iter().try_for_each(in_a_write)?; - PhysicalPlan::Delete(DeletePlan { - table: delete.table.clone(), - where_clause: delete.where_clause.clone(), - }) + PhysicalPlan::Delete(delete.clone()) } Statement::Update(update) => { let values = update.assignments.iter().map(|(_, value)| value); let mut written = values.chain(&update.where_clause); written.try_for_each(in_a_write)?; - PhysicalPlan::Update(UpdatePlan { - table: update.table.clone(), - assignments: update.assignments.clone(), - where_clause: update.where_clause.clone(), - }) + PhysicalPlan::Update(update.clone()) } Statement::Select(select) => plan_select(select)?, Statement::SetMemoryLimit(limit) => PhysicalPlan::SetMemoryLimit(limit.clone()), diff --git a/v2/contextdb/src/store/counts.rs b/v2/contextdb/src/store/counts.rs new file mode 100644 index 00000000..5392045b --- /dev/null +++ b/v2/contextdb/src/store/counts.rs @@ -0,0 +1,85 @@ +//! Kept counts: one record for each group of a count, under the count's number and the group's +//! values as a key, holding how many rows the group has. A count that reaches zero has no +//! record. + +use std::cmp::Ordering; + +use redb::{ReadableTable, Table}; + +use super::disk::{Stored, unsound}; +use super::keys::{Span, stored}; +use super::{ChangeSet, CountId, KeptIter, Key, Staged, Undo, staged_in}; +use crate::types::Result; + +impl ChangeSet { + /// Stages a change to one record of a kept count. + pub(crate) fn add_kept(&mut self, count: CountId, key: Key, delta: i64) -> Result<()> { + let bytes = self.counted(&key, 0)?; + *self + .kept + .entry(count.0) + .or_default() + .entry(key.clone()) + .or_default() += delta; + self.calls.push((Undo::Kept(count.0, key, delta), bytes)); + Ok(()) + } +} + +/// A kept count as its record holds it; `None` for a record that is not eight bytes. +pub(super) fn read(record: &[u8]) -> Option { + Some(u64::from_le_bytes(record.try_into().ok()?)) +} + +/// The records of a kept count inside a span with a transaction's staged changes added to +/// them, in key order; a count the changes bring to zero is left out. +pub(super) fn kept<'a>( + changes: &'a ChangeSet, + count: CountId, + span: &Span, +) -> Result> { + let mut stored = changes.base.moment.kept(count.0, span)?.peekable(); + let mut staged = staged_in(&changes.kept, count.0, span.clone()).peekable(); + Ok(Box::new(std::iter::from_fn(move || { + loop { + let stored_is = match (stored.peek(), staged.peek()) { + (None, None) => return None, + (Some(Ok((key, _))), Some((staged_key, _))) => key.cmp(staged_key), + // A failed read, or nothing staged from here on. + (Some(_), _) => Ordering::Less, + (None, Some(_)) => Ordering::Greater, + }; + let before = match stored_is { + Ordering::Less => return stored.next(), + Ordering::Equal => stored.next()?.map_or(0, |(_, count)| count), + Ordering::Greater => 0, + }; + let (key, change) = staged.next()?; + match before.saturating_add_signed(*change) { + 0 => {} + count => return Some(Ok((key.clone(), count))), + } + } + }))) +} + +/// Writes a commit's changes to the kept counts. +pub(super) fn write( + staged: &Staged, + counts: &mut Table<'_, &'static [u8], &'static [u8]>, +) -> Stored<()> { + for (count, staged) in staged { + for (key, change) in staged { + let key = stored(*count, &key.0); + let before = match counts.get(key.as_slice())? { + Some(record) => read(record.value()).ok_or_else(unsound)?, + None => 0, + }; + match before.saturating_add_signed(*change) { + 0 => drop(counts.remove(key.as_slice())?), + count => drop(counts.insert(key.as_slice(), count.to_le_bytes().as_slice())?), + } + } + } + Ok(()) +} diff --git a/v2/contextdb/src/store/disk.rs b/v2/contextdb/src/store/disk.rs index 51150c91..92840b64 100644 --- a/v2/contextdb/src/store/disk.rs +++ b/v2/contextdb/src/store/disk.rs @@ -1,15 +1,15 @@ //! What the data file holds, how one committed moment of it is read, and how a commit is //! written into it. //! -//! Seven tables of the storage library hold everything. Rows and index entries are keyed by +//! Eight tables of the storage library hold everything. Rows and index entries are keyed by //! the number of their table or index and then their key, so one table's rows are one run of //! keys. A large value is cut into slices of 243 bytes, each under a place number of its own: //! the slices of one value are under consecutive places, and its row says how many bytes there //! are and which place is the first. The vector part's blocks have a table to themselves, and -//! its partition entries and pending entries share another. The last table holds the few -//! numbers the store keeps: the newest commit, the commit that last changed the schema, the -//! next free place, the declared memory limit, each table's row count, and the vector part's -//! three numbers. +//! its partition entries and pending entries share another. Kept counts have one. The last +//! table holds the few numbers the store keeps: the newest commit, the commit that last +//! changed the schema, the next free place, the declared memory limit, each table's row count, +//! and the vector part's three numbers. use std::io::{self, ErrorKind}; use std::ops::{Bound, Range}; @@ -20,9 +20,12 @@ use redb::{ReadOnlyTable, ReadTransaction, ReadableDatabase, ReadableTable, Tabl use super::folder::Hold; use super::keys::{self, Span, stored}; -use super::{Beside, Cell, ChangeSet, EntryIter, IndexId, Key, Read, Row, RowIter, TableId, View}; +use super::{ + Beside, ChangeSet, CountId, EntryIter, IndexId, Joined, KeptIter, Key, Read, Row, RowIter, + TableId, View, +}; use super::{VectorNumber, VectorRecord}; -use super::{record, split}; +use super::{counts, drop, record, split}; use crate::memory::Reservation; use crate::types::{Error, Result, TxId, Value}; @@ -31,10 +34,14 @@ pub(super) type Stored = std::result::Result; /// Rows: table number and key, to the row as `record` writes it. const ROWS: TableDefinition<&[u8], &[u8]> = TableDefinition::new("rows"); -/// Index entries: index number, entry and row key, to the length of the entry. +/// Index entries: index number, entry and row key, to the length of the entry in four bytes +/// and, for an entry that holds a cut value, the place its first one's kept bytes end in four +/// more. const INDEX: TableDefinition<&[u8], &[u8]> = TableDefinition::new("index"); /// Large values: place, to one slice of the record `record` stores for the value. const BESIDE: TableDefinition = TableDefinition::new("beside"); +/// Kept counts: count number and the group's key, to the count in eight bytes. +const COUNTS: TableDefinition<&[u8], &[u8]> = TableDefinition::new("counts"); /// Schema records by name, as `schema` wrote them. const SCHEMA: TableDefinition<&str, &[u8]> = TableDefinition::new("schema"); /// The numbers the store keeps. @@ -60,7 +67,7 @@ const NEXT_PLACE: u64 = 2; const MEMORY_LIMIT: u64 = 3; /// Where a table's row count is kept among the numbers. -fn count_of(table: u32) -> u64 { +pub(super) fn count_of(table: u32) -> u64 { (1 << 32) | u64::from(table) } @@ -79,6 +86,17 @@ fn places(kept: &Beside) -> Range { kept.place..kept.place + kept.bytes.div_ceil(SLICE as u64) } +/// Removes the slices of a large value. +pub(super) fn remove_slices( + beside: &mut redb::Table<'_, u64, &'static [u8]>, + kept: &Beside, +) -> Stored<()> { + for place in places(kept) { + beside.remove(place)?; + } + Ok(()) +} + /// Tells a failure of the storage library or of a file as the engine tells it: as damage when /// stored data failed its check, and otherwise as a failure of `operation`, which is what the /// store was doing. @@ -112,7 +130,7 @@ pub(super) fn failed( } /// What a record that is not one the store wrote is told as: a failed check. -fn unsound() -> redb::Error { +pub(super) fn unsound() -> redb::Error { redb::Error::Corrupted("a stored record is not one the store wrote".to_owned()) } @@ -141,6 +159,7 @@ pub(super) fn prepare(data: &redb::Database) -> Stored<()> { write.open_table(ROWS)?; write.open_table(INDEX)?; write.open_table(BESIDE)?; + write.open_table(COUNTS)?; write.open_table(SCHEMA)?; write.open_table(NUMBERS)?; write.open_table(BLOCKS)?; @@ -149,11 +168,12 @@ pub(super) fn prepare(data: &redb::Database) -> Stored<()> { Ok(()) } -/// One committed moment: the seven tables as one read of the storage library shows them. +/// One committed moment: the eight tables as one read of the storage library shows them. pub(super) struct Moment { rows: ReadOnlyTable<&'static [u8], &'static [u8]>, index: ReadOnlyTable<&'static [u8], &'static [u8]>, beside: ReadOnlyTable, + counts: ReadOnlyTable<&'static [u8], &'static [u8]>, schema: ReadOnlyTable<&'static str, &'static [u8]>, numbers: ReadOnlyTable, blocks: ReadOnlyTable<&'static [u8], &'static [u8]>, @@ -175,6 +195,7 @@ impl Moment { rows: read.open_table(ROWS)?, index: read.open_table(INDEX)?, beside: read.open_table(BESIDE)?, + counts: read.open_table(COUNTS)?, schema: read.open_table(SCHEMA)?, blocks: read.open_table(BLOCKS)?, vectors: read.open_table(VECTORS)?, @@ -189,7 +210,13 @@ impl Moment { read.map_err(|error| failed(&self.folder, "read", error)) } - /// The records of one table or index inside a span, in key order, each read with `read`. + /// One of the numbers the store keeps; zero when it was never put. + pub(super) fn number(&self, at: u64) -> Result { + self.told(number(&self.numbers, at)) + } + + /// The records of one table, index or count inside a span, in key order, each read with + /// `read` from its key and its value. fn scan( &self, table: &ReadOnlyTable<&'static [u8], &'static [u8]>, @@ -198,7 +225,7 @@ impl Moment { read: F, ) -> Result> + use> where - F: Fn(&[u8]) -> Option, + F: Fn(&[u8], &[u8]) -> Option, { let folder = self.folder.clone(); let range = table @@ -206,8 +233,10 @@ impl Moment { .map_err(|error| failed(&folder, "read", error))?; Ok(range.map(move |entry| { let (key, record) = entry.map_err(|error| failed(&folder, "read", error))?; - let read = read(record.value()).ok_or_else(|| failed(&folder, "read", unsound()))?; - Ok((Key(key.value()[4..].to_vec()), read)) + let key = &key.value()[4..]; + let read = read(key, record.value()); + let read = read.ok_or_else(|| failed(&folder, "read", unsound()))?; + Ok((Key(key.to_vec()), read)) })) } @@ -217,7 +246,7 @@ impl Moment { table: u32, span: &Span, ) -> Result> + use<>> { - self.scan(&self.rows, table, span, |row| { + self.scan(&self.rows, table, span, |_, row| { Some(Row { cells: record::decode_row(row), }) @@ -225,17 +254,36 @@ impl Moment { } /// The entries of an index inside a span, each still joined to the key of its row, with - /// the length of the entry. + /// the length of the entry and the place its first cut value's kept bytes end. A record + /// of another size than four or eight bytes, an entry longer than its key and a place past + /// the entry are damage. pub(super) fn joined( &self, index: u32, span: &Span, - ) -> Result> + use<>> { - self.scan(&self.index, index, span, |length| { - Some(u32::from_le_bytes(*length.first_chunk()?) as usize) + ) -> Result> + use<>> { + self.scan(&self.index, index, span, |joined, record| { + let number = + |at: usize| Some(u32::from_le_bytes(*record.get(at..)?.first_chunk()?) as usize); + let length = number(0)?; + let cut = match record.len() { + 4 => None, + 8 => Some(number(4)?), + _ => return None, + }; + (length <= joined.len() && cut.is_none_or(|cut| cut <= length)).then_some((length, cut)) }) } + /// The records of a kept count inside a span. One that is not eight bytes is damage. + pub(super) fn kept( + &self, + count: u32, + span: &Span, + ) -> Result> + use<>> { + self.scan(&self.counts, count, span, |_, record| counts::read(record)) + } + /// Every schema record, in the order of their names. fn schema(&self) -> Stored>> { let mut records = Vec::new(); @@ -296,9 +344,13 @@ impl Read for View { Ok(Box::new(joined.map(split))) } + fn kept(&self, count: CountId, from: Bound<&Key>, to: Bound<&Key>) -> Result> { + let kept = self.moment.kept(count.0, &keys::span(from, to))?; + Ok(Box::new(kept)) + } + fn count(&self, table: TableId) -> Result { - self.moment - .told(number(&self.moment.numbers, count_of(table.0))) + self.moment.number(count_of(table.0)) } fn load(&self, beside: &Beside, reservation: &mut Reservation) -> Result { @@ -343,8 +395,7 @@ impl Read for View { } fn vector_number(&self, number: VectorNumber) -> Result { - let at = vector_number_at(number); - self.moment.told(self::number(&self.moment.numbers, at)) + self.moment.number(vector_number_at(number)) } fn damaged(&self, reason: &str) -> Error { @@ -392,11 +443,13 @@ pub(super) fn write(data: &redb::Database, changes: &mut ChangeSet) -> Stored Stored { - let row = + let left = &row.left; + let mut row = record::encode_row(values, &row.compact, |large| -> Stored { // The compressor's buffer is counted until the value is written. let room = record::compress(large, &mut changes.reserved); @@ -417,20 +471,21 @@ pub(super) fn write(data: &redb::Database, changes: &mut ChangeSet) -> Stored { - let before = rows.remove(key.as_slice())?; - before.map(|row| record::decode_row(row.value())) + record::leave(&mut row, left); + rows.insert(key.as_slice(), row.as_slice())? } + None => rows.remove(key.as_slice())?, }; - for cell in before.into_iter().flatten() { - if let Cell::Beside(kept) = cell { - for place in places(&kept) { - beside.remove(place)?; - } - } + let before: Vec = before + .iter() + .flat_map(|row| record::besides(row.value())) + .collect(); + // A value the new row left where it is stays; the others go with the old row. + for kept in before + .iter() + .filter(|kept| row.left.iter().all(|left| left.1 != **kept)) + { + remove_slices(&mut beside, kept)?; } } } @@ -438,14 +493,18 @@ pub(super) fn write(data: &redb::Database, changes: &mut ChangeSet) -> Stored { - let length = u32::try_from(*length).expect("an index entry is short"); - index.insert(key.as_slice(), length.to_le_bytes().as_slice())?; + Some((length, cut)) => { + let number = |number: usize| { + let number = u32::try_from(number).expect("an index entry is short"); + number.to_le_bytes() + }; + let record = [number(*length), number(cut.unwrap_or(0))]; + let record = &record.as_flattened()[..if cut.is_some() { 8 } else { 4 }]; + index.insert(key.as_slice(), record)?; } None => drop(index.remove(key.as_slice())?), } @@ -457,9 +516,27 @@ pub(super) fn write(data: &redb::Database, changes: &mut ChangeSet) -> Stored drop(schema.insert(name.as_str(), record.as_slice())?), + None => drop(schema.remove(name.as_str())?), + } } if !changes.schema.is_empty() { numbers.insert(SCHEMA_COMMIT, commit)?; @@ -471,8 +548,6 @@ pub(super) fn write(data: &redb::Database, changes: &mut ChangeSet) -> Stored {} } - let mut blocks = write.open_table(BLOCKS)?; - let mut vectors = write.open_table(VECTORS)?; for (key, value) in &changes.vectors { let table = if keys::with_blocks(key.0[0]) { &mut blocks @@ -484,7 +559,7 @@ pub(super) fn write(data: &redb::Database, changes: &mut ChangeSet) -> Stored drop(table.remove(key.0.as_slice())?), } } - for (at, number) in &changes.vector_numbers { + for (at, number) in &changes.numbers { numbers.insert(*at, *number)?; } } diff --git a/v2/contextdb/src/store/drop.rs b/v2/contextdb/src/store/drop.rs new file mode 100644 index 00000000..b8ed6d72 --- /dev/null +++ b/v2/contextdb/src/store/drop.rs @@ -0,0 +1,126 @@ +//! Dropping: what a transaction stages to remove whole, and how a commit removes it. A table's +//! rows go with their large values and an index's entries go, a storage page at a time with +//! no row held; one cell goes from every row of a table, one row in memory at a time; every +//! record and number of a VECTOR column goes, a storage page at a time. + +use redb::{ReadableTable, Table}; + +use super::disk::{self, Stored}; +use super::keys::{self, stored}; +use super::record; +use super::{ChangeSet, CountId, IndexId, STAGED_BYTES, TableId, Undo, VectorId, VectorNumber}; +use crate::types::Result; + +/// One thing a transaction has staged to drop. +pub(crate) enum Dropped { + /// Every row of a table, with its large values, its row count and its next row number. + Rows(TableId), + /// Every entry of an index. + Index(IndexId), + /// Every record of a kept count. + Kept(CountId), + /// The cell at this position of every row of a table, with the large value it keeps. A row + /// that has no cell there stays as it is. + Cell(TableId, usize), + /// Every record and number of a VECTOR column, and its rows' place among those waiting. + Vectors(VectorId), +} + +impl ChangeSet { + /// Stages the removal of one thing whole. + pub(crate) fn stage_drop(&mut self, dropped: Dropped) -> Result<()> { + self.reserved.grow(STAGED_BYTES)?; + self.dropped.push(dropped); + self.calls.push((Undo::Dropped, STAGED_BYTES)); + Ok(()) + } +} + +/// Removes what a commit drops, in the order it was staged. +pub(super) fn write( + dropped: &[Dropped], + rows: &mut Table<'_, &'static [u8], &'static [u8]>, + beside: &mut Table<'_, u64, &'static [u8]>, + index: &mut Table<'_, &'static [u8], &'static [u8]>, + counts: &mut Table<'_, &'static [u8], &'static [u8]>, + numbers: &mut Table<'_, u64, u64>, + [blocks, vectors]: [&mut Table<'_, &'static [u8], &'static [u8]>; 2], +) -> Stored<()> { + for dropped in dropped { + match *dropped { + Dropped::Rows(table) => { + let (from, end) = all(table.0); + let all = from.as_slice()..end.as_slice(); + for row in rows.extract_from_if::<&[u8], _>(all, |_, _| true)? { + for kept in record::besides(row?.1.value()) { + disk::remove_slices(beside, &kept)?; + } + } + numbers.remove(disk::count_of(table.0))?; + } + Dropped::Index(id) => { + let (from, end) = all(id.0); + index.retain_in::<&[u8], _>(from.as_slice()..end.as_slice(), |_, _| false)?; + } + Dropped::Kept(id) => { + let (from, end) = all(id.0); + counts.retain_in::<&[u8], _>(from.as_slice()..end.as_slice(), |_, _| false)?; + } + Dropped::Cell(table, slot) => { + let (mut from, end) = all(table.0); + loop { + let mut after = rows.range::<&[u8]>(from.as_slice()..end.as_slice())?; + let Some(next) = after.next() else { break }; + // The row is copied out, so that no page is lent while the table changes. + let (key, without) = { + let (key, row) = next?; + ( + key.value().to_vec(), + record::without_cell(row.value(), slot), + ) + }; + drop(after); + if let Some((without, kept)) = without { + rows.insert(key.as_slice(), without.as_slice())?; + if let Some(kept) = kept { + disk::remove_slices(beside, &kept)?; + } + } + from = key; + from.push(0); + } + } + Dropped::Vectors(column) => { + let mut pending = 0; + for kind in 0..=keys::LEFT { + let table = if keys::with_blocks(kind) { + &mut *blocks + } else { + &mut *vectors + }; + let [from, end] = [column.0, column.0 + 1] + .map(|at| [&[kind][..], &at.to_be_bytes()].concat()); + table.retain_in::<&[u8], _>(from.as_slice()..end.as_slice(), |_, _| { + pending += u64::from(kind == keys::PENDING); + false + })?; + } + let waiting = disk::vector_number_at(VectorNumber::Pending); + let left = numbers + .get(waiting)? + .map_or(0, |n| n.value()) + .saturating_sub(pending); + numbers.insert(waiting, left)?; + numbers.remove(disk::vector_number_at(VectorNumber::Partitions(column)))?; + numbers.remove(disk::vector_number_at(VectorNumber::NextPartition(column)))?; + } + } + } + Ok(()) +} + +/// The first stored key of a table or an index and the end of its keys: after the number, no +/// key begins with 0xFF. +fn all(id: u32) -> (Vec, Vec) { + (stored(id, &[]), stored(id, &[0xFF])) +} diff --git a/v2/contextdb/src/store/folder.rs b/v2/contextdb/src/store/folder.rs index 081ef588..1987b739 100644 --- a/v2/contextdb/src/store/folder.rs +++ b/v2/contextdb/src/store/folder.rs @@ -29,6 +29,10 @@ pub(super) struct DataFile(Box); impl DataFile { pub(super) fn over(file: File, storage: Option<&WrapStorage>) -> io::Result { + // What a folder reads as differs between systems, so nothing is asked of one. + if !file.metadata()?.is_file() { + return Err(io::Error::other("the store's data file is not a file")); + } Ok(DataFile(match storage { Some(wrap) => wrap(file)?, None => Box::new(Plain(file)), diff --git a/v2/contextdb/src/store/keys.rs b/v2/contextdb/src/store/keys.rs index d27f955a..fb400485 100644 --- a/v2/contextdb/src/store/keys.rs +++ b/v2/contextdb/src/store/keys.rs @@ -12,7 +12,7 @@ use std::ops::Bound; use super::{Key, VectorId, VectorRecord}; -use crate::types::{Error, Result, SortDirection, TxId, Uuid, Value}; +use crate::types::{ColumnType, Error, Result, SortDirection, TxId, Uuid, Value}; /// No value's form begins with this byte, turned or not, so a key followed by it sorts after /// every key that begins with that key and before every other key above it. @@ -21,9 +21,18 @@ const PAST: u8 = 0xFF; /// The bytes from the first, which a span includes, to the second, which it does not. pub(super) type Span = (Vec, Vec); -/// The form [`Key::from_values`] gives. -pub(super) fn encode(values: &[Value], directions: &[SortDirection]) -> Result> { +/// The form [`Key::from_values`] gives, where `room` is `usize::MAX`: every value whole. +/// +/// A text whose form is longer than `room` bytes is cut to exactly `room`: its leading bytes, +/// the mark and the digest of the whole text. The second answer is where in the key the kept +/// bytes of the first cut text end. +pub(super) fn encode( + values: &[Value], + directions: &[SortDirection], + room: usize, +) -> Result<(Vec, Option)> { let mut key = Vec::new(); + let mut first_cut = None; for (column, value) in values.iter().enumerate() { let at = key.len(); match value { @@ -34,14 +43,28 @@ pub(super) fn encode(values: &[Value], directions: &[SortDirection]) -> Result { key.push(4); // A zero byte inside the text is followed by 0xFF, and two zero bytes end it: - // a text sorts before every longer text it begins. + // a text sorts before every longer text it begins. The mark of a cut text + // sorts after that end and before any further byte. + let end = at.saturating_add(room - CUT_BYTES); + let mut cut = None; for byte in text.bytes() { + if key.len() + 1 + usize::from(byte == 0) > end { + cut = Some(key.len()); + break; + } key.push(byte); if byte == 0 { key.push(0xFF); } } - key.extend([0, 0]); + match cut { + None => key.extend([0, 0]), + Some(_) => { + key.extend([0, 1]); + key.extend(digest(text)); + } + } + first_cut = first_cut.or(cut); } Value::Uuid(id) => put(&mut key, 5, id.as_bytes()), Value::Timestamp(millis) => put(&mut key, 6, &signed(*millis)), @@ -56,7 +79,21 @@ pub(super) fn encode(values: &[Value], directions: &[SortDirection]) -> Result [u8; 16] { + let mut digest: u128 = 0x6c62_272e_07bb_0142_62b8_2175_6295_c58d; + for byte in text.bytes() { + digest ^= u128::from(byte); + digest = digest.wrapping_mul(0x0000_0000_0100_0000_0000_0000_0000_013b); + } + digest.to_be_bytes() } fn put(key: &mut Vec, tag: u8, bytes: &[u8]) { @@ -116,7 +153,7 @@ pub(super) fn stored(id: u32, key: &[u8]) -> Vec { /// of their own. pub(super) const BLOCK: u8 = 0; const PARTITION: u8 = 1; -const PENDING: u8 = 2; +pub(super) const PENDING: u8 = 2; const GROUPING: u8 = 3; /// The first byte of the stored key of a record of centres, which is kept with the blocks. pub(super) const CENTRES: u8 = 4; @@ -125,7 +162,7 @@ const ASSIGNED: u8 = 6; /// The first byte of the stored key of a record of sketches, which is kept with the blocks. pub(super) const SKETCH: u8 = 7; const MEAN: u8 = 8; -const LEFT: u8 = 9; +pub(super) const LEFT: u8 = 9; /// True for the kinds of record that are kept in the table of blocks. pub(super) fn with_blocks(kind: u8) -> bool { @@ -213,6 +250,58 @@ pub(super) fn vector_of<'a>(stored: &[u8], key: &'a mut Key) -> Option, + room: usize, +} + +/// One index entry, and where its first cut value's kept bytes end when it holds one. +pub(crate) struct Entry { + pub key: Key, + pub cut: Option, +} + +impl EntryShape { + /// The shape of the entries of an index over columns of these types and directions. + pub(crate) fn new(types: &[ColumnType], directions: &[SortDirection]) -> EntryShape { + let texts = types.iter().filter(|column| **column == ColumnType::Text); + let fixed = types.iter().map(|column| match column { + ColumnType::Text => 0, + ColumnType::Uuid => 17, + ColumnType::Boolean => 2, + _ => 9, + }); + EntryShape { + directions: directions.to_vec(), + room: ((KEY_LIMIT - fixed.sum::()) / texts.count().max(1)).min(ENTRY_TEXT), + } + } + + /// The entry of a row's values in the index's columns; given only leading values, the bytes + /// every entry with those values begins with. This is the only place an entry is made. + /// + /// A text that fits its column's room is whole, as [`Key::from_values`] writes it. A longer + /// one is cut, so entries are in the order of their values except that long texts with + /// the same kept bytes follow each other in the order of their digests. + pub(crate) fn entry(&self, values: &[Value]) -> Result { + let (key, cut) = encode(values, &self.directions, self.room)?; + Ok(Entry { key: Key(key), cut }) + } +} + /// The values a key of ascending columns was made from; `None` for bytes that are no such key. pub(super) fn decode(mut key: &[u8]) -> Option> { let whole = |bytes: [u8; 8]| i64::from_be_bytes(bytes) ^ i64::MIN; diff --git a/v2/contextdb/src/store/mod.rs b/v2/contextdb/src/store/mod.rs index c30e87b5..f74386e2 100644 --- a/v2/contextdb/src/store/mod.rs +++ b/v2/contextdb/src/store/mod.rs @@ -8,7 +8,9 @@ //! Two of the things the parts meet at are defined here: [`Read`], the view to read from, and //! [`ChangeSet`], the change set to write with. +mod counts; mod disk; +mod drop; mod folder; mod keys; mod record; @@ -27,10 +29,12 @@ pub use redb::StorageBackend; use redb::{ReadTransaction, ReadableDatabase}; use crate::memory::{Budget, Reservation}; -use crate::types::{Error, Result, SortDirection, TxId, Value}; +use crate::types::{Error, Result, SortDirection, TxId, Value, heap_bytes}; use disk::{Moment, Stored}; +pub(crate) use drop::Dropped; use folder::{DataFile, Hold, Locked}; use keys::Span; +pub(crate) use keys::{Entry, EntryShape, KEY_LIMIT}; /// The size of the page cache of a store opened under this memory limit, in bytes: a quarter /// of the limit and never less than 8 MiB, which is also its size when there is no limit. It is @@ -72,6 +76,10 @@ pub(crate) struct TableId(pub u32); #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub(crate) struct IndexId(pub u32); +/// The number of a kept count, given by `schema` like an index's. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub(crate) struct CountId(pub u32); + /// A VECTOR column's number inside one store, given by `schema` like a [`TableId`]. Every record /// the vector part keeps for the column begins with it. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] @@ -156,7 +164,7 @@ impl Key { /// The key of the leading columns of an index, made with their directions, is the bytes /// that every full key with those values begins with, and no other key begins with them. pub(crate) fn from_values(values: &[Value], directions: &[SortDirection]) -> Result { - keys::encode(values, directions).map(Key) + Ok(Key(keys::encode(values, directions, usize::MAX)?.0)) } /// The bytes of the key, for a part that keeps keys inside a record of its own. @@ -218,13 +226,22 @@ pub(crate) struct Beside { /// Rows in key order. pub(crate) type RowIter<'a> = Box> + 'a>; -/// Index entries in entry order, each with the key of the row it points at. -pub(crate) type EntryIter<'a> = Box> + 'a>; +/// Index entries in entry order, each with the key of the row it points at and the place its +/// first cut value's kept bytes end, when it holds one. +pub(crate) type EntryIter<'a> = Box)>> + 'a>; + +/// The records of a kept count in key order, each with its count. +pub(crate) type KeptIter<'a> = Box> + 'a>; -/// The rows a change set has staged: table, key, the new values or `None` for a delete, and -/// whether the transaction found no row at the key when it first staged it. +/// The rows a change set has staged: table, key, the new row or `None` for a delete, and +/// whether the transaction found no row at the key when it first staged it. A row is its +/// values, with NULL for each large value it left where it is, and those cells with their +/// places in the row. pub(crate) type StagedRows<'a> = - Box, bool)> + 'a>; + Box>, bool)> + 'a>; + +/// A staged row's values, and the large cells it left where they are. +pub(crate) type StagedValues<'a> = (&'a [Value], &'a [(usize, Beside)]); /// The view to read rows from. `query` and `write` read a store only through this. /// @@ -251,6 +268,9 @@ pub(crate) trait Read { /// range whatever row it points at. fn index(&self, index: IndexId, from: Bound<&Key>, to: Bound<&Key>) -> Result>; + /// The records of a kept count between two keys, in key order. + fn kept(&self, count: CountId, from: Bound<&Key>, to: Bound<&Key>) -> Result>; + /// How many rows the table holds at this moment. One small record, never a scan. fn count(&self, table: TableId) -> Result; @@ -318,31 +338,56 @@ pub(crate) struct ChangeSet { /// Staged rows by table and key. rows: Staged, /// Staged index entries by index, each entry joined to the key of its row: the length of - /// the entry, or `None` for a removal. - index: Staged>, + /// the entry and where its first cut value's kept bytes end, or `None` for a removal. + index: Staged>, /// The change to each table's kept row count. counts: BTreeMap, - /// Staged schema records by name. - schema: BTreeMap>, + /// The change to each record of a kept count, by count and key. + kept: Staged, + /// Staged schema records by name: the record, or `None` for a removal. + schema: BTreeMap>>, + /// What is to be dropped whole, in the order it was staged. + dropped: Vec, + /// The row guards, in the order they were staged. + guards: Vec, + /// The number the transaction took when it began; zero when it took none. + number: TxId, /// The memory limit the store is to remember, when one was staged: `Some(None)` for none. memory_limit: Option>, /// Staged records of the vector part by their stored key: the value, or `None` for a /// removal. vectors: BTreeMap>>, - /// Staged numbers of the vector part by their place among the store's numbers. - vector_numbers: BTreeMap, + /// Staged numbers of the vector part, by their place among the store's numbers. + numbers: BTreeMap, + /// How many rows of tables that declare no primary key the change set has made. + made: u64, /// Every staging call of a statement in order, each with what it put aside and the bytes it /// was counted as, so that the calls after a mark can be undone. calls: Vec<(Undo, u64)>, reserved: Reservation, } -/// Staged records of several tables or indexes, by number and then by key. +/// Staged records of several tables, indexes or counts, by number and then by key. type Staged = BTreeMap>; +/// The length of an index entry inside the key that joins it to its row, and where the +/// entry's first cut value's kept bytes end, when it holds one. +type Joined = (usize, Option); + +/// The values a row must still hold when its transaction is committed: the row's table and +/// key, and each guarded cell by its position. +pub(crate) struct RowGuard { + pub table: TableId, + pub key: Key, + pub columns: Vec<(usize, Value)>, +} + struct StagedRow { /// The new values, or `None` for a removal. values: Option>, + /// The large values that stay where they are, each by its cell's position; NULL stands at + /// that position in `values`. + left: Vec<(usize, Beside)>, /// The vector cells to write as a scale and codes. compact: Vec, /// What the transaction said when it first staged this key: it found no row there. @@ -353,11 +398,13 @@ struct StagedRow { /// `None` when nothing was. enum Undo { Row(u32, Key, Option), - Entry(u32, Key, Option>), + Entry(u32, Key, Option>), Count(u32, i64), - Schema(String, Option>), + Kept(u32, Key, i64), + Schema(String, Option>>), Vector(Key, Option>>), - VectorNumber(u64, Option), + Number(u64, Option), + Dropped, } /// What one staging call is counted as beside its key and its values: its place in the map @@ -372,10 +419,15 @@ impl ChangeSet { rows: BTreeMap::new(), index: BTreeMap::new(), counts: BTreeMap::new(), + kept: BTreeMap::new(), schema: BTreeMap::new(), + dropped: Vec::new(), + guards: Vec::new(), + number: TxId(0), memory_limit: None, vectors: BTreeMap::new(), - vector_numbers: BTreeMap::new(), + numbers: BTreeMap::new(), + made: 0, calls: Vec::new(), reserved: budget.reserve(0, "transaction")?, }) @@ -410,16 +462,22 @@ impl ChangeSet { put_back(self.index.entry(index).or_default(), joined, before); } Undo::Count(table, delta) => *self.counts.entry(table).or_default() -= delta, + Undo::Kept(count, key, delta) => { + *self.kept.entry(count).or_default().entry(key).or_default() -= delta; + } Undo::Schema(name, before) => put_back(&mut self.schema, name, before), Undo::Vector(key, before) => put_back(&mut self.vectors, key, before), - Undo::VectorNumber(at, before) => put_back(&mut self.vector_numbers, at, before), + Undo::Number(at, before) => put_back(&mut self.numbers, at, before), + Undo::Dropped => drop(self.dropped.pop()), } self.reserved.shrink(bytes); } } /// Stages a row, new or replacing the row with this key. Large text, JSON and vector values - /// are kept beside the row when it is written; the caller does not split them. + /// are kept beside the row when it is written; the caller does not split them. A + /// [`Cell::Beside`] is a value this row holds already, handed back as it was read: it is + /// written as the same place and none of its bytes is read or written. /// /// `compact` names the vector cells to keep in one byte a number, each with its scale and /// codes: such a cell is written as the tag `0x88`, its count of numbers, the scale and one @@ -431,17 +489,29 @@ impl ChangeSet { &mut self, table: TableId, key: Key, - values: Vec, + cells: Vec, compact: Vec, new_key: bool, ) -> Result<()> { - self.stage_row(table, key, Some(values), compact, new_key) + let mut left = Vec::new(); + let values = cells + .into_iter() + .enumerate() + .map(|(slot, cell)| match cell { + Cell::Here(value) => value, + Cell::Beside(kept) => { + left.push((slot, kept)); + Value::Null + } + }); + let values = Some(values.collect()); + self.stage_row(table, key, values, left, compact, new_key) } /// Stages the removal of a row and of the values kept beside it. A key first staged here /// is one a row was at. pub(crate) fn delete_row(&mut self, table: TableId, key: &Key) -> Result<()> { - self.stage_row(table, key.clone(), None, Vec::new(), false) + self.stage_row(table, key.clone(), None, Vec::new(), Vec::new(), false) } fn stage_row( @@ -449,28 +519,30 @@ impl ChangeSet { table: TableId, key: Key, values: Option>, + left: Vec<(usize, Beside)>, compact: Vec, new_key: bool, ) -> Result<()> { - let held: u64 = values.iter().flatten().map(value_bytes).sum(); + let held: u64 = values.iter().flatten().map(held).sum(); let codes: usize = compact.iter().map(|cell| cell.codes.len()).sum(); let bytes = self.counted(&key, held + codes as u64)?; let rows = self.rows.entry(table.0).or_default(); let new_key = rows.get(&key).map_or(new_key, |before| before.new_key); - let staged = StagedRow { + let row = StagedRow { values, + left, compact, new_key, }; - let before = rows.insert(key.clone(), staged); + let before = rows.insert(key.clone(), row); self.calls.push((Undo::Row(table.0, key, before), bytes)); Ok(()) } /// Stages an index entry pointing at a row. - pub(crate) fn put_index(&mut self, index: IndexId, entry: Key, row: Key) -> Result<()> { - let length = entry.0.len(); - self.stage_entry(index, Key([entry.0, row.0].concat()), Some(length)) + pub(crate) fn put_index(&mut self, index: IndexId, entry: Entry, row: Key) -> Result<()> { + let joined = (entry.key.0.len(), entry.cut); + self.stage_entry(index, Key([entry.key.0, row.0].concat()), Some(joined)) } /// Stages the removal of an index entry. @@ -478,7 +550,7 @@ impl ChangeSet { self.stage_entry(index, Key([entry.0.as_slice(), &row.0].concat()), None) } - fn stage_entry(&mut self, index: IndexId, joined: Key, entry: Option) -> Result<()> { + fn stage_entry(&mut self, index: IndexId, joined: Key, entry: Option) -> Result<()> { let bytes = self.counted(&joined, 0)?; let entries = self.index.entry(index.0).or_default(); let before = entries.insert(joined.clone(), entry); @@ -497,7 +569,17 @@ impl ChangeSet { /// Stages a schema record under a name, new or in place of the record with that name. pub(crate) fn put_schema(&mut self, name: &str, record: Vec) -> Result<()> { - let bytes = STAGED_BYTES + 2 * name.len() as u64 + record.len() as u64; + self.stage_schema(name, Some(record)) + } + + /// Stages the removal of the schema record under a name. + pub(crate) fn delete_schema(&mut self, name: &str) -> Result<()> { + self.stage_schema(name, None) + } + + fn stage_schema(&mut self, name: &str, record: Option>) -> Result<()> { + let held = record.as_ref().map_or(0, Vec::len); + let bytes = STAGED_BYTES + 2 * name.len() as u64 + held as u64; self.reserved.grow(bytes)?; let before = self.schema.insert(name.to_owned(), record); self.calls @@ -505,6 +587,57 @@ impl ChangeSet { Ok(()) } + /// How many rows of tables that declare no primary key this change set has made, the one + /// just made among them: with the transaction's number, the key such a row is kept under. + pub(crate) fn next_row_number(&mut self) -> u64 { + self.made += 1; + self.made + } + + /// Stages a row guard: the values the cells at these positions of a row, as this + /// transaction shows it, must still hold when the transaction is committed. Each position + /// is turned into the cell's position in the stored row, undoing, last first, the drops of + /// cells this transaction has staged for the table before the guard. It writes nothing, + /// so it is no staging call and no rewind takes it back; [`ChangeSet::guards`] hands it to + /// the rule check at commit. + pub(crate) fn guard_row( + &mut self, + table: TableId, + key: Key, + mut columns: Vec<(usize, Value)>, + ) -> Result<()> { + for (at, _) in &mut columns { + for dropped in self.dropped.iter().rev() { + if let Dropped::Cell(of, slot) = *dropped + && of == table + && slot <= *at + { + *at += 1; + } + } + } + let held = columns.iter().map(|(_, value)| held(value)).sum(); + self.counted(&key, held)?; + self.guards.push(RowGuard { + table, + key, + columns, + }); + Ok(()) + } + + /// Every staged row guard, for the rule check at commit. + pub(crate) fn guards(&self) -> &[RowGuard] { + &self.guards + } + + /// Records the number the transaction took when it began. Its commit is written under + /// that number when no commit has taken it or a higher one since, and under the next + /// number after the newest commit otherwise. + pub(crate) fn set_number(&mut self, number: TxId) { + self.number = number; + } + /// Stages the memory limit the store is to remember from this commit on, in bytes, or /// `None` for no limit. No statement of a transaction stages one, so a rewind leaves it. pub(crate) fn put_memory_limit(&mut self, limit: Option) -> Result<()> { @@ -538,9 +671,8 @@ impl ChangeSet { pub(crate) fn put_vector_number(&mut self, number: VectorNumber, value: u64) -> Result<()> { self.reserved.grow(STAGED_BYTES)?; let at = disk::vector_number_at(number); - let before = self.vector_numbers.insert(at, value); - self.calls - .push((Undo::VectorNumber(at, before), STAGED_BYTES)); + let before = self.numbers.insert(at, value); + self.calls.push((Undo::Number(at, before), STAGED_BYTES)); Ok(()) } @@ -552,20 +684,37 @@ impl ChangeSet { Ok(bytes) } - /// Every staged row, for the rule check at commit. + /// Every staged row, for the rule check at commit. A large value the row left where it is + /// was not read: it is NULL among the values and its cell is handed over beside them. pub(crate) fn rows_changed(&self) -> StagedRows<'_> { Box::new(self.rows.iter().flat_map(|(table, staged)| { - staged - .iter() - .map(|(key, row)| (TableId(*table), key, row.values.as_deref(), row.new_key)) + staged.iter().map(|(key, row)| { + let staged = row.values.as_deref().map(|values| (values, &*row.left)); + (TableId(*table), key, staged, row.new_key) + }) })) } - /// True when nothing is staged. The commit function asks before it commits, so that a - /// transaction that changed nothing writes nothing and spends no commit number. + /// True when nothing is staged to be written; a row guard writes nothing. The commit + /// function asks before it commits, so that a transaction that changed nothing writes + /// nothing and spends no commit number. pub(crate) fn is_empty(&self) -> bool { self.calls.is_empty() && self.memory_limit.is_none() } + + /// A stored row of `table` as it is once the cells this change set drops are gone, so that + /// a row is read by the same places inside the transaction as after its commit. + fn shaped(&self, table: TableId, mut row: Row) -> Row { + for dropped in &self.dropped { + if let Dropped::Cell(of, slot) = *dropped + && of == table + && slot < row.cells.len() + { + row.cells.remove(slot); + } + } + row + } } /// Puts a staged record back as it was before one staging call. @@ -576,28 +725,9 @@ fn put_back(records: &mut BTreeMap, key: K, before: Option) } } -/// What a value is counted as while a transaction holds it. -fn value_bytes(value: &Value) -> u64 { - let grown = match value { - Value::Text(text) => text.len() as u64, - Value::Vector(numbers) => 4 * numbers.len() as u64, - Value::Json(json) => json_bytes(json), - _ => 0, - }; - size_of::() as u64 + grown -} - -fn json_bytes(json: &serde_json::Value) -> u64 { - let grown = match json { - serde_json::Value::String(text) => text.len() as u64, - serde_json::Value::Array(items) => items.iter().map(json_bytes).sum(), - serde_json::Value::Object(members) => members - .iter() - .map(|(name, member)| name.len() as u64 + json_bytes(member)) - .sum(), - _ => 0, - }; - size_of::() as u64 + grown +/// What a value is counted as while a transaction holds it: its cell and what it takes outside. +fn held(value: &Value) -> u64 { + size_of::() as u64 + heap_bytes(value) } /// The staged records of one table or index inside a span, in key order. @@ -641,18 +771,27 @@ fn overlay<'a, T: 'a>( })) } -/// A staged row as a read returns it. Its values are in memory already, so none is beside. -fn shown(values: &[Value]) -> Row { - Row { - cells: values.iter().cloned().map(Cell::Here).collect(), +/// A staged row as a read returns it, `None` for a removal. Its values are in memory already, +/// so none is beside but those it left where they are. +fn shown(row: &StagedRow) -> Option { + let mut cells: Vec = row + .values + .iter() + .flatten() + .cloned() + .map(Cell::Here) + .collect(); + for (slot, kept) in &row.left { + cells[*slot] = Cell::Beside(kept.clone()); } + row.values.is_some().then_some(Row { cells }) } /// Parts an index entry from the key of its row. -fn split(joined: Result<(Key, usize)>) -> Result<(Key, Key)> { - let (Key(mut entry), length) = joined?; +fn split(joined: Result<(Key, Joined)>) -> Result<(Key, Key, Option)> { + let (Key(mut entry), (length, cut)) = joined?; let row = entry.split_off(length); - Ok((Key(entry), Key(row))) + Ok((Key(entry), Key(row), cut)) } impl Read for ChangeSet { @@ -662,8 +801,11 @@ impl Read for ChangeSet { fn row(&self, table: TableId, key: &Key) -> Result> { match self.rows.get(&table.0).and_then(|staged| staged.get(key)) { - Some(staged) => Ok(staged.values.as_deref().map(shown)), - None => self.base.row(table, key), + Some(staged) => Ok(shown(staged)), + None => Ok(self + .base + .row(table, key)? + .map(|row| self.shaped(table, row))), } } @@ -671,7 +813,12 @@ impl Read for ChangeSet { let span = keys::span(from, to); let stored = self.base.moment.rows(table.0, &span)?; let staged = staged_in(&self.rows, table.0, span); - let staged = staged.map(|(key, row)| (key, row.values.as_deref().map(shown))); + let staged = staged.map(|(key, row)| (key, shown(row))); + // A stored row is reshaped only while this change set drops something. + if self.dropped.is_empty() { + return Ok(overlay(stored, staged)); + } + let stored = stored.map(move |row| row.map(|(key, row)| (key, self.shaped(table, row)))); Ok(overlay(stored, staged)) } @@ -682,6 +829,10 @@ impl Read for ChangeSet { Ok(Box::new(overlay(stored, staged).map(split))) } + fn kept(&self, count: CountId, from: Bound<&Key>, to: Bound<&Key>) -> Result> { + counts::kept(self, count, &keys::span(from, to)) + } + fn count(&self, table: TableId) -> Result { let change = self.counts.get(&table.0).copied().unwrap_or(0); Ok(self.base.count(table)?.saturating_add_signed(change)) @@ -977,11 +1128,14 @@ impl Store { /// Writes a change set as one commit and waits for the disk. /// - /// Rows, the values kept beside them, index entries, counts, schema records, the vector + /// Whatever is dropped whole goes after the staged records, and takes with it those staged + /// under its number: every row of a table with its large values, every entry of an index, + /// or one cell of every row of a table, a storage page or a row at a time. Rows, the values kept beside them, index entries, counts, schema records, the vector /// part's records and numbers and the list of changed keys go in one storage transaction: /// after a kill either all of it is there or none. A large text or JSON value is compressed /// on its way in, in a buffer counted against the change set's reservation while it is - /// used. The caller holds the commit lock and has laid the change set over the newest + /// used. The commit takes the number the change set's transaction took, or the next after + /// the newest commit when that is higher. The caller holds the commit lock and has laid the change set over the newest /// moment; one over an older moment is refused with `StoreIo` and writes nothing. Returns /// the commit's number. Only a store opened for writing is given a change set to commit. pub(crate) fn commit(&self, mut changes: ChangeSet) -> Result { diff --git a/v2/contextdb/src/store/record.rs b/v2/contextdb/src/store/record.rs index 504a8e99..0bf76e9d 100644 --- a/v2/contextdb/src/store/record.rs +++ b/v2/contextdb/src/store/record.rs @@ -176,14 +176,81 @@ pub(super) fn decode_row(mut row: &[u8]) -> Vec { cells } +/// Steps over the first cell of a stored row without making its value, and answers with the +/// large value the cell keeps beside the row, when it keeps one. +fn skip_cell(row: &mut &[u8]) -> Option { + let width = match take(row, 1)[0] { + NULL => 0, + BOOL => 1, + UUID => 16, + TEXT | JSON | VECTOR => u64::from_le_bytes(take_array(row)) as usize, + REDUCED => u64::from_le_bytes(take_array(row)) as usize + 4, + BESIDE => { + return Some(Beside { + bytes: u64::from_le_bytes(take_array(row)), + place: u64::from_le_bytes(take_array(row)), + }); + } + _ => 8, + }; + take(row, width); + None +} + +/// The bytes of a stored row from its cell at `slot` on: none for a row with no cell there, +/// which was written before the column was added. +fn from_cell(mut row: &[u8], slot: usize) -> &[u8] { + for _ in 0..slot { + if !row.is_empty() { + skip_cell(&mut row); + } + } + row +} + +/// The large values a stored row keeps beside it. None of the row's values is made. +pub(super) fn besides(mut row: &[u8]) -> impl Iterator { + std::iter::from_fn(move || { + while !row.is_empty() { + if let Some(kept) = skip_cell(&mut row) { + return Some(kept); + } + } + None + }) +} + +/// A stored row without its cell at `slot`, and the large value that cell kept beside the row. +/// `None` for a row with no cell there. +pub(super) fn without_cell(row: &[u8], slot: usize) -> Option<(Vec, Option)> { + let mut rest = from_cell(row, slot); + let start = row.len() - rest.len(); + if rest.is_empty() { + return None; + } + let kept = skip_cell(&mut rest); + Some(([&row[..start], rest].concat(), kept)) +} + +/// Says again, in a row just written, where each large value that was left where it is is +/// kept. `left` names them by cell, and NULL was written in the cell of each. +pub(super) fn leave(row: &mut Vec, left: &[(usize, Beside)]) { + for (slot, kept) in left { + let at = row.len() - from_cell(row, *slot).len(); + let place = [kept.bytes.to_le_bytes(), kept.place.to_le_bytes()]; + row.splice(at..=at, [&[BESIDE][..], place.as_flattened()].concat()); + } +} + /// Makes a large value from its stored record, in either form. `reservation` already holds /// the record's bytes, and holds the bytes of the value's plain record when the value is made. /// /// A compressed record is decoded first. The reservation is grown by the bytes of the plain /// record before one of them exists, so a value that does not fit under the memory limit is /// refused and never produced; the stored bytes are given back once it is decoded. A text -/// keeps the plain record's buffer as its own. While a JSON or vector value is made, the -/// record and the value both exist, and the reservation holds the bytes twice for that moment. +/// keeps the plain record's buffer as its own. A JSON tree is counted, before it is made, as +/// the most its text can make. While a vector is made, the record and the vector both exist, +/// and the reservation holds the bytes twice for that moment. /// /// The inner error is the reason a record's own bytes do not support the length it gives: /// the store is damaged. Nothing is allocated from such a length. @@ -233,14 +300,16 @@ pub(super) fn load( let text = String::from_utf8(stored).map(Value::Text); return Ok(text.map_err(|_| "a large text is not text")); } + if stored[0] == JSON { + // The tree is counted at the most its text can make before it is made, and is held + // by the reservation as long as the value is. + reservation.grow(types::json_bytes(&stored[FRONT..]))?; + let json = serde_json::from_slice(&stored[FRONT..]).map(Value::Json); + return Ok(json.map_err(|_| "a large JSON value is not JSON")); + } let twice = stored.len() as u64; reservation.grow(twice)?; - let json = &stored[FRONT..]; - let value = match stored[0] { - JSON => serde_json::from_slice(json).ok().map(Value::Json), - VECTOR => Some(decode_value(&stored)), - _ => None, - }; + let value = (stored[0] == VECTOR).then(|| decode_value(&stored)); reservation.shrink(twice); Ok(value.ok_or("a large value is not text, JSON or a vector")) } diff --git a/v2/contextdb/src/store/tests.rs b/v2/contextdb/src/store/tests.rs index 16df5e8d..c3136ec6 100644 --- a/v2/contextdb/src/store/tests.rs +++ b/v2/contextdb/src/store/tests.rs @@ -1,22 +1,25 @@ //! Checks of the pure parts of the store: the order keys take as bytes, the span a pair of -//! bounds stands for, and a row written and read back. And the store's own check of a -//! compressed value's recorded length: the storage library checks every page it reads, so a -//! length changed on its way from the file never reaches that check, and the tests of damage -//! here write the wrong bytes into the data file as a commit of the storage library: a wrong -//! length, a large value with a slice missing, and a block that decodes to the recorded length -//! of bytes that are no value of its kind. +//! bounds stands for, and a row written and read back. And the store's own checks of stored +//! bytes that disagree with themselves: the storage library checks every page it reads, so these +//! tests write the wrong bytes into the data file as a commit of the storage library, which no +//! public call can: a large value with a slice missing, a block that decodes to the recorded +//! length of bytes that are no value of its kind, a reduced vector of another length, a kept +//! count, and an index entry with the place its first long value is cut at. And the one rule by +//! which an index entry is made: never longer than the limit, in the order of its values, and +//! the same bytes every time. use std::ops::Bound; use std::path::{Path, PathBuf}; use lz4_flex::block::{compress_into, get_maximum_output_size}; -use redb::ReadableTable; +use redb::{ReadableDatabase, ReadableTable}; use super::{ - Cell, ChangeSet, Compact, Key, Read, Store, TableId, VectorId, VectorRecord, keys, record, + Cell, ChangeSet, Compact, CountId, EntryShape, IndexId, KEY_LIMIT, Key, Read, Store, TableId, + VectorId, VectorRecord, keys, record, }; use crate::memory::Budget; -use crate::types::{Error, SortDirection, TxId, Uuid, Value}; +use crate::types::{ColumnType, Error, Result, SortDirection, TxId, Uuid, Value}; fn key(values: &[Value]) -> Key { Key::from_values(values, &[]).expect("these values can be part of a key") @@ -177,146 +180,6 @@ fn a_row_reads_back_as_written_with_its_large_values_beside_it() { /// The storage library's table of large values, as `disk` declares it: place to slice. const BESIDE: redb::TableDefinition = redb::TableDefinition::new("beside"); -#[test] -fn a_compressed_value_whose_recorded_length_is_wrong_is_damage_and_takes_no_memory() { - const MIB: u64 = 1 << 20; - // 16 KiB of one sentence, which the store keeps compressed in a block of about a hundredth - // of that. - const LEN: u64 = 16 << 10; - let folder = - std::env::temp_dir().join(format!("contextdb-wrong-length-{}", std::process::id())); - if folder.exists() { - std::fs::remove_dir_all(&folder).expect("clear the folder an earlier run left"); - } - let sentence = "A store keeps what it is given and hands it back as it was. "; - let body: String = sentence.chars().cycle().take(LEN as usize).collect(); - let (table, row_key) = (TableId(1), key(&[Value::Int64(1)])); - - // One row whose text goes beside it, written under no limit so that it is compressed. - let budget = Budget::new(None).expect("a budget"); - let store = Store::open_for_writing(&folder, &budget, None).expect("open a fresh store"); - let mut changes = ChangeSet::new(store.view().expect("a view"), &budget).expect("begin"); - let row = vec![Value::Int64(1), text(&body)]; - changes - .put_row(table, row_key.clone(), row, Vec::new(), true) - .expect("stage"); - store.commit(changes).expect("commit"); - drop(store); - - // Opens the store under a limit that leaves `room` bytes once it is open, and loads the - // text. Returns what the load gave, where the value is kept, the bytes the statement - // holds after the load, and whether what the budget counts is back where it was once - // the statement has let go. - let load = |room: u64| { - let budget = Budget::new(Some(9 * MIB + room)).expect("a budget"); - let store = Store::open_for_writing(&folder, &budget, None).expect("open the store"); - let view = store.view().expect("a view"); - let row = view.row(table, &row_key).expect("read the row"); - let Some(Cell::Beside(beside)) = row.expect("the row is there").cells.pop() else { - panic!("a text of {LEN} bytes was kept in its row"); - }; - let before = budget.used(); - let mut reservation = budget.reserve(0, "query").expect("a reservation"); - let loaded = view.load(&beside, &mut reservation); - let held = reservation.bytes(); - drop(reservation); - (loaded, beside, held, budget.used() == before) - }; - let (loaded, beside, held, given_back) = load(MIB); - assert!(loaded.expect("load the value") == text(&body)); - assert_eq!(held, 9 + LEN, "held for the loaded value"); - assert!(given_back, "counted once the value is let go"); - // The compressed record: nine bytes, then one block. - let block = beside.bytes - 9; - assert!( - beside.bytes < LEN / 50, - "{} bytes are stored for a repeating text of {LEN}: it is not compressed", - beside.bytes - ); - // The room every load below is given: what reading the value takes at its most, the - // stored record and the value's own record together, one byte and a little more. No - // block decodes to more than 255 times its own bytes, and that much does not fit. - let room = beside.bytes + 9 + LEN + 1 + 64; - assert!(LEN <= 255 * block && 9 + 255 * block > room); - - // Writes this length where the record keeps its own, as a commit of the storage library, - // which then hands the slice over as a page that passes its check. - let record_length = |wrong: u64| { - let data = redb::Database::open(folder.join("data")).expect("open the data file"); - let write = data.begin_write().expect("begin"); - { - let mut slices = write.open_table(BESIDE).expect("the large values"); - let first = slices.get(beside.place).expect("read the first slice"); - let mut first = first.expect("the first slice is there").value().to_vec(); - assert_eq!(first[0], 0x84, "the first byte of a compressed text"); - first[1..9].copy_from_slice(&wrong.to_le_bytes()); - slices - .insert(beside.place, first.as_slice()) - .expect("write the first slice"); - } - write.commit().expect("commit"); - }; - // The outcome of a load that must make no value: the reason of the store's damage, which - // names the folder and the large value, or `None` for the memory limit's refusal. Either - // way the statement holds nothing afterwards and nothing stays counted. - let damage = |wrong: u64| -> Option { - record_length(wrong); - let (loaded, _, held, given_back) = load(room); - assert_eq!(held, 0, "held after a recorded length of {wrong}"); - assert!(given_back, "counted after a recorded length of {wrong}"); - match loaded { - Err(Error::StoreCorrupted { path, reason }) => { - assert_eq!(path, folder, "a recorded length of {wrong}"); - assert!(reason.contains("large value"), "{wrong}: {reason}"); - Some(reason) - } - Err(Error::MemoryBudgetExceeded { operation, .. }) => { - assert_eq!(operation, "query", "a recorded length of {wrong}"); - None - } - other => panic!("a recorded length of {wrong} gave {other:?}"), - } - }; - - // The value reads under that room as it is. - let (loaded, _, held, _) = load(room); - assert!(loaded.expect("load the value under the room") == text(&body)); - assert_eq!(held, 9 + LEN); - - // A length the block cannot decode to, from one more than 255 times its bytes to the - // largest number there is, is damage before any memory is asked for. The limit has room - // for none of them: a length that was asked for would be the limit's refusal, and one - // that was taken without asking would end the test. - for wrong in [255 * block + 1, 1 << 40, 1 << 62, u64::MAX - 8, u64::MAX] { - assert!( - damage(wrong).is_some(), - "memory was asked for a recorded length of {wrong}" - ); - } - // The most a wrong length can ask for is 255 times the block, and it is asked for before - // it is taken: with no room for it, it is the limit's refusal, as for any value that - // does not fit. - assert!( - damage(255 * block).is_none(), - "no memory was asked for a recorded length of 255 times the block" - ); - // A length there is room for, and that the block does not decode to, is damage once the - // block is decoded: one byte more than the value has, one byte less, half of it, none. - for wrong in [LEN + 1, LEN - 1, LEN / 2, 0] { - assert!( - damage(wrong).is_some(), - "a recorded length of {wrong} was refused for want of room" - ); - } - - // With its own length back, the value is as it was written. - record_length(LEN); - let (loaded, _, held, _) = load(room); - assert!(loaded.expect("load the value") == text(&body)); - assert_eq!(held, 9 + LEN); - std::fs::remove_dir_all(&folder).expect("remove the test's folder"); -} - /// The bytes of one slice of a large value, as `disk` cuts them. const SLICE: u64 = 243; @@ -341,7 +204,7 @@ fn store_one_row(folder: &Path, value: Value) { let budget = Budget::new(None).expect("a budget"); let store = Store::open_for_writing(folder, &budget, None).expect("open a fresh store"); let mut changes = ChangeSet::new(store.view().expect("a view"), &budget).expect("begin"); - let row = vec![Value::Int64(1), value]; + let row = vec![Cell::Here(Value::Int64(1)), Cell::Here(value)]; changes .put_row(table, row_key, row, Vec::new(), true) .expect("stage"); @@ -502,6 +365,483 @@ fn a_compressed_json_that_decodes_to_bytes_that_are_not_json_is_damage() { a_block_of_other_bytes_is_damage("not-json", &json, wrong.as_bytes()); } +// ---- the kept count and the index entry, as records ---- + +/// The storage library's tables of kept counts and of index entries, as `disk` declares them. +const COUNTS: redb::TableDefinition<&[u8], &[u8]> = redb::TableDefinition::new("counts"); +const INDEX: redb::TableDefinition<&[u8], &[u8]> = redb::TableDefinition::new("index"); + +/// Every record of a kept count, in key order, or the refusal of the read. +fn kept(read: &dyn Read, count: CountId) -> Result> { + read.kept(count, Bound::Unbounded, Bound::Unbounded) + .and_then(|counts| counts.collect()) +} + +/// Every entry of an index, in entry order, each with its row key and the place its first long +/// value is cut at, or the refusal of the read. +fn entries(read: &dyn Read, index: IndexId) -> Result)>> { + read.index(index, Bound::Unbounded, Bound::Unbounded) + .and_then(|entries| entries.collect()) +} + +/// Opens the store in this folder under no memory limit and runs `read` on its newest moment. +fn on_the_newest_moment(folder: &Path, read: impl FnOnce(&dyn Read) -> T) -> T { + let budget = Budget::new(None).expect("a budget"); + let store = Store::open_for_writing(folder, &budget, None).expect("open the store"); + let view = store.view().expect("a view"); + read(&view) +} + +/// Stages with `stage` on the store's newest moment and commits. +fn commit_staged(folder: &Path, stage: impl FnOnce(&mut ChangeSet)) { + let budget = Budget::new(None).expect("a budget"); + let store = Store::open_for_writing(folder, &budget, None).expect("open the store"); + let mut changes = ChangeSet::new(store.view().expect("a view"), &budget).expect("begin"); + stage(&mut changes); + store.commit(changes).expect("commit"); +} + +/// Asserts that a read was refused as damage of the store in this folder. +fn assert_damage(read: Result, folder: &Path, what: &str) { + match read { + Err(Error::StoreCorrupted { path, .. }) => assert_eq!(path, folder, "{what}"), + other => panic!("{what} was read as {other:?}"), + } +} + +/// Every record of one of the storage library's tables, as (key, value). +fn records(folder: &Path, table: redb::TableDefinition<&[u8], &[u8]>) -> Vec<(Vec, Vec)> { + let data = redb::Database::open(folder.join("data")).expect("open the data file"); + let read = data.begin_read().expect("begin"); + let table = read.open_table(table).expect("the table"); + table + .iter() + .expect("read the table") + .map(|record| { + let (key, value) = record.expect("read a record"); + (key.value().to_vec(), value.value().to_vec()) + }) + .collect() +} + +/// Writes `value` under `key` in one of the storage library's tables, as a commit of that +/// library, which then hands the record over as a page that passes its check. +fn write_record( + folder: &Path, + table: redb::TableDefinition<&[u8], &[u8]>, + key: &[u8], + value: &[u8], +) { + let data = redb::Database::open(folder.join("data")).expect("open the data file"); + let write = data.begin_write().expect("begin"); + write + .open_table(table) + .expect("the table") + .insert(key, value) + .expect("write the record"); + write.commit().expect("commit"); +} + +#[test] +fn a_kept_count_that_reaches_zero_leaves_no_record() { + // No public call tells a zero record from none: the count reads 0 either way. + let folder = fresh_folder("kept-count-zero"); + let (count, group) = (CountId(3), key(&[Value::Int64(1)])); + commit_staged(&folder, |changes| { + changes.add_kept(count, group.clone(), 2).expect("stage"); + }); + assert_eq!(records(&folder, COUNTS).len(), 1); + commit_staged(&folder, |changes| { + changes.add_kept(count, group.clone(), -2).expect("stage"); + }); + assert!( + records(&folder, COUNTS).is_empty(), + "a count taken down to zero keeps no record" + ); + std::fs::remove_dir_all(&folder).expect("remove the test's folder"); +} + +#[test] +fn a_kept_count_whose_record_is_not_eight_bytes_is_damage() { + let folder = fresh_folder("kept-count-damaged"); + let count = CountId(1); + let group = key(&[text("group")]); + commit_staged(&folder, |changes| { + changes.add_kept(count, group.clone(), 7).expect("stage"); + }); + let [(stored_key, stored_value)] = records(&folder, COUNTS) + .try_into() + .expect("one count was written, so one record is there"); + assert_eq!(stored_value.len(), 8, "a kept count is eight bytes"); + + for wrong in [ + &stored_value[..7], + &[stored_value.as_slice(), &[0]].concat(), + &[], + ] { + write_record(&folder, COUNTS, &stored_key, wrong); + let read = on_the_newest_moment(&folder, |view| kept(view, count)); + assert_damage( + read, + &folder, + &format!("a kept count of {} bytes", wrong.len()), + ); + } + // With its own bytes back, the count is what was written. + write_record(&folder, COUNTS, &stored_key, &stored_value); + let read = on_the_newest_moment(&folder, |view| kept(view, count)); + assert_eq!(read.expect("read the count"), [(group, 7)]); + std::fs::remove_dir_all(&folder).expect("remove the test's folder"); +} + +/// The most bytes a TEXT column takes in an index entry (the engine's `ENTRY_TEXT`, which tests +/// cannot name). Every shape below leaves each of its TEXT columns at least this much. +const TEXT_ROOM: usize = 256; + +/// The longest text an entry keeps whole: of its room a cut text gives 19 to its tag, its mark +/// and its digest, and a longer one is kept as this many bytes and those. +const WHOLE: usize = TEXT_ROOM - 19; + +/// An index over an INTEGER and a TEXT column, both ascending. +fn integer_and_text() -> EntryShape { + let directions = [SortDirection::Asc, SortDirection::Asc]; + EntryShape::new(&[ColumnType::Integer, ColumnType::Text], &directions) +} + +/// Commits two entries of index 2: one whose text is short, for row 1, and one whose text is +/// cut, for row 2. Returns what reading the index must give back. +fn store_two_entries(folder: &Path) -> Vec<(Key, Key, Option)> { + let shape = integer_and_text(); + let values = [ + [Value::Int64(1), text("short")], + [Value::Int64(1), text(&"x".repeat(5000))], + ]; + let rows = [key(&[Value::Int64(1)]), key(&[Value::Int64(2)])]; + let entry = |at: usize| shape.entry(&values[at]).expect("an entry"); + assert_eq!(entry(0).cut, None, "a short text is kept whole"); + // Nine bytes of the INTEGER, the text's tag, and the bytes kept of the text. + let cut = Some(9 + 1 + WHOLE); + assert_eq!(entry(1).cut, cut, "where the long text is cut"); + let wanted = vec![ + (entry(0).key, rows[0].clone(), None), + (entry(1).key, rows[1].clone(), cut), + ]; + commit_staged(folder, |changes| { + for (at, row) in rows.iter().enumerate() { + changes + .put_index(IndexId(2), entry(at), row.clone()) + .expect("stage"); + } + // A change set reads the entries it has staged, each with its place. + assert_eq!(entries(changes, IndexId(2)).expect("read"), wanted); + }); + wanted +} + +#[test] +fn an_index_record_whose_value_does_not_fit_its_key_is_damage() { + let folder = fresh_folder("index-entry-damaged"); + let wanted = store_two_entries(&folder); + let stored = records(&folder, INDEX); + // The whole entry records its length in four bytes; the cut one records its place as well. + let lengths: Vec = stored.iter().map(|(_, value)| value.len()).collect(); + assert_eq!(lengths, [4, 8]); + + for (stored_key, stored_value) in &stored { + let mut wrongs = vec![ + stored_value[..3].to_vec(), + [stored_value.as_slice(), &[0]].concat(), + Vec::new(), + [stored_value.as_slice(), &[0; 8]].concat(), + ]; + if stored_value.len() == 8 { + // A place past the end of the entry, whichever way its four bytes are read. + wrongs.push([&stored_value[..4], &[0xFF; 4][..]].concat()); + } + for wrong in wrongs { + write_record(&folder, INDEX, stored_key, &wrong); + let read = on_the_newest_moment(&folder, |view| entries(view, IndexId(2))); + let what = format!( + "an index record of {} bytes where {} were written", + wrong.len(), + stored_value.len() + ); + assert_damage(read, &folder, &what); + } + write_record(&folder, INDEX, stored_key, stored_value); + } + // With their own bytes back, the entries are what was written. + let read = on_the_newest_moment(&folder, |view| entries(view, IndexId(2))); + assert_eq!(read.expect("read the entries"), wanted); + std::fs::remove_dir_all(&folder).expect("remove the test's folder"); +} + +// ---- how an index entry is made ---- + +/// The 128-bit FNV-1a digest of a text, most significant byte first. +fn digest(text: &str) -> [u8; 16] { + let mut digest: u128 = 0x6c62_272e_07bb_0142_62b8_2175_6295_c58d; + for byte in text.bytes() { + digest ^= u128::from(byte); + digest = digest.wrapping_mul(0x0000_0000_0100_0000_0000_0000_0000_013b); + } + digest.to_be_bytes() +} + +/// The bytes of the entry these values make. +fn entry_bytes(shape: &EntryShape, values: &[Value]) -> Vec { + let entry = shape.entry(values).expect("an entry"); + entry.key.as_bytes().to_vec() +} + +#[test] +fn an_entry_is_never_longer_than_the_limit_and_is_the_same_bytes_every_time() { + use ColumnType::{Boolean, Integer, Text, Timestamp, Uuid as UuidType}; + use SortDirection::{Asc, Desc}; + assert_eq!(KEY_LIMIT, 2704); + // Texts that an entry keeps whole and texts it must cut, for any share of an entry a TEXT + // column can have: short ones, zero bytes (each is two bytes in an entry), characters of + // several bytes, and lengths on both sides of every place a text is cut at below. + let mut texts = vec![ + String::new(), + "a".to_owned(), + "a\0b".to_owned(), + "\0".repeat(3000), + format!("{}\0{}", "z".repeat(60), "z".repeat(5000)), + "\u{65e5}\u{672c}".repeat(1000), + scattered(60_000), + ]; + for len in [ + 64, + 65, + 66, + WHOLE - 1, + WHOLE, + WHOLE + 1, + TEXT_ROOM, + KEY_LIMIT, + ] { + texts.push("x".repeat(len)); + } + let other = |column_type: ColumnType| match column_type { + Integer => Value::Int64(-7), + Boolean => Value::Bool(true), + Timestamp => Value::Timestamp(1), + _ => Value::Uuid(Uuid::from_u128(7)), + }; + let shapes: Vec<(Vec, Vec)> = vec![ + (vec![Text], vec![Asc]), + (vec![Text], vec![Desc]), + (vec![Text, Text], vec![Asc, Desc]), + (vec![UuidType, Text, Text, Text], vec![Asc; 4]), + ( + vec![Integer, Text, Timestamp, Text, Boolean], + vec![Desc, Asc, Asc, Desc, Asc], + ), + ( + vec![UuidType, UuidType, Text, UuidType, Text, Text], + vec![Asc; 6], + ), + (vec![Text; 32], vec![Asc; 32]), + ]; + for (types, directions) in shapes { + let shape = EntryShape::new(&types, &directions); + // Every text in every TEXT column at once, the same text in each, then each text + // beside the one after it, then NULL in every column. + let mut rows: Vec> = Vec::new(); + for at in 0..texts.len() { + let mut next = at; + let mut row = Vec::new(); + let mut same = Vec::new(); + for column_type in &types { + if *column_type == Text { + row.push(text(&texts[next % texts.len()])); + same.push(text(&texts[at])); + next += 1; + } else { + row.push(other(*column_type)); + same.push(other(*column_type)); + } + } + rows.extend([row, same]); + } + rows.push(vec![Value::Null; types.len()]); + for row in rows { + let entry = shape.entry(&row).expect("an entry"); + let bytes = entry.key.as_bytes(); + assert!( + bytes.len() <= KEY_LIMIT, + "an entry of {} bytes over {types:?}", + bytes.len() + ); + // The same values give the same bytes and the same place. + let again = shape.entry(&row).expect("an entry"); + assert!( + bytes == again.key.as_bytes(), + "two entries of one row differ" + ); + assert_eq!(entry.cut, again.cut); + let longest = row + .iter() + .any(|value| matches!(value, Value::Text(text) if text.len() > WHOLE)); + assert!( + !longest || entry.cut.is_some(), + "a text longer than any share of an entry was kept whole over {types:?}" + ); + // Given only its leading values, the entry begins every entry with those values. + for leading in 0..row.len() { + let begins = entry_bytes(&shape, &row[..leading]); + assert!( + bytes.starts_with(&begins), + "{leading} leading values of {types:?}" + ); + } + } + } +} + +#[test] +fn an_entry_keeps_a_text_whole_up_to_its_share_and_cuts_a_longer_one_to_its_digest() { + // One TEXT column has the room of an entry: a text of `WHOLE` bytes is kept whole, as a key + // keeps it, and one byte more is cut to `WHOLE` bytes, the mark and the digest of the whole + // text. + let shape = EntryShape::new(&[ColumnType::Text], &[SortDirection::Asc]); + let whole = "w".repeat(WHOLE); + let entry = shape.entry(&[text(&whole)]).expect("an entry"); + assert_eq!(entry.cut, None); + assert!( + entry.key == key(&[text(&whole)]), + "a whole text is not as a key keeps it" + ); + + let long = format!("{whole}w"); + let entry = shape.entry(&[text(&long)]).expect("an entry"); + let bytes = entry.key.as_bytes(); + let kept = 1 + WHOLE; + assert_eq!((bytes.len(), entry.cut), (TEXT_ROOM, Some(kept))); + assert!(bytes[..kept] == key(&[text(&whole)]).as_bytes()[..kept]); + assert_eq!(bytes[kept..kept + 2], [0, 1], "the mark of a cut text"); + assert_eq!( + bytes[kept + 2..], + digest(&long), + "the digest of the whole text" + ); + // A descending column turns every bit of the same form. + let descending = EntryShape::new(&[ColumnType::Text], &[SortDirection::Desc]); + let turned: Vec = bytes.iter().map(|byte| !byte).collect(); + assert!(entry_bytes(&descending, &[text(&long)]) == turned); + + // A zero byte is two bytes in an entry, and a cut never falls between the two. + let zeros = "\0".repeat(3000); + let entry = shape.entry(&[text(&zeros)]).expect("an entry"); + let cut = entry.cut.expect("3,000 zero bytes are cut"); + assert_eq!( + cut, + 1 + WHOLE / 2 * 2, + "whole zero bytes are kept, two bytes each" + ); + assert_eq!(entry.key.as_bytes().len(), cut + 2 + 16); + assert_eq!(entry.key.as_bytes()[cut + 2..], digest(&zeros)); + + // Three TEXT columns after a UUID each have `TEXT_ROOM` too, so a text of `WHOLE` bytes is + // whole and one byte more is cut. Two texts of 60,000 bytes after a short one make an entry + // of two full rooms after the short one, and the place it records is that of the first cut + // text. + let types = [ + ColumnType::Uuid, + ColumnType::Text, + ColumnType::Text, + ColumnType::Text, + ]; + let shape = EntryShape::new(&types, &[SortDirection::Asc; 4]); + let row = |value: &str| { + let id = Value::Uuid(Uuid::from_u128(7)); + [id, text("command"), text(value), text(value)] + }; + assert_eq!( + shape.entry(&row(&"v".repeat(WHOLE))).expect("an entry").cut, + None + ); + let entry = shape.entry(&row(&"v".repeat(WHOLE + 1))).expect("an entry"); + assert_eq!(entry.cut, Some(17 + 10 + 1 + WHOLE)); + let entry = shape.entry(&row(&scattered(60_000))).expect("an entry"); + assert_eq!( + (entry.key.as_bytes().len(), entry.cut), + (17 + 10 + 2 * TEXT_ROOM, Some(17 + 10 + 1 + WHOLE)) + ); +} + +#[test] +fn entries_are_in_the_order_of_their_values_whole_or_cut() { + // Rising texts, whole and cut: no two of the cut ones begin with the same kept bytes. + let rising = [ + Value::Text(String::new()), + text("a"), + text("a\0"), + text("a\0b"), + text("ab"), + text("b"), + text(&"b".repeat(WHOLE)), + text(&"b".repeat(5000)), + text("c"), + text(&"c".repeat(2001)), + text(&format!("{}{}", "c".repeat(2500), "a".repeat(3000))), + text(&format!("{}{}", "c".repeat(2000), "d".repeat(3000))), + text("d"), + Value::Null, + ]; + let ascending = EntryShape::new(&[ColumnType::Text], &[SortDirection::Asc]); + let descending = EntryShape::new(&[ColumnType::Text], &[SortDirection::Desc]); + let up: Vec> = rising + .iter() + .map(|value| entry_bytes(&ascending, std::slice::from_ref(value))) + .collect(); + assert!( + up.windows(2).all(|pair| pair[0] < pair[1]), + "entries of rising values do not rise" + ); + let down: Vec> = rising + .iter() + .map(|value| entry_bytes(&descending, std::slice::from_ref(value))) + .collect(); + assert!( + down.windows(2).all(|pair| pair[0] > pair[1]), + "entries of rising values in a descending column do not fall" + ); + // The column after a cut text decides only between rows whose text is the same text. + let two = EntryShape::new( + &[ColumnType::Text, ColumnType::Integer], + &[SortDirection::Asc, SortDirection::Asc], + ); + let long = "b".repeat(5000); + let longer = format!("{long}b"); + let pair = |value: &str, number: i64| entry_bytes(&two, &[text(value), Value::Int64(number)]); + assert!(pair(&long, 1) < pair(&long, 2)); + assert!(pair("b", 9) < pair(&long, 0) && pair(&long, 9) < pair("c", 0)); + assert!(pair(&long, 1) != pair(&longer, 1)); + + // Texts that begin with the same kept bytes differ by their digests alone, and follow each + // other in the order of those. + let alike: Vec = (0..6) + .map(|n| format!("{}{}", "m".repeat(3000), scattered(100 + n))) + .collect(); + let mut by_entry: Vec<&String> = alike.iter().collect(); + by_entry.sort_by_key(|value| entry_bytes(&ascending, &[text(value)])); + let mut by_digest: Vec<&String> = alike.iter().collect(); + by_digest.sort_by_key(|value| digest(value)); + assert!( + by_entry == by_digest, + "a run of cut entries is not in digest order" + ); + let first = ascending.entry(&[text(&alike[0])]).expect("an entry"); + let cut = first.cut.expect("a text of more than 3,000 bytes is cut"); + for value in &alike { + let entry = ascending.entry(&[text(value)]).expect("an entry"); + assert_eq!(entry.cut, Some(cut)); + assert!(entry.key.as_bytes()[..cut] == first.key.as_bytes()[..cut]); + } +} + #[test] fn a_key_gives_its_values_back() { let lists = [ @@ -555,69 +895,26 @@ fn reduced(count: usize) -> (Vec, f32, Vec) { (numbers, scale, codes) } -/// Writes a store of rows 1 to `rows`, each the number and then one vector of `count` reduced -/// numbers, handed to the store with its scale and codes when `compact`. Returns the bytes of -/// the data file once the store is closed. -fn store_vectors(folder: &Path, rows: i64, count: usize, compact: bool) -> u64 { +/// Writes a store of one row, the number 1 and then a vector of `count` reduced numbers, handed +/// to the store with its scale and codes. +fn store_reduced_vector(folder: &Path, count: usize) { let (numbers, scale, codes) = reduced(count); let budget = Budget::new(None).expect("a budget"); let store = Store::open_for_writing(folder, &budget, None).expect("open a fresh store"); let mut changes = ChangeSet::new(store.view().expect("a view"), &budget).expect("begin"); - for id in 1..=rows { - let row = vec![Value::Int64(id), Value::Vector(numbers.clone())]; - let kept = if compact { - vec![Compact { - at: 1, - scale, - codes: codes.clone(), - }] - } else { - Vec::new() - }; - changes - .put_row(TableId(1), key(&[Value::Int64(id)]), row, kept, true) - .expect("stage"); - } + let row = vec![ + Cell::Here(Value::Int64(1)), + Cell::Here(Value::Vector(numbers)), + ]; + let kept = vec![Compact { + at: 1, + scale, + codes, + }]; + changes + .put_row(TableId(1), key(&[Value::Int64(1)]), row, kept, true) + .expect("stage"); store.commit(changes).expect("commit"); - drop(store); - std::fs::metadata(folder.join("data")) - .expect("the data file") - .len() -} - -#[test] -fn a_reduced_vector_reads_back_number_for_number_in_a_quarter_of_the_room() { - let folder = fresh_folder("reduced-vector"); - // 4,000 rows, so that both files are well past the size a data file starts at. - let compact = store_vectors(&folder, 4000, 768, true); - // 781 bytes stay in the row, and read back as exactly the numbers that were put. - let (numbers, ..) = reduced(768); - let budget = Budget::new(None).expect("a budget"); - let store = Store::open_for_writing(&folder, &budget, None).expect("open the store"); - let view = store.view().expect("a view"); - for id in [1, 250, 500] { - let row = view.row(TableId(1), &key(&[Value::Int64(id)])); - let row = row.expect("read the row").expect("the row is there"); - match &row.cells[1] { - Cell::Here(Value::Vector(read)) => { - let bits = |numbers: &[f32]| -> Vec { - numbers.iter().map(|number| number.to_bits()).collect() - }; - assert_eq!(bits(read), bits(&numbers), "row {id}"); - } - other => panic!("the vector of row {id} was read as {other:?}"), - } - } - drop((view, store)); - let plain_folder = fresh_folder("reduced-vector-plain"); - let plain = store_vectors(&plain_folder, 4000, 768, false); - assert!( - 3 * compact < plain, - "4,000 vectors of 768 numbers take {compact} bytes of file reduced and {plain} as written" - ); - for folder in [folder, plain_folder] { - std::fs::remove_dir_all(&folder).expect("remove the test's folder"); - } } #[test] @@ -625,7 +922,7 @@ fn a_reduced_record_of_another_length_is_damage() { // 2,000 reduced numbers are a record of 2,013 bytes, kept beside the row. const COUNT: usize = 2000; let folder = fresh_folder("reduced-length"); - store_vectors(&folder, 1, COUNT, true); + store_reduced_vector(&folder, COUNT); let (loaded, beside, _, _) = load_the_value(&folder); assert!(loaded.expect("load the value as it was written") == Value::Vector(reduced(COUNT).0)); assert_eq!( @@ -689,3 +986,87 @@ fn the_records_of_a_grouping_keep_their_keys_apart() { let sketch = |under, block| keys::vector(Sketch(column, under, block)); assert!(sketch(6, 0) < sketch(6, 1) && sketch(6, 1) < sketch(7, 0)); } + +#[test] +fn a_recorded_length_of_255_times_the_block_is_the_limits_refusal_and_one_more_is_damage() { + // No public call writes a wrong recorded length. The most a block can decode to is 255 + // times its bytes: that much is asked of the memory limit before it is taken, and one byte + // more is damage before any memory is asked for. + const LEN: usize = 16 << 10; + let folder = fresh_folder("recorded-length-bound"); + store_one_row( + &folder, + text(&"A store keeps what it is given. ".repeat(LEN / 32)), + ); + let (_, beside, _, _) = load_the_value(&folder); + let block = beside.bytes - 9; + // Room for the stored record and a value of LEN bytes, and for no more than that. + let room = beside.bytes + 9 + LEN as u64 + 1 + 64; + assert!( + 9 + 255 * block > room, + "255 times the block does not fit the room" + ); + // A length one more or one less than the bytes the block decodes to is damage as well, + // though the page checksum holds: only the check of the decoded length can tell. + let cases = [ + (255 * block, false), + (255 * block + 1, true), + (LEN as u64 + 1, true), + (LEN as u64 - 1, true), + ]; + for (wrong, damaged) in cases { + record_length(&folder, beside.place, wrong); + let (table, row_key) = the_row(); + let budget = Budget::new(Some((9 << 20) + room)).expect("a budget"); + let store = Store::open_for_writing(&folder, &budget, None).expect("open the store"); + let view = store.view().expect("a view"); + let row = view.row(table, &row_key).expect("read").expect("the row"); + let Some(Cell::Beside(at)) = row.cells.into_iter().last() else { + panic!("the text was kept beside its row"); + }; + let before = budget.used(); + let mut reservation = budget.reserve(0, "query").expect("a reservation"); + let loaded = view.load(&at, &mut reservation); + assert_eq!(reservation.bytes(), 0, "held after a length of {wrong}"); + drop(reservation); + assert_eq!(budget.used(), before, "counted after a length of {wrong}"); + match loaded { + Err(Error::StoreCorrupted { path, .. }) if damaged => assert_eq!(path, folder), + Err(Error::MemoryBudgetExceeded { operation, .. }) if !damaged => { + assert_eq!(operation, "query"); + } + other => panic!("a recorded length of {wrong} gave {other:?}"), + } + } + std::fs::remove_dir_all(&folder).expect("remove the test's folder"); + + // A plain record whose recorded length is one more or one less than its bytes is damage. + let letters = scattered(2000); + for wrong in [2001_u64, 1999] { + let folder = fresh_folder("recorded-length-plain"); + store_one_row(&folder, text(&letters)); + let (_, beside, _, _) = load_the_value(&folder); + assert_eq!(beside.bytes, 9 + 2000, "the value is stored plain"); + record_length(&folder, beside.place, wrong); + loading_is_damage( + &folder, + &format!("a plain record that gives the length {wrong}"), + ); + std::fs::remove_dir_all(&folder).expect("remove the test's folder"); + } +} + +/// Puts `wrong` as the recorded length in the first slice of the large value at `place`, as a +/// commit of the storage library, so the page checksum is right for it. +fn record_length(folder: &Path, place: u64, wrong: u64) { + let data = redb::Database::open(folder.join("data")).expect("open the data file"); + let write = data.begin_write().expect("begin"); + { + let mut kept = write.open_table(BESIDE).expect("the large values"); + let first = kept.get(place).expect("read the first slice"); + let mut first = first.expect("the first slice is there").value().to_vec(); + first[1..9].copy_from_slice(&wrong.to_le_bytes()); + kept.insert(place, first.as_slice()).expect("write"); + } + write.commit().expect("commit"); +} diff --git a/v2/contextdb/src/types/mod.rs b/v2/contextdb/src/types/mod.rs index 2dc7cee3..37a43004 100644 --- a/v2/contextdb/src/types/mod.rs +++ b/v2/contextdb/src/types/mod.rs @@ -3,6 +3,7 @@ //! It never holds behaviour: nothing here reads a store, checks a rule or decides an answer. use std::fmt; +use std::ops::Range; use std::path::PathBuf; pub use uuid::Uuid; @@ -43,7 +44,7 @@ pub enum Value { /// A vector of `f32`. Its length must equal the dimension its `VECTOR(n)` column declares; /// any other length is refused with [`Error::ColumnTypeMismatch`]. Vector(Vec), - /// The position of a commit, as [`TxId`]. No column type of this version stores one. + /// The position of a commit, as [`TxId`]: what a `TXID` column holds. TxId(TxId), } @@ -95,6 +96,69 @@ impl From> for Value { } } +/// Values as a refusal shows them, joined by a comma and a space; a text longer than 64 bytes +/// is cut there, at a whole character, and followed by `...` and its length in bytes. +pub(crate) fn shown(values: &[Value]) -> String { + let each = values.iter().map(|value| match value { + Value::Text(text) if text.len() > 64 => { + let cut = (0..=64).rfind(|&at| text.is_char_boundary(at)).unwrap_or(0); + format!("{}... ({} bytes)", &text[..cut], text.len()) + } + Value::Text(text) => text.clone(), + Value::Int64(number) | Value::Timestamp(number) => number.to_string(), + Value::Uuid(uuid) => uuid.to_string(), + other => format!("{other:?}"), + }); + each.collect::>().join(", ") +} + +/// The memory a value takes outside its own cell. +pub(crate) fn heap_bytes(value: &Value) -> u64 { + fn json(value: &serde_json::Value) -> u64 { + let cell = size_of::(); + match value { + serde_json::Value::String(text) => text.len() as u64, + serde_json::Value::Array(items) => { + let items = items.iter(); + items.map(|item| cell as u64 + json(item)).sum() + } + serde_json::Value::Object(members) => { + let members = members.iter(); + members + .map(|(name, member)| (name.len() + cell) as u64 + json(member)) + .sum() + } + _ => 0, + } + } + match value { + Value::Text(text) => text.len() as u64, + Value::Vector(vector) => size_of_val(vector.as_slice()) as u64, + Value::Json(value) => json(value), + _ => 0, + } +} + +/// The most memory the JSON tree made from this text takes, worked out from the text alone, +/// so that it is counted before the tree is made. Outside strings, an object takes a table +/// node of its own and a share of the nodes its members fill, and every member and array +/// item takes a cell with the room its array may grow into; every string is its own bytes. +pub(crate) fn json_bytes(text: &[u8]) -> u64 { + let (mut bytes, mut quoted, mut escaped) = (text.len() as u64, false, false); + for &byte in text { + match byte { + _ if escaped => escaped = false, + b'\\' if quoted => escaped = true, + b'"' => quoted = !quoted, + _ if quoted => {} + b'{' => bytes += 768, + b'[' | b',' | b':' => bytes += 128, + _ => {} + } + } + bytes +} + /// Every refusal and failure of this library, by kind. /// /// A caller matches on the kind; the sentence a person reads comes from `Display`. A statement @@ -114,8 +178,9 @@ pub enum Error { /// The statement names a table that is not declared. Carries the bare table name. Returned /// by every `execute` and by `declaration`. TableNotFound(String), - /// The statement or a declaration names a column its table does not have. Returned by - /// every `execute`. + /// The statement or a declaration names a column its table does not have. For a name + /// written with a qualifier that is no table or alias of the statement, `table` is the + /// qualifier as written. Returned by every `execute`. ColumnNotFound { table: String, column: String }, /// An INSERT or UPDATE wrote a value that is not of its column's type. The only /// conversions are quoted text into a UUID, TIMESTAMP, JSON or VECTOR column and an INTEGER @@ -132,12 +197,16 @@ pub enum Error { ColumnNotNullable { table: String, column: String }, /// A row with this primary key, or with these values under a UNIQUE rule, is already /// there. `column` names every column of the rule in declared order, joined by a comma and - /// a space. + /// a space; `key` is the values the refused row has in them, as text. /// /// Returned by the INSERT or UPDATE when the other row is already visible to it. Returned /// by `Transaction::commit` when another commit took the key or the values while the /// transaction was open; then nothing of the transaction was committed. - UniqueViolation { table: String, column: String }, + UniqueViolation { + table: String, + column: String, + key: String, + }, /// A declaration that cannot stand. `reason` is a sentence for a person, not a table name: /// a `CREATE TABLE` of a name that is taken, a column declared twice, a second PRIMARY KEY, /// a DEFAULT that is not a constant or `NOW()`. Returned by the `execute` that declares. @@ -171,9 +240,10 @@ pub enum Error { /// Declarations and row writes commit in separate transactions. MixedSchemaAndRowWrite, /// `Transaction::commit` found that another commit had changed rows this transaction - /// updated or deleted since the transaction read them. `count` is the number of such rows. - /// Nothing of the transaction was committed; run the whole transaction again. Only a - /// transaction that was open across another commit can meet this. + /// updated or deleted since the transaction read them, or rows it asked to stay as they + /// were until it committed. `count` is the number of such rows. Nothing of the transaction + /// was committed; run the whole transaction again. Only a transaction that was open across + /// another commit can meet this. ConditionalUpdateConflict { count: u64 }, /// A call was made on a `Database` after its `close`. DatabaseClosed, @@ -259,6 +329,88 @@ pub enum Error { VectorIndexedRouteUnavailable { index: VectorIndexRef }, /// `USE VECTOR` was written in a SELECT that does not order by `<=>`. UseVectorRequiresVectorOrder, + /// An UPDATE, a DELETE or an `ON CONFLICT DO UPDATE` named a table declared `IMMUTABLE`. + ImmutableTable { table: String }, + /// A statement would change a column declared `IMMUTABLE`: an UPDATE that assigns it, or + /// an ALTER that drops or renames it. + ImmutableColumn { table: String, column: String }, + /// An UPDATE would move a row's state to one its table's `STATE MACHINE` does not list + /// after the state the row is in. + InvalidStateTransition { + table: String, + column: String, + from: String, + to: String, + }, + /// A commit would leave a reference without its parent row. `still_referenced` says the + /// parent row was removed or re-keyed while a row of `table` points at it; otherwise a + /// row of `table` names a parent that is not there. `columns` are the referencing columns, + /// joined by a comma and a space, and `key` the values they hold, as text. + ForeignKeyViolation { + table: String, + columns: String, + parent: String, + still_referenced: bool, + key: String, + }, + /// An edge of a type its table lists under `DAG` would close a cycle. + CycleDetected { table: String, edge_type: String }, + /// A propagation declared `ABORT ON FAILURE` met a row it could not move. + PropagationAborted { table: String, reason: String }, + /// A write would store a row key, a partition-key tuple or the values of a kept count + /// longer than the engine keeps: values that identify a row or a group and are kept whole. + /// `key` is `primary key`, the VECTOR column's name or the count's list as declared. A + /// value of any length may be indexed: an index is never the reason for this refusal. + KeyTooLong { + table: String, + key: String, + bytes: u64, + limit: u64, + }, + /// `DROP INDEX` named an index its table does not have, without `IF EXISTS`. + IndexNotFound { table: String, index: String }, + /// `DROP COLUMN` without `CASCADE` named a column that `index` is over. + ColumnInIndex { + table: String, + column: String, + index: String, + }, + /// An ALTER would drop or rename a column of a VECTOR column's `PARTITION_KEY`. + /// `index` is the VECTOR column. + VectorPartitionKeyInUse { + table: String, + column: String, + index: String, + }, + /// A DROP or an ALTER named a table, a column or an index that the rank policy `policy` + /// reads. + DropBlockedByRankPolicy { + table: String, + column: String, + policy: String, + }, + /// A `RANK_POLICY` whose declaration cannot stand; `reason` says why. + RankPolicyInvalid { + index: VectorIndexRef, + reason: String, + }, + /// A handle narrowed to some Contexts named, by its key, a row of another Context, or + /// wrote a row outside them. + ContextScopeViolation { table: String }, + /// A handle narrowed by scope labels named, by its key, a row its labels do not allow, or + /// wrote one. + ScopeLabelViolation { table: String }, + /// A handle's principal holds no grant for the row it named by its key. + AclDenied { table: String }, + /// A table with an `ACL REFERENCES` column was read through a narrowed handle that names + /// no principal able to hold grants. + PrincipalRequired { table: String }, + /// A narrowed handle ran a statement only the handle that declared no narrowing may run. + AdminRequired { statement: String }, + /// `CREATE PROCEDURE` or `CREATE FUNCTION`. + StoredProcNotSupported, + /// A statement wrote a `TXID` value above its own transaction's number. + TxIdOutOfRange { value: u64, max: u64 }, } impl fmt::Display for Error { @@ -282,8 +434,11 @@ impl fmt::Display for Error { Error::ColumnNotNullable { table, column } => { write!(f, "column {table}.{column} is NOT NULL") } - Error::UniqueViolation { table, column } => { - write!(f, "unique constraint violation: {table} ({column})") + Error::UniqueViolation { table, column, key } => { + write!( + f, + "unique constraint violation: {table} ({column}) = ({key})" + ) } Error::SchemaInvalid { reason } => write!(f, "schema invalid: {reason}"), Error::ColumnNotIndexable { @@ -397,6 +552,117 @@ impl fmt::Display for Error { Error::UseVectorRequiresVectorOrder => { f.write_str("USE VECTOR requires ORDER BY embedding <=> $param in the same query") } + Error::ImmutableTable { table } => write!( + f, + "table {table} is IMMUTABLE: its rows are inserted once and never updated or \ + deleted" + ), + Error::ImmutableColumn { table, column } => write!( + f, + "column {table}.{column} is IMMUTABLE: it is set when the row is inserted and \ + never changed" + ), + Error::InvalidStateTransition { + table: t, + column: c, + from, + to, + } => { + write!(f, "invalid state transition of {t}.{c}: {from} -> {to}") + } + Error::ForeignKeyViolation { + table, + columns, + parent, + still_referenced, + key, + } => match still_referenced { + true => write!( + f, + "foreign key violation: {parent} is still referenced by {table} ({columns}) \ + = ({key})" + ), + false => write!( + f, + "foreign key violation: {table} ({columns}) = ({key}) has no row in {parent}" + ), + }, + Error::CycleDetected { table, edge_type } => write!( + f, + "cycle detected: this {edge_type} edge of {table} would close a cycle" + ), + Error::PropagationAborted { table, reason } => { + write!(f, "propagation aborted on {table}: {reason}") + } + Error::KeyTooLong { + table, + key, + bytes, + limit, + } => write!( + f, + "key too long: the {key} of {table} takes {bytes} bytes and the limit is \ + {limit}; these values identify a row or a group and are kept whole, so use a \ + shorter one" + ), + Error::IndexNotFound { table, index } => write!(f, "index not found: {table}.{index}"), + Error::ColumnInIndex { + table, + column, + index, + } => write!( + f, + "column {table}.{column} is used by the index {index}; drop the index first, or \ + write CASCADE" + ), + Error::VectorPartitionKeyInUse { + table, + column, + index, + } => write!( + f, + "column {table}.{column} is in the PARTITION_KEY of {table}.{index}; to change \ + the key, declare a new table with the key you want and copy the rows" + ), + Error::DropBlockedByRankPolicy { + table, + column, + policy, + } => write!(f, "{table}.{column} is read by the rank policy {policy}"), + Error::RankPolicyInvalid { index, reason } => { + write!(f, "invalid rank policy on {index}: {reason}") + } + Error::ContextScopeViolation { table } => write!( + f, + "context scope violation: this row of {table} is outside the Contexts this \ + handle was given" + ), + Error::ScopeLabelViolation { table } => write!( + f, + "scope label violation: this row of {table} is outside the labels this handle \ + was given" + ), + Error::AclDenied { table } => write!( + f, + "access denied: this handle's principal holds no grant for this row of {table}" + ), + Error::PrincipalRequired { table } => write!( + f, + "table {table} is access-controlled: read it through a handle that names a \ + principal" + ), + Error::AdminRequired { statement } => write!( + f, + "{statement} reaches every Context, label and principal: run it through the \ + handle that declared no narrowing" + ), + Error::StoredProcNotSupported => { + f.write_str("stored procedures and functions are not supported") + } + Error::TxIdOutOfRange { value, max } => write!( + f, + "TXID {value} is above this transaction's own number, {max}" + ), } } } @@ -406,6 +672,163 @@ impl std::error::Error for Error {} #[cfg(test)] mod tests; +/// Reads an ISO 8601 moment with its offset (`2026-01-01T02:00:00+02:00`, +/// `1970-01-01T00:00:00.250Z`) as milliseconds since the first moment of 1970, UTC. Digits past +/// the millisecond are dropped. `None` when the text is not such a moment. +pub(crate) fn timestamp_millis(text: &str) -> Option { + let (year, month, day) = ( + digits(text, 0..4)?, + digits(text, 5..7)?, + digits(text, 8..10)?, + ); + let (hour, minute, second) = ( + digits(text, 11..13)?, + digits(text, 14..16)?, + digits(text, 17..19)?, + ); + let marks = text.as_bytes(); + let leap = year % 4 == 0 && (year % 100 != 0 || year % 400 == 0); + let month_days = match month { + 2 => 28 + i64::from(leap), + 4 | 6 | 9 | 11 => 30, + _ => 31, + }; + if [marks[4], marks[7], marks[13], marks[16]] != *b"--::" + || !matches!(marks[10], b'T' | b't') + || !(1..=12).contains(&month) + || !(1..=month_days).contains(&day) + || hour > 23 + || minute > 59 + || second > 59 + { + return None; + } + + let mut rest = &text[19..]; + let mut millis = 0; + if let Some(fraction) = rest.strip_prefix('.') { + let end = fraction + .find(|c: char| !c.is_ascii_digit()) + .unwrap_or(fraction.len()); + if end == 0 { + return None; + } + millis = format!("{:0<3}", &fraction[..end.min(3)]).parse().ok()?; + rest = &fraction[end..]; + } + let offset_minutes = match rest.as_bytes() { + b"Z" | b"z" => 0, + [sign @ (b'+' | b'-'), _, _, b':', _, _] => { + let minutes = digits(rest, 1..3)? * 60 + digits(rest, 4..6)?; + if *sign == b'-' { -minutes } else { minutes } + } + _ => return None, + }; + + // Days since 1970 of a date in the Gregorian calendar, counting years from March so that + // the leap day is the last day of its year. + let (year, month) = if month > 2 { + (year, month - 3) + } else { + (year - 1, month + 9) + }; + let leap_days = year.div_euclid(4) - year.div_euclid(100) + year.div_euclid(400); + let days = year * 365 + leap_days + (153 * month + 2) / 5 + day - 719_469; + Some(((days * 24 + hour) * 60 + minute - offset_minutes) * 60_000 + second * 1_000 + millis) +} + +/// The number written in these positions of the text, when every one of them is a digit. +fn digits(text: &str, positions: Range) -> Option { + let part = text.get(positions)?; + if part.bytes().all(|byte| byte.is_ascii_digit()) { + part.parse().ok() + } else { + None + } +} + +/// Reads the text spelling of a vector, `[0.1, 0.2, 0.3]`: the one reader of it, for a value +/// written into a VECTOR column and for the vector of a search. +pub(crate) fn vector_from_text(text: &str) -> Option> { + let inside = text.trim().strip_prefix('[')?.strip_suffix(']')?; + inside + .split(',') + .map(|number| number.trim().parse().ok()) + .collect() +} + +/// Brings a value to its column's type, or refuses it with `ColumnTypeMismatch`. +/// +/// NULL fits every type here; whether the column takes it is the NOT NULL rule's business. The +/// only conversions are quoted text into a UUID, TIMESTAMP, JSON or VECTOR column and an INTEGER +/// into a REAL column. A vector of the column's length with a number that is not finite is +/// refused with `InvalidVectorValue`, however it was written: no search could measure it. +pub(crate) fn typed(table: &str, column: &ColumnDef, value: Value) -> Result { + let converted = match (column.column_type, &value) { + (_, Value::Null) + | (ColumnType::Integer, Value::Int64(_)) + | (ColumnType::Real, Value::Float64(_)) + | (ColumnType::Text, Value::Text(_)) + | (ColumnType::Boolean, Value::Bool(_)) + | (ColumnType::Json, Value::Json(_)) + | (ColumnType::Uuid, Value::Uuid(_)) + | (ColumnType::Timestamp, Value::Timestamp(_)) + | (ColumnType::TxId, Value::TxId(_)) => return Ok(value), + (ColumnType::Vector(length), Value::Vector(vector)) if vector.len() == length => { + return finite(table, column, value); + } + (ColumnType::Real, Value::Int64(whole)) => Some(Value::Float64(*whole as f64)), + (ColumnType::Uuid, Value::Text(text)) => Uuid::parse_str(text).ok().map(Value::Uuid), + (ColumnType::Timestamp, Value::Text(text)) => timestamp_millis(text).map(Value::Timestamp), + (ColumnType::Json, Value::Text(text)) => serde_json::from_str(text).ok().map(Value::Json), + (ColumnType::Vector(length), Value::Text(text)) => vector_from_text(text) + .filter(|vector| vector.len() == length) + .map(Value::Vector), + _ => None, + }; + let converted = converted.ok_or_else(|| Error::ColumnTypeMismatch { + table: table.to_owned(), + column: column.name.clone(), + expected: column.column_type.to_string(), + actual: kind(&value), + })?; + finite(table, column, converted) +} + +/// The value itself, unless it is a vector that holds a number that is not finite. +fn finite(table: &str, column: &ColumnDef, value: Value) -> Result { + if let Value::Vector(vector) = &value + && let Some(position) = vector.iter().position(|number| !number.is_finite()) + { + return Err(Error::InvalidVectorValue { + index: VectorIndexRef { + table: table.to_owned(), + column: column.name.clone(), + }, + position, + }); + } + Ok(value) +} + +/// What a refusal calls the value it was given. A vector says how long it is, since its length +/// is what a VECTOR column refuses it for. +fn kind(value: &Value) -> String { + match value { + Value::Null => "Null", + Value::Bool(_) => "Bool", + Value::Int64(_) => "Int64", + Value::Float64(_) => "Float64", + Value::Text(_) => "Text", + Value::Uuid(_) => "Uuid", + Value::Timestamp(_) => "Timestamp", + Value::Json(_) => "Json", + Value::Vector(vector) => return format!("Vector({})", vector.len()), + Value::TxId(_) => "TxId", + } + .to_owned() +} + /// The declared type of a column, with the names SQL gives it. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ColumnType { @@ -425,6 +848,25 @@ pub enum ColumnType { Vector(usize), /// `TIMESTAMP`. Timestamp, + /// `TXID`: the number of a transaction, as [`TxId`]. + TxId, +} + +/// The type as SQL names it. +impl fmt::Display for ColumnType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + ColumnType::Integer => "INTEGER", + ColumnType::Real => "REAL", + ColumnType::Text => "TEXT", + ColumnType::Boolean => "BOOLEAN", + ColumnType::Json => "JSON", + ColumnType::Uuid => "UUID", + ColumnType::Vector(size) => return write!(f, "VECTOR({size})"), + ColumnType::Timestamp => "TIMESTAMP", + ColumnType::TxId => "TXID", + }) + } } /// A VECTOR column as a refusal names it: its table, then its own name. @@ -569,13 +1011,114 @@ pub struct ColumnDef { pub struct IndexDecl { pub name: String, pub columns: Vec<(String, SortDirection)>, + /// The keys of `(column -> 'key' -> ...)` when the index is over a path into its one JSON + /// column; empty for an index over columns. + pub json_path: Vec, } -/// A table as it was declared. +/// A table as it was declared, as far as this value has fields for: references, `PROPAGATE`, +/// `EXCLUDE VECTOR`, rank policies, the clauses of a VECTOR column, `DELIVERY MANIFEST`, +/// `EDGE DISCARD` and the rules of who may see a row are not in it; `Database::declaration` +/// returns them. #[derive(Debug, Clone, PartialEq)] #[non_exhaustive] pub struct TableMeta { pub columns: Vec, pub unique_constraints: Vec>, pub indexes: Vec, + /// The table was declared `IMMUTABLE`. + pub immutable: bool, + /// Each of these is `None`, or empty, when the table's declaration does not write it. + pub state_machine: Option, + /// The edge types of `DAG(...)`. + pub dag: Vec, + pub retain: Option, + pub history: Option, + pub sync: Option, + pub conflict: Option, + pub kept_counts: Option, +} + +/// `STATE MACHINE (column: state -> [next, ...], ...)`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StateMachine { + pub column: String, + pub transitions: Vec<(String, Vec)>, +} + +/// `RETAIN count unit [SYNC SAFE]`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Retain { + pub count: u64, + pub unit: RetainUnit, + pub sync_safe: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RetainUnit { + Seconds, + Minutes, + Hours, + Days, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum History { + All, + CurrentOnly, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SyncDirection { + Off, + PushOnly, + PullOnly, + TwoWay, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ConflictPolicy { + KeepFirst, + KeepLatest, +} + +/// `MAINTAIN COUNT NONE`, `ALL PREFIXES`, or the column lists written. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum KeptCounts { + None, + AllPrefixes, + Lists(Vec>), +} + +/// The Context a row belongs to, as its `CONTEXT_ID` column holds it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct ContextId(pub Uuid); + +impl ContextId { + /// The Context with this id. + pub fn new(uuid: Uuid) -> ContextId { + ContextId(uuid) + } +} + +/// One label of a table's `SCOPE_LABEL` column. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct ScopeLabel(pub String); + +impl ScopeLabel { + /// The label with this name. + pub fn new(label: impl Into) -> ScopeLabel { + ScopeLabel(label.into()) + } +} + +/// Who a handle acts for, as a table's access-control grants name them. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum Principal { + /// The system itself. + System, + /// An agent, by name. + Agent(String), + /// A person, by name. + Human(String), } diff --git a/v2/contextdb/src/types/tests.rs b/v2/contextdb/src/types/tests.rs index d9a21847..6327645e 100644 --- a/v2/contextdb/src/types/tests.rs +++ b/v2/contextdb/src/types/tests.rs @@ -20,9 +20,10 @@ fn a_unique_refusal_names_the_table_and_every_column_of_the_rule() { let refusal = Error::UniqueViolation { table: "t".to_owned(), column: "a, b, c".to_owned(), + key: "1, x, 1".to_owned(), }; assert_eq!( refusal.to_string(), - "unique constraint violation: t (a, b, c)" + "unique constraint violation: t (a, b, c) = (1, x, 1)" ); } diff --git a/v2/contextdb/src/vector/stage.rs b/v2/contextdb/src/vector/stage.rs index 8f16bf8e..62d924eb 100644 --- a/v2/contextdb/src/vector/stage.rs +++ b/v2/contextdb/src/vector/stage.rs @@ -219,18 +219,23 @@ fn moves(changes: &ChangeSet, catalog: &Catalog, held: &mut Reservation) -> Resu continue; } let staged = changes.rows_changed(); - for (_, key, after, _) in staged.filter(|staged| staged.0 == table.id) { + for (_, key, staged, _) in staged.filter(|staged| staged.0 == table.id) { let before = base.row(table.id, key)?; for column in &table.vectors { - let (leaves, enters) = tuples(table, column, before.as_ref(), after, base, held)?; + let (leaves, enters) = tuples(table, column, before.as_ref(), staged, base, held)?; if leaves == enters { - // The row has a vector under one tuple before and after, or none. - let (Some(row), Some(values), true) = (&before, after, leaves.is_some()) else { + // The row has a vector under one tuple before and after, or none. A vector + // the row left where it is has not changed. + let at = position(table, &column.column); + let kept = + staged.is_some_and(|(_, left)| left.iter().any(|(slot, _)| *slot == at)); + let (Some(row), Some((values, _)), true, false) = + (&before, staged, leaves.is_some(), kept) + else { continue; }; - let at = position(table, &column.column); let taken = held.bytes(); - let same = match &row.cells[at] { + let same = match &*table.cell(row, at) { Cell::Here(vector) => *vector == values[at], Cell::Beside(beside) => base.load(beside, held)? == values[at], }; diff --git a/v2/contextdb/src/vector/tally.rs b/v2/contextdb/src/vector/tally.rs index 3e898281..192f1288 100644 --- a/v2/contextdb/src/vector/tally.rs +++ b/v2/contextdb/src/vector/tally.rs @@ -6,8 +6,8 @@ use std::collections::btree_map::Entry; use super::directory; use crate::memory::{Budget, Reservation}; -use crate::schema::{Table, VectorColumn}; -use crate::store::{Cell, Key, Read, Row, VectorId, VectorNumber}; +use crate::schema::{Catalog, Table, VectorColumn}; +use crate::store::{Cell, Key, Read, Row, StagedValues, VectorId, VectorNumber}; use crate::types::{Error, Result, Value}; /// What the rows a transaction has staged mean for the partition count of each VECTOR column @@ -85,7 +85,7 @@ impl Tally { &mut self, table: &Table, before: Option<&Row>, - after: Option<&[Value]>, + after: Option>, read: &dyn Read, ) -> Result<()> { for column in &table.vectors { @@ -138,20 +138,18 @@ impl Tally { Ok(()) } - /// The last act of a statement that wrote rows. Refuses with - /// `VectorPartitionLimitExceeded` when the statement put a vector under a key tuple that - /// had none and the column would then have more partitions than its limit, and forgets - /// what the statement noted; otherwise keeps it. - pub(crate) fn settle(&mut self, table: &Table) -> Result<()> { - for column in &table.vectors { - let mut notes = self.noted.iter(); - let began = notes.any(|note| note.place.0 == column.id && note.began); + /// The last act of a statement that wrote rows, after the rows that follow it. Refuses + /// with `VectorPartitionLimitExceeded` when the statement put a vector under a key tuple + /// that had none and the column, found in `catalog`, would then have more partitions than + /// its limit; what the statement noted is then forgotten by `database`, as for any + /// refusal. Otherwise keeps it. + pub(crate) fn settle(&mut self, catalog: &Catalog) -> Result<()> { + let began = self.noted.iter().filter(|note| note.began); + for (table, column) in began.filter_map(|note| catalog.vector(note.place.0)) { let partitions = self.partitions.get(&column.id).copied().unwrap_or(0); if let Some(limit) = column.limit() - && began && partitions > u64::from(limit) { - self.forget(); return Err(Error::VectorPartitionLimitExceeded { index: table.vector_ref(column), max_partitions: limit, @@ -188,8 +186,10 @@ pub(super) fn position(table: &Table, name: &str) -> usize { /// The key tuple a row's vector was under in a VECTOR column, and the one it is under now. /// -/// `before` is the row as it was and `after` its values as they are staged; `None` is no row. -/// A row that is not there, and a row whose vector is NULL, has its vector under no tuple. +/// `before` is the row as it was and `after` its values as they are staged, with the large +/// cells it left where they are; `None` is no row. A row that is not there, a row whose vector +/// is NULL, and a row in a state its table declares `EXCLUDE VECTOR` has its vector under no +/// tuple. /// The tuple is the values of the PARTITION_KEY columns as [`Key::from_values`] encodes them, /// ascending; a column with no PARTITION_KEY has the one tuple of no values. A key column /// kept beside its row is loaded from `read` through `held`, which is given back what the @@ -198,18 +198,26 @@ pub(super) fn tuples( table: &Table, column: &VectorColumn, before: Option<&Row>, - after: Option<&[Value]>, + after: Option>, read: &dyn Read, held: &mut Reservation, ) -> Result<(Option, Option)> { let vector = position(table, &column.column); + let machine = table.meta.state_machine.as_ref(); + let state = machine.map(|machine| position(table, &machine.column)); + let excluded = |value: &Value| matches!(value, Value::Text(state) if table.rules.exclude_vector_states.contains(state)); let keys = column.partition_key.iter(); let keys: Vec = keys.map(|name| position(table, name)).collect(); let tuple = |values: Result>| Key::from_values(&values?, &[]).map(Some); let was = match before { - Some(row) if !matches!(row.cells[vector], Cell::Here(Value::Null)) => { + Some(row) + if !matches!(*table.cell(row, vector), Cell::Here(Value::Null)) + && !state.is_some_and( + |at| matches!(&*table.cell(row, at), Cell::Here(value) if excluded(value)), + ) => + { let taken = held.bytes(); - let values = keys.iter().map(|&at| match &row.cells[at] { + let values = keys.iter().map(|&at| match &*table.cell(row, at) { Cell::Here(value) => Ok(value.clone()), Cell::Beside(beside) => read.load(beside, held), }); @@ -220,7 +228,10 @@ pub(super) fn tuples( _ => None, }; let is = match after { - Some(values) if values[vector] != Value::Null => { + Some((values, left)) + if (values[vector] != Value::Null || left.iter().any(|(at, _)| *at == vector)) + && !state.is_some_and(|at| excluded(&values[at])) => + { tuple(Ok(keys.iter().map(|&at| values[at].clone()).collect()))? } _ => None, diff --git a/v2/contextdb/src/vector/tests.rs b/v2/contextdb/src/vector/tests.rs index fe285719..a3271c87 100644 --- a/v2/contextdb/src/vector/tests.rs +++ b/v2/contextdb/src/vector/tests.rs @@ -22,7 +22,7 @@ use super::train::{Mean, Round, first_centres, groups_for, nearest, product, sam use crate::Database; use crate::memory::Budget; use crate::schema::Catalog; -use crate::store::{ChangeSet, Key, Read, Store, TableId, VectorId, VectorRecord, View}; +use crate::store::{Cell, ChangeSet, Key, Read, Store, TableId, VectorId, VectorRecord, View}; use crate::types::{Error, Value}; /// A fixed scatter of numbers: a xorshift that starts from a seed. @@ -261,7 +261,7 @@ fn a_pending_record_of_a_row_with_no_vector_is_reported_as_damage() { let table = catalog.table("docs").expect("the table").id; let mut changes = ChangeSet::new(view, &budget).expect("a change set"); let key = Key::from_values(&[Value::Int64(damaged)], &[]).expect("a key"); - let row = vec![Value::Int64(damaged), Value::Null]; + let row = vec![Cell::Here(Value::Int64(damaged)), Cell::Here(Value::Null)]; changes .put_row(table, key, row, Vec::new(), false) .expect("stage"); @@ -321,17 +321,16 @@ fn a_vector_entry_named(test: &str, named: u8) -> Misnamed { let mut records = view.schema_records().expect("the schema records"); let mut record = records.pop().expect("the record of the one table"); assert!(records.is_empty(), "one table has one schema record"); - // A schema record ends with its VECTOR entries, and the one entry here ends with its - // column's name, a text of one byte, and then nine bytes: no PARTITION_KEY column, no - // MAX_PARTITIONS, no SEARCH_MODE. - // The VECTOR entry is the last thing in the record, and in it the column's name follows - // the length of the name: the last place the record holds that length and `v`. - let entry = [1, 0, 0, 0, b'v']; - let at = record - .windows(entry.len()) - .rposition(|window| window == entry) - .expect("the VECTOR entry of the column v"); - let name = at + entry.len() - 1; + // The one VECTOR entry of the record holds its column's name, a text of one byte, and then + // six bytes: a list of no PARTITION_KEY columns, no MAX_PARTITIONS, no SEARCH_MODE. The + // column `v` is named once more in the record, among the table's columns, and there its + // type follows the name, which is no zero byte. + let entry = [1, 0, 0, 0, b'v', 0, 0, 0, 0, 0, 0]; + let found: Vec = (0..record.len()) + .filter(|at| record[*at..].starts_with(&entry)) + .collect(); + assert_eq!(found.len(), 1, "the record's one VECTOR entry for `v`"); + let name = found[0] + 4; record[name] = named; let mut changes = ChangeSet::new(view, &budget).expect("a change set"); changes.put_schema("docs", record).expect("stage"); @@ -357,7 +356,9 @@ fn a_write_is_reported_as_damage(test: &str, named: u8, what: &str) { let mut changes = ChangeSet::new(view.clone(), &budget).expect("a change set"); let key = Key::from_values(&[Value::Int64(4)], &[]).expect("a key"); let numbers = Value::Vector(vec![0.0, 0.0, 0.0, 1.0]); - let row = vec![Value::Int64(4), Value::Int64(40), numbers]; + let row = [Value::Int64(4), Value::Int64(40), numbers] + .map(Cell::Here) + .to_vec(); changes .put_row(table, key, row, Vec::new(), true) .expect("stage"); diff --git a/v2/contextdb/src/write/commit.rs b/v2/contextdb/src/write/commit.rs new file mode 100644 index 00000000..7b1576cf --- /dev/null +++ b/v2/contextdb/src/write/commit.rs @@ -0,0 +1,275 @@ +//! The rules of a whole transaction, checked by the commit function over everything the +//! transaction staged: every reference, from the row that names a parent and from the parent +//! that is named, and, for a transaction another commit overtook, its keys, its unique values, +//! its edges, the rows it changed and the rows it guarded. + +use std::collections::HashMap; +use std::ops::Bound; + +use super::{Given, Rules, column, equal, pick}; +use crate::memory::{Budget, Reservation}; +use crate::query::{self, Reader}; +use crate::schema::{Catalog, Reference, Table}; +use crate::store::{Cell, ChangeSet, Key, Read, Row, RowGuard, View}; +use crate::types::{Error, Result, TxId, Value, shown}; + +/// Checks everything a transaction staged against the moment its change set is laid over. +/// +/// The commit function calls this, under its lock, for every commit. `before` is the moment +/// the change set was laid over until another commit overtook it, and `None` for a +/// transaction that none overtook. +/// +/// For every commit, a reference is checked both ways over the complete set, so a parent may +/// be written after the row that names it: a staged row whose reference has no NULL must find +/// its parent row, and a parent row that was removed, or lost the values it was named by, must +/// be named by no row. Both are `ForeignKeyViolation`. +/// +/// For an overtaken one, also: a key or unique value that the other commit took is refused +/// with `UniqueViolation`, and an edge that closes a cycle with the other commit's edges with +/// `CycleDetected`. A row this transaction changed or deleted that is no longer as it found +/// it, and a guarded row that is gone or no longer holds its guarded values, are counted and +/// refused together with `ConditionalUpdateConflict`. A transaction that none overtook met +/// each of these rules in its statements, over the moment it commits on. +/// +/// `rows_changed` yields one staged row per key. A row whose key an UPDATE changed is two of +/// them: its old key as a delete, and its new key as a row marked new. +/// +/// Costs one lookup of the parent per staged row and reference, and one row read and one +/// lookup in the child's index per changed or removed row and reference that names its table; +/// for an overtaken transaction two more lookups per changed row and per guard, one per new +/// row, one per staged row and unique column, and a walk per staged edge of a DAG type. +pub(crate) fn check_at_commit( + changes: &ChangeSet, + catalog: &Catalog, + before: Option<&View>, + budget: &Budget, +) -> Result<()> { + let (params, reader) = (HashMap::new(), Reader::admin()); + let given = Given { + params: ¶ms, + catalog, + budget, + reader: &reader, + now: 0, + number: TxId(0), + }; + let (tables, base) = (catalog.tables(), changes.base()); + let mut overtaken = 0; + // The table the staged rows now walked are of, with the references that name it as their + // parent, each with its own table. + let mut of: Option<(Rules, Vec<(&Table, _)>)> = None; + for (id, key, row, new_key) in changes.rows_changed() { + if of.as_ref().is_none_or(|(rules, _)| rules.table.id != id) { + of = None; + if let Some(table) = tables.iter().find(|table| table.id == id) { + let name = &table.name; + let children = tables.iter().flat_map(|child| { + let references = child.rules.references.iter(); + let named = references.filter(move |reference| reference.parent == *name); + named.map(move |reference| (*child, reference)) + }); + of = Some((Rules::of(table, &given)?, children.collect())); + } + } + let Some((rules, children)) = &of else { + continue; + }; + let table = rules.table; + let named = !new_key && !children.is_empty(); + let there = match before.is_some() || named { + true => base.row(id, key)?, + false => None, + }; + if let Some(before) = before { + if let (true, Some((row, _))) = (new_key, row) { + if there.is_some() { + return Err(rules.violation(rules.key_columns(), pick(&table.key, row))); + } + } else if before.row(id, key)? != there { + overtaken += 1; + } + } + // The values this row was named by until now: when it no longer holds them, no row + // may name them, unless another row of the table holds them now. + if let (true, Some(was)) = (named, &there) { + let mut loaded = budget.reserve(0, "query")?; + for &(child, reference) in children { + let at = reference.parent_columns.iter(); + let at: Vec = at.map(|name| column(table, name)).collect::>()?; + let old = at.iter().map(|&at| held(table, was, at, base, &mut loaded)); + let old: Vec = old.collect::>()?; + let whole = !old.contains(&Value::Null); + if whole + && !row.is_some_and(|(row, _)| pick(&at, row) == old) + && holds( + child, + reference.index, + &reference.columns, + &old, + changes, + &given, + )? + && !parent(table, &reference.parent_columns, &old, changes, &given)? + { + return Err(refused(child, reference, true, &old)); + } + } + } + // A large value the row left where it is reads as NULL here, which takes the row out + // of a rule over it: the statement that staged the row changed no entry of that rule + // and no value of that reference, so the row holds what it held. + let Some((row, _)) = row else { continue }; + for reference in &table.rules.references { + let values = pick(&reference.columns, row); + if values.contains(&Value::Null) { + continue; + } + let found = match catalog.table(&reference.parent) { + Some(to) => parent(to, &reference.parent_columns, &values, changes, &given)?, + None => false, + }; + if !found { + return Err(refused(table, reference, false, &values)); + } + } + if before.is_none() { + continue; + } + rules.acyclic(changes, rules.edge(None, row)?)?; + for index in &rules.indexes { + if let Some(columns) = index.rule(row) + && rules.taken(changes, index, &index.entry(row)?, row, key)? + { + return Err(rules.violation(columns.to_owned(), pick(&index.columns, row))); + } + } + } + if let Some(before) = before { + for guard in changes.guards() { + overtaken += u64::from(lost(guard, before, base, budget)?); + } + } + if overtaken > 0 { + return Err(Error::ConditionalUpdateConflict { count: overtaken }); + } + Ok(()) +} + +/// True when a guarded row is not as its guard found it: another commit removed it, or changed +/// a value the guard holds. `before` is the moment the guard was asked over and `now` the one +/// the commit is laid over. A row that is at both moments what it was, whatever its +/// transaction itself has done to it since, is as its guard found it. +fn lost(guard: &RowGuard, before: &View, now: &View, budget: &Budget) -> Result { + let row = now.row(guard.table, &guard.key)?; + if before.row(guard.table, &guard.key)? == row { + return Ok(false); + } + let Some(row) = row else { return Ok(true) }; + let mut loaded = budget.reserve(0, "query")?; + for (cell, value) in &guard.columns { + // A position the row has no cell at is a column added after the row was written: the + // row reads there what it read when the guard was asked. + let kept = match row.cells.get(*cell) { + Some(Cell::Here(held)) => held == value, + Some(Cell::Beside(beside)) => now.load(beside, &mut loaded)? == *value, + None => true, + }; + if !kept { + return Ok(true); + } + } + Ok(false) +} + +/// The value a stored row holds in a column: its cell's, loaded when it is kept beside the +/// row, or what the table says a row written before the column was added reads as. +fn held( + table: &Table, + row: &Row, + column: usize, + read: &dyn Read, + loaded: &mut Reservation, +) -> Result { + match table.cell(row, column).into_owned() { + Cell::Here(value) => Ok(value), + Cell::Beside(beside) => read.load(&beside, loaded), + } +} + +/// True when `read` shows a row of `table` that holds `values` in the columns `named`: the +/// columns of its key or of one of its UNIQUE rules, in the rule's order. +fn parent( + table: &Table, + named: &[String], + values: &[Value], + read: &dyn Read, + given: &Given, +) -> Result { + // A reference names the columns of its rule in the rule's order. + let rule = table.meta.indexes.iter().position(|index| { + (index.name.starts_with("__pk_") || index.name.starts_with("__unique_")) + && index.columns.iter().map(|(name, _)| name).eq(named) + }); + let Some(rule) = rule else { + return Ok(false); + }; + let at = named.iter().map(|name| column(table, name)); + let at: Vec = at.collect::>()?; + holds(table, rule, &at, values, read, given) +} + +/// True when `read` shows a row of `table` that holds `values` in `columns`, the first columns +/// of the index at `index` of the table's indexes. +/// +/// Costs one read of the first row or entry under those values. When one of them is too long +/// to be whole in an entry, the entry says only that such a row may be there: the rows are +/// then read through the table source and compared by their values. +fn holds( + table: &Table, + index: usize, + columns: &[usize], + values: &[Value], + read: &dyn Read, + given: &Given, +) -> Result { + // The index behind the primary key holds no entries: a row is kept under its key. + if table.meta.indexes[index].name.starts_with("__pk_") { + let key = Key::from_values(values, &[])?; + let at = Bound::Included(&key); + return Ok(read.rows(table.id, at, at)?.next().transpose()?.is_some()); + } + let entry = table.entry_shape(index).entry(values)?; + let at = Bound::Included(&entry.key); + let found = read.index(table.index_ids[index], at, at)?.next(); + if found.transpose()?.is_none() { + return Ok(false); + } + if entry.cut.is_none() { + return Ok(true); + } + let (filter, bound) = equal(table, columns, values); + let asked = query::Asked { + table, + filter: Some(&filter), + assigned: &[], + needed: &vec![false; table.meta.columns.len()], + reader: given.reader, + params: &bound, + now: given.now, + }; + let found = query::rows_for_write(&asked, None, given.catalog, read, given.budget)?; + Ok(found.is_some()) +} + +/// The refusal of a reference of `table`, whose columns hold `key`. +fn refused(table: &Table, reference: &Reference, still: bool, key: &[Value]) -> Error { + let columns = reference.columns.iter(); + let names = columns.map(|&at| &*table.meta.columns[at].name); + Error::ForeignKeyViolation { + table: table.name.clone(), + columns: names.collect::>().join(", "), + parent: reference.parent.clone(), + still_referenced: still, + key: shown(key), + } +} diff --git a/v2/contextdb/src/write/counts.rs b/v2/contextdb/src/write/counts.rs new file mode 100644 index 00000000..f7f385c4 --- /dev/null +++ b/v2/contextdb/src/write/counts.rs @@ -0,0 +1,102 @@ +//! Kept counts: for each list of columns a table declared with MAINTAIN COUNT, the number of +//! its rows under each set of values of those columns. +//! +//! A count changes with the row: where a row is written and where a row is removed, and +//! nowhere else. It is never worked out by reading rows when a row is written; the rows are +//! read once, when a count is declared on a table that has them. + +use std::collections::HashMap; + +use super::{Given, Rules, pick, split}; +use crate::memory::Budget; +use crate::query::Reader; +use crate::schema::Catalog; +use crate::store::{ChangeSet, CountId}; +use crate::types::{Result, SortDirection, TxId, Value}; + +impl Rules<'_> { + /// Stages what one row means for every count its table keeps: one more under the values + /// of `row`, the row as it is written, and one fewer under the values of `was`, the row as + /// it was. A new row has no `was` and a removed row no `row`. A count over columns in + /// which the two agree is left alone. + pub(super) fn count( + &self, + changes: &mut ChangeSet, + was: Option<&[Value]>, + row: Option<&[Value]>, + ) -> Result<()> { + for (at, (columns, _)) in self.counts.iter().enumerate() { + if let (Some(was), Some(row)) = (was, row) + && columns.iter().all(|&column| was[column] == row[column]) + { + continue; + } + for (values, by) in [(row, 1), (was, -1)] { + if let Some(values) = values { + self.add(changes, at, values, by)?; + } + } + } + Ok(()) + } + + /// Adds `by` to the count at this place among the table's counts, under the values `row` + /// has in the count's columns. + fn add(&self, changes: &mut ChangeSet, at: usize, row: &[Value], by: i64) -> Result<()> { + let (columns, ways) = &self.counts[at]; + // The count's list as declared. + let listed = || { + let named = columns.iter().zip(ways).map(|(&at, way)| { + let name = self.table.meta.columns[at].name.clone(); + match way { + SortDirection::Asc => name, + SortDirection::Desc => name + " DESC", + } + }); + named.collect::>().join(", ") + }; + let key = self.identity(listed, &pick(columns, row), ways)?; + changes.add_kept(self.table.rules.counts[at].id, key, by) + } +} + +/// Stages the records of counts a declaration has just made, for the rows their table already +/// has: the rows are read once for all of them, a batch at a time, so that what is held here +/// does not grow with the table. The records are counted against the change set like any +/// staged change, and counts that do not fit the memory limit are refused there. +pub(crate) fn fill_kept_count( + counts: &[CountId], + catalog: &Catalog, + changes: &mut ChangeSet, + budget: &Budget, +) -> Result<()> { + let (params, reader) = (HashMap::new(), Reader::admin()); + let given = Given { + params: ¶ms, + catalog, + budget, + reader: &reader, + now: 0, + number: TxId(0), + }; + for table in catalog.tables() { + let kept = table.rules.counts.iter().enumerate(); + let made = kept.filter(|(_, count)| counts.contains(&count.id)); + let made: Vec = made.map(|(at, _)| at).collect(); + if made.is_empty() { + continue; + } + let rules = Rules::of(table, &given)?; + let mut needed = vec![false; table.meta.columns.len()]; + for &column in made.iter().flat_map(|&at| &rules.counts[at].0) { + needed[column] = true; + } + let asked = rules.asked(None, ¶ms, &[], &needed, &reader); + rules.each(&asked, changes, |changes, _, cells| { + let row = split(cells).0; + made.iter() + .try_for_each(|&at| rules.add(changes, at, &row, 1)) + })?; + } + Ok(()) +} diff --git a/v2/contextdb/src/write/mod.rs b/v2/contextdb/src/write/mod.rs index 7f0d947b..4f0e5617 100644 --- a/v2/contextdb/src/write/mod.rs +++ b/v2/contextdb/src/write/mod.rs @@ -1,6 +1,8 @@ //! write: applies INSERT, UPDATE and DELETE to a transaction and checks every rule a table can -//! declare: column types, NOT NULL, the primary key and UNIQUE rules. It keeps each table's -//! index entries and row count in step with its rows. +//! declare: column types, NOT NULL, the primary key and UNIQUE rules, how long a key may be, +//! IMMUTABLE, the state machine, references, a DAG, PROPAGATE, and what the handle's reader +//! may write. It keeps each table's index entries and row count +//! in step with its rows. //! //! It never returns a query's result to a caller: it reads only to find the rows an UPDATE or //! DELETE names and to check a rule. @@ -9,33 +11,62 @@ //! NOT NULL, its key and its unique rules, and staged with its index entries before the next row //! is looked at, so a later row is checked against the earlier ones through the change set. The //! caller undoes a refused statement by rewinding the change set. +//! +//! The values that name a row or a group (a primary key, a kept count's values, a partition +//! key) are kept whole and refused past `KEY_LIMIT`. An index entry is made by +//! `EntryShape::entry`, which cuts a long value, so no row is refused for one. +mod commit; +mod counts; +mod propagate; +mod rules; #[cfg(test)] mod tests; -mod values; +pub(crate) use self::commit::check_at_commit; +pub(crate) use self::counts::fill_kept_count; -use std::collections::HashMap; +use std::cell::RefCell; +use std::collections::{BTreeSet, HashMap}; use std::ops::Bound; -use crate::memory::{Budget, Reservation}; -use crate::query; +use crate::memory::Budget; +use crate::query::{self, Reader, Visible}; use crate::schema::{Catalog, Quantization, Table}; -use crate::sql::Plan; -use crate::sql::plan::{DeletePlan, InsertPlan, UpdatePlan}; -use crate::store::{Cell, ChangeSet, Compact, IndexId, Key, Read, View}; -use crate::types::{ColumnDefault, Error, Result, SortDirection, Value}; +use crate::sql::ast::{BinOp, ColumnRef}; +use crate::sql::plan::InsertPlan; +use crate::sql::{Expr, Plan}; +use crate::store::{ + Beside, Cell, ChangeSet, Compact, Entry, EntryShape, IndexId, KEY_LIMIT, Key, Read, Row, +}; +use crate::types::{ + self, ColumnDefault, ColumnType, Error, Result, SortDirection, TxId, Value, shown, typed, +}; use crate::vector::{Reduced, Tally}; +/// What a statement that writes rows is run with. +pub(crate) struct Given<'a> { + pub params: &'a HashMap, + pub catalog: &'a Catalog, + pub budget: &'a Budget, + /// Who writes: a row this reader may not see is not matched, and a new or changed row it + /// may not write is refused. + pub reader: &'a Reader, + /// The statement's time in milliseconds: what `DEFAULT NOW()` stores and `NOW()` answers. + pub now: i64, + /// The transaction's number: the highest a `TXID` column takes from the statement. + pub number: TxId, +} + /// Applies one INSERT, UPDATE or DELETE to the change set and returns the number of rows: those -/// an INSERT stored, or those the WHERE of an UPDATE or DELETE matched. +/// an INSERT stored or, with `ON CONFLICT`, changed, or those the WHERE of an UPDATE or DELETE +/// matched. /// /// Reads through the change set, so the statement sees what its transaction already wrote. -/// Finds the rows of an UPDATE or DELETE with `query::matching`, which hands them back under a -/// reservation that is held until this returns, and works out values with `query::eval`. -/// Checks every rule that can be checked now and stages each row together with its index -/// entries and count change. A refusal may leave part of the statement staged: the caller takes -/// a mark on the change set before the call and rewinds to it on refusal. `now` is the -/// statement's time in milliseconds: what `DEFAULT NOW()` stores and `NOW()` answers. +/// Finds the rows of an UPDATE or DELETE with `query::rows_for_write`, a batch at a time, and +/// stages each batch before it asks for the next; works out values with `query::eval`. Checks +/// every rule that can be checked now and stages each row together with its index entries and +/// count change. A refusal may leave part of the statement staged: the caller takes a mark on +/// the change set before the call and rewinds to it on refusal. /// /// `partitions` is the transaction's tally of vector partitions. Every row staged in a table /// that has a VECTOR column with a PARTITION_KEY is noted in it, and the statement's last act @@ -47,240 +78,72 @@ use crate::vector::{Reduced, Tally}; /// When the plan is not an INSERT, UPDATE or DELETE. pub(crate) fn apply( plan: &Plan, - params: &HashMap, - catalog: &Catalog, + given: &Given, changes: &mut ChangeSet, partitions: &mut Tally, - budget: &Budget, - now: i64, ) -> Result { - match plan { - Plan::Insert(plan) => insert(plan, params, catalog, changes, partitions, budget, now), - Plan::Update(plan) => update(plan, params, catalog, changes, partitions, budget, now), - Plan::Delete(plan) => delete(plan, params, catalog, changes, partitions, budget, now), + let name = match plan { + Plan::Insert(plan) => &plan.table, + Plan::Update(plan) => &plan.table, + Plan::Delete(plan) => &plan.table, other => unreachable!("{other:?} writes no rows"), - } -} - -/// Stages an entry in a new index for every row the table already has. -/// -/// # Panics -/// -/// When the table does not list the index, or it is the index behind the primary key. -pub(crate) fn fill_index( - table: &Table, - index: IndexId, - changes: &mut ChangeSet, - budget: &Budget, -) -> Result<()> { - // The rows are read and their entries staged a batch at a time, so that what is held here - // does not grow with the table. - const BATCH: usize = 1024; - let rules = Rules::of(table)?; - let mut indexes = rules.indexes.iter(); - let index = indexes - .find(|known| known.id == index) - .expect("a new index is one of its table's indexes that hold entries"); - let mut last: Option = None; - loop { - let after = last.as_ref().map_or(Bound::Unbounded, Bound::Excluded); - let mut batch = Vec::with_capacity(BATCH); - // What a batch loads of values kept beside their rows is given back with the batch. - let mut loaded = budget.reserve(0, "CREATE INDEX")?; - for found in changes.rows(table.id, after, Bound::Unbounded)?.take(BATCH) { - let (key, row) = found?; - let values = index.columns.iter().map(|&at| match &row.cells[at] { - Cell::Here(value) => Ok(value.clone()), - Cell::Beside(beside) => changes.load(beside, &mut loaded), - }); - let values = values.collect::>>()?; - batch.push((Key::from_values(&values, &index.directions)?, key)); - } - let Some((_, key)) = batch.last() else { - return Ok(()); - }; - last = Some(key.clone()); - for (entry, key) in batch { - changes.put_index(index.id, entry, key)?; - } - } -} - -/// Checks the staged rows against the moment the change set is now laid over. -/// -/// The commit function calls this, under its lock, after rebasing a change set that another -/// commit overtook; `before` is the moment it was laid over until then. A key or unique value -/// that the other commit took is refused with `UniqueViolation`. A row this transaction changed -/// or deleted that is no longer as it found it is refused with `ConditionalUpdateConflict`, -/// which counts such rows. -/// -/// `rows_changed` yields one staged row per key. A row whose key an UPDATE changed is two of -/// them: its old key as a delete, and its new key as a row marked new. -/// -/// Costs two lookups per changed row, one per new row, and one per staged row and unique -/// column; the staged rows are walked once for each table of the catalog. -pub(crate) fn check_at_commit(changes: &ChangeSet, catalog: &Catalog, before: &View) -> Result<()> { - let mut overtaken = 0; - for table in catalog.tables() { - let rules = Rules::of(table)?; - let staged = changes.rows_changed(); - for (_, key, row, new_key) in staged.filter(|staged| staged.0 == table.id) { - let there = changes.base().row(table.id, key)?; - if new_key { - if there.is_some() { - return Err(rules.violation(rules.key_columns())); - } - } else if before.row(table.id, key)? != there { - overtaken += 1; - } - let Some(row) = row else { continue }; - for index in &rules.indexes { - if let Some(columns) = index.rule(row) - && taken(changes, index.id, &index.entry(row)?, key)? - { - return Err(rules.violation(columns.to_owned())); - } - } - } - } - if overtaken > 0 { - return Err(Error::ConditionalUpdateConflict { count: overtaken }); - } - Ok(()) -} - -fn insert( - plan: &InsertPlan, - params: &HashMap, - catalog: &Catalog, - changes: &mut ChangeSet, - partitions: &mut Tally, - budget: &Budget, - now: i64, -) -> Result { - let table = find(catalog, &plan.table)?; - let rules = Rules::of(table)?; - let mut held = budget.reserve(0, "INSERT")?; - let columns = &table.meta.columns; - let named: Vec = if plan.columns.is_empty() { - (0..columns.len()).collect() - } else { - let named = plan.columns.iter().map(|name| column(table, name)); - named.collect::>()? }; - // A column the statement leaves out starts at its default, or at NULL. - let mut blank = vec![Value::Null; columns.len()]; - for (at, column) in columns.iter().enumerate() { - if named.contains(&at) { - continue; + let table = given.catalog.table(name); + let table = table.ok_or_else(|| Error::TableNotFound(name.clone()))?; + let rules = Rules::of(table, given)?; + rules.changeable(plan)?; + // What the reader's filter holds is given back with the statement. + let mut seeing = given.budget.reserve(0, "query")?; + let mut visible = Visible::new(table, given.catalog, given.reader, &mut seeing)?; + let rows = match plan { + Plan::Insert(plan) => rules.insert(plan, changes, partitions, &mut visible)?, + Plan::Update(plan) => { + let (set, filter) = (rules.set(&plan.assignments)?, plan.where_clause.as_ref()); + let params = given.params; + rules.change(&set, filter, params, changes, partitions, &mut visible)? } - match &column.default { - Some(ColumnDefault::Literal(value)) => blank[at] = value.clone(), - Some(ColumnDefault::Now) => blank[at] = Value::Timestamp(now), - None => {} + Plan::Delete(plan) => { + let filter = plan.where_clause.as_ref(); + rules.delete(filter, changes, partitions, &mut visible)? } - } - - for given in &plan.values { - if given.len() != named.len() { - return Err(Error::PlanError(format!( - "INSERT INTO {} gives {} values for {} columns", - table.name, - given.len(), - named.len() - ))); - } - let mut row = blank.clone(); - for (&at, value) in named.iter().zip(given) { - row[at] = query::eval(value, table, &[], params, now)?; - } - let row = row.into_iter().enumerate(); - let row = row.map(|(at, value)| rules.settle(at, value)); - let row = row.collect::>()?; - rules.put(changes, partitions, &mut held, None, row)?; - } - changes.add_count(table.id, plan.values.len() as i64)?; - rules.settle_partitions(partitions)?; - Ok(plan.values.len() as u64) -} - -fn update( - plan: &UpdatePlan, - params: &HashMap, - catalog: &Catalog, - changes: &mut ChangeSet, - partitions: &mut Tally, - budget: &Budget, - now: i64, -) -> Result { - let table = find(catalog, &plan.table)?; - let rules = Rules::of(table)?; - let set = plan.assignments.iter(); - let set: Vec<_> = set - .map(|(name, value)| Ok((column(table, name)?, value))) - .collect::>()?; - let (found, mut held) = query::matching( - table, - plan.where_clause.as_ref(), - params, - &*changes, - budget, - now, - )?; - for (key, old) in &found { - // Every new value is worked out from the row as it was. - let mut row = old.clone(); - for &(at, value) in &set { - row[at] = rules.settle(at, query::eval(value, table, old, params, now)?)?; - } - // A row that already holds what it is set to is left alone: a statement that changes - // nothing stages nothing. - if row != *old { - rules.put(changes, partitions, &mut held, Some((key, old)), row)?; - } - } - rules.settle_partitions(partitions)?; - Ok(found.len() as u64) + other => unreachable!("{other:?} writes no rows"), + }; + propagate::follow(&rules, changes, partitions)?; + partitions.settle(given.catalog)?; + Ok(rows) } -fn delete( - plan: &DeletePlan, - params: &HashMap, +/// Stages an entry in a new index for every row the table already has, a batch of rows at a +/// time, so that what is held here does not grow with the table. An index that holds no +/// entries (the primary key's, or one over a JSON path) is passed over. +pub(crate) fn fill_index( + table: &Table, + index: IndexId, catalog: &Catalog, changes: &mut ChangeSet, - partitions: &mut Tally, budget: &Budget, - now: i64, -) -> Result { - let table = find(catalog, &plan.table)?; - let rules = Rules::of(table)?; - let (found, _held) = query::matching( - table, - plan.where_clause.as_ref(), - params, - &*changes, +) -> Result<()> { + let (params, reader) = (HashMap::new(), Reader::admin()); + let given = Given { + params: ¶ms, + catalog, budget, - now, - )?; - for (key, row) in &found { - rules.note(partitions, changes, Some(key), None)?; - for index in &rules.indexes { - changes.delete_index(index.id, &index.entry(row)?, key)?; - } - changes.delete_row(table.id, key)?; - } - // A statement that finds no row stages nothing, not even a count change of zero. - if !found.is_empty() { - changes.add_count(table.id, -(found.len() as i64))?; + reader: &reader, + now: 0, + number: TxId(0), + }; + let rules = Rules::of(table, &given)?; + let Some(index) = rules.indexes.iter().find(|known| known.id == index) else { + return Ok(()); + }; + let mut needed = vec![false; table.meta.columns.len()]; + for &at in &index.columns { + needed[at] = true; } - rules.settle_partitions(partitions)?; - Ok(found.len() as u64) -} - -fn find<'a>(catalog: &'a Catalog, table: &str) -> Result<&'a Table> { - catalog - .table(table) - .ok_or_else(|| Error::TableNotFound(table.to_owned())) + let asked = rules.asked(None, ¶ms, &[], &needed, &reader); + rules.each(&asked, changes, |changes, key, cells| { + changes.put_index(index.id, index.entry(&split(cells).0)?, key) + }) } /// Where a column a statement or an index names is in a row of the table. @@ -297,200 +160,611 @@ fn column(table: &Table, name: &str) -> Result { /// What a write needs to know about its table, worked out once for a statement. struct Rules<'a> { table: &'a Table, - /// Where the primary key's column is in a row: every column the table marks as its key. - key: Vec, + given: &'a Given<'a>, /// The table's indexes that hold entries, in the order the table lists them. The index /// behind the primary key (`__pk_...`) is left out: it holds none, a row is kept under its /// key. - indexes: Vec, - /// The table has a VECTOR column with a PARTITION_KEY: the rows a statement stages in it - /// are noted in the transaction's tally of partitions. - partitioned: bool, + indexes: Vec>, + /// The columns of each count the table keeps, and the direction of each. + counts: Vec<(Vec, Vec)>, + /// For each VECTOR column with a PARTITION_KEY: where it is in a row, then where the + /// columns of its key are. The rows a statement stages in such a table are noted in the + /// transaction's tally of partitions. + partitions: Vec>, /// Where the VECTOR columns that keep a vector in one byte a number are in a row. reduced: Vec, + /// The rows that are to follow the rows whose state the statement has changed so far. + followed: RefCell>, } /// One index of a table as a write uses it. -struct Index { +struct Index<'a> { id: IndexId, /// Where its columns are in a row, in index order. columns: Vec, - /// The direction each column was declared with. - directions: Vec, + /// How its entries are made. + shape: &'a EntryShape, /// The columns of the unique rule this index is behind, as a refusal names them. `schema` /// makes one index for each unique rule and names it `__unique_...`. unique: Option, } impl<'a> Rules<'a> { - fn of(table: &'a Table) -> Result> { - let meta = &table.meta; - let all = 0..meta.columns.len(); - let key = all.filter(|&at| meta.columns[at].primary_key).collect(); - let held = meta.indexes.iter().zip(&table.index_ids); - let held = held.filter(|(index, _)| !index.name.starts_with("__pk_")); - let indexes = held.map(|(index, &id)| { + fn of(table: &'a Table, given: &'a Given<'a>) -> Result> { + let held = table.meta.indexes.iter().enumerate().zip(&table.index_ids); + let held = held.filter(|((_, index), _)| { + !index.name.starts_with("__pk_") && index.json_path.is_empty() + }); + let indexes = held.map(|((at, index), &id)| { let names: Vec<&str> = index.columns.iter().map(|(name, _)| &**name).collect(); let columns = names.iter().map(|name| column(table, name)); Ok(Index { id, columns: columns.collect::>()?, - directions: index.columns.iter().map(|(_, way)| *way).collect(), + shape: table.entry_shape(at), unique: index .name .starts_with("__unique_") .then(|| names.join(", ")), }) }); - let mut vectors = table.vectors.iter(); - let reduced = vectors - .clone() - .filter(|vector| vector.quantization == Quantization::Sq8) - .map(|vector| column(table, &vector.column)); + let counts = table.rules.counts.iter(); + let reduced = table.vectors.iter(); + let reduced = reduced.filter(|vector| vector.quantization == Quantization::Sq8); + let vectors = table.vectors.iter(); + let vectors = vectors.filter(|vector| !vector.partition_key.is_empty()); + let partitions = vectors.map(|vector| { + let names = std::iter::once(&vector.column).chain(&vector.partition_key); + names.map(|name| column(table, name)).collect() + }); Ok(Rules { table, - key, + given, indexes: indexes.collect::>()?, - reduced: reduced.collect::>()?, - partitioned: vectors.any(|vector| !vector.partition_key.is_empty()), + counts: counts + .map(|count| count.columns.iter().copied().unzip()) + .collect(), + partitions: partitions.collect::>()?, + reduced: reduced + .map(|vector| column(table, &vector.column)) + .collect::>()?, + followed: RefCell::default(), }) } - /// Notes in the tally a row the statement is about to stage: `was` is the key the - /// transaction has had the row under until now, `None` for a new row, and `after` the - /// values it is staged with, `None` for a removal. - fn note( + /// What a statement asks of the table source. + fn asked<'b>( + &'b self, + filter: Option<&'b Expr>, + params: &'b HashMap, + assigned: &'b [usize], + needed: &'b [bool], + reader: &'b Reader, + ) -> query::Asked<'b> { + query::Asked { + table: self.table, + filter, + assigned, + needed, + reader, + params, + now: self.given.now, + } + } + + /// Hands `each` every row that was asked for, a batch at a time: a batch is staged, and + /// given back, before the next is asked for after its last key, so that one batch is held + /// beside the change set and never the matched rows twice. + fn each( &self, - partitions: &mut Tally, - changes: &ChangeSet, - was: Option<&Key>, - after: Option<&[Value]>, + asked: &query::Asked, + changes: &mut ChangeSet, + mut each: impl FnMut(&mut ChangeSet, Key, Vec) -> Result<()>, ) -> Result<()> { - if !self.partitioned { - return Ok(()); + let (catalog, budget) = (self.given.catalog, self.given.budget); + let mut after = None; + while let Some((batch, _held)) = + query::rows_for_write(asked, after.as_ref(), catalog, &*changes, budget)? + { + after = batch.last().map(|(key, _)| key.clone()); + for (key, cells) in batch { + each(changes, key, cells)?; + } } - let before = match was { - Some(key) => changes.row(self.table.id, key)?, + Ok(()) + } + + /// Where each assigned column is in a row, with its new value. + fn set<'e>(&self, assignments: &'e [(String, Expr)]) -> Result> { + self.targets(assignments.iter().map(|(name, value)| (name, value))) + } + + /// Where each column a statement writes is in a row, with what is written to it; one + /// named twice is refused. + fn targets<'n, T>( + &self, + named: impl Iterator, + ) -> Result> { + let mut targets: Vec<(usize, T)> = Vec::new(); + for (name, with) in named { + let at = column(self.table, name)?; + if targets.iter().any(|(target, _)| *target == at) { + let table = &self.table.name; + let refusal = format!("{name} is named twice as a column {table} is written in"); + return Err(Error::PlanError(refusal)); + } + targets.push((at, with)); + } + Ok(targets) + } + + fn insert( + &self, + plan: &InsertPlan, + changes: &mut ChangeSet, + partitions: &mut Tally, + visible: &mut Visible, + ) -> Result { + let (table, given) = (self.table, self.given); + let columns = &table.meta.columns; + let named: Vec = if plan.columns.is_empty() { + (0..columns.len()).collect() + } else { + let named = self.targets(plan.columns.iter().map(|name| (name, ())))?; + named.into_iter().map(|(at, ())| at).collect() + }; + // A column the statement leaves out starts at its default, or at NULL. + let mut blank = vec![Value::Null; columns.len()]; + for (at, column) in columns.iter().enumerate() { + if named.contains(&at) { + continue; + } + match &column.default { + Some(ColumnDefault::Literal(value)) => blank[at] = value.clone(), + Some(ColumnDefault::Now) => blank[at] = Value::Timestamp(given.now), + None => {} + } + } + // `ON CONFLICT` names the primary key or a whole UNIQUE rule, in any order. + let conflict = match &plan.on_conflict { + Some(conflict) => { + let on = conflict.columns.iter().map(|name| column(table, name)); + let mut on: Vec = on.collect::>()?; + on.sort_unstable(); + let is_on = |columns: &[usize]| { + let mut columns = columns.to_vec(); + columns.sort_unstable(); + columns == on + }; + let mut unique = self.indexes.iter().filter(|index| index.unique.is_some()); + if !is_on(&table.key) && !unique.any(|index| is_on(&index.columns)) { + return Err(Error::PlanError(format!( + "ON CONFLICT ({}) names neither the primary key nor a UNIQUE rule of {}", + conflict.columns.join(", "), + table.name + ))); + } + Some((on, self.set(&conflict.assignments)?)) + } None => None, }; - partitions.note(self.table, before.as_ref(), after, changes) + + let (mut stored, scope) = (0, table.scope()); + for values in &plan.values { + if values.len() != named.len() { + return Err(Error::PlanError(format!( + "INSERT INTO {} gives {} values for {} columns", + table.name, + values.len(), + named.len() + ))); + } + let mut row = blank.clone(); + for (&at, value) in named.iter().zip(values) { + let ctx = query::EvalContext::new(given.params, given.now); + row[at] = query::eval(value, &scope, &[], &ctx)?; + } + let row = row.into_iter().enumerate(); + let row = row.map(|(at, value)| self.settle(at, value)); + let row: Vec = row.collect::>()?; + // A row that meets one already there in the named columns changes that row and is + // not stored. A NULL meets nothing. + if let Some((on, set)) = &conflict + && !on.iter().any(|&at| row[at] == Value::Null) + { + let (filter, bound) = equal(table, on, &pick(on, &row)); + let filter = Some(&filter); + if self.change(set, filter, &bound, changes, partitions, visible)? > 0 { + continue; + } + } + self.put(changes, partitions, visible, None, row, Vec::new())?; + stored += 1; + } + // A statement that stores no row stages no count change, not even one of zero. + if stored > 0 { + changes.add_count(table.id, stored)?; + } + Ok(plan.values.len() as u64) } - /// The last act of a statement, once every row is staged and every other rule has passed: - /// refuses a statement that gave a column one partition too many. - fn settle_partitions(&self, partitions: &mut Tally) -> Result<()> { - if self.partitioned { - partitions.settle(self.table)?; + /// Changes every row that passes `filter`, whose parameters are `params`, by the + /// assignments of `set`, and returns how many rows passed. + /// + /// A large value is read only where the statement reads it: in an assignment, in the + /// reader's filter, or with an index, a kept count or a partition key that an assigned + /// column belongs to. Any other is handed back to the store as the cell it came as. + fn change( + &self, + set: &[(usize, &Expr)], + filter: Option<&Expr>, + params: &HashMap, + changes: &mut ChangeSet, + partitions: &mut Tally, + visible: &mut Visible, + ) -> Result { + let (table, given) = (self.table, self.given); + let assigned: Vec = set.iter().map(|&(at, _)| at).collect(); + let mut needed = vec![false; table.meta.columns.len()]; + for &(_, value) in set { + query::reads(value, table, given.params, &mut needed)?; } - Ok(()) + visible.reads(&mut needed); + let state = self.state(&assigned, &mut needed)?; + self.needs(&assigned, &mut needed); + // A row whose key changes is a new row at its new key, and one whose key or unique + // value is still held by a row this statement has yet to change is taken out and put + // back last: either is written whole. + let unique = self.indexes.iter().filter(|index| index.unique.is_some()); + let named = table + .key + .iter() + .chain(unique.flat_map(|index| &index.columns)); + if named.into_iter().any(|at| assigned.contains(at)) { + needed.fill(true); + } + let asked = self.asked(filter, params, &assigned, &needed, given.reader); + let scope = table.scope(); + // The keys this statement has moved rows to: a row met at one of them has been + // changed already and is passed by. + let mut reached = BTreeSet::new(); + let mut reserved = given.budget.reserve(0, "query")?; + // The rows taken out, as they are to be: a rule is held to what the statement leaves, + // so a row that meets one the statement has yet to change is no refusal yet. + let mut lifted = Vec::new(); + let bytes = |row: &[Value]| { + let held = row.iter().map(types::heap_bytes).sum::(); + held + size_of_val(row) as u64 + }; + // Every new value is worked out from the row as it was. + let assign = |old: &[Value]| -> Result> { + let mut row = old.to_vec(); + for &(at, value) in set { + let ctx = query::EvalContext::new(given.params, given.now); + row[at] = self.settle(at, query::eval(value, &scope, old, &ctx)?)?; + } + Ok(row) + }; + let mut count = 0; + self.each(&asked, changes, |changes, key, cells| { + if reached.contains(&key) { + return Ok(()); + } + count += 1; + let (old, mut left) = split(cells); + // Checked before anything else, so a row this reader may not write is refused even + // where the statement would leave it as it is. + visible.may_write(&old, &*changes)?; + let row = assign(&old)?; + self.transition(state, &old, &row)?; + let unread = left.len(); + left.retain(|(at, _)| !assigned.contains(at)); + // A row that already holds what it is set to is left alone: a statement that + // changes nothing stages nothing. + if row == old && left.len() == unread { + return Ok(()); + } + self.moved(&old, &row, &left, &*changes)?; + let (was, mark) = (Some((&key, &*old)), changes.mark()); + match self.put(changes, partitions, visible, was, row, left) { + Ok(Some(key)) => { + reserved.grow(key.as_bytes().len() as u64)?; + reached.insert(key); + } + Ok(None) => {} + Err(Error::UniqueViolation { .. }) => { + changes.rewind(mark); + let row = assign(&old)?; + reserved.grow(bytes(&row))?; + let cells = old.into_iter().map(Cell::Here).collect(); + self.remove(changes, partitions, visible, &key, cells)?; + lifted.push(row); + } + Err(other) => return Err(other), + } + Ok(()) + })?; + // Put back, a row is refused only by a row the statement leaves. + for row in lifted { + reserved.shrink(bytes(&row)); + self.put(changes, partitions, visible, None, row, Vec::new())?; + } + Ok(count) + } + + /// A write that changes a column reads every column of each index, count and partition + /// group that column is in. + fn needs(&self, changed: &[usize], needed: &mut [bool]) { + let indexes = self.indexes.iter().map(|index| &index.columns); + let counts = self.counts.iter().map(|(columns, _)| columns); + let groups = indexes.chain(counts).chain(&self.partitions); + let touched = groups.filter(|group| group.iter().any(|at| changed.contains(at))); + for &at in touched.flatten() { + needed[at] = true; + } + } + + /// Removes every row that passes `filter` and returns how many did. + fn delete( + &self, + filter: Option<&Expr>, + changes: &mut ChangeSet, + partitions: &mut Tally, + visible: &mut Visible, + ) -> Result { + let (table, given) = (self.table, self.given); + // A removed row's values are read only to find its index entries and its counts. + let mut needed = vec![false; table.meta.columns.len()]; + let indexes = self.indexes.iter().map(|index| &index.columns); + let counts = self.counts.iter().map(|(columns, _)| columns); + for &at in indexes.chain(counts).flatten() { + needed[at] = true; + } + let asked = self.asked(filter, given.params, &[], &needed, given.reader); + let mut count = 0_i64; + self.each(&asked, changes, |changes, key, cells| { + count += 1; + self.remove(changes, partitions, visible, &key, cells) + })?; + // A statement that finds no row stages nothing, not even a count change of zero. + if count > 0 { + changes.add_count(table.id, -count)?; + } + Ok(count as u64) + } + + /// Stages the removal of a row the reader may write, read with the columns of every index + /// and kept count, of its entries, and of its place in each count. + fn remove( + &self, + changes: &mut ChangeSet, + partitions: &mut Tally, + visible: &mut Visible, + key: &Key, + cells: Vec, + ) -> Result<()> { + let before = Row { cells }; + if !self.partitions.is_empty() { + partitions.note(self.table, Some(&before), None, &*changes)?; + } + let (row, _) = split(before.cells); + visible.may_write(&row, &*changes)?; + self.count(changes, Some(&row), None)?; + for index in &self.indexes { + changes.delete_index(index.id, &index.entry(&row)?.key, key)?; + } + changes.delete_row(self.table.id, key) + } + + /// The key of values that name a row or a group. It is kept whole, so one longer than + /// `KEY_LIMIT` is refused, under the name `what` gives. + fn identity( + &self, + what: impl FnOnce() -> String, + values: &[Value], + ways: &[SortDirection], + ) -> Result { + let key = Key::from_values(values, ways)?; + let bytes = key.as_bytes().len(); + if bytes > KEY_LIMIT { + return Err(Error::KeyTooLong { + table: self.table.name.clone(), + key: what(), + bytes: bytes as u64, + limit: KEY_LIMIT as u64, + }); + } + Ok(key) } /// Brings a value written into a column to the column's type, and refuses NULL where the - /// column needs a value: a NOT NULL column and the primary key. + /// column needs a value, a NOT NULL column and the primary key, and a transaction number + /// above the transaction's own. fn settle(&self, at: usize, value: Value) -> Result { let column = &self.table.meta.columns[at]; - let value = values::typed(&self.table.name, column, value)?; - if matches!(value, Value::Null) && (!column.nullable || column.primary_key) { + // A text becomes a JSON tree only once the most the tree can take is counted. + let _tree = match (&value, column.column_type) { + (Value::Text(text), ColumnType::Json) => Some( + self.given + .budget + .reserve(types::json_bytes(text.as_bytes()), "query")?, + ), + _ => None, + }; + let value = typed(&self.table.name, column, value)?; + if matches!(value, Value::Null) && (!column.nullable || self.table.key.contains(&at)) { return Err(Error::ColumnNotNullable { table: self.table.name.clone(), column: column.name.clone(), }); } - Ok(value) + let own = self.given.number; + match value { + Value::TxId(TxId(value)) if value > own.0 => { + Err(Error::TxIdOutOfRange { value, max: own.0 }) + } + value => Ok(value), + } } - /// Stages a row with its entry in every index, once its key and its unique values are found - /// free in what the change set shows. - /// - /// `was` is the key and the values the row had, for an UPDATE: what the row keeps from - /// before is its own and is not looked up, an entry that stays as it was is left alone, and - /// a row whose key changed leaves its old key. The row is noted in `partitions` first. + /// Stages a row with its entry in every index, once the reader may write it (`change` has + /// checked the row it was) and its key and its unique values are found free in what the + /// change set shows. Returns the row's key when an UPDATE moved the row to it. /// - /// Before anything else, a vector in a column that keeps one byte a number is replaced by - /// its reduced form, whose scale and codes go to the store with the row. `held` is grown - /// by a byte a number for the codes and four for the numbers until the row is staged. + /// `was` is the key and the values the row had, for an UPDATE, with NULL for each large + /// value that was not read; `left` holds those of them the statement did not assign, which + /// go back to the store as they came. What the row keeps from before is its own and is + /// not looked up, an entry whose values stay is left alone, and a row whose key changed + /// leaves its old key. A table with no primary key keeps a new row under the next row + /// number. fn put( &self, changes: &mut ChangeSet, partitions: &mut Tally, - held: &mut Reservation, + visible: &mut Visible, was: Option<(&Key, &[Value])>, mut row: Vec, - ) -> Result<()> { + left: Vec<(usize, Beside)>, + ) -> Result> { + let table = self.table; + // A vector in a column that keeps one byte a number is replaced by its reduced form, + // whose scale and codes go to the store with the row; held until the row is staged. let mut compact = Vec::new(); - let mut grown = 0; + let mut held = self.given.budget.reserve(0, "query")?; for &at in &self.reduced { let Value::Vector(vector) = &row[at] else { continue; }; - let bytes = 5 * vector.len() as u64; - held.grow(bytes)?; - grown += bytes; + held.grow(5 * vector.len() as u64)?; if let Some(reduced) = Reduced::of(vector) { row[at] = Value::Vector(reduced.numbers()); - compact.push(Compact { - at, - scale: reduced.scale, - codes: reduced.codes, - }); + let (scale, codes) = (reduced.scale, reduced.codes); + compact.push(Compact { at, scale, codes }); } } - self.note(partitions, changes, was.map(|(key, _)| key), Some(&row))?; - // No directions: a row's key is encoded ascending. - let key = Key::from_values(&pick(&self.key, &row), &[])?; + visible.may_write(&row, &*changes)?; + let edge = self.edge(was, &row)?; + // True when the row is new or has another value in one of these columns. + let differs = |columns: &[usize]| { + was.is_none_or(|(_, old)| columns.iter().any(|&at| row[at] != old[at])) + }; + let key = match was { + Some((old, _)) if !differs(&table.key) => old.clone(), + // The transaction's own number, which no other is given, and its count of such rows. + _ if table.key.is_empty() => { + let made = [self.given.number.0, changes.next_row_number()]; + Key::from_values(&made.map(|number| Value::Int64(number as i64)), &[])? + } + // No directions: a row's key is encoded ascending. + _ => self.identity(|| "primary key".to_owned(), &pick(&table.key, &row), &[])?, + }; // A new row, or a row whose key changed: the transaction must find no row at this key. let new_key = was.is_none_or(|(old, _)| *old != key); - if new_key && changes.row(self.table.id, &key)?.is_some() { - return Err(self.violation(self.key_columns())); + if new_key && changes.row(table.id, &key)?.is_some() { + return Err(self.violation(self.key_columns(), pick(&table.key, &row))); } - let own = was.map_or(&key, |(old_key, _)| old_key); + self.count(changes, was.map(|(_, old)| old), Some(&row))?; + let own = was.map_or(&key, |(old, _)| old); for index in &self.indexes { - let entry = index.entry(&row)?; - let old = was.map(|(_, old)| index.entry(old)).transpose()?; - let same = old.as_ref() == Some(&entry); - if same && !new_key { + let changed = differs(&index.columns); + if !changed && !new_key { continue; } - if !same + let entry = index.entry(&row)?; + if changed && let Some(columns) = index.rule(&row) - && taken(&*changes, index.id, &entry, own)? + && self.taken(&*changes, index, &entry, &row, own)? { - return Err(self.violation(columns.to_owned())); + return Err(self.violation(columns.to_owned(), pick(&index.columns, &row))); } - if let Some(old) = &old { - changes.delete_index(index.id, old, own)?; + if let Some((_, old)) = was { + changes.delete_index(index.id, &index.entry(old)?.key, own)?; } changes.put_index(index.id, entry, key.clone())?; } - if let Some((old_key, _)) = was - && new_key - { - changes.delete_row(self.table.id, old_key)?; + // Noted once nothing can refuse the row any more. + if new_key || self.partitions.iter().any(|group| differs(group)) { + for group in &self.partitions { + if row[group[0]] != Value::Null { + let vector = || table.meta.columns[group[0]].name.clone(); + self.identity(vector, &pick(&group[1..], &row), &[])?; + } + } + // The row as it was read: a statement that reaches here read all of it that the + // tally looks at. + let before = was.map(|(_, old)| Row { + cells: old.iter().cloned().map(Cell::Here).collect(), + }); + partitions.note(table, before.as_ref(), Some((&row, &left)), &*changes)?; } - changes.put_row(self.table.id, key, row, compact, new_key)?; - held.shrink(grown); - Ok(()) + let moved = match was { + Some((old, _)) if new_key => { + changes.delete_row(table.id, old)?; + Some(key.clone()) + } + _ => None, + }; + let mut cells: Vec = row.into_iter().map(Cell::Here).collect(); + for (at, beside) in left { + cells[at] = Cell::Beside(beside); + } + changes.put_row(table.id, key, cells, compact, new_key)?; + self.acyclic(&*changes, edge)?; + Ok(moved) + } + + /// True when a row other than `own` holds the values `row` has in the index's columns. + /// + /// An entry that holds a cut value says only that such a row may be there: the rows are + /// then read through the table source and compared by their values. + fn taken( + &self, + read: &dyn Read, + index: &Index, + entry: &Entry, + row: &[Value], + own: &Key, + ) -> Result { + let at = Bound::Included(&entry.key); + let mut others = read + .index(index.id, at, at)? + .filter(|found| !matches!(found, Ok(found) if found.1 == *own)); + if others.next().transpose()?.is_none() { + return Ok(false); + } + if entry.cut.is_none() { + return Ok(true); + } + drop(others); + let (filter, bound) = equal(self.table, &index.columns, &pick(&index.columns, row)); + let (needed, reader) = (vec![false; row.len()], Reader::admin()); + let asked = self.asked(Some(&filter), &bound, &[], &needed, &reader); + let given = self.given; + let found = query::rows_for_write(&asked, None, given.catalog, read, given.budget)?; + Ok(found.is_some_and(|(rows, _)| rows.iter().any(|(key, _)| key != own))) } /// The columns of the primary key, as a refusal names them. fn key_columns(&self) -> String { let columns = &self.table.meta.columns; - let names: Vec<&str> = self.key.iter().map(|&at| &*columns[at].name).collect(); + let key = self.table.key.iter(); + let names: Vec<&str> = key.map(|&at| &*columns[at].name).collect(); names.join(", ") } - fn violation(&self, column: String) -> Error { + fn violation(&self, column: String, key: Vec) -> Error { Error::UniqueViolation { table: self.table.name.clone(), column, + key: shown(&key), } } } -impl Index { +impl Index<'_> { /// The row's entry in this index. - fn entry(&self, row: &[Value]) -> Result { - Key::from_values(&pick(&self.columns, row), &self.directions) + fn entry(&self, row: &[Value]) -> Result { + self.shape.entry(&pick(&self.columns, row)) } /// The columns of the unique rule this index holds the row to. A NULL in any column of the @@ -507,12 +781,43 @@ fn pick(columns: &[usize], row: &[Value]) -> Vec { columns.iter().map(|&at| row[at].clone()).collect() } -/// True when the index has this entry for a row other than `own`. -fn taken(read: &dyn Read, index: IndexId, entry: &Key, own: &Key) -> Result { - for found in read.index(index, Bound::Included(entry), Bound::Included(entry))? { - if found?.1 != *own { - return Ok(true); +/// What passes exactly the rows that hold `values` in `columns`, side by side, and the +/// parameters it reads them from: each under its column's name. +fn equal(table: &Table, columns: &[usize], values: &[Value]) -> (Expr, HashMap) { + let mut bound = HashMap::new(); + let each = columns.iter().zip(values).map(|(&at, value)| { + let name = &table.meta.columns[at].name; + bound.insert(name.clone(), value.clone()); + let column = ColumnRef { + table: None, + column: name.clone(), + }; + Expr::BinaryOp { + left: Box::new(Expr::Column(column)), + op: BinOp::Eq, + right: Box::new(Expr::Parameter(name.clone())), } - } - Ok(false) + }); + let filter = each.reduce(|left, right| Expr::BinaryOp { + left: Box::new(left), + op: BinOp::And, + right: Box::new(right), + }); + let filter = filter.expect("a key and a UNIQUE rule have a column"); + (filter, bound) +} + +/// A row's values in column order, with NULL for each large value that was left beside the +/// row, and those cells with the columns they are of. +fn split(cells: Vec) -> (Vec, Vec<(usize, Beside)>) { + let mut left = Vec::new(); + let values = cells.into_iter().enumerate().map(|(at, cell)| match cell { + Cell::Here(value) => value, + Cell::Beside(beside) => { + left.push((at, beside)); + Value::Null + } + }); + let values = values.collect(); + (values, left) } diff --git a/v2/contextdb/src/write/propagate.rs b/v2/contextdb/src/write/propagate.rs new file mode 100644 index 00000000..64c58f06 --- /dev/null +++ b/v2/contextdb/src/write/propagate.rs @@ -0,0 +1,293 @@ +//! Propagation: a row that changes its state is followed, inside the statement that changed +//! it, by the rows that declared they follow that change. Those are the rows of a table whose +//! reference says `ON STATE s PROPAGATE SET t`, and the rows of the row's own table that an +//! edge joins it to under `PROPAGATE ON EDGE`. A row that moved by following is followed in +//! turn. +//! +//! The rows a statement names are all written first, and what follows them is then walked +//! nearest first. So a row is reached over the fewest edges there are, and a rule's +//! `MAX DEPTH` counts the steps (edges, or references) between a row and a row the statement +//! itself changed. +//! +//! A follower is found whoever wrote, and written as the statement's reader: one that reader +//! may not write refuses the statement. + +use std::collections::VecDeque; + +use super::{Given, Rules, column, equal, split}; +use crate::memory::Reservation; +use crate::query::{Reader, Visible}; +use crate::schema::Table; +use crate::sql::ast::{EdgePropagation, Propagation}; +use crate::store::{Beside, ChangeSet, Read}; +use crate::types::{Error, Result, Value}; +use crate::vector::Tally; + +/// How many edges from a changed row a rule reaches when it names no `MAX DEPTH`. +const MAX_DEPTH: u32 = 10; + +/// The rows that are to follow one row that changed its state. +pub(super) struct Step { + /// The table whose rows follow. + table: String, + via: Via, + /// What finds them: the values the reference's columns hold in them, or the id of the row + /// that changed. + values: Vec, + /// The memory the step takes, given back when it has been followed. + _held: Reservation, +} + +/// The declaration a step follows, by its place among those of the step's table. +enum Via { + Reference(usize), + Edge(usize), +} + +impl Rules<'_> { + /// Notes the rows that follow a row of the table which an UPDATE takes from `old` to + /// `row`: none unless the row changes its state. `left` holds the row's large values that + /// were not read. + pub(super) fn moved( + &self, + old: &[Value], + row: &[Value], + left: &[(usize, Beside)], + read: &dyn Read, + ) -> Result<()> { + let table = self.table; + let Some(machine) = &table.meta.state_machine else { + return Ok(()); + }; + let at = column(table, &machine.column)?; + let Value::Text(state) = &row[at] else { + return Ok(()); + }; + if old[at] == row[at] { + return Ok(()); + } + let step = |follower: &Table, via: Via, columns: &[usize]| -> Result<()> { + let bytes = size_of::() + columns.len() * size_of::(); + let mut held = self.given.budget.reserve(bytes as u64, "query")?; + let mut values = Vec::new(); + for &at in columns { + values.push(match left.iter().find(|unread| unread.0 == at) { + Some((_, beside)) => read.load(beside, &mut held)?, + None => { + held.grow(crate::types::heap_bytes(&row[at]))?; + row[at].clone() + } + }); + } + self.followed.borrow_mut().push(Step { + table: follower.name.clone(), + via, + values, + _held: held, + }); + Ok(()) + }; + for child in self.given.catalog.tables() { + for (place, reference) in child.rules.references.iter().enumerate() { + let rule = reference.on_state.as_ref(); + if reference.parent == table.name && rule.is_some_and(|r| r.on_state == *state) { + let columns = reference.parent_columns.iter(); + let columns = columns.map(|name| column(table, name)); + step( + child, + Via::Reference(place), + &columns.collect::>>()?, + )?; + } + } + } + for (place, edge) in table.rules.propagate_edges.iter().enumerate() { + if edge.rule.on_state == *state { + step(table, Via::Edge(place), &[column(table, "id")?])?; + } + } + Ok(()) + } +} + +/// One statement's propagation: the steps not yet followed, nearest first. +struct Walk<'a> { + /// What the statement is run with, read as a handle that is shown every row. + given: &'a Given<'a>, + /// Who writes the rows that follow. + writer: &'a Reader, + /// Each step with the number of edges between its rows and a row the statement changed. + queue: VecDeque<(u32, Step)>, + /// That number for the step being followed. + depth: u32, +} + +/// Follows every change of state the statement made to the rows of its table, and every change +/// made by following. Called once the statement's own rows are written, so the statement finds +/// its rows as they were before any of them was followed. +/// +/// # Panics +/// +/// When a step names a table that is not in the statement's catalog. +pub(super) fn follow(rules: &Rules, changes: &mut ChangeSet, partitions: &mut Tally) -> Result<()> { + let first = rules.followed.take(); + if first.is_empty() { + return Ok(()); + } + let admin = Reader::admin(); + let given = Given { + reader: &admin, + ..*rules.given + }; + let mut walk = Walk { + given: &given, + writer: rules.given.reader, + queue: first.into_iter().map(|step| (1, step)).collect(), + depth: 1, + }; + while let Some((depth, step)) = walk.queue.pop_front() { + walk.depth = depth; + let table = given.catalog.table(&step.table); + let table = table.expect("a step names a table of the statement's catalog"); + match step.via { + Via::Reference(place) => { + let reference = &table.rules.references[place]; + if let Some(rule) = &reference.on_state { + let columns = &reference.columns; + walk.shift(table, columns, &step.values, rule, changes, partitions)?; + } + } + Via::Edge(place) => { + let edge = &table.rules.propagate_edges[place]; + walk.along(table, edge, &step.values[0], changes, partitions)?; + } + } + } + Ok(()) +} + +impl Walk<'_> { + /// True when the rule reaches the rows of the step being followed. + fn reaches(&self, rule: &Propagation) -> bool { + self.depth <= rule.max_depth.unwrap_or(MAX_DEPTH) + } + + /// Moves the rows of `table` that an edge of the declared type joins to the row with this + /// id, over every table of edges: OUTGOING follows an edge from its source to its target, + /// INCOMING from its target to its source, and BOTH either way. + fn along( + &mut self, + table: &Table, + edge: &EdgePropagation, + id: &Value, + changes: &mut ChangeSet, + partitions: &mut Tally, + ) -> Result<()> { + if !self.reaches(&edge.rule) { + return Ok(()); + } + let given = self.given; + let key = [column(table, "id")?]; + for edges in given.catalog.tables() { + let mut indexes = edges.meta.indexes.iter(); + if !indexes.any(|index| index.name.starts_with("__graph_edge_")) { + continue; + } + let ends = [column(edges, "source_id")?, column(edges, "target_id")?]; + let kind = column(edges, "edge_type")?; + let rules = Rules::of(edges, given)?; + for (near, far) in [(0, 1), (1, 0)] { + if edge.direction == ["INCOMING", "OUTGOING"][near] { + continue; + } + let values = [id.clone(), Value::Text(edge.edge_type.clone())]; + let (filter, bound) = equal(edges, &[ends[near], kind], &values); + let needed = vec![false; edges.meta.columns.len()]; + let asked = rules.asked(Some(&filter), &bound, &[], &needed, given.reader); + rules.each(&asked, changes, |changes, _, cells| { + let other = split(cells).0.swap_remove(ends[far]); + self.shift(table, &key, &[other], &edge.rule, changes, partitions) + })?; + } + } + Ok(()) + } + + /// Moves to the state `rule` sets every row of `table` that holds `values` in `columns` + /// and whose state lists that state as a next one, and notes who follows each. A row + /// already in that state is passed by. A row in any other state is left as it is, or, + /// under `ABORT ON FAILURE`, refuses the statement; so is every row of a table whose rows + /// make no move: one with no state machine, and one that is IMMUTABLE or whose state is. + fn shift( + &mut self, + table: &Table, + columns: &[usize], + values: &[Value], + rule: &Propagation, + changes: &mut ChangeSet, + partitions: &mut Tally, + ) -> Result<()> { + if !self.reaches(rule) || values.contains(&Value::Null) { + return Ok(()); + } + let machine = table.meta.state_machine.as_ref(); + let state = match machine { + Some(machine) if !table.meta.immutable => Some(column(table, &machine.column)?), + _ => None, + }; + let state = state.filter(|at| !table.rules.immutable_columns.contains(at)); + if state.is_none() && !rule.abort_on_failure { + return Ok(()); + } + let given = self.given; + let rules = Rules::of(table, given)?; + let mut seeing = given.budget.reserve(0, "query")?; + let mut visible = Visible::new(table, given.catalog, self.writer, &mut seeing)?; + // What the write of a row reads when its state is all that changes. + let mut needed = vec![false; table.meta.columns.len()]; + visible.reads(&mut needed); + if let Some(state) = state { + needed[state] = true; + rules.needs(&[state], &mut needed); + } + let (filter, bound) = equal(table, columns, values); + let assigned = state.as_slice(); + let asked = rules.asked(Some(&filter), &bound, assigned, &needed, given.reader); + let to = Value::Text(rule.set.clone()); + rules.each(&asked, changes, |changes, key, cells| { + let (old, left) = split(cells); + if state.is_some_and(|at| old[at] == to) { + return Ok(()); + } + let leaves = |at: &usize| { + let mut listed = machine.into_iter().flat_map(|machine| &machine.transitions); + listed.any(|(from, next)| { + matches!(&old[*at], Value::Text(now) if now == from) && next.contains(&rule.set) + }) + }; + let Some(at) = state.filter(leaves) else { + if !rule.abort_on_failure { + return Ok(()); + } + return Err(Error::PropagationAborted { + table: table.name.clone(), + reason: format!( + "a row that follows cannot be moved to {}: its table lists no such \ + move from the state the row is in", + rule.set + ), + }); + }; + let mut row = old.clone(); + row[at] = to.clone(); + rules.moved(&old, &row, &left, &*changes)?; + let was = Some((&key, &*old)); + rules.put(changes, partitions, &mut visible, was, row, left)?; + Ok(()) + })?; + let next = self.depth + 1; + let noted = rules.followed.take().into_iter(); + self.queue.extend(noted.map(|step| (next, step))); + Ok(()) + } +} diff --git a/v2/contextdb/src/write/rules.rs b/v2/contextdb/src/write/rules.rs new file mode 100644 index 00000000..0513f6ba --- /dev/null +++ b/v2/contextdb/src/write/rules.rs @@ -0,0 +1,175 @@ +//! The rules a statement is held to as it writes a row: IMMUTABLE on a table and on a column, +//! the transitions of a STATE MACHINE, and the edge types of DAG, whose edges never close a +//! cycle. Each is checked here, once, for every way a row is written. + +use std::collections::BTreeSet; + +use super::{Rules, column, equal}; +use crate::query::{self, Reader}; +use crate::sql::{Expr, Plan}; +use crate::store::{Cell, Key, Read}; +use crate::types::{Error, Result, Uuid, Value}; + +/// What a node reached by a walk along edges is counted as: its id among the nodes reached and +/// in the list of those still to leave from. +const NODE_BYTES: u64 = 64; + +/// An edge of a type its table lists under DAG: its source, its target and its type. +pub(super) type Edge = (Uuid, Uuid, String); + +impl Rules<'_> { + /// Refuses an UPDATE, a DELETE and an `ON CONFLICT DO UPDATE` on a table declared + /// IMMUTABLE, and an assignment to a column declared IMMUTABLE. No row is read: the table + /// says so, whatever it holds. + pub(super) fn changeable(&self, plan: &Plan) -> Result<()> { + let table = self.table; + let assignments: &[(String, Expr)] = match plan { + Plan::Update(plan) => &plan.assignments, + Plan::Insert(plan) => match &plan.on_conflict { + Some(conflict) => &conflict.assignments, + None => return Ok(()), + }, + _ => &[], + }; + if table.meta.immutable { + return Err(Error::ImmutableTable { + table: table.name.clone(), + }); + } + for (name, _) in assignments { + if table + .rules + .immutable_columns + .contains(&column(table, name)?) + { + return Err(Error::ImmutableColumn { + table: table.name.clone(), + column: name.clone(), + }); + } + } + Ok(()) + } + + /// Where the column of the table's STATE MACHINE is in a row, when the statement assigns + /// it. The column is then marked as read: a move is judged from the state the row is in, + /// however long that state is. + pub(super) fn state(&self, assigned: &[usize], needed: &mut [bool]) -> Result> { + let Some(machine) = &self.table.meta.state_machine else { + return Ok(None); + }; + let at = column(self.table, &machine.column)?; + let moves = assigned.contains(&at); + needed[at] |= moves; + Ok(moves.then_some(at)) + } + + /// Refuses a row whose state, at `state`, is set to one the declaration does not list after + /// the state the row is in. A state does not follow itself unless it is listed so. A row + /// that has no state, or is given none, makes no move. + pub(super) fn transition( + &self, + state: Option, + old: &[Value], + new: &[Value], + ) -> Result<()> { + let (Some(at), Some(machine)) = (state, &self.table.meta.state_machine) else { + return Ok(()); + }; + let (Value::Text(from), Value::Text(to)) = (&old[at], &new[at]) else { + return Ok(()); + }; + let mut listed = machine.transitions.iter(); + if listed.any(|(state, next)| state == from && next.contains(to)) { + return Ok(()); + } + Err(Error::InvalidStateTransition { + table: self.table.name.clone(), + column: machine.column.clone(), + from: from.clone(), + to: to.clone(), + }) + } + + /// The edge a row is, as its source, its target and its type, when its table lists that + /// type under DAG. `was` is the row before an UPDATE: an edge the UPDATE leaves as it was + /// is none to look at. + pub(super) fn edge( + &self, + was: Option<(&Key, &[Value])>, + row: &[Value], + ) -> Result> { + if self.table.meta.dag.is_empty() { + return Ok(None); + } + let at = self.edge_columns()?; + let (Value::Uuid(from), Value::Uuid(to), Value::Text(edge_type)) = + (&row[at[0]], &row[at[1]], &row[at[2]]) + else { + return Ok(None); + }; + let same = was.is_some_and(|(_, old)| at.iter().all(|&at| old[at] == row[at])); + let listed = self.table.meta.dag.contains(edge_type); + Ok((listed && !same).then(|| (*from, *to, edge_type.clone()))) + } + + /// Where an edge's source, target and type are in a row. + fn edge_columns(&self) -> Result<[usize; 3]> { + let [source, target, kind] = + ["source_id", "target_id", "edge_type"].map(|name| column(self.table, name)); + Ok([source?, target?, kind?]) + } + + /// Refuses an edge when the edges of its type that `read` shows lead from its target back + /// to its source: it closes a cycle. Each type a table lists under DAG is held to this by + /// itself. + /// + /// It is asked once the edge's row is staged, so `read` shows the row as its statement + /// leaves it and nothing of the row it was: an UPDATE that takes a row out and puts it back + /// is judged where it puts it back. + /// + /// Costs the edges of that type that can be reached from the target, read through the + /// table source by the index that leads with an edge's source. The nodes reached are + /// counted against the memory limit as they are met. + pub(super) fn acyclic(&self, read: &dyn Read, edge: Option) -> Result<()> { + let Some((from, to, edge_type)) = edge else { + return Ok(()); + }; + let (table, given) = (self.table, self.given); + let at = self.edge_columns()?; + // The edges of this type that leave a node: the node is bound anew for each one. + let probe = [Value::Null, Value::Text(edge_type.clone())]; + let (filter, mut bound) = equal(table, &[at[0], at[2]], &probe); + let source = table.meta.columns[at[0]].name.clone(); + let mut needed = vec![false; table.meta.columns.len()]; + needed[at[1]] = true; + let reader = Reader::admin(); + let mut reserved = given.budget.reserve(0, "query")?; + let (mut reached, mut leave) = (BTreeSet::new(), vec![to]); + while let Some(node) = leave.pop() { + if node == from { + return Err(Error::CycleDetected { + table: table.name.clone(), + edge_type, + }); + } + bound.insert(source.clone(), Value::Uuid(node)); + let asked = self.asked(Some(&filter), &bound, &[], &needed, &reader); + let mut after = None; + while let Some((edges, _held)) = + query::rows_for_write(&asked, after.as_ref(), given.catalog, read, given.budget)? + { + after = edges.last().map(|(key, _)| key.clone()); + for (_, cells) in &edges { + if let Cell::Here(Value::Uuid(next)) = cells[at[1]] + && reached.insert(next) + { + reserved.grow(NODE_BYTES)?; + leave.push(next); + } + } + } + } + Ok(()) + } +} diff --git a/v2/contextdb/src/write/tests.rs b/v2/contextdb/src/write/tests.rs index c193d81f..6e1b1b40 100644 --- a/v2/contextdb/src/write/tests.rs +++ b/v2/contextdb/src/write/tests.rs @@ -2,7 +2,8 @@ use serde_json::json; -use super::values::{timestamp_millis, typed}; +use super::typed; +use crate::types::timestamp_millis; use crate::types::{ColumnDef, ColumnType, Error, Uuid, Value}; #[test] diff --git a/v2/contextdb/src/write/values.rs b/v2/contextdb/src/write/values.rs deleted file mode 100644 index 0396a1c7..00000000 --- a/v2/contextdb/src/write/values.rs +++ /dev/null @@ -1,153 +0,0 @@ -//! What a written value has to be before it is stored: of its column's type, by one of the few -//! documented conversions or not at all. - -use std::ops::Range; - -use crate::query::vector_from_text; -use crate::types::{ColumnDef, ColumnType, Error, Result, Uuid, Value, VectorIndexRef}; - -/// Brings a value to its column's type, or refuses it with `ColumnTypeMismatch`. -/// -/// NULL fits every type here; whether the column takes it is the NOT NULL rule's business. The -/// only conversions are quoted text into a UUID, TIMESTAMP, JSON or VECTOR column and an INTEGER -/// into a REAL column. A vector of the column's length with a number that is not finite is -/// refused with `InvalidVectorValue`, however it was written: no search could measure it. -pub(super) fn typed(table: &str, column: &ColumnDef, value: Value) -> Result { - let converted = match (column.column_type, &value) { - (_, Value::Null) - | (ColumnType::Integer, Value::Int64(_)) - | (ColumnType::Real, Value::Float64(_)) - | (ColumnType::Text, Value::Text(_)) - | (ColumnType::Boolean, Value::Bool(_)) - | (ColumnType::Json, Value::Json(_)) - | (ColumnType::Uuid, Value::Uuid(_)) - | (ColumnType::Timestamp, Value::Timestamp(_)) => return Ok(value), - (ColumnType::Vector(length), Value::Vector(vector)) if vector.len() == length => { - return finite(table, column, value); - } - (ColumnType::Real, Value::Int64(whole)) => Some(Value::Float64(*whole as f64)), - (ColumnType::Uuid, Value::Text(text)) => Uuid::parse_str(text).ok().map(Value::Uuid), - (ColumnType::Timestamp, Value::Text(text)) => timestamp_millis(text).map(Value::Timestamp), - (ColumnType::Json, Value::Text(text)) => serde_json::from_str(text).ok().map(Value::Json), - (ColumnType::Vector(length), Value::Text(text)) => vector_from_text(text) - .filter(|vector| vector.len() == length) - .map(Value::Vector), - _ => None, - }; - let converted = converted.ok_or_else(|| Error::ColumnTypeMismatch { - table: table.to_owned(), - column: column.name.clone(), - expected: format!("{:?}", column.column_type).to_uppercase(), - actual: kind(&value), - })?; - finite(table, column, converted) -} - -/// The value itself, unless it is a vector that holds a number that is not finite. -fn finite(table: &str, column: &ColumnDef, value: Value) -> Result { - if let Value::Vector(vector) = &value - && let Some(position) = vector.iter().position(|number| !number.is_finite()) - { - return Err(Error::InvalidVectorValue { - index: VectorIndexRef { - table: table.to_owned(), - column: column.name.clone(), - }, - position, - }); - } - Ok(value) -} - -/// What a refusal calls the value it was given. A vector says how long it is, since its length -/// is what a VECTOR column refuses it for. -fn kind(value: &Value) -> String { - match value { - Value::Null => "Null", - Value::Bool(_) => "Bool", - Value::Int64(_) => "Int64", - Value::Float64(_) => "Float64", - Value::Text(_) => "Text", - Value::Uuid(_) => "Uuid", - Value::Timestamp(_) => "Timestamp", - Value::Json(_) => "Json", - Value::Vector(vector) => return format!("Vector({})", vector.len()), - Value::TxId(_) => "TxId", - } - .to_owned() -} - -/// Reads an ISO 8601 moment with its offset (`2026-01-01T02:00:00+02:00`, -/// `1970-01-01T00:00:00.250Z`) as milliseconds since the first moment of 1970, UTC. Digits past -/// the millisecond are dropped. `None` when the text is not such a moment. -pub(super) fn timestamp_millis(text: &str) -> Option { - let (year, month, day) = ( - digits(text, 0..4)?, - digits(text, 5..7)?, - digits(text, 8..10)?, - ); - let (hour, minute, second) = ( - digits(text, 11..13)?, - digits(text, 14..16)?, - digits(text, 17..19)?, - ); - let marks = text.as_bytes(); - let leap = year % 4 == 0 && (year % 100 != 0 || year % 400 == 0); - let month_days = match month { - 2 => 28 + i64::from(leap), - 4 | 6 | 9 | 11 => 30, - _ => 31, - }; - if [marks[4], marks[7], marks[13], marks[16]] != *b"--::" - || !matches!(marks[10], b'T' | b't') - || !(1..=12).contains(&month) - || !(1..=month_days).contains(&day) - || hour > 23 - || minute > 59 - || second > 59 - { - return None; - } - - let mut rest = &text[19..]; - let mut millis = 0; - if let Some(fraction) = rest.strip_prefix('.') { - let end = fraction - .find(|c: char| !c.is_ascii_digit()) - .unwrap_or(fraction.len()); - if end == 0 { - return None; - } - millis = format!("{:0<3}", &fraction[..end.min(3)]).parse().ok()?; - rest = &fraction[end..]; - } - let offset_minutes = match rest.as_bytes() { - b"Z" | b"z" => 0, - [sign @ (b'+' | b'-'), _, _, b':', _, _] => { - let minutes = digits(rest, 1..3)? * 60 + digits(rest, 4..6)?; - if *sign == b'-' { -minutes } else { minutes } - } - _ => return None, - }; - - // Days since 1970 of a date in the Gregorian calendar, counting years from March so that - // the leap day is the last day of its year. - let (year, month) = if month > 2 { - (year, month - 3) - } else { - (year - 1, month + 9) - }; - let leap_days = year.div_euclid(4) - year.div_euclid(100) + year.div_euclid(400); - let days = year * 365 + leap_days + (153 * month + 2) / 5 + day - 719_469; - Some(((days * 24 + hour) * 60 + minute - offset_minutes) * 60_000 + second * 1_000 + millis) -} - -/// The number written in these positions of the text, when every one of them is a digit. -fn digits(text: &str, positions: Range) -> Option { - let part = text.get(positions)?; - if part.bytes().all(|byte| byte.is_ascii_digit()) { - part.parse().ok() - } else { - None - } -} diff --git a/v2/contextdb/tests/alter.rs b/v2/contextdb/tests/alter.rs new file mode 100644 index 00000000..30bdab47 --- /dev/null +++ b/v2/contextdb/tests/alter.rs @@ -0,0 +1,1308 @@ +//! ALTER TABLE and DROP: what a row written before an added column reads in it, what is left +//! in the store after a column or a table is dropped, what each refusal names, and that a +//! refused statement changes nothing. An altered table is printed back as what it now +//! declares, and the printed text declares that table again. + +mod support; + +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use contextdb::serde_json::json; +use contextdb::{Clock, Database, DatabaseOpenOptions, Error, ReadSession, Value, params}; +use support::Call; + +/// A clock that always shows the same second. +struct At(u64); + +impl Clock for At { + fn now(&self) -> SystemTime { + UNIX_EPOCH + Duration::from_secs(self.0) + } +} + +fn open_at(folder: &Path, second: u64) -> Database { + let mut options = DatabaseOpenOptions::default(); + options.clock = Some(Arc::new(At(second))); + support::open_with(folder, options).expect("open the store with a clock") +} + +/// The length of a store's data file, in bytes. +fn data_file(folder: &Path) -> u64 { + fs::metadata(folder.join("data")) + .expect("the store's data file") + .len() +} + +/// What a table is printed back as. +fn printed(db: &Database, table: &str) -> String { + db.declaration(table) + .unwrap_or_else(|error| panic!("the declaration of {table} was refused: {error:?}")) +} + +/// The words of a declaration, in lower case, and its quoted texts: what it says, whatever +/// the spacing and the punctuation it says it in. +fn words(sql: &str) -> Vec { + let mut found = Vec::new(); + let mut rest = sql.chars().peekable(); + while let Some(first) = rest.next() { + if first == '\'' || first == '"' { + let mut text = String::from(first); + loop { + let next = rest.next().expect("a quoted text ends with its quote"); + text.push(next); + if next == first && rest.next_if_eq(&first).is_none() { + break; + } + } + found.push(text); + } else if first.is_ascii_alphanumeric() || first == '_' { + let mut word = String::from(first.to_ascii_lowercase()); + while let Some(next) = rest.next_if(|c| c.is_ascii_alphanumeric() || *c == '_') { + word.push(next.to_ascii_lowercase()); + } + found.push(word); + } + } + found +} + +/// Whether a printed declaration says this clause: the clause's words, one after the other. +fn says(declaration: &str, clause: &str) -> bool { + let (text, clause) = (words(declaration), words(clause)); + text.windows(clause.len()).any(|window| window == clause) +} + +/// The statements of a printed declaration. +fn statements(text: &str) -> Vec<&str> { + let ended = text.split(";\n"); + ended + .filter(|statement| !statement.trim().is_empty()) + .collect() +} + +#[test] +fn an_added_column_reads_its_default_from_a_row_written_before_it() { + let folder = support::folder("added-column-default"); + let db = open_at(&folder, 1_700_000_000); + support::run( + &db, + "CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT NOT NULL)", + ); + support::run( + &db, + "INSERT INTO t (id, name) VALUES (1, 'a'), (2, 'b'), (3, 'c')", + ); + for column in [ + "plain TEXT", + "n INTEGER DEFAULT 7", + "word TEXT NOT NULL DEFAULT 'none'", + "ratio REAL DEFAULT 0.5", + "ok BOOLEAN DEFAULT TRUE", + "doc JSON DEFAULT '[]'", + "made TIMESTAMP DEFAULT NOW()", + ] { + support::run(&db, &format!("ALTER TABLE t ADD COLUMN {column}")); + } + // What a row written before the columns reads in them: nothing where the column declares + // no default, the default where it declares one, and for NOW() the time of the ALTER, the + // same for every such row. + let older = |id: i64, name: &str| { + vec![ + Value::Int64(id), + support::text(name), + Value::Null, + Value::Int64(7), + support::text("none"), + Value::Float64(0.5), + Value::Bool(true), + Value::Json(json!([])), + Value::Timestamp(1_700_000_000_000), + ] + }; + let all = "SELECT id, name, plain, n, word, ratio, ok, doc, made FROM t ORDER BY id"; + assert_eq!( + support::rows(&db, all), + [older(1, "a"), older(2, "b"), older(3, "c")] + ); + assert_eq!( + support::rows(&db, "SELECT * FROM t ORDER BY id"), + support::rows(&db, all) + ); + + // A filter, a sort and a new index read the same defaults. + let ids = |sql: &str| -> Vec { + let rows = support::rows(&db, sql); + rows.iter() + .map(|row| match row[0] { + Value::Int64(id) => id, + ref other => panic!("{sql}\nreturned {other:?} for an id"), + }) + .collect() + }; + assert_eq!( + ids("SELECT id FROM t WHERE n = 7 AND word = 'none' ORDER BY id"), + [1, 2, 3] + ); + assert_eq!( + ids("SELECT id FROM t WHERE plain IS NULL ORDER BY n, id"), + [1, 2, 3] + ); + support::run(&db, "CREATE INDEX t_by_n ON t (n, id)"); + assert_eq!(ids("SELECT id FROM t WHERE n = 7 ORDER BY id"), [1, 2, 3]); + assert_eq!(ids("SELECT id FROM t WHERE n = 8 ORDER BY id"), [0_i64; 0]); + + // A statement that changes another column of an older row leaves what the row reads in + // the added ones; one that assigns an added column changes that column alone. + support::run(&db, "UPDATE t SET name = 'renamed' WHERE id = 1"); + support::run(&db, "UPDATE t SET n = 8 WHERE id = 2"); + let mut second = older(2, "b"); + second[3] = Value::Int64(8); + assert_eq!( + support::rows(&db, all), + [older(1, "renamed"), second.clone(), older(3, "c")] + ); + assert_eq!(ids("SELECT id FROM t WHERE n = 7 ORDER BY id"), [1, 3]); + assert_eq!(ids("SELECT id FROM t WHERE n = 8 ORDER BY id"), [2]); + support::close(db); + + // Under a later clock the older rows read the time they read before, a row written now + // takes its defaults as any row does, and a NULL written is a NULL. + let db = open_at(&folder, 1_800_000_000); + support::run(&db, "INSERT INTO t (id, name) VALUES (4, 'd')"); + support::run( + &db, + "INSERT INTO t (id, name, n, ok) VALUES (5, 'e', NULL, NULL)", + ); + let mut fourth = older(4, "d"); + fourth[8] = Value::Timestamp(1_800_000_000_000); + let mut fifth = fourth.clone(); + (fifth[0], fifth[1]) = (Value::Int64(5), support::text("e")); + (fifth[3], fifth[6]) = (Value::Null, Value::Null); + assert_eq!( + support::rows(&db, all), + [older(1, "renamed"), second, older(3, "c"), fourth, fifth] + ); +} + +#[test] +fn adding_a_column_writes_no_row_again() { + // 2,000 rows of 500 letters: a million bytes of rows. + const ROWS: i64 = 2_000; + const LETTERS: usize = 500; + let folder = support::folder("added-column-writes"); + let (written, counting) = ( + Arc::new(AtomicU64::new(0)), + Arc::new(AtomicBool::new(false)), + ); + let mut options = DatabaseOpenOptions::default(); + let (bytes, on) = (written.clone(), counting.clone()); + options.storage = Some(support::storage(move |call| { + if let (Call::Write(_, data), true) = (call, on.load(Ordering::SeqCst)) { + bytes.fetch_add(data.len() as u64, Ordering::SeqCst); + } + })); + let db = support::open_with(&folder, options).expect("open the store through a door"); + support::run( + &db, + "CREATE TABLE t (id INTEGER PRIMARY KEY, body TEXT NOT NULL)", + ); + for first in (1..=ROWS).step_by(500) { + let tx = db.transaction().expect("begin"); + for id in first..first + 500 { + let body = support::letters(id, LETTERS); + tx.execute( + "INSERT INTO t (id, body) VALUES ($id, $body)", + ¶ms! { "id" => id, "body" => body }, + ) + .unwrap_or_else(|error| panic!("row {id} was refused: {error:?}")); + } + tx.commit().expect("commit five hundred rows"); + } + + counting.store(true, Ordering::SeqCst); + support::run( + &db, + "ALTER TABLE t ADD COLUMN tag TEXT NOT NULL DEFAULT 'new'", + ); + counting.store(false, Ordering::SeqCst); + let written = written.load(Ordering::SeqCst); + let rows = ROWS as u64 * LETTERS as u64; + assert!( + written > 0 && written < rows / 4, + "adding a column wrote {written} bytes to the data file of a table whose rows hold \ + {rows} bytes" + ); + assert_eq!( + support::int(&db, "SELECT COUNT(*) FROM t WHERE tag = 'new'"), + ROWS + ); + assert_eq!( + support::rows(&db, "SELECT body, tag FROM t WHERE id = 1234"), + [[ + support::text(&support::letters(1234, LETTERS)), + support::text("new") + ]] + ); +} + +#[test] +fn an_added_vector_column_is_null_on_older_rows_and_no_search_returns_them() { + let folder = support::folder("added-vector-column"); + let db = support::open(&folder); + support::run( + &db, + "CREATE TABLE docs (id INTEGER PRIMARY KEY, ctx TEXT NOT NULL, body TEXT)", + ); + support::run( + &db, + "INSERT INTO docs (id, ctx, body) VALUES (1, 'a', 'x'), (2, 'a', 'x'), (3, 'a', 'x'), \ + (4, 'b', 'x'), (5, 'b', 'x'), (6, 'b', 'x')", + ); + let column = "v VECTOR(4) PARTITION_KEY (ctx) MAX_PARTITIONS 4 SEARCH_MODE EXACT"; + support::run(&db, &format!("ALTER TABLE docs ADD COLUMN {column}")); + assert!( + says(&printed(&db, "docs"), column), + "{}", + printed(&db, "docs") + ); + + let vectors = |db: &Database| support::rows(db, "SELECT id, v FROM docs ORDER BY id"); + let none: Vec> = (1..=6) + .map(|id| vec![Value::Int64(id), Value::Null]) + .collect(); + assert_eq!(vectors(&db), none); + // No search returns a row that has no vector, with a filter or without. + let query = [1.0_f32, 0.0, 0.0, 0.0]; + let found = |db: &Database, filter: &str| -> Vec { + let sql = format!("SELECT id FROM docs {filter} ORDER BY v <=> $q LIMIT 10"); + let result = support::search(db, &sql, &query); + let result = result.unwrap_or_else(|error| panic!("{sql}\nwas refused: {error:?}")); + result.rows().iter().map(|row| row[0].clone()).collect() + }; + assert!(found(&db, "").is_empty()); + assert!(found(&db, "WHERE ctx = 'a'").is_empty()); + + // Rows given a vector, and a row written with one, are found; the others are not. + for (id, vector) in [ + (2_i64, vec![1.0_f32, 0.0, 0.0, 0.0]), + (5, vec![0.8, 0.6, 0.0, 0.0]), + ] { + db.execute( + "UPDATE docs SET v = $v WHERE id = $id", + ¶ms! { "id" => id, "v" => vector }, + ) + .expect("give an older row a vector"); + } + db.execute( + "INSERT INTO docs (id, ctx, body, v) VALUES (7, 'a', 'y', $v)", + ¶ms! { "v" => vec![0.0_f32, 1.0, 0.0, 0.0] }, + ) + .expect("a row with a vector"); + let ids = |ids: &[i64]| -> Vec { ids.iter().map(|id| Value::Int64(*id)).collect() }; + assert_eq!(found(&db, ""), ids(&[2, 5, 7])); + assert_eq!(found(&db, "WHERE ctx = 'a'"), ids(&[2, 7])); + assert_eq!(found(&db, "WHERE ctx = 'b'"), ids(&[5])); + + // The same once the vectors are in blocks, and after close and open. + assert_eq!(support::seal_all(&db), 3); + assert_eq!(found(&db, ""), ids(&[2, 5, 7])); + support::close(db); + let db = support::open(&folder); + assert_eq!(found(&db, ""), ids(&[2, 5, 7])); + assert_eq!(found(&db, "WHERE ctx = 'b'"), ids(&[5])); + let with_vector = vectors(&db) + .iter() + .filter(|row| row[1] != Value::Null) + .count(); + assert_eq!(with_vector, 3); +} + +/// How many rows the stores of the two tests below hold, and how many letters each row's +/// large value has: 3,200,000 bytes that do not shrink when stored. +const ROWS: i64 = 400; +const LETTERS: usize = 8_000; +const PUT_IN: u64 = ROWS as u64 * LETTERS as u64; + +/// Writes a large text into `column` of rows 1 to 400 of `table`, a hundred rows to a commit. +/// The rows are inserted when `insert` says so, and updated otherwise. The text of a row is a +/// function of its number and `seed`. +fn fill(db: &Database, table: &str, column: &str, insert: bool, seed: i64) { + let sql = if insert { + format!("INSERT INTO {table} (id, {column}) VALUES ($id, $text)") + } else { + format!("UPDATE {table} SET {column} = $text WHERE id = $id") + }; + for first in (1..=ROWS).step_by(100) { + let tx = db.transaction().expect("begin"); + for id in first..first + 100 { + let text = support::letters(seed + id, LETTERS); + tx.execute(&sql, ¶ms! { "id" => id, "text" => text }) + .unwrap_or_else(|error| panic!("{sql}\nwas refused for row {id}: {error:?}")); + } + tx.commit().expect("commit a hundred rows"); + } +} + +#[test] +fn a_dropped_columns_bytes_leave_the_store() { + // Two stores take the same writes: a large column filled, then a second large column + // added and filled. One of them drops the first column in between. + let store = |test: &str, drop: bool| -> u64 { + let folder = support::folder(test); + let db = support::open(&folder); + support::run( + &db, + "CREATE TABLE notes (id INTEGER PRIMARY KEY, body TEXT)", + ); + fill(&db, "notes", "body", true, 0); + if drop { + support::run(&db, "ALTER TABLE notes DROP COLUMN body"); + assert!(matches!( + support::refusal(&db, "SELECT body FROM notes"), + Error::ColumnNotFound { .. } + )); + } + support::run(&db, "ALTER TABLE notes ADD COLUMN other TEXT"); + fill(&db, "notes", "other", false, ROWS); + assert_eq!(support::int(&db, "SELECT COUNT(*) FROM notes"), ROWS); + assert_eq!( + support::rows(&db, "SELECT other FROM notes WHERE id = 77"), + [[support::text(&support::letters(ROWS + 77, LETTERS))]] + ); + support::close(db); + data_file(&folder) + }; + let kept = store("dropped-column-kept", false); + let dropped = store("dropped-column-dropped", true); + // The store that kept the column holds both columns' bytes. The one that dropped it has + // let the first column's bytes go, so the second column's took their room. + assert!( + kept >= 2 * PUT_IN, + "the store that dropped nothing is {kept} bytes long" + ); + assert!( + dropped + PUT_IN / 2 <= kept, + "after DROP COLUMN of {PUT_IN} bytes the data file is {dropped} bytes long, and the \ + same store with the column kept is {kept}: the dropped column's bytes are still held" + ); +} + +#[test] +fn a_dropped_table_leaves_no_record() { + // As above, with a table dropped where a column was: one table filled, then a second one. + let store = |test: &str, drop: bool| -> (PathBuf, u64) { + let folder = support::folder(test); + let db = support::open(&folder); + let declare = |table: &str| { + let columns = "id INTEGER PRIMARY KEY, body TEXT, tag TEXT, v VECTOR(2)"; + support::run(&db, &format!("CREATE TABLE {table} ({columns})")); + support::run(&db, &format!("CREATE INDEX by_tag ON {table} (tag, id)")); + }; + declare("one"); + fill(&db, "one", "body", true, 0); + support::run( + &db, + "UPDATE one SET tag = 'old', v = [1.0, 0.0] WHERE id <= 50", + ); + if drop { + let dropped = support::run(&db, "DROP TABLE one"); + assert_eq!(dropped.rows_affected(), 0); + assert!(db.table_names().is_empty()); + } + declare("two"); + fill(&db, "two", "body", true, ROWS); + support::close(db); + let length = data_file(&folder); + (folder, length) + }; + let (_, kept) = store("dropped-table-kept", false); + let (folder, dropped) = store("dropped-table-dropped", true); + assert!( + kept >= 2 * PUT_IN, + "the store that dropped nothing is {kept} bytes long" + ); + assert!( + dropped + PUT_IN / 2 <= kept, + "after DROP TABLE of {PUT_IN} bytes the data file is {dropped} bytes long, and the \ + same store with the table kept is {kept}: the dropped table's bytes are still held" + ); + + // Nothing answers for the dropped table: not the writer, not a reader, not after close + // and open; and a table declared under its name starts empty, with none of its indexes. + let db = support::open(&folder); + let reader = ReadSession::open(&folder).expect("open the store for reading"); + assert_eq!(db.table_names(), ["two"]); + assert_eq!(reader.table_names().expect("a reader's tables"), ["two"]); + let missing = |error: Error, what: &str| { + assert!( + matches!(&error, Error::TableNotFound(name) if name == "one"), + "{what} gave {error:?}" + ); + }; + missing( + db.declaration("one").expect_err("a declaration"), + "the declaration", + ); + missing( + reader.declaration("one").expect_err("a declaration"), + "a reader's declaration", + ); + for sql in [ + "SELECT id FROM one", + "INSERT INTO one (id) VALUES (1)", + "UPDATE one SET tag = 'x'", + "DELETE FROM one", + "DROP TABLE one", + "DROP INDEX by_tag ON one", + "ALTER TABLE one ADD COLUMN c TEXT", + "CREATE INDEX again ON one (tag)", + "SELECT id FROM one ORDER BY v <=> [1.0, 0.0] LIMIT 1", + ] { + missing(support::refusal(&db, sql), sql); + } + support::run( + &db, + "CREATE TABLE one (id INTEGER PRIMARY KEY, tag TEXT, v VECTOR(2))", + ); + assert_eq!(support::int(&db, "SELECT COUNT(*) FROM one"), 0); + assert!(support::rows(&db, "SELECT id FROM one WHERE tag = 'old'").is_empty()); + assert!(support::rows(&db, "SELECT id FROM one ORDER BY v <=> [1.0, 0.0] LIMIT 5").is_empty()); + support::run( + &db, + "INSERT INTO one (id, tag, v) VALUES (1, 'new', [0.0, 1.0])", + ); + support::run(&db, "CREATE INDEX by_tag ON one (tag)"); + assert_eq!( + support::rows(&db, "SELECT id FROM one ORDER BY v <=> [1.0, 0.0] LIMIT 5"), + [[Value::Int64(1)]] + ); + // The other table was not touched. + assert_eq!(support::int(&db, "SELECT COUNT(*) FROM two"), ROWS); +} + +/// A store with a table for each thing a drop or an ALTER may depend on, and rows in them. +fn store_with_dependents(test: &str) -> Database { + let db = support::open(&support::folder(test)); + for sql in [ + "CREATE TABLE parents (id INTEGER PRIMARY KEY, name TEXT)", + "CREATE TABLE t (id INTEGER PRIMARY KEY, a TEXT, b INTEGER, kind TEXT NOT NULL IMMUTABLE, \ + tenant TEXT NOT NULL, parent_id INTEGER REFERENCES parents(id), \ + v VECTOR(2) PARTITION_KEY (tenant) MAX_PARTITIONS 4)", + "CREATE INDEX t_first ON t (b, a)", + "CREATE INDEX t_second ON t (a)", + "CREATE TABLE edges (id INTEGER PRIMARY KEY, source_id UUID NOT NULL, \ + target_id UUID NOT NULL, edge_type TEXT NOT NULL) DAG('NEEDS')", + "CREATE TABLE task_results (id UUID PRIMARY KEY, task_id UUID NOT NULL, \ + success BOOLEAN NOT NULL)", + "CREATE INDEX task_results_task_id_idx ON task_results (task_id)", + "CREATE TABLE ranked (id UUID PRIMARY KEY, confidence REAL, \ + embedding VECTOR(4) RANK_POLICY (JOIN task_results ON task_id, \ + FORMULA 'coalesce({confidence}, 1.0) * coalesce({success}, 1.0)', \ + SORT_KEY effective_confidence))", + "INSERT INTO parents (id, name) VALUES (1, 'p')", + "INSERT INTO t (id, a, b, kind, tenant, parent_id, v) VALUES \ + (1, 'x', 10, 'k', 'north', 1, [1.0, 0.0]), (2, 'y', 20, 'k', 'south', 1, [0.0, 1.0])", + ] { + support::run(&db, sql); + } + db +} + +/// What [`state`] reads: the declarations, the rows of `t` read two ways, and the newest commit. +type State = ( + Vec, + Vec>, + Vec>, + contextdb::TxId, +); + +/// Everything a refused statement must leave as it was: each table's declaration, the rows, +/// and the number of the newest commit. +fn state(db: &Database) -> State { + let names = db.table_names(); + let declared = names.iter().map(|table| printed(db, table)).collect(); + let rows = support::rows( + db, + "SELECT id, a, b, kind, tenant, parent_id, v FROM t ORDER BY id", + ); + let parents = support::rows(db, "SELECT id, name FROM parents ORDER BY id"); + (declared, rows, parents, db.committed_watermark()) +} + +/// What a refused ALTER or DROP must come as. +enum Refused { + /// `ColumnInIndex` for this column of `t` and this index. + InIndex(&'static str, &'static str), + /// `VectorPartitionKeyInUse` for the column `tenant` of `t` and the VECTOR column `v`. + InPartitionKey, + /// `ImmutableColumn` for the column `kind` of `t`. + Immutable, + /// `SchemaInvalid`. + Invalid, + /// `DropBlockedByRankPolicy` for this table and the policy of `ranked`, and for this + /// column where the statement names one. + Ranked(&'static str, Option<&'static str>), + /// `TableNotFound` for the table `nope`. + NoTable, +} + +impl Refused { + fn is(&self, error: &Error) -> bool { + match (self, error) { + ( + Refused::InIndex(of, by), + Error::ColumnInIndex { + table, + column, + index, + }, + ) => table == "t" && column == of && index == by, + ( + Refused::InPartitionKey, + Error::VectorPartitionKeyInUse { + table, + column, + index, + }, + ) => table == "t" && column == "tenant" && index == "v", + (Refused::Immutable, Error::ImmutableColumn { table, column }) => { + table == "t" && column == "kind" + } + (Refused::Invalid, Error::SchemaInvalid { reason }) => !reason.is_empty(), + ( + Refused::Ranked(of, named), + Error::DropBlockedByRankPolicy { + table, + column, + policy, + }, + ) => { + table == of + && policy == "effective_confidence" + && named.is_none_or(|named| column == named) + } + (Refused::NoTable, Error::TableNotFound(name)) => name == "nope", + _ => false, + } + } +} + +#[test] +fn each_dependent_refusal_names_what_depends_and_changes_nothing() { + let db = store_with_dependents("dependent-refusals"); + // A table that keeps its rows to itself: SYNC SAFE cannot stand on it. + support::run(&db, "ALTER TABLE t SET SYNC OFF"); + let before = state(&db); + let cases = [ + // The first index, in the order they were made, that names the column. + ( + "ALTER TABLE t DROP COLUMN a", + Refused::InIndex("a", "t_first"), + ), + ( + "ALTER TABLE t DROP COLUMN a RESTRICT", + Refused::InIndex("a", "t_first"), + ), + ("ALTER TABLE t DROP b", Refused::InIndex("b", "t_first")), + ("ALTER TABLE t DROP COLUMN tenant", Refused::InPartitionKey), + ( + "ALTER TABLE t DROP COLUMN tenant CASCADE", + Refused::InPartitionKey, + ), + ( + "ALTER TABLE t RENAME COLUMN tenant TO owner", + Refused::InPartitionKey, + ), + ("ALTER TABLE t DROP COLUMN kind", Refused::Immutable), + ("ALTER TABLE t DROP COLUMN kind CASCADE", Refused::Immutable), + ( + "ALTER TABLE t RENAME COLUMN kind TO sort", + Refused::Immutable, + ), + // A column of the key, a table another references, and a column that makes a DAG + // table a table of edges. + ("ALTER TABLE t DROP COLUMN id", Refused::Invalid), + ("ALTER TABLE t RENAME COLUMN id TO key", Refused::Invalid), + ( + "ALTER TABLE parents DROP COLUMN id CASCADE", + Refused::Invalid, + ), + ("DROP TABLE parents", Refused::Invalid), + ("DROP TABLE IF EXISTS parents", Refused::Invalid), + ("ALTER TABLE edges DROP COLUMN source_id", Refused::Invalid), + ( + "ALTER TABLE edges DROP COLUMN target_id CASCADE", + Refused::Invalid, + ), + ( + "ALTER TABLE edges RENAME COLUMN edge_type TO kind", + Refused::Invalid, + ), + // What the rank policy of `ranked` reads. + ( + "DROP TABLE task_results", + Refused::Ranked("task_results", None), + ), + ( + "DROP INDEX task_results_task_id_idx ON task_results", + Refused::Ranked("task_results", None), + ), + ( + "ALTER TABLE task_results DROP COLUMN success", + Refused::Ranked("task_results", Some("success")), + ), + ( + "ALTER TABLE task_results DROP COLUMN task_id CASCADE", + Refused::Ranked("task_results", Some("task_id")), + ), + ( + "ALTER TABLE ranked DROP COLUMN confidence", + Refused::Ranked("ranked", Some("confidence")), + ), + // A column that may hold no NULL, with no default, on a table that has rows; a name + // that is taken; a second key. + ( + "ALTER TABLE t ADD COLUMN must TEXT NOT NULL", + Refused::Invalid, + ), + ("ALTER TABLE t ADD COLUMN a INTEGER", Refused::Invalid), + ( + "ALTER TABLE t ADD COLUMN k INTEGER PRIMARY KEY", + Refused::Invalid, + ), + ("ALTER TABLE t RENAME COLUMN a TO b", Refused::Invalid), + // Options that cannot stand. + ( + "ALTER TABLE t SET RETAIN 7 DAYS SYNC SAFE", + Refused::Invalid, + ), + ("ALTER TABLE t SET MAINTAIN COUNT (kind)", Refused::Invalid), + ("ALTER TABLE t SET MAINTAIN COUNT (id)", Refused::Invalid), + // What is not there. + ("DROP TABLE nope", Refused::NoTable), + ("ALTER TABLE nope ADD COLUMN c TEXT", Refused::NoTable), + ("ALTER TABLE nope DROP COLUMN c", Refused::NoTable), + ("DROP INDEX t_first ON nope", Refused::NoTable), + ]; + for (sql, refused) in &cases { + let error = support::refusal(&db, sql); + assert!(refused.is(&error), "{sql}\nwas refused as {error:?}"); + assert!( + state(&db) == before, + "{sql}\nwas refused and changed the store" + ); + } + // Keeping only current versions is refused on a table that sends its rows out and keeps + // the first write. + support::run(&db, "ALTER TABLE t SET SYNC TWO WAY"); + let before = state(&db); + let error = support::refusal(&db, "ALTER TABLE t SET HISTORY CURRENT ONLY"); + assert!(Refused::Invalid.is(&error), "{error:?}"); + assert!(state(&db) == before); + + // An index and a column that are not there, by name. + match support::refusal(&db, "DROP INDEX nope ON t") { + Error::IndexNotFound { table, index } => { + assert_eq!((table.as_str(), index.as_str()), ("t", "nope")); + } + other => panic!("dropping an index that is not there was refused as {other:?}"), + } + for sql in [ + "ALTER TABLE t DROP COLUMN nope", + "ALTER TABLE t RENAME COLUMN nope TO c", + ] { + match support::refusal(&db, sql) { + Error::ColumnNotFound { table, column } => { + assert_eq!((table.as_str(), column.as_str()), ("t", "nope")); + } + other => panic!("{sql}\nwas refused as {other:?}"), + } + } + // A limit on partitions below the partitions the column has. No document says which of + // the two kinds this is. + let error = support::refusal(&db, "ALTER TABLE t ALTER COLUMN v SET MAX_PARTITIONS 1"); + assert!( + matches!( + &error, + Error::VectorPartitionLimitExceeded { index, .. } + | Error::InvalidVectorPartitionDeclaration { index, .. } + if index.table == "t" && index.column == "v" + ), + "a limit of one partition on a column that has two was refused as {error:?}" + ); + assert!(state(&db) == before); + // With IF EXISTS, a table or an index that is not there is nothing to do, and changes + // no declaration and no row. + for sql in [ + "DROP TABLE IF EXISTS nope", + "DROP INDEX IF EXISTS nope ON t", + ] { + let result = support::run(&db, sql); + assert_eq!(result.rows_affected(), 0, "{sql}"); + assert!(result.rows().is_empty(), "{sql}"); + } + let after = state(&db); + assert!(after.0 == before.0 && after.1 == before.1 && after.2 == before.2); +} + +#[test] +fn an_altered_table_is_printed_as_what_it_now_declares() { + let folder = support::folder("altered-printed"); + let db = support::open(&folder); + for sql in [ + "CREATE TABLE parents (id INTEGER PRIMARY KEY, code TEXT UNIQUE)", + "CREATE TABLE t (id INTEGER PRIMARY KEY, a TEXT, tenant TEXT NOT NULL, \ + parent TEXT REFERENCES parents(code), \ + v VECTOR(2) PARTITION_KEY (tenant) MAX_PARTITIONS 2 SEARCH_MODE AUTO, \ + UNIQUE (a, tenant))", + "CREATE INDEX t_by_a ON t (a DESC, id)", + "INSERT INTO parents (id, code) VALUES (1, 'p')", + "INSERT INTO t (id, a, tenant, parent, v) VALUES (1, 'x', 'north', 'p', [1.0, 0.0])", + ] { + support::run(&db, sql); + } + // Each ALTER, what the table's declaration says after it, and what it no longer says. + let steps: &[(&str, &[&str], &[&str])] = &[ + ( + "ALTER TABLE t ADD COLUMN extra INTEGER NOT NULL DEFAULT 5", + &["extra INTEGER NOT NULL DEFAULT 5"], + &[], + ), + ( + "ALTER TABLE t RENAME COLUMN a TO title", + &[ + "title TEXT", + "UNIQUE (title, tenant)", + "ON t (title DESC, id)", + ], + &["a TEXT", "UNIQUE (a", "ON t (a"], + ), + ( + "ALTER TABLE parents RENAME COLUMN code TO label", + &["parent TEXT REFERENCES parents (label)"], + &["parents (code)"], + ), + ( + "ALTER TABLE t ALTER COLUMN v SET SEARCH_MODE EXACT", + &["PARTITION_KEY (tenant) MAX_PARTITIONS 2 SEARCH_MODE EXACT"], + &["SEARCH_MODE AUTO"], + ), + ( + "ALTER TABLE t ALTER COLUMN v SET MAX_PARTITIONS 9", + &["PARTITION_KEY (tenant) MAX_PARTITIONS 9 SEARCH_MODE EXACT"], + &["MAX_PARTITIONS 2"], + ), + ( + "ALTER TABLE t SET RETAIN 24 HOURS SYNC SAFE", + &["RETAIN 24 HOURS SYNC SAFE"], + &[], + ), + ( + "ALTER TABLE t SET RETAIN 7 DAYS", + &["RETAIN 7 DAYS"], + &["HOURS", "SYNC SAFE"], + ), + ("ALTER TABLE t DROP RETAIN", &[], &["RETAIN"]), + ("ALTER TABLE t SET SYNC PUSH ONLY", &["SYNC PUSH ONLY"], &[]), + ( + "ALTER TABLE t SET SYNC PULL ONLY", + &["SYNC PULL ONLY"], + &["PUSH ONLY"], + ), + ("ALTER TABLE t SET SYNC OFF", &["SYNC OFF"], &["PULL ONLY"]), + ( + "ALTER TABLE t SET SYNC TWO WAY", + &["SYNC TWO WAY"], + &["SYNC OFF"], + ), + ( + "ALTER TABLE t SET SYNC CONFLICT KEEP LATEST", + &["SYNC CONFLICT KEEP LATEST", "SYNC TWO WAY"], + &[], + ), + ( + "ALTER TABLE t SET HISTORY CURRENT ONLY", + &["HISTORY CURRENT ONLY"], + &[], + ), + ( + "ALTER TABLE t SET HISTORY ALL", + &["SYNC CONFLICT KEEP LATEST"], + &["CURRENT ONLY"], + ), + ( + "ALTER TABLE t SET SYNC CONFLICT KEEP FIRST", + &["SYNC CONFLICT KEEP FIRST"], + &["KEEP LATEST"], + ), + ( + "ALTER TABLE t SET MAINTAIN COUNT (title)", + &["MAINTAIN COUNT (title)"], + &[], + ), + ( + "ALTER TABLE t SET MAINTAIN COUNT ALL PREFIXES", + &["MAINTAIN COUNT ALL PREFIXES"], + &["MAINTAIN COUNT (title)"], + ), + ( + "ALTER TABLE t SET MAINTAIN COUNT NONE", + &["MAINTAIN COUNT NONE"], + &["ALL PREFIXES"], + ), + ( + "ALTER TABLE t SET MAINTAIN COUNT DEFAULT", + &["SYNC TWO WAY"], + &["MAINTAIN COUNT"], + ), + ( + "DROP INDEX t_by_a ON t", + &["UNIQUE (title, tenant)"], + &["t_by_a"], + ), + ( + "CREATE INDEX t_by_extra ON t (extra, id)", + &["t_by_extra"], + &[], + ), + ( + "ALTER TABLE t DROP COLUMN extra CASCADE", + &["title TEXT"], + &["extra", "t_by_extra"], + ), + ( + "ALTER TABLE t ADD COLUMN w VECTOR(3) SEARCH_MODE INDEXED", + &[ + "w VECTOR(3) SEARCH_MODE INDEXED", + "v VECTOR(2) PARTITION_KEY (tenant)", + ], + &[], + ), + ( + "ALTER TABLE t DROP COLUMN v", + &["w VECTOR(3)"], + &["VECTOR(2)", "PARTITION_KEY"], + ), + ]; + assert_eq!(steps.len(), 25); + let mut db = db; + for (number, (sql, now, no_longer)) in steps.iter().enumerate() { + support::run(&db, sql); + let text = printed(&db, "t"); + for clause in *now { + assert!( + says(&text, clause), + "after {sql}\nno `{clause}` in:\n{text}" + ); + } + for clause in *no_longer { + assert!( + !says(&text, clause), + "after {sql}\nstill `{clause}` in:\n{text}" + ); + } + // The printed texts declare the same two tables in an empty store. + let again = support::open(&support::folder(&format!("altered-printed-again-{number}"))); + for table in ["parents", "t"] { + let declared = printed(&db, table); + for statement in statements(&declared) { + support::run(&again, statement); + } + } + assert_eq!(printed(&again, "t"), text, "after {sql}, declared again"); + assert_eq!( + printed(&again, "parents"), + printed(&db, "parents"), + "after {sql}" + ); + // And the store says the same after close and open. + support::close(db); + db = support::open(&folder); + assert_eq!(printed(&db, "t"), text, "after {sql}, closed and opened"); + } + // The rows came through every one of them. + assert_eq!( + support::rows(&db, "SELECT id, title, tenant, parent FROM t"), + [[ + Value::Int64(1), + support::text("x"), + support::text("north"), + support::text("p") + ]] + ); +} + +#[test] +fn a_renamed_vector_column_keeps_its_declaration_and_its_vectors() { + let folder = support::folder("renamed-vector-column"); + let db = support::open(&folder); + support::run( + &db, + "CREATE TABLE docs (id INTEGER PRIMARY KEY, ctx TEXT NOT NULL, \ + v VECTOR(4) PARTITION_KEY (ctx) MAX_PARTITIONS 3 SEARCH_MODE EXACT)", + ); + let insert_under = |db: &Database, column: &str, id: i64, ctx: &str| { + db.execute( + &format!("INSERT INTO docs (id, ctx, {column}) VALUES ($id, $ctx, $v)"), + ¶ms! { "id" => id, "ctx" => ctx, "v" => support::vector(id, 4) }, + ) + }; + let insert = |db: &Database, id: i64, ctx: &str| insert_under(db, "v", id, ctx); + // Twenty vectors in blocks and five still waiting, in two partitions. + for id in 1..=20 { + insert(&db, id, if id % 2 == 0 { "a" } else { "b" }).expect("a row with a vector"); + } + assert_eq!(support::seal_all(&db), 20); + for id in 21..=25 { + insert(&db, id, "a").expect("a row with a vector"); + } + let query = support::vector(1000, 4); + let nearest = |db: &Database, column: &str, filter: &str| { + let sql = format!( + "SELECT id, {column} <=> $q AS distance FROM docs {filter} ORDER BY distance LIMIT 10" + ); + support::search(db, &sql, &query).map(|result| result.rows().to_vec()) + }; + let whole = nearest(&db, "v", "").expect("a search"); + let of_a = nearest(&db, "v", "WHERE ctx = 'a'").expect("a search"); + assert_eq!((whole.len(), of_a.len()), (10, 10)); + + support::run(&db, "ALTER TABLE docs RENAME COLUMN v TO embedding"); + // The same answers under the new name, from the blocks and from the waiting vectors, and + // none under the old one. + assert_eq!(nearest(&db, "embedding", "").expect("a search"), whole); + assert_eq!( + nearest(&db, "embedding", "WHERE ctx = 'a'").expect("a search"), + of_a + ); + assert!(nearest(&db, "v", "").is_err()); + let text = printed(&db, "docs"); + let clauses = "embedding VECTOR(4) PARTITION_KEY (ctx) MAX_PARTITIONS 3 SEARCH_MODE EXACT"; + assert!( + says(&text, clauses) && !says(&text, "v VECTOR(4)"), + "{text}" + ); + // The waiting vectors are the same five: the rename sealed nothing and lost nothing. + assert_eq!(support::seal_all(&db), 5); + assert_eq!(nearest(&db, "embedding", "").expect("a search"), whole); + // The limit on partitions still counts the two the column had. + insert_under(&db, "embedding", 26, "c").expect("a third partition"); + match insert_under(&db, "embedding", 27, "d") { + Err(Error::VectorPartitionLimitExceeded { + index, + max_partitions: 3, + }) => { + assert_eq!( + (index.table.as_str(), index.column.as_str()), + ("docs", "embedding") + ); + } + other => panic!("a fourth partition gave {other:?}"), + } + support::close(db); + let db = support::open(&folder); + assert_eq!( + nearest(&db, "embedding", "WHERE ctx = 'a'").expect("a search"), + of_a + ); +} + +#[test] +fn no_statement_drops_history_and_set_history_all_is_the_way_back() { + let db = support::open(&support::folder("no-drop-history")); + support::run( + &db, + "CREATE TABLE gauges (id TEXT PRIMARY KEY, state TEXT NOT NULL) \ + HISTORY CURRENT ONLY SYNC CONFLICT KEEP LATEST", + ); + let before = printed(&db, "gauges"); + assert!(says(&before, "HISTORY CURRENT ONLY"), "{before}"); + for sql in [ + "ALTER TABLE gauges DROP HISTORY", + "ALTER TABLE gauges SET HISTORY NONE", + ] { + if let Ok(result) = db.execute(sql, ¶ms! {}) { + panic!("{sql}\nwas accepted: {result:?}"); + } + assert_eq!(printed(&db, "gauges"), before, "after {sql}"); + } + support::run(&db, "ALTER TABLE gauges SET HISTORY ALL"); + assert!(!says(&printed(&db, "gauges"), "CURRENT ONLY")); + // What HISTORY CURRENT ONLY needed may go once every version is kept. + support::run(&db, "ALTER TABLE gauges SET SYNC CONFLICT KEEP FIRST"); +} + +#[test] +fn an_alter_or_a_drop_in_a_transaction_takes_hold_when_it_commits() { + let db = support::open(&support::folder("alter-in-transaction")); + support::run(&db, "CREATE TABLE t (id INTEGER PRIMARY KEY, a TEXT)"); + support::run(&db, "CREATE TABLE gone (id INTEGER PRIMARY KEY)"); + support::run(&db, "INSERT INTO t (id, a) VALUES (1, 'x')"); + let before = printed(&db, "t"); + + // Rolled back, it leaves the table as it was. + let tx = db.transaction().expect("begin"); + support::run(&tx, "ALTER TABLE t ADD COLUMN c TEXT DEFAULT 'new'"); + support::run(&tx, "DROP TABLE gone"); + tx.rollback(); + assert_eq!(printed(&db, "t"), before); + assert_eq!(db.table_names(), ["gone", "t"]); + assert!(matches!( + support::refusal(&db, "SELECT c FROM t"), + Error::ColumnNotFound { .. } + )); + + // A transaction either declares or writes rows, with ALTER and DROP as with CREATE. + let tx = db.transaction().expect("begin"); + support::run(&tx, "ALTER TABLE t ADD COLUMN c TEXT DEFAULT 'new'"); + assert!(matches!( + support::refusal(&tx, "INSERT INTO t (id, a) VALUES (2, 'y')"), + Error::MixedSchemaAndRowWrite + )); + support::run(&tx, "DROP TABLE gone"); + // Until it commits, nobody else sees either. + assert_eq!(db.table_names(), ["gone", "t"]); + assert_eq!(printed(&db, "t"), before); + tx.commit().expect("commit the two declarations"); + assert_eq!(db.table_names(), ["t"]); + assert_eq!( + support::rows(&db, "SELECT id, a, c FROM t"), + [[Value::Int64(1), support::text("x"), support::text("new")]] + ); + let tx = db.transaction().expect("begin"); + support::run(&tx, "INSERT INTO t (id, a) VALUES (2, 'y')"); + for sql in [ + "ALTER TABLE t DROP COLUMN c", + "DROP TABLE t", + "ALTER TABLE t SET SYNC OFF", + ] { + assert!( + matches!(support::refusal(&tx, sql), Error::MixedSchemaAndRowWrite), + "{sql}" + ); + } + tx.commit().expect("commit the row"); + assert_eq!(support::int(&db, "SELECT COUNT(*) FROM t"), 2); +} + +#[test] +fn a_vector_column_far_larger_than_the_memory_limit_is_dropped_under_it() { + const DIMENSION: usize = 64; + const VECTORS: i64 = 10_000; + const HELD: u64 = VECTORS as u64 * DIMENSION as u64 * 4; + const KIB: usize = 1 << 10; + const MIB: usize = 1 << 20; + // Two stores take the same writes: 10,000 vectors of 64 numbers, 2,560,000 bytes of them, + // in four partitions, all sealed, and then two tables of large text, 6,400,000 bytes. + // One of them drops the vector column in between, under a memory limit. + let store = |test: &str, drop: bool| -> u64 { + let folder = support::folder(test); + let db = support::open(&folder); + support::declare_docs(&db, DIMENSION, "PARTITION_KEY (ctx)"); + for first in (1..=VECTORS).step_by(1000) { + let tx = db.transaction().expect("begin"); + support::insert_docs(&tx, first..first + 1000, DIMENSION, |n| { + format!("p{}", n % 4) + }); + tx.commit().expect("commit a thousand vectors"); + } + assert_eq!(support::seal_all(&db), VECTORS as u64); + support::close(db); + if drop { + // 9 MiB is what a store needs to open at all. The 192 KiB above it are less than + // a tenth of the vectors, which is 256,000 bytes. + let mut options = DatabaseOpenOptions::default(); + options.memory_limit = Some(9 * MIB + 192 * KIB); + let db = support::open_with(&folder, options).expect("open under the limit"); + let sql = "ALTER TABLE docs DROP COLUMN v"; + if let Err(error) = db.execute(sql, ¶ms! {}) { + panic!("{sql}\nof {VECTORS} vectors was refused under the limit: {error:?}"); + } + // The column is gone, its rows are not, and a search on it is refused by name. + assert!(!says(&printed(&db, "docs"), "v VECTOR")); + let sql = support::nearest_sql("docs", "v", None, 10); + match support::search(&db, &sql, &support::vector(1, DIMENSION)) { + Err(Error::ColumnNotFound { table, column }) => { + assert_eq!((table.as_str(), column.as_str()), ("docs", "v")); + } + other => panic!("a search on the dropped column answered {other:?}"), + } + support::close(db); + } + let db = support::open(&folder); + assert_eq!(support::int(&db, "SELECT COUNT(*) FROM docs"), VECTORS); + for (table, seed) in [("n0", 0), ("n1", ROWS)] { + support::run( + &db, + &format!("CREATE TABLE {table} (id INTEGER PRIMARY KEY, body TEXT)"), + ); + fill(&db, table, "body", true, seed); + } + support::close(db); + data_file(&folder) + }; + let kept = store("drop-vector-column-kept", false); + let dropped = store("drop-vector-column-dropped", true); + // The store that dropped the column has let its vectors, blocks and waiting entries go, + // so the text written afterwards took their room. + assert!( + dropped + HELD / 2 <= kept, + "after DROP COLUMN of {HELD} bytes of vectors the data file is {dropped} bytes long, \ + and the same store with the column kept is {kept}: the column's bytes are still held" + ); +} + +#[test] +fn a_column_added_and_dropped_in_one_transaction_leaves_nothing_stored() { + // Two stores hold the same 400 rows. One of them adds a column whose default is 8,000 + // letters, 3,200,000 bytes over its rows, and drops it again in the same transaction. + let store = |test: &str, churn: bool| -> u64 { + let folder = support::folder(test); + let db = support::open(&folder); + support::run( + &db, + "CREATE TABLE notes (id INTEGER PRIMARY KEY, body TEXT)", + ); + fill(&db, "notes", "body", true, 0); + if churn { + let before = printed(&db, "notes"); + let default = support::letters(7, LETTERS); + let tx = db.transaction().expect("begin"); + support::run( + &tx, + &format!("ALTER TABLE notes ADD COLUMN extra TEXT DEFAULT '{default}'"), + ); + support::run(&tx, "ALTER TABLE notes DROP COLUMN extra"); + tx.commit().expect("commit the column added and dropped"); + // The table declares what it declared before, and has no such column. + assert_eq!(printed(&db, "notes"), before); + assert!(matches!( + support::refusal(&db, "SELECT extra FROM notes"), + Error::ColumnNotFound { .. } + )); + // A column of that name added now finds nothing kept for it in any row. + support::run(&db, "ALTER TABLE notes ADD COLUMN extra TEXT"); + assert_eq!( + support::int(&db, "SELECT COUNT(*) FROM notes WHERE extra IS NULL"), + ROWS + ); + support::run(&db, "ALTER TABLE notes DROP COLUMN extra"); + } + assert_eq!(support::int(&db, "SELECT COUNT(*) FROM notes"), ROWS); + assert_eq!( + support::rows(&db, "SELECT body FROM notes WHERE id = 77"), + [[support::text(&support::letters(77, LETTERS))]] + ); + support::close(db); + data_file(&folder) + }; + let plain = store("added-and-dropped-plain", false); + let churned = store("added-and-dropped-churned", true); + assert!( + churned <= plain + PUT_IN / 4, + "the store whose column was added and dropped in one transaction is {churned} bytes \ + long and the same store without it is {plain}: something is still held for the column" + ); +} + +#[test] +fn a_table_dropped_and_declared_again_in_one_transaction_reads_none_of_the_old_rows() { + let folder = support::folder("dropped-and-declared-again"); + let db = support::open(&folder); + support::run(&db, "CREATE TABLE t (id INTEGER PRIMARY KEY, a TEXT)"); + support::run( + &db, + "INSERT INTO t (id, a) VALUES (1, 'x'), (2, 'y'), (3, 'z')", + ); + let tx = db.transaction().expect("begin"); + support::run(&tx, "DROP TABLE t"); + support::run( + &tx, + "CREATE TABLE t (id INTEGER PRIMARY KEY, n INTEGER, b TEXT)", + ); + // Inside the transaction the new table is empty: no row of the old one is counted or read. + assert_eq!(support::int(&tx, "SELECT COUNT(*) FROM t"), 0); + // Until it commits, everyone else reads the old table. + assert_eq!(support::int(&db, "SELECT COUNT(*) FROM t"), 3); + tx.commit() + .expect("commit the table dropped and declared again"); + assert_eq!(support::int(&db, "SELECT COUNT(*) FROM t"), 0); + assert!(says(&printed(&db, "t"), "n INTEGER")); + assert!(matches!( + support::refusal(&db, "SELECT a FROM t"), + Error::ColumnNotFound { .. } + )); + // A row written under an old row's key is the only row there is. + support::run(&db, "INSERT INTO t (id, n, b) VALUES (1, 5, 'new')"); + assert_eq!( + support::rows(&db, "SELECT id, n, b FROM t"), + [[Value::Int64(1), Value::Int64(5), support::text("new")]] + ); +} + +#[test] +fn dropping_a_column_never_shifts_what_the_other_columns_hold() { + let db = support::open(&support::folder("alter-drop-no-shift")); + support::run(&db, "CREATE TABLE p (c INTEGER PRIMARY KEY)"); + support::run( + &db, + "CREATE TABLE t (id INTEGER PRIMARY KEY, b INTEGER, c INTEGER REFERENCES p(c), \ + UNIQUE (c, b))", + ); + support::run(&db, "INSERT INTO p (c) VALUES (200)"); + support::run(&db, "INSERT INTO t (id, b, c) VALUES (1, 100, 200)"); + + // Inside the transaction that ran the ALTER, the column still holds 200. + let tx = db.transaction().expect("begin"); + support::run(&tx, "ALTER TABLE t DROP COLUMN b CASCADE"); + assert_eq!( + support::rows(&tx, "SELECT c FROM t"), + [[Value::Int64(200)]], + "inside the transaction that dropped b" + ); + tx.commit().expect("commit the drop"); + + // After the commit, c still holds 200 (the reference to p(200) refuses its parent's removal). + assert!( + matches!( + support::refusal(&db, "DELETE FROM p WHERE c = 200"), + Error::ForeignKeyViolation { .. } + ), + "t still references p(200) after b was dropped" + ); + + // Rows shorter than the table, drops in the middle, adds after: each cell is its column's. + support::run( + &db, + "CREATE TABLE u (id INTEGER PRIMARY KEY, a INTEGER, b INTEGER)", + ); + support::run(&db, "INSERT INTO u (id, a, b) VALUES (1, 10, 20)"); + let tx = db.transaction().expect("begin"); + for change in [ + "ADD COLUMN c INTEGER DEFAULT 30", + "DROP COLUMN a", + "ADD COLUMN d INTEGER DEFAULT 40", + "DROP COLUMN c", + ] { + support::run(&tx, &format!("ALTER TABLE u {change}")); + } + let wanted = [[Value::Int64(1), Value::Int64(20), Value::Int64(40)]]; + assert_eq!( + support::rows(&tx, "SELECT id, b, d FROM u"), + wanted, + "inside the transaction" + ); + tx.commit().expect("commit the changes"); + support::run(&db, "INSERT INTO u (id, b, d) VALUES (2, 21, 41)"); + assert_eq!( + support::rows(&db, "SELECT id, b, d FROM u ORDER BY id"), + [ + wanted[0].clone(), + [Value::Int64(2), Value::Int64(21), Value::Int64(41)] + ] + ); +} diff --git a/v2/contextdb/tests/clock.rs b/v2/contextdb/tests/clock.rs index b54cf88b..f29bcc77 100644 --- a/v2/contextdb/tests/clock.rs +++ b/v2/contextdb/tests/clock.rs @@ -5,23 +5,30 @@ mod support; use std::path::Path; use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use contextdb::{Clock, Database, DatabaseOpenOptions, Value}; -/// A clock that always shows the same second. -struct At(u64); +/// A clock that shows the second it is set to. +struct At(Arc); impl Clock for At { fn now(&self) -> SystemTime { - UNIX_EPOCH + Duration::from_secs(self.0) + UNIX_EPOCH + Duration::from_secs(self.0.load(Ordering::SeqCst)) } } -fn open_at(folder: &Path, second: u64) -> Database { +fn open_ticking(folder: &Path, second: u64) -> (Database, Arc) { + let second = Arc::new(AtomicU64::new(second)); let mut options = DatabaseOpenOptions::default(); - options.clock = Some(Arc::new(At(second))); - support::open_with(folder, options).expect("open the store with a clock") + options.clock = Some(Arc::new(At(second.clone()))); + let db = support::open_with(folder, options).expect("open the store with a clock"); + (db, second) +} + +fn open_at(folder: &Path, second: u64) -> Database { + open_ticking(folder, second).0 } #[test] @@ -75,3 +82,28 @@ fn default_now_stores_the_time_the_clock_door_gives() { 1 ); } + +#[test] +fn now_in_a_declaration_is_read_once_even_when_the_transaction_commits_later() { + let (db, second) = open_ticking(&support::folder("clock-declaration-once"), 1_700_000_000); + support::run(&db, "CREATE TABLE t (id INTEGER PRIMARY KEY)"); + support::run(&db, "CREATE TABLE unrelated (id INTEGER PRIMARY KEY)"); + support::run(&db, "INSERT INTO t (id) VALUES (1)"); + + let tx = db.transaction().expect("begin"); + support::run( + &tx, + "ALTER TABLE t ADD COLUMN ts TIMESTAMP NOT NULL DEFAULT NOW()", + ); + let read = support::rows(&tx, "SELECT ts FROM t"); + assert_eq!(read, [[Value::Timestamp(1_700_000_000_000)]]); + // Another commit lands, the clock moves, and the transaction commits. + support::run(&db, "INSERT INTO unrelated (id) VALUES (1)"); + second.store(1_800_000_000, Ordering::SeqCst); + tx.commit().expect("commit the declaration"); + assert_eq!( + support::rows(&db, "SELECT ts FROM t"), + read, + "the time the declaration read is the time it stored" + ); +} diff --git a/v2/contextdb/tests/crash/main.rs b/v2/contextdb/tests/crash/main.rs index 0466f524..842171b7 100644 --- a/v2/contextdb/tests/crash/main.rs +++ b/v2/contextdb/tests/crash/main.rs @@ -1,8 +1,10 @@ //! Kill and reopen: a writer killed at any storage call of a commit leaves the whole commit or //! none of it, and the store opens again with nothing to mend by hand. That holds for a commit //! of rows with vectors and for the commit of a maintenance cycle that seals vectors: after -//! either, every search finds exactly the vectors of the rows that are there. Damage in the -//! file is refused as damage, by name. +//! either, every search finds exactly the vectors of the rows that are there. It holds for a +//! commit of rows that reference each other, are counted by a kept count and are indexed by +//! values too long for an index entry: the rows, the counts and the entries are there together +//! or not at all. Damage in the file is refused as damage, by name. //! //! A kill lands at a point the child names. The child's storage door counts the writes and //! syncs of its data file; before call number `stop_at` it announces `stopped` and waits there, @@ -245,8 +247,8 @@ fn damage_each_page_of_a_scan( refused } -/// Rows of padding in the test below, 800 bytes each: 32 MiB. -const PADDING_ROWS: i64 = 40_000; +/// Rows of padding in the test below, 800 bytes each: 16 MiB. +const PADDING_ROWS: i64 = 20_000; #[test] fn a_commit_on_disk_is_reported_and_served_though_the_read_after_it_fails() { @@ -256,31 +258,39 @@ fn a_commit_on_disk_is_reported_and_served_though_the_read_after_it_fails() { let syncs = Arc::new(AtomicU64::new(0)); let fail_after = Arc::new(AtomicU64::new(0)); let failed = Arc::new(AtomicU64::new(0)); - let mut options = DatabaseOpenOptions::default(); - options.storage = Some(support::failing_storage({ - let (syncs, fail_after, failed) = (syncs.clone(), fail_after.clone(), failed.clone()); - move |call| { - match call { - Call::Sync => { - syncs.fetch_add(1, Ordering::SeqCst); - } - Call::Read(..) => { - let after = fail_after.load(Ordering::SeqCst); - if after > 0 && syncs.load(Ordering::SeqCst) >= after { - failed.fetch_add(1, Ordering::SeqCst); - return Err(io::Error::other("a read this test makes fail")); + // When set, every read fails, whatever the syncs: reads that fail before a commit is on disk. + let fail_now = Arc::new(AtomicBool::new(false)); + let options = || { + let mut options = DatabaseOpenOptions::default(); + options.storage = Some(support::failing_storage({ + let (syncs, fail_after, failed) = (syncs.clone(), fail_after.clone(), failed.clone()); + let fail_now = fail_now.clone(); + move |call| { + match call { + Call::Sync => { + syncs.fetch_add(1, Ordering::SeqCst); } + Call::Read(..) => { + let after = fail_after.load(Ordering::SeqCst); + if fail_now.load(Ordering::SeqCst) + || (after > 0 && syncs.load(Ordering::SeqCst) >= after) + { + failed.fetch_add(1, Ordering::SeqCst); + return Err(io::Error::other("a read this test makes fail")); + } + } + Call::Write(..) => {} } - Call::Write(..) => {} + Ok(()) } - Ok(()) - } - })); - // Under a memory limit of 32 MiB the store's page cache is 8 MiB: a quarter of the limit. - options.memory_limit = Some(32 << 20); - let db = support::open_with(&folder, options).expect("open"); + })); + // Under a memory limit of 32 MiB the store's page cache is 8 MiB: a quarter of the limit. + options.memory_limit = Some(32 << 20); + options + }; + let db = support::open_with(&folder, options()).expect("open"); support::declare(&db); - // A second table four times the size of that 8 MiB page cache, in rows small enough to be + // A second table twice the size of that 8 MiB page cache, in rows small enough to be // kept in the table's own pages. Reading all of it leaves the cache full of its pages, so // the pages a commit writes afterwards have to be read back from the file. support::run( @@ -354,6 +364,64 @@ fn a_commit_on_disk_is_reported_and_served_though_the_read_after_it_fails() { // Closed and opened again with nothing failing, the store holds every commit that returned. let _ = db.close(); support::assert_rows_are(&support::open(&folder), 1..=rows); + + // A read that fails inside a statement, and inside a commit before it reaches the disk, is + // a failure of the storage and never damage; the handle that met it says so until it is + // opened again, nothing aborts, and a commit that was refused left no row. Each is made on a + // handle just opened, whose cache is empty, so that it has to read the file. + fail_after.store(0, Ordering::SeqCst); + let db = support::open_with(&folder, options()).expect("open again"); + fail_now.store(true, Ordering::SeqCst); + let failed_before = failed.load(Ordering::SeqCst); + match db.run("SELECT COUNT(name) FROM t", ¶ms! {}) { + Err(Error::StoreIo { .. }) => {} + other => panic!("a statement whose read failed got {other:?}, not a storage failure"), + } + assert!( + failed.load(Ordering::SeqCst) > failed_before, + "the statement made no read that failed" + ); + fail_now.store(false, Ordering::SeqCst); + let _ = db.close(); + let db = support::open_with(&folder, options()).expect("open again"); + fail_now.store(true, Ordering::SeqCst); + let failed_before = failed.load(Ordering::SeqCst); + let [id, name, tag] = + <[Value; 3]>::try_from(support::row(rows + 1)).expect("a row has three columns"); + let refused = db.execute( + "INSERT INTO t (id, name, tag) VALUES ($id, $name, $tag)", + ¶ms! { "id" => id, "name" => name, "tag" => tag }, + ); + let read_failed = failed.load(Ordering::SeqCst) > failed_before; + fail_now.store(false, Ordering::SeqCst); + match refused { + Err(Error::StoreIo { .. }) => {} + Ok(_) if !read_failed => panic!("the commit made no read that failed: nothing was shown"), + other => panic!("a commit whose read failed got {other:?}, not a storage failure"), + } + let _ = db.close(); + support::assert_rows_are(&support::open(&folder), 1..=rows); + + // An open, for writing and for reading, while every read fails is a failure of the storage. + let failing = || { + support::failing_storage(|call| match call { + Call::Read(..) => Err(io::Error::other("a read this test makes fail")), + _ => Ok(()), + }) + }; + let mut writing = DatabaseOpenOptions::default(); + writing.storage = Some(failing()); + match support::open_with(&folder, writing) { + Err(Error::StoreIo { .. }) => {} + other => panic!("an open for writing while reads fail got {:?}", other.err()), + } + let mut reading = ReadSessionOptions::default(); + reading.storage = Some(failing()); + match ReadSession::open_with_options(&folder, reading) { + Err(Error::StoreIo { .. }) => {} + other => panic!("an open for reading while reads fail got {:?}", other.err()), + } + support::assert_rows_are(&support::open(&folder), 1..=rows); } /// Asserts that the scan and the count of the table give rows 1 to `rows`, or are refused @@ -583,6 +651,206 @@ fn one_commit_of_vectors(role: support::Role) { // ---- shared by the tests of this file ---- +// ---- a commit of rows that a table's rules tie together ---- + +/// The body of a child: 3,400 bytes, of which the first 3,000 are the same for every child, so +/// that in the index over `body` the children differ only past what an entry keeps of a value. +fn body(n: i64) -> String { + format!("{}{}", "m".repeat(3000), support::letters(n, 400)) +} + +/// The children before the commit of the test below and after it, as (id, parent, body). +fn children(committed: bool) -> Vec<(i64, i64, String)> { + match committed { + false => (1..=6).map(|n| (n, n % 2 + 1, body(n))).collect(), + true => { + let mut children: Vec<_> = (3..=6).map(|n| (n, n % 2 + 1, body(n))).collect(); + children[0].2 = body(103); + children[1].1 = 3; + children.extend((7..=18).map(|n| (n, n % 3 + 1, body(n)))); + children + } + } +} + +/// A closed store with two parents and six children. A child references its parent, the table +/// keeps a count of the children of each parent, and `body` is indexed. +fn store_with_children(name: &str) -> PathBuf { + let folder = support::folder(name); + let db = support::open(&folder); + for sql in [ + "CREATE TABLE parents (id INTEGER PRIMARY KEY)", + "CREATE TABLE children (id INTEGER PRIMARY KEY, \ + parent_id INTEGER NOT NULL REFERENCES parents(id), body TEXT NOT NULL)", + "CREATE INDEX children_parent ON children (parent_id)", + "CREATE INDEX children_body ON children (body)", + "ALTER TABLE children SET MAINTAIN COUNT (parent_id)", + "INSERT INTO parents (id) VALUES (1), (2)", + ] { + support::run(&db, sql); + } + let tx = db.transaction().expect("begin"); + for child in children(false) { + insert_child(&tx, child).expect("insert a child"); + } + tx.commit().expect("commit"); + support::close(db); + folder +} + +fn insert_child( + on: &impl Run, + (id, parent, body): (i64, i64, String), +) -> contextdb::Result { + on.run( + "INSERT INTO children (id, parent_id, body) VALUES ($id, $parent, $body)", + ¶ms! { "id" => id, "parent" => parent, "body" => body }, + ) +} + +/// Asserts that the store holds exactly these children and that everything kept beside the rows +/// says the same as the rows: the parent of every child is there, the kept count of each parent +/// is the number of its children and reads no row, and every child is found by its body through +/// the index, alone and in the order of the bodies. +fn assert_children_are(db: &Database, wanted: &[(i64, i64, String)], when: &str) { + let rows: Vec> = wanted + .iter() + .map(|(id, parent, body)| { + vec![ + Value::Int64(*id), + Value::Int64(*parent), + support::text(body), + ] + }) + .collect(); + let found = support::rows(db, "SELECT id, parent_id, body FROM children ORDER BY id"); + support::assert_same_rows(&found, &rows, &format!("{when}: the children")); + + let parents = support::rows(db, "SELECT id FROM parents ORDER BY id"); + for parent in 1..=3_i64 { + let children = wanted.iter().filter(|child| child.1 == parent).count(); + assert!( + children == 0 || parents.contains(&vec![Value::Int64(parent)]), + "{when}: parent {parent} is gone and {children} children name it" + ); + let count = support::run( + db, + &format!("SELECT COUNT(*) FROM children WHERE parent_id = {parent}"), + ); + assert_eq!( + count.rows(), + [[Value::Int64(children as i64)]], + "{when}: the kept count of parent {parent}" + ); + assert_eq!(count.rows_examined(), 0, "{when}: rows read by the count"); + } + for (id, _, body) in wanted { + let by_body = db + .execute( + "SELECT id FROM children WHERE body = $body", + ¶ms! { "body" => body.as_str() }, + ) + .expect("look a child up by its body"); + assert_eq!( + by_body.rows(), + [[Value::Int64(*id)]], + "{when}: child {id} by its body" + ); + } + let mut by_body: Vec<&(i64, i64, String)> = wanted.iter().collect(); + by_body.sort_by(|a, b| a.2.cmp(&b.2)); + let in_order: Vec<[Value; 1]> = by_body + .iter() + .map(|child| [Value::Int64(child.0)]) + .collect(); + let found = support::rows(db, "SELECT id FROM children ORDER BY body"); + support::assert_same_rows(&found, &in_order, &format!("{when}: the children by body")); +} + +#[test] +fn a_kill_inside_a_commit_keeps_references_counts_and_long_entries_with_their_rows() { + if let Some(role) = support::role() { + return one_commit_of_children(role); + } + let test = "a_kill_inside_a_commit_keeps_references_counts_and_long_entries_with_their_rows"; + // Run once to the end to learn which storage calls belong to the commit. + let folder = store_with_children("kill-children-count"); + let lines = Child::start(test, "writer stop_at=0", &folder).end(); + let begins = support::last_number(&lines, "committing").expect("the writer announces it"); + let calls = support::last_number(&lines, "calls").expect("the writer announces its calls"); + assert!(calls > begins, "the commit made no write and no sync"); + assert_children_are( + &support::open(&folder), + &children(true), + "after the whole commit", + ); + + for k in begins + 1..=calls { + let folder = store_with_children(&format!("kill-children-at-{k}")); + let mut writer = Child::start(test, &format!("writer stop_at={k}"), &folder); + let before = writer.until("stopped"); + writer.kill(); + let returned = before.iter().any(|line| line == "committed"); + + // The store holds what it held before the commit, or everything the commit brought. + let db = support::open(&folder); + let when = format!("killed at storage call {k}"); + let committed = match support::int(&db, "SELECT COUNT(*) FROM children") { + 6 if !returned => false, + 16 => true, + rows => panic!( + "{when}: {rows} children are there; the commit had {}returned", + if returned { "" } else { "not " } + ), + }; + assert_children_are(&db, &children(committed), &when); + let parents = support::int(&db, "SELECT COUNT(*) FROM parents"); + assert_eq!( + parents, + if committed { 3 } else { 2 }, + "{when}: the parents" + ); + + // The store goes on from that state, and its rules with it. + let mut wanted = children(committed); + wanted.push((50, 1, body(50))); + insert_child(&db, (50, 1, body(50))).expect("insert a child after reopening"); + assert_children_are(&db, &wanted, &format!("{when}, and one more child")); + assert!(matches!( + insert_child(&db, (51, 404, body(51))), + Err(Error::ForeignKeyViolation { .. }) + )); + } +} + +/// In one transaction adds a parent and twelve children, removes two children, gives one another +/// body and one another parent. Announces `committing ` with the storage calls made so far +/// when the commit is about to begin, `committed` when it has returned, and `calls ` once +/// the store is closed. +fn one_commit_of_children(role: support::Role) { + let (storage, calls) = stopping_before_call(role.number("stop_at")); + let mut options = DatabaseOpenOptions::default(); + options.storage = Some(storage); + let db = support::open_with(&role.folder, options).expect("open"); + let tx = db.transaction().expect("begin"); + support::run(&tx, "INSERT INTO parents (id) VALUES (3)"); + for child in children(true).split_off(4) { + insert_child(&tx, child).expect("insert a child"); + } + support::run(&tx, "DELETE FROM children WHERE id <= 2"); + tx.execute( + "UPDATE children SET body = $body WHERE id = 3", + ¶ms! { "body" => body(103) }, + ) + .expect("change a body"); + support::run(&tx, "UPDATE children SET parent_id = 3 WHERE id = 4"); + support::say(format!("committing {}", calls.load(Ordering::SeqCst))); + tx.commit().expect("commit"); + support::say("committed"); + support::close(db); + support::say(format!("calls {}", calls.load(Ordering::SeqCst))); +} + /// A closed store with the table declared and rows 1 to `rows` committed. fn store_with_rows(name: &str, rows: i64) -> PathBuf { let folder = support::folder(name); diff --git a/v2/contextdb/tests/declarations.rs b/v2/contextdb/tests/declarations.rs new file mode 100644 index 00000000..3a2ad33e --- /dev/null +++ b/v2/contextdb/tests/declarations.rs @@ -0,0 +1,978 @@ +//! Declarations: every table and index text of a program built on this library is accepted, is +//! printed back with every clause its author wrote and no other, and the printed text declares +//! the same table again, in an empty store and after the store is closed and opened. And what a +//! declaration refuses: by its kind, saying why, and declaring nothing. + +mod support; + +use std::collections::{BTreeMap, BTreeSet}; + +use contextdb::{Database, Error, ReadSession}; + +/// The declarations of a program built on this library, as that program writes them. +const CONSUMER_SCHEMA: &str = include_str!("fixtures/consumer_schema.sql"); + +/// The statements of a text. A statement ends with `;` at the end of a line; a line that begins +/// with `--` between two statements is a comment. +fn statements(text: &str) -> Vec { + let mut found = Vec::new(); + let mut statement = String::new(); + for line in text.lines() { + if statement.is_empty() && (line.trim().is_empty() || line.starts_with("--")) { + continue; + } + statement.push_str(line); + statement.push('\n'); + if let Some(whole) = statement.trim_end().strip_suffix(';') { + found.push(whole.to_owned()); + statement.clear(); + } + } + assert!(statement.is_empty(), "a statement with no `;`: {statement}"); + found +} + +/// The fixture's statements: the ones a single store takes in the order written, and the +/// second texts for tables among them. +fn consumer_schema() -> (Vec, Vec) { + let (first, second) = CONSUMER_SCHEMA + .split_once("\n-- Second texts") + .expect("the fixture's second part"); + // What is left of the line that begins the second part is the rest of its comment. + let (_, second) = second.split_once('\n').expect("the second texts"); + (statements(first), statements(second)) +} + +/// A name as the fixture writes one: its letters up to the first character that is none. +fn name(text: &str) -> String { + text.chars() + .take_while(|c| c.is_ascii_alphanumeric() || *c == '_') + .collect() +} + +/// The table a `CREATE TABLE name (...)` or a `CREATE INDEX name ON table (...)` of the fixture +/// is for, and whether the statement is the table's own. +fn table_of(sql: &str) -> (String, bool) { + if let Some(rest) = sql.strip_prefix("CREATE TABLE ") { + return (name(rest), true); + } + let (_, rest) = sql + .split_once(" ON ") + .unwrap_or_else(|| panic!("neither a CREATE TABLE nor a CREATE INDEX: {sql}")); + (name(rest), false) +} + +/// The tables a statement of the fixture references, its own left out. +fn referenced(sql: &str) -> BTreeSet { + let own = table_of(sql).0; + let parents = sql.split("REFERENCES ").skip(1).map(name); + parents.filter(|parent| *parent != own).collect() +} + +/// The words of a declaration in lower case and its quoted texts as they are written: what the +/// declaration says, whatever the order, the spacing and the punctuation it says it in. A type +/// is given the name it is printed under, and a quoted name that needs no quotes loses them. +fn words(sql: &str) -> Vec { + let mut found = Vec::new(); + let mut rest = sql.chars().peekable(); + while let Some(first) = rest.next() { + if first == '\'' || first == '"' { + let mut text = String::from(first); + loop { + let next = rest.next().expect("a quoted text ends with its quote"); + text.push(next); + // The quote written twice is one quote of the text. + if next == first && rest.next_if_eq(&first).is_none() { + break; + } + } + let inner = &text[1..text.len() - 1]; + let plain = first == '"' + && inner.starts_with(|c: char| c.is_ascii_lowercase() || c == '_') + && inner + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_'); + found.push(if plain { inner.to_owned() } else { text }); + } else if first.is_ascii_alphanumeric() || first == '_' { + let mut word = String::from(first.to_ascii_lowercase()); + while let Some(next) = + rest.next_if(|c| c.is_ascii_alphanumeric() || *c == '_' || *c == '.') + { + word.push(next.to_ascii_lowercase()); + } + match word.as_str() { + "int" => found.push("integer".to_owned()), + "bool" => found.push("boolean".to_owned()), + "float" => found.push("real".to_owned()), + "state_machine" => found.extend(["state".to_owned(), "machine".to_owned()]), + _ => found.push(word), + } + } + } + found +} + +/// Asserts that the text a table is printed back as says everything its author wrote and +/// nothing else. `written` is the table's `CREATE TABLE` and each `CREATE INDEX` on it. +fn assert_prints_what_was_written(written: &[&str], printed: &str, table: &str) { + let written_words: Vec = written.iter().flat_map(|sql| words(sql)).collect(); + let mut counts: BTreeMap = BTreeMap::new(); + for word in &written_words { + *counts.entry(word.clone()).or_default() += 1; + } + for word in words(printed) { + *counts.entry(word).or_default() -= 1; + } + // The limit on partitions is printed for every column with a PARTITION_KEY, whether or not + // its author wrote one: 256 when none was written. + let count = |word: &str| written_words.iter().filter(|w| *w == word).count() as i64; + let defaulted = count("partition_key") - count("max_partitions"); + for word in ["max_partitions", "256"] { + *counts.entry(word.to_owned()).or_default() += defaulted; + } + let lost: Vec<&String> = counts + .iter() + .filter(|(_, n)| **n > 0) + .map(|(w, _)| w) + .collect(); + let added: Vec<&String> = counts + .iter() + .filter(|(_, n)| **n < 0) + .map(|(w, _)| w) + .collect(); + assert!( + lost.is_empty() && added.is_empty(), + "the declaration of {table} is printed back without {lost:?} and with {added:?}, which \ + its author did not write.\nwritten:\n{}\nprinted:\n{printed}", + written.join(";\n") + ); +} + +/// Declares statements that must be accepted. +fn declare>(db: &Database, statements: impl IntoIterator) { + for sql in statements { + support::run(db, sql.as_ref()); + } +} + +/// What a table is printed back as. +fn printed(db: &Database, table: &str) -> String { + db.declaration(table) + .unwrap_or_else(|error| panic!("the declaration of {table} was refused: {error:?}")) +} + +/// The statements for each table, the table's own first, with the tables in the order they +/// were declared. +fn by_table(statements: &[String]) -> Vec<(String, Vec<&str>)> { + let mut tables: Vec<(String, Vec<&str>)> = Vec::new(); + for sql in statements { + let (table, own) = table_of(sql); + if own { + tables.push((table, vec![sql.as_str()])); + } else { + let of_table = tables.iter_mut().find(|(name, _)| *name == table); + let (_, texts) = of_table.unwrap_or_else(|| panic!("an index before its table: {sql}")); + texts.push(sql.as_str()); + } + } + tables +} + +/// Declares `written` in a fresh store and asserts, of every table: that it is printed back +/// with what was written and nothing else; that what was printed, declared in an empty store, +/// is printed the same; and that it is printed the same after the store is closed and opened, +/// by the writer and by a reader. Gives back the printed texts by table. +fn assert_declared_and_printed_back(test: &str, written: &[String]) -> BTreeMap { + let folder = support::folder(test); + let db = support::open(&folder); + declare(&db, written); + let tables = by_table(written); + let mut names: Vec<&str> = tables.iter().map(|(table, _)| table.as_str()).collect(); + names.sort_unstable(); + assert_eq!(db.table_names(), names, "the tables of the store"); + + let mut texts = BTreeMap::new(); + for (table, written) in &tables { + let text = printed(&db, table); + assert_prints_what_was_written(written, &text, table); + texts.insert(table.clone(), text); + } + + let again = support::open(&support::folder(&format!("{test}-again"))); + for (table, _) in &tables { + declare(&again, statements(&texts[table])); + } + for (table, text) in &texts { + assert_eq!( + &printed(&again, table), + text, + "{table}, declared from its printed text in an empty store, is printed differently" + ); + } + + support::close(db); + let db = support::open(&folder); + let reader = ReadSession::open(&folder).expect("open the store for reading"); + for (table, text) in &texts { + assert_eq!(&printed(&db, table), text, "{table} after close and open"); + let read = reader.declaration(table).expect("a reader's declaration"); + assert_eq!(&read, text, "{table} as a reader prints it"); + } + texts +} + +#[test] +fn every_declaration_of_a_consumer_is_accepted_and_printed_back_as_written() { + let (first, second) = consumer_schema(); + let own = |statements: &[String]| statements.iter().filter(|sql| table_of(sql).1).count(); + // The fixture holds 93 table texts, 6 of them second texts, and 25 index texts. + assert_eq!((own(&first), first.len() - own(&first)), (87, 25)); + assert_eq!((own(&second), second.len()), (6, 6)); + let texts = assert_declared_and_printed_back("consumer-schema", &first); + assert_eq!(texts.len(), 87); +} + +#[test] +fn a_second_text_for_a_consumers_table_is_accepted_in_a_store_of_its_own() { + let (first, second) = consumer_schema(); + for (number, sql) in second.iter().enumerate() { + // The tables the text references, and the ones those reference, as first declared. + let mut wanted = referenced(sql); + loop { + let before = wanted.len(); + for parent in &first { + let (table, own) = table_of(parent); + if own && wanted.contains(&table) { + wanted.extend(referenced(parent)); + } + } + if wanted.len() == before { + break; + } + } + let mut store: Vec = first + .iter() + .filter(|parent| { + let (table, own) = table_of(parent); + own && wanted.contains(&table) + }) + .cloned() + .collect(); + store.push(sql.clone()); + let test = format!("consumer-schema-second-{number}"); + let texts = assert_declared_and_printed_back(&test, &store); + assert!(texts.contains_key(&table_of(sql).0)); + } +} + +/// Every clause a table and an index may declare that the consumer's schema does not hold, and +/// some it does, each on a table that says little else. +const EVERY_CLAUSE: &[&str] = &[ + "CREATE TABLE contexts (id UUID PRIMARY KEY, name TEXT NOT NULL UNIQUE)", + "CREATE TABLE acl_grants (id UUID PRIMARY KEY, principal_kind TEXT, principal_id TEXT, \ + acl_id UUID)", + "CREATE TABLE projects (id UUID PRIMARY KEY, \ + context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), \ + context_name TEXT REFERENCES contexts(name), name TEXT NOT NULL IMMUTABLE, \ + scope TEXT SCOPE_LABEL ('public', 'internal'), \ + acl_id UUID ACL REFERENCES acl_grants(acl_id), made_at TXID, \ + doc JSON NOT NULL DEFAULT '{}', tags JSON DEFAULT '[]', note TEXT DEFAULT '', \ + open BOOLEAN NOT NULL DEFAULT true, shut BOOLEAN DEFAULT false, UNIQUE (context_id, id)) \ + RETAIN 48 HOURS SYNC SAFE HISTORY ALL SYNC TWO WAY SYNC CONFLICT KEEP FIRST", + "CREATE INDEX projects_by_name ON projects (name DESC, id)", + "CREATE INDEX projects_by_kind ON projects ((doc -> 'kind'))", + "CREATE INDEX projects_by_owner ON projects ((doc -> 'owner' -> 'name'))", + "CREATE TABLE memos (id INTEGER PRIMARY KEY, \ + scope TEXT SCOPE_LABEL_READ ('reader', 'writer') WRITE ('writer'), body TEXT) \ + SYNC PULL ONLY SYNC CONFLICT KEEP LATEST HISTORY CURRENT ONLY RETAIN 90 SECONDS", + "CREATE TABLE plans (id UUID PRIMARY KEY, name TEXT NOT NULL)", + "CREATE TABLE steps (id UUID PRIMARY KEY, description TEXT NOT NULL, status TEXT NOT NULL, \ + confidence REAL, created_at TIMESTAMP DEFAULT NOW(), \ + plan_id UUID REFERENCES plans(id) ON STATE archived PROPAGATE SET invalidated, \ + tenant TEXT NOT NULL, embedding VECTOR(4) PARTITION_KEY (tenant) SEARCH_MODE INDEXED) \ + STATE MACHINE (status: active -> [invalidated, superseded]) \ + PROPAGATE ON EDGE CITES INCOMING STATE invalidated SET invalidated \ + PROPAGATE ON STATE invalidated EXCLUDE VECTOR \ + PROPAGATE ON STATE superseded EXCLUDE VECTOR", + "CREATE TABLE reviews (id UUID PRIMARY KEY, status TEXT NOT NULL, \ + plan_id UUID REFERENCES plans(id) ON STATE archived PROPAGATE SET stale MAX DEPTH 2 \ + ABORT ON FAILURE) STATE MACHINE (status: open -> [stale], stale -> [closed]) \ + PROPAGATE ON EDGE CITES OUTGOING STATE stale SET stale MAX DEPTH 3 ABORT ON FAILURE \ + PROPAGATE ON EDGE BLOCKS BOTH STATE stale SET stale", + "CREATE TABLE tasks (id UUID PRIMARY KEY, context_id UUID NOT NULL, \ + project_id UUID NOT NULL, \ + FOREIGN KEY (context_id, project_id) REFERENCES projects(context_id, id))", + "CREATE TABLE task_results (id UUID PRIMARY KEY, task_id UUID NOT NULL, \ + success BOOLEAN NOT NULL)", + "CREATE INDEX task_results_task_id_idx ON task_results (task_id)", + "CREATE TABLE ranked (id UUID PRIMARY KEY, description TEXT NOT NULL, confidence REAL, \ + embedding VECTOR(4) RANK_POLICY (JOIN task_results ON task_id, \ + FORMULA 'coalesce({confidence}, 1.0) * coalesce({success}, 1.0)', \ + SORT_KEY effective_confidence))", + "CREATE TABLE defaults (id INTEGER PRIMARY KEY, a TEXT DEFAULT NULL, b REAL DEFAULT 0.0, \ + c INTEGER DEFAULT -1, d REAL DEFAULT -2.5, e TEXT DEFAULT 'it''s', f BOOLEAN DEFAULT TRUE)", + "CREATE TABLE one_key (a INTEGER, b TEXT, PRIMARY KEY (a))", + "CREATE TABLE two_keys (a INTEGER NOT NULL, b TEXT NOT NULL, c TEXT, PRIMARY KEY (a, b))", + "CREATE TABLE no_key (line TEXT NOT NULL) SYNC OFF", + "CREATE TABLE links (id UUID PRIMARY KEY, source_id UUID NOT NULL, target_id UUID NOT NULL, \ + edge_type TEXT NOT NULL) DAG('DEPENDS_ON', 'BASED_ON')", + "CREATE TABLE counted (id UUID PRIMARY KEY, context_id UUID NOT NULL, \ + connector_id TEXT NOT NULL, UNIQUE (context_id, id), \ + UNIQUE (context_id, connector_id, id)) \ + MAINTAIN COUNT (context_id), (context_id, connector_id)", + "CREATE TABLE counted_all (id INTEGER PRIMARY KEY, kind TEXT, UNIQUE (kind, id)) \ + MAINTAIN COUNT ALL PREFIXES", + "CREATE TABLE counted_none (id INTEGER PRIMARY KEY, kind TEXT) MAINTAIN COUNT NONE", + "CREATE TABLE delivered (id INTEGER PRIMARY KEY) SYNC PUSH ONLY SYNC CONFLICT KEEP FIRST \ + IMMUTABLE DELIVERY MANIFEST OVER delivered_parts, delivered_marks \ + EDGE DISCARD AFTER OUTCOME", + "CREATE TABLE discard_never (id INTEGER PRIMARY KEY) SYNC PUSH ONLY \ + SYNC CONFLICT KEEP FIRST IMMUTABLE DELIVERY MANIFEST OVER discard_parts EDGE DISCARD NEVER", + "CREATE TABLE discard_always (id INTEGER PRIMARY KEY) SYNC PUSH ONLY \ + SYNC CONFLICT KEEP FIRST IMMUTABLE EDGE DISCARD ALWAYS", + "CREATE TABLE kept_minutes (id INTEGER PRIMARY KEY) RETAIN 30 MINUTES", + "CREATE TABLE kept_days (id INTEGER PRIMARY KEY) RETAIN 7 DAYS SYNC OFF", + "CREATE TABLE current_local (id TEXT PRIMARY KEY, state TEXT NOT NULL) \ + RETAIN 2 DAYS HISTORY CURRENT ONLY SYNC OFF", + "CREATE TABLE order_a (id INTEGER PRIMARY KEY, body TEXT) \ + IMMUTABLE SYNC OFF SYNC CONFLICT KEEP FIRST MAINTAIN COUNT NONE", + "CREATE TABLE order_b (id INTEGER PRIMARY KEY, body TEXT) \ + MAINTAIN COUNT NONE SYNC CONFLICT KEEP FIRST SYNC OFF IMMUTABLE", + "CREATE TABLE \"Mixed Case\" (\"Id\" INTEGER PRIMARY KEY, \"two words\" TEXT)", + "CREATE TABLE Folded (Id INTEGER PRIMARY KEY, Name TEXT)", + "CREATE INDEX Folded_By_Name ON Folded (Name)", +]; + +#[test] +fn every_clause_a_table_may_declare_is_printed_back_and_declares_the_same_table_again() { + // The fixture's reader of names knows plain lower-case names, so the two tables that are + // written in other letters are declared here under the names they are stored by. + let statements: Vec = EVERY_CLAUSE + .iter() + .map(|sql| { + sql.replace("\"Mixed Case\"", "mixed_case") + .replace("Folded_By_Name ON Folded", "folded_by_name ON folded") + .replace("TABLE Folded", "TABLE folded") + }) + .collect(); + let texts = assert_declared_and_printed_back("every-clause", &statements); + + // Options written in two orders are printed in one. + assert_eq!( + texts["order_a"].replace("order_a", "order"), + texts["order_b"].replace("order_b", "order") + ); + // A default is printed as its value, in the engine's spelling. + let projects = &texts["projects"]; + for default in [ + "doc JSON NOT NULL DEFAULT '{}'", + "tags JSON DEFAULT '[]'", + "note TEXT DEFAULT ''", + "open BOOLEAN NOT NULL DEFAULT TRUE", + "shut BOOLEAN DEFAULT FALSE", + ] { + assert!(projects.contains(default), "no `{default}` in:\n{projects}"); + } + // How long rows are kept is printed in the unit its author wrote. + assert!(projects.contains("RETAIN 48 HOURS SYNC SAFE"), "{projects}"); + assert!( + texts["memos"].contains("RETAIN 90 SECONDS"), + "{}", + texts["memos"] + ); + // A VECTOR column's clauses come in one order, the limit on partitions with its number + // though its author wrote none. + let steps = &texts["steps"]; + let clauses = + "embedding VECTOR(4) PARTITION_KEY (tenant) MAX_PARTITIONS 256 SEARCH_MODE INDEXED"; + assert!(steps.contains(clauses), "no `{clauses}` in:\n{steps}"); + // A table that declares none of these says none of them, whatever the table beside it + // declares. + assert_eq!( + texts["plans"], + "CREATE TABLE plans (\n id UUID PRIMARY KEY,\n name TEXT NOT NULL\n);\n" + ); + // The engine's own indexes are never printed: the ones behind a key, a UNIQUE rule and a + // reference, and the two of a table of edges. + for (table, text) in &texts { + assert!( + !text.contains("__"), + "{table} prints an index of the engine:\n{text}" + ); + } +} + +#[test] +fn a_name_without_quotes_is_stored_in_lower_case_and_a_quoted_name_as_written() { + let folder = support::folder("declared-names"); + let db = support::open(&folder); + declare(&db, EVERY_CLAUSE.iter().rev().take(3).rev()); + assert_eq!(db.table_names(), ["Mixed Case", "folded"]); + assert_eq!( + printed(&db, "folded"), + "CREATE TABLE folded (\n id INTEGER PRIMARY KEY,\n name TEXT\n);\n\ + CREATE INDEX folded_by_name ON folded (name);\n" + ); + assert_eq!( + printed(&db, "Mixed Case"), + "CREATE TABLE \"Mixed Case\" (\n \"Id\" INTEGER PRIMARY KEY,\n \"two words\" TEXT\n);\n" + ); + // The printed texts declare the same two tables in an empty store. + let again = support::open(&support::folder("declared-names-again")); + for table in ["Mixed Case", "folded"] { + declare(&again, statements(&printed(&db, table))); + assert_eq!(printed(&again, table), printed(&db, table)); + } + // The table is found by its stored name alone. + for missing in ["Folded", "FOLDED", "mixed case"] { + assert!( + matches!(db.declaration(missing), Err(Error::TableNotFound(name)) if name == missing), + "{missing}" + ); + } + // Every spelling without quotes is the one table, and a refusal names it as it is stored. + match support::refusal(&db, "CREATE TABLE FOLDED (id INTEGER PRIMARY KEY)") { + Error::SchemaInvalid { reason } => assert!(reason.contains("'folded'"), "{reason}"), + other => panic!("a second declaration of folded was refused as {other:?}"), + } + match support::refusal(&db, "SELECT Nope FROM Folded") { + Error::ColumnNotFound { table, column } => { + assert_eq!((table.as_str(), column.as_str()), ("folded", "nope")); + } + other => panic!("a column folded does not have was refused as {other:?}"), + } + match support::refusal(&db, "CREATE INDEX FOLDED_BY_NAME ON folded (id)") { + Error::DuplicateIndex { table, index } => { + assert_eq!( + (table.as_str(), index.as_str()), + ("folded", "folded_by_name") + ); + } + other => panic!("a second index of the name was refused as {other:?}"), + } +} + +#[test] +fn a_store_of_earlier_declarations_is_printed_as_it_was() { + let folder = support::folder("earlier-declarations"); + let db = support::open(&folder); + declare( + &db, + [ + "CREATE TABLE IF NOT EXISTS notes (id INTEGER PRIMARY KEY, \ + title TEXT NOT NULL DEFAULT 'it''s', body TEXT DEFAULT NULL, \ + stars INTEGER DEFAULT 3, slug TEXT UNIQUE, \ + \"made at\" TIMESTAMP NOT NULL DEFAULT NOW(), score REAL DEFAULT -2.0, \ + kept BOOLEAN DEFAULT TRUE, tenant TEXT NOT NULL, \ + embedding VECTOR(384) PARTITION_KEY (tenant) SEARCH_MODE EXACT, \ + plain VECTOR(3), UNIQUE (title, stars))", + "CREATE INDEX by_stars ON notes (stars DESC, \"made at\" ASC)", + "CREATE INDEX by_body ON notes (body)", + // A taken name with IF NOT EXISTS declares nothing. + "CREATE TABLE IF NOT EXISTS notes (id UUID PRIMARY KEY)", + ], + ); + let text = "\ +CREATE TABLE notes ( + id INTEGER PRIMARY KEY, + title TEXT NOT NULL DEFAULT 'it''s', + body TEXT DEFAULT NULL, + stars INTEGER DEFAULT 3, + slug TEXT UNIQUE, + \"made at\" TIMESTAMP NOT NULL DEFAULT NOW(), + score REAL DEFAULT -2.0, + kept BOOLEAN DEFAULT TRUE, + tenant TEXT NOT NULL, + embedding VECTOR(384) PARTITION_KEY (tenant) MAX_PARTITIONS 256 SEARCH_MODE EXACT, + plain VECTOR(3), + UNIQUE (title, stars) +); +CREATE INDEX by_stars ON notes (stars DESC, \"made at\"); +CREATE INDEX by_body ON notes (body); +"; + assert_eq!(printed(&db, "notes"), text); + support::close(db); + let db = support::open(&folder); + assert_eq!(printed(&db, "notes"), text, "after close and open"); +} + +/// What a refused declaration must come as. +enum Refused { + /// `SchemaInvalid`, with a sentence that holds each of these. + Schema(&'static [&'static str]), + /// `ParseError`, with a sentence that holds each of these. + Parse(&'static [&'static str]), + /// `ParseError` or `SchemaInvalid`: no document says which of the two. + ParseOrSchema, + /// `StoredProcNotSupported`. + StoredProcedure, +} + +#[test] +fn a_refused_declaration_says_why_and_declares_nothing() { + let folder = support::folder("refused-declarations"); + let db = support::open(&folder); + declare( + &db, + [ + "CREATE TABLE plans (id UUID PRIMARY KEY, name TEXT NOT NULL, context_id UUID, \ + UNIQUE (context_id, id))", + ], + ); + let columns: Vec = (1..=33).map(|n| format!("c{n:02}")).collect(); + let typed: Vec = columns.iter().map(|c| format!("{c} INTEGER")).collect(); + let wide = |rule: &str| { + format!( + "CREATE TABLE gauges ({}, {rule} ({}))", + typed.join(", "), + columns.join(", ") + ) + }; + let key = "id INTEGER PRIMARY KEY"; + let cases: Vec<(String, Refused)> = vec![ + // Keeping only current versions on a table that sends its rows out and keeps the first + // write. + ( + format!("CREATE TABLE gauges ({key}) HISTORY CURRENT ONLY"), + Refused::Schema(&["gauges", "HISTORY CURRENT ONLY"]), + ), + ( + format!("CREATE TABLE gauges ({key}) HISTORY CURRENT ONLY SYNC PUSH ONLY"), + Refused::Schema(&["gauges", "HISTORY CURRENT ONLY"]), + ), + ( + format!("CREATE TABLE gauges ({key}) IMMUTABLE RETAIN 7 DAYS"), + Refused::Schema(&["gauges", "IMMUTABLE", "RETAIN"]), + ), + ( + format!("CREATE TABLE gauges ({key}) IMMUTABLE HISTORY CURRENT ONLY SYNC OFF"), + Refused::Schema(&["gauges", "IMMUTABLE", "HISTORY CURRENT ONLY"]), + ), + ( + format!("CREATE TABLE gauges ({key}) RETAIN 7 DAYS SYNC SAFE SYNC OFF"), + Refused::Schema(&["gauges", "SYNC SAFE"]), + ), + ( + format!("CREATE TABLE gauges ({key}) RETAIN 7 DAYS SYNC SAFE SYNC PULL ONLY"), + Refused::Schema(&["gauges", "SYNC SAFE"]), + ), + // A reference to columns no key or UNIQUE rule of the parent covers. + ( + format!("CREATE TABLE gauges ({key}, plan TEXT REFERENCES plans(name))"), + Refused::Schema(&["gauges"]), + ), + ( + format!( + "CREATE TABLE gauges ({key}, context_id UUID, name TEXT, \ + FOREIGN KEY (context_id, name) REFERENCES plans(context_id, name))" + ), + Refused::Schema(&["gauges"]), + ), + ( + format!("CREATE TABLE gauges ({key}, kind TEXT) MAINTAIN COUNT (kind)"), + Refused::Schema(&["gauges", "MAINTAIN COUNT"]), + ), + ( + format!("CREATE TABLE gauges ({key}, kind TEXT) MAINTAIN COUNT (id)"), + Refused::Schema(&["gauges", "MAINTAIN COUNT"]), + ), + ( + format!("CREATE TABLE gauges ({key}, source_id UUID, target_id UUID) DAG('NEEDS')"), + Refused::Schema(&["gauges", "DAG"]), + ), + // EXCLUDE VECTOR needs a state machine, and a state of it: else the rows stay searchable unsaid. + ( + format!( + "CREATE TABLE gauges ({key}, status TEXT, v VECTOR(4)) PROPAGATE ON STATE gone EXCLUDE VECTOR" + ), + Refused::Schema(&["gauges", "gone"]), + ), + ( + format!( + "CREATE TABLE gauges ({key}, status TEXT, v VECTOR(4)) STATE MACHINE (status: on -> [off]) PROPAGATE ON STATE gone EXCLUDE VECTOR" + ), + Refused::Schema(&["gauges", "gone"]), + ), + // Anything that is not a literal or NOW() is no default this version keeps. + ( + format!("CREATE TABLE gauges ({key}, c INTEGER DEFAULT $p)"), + Refused::Schema(&[]), + ), + ( + format!("CREATE TABLE gauges ({key}, c INTEGER DEFAULT -'x')"), + Refused::Schema(&[]), + ), + // More than 32 columns under a key, a UNIQUE rule and an index. + (wide("PRIMARY KEY"), Refused::Schema(&["gauges", "32"])), + (wide("UNIQUE"), Refused::Schema(&["gauges", "32"])), + // A direction written inside RETAIN names the clause to write instead. + ( + format!("CREATE TABLE gauges ({key}) RETAIN 24 HOURS SYNC SAFE PUSH ONLY"), + Refused::Parse(&["SYNC PUSH ONLY"]), + ), + ( + format!("CREATE TABLE gauges ({key}) RETAIN 24 HOURS SYNC SAFE TWO WAY"), + Refused::Parse(&["SYNC TWO WAY"]), + ), + // One key to a table. + ( + "CREATE TABLE gauges (a INTEGER PRIMARY KEY, b TEXT, PRIMARY KEY (a, b))".to_owned(), + Refused::ParseOrSchema, + ), + ( + "CREATE TABLE gauges (a INTEGER, b TEXT, PRIMARY KEY (a, b), PRIMARY KEY (a))" + .to_owned(), + Refused::ParseOrSchema, + ), + ( + "CREATE PROCEDURE tidy () AS 'DELETE FROM plans'".to_owned(), + Refused::StoredProcedure, + ), + ( + "create function one () RETURNS INTEGER AS 'SELECT 1'".to_owned(), + Refused::StoredProcedure, + ), + ]; + let declared = printed(&db, "plans"); + for (sql, refused) in &cases { + let error = support::refusal(&db, sql); + let says = |text: &str, holds: &[&str]| holds.iter().all(|part| text.contains(part)); + let as_expected = match (refused, &error) { + (Refused::Schema(holds), Error::SchemaInvalid { reason }) => says(reason, holds), + (Refused::Parse(holds), Error::ParseError(text)) => says(text, holds), + (Refused::ParseOrSchema, Error::ParseError(_) | Error::SchemaInvalid { .. }) => true, + (Refused::StoredProcedure, Error::StoredProcNotSupported) => true, + _ => false, + }; + assert!(as_expected, "{sql}\nwas refused as {error:?}"); + assert_eq!(db.table_names(), ["plans"], "after {sql}"); + } + assert_eq!(printed(&db, "plans"), declared); + + // An index over 33 columns of a table that has them is refused the same way, and one over + // 32 is made. + let keyed = typed + .join(", ") + .replacen("INTEGER", "INTEGER PRIMARY KEY", 1); + support::run(&db, &format!("CREATE TABLE broad ({keyed})")); + let index = |count: usize| { + let columns = columns[..count].join(", "); + format!("CREATE INDEX broad_by ON broad ({columns})") + }; + match support::refusal(&db, &index(33)) { + Error::SchemaInvalid { reason } => assert!(reason.contains("32"), "{reason}"), + other => panic!("an index over 33 columns was refused as {other:?}"), + } + assert!(!printed(&db, "broad").contains("broad_by")); + support::run(&db, &index(32)); + assert!(printed(&db, "broad").contains("broad_by")); +} + +#[test] +fn a_declaration_that_says_a_thing_twice_or_two_ways_is_a_parse_error_that_names_it() { + let db = support::open(&support::folder("declared-twice")); + let key = "CREATE TABLE t (id UUID PRIMARY KEY)"; + let cases = [ + // A direction inside RETAIN names the clause to write instead. + ( + format!("{key} RETAIN 24 HOURS SYNC SAFE PUSH ONLY"), + "SYNC PUSH ONLY", + ), + ( + format!("{key} RETAIN 24 HOURS SYNC SAFE TWO WAY"), + "SYNC TWO WAY", + ), + ( + "ALTER TABLE t SET RETAIN 24 HOURS SYNC SAFE PUSH ONLY".to_owned(), + "SYNC PUSH ONLY", + ), + // An option written twice. + (format!("{key} SYNC OFF SYNC TWO WAY"), "SYNC"), + (format!("{key} RETAIN 1 DAYS RETAIN 2 DAYS"), "RETAIN"), + (format!("{key} HISTORY ALL HISTORY CURRENT ONLY"), "HISTORY"), + ( + format!("{key} SYNC CONFLICT KEEP FIRST SYNC CONFLICT KEEP LATEST"), + "SYNC CONFLICT", + ), + ( + format!("{key} MAINTAIN COUNT NONE MAINTAIN COUNT ALL PREFIXES"), + "MAINTAIN COUNT", + ), + // A column that says a thing twice, in a new table and in an added column. + ( + "CREATE TABLE t (id UUID PRIMARY KEY, o UUID REFERENCES p(id) REFERENCES q(id))" + .to_owned(), + "REFERENCES", + ), + ( + "ALTER TABLE t ADD COLUMN c TEXT NOT NULL NOT NULL".to_owned(), + "NOT NULL", + ), + // A list that names a column twice, or a column the table does not have. + ( + "CREATE TABLE t (a INTEGER, b TEXT, PRIMARY KEY (a, a))".to_owned(), + "'a'", + ), + ( + "CREATE TABLE t (a INTEGER, b TEXT, PRIMARY KEY (a, nope))".to_owned(), + "'nope'", + ), + ( + "CREATE TABLE t (a INTEGER, b TEXT, FOREIGN KEY (a, nope) REFERENCES p(a, b))" + .to_owned(), + "'nope'", + ), + // A reference of as many columns on one side as on the other. + ( + "CREATE TABLE t (a INTEGER, b TEXT, FOREIGN KEY (a, b) REFERENCES p(a))".to_owned(), + "FOREIGN KEY", + ), + // DEFAULT takes a declared list away, and a new table has none. + ( + format!("{key} MAINTAIN COUNT DEFAULT"), + "MAINTAIN COUNT DEFAULT", + ), + // What is neither this version's nor a later one's is a syntax error at the word. + ( + "ALTER TABLE t DROP COLUMN c RESTRICT CASCADE".to_owned(), + "'CASCADE'", + ), + ("ALTER TABLE t SET HISTORY NONE".to_owned(), "'NONE'"), + ( + "ALTER TABLE t ALTER COLUMN v SET PARTITION_KEY (a)".to_owned(), + "'PARTITION_KEY'", + ), + ("ALTER TABLE t RENAME a TO b".to_owned(), "'a'"), + ("DROP INDEX i".to_owned(), "ends too soon"), + ("DROP TABLE".to_owned(), "ends too soon"), + (format!("{key} SYNC SIDEWAYS"), "'SIDEWAYS'"), + (format!("{key} EDGE DISCARD SOMETIMES"), "'SOMETIMES'"), + (format!("{key} RETAIN 7 WEEKS"), "'WEEKS'"), + ]; + for (sql, names) in &cases { + match support::refusal(&db, sql) { + Error::ParseError(text) => { + assert!(text.contains(names), "{sql}\nwas refused with: {text}"); + } + other => panic!("{sql}\nwas not refused as a parse error: {other:?}"), + } + } + // A column that expires is refused by name. + let sql = "ALTER TABLE t ADD COLUMN until TIMESTAMP EXPIRES"; + match support::refusal(&db, sql) { + Error::PlanError(text) => assert_eq!( + text, "EXPIRES is not supported in this version of the library", + "{sql}" + ), + other => panic!("{sql}\nwas not refused by name: {other:?}"), + } + // The tuning clauses of an index no version reads are refused by name, and the sentence says + // what to write instead. + for (sql, name) in [ + ("ALTER TABLE t ALTER COLUMN v SET SEARCH DEFAULT", "SEARCH"), + ( + "ALTER TABLE t ALTER COLUMN v SET SEARCH (TARGET_ACCURACY = 95)", + "SEARCH", + ), + ("ALTER TABLE t ALTER COLUMN v SET IVF DEFAULT", "IVF"), + ( + "ALTER TABLE t ALTER COLUMN v SET IVF (LISTS = 64, PROBES = 8, REFINE_FACTOR = 2)", + "IVF", + ), + ] { + match support::refusal(&db, sql) { + Error::PlanError(text) => assert_eq!( + text, + format!( + "{name} (...) is not part of this library: the index sizes its own groups and \ + measures how far a search scans; for the true nearest rows write USE VECTOR \ + EXACT or SEARCH_MODE EXACT, and to move the size from which AUTO uses the \ + index write AUTO_INDEX_AT" + ), + "{sql}" + ), + other => panic!("{sql}\nwas not refused by name: {other:?}"), + } + } + for sql in [ + "create procedure tidy", + "CREATE FUNCTION one () RETURNS INTEGER AS 'SELECT 1'", + "Create Function one", + ] { + let error = support::refusal(&db, sql); + assert!( + matches!(error, Error::StoredProcNotSupported), + "{sql}: {error:?}" + ); + } + // A table may still be named for either word. + support::run( + &db, + "CREATE TABLE procedure (id INTEGER PRIMARY KEY, function TEXT)", + ); +} + +#[test] +fn a_rank_policy_finds_the_column_it_joins_by_one_rule_or_is_refused() { + let folder = support::folder("rank-policy-join"); + let db = support::open(&folder); + declare( + &db, + [ + "CREATE TABLE results (id UUID PRIMARY KEY, score REAL, label TEXT, task_id UUID)", + "CREATE INDEX results_by_task ON results (task_id)", + ], + ); + let policy = |join: &str, formula: &str| { + format!( + "embedding VECTOR(4) RANK_POLICY (JOIN {join}, FORMULA '{formula}', SORT_KEY by_score)" + ) + }; + let by_key = policy("results ON id", "coalesce({score}, 1.0)"); + let by_task = policy("results ON task_id", "coalesce({score}, 1.0)"); + + // Joined by the joined table's key: the anchor's column named for the joined table, in the + // singular or as it is; then the anchor's column of the same name. Joined by another + // column: the anchor's column of the same name. The column found is the one the policy + // reads, so it is the one that may not be dropped; the column beside it may. + let found = [ + ( + "singular", + format!("id UUID PRIMARY KEY, result_id UUID, other UUID, {by_key}"), + "result_id", + ), + ( + "whole", + format!("id UUID PRIMARY KEY, results_id UUID, other UUID, {by_key}"), + "results_id", + ), + ( + "named", + format!("id UUID PRIMARY KEY, task_id UUID, other UUID, {by_task}"), + "task_id", + ), + ]; + for (table, columns, joined) in &found { + support::run(&db, &format!("CREATE TABLE {table} ({columns})")); + match support::refusal(&db, &format!("ALTER TABLE {table} DROP COLUMN {joined}")) { + Error::DropBlockedByRankPolicy { + table: of, + column, + policy, + } => { + assert_eq!( + (of.as_str(), column.as_str(), policy.as_str()), + (*table, *joined, "by_score") + ); + } + other => panic!("dropping {table}.{joined} was refused as {other:?}"), + } + support::run(&db, &format!("ALTER TABLE {table} DROP COLUMN other")); + } + + // An `id` on both sides of the join is ambiguous in a formula, in a pair of tables whose + // ids are numbers. + support::run(&db, "CREATE TABLE counted (id INTEGER PRIMARY KEY, n REAL)"); + let sql = format!( + "CREATE TABLE ambiguous (id INTEGER PRIMARY KEY, counted_id INTEGER, {})", + policy("counted ON id", "{id} * 2.0") + ); + assert!( + matches!(support::refusal(&db, &sql), Error::RankPolicyInvalid { .. }), + "{sql}\nnames an `id` that both tables have" + ); + + // What cannot stand is refused as the policy's own kind, naming the VECTOR column, and + // declares nothing. + let tables = db.table_names(); + let refused = [ + // Two columns the first rule finds. + format!("id UUID PRIMARY KEY, result_id UUID, results_id UUID, {by_key}"), + ]; + let formulas = [ + // A table, and a column, that are not there; a column no index leads with. + ("nowhere ON id", "1.0"), + ("results ON nope", "1.0"), + ("results ON label", "1.0"), + // A formula over a column that is no number, over one that is not there, and one that + // does not read. + ("results ON id", "coalesce({label}, 1.0)"), + ("results ON id", "coalesce({nope}, 1.0)"), + ("results ON id", "coalesce({score}, "), + // A formula outside the documented grammar (`{column}`, `{vector_score}`, numbers, + // `+`, `*`, parentheses, `coalesce(expr, literal)`): subtraction, division, a function + // call, a name without braces; numbers the SQL grammar has no form for; a trailing + // operator; a brace never closed; a dotted reference; a column as the second value of + // `coalesce`; a minus sign; two values and no operator; a subquery. + ("results ON id", "{score} - 1.0"), + ("results ON id", "{score} / 2.0"), + ("results ON id", "{score} * NOW()"), + ("results ON id", "{score} * score"), + ("results ON id", "{score} * 1e1"), + ("results ON id", "({score} + )"), + ("results ON id", "{score + {score}"), + ("results ON id", "{results.score}"), + ("results ON id", "coalesce({score}, {score})"), + ("results ON id", "{score} {score}"), + ("results ON id", "{score} * (SELECT 1)"), + ]; + let refused = refused.into_iter().chain( + formulas.map(|(join, formula)| format!("id UUID PRIMARY KEY, {}", policy(join, formula))), + ); + for columns in refused { + let sql = format!("CREATE TABLE ranked ({columns})"); + match support::refusal(&db, &sql) { + Error::RankPolicyInvalid { index, reason } => { + assert_eq!( + (index.table.as_str(), index.column.as_str()), + ("ranked", "embedding") + ); + assert!(!reason.is_empty(), "{sql}"); + } + other => panic!("{sql}\nwas refused as {other:?}"), + } + assert_eq!(db.table_names(), tables, "after {sql}"); + } + + // Formulas that stand: `{vector_score}` beside `*` and a joined column; `coalesce` is a + // function, so a column that is named coalesce is not read by the policy and may be dropped; + // a column read through parentheses and `coalesce` is still read; capitals and a whole + // number are read. + support::run(&db, "CREATE TABLE lone (id UUID PRIMARY KEY, score REAL)"); + for (table, columns, join, formula) in [ + ( + "scored", + "result_id UUID, coalesce REAL", + "results", + "coalesce({score}, 1.0)", + ), + ( + "nested", + "lone_id UUID", + "lone", + "({vector_score} + 1) * coalesce({score}, 0.5)", + ), + ("capitals", "lone_id UUID", "lone", "COALESCE({score}, 1)"), + ] { + let sql = format!( + "CREATE TABLE {table} (id UUID PRIMARY KEY, {columns}, {})", + policy(&format!("{join} ON id"), formula) + ); + support::run(&db, &sql); + assert!( + printed(&db, table).contains(formula), + "{table} prints its formula as written" + ); + if table == "nested" { + // Read through the parentheses and the `coalesce`, so `score` may not be dropped + // from the table it is joined from. + match support::refusal(&db, "ALTER TABLE lone DROP COLUMN score") { + Error::DropBlockedByRankPolicy { table, column, .. } => { + assert_eq!((table.as_str(), column.as_str()), ("lone", "score")); + } + other => panic!("dropping lone.score was refused as {other:?}"), + } + } + } + support::run(&db, "ALTER TABLE scored DROP COLUMN coalesce"); +} diff --git a/v2/contextdb/tests/fixtures/consumer_schema.sql b/v2/contextdb/tests/fixtures/consumer_schema.sql new file mode 100644 index 00000000..f80baedf --- /dev/null +++ b/v2/contextdb/tests/fixtures/consumer_schema.sql @@ -0,0 +1,127 @@ +-- The table and index declarations of a program built on this library, each as that program +-- writes it. One statement to a line, ended by `;`. A table comes after the tables it +-- references. `tests/declarations.rs` declares every one and reads it back. + +CREATE TABLE cg_schema_version (id INTEGER PRIMARY KEY, version INTEGER NOT NULL, installed_at TXID NOT NULL) SYNC TWO WAY SYNC CONFLICT KEEP FIRST; +CREATE TABLE contexts (id UUID PRIMARY KEY CONTEXT_ID, name TEXT NOT NULL UNIQUE, labels JSON, properties JSON, created_at TXID NOT NULL) SYNC TWO WAY SYNC CONFLICT KEEP FIRST; +CREATE TABLE events (id UUID PRIMARY KEY, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), connector_id TEXT NOT NULL, adapter_version TEXT NOT NULL, anchor_declaration_version TEXT NOT NULL, source_instance_ref TEXT NOT NULL, session_ref TEXT NOT NULL, source_event_key TEXT NOT NULL, source_position JSON NOT NULL, seq INTEGER NOT NULL, content_hash TEXT NOT NULL, anchor_count INTEGER NOT NULL, anchor_set_hash TEXT NOT NULL, occurred_at TIMESTAMP, author TEXT NOT NULL, kind TEXT NOT NULL, render_role TEXT NOT NULL, interaction_ref TEXT, text TEXT, payload JSON NOT NULL DEFAULT '{}', anchors JSON NOT NULL DEFAULT '{}', source_remainder JSON NOT NULL DEFAULT '{}', coverage TEXT NOT NULL, UNIQUE (context_id, id), UNIQUE (context_id, connector_id, session_ref, source_event_key), UNIQUE (context_id, connector_id, source_instance_ref, seq)) IMMUTABLE SYNC OFF SYNC CONFLICT KEEP FIRST MAINTAIN COUNT (context_id), (context_id, connector_id, source_instance_ref); +CREATE TABLE event_anchors (id UUID PRIMARY KEY, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), event_id UUID NOT NULL REFERENCES events(id), correlation_scope_hash TEXT NOT NULL, declaration_version TEXT NOT NULL, connector_id TEXT NOT NULL, source_instance_ref TEXT NOT NULL, session_ref TEXT NOT NULL, render_role TEXT NOT NULL, interaction_ref TEXT, key TEXT NOT NULL, value TEXT NOT NULL, identity_value TEXT NOT NULL, original_value JSON NOT NULL, UNIQUE (context_id, id), UNIQUE (context_id, event_id, declaration_version, key, identity_value), FOREIGN KEY (context_id, event_id) REFERENCES events(context_id, id)) IMMUTABLE SYNC OFF SYNC CONFLICT KEEP FIRST MAINTAIN COUNT NONE; +CREATE TABLE event_local_commits (event_id UUID PRIMARY KEY REFERENCES events(id), context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), committed_at TXID NOT NULL, FOREIGN KEY (context_id, event_id) REFERENCES events(context_id, id)) IMMUTABLE SYNC OFF MAINTAIN COUNT NONE; +CREATE TABLE adapter_coverage_gaps (id UUID PRIMARY KEY, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), connector_id TEXT NOT NULL, captured_adapter_version TEXT NOT NULL, fingerprint_sha256 TEXT NOT NULL, first_seen_at TXID NOT NULL, last_seen_at TXID NOT NULL, UNIQUE (context_id, id), UNIQUE (context_id, connector_id, captured_adapter_version, fingerprint_sha256)) SYNC OFF MAINTAIN COUNT (context_id); +CREATE TABLE adapter_coverage_gap_events (context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), gap_id UUID NOT NULL REFERENCES adapter_coverage_gaps(id), event_id UUID NOT NULL REFERENCES events(id), PRIMARY KEY (context_id, gap_id, event_id), FOREIGN KEY (context_id, gap_id) REFERENCES adapter_coverage_gaps(context_id, id), FOREIGN KEY (context_id, event_id) REFERENCES events(context_id, id)) IMMUTABLE SYNC OFF; +CREATE TABLE event_interpretation_generations (id UUID PRIMARY KEY, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), connector_id TEXT NOT NULL, accepted_source_declarations JSON NOT NULL, target_interpretation_version TEXT NOT NULL, target_adapter_version TEXT NOT NULL, target_anchor_declaration_version TEXT NOT NULL, mapping_contract_id TEXT NOT NULL, mapping_contract_sha256 TEXT NOT NULL, event_count INTEGER NOT NULL, ordered_source_set_sha256 TEXT NOT NULL, source_content_sha256 TEXT NOT NULL, capture_ordinal INTEGER NOT NULL DEFAULT 0, build_ordinal INTEGER NOT NULL DEFAULT 0, claim_end_ordinal INTEGER, claim_membership_sha256 TEXT, lease_epoch INTEGER NOT NULL DEFAULT 0, lease_token TEXT, lease_expires_at TIMESTAMP, state TEXT NOT NULL, eligible_event_count INTEGER NOT NULL DEFAULT 0, interpreted_event_count INTEGER NOT NULL DEFAULT 0, failed_event_count INTEGER NOT NULL DEFAULT 0, effective_content_sha256 TEXT, sanitized_failure TEXT, started_at TXID NOT NULL, completed_at TXID, activated_at TXID, activation_receipt JSON, UNIQUE (context_id, id), UNIQUE (context_id, connector_id, id), UNIQUE (context_id, connector_id, target_interpretation_version, event_count, ordered_source_set_sha256)) SYNC OFF; +CREATE TABLE event_interpretation_source_events (context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), generation_id UUID NOT NULL REFERENCES event_interpretation_generations(id), ordinal INTEGER NOT NULL, event_id UUID NOT NULL REFERENCES events(id), source_instance_ref TEXT NOT NULL, source_sequence INTEGER NOT NULL, source_row_sha256 TEXT NOT NULL, PRIMARY KEY (context_id, generation_id, ordinal), UNIQUE (context_id, generation_id, event_id), FOREIGN KEY (context_id, generation_id) REFERENCES event_interpretation_generations(context_id, id), FOREIGN KEY (context_id, event_id) REFERENCES events(context_id, id)) IMMUTABLE SYNC OFF; +CREATE TABLE event_interpretation_values (context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), generation_id UUID NOT NULL REFERENCES event_interpretation_generations(id), event_id UUID NOT NULL REFERENCES events(id), captured_content_sha256 TEXT NOT NULL, occurred_at TIMESTAMP, author TEXT NOT NULL, kind TEXT NOT NULL, render_role TEXT NOT NULL, interaction_ref TEXT, text TEXT, payload JSON NOT NULL, anchors JSON NOT NULL, coverage TEXT NOT NULL, effective_sha256 TEXT NOT NULL, created_at TXID NOT NULL, PRIMARY KEY (context_id, generation_id, event_id), UNIQUE (context_id, generation_id, event_id), FOREIGN KEY (context_id, generation_id) REFERENCES event_interpretation_generations(context_id, id), FOREIGN KEY (context_id, event_id) REFERENCES events(context_id, id)) IMMUTABLE SYNC OFF; +CREATE TABLE event_interpretation_anchors (id UUID PRIMARY KEY, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), generation_id UUID NOT NULL REFERENCES event_interpretation_generations(id), event_id UUID NOT NULL REFERENCES events(id), declaration_version TEXT NOT NULL, key TEXT NOT NULL, canonical_value TEXT NOT NULL, identity_value TEXT NOT NULL, original_value JSON NOT NULL, UNIQUE (context_id, id), UNIQUE (context_id, generation_id, event_id, declaration_version, key, identity_value), FOREIGN KEY (context_id, generation_id, event_id) REFERENCES event_interpretation_values(context_id, generation_id, event_id)) IMMUTABLE SYNC OFF; +CREATE TABLE event_interpretation_heads (context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), connector_id TEXT NOT NULL, generation_id UUID NOT NULL REFERENCES event_interpretation_generations(id), activated_at TXID NOT NULL, PRIMARY KEY (context_id, connector_id), FOREIGN KEY (context_id, connector_id, generation_id) REFERENCES event_interpretation_generations(context_id, connector_id, id)) SYNC OFF; +CREATE TABLE event_context_revisions (context_id UUID CONTEXT_ID PRIMARY KEY REFERENCES contexts(id), revision_tx TXID NOT NULL) SYNC OFF; +CREATE TABLE event_context_counts (context_id UUID CONTEXT_ID PRIMARY KEY REFERENCES contexts(id), captured_events INTEGER NOT NULL, complete_events INTEGER NOT NULL, opaque_events INTEGER NOT NULL, payload_refused_events INTEGER NOT NULL, source_declared_missing_events INTEGER NOT NULL) SYNC OFF; +CREATE TABLE event_context_sessions (context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), connector_id TEXT NOT NULL, session_ref TEXT NOT NULL, event_count INTEGER NOT NULL, latest_occurred_at TIMESTAMP, PRIMARY KEY (context_id, connector_id, session_ref)) SYNC OFF; +CREATE TABLE event_search_request_headers (request_id UUID PRIMARY KEY, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), running_generation_id UUID, running_owner_token TEXT, UNIQUE (context_id, request_id)) SYNC OFF; +CREATE TABLE event_search_request_selections (context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), request_id UUID NOT NULL, ordinal INTEGER NOT NULL, connector_id TEXT NOT NULL, session_ref TEXT NOT NULL, source_instance_ref TEXT NOT NULL, frontier INTEGER NOT NULL, PRIMARY KEY (context_id, request_id, ordinal), FOREIGN KEY (context_id, request_id) REFERENCES event_search_request_headers(context_id, request_id)) SYNC OFF; +CREATE TABLE event_search_refusal_headers (request_id UUID PRIMARY KEY, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), UNIQUE (context_id, request_id)) SYNC OFF; +CREATE TABLE event_search_refusal_frontiers (context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), request_id UUID NOT NULL, ordinal INTEGER NOT NULL, connector_id TEXT NOT NULL, session_ref TEXT NOT NULL, source_instance_ref TEXT NOT NULL, frontier INTEGER NOT NULL, PRIMARY KEY (context_id, request_id, ordinal), FOREIGN KEY (context_id, request_id) REFERENCES event_search_refusal_headers(context_id, request_id)) SYNC OFF; +CREATE TABLE event_search_work_policies (context_id UUID CONTEXT_ID PRIMARY KEY REFERENCES contexts(id), search_order TEXT NOT NULL, search_sessions INTEGER) SYNC OFF; +CREATE TABLE event_placement (id INTEGER PRIMARY KEY, expected_policy TEXT NOT NULL, first_event_tx TXID, installation_table_baseline JSON) SYNC OFF; +CREATE TABLE context_bindings (namespace TEXT NOT NULL, opaque_reference TEXT NOT NULL, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), PRIMARY KEY (namespace, opaque_reference)) SYNC OFF; +CREATE TABLE event_source_locators (context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), connector_id TEXT NOT NULL, source_locator_ref TEXT NOT NULL, source_instance_ref TEXT NOT NULL, session_ref TEXT NOT NULL, provisional_empty BOOLEAN NOT NULL, adapter_version TEXT NOT NULL, anchor_declaration_version TEXT NOT NULL, PRIMARY KEY (context_id, connector_id, source_locator_ref)) SYNC OFF; +CREATE TABLE event_source_checkpoints (context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), connector_id TEXT NOT NULL, source_instance_ref TEXT NOT NULL, session_ref TEXT NOT NULL, adapter_version TEXT NOT NULL, anchor_declaration_version TEXT NOT NULL, checkpoint JSON NOT NULL, source_position JSON NOT NULL, committed_byte_offset INTEGER NOT NULL, prefix_sha256 TEXT NOT NULL, observed_source_len INTEGER NOT NULL, latest_event_seq INTEGER, last_successful_tx TXID NOT NULL, PRIMARY KEY (context_id, connector_id, source_instance_ref)) SYNC OFF; +CREATE TABLE event_source_ordering (declaration_hash TEXT NOT NULL UNIQUE, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), connector_id TEXT NOT NULL, source_instance_ref TEXT NOT NULL, adapter_version TEXT NOT NULL, source_position_order JSON NOT NULL, PRIMARY KEY (context_id, connector_id, source_instance_ref)) SYNC OFF; +CREATE TABLE intentions (id UUID PRIMARY KEY, description TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'active', context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), origin TEXT NOT NULL, origin_ref UUID, blueprint_id UUID, bindings JSON, blueprint_proposal TEXT, blueprint_catalog_id TEXT, properties JSON, tags JSON, created_at TXID NOT NULL, UNIQUE (context_id, id)) STATE MACHINE (status: active -> [paused, superseded, archived], paused -> [active, superseded, archived]) SYNC TWO WAY SYNC CONFLICT KEEP LATEST MAINTAIN COUNT (context_id); +CREATE TABLE decisions (id UUID PRIMARY KEY, decision_type TEXT NOT NULL IMMUTABLE, description TEXT NOT NULL IMMUTABLE, reasoning JSON, confidence REAL, status TEXT NOT NULL DEFAULT 'active', agent_id TEXT, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), properties JSON, tags JSON, created_at TXID NOT NULL, UNIQUE (context_id, id)) STATE MACHINE (status: active -> [superseded, archived, invalidated]) SYNC TWO WAY SYNC CONFLICT KEEP LATEST MAINTAIN COUNT (context_id); +CREATE TABLE outcomes (id UUID NOT NULL UNIQUE, decision_id UUID NOT NULL REFERENCES decisions(id), context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), success BOOLEAN NOT NULL, notes TEXT, recorded_at TXID NOT NULL, PRIMARY KEY (context_id, decision_id), UNIQUE (context_id, id), FOREIGN KEY (context_id, decision_id) REFERENCES decisions(context_id, id)) SYNC TWO WAY SYNC CONFLICT KEEP LATEST; +CREATE TABLE conflicts (id UUID NOT NULL UNIQUE, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), older_decision_id UUID NOT NULL REFERENCES decisions(id), newer_decision_id UUID NOT NULL REFERENCES decisions(id), threatened_intention_ids JSON NOT NULL DEFAULT '[]', intention_in_tension BOOLEAN NOT NULL, detected_at TXID NOT NULL, PRIMARY KEY (context_id, older_decision_id, newer_decision_id), FOREIGN KEY (context_id, older_decision_id) REFERENCES decisions(context_id, id), FOREIGN KEY (context_id, newer_decision_id) REFERENCES decisions(context_id, id)) IMMUTABLE SYNC TWO WAY SYNC CONFLICT KEEP FIRST; +CREATE TABLE entities (id UUID PRIMARY KEY, entity_type TEXT NOT NULL, name TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'active', context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), properties JSON, tags JSON, created_at TXID NOT NULL, UNIQUE (context_id, id)) STATE MACHINE (status: active -> [archived]) SYNC TWO WAY SYNC CONFLICT KEEP LATEST; +CREATE TABLE observations (id UUID PRIMARY KEY, observation_type TEXT NOT NULL, observed_properties JSON NOT NULL, state_delta JSON NOT NULL, properties JSON, source TEXT NOT NULL, entity_id UUID NOT NULL REFERENCES entities(id), context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), timestamp TIMESTAMP NOT NULL, created_at TXID NOT NULL, UNIQUE (context_id, id), UNIQUE (context_id, entity_id, id), FOREIGN KEY (context_id, entity_id) REFERENCES entities(context_id, id)) IMMUTABLE SYNC TWO WAY SYNC CONFLICT KEEP FIRST MAINTAIN COUNT (context_id); +CREATE TABLE invalidations (id UUID NOT NULL UNIQUE, status TEXT NOT NULL DEFAULT 'pending', context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), affected_decision_id UUID NOT NULL REFERENCES decisions(id), trigger_observation_id UUID NOT NULL REFERENCES observations(id), reason TEXT, basis_diff JSON NOT NULL, cascade_from UUID REFERENCES decisions(id), severity TEXT NOT NULL, threatened_intention_ids JSON NOT NULL DEFAULT '[]', intention_now_un_served BOOLEAN NOT NULL DEFAULT false, detected_at TXID NOT NULL, resolved_at TXID, resolution_action TEXT, resolution_decision_id UUID, PRIMARY KEY (context_id, affected_decision_id), FOREIGN KEY (context_id, affected_decision_id) REFERENCES decisions(context_id, id), FOREIGN KEY (context_id, trigger_observation_id) REFERENCES observations(context_id, id), FOREIGN KEY (context_id, cascade_from) REFERENCES decisions(context_id, id), FOREIGN KEY (context_id, resolution_decision_id) REFERENCES decisions(context_id, id)) STATE MACHINE (status: pending -> [acknowledged], acknowledged -> [resolved, dismissed]) SYNC TWO WAY SYNC CONFLICT KEEP LATEST; +CREATE TABLE evidence (id UUID PRIMARY KEY, observation_id UUID NOT NULL REFERENCES observations(id), context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), kind TEXT NOT NULL, source_ref TEXT NOT NULL, mime_type TEXT, content_hash TEXT, captured_at TIMESTAMP, captured_at_nanos INTEGER, byte_range JSON, time_range_ms JSON, frame_index INTEGER, region JSON, producer JSON, text_projection TEXT, derived_from UUID REFERENCES evidence(id), retention_status TEXT NOT NULL, media_retention_until TIMESTAMP, media_retention_until_nanos INTEGER, properties JSON, supply_order INTEGER NOT NULL, vector_text VECTOR(768), vector_vision VECTOR(512), UNIQUE (context_id, id), FOREIGN KEY (context_id, observation_id) REFERENCES observations(context_id, id), FOREIGN KEY (context_id, derived_from) REFERENCES evidence(context_id, id)) IMMUTABLE SYNC TWO WAY SYNC CONFLICT KEEP FIRST; +CREATE TABLE embedding_spaces (embedding_space_id TEXT PRIMARY KEY, dimension INTEGER NOT NULL, metric TEXT NOT NULL, model_family TEXT, model_name TEXT, model_version TEXT, supported_evidence_kinds JSON, default_for_evidence_kind TEXT, bindings JSON, created_at TXID NOT NULL) SYNC TWO WAY SYNC CONFLICT KEEP FIRST; +CREATE TABLE event_search_generations (id UUID PRIMARY KEY, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), interpretation_generation_id UUID, interpretation_snapshot JSON NOT NULL, interpretation_snapshot_sha256 TEXT NOT NULL, event_count INTEGER NOT NULL, ordered_source_set_sha256 TEXT NOT NULL, source_content_sha256 TEXT NOT NULL DEFAULT '', capture_ordinal INTEGER NOT NULL DEFAULT 0, renderer_id TEXT NOT NULL, renderer_version TEXT NOT NULL, renderer_profile JSON NOT NULL, embedding_space_id TEXT NOT NULL REFERENCES embedding_spaces(embedding_space_id), encoding_profile TEXT NOT NULL, state TEXT NOT NULL, required_event_count INTEGER NOT NULL DEFAULT 0, searchable_event_count INTEGER NOT NULL DEFAULT 0, searchable_excluded_event_count INTEGER NOT NULL DEFAULT 0, eligible_event_count INTEGER NOT NULL DEFAULT 0, rendered_event_count INTEGER NOT NULL DEFAULT 0, embedded_event_count INTEGER NOT NULL DEFAULT 0, excluded_event_count INTEGER NOT NULL DEFAULT 0, failed_event_count INTEGER NOT NULL DEFAULT 0, passage_count INTEGER NOT NULL DEFAULT 0, started_at TXID NOT NULL, completed_at TXID, finalized_event_revision_tx TXID, last_attempt_at TXID, last_attempt_token UUID NOT NULL, sanitized_failure TEXT, UNIQUE (context_id, id), FOREIGN KEY (context_id, interpretation_generation_id) REFERENCES event_interpretation_generations(context_id, id)) SYNC OFF; +CREATE TABLE event_search_source_events (membership_id UUID NOT NULL UNIQUE, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), generation_id UUID NOT NULL REFERENCES event_search_generations(id), ordinal INTEGER NOT NULL, event_id UUID NOT NULL REFERENCES events(id), connector_id TEXT NOT NULL, source_instance_ref TEXT NOT NULL, session_ref TEXT NOT NULL, source_sequence INTEGER NOT NULL, effective_interpretation_generation_id UUID, captured_source_sha256 TEXT NOT NULL, effective_source_sha256 TEXT NOT NULL, PRIMARY KEY (context_id, generation_id, ordinal), UNIQUE (context_id, generation_id, event_id), FOREIGN KEY (context_id, generation_id) REFERENCES event_search_generations(context_id, id), FOREIGN KEY (context_id, event_id) REFERENCES events(context_id, id), FOREIGN KEY (context_id, effective_interpretation_generation_id, event_id) REFERENCES event_interpretation_values(context_id, generation_id, event_id)) IMMUTABLE SYNC OFF; +CREATE TABLE event_search_session_progress (context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), generation_id UUID NOT NULL REFERENCES event_search_generations(id), connector_id TEXT NOT NULL, session_ref TEXT NOT NULL, event_count INTEGER NOT NULL, committed_event_count INTEGER NOT NULL DEFAULT 0, eligible_event_count INTEGER NOT NULL DEFAULT 0, excluded_event_count INTEGER NOT NULL DEFAULT 0, searchable_event_count INTEGER NOT NULL DEFAULT 0, no_searchable_text_event_count INTEGER NOT NULL DEFAULT 0, is_required BOOLEAN NOT NULL DEFAULT false, is_published BOOLEAN NOT NULL DEFAULT false, PRIMARY KEY (context_id, generation_id, connector_id, session_ref), FOREIGN KEY (context_id, generation_id) REFERENCES event_search_generations(context_id, id)) SYNC OFF; +CREATE TABLE event_search_passages (id UUID PRIMARY KEY, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), generation_id UUID NOT NULL REFERENCES event_search_generations(id), connector_id TEXT NOT NULL, session_ref TEXT NOT NULL, source_instance_ref TEXT NOT NULL, passage_ordinal INTEGER NOT NULL, first_event_sequence INTEGER NOT NULL, last_event_sequence INTEGER NOT NULL, rendered_text TEXT NOT NULL, content_sha256 TEXT NOT NULL, vector_text VECTOR(768) PARTITION_KEY (context_id, generation_id) MAX_PARTITIONS 4096 SEARCH_MODE AUTO, dimension INTEGER NOT NULL, is_current BOOLEAN NOT NULL, is_searchable BOOLEAN NOT NULL DEFAULT true, created_at TXID NOT NULL, UNIQUE (context_id, id), UNIQUE (context_id, generation_id, id), UNIQUE (context_id, generation_id, connector_id, source_instance_ref, passage_ordinal), FOREIGN KEY (context_id, generation_id) REFERENCES event_search_generations(context_id, id)) HISTORY CURRENT ONLY SYNC OFF; +CREATE TABLE event_search_passage_sources (context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), generation_id UUID NOT NULL REFERENCES event_search_generations(id), passage_id UUID NOT NULL REFERENCES event_search_passages(id), event_id UUID NOT NULL REFERENCES events(id), connector_id TEXT NOT NULL, session_ref TEXT NOT NULL, source_instance_ref TEXT NOT NULL, source_order INTEGER NOT NULL, event_start_byte INTEGER, event_end_byte INTEGER, passage_start_byte INTEGER NOT NULL, passage_end_byte INTEGER NOT NULL, PRIMARY KEY (context_id, passage_id, source_order), FOREIGN KEY (context_id, generation_id, passage_id) REFERENCES event_search_passages(context_id, generation_id, id), FOREIGN KEY (context_id, event_id) REFERENCES events(context_id, id)) IMMUTABLE SYNC OFF; +CREATE TABLE event_search_embedding_handles (id UUID PRIMARY KEY, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), generation_id UUID NOT NULL REFERENCES event_search_generations(id), passage_id UUID NOT NULL REFERENCES event_search_passages(id), embedding_space_id TEXT NOT NULL REFERENCES embedding_spaces(embedding_space_id), produced_by JSON NOT NULL, producer_system TEXT NOT NULL, created_at TXID NOT NULL, UNIQUE (context_id, id), UNIQUE (context_id, passage_id, embedding_space_id), FOREIGN KEY (context_id, generation_id, passage_id) REFERENCES event_search_passages(context_id, generation_id, id)) IMMUTABLE SYNC OFF; +CREATE TABLE event_search_heads (context_id UUID CONTEXT_ID PRIMARY KEY REFERENCES contexts(id), generation_id UUID NOT NULL REFERENCES event_search_generations(id), activated_at TXID NOT NULL, FOREIGN KEY (context_id, generation_id) REFERENCES event_search_generations(context_id, id)) SYNC OFF; +CREATE TABLE event_search_cursors (context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), generation_id UUID NOT NULL REFERENCES event_search_generations(id), connector_id TEXT NOT NULL, source_instance_ref TEXT NOT NULL, session_ref TEXT NOT NULL, committed_sequence INTEGER NOT NULL, open_tail_start_sequence INTEGER, open_tail_members JSON NOT NULL, open_tail_passage_id UUID, watermark_sha256 TEXT NOT NULL, updated_at TXID NOT NULL, PRIMARY KEY (context_id, generation_id, connector_id, source_instance_ref, session_ref), FOREIGN KEY (context_id, generation_id) REFERENCES event_search_generations(context_id, id)) SYNC OFF; +CREATE TABLE event_search_leases (context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), generation_id UUID NOT NULL REFERENCES event_search_generations(id), owner_token TEXT NOT NULL, lease_epoch INTEGER NOT NULL, claimed_connector_id TEXT, claimed_source_instance_ref TEXT, claimed_start_sequence INTEGER, claimed_end_sequence INTEGER, cursor_sha256 TEXT NOT NULL, expires_at TIMESTAMP NOT NULL, PRIMARY KEY (context_id, generation_id), FOREIGN KEY (context_id, generation_id) REFERENCES event_search_generations(context_id, id)) SYNC OFF; +CREATE TABLE embedding_space_bindings (embedding_space_id TEXT NOT NULL REFERENCES embedding_spaces(embedding_space_id), table_name TEXT NOT NULL, vector_column TEXT NOT NULL, PRIMARY KEY (embedding_space_id, table_name, vector_column)) SYNC TWO WAY SYNC CONFLICT KEEP FIRST; +CREATE TABLE evidence_embedding_handles (id UUID PRIMARY KEY, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), target_id UUID NOT NULL REFERENCES evidence(id), embedding_space_id TEXT NOT NULL REFERENCES embedding_spaces(embedding_space_id), produced_by JSON, producer_system TEXT NOT NULL, created_at TXID NOT NULL, FOREIGN KEY (context_id, target_id) REFERENCES evidence(context_id, id)) IMMUTABLE SYNC TWO WAY SYNC CONFLICT KEEP FIRST; +CREATE TABLE entity_reference_embedding_handles (id UUID PRIMARY KEY, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), target_id UUID NOT NULL, embedding_space_id TEXT NOT NULL REFERENCES embedding_spaces(embedding_space_id), produced_by JSON, producer_system TEXT NOT NULL, created_at TXID NOT NULL, FOREIGN KEY (context_id, target_id) REFERENCES entities(context_id, id)) IMMUTABLE SYNC TWO WAY SYNC CONFLICT KEEP FIRST; +CREATE TABLE entity_reference_embeddings_128 (id UUID PRIMARY KEY, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), entity_id UUID NOT NULL, label TEXT, vector VECTOR(128), embedding_space_id TEXT NOT NULL REFERENCES embedding_spaces(embedding_space_id), produced_by JSON, created_at TXID NOT NULL, UNIQUE (entity_id, label), UNIQUE (context_id, id), FOREIGN KEY (context_id, entity_id) REFERENCES entities(context_id, id)) SYNC TWO WAY SYNC CONFLICT KEEP FIRST; +CREATE TABLE entity_reference_embeddings_256 (id UUID PRIMARY KEY, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), entity_id UUID NOT NULL, label TEXT, vector VECTOR(256), embedding_space_id TEXT NOT NULL REFERENCES embedding_spaces(embedding_space_id), produced_by JSON, created_at TXID NOT NULL, UNIQUE (entity_id, label), UNIQUE (context_id, id), FOREIGN KEY (context_id, entity_id) REFERENCES entities(context_id, id)) SYNC TWO WAY SYNC CONFLICT KEEP FIRST; +CREATE TABLE entity_reference_embeddings_512 (id UUID PRIMARY KEY, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), entity_id UUID NOT NULL, label TEXT, vector VECTOR(512), embedding_space_id TEXT NOT NULL REFERENCES embedding_spaces(embedding_space_id), produced_by JSON, created_at TXID NOT NULL, UNIQUE (entity_id, label), UNIQUE (context_id, id), FOREIGN KEY (context_id, entity_id) REFERENCES entities(context_id, id)) SYNC TWO WAY SYNC CONFLICT KEEP FIRST; +CREATE TABLE entity_reference_embeddings_768 (id UUID PRIMARY KEY, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), entity_id UUID NOT NULL, label TEXT, vector VECTOR(768), embedding_space_id TEXT NOT NULL REFERENCES embedding_spaces(embedding_space_id), produced_by JSON, created_at TXID NOT NULL, UNIQUE (entity_id, label), UNIQUE (context_id, id), FOREIGN KEY (context_id, entity_id) REFERENCES entities(context_id, id)) SYNC TWO WAY SYNC CONFLICT KEEP FIRST; +CREATE TABLE reindex_cursors (embedding_space_id TEXT PRIMARY KEY REFERENCES embedding_spaces(embedding_space_id), status TEXT NOT NULL, owner_run_id TEXT NOT NULL, lease_epoch INTEGER NOT NULL, last_committed_cursor INTEGER NOT NULL, from_producer JSON, to_producer JSON, started_at TXID NOT NULL, last_advanced_at TXID NOT NULL) SYNC TWO WAY SYNC CONFLICT KEEP LATEST; +CREATE TABLE evidence_availability_log (id UUID PRIMARY KEY, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), evidence_id UUID NOT NULL REFERENCES evidence(id), status TEXT NOT NULL, recorded_at_txid TXID NOT NULL, recorded_at TIMESTAMP, FOREIGN KEY (context_id, evidence_id) REFERENCES evidence(context_id, id)) IMMUTABLE SYNC TWO WAY SYNC CONFLICT KEEP FIRST; +CREATE TABLE signatures (id UUID PRIMARY KEY, signature_type TEXT NOT NULL, description TEXT NOT NULL, match_criteria JSON NOT NULL, scope_labels JSON NOT NULL, confidence REAL NOT NULL, origin_context_ids JSON, status TEXT NOT NULL DEFAULT 'active', created_at TXID NOT NULL, last_matched_at TXID) SYNC TWO WAY SYNC CONFLICT KEEP FIRST; +CREATE TABLE signature_aggregates (signature_id TEXT PRIMARY KEY, signature_row_id UUID REFERENCES signatures(id), embedding_space_id TEXT NOT NULL, producer JSON NOT NULL, centroid JSON NOT NULL, member_count INTEGER NOT NULL, scope_labels JSON NOT NULL, created_at TXID NOT NULL) SYNC TWO WAY SYNC CONFLICT KEEP FIRST; +CREATE TABLE signature_producer_lineage (id UUID PRIMARY KEY, signature_id TEXT NOT NULL, signature_row_id UUID REFERENCES signatures(id), contribution_role TEXT NOT NULL, observation_id UUID NOT NULL, evidence_id UUID NOT NULL, embedding_id UUID NOT NULL, embedding_space_id TEXT NOT NULL, producer JSON NOT NULL, created_at TXID NOT NULL) SYNC TWO WAY SYNC CONFLICT KEEP FIRST; +CREATE TABLE digests (id UUID PRIMARY KEY, source TEXT NOT NULL, session_ref TEXT, summary TEXT NOT NULL, kind TEXT NOT NULL, vector_text VECTOR(768), dimension INTEGER NOT NULL, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), properties JSON, created_at TXID NOT NULL, UNIQUE (context_id, id)) IMMUTABLE SYNC TWO WAY SYNC CONFLICT KEEP FIRST; +CREATE TABLE digest_embedding_handles (id UUID PRIMARY KEY, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), target_id UUID NOT NULL REFERENCES digests(id), embedding_space_id TEXT NOT NULL REFERENCES embedding_spaces(embedding_space_id), produced_by JSON, producer_system TEXT NOT NULL, created_at TXID NOT NULL, FOREIGN KEY (context_id, target_id) REFERENCES digests(context_id, id)) IMMUTABLE SYNC TWO WAY SYNC CONFLICT KEEP FIRST; +CREATE TABLE blueprints (id UUID PRIMARY KEY, canonical_description TEXT NOT NULL, category TEXT NOT NULL, slots JSON NOT NULL, default_config_shape JSON, version TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'active', created_at TIMESTAMP NOT NULL) SYNC TWO WAY SYNC CONFLICT KEEP FIRST; +CREATE TABLE edges (id UUID PRIMARY KEY, source_id UUID NOT NULL, target_id UUID NOT NULL, edge_type TEXT NOT NULL, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), valid_from TXID NOT NULL, valid_to TXID, properties JSON, UNIQUE (source_id, target_id, edge_type)) DAG('CITES') SYNC TWO WAY SYNC CONFLICT KEEP FIRST; +CREATE TABLE entity_snapshots (id UUID PRIMARY KEY, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), entity_id UUID NOT NULL REFERENCES entities(id), data JSON NOT NULL, valid_from TXID NOT NULL, valid_to TXID, UNIQUE (context_id, id), UNIQUE (context_id, entity_id, id), FOREIGN KEY (context_id, entity_id) REFERENCES entities(context_id, id)) IMMUTABLE SYNC TWO WAY SYNC CONFLICT KEEP FIRST; +CREATE TABLE entity_snapshot_observations (id UUID PRIMARY KEY, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), entity_id UUID NOT NULL REFERENCES entities(id), snapshot_id UUID NOT NULL REFERENCES entity_snapshots(id), observation_id UUID NOT NULL REFERENCES observations(id), observation_order INTEGER NOT NULL, created_at TXID NOT NULL, UNIQUE (context_id, entity_id, snapshot_id, observation_order), UNIQUE (context_id, entity_id, snapshot_id, observation_id), FOREIGN KEY (context_id, entity_id) REFERENCES entities(context_id, id), FOREIGN KEY (context_id, entity_id, snapshot_id) REFERENCES entity_snapshots(context_id, entity_id, id), FOREIGN KEY (context_id, entity_id, observation_id) REFERENCES observations(context_id, entity_id, id)) SYNC TWO WAY SYNC CONFLICT KEEP FIRST; +CREATE TABLE decision_basis_snapshots (id UUID PRIMARY KEY, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), decision_id UUID NOT NULL REFERENCES decisions(id), entity_id UUID NOT NULL REFERENCES entities(id), snapshot_id UUID NOT NULL REFERENCES entity_snapshots(id), basis_order INTEGER NOT NULL, created_at TXID NOT NULL, UNIQUE (context_id, decision_id, basis_order), UNIQUE (context_id, decision_id, snapshot_id), FOREIGN KEY (context_id, decision_id) REFERENCES decisions(context_id, id), FOREIGN KEY (context_id, entity_id) REFERENCES entities(context_id, id), FOREIGN KEY (context_id, entity_id, snapshot_id) REFERENCES entity_snapshots(context_id, entity_id, id)) SYNC TWO WAY SYNC CONFLICT KEEP FIRST; +CREATE TABLE entity_current_snapshots (context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), entity_id UUID PRIMARY KEY REFERENCES entities(id), snapshot_id UUID NOT NULL REFERENCES entity_snapshots(id), updated_at TXID NOT NULL, UNIQUE (context_id, entity_id), FOREIGN KEY (context_id, entity_id) REFERENCES entities(context_id, id), FOREIGN KEY (context_id, entity_id, snapshot_id) REFERENCES entity_snapshots(context_id, entity_id, id)) SYNC TWO WAY SYNC CONFLICT KEEP LATEST; +CREATE TABLE entity_snapshot_links (id UUID PRIMARY KEY, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), entity_id UUID NOT NULL REFERENCES entities(id), predecessor_snapshot_id UUID NOT NULL REFERENCES entity_snapshots(id), successor_snapshot_id UUID NOT NULL REFERENCES entity_snapshots(id), created_at TXID NOT NULL, UNIQUE (context_id, entity_id, predecessor_snapshot_id, successor_snapshot_id), UNIQUE (context_id, entity_id, successor_snapshot_id), FOREIGN KEY (context_id, entity_id) REFERENCES entities(context_id, id), FOREIGN KEY (context_id, entity_id, predecessor_snapshot_id) REFERENCES entity_snapshots(context_id, entity_id, id), FOREIGN KEY (context_id, entity_id, successor_snapshot_id) REFERENCES entity_snapshots(context_id, entity_id, id)) IMMUTABLE SYNC TWO WAY SYNC CONFLICT KEEP FIRST; +CREATE TABLE intention_snapshots (id UUID PRIMARY KEY, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), intention_id UUID NOT NULL REFERENCES intentions(id), description TEXT NOT NULL, status TEXT NOT NULL, origin TEXT NOT NULL, properties JSON, tags JSON, valid_from TXID NOT NULL, valid_to TXID, UNIQUE (context_id, id), UNIQUE (context_id, intention_id, id), FOREIGN KEY (context_id, intention_id) REFERENCES intentions(context_id, id)) IMMUTABLE SYNC TWO WAY SYNC CONFLICT KEEP FIRST; +CREATE TABLE intention_current_snapshots (context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), intention_id UUID PRIMARY KEY REFERENCES intentions(id), snapshot_id UUID NOT NULL REFERENCES intention_snapshots(id), updated_at TXID NOT NULL, UNIQUE (context_id, intention_id), FOREIGN KEY (context_id, intention_id) REFERENCES intentions(context_id, id), FOREIGN KEY (context_id, intention_id, snapshot_id) REFERENCES intention_snapshots(context_id, intention_id, id)) SYNC TWO WAY SYNC CONFLICT KEEP LATEST; +CREATE TABLE intention_snapshot_links (id UUID PRIMARY KEY, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), intention_id UUID NOT NULL REFERENCES intentions(id), predecessor_snapshot_id UUID NOT NULL REFERENCES intention_snapshots(id), successor_snapshot_id UUID NOT NULL REFERENCES intention_snapshots(id), created_at TXID NOT NULL, UNIQUE (context_id, intention_id, predecessor_snapshot_id, successor_snapshot_id), UNIQUE (context_id, intention_id, successor_snapshot_id), FOREIGN KEY (context_id, intention_id) REFERENCES intentions(context_id, id), FOREIGN KEY (context_id, intention_id, predecessor_snapshot_id) REFERENCES intention_snapshots(context_id, intention_id, id), FOREIGN KEY (context_id, intention_id, successor_snapshot_id) REFERENCES intention_snapshots(context_id, intention_id, id)) IMMUTABLE SYNC TWO WAY SYNC CONFLICT KEEP FIRST; +CREATE TABLE decision_intention_citations (id UUID PRIMARY KEY, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), decision_id UUID NOT NULL REFERENCES decisions(id), intention_id UUID NOT NULL REFERENCES intentions(id), snapshot_id UUID NOT NULL REFERENCES intention_snapshots(id), created_at TXID NOT NULL, UNIQUE (context_id, decision_id, intention_id), FOREIGN KEY (context_id, decision_id) REFERENCES decisions(context_id, id), FOREIGN KEY (context_id, intention_id) REFERENCES intentions(context_id, id), FOREIGN KEY (context_id, intention_id, snapshot_id) REFERENCES intention_snapshots(context_id, intention_id, id)) IMMUTABLE SYNC TWO WAY SYNC CONFLICT KEEP FIRST; +CREATE TABLE approvals (id UUID PRIMARY KEY, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), decision_id UUID NOT NULL REFERENCES decisions(id), approver_id TEXT NOT NULL, level TEXT NOT NULL, rationale TEXT, created_at TXID NOT NULL, UNIQUE (context_id, id), FOREIGN KEY (context_id, decision_id) REFERENCES decisions(context_id, id)) SYNC TWO WAY SYNC CONFLICT KEEP FIRST; +CREATE TABLE audit_log (id UUID PRIMARY KEY, txid TXID NOT NULL, operation TEXT NOT NULL, actor TEXT NOT NULL, reason TEXT, target_kind TEXT NOT NULL, target_id UUID NOT NULL, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), outcome_recorded_at TXID, payload JSON) SYNC OFF; +CREATE TABLE precedent_matches (id UUID PRIMARY KEY, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), new_decision_id UUID NOT NULL REFERENCES decisions(id), precedent_decision_id UUID NOT NULL REFERENCES decisions(id), served_intention_ids JSON NOT NULL DEFAULT '[]', matched_precedent_outcome_success BOOLEAN NOT NULL, detected_at TXID NOT NULL, UNIQUE (context_id, new_decision_id, precedent_decision_id), FOREIGN KEY (context_id, new_decision_id) REFERENCES decisions(context_id, id), FOREIGN KEY (context_id, precedent_decision_id) REFERENCES decisions(context_id, id)) IMMUTABLE SYNC TWO WAY SYNC CONFLICT KEEP FIRST; +CREATE TABLE cg_learned_skills (id TEXT PRIMARY KEY, description TEXT NOT NULL, rule TEXT NOT NULL, effect TEXT, apply_strategy TEXT, provenance TEXT NOT NULL, active BOOLEAN NOT NULL DEFAULT true, held_for_review BOOLEAN NOT NULL DEFAULT false, fire_right INTEGER NOT NULL DEFAULT 0, fire_wrong INTEGER NOT NULL DEFAULT 0, created_at TXID NOT NULL) SYNC TWO WAY SYNC CONFLICT KEEP LATEST; +CREATE TABLE cg_teaching_events (id TEXT PRIMARY KEY, skill_id TEXT, corrected_item_id TEXT NOT NULL, kind TEXT NOT NULL, content TEXT NOT NULL, created_at TXID NOT NULL) SYNC TWO WAY SYNC CONFLICT KEEP FIRST; +CREATE TABLE cg_skill_firing_trace (id TEXT PRIMARY KEY, skill_id TEXT NOT NULL, capture_id TEXT NOT NULL, correct INTEGER, effect_entity_key TEXT, effect_entity_name TEXT, effect_from_type TEXT, effect_to_type TEXT, created_at TXID NOT NULL) SYNC OFF; +CREATE TABLE cg_review_cursor (id TEXT PRIMARY KEY, position INTEGER NOT NULL DEFAULT 0, is_local BOOLEAN NOT NULL DEFAULT true, updated_at TXID NOT NULL) SYNC OFF; +CREATE TABLE cg_overlay_entries (id TEXT PRIMARY KEY, item_id TEXT NOT NULL, kind TEXT NOT NULL, value TEXT, reversed BOOLEAN NOT NULL DEFAULT false, created_at TXID NOT NULL) SYNC TWO WAY SYNC CONFLICT KEEP FIRST; +CREATE TABLE cg_compensation_events (id TEXT PRIMARY KEY, reversed_overlay_id TEXT NOT NULL, kind TEXT NOT NULL, content TEXT NOT NULL, created_at TXID NOT NULL) SYNC TWO WAY SYNC CONFLICT KEEP FIRST; +CREATE TABLE cg_pending_review_items (id TEXT PRIMARY KEY, acted_on BOOLEAN NOT NULL DEFAULT false, content TEXT, legal_vocab TEXT, created_at TXID NOT NULL) SYNC OFF; +CREATE TABLE cg_source_chunks (id TEXT PRIMARY KEY, digest_id TEXT NOT NULL, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), seq INTEGER NOT NULL, content TEXT NOT NULL, content_hash TEXT NOT NULL, byte_len INTEGER NOT NULL, created_at TXID NOT NULL, created_at_ms INTEGER NOT NULL) SYNC OFF; +CREATE TABLE cg_retrieval_touches (digest_id TEXT PRIMARY KEY, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), touch_count INTEGER NOT NULL DEFAULT 0, last_touched_at TXID NOT NULL) SYNC OFF; +CREATE TABLE cg_understanding_completions (id TEXT PRIMARY KEY, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), model TEXT NOT NULL, receipt JSON, completed_at TXID NOT NULL) SYNC OFF; +CREATE TABLE cg_local_settings (key TEXT PRIMARY KEY, value TEXT NOT NULL, updated_at TXID NOT NULL) SYNC OFF; +CREATE TABLE cg_runtime_models (id UUID PRIMARY KEY, context_label TEXT NOT NULL, model_key TEXT NOT NULL, advertised_context_tokens INTEGER NOT NULL, default_for_context BOOLEAN NOT NULL, created_at TXID NOT NULL) SYNC OFF; +CREATE TABLE cg_changed_cursors (context_id UUID PRIMARY KEY, last_seen_tx TXID NOT NULL, updated_at TXID NOT NULL) SYNC OFF; +CREATE TABLE cg_subscription_receipts (sink_name TEXT NOT NULL, action_id TEXT NOT NULL, recorded_at TXID NOT NULL, PRIMARY KEY (sink_name, action_id)) SYNC OFF; +CREATE TABLE entity_reference_embeddings_1024 (id UUID PRIMARY KEY, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), entity_id UUID NOT NULL, label TEXT, vector VECTOR(1024), embedding_space_id TEXT NOT NULL REFERENCES embedding_spaces(embedding_space_id), produced_by JSON, created_at TXID NOT NULL, UNIQUE (entity_id, label), UNIQUE (context_id, id), FOREIGN KEY (context_id, entity_id) REFERENCES entities(context_id, id)); +CREATE TABLE work_inputs (job_id TEXT NOT NULL); + +CREATE INDEX idx_events_source_stream ON events (context_id, connector_id, source_instance_ref, seq); +CREATE INDEX idx_events_session ON events (context_id, connector_id, session_ref, seq); +CREATE INDEX idx_events_source_key ON events (context_id, connector_id, session_ref, source_event_key); +CREATE INDEX idx_events_source_time ON events (context_id, connector_id, occurred_at, seq); +CREATE INDEX idx_events_author_kind ON events (context_id, author, kind, seq); +CREATE INDEX idx_event_anchors_exact ON event_anchors (context_id, key, value, identity_value); +CREATE INDEX idx_event_anchors_physical_correlation_scope ON event_anchors (correlation_scope_hash, context_id, event_id); +CREATE INDEX idx_event_anchors_event ON event_anchors (event_id, context_id, declaration_version, key, identity_value); +CREATE INDEX idx_edges_source_context_type ON edges (source_id, context_id, edge_type); +CREATE INDEX idx_edges_target_context_type ON edges (target_id, context_id, edge_type); +CREATE INDEX idx_decisions_id_status ON decisions (id, status); +CREATE INDEX idx_evidence_embedding_handles_producer_system ON evidence_embedding_handles (producer_system); +CREATE INDEX idx_entity_reference_embedding_handles_producer_system ON entity_reference_embedding_handles (producer_system); +CREATE INDEX idx_digest_embedding_handles_producer_system ON digest_embedding_handles (producer_system); +CREATE INDEX idx_adapter_coverage_gaps_page ON adapter_coverage_gaps (context_id, connector_id, id); +CREATE INDEX idx_event_interpretation_source_events_generation ON event_interpretation_source_events (generation_id, context_id, ordinal); +CREATE INDEX idx_event_interpretation_anchors_exact ON event_interpretation_anchors (context_id, generation_id, key, canonical_value, identity_value); +CREATE INDEX idx_event_search_source_events_generation ON event_search_source_events (generation_id, context_id, ordinal); +CREATE INDEX idx_event_search_source_events_cursor_range ON event_search_source_events (context_id, generation_id, connector_id, source_instance_ref, source_sequence, ordinal); +CREATE INDEX event_search_passages_active_idx ON event_search_passages (context_id, generation_id, is_current, is_searchable); +CREATE INDEX event_search_passages_generation_ordinal_idx ON event_search_passages (generation_id, passage_ordinal); +CREATE INDEX event_search_passages_connector_idx ON event_search_passages (context_id, generation_id, is_current, connector_id); +CREATE INDEX event_search_passages_session_idx ON event_search_passages (context_id, generation_id, is_current, session_ref); +CREATE INDEX event_search_passages_source_idx ON event_search_passages (context_id, generation_id, is_current, source_instance_ref); +CREATE INDEX event_search_passage_sources_passage_idx ON event_search_passage_sources (passage_id, context_id, source_order); + +-- Second texts for a table declared above: the program declares one or the other in a store. +-- Each is declared in a store of its own, after the tables above that it references. + +CREATE TABLE events (id UUID PRIMARY KEY, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), connector_id TEXT NOT NULL, adapter_version TEXT NOT NULL, anchor_declaration_version TEXT NOT NULL, source_instance_ref TEXT NOT NULL, session_ref TEXT NOT NULL, source_event_key TEXT NOT NULL, source_position JSON NOT NULL, seq INTEGER NOT NULL, content_hash TEXT NOT NULL, anchor_count INTEGER NOT NULL, anchor_set_hash TEXT NOT NULL, occurred_at TIMESTAMP, author TEXT NOT NULL, kind TEXT NOT NULL, render_role TEXT NOT NULL, interaction_ref TEXT, text TEXT, payload JSON NOT NULL DEFAULT '{}', anchors JSON NOT NULL DEFAULT '{}', source_remainder JSON NOT NULL DEFAULT '{}', coverage TEXT NOT NULL, UNIQUE (context_id, id), UNIQUE (context_id, connector_id, session_ref, source_event_key), UNIQUE (context_id, connector_id, source_instance_ref, seq)) SYNC PUSH ONLY SYNC CONFLICT KEEP FIRST IMMUTABLE DELIVERY MANIFEST OVER event_anchors MAINTAIN COUNT (context_id), (context_id, connector_id, source_instance_ref); +CREATE TABLE event_anchors (id UUID PRIMARY KEY, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), event_id UUID NOT NULL REFERENCES events(id), correlation_scope_hash TEXT NOT NULL, declaration_version TEXT NOT NULL, connector_id TEXT NOT NULL, source_instance_ref TEXT NOT NULL, session_ref TEXT NOT NULL, render_role TEXT NOT NULL, interaction_ref TEXT, key TEXT NOT NULL, value TEXT NOT NULL, identity_value TEXT NOT NULL, original_value JSON NOT NULL, UNIQUE (context_id, id), UNIQUE (context_id, event_id, declaration_version, key, identity_value), FOREIGN KEY (context_id, event_id) REFERENCES events(context_id, id)) SYNC PUSH ONLY SYNC CONFLICT KEEP FIRST IMMUTABLE MAINTAIN COUNT NONE; +CREATE TABLE conflicts (id UUID NOT NULL UNIQUE, context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), older_decision_id UUID NOT NULL REFERENCES decisions(id), newer_decision_id UUID NOT NULL REFERENCES decisions(id), threatened_intention_ids JSON NOT NULL DEFAULT '[]', intention_in_tension BOOLEAN NOT NULL, detected_at TXID NOT NULL, PRIMARY KEY (context_id, older_decision_id, newer_decision_id), FOREIGN KEY (context_id, older_decision_id) REFERENCES decisions(context_id, id), FOREIGN KEY (context_id, newer_decision_id) REFERENCES decisions(context_id, id)) IMMUTABLE SYNC TWO WAY SYNC CONFLICT KEEP FIRST; +CREATE TABLE digests (id UUID PRIMARY KEY, embedding VECTOR(384)); +CREATE TABLE observations (id UUID PRIMARY KEY, data JSON NOT NULL); +CREATE TABLE cg_schema_version (id INTEGER PRIMARY KEY, version INTEGER NOT NULL, installed_at TXID NOT NULL); diff --git a/v2/contextdb/tests/kept_counts.rs b/v2/contextdb/tests/kept_counts.rs new file mode 100644 index 00000000..338e10a3 --- /dev/null +++ b/v2/contextdb/tests/kept_counts.rs @@ -0,0 +1,466 @@ +//! Counts with no row read. A table keeps the counts it declares with `MAINTAIN COUNT`, and a +//! `COUNT(*)` whose filter gives a declared list's columns is one stored number. A table that +//! declares none counts the entries of the one index range the filter gives. Either way the +//! count is the number of rows the same filter returns, and the statement reads no row. + +mod support; + +use contextdb::{Database, Error, Value, params}; +use support::Run; + +const MIB: usize = 1 << 20; + +/// The filters every check asks: each `ctx`, and each `ctx` with each `conn`, values that no +/// row has among them. +fn filters() -> Vec { + let mut filters = Vec::new(); + for ctx in 0..=4_i64 { + filters.push(format!("ctx = {ctx}")); + for conn in ["a", "b", "zz"] { + filters.push(format!("ctx = {ctx} AND conn = '{conn}'")); + } + } + filters +} + +/// Asserts that every count of `events` equals the number of rows its filter returns, and +/// returns the rows each count read, in the order of [`filters`]. The whole table's count is +/// checked the same way. +fn counts_agree(on: &impl Run, when: &str) -> Vec { + let mut read = Vec::new(); + for filter in filters() { + let rows = support::rows(on, &format!("SELECT id FROM events WHERE {filter}")); + let count = support::run(on, &format!("SELECT COUNT(*) FROM events WHERE {filter}")); + assert_eq!( + count.rows(), + [[Value::Int64(rows.len() as i64)]], + "{when}: COUNT(*) WHERE {filter}, beside the {} rows the filter returns", + rows.len() + ); + read.push(count.rows_examined()); + } + let rows = support::rows(on, "SELECT id FROM events"); + assert_eq!( + support::int(on, "SELECT COUNT(*) FROM events"), + rows.len() as i64, + "{when}: the count of the whole table" + ); + read +} + +/// Asserts the same, and that no count read a row. +fn counts_agree_and_read_nothing(db: &Database, when: &str) { + let read = counts_agree(db, when); + for (filter, rows) in filters().iter().zip(read) { + assert_eq!(rows, 0, "{when}: rows read by COUNT(*) WHERE {filter}"); + } +} + +/// Event `n`: its `ctx` is `n % 4` and its `conn` is `a` or `b`. +fn insert_events(on: &impl Run, numbers: impl IntoIterator) { + for n in numbers { + on.run( + "INSERT INTO events (id, ctx, conn, note) VALUES ($id, $ctx, $conn, $note)", + ¶ms! { + "id" => n, "ctx" => n % 4, "conn" => if n % 3 == 0 { "a" } else { "b" }, + "note" => format!("note-{n}"), + }, + ) + .expect("insert an event"); + } +} + +/// Puts `events` through inserts, updates that move rows between counted groups and updates +/// that do not, deletes, a refused statement, a rolled-back transaction and a transaction that +/// another commit overtook, and asserts after each that every count is the number of rows and +/// read none. +fn counts_follow_every_write(db: &Database) { + counts_agree_and_read_nothing(db, "on the empty table"); + let tx = db.transaction().expect("begin"); + insert_events(&tx, 1..=600); + // Inside the transaction that wrote the rows the counts are theirs already. + counts_agree(&tx, "inside the transaction that inserted"); + tx.commit().expect("commit the events"); + counts_agree_and_read_nothing(db, "after inserts"); + + // Rows move from one counted group to another, by either counted column and by both. + support::run(db, "UPDATE events SET ctx = 0 WHERE id <= 50"); + support::run( + db, + "UPDATE events SET conn = 'a' WHERE ctx = 2 AND id > 500", + ); + support::run(db, "UPDATE events SET ctx = 3, conn = 'b' WHERE id > 550"); + counts_agree_and_read_nothing(db, "after updates of counted columns"); + // A change to a column no count is over, and a change of a row's key. + support::run(db, "UPDATE events SET note = 'seen' WHERE ctx = 1"); + support::run(db, "UPDATE events SET id = id + 1000 WHERE id <= 20"); + counts_agree_and_read_nothing(db, "after updates of other columns"); + + support::run(db, "DELETE FROM events WHERE ctx = 3 AND conn = 'a'"); + support::run(db, "DELETE FROM events WHERE id > 100 AND id <= 130"); + counts_agree_and_read_nothing(db, "after deletes"); + + // A statement that is refused at its last row counted none of its rows. + let refusal = support::refusal( + db, + "INSERT INTO events (id, ctx, conn) VALUES (2001, 1, 'a'), (2002, 1, 'a'), (200, 1, 'a')", + ); + assert!( + matches!(refusal, Error::UniqueViolation { .. }), + "{refusal:?}" + ); + let refusal = support::refusal(db, "UPDATE events SET ctx = NULL WHERE ctx = 1"); + assert!( + matches!(refusal, Error::ColumnNotNullable { .. }), + "{refusal:?}" + ); + counts_agree_and_read_nothing(db, "after refused statements"); + + // A transaction that is rolled back counted nothing, whatever it had staged. + let tx = db.transaction().expect("begin"); + insert_events(&tx, 3001..=3050); + support::run(&tx, "DELETE FROM events WHERE ctx = 0"); + support::run(&tx, "UPDATE events SET ctx = 0 WHERE ctx = 2"); + counts_agree(&tx, "inside the transaction that is rolled back"); + tx.rollback(); + counts_agree_and_read_nothing(db, "after a rollback"); + + // Two transactions add to one group side by side: the group counts both. + let tx = db.transaction().expect("begin"); + insert_events(&tx, [4001, 4005]); + support::run(&tx, "DELETE FROM events WHERE id = 201"); + insert_events(db, [4009, 4013]); + support::run(db, "DELETE FROM events WHERE id = 205"); + tx.commit().expect("commit beside another commit"); + counts_agree_and_read_nothing(db, "after two commits side by side"); +} + +#[test] +fn a_kept_count_is_the_number_of_rows_after_every_kind_of_write_and_reads_no_row() { + let folder = support::folder("kept-counts"); + let db = support::open(&folder); + support::run( + &db, + "CREATE TABLE events (id INTEGER PRIMARY KEY, ctx INTEGER NOT NULL, conn TEXT NOT NULL, \ + note TEXT, UNIQUE (ctx, id), UNIQUE (ctx, conn, id)) \ + MAINTAIN COUNT (ctx), (ctx, conn)", + ); + counts_follow_every_write(&db); + + // The counts are in the store: they are the same after reopening, and still read no row. + support::close(db); + let db = support::open(&folder); + counts_agree_and_read_nothing(&db, "after reopening"); + // The filter that finds the rows reads them; the count beside it does not. + let rows = support::run(&db, "SELECT id FROM events WHERE ctx = 1"); + assert!(rows.rows().len() > 100); + assert_eq!(rows.rows_examined(), rows.rows().len() as u64); +} + +#[test] +fn a_table_that_declares_no_count_counts_an_index_range_and_reads_no_row() { + let db = support::open(&support::folder("counted-index-range")); + support::run( + &db, + "CREATE TABLE events (id INTEGER PRIMARY KEY, ctx INTEGER NOT NULL, conn TEXT NOT NULL, \ + note TEXT)", + ); + support::run(&db, "CREATE INDEX events_ctx_conn ON events (ctx, conn)"); + counts_follow_every_write(&db); + + // A count whose filter the index does not pin has to read rows: by the index's second + // column alone, and with a term over a column the index does not have. + for filter in ["conn = 'a'", "ctx = 1 AND note = 'seen'"] { + let rows = support::rows(&db, &format!("SELECT id FROM events WHERE {filter}")); + let count = support::run(&db, &format!("SELECT COUNT(*) FROM events WHERE {filter}")); + assert_eq!( + count.rows(), + [[Value::Int64(rows.len() as i64)]], + "{filter}" + ); + assert!(!rows.is_empty(), "{filter} returns no row"); + assert!( + count.rows_examined() >= rows.len() as u64, + "COUNT(*) WHERE {filter} read {} rows and counted {}", + count.rows_examined(), + rows.len() + ); + } +} + +/// The declaration of `groups`: an index on `tag`, and no kept count. +fn declare_groups(db: &Database) { + support::run( + db, + "CREATE TABLE groups (id INTEGER PRIMARY KEY, tag INTEGER NOT NULL)", + ); + support::run(db, "CREATE INDEX groups_tag ON groups (tag)"); +} + +#[test] +fn a_new_set_of_counts_on_a_table_with_rows_is_installed_whole_or_not_at_all() { + const ROWS: i64 = 10_000; + let folder = support::folder("kept-counts-installed"); + // The page cache is sized when the store is opened, so the limit of the second half is + // set on a store that was opened under a larger one. + let mut options = contextdb::DatabaseOpenOptions::default(); + options.memory_limit = Some(256 * MIB); + let db = support::open_with(&folder, options).expect("open the store under a limit"); + declare_groups(&db); + // Every row is a group of its own, so the new counts are as many as the rows. + for from in (0..ROWS).step_by(1000) { + let values: Vec = (from..from + 1000).map(|n| format!("({n}, {n})")).collect(); + support::run( + &db, + &format!("INSERT INTO groups (id, tag) VALUES {}", values.join(", ")), + ); + } + let declared = db.declaration("groups").expect("the declaration of groups"); + let count_of = |tag: i64| { + let count = support::run( + &db, + &format!("SELECT COUNT(*) FROM groups WHERE tag = {tag}"), + ); + assert_eq!(count.rows_examined(), 0, "the count of tag {tag}"); + match count.rows() { + [row] => row[0].clone(), + rows => panic!("a count came back as {} rows", rows.len()), + } + }; + + // With 64 KiB to spare, the 10,000 new counts do not fit: the statement is the memory + // limit's refusal, the declaration is the one the table had, and its counts still answer. + let limit = db.memory_used() + (64 << 10); + db.set_memory_limit(Some(limit)).expect("set the limit"); + match support::refusal(&db, "ALTER TABLE groups SET MAINTAIN COUNT (tag)") { + Error::MemoryBudgetExceeded { + budget_limit_bytes, .. + } => assert_eq!(budget_limit_bytes, limit as u64), + other => panic!("counts that do not fit the limit were refused as {other:?}"), + } + assert_eq!( + db.declaration("groups").expect("the declaration of groups"), + declared + ); + assert_eq!(count_of(7), Value::Int64(1)); + assert_eq!(count_of(ROWS), Value::Int64(0)); + + // With room, the same statement installs the counts, and they are right at once. + db.set_memory_limit(Some(256 * MIB)) + .expect("lift the limit"); + support::run(&db, "ALTER TABLE groups SET MAINTAIN COUNT (tag)"); + let installed = db.declaration("groups").expect("the declaration of groups"); + assert!( + installed.contains("MAINTAIN COUNT (tag)"), + "the declaration does not say MAINTAIN COUNT (tag):\n{installed}" + ); + support::run(&db, "UPDATE groups SET tag = 7 WHERE id < 10"); + assert_eq!(count_of(7), Value::Int64(10)); + assert_eq!(count_of(3), Value::Int64(0)); + assert_eq!(count_of(ROWS - 1), Value::Int64(1)); + + // Taking the counts away leaves the same answers, and the declaration it had before. + support::run(&db, "ALTER TABLE groups SET MAINTAIN COUNT DEFAULT"); + assert_eq!( + db.declaration("groups").expect("the declaration of groups"), + declared + ); + assert_eq!(count_of(7), Value::Int64(10)); + support::close(db); + let db = support::open(&folder); + assert_eq!( + support::int(&db, "SELECT COUNT(*) FROM groups WHERE tag = 7"), + 10 + ); +} + +#[test] +fn a_count_list_that_cannot_stand_is_refused_by_name_and_declares_nothing() { + let db = support::open(&support::folder("kept-count-refusals")); + support::run( + &db, + "CREATE TABLE t (id INTEGER PRIMARY KEY, a INTEGER, b INTEGER NOT NULL, \ + c INTEGER NOT NULL, UNIQUE (b, c))", + ); + support::run(&db, "CREATE INDEX t_by_b ON t (b)"); + let declared = db.declaration("t").expect("the declaration of t"); + let reason_of = |sql: &str| match support::refusal(&db, sql) { + Error::SchemaInvalid { reason } => reason, + other => panic!("{sql}\nwas refused as {other:?}"), + }; + let says = |reason: &str, words: &[&str]| { + for word in words { + assert!( + reason.contains(word), + "the refusal does not say `{word}`: {reason}" + ); + } + }; + + // No index begins with the list: the refusal names the list and the indexes it looked at, + // the engine's own among them. + let reason = reason_of("ALTER TABLE t SET MAINTAIN COUNT (a)"); + says(&reason, &["(a)", "t_by_b", "__pk_id"]); + let reason = reason_of("ALTER TABLE t SET MAINTAIN COUNT (b), (c, b)"); + says(&reason, &["(c, b)", "t_by_b"]); + // A key or a UNIQUE rule already holds the list to one row at most. + for list in ["(id)", "(b, c)"] { + let reason = reason_of(&format!("ALTER TABLE t SET MAINTAIN COUNT {list}")); + says(&reason, &[list]); + } + // The same list twice, and one column twice in a list. + let reason = reason_of("ALTER TABLE t SET MAINTAIN COUNT (b), (b)"); + says(&reason, &["(b)"]); + let reason = reason_of("ALTER TABLE t SET MAINTAIN COUNT (b, b)"); + says(&reason, &["b"]); + // A column the table does not have. + match support::refusal(&db, "ALTER TABLE t SET MAINTAIN COUNT (b, nope)") { + Error::ColumnNotFound { table, column } => { + assert_eq!((table.as_str(), column.as_str()), ("t", "nope")); + } + other => panic!("a count over a column that is not there was refused as {other:?}"), + } + assert_eq!(db.declaration("t").expect("the declaration of t"), declared); +} + +/// Asserts that every count of `notes` equals the number of rows its filter returns: each +/// `ctx`, and each `ctx` with no tag, with the tag `long` and with the tag `s`. +fn note_counts_agree(db: &Database, long: &str, when: &str) { + for ctx in 1..=3_i64 { + let filters = [ + (format!("ctx = {ctx}"), params! {}), + (format!("ctx = {ctx} AND tag IS NULL"), params! {}), + ( + format!("ctx = {ctx} AND tag = $tag"), + params! { "tag" => long }, + ), + ( + format!("ctx = {ctx} AND tag = $tag"), + params! { "tag" => "s" }, + ), + ]; + for (filter, given) in filters { + let rows = db + .run(&format!("SELECT id FROM notes WHERE {filter}"), &given) + .expect("the rows of a filter"); + let count = db + .run( + &format!("SELECT COUNT(*) FROM notes WHERE {filter}"), + &given, + ) + .expect("the count of a filter"); + assert_eq!( + count.rows(), + [[Value::Int64(rows.rows().len() as i64)]], + "{when}: COUNT(*) WHERE {filter}, tag of {} bytes", + given.get("tag").map_or(0, |tag| match tag { + Value::Text(tag) => tag.len(), + _ => 0, + }) + ); + } + } +} + +#[test] +fn a_count_over_a_null_and_over_a_value_past_1024_bytes_stays_equal_to_the_rows() { + let db = support::open(&support::folder("kept-count-null-and-long")); + support::run( + &db, + "CREATE TABLE notes (id INTEGER PRIMARY KEY, ctx INTEGER NOT NULL, tag TEXT)", + ); + support::run(&db, "CREATE INDEX notes_ctx_tag ON notes (ctx, tag)"); + support::run( + &db, + "ALTER TABLE notes SET MAINTAIN COUNT (ctx), (ctx, tag)", + ); + let long = support::letters(7, 1500); + // Note `n` is in ctx 1 or 2 and has no tag, the long tag or a short one. + let insert = |n: i64, tag: Value| { + db.execute( + "INSERT INTO notes (id, ctx, tag) VALUES ($id, $ctx, $tag)", + ¶ms! { "id" => n, "ctx" => n % 2 + 1, "tag" => tag }, + ) + }; + for n in 1..=12_i64 { + let tag = match n % 3 { + 0 => Value::Null, + 1 => Value::Text(long.clone()), + _ => Value::Text("s".to_owned()), + }; + insert(n, tag).expect("insert a note"); + } + assert_eq!(support::int(&db, "SELECT COUNT(*) FROM notes"), 12); + assert_eq!( + support::int( + &db, + "SELECT COUNT(*) FROM notes WHERE ctx = 1 AND tag IS NULL" + ), + 2, + "notes 6 and 12" + ); + let of_long = db + .execute( + "SELECT COUNT(*) FROM notes WHERE ctx = 2 AND tag = $tag", + ¶ms! { "tag" => long.as_str() }, + ) + .expect("the count of the long tag"); + assert_eq!(of_long.rows(), [[Value::Int64(2)]], "notes 1 and 7"); + note_counts_agree(&db, &long, "after the inserts"); + + // One note of each kind goes, then every note of ctx 1 that has no tag. + support::run(&db, "DELETE FROM notes WHERE id <= 3"); + note_counts_agree(&db, &long, "after deleting notes 1 to 3"); + support::run(&db, "DELETE FROM notes WHERE ctx = 1 AND tag IS NULL"); + note_counts_agree(&db, &long, "after deleting the notes of ctx 1 with no tag"); + db.execute( + "DELETE FROM notes WHERE tag = $tag", + ¶ms! { "tag" => long.as_str() }, + ) + .expect("delete the notes with the long tag"); + note_counts_agree(&db, &long, "after deleting the notes with the long tag"); + assert_eq!(support::int(&db, "SELECT COUNT(*) FROM notes"), 4); + + // A count's values identify a group and are kept whole: a tag past the limit is refused + // by name, and counts nothing. + match insert(13, Value::Text(support::letters(8, 3000))) { + Err(Error::KeyTooLong { + table, + key, + bytes, + limit, + }) => { + assert_eq!(table, "notes"); + assert!( + key.contains("tag"), + "the count that refused is named: {key}" + ); + assert!(bytes > limit, "{bytes} bytes against a limit of {limit}"); + } + other => panic!("a tag of 3,000 bytes in a kept count: {other:?}"), + } + assert_eq!(support::int(&db, "SELECT COUNT(*) FROM notes"), 4); + note_counts_agree(&db, &long, "after the refused insert"); +} + +#[test] +fn create_index_on_a_table_that_keeps_all_prefixes_fills_the_new_count_from_its_rows() { + let db = support::open(&support::folder("kept-count-new-index")); + support::run( + &db, + "CREATE TABLE every (id INTEGER PRIMARY KEY, a INTEGER, b INTEGER, UNIQUE (a, b, id)) \ + MAINTAIN COUNT ALL PREFIXES", + ); + support::run( + &db, + "INSERT INTO every (id, a, b) VALUES (1, 1, 7), (2, 2, 7), (3, 3, 7), (4, 1, 8)", + ); + // The index gives the table a new beginning, `b`, and so a new count, over rows that + // are already there. + support::run(&db, "CREATE INDEX every_b ON every (b)"); + for (b, rows) in [(7, 3), (8, 1), (9, 0)] { + let count = support::run(&db, &format!("SELECT COUNT(*) FROM every WHERE b = {b}")); + assert_eq!(count.rows(), [[Value::Int64(rows)]], "rows of b = {b}"); + assert_eq!(count.rows_examined(), 0, "rows read to count b = {b}"); + } +} diff --git a/v2/contextdb/tests/kept_counts_narrowed.rs b/v2/contextdb/tests/kept_counts_narrowed.rs new file mode 100644 index 00000000..ad379598 --- /dev/null +++ b/v2/contextdb/tests/kept_counts_narrowed.rs @@ -0,0 +1,67 @@ +//! A count through a handle that may not see every row: it is the number of rows the handle +//! sees, never the number there are, and the handle reads the rows to know. A count with no +//! row read is for a reader that sees them all. + +mod support; + +use std::collections::BTreeSet; + +use contextdb::{ContextId, Uuid, Value, params}; + +#[test] +fn a_handle_that_may_not_see_every_row_counts_the_rows_it_sees_by_reading_them() { + let (mine, theirs) = (Uuid::from_u128(1), Uuid::from_u128(2)); + // Once with a kept count over the filter's column and once with only an index: sixty + // rows of kind `k`, twenty of them in the Context the narrowed handle is given. + for (name, clause) in [ + ("narrowed-kept-count", "MAINTAIN COUNT (kind)"), + ("narrowed-index-count", ""), + ] { + let db = support::open(&support::folder(name)); + support::run( + &db, + "CREATE TABLE notes (id INTEGER PRIMARY KEY, context_id UUID CONTEXT_ID, \ + kind TEXT NOT NULL)", + ); + support::run(&db, "CREATE INDEX notes_kind ON notes (kind)"); + if !clause.is_empty() { + support::run(&db, &format!("ALTER TABLE notes SET {clause}")); + } + let tx = db.transaction().expect("begin"); + for n in 1..=90_i64 { + tx.execute( + "INSERT INTO notes (id, context_id, kind) VALUES ($id, $context, $kind)", + ¶ms! { + "id" => n, "context" => if n % 3 == 0 { mine } else { theirs }, + "kind" => if n <= 60 { "k" } else { "other" }, + }, + ) + .expect("insert a note"); + } + tx.commit().expect("commit the notes"); + let count = "SELECT COUNT(*) FROM notes WHERE kind = 'k'"; + + // The writer sees every row and reads none to count them. + let all = support::run(&db, count); + assert_eq!(all.rows(), [[Value::Int64(60)]], "{name}"); + assert_eq!(all.rows_examined(), 0, "{name}"); + + // The narrowed handle is told the number of rows it may see, never the number there + // are, and reads the rows to know. + let narrowed = db.scoped_with_contexts(BTreeSet::from([ContextId::new(mine)])); + let seen = support::rows(&narrowed, "SELECT id FROM notes WHERE kind = 'k'"); + assert_eq!(seen.len(), 20, "{name}"); + let some = support::run(&narrowed, count); + assert_eq!(some.rows(), [[Value::Int64(20)]], "{name}"); + assert!( + some.rows_examined() >= 20, + "{name}: a narrowed count of 20 rows read {} rows", + some.rows_examined() + ); + assert_eq!( + support::int(&narrowed, "SELECT COUNT(*) FROM notes"), + 30, + "{name}: the whole table through the narrowed handle" + ); + } +} diff --git a/v2/contextdb/tests/large_values.rs b/v2/contextdb/tests/large_values.rs index 8dfc170f..37e785df 100644 --- a/v2/contextdb/tests/large_values.rs +++ b/v2/contextdb/tests/large_values.rs @@ -137,17 +137,27 @@ fn assert_notes_are(on: &impl Run, kind: Kind, rows: i64, len: usize, what: &str // ---- every kind and size, three ways ---- /// The sizes of the first test: the last text that stays in its row and the first that does -/// not, two between, and the first that is too long to be compressed. -const SIZES: [usize; 5] = [1015, 1016, 4000, 70_000, LONGEST_COMPRESSED + 1]; +/// not, and two between. Each is taken by every kind of text. +const SIZES: [usize; 4] = [1015, 1016, 4000, 70_000]; -/// One row of `docs` for each kind of text at each size. -const DOCS: i64 = (Kind::ALL.len() * SIZES.len()) as i64; +/// The rows after those: the first size too long to be compressed, in one kind that shrinks and +/// one that does not. +const LONGEST: [Kind; 2] = [Kind::Repeating, Kind::Scattered]; + +/// One row of `docs` for each kind of text at each size, and one for each kind of `LONGEST`. +const DOCS: i64 = (Kind::ALL.len() * SIZES.len() + LONGEST.len()) as i64; /// The kind of text row `id` of `docs` is first written with, and the size of its body and /// of its meta. fn shape(id: i64) -> (Kind, usize) { let at = (id - 1) as usize; - (Kind::ALL[at % Kind::ALL.len()], SIZES[at / Kind::ALL.len()]) + match SIZES.get(at / Kind::ALL.len()) { + Some(size) => (Kind::ALL[at % Kind::ALL.len()], *size), + None => ( + LONGEST[at - Kind::ALL.len() * SIZES.len()], + LONGEST_COMPRESSED + 1, + ), + } } /// The body of row `id`: of the row's own kind when it is inserted, `changes` 0, and of the @@ -639,8 +649,15 @@ fn a_damaged_length_of_a_compressed_value_is_reported_as_damage() { insert_note(&db, 1, &body); support::close(db); - // A length no store holds, one byte more than the value has, and one byte less. - for wrong in [1_u64 << 40, LEN as u64 + 1, LEN as u64 - 1] { + // A length no store holds, one byte more than the value has, one byte less, half of it, none. + for wrong in [ + u64::MAX, + 1_u64 << 40, + LEN as u64 + 1, + LEN as u64 - 1, + LEN as u64 / 2, + 0, + ] { // A storage door that, in every read, puts the wrong length after that 0x84. let changed = Arc::new(AtomicU64::new(0)); let mut options = ReadSessionOptions::default(); diff --git a/v2/contextdb/tests/long_keys.rs b/v2/contextdb/tests/long_keys.rs new file mode 100644 index 00000000..dcd377af --- /dev/null +++ b/v2/contextdb/tests/long_keys.rs @@ -0,0 +1,807 @@ +//! How long a key may be, and what a long value in an index costs. +//! +//! The values that identify a row are kept whole and may take 2,704 bytes as the store writes +//! them; one byte more is refused by name. A value in an index has no limit: a text too long for +//! its room in an index entry (256 bytes) is kept there as its first bytes and a digest of the +//! whole. That must never show: equality finds exactly the rows that hold the value, an order and a range +//! are by the values themselves, a UNIQUE rule compares the values, and the index stays small. + +mod support; + +use std::fs; + +use contextdb::{Database, Error, QueryResult, Uuid, Value, params}; +use support::{Kind, Run}; + +/// The most bytes a key may take as the store writes it. +const KEY_LIMIT: u64 = 2704; +/// The most bytes a TEXT column takes in an index entry (the engine's `ENTRY_TEXT`, which tests +/// cannot name). +const ENTRY_TEXT: usize = 256; +/// The longest text an index keeps whole: a cut text gives 19 of its room to its tag, its mark and +/// its digest. A longer one is kept as this many bytes and a digest. +const WHOLE: usize = ENTRY_TEXT - 19; + +/// The ids a statement returned, in the order it returned them. +fn ids(result: &QueryResult) -> Vec { + result + .rows() + .iter() + .map(|row| match row[0] { + Value::Int64(id) => id, + ref other => panic!("an id came back as {other:?}"), + }) + .collect() +} + +/// A text of `len` bytes: `shared` times the letter `family`, and then letters fixed by `seed`. +/// Two texts of one family and one `shared` begin with the same `shared` bytes. +fn long(family: char, shared: usize, seed: i64, len: usize) -> String { + let mut text = family.to_string().repeat(shared); + text.push_str(&support::letters(seed, len - shared)); + text +} + +fn key_too_long(outcome: contextdb::Result, table: &str, bytes: u64, when: &str) { + match outcome { + Err(Error::KeyTooLong { + table: said_table, + key, + bytes: said_bytes, + limit, + }) => assert_eq!( + (said_table.as_str(), key.as_str(), said_bytes, limit), + (table, "primary key", bytes, KEY_LIMIT), + "{when}" + ), + other => panic!("{when}: {other:?}"), + } +} + +#[test] +fn a_key_of_2704_bytes_is_stored_and_one_of_2705_is_refused_by_name() { + let db = support::open(&support::folder("key-length")); + support::run(&db, "CREATE TABLE names (name TEXT PRIMARY KEY, n INTEGER)"); + let insert = |on: &dyn Run, name: &str, n: i64| { + on.run( + "INSERT INTO names (name, n) VALUES ($name, $n)", + ¶ms! { "name" => name, "n" => n }, + ) + }; + // A text takes its bytes and three more, and one more for each zero byte in it. + let longest = "k".repeat(2701); + let with_zero = format!("{}\0{}", "z".repeat(1000), "z".repeat(1699)); + insert(&db, &longest, 1).expect("a key of 2,704 bytes"); + insert(&db, &with_zero, 2).expect("a key of 2,704 bytes, one of them for a zero byte"); + key_too_long( + insert(&db, &"k".repeat(2702), 3), + "names", + 2705, + "a key of 2,705 bytes", + ); + key_too_long( + insert(&db, &format!("{with_zero}z"), 3), + "names", + 2705, + "a key of 2,705 bytes, one of them for a zero byte", + ); + key_too_long( + insert(&db, &"k".repeat(60_000), 3), + "names", + 60_003, + "a key of 60,003 bytes", + ); + // Inside a transaction the refusal leaves the statements around it standing. + let tx = db.transaction().expect("begin"); + insert(&tx, "short", 4).expect("a short key"); + key_too_long( + insert(&tx, &"q".repeat(2702), 5), + "names", + 2705, + "a key of 2,705 bytes inside a transaction", + ); + // A change of a key is held to the same length. + key_too_long( + tx.execute( + "UPDATE names SET name = $name WHERE n = 4", + ¶ms! { "name" => "s".repeat(2702) }, + ), + "names", + 2705, + "an UPDATE to a key of 2,705 bytes", + ); + tx.commit().expect("commit"); + assert_eq!( + support::rows(&db, "SELECT n FROM names ORDER BY n"), + [1_i64, 2, 4].map(|n| [Value::Int64(n)]) + ); + + // The longest key is found by its value and reads one row. A value longer than any key + // can be names no row: that is an empty answer, found without reading anything. + let find = |name: &str| { + db.execute( + "SELECT n FROM names WHERE name = $name", + ¶ms! { "name" => name }, + ) + .expect("look a row up by its key") + }; + let found = find(&longest); + assert_eq!(found.rows(), [[Value::Int64(1)]]); + assert_eq!(found.rows_examined(), 1); + assert_eq!(find(&with_zero).rows(), [[Value::Int64(2)]]); + for missing in ["k".repeat(2702), "k".repeat(60_000)] { + let found = find(&missing); + assert!(found.rows().is_empty()); + assert_eq!(found.rows_examined(), 0); + } + + // A key of several columns is measured as a whole: a UUID takes 17 bytes. + support::run( + &db, + "CREATE TABLE labels (owner UUID NOT NULL, label TEXT NOT NULL, PRIMARY KEY (owner, label))", + ); + let label = |len: usize| { + db.execute( + "INSERT INTO labels (owner, label) VALUES ($owner, $label)", + ¶ms! { "owner" => Uuid::from_u128(1), "label" => "l".repeat(len) }, + ) + }; + label(2684).expect("a key of 17 and 2,687 bytes"); + key_too_long(label(2685), "labels", 2705, "a key of 17 and 2,688 bytes"); + assert_eq!(support::int(&db, "SELECT COUNT(*) FROM labels"), 1); +} + +/// The table of the tests below: a long value in a plain index (`body`), under a UNIQUE rule +/// (`sole`), and twice in an index of four columns after two short ones (`value` and +/// `identity_value`). +fn declare_anchors(db: &Database) { + support::run( + db, + "CREATE TABLE anchors (id INTEGER PRIMARY KEY, context_id UUID NOT NULL, name TEXT NOT NULL, \ + value TEXT NOT NULL, identity_value TEXT NOT NULL, body TEXT, sole TEXT UNIQUE)", + ); + support::run(db, "CREATE INDEX anchors_body ON anchors (body)"); + support::run( + db, + "CREATE INDEX anchors_exact ON anchors (context_id, name, value, identity_value)", + ); +} + +/// Writes anchor `id`, whose every long column holds `text`. +fn insert_anchor(on: &impl Run, id: i64, text: &str) -> contextdb::Result { + on.run( + "INSERT INTO anchors (id, context_id, name, value, identity_value, body, sole) \ + VALUES ($id, $context, 'command', $text, $text, $text, $text)", + ¶ms! { "id" => id, "context" => Uuid::from_u128(7), "text" => text }, + ) +} + +/// The three statements that find an anchor by a long value: through the plain index, through +/// the UNIQUE rule's index, and through the index of four columns. +const BY_VALUE: [&str; 3] = [ + "SELECT id FROM anchors WHERE body = $text", + "SELECT id FROM anchors WHERE sole = $text", + "SELECT id FROM anchors WHERE context_id = $context AND name = 'command' AND value = $text \ + AND identity_value = $text", +]; + +/// Asserts that each of the three statements, asked for `text`, returns exactly these rows and +/// read no other row to do it. +fn assert_found_by_value(db: &Database, text: &str, wanted: &[i64], what: &str) { + for sql in BY_VALUE { + let mut values = params! { "text" => text }; + if sql.contains("$context") { + values.insert("context".to_owned(), Value::Uuid(Uuid::from_u128(7))); + } + let found = db + .execute(sql, &values) + .expect("look rows up by a long value"); + assert_eq!(ids(&found), wanted, "{what}: {sql}"); + assert_eq!( + found.rows_examined(), + wanted.len() as u64, + "{what}: rows read by {sql}" + ); + } +} + +#[test] +fn a_row_whose_indexed_value_is_60_000_bytes_is_stored_and_found_reading_one_row() { + const LEN: usize = 60_000; + let folder = support::folder("long-indexed-values"); + let db = support::open(&folder); + declare_anchors(&db); + // Ten values of 60,000 bytes that differ from their first byte on, and short ones beside. + let text = |id: i64| support::large_text(Kind::Scattered, id, LEN); + for id in 1..=10 { + insert_anchor(&db, id, &text(id)) + .expect("a row with a value of 60,000 bytes in three indexes"); + insert_anchor(&db, 100 + id, &format!("short-{id}")).expect("a short row"); + } + assert_eq!(support::int(&db, "SELECT COUNT(*) FROM anchors"), 20); + + let check = |db: &Database| { + for id in 1..=10 { + assert_found_by_value(db, &text(id), &[id], &format!("the value of row {id}")); + assert_found_by_value(db, &format!("short-{id}"), &[100 + id], "a short value"); + } + // A value of the same length that no row holds finds nothing and reads nothing. + assert_found_by_value(db, &text(11), &[], "a value no row holds"); + // The value comes back byte for byte. + let stored = support::rows(db, "SELECT body, value FROM anchors WHERE id = 3"); + support::assert_text_is(&stored[0][0], &text(3), "the body of row 3"); + support::assert_text_is(&stored[0][1], &text(3), "the value of row 3"); + }; + check(&db); + + // An UPDATE moves the row to its new value in every index, and a DELETE takes it out. + db.execute( + "UPDATE anchors SET body = $text, sole = $text, value = $text, identity_value = $text WHERE id = 1", + ¶ms! { "text" => text(11) }, + ) + .expect("change a long value"); + assert_found_by_value(&db, &text(1), &[], "the value row 1 held"); + assert_found_by_value(&db, &text(11), &[1], "the value row 1 holds now"); + db.execute( + "UPDATE anchors SET body = $text, sole = $text, value = $text, identity_value = $text WHERE id = 1", + ¶ms! { "text" => text(1) }, + ) + .expect("change it back"); + support::run(&db, "DELETE FROM anchors WHERE id = 10"); + assert_found_by_value(&db, &text(10), &[], "the value of the deleted row"); + insert_anchor(&db, 10, &text(10)).expect("insert the row again"); + + // An index declared over rows that are already there holds them the same way. + support::run(&db, "CREATE INDEX anchors_value ON anchors (value, id)"); + let found = db + .execute( + "SELECT id FROM anchors WHERE value = $text", + ¶ms! { "text" => text(5) }, + ) + .expect("look a row up through the new index"); + assert_eq!((ids(&found), found.rows_examined()), (vec![5], 1)); + + support::close(db); + check(&support::open(&folder)); +} + +#[test] +fn long_values_that_begin_alike_are_each_found_by_their_own_equality_and_by_no_other() { + const LEN: usize = 60_000; + let db = support::open(&support::folder("long-values-alike")); + declare_anchors(&db); + // Rows 1 and 2 share their first 100 bytes, less than any index keeps of a value: they + // differ in the bytes kept. Rows 3, 4 and 5 share their first 3,000 bytes, which is more + // than any index keeps: in every index they differ only by their digest. + // Row 6 is the first 3,000 bytes alone, and row 7 is row 3's value without its last byte. + let mut values = vec![ + long('p', 100, 1, LEN), + long('p', 100, 2, LEN), + long('q', 3000, 3, LEN), + long('q', 3000, 4, LEN), + long('q', 3000, 5, LEN), + "q".repeat(3000), + ]; + values.push(values[2][..LEN - 1].to_owned()); + assert_eq!(values[0][..100], values[1][..100]); + assert_ne!(values[0][..WHOLE], values[1][..WHOLE]); + assert_eq!(values[2][..3000], values[4][..3000]); + for (id, value) in (1..).zip(&values) { + insert_anchor(&db, id, value).expect("a row whose long values begin like another's"); + } + assert_eq!(support::int(&db, "SELECT COUNT(*) FROM anchors"), 7); + + for (id, value) in (1..).zip(&values) { + assert_found_by_value(&db, value, &[id], &format!("the value of row {id}")); + } + // Values that begin the same and that no row holds find nothing, and no row is read for + // them: one more of each family, and a stored value with one byte more. + for missing in [ + long('p', 100, 8, LEN), + long('q', 3000, 9, LEN), + format!("{}x", values[3]), + "q".repeat(3001), + ] { + assert_found_by_value(&db, &missing, &[], "a value that begins like stored ones"); + } +} + +/// The rows of the ordering tests, as (id, body): short values, long values that differ early, +/// and thirteen long values that begin with the same 3,000 bytes, which is more than any index +/// keeps of a value. Among those thirteen, the order the ids are in, the order of the values +/// and the order of their digests are three different orders. Row 30 has no body. +fn lines() -> Vec<(i64, Option)> { + let mut lines = vec![ + (1, Some("a".to_owned())), + (2, Some("m".to_owned())), + (3, Some("zz".to_owned())), + (4, Some("b".repeat(5000))), + (5, Some("y".repeat(5000))), + // The longest value an index of one column keeps whole, and the shortest it cuts: + // the second begins the thirteen. + (6, Some("m".repeat(WHOLE))), + (7, Some("m".repeat(3000))), + (30, None), + ]; + for n in 0..12_i64 { + // Ids rise while the values neither rise nor fall. + lines.push((10 + n, Some(long('m', 3000, (n * 7) % 12, 3400)))); + } + lines +} + +/// The thirteen bodies that begin alike, in id order. +fn run_of_lines() -> Vec { + lines() + .into_iter() + .filter(|(id, _)| *id == 7 || (10..22).contains(id)) + .filter_map(|(_, body)| body) + .collect() +} + +/// The ids of the lines in the order of their bodies: rising, NULL last; or falling, NULL first. +fn in_order(descending: bool) -> Vec { + let mut lines = lines(); + // A body that is there sorts before one that is not; bodies compare as their bytes do. + lines.sort_by(|(_, a), (_, b)| match (a, b) { + (Some(a), Some(b)) => a.as_bytes().cmp(b.as_bytes()), + (a, b) => b.is_some().cmp(&a.is_some()), + }); + if descending { + lines.reverse(); + } + lines.into_iter().map(|(id, _)| id).collect() +} + +/// Declares `table` (id, kind, body) with this index, and writes the lines under kind `k` and +/// three rows of another kind. +fn store_of_lines(db: &Database, table: &str, index_columns: &str) { + support::run( + db, + &format!("CREATE TABLE {table} (id INTEGER PRIMARY KEY, kind TEXT NOT NULL, body TEXT)"), + ); + support::run( + db, + &format!("CREATE INDEX {table}_index ON {table} ({index_columns})"), + ); + let insert = format!("INSERT INTO {table} (id, kind, body) VALUES ($id, $kind, $body)"); + let tx = db.transaction().expect("begin"); + for (id, body) in lines() { + let body = body.map_or(Value::Null, Value::Text); + tx.execute( + &insert, + ¶ms! { "id" => id, "kind" => "k", "body" => body }, + ) + .expect("insert a line"); + } + for (id, body) in [(101_i64, "n"), (102, "c"), (103, "x")] { + tx.execute( + &insert, + ¶ms! { "id" => 1000 + id, "kind" => "j", "body" => body.repeat(4000) }, + ) + .expect("insert a line of another kind"); + } + tx.commit().expect("commit the lines"); +} + +/// Asserts that `sql` returns the lines in this order, whole and with every `LIMIT`. With +/// `run` it also asserts the rows read: a statement with `LIMIT n` reads `n` rows, and when +/// its last row is one of the thirteen that begin alike (they are at these places in the +/// order) it reads all thirteen, because only their values say which of them come first. +fn assert_order(db: &Database, sql: &str, wanted: &[i64], run: Option>) { + let whole = support::run(db, sql); + assert_eq!(ids(&whole), wanted, "{sql}"); + for limit in 1..=wanted.len() { + let limited = format!("{sql} LIMIT {limit}"); + let found = support::run(db, &limited); + assert_eq!(ids(&found), wanted[..limit], "{limited}"); + if let Some(run) = &run { + let read = match run.contains(&(limit - 1)) { + true => run.end, + false => limit, + }; + assert_eq!(found.rows_examined(), read as u64, "rows read by {limited}"); + } + } +} + +#[test] +fn an_order_by_through_an_index_returns_long_values_in_value_order_with_a_limit_too() { + let db = support::open(&support::folder("long-values-ordered")); + store_of_lines(&db, "lines", "body"); + // The kind `j` rows sort among the others by their bodies. + let mut wanted = in_order(false); + let place = |wanted: &[i64], id: i64| wanted.iter().position(|found| *found == id).unwrap(); + // c… after b…, n… after every m…, x… before y…. + let at = place(&wanted, 4) + 1; + wanted.insert(at, 1102); + let at = place(&wanted, 5); + wanted.insert(at, 1103); + let at = place(&wanted, 1103); + wanted.insert(at, 1101); + assert_eq!(wanted.len(), 23); + let first = place(&wanted, 7); + assert_eq!( + wanted[first - 1], + 6, + "the longest whole value comes just before the thirteen" + ); + + assert_order( + &db, + "SELECT id FROM lines ORDER BY body", + &wanted, + Some(first..first + 13), + ); + // The same order with the index read backwards, and for one kind through a filter the + // index does not serve. + let falling: Vec = wanted.iter().rev().copied().collect(); + assert_order( + &db, + "SELECT id FROM lines ORDER BY body DESC", + &falling, + None, + ); + assert_order( + &db, + "SELECT id FROM lines WHERE kind = 'k' ORDER BY body", + &in_order(false), + None, + ); +} + +#[test] +fn an_order_by_through_a_descending_index_returns_long_values_descending_with_a_limit_too() { + let db = support::open(&support::folder("long-values-ordered-descending")); + // An index whose one column is declared DESC. + store_of_lines(&db, "lines", "body DESC"); + let kind_k = "SELECT id FROM lines WHERE kind = 'k' ORDER BY body DESC"; + let falling = in_order(true); + assert_eq!( + falling[0], 30, + "the row with no body comes first going down" + ); + let run_at = falling.iter().position(|id| *id == 7).unwrap() - 12; + assert_eq!( + falling[run_at + 13], + 6, + "the longest whole value comes just after the thirteen" + ); + // The thirteen come back falling, never rising and never as their digests fall. + let rising_run: Vec = in_order(false) + .into_iter() + .filter(|id| *id == 7 || (10..22).contains(id)) + .collect(); + let found = ids(&support::run(&db, kind_k)); + let found_run: Vec = found[run_at..run_at + 13].to_vec(); + assert_ne!( + found_run, rising_run, + "the run of long values came back ascending" + ); + assert_eq!(found, falling, "{kind_k}"); + + // Every row of the table, in the index's own order, with the rows read for each LIMIT. The + // rows of kind `j` sort among the others by their bodies: x… after y…, n… before every m…, + // c… before b…. + let mut wanted = falling.clone(); + let place = |wanted: &[i64], id: i64| wanted.iter().position(|found| *found == id).unwrap(); + let at = place(&wanted, 5) + 1; + wanted.insert(at, 1103); + let at = place(&wanted, 1103) + 1; + wanted.insert(at, 1101); + let at = place(&wanted, 4); + wanted.insert(at, 1102); + let first = place(&wanted, 7) - 12; + assert_order( + &db, + "SELECT id FROM lines ORDER BY body DESC", + &wanted, + Some(first..first + 13), + ); + // The same index read backwards gives the rising order. + let rising: Vec = wanted.iter().rev().copied().collect(); + assert_order(&db, "SELECT id FROM lines ORDER BY body", &rising, None); + + // An index of two columns, the second declared DESC, with the first given by the filter: + // the long values are cut in the column after the one the filter fixed. + store_of_lines(&db, "kinds", "kind ASC, body DESC"); + let through_two = "SELECT id FROM kinds WHERE kind = 'k' ORDER BY body DESC"; + assert_order(&db, through_two, &falling, None); + let found = ids(&support::run( + &db, + &format!("{through_two} LIMIT {}", run_at + 13), + )); + assert_ne!( + found[run_at..].to_vec(), + rising_run, + "the run of long values came back ascending through the index of two columns" + ); + // With a LIMIT the index gives the order: far fewer rows are read than the kind has. + let few = support::run(&db, &format!("{through_two} LIMIT 2")); + assert_eq!(ids(&few), falling[..2]); + assert_eq!(few.rows_examined(), 2); +} + +/// Whether a body is on the kept side of a bound. +type Keeps = fn(&[u8], &[u8]) -> bool; + +#[test] +fn a_range_bounded_by_a_long_value_returns_exactly_the_rows_in_range() { + let db = support::open(&support::folder("long-values-ranged")); + store_of_lines(&db, "lines", "body"); + let lines = lines(); + let run = run_of_lines(); + let total = support::int(&db, "SELECT COUNT(*) FROM lines WHERE kind = 'k'") as usize; + // Bounds that are stored values, short and long, inside the thirteen and outside them, and + // bounds that no row holds: one that begins like the thirteen, and one longer than all. + let bounds = [ + "m".to_owned(), + "m".repeat(WHOLE), + "m".repeat(3000), + run[4].clone(), + run[9].clone(), + "b".repeat(5000), + format!("{}zzz", "m".repeat(3000)), + "m".repeat(2000), + "y".repeat(5001), + ]; + for (number, bound) in bounds.iter().enumerate() { + let operators: [(&str, Keeps); 4] = [ + (">", |body, bound| body > bound), + (">=", |body, bound| body >= bound), + ("<", |body, bound| body < bound), + ("<=", |body, bound| body <= bound), + ]; + for (operator, keeps) in operators { + let sql = format!( + "SELECT id FROM lines WHERE kind = 'k' AND body {operator} $bound ORDER BY id" + ); + let found = db + .execute(&sql, ¶ms! { "bound" => bound.as_str() }) + .expect("read a range bounded by a long value"); + let mut wanted: Vec = lines + .iter() + .filter(|(_, body)| { + body.as_ref() + .is_some_and(|body| keeps(body.as_bytes(), bound.as_bytes())) + }) + .map(|(id, _)| *id) + .collect(); + wanted.sort_unstable(); + assert_eq!( + ids(&found), + wanted, + "bound {number}: body {operator} a value" + ); + // The rows read are the rows of the range, of either kind, and at most the rest of + // the thirteen that begin like the bound: never the table for a range of a few rows. + let most = (wanted.len() + 3 + 13).min(total + 3) as u64; + assert!( + found.rows_examined() <= most, + "bound {number}: body {operator} a value read {} rows for a range of {}", + found.rows_examined(), + wanted.len() + ); + } + } + + // Between two of the thirteen, either end taken in or left out. + let (low, high) = match run[4] < run[9] { + true => (&run[4], &run[9]), + false => (&run[9], &run[4]), + }; + for (lower, upper) in [(">", "<"), (">=", "<="), (">", "<="), (">=", "<")] { + let sql = format!( + "SELECT id FROM lines WHERE body {lower} $low AND body {upper} $high ORDER BY id" + ); + let found = db + .execute( + &sql, + ¶ms! { "low" => low.as_str(), "high" => high.as_str() }, + ) + .expect("read a range between two long values"); + let mut wanted: Vec = lines + .iter() + .filter(|(_, body)| { + body.as_ref().is_some_and(|body| { + let above = if lower == ">" { + body > low + } else { + body >= low + }; + let below = if upper == "<" { + body < high + } else { + body <= high + }; + above && below + }) + }) + .map(|(id, _)| *id) + .collect(); + wanted.sort_unstable(); + assert_eq!(ids(&found), wanted, "{sql}"); + assert!( + found.rows_examined() <= 13, + "a range inside the thirteen read {} rows", + found.rows_examined() + ); + } +} + +#[test] +fn a_unique_rule_over_long_values_refuses_the_same_value_and_takes_one_that_differs_in_a_byte() { + const LEN: usize = 60_000; + let db = support::open(&support::folder("long-values-unique")); + support::run( + &db, + "CREATE TABLE commands (id INTEGER PRIMARY KEY, command TEXT UNIQUE, a TEXT, b TEXT, UNIQUE (a, b))", + ); + let insert = |on: &dyn Run, id: i64, text: &str, a: &str, b: &str| { + on.run( + "INSERT INTO commands (id, command, a, b) VALUES ($id, $text, $a, $b)", + ¶ms! { "id" => id, "text" => text, "a" => a, "b" => b }, + ) + }; + let refused = |outcome: contextdb::Result, columns: &str, when: &str| match outcome + { + Err(Error::UniqueViolation { table, column, .. }) => { + assert_eq!( + (table.as_str(), column.as_str()), + ("commands", columns), + "{when}" + ); + } + other => panic!("{when}: {other:?}"), + }; + let value = long('c', 3000, 1, LEN); + // The same value but for its last byte, and but for its first byte past what an index keeps. + let last_differs = format!("{}#", &value[..LEN - 1]); + let early_differs = format!("{}#{}", &value[..3000], &value[3001..]); + let (pa, pb) = (long('a', 3000, 2, LEN), long('b', 3000, 3, LEN)); + let pb_differs = format!("{}#", &pb[..LEN - 1]); + + insert(&db, 1, &value, &pa, &pb).expect("the first row"); + let again = insert(&db, 2, &value, "x", "y"); + // The refusal of a long value carries its first 64 bytes and its length, not the value. + let said = again + .as_ref() + .err() + .map(ToString::to_string) + .unwrap_or_default(); + assert!( + said.contains(&value[..64]) && !said.contains(&value[..65]) && said.contains("60000"), + "the refusal must carry the first 64 bytes of the value and its length: {said}" + ); + assert!( + said.len() < 200, + "a refusal of a 60,000-byte value is {} bytes", + said.len() + ); + refused(again, "command", "the same long value a second time"); + refused( + insert(&db, 2, "t", &pa, &pb), + "a, b", + "the same pair of long values a second time", + ); + insert(&db, 2, &last_differs, &pa, &pb_differs).expect("values that differ in their last byte"); + insert(&db, 3, &early_differs, &pb, &pa).expect("values that differ further in"); + // A change to a value another row holds is refused; a change to one no row holds is taken. + refused( + db.execute( + "UPDATE commands SET command = $text WHERE id = 3", + ¶ms! { "text" => last_differs.as_str() }, + ), + "command", + "an UPDATE to a long value another row holds", + ); + refused( + db.execute( + "UPDATE commands SET b = $b WHERE id = 2", + ¶ms! { "b" => pb.as_str() }, + ), + "a, b", + "an UPDATE to a pair of long values another row holds", + ); + assert_eq!(support::int(&db, "SELECT COUNT(*) FROM commands"), 3); + for (id, text) in [(1_i64, &value), (2, &last_differs), (3, &early_differs)] { + let found = db + .execute( + "SELECT id FROM commands WHERE command = $text", + ¶ms! { "text" => text.as_str() }, + ) + .expect("look a row up by its long value"); + assert_eq!((ids(&found), found.rows_examined()), (vec![id], 1)); + } + + // A child table that references the long UNIQUE column finds a stored value, and finds none + // for a value that differs in its last byte. + support::run( + &db, + "CREATE TABLE notes (id INTEGER PRIMARY KEY, command TEXT REFERENCES commands(command))", + ); + let note = |id: i64, text: &str| { + db.execute( + "INSERT INTO notes (id, command) VALUES ($id, $text)", + ¶ms! { "id" => id, "text" => text }, + ) + }; + note(1, &value).expect("a child naming a stored long value"); + let missing = format!("{}#", &early_differs[..LEN - 1]); + match note(2, &missing) { + Err(Error::ForeignKeyViolation { table, .. }) => assert_eq!(table, "notes"), + other => panic!("a child naming a long value no row holds gave {other:?}"), + } + assert_eq!(support::int(&db, "SELECT COUNT(*) FROM notes"), 1); + + // Two transactions write one long value side by side: the second to commit is refused. + // One that wrote a value that only begins the same commits. + let taken = long('c', 3000, 7, LEN); + let beside = long('c', 3000, 8, LEN); + let losing = db.transaction().expect("begin"); + let winning = db.transaction().expect("begin"); + insert(&losing, 10, &taken, "l", "l").expect("stage the value"); + insert(&winning, 11, &beside, "w", "w").expect("stage a value that begins the same"); + insert(&db, 12, &taken, "d", "d").expect("commit the value meanwhile"); + winning + .commit() + .expect("commit a value that only begins like the others"); + match losing.commit() { + Err(Error::UniqueViolation { table, column, .. }) => { + assert_eq!((table.as_str(), column.as_str()), ("commands", "command")); + } + other => panic!("a commit of a long value another commit took gave {other:?}"), + } + assert_eq!( + support::rows(&db, "SELECT id FROM commands ORDER BY id"), + [1_i64, 2, 3, 11, 12].map(|id| [Value::Int64(id)]) + ); +} + +#[test] +fn two_hundred_rows_of_50_000_byte_indexed_values_make_a_store_smaller_than_twice_their_bytes() { + const ROWS: i64 = 200; + const LEN: usize = 50_000; + let folder = support::folder("long-values-store-size"); + let db = support::open(&folder); + support::run( + &db, + "CREATE TABLE notes (id INTEGER PRIMARY KEY, body TEXT NOT NULL)", + ); + support::run(&db, "CREATE INDEX notes_body ON notes (body)"); + // Text that does not shrink, so each value is stored once in as many bytes as it has: an + // index that held the values too would hold as many again. + let body = |id: i64| support::large_text(Kind::Scattered, id, LEN); + for first in (1..=ROWS).step_by(10) { + let tx = db.transaction().expect("begin"); + for id in first..first + 10 { + tx.execute( + "INSERT INTO notes (id, body) VALUES ($id, $body)", + ¶ms! { "id" => id, "body" => body(id) }, + ) + .expect("insert a note"); + } + tx.commit().expect("commit ten notes"); + } + let found = db + .execute( + "SELECT id FROM notes WHERE body = $body", + ¶ms! { "body" => body(137) }, + ) + .expect("look a note up by its body"); + assert_eq!((ids(&found), found.rows_examined()), (vec![137], 1)); + support::close(db); + + let values = ROWS as u64 * LEN as u64; + let stored = fs::metadata(folder.join("data")) + .expect("the store's data file") + .len(); + assert!( + stored >= values, + "the store is {stored} bytes and holds {values} bytes of values that do not shrink" + ); + assert!( + stored < 2 * values, + "the store is {stored} bytes for {values} bytes of values: an index entry holds a whole value" + ); +} diff --git a/v2/contextdb/tests/memory_ceiling.rs b/v2/contextdb/tests/memory_ceiling.rs index 898eeca1..c7389f55 100644 --- a/v2/contextdb/tests/memory_ceiling.rs +++ b/v2/contextdb/tests/memory_ceiling.rs @@ -15,7 +15,7 @@ mod support; use std::alloc::{GlobalAlloc, Layout, System}; use std::fs; -use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use contextdb::{ Database, DatabaseOpenOptions, Error, ReadSession, ReadSessionOptions, Value, params, @@ -25,6 +25,11 @@ const LIMIT: usize = 64 << 20; const ROWS: i64 = 1_000_000; const ROWS_IN_A_COMMIT: i64 = 10_000; +/// The most pages one UPDATE of one small column may write: the pages from the root to the row +/// and to its two index entries, the commit's own records, and room beside them. +const UPDATE_PAGES: u64 = 32; +const PAGE: u64 = 4096; + /// The system allocator, counting the bytes allocated now and the most there have been. struct Counting { now: AtomicUsize, @@ -109,6 +114,25 @@ fn a_million_rows_load_and_scan_under_64_mib() { {} bytes on the store's account under a limit of {LIMIT}", peak - before ); + + // A commit writes what it changed, not what the store holds: the hundredth commit of + // 10,000 new rows writes about what the first did, whatever the tree has grown to. + let first = + support::last_number(&lines, "first_commit_bytes").expect("the loader announces it"); + let last = support::last_number(&lines, "last_commit_bytes").expect("the loader announces it"); + assert!( + last <= 2 * first, + "the first commit of {ROWS_IN_A_COMMIT} rows wrote {first} bytes and the hundredth wrote \ + {last}: a commit's writes grow with the size of the store" + ); + // One UPDATE of one small column of one row, in a store of a million rows, changes the + // pages on the way to that row and to its index entry, and nothing else. + let update = support::last_number(&lines, "update_bytes").expect("the loader announces it"); + assert!( + update <= UPDATE_PAGES * PAGE, + "one UPDATE of one column of one row in a store of {ROWS} rows wrote {update} bytes, more \ + than {UPDATE_PAGES} pages of {PAGE} bytes" + ); } /// Opens the store with the limit, loads the rows, scans them, and announces `before `, @@ -122,13 +146,31 @@ fn load_and_scan(role: support::Role) { ALLOCATED.peak.store(before, Ordering::Relaxed); let mut options = DatabaseOpenOptions::default(); options.memory_limit = Some(LIMIT); + // The bytes written to the data file, counted by the storage door. + let written = std::sync::Arc::new(AtomicU64::new(0)); + options.storage = Some(support::storage({ + let written = written.clone(); + move |call| { + if let support::Call::Write(_, bytes) = call { + written.fetch_add(bytes.len() as u64, Ordering::SeqCst); + } + } + })); let db = support::open_with(&role.folder, options).expect("open the store under the limit"); for first in (1..=ROWS).step_by(ROWS_IN_A_COMMIT as usize) { + written.store(0, Ordering::SeqCst); let tx = db.transaction().expect("begin"); support::insert(&tx, first..first + ROWS_IN_A_COMMIT); tx.commit() .expect("commit ten thousand rows under the limit"); + let bytes = written.load(Ordering::SeqCst); + if first == 1 { + support::say(format!("first_commit_bytes {bytes}")); + } + if first + ROWS_IN_A_COMMIT > ROWS { + support::say(format!("last_commit_bytes {bytes}")); + } support::say(format!("committed {}", first + ROWS_IN_A_COMMIT - 1)); } @@ -148,6 +190,11 @@ fn load_and_scan(role: support::Role) { let wanted: Vec<_> = (ROWS - 99..=ROWS).map(support::row).collect(); support::assert_same_rows(last_page.rows(), &wanted, "the last hundred rows"); + // One UPDATE of one small column of one row, and the bytes it writes. + written.store(0, Ordering::SeqCst); + support::run(&db, "UPDATE t SET tag = 7 WHERE id = 500000"); + support::say(format!("update_bytes {}", written.load(Ordering::SeqCst))); + // Memory the host takes for itself is not the store's: with twice the limit allocated // beside it, the store counts what it counted and still answers under its limit. let loaded = ALLOCATED.peak.load(Ordering::Relaxed); @@ -178,10 +225,17 @@ fn load_and_scan(role: support::Role) { const VALUE: usize = 1 << 20; const VALUE_LIMIT: usize = (9 << 20) + (64 << 10); +/// The JSON cases of the test below: the same 1 MiB as text, in the shape that shrinks most +/// (an array of small objects), whose tree in memory is some twenty times its text. The limit +/// leaves 4 MiB beyond the open's 9 MiB: room for the text, none for the tree. +const JSON_ROOM: usize = 4 << 20; +const JSON_LIMIT: usize = (9 << 20) + JSON_ROOM; + #[test] fn a_refused_compressed_value_is_never_produced() { if let Some(role) = support::role() { - return refused_select(&role); + refused_select(&role); + return refused_json(&role); } let folder = support::folder("refused-value-never-produced"); let db = support::open(&folder); @@ -189,6 +243,15 @@ fn a_refused_compressed_value_is_never_produced() { &db, "CREATE TABLE notes (id INTEGER PRIMARY KEY, body TEXT NOT NULL)", ); + support::run( + &db, + "CREATE TABLE records (id INTEGER PRIMARY KEY, meta JSON NOT NULL)", + ); + db.execute( + "INSERT INTO records (id, meta) VALUES ($id, $meta)", + ¶ms! { "id" => 1_i64, "meta" => Value::Json(support::large_json(true, 1, VALUE)) }, + ) + .expect("store the JSON value"); let large = support::large_text(support::Kind::Repeating, 1, VALUE); for (id, body) in [(1_i64, large.as_str()), (2, "small")] { db.execute( @@ -228,6 +291,43 @@ fn a_refused_compressed_value_is_never_produced() { "while a value of {VALUE} bytes was refused, the process allocated {rise} bytes more \ than it had before the select" ); + + // A JSON value is counted as the tree it becomes, not as its text: a tree of some twenty + // times the text, under a limit with room for the text only, is refused by name when it is + // selected and when the same text is written, and is made in neither case. + for (prefix, what) in [("select_", "selected"), ("write_", "written")] { + assert_eq!( + support::last_number(&lines, &format!("{prefix}refused")), + Some(1), + "a JSON value of {VALUE} bytes of text whose tree is far larger than the \ + {JSON_ROOM} bytes of room its limit leaves was not refused when {what}: the limit \ + counts the tree, not the text" + ); + let rise = support::last_number(&lines, &format!("{prefix}rise")).expect("announced"); + assert!( + rise < JSON_ROOM as u64, + "while a JSON value was refused when {what}, the process allocated {rise} bytes more \ + than it had before: the tree was made though the limit left {JSON_ROOM} bytes" + ); + } +} + +/// Runs one statement and announces `refused 1` when the memory limit refused it (0 when +/// it was answered) and `rise `: how far the most this process ever had allocated +/// rose above what it had allocated just before the statement. Any other outcome fails. +fn measured(prefix: &str, statement: impl FnOnce() -> contextdb::Result) -> bool { + let before = ALLOCATED.now.load(Ordering::Relaxed); + ALLOCATED.peak.store(before, Ordering::Relaxed); + let outcome = statement(); + let peak = ALLOCATED.peak.load(Ordering::Relaxed); + let refused = match outcome { + Err(Error::MemoryBudgetExceeded { .. }) => true, + Err(other) => panic!("the statement was refused as {other:?}"), + Ok(_) => false, + }; + support::say(format!("{prefix}refused {}", u8::from(refused))); + support::say(format!("{prefix}rise {}", peak - before)); + refused } /// Opens the store for reading under the limit and reads the small row and the count, so that @@ -246,26 +346,49 @@ fn refused_select(role: &support::Role) { [[support::text("small")]] ); - let before = ALLOCATED.now.load(Ordering::Relaxed); - ALLOCATED.peak.store(before, Ordering::Relaxed); - let outcome = reader.execute("SELECT body FROM notes WHERE id = 1", ¶ms! {}); - let peak = ALLOCATED.peak.load(Ordering::Relaxed); - match outcome { - Err(Error::MemoryBudgetExceeded { .. }) => support::say("refused 1"), - Err(other) => panic!("the select of the large value was refused as {other:?}"), - Ok(result) => panic!( - "the select of the large value was answered with {} rows", - result.rows().len() - ), - } - support::say(format!("rise {}", peak - before)); + measured("", || { + reader.execute("SELECT body FROM notes WHERE id = 1", ¶ms! {}) + }); +} + +/// Selects the JSON value under [`JSON_LIMIT`] from a reader, then writes the text of a second +/// value of the same shape from a writer under the same limit, and announces each as +/// [`measured`] does, with the prefixes `select_` and `write_`. A refused write left no row. +fn refused_json(role: &support::Role) { + let mut options = ReadSessionOptions::default(); + options.memory_limit = Some(JSON_LIMIT); + let reader = ReadSession::open_with_options(&role.folder, options) + .expect("open the store for reading under the limit"); + assert_eq!(support::int(&reader, "SELECT COUNT(*) FROM records"), 1); + measured("select_", || { + reader.execute("SELECT meta FROM records WHERE id = 1", ¶ms! {}) + }); + drop(reader); + + let mut options = DatabaseOpenOptions::default(); + options.memory_limit = Some(JSON_LIMIT); + let db = support::open_with(&role.folder, options).expect("open the store under the limit"); + let text = contextdb::serde_json::to_string(&support::large_json(true, 2, VALUE)) + .expect("JSON writes as text"); + measured("write_", || { + db.execute( + "INSERT INTO records (id, meta) VALUES ($id, $meta)", + ¶ms! { "id" => 2_i64, "meta" => text }, + ) + }); + assert_eq!( + support::int(&db, "SELECT COUNT(*) FROM records"), + 1, + "the refused write left a row" + ); + support::close(db); } // ---- a full cache ---- /// Rows of a table beside the one a test measures, each with a text of 900 bytes that does not -/// shrink: 18 MiB, more than twice the 8 MiB cache of a store whose limit is under 32 MiB. -const FILLER_ROWS: i64 = 20_000; +/// shrink: 9 MiB, more than the 8 MiB cache of a store whose limit is under 32 MiB. +const FILLER_ROWS: i64 = 10_000; const FILLER_TEXT: usize = 900; /// Writes the filler rows, in a table of their own and one commit. @@ -465,7 +588,7 @@ fn an_indexed_search_and_a_build_stay_under_the_limit() { let db = support::open(&folder); write_the_filler(&db); support::declare_docs(&db, 64, "AUTO_INDEX_AT 1000"); - support::load_docs(&db, 1..=2000, "a", |n| support::gathered(n, 64)); + support::load_docs(&db, 1..=1100, "a", |n| support::gathered(n, 64)); support::close(db); let lines = support::Child::start( @@ -481,9 +604,19 @@ fn an_indexed_search_and_a_build_stay_under_the_limit() { assert!(refused.is_empty(), "under {BUILD_LIMIT} bytes: {refused:?}"); let told = |key: &str| support::last_number(&lines, key); assert_eq!( - (told("stalled"), told("lists"), told("rows")), - (Some(0), Some(45), Some(10)), - "the build's stalled partitions and groups, and the rows of the indexed search" + ( + told("stalled"), + told("lists"), + told("rows"), + told("exact_rows") + ), + (Some(0), Some(33), Some(10), Some(10)), + "the build's stalled partitions and groups, and the rows of the indexed and of the \ + exact search" + ); + assert!( + lines.iter().any(|line| line == "committed 200"), + "the commit of vectors under the limit was not taken: {lines:?}" ); // What the process took for the store, from before the open to the end, every step of // the build and the search included, is within the limit. Each began with the cache full. @@ -507,6 +640,20 @@ fn a_build_under_the_limit(role: &support::Role) { let mut options = DatabaseOpenOptions::default(); options.memory_limit = Some(BUILD_LIMIT); let db = support::open_with(&role.folder, options).expect("open the store under the limit"); + // One commit of vectors under the limit: 200 rows take a nearby vector of the same + // direction, so the column keeps its 1,100 vectors and the build below is the same. + fill_the_cache(&db); + let tx = db.transaction().expect("begin"); + for id in 1..=200_i64 { + let values = params! { "id" => id, "v" => support::gathered(id + 8, 64) }; + if let Err(error) = tx.execute("UPDATE docs SET v = $v WHERE id = $id", &values) { + support::say(format!("refused {error:?}")); + } + } + match tx.commit() { + Ok(_) => support::say("committed 200"), + Err(error) => support::say(format!("refused {error:?}")), + } for _ in 0..100 { fill_the_cache(&db); match db.run_maintenance_cycle() { @@ -534,6 +681,11 @@ fn a_build_under_the_limit(role: &support::Role) { Ok(result) => support::say(format!("rows {}", result.rows().len())), Err(error) => support::say(format!("refused {error:?}")), } + fill_the_cache(&db); + match support::search(&db, &support::by_mode(None, "EXACT", 10), &query) { + Ok(result) => support::say(format!("exact_rows {}", result.rows().len())), + Err(error) => support::say(format!("refused {error:?}")), + } support::close(db); support::say(format!("before {before}")); support::say(format!("peak {}", ALLOCATED.peak.load(Ordering::Relaxed))); diff --git a/v2/contextdb/tests/propagation.rs b/v2/contextdb/tests/propagation.rs new file mode 100644 index 00000000..498d20b6 --- /dev/null +++ b/v2/contextdb/tests/propagation.rs @@ -0,0 +1,379 @@ +//! Propagation: rows that declared they follow a transition make it inside the statement that +//! makes the transition. So the transaction that ran the statement reads the rows as moved, +//! nobody else does until it commits, and a rollback undoes all of it together. + +mod support; + +use contextdb::{Database, Error, Uuid, Value, params}; +use support::Run; + +/// The status of every row of a table, in key order. +fn statuses(on: &impl Run, table: &str) -> Vec { + support::rows(on, &format!("SELECT status FROM {table} ORDER BY id")) + .into_iter() + .map(|row| match row.as_slice() { + [Value::Text(status)] => status.clone(), + other => panic!("a status of {table} came back as {other:?}"), + }) + .collect() +} + +/// `count` times one status. +fn all(status: &str, count: usize) -> Vec { + vec![status.to_owned(); count] +} + +/// A store with projects 1 to 3, all active, and 3,000 tasks: task `n` names project +/// `n % 3 + 1`. A task follows its project into the archive. `clause` is written after the +/// reference's `PROPAGATE SET invalidated`. +fn store_of_projects(name: &str, clause: &str) -> Database { + let db = support::open(&support::folder(name)); + support::run( + &db, + "CREATE TABLE projects (id INTEGER PRIMARY KEY, status TEXT NOT NULL) \ + STATE MACHINE (status: active -> [archived])", + ); + support::run( + &db, + &format!( + "CREATE TABLE tasks (id INTEGER PRIMARY KEY, status TEXT NOT NULL, \ + project_id INTEGER REFERENCES projects(id) ON STATE archived PROPAGATE SET invalidated \ + {clause}) STATE MACHINE (status: active -> [invalidated, superseded])" + ), + ); + support::run( + &db, + "INSERT INTO projects (id, status) VALUES (1, 'active'), (2, 'active'), (3, 'active')", + ); + let tx = db.transaction().expect("begin"); + for n in 0..3000_i64 { + tx.execute( + "INSERT INTO tasks (id, status, project_id) VALUES ($id, 'active', $project)", + ¶ms! { "id" => n, "project" => n % 3 + 1 }, + ) + .expect("insert a task"); + } + tx.commit().expect("commit the tasks"); + db +} + +/// The statuses the tasks have once the projects in `archived` are archived and nothing else +/// has happened. +fn tasks_after(archived: &[i64]) -> Vec { + (0..3000_i64) + .map(|n| match archived.contains(&(n % 3 + 1)) { + true => "invalidated".to_owned(), + false => "active".to_owned(), + }) + .collect() +} + +#[test] +fn rows_follow_inside_the_statement_and_are_seen_and_undone_with_its_transaction() { + let db = store_of_projects("propagation-in-transaction", ""); + + // Rolled back: the project and every task that followed it are as they were. + let tx = db.transaction().expect("begin"); + let archived = support::run(&tx, "UPDATE projects SET status = 'archived' WHERE id = 2"); + assert_eq!(archived.rows_affected(), 1, "the rows the statement named"); + // The transaction reads its own propagation, a thousand rows of it and no other row. + assert_eq!(statuses(&tx, "tasks"), tasks_after(&[2])); + assert_eq!( + support::int( + &tx, + "SELECT COUNT(*) FROM tasks WHERE status = 'invalidated'" + ), + 1000 + ); + // Nobody else does. + assert_eq!(statuses(&db, "tasks"), all("active", 3000)); + assert_eq!(statuses(&db, "projects"), all("active", 3)); + tx.rollback(); + assert_eq!(statuses(&db, "tasks"), all("active", 3000)); + assert_eq!(statuses(&db, "projects"), all("active", 3)); + + // Committed: the transition and the rows that followed it arrive together. + let tx = db.transaction().expect("begin"); + support::run(&tx, "UPDATE projects SET status = 'archived' WHERE id = 2"); + // A task that follows may still be changed by the transaction's next statement. + support::run(&tx, "UPDATE tasks SET status = 'superseded' WHERE id = 0"); + tx.commit().expect("commit the transition"); + let mut wanted = tasks_after(&[2]); + wanted[0] = "superseded".to_owned(); + assert_eq!(statuses(&db, "tasks"), wanted); + assert_eq!(statuses(&db, "projects"), ["active", "archived", "active"]); + + // One statement that moves two projects moves the tasks of both. Task 0, of project 1, was + // superseded, which lists no next state: it is left as it is and the rest follow. + let both = support::run(&db, "UPDATE projects SET status = 'archived' WHERE id <> 2"); + assert_eq!(both.rows_affected(), 2); + let mut wanted = all("invalidated", 3000); + wanted[0] = "superseded".to_owned(); + assert_eq!(statuses(&db, "tasks"), wanted); + + assert_eq!( + support::int( + &db, + "SELECT COUNT(*) FROM tasks WHERE status = 'invalidated'" + ), + 2999 + ); +} + +#[test] +fn an_aborting_propagation_names_its_table_and_undoes_its_statement_and_no_other() { + let db = store_of_projects("propagation-aborted", "ABORT ON FAILURE"); + // One task of project 1 cannot follow: superseded lists no next state. + support::run( + &db, + "UPDATE tasks SET status = 'superseded' WHERE id = 1500", + ); + let mut before = all("active", 3000); + before[1500] = "superseded".to_owned(); + + let aborted = |error: Error, how: &str| match error { + Error::PropagationAborted { table, reason } => { + assert_eq!(table, "tasks", "{how}"); + assert!(!reason.is_empty(), "{how}: the refusal gives no reason"); + } + other => panic!("a propagation that cannot move a row, {how}, gave {other:?}"), + }; + // On the writer: the project has not moved and neither has any of its 999 other tasks. + aborted( + support::refusal(&db, "UPDATE projects SET status = 'archived' WHERE id = 1"), + "on the writer", + ); + assert_eq!(statuses(&db, "projects"), all("active", 3)); + assert_eq!(statuses(&db, "tasks"), before); + + // Inside a transaction the refused statement is undone whole, the statements before it + // stand, and the transaction commits them. + let tx = db.transaction().expect("begin"); + support::run(&tx, "UPDATE projects SET status = 'archived' WHERE id = 2"); + aborted( + support::refusal( + &tx, + "UPDATE projects SET status = 'archived' WHERE id IN (1, 3)", + ), + "inside a transaction", + ); + assert_eq!(statuses(&tx, "projects"), ["active", "archived", "active"]); + let mut wanted = tasks_after(&[2]); + wanted[1500] = "superseded".to_owned(); + assert_eq!(statuses(&tx, "tasks"), wanted); + tx.commit().expect("commit the statement that stood"); + assert_eq!(statuses(&db, "projects"), ["active", "archived", "active"]); + assert_eq!(statuses(&db, "tasks"), wanted); + + // A project all of whose tasks can follow is archived with them. + support::run(&db, "UPDATE projects SET status = 'archived' WHERE id = 3"); + let mut wanted = tasks_after(&[2, 3]); + wanted[1500] = "superseded".to_owned(); + assert_eq!(statuses(&db, "tasks"), wanted); +} + +/// Declares `claims`, whose rows follow each other into `invalidated` along incoming `CITES` +/// edges, with this `clause` after the declaration, and the edge table beside it. Claim `n` +/// has the id `Uuid::from_u128(n)`. +fn store_of_claims(name: &str, claims: u128, clause: &str) -> Database { + let db = support::open(&support::folder(name)); + support::run( + &db, + "CREATE TABLE edges (id INTEGER PRIMARY KEY, source_id UUID NOT NULL, \ + target_id UUID NOT NULL, edge_type TEXT NOT NULL)", + ); + support::run( + &db, + &format!( + "CREATE TABLE claims (id UUID PRIMARY KEY, status TEXT NOT NULL) \ + STATE MACHINE (status: active -> [invalidated]) \ + PROPAGATE ON EDGE CITES INCOMING STATE invalidated SET invalidated {clause}" + ), + ); + let tx = db.transaction().expect("begin"); + for n in 1..=claims { + tx.execute( + "INSERT INTO claims (id, status) VALUES ($id, 'active')", + ¶ms! { "id" => Uuid::from_u128(n) }, + ) + .expect("insert a claim"); + } + tx.commit().expect("commit the claims"); + db +} + +/// Writes the edge `id`: claim `source` cites claim `target`. +fn cites(on: &impl Run, id: i64, source: u128, target: u128) { + on.run( + "INSERT INTO edges (id, source_id, target_id, edge_type) VALUES ($id, $source, $target, 'CITES')", + ¶ms! { + "id" => id, "source" => Uuid::from_u128(source), "target" => Uuid::from_u128(target), + }, + ) + .expect("insert an edge"); +} + +fn invalidate(on: &impl Run, claim: u128) { + on.run( + "UPDATE claims SET status = 'invalidated' WHERE id = $id", + ¶ms! { "id" => Uuid::from_u128(claim) }, + ) + .expect("invalidate a claim"); +} + +#[test] +fn max_depth_reaches_rows_at_most_that_many_edges_from_the_changed_row() { + // A chain of three: claim 2 cites claim 1, and claim 3 cites claim 2. + let chain = |name: &str, clause: &str| { + let db = store_of_claims(name, 3, clause); + cites(&db, 1, 2, 1); + cites(&db, 2, 3, 2); + invalidate(&db, 1); + statuses(&db, "claims") + }; + // One edge away follows; two edges away is past MAX DEPTH 1 and does not. + assert_eq!( + chain("propagation-depth-one", "MAX DEPTH 1"), + ["invalidated", "invalidated", "active"] + ); + // With MAX DEPTH 2 both follow. + assert_eq!( + chain("propagation-depth-two", "MAX DEPTH 2"), + ["invalidated", "invalidated", "invalidated"] + ); +} + +/// Row `n` of a table whose rows follow each other along edges. +fn node(n: u128) -> Uuid { + Uuid::from_u128(n) +} + +/// The table of edges, and `nodes` with rows 1 to `count` in the state `first`, declared with +/// `rules` after its columns. +fn store_of_nodes(name: &str, rules: &str, first: &str, count: u128) -> Database { + let db = support::open(&support::folder(name)); + support::run( + &db, + "CREATE TABLE edges (id INTEGER PRIMARY KEY, source_id UUID NOT NULL, \ + target_id UUID NOT NULL, edge_type TEXT NOT NULL)", + ); + support::run( + &db, + &format!( + "CREATE TABLE nodes (id UUID PRIMARY KEY, status TEXT NOT NULL, \ + batch INTEGER NOT NULL) {rules}" + ), + ); + for n in 1..=count { + db.execute( + "INSERT INTO nodes (id, status, batch) VALUES ($id, $status, 0)", + ¶ms! { "id" => node(n), "status" => first }, + ) + .expect("insert a node"); + } + db +} + +/// Joins node `from` to node `to` by an edge of type FEEDS. +fn feed(db: &Database, edge: i64, from: u128, to: u128) { + db.execute( + "INSERT INTO edges (id, source_id, target_id, edge_type) \ + VALUES ($id, $from, $to, 'FEEDS')", + ¶ms! { "id" => edge, "from" => node(from), "to" => node(to) }, + ) + .expect("insert an edge"); +} + +#[test] +fn a_follower_reachable_from_two_changed_rows_changes_once_by_the_shorter_way() { + let db = store_of_nodes( + "propagation-shorter-way", + "STATE MACHINE (status: running -> [stopped]) \ + PROPAGATE ON EDGE FEEDS OUTGOING STATE stopped SET stopped MAX DEPTH 2", + "running", + 6, + ); + support::run(&db, "CREATE INDEX nodes_status ON nodes (status)"); + support::run(&db, "ALTER TABLE nodes SET MAINTAIN COUNT (status)"); + // 1 feeds 2, 2 feeds 3, 3 feeds 4 and 4 feeds 5; 6 feeds 3 as well. So node 3 is two + // edges from node 1 and one edge from node 6. + for (edge, from, to) in [(1, 1, 2), (2, 2, 3), (3, 3, 4), (4, 4, 5), (5, 6, 3)] { + feed(&db, edge, from, to); + } + for n in [1, 6] { + db.execute( + "UPDATE nodes SET batch = 1 WHERE id = $id", + ¶ms! { "id" => node(n) }, + ) + .expect("mark a node"); + } + let stopped = support::run(&db, "UPDATE nodes SET status = 'stopped' WHERE batch = 1"); + assert_eq!(stopped.rows_affected(), 2, "the rows the statement named"); + + // Node 3 is one edge from a changed row, so node 4 is two and follows under MAX DEPTH 2. + // Had node 3 been reached by way of node 2, node 4 would be three edges away and stay. + // Node 5 is three edges away by the shortest way and stays. + assert_eq!( + statuses(&db, "nodes"), + [ + "stopped", "stopped", "stopped", "stopped", "running", "stopped" + ] + ); + // Node 3 changed once: the kept count of each state is the number of rows in it. + for (status, rows) in [("stopped", 5), ("running", 1)] { + let count = support::run( + &db, + &format!("SELECT COUNT(*) FROM nodes WHERE status = '{status}'"), + ); + assert_eq!( + count.rows(), + [[Value::Int64(rows)]], + "rows that are {status}" + ); + assert_eq!(count.rows_examined(), 0, "rows read to count {status}"); + } +} + +#[test] +fn a_follower_the_handle_may_not_write_refuses_the_statement_by_kind_and_nothing_changes() { + let (mine, theirs) = (Uuid::from_u128(1), Uuid::from_u128(2)); + let db = support::open(&support::folder("propagation-narrowed-writer")); + support::run( + &db, + "CREATE TABLE projects (id INTEGER PRIMARY KEY, context_id UUID CONTEXT_ID, \ + status TEXT NOT NULL) STATE MACHINE (status: active -> [archived])", + ); + support::run( + &db, + "CREATE TABLE tasks (id INTEGER PRIMARY KEY, context_id UUID CONTEXT_ID, \ + status TEXT NOT NULL, \ + project_id INTEGER REFERENCES projects(id) ON STATE archived PROPAGATE SET invalidated) \ + STATE MACHINE (status: active -> [invalidated])", + ); + db.execute( + "INSERT INTO projects (id, context_id, status) VALUES (1, $mine, 'active')", + ¶ms! { "mine" => mine }, + ) + .expect("insert the project"); + // Task 1 is in the Context the narrowed handle is given and task 2 is in another. + for (id, context) in [(1_i64, mine), (2, theirs)] { + db.execute( + "INSERT INTO tasks (id, context_id, status, project_id) \ + VALUES ($id, $context, 'active', 1)", + ¶ms! { "id" => id, "context" => context }, + ) + .expect("insert a task"); + } + + let contexts = std::collections::BTreeSet::from([contextdb::ContextId::new(mine)]); + let narrowed = db.scoped_with_contexts(contexts); + match support::refusal( + &narrowed, + "UPDATE projects SET status = 'archived' WHERE id = 1", + ) { + Error::ContextScopeViolation { table } => assert_eq!(table, "tasks"), + other => panic!("a follower in another Context refused the statement as {other:?}"), + } + assert_eq!(statuses(&db, "projects"), all("active", 1)); + assert_eq!(statuses(&db, "tasks"), all("active", 2)); +} diff --git a/v2/contextdb/tests/readers.rs b/v2/contextdb/tests/readers.rs index aac309d2..36bd862d 100644 --- a/v2/contextdb/tests/readers.rs +++ b/v2/contextdb/tests/readers.rs @@ -8,7 +8,7 @@ use std::path::Path; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use contextdb::{DatabaseOpenOptions, ReadSession, ReadSessionOptions, Value}; +use contextdb::{DatabaseOpenOptions, Error, ReadSession, ReadSessionOptions, Value, params}; use support::{Call, Child}; @@ -219,6 +219,21 @@ fn a_killed_reader_leaves_one_file_that_the_next_open_clears() { support::listing(&readers).is_empty(), "a reader that ended left its file behind" ); + + // With `readers` replaced by a plain file a reader cannot make its file: it is refused as a + // failure of the storage, and no row comes from it. + std::fs::remove_dir(&readers).expect("the empty readers folder"); + std::fs::write(&readers, b"not a folder").expect("a plain file where readers was"); + let outcome = ReadSession::open_with_options(&folder, ReadSessionOptions::default()) + .and_then(|session| session.execute(support::SCAN, ¶ms! {})); + match outcome { + Err(Error::StoreIo { .. }) => {} + Err(other) => panic!("a reader beside a plain file named readers was refused as {other:?}"), + Ok(result) => panic!( + "a reader beside a plain file named readers returned {} rows", + result.rows().len() + ), + } } #[test] diff --git a/v2/contextdb/tests/refusals.rs b/v2/contextdb/tests/refusals.rs index f0eb7504..e2d06e04 100644 --- a/v2/contextdb/tests/refusals.rs +++ b/v2/contextdb/tests/refusals.rs @@ -19,7 +19,7 @@ fn a_unique_refusal_names_the_table_and_every_column_of_the_rule() { support::run(&db, "INSERT INTO t (id, a, b, c) VALUES (2, 1, 'x', 2)"); match support::refusal(&db, "INSERT INTO t (id, a, b, c) VALUES (3, 1, 'x', 1)") { - Error::UniqueViolation { table, column } => { + Error::UniqueViolation { table, column, .. } => { assert_eq!((table.as_str(), column.as_str()), ("t", "a, b, c")); } other => panic!("a duplicate under UNIQUE (a, b, c) was refused as {other:?}"), diff --git a/v2/contextdb/tests/row_guards.rs b/v2/contextdb/tests/row_guards.rs new file mode 100644 index 00000000..af65df39 --- /dev/null +++ b/v2/contextdb/tests/row_guards.rs @@ -0,0 +1,427 @@ +//! The row guard: a transaction asks whether a row holds some values now, and asks its commit to +//! stand only if the row still holds them then. A guard writes nothing, so a guarded row is not +//! a changed row. + +mod support; + +use std::collections::BTreeSet; + +use contextdb::{ContextId, Database, Error, Transaction, Uuid, Value, params}; + +const LEASES: &str = "CREATE TABLE leases (id INTEGER PRIMARY KEY, generation UUID UNIQUE, \ + owner TEXT, epoch INTEGER NOT NULL)"; +const ALL_LEASES: &str = "SELECT id, owner, epoch FROM leases ORDER BY id"; + +/// A store with `leases` 1 (owner `a`, epoch 1) and 2 (no owner, epoch 5), each with a +/// generation of its own number. +fn store_of_leases(name: &str) -> Database { + let db = support::open(&support::folder(name)); + support::run(&db, LEASES); + for (id, owner, epoch) in [(1_i64, Value::from("a"), 1_i64), (2, Value::Null, 5)] { + db.execute( + "INSERT INTO leases (id, generation, owner, epoch) VALUES ($id, $generation, $owner, $epoch)", + ¶ms! { + "id" => id, "generation" => Uuid::from_u128(id as u128), "owner" => owner, + "epoch" => epoch, + }, + ) + .expect("insert a lease"); + } + db +} + +/// Asks the transaction to stand only if lease `id` still holds these values when it commits. +fn guard(tx: &Transaction<'_>, id: i64, values: &[(&str, Value)]) -> bool { + let predicates: Vec<(String, Value)> = values + .iter() + .map(|(column, value)| ((*column).to_owned(), value.clone())) + .collect(); + tx.guard_row_conditions("leases", "id", &Value::Int64(id), &predicates) + .expect("ask for a row guard") +} + +fn assert_guard_conflict(outcome: contextdb::Result<()>, when: &str) { + match outcome { + Err(Error::ConditionalUpdateConflict { count }) => assert_eq!(count, 1, "{when}"), + other => panic!("{when}: {other:?}"), + } +} + +#[test] +fn a_row_guard_answers_true_only_for_a_row_that_is_there_and_holds_every_value() { + let db = store_of_leases("guard-answers"); + let tx = db.transaction().expect("begin"); + + assert!(guard( + &tx, + 1, + &[("owner", "a".into()), ("epoch", 1_i64.into())] + )); + assert!(guard(&tx, 1, &[("epoch", 1_i64.into())])); + // A NULL that is there is a value the row holds. + assert!(guard( + &tx, + 2, + &[("owner", Value::Null), ("epoch", 5_i64.into())] + )); + // With nothing to hold, the guard asks only that the row is there. + assert!(guard(&tx, 1, &[])); + + // One value of two is another, a value where the row has NULL, NULL where it has a value. + assert!(!guard( + &tx, + 1, + &[("owner", "a".into()), ("epoch", 2_i64.into())] + )); + assert!(!guard(&tx, 2, &[("owner", "a".into())])); + assert!(!guard(&tx, 1, &[("owner", Value::Null)])); + // No such row. + assert!(!guard(&tx, 404, &[])); + assert!(!guard(&tx, 404, &[("epoch", 1_i64.into())])); + + // The row may be named by another column that names one row. + let by_generation = |generation: u128, epoch: i64| { + tx.guard_row_conditions( + "leases", + "generation", + &Value::Uuid(Uuid::from_u128(generation)), + &[("epoch".to_owned(), Value::Int64(epoch))], + ) + .expect("ask for a row guard by another column") + }; + assert!(by_generation(2, 5)); + assert!(!by_generation(2, 6)); + assert!(!by_generation(9, 5)); + + // A table or a column that is not there is refused by name, and the transaction goes on. + assert!(matches!( + tx.guard_row_conditions("nowhere", "id", &Value::Int64(1), &[]), + Err(Error::TableNotFound(table)) if table == "nowhere" + )); + assert!(matches!( + tx.guard_row_conditions("leases", "id", &Value::Int64(1), &[("nope".to_owned(), Value::Null)]), + Err(Error::ColumnNotFound { table, column }) if table == "leases" && column == "nope" + )); + assert!(guard(&tx, 1, &[("epoch", 1_i64.into())])); + + // The guard reads through the transaction: a value the transaction itself wrote is the + // value the row holds. + support::run(&tx, "UPDATE leases SET epoch = 6 WHERE id = 2"); + assert!(guard(&tx, 2, &[("epoch", 6_i64.into())])); + assert!(!guard(&tx, 2, &[("epoch", 5_i64.into())])); + tx.rollback(); + assert_eq!( + support::rows(&db, ALL_LEASES), + [ + vec![Value::Int64(1), support::text("a"), Value::Int64(1)], + vec![Value::Int64(2), Value::Null, Value::Int64(5)], + ] + ); +} + +#[test] +fn a_row_guard_answers_false_for_a_row_hidden_from_its_handle() { + let db = support::open(&support::folder("guard-hidden")); + support::run( + &db, + "CREATE TABLE notes (id INTEGER PRIMARY KEY, context_id UUID CONTEXT_ID, body TEXT)", + ); + let (mine, theirs) = (Uuid::from_u128(1), Uuid::from_u128(2)); + for (id, context) in [(1_i64, mine), (2, theirs)] { + db.execute( + "INSERT INTO notes (id, context_id, body) VALUES ($id, $context, 'kept')", + ¶ms! { "id" => id, "context" => context }, + ) + .expect("insert a note"); + } + let holds = |tx: &Transaction<'_>, id: i64| { + tx.guard_row_conditions( + "notes", + "id", + &Value::Int64(id), + &[("body".to_owned(), support::text("kept"))], + ) + .expect("ask for a row guard") + }; + + // The writer sees both rows. A handle given one Context sees its own row and not the other, + // though the other is there and holds the value. + let tx = db.transaction().expect("begin"); + assert!(holds(&tx, 1) && holds(&tx, 2)); + tx.rollback(); + let narrowed = db.scoped_with_contexts(BTreeSet::from([ContextId::new(mine)])); + let tx = narrowed + .transaction() + .expect("begin on the narrowed handle"); + assert!(holds(&tx, 1)); + assert!( + !holds(&tx, 2), + "a guard answered for a row its handle cannot see" + ); + tx.commit().expect("commit a transaction that only guarded"); +} + +#[test] +fn a_commit_is_refused_when_a_guarded_value_was_changed_meanwhile_and_writes_nothing() { + let db = store_of_leases("guard-conflict"); + let insert_three = "INSERT INTO leases (id, owner, epoch) VALUES (3, 'c', 1)"; + + // The guarded value was changed by another commit. + let tx = db.transaction().expect("begin"); + assert!(guard( + &tx, + 1, + &[("owner", "a".into()), ("epoch", 1_i64.into())] + )); + support::run(&tx, insert_three); + support::run(&db, "UPDATE leases SET epoch = 2 WHERE id = 1"); + assert_guard_conflict(tx.commit(), "a commit over a guarded value that changed"); + assert_eq!(support::int(&db, "SELECT COUNT(*) FROM leases"), 2); + + // The guarded row was deleted by another commit. + let tx = db.transaction().expect("begin"); + assert!(guard(&tx, 2, &[("epoch", 5_i64.into())])); + support::run(&tx, insert_three); + support::run(&db, "DELETE FROM leases WHERE id = 2"); + assert_guard_conflict(tx.commit(), "a commit over a guarded row that is gone"); + assert_eq!(support::int(&db, "SELECT COUNT(*) FROM leases"), 1); + + // A column the guard did not name changed: the guarded values are still there, and the + // commit stands. + let tx = db.transaction().expect("begin"); + assert!(guard(&tx, 1, &[("epoch", 2_i64.into())])); + support::run(&tx, insert_three); + support::run(&db, "UPDATE leases SET owner = 'b' WHERE id = 1"); + tx.commit() + .expect("commit over a guarded row whose other column changed"); + assert_eq!( + support::rows(&db, ALL_LEASES), + [ + vec![Value::Int64(1), support::text("b"), Value::Int64(2)], + vec![Value::Int64(3), support::text("c"), Value::Int64(1)], + ] + ); + + // A guard that answered false asked for nothing: the commit stands whatever the row does. + let tx = db.transaction().expect("begin"); + assert!(!guard(&tx, 1, &[("epoch", 99_i64.into())])); + support::run( + &tx, + "INSERT INTO leases (id, owner, epoch) VALUES (4, 'd', 1)", + ); + support::run(&db, "UPDATE leases SET epoch = 3 WHERE id = 1"); + tx.commit().expect("commit after a guard that did not hold"); + assert_eq!(support::int(&db, "SELECT COUNT(*) FROM leases"), 3); +} + +#[test] +fn a_guard_taken_after_a_declaration_refuses_the_commit_when_its_value_was_changed_meanwhile() { + let db = store_of_leases("guard-after-declaration"); + let before = db.table_names(); + let tx = db.transaction().expect("begin"); + support::run( + &tx, + "CREATE TABLE audits (id INTEGER PRIMARY KEY, note TEXT)", + ); + assert!(guard( + &tx, + 1, + &[("owner", "a".into()), ("epoch", 1_i64.into())] + )); + // The store's own handle commits a change to the guarded value. + support::run(&db, "UPDATE leases SET epoch = 2 WHERE id = 1"); + assert_guard_conflict( + tx.commit(), + "a commit of a declaration over a guarded value that changed", + ); + // Nothing the transaction declared is stored. + assert_eq!(db.table_names(), before); + assert!(matches!( + support::refusal(&db, "SELECT COUNT(*) FROM audits"), + Error::TableNotFound(table) if table == "audits" + )); + assert_eq!( + support::rows(&db, ALL_LEASES), + [ + vec![Value::Int64(1), support::text("a"), Value::Int64(2)], + vec![Value::Int64(2), Value::Null, Value::Int64(5)], + ] + ); + + // A guard names places in the row as the transaction shows it, and holds the same values + // whatever column the transaction dropped before or after it. Each case has a table of + // its own: the transaction drops `a`, guards `epoch` (before the drop in the + // third case), and the store's own handle changes one cell before the commit. + for (table, guard_first, meanwhile, held) in [ + ("seats_a", false, "a = 9", true), + ("seats_b", false, "epoch = 2", false), + ("seats_c", true, "a = 9", true), + ] { + support::run( + &db, + &format!("CREATE TABLE {table} (id INTEGER PRIMARY KEY, a INTEGER, epoch INTEGER)"), + ); + support::run(&db, &format!("INSERT INTO {table} VALUES (1, 1, 1)")); + let tx = db.transaction().expect("begin"); + let ask = || { + tx.guard_row_conditions( + table, + "id", + &Value::Int64(1), + &[("epoch".into(), 1_i64.into())], + ) + .expect("ask for a row guard") + }; + let before_drop = guard_first.then(ask); + support::run(&tx, &format!("ALTER TABLE {table} DROP COLUMN a")); + assert!( + before_drop.unwrap_or_else(ask), + "{table}: the guard on epoch after the drop" + ); + support::run(&db, &format!("UPDATE {table} SET {meanwhile} WHERE id = 1")); + if held { + tx.commit() + .unwrap_or_else(|e| panic!("{table}: a guard that still held: {e:?}")); + assert_eq!( + support::rows(&db, &format!("SELECT * FROM {table}")), + [[Value::Int64(1), Value::Int64(1)]], + "{table}: the drop stands and no cell moved" + ); + } else { + assert_guard_conflict(tx.commit(), table); + assert_eq!( + support::rows(&db, &format!("SELECT a FROM {table}")), + [[Value::Int64(1)]], + "{table}: the refused commit dropped nothing" + ); + } + } +} + +#[test] +fn a_guarded_row_is_not_a_changed_row() { + let db = store_of_leases("guard-is-no-write"); + let before = support::rows(&db, ALL_LEASES); + let watermark = db.committed_watermark(); + + // A transaction that only guards commits and writes nothing. + let tx = db.transaction().expect("begin"); + assert!(guard(&tx, 1, &[("epoch", 1_i64.into())])); + tx.commit().expect("commit a transaction that only guarded"); + assert_eq!(support::rows(&db, ALL_LEASES), before); + assert_eq!( + db.committed_watermark(), + watermark, + "a transaction that only guarded made a commit" + ); + + // Two transactions guard the same row and write other rows. Two that had changed the row + // could not both commit; a guard changes nothing, so both do. + let first = db.transaction().expect("begin"); + let second = db.transaction().expect("begin"); + for (tx, id) in [(&first, 3_i64), (&second, 4)] { + assert!(guard(tx, 1, &[("owner", "a".into())])); + tx.execute( + "INSERT INTO leases (id, owner, epoch) VALUES ($id, 'n', 1)", + ¶ms! { "id" => id }, + ) + .expect("insert beside the guarded row"); + } + first.commit().expect("the first guarded commit"); + second + .commit() + .expect("the second commit over the same guarded row"); + + // And a transaction that changed the guarded row's other column commits beside one that + // guarded it: the guard holds a value, not the row. + let guarding = db.transaction().expect("begin"); + assert!(guard(&guarding, 1, &[("epoch", 1_i64.into())])); + support::run(&guarding, "UPDATE leases SET owner = 'z' WHERE id = 2"); + let writing = db.transaction().expect("begin"); + support::run(&writing, "UPDATE leases SET owner = 'w' WHERE id = 1"); + writing.commit().expect("commit a change to the row"); + guarding + .commit() + .expect("commit over a guard whose value is still held"); + assert_eq!( + support::rows( + &db, + "SELECT id, owner, epoch FROM leases WHERE id <= 2 ORDER BY id" + ), + [ + vec![Value::Int64(1), support::text("w"), Value::Int64(1)], + vec![Value::Int64(2), support::text("z"), Value::Int64(5)], + ] + ); +} + +#[test] +fn a_guard_expects_null_in_a_column_added_after_the_row_was_written() { + let db = store_of_leases("guard-added-column"); + support::run(&db, "ALTER TABLE leases ADD COLUMN holder TEXT"); + let insert_three = "INSERT INTO leases (id, owner, epoch) VALUES (3, 'c', 1)"; + + // The row was written before the column was there, and holds NULL in it. + let tx = db.transaction().expect("begin"); + assert!(guard( + &tx, + 1, + &[("holder", Value::Null), ("epoch", 1_i64.into())] + )); + assert!(!guard(&tx, 1, &[("holder", "x".into())])); + support::run(&tx, insert_three); + tx.commit() + .expect("commit over a guard that expects NULL in the added column"); + assert_eq!(support::int(&db, "SELECT COUNT(*) FROM leases"), 3); + + // Another commit gives the row a value there: the guarded commit is refused. + let tx = db.transaction().expect("begin"); + assert!(guard(&tx, 2, &[("holder", Value::Null)])); + support::run( + &tx, + "INSERT INTO leases (id, owner, epoch) VALUES (4, 'd', 1)", + ); + support::run(&db, "UPDATE leases SET holder = 'x' WHERE id = 2"); + assert_guard_conflict( + tx.commit(), + "a commit over a guard on the added column, after the column was written", + ); + assert_eq!(support::int(&db, "SELECT COUNT(*) FROM leases"), 3); + assert_eq!( + support::rows(&db, "SELECT holder FROM leases ORDER BY id"), + [[Value::Null], [support::text("x")], [Value::Null]] + ); +} + +#[test] +fn a_guard_is_checked_again_at_commit_though_the_transaction_wrote_nothing() { + let db = store_of_leases("guard-rechecked-at-commit"); + + // The transaction only guards. Another commit changes the guarded value before it commits: + // the guard no longer holds, and the commit is refused though it has nothing to write. + let tx = db.transaction().expect("begin"); + assert!(guard(&tx, 1, &[("epoch", 1_i64.into())])); + support::run(&db, "UPDATE leases SET epoch = 2 WHERE id = 1"); + assert_guard_conflict( + tx.commit(), + "a commit that only guarded, over a value another commit changed", + ); + + // The guard is checked against the row as it is at the commit, not as it was on the way: a + // value changed and put back is a value the row holds. + let tx = db.transaction().expect("begin"); + assert!(guard(&tx, 2, &[("epoch", 5_i64.into())])); + support::run(&db, "UPDATE leases SET epoch = 6 WHERE id = 2"); + support::run(&db, "UPDATE leases SET epoch = 5 WHERE id = 2"); + tx.commit() + .expect("commit over a guarded value that is held again"); + + // Only the other commits wrote. + assert_eq!( + support::rows(&db, ALL_LEASES), + [ + vec![Value::Int64(1), support::text("a"), Value::Int64(2)], + vec![Value::Int64(2), Value::Null, Value::Int64(5)], + ] + ); +} diff --git a/v2/contextdb/tests/rules.rs b/v2/contextdb/tests/rules.rs new file mode 100644 index 00000000..403b3f6c --- /dev/null +++ b/v2/contextdb/tests/rules.rs @@ -0,0 +1,679 @@ +//! The rules a table declares, as a host meets them: each refusal comes as its kind with the +//! fields that say what was refused, and leaves the store as it was. A reference is checked +//! when its transaction commits. + +mod support; + +use contextdb::{Database, Error, Uuid, Value, params}; +use support::Run; + +/// A statement and the values it binds, run on a writer or inside one of its transactions. +fn bound( + on: &impl Run, + sql: &str, + values: &std::collections::HashMap, +) -> contextdb::Result { + on.run(sql, values) +} + +/// Runs `refused` three ways, each of which must be refused, and hands every refusal to `check`: +/// as text on the writer, with its values bound on the writer, and inside a transaction that +/// stays usable and is then rolled back. A rule is a rule about rows, whichever way a write +/// arrives. +fn refused_every_way( + db: &Database, + text: &str, + with_values: (&str, std::collections::HashMap), + check: impl Fn(Error, &str), +) { + check(support::refusal(db, text), "written into the statement"); + let (sql, values) = with_values; + match bound(db, sql, &values) { + Err(error) => check(error, "with its values bound"), + Ok(result) => panic!("{sql}\nwas not refused with its values bound: {result:?}"), + } + let tx = db.transaction().expect("begin"); + check(support::refusal(&tx, text), "inside a transaction"); + // The transaction is as it was: it reads, and it has staged nothing. + tx.rollback(); +} + +const TASKS: &str = "CREATE TABLE tasks (id INTEGER PRIMARY KEY, kind TEXT NOT NULL IMMUTABLE, \ + status TEXT NOT NULL, note TEXT) \ + STATE MACHINE (status: draft -> [active, rejected], active -> [superseded])"; +const ALL_TASKS: &str = "SELECT id, kind, status, note FROM tasks ORDER BY id"; + +/// A store with `tasks` (a frozen column and a state machine) holding three rows, and `log`, a +/// table that is IMMUTABLE as a whole, holding one. +fn store_of_tasks(name: &str) -> Database { + let db = support::open(&support::folder(name)); + support::run(&db, TASKS); + support::run( + &db, + "INSERT INTO tasks (id, kind, status, note) VALUES (1, 'plan', 'draft', 'a'), \ + (2, 'plan', 'active', 'b'), (3, 'fix', 'rejected', 'c')", + ); + support::run( + &db, + "CREATE TABLE log (id INTEGER PRIMARY KEY, entry TEXT NOT NULL) IMMUTABLE", + ); + support::run(&db, "INSERT INTO log (id, entry) VALUES (1, 'first')"); + db +} + +#[test] +fn an_immutable_refusal_names_the_table_or_the_column_and_changes_nothing() { + let db = store_of_tasks("immutable-refusals"); + let tasks = support::rows(&db, ALL_TASKS); + let log = support::rows(&db, "SELECT id, entry FROM log ORDER BY id"); + let declared = db.declaration("tasks").expect("the declaration of tasks"); + + let whole_table = |error: Error, how: &str| match error { + Error::ImmutableTable { table } => assert_eq!(table, "log", "{how}"), + other => panic!("a change to an IMMUTABLE table, {how}, was refused as {other:?}"), + }; + refused_every_way( + &db, + "UPDATE log SET entry = 'changed' WHERE id = 1", + ( + "UPDATE log SET entry = $entry WHERE id = $id", + params! { "entry" => "changed", "id" => 1_i64 }, + ), + whole_table, + ); + refused_every_way( + &db, + "DELETE FROM log WHERE id = 1", + ("DELETE FROM log WHERE id = $id", params! { "id" => 1_i64 }), + whole_table, + ); + refused_every_way( + &db, + "INSERT INTO log (id, entry) VALUES (1, 'again') ON CONFLICT (id) DO UPDATE SET entry = 'again'", + ( + "INSERT INTO log (id, entry) VALUES ($id, $entry) ON CONFLICT (id) DO UPDATE SET entry = $entry", + params! { "id" => 1_i64, "entry" => "again" }, + ), + whole_table, + ); + + let one_column = |error: Error, how: &str| match error { + Error::ImmutableColumn { table, column } => { + assert_eq!( + (table.as_str(), column.as_str()), + ("tasks", "kind"), + "{how}" + ); + } + other => panic!("a change to an IMMUTABLE column, {how}, was refused as {other:?}"), + }; + refused_every_way( + &db, + "UPDATE tasks SET note = 'z', kind = 'other' WHERE id = 1", + ( + "UPDATE tasks SET note = $note, kind = $kind WHERE id = $id", + params! { "note" => "z", "kind" => "other", "id" => 1_i64 }, + ), + one_column, + ); + refused_every_way( + &db, + "INSERT INTO tasks (id, kind, status) VALUES (1, 'plan', 'draft') \ + ON CONFLICT (id) DO UPDATE SET kind = 'other'", + ( + "INSERT INTO tasks (id, kind, status) VALUES ($id, $kind, $status) \ + ON CONFLICT (id) DO UPDATE SET kind = $kind", + params! { "id" => 1_i64, "kind" => "other", "status" => "draft" }, + ), + one_column, + ); + // The column cannot be taken out of the table or renamed either. + for sql in [ + "ALTER TABLE tasks DROP COLUMN kind", + "ALTER TABLE tasks DROP COLUMN kind CASCADE", + "ALTER TABLE tasks RENAME COLUMN kind TO sort", + ] { + one_column(support::refusal(&db, sql), sql); + } + + assert_eq!(support::rows(&db, ALL_TASKS), tasks); + assert_eq!( + support::rows(&db, "SELECT id, entry FROM log ORDER BY id"), + log + ); + assert_eq!( + db.declaration("tasks").expect("the declaration of tasks"), + declared + ); +} + +#[test] +fn a_state_refusal_names_the_table_the_column_and_both_states() { + let db = store_of_tasks("state-refusals"); + let tasks = support::rows(&db, ALL_TASKS); + + // (the row, the state asked for, the state the row is in) + for (id, to, from) in [ + (2_i64, "draft", "active"), + (1, "superseded", "draft"), + (3, "active", "rejected"), + (1, "paused", "draft"), + (2, "active", "active"), + ] { + refused_every_way( + &db, + &format!("UPDATE tasks SET status = '{to}' WHERE id = {id}"), + ( + "UPDATE tasks SET status = $to WHERE id = $id", + params! { "to" => to, "id" => id }, + ), + |error, how| { + // The sentence a person reads names the table and the column too. + assert_eq!( + error.to_string(), + format!("invalid state transition of tasks.status: {from} -> {to}"), + "row {id}, {how}" + ); + match error { + Error::InvalidStateTransition { + table, + column, + from: said_from, + to: said_to, + } => assert_eq!( + ( + table.as_str(), + column.as_str(), + said_from.as_str(), + said_to.as_str() + ), + ("tasks", "status", from, to), + "row {id}, {how}" + ), + other => panic!("moving row {id} from {from} to {to}, {how}, gave {other:?}"), + } + }, + ); + } + assert_eq!(support::rows(&db, ALL_TASKS), tasks); + + // A statement over several rows is refused by the one that may not move, and names it. + match support::refusal(&db, "UPDATE tasks SET status = 'active' WHERE id IN (1, 3)") { + Error::InvalidStateTransition { from, to, .. } => { + assert_eq!((from.as_str(), to.as_str()), ("rejected", "active")); + } + other => panic!("moving a rejected row was refused as {other:?}"), + } + assert_eq!(support::rows(&db, ALL_TASKS), tasks); + + // Inside a transaction a refused move leaves the moves before it standing. + let tx = db.transaction().expect("begin"); + support::run(&tx, "UPDATE tasks SET status = 'active' WHERE id = 1"); + assert!(matches!( + support::refusal(&tx, "UPDATE tasks SET status = 'draft' WHERE id = 1"), + Error::InvalidStateTransition { .. } + )); + support::run(&tx, "UPDATE tasks SET status = 'superseded' WHERE id = 1"); + tx.commit().expect("commit the two listed moves"); + assert_eq!( + support::rows(&db, "SELECT status FROM tasks WHERE id = 1"), + [[support::text("superseded")]] + ); + + // A declaration prints the states a row may leave, and those it may go to, in the order + // they were written. + support::run( + &db, + "CREATE TABLE jobs (id INTEGER PRIMARY KEY, status TEXT NOT NULL) \ + STATE MACHINE (status: zeta -> [b, a], draft -> active, active -> [superseded, archived])", + ); + let declared = db.declaration("jobs").expect("the declaration of jobs"); + let wanted = "STATE MACHINE (status: zeta -> [b, a], draft -> [active], \ + active -> [superseded, archived])"; + assert!(declared.contains(wanted), "not\n{wanted}\nbut\n{declared}"); +} + +const PROJECTS: &str = "CREATE TABLE projects (id INTEGER PRIMARY KEY, title TEXT)"; +const WORK: &str = "CREATE TABLE work (id INTEGER PRIMARY KEY, project_id INTEGER REFERENCES projects(id), \ + note TEXT)"; + +/// A store with `projects` 1 and 2 and `work`, whose rows reference a project: work 1 names +/// project 1. +fn store_of_work(name: &str) -> Database { + let db = support::open(&support::folder(name)); + support::run(&db, PROJECTS); + support::run(&db, WORK); + support::run( + &db, + "INSERT INTO projects (id, title) VALUES (1, 'one'), (2, 'two')", + ); + support::run(&db, "INSERT INTO work (id, project_id) VALUES (1, 1)"); + db +} + +/// Asserts that an outcome is the refusal of a reference from `work`: with `still_referenced`, +/// a project that a row of `work` still names; without, a row of `work` whose project is not +/// there. +fn assert_reference_refused( + outcome: contextdb::Result, + still_referenced: bool, + when: &str, +) { + match outcome { + Err(Error::ForeignKeyViolation { + table, + columns, + parent, + still_referenced: said, + .. + }) => assert_eq!( + (table.as_str(), columns.as_str(), parent.as_str(), said), + ("work", "project_id", "projects", still_referenced), + "{when}" + ), + other => panic!("{when}: {other:?}"), + } +} + +const ALL_WORK: &str = "SELECT id, project_id FROM work ORDER BY id"; +const ALL_PROJECTS: &str = "SELECT id FROM projects ORDER BY id"; + +#[test] +fn a_reference_is_checked_when_its_transaction_commits_and_a_refused_commit_leaves_nothing() { + let db = store_of_work("reference-at-commit"); + + // The row that references comes first and its parent after it: the commit sees both. + let tx = db.transaction().expect("begin"); + support::run(&tx, "INSERT INTO work (id, project_id) VALUES (2, 3)"); + support::run(&tx, "INSERT INTO projects (id, title) VALUES (3, 'three')"); + tx.commit().expect("commit a row and the parent it names"); + + // A row whose parent never comes is taken by its statement, with the rows around it, and + // refused by the commit. The transaction is gone and none of it was written. + let tx = db.transaction().expect("begin"); + support::run(&tx, "INSERT INTO projects (id, title) VALUES (4, 'four')"); + support::run(&tx, "INSERT INTO work (id, project_id) VALUES (3, 4)"); + support::run(&tx, "INSERT INTO work (id, project_id) VALUES (4, 404)"); + support::run(&tx, "UPDATE projects SET title = 'changed' WHERE id = 1"); + assert_reference_refused( + tx.commit(), + false, + "a commit with a row whose parent is missing", + ); + assert_eq!( + support::rows(&db, ALL_WORK), + [ + [Value::Int64(1), Value::Int64(1)], + [Value::Int64(2), Value::Int64(3)] + ] + ); + assert_eq!( + support::rows(&db, "SELECT id, title FROM projects ORDER BY id"), + [ + vec![Value::Int64(1), support::text("one")], + vec![Value::Int64(2), support::text("two")], + vec![Value::Int64(3), support::text("three")], + ] + ); + + // A statement on the writer is a transaction of its own, so it brings the refusal itself. + let refused = db.execute( + "INSERT INTO work (id, project_id) VALUES ($id, $project)", + ¶ms! { "id" => 9_i64, "project" => 404_i64 }, + ); + assert_eq!( + refused.as_ref().err().map(ToString::to_string).as_deref(), + Some("foreign key violation: work (project_id) = (404) has no row in projects"), + "the refusal says which value has no parent" + ); + assert_reference_refused( + refused, + false, + "an insert on the writer of a row whose parent is missing", + ); + // A reference that is NULL names nothing. + support::run(&db, "INSERT INTO work (id, project_id) VALUES (9, NULL)"); + + // A parent that another commit removed while this transaction was open is missing too. + let tx = db.transaction().expect("begin"); + support::run(&tx, "INSERT INTO work (id, project_id) VALUES (10, 2)"); + support::run(&db, "DELETE FROM projects WHERE id = 2"); + assert_reference_refused( + tx.commit(), + false, + "a commit of a row whose parent was deleted meanwhile", + ); + assert_eq!(support::int(&db, "SELECT COUNT(*) FROM work"), 3); +} + +#[test] +fn a_parent_is_not_removed_or_given_another_key_under_a_row_that_names_it() { + let db = store_of_work("parent-under-child"); + let work = support::rows(&db, ALL_WORK); + let projects = support::rows(&db, ALL_PROJECTS); + + assert_reference_refused( + db.execute("DELETE FROM projects WHERE id = 1", ¶ms! {}), + true, + "deleting a project that work names", + ); + assert_reference_refused( + db.execute("UPDATE projects SET id = 7 WHERE id = 1", ¶ms! {}), + true, + "giving a project that work names another key", + ); + // In a transaction the statement is taken and the commit refuses. + let tx = db.transaction().expect("begin"); + support::run(&tx, "DELETE FROM projects WHERE id = 1"); + support::run(&tx, "INSERT INTO projects (id, title) VALUES (5, 'five')"); + assert_reference_refused(tx.commit(), true, "a commit that deletes a named project"); + assert_eq!(support::rows(&db, ALL_WORK), work); + assert_eq!(support::rows(&db, ALL_PROJECTS), projects); + + // A project nothing named when it was deleted, and that a row committed meanwhile names. + let tx = db.transaction().expect("begin"); + support::run(&tx, "DELETE FROM projects WHERE id = 2"); + support::run(&db, "INSERT INTO work (id, project_id) VALUES (2, 2)"); + assert_reference_refused( + tx.commit(), + true, + "a commit that deletes a project a row was pointed at meanwhile", + ); + assert_eq!(support::rows(&db, ALL_PROJECTS), projects); + + // The rows that name it and the parent go together, in either order, in one transaction; + // and a parent may be moved to another key together with the rows that name it. + let tx = db.transaction().expect("begin"); + support::run(&tx, "DELETE FROM projects WHERE id = 2"); + support::run(&tx, "DELETE FROM work WHERE id = 2"); + support::run(&tx, "UPDATE projects SET id = 7 WHERE id = 1"); + support::run(&tx, "UPDATE work SET project_id = 7 WHERE id = 1"); + tx.commit() + .expect("commit a parent with the rows that name it"); + assert_eq!( + support::rows(&db, ALL_WORK), + [[Value::Int64(1), Value::Int64(7)]] + ); + assert_eq!(support::rows(&db, ALL_PROJECTS), [[Value::Int64(7)]]); +} + +#[test] +fn a_reference_refusal_names_every_column_of_the_reference() { + // The kind is the same for every shape of reference (the scripts in `tests/sql/` hold the + // shapes); the fields that say which reference was refused are checked here. + let db = support::open(&support::folder("reference-columns-named")); + support::run( + &db, + "CREATE TABLE windows (machine TEXT NOT NULL, slot INTEGER NOT NULL, \ + PRIMARY KEY (machine, slot))", + ); + support::run( + &db, + "CREATE TABLE samples (id INTEGER PRIMARY KEY, machine TEXT, slot INTEGER, \ + FOREIGN KEY (machine, slot) REFERENCES windows(machine, slot))", + ); + support::run( + &db, + "CREATE TABLE parts (id INTEGER PRIMARY KEY, parent_id INTEGER REFERENCES parts(id))", + ); + support::run( + &db, + "INSERT INTO windows (machine, slot) VALUES ('m1', 1), ('m2', 2)", + ); + support::run( + &db, + "INSERT INTO samples (id, machine, slot) VALUES (1, 'm1', 1)", + ); + for (sql, table, columns, parent, still) in [ + // Each value has a parent of its own; the pair has none. + ( + "INSERT INTO samples VALUES (4, 'm1', 2)", + "samples", + "machine, slot", + "windows", + false, + ), + ( + "DELETE FROM windows WHERE machine = 'm1'", + "samples", + "machine, slot", + "windows", + true, + ), + ( + "INSERT INTO parts VALUES (3, 404)", + "parts", + "parent_id", + "parts", + false, + ), + ] { + match support::refusal(&db, sql) { + Error::ForeignKeyViolation { + table: said_table, + columns: said_columns, + parent: said_parent, + still_referenced, + .. + } => assert_eq!( + ( + said_table.as_str(), + said_columns.as_str(), + said_parent.as_str(), + still_referenced + ), + (table, columns, parent, still), + "{sql}" + ), + other => panic!("{sql}\nwas refused as {other:?}"), + } + } +} + +const EDGES: &str = "CREATE TABLE edges (id INTEGER PRIMARY KEY, source_id UUID NOT NULL, \ + target_id UUID NOT NULL, edge_type TEXT NOT NULL) DAG('DEPENDS_ON')"; + +/// Inserts the edge `id`, from node `source` to node `target`. +fn insert_edge( + on: &impl Run, + id: i64, + source: u128, + target: u128, + edge_type: &str, +) -> contextdb::Result { + on.run( + "INSERT INTO edges (id, source_id, target_id, edge_type) VALUES ($id, $source, $target, $type)", + ¶ms! { + "id" => id, "source" => Uuid::from_u128(source), "target" => Uuid::from_u128(target), + "type" => edge_type, + }, + ) +} + +fn assert_cycle(outcome: contextdb::Result, when: &str) { + match outcome { + Err(Error::CycleDetected { table, edge_type }) => assert_eq!( + (table.as_str(), edge_type.as_str()), + ("edges", "DEPENDS_ON"), + "{when}" + ), + other => panic!("{when}: {other:?}"), + } +} + +#[test] +fn a_cycle_is_refused_by_the_statement_and_by_the_commit_that_would_close_it() { + let db = support::open(&support::folder("cycles")); + support::run(&db, EDGES); + insert_edge(&db, 1, 1, 2, "DEPENDS_ON").expect("1 -> 2"); + insert_edge(&db, 2, 2, 3, "DEPENDS_ON").expect("2 -> 3"); + let ids = "SELECT id FROM edges ORDER BY id"; + + assert_cycle( + insert_edge(&db, 3, 3, 1, "DEPENDS_ON"), + "an edge back to the start, on the writer", + ); + // Inside a transaction the statement that would close the cycle is refused, over the edges + // the transaction itself has staged, and the transaction goes on. + let tx = db.transaction().expect("begin"); + insert_edge(&tx, 3, 3, 4, "DEPENDS_ON").expect("3 -> 4"); + assert_cycle( + insert_edge(&tx, 4, 4, 1, "DEPENDS_ON"), + "an edge that closes a cycle over a staged edge", + ); + insert_edge(&tx, 4, 4, 5, "DEPENDS_ON").expect("4 -> 5"); + tx.commit().expect("commit the edges that close no cycle"); + assert_eq!( + support::rows(&db, ids), + [1_i64, 2, 3, 4].map(|id| [Value::Int64(id)]) + ); + + // Two transactions that each add one edge, neither of which closes a cycle where it was + // written: the second to commit would close one, and is refused whole. + let tx = db.transaction().expect("begin"); + insert_edge(&tx, 5, 5, 6, "DEPENDS_ON").expect("5 -> 6, staged"); + insert_edge(&tx, 6, 8, 9, "DEPENDS_ON").expect("8 -> 9, staged"); + insert_edge(&db, 7, 6, 1, "DEPENDS_ON").expect("6 -> 1, committed meanwhile"); + assert_cycle( + tx.commit(), + "a commit whose edge closes a cycle with a newer commit", + ); + assert_eq!( + support::rows(&db, ids), + [1_i64, 2, 3, 4, 7].map(|id| [Value::Int64(id)]) + ); + // The same edges, written now, are refused by the statement. + assert_cycle( + insert_edge(&db, 5, 5, 6, "DEPENDS_ON"), + "the same edge on the writer", + ); +} + +fn insert_fan(db: &Database, hub: u128, first_leaf: u128, first_id: i64, count: usize) { + let numbers: Vec = (0..count).collect(); + for batch in numbers.chunks(1000) { + let rows: Vec = batch + .iter() + .map(|n| { + format!( + "({}, '{}', '{}', 'DEPENDS_ON')", + first_id + *n as i64, + Uuid::from_u128(hub), + Uuid::from_u128(first_leaf + *n as u128) + ) + }) + .collect(); + support::run( + db, + &format!( + "INSERT INTO edges (id, source_id, target_id, edge_type) VALUES {}", + rows.join(", ") + ), + ); + } +} + +#[test] +fn a_cycle_through_a_node_with_1_500_edges_is_refused_and_an_edge_that_closes_none_is_accepted() { + // Node 2 has 1,500 edges out, to the nodes 1000..2500, and node 1 has one edge, to node 2. + const FAN: usize = 1_500; + const LAST_LEAF: u128 = 1000 + FAN as u128 - 1; + let db = support::open(&support::folder("cycle-through-wide-node")); + support::run(&db, EDGES); + insert_fan(&db, 2, 1000, 1000, FAN); + insert_edge(&db, 1, 1, 2, "DEPENDS_ON").expect("1 -> 2"); + let count = || support::int(&db, "SELECT COUNT(*) FROM edges"); + assert_eq!(count(), FAN as i64 + 1); + + // The way round goes through the wide node and out by the last of its edges. + assert_cycle( + insert_edge(&db, 5000, LAST_LEAF, 1, "DEPENDS_ON"), + "an edge from the last node the wide node reaches, back to the node before it", + ); + assert_cycle( + insert_edge(&db, 5000, LAST_LEAF, 2, "DEPENDS_ON"), + "an edge from the last node the wide node reaches, back to the wide node", + ); + assert_eq!(count(), FAN as i64 + 1); + + // An edge on from that node, and one more edge into the wide node, close no cycle. + insert_edge(&db, 5000, LAST_LEAF, 9000, "DEPENDS_ON").expect("on from the last node"); + insert_edge(&db, 5001, 9001, 2, "DEPENDS_ON").expect("one more edge into the wide node"); + insert_edge(&db, 5002, 9000, 9001 + 1, "DEPENDS_ON").expect("on again"); + assert_eq!(count(), FAN as i64 + 4); + assert_eq!( + support::rows(&db, "SELECT id FROM edges WHERE id >= 5000 ORDER BY id"), + [5000_i64, 5001, 5002].map(|id| [Value::Int64(id)]) + ); +} + +#[test] +fn a_cycle_check_too_large_for_the_memory_limit_is_refused_by_the_limit_and_writes_nothing() { + // Node 2 reaches 20,000 nodes. An edge into node 2 is checked by going over all of them, + // and what that holds does not fit in the room the limit leaves. + const FAN: usize = 20_000; + const ROOM: usize = 64 * 1024; + let db = support::open(&support::folder("cycle-check-memory-limit")); + support::run(&db, EDGES); + insert_fan(&db, 2, 1000, 1000, FAN); + let count = || support::int(&db, "SELECT COUNT(*) FROM edges"); + assert_eq!(count(), FAN as i64); + + db.set_memory_limit(Some(db.memory_used() + ROOM)) + .expect("set the limit"); + // The limit has room for an edge whose check goes nowhere. + insert_edge(&db, 1, 500, 501, "DEPENDS_ON").expect("an edge between two new nodes"); + match insert_edge(&db, 2, 1, 2, "DEPENDS_ON") { + Err(Error::MemoryBudgetExceeded { .. }) => {} + other => panic!( + "an edge whose check goes over {FAN} nodes with {ROOM} bytes of room gave {other:?}" + ), + } + assert_eq!(count(), FAN as i64 + 1); + assert!(support::rows(&db, "SELECT id FROM edges WHERE id = 2").is_empty()); + + // It was the limit that refused it: the edge closes no cycle, and is written without one. + db.set_memory_limit(None).expect("lift the limit"); + insert_edge(&db, 2, 1, 2, "DEPENDS_ON").expect("the same edge with no limit"); + assert_eq!(count(), FAN as i64 + 2); +} + +#[test] +fn a_table_another_table_references_is_not_dropped_until_the_reference_is_gone() { + let db = store_of_work("drop-referenced-parent"); + let declared = db + .declaration("projects") + .expect("the declaration of projects"); + let projects = support::rows(&db, "SELECT id FROM projects ORDER BY id"); + let work = support::int(&db, "SELECT COUNT(*) FROM work"); + + for sql in ["DROP TABLE projects", "DROP TABLE IF EXISTS projects"] { + match support::refusal(&db, sql) { + Error::SchemaInvalid { .. } => {} + other => panic!("{sql}\nwhile work references projects was refused as {other:?}"), + } + } + assert_eq!( + db.declaration("projects") + .expect("the declaration of projects"), + declared + ); + assert_eq!( + support::rows(&db, "SELECT id FROM projects ORDER BY id"), + projects + ); + assert_eq!(support::int(&db, "SELECT COUNT(*) FROM work"), work); + // The reference still stands over the table that was not dropped. + assert!(matches!( + support::refusal(&db, "INSERT INTO work (id, project_id) VALUES (90, 404)"), + Error::ForeignKeyViolation { .. } + )); + + // With the referencing table gone, the table goes. + support::run(&db, "DROP TABLE work"); + support::run(&db, "DROP TABLE projects"); + assert!(matches!( + support::refusal(&db, "SELECT id FROM projects"), + Error::TableNotFound(table) if table == "projects" + )); +} diff --git a/v2/contextdb/tests/sql/alter_table.sql b/v2/contextdb/tests/sql/alter_table.sql new file mode 100644 index 00000000..3543d13e --- /dev/null +++ b/v2/contextdb/tests/sql/alter_table.sql @@ -0,0 +1,466 @@ +# ALTER TABLE: a column added, dropped and renamed on a table that has rows, the options of a +# table changed after it was declared, and the two clauses of a VECTOR column that may change. +# A refused ALTER changes nothing. What is left in the store after a drop, and how an altered +# table is printed back, is checked by `tests/alter.rs`. + +CREATE TABLE t (id INTEGER PRIMARY KEY, a TEXT, b INTEGER); +INSERT INTO t (id, a, b) VALUES (1, 'one', 10), (2, 'two', 20); + +# A column is added with or without the word COLUMN. A row written before reads NULL in it, or +# the column's default when it declares one, and every other column is as it was. +ALTER TABLE t ADD COLUMN c TEXT; +ALTER TABLE t ADD d INTEGER DEFAULT 7; +ALTER TABLE t ADD COLUMN e TEXT NOT NULL DEFAULT 'none'; +SELECT id, a, b, c, d, e FROM t ORDER BY id; +---- +id | a | b | c | d | e +1 | one | 10 | NULL | 7 | none +2 | two | 20 | NULL | 7 | none + +# A row written after is written like any row, and a filter reads an older row's default. +INSERT INTO t (id, a) VALUES (3, 'three'); +UPDATE t SET c = 'set' WHERE id = 1; +SELECT id, c, d, e FROM t ORDER BY id; +---- +id | c | d | e +1 | set | 7 | none +2 | NULL | 7 | none +3 | NULL | 7 | none + +SELECT id FROM t WHERE d = 7 AND c IS NULL ORDER BY id; +---- +id +2 +3 + +# A column that may hold no NULL needs a default when the table has rows. +ALTER TABLE t ADD COLUMN f TEXT NOT NULL; +---- +error: SchemaInvalid + +# A name the table has, a second key, and a table that is not there. +ALTER TABLE t ADD COLUMN a TEXT; +---- +error: SchemaInvalid + +ALTER TABLE t ADD COLUMN k INTEGER PRIMARY KEY; +---- +error: SchemaInvalid + +ALTER TABLE nope ADD COLUMN a TEXT; +---- +error: TableNotFound + +SELECT f FROM t; +---- +error: ColumnNotFound + +# An added column may carry a rule, and the rule holds from then on. +ALTER TABLE t ADD COLUMN slug TEXT UNIQUE; +UPDATE t SET slug = 'first' WHERE id = 1; +UPDATE t SET slug = 'first' WHERE id = 2; +---- +error: UniqueViolation + +# A column is dropped with or without the word COLUMN, and the others keep their values. +ALTER TABLE t DROP b; +ALTER TABLE t DROP COLUMN d RESTRICT; +SELECT b FROM t; +---- +error: ColumnNotFound + +SELECT id, a, c, e FROM t ORDER BY id; +---- +id | a | c | e +1 | one | set | none +2 | two | NULL | none +3 | three | NULL | none + +# A column an index names is not dropped unless the statement says CASCADE; RESTRICT is the +# default. CASCADE drops the index with the column. +CREATE INDEX t_by_c ON t (c, e); +ALTER TABLE t DROP COLUMN c; +---- +error: ColumnInIndex + +ALTER TABLE t DROP COLUMN c RESTRICT; +---- +error: ColumnInIndex + +SELECT id, c FROM t WHERE c = 'set'; +---- +id | c +1 | set + +CREATE INDEX t_by_c_late ON t (a, c); +ALTER TABLE t DROP COLUMN c CASCADE; +---- +dropped_indexes +t_by_c +t_by_c_late + +DROP INDEX t_by_c ON t; +---- +error: IndexNotFound + +ALTER TABLE t ADD COLUMN spare TEXT; +ALTER TABLE t DROP COLUMN spare CASCADE; +---- +rows affected: 0 + +SELECT c FROM t; +---- +error: ColumnNotFound + +# The name is free again, and no row reads what the dropped column held. +ALTER TABLE t ADD COLUMN c TEXT; +SELECT id, c FROM t ORDER BY id; +---- +id | c +1 | NULL +2 | NULL +3 | NULL + +# A column is renamed, and an index over it follows the name. +CREATE INDEX t_by_a ON t (a); +ALTER TABLE t RENAME COLUMN a TO title; +SELECT a FROM t; +---- +error: ColumnNotFound + +SELECT id, title FROM t WHERE title = 'two'; +---- +id | title +2 | two + +ALTER TABLE t DROP COLUMN title; +---- +error: ColumnInIndex + +ALTER TABLE t RENAME COLUMN title TO e; +---- +error: SchemaInvalid + +ALTER TABLE t RENAME COLUMN nope TO other; +---- +error: ColumnNotFound + +# A column of the primary key is neither dropped nor renamed. +ALTER TABLE t DROP COLUMN id; +---- +error: SchemaInvalid + +ALTER TABLE t DROP COLUMN id CASCADE; +---- +error: SchemaInvalid + +ALTER TABLE t RENAME COLUMN id TO key; +---- +error: SchemaInvalid + +SELECT id, title, c, e, slug FROM t ORDER BY id; +---- +id | title | c | e | slug +1 | one | NULL | none | first +2 | two | NULL | none | NULL +3 | three | NULL | none | NULL + +# An IMMUTABLE column is neither dropped nor renamed; the column beside it is. +CREATE TABLE frozen (id INTEGER PRIMARY KEY, kind TEXT NOT NULL IMMUTABLE, note TEXT); +ALTER TABLE frozen DROP COLUMN kind; +---- +error: ImmutableColumn + +ALTER TABLE frozen DROP COLUMN kind CASCADE; +---- +error: ImmutableColumn + +ALTER TABLE frozen RENAME COLUMN kind TO sort; +---- +error: ImmutableColumn + +ALTER TABLE frozen DROP COLUMN note; + +# A column of a VECTOR column's PARTITION_KEY is neither dropped nor renamed, with CASCADE or +# without, and no ALTER changes a partition key in place. +CREATE TABLE docs ( + id INTEGER PRIMARY KEY, + tenant TEXT NOT NULL, + v VECTOR(3) PARTITION_KEY (tenant) MAX_PARTITIONS 1 SEARCH_MODE INDEXED, + loose VECTOR(3) +); +ALTER TABLE docs DROP COLUMN tenant; +---- +error: VectorPartitionKeyInUse + +ALTER TABLE docs DROP COLUMN tenant CASCADE; +---- +error: VectorPartitionKeyInUse + +ALTER TABLE docs RENAME COLUMN tenant TO owner; +---- +error: VectorPartitionKeyInUse + +ALTER TABLE docs ALTER COLUMN v SET PARTITION_KEY (id); +---- +error: ParseError + +ALTER TABLE docs ALTER COLUMN loose SET PARTITION_KEY (tenant); +---- +error: ParseError + +# The limit on partitions is raised, and the next partition is then accepted. +INSERT INTO docs (id, tenant, v) VALUES (1, 'north', [1.0, 0.0, 0.0]); +INSERT INTO docs (id, tenant, v) VALUES (2, 'south', [0.0, 1.0, 0.0]); +---- +error: VectorPartitionLimitExceeded + +ALTER TABLE docs ALTER COLUMN v SET MAX_PARTITIONS 2; +INSERT INTO docs (id, tenant, v) VALUES (2, 'south', [0.0, 1.0, 0.0]); +ALTER TABLE docs ALTER COLUMN v SET MAX_PARTITIONS 0; +---- +error: InvalidVectorPartitionDeclaration + +ALTER TABLE docs ALTER COLUMN v SET MAX_PARTITIONS -1; +---- +error: InvalidVectorPartitionDeclaration + +ALTER TABLE docs ALTER COLUMN loose SET MAX_PARTITIONS 4; +---- +error: InvalidVectorPartitionDeclaration + +# The column's search mode is changed, and the next search runs in it. A column too small to +# have a grouping has nothing for INDEXED to choose among, and INDEXED searches it as EXACT does. +SELECT id FROM docs ORDER BY v <=> [1.0, 0.0, 0.0] LIMIT 1; +---- +id +1 + +ALTER TABLE docs ALTER COLUMN v SET SEARCH_MODE EXACT; +SELECT id FROM docs ORDER BY v <=> [1.0, 0.0, 0.0] LIMIT 1; +---- +id +1 + +ALTER TABLE docs ALTER COLUMN v SET SEARCH_MODE INDEXED; +SELECT id FROM docs ORDER BY v <=> [1.0, 0.0, 0.0] LIMIT 1; +---- +id +1 + +ALTER TABLE docs ALTER COLUMN v SET SEARCH_MODE AUTO; +SELECT id FROM docs ORDER BY v <=> [0.0, 1.0, 0.0] LIMIT 1; +---- +id +2 + +ALTER TABLE docs ALTER COLUMN nope SET SEARCH_MODE EXACT; +---- +error: ColumnNotFound + +# The thresholds of the search index are changed online, and the next search and upkeep cycle +# read them: the declaration says what was written and what is in force, and DEFAULT takes the +# declaration away. +ALTER TABLE docs ALTER COLUMN v SET AUTO_INDEX_AT 5000; +ALTER TABLE docs ALTER COLUMN v SET CONSOLIDATION (CHANGE_PERCENT = 30, TOMBSTONE_PERCENT = 5); +SHOW VECTOR_INDEXES; +---- +table | column | dimension | quantization | vector_count | bytes | partition_key_columns | max_partitions | live_partitions | search_mode | declared_auto_index_at | effective_auto_index_at | declared_consolidation_mode | declared_consolidation_change_percent | declared_consolidation_tombstone_percent | effective_consolidation_mode | effective_consolidation_change_percent | effective_consolidation_tombstone_percent | pending_inserts | tombstones | durable_vector_bytes | durable_index_bytes | indexed_available | maintenance_state | maintenance_vectors_total | maintenance_vectors_done | maintenance_vectors_remaining | unavailable_partitions | stalled_partitions | broad_route | recovery_action +docs | loose | * | * | * | * | * | * | * | * | NULL | 100000 | NULL | NULL | NULL | PERCENT | 20 | 10 | * | * | * | * | * | * | * | * | * | * | * | * | * +docs | v | * | * | * | * | * | * | * | * | 5000 | 5000 | PERCENT | 30 | 5 | PERCENT | 30 | 5 | * | * | * | * | * | * | * | * | * | * | * | * | * + +ALTER TABLE docs ALTER COLUMN v SET AUTO_INDEX_AT DEFAULT; +ALTER TABLE docs ALTER COLUMN v SET CONSOLIDATION NONE; +SHOW VECTOR_INDEXES; +---- +table | column | dimension | quantization | vector_count | bytes | partition_key_columns | max_partitions | live_partitions | search_mode | declared_auto_index_at | effective_auto_index_at | declared_consolidation_mode | declared_consolidation_change_percent | declared_consolidation_tombstone_percent | effective_consolidation_mode | effective_consolidation_change_percent | effective_consolidation_tombstone_percent | pending_inserts | tombstones | durable_vector_bytes | durable_index_bytes | indexed_available | maintenance_state | maintenance_vectors_total | maintenance_vectors_done | maintenance_vectors_remaining | unavailable_partitions | stalled_partitions | broad_route | recovery_action +docs | loose | * | * | * | * | * | * | * | * | NULL | 100000 | NULL | NULL | NULL | PERCENT | 20 | 10 | * | * | * | * | * | * | * | * | * | * | * | * | * +docs | v | * | * | * | * | * | * | * | * | NULL | 100000 | NONE | NULL | NULL | NONE | NULL | NULL | * | * | * | * | * | * | * | * | * | * | * | * | * + +ALTER TABLE docs ALTER COLUMN v SET CONSOLIDATION DEFAULT; +SHOW VECTOR_INDEXES; +---- +table | column | dimension | quantization | vector_count | bytes | partition_key_columns | max_partitions | live_partitions | search_mode | declared_auto_index_at | effective_auto_index_at | declared_consolidation_mode | declared_consolidation_change_percent | declared_consolidation_tombstone_percent | effective_consolidation_mode | effective_consolidation_change_percent | effective_consolidation_tombstone_percent | pending_inserts | tombstones | durable_vector_bytes | durable_index_bytes | indexed_available | maintenance_state | maintenance_vectors_total | maintenance_vectors_done | maintenance_vectors_remaining | unavailable_partitions | stalled_partitions | broad_route | recovery_action +docs | loose | * | * | * | * | * | * | * | * | NULL | 100000 | NULL | NULL | NULL | PERCENT | 20 | 10 | * | * | * | * | * | * | * | * | * | * | * | * | * +docs | v | * | * | * | * | * | * | * | * | NULL | 100000 | NULL | NULL | NULL | PERCENT | 20 | 10 | * | * | * | * | * | * | * | * | * | * | * | * | * + +# A value CREATE TABLE refuses is refused here, and the declaration stays as it was. +ALTER TABLE docs ALTER COLUMN v SET AUTO_INDEX_AT 0; +---- +error: InvalidVectorPolicyDeclaration + +ALTER TABLE docs ALTER COLUMN v SET CONSOLIDATION (CHANGE_PERCENT = 101); +---- +error: InvalidVectorPolicyDeclaration + +# SEARCH and IVF tune what the lean index measures for itself: no version reads them. +ALTER TABLE docs ALTER COLUMN v SET SEARCH DEFAULT; +---- +error: PlanError + +ALTER TABLE docs ALTER COLUMN v SET IVF (LISTS = 64, PROBES = 8); +---- +error: PlanError + +# A VECTOR column is added with its whole declaration. A row written before has no vector and +# is in no search until it is given one. +CREATE TABLE passages (id INTEGER PRIMARY KEY, tenant TEXT NOT NULL, body TEXT); +INSERT INTO passages (id, tenant, body) VALUES (1, 'north', 'old'); +ALTER TABLE passages ADD COLUMN v VECTOR(2) PARTITION_KEY (tenant) MAX_PARTITIONS 2 SEARCH_MODE EXACT; +SELECT id, v FROM passages ORDER BY id; +---- +id | v +1 | NULL + +SELECT id FROM passages ORDER BY v <=> [1.0, 0.0] LIMIT 5; +---- +id + +INSERT INTO passages (id, tenant, body, v) VALUES (2, 'north', 'new', [1.0, 0.0]); +SELECT id FROM passages ORDER BY v <=> [1.0, 0.0] LIMIT 5; +---- +id +2 + +UPDATE passages SET v = [0.0, 1.0] WHERE id = 1; +SELECT id FROM passages ORDER BY v <=> [0.0, 1.0] LIMIT 5; +---- +id +1 +2 + +INSERT INTO passages (id, tenant, body, v) VALUES (3, 'south', 'new', [1.0, 0.0]); +INSERT INTO passages (id, tenant, body, v) VALUES (4, 'east', 'new', [1.0, 0.0]); +---- +error: VectorPartitionLimitExceeded + +# An added VECTOR column is refused what a declared one is. +ALTER TABLE passages ADD COLUMN w VECTOR(2) PARTITION_KEY (body); +---- +error: InvalidVectorPartitionDeclaration + +ALTER TABLE passages ADD COLUMN w VECTOR(0); +---- +error: InvalidVectorPolicyDeclaration + +SELECT w FROM passages; +---- +error: ColumnNotFound + +# How long rows are kept is set and dropped. SYNC SAFE needs a direction that delivers, when +# either of the two is set. +CREATE TABLE scratch (id INTEGER PRIMARY KEY, body TEXT); +INSERT INTO scratch (id, body) VALUES (1, 'kept'); +ALTER TABLE scratch SET RETAIN 7 DAYS; +ALTER TABLE scratch SET RETAIN 24 HOURS SYNC SAFE; +ALTER TABLE scratch SET SYNC PULL ONLY; +---- +error: SchemaInvalid + +ALTER TABLE scratch SET SYNC OFF; +---- +error: SchemaInvalid + +ALTER TABLE scratch SET SYNC PUSH ONLY; +ALTER TABLE scratch DROP RETAIN; +ALTER TABLE scratch SET SYNC PULL ONLY; +ALTER TABLE scratch SET RETAIN 1 DAYS SYNC SAFE; +---- +error: SchemaInvalid + +ALTER TABLE scratch SET RETAIN 1 DAYS; +ALTER TABLE scratch SET SYNC OFF; +ALTER TABLE scratch SET SYNC TWO WAY; +ALTER TABLE scratch SET RETAIN 24 HOURS SYNC SAFE PUSH ONLY; +---- +error: ParseError + +# Which write wins, and how many versions are kept. Keeping only the current version is +# refused, by every statement that could bring it about, on a table that sends its rows out +# and keeps the first write. +ALTER TABLE scratch SET HISTORY CURRENT ONLY; +---- +error: SchemaInvalid + +ALTER TABLE scratch SET SYNC CONFLICT KEEP LATEST; +ALTER TABLE scratch SET HISTORY CURRENT ONLY; +ALTER TABLE scratch SET SYNC CONFLICT KEEP FIRST; +---- +error: SchemaInvalid + +ALTER TABLE scratch SET HISTORY ALL; +ALTER TABLE scratch SET SYNC CONFLICT KEEP FIRST; +ALTER TABLE scratch SET SYNC OFF; +ALTER TABLE scratch SET HISTORY CURRENT ONLY; +ALTER TABLE scratch SET SYNC TWO WAY; +---- +error: SchemaInvalid + +ALTER TABLE scratch SET SYNC PUSH ONLY; +---- +error: SchemaInvalid + +SELECT id, body FROM scratch; +---- +id | body +1 | kept + +# An IMMUTABLE table takes neither RETAIN nor HISTORY CURRENT ONLY later. +CREATE TABLE ledger (id INTEGER PRIMARY KEY, body TEXT) IMMUTABLE SYNC OFF; +ALTER TABLE ledger SET RETAIN 7 DAYS; +---- +error: SchemaInvalid + +ALTER TABLE ledger SET HISTORY CURRENT ONLY; +---- +error: SchemaInvalid + +ALTER TABLE ledger SET HISTORY ALL; + +# The counts a table keeps are replaced whole by one statement, on a table that has rows. +CREATE TABLE events ( + id INTEGER PRIMARY KEY, + context_id INTEGER NOT NULL, + connector_id TEXT NOT NULL, + UNIQUE (context_id, id), + UNIQUE (context_id, connector_id, id) +); +INSERT INTO events (id, context_id, connector_id) VALUES (1, 1, 'a'), (2, 1, 'a'), (3, 1, 'b'), (4, 2, 'a'); +ALTER TABLE events SET MAINTAIN COUNT (context_id); +SELECT COUNT(*) AS n FROM events WHERE context_id = 1; +---- +n +3 + +ALTER TABLE events SET MAINTAIN COUNT (context_id), (context_id, connector_id); +SELECT COUNT(*) AS n FROM events WHERE context_id = 1 AND connector_id = 'a'; +---- +n +2 + +ALTER TABLE events SET MAINTAIN COUNT ALL PREFIXES; +ALTER TABLE events SET MAINTAIN COUNT NONE; +ALTER TABLE events SET MAINTAIN COUNT DEFAULT; +SELECT COUNT(*) AS n FROM events WHERE context_id = 1; +---- +n +3 + +ALTER TABLE events SET MAINTAIN COUNT (connector_id); +---- +error: SchemaInvalid + +ALTER TABLE events SET MAINTAIN COUNT (id); +---- +error: SchemaInvalid + +# A column named like a clause is still a column: it is added, renamed and dropped by its name. +CREATE TABLE words (id INTEGER PRIMARY KEY, state TEXT, retain TEXT); +ALTER TABLE words ADD COLUMN history TEXT; +ALTER TABLE words RENAME COLUMN state TO sync; +ALTER TABLE words DROP COLUMN retain; +INSERT INTO words (id, sync, history) VALUES (1, 'a', 'b'); +SELECT id, sync, history FROM words; +---- +id | sync | history +1 | a | b diff --git a/v2/contextdb/tests/sql/compare_types.sql b/v2/contextdb/tests/sql/compare_types.sql new file mode 100644 index 00000000..14943766 --- /dev/null +++ b/v2/contextdb/tests/sql/compare_types.sql @@ -0,0 +1,158 @@ +# A comparison between a column and a value of another type is refused when the statement is +# read, whatever the table holds: on an empty table, in a SELECT, an UPDATE and a DELETE. It is +# never answered by leaving rows out. The conversions a write has still apply: quoted text is +# read as a UUID or a TIMESTAMP, and an INTEGER as a REAL. + +CREATE TABLE things ( + id INTEGER PRIMARY KEY, + n INTEGER, + ratio REAL, + label TEXT, + flag BOOLEAN, + ref UUID, + at TIMESTAMP +); + +# The table is empty, and each of these is refused all the same. +SELECT id FROM things WHERE n = 'seven'; +---- +error: PlanError + +SELECT id FROM things WHERE label = 7; +---- +error: PlanError + +SELECT id FROM things WHERE label < 7; +---- +error: PlanError + +SELECT id FROM things WHERE flag = 1; +---- +error: PlanError + +SELECT id FROM things WHERE n = TRUE; +---- +error: PlanError + +SELECT id FROM things WHERE ref = 7; +---- +error: PlanError + +SELECT id FROM things WHERE at = 7; +---- +error: PlanError + +SELECT id FROM things WHERE id = 'one'; +---- +error: PlanError + +SELECT id FROM things WHERE n IN (1, 'two'); +---- +error: PlanError + +SELECT id FROM things WHERE n = label; +---- +error: PlanError + +SELECT id FROM things WHERE label = 'a' AND n > 'b'; +---- +error: PlanError + +UPDATE things SET n = 1 WHERE label = 7; +---- +error: PlanError + +UPDATE things SET label = 'x' WHERE n >= 'seven'; +---- +error: PlanError + +DELETE FROM things WHERE n = 'seven'; +---- +error: PlanError + +DELETE FROM things WHERE flag = 'yes'; +---- +error: PlanError + +# A table that is not there is said first. +SELECT id FROM nowhere WHERE id = 'one'; +---- +error: TableNotFound + +# What does compare, on the empty table: an answer of no rows and no refusal. +SELECT id FROM things WHERE n = 7 AND ratio > 1 AND label = 'seven' AND flag = TRUE; +---- +id + +SELECT id FROM things WHERE n = NULL; +---- +id + +INSERT INTO things (id, n, ratio, label, flag, ref, at) +VALUES (1, 7, 1.5, 'seven', TRUE, '550e8400-e29b-41d4-a716-446655440001', '2026-01-01T00:00:00Z'), + (2, 8, 2.0, 'eight', FALSE, '550e8400-e29b-41d4-a716-446655440002', '2026-01-02T00:00:00Z'), + (3, NULL, NULL, NULL, NULL, NULL, NULL); + +# With rows in the table the refusals are the same, and no row is changed or removed. +SELECT id FROM things WHERE n = 'seven'; +---- +error: PlanError + +SELECT id FROM things WHERE label = 7; +---- +error: PlanError + +UPDATE things SET n = 0 WHERE label = 7; +---- +error: PlanError + +DELETE FROM things WHERE n = 'seven'; +---- +error: PlanError + +# Quoted text is read as a UUID, in either case, and as a TIMESTAMP. +SELECT id FROM things WHERE ref = '550E8400-E29B-41D4-A716-446655440002'; +---- +id +2 + +SELECT id FROM things WHERE at = '2026-01-01T00:00:00Z'; +---- +id +1 + +SELECT id FROM things WHERE at > '2026-01-01T12:00:00Z' ORDER BY id; +---- +id +2 + +# An INTEGER is read as a REAL. +SELECT id FROM things WHERE ratio = 2; +---- +id +2 + +SELECT id FROM things WHERE ratio > 1 ORDER BY id; +---- +id +1 +2 + +# The same conversions in an UPDATE and a DELETE. +UPDATE things SET label = 'two' WHERE ratio = 2; +---- +rows affected: 1 + +UPDATE things SET n = 9 WHERE ref = '550e8400-e29b-41d4-a716-446655440001'; +---- +rows affected: 1 + +DELETE FROM things WHERE at = '2026-01-02T00:00:00Z'; +---- +rows affected: 1 + +SELECT id, n, label FROM things ORDER BY id; +---- +id | n | label +1 | 9 | seven +3 | NULL | NULL diff --git a/v2/contextdb/tests/sql/composite_keys.sql b/v2/contextdb/tests/sql/composite_keys.sql new file mode 100644 index 00000000..4b32e915 --- /dev/null +++ b/v2/contextdb/tests/sql/composite_keys.sql @@ -0,0 +1,233 @@ +# A primary key of several columns: the whole tuple names the row. And the other things a key +# may be declared with: a key written as an element of the table over one column, the types a +# partition key takes, the TXID type, and the index names the engine keeps for itself. + +CREATE TABLE metric_windows ( + machine_id TEXT NOT NULL, + sensor_id TEXT NOT NULL, + metric TEXT NOT NULL, + window_start INTEGER NOT NULL, + value INTEGER NOT NULL, + PRIMARY KEY (machine_id, sensor_id, metric, window_start) +); +INSERT INTO metric_windows (machine_id, sensor_id, metric, window_start, value) VALUES + ('m1', 's1', 'temp', 10, 1), + ('m1', 's1', 'temp', 20, 2), + ('m1', 's2', 'temp', 10, 3), + ('m2', 's1', 'temp', 10, 4); + +# The same tuple again is the same row and is refused; a tuple that differs in one column, +# the last one too, is another row. +INSERT INTO metric_windows (machine_id, sensor_id, metric, window_start, value) VALUES ('m1', 's1', 'temp', 10, 9); +---- +error: UniqueViolation + +INSERT INTO metric_windows (machine_id, sensor_id, metric, window_start, value) VALUES ('m1', 's1', 'temp', 30, 5); +INSERT INTO metric_windows (machine_id, sensor_id, metric, window_start, value) VALUES ('m1', 's1', 'load', 10, 6); +SELECT COUNT(*) AS n FROM metric_windows; +---- +n +6 + +# Every column of the key is a column of every row. +INSERT INTO metric_windows (machine_id, sensor_id, metric, value) VALUES ('m1', 's1', 'temp', 7); +---- +error: ColumnNotNullable + +# The whole key names one row; its leading columns name the rows that share them, in key order. +SELECT value FROM metric_windows WHERE machine_id = 'm1' AND sensor_id = 's1' AND metric = 'temp' AND window_start = 20; +---- +value +2 + +SELECT sensor_id, metric, window_start, value FROM metric_windows WHERE machine_id = 'm1' ORDER BY sensor_id, metric, window_start; +---- +sensor_id | metric | window_start | value +s1 | load | 10 | 6 +s1 | temp | 10 | 1 +s1 | temp | 20 | 2 +s1 | temp | 30 | 5 +s2 | temp | 10 | 3 + +SELECT machine_id, value FROM metric_windows WHERE sensor_id = 's1' AND window_start = 10 ORDER BY machine_id, value; +---- +machine_id | value +m1 | 1 +m1 | 6 +m2 | 4 + +# An UPDATE and a DELETE reach a row by its whole key, and change that row alone. +UPDATE metric_windows SET value = 20 WHERE machine_id = 'm1' AND sensor_id = 's1' AND metric = 'temp' AND window_start = 20; +---- +rows affected: 1 + +DELETE FROM metric_windows WHERE machine_id = 'm1' AND sensor_id = 's1' AND metric = 'load' AND window_start = 10; +---- +rows affected: 1 + +# A column of the key may be assigned: the row moves to the new tuple, unless a row is there. +UPDATE metric_windows SET window_start = 20 WHERE machine_id = 'm1' AND sensor_id = 's1' AND metric = 'temp' AND window_start = 10; +---- +error: UniqueViolation + +UPDATE metric_windows SET window_start = 40 WHERE machine_id = 'm1' AND sensor_id = 's1' AND metric = 'temp' AND window_start = 10; +---- +rows affected: 1 + +SELECT machine_id, sensor_id, metric, window_start, value FROM metric_windows ORDER BY machine_id, sensor_id, metric, window_start; +---- +machine_id | sensor_id | metric | window_start | value +m1 | s1 | temp | 20 | 20 +m1 | s1 | temp | 30 | 5 +m1 | s1 | temp | 40 | 1 +m1 | s2 | temp | 10 | 3 +m2 | s1 | temp | 10 | 4 + +# Rows of one stream after a position, in order, and never a row of the stream beside it. +CREATE TABLE stream_events (stream_id TEXT NOT NULL, ordinal INTEGER NOT NULL, body TEXT, PRIMARY KEY (stream_id, ordinal)); +INSERT INTO stream_events (stream_id, ordinal, body) VALUES + ('a', 1, 'a1'), ('a', 2, 'a2'), ('a', 3, 'a3'), ('a', 4, 'a4'), + ('b', 1, 'b1'), ('b', 2, 'b2'), ('b', 3, 'b3'), ('b', 4, 'b4'), ('b', 5, 'b5'), + ('c', 3, 'c3'), ('c', 4, 'c4'); +SELECT ordinal, body FROM stream_events WHERE stream_id = 'b' AND ordinal > 2 ORDER BY ordinal ASC LIMIT 2; +---- +ordinal | body +3 | b3 +4 | b4 + +SELECT ordinal, body FROM stream_events WHERE stream_id = 'b' AND ordinal > 4 ORDER BY ordinal ASC LIMIT 2; +---- +ordinal | body +5 | b5 + +SELECT ordinal, body FROM stream_events WHERE stream_id = 'b' AND ordinal > 5 ORDER BY ordinal ASC LIMIT 2; +---- +ordinal | body + +# A write that meets its own key changes that row and adds none. +INSERT INTO stream_events (stream_id, ordinal, body) VALUES ('b', 5, 'again') +ON CONFLICT (stream_id, ordinal) DO UPDATE SET body = 'b5 again'; +SELECT COUNT(*) AS n FROM stream_events WHERE stream_id = 'b'; +---- +n +5 + +SELECT body FROM stream_events WHERE stream_id = 'b' AND ordinal = 5; +---- +body +b5 again + +# A key written as an element of the table may name one column. +CREATE TABLE one_key (a INTEGER, b TEXT, PRIMARY KEY (a)); +INSERT INTO one_key (a, b) VALUES (1, 'x'); +INSERT INTO one_key (a, b) VALUES (1, 'y'); +---- +error: UniqueViolation + +SELECT a, b FROM one_key WHERE a = 1; +---- +a | b +1 | x + +# A key names columns the table has, each once, of a type whose values are equal or not and +# nothing between. +CREATE TABLE refused (a INTEGER, b TEXT, PRIMARY KEY (a, nope)); +---- +error: ParseError + +CREATE TABLE refused (a INTEGER, b TEXT, PRIMARY KEY (a, a)); +---- +error: ParseError + +CREATE TABLE refused (a INTEGER, r REAL, PRIMARY KEY (a, r)); +---- +error: ColumnNotIndexable + +CREATE TABLE refused (a INTEGER, doc JSON, PRIMARY KEY (a, doc)); +---- +error: ColumnNotIndexable + +CREATE TABLE refused (a INTEGER, v VECTOR(2), PRIMARY KEY (a, v)); +---- +error: ColumnNotIndexable + +SELECT a FROM refused; +---- +error: TableNotFound + +# A table with a key of several columns takes UNIQUE rules, indexes and references like any +# other, and a reference may name the whole key of its parent. +CREATE TABLE readings ( + id INTEGER PRIMARY KEY, + machine_id TEXT, + sensor_id TEXT, + metric TEXT, + window_start INTEGER, + note TEXT, + UNIQUE (machine_id, note), + FOREIGN KEY (machine_id, sensor_id, metric, window_start) + REFERENCES metric_windows(machine_id, sensor_id, metric, window_start) +); +CREATE INDEX windows_by_value ON metric_windows (value DESC, machine_id); +SELECT machine_id, value FROM metric_windows WHERE value = 20; +---- +machine_id | value +m1 | 20 + +# The names the engine gives the indexes it makes are kept for it. +CREATE INDEX __fk_readings ON readings (machine_id); +---- +error: ReservedIndexName + +CREATE INDEX __pk_machine_id ON metric_windows (machine_id); +---- +error: ReservedIndexName + +CREATE INDEX __unique_note ON readings (note); +---- +error: ReservedIndexName + +# A partition key takes a column of any type whose values are equal or not and nothing +# between: UUID, TEXT, INTEGER, BOOLEAN, TIMESTAMP and TXID. +CREATE TABLE partitioned ( + id INTEGER PRIMARY KEY, + u UUID NOT NULL, + t TEXT NOT NULL, + i INTEGER NOT NULL, + b BOOLEAN NOT NULL, + at TIMESTAMP NOT NULL, + v VECTOR(2) PARTITION_KEY (u, t, i, b, at) MAX_PARTITIONS 2 +); +INSERT INTO partitioned (id, u, t, i, b, at, v) VALUES + (1, '00000000-0000-0000-0000-000000000001', 'x', 1, TRUE, '2026-01-01T00:00:00Z', [1.0, 0.0]), + (2, '00000000-0000-0000-0000-000000000001', 'x', 1, TRUE, '2026-01-01T00:00:00Z', [0.0, 1.0]), + (3, '00000000-0000-0000-0000-000000000001', 'x', 1, FALSE, '2026-01-01T00:00:00Z', [1.0, 0.0]); +INSERT INTO partitioned (id, u, t, i, b, at, v) VALUES + (4, '00000000-0000-0000-0000-000000000001', 'x', 2, TRUE, '2026-01-01T00:00:00Z', [1.0, 0.0]); +---- +error: VectorPartitionLimitExceeded + +SELECT id FROM partitioned WHERE b = TRUE ORDER BY v <=> [0.0, 1.0] LIMIT 5; +---- +id +2 +1 + +CREATE TABLE partitioned_by_commit (id INTEGER PRIMARY KEY, made_at TXID NOT NULL, v VECTOR(2) PARTITION_KEY (made_at)); + +# A TXID is given by the engine and bound by the program that calls the library: no SQL text +# writes one. +CREATE TABLE stamped (id INTEGER PRIMARY KEY, made_at TXID); +INSERT INTO stamped (id, made_at) VALUES (1, 5); +---- +error: ColumnTypeMismatch + +INSERT INTO stamped (id, made_at) VALUES (1, '5'); +---- +error: ColumnTypeMismatch + +INSERT INTO stamped (id) VALUES (1); +SELECT id, made_at FROM stamped; +---- +id | made_at +1 | NULL diff --git a/v2/contextdb/tests/sql/dag.sql b/v2/contextdb/tests/sql/dag.sql new file mode 100644 index 00000000..87a354c2 --- /dev/null +++ b/v2/contextdb/tests/sql/dag.sql @@ -0,0 +1,162 @@ +# DAG('TYPE', ...): the edges of each listed type never form a cycle. An edge that would close +# one is refused when it is written, by an INSERT or by an UPDATE. + +CREATE TABLE edges ( + id INTEGER PRIMARY KEY, + source_id UUID NOT NULL, + target_id UUID NOT NULL, + edge_type TEXT NOT NULL +) DAG('DEPENDS_ON', 'BASED_ON'); + +# 1 -> 2 -> 3, and 1 -> 3 beside it: two ways to one node are no cycle. +INSERT INTO edges (id, source_id, target_id, edge_type) +VALUES (1, '00000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000002', 'DEPENDS_ON'), + (2, '00000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000003', 'DEPENDS_ON'), + (3, '00000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000003', 'DEPENDS_ON'); +---- +rows affected: 3 + +# Back to the start of the chain, over two edges and over one. +INSERT INTO edges (id, source_id, target_id, edge_type) +VALUES (4, '00000000-0000-0000-0000-000000000003', '00000000-0000-0000-0000-000000000001', 'DEPENDS_ON'); +---- +error: CycleDetected + +INSERT INTO edges (id, source_id, target_id, edge_type) +VALUES (4, '00000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000001', 'DEPENDS_ON'); +---- +error: CycleDetected + +# An edge from a node to itself is the shortest cycle. +INSERT INTO edges (id, source_id, target_id, edge_type) +VALUES (4, '00000000-0000-0000-0000-000000000009', '00000000-0000-0000-0000-000000000009', 'DEPENDS_ON'); +---- +error: CycleDetected + +# Two edges of one statement that close a cycle between them: neither is written. +INSERT INTO edges (id, source_id, target_id, edge_type) +VALUES (4, '00000000-0000-0000-0000-000000000005', '00000000-0000-0000-0000-000000000006', 'DEPENDS_ON'), + (5, '00000000-0000-0000-0000-000000000006', '00000000-0000-0000-0000-000000000005', 'DEPENDS_ON'); +---- +error: CycleDetected + +SELECT id FROM edges ORDER BY id; +---- +id +1 +2 +3 + +# Each listed type is held acyclic by itself: an edge of the second type may run against the +# edges of the first, and may not run against its own. +INSERT INTO edges (id, source_id, target_id, edge_type) +VALUES (10, '00000000-0000-0000-0000-000000000003', '00000000-0000-0000-0000-000000000001', 'BASED_ON'); +---- +rows affected: 1 + +INSERT INTO edges (id, source_id, target_id, edge_type) +VALUES (11, '00000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000003', 'BASED_ON'); +---- +error: CycleDetected + +# A type the table does not list may form any cycle. +INSERT INTO edges (id, source_id, target_id, edge_type) +VALUES (20, '00000000-0000-0000-0000-000000000003', '00000000-0000-0000-0000-000000000001', 'CITES'), + (21, '00000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000003', 'CITES'), + (22, '00000000-0000-0000-0000-000000000004', '00000000-0000-0000-0000-000000000004', 'CITES'); +---- +rows affected: 3 + +# An UPDATE is held to the same rule: turning an edge round, and giving an edge a listed type. +UPDATE edges SET source_id = '00000000-0000-0000-0000-000000000003', target_id = '00000000-0000-0000-0000-000000000001' +WHERE id = 3; +---- +error: CycleDetected + +UPDATE edges SET edge_type = 'DEPENDS_ON' WHERE id = 20; +---- +error: CycleDetected + +UPDATE edges SET edge_type = 'DEPENDS_ON' WHERE id = 22; +---- +error: CycleDetected + +SELECT id, edge_type FROM edges ORDER BY id; +---- +id | edge_type +1 | DEPENDS_ON +2 | DEPENDS_ON +3 | DEPENDS_ON +10 | BASED_ON +20 | CITES +21 | CITES +22 | CITES + +# Once the edge that made the way round is gone, the edge back is no cycle. +DELETE FROM edges WHERE id IN (2, 3); +---- +rows affected: 2 + +INSERT INTO edges (id, source_id, target_id, edge_type) +VALUES (4, '00000000-0000-0000-0000-000000000003', '00000000-0000-0000-0000-000000000001', 'DEPENDS_ON'); +---- +rows affected: 1 + +# 3 -> 1 -> 2 now: the edge 2 -> 3 that was deleted would close it. +INSERT INTO edges (id, source_id, target_id, edge_type) +VALUES (2, '00000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000003', 'DEPENDS_ON'); +---- +error: CycleDetected + +# The table declares no UNIQUE rule over an edge's three values, so a second row with the same +# three is a second row. It closes no cycle either. +INSERT INTO edges (id, source_id, target_id, edge_type) +VALUES (30, '00000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000002', 'DEPENDS_ON'); +---- +rows affected: 1 + +SELECT id FROM edges WHERE edge_type = 'DEPENDS_ON' ORDER BY id; +---- +id +1 +4 +30 + +# DAG is a rule about edges: a table without the three edge columns cannot declare it. +CREATE TABLE plain (id INTEGER PRIMARY KEY, note TEXT) DAG('DEPENDS_ON'); +---- +error: SchemaInvalid + +CREATE TABLE almost (id INTEGER PRIMARY KEY, source_id UUID NOT NULL, target_id UUID NOT NULL) DAG('DEPENDS_ON'); +---- +error: SchemaInvalid + +SELECT id FROM plain; +---- +error: TableNotFound + +# An edge that is the only one in the way is not held against its own replacement: 2 -> 1 would +# close a cycle with 1 -> 2, but 1 -> 2 is the edge this update turns round. +CREATE TABLE one_edge ( + id INTEGER PRIMARY KEY, + source_id UUID NOT NULL, + target_id UUID NOT NULL, + edge_type TEXT NOT NULL +) DAG('DEPENDS_ON'); +INSERT INTO one_edge (id, source_id, target_id, edge_type) +VALUES (1, '00000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000002', 'DEPENDS_ON'); +UPDATE one_edge SET source_id = '00000000-0000-0000-0000-000000000002', target_id = '00000000-0000-0000-0000-000000000001' +WHERE id = 1; +---- +rows affected: 1 + +# The edge is now 2 -> 1, and the edge it was would close a cycle with it. +INSERT INTO one_edge (id, source_id, target_id, edge_type) +VALUES (2, '00000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000002', 'DEPENDS_ON'); +---- +error: CycleDetected + +SELECT id, source_id, target_id FROM one_edge; +---- +id | source_id | target_id +1 | 00000000-0000-0000-0000-000000000002 | 00000000-0000-0000-0000-000000000001 diff --git a/v2/contextdb/tests/sql/declarations.sql b/v2/contextdb/tests/sql/declarations.sql new file mode 100644 index 00000000..d67f2b70 --- /dev/null +++ b/v2/contextdb/tests/sql/declarations.sql @@ -0,0 +1,359 @@ +# What a table and an index may declare, and what a declaration refuses. A refused declaration +# declares nothing. What a rule then does to a write is the subject of the script named after +# the rule; how a declaration is printed back is checked by `tests/declarations.rs`. + +# What earlier versions read is still read, with its meaning. IF NOT EXISTS declares a free name +# and leaves a taken one as it is, whatever the second declaration says. +CREATE TABLE IF NOT EXISTS notes (id INTEGER PRIMARY KEY, title TEXT NOT NULL DEFAULT 'none', body TEXT NULL, UNIQUE (title, body)); +INSERT INTO notes (id, body) VALUES (1, 'first'); +CREATE TABLE IF NOT EXISTS notes (id UUID PRIMARY KEY, other REAL) IMMUTABLE; +---- +rows affected: 0 + +SELECT other FROM notes; +---- +error: ColumnNotFound + +UPDATE notes SET body = 'again' WHERE id = 1; +---- +rows affected: 1 + +SELECT id, title, body FROM notes ORDER BY id; +---- +id | title | body +1 | none | again + +# The clauses of a VECTOR column: its partition key, its limit and its search mode. +CREATE TABLE docs ( + id INTEGER PRIMARY KEY, + tenant TEXT NOT NULL, + v VECTOR(3) PARTITION_KEY (tenant) MAX_PARTITIONS 2 SEARCH_MODE EXACT, + w VECTOR(2) SEARCH_MODE AUTO, + plain VECTOR(2) +); +INSERT INTO docs (id, tenant, v) VALUES (1, 'north', [1.0, 0.0, 0.0]), (2, 'south', [0.0, 1.0, 0.0]); +INSERT INTO docs (id, tenant, v) VALUES (3, 'east', [0.0, 0.0, 1.0]); +---- +error: VectorPartitionLimitExceeded + +SELECT id FROM docs ORDER BY v <=> [1.0, 0.0, 0.0] LIMIT 1; +---- +id +1 + +# A plain index with a direction for each column. +CREATE INDEX notes_by_title ON notes (title DESC, id ASC); +INSERT INTO notes (id, title, body) VALUES (2, 'b', 'x'), (3, 'a', 'y'); +SELECT id, title FROM notes ORDER BY title DESC, id; +---- +id | title +1 | none +2 | b +3 | a + +# A name written without quotes is its lower-case letters, and a name in quotes is the letters +# as written: `Items`, `ITEMS` and `items` are one table, and "Items" is another. +CREATE TABLE Items (Id INTEGER PRIMARY KEY, Name TEXT); +INSERT INTO ITEMS (ID, NAME) VALUES (1, 'a'); +SELECT Id, NAME FROM items; +---- +id | name +1 | a + +CREATE TABLE ITEMS (id INTEGER PRIMARY KEY); +---- +error: SchemaInvalid + +SELECT id FROM "Items"; +---- +error: TableNotFound + +SELECT "Id" FROM items; +---- +error: ColumnNotFound + +CREATE TABLE "Items" ("Id" INTEGER PRIMARY KEY, "Name" TEXT); +INSERT INTO "Items" ("Id", "Name") VALUES (7, 'quoted'); +SELECT "Id", "Name" FROM "Items"; +---- +Id | Name +7 | quoted + +SELECT id, name FROM items; +---- +id | name +1 | a + +# An index's name folds the same way, and belongs to its table: a second index of the name on +# the same table is refused, and another table may use the name. +CREATE INDEX Items_By_Name ON Items (Name); +CREATE INDEX items_by_name ON items (id); +---- +error: DuplicateIndex + +CREATE INDEX items_by_name ON notes (body); + +# What a column may declare: a reference to a parent's key or unique column, IMMUTABLE, the +# column that holds the row's Context, the labels a reader needs, and the type TXID. +CREATE TABLE contexts (id UUID PRIMARY KEY, name TEXT NOT NULL UNIQUE); +CREATE TABLE projects ( + id UUID PRIMARY KEY, + context_id UUID CONTEXT_ID NOT NULL REFERENCES contexts(id), + context_name TEXT REFERENCES contexts(name), + name TEXT NOT NULL IMMUTABLE, + scope TEXT SCOPE_LABEL ('public', 'internal'), + made_at TXID, + UNIQUE (context_id, id) +); +CREATE TABLE memos ( + id INTEGER PRIMARY KEY, + scope TEXT SCOPE_LABEL_READ ('reader', 'writer') WRITE ('writer'), + body TEXT +); + +# The column that holds a row's access id, checked against a table of grants. +CREATE TABLE acl_grants (id UUID PRIMARY KEY, principal_kind TEXT, principal_id TEXT, acl_id UUID); +CREATE TABLE guarded (id INTEGER PRIMARY KEY, acl_id UUID ACL REFERENCES acl_grants(acl_id), payload TEXT); + +# A reference of several columns is an element of the table, to a parent's key or unique rule. +CREATE TABLE tasks ( + id UUID PRIMARY KEY, + context_id UUID NOT NULL, + project_id UUID NOT NULL, + status TEXT NOT NULL DEFAULT 'active', + FOREIGN KEY (context_id, project_id) REFERENCES projects(context_id, id) +) STATE MACHINE (status: active -> [done, dropped], done -> [archived]); + +# A reference may move its row's state when the parent's state changes. +CREATE TABLE plans (id UUID PRIMARY KEY, name TEXT NOT NULL); +CREATE TABLE steps ( + id UUID PRIMARY KEY, + description TEXT NOT NULL, + status TEXT NOT NULL, + confidence REAL, + created_at TIMESTAMP DEFAULT NOW(), + plan_id UUID REFERENCES plans(id) + ON STATE archived PROPAGATE SET invalidated, + embedding VECTOR(4) +) STATE MACHINE (status: active -> [invalidated, superseded]) + PROPAGATE ON EDGE CITES INCOMING STATE invalidated SET invalidated + PROPAGATE ON STATE invalidated EXCLUDE VECTOR + PROPAGATE ON STATE superseded EXCLUDE VECTOR; +CREATE TABLE reviews ( + id UUID PRIMARY KEY, + status TEXT NOT NULL, + plan_id UUID REFERENCES plans(id) ON STATE archived PROPAGATE SET stale MAX DEPTH 2 ABORT ON FAILURE +) STATE MACHINE (status: open -> [stale]) + PROPAGATE ON EDGE CITES OUTGOING STATE stale SET stale MAX DEPTH 3 ABORT ON FAILURE + PROPAGATE ON EDGE BLOCKS BOTH STATE stale SET stale; + +# The primary key as an element of the table, over one column or several, and a table with no +# primary key, whose rows are kept as they arrive: two that hold the same values are two rows. +CREATE TABLE one_key (a INTEGER, b TEXT, PRIMARY KEY (a)); +CREATE TABLE two_keys (a INTEGER NOT NULL, b TEXT NOT NULL, c TEXT, PRIMARY KEY (a, b)); +CREATE TABLE lines (line TEXT NOT NULL); +INSERT INTO lines (line) VALUES ('same'), ('same'); +SELECT COUNT(*) AS kept FROM lines; +---- +kept +2 + +# Every option of a table, alone. +CREATE TABLE o_immutable (id INTEGER PRIMARY KEY, body TEXT) IMMUTABLE; +CREATE TABLE o_dag ( + id UUID PRIMARY KEY, + source_id UUID NOT NULL, + target_id UUID NOT NULL, + edge_type TEXT NOT NULL +) DAG('DEPENDS_ON', 'BASED_ON'); +CREATE TABLE o_retain_seconds (id INTEGER PRIMARY KEY) RETAIN 90 SECONDS; +CREATE TABLE o_retain_minutes (id INTEGER PRIMARY KEY) RETAIN 30 MINUTES; +CREATE TABLE o_retain_hours (id INTEGER PRIMARY KEY) RETAIN 24 HOURS SYNC SAFE; +CREATE TABLE o_retain_days (id INTEGER PRIMARY KEY) RETAIN 7 DAYS; +CREATE TABLE o_history_all (id INTEGER PRIMARY KEY) HISTORY ALL; +CREATE TABLE o_sync_off (id INTEGER PRIMARY KEY) SYNC OFF; +CREATE TABLE o_sync_push (id INTEGER PRIMARY KEY) SYNC PUSH ONLY; +CREATE TABLE o_sync_pull (id INTEGER PRIMARY KEY) SYNC PULL ONLY; +CREATE TABLE o_sync_two_way (id INTEGER PRIMARY KEY) SYNC TWO WAY; +CREATE TABLE o_keep_first (id INTEGER PRIMARY KEY) SYNC CONFLICT KEEP FIRST; +CREATE TABLE o_keep_latest (id INTEGER PRIMARY KEY) SYNC CONFLICT KEEP LATEST; +CREATE TABLE o_counts ( + id UUID PRIMARY KEY, + context_id UUID NOT NULL, + connector_id TEXT NOT NULL, + source_instance_ref TEXT NOT NULL, + UNIQUE (context_id, id), + UNIQUE (context_id, connector_id, source_instance_ref, id) +) MAINTAIN COUNT (context_id), (context_id, connector_id, source_instance_ref); +CREATE TABLE o_counts_all (id INTEGER PRIMARY KEY, kind TEXT, UNIQUE (kind, id)) MAINTAIN COUNT ALL PREFIXES; +CREATE TABLE o_counts_none (id INTEGER PRIMARY KEY, kind TEXT) MAINTAIN COUNT NONE; +CREATE TABLE o_manifest (id INTEGER PRIMARY KEY) SYNC PUSH ONLY SYNC CONFLICT KEEP FIRST IMMUTABLE + DELIVERY MANIFEST OVER o_manifest_parts, o_manifest_marks + EDGE DISCARD AFTER OUTCOME; +CREATE TABLE o_discard_never (id INTEGER PRIMARY KEY) SYNC PUSH ONLY SYNC CONFLICT KEEP FIRST IMMUTABLE + DELIVERY MANIFEST OVER o_discard_parts EDGE DISCARD NEVER; +CREATE TABLE o_discard_always (id INTEGER PRIMARY KEY) SYNC PUSH ONLY SYNC CONFLICT KEEP FIRST IMMUTABLE + EDGE DISCARD ALWAYS; + +# How long rows are kept, how many versions, where rows travel and which write wins are four +# separate clauses, and compose on one table. Keeping only the current version of each row is +# its own clause, with the two declarations that let it stand. +CREATE TABLE windows (id UUID PRIMARY KEY, body TEXT) + RETAIN 48 HOURS SYNC SAFE HISTORY ALL SYNC TWO WAY SYNC CONFLICT KEEP FIRST; +CREATE TABLE current_latest (id TEXT PRIMARY KEY, state TEXT NOT NULL) HISTORY CURRENT ONLY SYNC CONFLICT KEEP LATEST; +CREATE TABLE current_local (id TEXT PRIMARY KEY, state TEXT NOT NULL) RETAIN 2 DAYS HISTORY CURRENT ONLY SYNC OFF; + +# A partitioned VECTOR column stands under every direction. +CREATE TABLE p_off (id INTEGER PRIMARY KEY, t TEXT NOT NULL, v VECTOR(2) PARTITION_KEY (t)) SYNC OFF; +CREATE TABLE p_push (id INTEGER PRIMARY KEY, t TEXT NOT NULL, v VECTOR(2) PARTITION_KEY (t)) SYNC PUSH ONLY; +CREATE TABLE p_pull (id INTEGER PRIMARY KEY, t TEXT NOT NULL, v VECTOR(2) PARTITION_KEY (t)) SYNC PULL ONLY; +CREATE TABLE p_two_way (id INTEGER PRIMARY KEY, t TEXT NOT NULL, v VECTOR(2) PARTITION_KEY (t)) SYNC TWO WAY; + +# A table that keeps only current versions, sends its rows out and keeps the first write would +# hand another machine a newest value as the first: refused. KEEP LATEST or SYNC OFF is the way +# out, as above. +CREATE TABLE refused (id TEXT PRIMARY KEY, state TEXT NOT NULL) HISTORY CURRENT ONLY; +---- +error: SchemaInvalid + +CREATE TABLE refused (id TEXT PRIMARY KEY, state TEXT NOT NULL) HISTORY CURRENT ONLY SYNC PUSH ONLY SYNC CONFLICT KEEP FIRST; +---- +error: SchemaInvalid + +# An IMMUTABLE table has no row to expire and no older version to drop. +CREATE TABLE refused (id INTEGER PRIMARY KEY) IMMUTABLE RETAIN 7 DAYS; +---- +error: SchemaInvalid + +CREATE TABLE refused (id INTEGER PRIMARY KEY) IMMUTABLE HISTORY CURRENT ONLY SYNC OFF; +---- +error: SchemaInvalid + +# SYNC SAFE waits for delivery, so it needs a direction that delivers. Plain RETAIN takes any. +CREATE TABLE refused (id INTEGER PRIMARY KEY) RETAIN 7 DAYS SYNC SAFE SYNC OFF; +---- +error: SchemaInvalid + +CREATE TABLE refused (id INTEGER PRIMARY KEY) RETAIN 7 DAYS SYNC SAFE SYNC PULL ONLY; +---- +error: SchemaInvalid + +CREATE TABLE retain_local (id INTEGER PRIMARY KEY) RETAIN 7 DAYS SYNC OFF; +CREATE TABLE retain_pull (id INTEGER PRIMARY KEY) RETAIN 7 DAYS SYNC PULL ONLY; + +# RETAIN names no direction: one written inside it is a syntax error. +CREATE TABLE refused (id INTEGER PRIMARY KEY) RETAIN 24 HOURS SYNC SAFE PUSH ONLY; +---- +error: ParseError + +# A reference names columns that a key or a UNIQUE rule of the parent covers. +CREATE TABLE refused (id INTEGER PRIMARY KEY, plan_name TEXT REFERENCES plans(name)); +---- +error: SchemaInvalid + +CREATE TABLE refused ( + id INTEGER PRIMARY KEY, + context_id UUID, + name TEXT, + FOREIGN KEY (context_id, name) REFERENCES projects(context_id, name) +); +---- +error: SchemaInvalid + +# A kept count needs an index that leads with its list, and is not kept where a key already +# says the count is at most one. +CREATE TABLE refused (id INTEGER PRIMARY KEY, kind TEXT) MAINTAIN COUNT (kind); +---- +error: SchemaInvalid + +CREATE TABLE refused (id INTEGER PRIMARY KEY, kind TEXT) MAINTAIN COUNT (id); +---- +error: SchemaInvalid + +# DAG is for a table of edges: source_id, target_id and edge_type. +CREATE TABLE refused (id INTEGER PRIMARY KEY, source_id UUID NOT NULL, target_id UUID NOT NULL) DAG('DEPENDS_ON'); +---- +error: SchemaInvalid + +SELECT id FROM refused; +---- +error: TableNotFound + +# A key, a UNIQUE rule and an index name at most 32 columns. +CREATE TABLE refused (c01 INTEGER, c02 INTEGER, c03 INTEGER, c04 INTEGER, c05 INTEGER, c06 INTEGER, c07 INTEGER, c08 INTEGER, c09 INTEGER, c10 INTEGER, c11 INTEGER, c12 INTEGER, c13 INTEGER, c14 INTEGER, c15 INTEGER, c16 INTEGER, c17 INTEGER, c18 INTEGER, c19 INTEGER, c20 INTEGER, c21 INTEGER, c22 INTEGER, c23 INTEGER, c24 INTEGER, c25 INTEGER, c26 INTEGER, c27 INTEGER, c28 INTEGER, c29 INTEGER, c30 INTEGER, c31 INTEGER, c32 INTEGER, c33 INTEGER, PRIMARY KEY (c01, c02, c03, c04, c05, c06, c07, c08, c09, c10, c11, c12, c13, c14, c15, c16, c17, c18, c19, c20, c21, c22, c23, c24, c25, c26, c27, c28, c29, c30, c31, c32, c33)); +---- +error: SchemaInvalid + +CREATE TABLE refused (c01 INTEGER, c02 INTEGER, c03 INTEGER, c04 INTEGER, c05 INTEGER, c06 INTEGER, c07 INTEGER, c08 INTEGER, c09 INTEGER, c10 INTEGER, c11 INTEGER, c12 INTEGER, c13 INTEGER, c14 INTEGER, c15 INTEGER, c16 INTEGER, c17 INTEGER, c18 INTEGER, c19 INTEGER, c20 INTEGER, c21 INTEGER, c22 INTEGER, c23 INTEGER, c24 INTEGER, c25 INTEGER, c26 INTEGER, c27 INTEGER, c28 INTEGER, c29 INTEGER, c30 INTEGER, c31 INTEGER, c32 INTEGER, c33 INTEGER, PRIMARY KEY (c01), UNIQUE (c01, c02, c03, c04, c05, c06, c07, c08, c09, c10, c11, c12, c13, c14, c15, c16, c17, c18, c19, c20, c21, c22, c23, c24, c25, c26, c27, c28, c29, c30, c31, c32, c33)); +---- +error: SchemaInvalid + +SELECT c01 FROM refused; +---- +error: TableNotFound + +CREATE TABLE wide (c01 INTEGER, c02 INTEGER, c03 INTEGER, c04 INTEGER, c05 INTEGER, c06 INTEGER, c07 INTEGER, c08 INTEGER, c09 INTEGER, c10 INTEGER, c11 INTEGER, c12 INTEGER, c13 INTEGER, c14 INTEGER, c15 INTEGER, c16 INTEGER, c17 INTEGER, c18 INTEGER, c19 INTEGER, c20 INTEGER, c21 INTEGER, c22 INTEGER, c23 INTEGER, c24 INTEGER, c25 INTEGER, c26 INTEGER, c27 INTEGER, c28 INTEGER, c29 INTEGER, c30 INTEGER, c31 INTEGER, c32 INTEGER, c33 INTEGER, PRIMARY KEY (c01), UNIQUE (c01, c02, c03, c04, c05, c06, c07, c08, c09, c10, c11, c12, c13, c14, c15, c16, c17, c18, c19, c20, c21, c22, c23, c24, c25, c26, c27, c28, c29, c30, c31, c32)); +CREATE INDEX wide_33 ON wide (c01, c02, c03, c04, c05, c06, c07, c08, c09, c10, c11, c12, c13, c14, c15, c16, c17, c18, c19, c20, c21, c22, c23, c24, c25, c26, c27, c28, c29, c30, c31, c32, c33); +---- +error: SchemaInvalid + +CREATE INDEX wide_32 ON wide (c01, c02, c03, c04, c05, c06, c07, c08, c09, c10, c11, c12, c13, c14, c15, c16, c17, c18, c19, c20, c21, c22, c23, c24, c25, c26, c27, c28, c29, c30, c31, c32); +CREATE TABLE wide_key (c01 INTEGER, c02 INTEGER, c03 INTEGER, c04 INTEGER, c05 INTEGER, c06 INTEGER, c07 INTEGER, c08 INTEGER, c09 INTEGER, c10 INTEGER, c11 INTEGER, c12 INTEGER, c13 INTEGER, c14 INTEGER, c15 INTEGER, c16 INTEGER, c17 INTEGER, c18 INTEGER, c19 INTEGER, c20 INTEGER, c21 INTEGER, c22 INTEGER, c23 INTEGER, c24 INTEGER, c25 INTEGER, c26 INTEGER, c27 INTEGER, c28 INTEGER, c29 INTEGER, c30 INTEGER, c31 INTEGER, c32 INTEGER, c33 INTEGER, PRIMARY KEY (c01, c02, c03, c04, c05, c06, c07, c08, c09, c10, c11, c12, c13, c14, c15, c16, c17, c18, c19, c20, c21, c22, c23, c24, c25, c26, c27, c28, c29, c30, c31, c32)); + +# Clauses of a later version are still refused by their names, and stored procedures and +# functions by their own kind. +CREATE TABLE refused (id INTEGER PRIMARY KEY, until TIMESTAMP EXPIRES); +---- +error: PlanError + +CREATE PROCEDURE tidy () AS 'DELETE FROM notes'; +---- +error: StoredProcNotSupported + +CREATE FUNCTION one () RETURNS INTEGER AS 'SELECT 1'; +---- +error: StoredProcNotSupported + +SELECT id FROM refused; +---- +error: TableNotFound + +# An index names columns the table has and can index: a JSON column is not indexable, and a REAL +# may be indexed but is no key and under no UNIQUE rule. +CREATE TABLE people (id INTEGER PRIMARY KEY, score REAL, born INTEGER, doc JSON); +CREATE INDEX by_age ON people (age DESC); +---- +error: ColumnNotFound + +CREATE INDEX by_doc ON people (doc DESC); +---- +error: ColumnNotIndexable + +CREATE INDEX by_score ON people (score DESC); +CREATE TABLE ratios (id INTEGER PRIMARY KEY, ratio REAL UNIQUE); +---- +error: ColumnNotIndexable + +CREATE TABLE ratios (id INTEGER PRIMARY KEY, ratio REAL, UNIQUE (id, ratio)); +---- +error: ColumnNotIndexable + +# A JSON-path index is declared and stored but filled by a later step: a table with one is +# written to like any other, and the index is made on a table that already has rows. +CREATE TABLE kinds (id INTEGER PRIMARY KEY, doc JSON); +CREATE INDEX kinds_by_kind ON kinds ((doc -> 'kind')); +INSERT INTO kinds (id, doc) VALUES (1, '{"kind": "a"}'), (2, '{"kind": "b"}'); +UPDATE kinds SET doc = '{"kind": "c"}' WHERE id = 2; +SELECT id FROM kinds ORDER BY id; +---- +id +1 +2 + +CREATE TABLE filled (id INTEGER PRIMARY KEY, doc JSON); +INSERT INTO filled (id, doc) VALUES (1, '{"kind": "a"}'); +CREATE INDEX filled_by_kind ON filled ((doc -> 'kind')); +SELECT id FROM filled; +---- +id +1 diff --git a/v2/contextdb/tests/sql/drop.sql b/v2/contextdb/tests/sql/drop.sql new file mode 100644 index 00000000..27292a44 --- /dev/null +++ b/v2/contextdb/tests/sql/drop.sql @@ -0,0 +1,237 @@ +# DROP INDEX and DROP TABLE, and what a drop is refused for. A refused drop changes nothing. +# That a drop leaves none of the dropped thing's bytes in the store is checked by +# `tests/alter.rs`. + +CREATE TABLE notes (id INTEGER PRIMARY KEY, tag TEXT, body TEXT); +CREATE TABLE tags (id INTEGER PRIMARY KEY, tag TEXT); +INSERT INTO notes (id, tag, body) VALUES (1, 'a', 'first'), (2, 'b', 'second'), (3, 'a', 'third'); +INSERT INTO tags (id, tag) VALUES (1, 'a'); + +# An index is dropped by its name and its table. The rows are as they were, and a filter the +# index served is still answered. +CREATE INDEX by_tag ON notes (tag); +CREATE INDEX by_tag ON tags (tag); +DROP INDEX by_tag ON notes; +SELECT id FROM notes WHERE tag = 'a' ORDER BY id; +---- +id +1 +3 + +# An index that is not there is refused by name, and with IF EXISTS is nothing to do. +DROP INDEX by_tag ON notes; +---- +error: IndexNotFound + +DROP INDEX IF EXISTS by_tag ON notes; +---- +rows affected: 0 + +DROP INDEX IF EXISTS by_tag ON notes; +---- +rows affected: 0 + +DROP INDEX never_made ON notes; +---- +error: IndexNotFound + +DROP INDEX by_tag ON nope; +---- +error: TableNotFound + +# The index of the same name on another table was not touched, and the name is free again on +# this one. +DROP INDEX by_tag ON tags; +DROP INDEX by_tag ON tags; +---- +error: IndexNotFound + +CREATE INDEX by_tag ON notes (tag DESC); +DROP INDEX IF EXISTS By_Tag ON Notes; +DROP INDEX by_tag ON notes; +---- +error: IndexNotFound + +# A table is dropped with its rows and its indexes. +CREATE INDEX by_body ON notes (body); +DROP TABLE notes; +SELECT id FROM notes; +---- +error: TableNotFound + +INSERT INTO notes (id, tag, body) VALUES (9, 'z', 'late'); +---- +error: TableNotFound + +DROP INDEX by_body ON notes; +---- +error: TableNotFound + +# A table that is not there is refused by name, and with IF EXISTS is nothing to do. +DROP TABLE notes; +---- +error: TableNotFound + +DROP TABLE IF EXISTS notes; +---- +rows affected: 0 + +DROP TABLE IF EXISTS never_made; +---- +rows affected: 0 + +# The other table is as it was. +SELECT id, tag FROM tags ORDER BY id; +---- +id | tag +1 | a + +# The name is free: a table declared under it has none of the old columns, rows or indexes. +CREATE TABLE notes (id INTEGER PRIMARY KEY, other TEXT); +SELECT tag FROM notes; +---- +error: ColumnNotFound + +SELECT COUNT(*) AS n FROM notes; +---- +n +0 + +INSERT INTO notes (id, other) VALUES (1, 'new'); +CREATE INDEX by_body ON notes (other); +SELECT id, other FROM notes ORDER BY id; +---- +id | other +1 | new + +# A name folds in DROP as it does everywhere, and a quoted name is the letters as written. +CREATE TABLE "Mixed Case" (id INTEGER PRIMARY KEY); +DROP TABLE Notes; +DROP TABLE "mixed case"; +---- +error: TableNotFound + +DROP TABLE "Mixed Case"; +SELECT id FROM "Mixed Case"; +---- +error: TableNotFound + +# A table another table references is not dropped while the reference stands. +CREATE TABLE parents (id INTEGER PRIMARY KEY); +CREATE TABLE children (id INTEGER PRIMARY KEY, parent_id INTEGER REFERENCES parents(id)); +INSERT INTO parents (id) VALUES (1); +DROP TABLE parents; +---- +error: SchemaInvalid + +DROP TABLE IF EXISTS parents; +---- +error: SchemaInvalid + +SELECT id FROM parents; +---- +id +1 + +DROP TABLE children; +DROP TABLE parents; + +# The three columns that make a table a table of edges are not dropped or renamed while the +# table declares DAG. On a table that declares none, a drop of one is a drop like any other. +CREATE TABLE edges ( + id INTEGER PRIMARY KEY, + source_id UUID NOT NULL, + target_id UUID NOT NULL, + edge_type TEXT NOT NULL, + note TEXT +) DAG('DEPENDS_ON'); +ALTER TABLE edges DROP COLUMN edge_type; +---- +error: SchemaInvalid + +ALTER TABLE edges DROP COLUMN source_id CASCADE; +---- +error: SchemaInvalid + +ALTER TABLE edges RENAME COLUMN target_id TO goal; +---- +error: SchemaInvalid + +ALTER TABLE edges DROP COLUMN note; +CREATE TABLE links ( + id INTEGER PRIMARY KEY, + source_id UUID NOT NULL, + target_id UUID NOT NULL, + edge_type TEXT NOT NULL +); +INSERT INTO links (id, source_id, target_id, edge_type) +VALUES (1, '00000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000002', 'CITES'); +ALTER TABLE links DROP COLUMN edge_type; +SELECT id, source_id, target_id FROM links; +---- +id | source_id | target_id +1 | 00000000-0000-0000-0000-000000000001 | 00000000-0000-0000-0000-000000000002 + +# What a rank policy reads is not dropped while the policy stands: the table it joins, the +# index it joins through, and the columns its formula and its join name. +CREATE TABLE task_results (id UUID PRIMARY KEY, task_id UUID NOT NULL, success BOOLEAN NOT NULL, note TEXT); +CREATE INDEX task_results_task_id_idx ON task_results (task_id); +CREATE TABLE ranked ( + id UUID PRIMARY KEY, + description TEXT NOT NULL, + confidence REAL, + embedding VECTOR(4) RANK_POLICY ( + JOIN task_results ON task_id, + FORMULA 'coalesce({confidence}, 1.0) * coalesce({success}, 1.0)', + SORT_KEY effective_confidence + ) +); +DROP TABLE task_results; +---- +error: DropBlockedByRankPolicy + +DROP INDEX task_results_task_id_idx ON task_results; +---- +error: DropBlockedByRankPolicy + +ALTER TABLE task_results DROP COLUMN success; +---- +error: DropBlockedByRankPolicy + +ALTER TABLE task_results DROP COLUMN task_id CASCADE; +---- +error: DropBlockedByRankPolicy + +ALTER TABLE ranked DROP COLUMN confidence; +---- +error: DropBlockedByRankPolicy + +ALTER TABLE task_results DROP COLUMN note; +DROP TABLE ranked; +DROP INDEX task_results_task_id_idx ON task_results; +DROP TABLE task_results; + +# A dropped VECTOR column, and a dropped table with one, leave no vector behind: the column +# declared again has no partition, so a limit of one takes a row under another key, and a +# search finds only what was written since. +CREATE TABLE docs (id INTEGER PRIMARY KEY, tenant TEXT NOT NULL, v VECTOR(2) PARTITION_KEY (tenant) MAX_PARTITIONS 1); +INSERT INTO docs (id, tenant, v) VALUES (1, 'north', [1.0, 0.0]); +INSERT INTO docs (id, tenant, v) VALUES (2, 'south', [0.0, 1.0]); +---- +error: VectorPartitionLimitExceeded + +ALTER TABLE docs DROP COLUMN v; +ALTER TABLE docs ADD COLUMN v VECTOR(2) PARTITION_KEY (tenant) MAX_PARTITIONS 1; +INSERT INTO docs (id, tenant, v) VALUES (2, 'south', [0.0, 1.0]); +SELECT id FROM docs ORDER BY v <=> [1.0, 0.0] LIMIT 5; +---- +id +2 + +DROP TABLE docs; +CREATE TABLE docs (id INTEGER PRIMARY KEY, tenant TEXT NOT NULL, v VECTOR(2) PARTITION_KEY (tenant) MAX_PARTITIONS 1); +INSERT INTO docs (id, tenant, v) VALUES (7, 'east', [1.0, 0.0]); +SELECT id FROM docs ORDER BY v <=> [1.0, 0.0] LIMIT 5; +---- +id +7 diff --git a/v2/contextdb/tests/sql/foreign_keys.sql b/v2/contextdb/tests/sql/foreign_keys.sql new file mode 100644 index 00000000..7c8c669b --- /dev/null +++ b/v2/contextdb/tests/sql/foreign_keys.sql @@ -0,0 +1,312 @@ +# REFERENCES and FOREIGN KEY: a row that names a parent row is only committed when that row is +# there, and a parent row is not removed or given another key while a row still names it. Each +# statement here is a transaction of its own, so each refusal comes from the statement. + +CREATE TABLE projects (id INTEGER PRIMARY KEY, code TEXT UNIQUE, title TEXT); +CREATE TABLE tasks ( + id INTEGER PRIMARY KEY, + project_id INTEGER REFERENCES projects(id), + project_code TEXT REFERENCES projects(code), + note TEXT +); +INSERT INTO projects (id, code, title) VALUES (1, 'alpha', 'first'), (2, 'beta', 'second'); + +# A row whose parents are there, by the key and by the unique column. +INSERT INTO tasks (id, project_id, project_code) VALUES (1, 1, 'alpha'), (2, 2, NULL); +---- +rows affected: 2 + +# A reference that is NULL names no parent and is not checked. +INSERT INTO tasks (id, project_id, project_code) VALUES (3, NULL, NULL); +---- +rows affected: 1 + +# No such parent, by either reference. +INSERT INTO tasks (id, project_id) VALUES (4, 404); +---- +error: ForeignKeyViolation + +INSERT INTO tasks (id, project_code) VALUES (4, 'gamma'); +---- +error: ForeignKeyViolation + +# One row without its parent refuses the statement for every row. +INSERT INTO tasks (id, project_id) VALUES (5, 1), (6, 404); +---- +error: ForeignKeyViolation + +# A change that points a row at a parent that is not there, and one that takes the reference away. +UPDATE tasks SET project_id = 404 WHERE id = 1; +---- +error: ForeignKeyViolation + +UPDATE tasks SET project_code = NULL WHERE id = 1; +---- +rows affected: 1 + +SELECT id, project_id, project_code FROM tasks ORDER BY id; +---- +id | project_id | project_code +1 | 1 | NULL +2 | 2 | NULL +3 | NULL | NULL + +# A parent that a row still names is not deleted and does not change its key; another column +# of it changes freely. +DELETE FROM projects WHERE id = 1; +---- +error: ForeignKeyViolation + +UPDATE projects SET id = 10 WHERE id = 1; +---- +error: ForeignKeyViolation + +UPDATE projects SET title = 'renamed' WHERE id = 1; +---- +rows affected: 1 + +# No row names 'alpha' or 'beta' any more, so the unique column they were named by is free to +# change. +UPDATE projects SET code = 'alpha-2' WHERE id = 1; +---- +rows affected: 1 + +# Once nothing names it, the parent goes. +DELETE FROM tasks WHERE id = 1; +DELETE FROM projects WHERE id = 1; +---- +rows affected: 1 + +SELECT id, code, title FROM projects ORDER BY id; +---- +id | code | title +2 | beta | second + +# A reference of two columns to a parent whose key is those two columns in that order. +CREATE TABLE windows ( + machine TEXT NOT NULL, + slot INTEGER NOT NULL, + label TEXT, + PRIMARY KEY (machine, slot) +); +CREATE TABLE samples ( + id INTEGER PRIMARY KEY, + machine TEXT, + slot INTEGER, + FOREIGN KEY (machine, slot) REFERENCES windows(machine, slot) +); +INSERT INTO windows (machine, slot, label) VALUES ('m1', 1, 'a'), ('m1', 2, 'b'), ('m2', 1, 'c'); +INSERT INTO samples (id, machine, slot) VALUES (1, 'm1', 1), (2, 'm2', 1); +---- +rows affected: 2 + +# Each column has a parent of its own, and the pair has none. +INSERT INTO samples (id, machine, slot) VALUES (3, 'm2', 2); +---- +error: ForeignKeyViolation + +# A reference with a NULL in any of its columns is not whole and is not checked. +INSERT INTO samples (id, machine, slot) VALUES (4, 'm9', NULL), (5, NULL, 9); +---- +rows affected: 2 + +DELETE FROM windows WHERE machine = 'm1' AND slot = 1; +---- +error: ForeignKeyViolation + +DELETE FROM windows WHERE machine = 'm1' AND slot = 2; +---- +rows affected: 1 + +UPDATE windows SET slot = 7 WHERE machine = 'm2'; +---- +error: ForeignKeyViolation + +SELECT machine, slot FROM windows ORDER BY machine, slot; +---- +machine | slot +m1 | 1 +m2 | 1 + +# A reference to two columns that a UNIQUE rule covers. +CREATE TABLE seats (id INTEGER PRIMARY KEY, room TEXT, seat INTEGER, UNIQUE (room, seat)); +CREATE TABLE bookings ( + id INTEGER PRIMARY KEY, + room TEXT, + seat INTEGER, + FOREIGN KEY (room, seat) REFERENCES seats(room, seat) +); +INSERT INTO seats (id, room, seat) VALUES (1, 'east', 1); +INSERT INTO bookings (id, room, seat) VALUES (1, 'east', 1); +INSERT INTO bookings (id, room, seat) VALUES (2, 'east', 2); +---- +error: ForeignKeyViolation + +# A reference must name what a key or a UNIQUE rule covers, in the rule's order, in a table that +# is there. Nothing is declared by a refused statement. +CREATE TABLE bad_column (id INTEGER PRIMARY KEY, title TEXT REFERENCES projects(title)); +---- +error: SchemaInvalid + +CREATE TABLE bad_order ( + id INTEGER PRIMARY KEY, machine TEXT, slot INTEGER, + FOREIGN KEY (slot, machine) REFERENCES windows(slot, machine) +); +---- +error: SchemaInvalid + +CREATE TABLE bad_part (id INTEGER PRIMARY KEY, machine TEXT, FOREIGN KEY (machine) REFERENCES windows(machine)); +---- +error: SchemaInvalid + +CREATE TABLE bad_parent (id INTEGER PRIMARY KEY, owner INTEGER REFERENCES nowhere(id)); +---- +error: TableNotFound + +SELECT id FROM bad_column; +---- +error: TableNotFound + +SELECT id FROM bad_parent; +---- +error: TableNotFound + +# The engine names the index behind a reference itself, and keeps that beginning of a name. +CREATE INDEX __fk_tasks_note ON tasks (note); +---- +error: ReservedIndexName + +# A column an index names is not dropped unless the statement says CASCADE, and the refusal +# leaves the column and its indexes in place. +CREATE TABLE notes (id INTEGER PRIMARY KEY, tag TEXT, body TEXT); +CREATE INDEX notes_tag ON notes (tag); +CREATE INDEX notes_tag_body ON notes (tag, body); +INSERT INTO notes (id, tag, body) VALUES (1, 'a', 'x'), (2, 'b', 'y'); +ALTER TABLE notes DROP COLUMN tag; +---- +error: ColumnInIndex + +ALTER TABLE notes DROP COLUMN tag RESTRICT; +---- +error: ColumnInIndex + +SELECT id, tag, body FROM notes WHERE tag = 'b'; +---- +id | tag | body +2 | b | y + +ALTER TABLE notes DROP COLUMN tag CASCADE; +SELECT tag FROM notes; +---- +error: ColumnNotFound + +SELECT id, body FROM notes ORDER BY id; +---- +id | body +1 | x +2 | y + +# A reference joins columns of one type: an INTEGER never matches a UUID key, so the table is +# refused when it is declared. +CREATE TABLE typed_parents (id UUID PRIMARY KEY); +CREATE TABLE typed_children (id INTEGER PRIMARY KEY, pid INTEGER REFERENCES typed_parents(id)); +---- +error: SchemaInvalid + +SELECT id FROM typed_children; +---- +error: TableNotFound + +# A table that references itself keeps the reference when the column is renamed. +CREATE TABLE tree (id INTEGER PRIMARY KEY, parent INTEGER REFERENCES tree(id)); +INSERT INTO tree (id, parent) VALUES (1, NULL), (2, 1); +ALTER TABLE tree RENAME COLUMN parent TO up; +INSERT INTO tree (id, up) VALUES (3, 2); +---- +rows affected: 1 + +INSERT INTO tree (id, up) VALUES (4, 404); +---- +error: ForeignKeyViolation + +DELETE FROM tree WHERE id = 1; +---- +error: ForeignKeyViolation + +# Renaming the referenced UNIQUE column (not the key) leaves the reference in force, as in +# Postgres: a missing parent is still refused and a present one taken. +CREATE TABLE n (id INTEGER PRIMARY KEY, code TEXT UNIQUE, parent TEXT REFERENCES n(code)); +INSERT INTO n (id, code) VALUES (1, 'a'); +ALTER TABLE n RENAME COLUMN code TO tag; +INSERT INTO n (id, tag, parent) VALUES (2, 'b', 'a'); +---- +rows affected: 1 + +INSERT INTO n (id, tag, parent) VALUES (3, 'c', 'zz'); +---- +error: ForeignKeyViolation + +# Propagation along an edge needs the table to have the `id` column the edges name, so a table +# without one is refused when it is declared, not when a row first changes state. +CREATE TABLE no_id (k UUID PRIMARY KEY, s TEXT) STATE MACHINE (s: a -> [b]) PROPAGATE ON EDGE E INCOMING STATE b SET b; +---- +error: SchemaInvalid + +# The child's key is the reference: a profile is the profile of the person with its key. +CREATE TABLE people (id INTEGER PRIMARY KEY); +CREATE TABLE profiles ( + person_id INTEGER PRIMARY KEY, + bio TEXT, + FOREIGN KEY (person_id) REFERENCES people(id) +); +INSERT INTO people (id) VALUES (1), (2); +INSERT INTO profiles (person_id, bio) VALUES (404, 'nobody'); +---- +error: ForeignKeyViolation + +INSERT INTO profiles (person_id, bio) VALUES (1, 'first'); +---- +rows affected: 1 + +SELECT person_id, bio FROM profiles; +---- +person_id | bio +1 | first + +# A table that references itself: a part names the part it belongs to. +CREATE TABLE parts (id INTEGER PRIMARY KEY, parent_id INTEGER REFERENCES parts(id)); +INSERT INTO parts (id, parent_id) VALUES (1, NULL); +INSERT INTO parts (id, parent_id) VALUES (3, 404); +---- +error: ForeignKeyViolation + +INSERT INTO parts (id, parent_id) VALUES (2, 1); +---- +rows affected: 1 + +SELECT id, parent_id FROM parts ORDER BY id; +---- +id | parent_id +1 | NULL +2 | 1 + +# A reference onto a TEXT column whose values are longer than an index entry keeps (256 bytes): +# the parent named differs from another only after byte 256, and the reference tells them apart. +CREATE TABLE long_parents (id INTEGER PRIMARY KEY, name TEXT UNIQUE); +CREATE TABLE long_children (id INTEGER PRIMARY KEY, parent_name TEXT REFERENCES long_parents(name)); +INSERT INTO long_parents (id, name) VALUES (1, 'ppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppaxxxxxxxxxxxxxxxxxxx'), (2, 'ppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppbxxxxxxxxxxxxxxxxxxx'); +INSERT INTO long_children (id, parent_name) VALUES (1, 'ppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppaxxxxxxxxxxxxxxxxxxx'); +---- +rows affected: 1 + +INSERT INTO long_children (id, parent_name) VALUES (2, 'ppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppcxxxxxxxxxxxxxxxxxxx'); +---- +error: ForeignKeyViolation + +DELETE FROM long_parents WHERE id = 1; +---- +error: ForeignKeyViolation + +DELETE FROM long_parents WHERE id = 2; +---- +rows affected: 1 diff --git a/v2/contextdb/tests/sql/immutable.sql b/v2/contextdb/tests/sql/immutable.sql new file mode 100644 index 00000000..6a4c9b72 --- /dev/null +++ b/v2/contextdb/tests/sql/immutable.sql @@ -0,0 +1,175 @@ +# IMMUTABLE on a table: its rows are inserted once and never updated or deleted. IMMUTABLE on a +# column: the value is set when the row is inserted and never changed, while the row's other +# columns stay as free as before. + +CREATE TABLE documents ( + id UUID PRIMARY KEY, + data JSON, + embedding VECTOR(3), + title TEXT, + created_at TIMESTAMP DEFAULT NOW() +) IMMUTABLE; +INSERT INTO documents (id, data, embedding, title) +VALUES ('00000000-0000-0000-0000-000000000001', '{"k": 1}', [1.0, 0.0, 0.0], 'first'); +---- +rows affected: 1 + +# Refused whether or not the statement would reach a row: the table says so, not the data. +UPDATE documents SET title = 'changed' WHERE id = '00000000-0000-0000-0000-000000000001'; +---- +error: ImmutableTable + +UPDATE documents SET title = 'changed' WHERE id = '00000000-0000-0000-0000-000000000404'; +---- +error: ImmutableTable + +DELETE FROM documents WHERE id = '00000000-0000-0000-0000-000000000001'; +---- +error: ImmutableTable + +DELETE FROM documents WHERE title = 'no such title'; +---- +error: ImmutableTable + +DELETE FROM documents; +---- +error: ImmutableTable + +# An insert that may turn into an update is refused as well, on a key that is taken and on one +# that is free. +INSERT INTO documents (id, title) VALUES ('00000000-0000-0000-0000-000000000001', 'again') +ON CONFLICT (id) DO UPDATE SET title = 'again'; +---- +error: ImmutableTable + +INSERT INTO documents (id, title) VALUES ('00000000-0000-0000-0000-000000000009', 'ninth') +ON CONFLICT (id) DO UPDATE SET title = 'ninth'; +---- +error: ImmutableTable + +# A plain insert is what the table takes, and a taken key is refused as on any table. +INSERT INTO documents (id, data, title) VALUES ('00000000-0000-0000-0000-000000000002', '[2]', 'second'); +INSERT INTO documents (id, title) VALUES ('00000000-0000-0000-0000-000000000002', 'twice'); +---- +error: UniqueViolation + +SELECT id, data, title FROM documents ORDER BY id; +---- +id | data | title +00000000-0000-0000-0000-000000000001 | {"k":1} | first +00000000-0000-0000-0000-000000000002 | [2] | second + +# A table that is never changed has no rows to expire and no older versions to drop, so IMMUTABLE +# stands with neither clause. Each clause stands without it. +CREATE TABLE expiring_log (id INTEGER PRIMARY KEY, note TEXT) IMMUTABLE RETAIN 1 DAYS; +---- +error: SchemaInvalid + +CREATE TABLE current_log (id INTEGER PRIMARY KEY, note TEXT) IMMUTABLE HISTORY CURRENT ONLY SYNC OFF; +---- +error: SchemaInvalid + +CREATE TABLE expiring (id INTEGER PRIMARY KEY, note TEXT) RETAIN 1 DAYS; +CREATE TABLE current (id INTEGER PRIMARY KEY, note TEXT) HISTORY CURRENT ONLY SYNC OFF; +SELECT id FROM expiring_log; +---- +error: TableNotFound + +SELECT id FROM current_log; +---- +error: TableNotFound + +# IMMUTABLE on two columns of a table whose rows may otherwise change. +CREATE TABLE tasks ( + id INTEGER PRIMARY KEY, + task_type TEXT NOT NULL IMMUTABLE, + description TEXT NOT NULL IMMUTABLE, + confidence REAL, + status TEXT NOT NULL DEFAULT 'active' +); +INSERT INTO tasks (id, task_type, description) VALUES (1, 'sql-migration', 'migrate the store'); + +UPDATE tasks SET task_type = 'other' WHERE id = 1; +---- +error: ImmutableColumn + +UPDATE tasks SET description = 'migrate the store' WHERE id = 1; +---- +error: ImmutableColumn + +UPDATE tasks SET task_type = 'other' WHERE id = 404; +---- +error: ImmutableColumn + +# One assignment to such a column refuses the whole statement: status is not changed either. +UPDATE tasks SET status = 'archived', description = 'rewritten' WHERE id = 1; +---- +error: ImmutableColumn + +INSERT INTO tasks (id, task_type, description) VALUES (1, 'sql-migration', 'second try') +ON CONFLICT (id) DO UPDATE SET description = 'second try'; +---- +error: ImmutableColumn + +SELECT id, task_type, description, confidence, status FROM tasks ORDER BY id; +---- +id | task_type | description | confidence | status +1 | sql-migration | migrate the store | NULL | active + +# The other columns change, by an UPDATE and by an insert that meets the row. +UPDATE tasks SET confidence = 0.5 WHERE id = 1; +---- +rows affected: 1 + +INSERT INTO tasks (id, task_type, description) VALUES (1, 'sql-migration', 'migrate the store') +ON CONFLICT (id) DO UPDATE SET confidence = 0.75; +---- +rows affected: 1 + +# A record that turned out wrong is corrected by a second row, and the first is marked: both +# stay, and both are found. +INSERT INTO tasks (id, task_type, description) VALUES (2, 'sql-migration', 'migrate the store (rev 2)'); +UPDATE tasks SET status = 'superseded' WHERE id = 1; +---- +rows affected: 1 + +SELECT id, task_type, description, confidence, status FROM tasks ORDER BY id; +---- +id | task_type | description | confidence | status +1 | sql-migration | migrate the store | 0.75 | superseded +2 | sql-migration | migrate the store (rev 2) | NULL | active + +SELECT id FROM tasks WHERE status = 'superseded'; +---- +id +1 + +# The column cannot be taken out of the table or given another name, with CASCADE or without. +ALTER TABLE tasks DROP COLUMN task_type; +---- +error: ImmutableColumn + +ALTER TABLE tasks DROP COLUMN task_type CASCADE; +---- +error: ImmutableColumn + +ALTER TABLE tasks RENAME COLUMN description TO summary; +---- +error: ImmutableColumn + +SELECT task_type, description FROM tasks WHERE id = 2; +---- +task_type | description +sql-migration | migrate the store (rev 2) + +# A column that is not IMMUTABLE goes or is renamed, and a row of the table may be deleted. +ALTER TABLE tasks RENAME COLUMN confidence TO certainty; +ALTER TABLE tasks DROP COLUMN certainty; +DELETE FROM tasks WHERE id = 1; +---- +rows affected: 1 + +SELECT id, task_type, description, status FROM tasks ORDER BY id; +---- +id | task_type | description | status +2 | sql-migration | migrate the store (rev 2) | active diff --git a/v2/contextdb/tests/sql/insert.sql b/v2/contextdb/tests/sql/insert.sql index 5cb686b3..11b59618 100644 --- a/v2/contextdb/tests/sql/insert.sql +++ b/v2/contextdb/tests/sql/insert.sql @@ -59,3 +59,13 @@ id 3 4 5 + +# A column named twice as a target is refused, as Postgres refuses it, and nothing is written. +INSERT INTO items (id, name, name) VALUES (7, 'seven', 'again'); +---- +error: PlanError + +SELECT COUNT(*) FROM items WHERE id = 7; +---- +COUNT +0 diff --git a/v2/contextdb/tests/sql/key_length.sql b/v2/contextdb/tests/sql/key_length.sql new file mode 100644 index 00000000..e7a0e68d --- /dev/null +++ b/v2/contextdb/tests/sql/key_length.sql @@ -0,0 +1,207 @@ +# How long a key may be. The values that identify a row or a group are kept whole and may take +# 2,704 bytes as the store writes them: a TEXT value takes its bytes and three more, an INTEGER +# nine. One byte more is refused by name and nothing is written. A value in an index has no such +# limit: a row is never refused because a value it indexes is long. + +# A key of one TEXT column: 2,701 bytes are 2,704 as written, and 2,702 are one too many. +CREATE TABLE names (name TEXT PRIMARY KEY, n INTEGER); +INSERT INTO names (name, n) VALUES ('kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk', 1); +---- +rows affected: 1 + +INSERT INTO names (name, n) VALUES ('kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk', 2); +---- +error: KeyTooLong + +INSERT INTO names (name, n) VALUES ('short', 3); +SELECT n FROM names ORDER BY n; +---- +n +1 +3 + +# The longest key is found by its value, and a value longer than any key can be finds nothing +# and is no refusal. +SELECT n FROM names WHERE name = 'kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk'; +---- +n +1 + +SELECT n FROM names WHERE name = 'kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk'; +---- +n + +SELECT n FROM names WHERE name = 'kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk'; +---- +n + +# An UPDATE that would make the key too long is refused and leaves the row. +UPDATE names SET name = 'ssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss' WHERE n = 3; +---- +error: KeyTooLong + +SELECT name FROM names WHERE n = 3; +---- +name +short + +# A key of two columns is measured as a whole: a TEXT of 2,692 bytes and an INTEGER are 2,704. +CREATE TABLE pairs (label TEXT NOT NULL, seq INTEGER NOT NULL, n INTEGER, PRIMARY KEY (label, seq)); +INSERT INTO pairs (label, seq, n) VALUES ('pppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppp', 1, 1); +---- +rows affected: 1 + +INSERT INTO pairs (label, seq, n) VALUES ('ppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppp', 1, 2); +---- +error: KeyTooLong + +SELECT n FROM pairs ORDER BY n; +---- +n +1 + +# The values of a kept count name a group and are held to the same limit. +CREATE TABLE tagged (id INTEGER PRIMARY KEY, tag TEXT NOT NULL); +CREATE INDEX tagged_tag ON tagged (tag); +ALTER TABLE tagged SET MAINTAIN COUNT (tag); +INSERT INTO tagged (id, tag) VALUES (1, 'ttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttt'); +---- +rows affected: 1 + +INSERT INTO tagged (id, tag) VALUES (2, 'tttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttt'); +---- +error: KeyTooLong + +UPDATE tagged SET tag = 'uuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuu' WHERE id = 1; +---- +error: KeyTooLong + +SELECT COUNT(*) AS n FROM tagged WHERE tag = 'ttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttt'; +---- +n +1 + +SELECT id FROM tagged ORDER BY id; +---- +id +1 + +# The values of a PARTITION_KEY name a partition and are held to it as well, for a row that has +# a vector. +CREATE TABLE shapes (id INTEGER PRIMARY KEY, family TEXT NOT NULL, v VECTOR(2) PARTITION_KEY (family)); +INSERT INTO shapes (id, family, v) VALUES (1, 'fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff', [1.0, 0.0]); +---- +rows affected: 1 + +INSERT INTO shapes (id, family, v) VALUES (2, 'ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff', [0.0, 1.0]); +---- +error: KeyTooLong + +SELECT id FROM shapes ORDER BY id; +---- +id +1 + +# The same values in a plain index, in a UNIQUE rule and beside a key are stored: 2,702 bytes, +# and 5,000. +CREATE TABLE notes (id INTEGER PRIMARY KEY, body TEXT, title TEXT UNIQUE); +CREATE INDEX notes_body ON notes (body); +INSERT INTO notes (id, body, title) VALUES (1, 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', 'cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc'); +---- +rows affected: 1 + +INSERT INTO notes (id, body, title) VALUES (2, 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', 'cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc'); +---- +rows affected: 1 + +UPDATE notes SET body = 'dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd' WHERE id = 1; +---- +rows affected: 1 + +SELECT id FROM notes WHERE body = 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'; +---- +id +2 + +SELECT id FROM notes WHERE title = 'cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc'; +---- +id +1 + +# The UNIQUE rule still holds over the long value. +INSERT INTO notes (id, body, title) VALUES (3, 'short', 'cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc'); +---- +error: UniqueViolation + +SELECT id FROM notes ORDER BY id; +---- +id +1 +2 + +# Texts of 300 bytes, between the 256 an index entry keeps and the key limit, that differ only at +# byte 280. Each is found by its own value, a UNIQUE rule takes all six and refuses a repeat, +# a range and an ORDER BY give them in the order of their values, and COUNT(*) counts the rows +# that hold one of them. Inserted out of order, so the order of the ids is not the order sought. +CREATE TABLE mid (id INTEGER PRIMARY KEY, t TEXT UNIQUE, u TEXT); +CREATE INDEX mid_u ON mid (u); +INSERT INTO mid (id, t, u) VALUES (1, 'ppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppcxxxxxxxxxxxxxxxxxxx', 'ppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppcxxxxxxxxxxxxxxxxxxx'), (2, 'ppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppaxxxxxxxxxxxxxxxxxxx', 'ppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppaxxxxxxxxxxxxxxxxxxx'), (3, 'ppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppfxxxxxxxxxxxxxxxxxxx', 'ppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppfxxxxxxxxxxxxxxxxxxx'), (4, 'ppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppbxxxxxxxxxxxxxxxxxxx', 'ppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppbxxxxxxxxxxxxxxxxxxx'), (5, 'ppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppexxxxxxxxxxxxxxxxxxx', 'ppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppexxxxxxxxxxxxxxxxxxx'), (6, 'ppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppdxxxxxxxxxxxxxxxxxxx', 'ppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppdxxxxxxxxxxxxxxxxxxx'); +---- +rows affected: 6 + +SELECT id FROM mid WHERE t = 'ppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppaxxxxxxxxxxxxxxxxxxx'; +---- +id +2 + +INSERT INTO mid (id, t, u) VALUES (8, 'ppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppdxxxxxxxxxxxxxxxxxxx', 'other'); +---- +error: UniqueViolation + +INSERT INTO mid (id, t, u) VALUES (7, 'p', 'ppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppaxxxxxxxxxxxxxxxxxxx'); +---- +rows affected: 1 + +SELECT COUNT(*) FROM mid WHERE u = 'ppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppaxxxxxxxxxxxxxxxxxxx'; +---- +COUNT +2 + +SELECT COUNT(*) FROM mid WHERE u = 'ppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppexxxxxxxxxxxxxxxxxxx'; +---- +COUNT +1 + +SELECT id FROM mid WHERE t > 'ppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppcxxxxxxxxxxxxxxxxxxx' ORDER BY id; +---- +id +3 +5 +6 + +SELECT id FROM mid WHERE t <= 'ppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppcxxxxxxxxxxxxxxxxxxx' ORDER BY id; +---- +id +1 +2 +4 +7 + +SELECT id FROM mid ORDER BY t LIMIT 7; +---- +id +7 +2 +4 +1 +6 +5 +3 + +SELECT id FROM mid ORDER BY t DESC LIMIT 4; +---- +id +3 +5 +6 +1 diff --git a/v2/contextdb/tests/sql/maintain_count.sql b/v2/contextdb/tests/sql/maintain_count.sql new file mode 100644 index 00000000..879d7a33 --- /dev/null +++ b/v2/contextdb/tests/sql/maintain_count.sql @@ -0,0 +1,242 @@ +# MAINTAIN COUNT: the lists of columns for which a table keeps the number of its rows under +# each set of values, so that COUNT(*) with those columns given is one stored number. A count +# is always the number of rows the same filter returns, kept or not. + +CREATE TABLE events ( + id INTEGER PRIMARY KEY, + ctx INTEGER NOT NULL, + conn TEXT NOT NULL, + note TEXT, + UNIQUE (ctx, id), + UNIQUE (ctx, conn, id) +) MAINTAIN COUNT (ctx), (ctx, conn); +INSERT INTO events (id, ctx, conn) +VALUES (1, 1, 'a'), (2, 1, 'a'), (3, 1, 'b'), (4, 2, 'a'), (5, 2, 'b'), (6, 2, 'b'); + +SELECT COUNT(*) AS n FROM events WHERE ctx = 1; +---- +n +3 + +SELECT COUNT(*) AS n FROM events WHERE ctx = 2 AND conn = 'b'; +---- +n +2 + +# Values no row has are a count of nothing. +SELECT COUNT(*) AS n FROM events WHERE ctx = 3; +---- +n +0 + +SELECT COUNT(*) AS n FROM events WHERE ctx = 1 AND conn = 'z'; +---- +n +0 + +# The counts follow an insert, an UPDATE that moves a row from one set of values to another, +# an UPDATE that touches no counted column, and a DELETE. +INSERT INTO events (id, ctx, conn) VALUES (7, 1, 'b'); +UPDATE events SET ctx = 1 WHERE id = 4; +UPDATE events SET note = 'seen' WHERE ctx = 2; +DELETE FROM events WHERE id = 1; +SELECT COUNT(*) AS n FROM events WHERE ctx = 1; +---- +n +4 + +SELECT COUNT(*) AS n FROM events WHERE ctx = 2; +---- +n +2 + +SELECT COUNT(*) AS n FROM events WHERE ctx = 1 AND conn = 'a'; +---- +n +2 + +SELECT COUNT(*) AS n FROM events WHERE ctx = 1 AND conn = 'b'; +---- +n +2 + +SELECT COUNT(*) AS n FROM events WHERE ctx = 2 AND conn = 'a'; +---- +n +0 + +# A refused statement counts nothing: the second row of this one has a key that is taken. +INSERT INTO events (id, ctx, conn) VALUES (8, 2, 'a'), (2, 2, 'a'); +---- +error: UniqueViolation + +SELECT COUNT(*) AS n FROM events WHERE ctx = 2 AND conn = 'a'; +---- +n +0 + +SELECT COUNT(*) AS n FROM events; +---- +n +6 + +# A table that declares no count answers the same statements with the same numbers. +CREATE TABLE plain (id INTEGER PRIMARY KEY, ctx INTEGER NOT NULL, conn TEXT NOT NULL); +CREATE INDEX plain_ctx_conn ON plain (ctx, conn); +INSERT INTO plain (id, ctx, conn) +VALUES (2, 1, 'a'), (3, 1, 'b'), (4, 1, 'a'), (5, 2, 'b'), (6, 2, 'b'), (7, 1, 'b'); +SELECT COUNT(*) AS n FROM plain WHERE ctx = 1; +---- +n +4 + +SELECT COUNT(*) AS n FROM plain WHERE ctx = 1 AND conn = 'b'; +---- +n +2 + +SELECT COUNT(*) AS n FROM plain WHERE ctx = 3; +---- +n +0 + +# ALTER TABLE ... SET MAINTAIN COUNT replaces the whole declared set, on a table that has rows: +# the counts it names are right at once. +ALTER TABLE plain SET MAINTAIN COUNT (ctx), (ctx, conn); +SELECT COUNT(*) AS n FROM plain WHERE ctx = 1; +---- +n +4 + +SELECT COUNT(*) AS n FROM plain WHERE ctx = 2 AND conn = 'b'; +---- +n +2 + +INSERT INTO plain (id, ctx, conn) VALUES (8, 2, 'b'); +DELETE FROM plain WHERE id = 2; +SELECT COUNT(*) AS n FROM plain WHERE ctx = 1; +---- +n +3 + +SELECT COUNT(*) AS n FROM plain WHERE ctx = 2 AND conn = 'b'; +---- +n +3 + +# ALL PREFIXES keeps a count for every beginning of every index, NONE says in words what a +# table that declares nothing does, and DEFAULT takes the declaration away. Under each, the +# same statements give the same numbers. +ALTER TABLE plain SET MAINTAIN COUNT ALL PREFIXES; +INSERT INTO plain (id, ctx, conn) VALUES (9, 1, 'a'); +SELECT COUNT(*) AS n FROM plain WHERE ctx = 1; +---- +n +4 + +SELECT COUNT(*) AS n FROM plain WHERE ctx = 1 AND conn = 'a'; +---- +n +2 + +ALTER TABLE plain SET MAINTAIN COUNT NONE; +INSERT INTO plain (id, ctx, conn) VALUES (10, 1, 'a'); +SELECT COUNT(*) AS n FROM plain WHERE ctx = 1 AND conn = 'a'; +---- +n +3 + +ALTER TABLE plain SET MAINTAIN COUNT (ctx); +ALTER TABLE plain SET MAINTAIN COUNT DEFAULT; +DELETE FROM plain WHERE id = 10; +SELECT COUNT(*) AS n FROM plain WHERE ctx = 1; +---- +n +4 + +# CREATE TABLE takes ALL PREFIXES and NONE as well. +CREATE TABLE every (id INTEGER PRIMARY KEY, a INTEGER, b INTEGER, UNIQUE (a, b, id)) MAINTAIN COUNT ALL PREFIXES; +CREATE TABLE silent (id INTEGER PRIMARY KEY, a INTEGER, UNIQUE (a, id)) MAINTAIN COUNT NONE; +INSERT INTO every (id, a, b) VALUES (1, 1, 1), (2, 1, 1), (3, 1, 2); +SELECT COUNT(*) AS n FROM every WHERE a = 1 AND b = 1; +---- +n +2 + +# A list is written with ASC or DESC on each column, as an index is, and is led by an index +# that runs the same way. +CREATE TABLE marks (id INTEGER PRIMARY KEY, day INTEGER NOT NULL, kind TEXT NOT NULL); +CREATE INDEX marks_day_kind ON marks (day DESC, kind ASC); +ALTER TABLE marks SET MAINTAIN COUNT (day DESC), (day DESC, kind ASC); +INSERT INTO marks (id, day, kind) VALUES (1, 5, 'x'), (2, 5, 'y'), (3, 6, 'x'); +SELECT COUNT(*) AS n FROM marks WHERE day = 5; +---- +n +2 + +ALTER TABLE marks SET MAINTAIN COUNT (day ASC); +---- +error: SchemaInvalid + +# What is refused when it is declared. A list no index begins with: nothing could read it. +CREATE TABLE no_index (id INTEGER PRIMARY KEY, a INTEGER) MAINTAIN COUNT (a); +---- +error: SchemaInvalid + +ALTER TABLE events SET MAINTAIN COUNT (conn); +---- +error: SchemaInvalid + +# A list a key or a UNIQUE rule already holds to one row at most. +CREATE TABLE by_key (id INTEGER PRIMARY KEY, a INTEGER) MAINTAIN COUNT (id); +---- +error: SchemaInvalid + +ALTER TABLE events SET MAINTAIN COUNT (ctx, id); +---- +error: SchemaInvalid + +# The same list twice, a column twice in one list, and a column the table does not have. +CREATE TABLE twice (id INTEGER PRIMARY KEY, a INTEGER, UNIQUE (a, id)) MAINTAIN COUNT (a), (a); +---- +error: SchemaInvalid + +CREATE TABLE repeated (id INTEGER PRIMARY KEY, a INTEGER, UNIQUE (a, id)) MAINTAIN COUNT (a, a); +---- +error: SchemaInvalid + +CREATE TABLE unknown (id INTEGER PRIMARY KEY, a INTEGER, UNIQUE (a, id)) MAINTAIN COUNT (nope); +---- +error: ColumnNotFound + +SELECT id FROM no_index; +---- +error: TableNotFound + +SELECT id FROM twice; +---- +error: TableNotFound + +# A refused ALTER left the table's counts as they were declared, and right. +INSERT INTO events (id, ctx, conn) VALUES (9, 2, 'a'); +SELECT COUNT(*) AS n FROM events WHERE ctx = 2 AND conn = 'a'; +---- +n +1 + +SELECT COUNT(*) AS n FROM events WHERE ctx = 2; +---- +n +3 + +# A column a kept count is declared by cannot be dropped, even with CASCADE, which drops the +# indexes that name it but not the rule that counts by it. +ALTER TABLE events DROP COLUMN conn CASCADE; +---- +error: SchemaInvalid + +SELECT COUNT(*) AS n FROM events WHERE ctx = 2 AND conn = 'a'; +---- +n +1 diff --git a/v2/contextdb/tests/sql/on_conflict.sql b/v2/contextdb/tests/sql/on_conflict.sql new file mode 100644 index 00000000..475b6579 --- /dev/null +++ b/v2/contextdb/tests/sql/on_conflict.sql @@ -0,0 +1,199 @@ +# INSERT ... ON CONFLICT (columns) DO UPDATE SET ...: a row whose key or unique values are free +# is inserted as written; a row that meets one already there changes that row by the SET list +# and inserts nothing. Either way the statement counts the row once. + +CREATE TABLE items ( + id INTEGER PRIMARY KEY, + code TEXT UNIQUE, + name TEXT NOT NULL, + qty INTEGER, + shelf TEXT, + slot INTEGER, + UNIQUE (shelf, slot) +); + +# A free key: the row goes in as written and the SET list is not used. +INSERT INTO items (id, code, name, qty, shelf, slot) VALUES (1, 'k1', 'bolt', 5, 'a', 1) +ON CONFLICT (id) DO UPDATE SET name = 'changed'; +---- +rows affected: 1 + +SELECT id, code, name, qty, shelf, slot FROM items ORDER BY id; +---- +id | code | name | qty | shelf | slot +1 | k1 | bolt | 5 | a | 1 + +# A taken key: only the columns of the SET list change, and nothing else of the new row is kept. +INSERT INTO items (id, code, name, qty, shelf, slot) VALUES (1, 'k9', 'nut', 99, 'z', 9) +ON CONFLICT (id) DO UPDATE SET name = 'bolt m4'; +---- +rows affected: 1 + +SELECT id, code, name, qty, shelf, slot FROM items ORDER BY id; +---- +id | code | name | qty | shelf | slot +1 | k1 | bolt m4 | 5 | a | 1 + +# A column in the SET list reads the row that is there. +INSERT INTO items (id, code, name, qty) VALUES (1, 'k1', 'bolt', 0) +ON CONFLICT (id) DO UPDATE SET qty = qty + 10, name = 'bolt m5'; +---- +rows affected: 1 + +SELECT id, name, qty FROM items ORDER BY id; +---- +id | name | qty +1 | bolt m5 | 15 + +# By a UNIQUE column: the row with that code is changed, whatever key the new row carried. +INSERT INTO items (id, code, name, qty) VALUES (2, 'k1', 'other', 0) +ON CONFLICT (code) DO UPDATE SET qty = 20; +---- +rows affected: 1 + +# By a UNIQUE rule of two columns. +INSERT INTO items (id, code, name, qty, shelf, slot) VALUES (3, 'k3', 'washer', 0, 'a', 1) +ON CONFLICT (shelf, slot) DO UPDATE SET qty = 30; +---- +rows affected: 1 + +SELECT id, code, name, qty, shelf, slot FROM items ORDER BY id; +---- +id | code | name | qty | shelf | slot +1 | k1 | bolt m5 | 30 | a | 1 + +# Several rows in one statement: each is inserted or changes the row it meets. +INSERT INTO items (id, code, name, qty, shelf, slot) +VALUES (2, 'k2', 'nut', 1, 'a', 2), (1, 'k7', 'gone', 7, 'b', 7), (3, 'k3', 'washer', 3, 'b', 1) +ON CONFLICT (id) DO UPDATE SET qty = 0; +---- +rows affected: 3 + +SELECT id, code, name, qty, shelf, slot FROM items ORDER BY id; +---- +id | code | name | qty | shelf | slot +1 | k1 | bolt m5 | 0 | a | 1 +2 | k2 | nut | 1 | a | 2 +3 | k3 | washer | 3 | b | 1 + +# The clause answers a meeting on the columns it names. A row that is free there and meets +# another rule is refused as any insert is. +INSERT INTO items (id, code, name) VALUES (4, 'k2', 'pin') +ON CONFLICT (id) DO UPDATE SET name = 'pin'; +---- +error: UniqueViolation + +# The change it makes is held to the table's rules as an UPDATE is: NOT NULL, and the other +# UNIQUE rules. +INSERT INTO items (id, code, name) VALUES (2, 'k2', 'nut') +ON CONFLICT (id) DO UPDATE SET name = NULL; +---- +error: ColumnNotNullable + +INSERT INTO items (id, code, name) VALUES (2, 'k2', 'nut') +ON CONFLICT (id) DO UPDATE SET code = 'k3'; +---- +error: UniqueViolation + +SELECT id, code, name, qty, shelf, slot FROM items ORDER BY id; +---- +id | code | name | qty | shelf | slot +1 | k1 | bolt m5 | 0 | a | 1 +2 | k2 | nut | 1 | a | 2 +3 | k3 | washer | 3 | b | 1 + +# A NULL in a UNIQUE column meets nothing, so the row is inserted each time. +INSERT INTO items (id, code, name) VALUES (5, NULL, 'loose') +ON CONFLICT (code) DO UPDATE SET name = 'met'; +INSERT INTO items (id, code, name) VALUES (6, NULL, 'loose') +ON CONFLICT (code) DO UPDATE SET name = 'met'; +SELECT id, name FROM items WHERE id >= 5 ORDER BY id; +---- +id | name +5 | loose +6 | loose + +# A key of two columns. +CREATE TABLE windows ( + machine TEXT NOT NULL, + slot INTEGER NOT NULL, + value INTEGER NOT NULL, + PRIMARY KEY (machine, slot) +); +INSERT INTO windows (machine, slot, value) VALUES ('m1', 1, 10), ('m1', 2, 20); +INSERT INTO windows (machine, slot, value) VALUES ('m1', 2, 99), ('m2', 2, 22) +ON CONFLICT (machine, slot) DO UPDATE SET value = value + 1; +---- +rows affected: 2 + +SELECT machine, slot, value FROM windows ORDER BY machine, slot; +---- +machine | slot | value +m1 | 1 | 10 +m1 | 2 | 21 +m2 | 2 | 22 + +# Without the clause, a taken key is refused. +INSERT INTO windows (machine, slot, value) VALUES ('m1', 2, 99); +---- +error: UniqueViolation + +# A column named twice in the SET list is refused as well. +INSERT INTO items (id, code, name, qty) VALUES (1, 'k1', 'bolt', 0) +ON CONFLICT (id) DO UPDATE SET qty = 1, qty = 2; +---- +error: PlanError + +# The columns must be the key or a whole UNIQUE rule: an index is not a rule, a part of a rule +# is not the rule, and neither is a rule with a column more. The refusal comes when the +# statement is planned, though the row to insert meets nothing. +CREATE TABLE stock (id INTEGER PRIMARY KEY, code TEXT UNIQUE, name TEXT NOT NULL, shelf TEXT, slot INTEGER, UNIQUE (shelf, slot)); +CREATE INDEX stock_name ON stock (name); +INSERT INTO stock (id, code, name, shelf, slot) VALUES (1, 'k1', 'bolt', 'a', 1); +INSERT INTO stock (id, code, name, shelf, slot) VALUES (2, 'k2', 'nut', 'b', 2) ON CONFLICT (name) DO UPDATE SET name = 'changed'; +---- +error: PlanError + +INSERT INTO stock (id, code, name, shelf, slot) VALUES (2, 'k2', 'nut', 'b', 2) ON CONFLICT (shelf) DO UPDATE SET name = 'changed'; +---- +error: PlanError + +INSERT INTO stock (id, code, name, shelf, slot) VALUES (2, 'k2', 'nut', 'b', 2) ON CONFLICT (slot) DO UPDATE SET name = 'changed'; +---- +error: PlanError + +INSERT INTO stock (id, code, name, shelf, slot) VALUES (2, 'k2', 'nut', 'b', 2) ON CONFLICT (id, name) DO UPDATE SET name = 'changed'; +---- +error: PlanError + +INSERT INTO stock (id, code, name, shelf, slot) VALUES (2, 'k2', 'nut', 'b', 2) ON CONFLICT (shelf, slot, name) DO UPDATE SET name = 'changed'; +---- +error: PlanError + +SELECT id, name FROM stock ORDER BY id; +---- +id | name +1 | bolt + +# The key and each UNIQUE rule are accepted. +INSERT INTO stock (id, code, name, shelf, slot) VALUES (2, 'k2', 'nut', 'b', 2) ON CONFLICT (id) DO UPDATE SET name = 'changed'; +---- +rows affected: 1 + +INSERT INTO stock (id, code, name, shelf, slot) VALUES (3, 'k3', 'nut', 'b', 3) ON CONFLICT (code) DO UPDATE SET name = 'changed'; +---- +rows affected: 1 + +INSERT INTO stock (id, code, name, shelf, slot) VALUES (4, 'k4', 'nut', 'b', 4) ON CONFLICT (shelf, slot) DO UPDATE SET name = 'changed'; +---- +rows affected: 1 + +SELECT COUNT(*) AS n FROM stock WHERE name = 'changed'; +---- +n +0 + +SELECT COUNT(*) AS n FROM stock; +---- +n +4 diff --git a/v2/contextdb/tests/sql/order_limit.sql b/v2/contextdb/tests/sql/order_limit.sql index ec0fc7c6..ecd10495 100644 --- a/v2/contextdb/tests/sql/order_limit.sql +++ b/v2/contextdb/tests/sql/order_limit.sql @@ -120,3 +120,18 @@ id | points 5 | 20 1 | 30 2 | NULL + +# With no ORDER BY, a read through an index comes in the index's order: its values, then the +# row's key, whatever order the rows were written in. +CREATE TABLE unordered (id INTEGER PRIMARY KEY, grp INTEGER NOT NULL); +CREATE INDEX unordered_by_grp ON unordered (grp); +INSERT INTO unordered (id, grp) VALUES (9, 2), (3, 1), (7, 2), (1, 1), (5, 2), (4, 1); +SELECT id, grp FROM unordered WHERE grp >= 1; +---- +id | grp +1 | 1 +3 | 1 +4 | 1 +5 | 2 +7 | 2 +9 | 2 diff --git a/v2/contextdb/tests/sql/propagate.sql b/v2/contextdb/tests/sql/propagate.sql new file mode 100644 index 00000000..326baba2 --- /dev/null +++ b/v2/contextdb/tests/sql/propagate.sql @@ -0,0 +1,350 @@ +# PROPAGATE: a row follows the transition of a row it references, or of a row an edge joins it +# to, inside the statement that makes the transition. A row that cannot make the move is left +# as it is, unless the declaration says ABORT ON FAILURE: then the statement is refused whole. + +CREATE TABLE projects (id INTEGER PRIMARY KEY, status TEXT NOT NULL) +STATE MACHINE (status: active -> [archived]); +CREATE TABLE tasks ( + id INTEGER PRIMARY KEY, + status TEXT NOT NULL, + project_id INTEGER REFERENCES projects(id) ON STATE archived PROPAGATE SET invalidated +) STATE MACHINE (status: active -> [invalidated, superseded]); +CREATE TABLE steps ( + id INTEGER PRIMARY KEY, + status TEXT NOT NULL, + task_id INTEGER REFERENCES tasks(id) ON STATE invalidated PROPAGATE SET invalidated +) STATE MACHINE (status: active -> [invalidated]); +INSERT INTO projects (id, status) VALUES (1, 'active'), (2, 'active'); +INSERT INTO tasks (id, status, project_id) +VALUES (1, 'active', 1), (2, 'superseded', 1), (3, 'active', 2), (4, 'active', NULL); +INSERT INTO steps (id, status, task_id) +VALUES (1, 'active', 1), (2, 'active', 1), (3, 'active', 2), (4, 'active', 3); + +# The statement says how many rows it named, not how many followed. +UPDATE projects SET status = 'archived' WHERE id = 1; +---- +rows affected: 1 + +# Task 1 followed its project. Task 2 was superseded, a state that lists no next state, and is +# left as it is. Tasks 3 and 4 name another project and none. +SELECT id, status FROM tasks ORDER BY id; +---- +id | status +1 | invalidated +2 | superseded +3 | active +4 | active + +# The steps of task 1 followed task 1: a row that moved by following is followed in turn. The +# step of task 2 did not move, because task 2 did not. +SELECT id, status FROM steps ORDER BY id; +---- +id | status +1 | invalidated +2 | invalidated +3 | active +4 | active + +# A task that is moved by its own statement is followed the same way. +UPDATE tasks SET status = 'invalidated' WHERE id = 3; +---- +rows affected: 1 + +SELECT id, status FROM steps ORDER BY id; +---- +id | status +1 | invalidated +2 | invalidated +3 | active +4 | invalidated + +# A transition to a state the reference does not name moves nothing. +UPDATE tasks SET status = 'superseded' WHERE id = 4; +INSERT INTO steps (id, status, task_id) VALUES (5, 'active', 4); +SELECT id, status FROM steps WHERE id = 5; +---- +id | status +5 | active + +# ABORT ON FAILURE: one row that cannot follow refuses the statement, and the row that was to +# make the transition has not made it. +CREATE TABLE strict_tasks ( + id INTEGER PRIMARY KEY, + status TEXT NOT NULL, + project_id INTEGER REFERENCES projects(id) ON STATE archived PROPAGATE SET invalidated ABORT ON FAILURE +) STATE MACHINE (status: active -> [invalidated, superseded]); +INSERT INTO projects (id, status) VALUES (3, 'active'), (4, 'active'); +INSERT INTO strict_tasks (id, status, project_id) +VALUES (1, 'active', 3), (2, 'superseded', 3), (3, 'active', 4); +UPDATE projects SET status = 'archived' WHERE id = 3; +---- +error: PropagationAborted + +SELECT id, status FROM projects WHERE id >= 3 ORDER BY id; +---- +id | status +3 | active +4 | active + +SELECT id, status FROM strict_tasks ORDER BY id; +---- +id | status +1 | active +2 | superseded +3 | active + +# Where every row can follow, the statement stands. +UPDATE projects SET status = 'archived' WHERE id = 4; +---- +rows affected: 1 + +SELECT id, status FROM strict_tasks ORDER BY id; +---- +id | status +1 | active +2 | superseded +3 | invalidated + +# PROPAGATE ON EDGE: the rows of a table follow each other along the edges of one type. +# INCOMING follows an edge from its target to its source: here, a claim that cites an +# invalidated claim becomes invalidated, and so does a claim that cites that one. +CREATE TABLE edges ( + id INTEGER PRIMARY KEY, + source_id UUID NOT NULL, + target_id UUID NOT NULL, + edge_type TEXT NOT NULL +); +CREATE TABLE claims (id UUID PRIMARY KEY, status TEXT NOT NULL) +STATE MACHINE (status: active -> [invalidated, superseded]) +PROPAGATE ON EDGE CITES INCOMING STATE invalidated SET invalidated; +INSERT INTO claims (id, status) +VALUES ('00000000-0000-0000-0000-000000000001', 'active'), + ('00000000-0000-0000-0000-000000000002', 'active'), + ('00000000-0000-0000-0000-000000000003', 'active'), + ('00000000-0000-0000-0000-000000000004', 'active'), + ('00000000-0000-0000-0000-000000000005', 'active'); +# 2 cites 1, 3 cites 2, 1 cites 4, and 5 is joined to 1 by an edge of another type. +INSERT INTO edges (id, source_id, target_id, edge_type) +VALUES (1, '00000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000001', 'CITES'), + (2, '00000000-0000-0000-0000-000000000003', '00000000-0000-0000-0000-000000000002', 'CITES'), + (3, '00000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000004', 'CITES'), + (4, '00000000-0000-0000-0000-000000000005', '00000000-0000-0000-0000-000000000001', 'MENTIONS'); +UPDATE claims SET status = 'invalidated' WHERE id = '00000000-0000-0000-0000-000000000001'; +---- +rows affected: 1 + +# Claim 4 is cited by claim 1, which is the other way along the edge, and stays. +SELECT id, status FROM claims ORDER BY id; +---- +id | status +00000000-0000-0000-0000-000000000001 | invalidated +00000000-0000-0000-0000-000000000002 | invalidated +00000000-0000-0000-0000-000000000003 | invalidated +00000000-0000-0000-0000-000000000004 | active +00000000-0000-0000-0000-000000000005 | active + +# OUTGOING follows an edge from its source to its target, and MAX DEPTH ends the walk: 11 +# feeds 12, 12 feeds 13 and 13 feeds 14. With MAX DEPTH 1 the pump one edge away stops and the +# pump three edges away runs on. +CREATE TABLE pumps (id UUID PRIMARY KEY, status TEXT NOT NULL) +STATE MACHINE (status: running -> [stopped]) +PROPAGATE ON EDGE FEEDS OUTGOING STATE stopped SET stopped MAX DEPTH 1; +INSERT INTO pumps (id, status) +VALUES ('00000000-0000-0000-0000-000000000011', 'running'), + ('00000000-0000-0000-0000-000000000012', 'running'), + ('00000000-0000-0000-0000-000000000013', 'running'), + ('00000000-0000-0000-0000-000000000014', 'running'); +INSERT INTO edges (id, source_id, target_id, edge_type) +VALUES (11, '00000000-0000-0000-0000-000000000011', '00000000-0000-0000-0000-000000000012', 'FEEDS'), + (12, '00000000-0000-0000-0000-000000000012', '00000000-0000-0000-0000-000000000013', 'FEEDS'), + (13, '00000000-0000-0000-0000-000000000013', '00000000-0000-0000-0000-000000000014', 'FEEDS'); +UPDATE pumps SET status = 'stopped' WHERE id = '00000000-0000-0000-0000-000000000011'; +SELECT id, status FROM pumps WHERE id <> '00000000-0000-0000-0000-000000000013' ORDER BY id; +---- +id | status +00000000-0000-0000-0000-000000000011 | stopped +00000000-0000-0000-0000-000000000012 | stopped +00000000-0000-0000-0000-000000000014 | running + +# BOTH follows an edge either way: 21 is tied to 22, and 23 is tied to 21. +CREATE TABLE peers (id UUID PRIMARY KEY, status TEXT NOT NULL) +STATE MACHINE (status: up -> [down]) +PROPAGATE ON EDGE TIED BOTH STATE down SET down; +INSERT INTO peers (id, status) +VALUES ('00000000-0000-0000-0000-000000000021', 'up'), + ('00000000-0000-0000-0000-000000000022', 'up'), + ('00000000-0000-0000-0000-000000000023', 'up'), + ('00000000-0000-0000-0000-000000000024', 'up'); +INSERT INTO edges (id, source_id, target_id, edge_type) +VALUES (21, '00000000-0000-0000-0000-000000000021', '00000000-0000-0000-0000-000000000022', 'TIED'), + (22, '00000000-0000-0000-0000-000000000023', '00000000-0000-0000-0000-000000000021', 'TIED'); +UPDATE peers SET status = 'down' WHERE id = '00000000-0000-0000-0000-000000000021'; +SELECT id, status FROM peers ORDER BY id; +---- +id | status +00000000-0000-0000-0000-000000000021 | down +00000000-0000-0000-0000-000000000022 | down +00000000-0000-0000-0000-000000000023 | down +00000000-0000-0000-0000-000000000024 | up + +# ABORT ON FAILURE along an edge: 32 holds 31 and is already released, a state that lists no +# next state, so 31 cannot be revoked. 33 holds nothing and can. +CREATE TABLE grants (id UUID PRIMARY KEY, status TEXT NOT NULL) +STATE MACHINE (status: live -> [revoked, released]) +PROPAGATE ON EDGE HOLDS INCOMING STATE revoked SET revoked ABORT ON FAILURE; +INSERT INTO grants (id, status) +VALUES ('00000000-0000-0000-0000-000000000031', 'live'), + ('00000000-0000-0000-0000-000000000032', 'released'), + ('00000000-0000-0000-0000-000000000033', 'live'); +INSERT INTO edges (id, source_id, target_id, edge_type) +VALUES (31, '00000000-0000-0000-0000-000000000032', '00000000-0000-0000-0000-000000000031', 'HOLDS'); +UPDATE grants SET status = 'revoked' WHERE id = '00000000-0000-0000-0000-000000000031'; +---- +error: PropagationAborted + +UPDATE grants SET status = 'revoked' WHERE id = '00000000-0000-0000-0000-000000000033'; +---- +rows affected: 1 + +SELECT id, status FROM grants ORDER BY id; +---- +id | status +00000000-0000-0000-0000-000000000031 | live +00000000-0000-0000-0000-000000000032 | released +00000000-0000-0000-0000-000000000033 | revoked + +# A ring of edges ends: 101 cites 102, 102 cites 103, 103 cites 101, and 104 is joined to +# nothing. Each claim follows once and the statement ends. An edge whose source is no claim +# is passed over. +CREATE TABLE ring_claims (id UUID PRIMARY KEY, status TEXT NOT NULL) +STATE MACHINE (status: active -> [invalidated]) +PROPAGATE ON EDGE CITES INCOMING STATE invalidated SET invalidated; +INSERT INTO ring_claims (id, status) +VALUES ('00000000-0000-0000-0000-000000000101', 'active'), ('00000000-0000-0000-0000-000000000102', 'active'), ('00000000-0000-0000-0000-000000000103', 'active'), ('00000000-0000-0000-0000-000000000104', 'active'); +INSERT INTO edges (id, source_id, target_id, edge_type) +VALUES (101, '00000000-0000-0000-0000-000000000101', '00000000-0000-0000-0000-000000000102', 'CITES'), (102, '00000000-0000-0000-0000-000000000102', '00000000-0000-0000-0000-000000000103', 'CITES'), (103, '00000000-0000-0000-0000-000000000103', '00000000-0000-0000-0000-000000000101', 'CITES'), (104, '00000000-0000-0000-0000-000000000999', '00000000-0000-0000-0000-000000000101', 'CITES'); +UPDATE ring_claims SET status = 'invalidated' WHERE id = '00000000-0000-0000-0000-000000000101'; +---- +rows affected: 1 + +SELECT id, status FROM ring_claims ORDER BY id; +---- +id | status +00000000-0000-0000-0000-000000000101 | invalidated +00000000-0000-0000-0000-000000000102 | invalidated +00000000-0000-0000-0000-000000000103 | invalidated +00000000-0000-0000-0000-000000000104 | active + +# A ring under ABORT ON FAILURE ends the same way: the row the ring comes back to is already in +# the state asked for, so it is left alone, and not refused for lacking a move out of it. +CREATE TABLE ring_firm (id UUID PRIMARY KEY, status TEXT NOT NULL) +STATE MACHINE (status: active -> [invalidated]) +PROPAGATE ON EDGE CITES_FIRM INCOMING STATE invalidated SET invalidated ABORT ON FAILURE; +INSERT INTO ring_firm (id, status) VALUES ('00000000-0000-0000-0000-000000000111', 'active'), ('00000000-0000-0000-0000-000000000112', 'active'), ('00000000-0000-0000-0000-000000000113', 'active'); +INSERT INTO edges (id, source_id, target_id, edge_type) +VALUES (111, '00000000-0000-0000-0000-000000000111', '00000000-0000-0000-0000-000000000112', 'CITES_FIRM'), (112, '00000000-0000-0000-0000-000000000112', '00000000-0000-0000-0000-000000000113', 'CITES_FIRM'), (113, '00000000-0000-0000-0000-000000000113', '00000000-0000-0000-0000-000000000111', 'CITES_FIRM'); +UPDATE ring_firm SET status = 'invalidated' WHERE id = '00000000-0000-0000-0000-000000000111'; +---- +rows affected: 1 + +SELECT id, status FROM ring_firm ORDER BY id; +---- +id | status +00000000-0000-0000-0000-000000000111 | invalidated +00000000-0000-0000-0000-000000000112 | invalidated +00000000-0000-0000-0000-000000000113 | invalidated + +# With no MAX DEPTH a change is followed ten edges and the eleventh row stays: 201 feeds 202, +# and so on to 20c, twelve nodes in a chain. +CREATE TABLE chain_nodes (id UUID PRIMARY KEY, status TEXT NOT NULL) +STATE MACHINE (status: running -> [stopped]) +PROPAGATE ON EDGE FEEDS OUTGOING STATE stopped SET stopped; +INSERT INTO chain_nodes (id, status) +VALUES ('00000000-0000-0000-0000-000000000201', 'running'), ('00000000-0000-0000-0000-000000000202', 'running'), ('00000000-0000-0000-0000-000000000203', 'running'), ('00000000-0000-0000-0000-000000000204', 'running'), ('00000000-0000-0000-0000-000000000205', 'running'), ('00000000-0000-0000-0000-000000000206', 'running'), ('00000000-0000-0000-0000-000000000207', 'running'), ('00000000-0000-0000-0000-000000000208', 'running'), ('00000000-0000-0000-0000-000000000209', 'running'), ('00000000-0000-0000-0000-00000000020a', 'running'), ('00000000-0000-0000-0000-00000000020b', 'running'), ('00000000-0000-0000-0000-00000000020c', 'running'); +INSERT INTO edges (id, source_id, target_id, edge_type) +VALUES (301, '00000000-0000-0000-0000-000000000201', '00000000-0000-0000-0000-000000000202', 'FEEDS'), (302, '00000000-0000-0000-0000-000000000202', '00000000-0000-0000-0000-000000000203', 'FEEDS'), (303, '00000000-0000-0000-0000-000000000203', '00000000-0000-0000-0000-000000000204', 'FEEDS'), (304, '00000000-0000-0000-0000-000000000204', '00000000-0000-0000-0000-000000000205', 'FEEDS'), (305, '00000000-0000-0000-0000-000000000205', '00000000-0000-0000-0000-000000000206', 'FEEDS'), (306, '00000000-0000-0000-0000-000000000206', '00000000-0000-0000-0000-000000000207', 'FEEDS'), (307, '00000000-0000-0000-0000-000000000207', '00000000-0000-0000-0000-000000000208', 'FEEDS'), (308, '00000000-0000-0000-0000-000000000208', '00000000-0000-0000-0000-000000000209', 'FEEDS'), (309, '00000000-0000-0000-0000-000000000209', '00000000-0000-0000-0000-00000000020a', 'FEEDS'), (310, '00000000-0000-0000-0000-00000000020a', '00000000-0000-0000-0000-00000000020b', 'FEEDS'), (311, '00000000-0000-0000-0000-00000000020b', '00000000-0000-0000-0000-00000000020c', 'FEEDS'); +UPDATE chain_nodes SET status = 'stopped' WHERE id = '00000000-0000-0000-0000-000000000201'; +---- +rows affected: 1 + +SELECT id, status FROM chain_nodes ORDER BY id; +---- +id | status +00000000-0000-0000-0000-000000000201 | stopped +00000000-0000-0000-0000-000000000202 | stopped +00000000-0000-0000-0000-000000000203 | stopped +00000000-0000-0000-0000-000000000204 | stopped +00000000-0000-0000-0000-000000000205 | stopped +00000000-0000-0000-0000-000000000206 | stopped +00000000-0000-0000-0000-000000000207 | stopped +00000000-0000-0000-0000-000000000208 | stopped +00000000-0000-0000-0000-000000000209 | stopped +00000000-0000-0000-0000-00000000020a | stopped +00000000-0000-0000-0000-00000000020b | stopped +00000000-0000-0000-0000-00000000020c | running + +# An upsert that changes a state is followed as an UPDATE is. +CREATE TABLE up_projects (id INTEGER PRIMARY KEY, status TEXT NOT NULL) +STATE MACHINE (status: active -> [archived]); +CREATE TABLE up_tasks ( + id INTEGER PRIMARY KEY, + status TEXT NOT NULL, + project_id INTEGER REFERENCES up_projects(id) ON STATE archived PROPAGATE SET invalidated +) STATE MACHINE (status: active -> [invalidated]); +INSERT INTO up_projects (id, status) VALUES (1, 'active'), (2, 'active'); +INSERT INTO up_tasks (id, status, project_id) +VALUES (1, 'active', 1), (2, 'active', 2), (3, 'active', 1); +INSERT INTO up_projects (id, status) VALUES (1, 'active') +ON CONFLICT (id) DO UPDATE SET status = 'archived'; +---- +rows affected: 1 + +SELECT id, status FROM up_tasks ORDER BY id; +---- +id | status +1 | invalidated +2 | active +3 | invalidated + +# A follower whose state column is IMMUTABLE cannot follow. Without ABORT ON FAILURE the +# statement stands and the follower is left as it is; with it the statement is refused whole. +CREATE TABLE gate_projects (id INTEGER PRIMARY KEY, status TEXT NOT NULL) +STATE MACHINE (status: active -> [archived]); +CREATE TABLE lax_tasks ( + id INTEGER PRIMARY KEY, + status TEXT NOT NULL IMMUTABLE, + project_id INTEGER REFERENCES gate_projects(id) ON STATE archived PROPAGATE SET invalidated +) STATE MACHINE (status: active -> [invalidated]); +CREATE TABLE firm_tasks ( + id INTEGER PRIMARY KEY, + status TEXT NOT NULL IMMUTABLE, + project_id INTEGER REFERENCES gate_projects(id) ON STATE archived PROPAGATE SET invalidated ABORT ON FAILURE +) STATE MACHINE (status: active -> [invalidated]); +INSERT INTO gate_projects (id, status) VALUES (1, 'active'), (2, 'active'); +INSERT INTO lax_tasks (id, status, project_id) VALUES (1, 'active', 1), (2, 'active', 1); +INSERT INTO firm_tasks (id, status, project_id) VALUES (1, 'active', 2), (2, 'active', 2); +UPDATE gate_projects SET status = 'archived' WHERE id = 1; +---- +rows affected: 1 + +SELECT id, status FROM lax_tasks ORDER BY id; +---- +id | status +1 | active +2 | active + +UPDATE gate_projects SET status = 'archived' WHERE id = 2; +---- +error: PropagationAborted + +SELECT id, status FROM gate_projects ORDER BY id; +---- +id | status +1 | archived +2 | active + +SELECT id, status FROM firm_tasks ORDER BY id; +---- +id | status +1 | active +2 | active diff --git a/v2/contextdb/tests/sql/state_machine.sql b/v2/contextdb/tests/sql/state_machine.sql new file mode 100644 index 00000000..46da245a --- /dev/null +++ b/v2/contextdb/tests/sql/state_machine.sql @@ -0,0 +1,187 @@ +# STATE MACHINE: a column whose value may only move along the transitions its table declares. +# An insert sets the first state; every UPDATE of the column is held to the list. + +CREATE TABLE tasks ( + id INTEGER PRIMARY KEY, + title TEXT, + status TEXT NOT NULL +) STATE MACHINE (status: draft -> [active, rejected], active -> [superseded]); +INSERT INTO tasks (id, title, status) +VALUES (1, 'one', 'draft'), (2, 'two', 'draft'), (3, 'three', 'active'), (4, 'four', 'draft'); + +# A listed transition. +UPDATE tasks SET status = 'active' WHERE id = 1; +---- +rows affected: 1 + +UPDATE tasks SET status = 'rejected' WHERE id = 2; +---- +rows affected: 1 + +# Not listed: backwards, past a state, out of a state that lists nothing, and to a state the +# declaration never names. +UPDATE tasks SET status = 'draft' WHERE id = 1; +---- +error: InvalidStateTransition + +UPDATE tasks SET status = 'superseded' WHERE id = 4; +---- +error: InvalidStateTransition + +UPDATE tasks SET status = 'active' WHERE id = 2; +---- +error: InvalidStateTransition + +UPDATE tasks SET status = 'paused' WHERE id = 4; +---- +error: InvalidStateTransition + +# A state does not list itself, so writing the state a row is already in is no transition. +UPDATE tasks SET status = 'active' WHERE id = 3; +---- +error: InvalidStateTransition + +# One row that may not move refuses the statement for every row: row 4 could have moved. +UPDATE tasks SET status = 'active' WHERE id IN (2, 4); +---- +error: InvalidStateTransition + +SELECT id, status FROM tasks ORDER BY id; +---- +id | status +1 | active +2 | rejected +3 | active +4 | draft + +# The column the table's rule is declared by cannot be dropped, with or without CASCADE, and a +# refused drop leaves the column and its rows. +ALTER TABLE tasks DROP COLUMN status; +---- +error: SchemaInvalid + +ALTER TABLE tasks DROP COLUMN status CASCADE; +---- +error: SchemaInvalid + +SELECT id, status FROM tasks ORDER BY id; +---- +id | status +1 | active +2 | rejected +3 | active +4 | draft + +# Every row the statement reaches may move: both do. +UPDATE tasks SET status = 'superseded' WHERE status = 'active'; +---- +rows affected: 2 + +# Another column of the row changes whatever state the row is in, and a statement that reaches +# no row is refused for nothing. +UPDATE tasks SET title = 'renamed' WHERE id = 2; +---- +rows affected: 1 + +UPDATE tasks SET status = 'draft' WHERE id = 404; +---- +rows affected: 0 + +SELECT id, title, status FROM tasks ORDER BY id; +---- +id | title | status +1 | one | superseded +2 | renamed | rejected +3 | three | superseded +4 | four | draft + +# An insert that meets a row moves it by the same list. +INSERT INTO tasks (id, title, status) VALUES (4, 'four', 'draft') +ON CONFLICT (id) DO UPDATE SET status = 'superseded'; +---- +error: InvalidStateTransition + +INSERT INTO tasks (id, title, status) VALUES (4, 'four', 'draft') +ON CONFLICT (id) DO UPDATE SET status = 'active'; +---- +rows affected: 1 + +SELECT id, status FROM tasks WHERE id = 4; +---- +id | status +4 | active + +# One next state may be written without brackets. +CREATE TABLE switches (id INTEGER PRIMARY KEY, state TEXT NOT NULL) +STATE MACHINE (state: idle -> busy, busy -> idle); +INSERT INTO switches (id, state) VALUES (1, 'idle'); +UPDATE switches SET state = 'busy' WHERE id = 1; +UPDATE switches SET state = 'idle' WHERE id = 1; +UPDATE switches SET state = 'broken' WHERE id = 1; +---- +error: InvalidStateTransition + +SELECT id, state FROM switches; +---- +id | state +1 | idle + +# A table that keeps only the current version of a row keeps its state and holds every change +# to the same list. +CREATE TABLE devices (id INTEGER PRIMARY KEY, state TEXT NOT NULL) +STATE MACHINE (state: starting -> [running], running -> [stopped]) +HISTORY CURRENT ONLY SYNC CONFLICT KEEP LATEST; +INSERT INTO devices (id, state) VALUES (1, 'starting'); +UPDATE devices SET state = 'stopped' WHERE id = 1; +---- +error: InvalidStateTransition + +UPDATE devices SET state = 'running' WHERE id = 1; +UPDATE devices SET state = 'starting' WHERE id = 1; +---- +error: InvalidStateTransition + +UPDATE devices SET state = 'stopped' WHERE id = 1; +SELECT id, state FROM devices; +---- +id | state +1 | stopped + +# A row that references another follows that row's transition: when a project becomes +# archived, the tasks that name it become invalidated. A task of another project, and a task +# that names no project, stay as they are. +CREATE TABLE projects (id INTEGER PRIMARY KEY, status TEXT NOT NULL) +STATE MACHINE (status: active -> [paused, archived], paused -> [active, archived]); +CREATE TABLE project_tasks ( + id INTEGER PRIMARY KEY, + status TEXT NOT NULL, + project_id INTEGER REFERENCES projects(id) ON STATE archived PROPAGATE SET invalidated +) STATE MACHINE (status: active -> [invalidated, superseded]); +INSERT INTO projects (id, status) VALUES (1, 'active'), (2, 'active'); +INSERT INTO project_tasks (id, status, project_id) +VALUES (1, 'active', 1), (2, 'active', 1), (3, 'active', 2), (4, 'active', NULL); + +# A transition to another state than the one the reference names moves no task. +UPDATE projects SET status = 'paused' WHERE id = 1; +---- +rows affected: 1 + +SELECT id, status FROM project_tasks ORDER BY id; +---- +id | status +1 | active +2 | active +3 | active +4 | active + +UPDATE projects SET status = 'archived' WHERE id = 1; +---- +rows affected: 1 + +SELECT id, status FROM project_tasks ORDER BY id; +---- +id | status +1 | invalidated +2 | invalidated +3 | active +4 | active diff --git a/v2/contextdb/tests/sql/update_delete.sql b/v2/contextdb/tests/sql/update_delete.sql index 0d5a19aa..e498f807 100644 --- a/v2/contextdb/tests/sql/update_delete.sql +++ b/v2/contextdb/tests/sql/update_delete.sql @@ -63,3 +63,44 @@ rows affected: 2 SELECT id FROM tasks ORDER BY id; ---- id + +# A column named twice in the SET list is refused, as Postgres refuses it, whatever it matches. +INSERT INTO tasks (id, title) VALUES (1, 'again'); +UPDATE tasks SET title = 'first', title = 'second' WHERE id = 1; +---- +error: PlanError + +# An UPDATE that swaps two keys: each key is taken while the statement runs and free when it +# ends, so each row is left under the other's key and is found by it. +CREATE TABLE swapped (k INTEGER PRIMARY KEY, v TEXT NOT NULL); +INSERT INTO swapped (k, v) VALUES (1, 'one'), (2, 'two'), (3, 'three'); +UPDATE swapped SET k = 3 - k WHERE k IN (1, 2); +---- +rows affected: 2 + +SELECT k, v FROM swapped ORDER BY k; +---- +k | v +1 | two +2 | one +3 | three + +SELECT v FROM swapped WHERE k = 1; +---- +v +two + +# A row moved to a key is not met again at that key: every row of a shift is changed once, +# although the key it moved to still passes the WHERE. +CREATE TABLE shifted (k INTEGER PRIMARY KEY, v TEXT NOT NULL); +INSERT INTO shifted (k, v) VALUES (98, 'one'), (99, 'two'), (100, 'three'); +UPDATE shifted SET k = k + 1 WHERE k >= 1 AND k <= 100; +---- +rows affected: 3 + +SELECT k, v FROM shifted ORDER BY k; +---- +k | v +99 | one +100 | two +101 | three diff --git a/v2/contextdb/tests/sql/vector_declarations.sql b/v2/contextdb/tests/sql/vector_declarations.sql index 83a945d5..bbbb02bb 100644 --- a/v2/contextdb/tests/sql/vector_declarations.sql +++ b/v2/contextdb/tests/sql/vector_declarations.sql @@ -87,8 +87,9 @@ SELECT id FROM docs; ---- error: TableNotFound -# The name was free all along. ALTER is a later version's too: refused, each leaves the column -# as it was declared, with room for one partition and searched INDEXED, which answers. +# The name was free all along. The column is declared with room for one partition and searched +# INDEXED; ALTER changes each: searched EXACT, an ordinary search answers, and with the limit +# raised a second partition is accepted. CREATE TABLE docs ( id INTEGER PRIMARY KEY, tenant TEXT NOT NULL, @@ -96,16 +97,16 @@ CREATE TABLE docs ( ); ALTER TABLE docs ALTER COLUMN v SET SEARCH_MODE EXACT; ---- -error: PlanError +rows affected: 0 ALTER TABLE docs ALTER COLUMN v SET MAX_PARTITIONS 512; ---- -error: PlanError +rows affected: 0 INSERT INTO docs (id, tenant, v) VALUES (1, 'north', [1.0, 0.0, 0.0]); INSERT INTO docs (id, tenant, v) VALUES (2, 'south', [0.0, 1.0, 0.0]); ---- -error: VectorPartitionLimitExceeded +rows affected: 1 SELECT id FROM docs ORDER BY v <=> [1.0, 0.0, 0.0] LIMIT 1; ---- diff --git a/v2/contextdb/tests/sql/vector_search.sql b/v2/contextdb/tests/sql/vector_search.sql index 8aec8f22..1813056b 100644 --- a/v2/contextdb/tests/sql/vector_search.sql +++ b/v2/contextdb/tests/sql/vector_search.sql @@ -175,3 +175,85 @@ id | distance 4 | 0.0 1 | 1.0 2 | 1.0 + +# A table may leave the rows in some states out of every search: `PROPAGATE ON STATE x EXCLUDE +# VECTOR`. Such a row stays a row, and a plain SELECT still returns it; a similarity search does +# not. A row moved into an excluded state by an UPDATE, or by propagation from the row it +# references, is out of the next search, EXACT or INDEXED, and out of what SHOW counts; moved +# back to a state the table does not exclude, it is found again. From [1, 0] the order is 1, 2, +# 5, 3, 4: the rows left out are the nearest, so a search that cut its list before leaving +# them out would come back short. +CREATE TABLE sources (id INTEGER PRIMARY KEY, status TEXT NOT NULL) +STATE MACHINE (status: active -> [archived]); +CREATE TABLE claims ( + id INTEGER PRIMARY KEY, + status TEXT NOT NULL, + source_id INTEGER REFERENCES sources(id) ON STATE archived PROPAGATE SET superseded, + v VECTOR(2) +) STATE MACHINE (status: active -> [invalidated, superseded], invalidated -> [active], superseded -> [active]) + PROPAGATE ON STATE invalidated EXCLUDE VECTOR + PROPAGATE ON STATE superseded EXCLUDE VECTOR; +INSERT INTO sources (id, status) VALUES (1, 'active'), (2, 'active'); +INSERT INTO claims (id, status, source_id, v) +VALUES (3, 'active', 2, [0.0, 1.0]), + (1, 'active', 1, [1.0, 0.0]), + (4, 'active', 2, [-1.0, 0.0]), + (2, 'active', 1, [0.9, 0.1]), + (5, 'active', 2, [0.5, 0.5]); + +SELECT id FROM claims ORDER BY v <=> [1.0, 0.0] LIMIT 10; +---- +id +1 +2 +5 +3 +4 + +# Row 3 by an UPDATE; rows 1 and 2 by propagation, as source 1 is archived. +UPDATE claims SET status = 'invalidated' WHERE id = 3; +---- +rows affected: 1 + +UPDATE sources SET status = 'archived' WHERE id = 1; +---- +rows affected: 1 + +SELECT id FROM claims ORDER BY v <=> [1.0, 0.0] LIMIT 10; +---- +id +5 +4 + +SELECT id FROM claims ORDER BY v <=> [1.0, 0.0] USE VECTOR INDEXED LIMIT 2; +---- +id +5 +4 + +SELECT id, status FROM claims ORDER BY id; +---- +id | status +1 | superseded +2 | superseded +3 | invalidated +4 | active +5 | active + +SHOW VECTOR_PARTITIONS FOR claims.v; +---- +table | column | partition_key | live_rows | pending_inserts | tombstones | durable_vector_bytes | durable_index_bytes | indexed_available | availability_reason | maintenance_state | maintenance_reason | maintenance_vectors_total | maintenance_vectors_done | maintenance_vectors_remaining | lists | probes | recovery_action +claims | v | {} | 2 | 2 | 0 | 0 | 0 | true | none | idle | none | 0 | 0 | 0 | NULL | NULL | none + +UPDATE claims SET status = 'active' WHERE id IN (1, 2, 3); +---- +rows affected: 3 + +SELECT id FROM claims ORDER BY v <=> [1.0, 0.0] LIMIT 10; +---- +id +1 +2 +5 +3 +4 diff --git a/v2/contextdb/tests/table_meta.rs b/v2/contextdb/tests/table_meta.rs new file mode 100644 index 00000000..b74c657b --- /dev/null +++ b/v2/contextdb/tests/table_meta.rs @@ -0,0 +1,128 @@ +//! What `Database::table_meta` says of a declared table: its columns, its UNIQUE rules, its +//! indexes and every rule it declares, the same after the store is closed and opened. + +mod support; + +use contextdb::{ColumnType, Error}; + +#[test] +fn table_meta_gives_each_tables_rules() { + let folder = support::folder("table-meta"); + let db = support::open(&folder); + let columns = "id UUID PRIMARY KEY, kind TEXT NOT NULL IMMUTABLE, slug TEXT UNIQUE, \ + status TEXT NOT NULL DEFAULT 'active', source_id UUID NOT NULL, \ + target_id UUID NOT NULL, edge_type TEXT NOT NULL, UNIQUE (kind, slug)"; + // One table that declares no option, and one for each option, the same in everything else. + let options = [ + ("plain", ""), + ("frozen", "IMMUTABLE"), + ("local", "SYNC OFF"), + ("pushed", "SYNC PUSH ONLY"), + ("latest", "SYNC CONFLICT KEEP LATEST"), + ("current", "SYNC CONFLICT KEEP LATEST HISTORY CURRENT ONLY"), + ("kept", "RETAIN 7 DAYS"), + ("kept_safe", "RETAIN 7 DAYS SYNC SAFE"), + ("moving", "STATE MACHINE (status: active -> [done])"), + ("acyclic", "DAG('NEEDS')"), + ("counted", "MAINTAIN COUNT (kind)"), + ("counted_all", "MAINTAIN COUNT ALL PREFIXES"), + ]; + for (table, option) in options { + support::run(&db, &format!("CREATE TABLE {table} ({columns}) {option}")); + support::run( + &db, + &format!("CREATE INDEX by_status ON {table} (status DESC, id)"), + ); + } + + // What every one of them says of its columns, its UNIQUE rules and its indexes. + let plain = db + .table_meta("plain") + .expect("the description of a declared table"); + let names: Vec<&str> = plain.columns.iter().map(|c| c.name.as_str()).collect(); + assert_eq!( + names, + [ + "id", + "kind", + "slug", + "status", + "source_id", + "target_id", + "edge_type" + ] + ); + let id = &plain.columns[0]; + assert_eq!(id.column_type, ColumnType::Uuid); + assert!(id.primary_key && !plain.columns[1].primary_key); + assert!(!plain.columns[1].nullable && plain.columns[2].nullable); + assert!(plain.columns[2].unique && !plain.columns[3].unique); + assert!(plain.columns[3].default.is_some() && plain.columns[2].default.is_none()); + assert_eq!(plain.unique_constraints, [["kind", "slug"]]); + let indexes: Vec<&str> = plain.indexes.iter().map(|i| i.name.as_str()).collect(); + assert!( + indexes.contains(&"__pk_id") && indexes.contains(&"by_status"), + "{indexes:?}" + ); + + // Rules whose columns spell one name stay separate rules, each with its own index. + support::run( + &db, + "CREATE TABLE pairs (id INTEGER PRIMARY KEY, a INTEGER, b INTEGER, a_b INTEGER, \ + UNIQUE (a_b), UNIQUE (a, b), UNIQUE (a, b))", + ); + let pairs = db.table_meta("pairs").expect("a description"); + let names: Vec<&str> = pairs.indexes.iter().map(|i| i.name.as_str()).collect(); + assert_eq!( + names, + [ + "__pk_id", + "__unique_a_b", + "__unique_a_b_2", + "__unique_a_b_3" + ], + "UNIQUE (a_b) and UNIQUE (a, b) are two rules with two indexes, and a repeat is a third" + ); + + // A key written as an element of the table describes its column as the column-level form + // (`id` above) does: not nullable, as in Postgres. + assert!(!id.nullable, "a PRIMARY KEY column is NOT NULL"); + support::run(&db, "CREATE TABLE by_element (a INTEGER, PRIMARY KEY (a))"); + let by_element = db.table_meta("by_element").expect("a description"); + assert!( + !by_element.columns[0].nullable, + "PRIMARY KEY (a) makes `a` NOT NULL exactly as `a INTEGER PRIMARY KEY` does" + ); + + // IMMUTABLE on a table, and no other table says it. + for (table, _) in options { + let meta = db + .table_meta(table) + .expect("the description of a declared table"); + assert_eq!(meta.immutable, table == "frozen", "{table}"); + } + // Every other option shows in the description: a table that declares one is described + // differently from the table that declares none, and from every other. + let described: Vec = options + .iter() + .map(|(table, _)| format!("{:?}", db.table_meta(table).expect("a description"))) + .collect(); + for (first, one) in described.iter().enumerate() { + for (second, other) in described.iter().enumerate().skip(first + 1) { + assert_ne!( + one, other, + "{} and {} are described alike", + options[first].0, options[second].0 + ); + } + } + + // A table that is not there, by name; and the same descriptions after close and open. + assert!(matches!(db.table_meta("nope"), Err(Error::TableNotFound(name)) if name == "nope")); + support::close(db); + let db = support::open(&folder); + for ((table, _), before) in options.iter().zip(&described) { + let after = format!("{:?}", db.table_meta(table).expect("a description")); + assert_eq!(&after, before, "{table} after close and open"); + } +} diff --git a/v2/contextdb/tests/table_source.rs b/v2/contextdb/tests/table_source.rs new file mode 100644 index 00000000..26a333ec --- /dev/null +++ b/v2/contextdb/tests/table_source.rs @@ -0,0 +1,198 @@ +//! The table source: the one place a statement's rows come from. A narrowed reader that names a +//! hidden row whole is told which rule hid it, a list of keys and an `IS NULL` over an index +//! read only the rows they return, and a large value no statement asked for is never loaded. + +mod support; + +use std::collections::BTreeSet; + +use contextdb::{ContextId, Error, Principal, ScopeLabel, Uuid, Value, params}; + +#[test] +fn a_narrowed_reader_naming_a_hidden_row_by_its_key_is_refused_by_the_rule_that_hid_it() { + let folder = support::folder("table-source-hidden-row-by-key"); + let db = support::open(&folder); + let context = |n: u128| Uuid::from_u128(0x00C0_0000 + n); + let acl = |n: u128| Uuid::from_u128(0x00AC_0000 + n); + + // Row 1 of each table is in the reader's view and row 2 is hidden by the table's rule. + support::run( + &db, + "CREATE TABLE placed (id INTEGER PRIMARY KEY, context_id UUID CONTEXT_ID, payload TEXT)", + ); + support::run( + &db, + "CREATE TABLE labelled (id INTEGER PRIMARY KEY, reach TEXT SCOPE_LABEL_READ ('public', \ + 'internal') WRITE ('public', 'internal'), payload TEXT)", + ); + support::run( + &db, + "CREATE TABLE acl_grants (id INTEGER PRIMARY KEY, principal_kind TEXT, \ + principal_id TEXT, acl_id UUID)", + ); + support::run( + &db, + "CREATE TABLE guarded (id INTEGER PRIMARY KEY, \ + acl_id UUID ACL REFERENCES acl_grants(acl_id), payload TEXT)", + ); + for (id, label) in [(1_i64, "public"), (2, "internal")] { + let n = u128::try_from(id).expect("a small id"); + db.execute( + "INSERT INTO placed (id, context_id, payload) VALUES ($id, $context, 'kept')", + ¶ms! { "id" => id, "context" => context(n) }, + ) + .expect("insert a placed row"); + db.execute( + "INSERT INTO labelled (id, reach, payload) VALUES ($id, $reach, 'kept')", + ¶ms! { "id" => id, "reach" => label }, + ) + .expect("insert a labelled row"); + db.execute( + "INSERT INTO guarded (id, acl_id, payload) VALUES ($id, $acl, 'kept')", + ¶ms! { "id" => id, "acl" => acl(n) }, + ) + .expect("insert a guarded row"); + } + db.execute( + "INSERT INTO acl_grants (id, principal_kind, principal_id, acl_id) \ + VALUES (1, 'Agent', 'a', $acl)", + ¶ms! { "acl" => acl(1) }, + ) + .expect("grant access id 1"); + + let contexts: BTreeSet = [ContextId::new(context(1))].into(); + let labels: BTreeSet = [ScopeLabel::new("public")].into(); + let agent = Principal::Agent("a".to_owned()); + let in_one = db.scoped_with_constraints(Some(contexts), None, None); + let public = db.scoped_with_constraints(None, Some(labels), None); + let granted = db.scoped_with_constraints(None, None, Some(agent)); + + let kept = vec![vec![Value::Text("kept".to_owned())]]; + for (reader, table) in [ + (&in_one, "placed"), + (&public, "labelled"), + (&granted, "guarded"), + ] { + // The row in view is returned, and a key no row has is an empty answer, not a refusal. + let shown = format!("SELECT payload FROM {table} WHERE id = 1"); + assert_eq!(support::rows(reader, &shown), kept, "{shown}"); + let absent = format!("SELECT payload FROM {table} WHERE id = 99"); + assert_eq!( + support::rows(reader, &absent), + Vec::>::new(), + "{absent}" + ); + + let hidden = format!("SELECT payload FROM {table} WHERE id = 2"); + let refused = support::refusal(reader, &hidden); + let named = match (table, &refused) { + ("placed", Error::ContextScopeViolation { table }) => table, + ("labelled", Error::ScopeLabelViolation { table }) => table, + ("guarded", Error::AclDenied { table }) => table, + _ => panic!("{hidden}\nwas refused with {refused:?}, not by the rule of {table}"), + }; + assert_eq!(named, table, "{hidden}"); + // To a reader not narrowed the row is there. + assert_eq!(support::rows(&db, &hidden), kept, "{hidden}, not narrowed"); + } +} + +#[test] +fn a_list_of_keys_and_an_is_null_over_an_index_read_only_the_rows_they_return() { + let folder = support::folder("table-source-in-and-is-null"); + let db = support::open(&folder); + support::run(&db, "CREATE TABLE n (id INTEGER PRIMARY KEY, c INTEGER)"); + support::run(&db, "CREATE INDEX n_c ON n (c)"); + // Of a thousand rows, the ten whose id is a multiple of 100 have no `c`. + let tx = db.transaction().expect("begin"); + for id in 1..=1000_i64 { + let c = if id % 100 == 0 { + Value::Null + } else { + Value::Int64(id % 7) + }; + tx.execute( + "INSERT INTO n (id, c) VALUES ($id, $c)", + ¶ms! { "id" => id, "c" => c }, + ) + .expect("insert a row"); + } + tx.commit().expect("commit"); + + let listed = support::run( + &db, + "SELECT id FROM n WHERE id IN (3, 500, 997) ORDER BY id", + ); + let wanted = [3_i64, 500, 997].map(|id| vec![Value::Int64(id)]); + assert_eq!(listed.rows(), wanted, "the rows of the three keys"); + assert_eq!(listed.rows_examined(), 3, "rows read for three keys"); + + // A key in the list that no row has is not a row read. + let listed = support::run(&db, "SELECT id FROM n WHERE id IN (3, 5000) ORDER BY id"); + assert_eq!( + listed.rows(), + [[Value::Int64(3)]], + "the one key that is there" + ); + assert_eq!( + listed.rows_examined(), + 1, + "rows read for one key that is there" + ); + + let nulls = support::run(&db, "SELECT id FROM n WHERE c IS NULL ORDER BY id"); + let wanted: Vec> = (1..=10).map(|n| vec![Value::Int64(n * 100)]).collect(); + assert_eq!(nulls.rows(), wanted, "the rows with no `c`"); + assert_eq!( + nulls.rows_examined(), + 10, + "rows read for ten rows with no `c`" + ); +} + +#[test] +fn a_large_value_no_statement_selects_is_not_loaded() { + const TEXT: usize = 2 << 20; + let folder = support::folder("table-source-unselected-large-value"); + let db = support::open(&folder); + support::run( + &db, + "CREATE TABLE big (id INTEGER PRIMARY KEY, title TEXT, body TEXT)", + ); + let body = support::letters(7, TEXT); + db.execute( + "INSERT INTO big (id, title, body) VALUES (1, 'first', $body)", + ¶ms! { "body" => body.clone() }, + ) + .expect("insert the row with the large text"); + support::close(db); + + // The limit is what the open store has in use after the same small read, and half the + // text's size more: room for the row's small values and none for its text. + let db = support::open(&folder); + let title = "SELECT id, title FROM big WHERE id = 1"; + let small = vec![vec![Value::Int64(1), Value::Text("first".to_owned())]]; + assert_eq!(support::rows(&db, title), small, "before the limit"); + db.set_memory_limit(Some(db.memory_used() + TEXT / 2)) + .expect("set a limit above what is in use"); + + assert_eq!( + support::rows(&db, title), + small, + "the row read without its text, under the limit" + ); + match support::refusal(&db, "SELECT body FROM big WHERE id = 1") { + Error::MemoryBudgetExceeded { .. } => {} + other => panic!("selecting the text under the limit was refused with {other:?}"), + } + assert_eq!( + support::rows(&db, title), + small, + "the row read without its text, after the refusal" + ); + + // With the limit lifted the text is there, whole. + db.set_memory_limit(None).expect("lift the limit"); + let found = support::rows(&db, "SELECT body FROM big WHERE id = 1"); + support::assert_text_is(&found[0][0], &body, "the text, with no limit"); +} diff --git a/v2/contextdb/tests/transaction_numbers.rs b/v2/contextdb/tests/transaction_numbers.rs new file mode 100644 index 00000000..c0b27a43 --- /dev/null +++ b/v2/contextdb/tests/transaction_numbers.rs @@ -0,0 +1,209 @@ +//! The number a transaction has from the moment it begins, the `TXID` column that holds it, and +//! the number its commit carries: the transaction's own, or the next after the newest commit +//! when another commit overtook it, and never below a `TXID` value the commit holds. + +mod support; + +use contextdb::{Database, Error, TxId, Value, params}; +use support::Run; + +const LEDGER: &str = "CREATE TABLE ledger (id INTEGER PRIMARY KEY, at TXID, note TEXT)"; + +/// Writes ledger row `id`, stamped `at`. +fn stamp(on: &impl Run, id: i64, at: TxId) -> contextdb::Result { + on.run( + "INSERT INTO ledger (id, at) VALUES ($id, $at)", + ¶ms! { "id" => id, "at" => Value::TxId(at) }, + ) +} + +/// The stamp of ledger row `id`. +fn stamp_of(on: &impl Run, id: i64) -> Value { + let rows = support::rows(on, &format!("SELECT at FROM ledger WHERE id = {id}")); + match rows.as_slice() { + [row] => row[0].clone(), + rows => panic!("ledger row {id} came back as {} rows", rows.len()), + } +} + +#[test] +fn a_txid_column_holds_its_transactions_number_and_a_commit_is_never_numbered_below_it() { + let folder = support::folder("transaction-numbers"); + let db = support::open(&folder); + support::run(&db, LEDGER); + support::run(&db, "CREATE INDEX ledger_at ON ledger (at)"); + let newest = db.committed_watermark(); + + // A transaction has its number from the moment it begins, above every commit there is. + let tx = db.transaction().expect("begin"); + let first = tx.id(); + assert!( + first > newest, + "{first:?} is not above the newest commit, {newest:?}" + ); + assert_eq!(tx.id(), first, "a transaction has one number"); + stamp(&tx, 1, first).expect("stamp a row with the transaction's own number"); + // A number of an earlier commit is a value like any other, and so is no value. + stamp(&tx, 2, newest).expect("stamp a row with an earlier number"); + support::run(&tx, "INSERT INTO ledger (id, at) VALUES (3, NULL)"); + assert_eq!(stamp_of(&tx, 1), Value::TxId(first)); + tx.commit().expect("commit"); + // No other commit came between: the commit carries the transaction's number. + assert_eq!(db.committed_watermark(), first); + assert_eq!(stamp_of(&db, 1), Value::TxId(first)); + assert_eq!(stamp_of(&db, 2), Value::TxId(newest)); + assert_eq!(stamp_of(&db, 3), Value::Null); + + // Two transactions side by side have two numbers. The second to begin commits first, under + // its own number; the first was overtaken, and commits under the next number after that. + let slow = db.transaction().expect("begin"); + let quick = db.transaction().expect("begin"); + assert!(slow.id() > first && quick.id() > slow.id()); + let (slow_id, quick_id) = (slow.id(), quick.id()); + stamp(&slow, 4, slow_id).expect("stamp in the slow transaction"); + stamp(&quick, 5, quick_id).expect("stamp in the quick transaction"); + quick.commit().expect("commit the quick transaction"); + assert_eq!(db.committed_watermark(), quick_id); + slow.commit().expect("commit the slow transaction"); + let overtaken = db.committed_watermark(); + assert_eq!( + overtaken, + TxId(quick_id.0 + 1), + "an overtaken commit takes the next number after the newest" + ); + // The row keeps the number its transaction had, which is below the commit's number. + assert_eq!(stamp_of(&db, 4), Value::TxId(slow_id)); + assert!(slow_id < overtaken); + + // A transaction that commits nothing does not hand its number to the next one. + let unused = db.transaction().expect("begin"); + let unused_id = unused.id(); + assert!(unused_id > overtaken); + unused.rollback(); + let next = db.transaction().expect("begin"); + assert!(next.id() > unused_id); + stamp(&next, 6, next.id()).expect("stamp"); + let next_id = next.id(); + next.commit().expect("commit"); + assert_eq!(db.committed_watermark(), next_id); + + // A stamp is found by its value through the index, and kept across a reopen. + let stamped = |on: &Database, at: TxId| { + on.execute( + "SELECT id FROM ledger WHERE at = $at", + ¶ms! { "at" => Value::TxId(at) }, + ) + .expect("find rows by their stamp") + }; + let found = stamped(&db, slow_id); + assert_eq!(found.rows(), [[Value::Int64(4)]]); + assert_eq!(found.rows_examined(), 1); + support::close(db); + let db = support::open(&folder); + assert_eq!(stamped(&db, next_id).rows(), [[Value::Int64(6)]]); + assert_eq!(stamp_of(&db, 5), Value::TxId(quick_id)); + // Numbers go on rising from the newest commit. + assert!(db.transaction().expect("begin").id() > next_id); +} + +#[test] +fn a_stamp_above_the_transactions_own_number_and_a_stamp_written_as_a_number_are_refused() { + let db = support::open(&support::folder("transaction-number-refusals")); + support::run(&db, LEDGER); + let tx = db.transaction().expect("begin"); + let own = tx.id(); + stamp(&tx, 1, own).expect("stamp a row with the transaction's own number"); + + let out_of_range = + |outcome: contextdb::Result, wrote: u64| match outcome { + Err(Error::TxIdOutOfRange { value, max }) => { + assert_eq!((value, max), (wrote, own.0)); + } + other => panic!("a stamp of {wrote} in transaction {} gave {other:?}", own.0), + }; + out_of_range(stamp(&tx, 2, TxId(own.0 + 1)), own.0 + 1); + out_of_range(stamp(&tx, 2, TxId(u64::MAX)), u64::MAX); + out_of_range( + tx.execute( + "UPDATE ledger SET at = $at WHERE id = 1", + ¶ms! { "at" => Value::TxId(TxId(own.0 + 7)) }, + ), + own.0 + 7, + ); + + // SQL has no way to write a transaction's number: a number and a text are of another type. + for sql in [ + "INSERT INTO ledger (id, at) VALUES (3, 7)", + "INSERT INTO ledger (id, at) VALUES (3, '7')", + "UPDATE ledger SET at = 7 WHERE id = 1", + ] { + match support::refusal(&tx, sql) { + Error::ColumnTypeMismatch { table, column, .. } => { + assert_eq!((table.as_str(), column.as_str()), ("ledger", "at"), "{sql}"); + } + other => panic!("{sql}\nwas refused as {other:?}"), + } + } + // The transaction holds the one row it stamped, as it stamped it. + assert_eq!( + support::rows(&tx, "SELECT id, at FROM ledger ORDER BY id"), + [[Value::Int64(1), Value::TxId(own)]] + ); + tx.commit().expect("commit"); + + // A statement on the writer has a number of its own, and is held to it the same way. + match stamp(&db, 4, TxId(u64::MAX)) { + Err(Error::TxIdOutOfRange { value, .. }) => assert_eq!(value, u64::MAX), + other => panic!("a stamp of the largest number on the writer gave {other:?}"), + } + stamp(&db, 4, db.committed_watermark()).expect("stamp with the newest commit's number"); + assert_eq!(support::int(&db, "SELECT COUNT(*) FROM ledger"), 2); +} + +#[test] +fn a_vector_column_is_partitioned_by_the_number_of_the_transaction_that_wrote_the_row() { + const DIMENSION: usize = 4; + let db = support::open(&support::folder("partitioned-by-txid")); + support::run( + &db, + &format!( + "CREATE TABLE marks (id INTEGER PRIMARY KEY, at TXID NOT NULL, \ + v VECTOR({DIMENSION}) PARTITION_KEY (at))" + ), + ); + // Two transactions of five rows each: each transaction's rows are one partition. + let mut numbers = Vec::new(); + for batch in 0..2_i64 { + let tx = db.transaction().expect("begin"); + for n in 1..=5_i64 { + let id = batch * 5 + n; + tx.execute( + "INSERT INTO marks (id, at, v) VALUES ($id, $at, $v)", + ¶ms! { "id" => id, "at" => Value::TxId(tx.id()), "v" => support::vector(id, DIMENSION) }, + ) + .expect("insert a mark"); + } + numbers.push(tx.id()); + tx.commit().expect("commit"); + } + let query = support::vector(77, DIMENSION); + for (batch, at) in numbers.into_iter().enumerate() { + let found = db + .execute( + "SELECT id FROM marks WHERE at = $at ORDER BY v <=> $query LIMIT 5", + ¶ms! { "at" => Value::TxId(at), "query" => query.clone() }, + ) + .expect("search one transaction's rows"); + let mut ids: Vec = found + .rows() + .iter() + .map(|row| match row[0] { + Value::Int64(id) => id, + ref other => panic!("an id came back as {other:?}"), + }) + .collect(); + ids.sort_unstable(); + let first = batch as i64 * 5 + 1; + assert_eq!(ids, (first..first + 5).collect::>()); + } +} diff --git a/v2/contextdb/tests/transactions.rs b/v2/contextdb/tests/transactions.rs index be422b4c..fd098510 100644 --- a/v2/contextdb/tests/transactions.rs +++ b/v2/contextdb/tests/transactions.rs @@ -155,7 +155,7 @@ fn of_two_transactions_inserting_one_key_exactly_one_commits() { .commit() .expect("the first of two transactions commits"); match second.commit() { - Err(Error::UniqueViolation { table, column }) => { + Err(Error::UniqueViolation { table, column, .. }) => { assert_eq!((table.as_str(), column.as_str()), ("people", "id")); } other => panic!("the second transaction to take key 1 got {other:?}"), @@ -176,7 +176,7 @@ fn of_two_transactions_inserting_one_key_exactly_one_commits() { .commit() .expect("the first of two transactions commits"); match second.commit() { - Err(Error::UniqueViolation { table, column }) => { + Err(Error::UniqueViolation { table, column, .. }) => { assert_eq!((table.as_str(), column.as_str()), ("people", "email")); } other => panic!("the second transaction to take the email got {other:?}"), @@ -201,6 +201,39 @@ fn of_two_transactions_inserting_one_key_exactly_one_commits() { .commit() .expect("a transaction that conflicts with nothing commits"); + // A table with no primary key has no key for rows to collide on: both commit, as in + // Postgres, and both rows are there. + support::run(&db, "CREATE TABLE notes (body TEXT)"); + let first = db.transaction().expect("begin"); + let second = db.transaction().expect("begin"); + support::run(&first, "INSERT INTO notes (body) VALUES ('one')"); + support::run(&second, "INSERT INTO notes (body) VALUES ('two')"); + first + .commit() + .expect("the first insert into a table with no key"); + second + .commit() + .unwrap_or_else(|error| panic!("the second insert into a table with no key: {error:?}")); + assert_eq!( + support::rows(&db, "SELECT body FROM notes ORDER BY body"), + [ + [Value::Text("one".to_owned())], + [Value::Text("two".to_owned())] + ] + ); + + // A transaction that inserts a key and deletes it again leaves the key empty, but another + // handle took the key meanwhile: its commit is refused as a conflict, and the other handle's + // row stands among the winners below. + let tx = db.transaction().expect("begin"); + support::run(&tx, "INSERT INTO people VALUES (6, 'six@example.org')"); + support::run(&tx, "DELETE FROM people WHERE id = 6"); + support::run(&db, "INSERT INTO people VALUES (6, 'six@example.org')"); + match tx.commit() { + Err(Error::ConditionalUpdateConflict { count }) => assert_eq!(count, 1), + other => panic!("a transaction whose key was taken meanwhile: {other:?}"), + } + // The stored rows are exactly the winners'. let text = |s: &str| Value::Text(s.to_owned()); assert_eq!( @@ -210,6 +243,7 @@ fn of_two_transactions_inserting_one_key_exactly_one_commits() { [Value::Int64(2), text("shared@example.org")], [Value::Int64(4), text("four@example.org")], [Value::Int64(5), text("five@example.org")], + [Value::Int64(6), text("six@example.org")], ] ); } @@ -372,7 +406,7 @@ fn a_refused_statement_changes_nothing_and_leaves_the_transaction_usable() { for taken in [1, 2] { let sql = format!("INSERT INTO t (id, name, tag) VALUES ({taken}, 'again', 0)"); match support::refusal(&tx, &sql) { - Error::UniqueViolation { table, column } => { + Error::UniqueViolation { table, column, .. } => { assert_eq!((table.as_str(), column.as_str()), ("t", "id")); } other => panic!("a taken key was refused as {other:?}"), @@ -465,6 +499,32 @@ fn a_transaction_that_declares_a_table_may_not_also_write_rows() { support::refusal(&db, "SELECT * FROM late"), Error::TableNotFound(_) )); + + // The same for a change and a drop of the table: after a row, each is refused and the row + // commits with the table as it was; before a row, the drop of the column commits. + let tx = db.transaction().expect("begin"); + support::insert(&tx, 3..=3); + for sql in ["ALTER TABLE t DROP COLUMN tag CASCADE", "DROP TABLE t"] { + let refusal = support::refusal(&tx, sql); + assert!( + matches!(refusal, Error::MixedSchemaAndRowWrite), + "{sql}\nafter a row was refused as {refusal:?}" + ); + } + tx.commit().expect("commit the row"); + support::assert_rows_are(&db, 1..=3); + let tx = db.transaction().expect("begin"); + support::run(&tx, "ALTER TABLE t DROP COLUMN tag CASCADE"); + let refusal = support::refusal(&tx, "UPDATE t SET name = 'x' WHERE id = 1"); + assert!( + matches!(refusal, Error::MixedSchemaAndRowWrite), + "a row change after a dropped column was refused as {refusal:?}" + ); + tx.commit().expect("commit the drop"); + assert!(matches!( + support::refusal(&db, "SELECT tag FROM t"), + Error::ColumnNotFound { .. } + )); } #[test] diff --git a/v2/contextdb/tests/updates.rs b/v2/contextdb/tests/updates.rs new file mode 100644 index 00000000..597bb4e2 --- /dev/null +++ b/v2/contextdb/tests/updates.rs @@ -0,0 +1,503 @@ +//! What an UPDATE and a DELETE cost, said as bytes written and memory held. An UPDATE leaves a large value it does not assign where it +//! is; an UPDATE or DELETE of many rows holds one batch of them beside what it has staged, +//! never all of them twice; a row a statement has changed is not met by it again. And three +//! rules of writing rows: assigning what a statement finds its rows by, the columns ON CONFLICT +//! may name, and a long indexed value beside an assigned one. + +mod support; + +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + +use contextdb::{Database, DatabaseOpenOptions, Error, Value, params}; +use support::{Call, Kind}; + +const MIB: usize = 1 << 20; + +#[test] +fn an_update_of_another_column_leaves_a_large_value_where_it_is() { + // A text and a JSON value that do not shrink, so each is stored in as many bytes as it has. + const LEN: usize = 2 * MIB; + let folder = support::folder("update-leaves-large-values"); + let written = Arc::new(AtomicU64::new(0)); + let mut options = DatabaseOpenOptions::default(); + options.storage = Some(support::storage({ + let written = written.clone(); + move |call| { + if let Call::Write(_, bytes) = call { + written.fetch_add(bytes.len() as u64, Ordering::SeqCst); + } + } + })); + let db = support::open_with(&folder, options).expect("open the store"); + support::run( + &db, + "CREATE TABLE docs (id INTEGER PRIMARY KEY, title TEXT, body TEXT, meta JSON)", + ); + support::run(&db, "CREATE INDEX docs_title ON docs (title)"); + let body = |id: i64| support::large_text(Kind::Scattered, id, LEN); + let meta = |id: i64| support::large_json(false, id, LEN); + for id in 1..=4_i64 { + db.execute( + "INSERT INTO docs (id, title, body, meta) VALUES ($id, $title, $body, $meta)", + ¶ms! { "id" => id, "title" => format!("title-{id}"), "body" => body(id), "meta" => meta(id) }, + ) + .expect("insert a document"); + } + // The bytes the data file is written while `change` runs. + let written_by = |change: &dyn Fn()| { + let before = written.load(Ordering::SeqCst); + change(); + written.load(Ordering::SeqCst) - before + }; + let assert_doc = |id: i64, title: &str| { + let found = db + .execute( + "SELECT title, body, meta FROM docs WHERE id = $id", + ¶ms! { "id" => id }, + ) + .expect("read a document"); + let [row] = found.rows() else { + panic!("document {id} came back as {} rows", found.rows().len()); + }; + assert_eq!(row[0], support::text(title), "the title of document {id}"); + support::assert_text_is(&row[1], &body(id), &format!("the body of document {id}")); + support::assert_json_is(&row[2], &meta(id), &format!("the meta of document {id}")); + }; + + // The door sees a rewrite: assigning the large value of row 4 writes at least its bytes. + let rewritten = written_by(&|| { + db.execute( + "UPDATE docs SET body = $body WHERE id = 4", + ¶ms! { "body" => body(40) }, + ) + .expect("assign the large value"); + }); + assert!( + rewritten >= LEN as u64, + "assigning a text of {LEN} bytes wrote {rewritten} bytes" + ); + + // Assigning another column writes less than one of the row's two large values holds, + // whether the row is found by its key, by an index or by reading the table, alone or in a + // transaction with other statements. + let by_key = written_by(&|| { + support::run(&db, "UPDATE docs SET title = 'by key' WHERE id = 1"); + }); + assert!( + by_key < LEN as u64, + "an UPDATE of the title wrote {by_key} bytes beside two values of {LEN} bytes" + ); + assert_doc(1, "by key"); + + let by_index = written_by(&|| { + support::run( + &db, + "UPDATE docs SET title = 'by index' WHERE title = 'title-2'", + ); + }); + assert!(by_index < LEN as u64, "found by an index: {by_index} bytes"); + assert_doc(2, "by index"); + + let every_row = written_by(&|| { + let tx = db.transaction().expect("begin"); + let changed = support::run(&tx, "UPDATE docs SET title = 'all' WHERE id <= 3"); + assert_eq!(changed.rows_affected(), 3); + support::run( + &tx, + "UPDATE docs SET title = 'all again' WHERE title = 'all'", + ); + tx.commit().expect("commit"); + }); + assert!( + every_row < LEN as u64, + "two UPDATEs of the titles of three rows wrote {every_row} bytes" + ); + for id in 1..=3 { + assert_doc(id, "all again"); + } + + // An UPDATE that assigns one large value leaves the other. + let one_of_two = written_by(&|| { + db.execute( + "UPDATE docs SET meta = $meta WHERE id = 4", + ¶ms! { "meta" => meta(41) }, + ) + .expect("assign the JSON value"); + }); + assert!( + (LEN as u64..rewritten + LEN as u64 / 2).contains(&one_of_two), + "assigning one of two values of {LEN} bytes wrote {one_of_two} bytes, and assigning \ + the other alone wrote {rewritten}" + ); + let other = support::rows(&db, "SELECT body, meta FROM docs WHERE id = 4"); + support::assert_text_is(&other[0][0], &body(40), "the body of document 4"); + support::assert_json_is(&other[0][1], &meta(41), "the meta of document 4"); + + // The values are where they were after reopening too. + support::close(db); + let db = support::open(&folder); + let titles = support::rows(&db, "SELECT title FROM docs WHERE id <= 3 ORDER BY id"); + assert_eq!(titles, vec![[support::text("all again")]; 3]); + let found = support::rows(&db, "SELECT body FROM docs WHERE id = 2"); + support::assert_text_is(&found[0][0], &body(2), "the body of document 2, reopened"); +} + +/// Asserts that a statement over many rows was not refused, and gives its result. A refusal by +/// the memory limit says who asked and for how much: that is what holding the rows twice +/// looks like. +fn fits(outcome: contextdb::Result, what: &str) -> contextdb::QueryResult { + match outcome { + Ok(result) => result, + Err(Error::MemoryBudgetExceeded { + operation, + requested_bytes, + available_bytes, + .. + }) => panic!( + "{what} did not fit a limit with room for what it stages and an eighth more: \ + `{operation}` asked for {requested_bytes} bytes with {available_bytes} left" + ), + Err(other) => panic!("{what} was refused: {other:?}"), + } +} + +#[test] +fn an_update_and_a_delete_of_40_000_rows_hold_one_batch_beside_what_they_stage() { + // Forty batches of a thousand rows: an eighth of what the statement stages is room for + // five batches and for no more than an eighth of a second copy of the matched rows. + const ROWS: i64 = 40_000; + const ROOMY: usize = 512 * MIB; + let folder = support::folder("many-rows-one-batch"); + let mut options = DatabaseOpenOptions::default(); + options.memory_limit = Some(ROOMY); + let db = support::open_with(&folder, options).expect("open the store under a limit"); + support::run( + &db, + "CREATE TABLE small (id INTEGER PRIMARY KEY, n INTEGER NOT NULL, tag TEXT NOT NULL)", + ); + // Row `id` holds `n = id % 1000`. + for from in (0..ROWS).step_by(1000) { + let rows: Vec = (from..from + 1000) + .map(|id| format!("({id}, {}, 'tag-{}')", id % 1000, id % 7)) + .collect(); + let tx = db.transaction().expect("begin"); + support::run( + &tx, + &format!("INSERT INTO small (id, n, tag) VALUES {}", rows.join(", ")), + ); + tx.commit().expect("commit a thousand rows"); + } + let sum = |on: &Database| support::int(on, "SELECT SUM(n) FROM small"); + let before = sum(&db); + assert_eq!(before, ROWS / 1000 * (999 * 1000 / 2)); + + // What a statement stages is held by its transaction until the commit. This runs the + // statement in a transaction under the roomy limit, reads what the transaction holds once + // the statement has returned, and rolls back. + let staged_by = |sql: &str| -> usize { + let tx = db.transaction().expect("begin"); + let idle = db.memory_used(); + let changed = fits(tx.execute(sql, ¶ms! {}), sql); + assert_eq!(changed.rows_affected(), ROWS as u64, "{sql}"); + let staged = db.memory_used() - idle; + tx.rollback(); + staged + }; + // Sets a limit with room for what the statement stages and an eighth of that again: one + // batch of a thousand rows is a fortieth of the rows and fits in the eighth, and a second + // copy of the 40,000 matched rows does not fit at all. + let limit_for = |staged: usize| { + assert!( + staged > ROWS as usize * 8, + "a statement over {ROWS} rows staged {staged} bytes" + ); + let limit = db.memory_used() + staged + staged / 8; + db.set_memory_limit(Some(limit)).expect("set the limit"); + }; + + let update = "UPDATE small SET n = n + 1 WHERE tag <> 'none'"; + limit_for(staged_by(update)); + let changed = fits(db.execute(update, ¶ms! {}), update); + assert_eq!(changed.rows_affected(), ROWS as u64); + // Each row was changed, and changed once. + assert_eq!(sum(&db), before + ROWS); + assert_eq!(support::int(&db, "SELECT MIN(n) FROM small"), 1); + assert_eq!(support::int(&db, "SELECT MAX(n) FROM small"), 1000); + assert_eq!(support::int(&db, "SELECT COUNT(*) FROM small"), ROWS); + + db.set_memory_limit(Some(ROOMY)).expect("lift the limit"); + let delete = "DELETE FROM small WHERE n > 0"; + limit_for(staged_by(delete)); + let removed = fits(db.execute(delete, ¶ms! {}), delete); + assert_eq!(removed.rows_affected(), ROWS as u64); + assert_eq!(support::int(&db, "SELECT COUNT(*) FROM small"), 0); + assert!(support::rows(&db, "SELECT id FROM small LIMIT 1").is_empty()); +} + +#[test] +fn an_update_or_delete_over_more_than_a_batch_changes_each_row_once_and_keeps_every_count() { + // 1,200 rows are two batches (the first reads 1,024): a row the first batch moves lands + // where the second still has to read, and a statement that met it again would move it again. + const ROWS: i64 = 1200; + let db = support::open(&support::folder("update-along-what-it-finds-by")); + support::run( + &db, + "CREATE TABLE moves (k INTEGER PRIMARY KEY, origin INTEGER NOT NULL, \ + place INTEGER NOT NULL, g INTEGER NOT NULL, UNIQUE (g, k)) MAINTAIN COUNT (g)", + ); + support::run(&db, "CREATE INDEX moves_place ON moves (place)"); + let tx = db.transaction().expect("begin"); + for n in 1..=ROWS { + tx.execute( + "INSERT INTO moves (k, origin, place, g) VALUES ($n, $n, $n, $g)", + ¶ms! { "n" => n, "g" => n % 7 }, + ) + .expect("insert a row"); + } + tx.commit().expect("commit"); + // The rows the store should hold, as (k, origin, place, g), told apart by `origin`. + let mut model: Vec<[i64; 4]> = (1..=ROWS).map(|n| [n, n, n, n % 7]).collect(); + // Asserts the rows, and that each count of `g` is the rows of that `g` and read none. + let agree = |model: &[[i64; 4]], when: &str| { + let wanted: Vec> = model + .iter() + .map(|r| vec![Value::Int64(r[1]), Value::Int64(r[0]), Value::Int64(r[2])]) + .collect(); + let found = support::rows(&db, "SELECT origin, k, place FROM moves ORDER BY origin"); + support::assert_same_rows(&found, &wanted, when); + for g in 0..7_i64 { + let count = support::run(&db, &format!("SELECT COUNT(*) FROM moves WHERE g = {g}")); + let rows = model.iter().filter(|r| r[3] == g).count() as i64; + assert_eq!( + count.rows(), + [[Value::Int64(rows)]], + "{when}: count of g = {g}" + ); + assert_eq!( + count.rows_examined(), + 0, + "{when}: rows read to count g = {g}" + ); + } + }; + + // Each statement finds its rows by the key or by the index it assigns, and moves them + // inside the range it reads; `Some(n)` is the delta to the key, `None` a DELETE. + type Statement = (&'static str, fn(&[i64; 4]) -> bool, Option<(i64, i64)>); + let statements: [Statement; 6] = [ + ( + "UPDATE moves SET k = k + 1000 WHERE k >= 1 AND k <= 1200", + |_| true, + Some((1000, 0)), + ), + ( + "UPDATE moves SET place = place + 1000 WHERE place >= 1 AND place <= 1200", + |_| true, + Some((0, 1000)), + ), + ( + "UPDATE moves SET place = place + 100 WHERE place > 0", + |_| true, + Some((0, 100)), + ), + ( + "UPDATE moves SET place = place + 300 WHERE place >= 1400 AND place < 2000", + |r| (1400..2000).contains(&r[2]), + Some((0, 300)), + ), + ( + "UPDATE moves SET place = place - 50 WHERE place > 50", + |_| true, + Some((0, -50)), + ), + ( + "DELETE FROM moves WHERE place >= 2100", + |r| r[2] >= 2100, + None, + ), + ]; + for (sql, matches, delta) in statements { + let hit = model.iter().filter(|r| matches(r)).count() as u64; + let done = support::run(&db, sql); + assert_eq!(done.rows_affected(), hit, "rows changed by {sql}"); + match delta { + Some((k, place)) => { + for r in model.iter_mut().filter(|r| matches(r)) { + r[0] += k; + r[2] += place; + } + } + None => model.retain(|r| !matches(r)), + } + assert!(hit > 0, "{sql} matched no row"); + agree(&model, sql); + assert_eq!( + support::int(&db, "SELECT COUNT(*) FROM moves"), + model.len() as i64 + ); + } + // Two statements in one transaction, downwards. + let tx = db.transaction().expect("begin"); + for _ in 0..2 { + let done = support::run(&tx, "UPDATE moves SET place = place - 50 WHERE place > 50"); + assert_eq!(done.rows_affected(), model.len() as u64); + model.iter_mut().for_each(|r| r[2] -= 50); + } + tx.commit().expect("commit"); + agree(&model, "after two moves down in a transaction"); +} + +#[test] +fn paging_one_stream_by_its_two_column_key_reads_a_page_however_long_the_stream_is() { + let db = support::open(&support::folder("paging-a-stream")); + support::run( + &db, + "CREATE TABLE stream_events (stream TEXT NOT NULL, ordinal INTEGER NOT NULL, body TEXT, \ + PRIMARY KEY (stream, ordinal))", + ); + // Adds the events `from..=to` to a stream. + let append = |stream: &str, from: i64, to: i64| { + let tx = db.transaction().expect("begin"); + for ordinal in from..=to { + tx.execute( + "INSERT INTO stream_events (stream, ordinal, body) VALUES ($stream, $ordinal, $body)", + ¶ms! { "stream" => stream, "ordinal" => ordinal, "body" => format!("{stream}-{ordinal}") }, + ) + .expect("insert an event"); + } + tx.commit().expect("commit"); + }; + // The page of at most ten events of stream `b` after `committed`. + let page = |committed: i64| { + db.execute( + "SELECT ordinal, body FROM stream_events WHERE stream = $stream AND ordinal > $committed \ + ORDER BY ordinal ASC LIMIT 10", + ¶ms! { "stream" => "b", "committed" => committed }, + ) + .expect("read a page of the stream") + }; + let events = |ordinals: std::ops::RangeInclusive| -> Vec> { + ordinals + .map(|ordinal| vec![Value::Int64(ordinal), Value::Text(format!("b-{ordinal}"))]) + .collect() + }; + // Streams on either side of `b` in key order, each with ordinals the pages ask for. + append("a", 1, 4000); + append("c", 1, 4000); + + // A thousand events, of which twenty are still to be read: the page is the first ten of + // them and costs ten rows. + append("b", 1, 1000); + let first = page(980); + support::assert_same_rows(first.rows(), &events(981..=990), "the page after 980"); + assert_eq!(first.rows_examined(), 10); + + // Two thousand more, read up to twenty before the end again: the same ten rows of work. + append("b", 1001, 3000); + let second = page(2980); + support::assert_same_rows(second.rows(), &events(2981..=2990), "the page after 2980"); + assert_eq!(second.rows_examined(), first.rows_examined()); + + // A tail shorter than the page ends at the stream's end, not in the next stream. + let tail = page(2995); + support::assert_same_rows(tail.rows(), &events(2996..=3000), "the tail of the stream"); + assert_eq!(tail.rows_examined(), 5); + let end = page(3000); + assert!(end.rows().is_empty()); + assert_eq!(end.rows_examined(), 0); +} + +#[test] +fn an_update_of_one_indexed_column_leaves_a_long_value_of_another_found_by_equality() { + // Values of 3,000 bytes that differ only in their last ten. + let long = |id: i64| format!("{}{id:010}", "v".repeat(2990)); + let db = support::open(&support::folder("update-beside-a-long-indexed-value")); + support::run( + &db, + "CREATE TABLE notes (id INTEGER PRIMARY KEY, label TEXT NOT NULL, body TEXT NOT NULL)", + ); + support::run(&db, "CREATE INDEX notes_label ON notes (label)"); + support::run(&db, "CREATE INDEX notes_body ON notes (body)"); + for id in 1..=3_i64 { + db.execute( + "INSERT INTO notes (id, label, body) VALUES ($id, $label, $body)", + ¶ms! { "id" => id, "label" => format!("label-{id}"), "body" => long(id) }, + ) + .expect("insert a note with a long indexed value"); + } + // The ids and labels of the notes whose body equals the long value of `id`. + let by_body = |id: i64| { + db.execute( + "SELECT id, label FROM notes WHERE body = $body", + ¶ms! { "body" => long(id) }, + ) + .expect("read by the long value") + .rows() + .to_vec() + }; + let note = |id: i64, label: &str| vec![vec![Value::Int64(id), support::text(label)]]; + for id in 1..=3 { + support::assert_same_rows( + &by_body(id), + ¬e(id, &format!("label-{id}")), + "as inserted", + ); + } + + let changed = support::run(&db, "UPDATE notes SET label = 'moved' WHERE id = 2"); + assert_eq!(changed.rows_affected(), 1); + let changed = support::run( + &db, + "UPDATE notes SET label = 'found' WHERE label = 'label-3'", + ); + assert_eq!(changed.rows_affected(), 1); + + support::assert_same_rows(&by_body(1), ¬e(1, "label-1"), "the note left alone"); + support::assert_same_rows(&by_body(2), ¬e(2, "moved"), "the note changed by key"); + support::assert_same_rows(&by_body(3), ¬e(3, "found"), "the note changed by index"); + assert_eq!(support::int(&db, "SELECT COUNT(*) FROM notes"), 3); + assert_eq!( + support::int(&db, "SELECT id FROM notes WHERE label = 'moved'"), + 2 + ); + assert!(support::rows(&db, "SELECT id FROM notes WHERE label = 'label-2'").is_empty()); + let body = support::rows(&db, "SELECT body FROM notes WHERE id = 2"); + assert_eq!(body, vec![vec![support::text(&long(2))]]); +} + +const KEYED: &str = "SELECT k, v FROM t ORDER BY k"; + +/// A store with `t`, keyed by `k`: rows 1 `one`, 2 `two` and 3 `three`. +fn store_of_three_keys(name: &str) -> Database { + let db = support::open(&support::folder(name)); + support::run( + &db, + "CREATE TABLE t (k INTEGER PRIMARY KEY, v TEXT NOT NULL)", + ); + support::run( + &db, + "INSERT INTO t (k, v) VALUES (1, 'one'), (2, 'two'), (3, 'three')", + ); + db +} + +#[test] +fn an_update_that_gives_two_rows_one_key_is_a_unique_violation_and_changes_nothing() { + let db = store_of_three_keys("two-rows-one-key"); + let before = support::rows(&db, KEYED); + match support::refusal(&db, "UPDATE t SET k = 7 WHERE k IN (1, 2)") { + error @ Error::UniqueViolation { .. } => assert_eq!( + error.to_string(), + "unique constraint violation: t (k) = (7)", + "the refusal says which key two rows were given, as Postgres does" + ), + other => panic!("an UPDATE that gives two rows the key 7 was refused as {other:?}"), + } + assert_eq!(support::rows(&db, KEYED), before); + assert!(support::rows(&db, "SELECT v FROM t WHERE k = 7").is_empty()); + assert_eq!( + support::rows(&db, "SELECT v FROM t WHERE k = 2"), + [[support::text("two")]] + ); +} diff --git a/v2/contextdb/tests/vectors.rs b/v2/contextdb/tests/vectors.rs index 75f380ef..e18aac43 100644 --- a/v2/contextdb/tests/vectors.rs +++ b/v2/contextdb/tests/vectors.rs @@ -58,7 +58,7 @@ fn told_apart(n: i64, dimension: usize) -> Option> { #[test] fn exact_search_returns_the_true_nearest_vectors() { - for (dimension, rows) in [(3, 5000_i64), (16, 5000), (33, 600), (384, 300), (768, 300)] { + for (dimension, rows) in [(3, 5000_i64), (16, 300), (33, 300), (384, 300), (768, 300)] { let folder = support::folder(&format!("exact-nearest-{dimension}")); let db = support::open(&folder); support::declare_docs(&db, dimension, ""); diff --git a/v2/contextdb/tests/vectors_inspection.rs b/v2/contextdb/tests/vectors_inspection.rs index d06b1a0c..059f1003 100644 --- a/v2/contextdb/tests/vectors_inspection.rs +++ b/v2/contextdb/tests/vectors_inspection.rs @@ -4,9 +4,10 @@ mod support; +use std::collections::BTreeSet; use std::sync::atomic::Ordering; -use contextdb::{DatabaseOpenOptions, Error, ReadSession, Value, VectorIndexRef}; +use contextdb::{ContextId, DatabaseOpenOptions, Error, ReadSession, Uuid, Value, VectorIndexRef}; use support::{Run, Shown}; @@ -355,6 +356,31 @@ fn explain_names_how_each_partition_is_searched() { assert!(lines.iter().any(unbound), "{explained}"); assert_eq!(routes(&lines).len(), 2); + // A handle narrowed to Context 1 is told nothing of the partitions of other Contexts: no + // partition line, no count, no id of Context 2; the unnarrowed handle's text is unchanged. + support::run( + &db, + "CREATE TABLE scoped (id INTEGER PRIMARY KEY, context_id UUID NOT NULL CONTEXT_ID, \ + v VECTOR(2) PARTITION_KEY (context_id))", + ); + let (one, two) = (Uuid::from_u128(0x00C0_0001), Uuid::from_u128(0x00C0_0002)); + for (id, context) in [(1, one), (2, two)] { + let sql = format!("INSERT INTO scoped VALUES ({id}, '{context}', [1.0, 0.0])"); + support::run(&db, &sql); + } + let sql = "SELECT id, v <=> [1.0, 0.0] AS distance FROM scoped ORDER BY distance LIMIT 10"; + let whole = db.explain(sql).expect("explain"); + assert!(whole.contains(&two.to_string()), "{whole}"); + let narrowed = db.scoped_with_contexts(BTreeSet::from([ContextId::new(one)])); + let seen = narrowed.explain(sql).expect("a narrowed handle explains"); + assert!( + !seen.contains("partition:") + && !seen.contains("partitions:") + && !seen.contains(&two.to_string()), + "a handle narrowed to Context 1 was told of other partitions:\n{seen}" + ); + assert_eq!(db.explain(sql).expect("explain"), whole); + // Any other statement is a later version's to explain. match db.explain("SELECT id FROM docs") { Err(error) => assert!( diff --git a/v2/contextdb/tests/vectors_memory.rs b/v2/contextdb/tests/vectors_memory.rs index 1db16408..0645d541 100644 --- a/v2/contextdb/tests/vectors_memory.rs +++ b/v2/contextdb/tests/vectors_memory.rs @@ -198,17 +198,16 @@ fn vector_search_ends_complete_or_refused() { #[test] fn memory_stays_bounded_as_data_grows_vector() { - const DIMENSION: usize = 768; - // Vectors in the large partition. In a block each takes 8 + 768 bytes of slot, 4 of key - // end and 9 of key, so these are 25 MB of blocks: three times the page cache of 8 MiB - // that the reader's limit gives it, and the fewest that are. - const LARGE: i64 = 31_896; + const DIMENSION: usize = 3072; + // The large partition, in blocks of 8 + 3,072 bytes of slot, 4 of key end and 9 of key each: + // 12.6 MB, one and a half times the reader's 8 MiB page cache, and the fewest that are. + const LARGE: i64 = 4_069; const SMALL: i64 = 200; const LIMIT: usize = 16 * MIB; const CACHE: usize = 8 * MIB; const BLOCK: u64 = 262_080; - const { assert!(LARGE as u64 * (8 + 768 + 4 + 9) > 3 * CACHE as u64) }; - const { assert!((LARGE as u64 - 1) * (8 + 768 + 4 + 9) <= 3 * CACHE as u64) }; + const { assert!(LARGE as u64 * (8 + 3072 + 4 + 9) > 3 * CACHE as u64 / 2) }; + const { assert!((LARGE as u64 - 1) * (8 + 3072 + 4 + 9) <= 3 * CACHE as u64 / 2) }; let folder = support::folder("memory-bounded-vectors"); let db = support::open(&folder); @@ -448,7 +447,41 @@ fn a_limit_below_the_sessions_cache_is_refused_and_one_above_it_is_taken() { // Under the 1 GiB the store remembers the cache is 256 MiB, and 64 MiB is refused. refused_below_the_cache(set(&db, 64), 64 * MIB, 257 * MIB); assert_eq!(db.memory_limit(), Some(1024 * MIB)); + support::close(db); + } + + // An open under 1 MiB, for writing and for reading, is refused by the ordinary memory + // refusal with the bytes it needed and the limit; it leaves the store as it was. + let folder = support::folder("open-below-1-mib"); + support::close(support::open(&folder)); + let (mut write, mut read) = ( + DatabaseOpenOptions::default(), + ReadSessionOptions::default(), + ); + (write.memory_limit, read.memory_limit) = (Some(MIB), Some(MIB)); + let refusals = [ + ("writing", support::open_with(&folder, write).err()), + ( + "reading", + ReadSession::open_with_options(&folder, read).err(), + ), + ]; + for (how, refusal) in refusals { + let Some(Error::MemoryBudgetExceeded { + requested_bytes, + budget_limit_bytes, + .. + }) = refusal + else { + panic!("an open for {how} under 1 MiB was not a memory refusal: {refusal:?}"); + }; + assert!( + budget_limit_bytes == MIB as u64 && requested_bytes > MIB as u64, + "an open for {how}: the refusal names the limit ({budget_limit_bytes}) and the \ + bytes the session needs ({requested_bytes}), which are more than the limit" + ); } + support::close(support::open(&folder)); } #[test] diff --git a/v2/contextdb/tests/vectors_partitions.rs b/v2/contextdb/tests/vectors_partitions.rs index e36fba63..68659513 100644 --- a/v2/contextdb/tests/vectors_partitions.rs +++ b/v2/contextdb/tests/vectors_partitions.rs @@ -5,7 +5,12 @@ mod support; -use contextdb::{Database, Error, ReadSession, ReadSessionOptions, Value, VectorIndexRef, params}; +use std::sync::atomic::Ordering; + +use contextdb::{ + Database, DatabaseOpenOptions, Error, ReadSession, ReadSessionOptions, Value, VectorIndexRef, + params, +}; use support::{Child, Near, Run}; @@ -456,6 +461,24 @@ fn vector_partition_limit_counts_and_refuses() { [[Value::Int64(30)], [Value::Int64(32)]] ); + // A statement refused after it opened a tuple puts the count back, whatever refused it: + // with b emptied, a two-row INSERT opens c and then breaks the key on its second row, and + // b is still the tuple there is room for. + let tx = db.transaction().expect("begin"); + support::run(&tx, "DELETE FROM docs WHERE ctx = 'b'"); + let broken = tx + .execute( + "INSERT INTO docs (id, ctx, tag, v) VALUES (90, 'c', 0, $v), (1, 'a', 0, $v)", + ¶ms! { "v" => support::vector(90, 8) }, + ) + .expect_err("the second row repeats a key"); + assert!( + !matches!(broken, Error::VectorPartitionLimitExceeded { .. }), + "the second row breaks the key, not the limit: {broken:?}" + ); + insert_one(&tx, 91, "b").expect("b once the refused statement left its count"); + tx.rollback(); + // In one transaction at the limit: deleting a partition's last vector and then inserting // under a new tuple succeeds, and inserting under the emptied tuple after that is refused. let tx = db.transaction().expect("begin"); @@ -563,6 +586,57 @@ fn vector_partition_limit_counts_and_refuses() { ); assert_eq!(work(&db), read_before); assert!(state_of(&db).2 == sealed.2); + + // The storage reads of one INSERT under an existing partition, from a cold cache, are the + // same with 8 partitions and with 512, within the depth of the file's tree: the insert + // finds its partition by its key and does not visit the others. The key tuples are long, so + // that 512 of them cannot sit in the one or two pages that 8 do. + let reads_of_one_insert = |partitions: i64| -> u64 { + let folder = support::folder(&format!("insert-reads-{partitions}-partitions")); + let db = support::open(&folder); + support::run( + &db, + "CREATE TABLE long_keys (id INTEGER PRIMARY KEY, ctx TEXT NOT NULL, \ + v VECTOR(4) PARTITION_KEY (ctx) MAX_PARTITIONS 4096)", + ); + let insert = "INSERT INTO long_keys (id, ctx, v) VALUES ($id, $ctx, $v)"; + let tx = db.transaction().expect("begin"); + for n in 1..=partitions { + let ctx = format!("{n:0>200}"); + tx.execute( + insert, + ¶ms! { "id" => n, "ctx" => ctx, "v" => support::vector(n, 4) }, + ) + .unwrap_or_else(|error| panic!("partition {n} was refused: {error:?}")); + } + tx.commit().expect("commit the partitions"); + support::close(db); + + let (storage, bytes) = support::counting_bytes_read(); + let mut options = DatabaseOpenOptions::default(); + options.storage = Some(storage); + let db = support::open_with(&folder, options).expect("open with a counting door"); + let opened = bytes.load(Ordering::SeqCst); + db.execute( + insert, + ¶ms! { + "id" => partitions + 1, + "ctx" => format!("{:0>200}", 1), + "v" => support::vector(partitions + 1, 4), + }, + ) + .expect("a row under an existing partition"); + bytes.load(Ordering::SeqCst) - opened + }; + // The promise: the bytes read for one insert at 512 partitions are at most four times + // those at 8: a walk of the partition list reads 64 times as much, a lookup through a few + // trees that each grow a level reads a few more pages. + let (few, most) = (reads_of_one_insert(8), reads_of_one_insert(512)); + assert!( + most <= few * 4, + "one insert read {few} bytes of the file with 8 partitions and {most} with 512: its \ + reads grow with the number of partitions as a walk of the partition list does" + ); } /// Rows in each of the two partitions of the next test. diff --git a/v2/contextdb/tests/vectors_statements.rs b/v2/contextdb/tests/vectors_statements.rs index 1a0abe60..5d3696ad 100644 --- a/v2/contextdb/tests/vectors_statements.rs +++ b/v2/contextdb/tests/vectors_statements.rs @@ -1125,12 +1125,16 @@ fn a_distance_on_a_column_of_another_table_is_a_column_that_is_not_found() { let mut wrong = Vec::new(); for qualifier in ["other", "nowhere"] { // What any column written under that name is refused as: not found, naming the - // statement's table and the column. + // qualifier as written and the column. let plain = format!("SELECT {qualifier}.v FROM docs LIMIT 3"); let not_found = refused_at_every_door(&db, &reader, &plain, &query); match ¬_found { Error::ColumnNotFound { table, column } => { - assert_eq!((table.as_str(), column.as_str()), ("docs", "v"), "{plain}"); + assert_eq!( + (table.as_str(), column.as_str()), + (qualifier, "v"), + "{plain}" + ); } other => panic!("{plain}\nwas refused as {other:?}"), } @@ -1437,24 +1441,36 @@ fn the_index_clauses_are_refused_by_kind() { other => panic!("{sql}\nwas refused as {other:?}"), } } - // What a later version reads is refused by its name. - for (declared, name) in [ + // What no version reads is refused by its name, and the sentence says what to write instead. + let tuning = |name: &str| { + format!( + "{name} (...) is not part of this library: the index sizes its own groups and measures \ + how far a search scans; for the true nearest rows write USE VECTOR EXACT or \ + SEARCH_MODE EXACT, and to move the size from which AUTO uses the index write \ + AUTO_INDEX_AT" + ) + }; + for (declared, sentence) in [ ( "VECTOR(4) WITH (quantization = 'SQ4')", - "WITH (quantization = 'SQ4')", + "WITH (quantization = 'SQ4') is not part of this library: write 'SQ8' to store one \ + byte a component, or leave the clause out to store each vector as given" + .to_owned(), ), - ("VECTOR(4) IVF (LISTS = 4, PROBES = 2)", "IVF"), ( - "VECTOR(4) SEARCH (TARGET_ACCURACY = 0.9, EXPECTED_VECTORS = 100)", - "SEARCH", + "VECTOR(4) IVF (LISTS = 16, PROBES = 4, REFINE_FACTOR = 2)", + tuning("IVF"), + ), + ( + "VECTOR(4) SEARCH (TARGET_ACCURACY = 0.95, EXPECTED_VECTORS = 1000000)", + tuning("SEARCH"), ), ] { let sql = with_vector(declared); - let sentence = support::refusal(&db, &sql).to_string(); - assert!( - sentence.ends_with(&format!("{name} {LATER_VERSION}")), - "{sql}\nwas refused in the sentence: {sentence}" - ); + match support::refusal(&db, &sql) { + Error::PlanError(text) => assert_eq!(text, sentence, "{sql}"), + other => panic!("{sql}\nwas not refused by name: {other:?}"), + } } // A CONSOLIDATION list that is empty, names a member twice or names another member, and // a clause out of its place, are syntax errors. diff --git a/v2/contextdb/tests/vectors_upkeep.rs b/v2/contextdb/tests/vectors_upkeep.rs index 638714eb..2de8f158 100644 --- a/v2/contextdb/tests/vectors_upkeep.rs +++ b/v2/contextdb/tests/vectors_upkeep.rs @@ -709,14 +709,14 @@ fn a_partition_is_grouped_again_only_when_its_changes_reach_the_declared_share() #[test] fn a_cycle_under_a_tight_limit_records_the_stall_and_the_old_grouping_serves() { - // A round over 49 centres of 256 numbers takes 150,528 bytes for its sums and centres; + // A round over 22 centres of 512 numbers takes 135,168 bytes for its sums and centres; // the limit leaves 98,304 over what an open sets aside. - const WIDE: usize = 256; + const WIDE: usize = 512; const LIMIT: usize = (9 << 20) + (96 << 10); - let (folder, db) = support::built("stalled-build", WIDE, AT_1000, 2000); - support::load_docs(&db, 2001..=2400, "a", |n| support::gathered(n, WIDE)); - // This cycle seals the 400 and begins the regrouping they make due. - assert_eq!(support::cycle(&db).maintenance_vectors_done, 400); + let (folder, db) = support::built("stalled-build", WIDE, "AUTO_INDEX_AT 200", 400); + support::load_docs(&db, 401..=480, "a", |n| support::gathered(n, WIDE)); + // This cycle seals the 80 and begins the regrouping they make due. + assert_eq!(support::cycle(&db).maintenance_vectors_done, 80); let stored = support::stored(&db, support::DOCS); support::close(db); @@ -749,7 +749,7 @@ fn a_cycle_under_a_tight_limit_records_the_stall_and_the_old_grouping_serves() { ); // The grouping in force still serves. - assert_eq!(lists_of(&partition), Some(45)); + assert_eq!(lists_of(&partition), Some(20)); let query = support::gathered(60_003, WIDE); let result = support::answer(&db, &support::by_mode(None, "INDEXED", 10), &query); support::assert_sound(&result, &stored, |_| true, &query, 10, "while stalled"); @@ -758,6 +758,6 @@ fn a_cycle_under_a_tight_limit_records_the_stall_and_the_old_grouping_serves() { support::upkeep(&db); let partition = support::the_partition(&db); assert_eq!(state_of(&partition), ("idle".to_owned(), "none".to_owned())); - assert_eq!(lists_of(&partition), Some(49), "round(sqrt(2400))"); + assert_eq!(lists_of(&partition), Some(22), "round(sqrt(480))"); assert_eq!(support::cycle(&db).stalled_partitions, 0); } diff --git a/v2/contextdb/tests/visibility.rs b/v2/contextdb/tests/visibility.rs new file mode 100644 index 00000000..6d3c7890 --- /dev/null +++ b/v2/contextdb/tests/visibility.rs @@ -0,0 +1,1600 @@ +//! Who may see and write a row. A handle narrowed to Contexts, to scope labels or to a principal +//! reads the rows it may and no others, changes none it may not see, is refused a row it names +//! by its key by the rule that hid it, and is told nothing of what it may not see: not by a +//! count, not by a search, not by the words of a refusal. + +mod support; + +use std::collections::{BTreeSet, HashMap}; + +use contextdb::{ + ContextId, Database, DatabaseOpenOptions, Error, Principal, ScopeLabel, Uuid, Value, params, +}; + +use support::Run; + +// ---- who is asking ---- + +/// Context number `n`. +fn context(n: u128) -> Uuid { + Uuid::from_u128(0x00C0_0000 + n) +} + +/// Access-control id number `n`. +fn acl(n: u128) -> Uuid { + Uuid::from_u128(0x00AC_0000 + n) +} + +fn context_set(numbers: &[u128]) -> BTreeSet { + numbers + .iter() + .map(|n| ContextId::new(context(*n))) + .collect() +} + +fn label_set(names: &[&str]) -> BTreeSet { + names.iter().map(|name| ScopeLabel::new(*name)).collect() +} + +/// A handle narrowed to the Contexts with these numbers. +fn in_contexts(db: &Database, numbers: &[u128]) -> Database { + db.scoped_with_contexts(context_set(numbers)) +} + +/// A handle narrowed to these scope labels. +fn with_labels(db: &Database, names: &[&str]) -> Database { + db.scoped_with_constraints(None, Some(label_set(names)), None) +} + +/// A handle that names this principal and nothing else. +fn named(db: &Database, principal: Principal) -> Database { + db.scoped_with_constraints(None, None, Some(principal)) +} + +fn agent(name: &str) -> Principal { + Principal::Agent(name.to_owned()) +} + +fn human(name: &str) -> Principal { + Principal::Human(name.to_owned()) +} + +// ---- what a statement answered ---- + +/// The first column of every row a statement returns, which must be an INTEGER. +fn ids(on: &impl Run, sql: &str) -> Vec { + ids_with(on, sql, ¶ms! {}) +} + +fn ids_with(on: &impl Run, sql: &str, values: &HashMap) -> Vec { + let result = match on.run(sql, values) { + Ok(result) => result, + Err(error) => panic!("{sql}\nwas refused: {error:?}"), + }; + let first = |row: &Vec| match row.first() { + Some(Value::Int64(id)) => *id, + other => panic!("{sql}\nreturned {other:?} where an INTEGER was wanted"), + }; + result.rows().iter().map(first).collect() +} + +/// The ids `SELECT id FROM
` returns. +fn seen(on: &impl Run, table: &str, tail: &str) -> Vec { + ids(on, &format!("SELECT id FROM {table} {tail}")) +} + +/// The first column of every row a statement returns, which must be TEXT. +fn texts(on: &impl Run, sql: &str) -> Vec { + let first = |row: &Vec| match row.first() { + Some(Value::Text(text)) => text.clone(), + other => panic!("{sql}\nreturned {other:?} where a TEXT was wanted"), + }; + support::rows(on, sql).iter().map(first).collect() +} + +/// How many rows a write changed. The write must not be refused. +fn changed(on: &impl Run, sql: &str) -> u64 { + support::run(on, sql).rows_affected() +} + +/// The rule a narrowed handle was refused by. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Rule { + Context, + Label, + Grant, + NoPrincipal, +} + +/// Asserts that a statement is refused by this rule, naming this table, and returns the +/// refusal. +fn refused_with( + on: &impl Run, + sql: &str, + values: &HashMap, + rule: Rule, + table: &str, +) -> Error { + let refusal = match on.run(sql, values) { + Err(refusal) => refusal, + Ok(result) => panic!("{sql}\nwas not refused ({rule:?} was wanted): {result:?}"), + }; + let found = match &refusal { + Error::ContextScopeViolation { table } => (Rule::Context, table.as_str()), + Error::ScopeLabelViolation { table } => (Rule::Label, table.as_str()), + Error::AclDenied { table } => (Rule::Grant, table.as_str()), + Error::PrincipalRequired { table } => (Rule::NoPrincipal, table.as_str()), + other => panic!("{sql}\nwas refused with {other:?}; {rule:?} was wanted"), + }; + assert_eq!(found, (rule, table), "{sql}"); + refusal +} + +fn refused(on: &impl Run, sql: &str, rule: Rule, table: &str) -> Error { + refused_with(on, sql, ¶ms! {}, rule, table) +} + +// ---- the documented tables: a grant table and notes under a Context and an access id ---- + +const GRANTS: &str = "CREATE TABLE acl_grants (id UUID PRIMARY KEY, principal_kind TEXT, \ + principal_id TEXT, acl_id UUID)"; + +const NOTES: &str = "CREATE TABLE notes (id INTEGER PRIMARY KEY, context_id UUID CONTEXT_ID, \ + acl_id UUID ACL REFERENCES acl_grants(acl_id), payload TEXT)"; + +/// Grants a principal one access-control id. `number` makes the grant row's own key. +fn grant(db: &Database, number: u128, kind: &str, name: &str, acl_id: Uuid) { + let values = params! { + "id" => Uuid::from_u128(0x0060_0000 + number), "kind" => kind, "name" => name, + "acl" => acl_id, + }; + let sql = "INSERT INTO acl_grants (id, principal_kind, principal_id, acl_id) \ + VALUES ($id, $kind, $name, $acl)"; + if let Err(error) = db.execute(sql, &values) { + panic!("granting {acl_id} to {kind} {name} was refused: {error:?}"); + } +} + +/// Writes note `id`, with the payload `note-`, under this Context and this access id, or +/// under none of either for `None`. +fn note( + on: &impl Run, + id: i64, + context_number: Option, + acl_number: Option, +) -> contextdb::Result { + let uuid = |id: Option| id.map_or(Value::Null, Value::Uuid); + on.run( + "INSERT INTO notes (id, context_id, acl_id, payload) VALUES ($id, $context, $acl, $payload)", + ¶ms! { + "id" => id, "context" => uuid(context_number.map(context)), + "acl" => uuid(acl_number.map(acl)), "payload" => format!("note-{id}"), + }, + ) +} + +/// A store with the documented tables. The agent `report-reader` holds access id 1 and the +/// human `ana` holds access id 2; nobody holds access id 3. The notes: +/// +/// | id | Context | access id | +/// |---|---|---| +/// | 1 | 1 | 1 | +/// | 2 | 1 | 2 | +/// | 3 | 2 | 1 | +/// | 4 | 2 | 2 | +/// | 5 | 1 | 3 | +/// | 6 | 1 | none | +/// | 7 | none | 1 | +fn notes(name: &str) -> Database { + let db = support::open(&support::folder(name)); + support::run(&db, GRANTS); + support::run(&db, NOTES); + grant(&db, 1, "Agent", "report-reader", acl(1)); + grant(&db, 2, "Human", "ana", acl(2)); + for (id, context_number, acl_number) in [ + (1, Some(1), Some(1)), + (2, Some(1), Some(2)), + (3, Some(2), Some(1)), + (4, Some(2), Some(2)), + (5, Some(1), Some(3)), + (6, Some(1), None), + (7, None, Some(1)), + ] { + if let Err(error) = note(&db, id, context_number, acl_number) { + panic!("note {id} was refused: {error:?}"); + } + } + db +} + +const ALL_NOTES: &str = "SELECT id FROM notes ORDER BY id"; + +#[test] +fn a_handle_that_declares_nothing_reads_every_row_of_an_access_controlled_table() { + let db = notes("visibility-declares-nothing"); + assert_eq!(ids(&db, ALL_NOTES), [1, 2, 3, 4, 5, 6, 7]); + // A handle made with no Context, no label and no principal declared nothing either. + let unnarrowed = db.scoped_with_constraints(None, None, None); + assert_eq!(ids(&unnarrowed, ALL_NOTES), [1, 2, 3, 4, 5, 6, 7]); + assert_eq!( + texts(&unnarrowed, "SELECT payload FROM notes WHERE id = 5"), + ["note-5"] + ); + assert_eq!(support::int(&unnarrowed, "SELECT COUNT(*) FROM notes"), 7); +} + +#[test] +fn a_principal_reads_the_rows_it_holds_a_grant_for_and_no_others() { + let db = notes("visibility-grants"); + let reader = named(&db, agent("report-reader")); + assert_eq!(ids(&reader, ALL_NOTES), [1, 3, 7]); + assert_eq!( + texts(&reader, "SELECT payload FROM notes ORDER BY id"), + ["note-1", "note-3", "note-7"] + ); + assert_eq!(support::int(&reader, "SELECT COUNT(*) FROM notes"), 3); + assert_eq!(ids(&named(&db, human("ana")), ALL_NOTES), [2, 4]); + + // The kind is part of who a principal is, and a principal nobody granted anything reads an + // empty table, without a refusal. + for nobody in [agent("ana"), human("report-reader"), agent("nobody")] { + let handle = named(&db, nobody.clone()); + assert_eq!(ids(&handle, ALL_NOTES), [0_i64; 0], "{nobody:?}"); + assert_eq!( + support::int(&handle, "SELECT COUNT(*) FROM notes"), + 0, + "{nobody:?}" + ); + } + + // A grant is read when a statement runs: one given later shows its rows to the handle that + // was made before it, and one taken away hides them again. + grant(&db, 3, "Agent", "report-reader", acl(3)); + assert_eq!(ids(&reader, ALL_NOTES), [1, 3, 5, 7]); + let first_grant = params! { "id" => Uuid::from_u128(0x0060_0001) }; + let taken = db + .execute("DELETE FROM acl_grants WHERE id = $id", &first_grant) + .expect("take the first grant away"); + assert_eq!(taken.rows_affected(), 1); + assert_eq!(ids(&reader, ALL_NOTES), [5]); + assert_eq!(ids(&db, ALL_NOTES), [1, 2, 3, 4, 5, 6, 7]); +} + +#[test] +fn a_handle_with_a_context_and_a_principal_sees_only_granted_rows_of_its_context() { + let db = notes("visibility-context-and-principal"); + let both = |numbers: &[u128], principal: Principal| { + db.scoped_with_constraints(Some(context_set(numbers)), None, Some(principal)) + }; + let reader = both(&[1], agent("report-reader")); + assert_eq!(ids(&reader, ALL_NOTES), [1]); + assert_eq!(support::int(&reader, "SELECT COUNT(*) FROM notes"), 1); + assert_eq!(ids(&both(&[2], agent("report-reader")), ALL_NOTES), [3]); + assert_eq!(ids(&both(&[1, 2], human("ana")), ALL_NOTES), [2, 4]); + assert_eq!(ids(&both(&[1, 2], agent("nobody")), ALL_NOTES), [0_i64; 0]); + + // Named by its key, a hidden row is refused by the rule that hid it, and the Context is + // asked before the grant: note 3 is granted and outside the Context, note 2 is inside it + // and not granted, note 4 is neither. + let by_key = |id: i64| format!("SELECT payload FROM notes WHERE id = {id}"); + assert_eq!(texts(&reader, &by_key(1)), ["note-1"]); + refused(&reader, &by_key(3), Rule::Context, "notes"); + refused(&reader, &by_key(2), Rule::Grant, "notes"); + refused(&reader, &by_key(4), Rule::Context, "notes"); + // Note 7 is under no Context: it is outside every Context a handle can be narrowed to. + refused(&reader, &by_key(7), Rule::Context, "notes"); + assert!(texts(&reader, &by_key(99)).is_empty()); + + // All three rules on one table: a row that fails two is refused by the first asked, the + // Context, then the label, then the grant. + support::run( + &db, + "CREATE TABLE guarded (id INTEGER PRIMARY KEY, context_id UUID CONTEXT_ID, label TEXT \ + SCOPE_LABEL_READ ('public') WRITE ('public'), acl_id UUID ACL REFERENCES \ + acl_grants(acl_id))", + ); + let triple = db.scoped_with_constraints( + Some(context_set(&[1])), + Some(label_set(&["public"])), + Some(agent("report-reader")), + ); + for (id, context_number, label, acl_number, wanted) in [ + (1, 1, "public", 1, None), + (2, 2, "internal", 1, Some(Rule::Context)), + (3, 2, "public", 2, Some(Rule::Context)), + (4, 1, "internal", 2, Some(Rule::Label)), + (5, 1, "public", 2, Some(Rule::Grant)), + ] { + support::run( + &db, + &format!( + "INSERT INTO guarded VALUES ({id}, '{}', '{label}', '{}')", + context(context_number), + acl(acl_number) + ), + ); + let by_key = format!("SELECT id FROM guarded WHERE id = {id}"); + match wanted { + None => assert_eq!(ids(&triple, &by_key), [id], "row {id}"), + Some(rule) => { + refused(&triple, &by_key, rule, "guarded"); + } + } + } +} + +#[test] +fn a_narrowed_handle_with_no_principal_that_holds_grants_is_refused_an_access_controlled_table() { + let db = notes("visibility-principal-required"); + // The same declaration with no row in it: the refusal is of the table, not of its rows. + support::run( + &db, + "CREATE TABLE empty_notes (id INTEGER PRIMARY KEY, context_id UUID CONTEXT_ID, \ + acl_id UUID ACL REFERENCES acl_grants(acl_id), payload TEXT)", + ); + let handles = [ + ("a Context", in_contexts(&db, &[1])), + ("a label", with_labels(&db, &["public"])), + ( + "a Context and a label", + db.scoped_with_constraints(Some(context_set(&[1])), Some(label_set(&["public"])), None), + ), + ("the system principal", named(&db, Principal::System)), + ( + "a Context and the system principal", + db.scoped_with_constraints(Some(context_set(&[1])), None, Some(Principal::System)), + ), + ]; + let new_note = "INSERT INTO notes (id, context_id, acl_id, payload) \ + VALUES (50, $context, $acl, 'new')"; + let new_values = params! { "context" => context(1), "acl" => acl(1) }; + for (what, handle) in &handles { + for sql in [ + ALL_NOTES, + "SELECT payload FROM notes WHERE id = 1", + "SELECT payload FROM notes WHERE id = 99", + "SELECT id FROM notes WHERE payload = 'note-1'", + "SELECT COUNT(*) FROM notes", + "UPDATE notes SET payload = 'changed' WHERE id = 1", + "UPDATE notes SET payload = 'changed'", + "DELETE FROM notes WHERE id = 1", + "DELETE FROM notes", + ] { + let refusal = refused(handle, sql, Rule::NoPrincipal, "notes"); + let said = refusal.to_string(); + assert!( + !said.chars().any(|c| c.is_ascii_digit()), + "narrowed by {what}, {sql}\nwas refused with a number in its words: {said}" + ); + } + refused_with(handle, new_note, &new_values, Rule::NoPrincipal, "notes"); + refused( + handle, + "SELECT id FROM empty_notes ORDER BY id", + Rule::NoPrincipal, + "empty_notes", + ); + refused( + handle, + "SELECT COUNT(*) FROM empty_notes", + Rule::NoPrincipal, + "empty_notes", + ); + } + // Nothing was changed by any of them. + assert_eq!( + texts(&db, "SELECT payload FROM notes ORDER BY id"), + [ + "note-1", "note-2", "note-3", "note-4", "note-5", "note-6", "note-7" + ] + ); +} + +#[test] +fn a_table_is_narrowed_only_by_what_it_declares() { + let db = support::open(&support::folder("visibility-undeclared")); + support::run( + &db, + "CREATE TABLE plain (id INTEGER PRIMARY KEY, payload TEXT)", + ); + support::run( + &db, + "CREATE TABLE scoped (id INTEGER PRIMARY KEY, context_id UUID CONTEXT_ID, payload TEXT)", + ); + support::run( + &db, + "INSERT INTO plain (id, payload) VALUES (1, 'a'), (2, 'b'), (3, 'c')", + ); + for (id, number) in [(1_i64, 1), (2, 2), (3, 1), (4, 2)] { + db.execute( + "INSERT INTO scoped (id, context_id, payload) VALUES ($id, $context, 'p')", + ¶ms! { "id" => id, "context" => context(number) }, + ) + .expect("insert a scoped row"); + } + let in_context_one = in_contexts(&db, &[1]); + let others = [ + ("a label", with_labels(&db, &["public"])), + ("a principal", named(&db, agent("report-reader"))), + ("the system principal", named(&db, Principal::System)), + ]; + + // A table that declares no Context column, no label and no access column is read whole by + // every handle, and written by it. + assert_eq!( + ids(&in_context_one, "SELECT id FROM plain ORDER BY id"), + [1, 2, 3] + ); + for (what, handle) in &others { + assert_eq!( + ids(handle, "SELECT id FROM plain ORDER BY id"), + [1, 2, 3], + "{what}" + ); + assert_eq!( + support::int(handle, "SELECT COUNT(*) FROM plain"), + 3, + "{what}" + ); + } + assert_eq!( + changed( + &in_context_one, + "INSERT INTO plain (id, payload) VALUES (4, 'd')" + ), + 1 + ); + assert_eq!( + changed(&others[1].1, "UPDATE plain SET payload = 'e' WHERE id = 4"), + 1 + ); + assert_eq!(texts(&db, "SELECT payload FROM plain WHERE id = 4"), ["e"]); + + // A table that declares a Context column and no access column is narrowed by a handle's + // Contexts and by nothing else it declares: no principal is asked for. + assert_eq!( + ids(&in_context_one, "SELECT id FROM scoped ORDER BY id"), + [1, 3] + ); + for (what, handle) in &others { + assert_eq!( + ids(handle, "SELECT id FROM scoped ORDER BY id"), + [1, 2, 3, 4], + "{what}" + ); + } + let principal_in_context = + db.scoped_with_constraints(Some(context_set(&[2])), None, Some(Principal::System)); + assert_eq!( + ids(&principal_in_context, "SELECT id FROM scoped ORDER BY id"), + [2, 4] + ); +} + +// ---- a table under a Context column ---- + +/// A store with `items`: item `n` of 1 to 12 is `(n, , +/// "k", n * 10)`, and item 13 is under no Context. One index, over `(kind, n)`. +fn items(name: &str) -> Database { + let db = support::open(&support::folder(name)); + support::run( + &db, + "CREATE TABLE items (id INTEGER PRIMARY KEY, context_id UUID CONTEXT_ID, \ + kind TEXT NOT NULL, n INTEGER NOT NULL)", + ); + support::run(&db, "CREATE INDEX items_kind ON items (kind, n)"); + for id in 1..=13_i64 { + let of = match id { + 13 => Value::Null, + id if id % 2 == 1 => Value::Uuid(context(1)), + _ => Value::Uuid(context(2)), + }; + if let Err(error) = item(&db, id, of) { + panic!("item {id} was refused: {error:?}"); + } + } + db +} + +fn item(on: &impl Run, id: i64, context_id: Value) -> contextdb::Result { + on.run( + "INSERT INTO items (id, context_id, kind, n) VALUES ($id, $context, $kind, $n)", + ¶ms! { + "id" => id, "context" => context_id, "kind" => format!("k{}", id % 3), "n" => id * 10, + }, + ) +} + +const ALL_ITEMS: &str = "SELECT id FROM items ORDER BY id"; + +#[test] +fn a_handle_narrowed_to_contexts_lists_only_the_rows_of_its_contexts() { + let db = items("visibility-context-lists"); + let one = in_contexts(&db, &[1]); + let two = in_contexts(&db, &[2]); + let lists: [(&Database, &str, &[i64]); 12] = [ + (&one, "ORDER BY id", &[1, 3, 5, 7, 9, 11]), + (&two, "ORDER BY id", &[2, 4, 6, 8, 10, 12]), + // A filter that is no key. + (&one, "WHERE kind = 'k0' ORDER BY id", &[3, 9]), + (&one, "WHERE n >= 100 ORDER BY id", &[11]), + // A range of the key, and a list of keys, hidden ones among them. + (&one, "WHERE id >= 2 AND id <= 9 ORDER BY id", &[3, 5, 7, 9]), + (&one, "WHERE id IN (1, 2, 3, 4, 13) ORDER BY id", &[1, 3]), + (&one, "WHERE id IN (2)", &[]), + // The first rows in order are hidden: the limit is filled with rows the handle sees. + (&two, "ORDER BY id LIMIT 1", &[2]), + (&one, "ORDER BY id DESC LIMIT 2", &[11, 9]), + // In the order an index gives: of kind k1 are items 1, 4, 7, 10 and 13. + (&two, "WHERE kind = 'k1' ORDER BY n LIMIT 2", &[4, 10]), + (&one, "WHERE kind = 'k1' ORDER BY n LIMIT 1", &[1]), + (&one, "WHERE kind = 'k1' ORDER BY n DESC LIMIT 1", &[7]), + ]; + for (handle, sql, wanted) in lists { + assert_eq!(seen(handle, "items", sql), wanted, "{sql}"); + } + + // The two ways to narrow to Contexts make the same handle. + let also_one = db.scoped_with_constraints(Some(context_set(&[1])), None, None); + assert_eq!(ids(&also_one, ALL_ITEMS), [1, 3, 5, 7, 9, 11]); + // Several Contexts are read together; item 13, under none, is in no handle's Contexts. + let both = in_contexts(&db, &[1, 2]); + assert_eq!(ids(&both, ALL_ITEMS), (1..=12).collect::>()); + // A handle given no Context at all is narrowed to none: it sees no row. + let none = in_contexts(&db, &[]); + assert_eq!(ids(&none, ALL_ITEMS), [0_i64; 0]); + assert_eq!(support::int(&none, "SELECT COUNT(*) FROM items"), 0); + // A Context no row is under. + assert_eq!(ids(&in_contexts(&db, &[3]), ALL_ITEMS), [0_i64; 0]); + assert_eq!(ids(&db, ALL_ITEMS), (1..=13).collect::>()); +} + +#[test] +fn a_new_or_changed_row_outside_the_handles_contexts_is_refused_and_nothing_is_written() { + let db = items("visibility-context-writes"); + let one = in_contexts(&db, &[1]); + let before = support::rows(&db, "SELECT id, context_id, kind, n FROM items ORDER BY id"); + let unchanged = |what: &str| { + let now = support::rows(&db, "SELECT id, context_id, kind, n FROM items ORDER BY id"); + support::assert_same_rows(&now, &before, what); + }; + + // A new row under another Context, or under none. + for (id, of) in [(20_i64, Value::Uuid(context(2))), (21, Value::Null)] { + match item(&one, id, of.clone()) { + Err(Error::ContextScopeViolation { table }) => assert_eq!(table, "items"), + other => panic!("a new row under {of:?} gave {other:?}"), + } + unchanged("after a refused insert"); + } + // One statement with a row inside and a row outside writes neither. + refused_with( + &one, + "INSERT INTO items (id, context_id, kind, n) VALUES (22, $mine, 'k', 1), (23, $other, 'k', 2)", + ¶ms! { "mine" => context(1), "other" => context(2) }, + Rule::Context, + "items", + ); + unchanged("after a refused insert of two rows"); + // A row the handle sees may not be moved out of its Contexts. + let to_two = params! { "other" => context(2) }; + for sql in [ + "UPDATE items SET context_id = $other WHERE id = 1", + "UPDATE items SET context_id = $other WHERE kind = 'k0'", + "UPDATE items SET context_id = NULL WHERE id = 3", + ] { + let values = if sql.contains("$other") { + to_two.clone() + } else { + params! {} + }; + refused_with(&one, sql, &values, Rule::Context, "items"); + unchanged(sql); + } + + // Inside its Contexts it writes, and the row is there for every handle that may see it. + item(&one, 24, Value::Uuid(context(1))).expect("a new row inside the handle's Context"); + assert_eq!(ids(&one, "SELECT id FROM items WHERE id = 24"), [24]); + assert_eq!(ids(&db, "SELECT id FROM items WHERE id = 24"), [24]); + assert_eq!( + ids( + &in_contexts(&db, &[2]), + "SELECT id FROM items WHERE id IN (24)" + ), + [0_i64; 0] + ); + let both = in_contexts(&db, &[1, 2]); + assert_eq!( + both.execute( + "UPDATE items SET context_id = $other WHERE id = 24", + &to_two + ) + .expect("move a row between two Contexts of the handle") + .rows_affected(), + 1 + ); + assert_eq!( + ids(&one, "SELECT id FROM items WHERE id IN (24)"), + [0_i64; 0] + ); + + // A transaction of the handle reads and writes within the same reach, and a refused + // statement leaves it usable. + let tx = one.transaction().expect("begin"); + item(&tx, 30, Value::Uuid(context(1))).expect("a new row inside the handle's Context"); + match item(&tx, 31, Value::Uuid(context(2))) { + Err(Error::ContextScopeViolation { table }) => assert_eq!(table, "items"), + other => panic!("a new row outside the Context, in a transaction, gave {other:?}"), + } + assert_eq!(ids(&tx, ALL_ITEMS), [1, 3, 5, 7, 9, 11, 30]); + assert_eq!(changed(&tx, "UPDATE items SET n = 0 WHERE id = 2"), 0); + assert_eq!(changed(&tx, "DELETE FROM items WHERE kind = 'k2'"), 2); + tx.commit().expect("commit"); + assert_eq!( + ids(&db, ALL_ITEMS), + [1, 2, 3, 4, 6, 7, 8, 9, 10, 12, 13, 24, 30] + ); + assert_eq!(ids(&db, "SELECT n FROM items WHERE id = 2"), [20]); +} + +// ---- one table under each rule that hides rows ---- + +/// `ledger`'s rows: id, slug, a, b, whether the handle of [`Gate::handle`] may see the row, +/// payload. `slug` is UNIQUE and so is `(a, b)`. +const LEDGER: [(i64, &str, i64, i64, bool, &str); 4] = [ + (1, "open-one", 1, 1, true, "seen-one"), + (2, "shut-two", 1, 2, false, "secret-two"), + (3, "open-three", 2, 1, true, "seen-three"), + (4, "shut-four", 2, 2, false, "secret-four"), +]; + +const LEDGER_ROWS: &str = "SELECT id, payload FROM ledger ORDER BY id"; + +/// One rule that hides rows, with the table `ledger` declared under it. Rows 1 and 3 are +/// within the reach of the rule's handle; rows 2 and 4 are hidden from it by this rule and by +/// no other. +#[derive(Clone, Copy, Debug)] +enum Gate { + Context, + Label, + Grant, +} + +impl Gate { + const ALL: [Gate; 3] = [Gate::Context, Gate::Label, Gate::Grant]; + + /// The column `reach` as this rule declares it. + fn column(self) -> &'static str { + match self { + Gate::Context => "reach UUID CONTEXT_ID", + Gate::Label => { + "reach TEXT SCOPE_LABEL_READ ('public', 'internal') WRITE ('public', 'internal')" + } + Gate::Grant => "reach UUID ACL REFERENCES acl_grants(acl_id)", + } + } + + /// What `reach` holds in a row the handle may see, and in one it may not. + fn reach(self, seen: bool) -> Value { + match (self, seen) { + (Gate::Context, true) => Value::Uuid(context(1)), + (Gate::Context, false) => Value::Uuid(context(2)), + (Gate::Label, true) => support::text("public"), + (Gate::Label, false) => support::text("internal"), + (Gate::Grant, true) => Value::Uuid(acl(1)), + (Gate::Grant, false) => Value::Uuid(acl(2)), + } + } + + fn handle(self, db: &Database) -> Database { + match self { + Gate::Context => in_contexts(db, &[1]), + Gate::Label => with_labels(db, &["public"]), + Gate::Grant => named(db, agent("report-reader")), + } + } + + /// The refusal of this rule. + fn rule(self) -> Rule { + match self { + Gate::Context => Rule::Context, + Gate::Label => Rule::Label, + Gate::Grant => Rule::Grant, + } + } + + /// A store with `ledger` under this rule and its four rows. + fn store(self, name: &str) -> Database { + let db = support::open(&support::folder(&format!("{name}-{self:?}"))); + if let Gate::Grant = self { + support::run(&db, GRANTS); + grant(&db, 1, "Agent", "report-reader", acl(1)); + grant(&db, 2, "Human", "someone-else", acl(2)); + } + support::run( + &db, + &format!( + "CREATE TABLE ledger (id INTEGER PRIMARY KEY, slug TEXT UNIQUE, a INTEGER, \ + b INTEGER, {}, payload TEXT, UNIQUE (a, b))", + self.column() + ), + ); + for (id, slug, a, b, seen, payload) in LEDGER { + db.execute( + "INSERT INTO ledger (id, slug, a, b, reach, payload) \ + VALUES ($id, $slug, $a, $b, $reach, $payload)", + ¶ms! { + "id" => id, "slug" => slug, "a" => a, "b" => b, "reach" => self.reach(seen), + "payload" => payload, + }, + ) + .unwrap_or_else(|error| panic!("{self:?}: ledger row {id} was refused: {error:?}")); + } + db + } +} + +/// The `(id, payload)` rows of `ledger` with these ids, as they were first written. +fn ledger_rows(of: &[i64]) -> Vec> { + LEDGER + .iter() + .filter(|row| of.contains(&row.0)) + .map(|row| vec![Value::Int64(row.0), support::text(row.5)]) + .collect() +} + +/// A hidden row named whole (by its key, by its one UNIQUE column, by every column of the +/// composite UNIQUE rule) is refused by the rule that hid it; every other read leaves it out +/// and says nothing. +fn a_hidden_row_named_whole_is_refused_and_a_list_read_leaves_it_out(gate: Gate) { + let db = gate.store("visibility-anchor"); + let handle = gate.handle(&db); + let rule = gate.rule(); + + for sql in [ + "SELECT payload FROM ledger WHERE id = 2", + "SELECT payload FROM ledger WHERE slug = 'shut-two'", + "SELECT payload FROM ledger WHERE a = 1 AND b = 2", + "SELECT payload FROM ledger WHERE b = 2 AND a = 2", + "SELECT id, slug, a, b, payload FROM ledger WHERE id = 4", + "SELECT id FROM ledger WHERE slug = 'shut-four'", + ] { + refused(&handle, sql, rule, "ledger"); + } + // The same when the key is a bound value. + let bound: [(&str, HashMap); 3] = [ + ( + "SELECT payload FROM ledger WHERE id = $id", + params! { "id" => 2_i64 }, + ), + ( + "SELECT payload FROM ledger WHERE slug = $slug", + params! { "slug" => "shut-four" }, + ), + ( + "SELECT payload FROM ledger WHERE a = $a AND b = $b", + params! { "a" => 2_i64, "b" => 2_i64 }, + ), + ]; + for (sql, values) in &bound { + refused_with(&handle, sql, values, rule, "ledger"); + } + + // The same shapes over a row within reach answer with it, and over no row with nothing. + for (sql, wanted) in [ + ("SELECT payload FROM ledger WHERE id = 1", "seen-one"), + ( + "SELECT payload FROM ledger WHERE slug = 'open-three'", + "seen-three", + ), + ( + "SELECT payload FROM ledger WHERE a = 2 AND b = 1", + "seen-three", + ), + ] { + assert_eq!(texts(&handle, sql), [wanted], "{gate:?}: {sql}"); + } + for sql in [ + "SELECT payload FROM ledger WHERE id = 99", + "SELECT payload FROM ledger WHERE slug = 'no-such'", + "SELECT payload FROM ledger WHERE a = 9 AND b = 9", + ] { + assert!(texts(&handle, sql).is_empty(), "{gate:?}: {sql}"); + } + + // Every other shape is a list, a key in a list and a key in a range among them. + let lists: [(&str, &[i64]); 12] = [ + ("ORDER BY id", &[1, 3]), + ("WHERE a = 1 ORDER BY id", &[1]), + ("WHERE b = 2 ORDER BY id", &[]), + ("WHERE id IN (2)", &[]), + ("WHERE id IN (1, 2, 4) ORDER BY id", &[1]), + ("WHERE id >= 2 AND id <= 2", &[]), + ("WHERE id > 1 ORDER BY id", &[3]), + ("WHERE payload = 'secret-two'", &[]), + ("WHERE slug IN ('shut-two', 'open-one') ORDER BY id", &[1]), + ("ORDER BY id DESC LIMIT 1", &[3]), + ("ORDER BY slug DESC LIMIT 1", &[3]), + ("ORDER BY slug LIMIT 4", &[1, 3]), + ]; + for (sql, wanted) in lists { + assert_eq!(seen(&handle, "ledger", sql), wanted, "{gate:?}: {sql}"); + } + + // The handle that declares nothing reads all four, however it names them. + assert_eq!(ids(&db, "SELECT id FROM ledger ORDER BY id"), [1, 2, 3, 4]); + assert_eq!( + texts(&db, "SELECT payload FROM ledger WHERE id = 2"), + ["secret-two"] + ); + assert_eq!( + texts(&db, "SELECT payload FROM ledger WHERE a = 2 AND b = 2"), + ["secret-four"] + ); +} + +#[test] +fn a_row_of_another_context_named_by_key_is_a_context_refusal_and_a_list_read_leaves_it_out() { + a_hidden_row_named_whole_is_refused_and_a_list_read_leaves_it_out(Gate::Context); +} + +#[test] +fn a_row_under_another_label_named_by_key_is_a_label_refusal_and_a_list_read_leaves_it_out() { + a_hidden_row_named_whole_is_refused_and_a_list_read_leaves_it_out(Gate::Label); +} + +#[test] +fn a_row_with_no_grant_named_by_key_is_an_access_refusal_and_a_list_read_leaves_it_out() { + a_hidden_row_named_whole_is_refused_and_a_list_read_leaves_it_out(Gate::Grant); +} + +#[test] +fn an_update_or_a_delete_changes_no_row_the_handle_may_not_see() { + for gate in Gate::ALL { + let db = gate.store("visibility-hidden-writes"); + let handle = gate.handle(&db); + let ledger_is = |wanted: &[Vec], what: &str| { + let now = support::rows(&db, LEDGER_ROWS); + support::assert_same_rows(&now, wanted, &format!("{gate:?}: the ledger {what}")); + }; + + // By its key, by a whole UNIQUE rule, by a filter that only hidden rows pass: no row + // is changed, and the statement is not refused. + for sql in [ + "UPDATE ledger SET payload = 'changed' WHERE id = 2", + "UPDATE ledger SET payload = 'changed' WHERE slug = 'shut-four'", + "UPDATE ledger SET payload = 'changed' WHERE a = 1 AND b = 2", + "UPDATE ledger SET payload = 'changed' WHERE b = 2", + "UPDATE ledger SET payload = 'changed' WHERE payload = 'secret-two'", + "UPDATE ledger SET payload = 'changed' WHERE id IN (2, 4)", + "DELETE FROM ledger WHERE id = 2", + "DELETE FROM ledger WHERE slug = 'shut-four'", + "DELETE FROM ledger WHERE a = 2 AND b = 2", + "DELETE FROM ledger WHERE b = 2", + "DELETE FROM ledger WHERE payload = 'secret-four'", + "DELETE FROM ledger WHERE id >= 2 AND id <= 2", + ] { + assert_eq!(changed(&handle, sql), 0, "{gate:?}: {sql}"); + ledger_is(&ledger_rows(&[1, 2, 3, 4]), &format!("after {sql}")); + } + + // A filter that hidden rows and seen rows pass changes the seen ones only. + assert_eq!( + changed(&handle, "UPDATE ledger SET payload = 'mine' WHERE a = 1"), + 1, + "{gate:?}" + ); + assert_eq!( + changed(&handle, "UPDATE ledger SET payload = 'mine'"), + 2, + "{gate:?}" + ); + let mine = |id: i64| vec![Value::Int64(id), support::text("mine")]; + let hidden = ledger_rows(&[2, 4]); + ledger_is( + &[mine(1), hidden[0].clone(), mine(3), hidden[1].clone()], + "after the handle updated every row it sees", + ); + assert_eq!( + changed(&handle, "DELETE FROM ledger WHERE a = 2"), + 1, + "{gate:?}" + ); + assert_eq!(changed(&handle, "DELETE FROM ledger"), 1, "{gate:?}"); + ledger_is(&hidden, "after the handle deleted every row it sees"); + assert_eq!( + support::int(&db, "SELECT COUNT(*) FROM ledger"), + 2, + "{gate:?}" + ); + } +} + +#[test] +fn a_row_guard_answers_false_for_a_row_the_handle_may_not_see() { + for gate in Gate::ALL { + let db = gate.store("visibility-guard"); + let handle = gate.handle(&db); + let holds = |payload: &str| vec![("payload".to_owned(), support::text(payload))]; + + // Row 2 is there and holds this value: the handle that declares nothing is told so. + let tx = db.transaction().expect("begin"); + let asked = tx.guard_row_conditions("ledger", "id", &Value::Int64(2), &holds("secret-two")); + assert!( + asked.expect("a guard through the handle that declares nothing"), + "{gate:?}: row 2 does not hold what it was written with" + ); + tx.rollback(); + + let tx = handle.transaction().expect("begin"); + let guard = |key_column: &str, key: Value, predicates: &[(String, Value)]| match tx + .guard_row_conditions("ledger", key_column, &key, predicates) + { + Ok(answer) => answer, + Err(error) => { + panic!("{gate:?}: a guard on {key_column} = {key:?} was refused: {error:?}") + } + }; + // A row within reach: the guard answers by what the row holds. + assert!(guard("id", Value::Int64(1), &holds("seen-one")), "{gate:?}"); + assert!(guard("id", Value::Int64(1), &[]), "{gate:?}"); + assert!(!guard("id", Value::Int64(1), &holds("other")), "{gate:?}"); + // A hidden row is to the handle a row that is not there: false whatever is asked of + // it, by its key or by another column that names it. + assert!( + !guard("id", Value::Int64(2), &holds("secret-two")), + "{gate:?}" + ); + assert!(!guard("id", Value::Int64(2), &[]), "{gate:?}"); + assert!( + !guard("slug", support::text("shut-four"), &holds("secret-four")), + "{gate:?}" + ); + assert!(!guard("slug", support::text("shut-four"), &[]), "{gate:?}"); + assert!(!guard("id", Value::Int64(99), &[]), "{gate:?}"); + tx.rollback(); + let now = support::rows(&db, LEDGER_ROWS); + support::assert_same_rows( + &now, + &ledger_rows(&[1, 2, 3, 4]), + "the ledger after the guards", + ); + } +} + +#[test] +fn no_refusal_holds_a_value_or_a_number_of_a_row_the_handle_may_not_see() { + for gate in Gate::ALL { + let db = gate.store("visibility-refusal-words"); + let handle = gate.handle(&db); + let rule = gate.rule(); + let mut refusals = Vec::new(); + for sql in [ + "SELECT payload FROM ledger WHERE id = 2", + "SELECT payload FROM ledger WHERE slug = 'shut-two'", + "SELECT payload FROM ledger WHERE a = 2 AND b = 2", + ] { + refusals.push((sql, refused(&handle, sql, rule, "ledger"))); + } + let by_key = "SELECT payload FROM ledger WHERE id = $id"; + let key = params! { "id" => 4_i64 }; + refusals.push((by_key, refused_with(&handle, by_key, &key, rule, "ledger"))); + // A write outside the handle's reach: a new row, and a seen row moved out. + let outside = params! { "reach" => gate.reach(false) }; + for sql in [ + "INSERT INTO ledger (id, slug, a, b, reach, payload) \ + VALUES (5, 'shut-five', 3, 3, $reach, 'secret-five')", + "UPDATE ledger SET reach = $reach WHERE id = 1", + ] { + refusals.push((sql, refused_with(&handle, sql, &outside, rule, "ledger"))); + } + + // The table's name has no digit in it, so a digit in a refusal is a key, a count, a + // Context or an access id; and no word of a hidden row's values is in one. + for (sql, refusal) in &refusals { + for said in [refusal.to_string(), format!("{refusal:?}")] { + assert!( + !said.chars().any(|c| c.is_ascii_digit()), + "{gate:?}: {sql}\nwas refused with a number in its words: {said}" + ); + for hidden in ["secret", "shut", "internal", "someone-else"] { + assert!( + !said.contains(hidden), + "{gate:?}: {sql}\nwas refused with `{hidden}` in its words: {said}" + ); + } + } + } + let now = support::rows(&db, LEDGER_ROWS); + support::assert_same_rows( + &now, + &ledger_rows(&[1, 2, 3, 4]), + "the ledger after refusals", + ); + } +} + +// ---- scope labels ---- + +#[test] +fn a_simple_label_is_checked_when_a_row_is_written_and_hides_no_row() { + let db = support::open(&support::folder("visibility-simple-label")); + support::run( + &db, + "CREATE TABLE memos (id INTEGER PRIMARY KEY, label TEXT SCOPE_LABEL ('public', \ + 'internal'), payload TEXT)", + ); + let new = |id: i64, label: &str| { + format!("INSERT INTO memos (id, label, payload) VALUES ({id}, {label}, 'new')") + }; + support::run(&db, &new(1, "'public'")); + support::run(&db, &new(2, "'internal'")); + let memos = "SELECT id, label, payload FROM memos ORDER BY id"; + let public = with_labels(&db, &["public"]); + let other = with_labels(&db, &["other"]); + + // The simple form has no read side: a handle with labels reads every row, by key too. + assert_eq!(seen(&public, "memos", "ORDER BY id"), [1, 2]); + assert_eq!(seen(&public, "memos", "WHERE id = 2"), [2]); + + // It writes rows under a label the table lists and the handle holds. + assert_eq!(changed(&public, &new(3, "'public'")), 1); + let renamed = "UPDATE memos SET payload = 'uno' WHERE id = 1"; + assert_eq!(changed(&public, renamed), 1); + let before = support::rows(&db, memos); + // Refused: a label the table lists and the handle does not hold, a label the handle + // holds and the table does not list, and no label. + for (handle, id, label) in [ + (&public, 4, "'internal'"), + (&other, 5, "'other'"), + (&public, 6, "NULL"), + ] { + refused(handle, &new(id, label), Rule::Label, "memos"); + support::assert_same_rows(&support::rows(&db, memos), &before, label); + } + // Refused: a row it reads and may not write, changed, relabelled or removed. + for sql in [ + "UPDATE memos SET payload = 'dos' WHERE id = 2", + // Setting the value the row holds is the same write to the same row. + "UPDATE memos SET payload = 'new' WHERE id = 2", + "INSERT INTO memos (id, label, payload) VALUES (2, 'public', 'x') \ + ON CONFLICT (id) DO UPDATE SET payload = 'dos'", + "INSERT INTO memos (id, label, payload) VALUES (2, 'public', 'x') \ + ON CONFLICT (id) DO UPDATE SET payload = 'new'", + "UPDATE memos SET label = 'internal' WHERE id = 1", + "DELETE FROM memos WHERE id = 2", + "DELETE FROM memos", + ] { + refused(&public, sql, Rule::Label, "memos"); + support::assert_same_rows(&support::rows(&db, memos), &before, sql); + } + assert_eq!(seen(&db, "memos", "ORDER BY id"), [1, 2, 3]); +} + +#[test] +fn a_label_with_a_read_side_hides_rows_and_is_checked_when_a_row_is_written() { + let db = support::open(&support::folder("visibility-split-label")); + // Read under 'public' and 'internal'; written under 'internal' and 'draft'. + support::run( + &db, + "CREATE TABLE briefs (id INTEGER PRIMARY KEY, label TEXT SCOPE_LABEL_READ ('public', \ + 'internal') WRITE ('internal', 'draft'), payload TEXT)", + ); + let new = |id: i64, label: &str| { + format!("INSERT INTO briefs (id, label, payload) VALUES ({id}, {label}, 'new')") + }; + for (id, label) in [ + (1, "'public'"), + (2, "'internal'"), + (3, "'draft'"), + (4, "NULL"), + ] { + support::run(&db, &new(id, label)); + } + let all = "ORDER BY id"; + let briefs = "SELECT id, label, payload FROM briefs ORDER BY id"; + + // A row is read under a label the table lists for reading and the handle holds: 'draft' + // is no label the table is read under, whoever holds it. + let reads: [(&[&str], &[i64]); 7] = [ + (&["public"], &[1]), + (&["internal"], &[2]), + (&["public", "internal"], &[1, 2]), + (&["draft"], &[]), + (&["public", "internal", "draft"], &[1, 2]), + (&["other"], &[]), + (&[], &[]), + ]; + for (names, wanted) in reads { + let handle = with_labels(&db, names); + assert_eq!(seen(&handle, "briefs", all), wanted, "labels {names:?}"); + } + // A handle that declares no label is not narrowed by a table's labels. + assert_eq!(seen(&in_contexts(&db, &[1]), "briefs", all), [1, 2, 3, 4]); + assert_eq!(seen(&db, "briefs", all), [1, 2, 3, 4]); + + // Named by key, a row under a label the handle may not read is the label's refusal: one + // it does not hold, one the table is not read under, and a row with no label. + let every = with_labels(&db, &["public", "internal", "draft"]); + let public = with_labels(&db, &["public"]); + let by_key = |id: i64| format!("SELECT payload FROM briefs WHERE id = {id}"); + refused(&public, &by_key(2), Rule::Label, "briefs"); + refused(&every, &by_key(3), Rule::Label, "briefs"); + refused(&every, &by_key(4), Rule::Label, "briefs"); + assert_eq!(seen(&every, "briefs", "WHERE id = 2"), [2]); + + // Written under a label the table lists for writing and the handle holds, though a row + // under 'draft' cannot be read back. + assert_eq!(changed(&every, &new(5, "'internal'")), 1); + assert_eq!(changed(&every, &new(6, "'draft'")), 1); + let renamed = "UPDATE briefs SET payload = 'dos' WHERE id = 2"; + assert_eq!(changed(&every, renamed), 1); + assert_eq!(seen(&every, "briefs", all), [1, 2, 5]); + // A hidden row is not matched, whatever label it is under. + let hidden = "UPDATE briefs SET payload = 'tres' WHERE id = 3"; + assert_eq!(changed(&every, hidden), 0); + assert_eq!(changed(&every, "DELETE FROM briefs WHERE id = 6"), 0); + + let before = support::rows(&db, briefs); + // Refused: 'public' is read and not written; 'internal' is written and `public` does not + // hold it; and no label. + for (handle, id, label) in [ + (&every, 7, "'public'"), + (&public, 8, "'internal'"), + (&public, 9, "NULL"), + ] { + refused(handle, &new(id, label), Rule::Label, "briefs"); + support::assert_same_rows(&support::rows(&db, briefs), &before, label); + } + for sql in [ + "UPDATE briefs SET payload = 'uno' WHERE id = 1", + "DELETE FROM briefs WHERE id = 1", + "UPDATE briefs SET label = 'public' WHERE id = 2", + ] { + refused(&every, sql, Rule::Label, "briefs"); + support::assert_same_rows(&support::rows(&db, briefs), &before, sql); + } + assert_eq!(seen(&db, "briefs", all), [1, 2, 3, 4, 5, 6]); +} + +// ---- access-controlled rows, written ---- + +#[test] +fn a_principal_writes_only_rows_under_an_id_it_holds_a_grant_for() { + let db = notes("visibility-grant-writes"); + let reader = named(&db, agent("report-reader")); + let payloads = "SELECT id, acl_id, payload FROM notes ORDER BY id"; + + note(&reader, 60, Some(1), Some(1)).expect("a new row under an id the principal holds"); + assert_eq!(ids(&reader, ALL_NOTES), [1, 3, 7, 60]); + assert_eq!( + changed(&reader, "UPDATE notes SET payload = 'mine' WHERE id = 60"), + 1 + ); + + let before = support::rows(&db, payloads); + // A new row under an id granted to somebody else, to nobody, and under no id. + for (id, acl_number) in [(61_i64, Some(2)), (62, Some(3)), (63, None)] { + match note(&reader, id, Some(1), acl_number) { + Err(Error::AclDenied { table }) => assert_eq!(table, "notes"), + other => panic!("a new row under access id {acl_number:?} gave {other:?}"), + } + support::assert_same_rows( + &support::rows(&db, payloads), + &before, + "after a refused insert", + ); + } + // A row it sees may not be moved under an id it does not hold. + refused_with( + &reader, + "UPDATE notes SET acl_id = $acl WHERE id = 1", + ¶ms! { "acl" => acl(2) }, + Rule::Grant, + "notes", + ); + support::assert_same_rows( + &support::rows(&db, payloads), + &before, + "after a refused update", + ); + + // Rows under other ids are not matched by its writes. + assert_eq!( + changed(&reader, "UPDATE notes SET payload = 'mine' WHERE id = 2"), + 0 + ); + assert_eq!( + changed(&reader, "DELETE FROM notes WHERE id IN (2, 4, 5, 6)"), + 0 + ); + assert_eq!(changed(&reader, "DELETE FROM notes WHERE id >= 1"), 4); + assert_eq!(ids(&db, ALL_NOTES), [2, 4, 5, 6]); + assert_eq!( + texts(&db, "SELECT payload FROM notes WHERE id = 2"), + ["note-2"] + ); +} + +// ---- what a narrowed handle is told by a count ---- + +#[test] +fn a_narrowed_handle_counts_its_own_rows_where_the_whole_table_is_counted_with_no_row_read() { + let db = support::open(&support::folder("visibility-counts")); + support::run( + &db, + "CREATE TABLE events (id INTEGER PRIMARY KEY, context_id UUID NOT NULL CONTEXT_ID, \ + kind TEXT NOT NULL, UNIQUE (context_id, id)) MAINTAIN COUNT (context_id)", + ); + support::run(&db, "CREATE INDEX events_kind ON events (kind)"); + // Events 1 to 10 are under Context 1 and 11 to 15 under Context 2; event n is of kind + // k. Of kind k1 are five events of Context 1 and three of Context 2. + let tx = db.transaction().expect("begin"); + for id in 1..=15_i64 { + tx.execute( + "INSERT INTO events (id, context_id, kind) VALUES ($id, $context, $kind)", + ¶ms! { + "id" => id, "context" => context(if id <= 10 { 1 } else { 2 }), + "kind" => format!("k{}", id % 2), + }, + ) + .expect("insert an event"); + } + tx.commit().expect("commit"); + + // A count through a handle is the number of its own rows, by the table's count, by the + // kept count of a Context and by an index range: never the table's 15, the other Context's + // 5 or the index range's 8. (That the whole table is counted with no row read is proved in + // `counted_work.rs` and `kept_counts.rs`; that a narrowed count reads its rows, in `kept_counts_narrowed.rs`.) + let count = |on: &Database, filter: &str, of: Option| { + let sql = format!("SELECT COUNT(*) FROM events{filter}"); + let values = of.map_or_else(|| params! {}, |n| params! { "context" => context(n) }); + match on.execute(&sql, &values).expect("a count").rows() { + [row] => match row.as_slice() { + [Value::Int64(count)] => *count, + other => panic!("{sql}\nreturned {other:?}, not one count"), + }, + other => panic!("{sql}\nreturned {} rows, not one", other.len()), + } + }; + let in_context = " WHERE context_id = $context"; + let one = in_contexts(&db, &[1]); + assert_eq!(count(&one, "", None), 10); + assert_eq!(count(&one, in_context, Some(1)), 10); + assert_eq!(count(&one, in_context, Some(2)), 0); + assert_eq!(count(&one, " WHERE kind = 'k1'", None), 5); + assert_eq!(count(&one, " WHERE id > 8", None), 2); + assert_eq!( + count(&in_contexts(&db, &[2]), " WHERE kind = 'k1'", None), + 3 + ); + assert_eq!(count(&in_contexts(&db, &[3]), "", None), 0); + + // A count that names a hidden row by its key says the row is not the handle's, by the + // refusal or by 0, and never counts it. + let hidden = "SELECT COUNT(*) FROM events WHERE id = 11"; + match one.execute(hidden, ¶ms! {}) { + Ok(result) => assert_eq!(result.rows(), [[Value::Int64(0)]], "{hidden}"), + Err(Error::ContextScopeViolation { table }) => assert_eq!(table, "events"), + Err(other) => panic!("{hidden}\nwas refused with {other:?}"), + } + + // The counts follow the rows the handle writes, for it and for the whole table. + assert_eq!(changed(&one, "DELETE FROM events WHERE kind = 'k1'"), 5); + assert_eq!(count(&one, "", None), 5); + assert_eq!(count(&db, "", None), 10); + assert_eq!(count(&db, in_context, Some(2)), 5); + assert_eq!(count(&db, " WHERE kind = 'k1'", None), 3); +} + +// ---- what a narrowed handle is told by a search ---- + +#[test] +fn a_search_returns_the_nearest_rows_the_handle_may_see_and_no_hidden_row() { + const DIMENSION: usize = 8; + let db = support::open(&support::folder("visibility-search")); + support::run(&db, GRANTS); + grant(&db, 1, "Agent", "report-reader", acl(1)); + grant(&db, 2, "Human", "someone-else", acl(2)); + support::run( + &db, + &format!( + "CREATE TABLE passages (id INTEGER PRIMARY KEY, context_id UUID CONTEXT_ID, \ + acl_id UUID ACL REFERENCES acl_grants(acl_id), v VECTOR({DIMENSION}))" + ), + ); + let query = support::vector(1_000, DIMENSION); + let passage = |id: i64, context_number: u128, acl_number: u128, v: Vec| { + db.execute( + "INSERT INTO passages (id, context_id, acl_id, v) VALUES ($id, $context, $acl, $v)", + ¶ms! { + "id" => id, "context" => context(context_number), "acl" => acl(acl_number), + "v" => v, + }, + ) + .unwrap_or_else(|error| panic!("passage {id} was refused: {error:?}")); + }; + // Passage n of 1 to 40: Context 1 for odd n and 2 for even; access id 2 when n is a + // multiple of 3 and 1 otherwise. Three more hold the very vector searched with, so that + // each would be the first answer: 101 is hidden by its Context, 102 by its access id, + // 103 by both. + for n in 1..=40_i64 { + let context_number = if n % 2 == 1 { 1 } else { 2 }; + let acl_number = if n % 3 == 0 { 2 } else { 1 }; + passage(n, context_number, acl_number, support::vector(n, DIMENSION)); + } + passage(101, 2, 1, query.clone()); + passage(102, 1, 2, query.clone()); + passage(103, 2, 2, query.clone()); + + let reader = named(&db, agent("report-reader")); + let reader_in_one = + db.scoped_with_constraints(Some(context_set(&[1])), None, Some(agent("report-reader"))); + let in_context_one = |row: &[Value]| row[1] == Value::Uuid(context(1)); + let granted = |row: &[Value]| row[2] == Value::Uuid(acl(1)); + + // Twice: with every vector waiting on its list, and with every vector in a block. + for sealed in [false, true] { + if sealed { + assert_eq!(support::seal_all(&db), 43, "vectors sealed"); + } + let stored = support::stored(&db, "SELECT id, context_id, acl_id, v FROM passages"); + for k in [1, 5, 43] { + for filter in [None, Some("id > 10")] { + let sql = support::nearest_sql("passages", "v", filter, k); + let passes = |row: &[Value]| { + filter.is_none() || matches!(row[0], Value::Int64(id) if id > 10) + }; + let what = |who: &str| format!("{who}, sealed: {sealed}"); + + // The handle that declares nothing finds the planted rows first. + let wanted = stored.nearest(passes, &query, k); + support::assert_nearest(&db, &sql, &query, &wanted, &what("declaring nothing")); + assert!( + matches!(wanted[0].key, Value::Int64(101)), + "the planted rows are not the nearest: {wanted:?}" + ); + + // A narrowed handle is answered with the nearest rows it may see, each at its + // true distance; a hidden row takes no place among them. + let wanted = stored.nearest(|row| passes(row) && granted(row), &query, k); + support::assert_nearest(&reader, &sql, &query, &wanted, &what("the principal")); + let wanted = stored.nearest( + |row| passes(row) && granted(row) && in_context_one(row), + &query, + k, + ); + let what = what("the principal in Context 1"); + support::assert_nearest(&reader_in_one, &sql, &query, &wanted, &what); + } + } + } + + // With no principal that holds grants the search is refused like any read of the table. + let sql = support::nearest_sql("passages", "v", None, 5); + match support::search(&in_contexts(&db, &[1]), &sql, &query) { + Err(Error::PrincipalRequired { table }) => assert_eq!(table, "passages"), + other => panic!("{sql}\nthrough a handle with no principal gave {other:?}"), + } +} + +// ---- a handle made from a narrowed handle ---- + +#[test] +fn a_handle_made_from_a_narrowed_handle_sees_no_more_than_it() { + let db = items("visibility-narrowed-again"); + let one = in_contexts(&db, &[1]); + // Asking for more Contexts, or for none, gives no more than the handle already has. + let widened = one.scoped_with_contexts(context_set(&[1, 2])); + assert_eq!(ids(&widened, ALL_ITEMS), [1, 3, 5, 7, 9, 11]); + let undeclared = one.scoped_with_constraints(None, None, None); + assert_eq!(ids(&undeclared, ALL_ITEMS), [1, 3, 5, 7, 9, 11]); + refused( + &undeclared, + "SELECT kind FROM items WHERE id = 2", + Rule::Context, + "items", + ); + // Asking for other Contexts leaves none. + let other = one.scoped_with_contexts(context_set(&[2])); + assert_eq!(ids(&other, ALL_ITEMS), [0_i64; 0]); + // Narrowing a handle of two Contexts to one of them keeps that one. + let both = in_contexts(&db, &[1, 2]); + assert_eq!( + ids(&both.scoped_with_contexts(context_set(&[2, 3])), ALL_ITEMS), + [2, 4, 6, 8, 10, 12] + ); + + // A handle that names a principal keeps it: another name asked of it does not replace it. + let store = notes("visibility-narrowed-again-notes"); + let reader = named(&store, agent("report-reader")); + let renamed = reader.scoped_with_constraints(None, None, Some(human("ana"))); + assert_eq!(ids(&renamed, ALL_NOTES), [1, 3, 7]); + let unnamed = reader.scoped_with_constraints(Some(context_set(&[1])), None, None); + assert_eq!(ids(&unnamed, ALL_NOTES), [1]); +} + +// ---- what the filter holds in memory ---- + +// Its size is fixed by the room: 24,000 ids are 23 times the 1,024 a statement remembers, and +// their 384,000 bytes are three times the 128 KiB the limit leaves. +#[test] +fn twenty_four_thousand_access_ids_are_read_under_a_limit_with_no_room_to_remember_them() { + const ROWS: i64 = 24_000; + const BATCH: i64 = 500; + const KIB: usize = 1 << 10; + const MIB: usize = 1 << 20; + let paper_acl = |n: i64| Uuid::from_u128(0x0A00_0000_0000 + n as u128); + + // Paper n of 24,000 is under an access id of its own, and the agent holds the id of + // every even paper. Loaded under no limit, 500 rows a statement. + let folder = support::folder("visibility-many-access-ids"); + let db = support::open(&folder); + support::run(&db, GRANTS); + // Whichever way a grant is looked up, an index that leads with the id serves it. + support::run(&db, "CREATE INDEX acl_grants_acl ON acl_grants (acl_id)"); + support::run( + &db, + "CREATE TABLE papers (id INTEGER PRIMARY KEY, acl_id UUID ACL REFERENCES \ + acl_grants(acl_id), n INTEGER NOT NULL)", + ); + for first in (1..=ROWS).step_by((BATCH * 8) as usize) { + let tx = db.transaction().expect("begin"); + for start in (first..first + BATCH * 8).step_by(BATCH as usize) { + let batch = start..start + BATCH; + let papers: Vec = batch + .clone() + .map(|n| format!("({n}, '{}', {n})", paper_acl(n))) + .collect(); + let grants: Vec = batch + .filter(|n| n % 2 == 0) + .map(|n| { + let id = Uuid::from_u128(0x0B00_0000_0000 + n as u128); + format!("('{id}', 'Agent', 'reader', '{}')", paper_acl(n)) + }) + .collect(); + for sql in [ + format!( + "INSERT INTO papers (id, acl_id, n) VALUES {}", + papers.join(", ") + ), + format!( + "INSERT INTO acl_grants (id, principal_kind, principal_id, acl_id) VALUES {}", + grants.join(", ") + ), + ] { + if let Err(error) = tx.execute(&sql, ¶ms! {}) { + panic!("loading the rows from {start} was refused: {error:?}"); + } + } + } + tx.commit().expect("commit a part of the load"); + } + support::close(db); + + // 9 MiB is what a store needs to open at all. The 128 KiB above it hold the filter's + // 64 KiB and a row eight times over, and do not hold an entry for each id: 24,000 ids of + // 16 bytes alone are 384,000 bytes, and the 12,000 that are granted are 192,000. + let limit = 9 * MIB + 128 * KIB; + let mut options = DatabaseOpenOptions::default(); + options.memory_limit = Some(limit); + let db = support::open_with(&folder, options).expect("open the store under the limit"); + let reader = named(&db, agent("reader")); + let in_use = db.memory_used(); + let granted = ROWS / 2; + assert_eq!( + support::int(&reader, "SELECT COUNT(*) FROM papers"), + granted + ); + // The even numbers up to 24,000: which rows were counted, not only how many. + assert_eq!( + support::int(&reader, "SELECT SUM(n) FROM papers"), + granted * (granted + 1) + ); + assert_eq!( + db.memory_used(), + in_use, + "memory in use after the statements" + ); + // A principal that holds none of the ids reads the table whole as well, and sees no row. + let stranger = named(&db, agent("stranger")); + assert_eq!(support::int(&stranger, "SELECT COUNT(*) FROM papers"), 0); + assert_eq!(support::int(&db, "SELECT COUNT(*) FROM papers"), ROWS); +} + +// ---- a narrowed handle declares nothing ---- + +#[test] +fn a_narrowed_handle_is_refused_every_declaration_and_setting_and_changes_nothing() { + // The statements that declare a table or an index or set the store's limit apply to every + // Context, label and principal at once, so only the handle that declared no narrowing may + // run them. The refusal names the statement as a person writes it. The order lets each + // statement run, unnarrowed, after the ones before it. + let statements = [ + ("CREATE INDEX notes_by_tag ON notes (tag)", "CREATE INDEX"), + ("ALTER TABLE notes ADD COLUMN extra TEXT", "ALTER TABLE"), + ("SET MEMORY_LIMIT 64M", "SET MEMORY_LIMIT"), + ("DROP INDEX notes_by_s ON notes", "DROP INDEX"), + ("ALTER TABLE notes DROP COLUMN s CASCADE", "ALTER TABLE"), + ("DROP TABLE notes", "DROP TABLE"), + ]; + // The two readings of the vector part's state are the same: for every Context at once. + let readings = [ + ("SHOW VECTOR_INDEXES", "SHOW VECTOR_INDEXES"), + ("SHOW VECTOR_PARTITIONS", "SHOW VECTOR_PARTITIONS"), + ]; + type Narrow = fn(&Database) -> Database; + let handles: [(&str, Narrow); 3] = [ + ("a Context", |db| in_contexts(db, &[1])), + ("a scope label", |db| with_labels(db, &["public"])), + ("a principal", |db| named(db, agent("someone"))), + ]; + let db = support::open(&support::folder("visibility-declare")); + support::run( + &db, + "CREATE TABLE notes (id INTEGER PRIMARY KEY, context_id UUID CONTEXT_ID, s TEXT, tag TEXT)", + ); + support::run(&db, "CREATE INDEX notes_by_s ON notes (s)"); + for (id, number) in [(1_i64, 1_u128), (2, 2)] { + let values = params! { "id" => id, "context" => context(number) }; + db.execute( + "INSERT INTO notes (id, context_id, s, tag) VALUES ($id, $context, 'a', 't')", + &values, + ) + .expect("insert a note"); + } + let limit = |db: &Database| support::rows(db, "SHOW MEMORY_LIMIT")[0][0].clone(); + let (declared, limit_before) = (support::rows(&db, "SELECT * FROM notes"), limit(&db)); + for (sql, name) in statements.iter().chain(&readings) { + for (narrowed_by, narrow) in &handles { + match support::refusal(&narrow(&db), sql) { + Error::AdminRequired { statement } => assert_eq!( + statement, *name, + "{sql}\nrefused to a handle narrowed by {narrowed_by} under another name" + ), + other => panic!( + "{sql}\nto a handle narrowed by {narrowed_by}, which may not change what \ + every Context's rows are kept in, was refused as {other:?}" + ), + } + } + } + // The Rust call that sets the same limit is refused as the statement is: a program holding + // only a narrowed handle must not lower the limit of the whole store through the other door. + let bytes_before = db.memory_limit(); + for (narrowed_by, narrow) in &handles { + match narrow(&db).set_memory_limit(Some(64 << 20)) { + Err(Error::AdminRequired { statement }) => assert_eq!( + statement, "SET MEMORY_LIMIT", + "set_memory_limit on a handle narrowed by {narrowed_by} named another statement" + ), + other => panic!( + "set_memory_limit on a handle narrowed by {narrowed_by} sets the limit of every \ + Context, so it must be AdminRequired, got {other:?}" + ), + } + } + assert_eq!( + db.memory_limit(), + bytes_before, + "a refused call changed the limit" + ); + // The unnarrowed handle still sees the table, its column, its index and the row of the + // Context the narrowed handles could not see, and the limit is what it was. + assert_eq!( + support::rows(&db, "SELECT * FROM notes"), + declared, + "the table changed" + ); + let text = db.declaration("notes").expect("the table is still there"); + assert!( + text.contains("notes_by_s") && !text.contains("notes_by_tag") && !text.contains("extra"), + "a refused statement changed the declaration:\n{text}" + ); + assert_eq!(limit(&db), limit_before, "the memory limit"); + assert_eq!(seen(&db, "notes", "ORDER BY id"), [1, 2]); + // Each is a statement the unnarrowed handle may run. + for (sql, _) in statements.iter().chain(&readings) { + support::run(&db, sql); + } +} diff --git a/v2/contextdb/tests/visibility_grants.rs b/v2/contextdb/tests/visibility_grants.rs new file mode 100644 index 00000000..fdfebe9d --- /dev/null +++ b/v2/contextdb/tests/visibility_grants.rs @@ -0,0 +1,276 @@ +//! Which grants count. A row under an access id is shown to a principal only through a grant row +//! that the reader may itself see, in a grant table that is there and has the columns a grant is +//! read from; a grant row counts whatever its table's columns have become since it was written +//! and however long its principal's name is; and one statement sees one set of grants. + +mod support; + +use std::collections::{BTreeSet, HashMap}; + +use contextdb::{ContextId, Database, Error, Principal, ScopeLabel, Uuid, Value, params}; + +use support::Run; + +// ---- who is asking ---- + +/// Context number `n`. +fn context(n: u128) -> Uuid { + Uuid::from_u128(0x00C0_0000 + n) +} + +/// Access-control id number `n`. +fn acl(n: u128) -> Uuid { + Uuid::from_u128(0x00AC_0000 + n) +} + +fn context_set(numbers: &[u128]) -> BTreeSet { + numbers + .iter() + .map(|n| ContextId::new(context(*n))) + .collect() +} + +fn label_set(names: &[&str]) -> BTreeSet { + names.iter().map(|name| ScopeLabel::new(*name)).collect() +} + +/// A handle that names this agent and nothing else. +fn agent(db: &Database, name: &str) -> Database { + db.scoped_with_constraints(None, None, Some(Principal::Agent(name.to_owned()))) +} + +// ---- what a statement answered ---- + +/// Runs a statement that must not be refused. +fn put(on: &impl Run, sql: &str, values: &HashMap) { + if let Err(error) = on.run(sql, values) { + panic!("{sql}\nwas refused: {error:?}"); + } +} + +/// The ids `SELECT id FROM
ORDER BY id` returns, which must be INTEGERs. +fn seen(on: &impl Run, table: &str) -> Vec { + let sql = format!("SELECT id FROM {table} ORDER BY id"); + let first = |row: &Vec| match row.first() { + Some(Value::Int64(id)) => *id, + other => panic!("{sql}\nreturned {other:?} where an INTEGER was wanted"), + }; + support::rows(on, &sql).iter().map(first).collect() +} + +/// Asserts that a statement is refused for want of a grant on this table. +fn denied(on: &impl Run, sql: &str, values: &HashMap, table: &str) { + match on.run(sql, values) { + Err(Error::AclDenied { table: named }) => assert_eq!(named, table, "{sql}"), + Err(other) => panic!("{sql}\nwas refused with {other:?}; AclDenied was wanted"), + Ok(result) => panic!("{sql}\nwas not refused: {result:?}"), + } +} + +// ---- tables of grants and the rows they guard ---- + +/// Declares a grant table with these columns after the four every grant table here has. +fn declare_grants(db: &Database, table: &str, more: &str) { + support::run( + db, + &format!( + "CREATE TABLE {table} (id INTEGER PRIMARY KEY, principal_kind TEXT, \ + principal_id TEXT, acl_id UUID{more})" + ), + ); +} + +/// Declares a table whose rows are guarded by grants of `grants`, with a row under each of +/// these access ids, keyed by the id's number. +fn declare_guarded(db: &Database, table: &str, grants: &str, under: &[u128]) { + support::run( + db, + &format!( + "CREATE TABLE {table} (id INTEGER PRIMARY KEY, \ + acl_id UUID ACL REFERENCES {grants}(acl_id), payload TEXT)" + ), + ); + for n in under { + let id = i64::try_from(*n).expect("a small access id number"); + put( + db, + &format!("INSERT INTO {table} (id, acl_id, payload) VALUES ($id, $acl, 'kept')"), + ¶ms! { "id" => id, "acl" => acl(*n) }, + ); + } +} + +/// Writes grant row `id` of `table`: this agent holds access id number `n`. `more` names +/// further columns, each filled from the parameter of its own name in `values`. +fn grant(db: &Database, table: &str, id: i64, name: &str, n: u128, more: &[(&str, Value)]) { + let mut columns = String::from("id, principal_kind, principal_id, acl_id"); + let mut places = String::from("$id, 'Agent', $name, $acl"); + let mut values = params! { "id" => id, "name" => name, "acl" => acl(n) }; + for (column, value) in more { + columns.push_str(&format!(", {column}")); + places.push_str(&format!(", ${column}")); + values.insert((*column).to_owned(), value.clone()); + } + put( + db, + &format!("INSERT INTO {table} ({columns}) VALUES ({places})"), + &values, + ); +} + +// ---- the five cases ---- + +#[test] +fn a_grant_row_the_reader_may_not_see_gives_no_access() { + let folder = support::folder("grants-hidden-grant-row"); + let db = support::open(&folder); + let a = Some(Principal::Agent("a".to_owned())); + + // Hidden by its Context: the grant of access id 2 is in a Context the reader is not in. + declare_grants(&db, "placed_grants", ", context_id UUID CONTEXT_ID"); + declare_guarded(&db, "placed", "placed_grants", &[1, 2]); + for (id, n) in [(1, 1), (2, 2)] { + let place = [("context_id", Value::Uuid(context(n)))]; + grant(&db, "placed_grants", id, "a", n, &place); + } + let in_one = db.scoped_with_constraints(Some(context_set(&[1])), None, a.clone()); + let in_both = db.scoped_with_constraints(Some(context_set(&[1, 2])), None, a.clone()); + assert_eq!(seen(&in_one, "placed"), [1], "a grant in another Context"); + assert_eq!(seen(&in_both, "placed"), [1, 2], "both grants in view"); + + // Hidden by its label: the grant of access id 2 is under a label the reader does not hold. + declare_grants( + &db, + "labelled_grants", + ", reach TEXT SCOPE_LABEL_READ ('public', 'internal') WRITE ('public', 'internal')", + ); + declare_guarded(&db, "labelled", "labelled_grants", &[1, 2]); + for (id, n, label) in [(1, 1, "public"), (2, 2, "internal")] { + let reach = [("reach", Value::Text(label.to_owned()))]; + grant(&db, "labelled_grants", id, "a", n, &reach); + } + let public = db.scoped_with_constraints(None, Some(label_set(&["public"])), a.clone()); + let both = db.scoped_with_constraints(None, Some(label_set(&["public", "internal"])), a); + assert_eq!( + seen(&public, "labelled"), + [1], + "a grant under another label" + ); + assert_eq!(seen(&both, "labelled"), [1, 2], "both grants in view"); +} + +#[test] +fn a_grant_table_that_is_missing_or_lacks_the_grant_column_grants_nobody() { + let folder = support::folder("grants-no-grant-table"); + let db = support::open(&folder); + let reader = agent(&db, "a"); + + for (table, grants, remove) in [ + ("orphans", "gone_grants", "DROP TABLE gone_grants"), + ( + "strays", + "cut_grants", + "ALTER TABLE cut_grants DROP COLUMN acl_id", + ), + ] { + declare_grants(&db, grants, ""); + declare_guarded(&db, table, grants, &[1, 2]); + grant(&db, grants, 1, "a", 1, &[]); + assert_eq!(seen(&reader, table), [1], "{table}, while the grant stands"); + + support::run(&db, remove); + assert_eq!(seen(&reader, table), [0_i64; 0], "{table}, after {remove}"); + assert_eq!( + seen(&db, table), + [1, 2], + "{table}, to a reader not narrowed" + ); + let changed = support::run(&reader, &format!("UPDATE {table} SET payload = 'taken'")); + assert_eq!( + changed.rows_affected(), + 0, + "{table}, an UPDATE of hidden rows" + ); + denied( + &reader, + &format!("INSERT INTO {table} (id, acl_id, payload) VALUES (3, $acl, 'new')"), + ¶ms! { "acl" => acl(1) }, + table, + ); + assert_eq!( + seen(&db, table), + [1, 2], + "{table}, after the refused writes" + ); + } +} + +#[test] +fn a_grant_row_older_than_an_added_column_or_carrying_a_dropped_one_still_counts() { + let folder = support::folder("grants-altered-grant-table"); + let db = support::open(&folder); + let reader = agent(&db, "a"); + declare_grants(&db, "acl_grants", ", memo TEXT"); + declare_guarded(&db, "notes", "acl_grants", &[1, 2, 3, 4]); + + // Grant 1 carries a cell of `memo`, which is then dropped. + let memo = [("memo", Value::Text("written before the drop".to_owned()))]; + grant(&db, "acl_grants", 1, "a", 1, &memo); + support::run(&db, "ALTER TABLE acl_grants DROP COLUMN memo"); + assert_eq!(seen(&reader, "notes"), [1], "after the column was dropped"); + + // Grants 1 and 2 are written before `reason` is added; grant 3 after, with a value in it. + grant(&db, "acl_grants", 2, "a", 2, &[]); + support::run(&db, "ALTER TABLE acl_grants ADD COLUMN reason TEXT"); + assert_eq!(seen(&reader, "notes"), [1, 2], "after the column was added"); + let reason = [("reason", Value::Text("written after the add".to_owned()))]; + grant(&db, "acl_grants", 3, "a", 3, &reason); + assert_eq!(seen(&reader, "notes"), [1, 2, 3], "old and new grant rows"); +} + +#[test] +fn a_statement_that_writes_its_own_grant_table_sees_the_grants_it_first_looked_up() { + let folder = support::folder("grants-self-written"); + let db = support::open(&folder); + // A grant table whose own rows are guarded by its grants. Every row is under access id 1, + // and only row 1, the first by key, grants that id to the reader. + declare_grants(&db, "seats", ", guard UUID ACL REFERENCES seats(acl_id)"); + for (id, name) in [(1, "a"), (2, "b"), (3, "b")] { + grant(&db, "seats", id, name, 1, &[("guard", Value::Uuid(acl(1)))]); + } + let reader = agent(&db, "a"); + assert_eq!(seen(&reader, "seats"), [1, 2, 3], "before the statement"); + + // The statement takes the reader's only grant away in the first row it changes, and still + // reaches every row that grant showed it. + let changed = support::run(&reader, "UPDATE seats SET principal_id = 'nobody'"); + assert_eq!(changed.rows_affected(), 3, "rows the one statement changed"); + let names = support::rows(&db, "SELECT principal_id FROM seats ORDER BY id"); + let nobody = vec![Value::Text("nobody".to_owned())]; + assert_eq!(names, vec![nobody; 3], "every row was rewritten"); + + // The next statement looks the grants up afresh and finds none. + assert_eq!(seen(&reader, "seats"), [0_i64; 0], "after the statement"); +} + +#[test] +fn a_grant_to_a_principal_whose_name_is_2000_bytes_long_is_honoured() { + let folder = support::folder("grants-long-principal-name"); + let db = support::open(&folder); + declare_grants(&db, "acl_grants", ""); + declare_guarded(&db, "notes", "acl_grants", &[1, 2]); + let name = "p".repeat(2_000); + let near = format!("{}q", &name[1..]); + grant(&db, "acl_grants", 1, &name, 1, &[]); + + assert_eq!( + seen(&agent(&db, &name), "notes"), + [1], + "the principal granted" + ); + assert_eq!( + seen(&agent(&db, &near), "notes"), + [0_i64; 0], + "a principal whose name differs in its last byte" + ); +} diff --git a/v2/numbers.sh b/v2/numbers.sh index 6360be0d..cfa37b94 100755 --- a/v2/numbers.sh +++ b/v2/numbers.sh @@ -127,6 +127,42 @@ passages passages_measurements "${PASSAGES:-}" sq8 passages passages_large_measurements "${PASSAGES_LARGE:-}" sq8,tail passages passages_larger_measurements "${PASSAGES_LARGER:-}" tail +# Context Graph's declarations and statements (the head of contextdb/examples/queries.rs): +# its declarations; the second dump in its passages table; the anchors of every session file +# of the SESSIONS folder's `files`; and two generated cases. +queries() { + example queries -- "$@" "$scratch/queries" || echo "queries_${1}_measurements unavailable -" + rm -rf -- "$scratch/queries" +} +queries declarations +queries dag +queries grants +if [ -n "${PASSAGES_LARGE:-}" ]; then + queries declared "${PASSAGES_LARGE%%:*}" "${PASSAGES_LARGE#*:}" +else + echo "declared_measurements unavailable -" +fi +if [ -n "${SESSIONS:-}" ]; then + queries anchors "${SESSIONS%%:*}/files" +else + echo "anchors_measurements unavailable -" +fi + +# The Context Graph branch, CG_BRANCH=, checked against this tree's library: the +# compiler's errors, and the call sites still fenced for each later step. +if [ -n "${CG_BRANCH:-}" ]; then + mkdir -p "$scratch/cg" + tar -C "$CG_BRANCH" --exclude=./target -cf - . | tar -xf - -C "$scratch/cg" + sed -i "s#path = \"[^\"]*/v2/contextdb\"#path = \"$PWD/v2/contextdb\"#" "$scratch/cg/Cargo.toml" + cargo check --manifest-path "$scratch/cg/Cargo.toml" --workspace --all-targets --keep-going > "$scratch/cg.log" 2>&1 + echo "cg_branch_errors $(grep -E '^error(\[|:)' "$scratch/cg.log" | grep -cv 'could not compile') errors" + for step in 3 5 6 7 8; do + echo "cg_branch_fenced_step$step $(grep -r -o "cfg(lean_step$step)" "$scratch/cg/crates" | wc -l) sites" + done +else + echo "cg_branch_measurements unavailable -" +fi + # From a fresh clone to a green `cargo test`: nothing built before. start=$SECONDS if CARGO_TARGET_DIR="$scratch/fresh" cargo test --manifest-path v2/Cargo.toml --workspace --locked > "$scratch/fresh.log" 2>&1; then