From 22dd2659d72c6ca9f16c67208d5775c3b46ba231 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 5 Oct 2026 00:16:14 +0200 Subject: [PATCH] Split the API into /clickhouse and /postgres entries with one table Each database gets one import that covers it: the whole query builder plus that dialect's column types, functions, table DDL and compile. The root entry and /types are removed. `table` is now the only way to declare a table, and it carries its DDL (CH.table with engine and keys, PG.table with primary key, indexes and foreign keys). The plain table(name, columns, options) is gone from the public API; insert typing comes from column options. `external: true` covers what the schema does not own: system tables, table functions, subqueries and tables another tool migrates. It has no DDL, so generate skips it. /schema keeps only the migration tooling. Docs, reference catalog and checks are updated for both entries. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 20 ++ README.md | 90 +++-- docs/README.md | 55 +-- docs/benchmarking.md | 13 +- docs/database.md | 10 +- docs/decoding-results.md | 10 +- docs/expressions.md | 11 +- docs/extending.md | 44 ++- docs/getting-started.md | 84 +++-- docs/inserts.md | 89 +++-- docs/joins-and-subqueries.md | 6 +- docs/migrations.md | 68 ++-- docs/params-and-compilation.md | 25 +- docs/postgres.md | 54 +-- docs/queries.md | 7 +- docs/recipes.md | 20 +- docs/reference.md | 367 +++++++++++++++------ docs/running-queries.md | 11 +- docs/tables-and-types.md | 277 +++++++++++----- docs/tenant-scoping.md | 13 +- docs/testing.md | 5 +- docs/troubleshooting.md | 12 +- docs/unions-and-ctes.md | 9 +- docs/updates-and-deletes.md | 22 +- package.json | 10 +- scripts/check-doc-examples.mjs | 2 +- scripts/check-exports-documented.mjs | 81 ++--- scripts/check-package.ts | 2 +- src/benchmark/benchmark.test.ts | 2 +- src/ch/any-boundaries.test-d.ts | 2 +- src/ch/any-boundaries.test.ts | 2 +- src/ch/brand.test-d.ts | 4 +- src/ch/brand.test.ts | 2 +- src/ch/dialect.ts | 2 +- src/ch/functions/builtin.ts | 2 +- src/ch/index.ts | 316 +----------------- src/ch/insert.test-d.ts | 16 +- src/ch/insert.test.ts | 14 +- src/ch/table.ts | 4 +- src/clickhouse.ts | 191 +++++++++++ src/core.ts | 147 +++++++++ src/database/database.test-d.ts | 2 +- src/database/database.test.ts | 2 +- src/docs-examples.test.ts | 19 +- src/index.ts | 8 - src/kit/generate.ts | 2 +- src/kit/graph.test.ts | 6 +- src/kit/kit.test.ts | 14 +- src/migrate/pg-migrate.test.ts | 24 +- src/pg/postgres.test.ts | 2 +- src/postgres.ts | 47 ++- src/schema.ts | 35 +- src/schema/define.ts | 66 +++- src/schema/entities.ts | 2 +- src/schema/external.test.ts | 57 ++++ src/schema/pg-define.ts | 30 +- src/schema/pg-schema.test.ts | 54 +-- src/schema/schema.test.ts | 46 +-- src/types.ts | 57 ---- tests/clickhouse-support.ts | 2 +- tests/core-cases.ts | 47 ++- tests/database.clickhouse.test.ts | 24 +- tests/deep-builder.clickhouse.test.ts | 12 +- tests/deep-codecs.clickhouse.test.ts | 13 +- tests/dialect-cases.postgres.ts | 7 +- tests/dialect-cases.ts | 64 ++-- tests/dialect-coverage.test.ts | 49 ++- tests/migrate.clickhouse.test.ts | 28 +- tests/package-consumer.mts | 23 +- tests/postgres-support.ts | 2 +- tests/publish-readiness.clickhouse.test.ts | 27 +- tsdown.config.ts | 3 +- 72 files changed, 1743 insertions(+), 1152 deletions(-) create mode 100644 src/clickhouse.ts create mode 100644 src/core.ts delete mode 100644 src/index.ts create mode 100644 src/schema/external.test.ts delete mode 100644 src/types.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bf7bee..9188c8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,26 @@ ## Unreleased +- **Breaking:** one entry per database, drizzle style. `@maple-dev/effect-orm/clickhouse` and + `@maple-dev/effect-orm/postgres` each export the whole query builder plus that database's + column types, functions, table definitions and `compile`; one import covers a dialect. + - The root entry (`@maple-dev/effect-orm`) and `/types` are removed. Import from + `/clickhouse` instead (`CH.string` rather than `T.string`); Postgres code imports only + `/postgres` (`PG.from`, `PG.param`, `PG.insertInto`). + - `table` is the only way to declare a table, and it carries its DDL: `CH.table(name, + { columns, engine, orderBy, ... })` and `PG.table(name, { columns, primaryKey, ... })`. + The plain `table(name, columns, { tenantColumn, defaults, computed })` is removed; which + columns an insert may omit or may not write now comes from `column(type, options)`. + - `S.defineTable`, `S.column`, `S.engine`, `S.index`, `S.materializedView` and + `S.ttlAfterDays` move to `/clickhouse` (`CH.table`, `CH.column`, ...); `S.pg.*` moves to + `/postgres` (`PG.table`, `PG.column`, `PG.index`, `PG.uniqueIndex`, `PG.foreignKey`). + `/schema` keeps the tooling: rendering, snapshots and the diff. + - Add `external: true` to `table` for what the schema does not own: system tables, table + functions, subqueries, views, tables another tool migrates. It queries like any table, + carries no DDL (so `generate` skips it), and its name is written verbatim as the FROM target. +- Postgres `GENERATED ALWAYS AS` columns are not modeled yet; the removed `computed` option was + the only way to mark one read-only. + - **Breaking:** branded column types compare strictly. A comparison, an insert or update value, and an `INSERT ... SELECT` into a column that decodes to a branded type (`Brand<...>`) take that brand: a value of it, a column of the same brand, or a param declared with the type diff --git a/README.md b/README.md index 1b428a5..8c49f26 100644 --- a/README.md +++ b/README.md @@ -40,24 +40,27 @@ prereleases are incompatible. ## Quick start +One import per database. Each holds the whole query builder plus that database's column +types, functions, table definitions, and `compile`: + ```ts -import * as CH from "@maple-dev/effect-orm" -import * as T from "@maple-dev/effect-orm/types" - -// 1. Describe a table -const Events = CH.table( - "events", - { - OrgId: T.string, - Name: T.string, - Timestamp: T.dateTime, - DurationMs: T.uint64, - Attributes: T.map(T.string, T.string), +import * as CH from "@maple-dev/effect-orm/clickhouse" + +// 1. Describe a table. The engine and sorting key are its DDL, for migrations. +const Events = CH.table("events", { + columns: { + OrgId: CH.string, + Name: CH.string, + Timestamp: CH.dateTime, + DurationMs: CH.uint64, + Attributes: CH.map(CH.string, CH.string), }, + engine: CH.engine.mergeTree(), + orderBy: ["OrgId", "Timestamp"], // Optional: name the column carrying row-level tenancy and every compiled // query reports whether it pinned it. See docs/tenant-scoping.md. - { tenantColumn: "OrgId" }, -) + tenantColumn: "OrgId", +}) // 2. Build a query const query = CH.from(Events) @@ -81,9 +84,39 @@ const compiled = CH.compileUnsafe(query, { startTime: "2026-01-01 00:00:00", }) -compiled.sql // -> SELECT Name AS name, quantile(0.95)(DurationMs) AS p95, ... +compiled.sql // -> SELECT events.Name AS name, quantile(0.95)(events.DurationMs) AS p95, ... ``` +The same query against Postgres imports only `PG`. Params become `$1`, `$2`, and the functions +are Postgres's own: + +```ts +import * as PG from "@maple-dev/effect-orm/postgres" + +const Requests = PG.table("requests", { + columns: { + id: PG.column(PG.int8, { identity: "always" }), + org_id: PG.text, + route: PG.text, + duration_ms: PG.int8, + at: PG.column(PG.timestamptz, { defaultExpr: "now()" }), + }, + primaryKey: ["id"], + tenantColumn: "org_id", +}) + +const byRoute = PG.from(Requests) + .select(($) => ({ route: $.route, count: PG.count(), p50: PG.percentileCont(0.5, $.duration_ms) })) + .where(($) => [$.org_id.eq(PG.param.string("orgId"))]) + .groupBy("route") + +PG.compileUnsafe(byRoute, { orgId: "org_123" }).parameters // -> ["org_123"] +``` + +A table that this schema does not own (a system table, a table function, a view) is declared +with `external: true` and no DDL: `CH.table("system.one", { external: true, columns: {} })`. +See [Tables and column types](./docs/tables-and-types.md). + ## Decoding results Run the SQL with your own ClickHouse client, then hand the rows back to @@ -144,7 +177,7 @@ Full guides live in [`docs/`](./docs/README.md): | Guide | What it covers | | ---------------------------------------------------------- | --------------------------------------------------------------- | | [Getting started](./docs/getting-started.md) | Install, define a table, build → compile → decode | -| [Tables and column types](./docs/tables-and-types.md) | `table()`, column-type constructors, `Map`/`Array`/`Nullable` | +| [Tables and column types](./docs/tables-and-types.md) | `table()`, column options, external tables, column types | | [Building queries](./docs/queries.md) | `select`, `where`, `groupBy`, `orderBy`, `limit`, immutability | | [Expressions and conditions](./docs/expressions.md) | Comparisons, arithmetic, optional predicates, aggregates | | [Joins and subqueries](./docs/joins-and-subqueries.md) | The join family, `fromQuery`, correlated subqueries | @@ -156,6 +189,7 @@ Full guides live in [`docs/`](./docs/README.md): | [Running a query](./docs/running-queries.md) | Executing the SQL with a real client, wire settings, `SETTINGS` | | [Tenant scoping](./docs/tenant-scoping.md) | `tenantColumn`, what marks a query scoped, `crossTenant()` | | [Postgres](./docs/postgres.md) | The Postgres dialect, its column types and functions | +| [Schema and migrations](./docs/migrations.md) | DDL from `table`, `effect-orm generate`, applying migrations | | [Extending the DSL](./docs/extending.md) | `defineFn`, raw escape hatches, handwritten SQL | | [API reference](./docs/reference.md) | Full export catalog by module, plus error types | @@ -165,29 +199,31 @@ regressions live in [`src/docs-examples.test.ts`](./src/docs-examples.test.ts). ## Entry points -| Import | Contents | -| ------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `@maple-dev/effect-orm` | Curated public API: `from`, `compile`, `param`, expression helpers, and ClickHouse functions under friendly names (`min`, `max`, `count`, `quantile`, …). | -| `@maple-dev/effect-orm/types` | Column-type constructors (`string`, `uint64`, `dateTime`, `map`, `array`, `nullable`, …) and the `CH*` type descriptors. | -| `@maple-dev/effect-orm/expr` | Kitchen-sink namespace: every expression helper plus all ClickHouse functions under their raw names (`min_`, `toString_`, `toStartOfInterval`, `dynamicColumn`, …). Handy for `import * as CH`. | -| `@maple-dev/effect-orm/sql` | The low-level `SqlFragment` AST (`raw`, `ident`, `compile`, …) for hand-rolling fragments. | -| `@maple-dev/effect-orm/postgres` | The Postgres dialect: `postgresDialect`, Postgres column types and functions, and a `compile` that defaults to Postgres. | +| Import | Contents | +| ----------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `@maple-dev/effect-orm/clickhouse` | Everything for ClickHouse: the query builder (`from`, `param`, `insertInto`, …), column types (`string`, `uint64`, `map`, …), functions (`count`, `quantile`, …), `table` with its DDL, and `compile`. | +| `@maple-dev/effect-orm/postgres` | The same for Postgres: the builder, Postgres column types (`text`, `int8`, `timestamptz`, …) and functions, `table` with keys, indexes and foreign keys, and a `compile` for Postgres. | +| `@maple-dev/effect-orm/expr` | Kitchen-sink namespace: every expression helper plus all ClickHouse functions under their raw names (`min_`, `toString_`, `toStartOfInterval`, `dynamicColumn`, …). | +| `@maple-dev/effect-orm/sql` | The low-level `SqlFragment` AST (`raw`, `ident`, `compile`, …) for hand-rolling fragments. | +| `@maple-dev/effect-orm/schema` | Migration tooling over `table` values: DDL rendering, snapshots, the schema diff. Pure. | +| `@maple-dev/effect-orm/kit`, `/migrate` | `effect-orm generate` and `check`; applying migrations through a driver you provide. See [Schema and migrations](./docs/migrations.md). | +| `@maple-dev/effect-orm/database` | `Database` over your `SqlClient`: `run`, `execute`, `transaction` with retry. | ## Extending with custom functions ```ts import type { DateTime } from "effect" -import { defineFn, sameAs } from "@maple-dev/effect-orm" +import * as CH from "@maple-dev/effect-orm/clickhouse" // Declare any ClickHouse function not already wrapped. The second argument is // the ClickHouse type it returns — required, because that is what lets a query // using it still derive its row schema. -const toStartOfFiveMinute = defineFn<[CH.Expr], DateTime.Utc>("toStartOfFiveMinute", T.dateTime) +const toStartOfFiveMinute = CH.defineFn<[CH.Expr], DateTime.Utc>("toStartOfFiveMinute", CH.dateTime) // When the result type depends on the arguments — `min`, `argMax`, `coalesce`, // `arrayJoin` all hand back one of their inputs — pass a rule instead: // `sameAs(i)`, `firstTyped()`, `elementOf(i)`, `arrayOfArg(i)`. -const anyLast = defineFn<[CH.Expr], string>("anyLast", sameAs(0)) +const anyLast = CH.defineFn<[CH.Expr], string>("anyLast", CH.sameAs(0)) ``` ## Validation @@ -216,4 +252,4 @@ MIT The optional `@maple-dev/effect-orm/benchmark` entry point and bundled `ch-bench` CLI measure real queries, compare fixed workloads, and save evidence. See [Benchmarking](docs/benchmarking.md) and the -[agent playbook](docs/benchmark-agent.md). The root SQL builder remains driver-free. +[agent playbook](docs/benchmark-agent.md). The `/clickhouse` and `/postgres` builders remain driver-free. diff --git a/docs/README.md b/docs/README.md index 34cf2c9..5d98d99 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,24 +1,31 @@ # Effect ORM -`@maple-dev/effect-orm` builds ClickHouse SQL from typed TypeScript. You describe a +`@maple-dev/effect-orm` builds ClickHouse and Postgres SQL from typed TypeScript. You describe a table once, and the builder infers column types, output row shapes, and join accessors from it. Queries are immutable values — every method returns a new query — and nothing touches the network: the end product is a `CompiledQuery` holding a SQL string plus a typed decoder. You -bring your own ClickHouse client. +bring your own database client. + +Each database has one entry that holds everything for it, the way drizzle and kysely split +dialects: `import * as CH from "@maple-dev/effect-orm/clickhouse"` or +`import * as PG from "@maple-dev/effect-orm/postgres"`. The query builder is the same in both; +the column types, functions, table DDL, and `compile` are the database's own. ## Is this for your project? -Use the builder when you want typed ClickHouse SELECT queries, reusable query definitions, +Use the builder when you want typed ClickHouse or Postgres queries, reusable query definitions, and runtime result decoding in TypeScript. It works with an ordinary async application as well as an Effect application. The database client remains your choice. You do not need a Maple account, Maple's schema, or tenant columns. Tenant analysis is an optional feature for applications that share tables between tenants. -The root builder does not manage connections, create tables, or run migrations. It builds -SELECTs, [INSERTs](./inserts.md), and [UPDATEs and DELETEs](./updates-and-deletes.md). Opt-in [schema and migration entry points](./migrations.md) add DDL and migrations for -ClickHouse. It does not validate SQL against a live server, choose query plans, enforce authorization, -or supply retries. Existing ClickHouse tables and your executor own those responsibilities. +The builder does not manage connections, create tables, or run migrations. It builds +SELECTs, [INSERTs](./inserts.md), and [UPDATEs and DELETEs](./updates-and-deletes.md). Every +`table` carries its DDL, and opt-in [schema and migration entry points](./migrations.md) turn it +into migrations for ClickHouse and Postgres. It does not validate SQL against a live server, +choose query plans, enforce authorization, or supply retries. Your existing tables and your +executor own those responsibilities. [Getting started](./getting-started.md) covers npm installation and building from source. ## Start here @@ -41,7 +48,7 @@ Roughly in reading order. | Guide | What it covers | | ----------------------------------------------------- | ----------------------------------------------------------------------------------- | | [Getting started](./getting-started.md) | Install, define a table, build → compile → decode | -| [Tables and column types](./tables-and-types.md) | `table()`, the column-type constructors, `Map`/`Array`/`Nullable` | +| [Tables and column types](./tables-and-types.md) | `table()`, column options, external tables, the column-type constructors | | [Building queries](./queries.md) | `select`, `where`, `groupBy`, `orderBy`, `limit`, `format`, immutability | | [Expressions and conditions](./expressions.md) | Comparisons, arithmetic, optional predicates, aggregates | | [Joins and subqueries](./joins-and-subqueries.md) | The join family, `fromQuery`, correlated subqueries | @@ -56,7 +63,7 @@ Roughly in reading order. | [Tenant scoping](./tenant-scoping.md) | `tenantScope`, what marks a query scoped, `crossTenant()` | | [Extending the DSL](./extending.md) | `defineFn`, raw escape hatches, handwritten SQL | | [Postgres](./postgres.md) | The Postgres dialect, its column types and functions | -| [Schema and migrations](./migrations.md) | `defineTable`, `materializedView`, `S.pg.table`, `effect-orm generate`, applying migrations, adopting drizzle-kit | +| [Schema and migrations](./migrations.md) | DDL from `CH.table` / `PG.table`, `materializedView`, `effect-orm generate`, applying migrations, adopting drizzle-kit | | [Statements and transactions](./database.md) | `Database` over your `SqlClient`: `run`, `execute`, `transaction`, retry | ## Reference @@ -70,24 +77,24 @@ Roughly in reading order. | Import | Contents | | --------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `@maple-dev/effect-orm` | Curated public API — `from`, `compile`, `param`, expression helpers, and ClickHouse functions under friendly names (`min`, `max`, `count`, `quantile`, …) | -| `@maple-dev/effect-orm/types` | Column-type constructors (`string`, `uint64`, `dateTime`, `map`, `array`, `nullable`, …) and the `CH*` type descriptors | -| `@maple-dev/effect-orm/expr` | Kitchen-sink namespace: every expression helper plus all ClickHouse functions under their raw names (`min_`, `toString_`, `dynamicColumn`, `not`, …) | -| `@maple-dev/effect-orm/sql` | The low-level `SqlFragment` AST (`raw`, `ident`, `compile`, …) for hand-rolling fragments | -| `@maple-dev/effect-orm/benchmark` | Driver-free suite definitions, runner, report schemas, and comparisons | -| `@maple-dev/effect-orm/benchmark/http` | ClickHouse HTTP transport, environment configuration, and query-log collection | -| `@maple-dev/effect-orm/benchmark/cli` | `runCli(args)` for embedding the bundled `ch-bench` commands | -| `@maple-dev/effect-orm/schema` | `defineTable`, `materializedView`, DDL rendering, snapshots, and the schema diff. Pure | -| `@maple-dev/effect-orm/kit` | `generate` and `check` over a migrations folder, `defineConfig`, and `runCli` for the bundled `effect-orm` command. Node or Bun | -| `@maple-dev/effect-orm/migrate` | `run`, `status`, `verify`, `baseline`, and `MigrationDriver`: applies ClickHouse or Postgres migrations through a driver you provide | -| `@maple-dev/effect-orm/database` | `Database` over your `SqlClient`: `run` compiled queries, `execute` statements, `transaction` with settings and contention retry | - -The root barrel is curated, not exhaustive — see -[the reference](./reference.md#whats-only-on-a-subpath) for what lives only on a subpath. +| `@maple-dev/effect-orm/clickhouse` | Everything for ClickHouse: the query builder (`from`, `param`, `insertInto`, `compile`, …), column types (`string`, `uint64`, `map`, …), ClickHouse functions under friendly names (`min`, `max`, `count`, `quantile`, …), and `table`, `column`, `engine`, `index`, `materializedView` | +| `@maple-dev/effect-orm/postgres` | Everything for Postgres: the same builder, Postgres column types (`text`, `int8`, `timestamptz`, `jsonb`, …) and functions, `table`, `column`, `index`, `uniqueIndex`, `foreignKey`, and a `compile` for Postgres | +| `@maple-dev/effect-orm/expr` | Kitchen-sink namespace: every expression helper plus all ClickHouse functions under their raw names (`min_`, `toString_`, `dynamicColumn`, `not`, …) | +| `@maple-dev/effect-orm/sql` | The low-level `SqlFragment` AST (`raw`, `ident`, `compile`, …) for hand-rolling fragments | +| `@maple-dev/effect-orm/benchmark` | Driver-free suite definitions, runner, report schemas, and comparisons | +| `@maple-dev/effect-orm/benchmark/http` | ClickHouse HTTP transport, environment configuration, and query-log collection | +| `@maple-dev/effect-orm/benchmark/cli` | `runCli(args)` for embedding the bundled `ch-bench` commands | +| `@maple-dev/effect-orm/schema` | Tooling over `table` values: `renderSchema` / `renderPgSchema`, `entitiesOf` / `pgEntitiesOf`, snapshots, and the schema diff. Pure | +| `@maple-dev/effect-orm/kit` | `generate` and `check` over a migrations folder, `defineConfig`, and `runCli` for the bundled `effect-orm` command. Node or Bun | +| `@maple-dev/effect-orm/migrate` | `run`, `status`, `verify`, `baseline`, and `MigrationDriver`: applies ClickHouse or Postgres migrations through a driver you provide | +| `@maple-dev/effect-orm/database` | `Database` over your `SqlClient`: `run` compiled queries, `execute` statements, `transaction` with settings and contention retry | + +There is no root import: pick the dialect entry. [The reference](./reference.md) lists every +export of both. ## Query benchmarks The optional `@maple-dev/effect-orm/benchmark` entry point and bundled `ch-bench` CLI measure real queries, compare fixed workloads, and save evidence. See [Benchmarking](./benchmarking.md) and the -[agent playbook](./benchmark-agent.md). The root SQL builder remains driver-free. +[agent playbook](./benchmark-agent.md). The SQL builders remain driver-free. diff --git a/docs/benchmarking.md b/docs/benchmarking.md index ccb65cf..59788a2 100644 --- a/docs/benchmarking.md +++ b/docs/benchmarking.md @@ -1,8 +1,8 @@ # Benchmarking ClickHouse queries The package ships a driver-free `@maple-dev/effect-orm/benchmark` API, -an HTTP adapter at `/benchmark/http`, and the `ch-bench` executable. The builder's -root import still does no networking. No Maple account or schema is required. +an HTTP adapter at `/benchmark/http`, and the `ch-bench` executable. The query +builder entries (`/clickhouse`, `/postgres`) still do no networking. No Maple account or schema is required. The executable runs on Node 22.18+ or Bun. For TypeScript suites with extensionless imports or application path aliases, use Bun (`bun run ch-bench …`). With Node, use @@ -36,11 +36,14 @@ it does not create a dataset or determine whether your query inputs are represen ## Define a workload ```ts title="benchmark-suite.ts" -import { compile, from, param, table } from "@maple-dev/effect-orm" -import { uint32, string } from "@maple-dev/effect-orm/types" +import { compile, engine, from, param, string, table, uint32 } from "@maple-dev/effect-orm/clickhouse" import * as Bench from "@maple-dev/effect-orm/benchmark" -const events = table("events", { id: uint32, name: string }) +const events = table("events", { + columns: { id: uint32, name: string }, + engine: engine.mergeTree(), + orderBy: ["id"], +}) const byName = from(events) .select("id", "name") .where(($) => [$.name.eq(param.string("name"))]) diff --git a/docs/database.md b/docs/database.md index ff1321d..683253d 100644 --- a/docs/database.md +++ b/docs/database.md @@ -25,11 +25,10 @@ This runs on PGlite, Postgres compiled to WASM, so it needs no server. Swap ```ts title="database-transaction.ts" import { PgliteClient } from "@effect/sql-pglite" import { Effect, Layer, Schema } from "effect" -import * as CH from "@maple-dev/effect-orm" import * as Db from "@maple-dev/effect-orm/database" import * as PG from "@maple-dev/effect-orm/postgres" -const Accounts = CH.table("accounts", { id: PG.int4, balance: PG.int8 }) +const Accounts = PG.table("accounts", { columns: { id: PG.int4, balance: PG.int8 }, primaryKey: ["id"] }) class InsufficientFunds extends Schema.TaggedError()("InsufficientFunds", { account: Schema.Number, @@ -37,7 +36,7 @@ class InsufficientFunds extends Schema.TaggedError()("Insuffi const balanceOf = (id: number) => Db.run( - CH.from(Accounts) + PG.from(Accounts) .select("balance") .where(($) => [$.id.eq(id)]), ).pipe(Effect.map((rows) => rows[0]?.balance ?? 0)) @@ -103,7 +102,8 @@ Calling `withdraw(1, 30)` outside `transfer` does not compile: `requireTransacti query compiled elsewhere. It compiles with the database's dialect, so you never pick a `compile`; `params` fills the query's `param.*` markers, and a missing one fails with `QueryBuilderError`. A query compiled elsewhere must -have been compiled for the same dialect, or `run` dies: the root `compile` is ClickHouse's. +have been compiled for the same dialect, or `run` dies: `CH.compile` from `/clickhouse` writes +ClickHouse, `PG.compile` from `/postgres` writes Postgres. `sql` writes the statements the builder does not have yet (DDL, bulk `UPDATE ... FROM`, advisory locks). Each `${value}` is bound, as `$1, $2, ...` on Postgres and as an escaped literal on @@ -146,7 +146,7 @@ For SQL inside a builder query rather than a whole statement, use | `observe` | Called with every statement before it runs, including the `SET TRANSACTION` a transaction's settings become | `run` refuses a query compiled for another dialect (`CompiledQuery.dialect`), as a defect: a -query built with the root `compile`, which is ClickHouse's, can run on Postgres with the wrong +query compiled with the `/clickhouse` entry's `compile` would run on Postgres with the wrong quoting and inlined params. Rows come back without the client's name transforms, because the decoder reads the aliases the compiler wrote. diff --git a/docs/decoding-results.md b/docs/decoding-results.md index 5ec69f6..bbf76c7 100644 --- a/docs/decoding-results.md +++ b/docs/decoding-results.md @@ -19,7 +19,7 @@ Provide the `ClickhouseClient` layer at the application boundary. ## The row schema is derived from the SELECT -Column types _are_ Effect schemas — `T.uint64` is "a 64-bit integer as ClickHouse actually sends +Column types _are_ Effect schemas — `CH.uint64` is "a 64-bit integer as ClickHouse actually sends it", not a phantom tag. So a query built from typed pieces already knows how its rows decode, and `compile` folds those schemas into one: @@ -40,7 +40,7 @@ await Effect.runPromise(compiled.decodeRows([{ name: "checkout", calls: "42" }]) Note `calls`. ClickHouse's `FORMAT JSON` quotes 64-bit integers, a client that sets `output_format_json_quote_64bit_integers=0` gets them bare, and a gateway that refuses `output_format_json_quote_64bit_integers=0` quotes them whatever you asked for. -`T.uint64` accepts both wire representations and decodes them to JavaScript numbers. +`CH.uint64` accepts both wire representations and decodes them to JavaScript numbers. ## When there is nothing to derive from @@ -62,8 +62,8 @@ await Effect.runPromise(compiled.decodeRows([{ name: 42, odd: 1 }])) Inventing a permissive schema for that one field would hand back something that _looks_ validated and is not, so the query keeps its honest answer instead. `untypedColumns` names what to fix; -close the gap by typing the escape hatch — `CH.rawExpr("anyLast(Whatever)", T.string)`, -`CH.defineFn("myFn", T.uint64)` — or by declaring the whole schema yourself. +close the gap by typing the escape hatch — `CH.rawExpr("anyLast(Whatever)", CH.string)`, +`CH.defineFn("myFn", CH.uint64)` — or by declaring the whole schema yourself. ## Going back to the wire @@ -170,6 +170,6 @@ Only relevant when you declare one by hand; the column types already handle thes - **64-bit integers** — accept both wire shapes. A `UInt64` above `2^53` cannot survive as a JavaScript number at all; have such columns emitted as strings (`toString(...)`) in the SELECT and use a string schema for the projected field. -- **`DateTime` columns** — `T.dateTime` parses them as UTC; `T.dateTimeString` leaves them as +- **`DateTime` columns** — `CH.dateTime` parses them as UTC; `CH.dateTimeString` leaves them as sent. See [Tables and column types](./tables-and-types.md#column-types). - **`leftJoin` columns** — nullable on the SQL side, so pair them with `Schema.NullOr`. diff --git a/docs/expressions.md b/docs/expressions.md index e3f3a98..3f506eb 100644 --- a/docs/expressions.md +++ b/docs/expressions.md @@ -25,10 +25,13 @@ compilation with a `QueryBuilderError`. Use `.isNull()` (or `.isNotNull()` for p values), which write `IS NULL` and work on every dialect: ```ts title="null-filter.ts" -import * as CH from "@maple-dev/effect-orm" -import * as T from "@maple-dev/effect-orm/types" +import * as CH from "@maple-dev/effect-orm/clickhouse" -const Notes = CH.table("notes", { Note: T.nullable(T.string) }) +const Notes = CH.table("notes", { + columns: { Note: CH.nullable(CH.string) }, + engine: CH.engine.mergeTree(), + orderBy: [], +}) export const compiled = CH.compileUnsafe(CH.from(Notes).select("Note").where(($) => [$.Note.isNull()])) console.log(compiled.sql) // SELECT Note AS Note FROM notes WHERE Note IS NULL ``` @@ -84,7 +87,7 @@ list is written as the constant it means, `1 = 0` for `in_()` and `1 = 1` for `n ``` `.and()` / `.or()` parenthesise their result, so precedence is explicit. `CH.not(condition)` wraps -in `NOT (…)` and is available from the root and `/expr` subpath. +in `NOT (…)` and is available from both dialect entries and the `/expr` subpath. `CH.and(...)` and `CH.or(...)` take any number of conditions, skip `undefined` ones, and write one flat group. With none left they return `undefined`, which `where` skips, so optional diff --git a/docs/extending.md b/docs/extending.md index 8c9de15..b953b76 100644 --- a/docs/extending.md +++ b/docs/extending.md @@ -13,7 +13,7 @@ import type { DateTime } from "effect" const toStartOfFiveMinute = CH.defineFn<[CH.Expr], DateTime.Utc>( "toStartOfFiveMinute", - T.dateTime, + CH.dateTime, ) CH.from(Events) @@ -74,13 +74,16 @@ _(Backed by `docs/extending.md > defineCondFn declares a predicate`.)_ When the signature is too irregular for `defineFn`, write the wrapper yourself: ```ts title="typed-function.ts" -import * as CH from "@maple-dev/effect-orm" -import * as T from "@maple-dev/effect-orm/types" +import * as CH from "@maple-dev/effect-orm/clickhouse" const greatestOf = (first: CH.Expr, ...rest: CH.Expr[]) => - CH.compileTypedFnCall("greatest", T.float64.schema, first, ...rest) + CH.compileTypedFnCall("greatest", CH.float64.schema, first, ...rest) -const Events = CH.table("events", { Name: T.string, DurationMs: T.uint64 }) +const Events = CH.table("events", { + columns: { Name: CH.string, DurationMs: CH.uint64 }, + engine: CH.engine.mergeTree(), + orderBy: ["Name"], +}) export const compiled = CH.compileUnsafe( CH.from(Events).select(($) => ({ name: $.Name, durationMs: greatestOf($.DurationMs, CH.lit(1)) })), {}, @@ -101,13 +104,13 @@ For functions whose call syntax is not `fn(a, b)` at all — parametric aggregat anything bespoke: ```ts -import { makeExpr } from "@maple-dev/effect-orm" +import { makeExpr } from "@maple-dev/effect-orm/clickhouse" import { raw, compile } from "@maple-dev/effect-orm/sql" const quantileExact = (q: number) => (expr: CH.Expr) => - makeExpr(raw(`quantileExact(${q})(${compile(expr.toFragment())})`), T.float64.schema, undefined, [expr]) + makeExpr(raw(`quantileExact(${q})(${compile(expr.toFragment())})`), CH.float64.schema, undefined, [expr]) ``` This is how the bundled `quantile` is built. The last argument, `uses`, lists the expressions @@ -150,16 +153,20 @@ it never makes a valid query fail. ## A column type of your own -`T.custom(sql, schema)` is the extension point the built-in types are built from — `T.uint64` is +`CH.custom(sql, schema)` is the extension point the built-in types are built from — `CH.uint64` is `custom("UInt64", CHNumber)`. Declare one for a ClickHouse type this package does not model and it works everywhere a built-in does: rows decode through it, literals encode through it, and `param.of(type, name)` takes it as a param. ```ts -const Level = T.custom("Enum8('warn' = 1, 'error' = 2)", Schema.Literals(["warn", "error"])) -const Decimal = T.custom("Decimal(18, 4)", Schema.String) +const Level = CH.custom("Enum8('warn' = 1, 'error' = 2)", Schema.Literals(["warn", "error"])) +const Decimal = CH.custom("Decimal(18, 4)", Schema.String) -const Logs = CH.table("logs", { OrgId: T.string, Level, Amount: Decimal }) +const Logs = CH.table("logs", { + columns: { OrgId: CH.string, Level, Amount: Decimal }, + engine: CH.engine.mergeTree(), + orderBy: ["OrgId"], +}) ``` This Decimal declaration expects decimal text from your client and preserves it as a string. @@ -176,15 +183,16 @@ _(Backed by `src/ch/literal.test.ts > param.of`.)_ For SQL the builder has no syntax for — a cast, an operator, a Postgres function — write a template. It is an expression (or, with `.cond`, a condition), so it goes anywhere the builder -takes one: a select, a `where`, a join's ON, an UPDATE's SET. +takes one: a select, a `where`, a join's ON, an UPDATE's SET. `sql` is on both entries; this +example is Postgres. ```ts -CH.from(Keys) +PG.from(Keys) .select(($) => ({ - txid: CH.sql(PG.text)`pg_current_xact_id()::xid::text`, - next: CH.sql(PG.int8)`${$.uses} + ${1}`, + txid: PG.sql(PG.text)`pg_current_xact_id()::xid::text`, + next: PG.sql(PG.int8)`${$.uses} + ${1}`, })) - .where(($) => [CH.sql.cond`${$.meta} @> ${CH.param.string("filter")}::jsonb`]) + .where(($) => [PG.sql.cond`${$.meta} @> ${PG.param.string("filter")}::jsonb`]) // SELECT (pg_current_xact_id()::xid::text) AS "txid", ("keys"."uses" + 1) AS "next" … // WHERE ("keys"."meta" @> $1::jsonb) ``` @@ -226,7 +234,7 @@ SQL produces, so the row it lands in can still be decoded: ```ts CH.from(Events) - .select(($) => ({ odd: CH.rawExpr("DurationMs % 2", T.float64) })) + .select(($) => ({ odd: CH.rawExpr("DurationMs % 2", CH.float64) })) .where(($) => [$.OrgId.eq("org_123"), CH.rawCond("Name GLOBAL IN (SELECT 1)")]) ``` @@ -237,7 +245,7 @@ CH.from(Events) that is only ever an `argMin` tiebreaker, never a selected value. Selecting one costs the query its row schema, so it is deliberately a separate name. -`dynamicColumn(name, type?)` (on the root and `/expr` subpath) is the same idea for a column name only +`dynamicColumn(name, type?)` (on both dialect entries and `/expr`) is the same idea for a column name only known at runtime; pass the type where you know it. _(Backed by `docs/extending.md > rawExpr and rawCond are the last resort`.)_ diff --git a/docs/getting-started.md b/docs/getting-started.md index f086815..e636449 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -41,6 +41,20 @@ Keep the Effect 4 range explicit when installing. Use an ESM project and a TypeScript runner such as Bun for the `.ts` files below. A database client is a separate dependency, needed only when you execute SQL. +## Pick your database + +Each database has one entry that holds everything for it: the query builder, its column types, +its functions, table definitions, and a `compile` that writes its dialect. + +```ts +import * as CH from "@maple-dev/effect-orm/clickhouse" +import * as PG from "@maple-dev/effect-orm/postgres" +``` + +The builder (`from`, `param`, `and`, `insertInto`, …) is the same in both. Import the one for the +database you query; the examples below use ClickHouse, and [Postgres](./postgres.md) shows the +same flow with `PG`. + ## A complete first example Save this as `quick-start.ts` and run `bun quick-start.ts`. It builds SQL and decodes a sample @@ -48,12 +62,15 @@ wire response; it does not need a server, credentials, or an existing table. ```ts title="quick-start.ts" import { Effect } from "effect" -import * as CH from "@maple-dev/effect-orm" -import * as T from "@maple-dev/effect-orm/types" +import * as CH from "@maple-dev/effect-orm/clickhouse" const Events = CH.table("events", { - Name: T.string, - DurationMs: T.uint64, + columns: { + Name: CH.string, + DurationMs: CH.uint64, + }, + engine: CH.engine.mergeTree(), + orderBy: ["Name"], }) const query = CH.from(Events) @@ -78,15 +95,20 @@ console.log(rows) // [{ name: "checkout", p95: 420, count: 3 }] The generated SQL is: ```sql -SELECT Name AS name, quantile(0.95)(DurationMs) AS p95, count() AS count +SELECT events.Name AS name, quantile(0.95)(events.DurationMs) AS p95, count() AS count FROM events -WHERE DurationMs >= 100 +WHERE events.DurationMs >= 100 GROUP BY name ORDER BY count DESC, name ASC LIMIT 50 ``` -`table()` describes a table; it does not create it or check that the database has those columns. +`table()` describes a table; compiling a query does not create it or check that the database has +those columns. The `engine` and `orderBy` are what [migrations](./migrations.md) turn into +`CREATE TABLE`; the query builder only reads the name and `columns`. A MergeTree table must +declare `orderBy` (`[]` for `ORDER BY tuple()`), so a definition that could not become DDL fails +when the module loads, not at deploy time. + The keys returned by `select` become both SQL aliases and result properties. This query infers `{ name: string; p95: number | null; count: number }`: ClickHouse can return JSON `null` for an aggregate with a non-finite result. @@ -101,35 +123,36 @@ The result schema is derived from the typed SELECT, so you do not need to write ## Shared tables used by the guides -The later guides use `CH`, `T`, `Effect`, and these illustrative tables. Save this as `schema.ts` +The later guides use `CH`, `Effect`, and these illustrative tables. Save this as `schema.ts` when trying their query snippets. Their table and column names are case-sensitive contracts with your own database. Replace them with your real schema before executing. ```ts title="schema.ts" -import * as CH from "@maple-dev/effect-orm" -import * as T from "@maple-dev/effect-orm/types" - -export const Events = CH.table( - "events", - { - OrgId: T.string, - Name: T.string, - Timestamp: T.dateTime, - DurationMs: T.uint64, - Attributes: T.map(T.string, T.string), +import * as CH from "@maple-dev/effect-orm/clickhouse" + +export const Events = CH.table("events", { + columns: { + OrgId: CH.string, + Name: CH.string, + Timestamp: CH.dateTime, + DurationMs: CH.uint64, + Attributes: CH.map(CH.string, CH.string), }, - { tenantColumn: "OrgId" }, -) - -export const Services = CH.table( - "services", - { - OrgId: T.string, - Name: T.string, - Team: T.string, + engine: CH.engine.mergeTree(), + orderBy: ["OrgId", "Timestamp"], + tenantColumn: "OrgId", +}) + +export const Services = CH.table("services", { + columns: { + OrgId: CH.string, + Name: CH.string, + Team: CH.string, }, - { tenantColumn: "OrgId" }, -) + engine: CH.engine.replacingMergeTree(), + orderBy: ["OrgId", "Name"], + tenantColumn: "OrgId", +}) ``` Tenant scoping is optional. The first example has no tenant column; the shared tables do. @@ -141,4 +164,5 @@ Always supply the tenant from your trusted application context. See [Tenant scop - [Running a query](./running-queries.md): a complete client example using `system.numbers`, with no table setup. - [Recipes](./recipes.md): time buckets, optional filters, aggregate filters, pagination, and lossless IDs. - [Tables and column types](./tables-and-types.md): model your actual schema and wire formats. +- [Postgres](./postgres.md): the same builder against Postgres, from `@maple-dev/effect-orm/postgres`. - [Troubleshooting](./troubleshooting.md): installation, compilation, decoding, and unexpected results. diff --git a/docs/inserts.md b/docs/inserts.md index d1099a2..bba862e 100644 --- a/docs/inserts.md +++ b/docs/inserts.md @@ -5,26 +5,25 @@ read. Like a query, it is an immutable value: nothing is sent until you run it, writes it for the dialect you compile with. ```ts -import * as CH from "@maple-dev/effect-orm" import * as Db from "@maple-dev/effect-orm/database" import * as PG from "@maple-dev/effect-orm/postgres" -const ApiKeys = CH.table( - "api_keys", - { +const ApiKeys = PG.table("api_keys", { + columns: { id: PG.uuid, org_id: PG.text, name: PG.text, - created_at: PG.timestamptz, - revoked: PG.bool, + created_at: PG.column(PG.timestamptz, { defaultExpr: "now()" }), + revoked: PG.column(PG.bool, { default: false }), note: PG.nullable(PG.text), }, - { tenantColumn: "org_id", defaults: ["created_at", "revoked"] }, -) + primaryKey: ["id"], + tenantColumn: "org_id", +}) -const insertKey = CH.insertInto(ApiKeys).values({ - id: CH.param.string("id"), - org_id: CH.param.string("orgId"), +const insertKey = PG.insertInto(ApiKeys).values({ + id: PG.param.string("id"), + org_id: PG.param.string("orgId"), name: "default", }) @@ -38,30 +37,36 @@ const insertKey = CH.insertInto(ApiKeys).values({ Each row is typed from the table: -- A column is **required** unless it is nullable or listed in `defaults`. +- A column is **required** unless it is nullable or its column options give it a default. - Leaving an optional column out, or passing `undefined`, writes the column's default. - `null` writes NULL, and only type-checks on a nullable column. - A value can be a plain value of the column's type, a `param.*` of it, or any expression of it, - such as `CH.rawExpr("now()", CH.dateTime)`. A `DateTime` column also takes a `Date` or the + such as `PG.now()`. A `DateTime` column also takes a `Date` or the `'YYYY-MM-DD hh:mm:ss'` string, as in a comparison. `InsertRowOf` names the row type, for a function that builds rows. ### Which columns have defaults -`table()` cannot see your DDL, so you list the columns the database fills in with -`defaults`: a Postgres `serial` or `DEFAULT now()`, a ClickHouse `DEFAULT`. A table declared -with [`defineTable`](./migrations.md) works this out from its column options: a column with -`default` or `defaultExpr` is optional, and a `materialized` or `alias` column cannot be inserted -at all (it is not in the row type, and a row that names it anyway fails to compile). On `table()`, -list such columns (a Postgres `GENERATED ALWAYS` column) with `computed`; they stay readable. +The row type is derived from the column options that also write the [DDL](./migrations.md), so +it cannot drift from what the database fills in: + +- `default` or `defaultExpr` (both dialects) or `identity` (Postgres) makes a column optional. +- `materialized` or `alias` (ClickHouse) makes a column not writable: it is not in the row type, + and a row that names it anyway fails to compile. It stays readable. + +Postgres `GENERATED ALWAYS AS (...) STORED` columns are not modeled yet: there is no column +option for them, so such a column reads as an ordinary required column. + +An [external table](./migrations.md) (`external: true`, for a table another tool migrates) has +no DDL, but its column options still drive the row type the same way. `insertInto(table)` offers only `values` and `select` until it has rows (its type is `CHInsertStart`), so an insert cannot be compiled or run before it says what to insert. ClickHouse fills every column it is not given with a default, even without a `DEFAULT` clause: -`0` for a number, `''` for a string. The row type still requires those columns unless you list -them, so a forgotten value is a type error rather than a silent zero. +`0` for a number, `''` for a string. The row type still requires those columns unless they +declare a default, so a forgotten value is a type error rather than a silent zero. ## What it compiles to @@ -70,11 +75,12 @@ in different orders cannot swap values. A column that some rows give and others `DEFAULT` in the rows that leave it out: ```ts -const Events = CH.table( - "events", - { OrgId: CH.string, Id: CH.uint64, At: CH.dateTime }, - { tenantColumn: "OrgId", defaults: ["Id"] }, -) +const Events = CH.table("events", { + columns: { OrgId: CH.string, Id: CH.column(CH.uint64, { default: 0 }), At: CH.dateTime }, + engine: CH.engine.mergeTree(), + orderBy: ["OrgId", "At"], + tenantColumn: "OrgId", +}) CH.compileUnsafe( CH.insertInto(Events).values([ @@ -100,8 +106,18 @@ the codec rejects fails to compile with a `QueryBuilderError` that names the row selected alias names the column it goes into, so select under the target's column names: ```ts -const Spans = CH.table("spans", { OrgId: CH.string, Name: CH.string, Ms: CH.uint64 }, { tenantColumn: "OrgId" }) -const Daily = CH.table("daily", { OrgId: CH.string, Name: CH.string, Total: CH.uint64 }, { tenantColumn: "OrgId" }) +const Spans = CH.table("spans", { + columns: { OrgId: CH.string, Name: CH.string, Ms: CH.uint64 }, + engine: CH.engine.mergeTree(), + orderBy: ["OrgId", "Name"], + tenantColumn: "OrgId", +}) +const Daily = CH.table("daily", { + columns: { OrgId: CH.string, Name: CH.string, Total: CH.uint64 }, + engine: CH.engine.summingMergeTree(), + orderBy: ["OrgId", "Name"], + tenantColumn: "OrgId", +}) CH.insertInto(Daily).select( CH.from(Spans) @@ -114,7 +130,7 @@ CH.insertInto(Daily).select( ``` The selected row is checked against the table: selecting a column the table does not have (or -a computed one), selecting a value of another type, or leaving out a required column is a type +a `materialized` or `alias` one), selecting a value of another type, or leaving out a required column is a type error naming the columns (`targetCannotTake`, `missingColumns`). A nullable result, such as a Postgres `sum`, does not fit a NOT NULL column; wrap it in `coalesce`. @@ -132,8 +148,8 @@ decoded. With no arguments it returns every column, as Drizzle's bare `.returnin also takes column names, or a callback building one expression per alias, as `select` does: ```ts -const created = CH.insertInto(ApiKeys) - .values({ id: CH.param.string("id"), org_id: CH.param.string("orgId"), name: "default" }) +const created = PG.insertInto(ApiKeys) + .values({ id: PG.param.string("id"), org_id: PG.param.string("orgId"), name: "default" }) .returning(($) => ({ id: $.id, createdAt: $.created_at })) // const [row] = yield* Db.run(created, { id, orgId }) // { id: string; createdAt: DateTime.Utc } @@ -150,13 +166,16 @@ On Postgres, `onConflictDoNothing` and `onConflictDoUpdate` add an `ON CONFLICT` options follow Drizzle's, so code moving from Drizzle changes little. ```ts -const Counters = CH.table("counters", { key: PG.text, count: PG.int8, locked: PG.bool }, { defaults: ["locked"] }) +const Counters = PG.table("counters", { + columns: { key: PG.text, count: PG.int8, locked: PG.column(PG.bool, { default: false }) }, + primaryKey: ["key"], +}) // Skip a row whose key exists. Without `target`, any unique index or constraint counts. -CH.insertInto(Counters).values({ key: "a", count: 1 }).onConflictDoNothing({ target: ["key"] }) +PG.insertInto(Counters).values({ key: "a", count: 1 }).onConflictDoNothing({ target: ["key"] }) // Upsert: add to the existing count, unless the row is locked. -CH.insertInto(Counters) +PG.insertInto(Counters) .values({ key: "a", count: 1 }) .onConflictDoUpdate({ target: ["key"], @@ -216,7 +235,7 @@ without a tenant column gives `"untenanted"`. | Case | Result | | ------------------------------------------------- | ---------------------------------------- | | `values([])`, a row with no values | `QueryBuilderError` `InvalidArguments` | -| A key that is not a column, or a computed column | `QueryBuilderError` `InvalidArguments` | +| A key that is not a column, or a non-writable one | `QueryBuilderError` `InvalidArguments` | | A value the column's codec rejects | `QueryBuilderError` `InvalidLiteral` | | A param with no value | `QueryBuilderError` `UnresolvedParam` | | Over the dialect's bound-value limit | `QueryBuilderError` `InvalidArguments` | diff --git a/docs/joins-and-subqueries.md b/docs/joins-and-subqueries.md index c98a524..788e67b 100644 --- a/docs/joins-and-subqueries.md +++ b/docs/joins-and-subqueries.md @@ -145,7 +145,7 @@ That is what `subqueryExpr` is for. It takes the inner query, the column type it as, and a `wrap` function that receives the inner SQL and returns the expression text: ```ts -import { subqueryCond, subqueryExpr } from "@maple-dev/effect-orm" +import { subqueryCond, subqueryExpr } from "@maple-dev/effect-orm/clickhouse" // Stage 1: a cheap scan reading only the sort column. const cheapScan = CH.from(Events) @@ -154,7 +154,7 @@ const cheapScan = CH.from(Events) .orderBy(["ts", "desc"]) .limit(100) -const cutoff = subqueryExpr(cheapScan, T.dateTime, (sql) => `(SELECT min(ts) FROM (${sql}))`) +const cutoff = subqueryExpr(cheapScan, CH.dateTime, (sql) => `(SELECT min(ts) FROM (${sql}))`) // Stage 2: the heavy columns, read only for rows at or after the cutoff. const query = CH.from(Events) @@ -198,7 +198,7 @@ _(Backed by `docs/joins-and-subqueries.md > subqueryExpr splices an inner query, - `inList(expr, values)` — `expr IN ('a', 'b')` for a string list - `inExprList(expr, exprs)` — same, for expression lists -- `notInList(expr, values)` — available from the root and `/expr` subpath +- `notInList(expr, values)` — available from both dialect entries and the `/expr` subpath These predate `.in_()` and remain useful when you have an array in hand rather than varargs. diff --git a/docs/migrations.md b/docs/migrations.md index 83bb04a..473161b 100644 --- a/docs/migrations.md +++ b/docs/migrations.md @@ -1,11 +1,13 @@ # Schema and migrations -The query builder works with tables you manage elsewhere. If you would rather keep the schema -in TypeScript too, three entry points add that, all opt-in: +Every table is declared with its DDL: `table` from `/clickhouse` or `/postgres` carries the +engine, keys, indexes, and defaults beside the columns the query builder reads. You can ignore +that and manage the database elsewhere, or let three opt-in entry points turn the definitions +into migrations: | Entry | Runs where | What it does | | ---------------------------------- | ------------------ | ----------------------------------------------------------------------------- | -| `@maple-dev/effect-orm/schema` | anywhere, pure | `defineTable` / `materializedView` / `pg.table`, DDL rendering, snapshots, the diff | +| `@maple-dev/effect-orm/schema` | anywhere, pure | reads `table` / `materializedView` values: DDL rendering, snapshots, the diff | | `@maple-dev/effect-orm/kit` | Node or Bun | `generate` and `check` over a migrations folder; the `effect-orm` command | | `@maple-dev/effect-orm/migrate` | anywhere Effect runs | applies migrations through a driver you provide, `status`, `verify` | @@ -18,38 +20,39 @@ sections below describe ClickHouse first; [Postgres](#postgres) covers what diff ## Defining tables -`defineTable` returns a `Table`, so every query API accepts it. Columns are the usual column -types, or `S.column(type, options)` for a default, a codec, or a comment. Keys, TTL, defaults, -and index expressions are SQL strings or DSL callbacks. +`CH.table` returns a `Table`, so every query API accepts it. Columns are the usual column +types, or `CH.column(type, options)` for a default, a codec, or a comment. Keys, TTL, defaults, +and index expressions are SQL strings or DSL callbacks. `/schema` only reads these values: it +renders them, snapshots them, and diffs them. ```ts title="migrations-schema.ts" -import * as CH from "@maple-dev/effect-orm" +import * as CH from "@maple-dev/effect-orm/clickhouse" import * as S from "@maple-dev/effect-orm/schema" -export const Requests = S.defineTable("requests", { +export const Requests = CH.table("requests", { columns: { OrgId: CH.custom("LowCardinality(String)", CH.string.schema), Timestamp: CH.dateTime, Route: CH.string, - Status: S.column(CH.uint16, { default: 200 }), + Status: CH.column(CH.uint16, { default: 200 }), }, - engine: S.engine.mergeTree(), + engine: CH.engine.mergeTree(), orderBy: ["OrgId", "Route", "Timestamp"], partitionBy: "toDate(Timestamp)", - ttl: S.ttlAfterDays("toDate(Timestamp)", 30), - indexes: [S.index("idx_status", ($) => $.Status, "set(100)")], + ttl: CH.ttlAfterDays("toDate(Timestamp)", 30), + indexes: [CH.index("idx_status", ($) => $.Status, "set(100)")], tenantColumn: "OrgId", }) -export const RoutesHourly = S.defineTable("routes_hourly", { +export const RoutesHourly = CH.table("routes_hourly", { columns: { OrgId: CH.string, Hour: CH.dateTime, Route: CH.string, Requests: CH.uint64 }, - engine: S.engine.summingMergeTree(), + engine: CH.engine.summingMergeTree(), orderBy: ["OrgId", "Hour", "Route"], }) // The body is a DSL query. An output column the target lacks, or of another // type, is a type error here rather than a failed insert later. -export const RoutesHourlyMv = S.materializedView("routes_hourly_mv", { +export const RoutesHourlyMv = CH.materializedView("routes_hourly_mv", { to: RoutesHourly, as: CH.from(Requests) .select(($) => ({ OrgId: $.OrgId, Hour: CH.toStartOfHour($.Timestamp), Route: $.Route, Requests: CH.count() })) @@ -68,6 +71,11 @@ Write engines as the plain family. Replicated engines and `ON CLUSTER` are rende (`{ replicated: {}, cluster: "main" }`), so one schema serves a single server, a cluster, and ClickHouse Cloud. +A table declared with `external: true` (a system table, a table function, a table another tool +migrates) carries no DDL: `S.isSchemaObject` is false for it, so `generate` skips it and +`entitiesOf` does not take it. See +[External tables](./tables-and-types.md#external-tables). + ## Generating migrations Create `effect-orm.config.ts`: @@ -182,55 +190,55 @@ its own runner.)_ ## Postgres -Set `dialect: "postgres"` in the config and define tables with `S.pg.table`. `generate`, `check`, +Set `dialect: "postgres"` in the config and define tables with `PG.table`. `generate`, `check`, `migrate`, `status` and `verify` then work as above, with the differences below. ```ts title="migrations-postgres.ts" -import * as CH from "@maple-dev/effect-orm" import * as PG from "@maple-dev/effect-orm/postgres" import * as S from "@maple-dev/effect-orm/schema" -export const Dashboards = S.pg.table("dashboards", { +export const Dashboards = PG.table("dashboards", { columns: { org_id: PG.text, id: PG.text, - status: S.pg.column(PG.text, { default: "open" }), - created_at: S.pg.column(PG.timestamptz, { defaultExpr: "now()" }), + status: PG.column(PG.text, { default: "open" }), + created_at: PG.column(PG.timestamptz, { defaultExpr: "now()" }), archived_at: PG.nullable(PG.timestamptz), }, primaryKey: ["org_id", "id"], - indexes: [S.pg.index("dashboards_open_idx", ["org_id"], { where: ($) => $.archived_at.isNull() })], + indexes: [PG.index("dashboards_open_idx", ["org_id"], { where: ($) => $.archived_at.isNull() })], tenantColumn: "org_id", }) -export const Shares = S.pg.table("dashboard_shares", { +export const Shares = PG.table("dashboard_shares", { columns: { org_id: PG.text, id: PG.text, dashboard_id: PG.text, widget_id: PG.nullable(PG.text), revoked_at: PG.nullable(PG.timestamptz) }, primaryKey: ["org_id", "id"], indexes: [ // At most one live share per dashboard and widget: a partial unique index on an expression. - S.pg.uniqueIndex("dashboard_shares_live_unq", ($) => [$.org_id, $.dashboard_id, CH.coalesce($.widget_id, CH.lit(""))], { + PG.uniqueIndex("dashboard_shares_live_unq", ($) => [$.org_id, $.dashboard_id, PG.coalesce($.widget_id, PG.lit(""))], { where: "revoked_at is null", }), ], foreignKeys: [ - S.pg.foreignKey({ columns: ["org_id", "dashboard_id"], references: Dashboards, foreignColumns: ["org_id", "id"], onDelete: "cascade" }), + PG.foreignKey({ columns: ["org_id", "dashboard_id"], references: Dashboards, foreignColumns: ["org_id", "id"], onDelete: "cascade" }), ], }) export const ddl = S.renderPgSchema(S.pgEntitiesOf([Dashboards, Shares])) ``` -**Definitions.** A column is `NOT NULL` unless its type is `PG.nullable(...)`. `S.pg.column(type, +**Definitions.** A column is `NOT NULL` unless its type is `PG.nullable(...)`. `PG.column(type, options)` adds a `default` (a value of the column's type), a `defaultExpr` (SQL or a DSL expression) or an `identity` (`"always"` or `"by default"`); any of them makes the column optional on insert. `primaryKey` takes column names, or `{ columns, name }`; the default name is -`_pkey`. Indexes are `S.pg.index` / `S.pg.uniqueIndex` over column names or expressions, +`
_pkey`. Indexes are `PG.index` / `PG.uniqueIndex` over column names or expressions, with `where` for a partial index and `using` for the access method. A foreign key without a `name` gets drizzle-orm's, `
____fk`, shortened with drizzle-kit's hash to `
__fk` when it would pass 63 characters. Types are stored as Postgres names them (`int4` is `integer`), so snapshots compare with the catalog and -with drizzle-kit. Check and unique constraints, enums, views, sequences and other schemas are -not modeled yet; write them in a `--custom` migration. +with drizzle-kit. Check and unique constraints, generated columns, enums, views, sequences and other schemas +are not modeled yet; write them in a `--custom` migration, and declare a view you query as an +[external table](./tables-and-types.md#external-tables). **Generating.** Postgres changes a column's type, nullability, default or identity in place (`ALTER COLUMN`), and a primary key, an index or a foreign key by dropping and re-creating it, @@ -262,13 +270,13 @@ and set aside, so the folder runs as it is. Adoption is two steps: 1. `effect-orm generate --baseline --from-drizzle` writes a migration that runs nothing, whose snapshot is drizzle-kit's last one converted to entities. Anything the conversion cannot model is listed and nothing is written. Without `--from-drizzle` the snapshot comes from your - `S.pg.table` definitions instead. Migrations before the baseline are legacy: they run, but + `PG.table` definitions instead. Migrations before the baseline are legacy: they run, but nothing diffs against them, and a plain `generate` refuses to run until a baseline exists. 2. On a database drizzle-kit (or anything else) already migrated, `effect-orm baseline ` (`Migrate.baseline`) records the baseline and every migration before it as applied, without running them. A fresh database, such as a test's, simply runs everything. -The first `generate` after the baseline diffs your `S.pg.table` definitions against what +The first `generate` after the baseline diffs your `PG.table` definitions against what drizzle-kit recorded, so every place they disagree (a constraint name, a default) shows up as an op to accept or fix. `verify` against the baseline also finds objects the database has and drizzle-kit's snapshot does not, such as a table a hand-written migration created and nothing diff --git a/docs/params-and-compilation.md b/docs/params-and-compilation.md index 70c1450..2153877 100644 --- a/docs/params-and-compilation.md +++ b/docs/params-and-compilation.md @@ -10,7 +10,7 @@ CH.param.bool("includeDrafts") CH.param.dateTime("startTime") CH.param.dateTimeString("stringTimestamp") CH.param.dateTimeSeconds("secondPrecisionTimestamp") -CH.param.of(T.uint64, "customNumber") +CH.param.of(CH.uint64, "customNumber") ``` A param is an `Expr` placeholder usable anywhere an expression @@ -73,7 +73,7 @@ directions cannot drift, because there is only one of them. `param.dateTime` and `param.dateTimeString` preserve milliseconds in `Date` and `DateTime.Utc` values, as well as fractions already present in strings. Use `param.dateTimeSeconds` for a -whole-second DateTime bound, or `param.of(T.dateTime, name)` for the parsed UTC flavour. +whole-second DateTime bound, or `param.of(CH.dateTime, name)` for the parsed UTC flavour. `param.dateTimeSeconds` converts zoned strings to UTC before flooring: for example, `2026-01-02T00:00:00.500+02:00` becomes `2026-01-01 22:00:00`. @@ -81,11 +81,11 @@ whole-second DateTime bound, or `param.of(T.dateTime, name)` for the parsed UTC Reuse type definitions across queries when practical. `param.of(type, name)` takes it further: any column type, including one you declared with -`T.custom`, works as a param. +`CH.custom`, works as a param. ```ts -const Level = T.custom("Enum8('warn' = 1, 'error' = 2)", Schema.Literals(["warn", "error"])) -const Logs = CH.table("logs", { Level }) +const Level = CH.custom("Enum8('warn' = 1, 'error' = 2)", Schema.Literals(["warn", "error"])) +const Logs = CH.table("logs", { columns: { Level }, engine: CH.engine.mergeTree(), orderBy: ["Level"] }) const query = CH.from(Logs) .select("Level") .where(($) => [$.Level.eq(CH.param.of(Level, "level"))]) @@ -172,9 +172,9 @@ CH.compileUnsafe(query, params, options?) // CompiledQuery, throws | `options.rowSchema` | Effect `Schema` used by `decodeRows` / `decodeFirstRow` | | `options.skipFormat` | Omit a trailing `FORMAT` clause (used internally for subqueries) | | `options.deferParams` | Leave placeholders unresolved, for SQL spliced into an outer compile | -| `options.dialect` | How params reach the server; `clickhouseDialect` when omitted | +| `options.dialect` | Overrides the entry's dialect (`clickhouseDialect` from `/clickhouse`, `postgresDialect` from `/postgres`) | -`compileCH` is the internal name; the package exports it as `compile`. Unions use +Each dialect entry exports its own `compile`, defaulting to that dialect. Unions use `compileUnion(union, params)`. ## The `CompiledQuery` @@ -217,9 +217,9 @@ Use `decodeRows` to validate wire values against the row schema. A `Dialect` is the database a query is compiled for: how identifiers and literals are written, how resolved params reach the server, and which clauses exist. It is installed for the length of the compile, so every column reference, string fragment, compared value and inline param goes -through it. The default, `clickhouseDialect`, writes names bare and each param value into the SQL -as a ClickHouse literal, and leaves `parameters` empty. `postgresDialect`, from the `/postgres` -entry point, is the other built-in one; see [Postgres](./postgres.md). +through it. `clickhouseDialect`, the default of the `/clickhouse` entry's `compile`, writes names bare and each param value into the SQL +as a ClickHouse literal, and leaves `parameters` empty. `postgresDialect`, the default of the `/postgres` +entry's `compile`, is the other built-in one; see [Postgres](./postgres.md). A dialect whose `params` style is `bind` leaves a placeholder instead and returns the encoded values in `parameters`, numbered once across the whole statement, unions and subqueries @@ -296,10 +296,9 @@ a required parameter and recovers only that typed builder failure; defects are n ```ts title="compile-errors.ts" import { Effect } from "effect" -import * as CH from "@maple-dev/effect-orm" -import * as T from "@maple-dev/effect-orm/types" +import * as CH from "@maple-dev/effect-orm/clickhouse" -const Events = CH.table("events", { Name: T.string }) +const Events = CH.table("events", { columns: { Name: CH.string }, engine: CH.engine.mergeTree(), orderBy: ["Name"] }) const query = CH.from(Events) .select("Name") .where(($) => [$.Name.eq(CH.param.string("name"))]) diff --git a/docs/postgres.md b/docs/postgres.md index 401dc26..65ca8af 100644 --- a/docs/postgres.md +++ b/docs/postgres.md @@ -1,30 +1,37 @@ # Postgres The same builder writes Postgres SQL. Queries, params, tenant scoping and row decoding work -as they do for ClickHouse; the `/postgres` entry point supplies the parts that differ: column -types whose codecs read what Postgres drivers send, functions spelled the Postgres way, and a -`compile` that defaults to the Postgres dialect. +as they do for ClickHouse. `@maple-dev/effect-orm/postgres` is the one import for it: the +builder, column types whose codecs read what Postgres drivers send, functions spelled the +Postgres way, `table` with its keys, indexes and foreign keys, and a `compile` that defaults to +the Postgres dialect. Nothing from `/clickhouse` is needed. ```ts title="postgres-quickstart.ts" import { PGlite } from "@electric-sql/pglite" import { Effect } from "effect" -import * as CH from "@maple-dev/effect-orm" import * as PG from "@maple-dev/effect-orm/postgres" - -const Requests = CH.table( - "requests", - { OrgId: PG.text, Route: PG.text, DurationMs: PG.int8, At: PG.timestamptz }, - { tenantColumn: "OrgId" }, -) - -const query = CH.from(Requests) +import * as S from "@maple-dev/effect-orm/schema" + +const Requests = PG.table("requests", { + columns: { + Id: PG.column(PG.int8, { identity: "always" }), + OrgId: PG.text, + Route: PG.text, + DurationMs: PG.int8, + At: PG.timestamptz, + }, + primaryKey: ["Id"], + tenantColumn: "OrgId", +}) + +const query = PG.from(Requests) .select(($) => ({ route: $.Route, count: PG.count(), slow: PG.countIf($.DurationMs.gte(500)), p50: PG.percentileCont(0.5, $.DurationMs), })) - .where(($) => [$.OrgId.eq(CH.param.string("orgId")), $.At.gte(CH.param.dateTime("since"))]) + .where(($) => [$.OrgId.eq(PG.param.string("orgId")), $.At.gte(PG.param.dateTime("since"))]) .groupBy("route") .orderBy(["count", "desc"]) @@ -33,9 +40,10 @@ export const compiled = PG.compileUnsafe(query, { orgId: "org_1", since: new Dat // compiled.parameters: ["org_1", "2026-01-01T00:00:00.000Z"] const db = new PGlite() +// The CREATE TABLE comes from the definition itself; migrations.md shows the managed way. +for (const statement of S.renderPgSchema(S.pgEntitiesOf([Requests]))) await db.exec(statement) await db.exec(` - CREATE TABLE requests ("OrgId" text, "Route" text, "DurationMs" int8, "At" timestamptz); - INSERT INTO requests VALUES + INSERT INTO requests ("OrgId", "Route", "DurationMs", "At") VALUES ('org_1', '/checkout', 120, '2026-01-01T10:00:00Z'), ('org_1', '/checkout', 900, '2026-01-01T10:01:00Z'), ('org_1', '/search', 40, '2026-01-01T10:02:00Z'); @@ -47,7 +55,10 @@ await db.close() ``` `PGlite` stands in for any driver that takes `(sql, values)`: node-postgres, postgres.js, or -`@effect/sql-pg`'s `unsafe`. The builder never runs the query. +`@effect/sql-pg`'s `unsafe`. The builder never runs the query. `Id` is an identity column, so an +insert may leave it out: `PG.InsertRowOf` makes it optional. See +[Tables and column types](./tables-and-types.md) for column options and external tables, and +[Schema and migrations](./migrations.md#postgres) for indexes, foreign keys, and `generate`. ## What the dialect changes @@ -82,6 +93,7 @@ literal that still contained it would fail the compile with `InvalidLiteral`. | `array(type)` | `type[]` | `ReadonlyArray` | array | | `nullable(type)` | the same type | `T \| null` | the same, or `null` | | `custom(sql, schema, literalSchema?)` | anything | the schema's type | whatever the schema reads | +| `brand(type, schema)` | the base type | the schema's type | what the base type reads | `int8` and `numeric` decode to `number`, so values beyond 2^53 or a double's precision lose digits. Where exact digits matter, declare @@ -108,12 +120,12 @@ which no session time zone can reinterpret; a zoneless string is read as UTC. | `jsonText(x, key)` | `(x ->> key)` | `null` when absent | The shared operators (`eq`, `in_`, `like`, `ilike`, `and`, `or`, `not`, arithmetic, `lit`) work -unchanged. The ClickHouse function catalog on the root entry (`quantile`, `toStartOfInterval`, -`count()`, …) writes ClickHouse SQL, so compiling a query that uses one for Postgres is a +unchanged. `/postgres` exports only functions Postgres has, plus `nullIf`, which renders the +same on both. The ClickHouse catalog on `/clickhouse` (`quantile`, `toStartOfInterval`, its +`count()`, …) writes ClickHouse SQL, so a query that uses one and is compiled for Postgres is a `QueryBuilderDefect` naming the function; the Postgres functions above fail the same way on -ClickHouse. `coalesce`, `nullIf` and `lower` from the root entry render the same on both and -are allowed on either. A custom `Dialect` opts in with `functions: "clickhouse"` or -`"postgres"`; without it, nothing is checked. +ClickHouse. A custom `Dialect` opts in with `functions: "clickhouse"` or `"postgres"`; without +it, nothing is checked. ## Known differences diff --git a/docs/queries.md b/docs/queries.md index 4ef51b6..9cb4bca 100644 --- a/docs/queries.md +++ b/docs/queries.md @@ -98,7 +98,7 @@ const query = CH.from(Events) .select(($) => ({ name: $.Name, count: CH.count() })) .where(($) => [$.OrgId.eq(CH.param.string("orgId"))]) .groupBy("name") - .having(() => [CH.dynamicColumn("count", T.uint64).gte(CH.param.int("minimumCount"))]) + .having(() => [CH.dynamicColumn("count", CH.uint64).gte(CH.param.int("minimumCount"))]) ``` Compile with `{ orgId: "org_123", minimumCount: 10 }` to emit `HAVING count >= 10`. @@ -161,7 +161,7 @@ On Postgres, `forUpdate`, `forNoKeyUpdate`, `forShare` and `forKeyShare` add a l after LIMIT. Each takes `{ skipLocked?, noWait?, of? }`. The usual job-queue claim: ```ts -CH.from(Jobs) +PG.from(Jobs) .select("id") .where(($) => [$.state.eq("queued")]) .orderBy(["id", "asc"]) @@ -191,5 +191,6 @@ changing the SQL. Both are covered in [Tenant scoping](./tenant-scoping.md). const compiled = CH.compileUnsafe(query, params) ``` -`compile` is an alias of `compileCH`; both are exported. Unions compile with `compileUnion`. +`compile` writes its entry's dialect: ClickHouse from `/clickhouse`, Postgres from `/postgres`. +Unions compile with `compileUnion`. See [Params and compilation](./params-and-compilation.md). diff --git a/docs/recipes.md b/docs/recipes.md index db7b9db..3b892c9 100644 --- a/docs/recipes.md +++ b/docs/recipes.md @@ -13,7 +13,7 @@ These examples assume UTC timestamps. ```ts title="time-buckets.ts" import { Effect } from "effect" -import * as CH from "@maple-dev/effect-orm" +import * as CH from "@maple-dev/effect-orm/clickhouse" import { Events } from "./schema" const query = CH.from(Events) @@ -57,7 +57,7 @@ in your product, return an empty result before executing instead. ```ts title="optional-filters.ts" import { Effect } from "effect" -import * as CH from "@maple-dev/effect-orm" +import * as CH from "@maple-dev/effect-orm/clickhouse" import { Events } from "./schema" export const buildQuery = (names: readonly string[], minDurationMs?: number) => @@ -93,15 +93,14 @@ Use `where` to choose events, then `having` to choose groups by their aggregated ```ts title="aggregate-filter.ts" import { Effect } from "effect" -import * as CH from "@maple-dev/effect-orm" -import * as T from "@maple-dev/effect-orm/types" +import * as CH from "@maple-dev/effect-orm/clickhouse" import { Events } from "./schema" const query = CH.from(Events) .select(($) => ({ name: $.Name, count: CH.count() })) .where(($) => [$.OrgId.eq(CH.param.string("orgId"))]) .groupBy("name") - .having(() => [CH.dynamicColumn("count", T.uint64).gte(CH.param.int("minimumCount"))]) + .having(() => [CH.dynamicColumn("count", CH.uint64).gte(CH.param.int("minimumCount"))]) .orderBy(["count", "desc"], ["name", "asc"]) .limit(20) @@ -124,7 +123,7 @@ makes `name` unique in this result, so it breaks ties between equal counts. ```ts title="pagination.ts" import { Effect } from "effect" -import * as CH from "@maple-dev/effect-orm" +import * as CH from "@maple-dev/effect-orm/clickhouse" import { Events } from "./schema" const pageSize = 25 @@ -153,10 +152,13 @@ JSON parsing could lose precision. This also works for IDs produced by numeric h ```ts title="large-ids.ts" import { Effect } from "effect" -import * as CH from "@maple-dev/effect-orm" -import * as T from "@maple-dev/effect-orm/types" +import * as CH from "@maple-dev/effect-orm/clickhouse" -const Records = CH.table("records", { Id: T.uint64, Name: T.string }) +const Records = CH.table("records", { + columns: { Id: CH.uint64, Name: CH.string }, + engine: CH.engine.mergeTree(), + orderBy: ["Id"], +}) const query = CH.from(Records) .select(($) => ({ id: CH.toString($.Id), name: $.Name })) .limit(1) diff --git a/docs/reference.md b/docs/reference.md index c2f27f2..f978da7 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -1,64 +1,45 @@ # API reference -Everything on this page is exported from the root entry point -(`@maple-dev/effect-orm`) unless marked otherwise. +The package has one entry per database, each self-contained: + +```ts +import * as CH from "@maple-dev/effect-orm/clickhouse" +import * as PG from "@maple-dev/effect-orm/postgres" +``` + +Both carry the whole [query builder](#query-builder-both-entries). Each adds its own column +types, functions, table definitions and a `compile` that defaults to its dialect: +[`/clickhouse`](#clickhouse) and [`/postgres`](#postgres). Everything else is on a +[subpath](#other-subpaths) for a narrower job. ## Naming conventions Some ClickHouse functions collide with JavaScript reserved words or globals. The source defines -those with a trailing underscore, and **the root barrel drops it**: `min_`, `max_`, `any_`, +those with a trailing underscore, and **`/clickhouse` drops it**: `min_`, `max_`, `any_`, `toString_`, `length_`, `left_`, `extract_`, `least_`, `greatest_`, `position_`, `lower_`, -`round_`, `path_` and `domain_` are all exported from the root under their bare names. +`round_`, `path_` and `domain_` are all exported from `/clickhouse` under their bare names. One exception, because it cannot be anything else: -| Root barrel name | Also on `/expr` as | Note | -| ---------------- | ------------------ | -------------------------------- | -| `if_` | `if_` | `if` is a reserved word | -| `in_` / `notIn` | — | `Expr` methods; `in` is reserved | +| `/clickhouse` name | Also on `/expr` as | Note | +| ------------------ | ------------------ | -------------------------------- | +| `if_` | `if_` | `if` is a reserved word | +| `in_` / `notIn` | — | `Expr` methods; `in` is reserved | Importing the kitchen-sink namespace (`import * as CH from "@maple-dev/effect-orm/expr"`) gives you the raw underscored names uniformly, which some codebases prefer for exactly this reason. -## What's only on a subpath - -The root barrel is curated. These are exported by the package but not from it: - -| Symbol | Subpath | -| ------------------------------------------------------------------------------------------------ | ----------------- | -| `toFragment` — value → `SqlFragment`, for hand-rolled function wrappers | `/expr` | -| `raw`, `str`, `ident`, `int`, `join`, `as_`, `lazy`, `when`, `compile`, `escapeClickHouseString` | `/sql` | -| `SqlQuery`, `compileQuery` | `/sql` | -| `ClickHouseStatement`, `parseStatement`, `renderStatement`, `withSettings`, `withFormat` | `/sql` | -| `ClickHouseStatementFromString`, `splitTerminalClauses`, `maskLiteralsAndComments` | `/sql` | -| `defineSuite`, `query`, `caseFromCompiled`, `runSuite`, `compareRuns`, `compareBudgets` | `/benchmark` | -| `Suite`, `RunOutput`, `BenchmarkError` and benchmark contracts | `/benchmark` | -| `makeHttpClient`, `makeHttpTransport`, `httpConfigFromEnv` | `/benchmark/http` | -| `runCli` | `/benchmark/cli` | -| `postgresDialect`, Postgres column types and functions, Postgres-default `compile` | `/postgres` | -| `Database`, `run`, `sql`, `query`, `execute`, `transaction`, `requireTransaction`, `retryContention`, `Transaction` | `/database` | -| `DatabaseError`, `TransactionCommitFailed`, `TransactionRollbackFailed` and the other transaction errors | `/database` | - -Every column-type constructor and every expression helper is on the root as well as on its -subpath. See [Running a query](./running-queries.md) for what the `/sql` statement helpers are -for. See [Benchmarking](./benchmarking.md) for the complete benchmark API, command -workflow, result verification, and JSON protocol. - -Note `/sql` exports a `compile` (fragment → string) distinct from the root `compile` -(query → `CompiledQuery`), and a `when` distinct from the root `when` (optional conditions). - --- -## Entry points +## Query builder (both entries) + +Everything in this section is exported, identically, from both `/clickhouse` and `/postgres`. ### Query construction | Export | Signature | | ----------- | ----------------------------------------- | -| `table` | `(name, columns, options?) => Table` | -| `custom` | `(sql, schema, literalSchema?) => CHType` | -| `brand` | `(type, schema) => CHType`: the type narrowed by `schema` (a branded id, a literal union); see [Branded columns](./tables-and-types.md#branded-columns) | | `from` | `(table, alias?) => CHQuery` | | `fromQuery` | `(query, alias) => CHQuery` | | `fromUnion` | `(union, alias) => CHQuery` | @@ -67,6 +48,9 @@ Note `/sql` exports a `compile` (fragment → string) distinct from the root `co | `deleteFrom` | `(table) => CHDelete`: `.where(fn)` or `.allRows()`, `.returning(...)`, `.settings(record)` | | `insertInto` | `(table) => CHInsertStart`, then `CHInsert`; `.values(row \| rows)` or `.select(query)` sets its rows, `.settings(record)` ClickHouse `SETTINGS`, `.returning(...)` the RETURNING list, `.onConflictDoNothing(options?)` / `.onConflictDoUpdate(options)` the ON CONFLICT clause (Postgres). See [Inserting rows](./inserts.md) | +Tables come from the dialect's `table`: [`CH.table`](#tables-and-ddl) or +[`PG.table`](#tables-and-ddl-1). + ### `CHQuery` methods | Method | Notes | @@ -90,6 +74,9 @@ Note `/sql` exports a `compile` (fragment → string) distinct from the root `co ### Compilation +Each entry exports `compile`, `compileUnsafe`, `compileUnion` and `compileUnionUnsafe`, defaulting +to its own dialect; `options.dialect` overrides it. + | Export | Signature | | -------------------- | ------------------------------------------------------------------------------------ | | `compile` | `(query, params?, options?) => Effect, QueryBuilderError>`; also `(insert, params?, options?)`, whose options (`InsertCompileOptions`) are only `dialect` | @@ -97,7 +84,6 @@ Note `/sql` exports a `compile` (fragment → string) distinct from the root `co | `compileUnion` | `(union, params, options?) => Effect, QueryBuilderError>` | | `compileUnionUnsafe` | The same, throwing instead | | `rawCompiledQuery` | `({ sql, tenantScope, reason, justification, rowSchema?, route?, dialect?, kind? }) => CompiledQuery` | -| `clickhouseDialect` | The default `Dialect`: params written into the SQL as ClickHouse literals | `Dialect`, `DialectClauses` and `ParamStyle` describe a database: how identifiers and literals are written, how params reach the server, and which clauses exist. Pass one as @@ -112,9 +98,7 @@ which transactions the database supports; see [Database](./database.md). See [Pa type. Each checks the value it is handed at compile time; see [Params and compilation](./params-and-compilation.md#what-each-kind-accepts). ---- - -## Expressions +### Expressions | Export | Purpose | | ------------------------- | ---------------------------------------------------------- | @@ -143,7 +127,7 @@ as `number | null` — ClickHouse sends `inf`/`nan` as JSON `null` — except by magnitude ≥ 1 (`Quotient`), which keeps the dividend's nullability; use `ifNull(ifNotFinite(expr, 0), lit(0))` for a guaranteed number otherwise. -### Spliced sub-SELECTs +#### Spliced sub-SELECTs For SQL the builder has no syntax for — an inner query's text inside an aggregate or a tuple comparison. The inner query is compiled by the **outer** `compile`, so its params resolve from the @@ -163,7 +147,7 @@ parentheses, which is the plain "this value is a sub-SELECT" case. `ColumnRef` adds `.get(key)` for `Map` columns; the result decodes as the map's value type. -## Extensibility +### Extensibility | Export | Purpose | | -------------------------------------- | -------------------------------------------------- | @@ -187,11 +171,146 @@ parentheses, which is the plain "this value is a sub-SELECT" case. | `withoutNull(schema)` | A codec minus its `null` arm, or `undefined` | | `paramPlaceholder(kind, name)` | The `__PARAM_…__` text, for handwritten fragments | +### Types + +**Column plumbing** — `CHType` (every column type, on either dialect, is one), `InferTS` (the +decoded type of a column), `InferEncoded` (its wire type), `ColumnDefs`, `NullableColumnDefs`, +`OutputToColumnDefs`. + +**Inference** — `InferOutput`, `InferQueryOutput`, `InferUnionOutput`, `SelectRowOf`, +`InsertRow`, `InsertRowOf`, `UpdateSet`, `UpdateSetOf`. + +**Everything else** — `Table` (what `from` and the write builders accept; every `table` value is +one), `Expr`, `ColumnRef`, `Condition`, `Comparable` (what a value of a type may be compared +against), `MapValueOf`, `Subquery`, `ParamMarker`, `ParamKind`, `CHQuery`, `CHUnionQuery`, +`CHInsert`, `CHInsertStart`, `CHUpdate`, `CHUpdateStart`, `CHDelete`, `CHWrite`, `InsertValue`, +`InsertSelectMisfits`, `InsertSelectMissing`, `InsertSettingValue`, `ConflictTarget`, +`ConflictSet`, `OnConflictDoNothing`, `OnConflictDoUpdate`, `ColumnAccessor`, +`JoinedColumnAccessor`, `JoinOnCallback`, `LockOptions`, `CompiledQuery`, `CompiledQueryInput`, +`CompiledQueryRowSchema`, `RowSchemaMismatch`, `TenantScope`, `Dialect`, `DialectClauses`, +`DialectTransactions`, `IsolationLevel`, `TransactionSettings`, `ParamStyle`, `FnResult`. + +### Errors + +These errors are Effect `Schema.TaggedError` classes. Expected failures can be caught by +their full namespaced tag; `QueryBuilderDefect` remains a defect rather than a typed failure. + +#### `QueryBuilderError` + +Tag `"@maple-dev/effect-orm/QueryBuilderError"`. Raised while compiling, and surfaced in +`compile`'s error channel (thrown by `compileUnsafe`). + +| `code` | Cause | +| ------------------ | ------------------------------------------------------------------------ | +| `UnresolvedParam` | A param the params bag has no value for | +| `InvalidLiteral` | A param value, or a comparison operand, the column's codec rejects | +| `InvalidArguments` | Arguments a function cannot use — an empty condition list, a bad pattern, an insert with no rows or an unknown column, more bound values than the dialect allows | + +#### `QueryBuilderDefect` + +Tag `"@maple-dev/effect-orm/QueryBuilderDefect"`. A DSL misuse no runtime value can cause +— a query with no `select()`, an `orderBy` entry that is not a tuple, a bad param name, a +comparison called on a param marker. Always +a defect: `compile` maps only `QueryBuilderError` into the error channel. See +[Failures and defects](./params-and-compilation.md#failures-and-defects). + +#### `CompiledQueryEncodeError` + +Tag `"@maple-dev/effect-orm/CompiledQueryEncodeError"`. Fails the `encodeRows` Effect +when a decoded row cannot be written back to its wire shape. Fields: `message`, `rowIndex`, +`cause`. + +#### `CompiledQueryDecodeError` + +Tag `"@maple-dev/effect-orm/CompiledQueryDecodeError"`. Fails the `decodeRows` / +`decodeFirstRow` Effect. Fields: `message`, `rowIndex`, `cause`. + +#### `SchemaDefinitionDefect` + +Tag `"@maple-dev/effect-orm/SchemaDefinitionDefect"`. Thrown by `table` (either dialect) and +`CH.materializedView` for a definition that cannot render: a name that is not a plain +identifier, a MergeTree-family engine with no `orderBy`, a view reading from a union. + --- -## ClickHouse functions +## `/clickhouse` + +`import * as CH from "@maple-dev/effect-orm/clickhouse"`: the query builder above, plus the +following. -### Aggregate +### Tables and DDL + +`table` is the only way to declare a table. A managed table carries the DDL +`effect-orm generate` diffs; see [Schema and migrations](./migrations.md). + +```ts +const Events = CH.table("events", { + columns: { + OrgId: CH.string, + Name: CH.string, + Timestamp: CH.dateTime, + DurationMs: CH.uint64, + Status: CH.column(CH.uint16, { default: 200 }), + }, + engine: CH.engine.mergeTree(), + orderBy: ["OrgId", "Timestamp"], + tenantColumn: "OrgId", +}) +``` + +| Export | Signature | +| ------------------ | ---------------------------------------------------------------------------------------- | +| `table` | `(name, TableDefinition) => SchemaTable`, or `(name, ExternalTableDefinition) => Table` | +| `column` | `(type, ColumnOptions) => ColumnSpec`: a column with options; a bare type works where none are needed | +| `engine` | `mergeTree()`, `replacingMergeTree({ version?, isDeleted? })`, `summingMergeTree({ columns? })`, `aggregatingMergeTree()`, `collapsingMergeTree(sign)`, `versionedCollapsingMergeTree(sign, version)`, `null()`, `memory()` | +| `index` | `(name, expr, type, granularity = 1) => IndexSpec`: a data-skipping index | +| `ttlAfterDays` | `(expr, days) => DdlExpr`: ` + toIntervalDay(days)`, the usual row TTL | +| `materializedView` | `(name, { to, as }) => MaterializedView`: `as` is a DSL query; its output is checked against `to`'s columns (`MisfitColumns`). Never `POPULATE` | + +**`TableDefinition`** — `columns` (required), `engine` (required), `orderBy` (required for the +MergeTree family; `[]` for `ORDER BY tuple()`), `partitionBy`, `primaryKey`, `ttl`, `settings`, +`indexes`, `comment`, `tenantColumn`. Key and expression options are `DdlKey` / `DdlExpr`: SQL +text, or a callback building it from the column accessor. + +**`ExternalTableDefinition`** — `{ external: true, columns, tenantColumn? }`. A table the schema +does not own: a system table, a table function, a subquery, a CTE name, a table another tool +migrates. No DDL, so `generate` never sees it; the name is written verbatim as the FROM target; +`columns` may be empty. + +```ts +const One = CH.table("system.one", { external: true, columns: {} }) +const Numbers = CH.table("numbers(10)", { external: true, columns: { number: CH.uint64 } }) +``` + +**`ColumnOptions`** — at most one of `default` (a literal, encoded through the column's type), +`defaultExpr`, `materialized`, `alias`; plus `codec` and `comment`. They type inserts: a column +with `default` or `defaultExpr` may be left out (`DefaultedColumnsOf`), a `materialized` or +`alias` column may not be written (`ComputedColumnsOf`). + +**Types** — `ColumnInput` (a type or a `ColumnSpec`), `ColumnsOf` (the query-side column types +of a `columns` record), `TableDdl`, `SchemaTable`, `MaterializedView`, `IndexSpec`. A definition +that cannot render throws `SchemaDefinitionDefect` while the module loads. + +### Column types + +**Constructors** — `string`, `bool`, `uint8`, `uint16`, `uint32`, `uint64`, `int32`, +`int64`, `float64`, `dateTime`, `dateTime64`, `dateTimeString`, `dateTime64String`, `map`, +`array`, `nullable`, `aggregateState(fn, ...args)`, `custom(sql, schema, literalSchema?)`, +`brand(type, schema)` (the type narrowed by `schema`: a branded id, a literal union; see +[Branded columns](./tables-and-types.md#branded-columns)), and `untyped(sql)` for a wire value +passed through unvalidated. See [Tables and column types](./tables-and-types.md). + +**Type descriptors** — `CHString`, `CHBool`, `CHUInt8`, `CHUInt16`, `CHUInt32`, +`CHUInt64`, `CHInt32`, `CHInt64`, `CHFloat64`, `CHDateTime`, `CHDateTime64`, `CHDateTimeString`, +`CHDateTime64String`, `CHMap`, `CHArray`, `CHNullable`, and `CHStringLike` (any `String` column, +branded or not, for a helper that accepts either). + +`CHNumber` is the codec the 64-bit integer types decode with: a JSON number, or the same value +quoted. + +### Functions + +#### Aggregate `count()`, `countIf(cond)`, `avg(e)`, `sum(e)`, `min(e)`, `max(e)`, `any(e)`, `uniq(e)`, `sumIf(e, cond)`, `avgIf(e, cond)`, `minIf(e, cond)`, `maxIf(e, cond)`, `anyIf(e, cond)`, @@ -202,7 +321,7 @@ _(both curried; `WindowFunnelMode` is the mode union)_. `min`/`max` return `Expr>`; `groupUniqArray` returns `Expr>`. -### String +#### String `toString(e)`, `length(e)`, `lower(e)`, `hex(e)`, `match(e, pattern)`, `matchCond(e, pattern)` → `Condition`, `domain(url)`, `path(url)`, `cutQueryString(url)`, `position(haystack, needle)`, @@ -212,41 +331,41 @@ _(both curried; `WindowFunnelMode` is the mode union)_. `hasToken` and `hasAllTokens` return `Condition`. -### Numeric +#### Numeric `toFloat64(e)`, `toFloat64OrZero(e)`, `toUInt16OrZero(e)`, `toUInt64(e)`, `toInt64(e)`, `intDiv(a, b)`, `round(e, decimals?)`, `least(...exprs)`, `greatest(...exprs)`, `cityHash64(...exprs)`. -### Date/time +#### Date/time `toStartOfInterval(col, seconds)`, `toStartOfHour(col)`, `toUnixTimestamp(col)`, `toUnixTimestamp64Nano(col)`, `intervalSub(col, seconds)`, `intervalAdd(col, seconds)`, `formatDateTime(col, format)`, `toDateTime(col)`, `toStartOfMinute(col)`, `toHour(col)`. -### Conditional +#### Conditional `if_(cond, then, else)`, `multiIf([[cond, value], …], fallback)`, `coalesce(...exprs)`, -`nullIf(expr, value)`, `ifNotFinite(expr, fallback)` (`expr` unless it is `nan`/`inf` — the SQL-side -guard for division; preserves SQL NULL). `nullIf` returns `Expr`. -`avg`, `avgIf`, and `quantile` return `Expr` for empty input. +`ifNull(expr, fallback)`, `nullIf(expr, value)`, `ifNotFinite(expr, fallback)` (`expr` unless it +is `nan`/`inf` — the SQL-side guard for division; preserves SQL NULL). `nullIf` returns +`Expr`. `avg`, `avgIf`, and `quantile` return `Expr` for empty input. -### Array +#### Array `arrayOf(...exprs)`, `arrayStringConcat(arr, sep)`, `arrayFilter(fn, arr)`, `arrayJoin(arr)`, `arraySort(arr)`, `arrayReverseSort(arr)`, `arrayDistinct(arr)`, `arrayPushFront(arr, value)`, `arrayElement(arr, index)`, `has(arr, value)` → `Condition`. -### Map +#### Map `mapContains(map, key)` → `Condition`, `mapGet(map, key)`, `mapKeys(map)`, `mapValues(map)`, `mapLiteral(...[key, expr])`. Prefer `$.Column.get(key)` for a declared `Map` column. -### JSON +#### JSON `toJSONString(e)`. -### Window +#### Window `over(expr, spec)`, `windowSpec({ partitionBy?, orderBy?, frame? })`, `rowsBetween(start, end)`, `lagInFrame(expr, offset, defaultValue)` _(all three arguments @@ -267,62 +386,110 @@ CH.over( Types: `WindowSpec`, `CompiledWindowSpec`, `WindowFrameBound`, `WindowRowsFrame`, `WindowOrderDirection`. +### Dialect + +`compile` and the other compile functions default to `clickhouseDialect`: params written into the +SQL as ClickHouse literals. + --- -## Types +## `/postgres` -**Column-type constructors** — `string`, `bool`, `uint8`, `uint16`, `uint32`, `uint64`, `int32`, -`int64`, `float64`, `dateTime`, `dateTime64`, `dateTimeString`, `dateTime64String`, `map`, -`array`, `nullable`, `aggregateState(fn, ...args)`, `custom(sql, schema, literalSchema?)`, and -`untyped(sql)` for a wire value passed through unvalidated. See -[Tables and column types](./tables-and-types.md). +`import * as PG from "@maple-dev/effect-orm/postgres"`: the query builder above, plus the +following. See [Postgres](./postgres.md). -**Type descriptors** — `CHType`, `CHString`, `CHBool`, `CHUInt8`, `CHUInt16`, `CHUInt32`, -`CHUInt64`, `CHInt32`, `CHInt64`, `CHFloat64`, `CHDateTime`, `CHDateTime64`, `CHDateTimeString`, -`CHDateTime64String`, `CHMap`, `CHArray`, `CHNullable`. +### Tables and DDL -**Inference** — `InferTS` (the decoded type of a column), `InferEncoded` (its wire type), -`InferOutput`, `InferQueryOutput`, `InferUnionOutput`, `OutputToColumnDefs`, -`NullableColumnDefs`, `ColumnDefs`. +```ts +const Users = PG.table("users", { + columns: { + id: PG.column(PG.int8, { identity: "always" }), + orgId: PG.text, + email: PG.text, + createdAt: PG.column(PG.timestamptz, { defaultExpr: "now()" }), + }, + primaryKey: ["id"], + indexes: [PG.uniqueIndex("users_org_email_idx", ["orgId", "email"])], + tenantColumn: "orgId", +}) +``` -**Everything else** — `Table`, `TableOptions`, `Expr`, `ColumnRef`, `Condition`, `Comparable` -(what a value of a type may be compared against), `MapValueOf`, `Subquery`, `ParamMarker`, -`ParamKind`, `CHQuery`, `CHUnionQuery`, `CHInsert`, `CHInsertStart`, `CHUpdate`, `CHUpdateStart`, `CHDelete`, `CHWrite`, `UpdateSet`, `UpdateSetOf`, `InsertRow`, `InsertRowOf`, `SelectRowOf`, `InsertValue`, `InsertSelectMisfits`, `InsertSelectMissing`, `InsertSettingValue`, `ConflictTarget`, `ConflictSet`, `OnConflictDoNothing`, `OnConflictDoUpdate`, `ColumnAccessor`, `JoinedColumnAccessor`, -`JoinOnCallback`, `CompiledQuery`, `CompiledQueryInput`, `CompiledQueryRowSchema`, `RowSchemaMismatch`, `TenantScope`, `Dialect`, `DialectClauses`, `DialectTransactions`, `IsolationLevel`, `TransactionSettings`, `ParamStyle`, `FnResult`, -`WindowFunnelMode`, `WindowSpec`, `WindowRowsFrame`, `WindowFrameBound`, -`WindowOrderDirection`, `CompiledWindowSpec`. +| Export | Signature | +| ----------------------- | ---------------------------------------------------------------------------------- | +| `table` | `(name, TableDefinition) => PgSchemaTable`, or `(name, ExternalTableDefinition) => Table` | +| `column` | `(type, ColumnOptions) => ColumnSpec` | +| `index` | `(name, on, IndexOptions?) => IndexSpec`: columns, or a callback building expressions | +| `uniqueIndex` | The same, `UNIQUE`; with `where`, a partial unique index | +| `foreignKey` | `({ columns, references, foreignColumns, onDelete?, onUpdate?, name? }) => ForeignKeySpec`; `references` is a `Table` (its columns are checked) or a name | +| `defaultForeignKeyName` | `(table, columns, foreignTable, foreignColumns) => string`: the name drizzle-kit gives, shortened past 63 characters as drizzle-kit does | -## Errors +**`TableDefinition`** — `columns` (required), `primaryKey` (column names, or +`{ columns, name? }`), `indexes`, `foreignKeys`, `tenantColumn`. -These errors are Effect `Schema.TaggedError` classes. Expected failures can be caught by -their full namespaced tag; `QueryBuilderDefect` remains a defect rather than a typed failure. +**`ExternalTableDefinition`** — `{ external: true, columns, tenantColumn? }`, as on +`/clickhouse`: a view, a catalog table, one another tool migrates. No DDL. -### `QueryBuilderError` +**`ColumnOptions`** — one of `default`, `defaultExpr`, `identity` (`"always"` or +`"by default"`); any of them lets an insert leave the column out (`DefaultedColumnsOf`). -Tag `"@maple-dev/effect-orm/QueryBuilderError"`. Raised while compiling, and surfaced in -`compile`'s error channel (thrown by `compileUnsafe`). +**`IndexOptions`** — `where` (a `DdlPredicate`: SQL text or a callback returning a condition), +`using` (the access method; default `btree`). -| `code` | Cause | -| ------------------ | ------------------------------------------------------------------------ | -| `UnresolvedParam` | A param the params bag has no value for | -| `InvalidLiteral` | A param value, or a comparison operand, the column's codec rejects | -| `InvalidArguments` | Arguments a function cannot use — an empty condition list, a bad pattern, an insert with no rows or an unknown column, more bound values than the dialect allows | +**Types** — `ColumnInput`, `ColumnSpec`, `ColumnsOf`, `IndexSpec`, `ForeignKeySpec`, +`ReferentialAction` (drizzle's lowercase spelling or the catalog's), `TableDdl`, `PgSchemaTable`, +`DdlExpr`, `DdlKey`. A definition that cannot render throws `SchemaDefinitionDefect`. -### `QueryBuilderDefect` +### Column types -Tag `"@maple-dev/effect-orm/QueryBuilderDefect"`. A DSL misuse no runtime value can cause -— a query with no `select()`, an `orderBy` entry that is not a tuple, a bad param name, a -comparison called on a param marker. Always -a defect: `compile` maps only `QueryBuilderError` into the error channel. See -[Failures and defects](./params-and-compilation.md#failures-and-defects). +`text`, `uuid`, `bool`, `int2`, `int4`, `int8`, `float4`, `float8`, `numeric`, `timestamptz`, +`jsonb(schema?)`, `array(type)`, `nullable(type)`, `custom(sql, schema, literalSchema?)`, +`brand(type, schema)`. See [Postgres column types](./postgres.md#column-types). -### `CompiledQueryEncodeError` +Types: `PgType` (a Postgres column type; a `CHType`), `PgArray`, `PgNullable`. Codecs: +`PgNumber` (a number, numeric string or `bigint`), `PgTimestampLiteral` (an instant written as +ISO-8601), `timestampLiteral(format)` (the same with another format), and `pgTimestampToIso`, +which normalizes Postgres timestamp text to ISO-8601. -Tag `"@maple-dev/effect-orm/CompiledQueryEncodeError"`. Fails the `encodeRows` Effect -when a decoded row cannot be written back to its wire shape. Fields: `message`, `rowIndex`, -`cause`. +### Functions -### `CompiledQueryDecodeError` +`count()`, `countDistinct(x)`, `countIf(c)`, `sum(x)`, `sumIf(x, c)`, `avg(x)`, `min(x)`, +`max(x)`, `percentileCont(f, x)`, `arrayAgg(x)`, `dateTrunc(unit, ts)` (`DateTruncUnit` is the +unit union), `dateBin(seconds, ts)`, `now()`, `lower(x)`, `upper(x)`, `length(x)`, +`coalesce(x, fallback)`, `nullIf(x, value)`, `jsonText(x, key)`. See +[Postgres functions](./postgres.md#functions) for the SQL each writes. -Tag `"@maple-dev/effect-orm/CompiledQueryDecodeError"`. Fails the `decodeRows` / -`decodeFirstRow` Effect. Fields: `message`, `rowIndex`, `cause`. +### Dialect + +`compile` and the other compile functions default to `postgresDialect`: numbered `$n` +placeholders, double-quoted identifiers. + +--- + +## Other subpaths + +| Symbol | Subpath | +| ------------------------------------------------------------------------------------------------ | ----------------- | +| The ClickHouse functions under their raw underscored names, the expression helpers and function factories, and `toFragment` — value → `SqlFragment`, for hand-rolled function wrappers | `/expr` | +| `raw`, `str`, `ident`, `int`, `join`, `as_`, `lazy`, `when`, `compile`, `escapeClickHouseString` | `/sql` | +| `SqlQuery`, `compileQuery` | `/sql` | +| `ClickHouseStatement`, `parseStatement`, `renderStatement`, `withSettings`, `withFormat` | `/sql` | +| `ClickHouseStatementFromString`, `splitTerminalClauses`, `maskLiteralsAndComments` | `/sql` | +| Schema tooling: `isSchemaObject`, `makeSnapshot`, `renderSchema`, `diffSchemas`, `diffPgSchemas`, `fromDrizzleSnapshot`, snapshot and entity schemas | `/schema` | +| `run`, `status`, `verify`, `baseline`, `MigrationDriver` and the migrate errors | `/migrate` | +| `Database`, `run`, `sql`, `query`, `execute`, `transaction`, `requireTransaction`, `retryContention`, `Transaction` | `/database` | +| `DatabaseError`, `TransactionCommitFailed`, `TransactionRollbackFailed` and the other transaction errors | `/database` | +| `defineConfig`, `generate`, `check`, `loadSchema`, `readMigrations`, `analyze`, `KitError` | `/kit` | +| `defineSuite`, `query`, `caseFromCompiled`, `runSuite`, `compareRuns`, `compareBudgets` | `/benchmark` | +| `Suite`, `RunOutput`, `BenchmarkError` and benchmark contracts | `/benchmark` | +| `makeHttpClient`, `makeHttpTransport`, `httpConfigFromEnv` | `/benchmark/http` | +| `runCli` | `/benchmark/cli` | + +`/schema` reads tables; it does not declare them. Declare tables with `table` from `/clickhouse` +or `/postgres`. See [Schema and migrations](./migrations.md), [Database](./database.md), +[Running a query](./running-queries.md) for what the `/sql` statement helpers are for, and +[Benchmarking](./benchmarking.md) for the complete benchmark API, command workflow, result +verification, and JSON protocol. + +Note `/sql` exports a `compile` (fragment → string) distinct from the dialect entries' `compile` +(query → `CompiledQuery`), and a `when` distinct from the query builder's `when` (optional +conditions). diff --git a/docs/running-queries.md b/docs/running-queries.md index ccb630e..0b020de 100644 --- a/docs/running-queries.md +++ b/docs/running-queries.md @@ -14,15 +14,14 @@ release as your `effect` dependency; this example is checked against `4.0.0`: npm install effect@4.0.0 @effect/sql-clickhouse@4.0.0 ``` -This example reads five rows from ClickHouse's built-in `system.numbers` table. It creates no -schema and writes no data. Set `CLICKHOUSE_URL`, `CLICKHOUSE_USERNAME`, and +This example reads five rows from ClickHouse's built-in `system.numbers` table, declared +`external: true` because this schema does not own it. It creates no schema and writes no data. Set `CLICKHOUSE_URL`, `CLICKHOUSE_USERNAME`, and `CLICKHOUSE_PASSWORD` for your server; the defaults target a local server. ```ts title="run-query.ts" import { ClickhouseClient } from "@effect/sql-clickhouse" import { Config, Effect, Redacted } from "effect" -import * as CH from "@maple-dev/effect-orm" -import * as T from "@maple-dev/effect-orm/types" +import * as CH from "@maple-dev/effect-orm/clickhouse" const ClickHouseLive = ClickhouseClient.layerConfig({ url: Config.String("CLICKHOUSE_URL").pipe(Config.withDefault("http://localhost:8123")), @@ -35,7 +34,7 @@ const ClickHouseLive = ClickhouseClient.layerConfig({ const program = Effect.gen(function* () { const client = yield* ClickhouseClient.ClickhouseClient - const Numbers = CH.table("system.numbers", { number: T.uint64 }) + const Numbers = CH.table("system.numbers", { external: true, columns: { number: CH.uint64 } }) const query = CH.from(Numbers).select("number").limit(5) const compiled = yield* CH.compile(query, {}) const wire = yield* client.unsafe>(compiled.sql).pipe( @@ -84,7 +83,7 @@ You do not need `output_format_json_quote_64bit_integers: 0` for decoding: the n accept both quoted and unquoted numbers. Both decode into JavaScript `number`, so **neither choice preserves arbitrary 64-bit integers**. For IDs, hashes, or exact large counters, select `CH.toString($.Id)` and keep the result as a string. Do not relabel a numeric database column as -`T.string` without converting its SELECT expression. See the [lossless ID recipe](./recipes.md#preserve-large-integer-ids). +`CH.string` without converting its SELECT expression. See the [lossless ID recipe](./recipes.md#preserve-large-integer-ids). ## Error boundaries diff --git a/docs/tables-and-types.md b/docs/tables-and-types.md index 9107d03..e9c9efc 100644 --- a/docs/tables-and-types.md +++ b/docs/tables-and-types.md @@ -1,60 +1,191 @@ # Tables and column types -## `table(name, columns)` +A table is declared once, with the dialect entry's `table`. The same value serves the query +builder (its name and columns), inserts (which columns may be left out), and +[migrations](./migrations.md) (its DDL). There is no second, lighter way to declare one: a table +grows options as it needs them. + +## `table(name, definition)` + +The smallest ClickHouse table is its columns, an engine, and the sorting key the MergeTree family +requires: ```ts -import * as CH from "@maple-dev/effect-orm" -import * as T from "@maple-dev/effect-orm/types" +import * as CH from "@maple-dev/effect-orm/clickhouse" const Events = CH.table("events", { - OrgId: T.string, - Name: T.string, - Timestamp: T.dateTime, - DurationMs: T.uint64, - Attributes: T.map(T.string, T.string), + columns: { + OrgId: CH.string, + Name: CH.string, + Timestamp: CH.dateTime, + DurationMs: CH.uint64, + Attributes: CH.map(CH.string, CH.string), + }, + engine: CH.engine.mergeTree(), + orderBy: ["OrgId", "Timestamp"], +}) +``` + +The `columns` record is what every accessor, output type, and join is inferred from. `name` is +emitted as the FROM target. `orderBy: []` writes `ORDER BY tuple()`. + +The value is a plain `Table` with its DDL beside it on `ddl`. It is never checked against a live +server, so a column that does not exist in ClickHouse will typecheck happily and fail at query +time. [`effect-orm generate`](./migrations.md#generating-migrations) keeps the two in step when +this definition is your schema; otherwise treat it as a contract you keep in sync with your +migrations by hand. + +A definition that cannot become DDL (a MergeTree table without `orderBy`, a name that is not a +plain identifier, two of `default`/`materialized` on one column) throws `SchemaDefinitionDefect` +when the module loads. + +The Postgres table has the same shape, with a primary key in place of an engine: + +```ts +import * as PG from "@maple-dev/effect-orm/postgres" + +const Users = PG.table("users", { + columns: { + id: PG.column(PG.int8, { identity: "always" }), + email: PG.text, + name: PG.nullable(PG.text), + created_at: PG.column(PG.timestamptz, { defaultExpr: "now()" }), + }, + primaryKey: ["id"], +}) +``` + +A Postgres column is `NOT NULL` unless its type is `PG.nullable(...)`. See +[Postgres](./postgres.md) for its types and [migrations](./migrations.md#postgres) for its +indexes and foreign keys. + +## Keys, defaults, and tenancy + +`column(type, options)` wraps a column type with what a bare type cannot say. The options also +decide the insert row type, so it is derived from the DDL rather than declared twice: + +| Option (ClickHouse) | DDL | On insert | +| ------------------------- | ---------------------- | ---------------------------- | +| `default: 200` | `DEFAULT 200` | optional | +| `defaultExpr: "now()"` | `DEFAULT now()` | optional | +| `materialized: "…"` | `MATERIALIZED …` | not writable | +| `alias: "…"` | `ALIAS …` | not writable | +| `codec`, `comment` | `CODEC(…)`, `COMMENT` | unchanged | + +Postgres columns take `default`, `defaultExpr`, and `identity` (`"always"` or `"by default"`); +each makes the column optional on insert. Postgres generated columns are not modeled yet. + +```ts +const Requests = CH.table("requests", { + columns: { + OrgId: CH.string, + Timestamp: CH.column(CH.dateTime, { defaultExpr: "now()" }), + Route: CH.string, + Status: CH.column(CH.uint16, { default: 200 }), + Hour: CH.column(CH.dateTime, { materialized: "toStartOfHour(Timestamp)" }), + }, + engine: CH.engine.mergeTree(), + orderBy: ["OrgId", "Route", "Timestamp"], + partitionBy: "toDate(Timestamp)", + ttl: CH.ttlAfterDays("toDate(Timestamp)", 30), + tenantColumn: "OrgId", +}) + +type NewRequest = CH.InsertRowOf +// { OrgId: string; Route: string; Timestamp?: …; Status?: number } — Hour cannot be written +``` + +`tenantColumn` names the column that carries tenancy; see [Tenant scoping](./tenant-scoping.md). +The other table options are `primaryKey`, `settings`, and `comment`. Keys, partitions, and TTLs +are SQL strings or callbacks over the columns (`($) => [$.OrgId, CH.toStartOfHour($.Timestamp)]`). +[Inserting rows](./inserts.md#which-columns-have-defaults) covers how the insert type is used. + +## Indexes and materialized views + +A data-skipping index is `CH.index(name, expr, type, granularity?)`. A materialized view is +`CH.materializedView(name, { to, as })`, whose body is a query: an output column the target table +lacks, or of another type, is a type error. + +```ts +const RequestsIndexed = CH.table("requests", { + columns: { OrgId: CH.string, Timestamp: CH.dateTime, Route: CH.string, Status: CH.uint16 }, + engine: CH.engine.mergeTree(), + orderBy: ["OrgId", "Timestamp"], + indexes: [CH.index("idx_status", ($) => $.Status, "set(100)")], +}) + +const RoutesHourly = CH.table("routes_hourly", { + columns: { OrgId: CH.string, Hour: CH.dateTime, Route: CH.string, Requests: CH.uint64 }, + engine: CH.engine.summingMergeTree(), + orderBy: ["OrgId", "Hour", "Route"], +}) + +const RoutesHourlyMv = CH.materializedView("routes_hourly_mv", { + to: RoutesHourly, + as: CH.from(RequestsIndexed) + .select(($) => ({ OrgId: $.OrgId, Hour: CH.toStartOfHour($.Timestamp), Route: $.Route, Requests: CH.count() })) + .groupBy("OrgId", "Hour", "Route"), }) ``` -`name` is emitted verbatim as the FROM target, so it can also name a CTE or a view. The -`columns` record is what every accessor, output type, and join is inferred from. +Postgres indexes are `PG.index` and `PG.uniqueIndex`, over column names or expressions, with +`where` for a partial index; foreign keys are `PG.foreignKey`. Both are covered in +[Schema and migrations](./migrations.md#postgres). + +## External tables + +Not every FROM target is a table this schema owns. A system table, a table function, a view, a +CTE, or a table another tool migrates is declared with `external: true`: -A table is a plain value — `{ _tag: "Table", name, columns }`. It is never checked against a -live server, so a column that does not exist in ClickHouse will typecheck happily and fail at -query time. Treat the declaration as a contract you keep in sync with your migrations. +```ts +const One = CH.table("system.one", { external: true, columns: {} }) +const Numbers = CH.table("numbers(10)", { external: true, columns: { number: CH.uint64 } }) +const Stats = PG.table("pg_stat_user_tables", { external: true, columns: { relname: PG.text } }) + +CH.from(Numbers).select("number") +// SELECT __ch_source.number AS number FROM numbers(10) AS __ch_source +``` -The third argument takes options. `tenantColumn` names the column that carries tenancy (see -[Tenant scoping](./tenant-scoping.md)); `defaults` lists the columns the database fills in when -an insert leaves them out, and `computed` the ones an insert may not write at all (see -[Inserting rows](./inserts.md#which-columns-have-defaults)). +An external table carries no DDL, so `generate` never creates, alters, or drops it. Its name is +written verbatim as the FROM target (one that is not a plain identifier, like `numbers(10)`, gets +an alias to qualify its columns), it may have no columns (for a FROM that only anchors +constants), and it takes only `columns` and `tenantColumn`. Column options still drive insert +typing: `CH.column(CH.uint16, { default: 200 })` is optional on insert here too. ## Column types -A column type is an Effect `Schema` plus the ClickHouse type name it stands for. That schema is +A column type is an Effect `Schema` plus the database type name it stands for. That schema is the single source of truth: the TypeScript column type is read off it, and `compile` folds the selected columns' schemas into the row schema `decodeRows` validates against — see -[Decoding results](./decoding-results.md). +[Decoding results](./decoding-results.md). Each dialect entry has its own; the ClickHouse ones +are below, the Postgres ones in [Postgres](./postgres.md#column-types). The constructors are values, not calls (except the parameterised ones): -| Constructor | ClickHouse type | Decodes to | From the wire | -| -------------------- | ----------------- | ------------------- | ------------------------- | -| `T.string` | `String` | `string` | `string` | -| `T.uint8` | `UInt8` | `number` | number or quoted number | -| `T.uint16` | `UInt16` | `number` | number or quoted number | -| `T.uint32` | `UInt32` | `number` | number or quoted number | -| `T.uint64` | `UInt64` | `number` | number or quoted number | -| `T.int64` | `Int64` | `number` | number or quoted number | -| `T.int32` | `Int32` | `number` | number or quoted number | -| `T.float64` | `Float64` | `number` | number or quoted number | -| `T.bool` | `Bool` | `boolean` | `true`/`false` or `1`/`0` | -| `T.dateTime` | `DateTime` | `DateTime.Utc` | `YYYY-MM-DD hh:mm:ss` | -| `T.dateTime64` | `DateTime64` | `DateTime.Utc` | with a fractional part | -| `T.dateTimeString` | `DateTime` | `string` | unparsed, as sent | -| `T.dateTime64String` | `DateTime64` | `string` | unparsed, as sent | -| `T.map(k, v)` | `Map(K, V)` | `Record` | object | -| `T.array(e)` | `Array(E)` | `ReadonlyArray` | array | -| `T.nullable(t)` | `Nullable(T)` | `T \| null` | value or `null` | -| `T.untyped(sql)` | whatever you name | `unknown` | unvalidated | +| Constructor | ClickHouse type | Decodes to | From the wire | +| --------------------- | ----------------- | ------------------- | ------------------------- | +| `CH.string` | `String` | `string` | `string` | +| `CH.uint8` | `UInt8` | `number` | number or quoted number | +| `CH.uint16` | `UInt16` | `number` | number or quoted number | +| `CH.uint32` | `UInt32` | `number` | number or quoted number | +| `CH.uint64` | `UInt64` | `number` | number or quoted number | +| `CH.int64` | `Int64` | `number` | number or quoted number | +| `CH.int32` | `Int32` | `number` | number or quoted number | +| `CH.float64` | `Float64` | `number` | number or quoted number | +| `CH.bool` | `Bool` | `boolean` | `true`/`false` or `1`/`0` | +| `CH.dateTime` | `DateTime` | `DateTime.Utc` | `YYYY-MM-DD hh:mm:ss` | +| `CH.dateTime64` | `DateTime64` | `DateTime.Utc` | with a fractional part | +| `CH.dateTimeString` | `DateTime` | `string` | unparsed, as sent | +| `CH.dateTime64String` | `DateTime64` | `string` | unparsed, as sent | +| `CH.map(k, v)` | `Map(K, V)` | `Record` | object | +| `CH.array(e)` | `Array(E)` | `ReadonlyArray` | array | +| `CH.nullable(t)` | `Nullable(T)` | `T \| null` | value or `null` | +| `CH.untyped(sql)` | whatever you name | `unknown` | unvalidated | + +Column types, functions, and the query builder share the one `CH` namespace, so a schema module +and a query module import the same thing. + +_(Backed by `docs/tables-and-types.md > Column types come from the dialect entry`.)_ Two of those deserve a note. @@ -62,18 +193,18 @@ Two of those deserve a note. `output_format_json_quote_64bit_integers=0` gets them bare, and a gateway that refuses `output_format_json_quote_64bit_integers=0` quotes them regardless. Every integer type accepts both and decodes to a `number` — which also means a `UInt64` above `2^53` cannot -survive: select `CH.toString($.Id)` while leaving the actual table column declared `T.uint64`. +survive: select `CH.toString($.Id)` while leaving the actual table column declared `CH.uint64`. The resulting expression has a string codec; see the [ID recipe](./recipes.md#preserve-large-integer-ids). **DateTimes.** The parsed codecs interpret zone-less strings such as `2026-05-24 14:30:00` as UTC. ClickHouse does **not** guarantee that all timestamp strings are UTC: text output follows the column/server timezone. Use UTC columns or normalize the selected expression to UTC before -using `T.dateTime` / `T.dateTime64`. For an unchanged wire string, use `T.dateTimeString` / -`T.dateTime64String`. See [ClickHouse DateTime timezones](https://clickhouse.com/docs/reference/data-types/datetime). +using `CH.dateTime` / `CH.dateTime64`. For an unchanged wire string, use `CH.dateTimeString` / +`CH.dateTime64String`. See [ClickHouse DateTime timezones](https://clickhouse.com/docs/reference/data-types/datetime). **Numeric validation.** Built-in numeric codecs accept finite numbers and quoted finite numbers. They do not enforce each ClickHouse integer's sign, bit width, safe-integer range, or integrality. -Use schema checks through `T.custom` when your application needs those constraints; `param.int` +Use schema checks through `CH.custom` when your application needs those constraints; `param.int` separately requires a safe integer. A successful decode does not prove an unsafe large number retained precision. @@ -87,24 +218,20 @@ unions, result encoding prefers `DateTime64` and retains milliseconds. This also through nullable and array wrappers. Custom codecs retain their declared encoding behavior; provide an explicit result schema when different custom transforms need a particular encoding. -> **Import the namespace.** Every constructor is on the root barrel too, but -> `import * as T from "@maple-dev/effect-orm/types"` — as above — reads better than -> `CH.string` and keeps column types visually distinct from the query DSL. - -_(Backed by `docs/tables-and-types.md > Column types come from /types as a namespace`.)_ - ## `InferTS` `InferTS` maps a column type to its TypeScript type. You rarely need it directly — `select` already infers output rows — but it is exported for writing your own helpers: ```ts -import type { InferTS } from "@maple-dev/effect-orm" +import type { InferTS } from "@maple-dev/effect-orm/clickhouse" -type Ms = InferTS // number +type Ms = InferTS // number ``` -`InferEncoded` is its counterpart — the wire type the schema decodes _from_. +`InferEncoded` is its counterpart — the wire type the schema decodes _from_. For a whole +table, `SelectRowOf` is the decoded row and `InsertRowOf` the row an +insert takes. Related utilities: `ColumnDefs` (the shape of a `columns` record), `OutputToColumnDefs` (converts a query's output row back into column defs, used by `fromQuery`), and @@ -119,7 +246,7 @@ const query = CH.from(Events) .select(($) => ({ method: $.Attributes.get("http.method") })) .where(($) => [$.OrgId.eq("org_123")]) -// SELECT Attributes['http.method'] AS method FROM events WHERE OrgId = 'org_123' +// SELECT events.Attributes['http.method'] AS method FROM events WHERE events.OrgId = 'org_123' ``` `.get()` yields the map's _value_ type — `Expr` for a `Map(String, String)`, `Expr` for a `Map(String, UInt64)`. For the other map operations — `mapContains`, @@ -139,27 +266,26 @@ CH.from(Events, "e") // FROM events AS e, columns emit as e.Name See [Joins and subqueries](./joins-and-subqueries.md). -`T.dateTime64` preserves milliseconds when encoding `Date`/`DateTime.Utc` comparison bounds -and decoded rows. JavaScript timestamps have millisecond precision; use `T.dateTime64String` -when forwarding microseconds or nanoseconds unchanged. `T.dateTime` encodes whole seconds. +`CH.dateTime64` preserves milliseconds when encoding `Date`/`DateTime.Utc` comparison bounds +and decoded rows. JavaScript timestamps have millisecond precision; use `CH.dateTime64String` +when forwarding microseconds or nanoseconds unchanged. `CH.dateTime` encodes whole seconds. ## Types not in the built-in list -`T.custom(sqlType, schema)` models types such as UUID, LowCardinality, enums, or decimals using +`CH.custom(sqlType, schema)` models types such as UUID, LowCardinality, enums, or decimals using their JSON representation. Match your existing database schema rather than redesigning the physical table to fit this library's constructors. For a `LowCardinality(String)` column, for -example, `T.custom("LowCardinality(String)", Schema.String)` decodes the ordinary string it emits. +example, `CH.custom("LowCardinality(String)", Schema.String)` decodes the ordinary string it emits. ## Branded columns -`T.brand(type, schema)` narrows a column type with an Effect schema: a branded id, a literal -union, a refined number. It keeps the base type's SQL type and wire codec, so `brand(PG.int8, +`brand(type, schema)` narrows a column type with an Effect schema: a branded id, a literal +union, a refined number. It keeps the base type's SQL type and wire codec, so `PG.brand(PG.int8, Cents)` still reads the string node-postgres sends, and wraps like any type: -`nullable(brand(...))`, `array(brand(...))`. +`nullable(brand(...))`, `array(brand(...))`. Both entries have it: `CH.brand`, `PG.brand`. ```ts title="branded-columns.ts" import { Schema } from "effect" -import * as CH from "@maple-dev/effect-orm" import * as PG from "@maple-dev/effect-orm/postgres" const OrgId = Schema.String.check(Schema.isMinLength(1)).pipe(Schema.brand("OrgId")) @@ -168,47 +294,50 @@ const UserId = Schema.String.pipe(Schema.brand("UserId")) // Declare the column type once; tables and params both use it. const orgId = PG.brand(PG.text, OrgId) -const Dashboards = CH.table("dashboards", { - org_id: orgId, - id: PG.text, - owner: PG.nullable(PG.brand(PG.text, UserId)), +const Dashboards = PG.table("dashboards", { + columns: { + org_id: orgId, + id: PG.text, + owner: PG.nullable(PG.brand(PG.text, UserId)), + }, + primaryKey: ["org_id", "id"], }) -export type Dashboard = CH.SelectRowOf +export type Dashboard = PG.SelectRowOf // { readonly org_id: OrgId; readonly id: string; readonly owner: UserId | null } -export const byOrg = CH.from(Dashboards) +export const byOrg = PG.from(Dashboards) .select("id", "owner") - .where(($) => [$.org_id.eq(CH.param.of(orgId, "orgId"))]) + .where(($) => [$.org_id.eq(PG.param.of(orgId, "orgId"))]) export const compiled = PG.compileUnsafe(byOrg, { orgId: OrgId.make("org_1") }) declare const userId: typeof UserId.Type // @ts-expect-error a UserId is not an OrgId -CH.from(Dashboards).select("id").where(($) => [$.org_id.eq(userId)]) +PG.from(Dashboards).select("id").where(($) => [$.org_id.eq(userId)]) ``` A brand is strict everywhere it is written or compared: - **Rows** decode to the brand, and `SelectRowOf` names the whole row. - **Comparisons** (`eq`, `in_`, `between`, joins) take a value of the brand, a column of the same - brand, or a param declared with the type: `CH.param.of(orgId, "orgId")`, whose value + brand, or a param declared with the type: `PG.param.of(orgId, "orgId")`, whose value `compile` then requires to be an `OrgId`. A plain string, another brand, `param.string`, or an unbranded column is a type error. - **Inserts and updates** take the brand, a param of it, or an expression of it. - **Checks run both ways.** A row that fails the schema's checks is a decode error; a literal or param value that fails them is a `QueryBuilderError` from `compile`. -A literal union (`brand(PG.text, Schema.Literals(["open", "closed"]))`) is not a brand: it +A literal union (`PG.brand(PG.text, Schema.Literals(["open", "closed"]))`) is not a brand: it compares against any string, and the database checks the value. -`T.custom("String", OrgId)` brands the same way, but replaces the wire codec with `OrgId` itself; +`CH.custom("String", OrgId)` brands the same way, but replaces the wire codec with `OrgId` itself; prefer `brand` over a built-in type whose codec does work (numbers, timestamps). -`T.untyped(sqlType)` accepts an unknown field without validating it. Unlike `CH.untypedExpr`, it +`CH.untyped(sqlType)` accepts an unknown field without validating it. Unlike `CH.untypedExpr`, it supplies a `Schema.Unknown` codec, so other selected fields can still be validated. The unknown field itself has no guarantee. Prefer a real custom codec where you know the wire representation. -`T.aggregateState(fn, ...argumentTypes)` describes an opaque aggregate-state value passed from +`CH.aggregateState(fn, ...argumentTypes)` describes an opaque aggregate-state value passed from an inner query into a matching merge function. It is not a decoder for inspecting state bytes. See [Extending the DSL](./extending.md#a-column-type-of-your-own). diff --git a/docs/tenant-scoping.md b/docs/tenant-scoping.md index 9de884c..30bff3b 100644 --- a/docs/tenant-scoping.md +++ b/docs/tenant-scoping.md @@ -20,7 +20,12 @@ Row-per-tenant is the usual ClickHouse multi-tenancy shape, but whether a table what the column is called — is a schema decision, so it is declared on the table: ```ts -const Events = CH.table("events", { OrgId: T.string, Name: T.string }, { tenantColumn: "OrgId" }) +const Events = CH.table("events", { + columns: { OrgId: CH.string, Name: CH.string }, + engine: CH.engine.mergeTree(), + orderBy: ["OrgId"], + tenantColumn: "OrgId", +}) ``` The option is checked against the column names you just declared, so a typo is a type error @@ -28,7 +33,11 @@ rather than a query that silently never scopes. A table with **no** `tenantColum pin, and compiles to the third scope: ```ts -const Untenanted = CH.table("untenanted", { tenant_id: T.string, Name: T.string }) +const Untenanted = CH.table("untenanted", { + columns: { tenant_id: CH.string, Name: CH.string }, + engine: CH.engine.mergeTree(), + orderBy: [], +}) CH.from(Untenanted) .select(($) => ({ name: $.Name })) diff --git a/docs/testing.md b/docs/testing.md index 68fbb2b..bc3a5dd 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -34,7 +34,7 @@ The user defaults to `default` and the password to an empty string. `bun run test:package` builds and packs the package, installs the tarball outside the workspace with its Effect peer, typechecks a consumer with strict declarations, and -executes imports from all seven public entry points under Node. This needs npm registry +executes imports from the public entry points under Node. This needs npm registry access. It uses the installed Effect and TypeScript versions, with explicit Node, DOM, and disposable type libraries required by the Effect declarations. @@ -86,7 +86,8 @@ with a reason. The core manifest in `tests/dialect-coverage.test.ts` requires ev and union method, expression and condition operator, and param kind to have a core case. Postgres functions and types have their own manifest: every export of the `./postgres` -entry is run by a case in `tests/dialect-cases.postgres.ts` (`tests/dialect.postgres.test.ts`) +entry that is its own (not the shared query builder, which the ClickHouse manifest covers) is +run by a case in `tests/dialect-cases.postgres.ts` (`tests/dialect.postgres.test.ts`) or exempted with a reason. Tests preserve documented behavior: arithmetic chains follow SQL precedence, not call diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 1de7635..c3f39b8 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -10,8 +10,10 @@ or row decoding. [Running a query](./running-queries.md#error-boundaries) separa | npm returns 404 for the builder | The package name or configured registry is incorrect. | Check `@maple-dev/effect-orm` and your npm registry; see [Getting started](./getting-started.md#installation-and-compatibility). | | `Schema.TaggedError is not a function` | Effect 3 is installed or resolving ahead of Effect 4. | Install the documented Effect 4 version and inspect the resolved dependency tree. | | `ERR_REQUIRE_ESM` or an import cannot be loaded by `require` | The builder ships ESM. | Use ESM imports and `"type": "module"`, or your bundler's ESM support. | -| A helper exists in source but not in the package | A deep source import or stale local build. | Use the seven public entry points and rebuild/reinstall your tarball. | -| An example's `Events`, `Services`, `CH`, or `T` is undefined | A guide fragment expects the shared schema/imports. | Start with the complete example and shared `schema.ts`; recipe files include their own imports. | +| A helper exists in source but not in the package | A deep source import or stale local build. | Use the public entry points in `package.json` `exports` and rebuild/reinstall your tarball. | +| `@maple-dev/effect-orm` or `@maple-dev/effect-orm/types` cannot be resolved | Code written for the removed root and `/types` entries. | Import everything for one database from `@maple-dev/effect-orm/clickhouse` or `@maple-dev/effect-orm/postgres`; write `CH.string`, not `T.string`. | +| An example's `Events`, `Services`, or `CH` is undefined | A guide fragment expects the shared schema/imports. | Start with the complete example and shared `schema.ts`; recipe files include their own imports. | +| `SchemaDefinitionDefect` when a module loads | A `table` definition is invalid, such as a MergeTree table without `orderBy`. | Read the message; pass `orderBy: []` for `ORDER BY tuple()`, or `external: true` for a table this schema does not own. | ## Compilation failures @@ -61,12 +63,12 @@ alongside `compiled.rowSchemaSource`, `compiled.untypedColumns`, and `compiled.r | A field disappeared after decoding | An explicit schema replaced the derived shape. Inspect `rowSchemaMismatch`. | | An ID's last digits changed | The value passed through JavaScript `number`. Project `toString(...)` in SQL before parsing JSON. | | Timestamps shifted by several hours | The codec assumes zone-less text is UTC, but the server/column emitted another timezone. Normalize SQL output or use a matching custom codec. | -| Microseconds/nanoseconds disappeared | Parsed `DateTime.Utc` has millisecond precision. Preserve the text with `T.dateTime64String`. | +| Microseconds/nanoseconds disappeared | Parsed `DateTime.Utc` has millisecond precision. Preserve the text with `CH.dateTime64String`. | | An average/percentile is NULL | Empty/non-finite aggregate results can serialize as JSON null. Keep the nullable type or explicitly define a fallback. | | “No rows” crashes a point lookup | Use `decodeFirstRow` and handle `Option.none`; an empty result is not a decode failure. | -The built-in numeric codecs are wire decoders, not full range validators. `T.uint64` does not -make JavaScript numbers lossless or enforce UInt64 bounds. `T.untyped` validates nothing for its +The built-in numeric codecs are wire decoders, not full range validators. `CH.uint64` does not +make JavaScript numbers lossless or enforce UInt64 bounds. `CH.untyped` validates nothing for its field even when the rest of the row has a derived schema. ## Reporting a problem diff --git a/docs/unions-and-ctes.md b/docs/unions-and-ctes.md index 1098879..8c3f5e2 100644 --- a/docs/unions-and-ctes.md +++ b/docs/unions-and-ctes.md @@ -57,7 +57,7 @@ in-progress hour, then re-aggregating over both. `compile` time and its tenant scope is **derived**, so nobody has to assert it: ```ts -const Recent = CH.table("recent", { Name: T.string }) +const Recent = CH.table("recent", { external: true, columns: { Name: CH.string } }) const cte = CH.from(Events) .select(($) => ({ Name: $.Name })) @@ -73,8 +73,9 @@ const compiled = CH.compileUnsafe(query, {}) compiled.tenantScope // "single-tenant" — read off the CTE, not declared ``` -To _read_ a CTE, declare a table whose name matches it and start the query there. That is what -gives you typed accessors over the CTE's columns. +To _read_ a CTE, declare an external table (`external: true`) whose name matches it and start the +query there. That is what gives you typed accessors over the CTE's columns; being external, it +carries no DDL, so `effect-orm generate` never tries to create it. _(Backed by `docs/unions-and-ctes.md > Selecting from a CTE`.)_ @@ -86,7 +87,7 @@ all you have: ```ts const cteSql = "SELECT Name FROM events WHERE OrgId = 'org_123'" -CH.from(CH.table("recent", { Name: T.string })) +CH.from(CH.table("recent", { external: true, columns: { Name: CH.string } })) .withCTE("recent", cteSql, { tenantScope: "single-tenant" }) .select(($) => ({ name: $.Name })) ``` diff --git a/docs/updates-and-deletes.md b/docs/updates-and-deletes.md index 0d0d927..d0abb33 100644 --- a/docs/updates-and-deletes.md +++ b/docs/updates-and-deletes.md @@ -5,21 +5,20 @@ like [`insertInto`](./inserts.md). They are immutable values; `Database.run` com its database's dialect and runs them. ```ts -import * as CH from "@maple-dev/effect-orm" import * as PG from "@maple-dev/effect-orm/postgres" -const Tickets = CH.table( - "tickets", - { id: PG.int4, org: PG.text, seats: PG.int4, tags: PG.array(PG.text) }, - { tenantColumn: "org" }, -) +const Tickets = PG.table("tickets", { + columns: { id: PG.int4, org: PG.text, seats: PG.int4, tags: PG.array(PG.text) }, + primaryKey: ["id"], + tenantColumn: "org", +}) -const bump = CH.update(Tickets) +const bump = PG.update(Tickets) .set(($) => ({ seats: $.seats.add(1) })) - .where(($) => [$.org.eq(CH.param.string("org")), $.seats.lt(5)]) + .where(($) => [$.org.eq(PG.param.string("org")), $.seats.lt(5)]) .returning("id", "seats") -const revoke = CH.deleteFrom(Tickets).where(($) => [$.id.eq(CH.param.int("id"))]) +const revoke = PG.deleteFrom(Tickets).where(($) => [$.id.eq(PG.param.int("id"))]) // yield* Db.run(bump, { org }) // [{ id, seats }] // yield* Db.run(revoke, { id }) // [] @@ -29,8 +28,9 @@ const revoke = CH.deleteFrom(Tickets).where(($) => [$.id.eq(CH.param.int("id"))] `set` takes a record of values, params or expressions, or a callback that gets the row's columns as `$`. A key left out (or `undefined`) keeps the existing value. Values are encoded -through the column's codec and bound on Postgres, as in an insert. A computed column (see -[`computed`](./inserts.md#which-columns-have-defaults)) cannot be set. `UpdateSetOf` +through the column's codec and bound on Postgres, as in an insert. A column that is not +writable (a ClickHouse `materialized` or `alias` column, see +[the row type](./inserts.md#which-columns-have-defaults)) cannot be set. `UpdateSetOf` names the record type. `update(table)` offers only `set` until it has one (its type is `CHUpdateStart`). diff --git a/package.json b/package.json index 85f73ff..4f44ae7 100644 --- a/package.json +++ b/package.json @@ -33,18 +33,14 @@ "type": "module", "sideEffects": false, "exports": { - ".": { - "types": "./dist/index.d.mts", - "import": "./dist/index.mjs" + "./clickhouse": { + "types": "./dist/clickhouse.d.mts", + "import": "./dist/clickhouse.mjs" }, "./expr": { "types": "./dist/expr.d.mts", "import": "./dist/expr.mjs" }, - "./types": { - "types": "./dist/types.d.mts", - "import": "./dist/types.mjs" - }, "./sql": { "types": "./dist/sql.d.mts", "import": "./dist/sql.mjs" diff --git a/scripts/check-doc-examples.mjs b/scripts/check-doc-examples.mjs index 345a3cf..06e588e 100644 --- a/scripts/check-doc-examples.mjs +++ b/scripts/check-doc-examples.mjs @@ -46,7 +46,7 @@ try { ` import assert from "node:assert/strict" import { Effect } from "effect" -import * as CH from "@maple-dev/effect-orm" +import * as CH from "@maple-dev/effect-orm/clickhouse" import { Events } from "./schema" const sql = (query: { sql: string }) => query.sql.replace(/\\s+/g, " ").trim() const quick = await import("./quick-start") diff --git a/scripts/check-exports-documented.mjs b/scripts/check-exports-documented.mjs index 872badd..93bec71 100644 --- a/scripts/check-exports-documented.mjs +++ b/scripts/check-exports-documented.mjs @@ -1,53 +1,58 @@ #!/usr/bin/env node -// Every root export is named in docs/reference.md, whose first line claims to be -// the catalog. It was not: 42 exports appeared nowhere on the page and two rows -// listed root exports as subpath-only. A claim like that decays the moment +// Every export of the two dialect entries is named in docs/reference.md, whose +// first line claims to be the catalog. A claim like that decays the moment // anything is added, so it is checked rather than promised. // // Reads the BUILT types, not the source: what a consumer can import is what -// `dist/index.d.mts` says, and the barrel's re-export chains are already +// `dist/.d.mts` says, and the barrel's re-export chains are already // resolved there. import { readFileSync, existsSync } from "node:fs" import { dirname, join } from "node:path" import { fileURLToPath } from "node:url" const root = join(dirname(fileURLToPath(import.meta.url)), "..") -const dts = join(root, "dist/index.d.mts") -if (!existsSync(dts)) { - console.error("dist/index.d.mts missing — run `bun run build` first.") - process.exit(1) -} +const entries = ["clickhouse", "postgres"] -// The final `export { … }` statement of the bundle is the whole public surface. -const source = readFileSync(dts, "utf8") -const statements = [...source.matchAll(/export \{([^}]*)\};/g)] -const last = statements.at(-1) -if (!last) { - console.error("No `export { … }` statement found in dist/index.d.mts.") - process.exit(1) +const exportsOf = (entry) => { + const dts = join(root, `dist/${entry}.d.mts`) + if (!existsSync(dts)) { + console.error(`dist/${entry}.d.mts missing — run \`bun run build\` first.`) + process.exit(1) + } + // The final `export { … }` statement of the bundle is the whole public surface. + const source = readFileSync(dts, "utf8") + const last = [...source.matchAll(/export \{([^}]*)\};/g)].at(-1) + if (!last) { + console.error(`No \`export { … }\` statement found in dist/${entry}.d.mts.`) + process.exit(1) + } + return ( + last[1] + .split(",") + .map((entry) => entry.trim()) + .filter(Boolean) + // `type Foo`, `bar as baz` — the importable name is what follows `as`, and + // the `type` marker is not part of it. + .map((entry) => { + const aliased = entry.split(/\s+as\s+/) + return (aliased.at(-1) ?? entry).replace(/^type\s+/, "").trim() + }) + .filter((name) => name !== "default") + ) } -const names = last[1] - .split(",") - .map((entry) => entry.trim()) - .filter(Boolean) - // `type Foo`, `bar as baz` — the importable name is what follows `as`, and - // the `type` marker is not part of it. - .map((entry) => { - const aliased = entry.split(/\s+as\s+/) - return (aliased.at(-1) ?? entry).replace(/^type\s+/, "").trim() - }) - .filter((name) => name !== "default") - const reference = readFileSync(join(root, "docs/reference.md"), "utf8") -const missing = names.filter((name) => !new RegExp(`\\b${name.replace(/\$/g, "\\$")}\\b`).test(reference)) - -if (missing.length > 0) { - console.error( - `docs/reference.md claims to be the export catalog but does not mention ${missing.length} root export${ - missing.length > 1 ? "s" : "" - }:\n ` + missing.join("\n "), - ) - process.exit(1) +let failed = false +for (const entry of entries) { + const names = exportsOf(entry) + const missing = names.filter((name) => !new RegExp(`\\b${name.replace(/\$/g, "\\$")}\\b`).test(reference)) + if (missing.length > 0) { + failed = true + console.error( + `docs/reference.md claims to be the export catalog but does not mention ${missing.length} /${entry} export${ + missing.length > 1 ? "s" : "" + }:\n ` + missing.join("\n "), + ) + } else console.log(`export catalog ok (${names.length} /${entry} exports)`) } -console.log(`export catalog ok (${names.length} root exports)`) +if (failed) process.exit(1) diff --git a/scripts/check-package.ts b/scripts/check-package.ts index 8c100a8..45a3ae4 100644 --- a/scripts/check-package.ts +++ b/scripts/check-package.ts @@ -33,7 +33,7 @@ const program = Effect.gen(function* () { const packed = (yield* Schema.decodeEffect(PackResult)(output))[0] if (!packed) return yield* check(false, "tarball", "npm pack produced no artifact") yield* check( - packed.files.some((file) => file.path === "dist/index.mjs"), + packed.files.some((file) => file.path === "dist/clickhouse.mjs") && packed.files.some((file) => file.path === "dist/postgres.mjs"), "tarball", "Missing built entry point", ) diff --git a/src/benchmark/benchmark.test.ts b/src/benchmark/benchmark.test.ts index 1225cc7..9114d37 100644 --- a/src/benchmark/benchmark.test.ts +++ b/src/benchmark/benchmark.test.ts @@ -1,4 +1,4 @@ -import * as T from "../types" +import * as T from "../ch/types" import * as CH from "../ch/index" import { describe, expect, it } from "vitest" import { Effect, Schema } from "effect" diff --git a/src/ch/any-boundaries.test-d.ts b/src/ch/any-boundaries.test-d.ts index d55c18f..6c961ce 100644 --- a/src/ch/any-boundaries.test-d.ts +++ b/src/ch/any-boundaries.test-d.ts @@ -1,6 +1,6 @@ import { Schema, type DateTime } from "effect" import { expectTypeOf } from "expect-type" -import * as CH from "../index" +import * as CH from "./index" const values = CH.arrayFilter("x -> x > 0", CH.arrayOf(CH.lit(1))) expectTypeOf(values).toEqualTypeOf>>() diff --git a/src/ch/any-boundaries.test.ts b/src/ch/any-boundaries.test.ts index c47f702..7dd5569 100644 --- a/src/ch/any-boundaries.test.ts +++ b/src/ch/any-boundaries.test.ts @@ -1,6 +1,6 @@ import { Effect } from "effect" import { describe, expect, it } from "@effect/vitest" -import * as CH from "../index" +import * as CH from "./index" describe("typed arrayFilter", () => { it.effect("preserves element decoding and encoding in selected rows", () => Effect.gen(function* () { diff --git a/src/ch/brand.test-d.ts b/src/ch/brand.test-d.ts index edb1ec3..4e63c98 100644 --- a/src/ch/brand.test-d.ts +++ b/src/ch/brand.test-d.ts @@ -20,14 +20,14 @@ const Status = Schema.Literals(["open", "closed"]) const orgId = PG.brand(PG.text, OrgId) const userId = PG.brand(PG.text, UserId) -const Dashboards = S.pg.table("dashboards", { +const Dashboards = PG.table("dashboards", { columns: { org_id: orgId, id: PG.text, owner: PG.nullable(userId), editors: PG.array(userId), budget: PG.brand(PG.int8, Cents), - status: S.pg.column(PG.brand(PG.text, Status), { default: "open" }), + status: PG.column(PG.brand(PG.text, Status), { default: "open" }), created_at: PG.timestamptz, }, primaryKey: ["org_id", "id"], diff --git a/src/ch/brand.test.ts b/src/ch/brand.test.ts index 98a7e2d..7e98853 100644 --- a/src/ch/brand.test.ts +++ b/src/ch/brand.test.ts @@ -12,7 +12,7 @@ const Cents = Schema.Number.check(Schema.isGreaterThanOrEqualTo(0)).pipe(Schema. const orgId = PG.brand(PG.text, OrgId) -const Accounts = S.pg.table("accounts", { +const Accounts = PG.table("accounts", { columns: { org_id: orgId, id: PG.text, diff --git a/src/ch/dialect.ts b/src/ch/dialect.ts index 06c9384..ed39226 100644 --- a/src/ch/dialect.ts +++ b/src/ch/dialect.ts @@ -145,7 +145,7 @@ export interface Dialect extends SqlSyntax { readonly transactions?: DialectTransactions /** * Which built-in function set renders correctly here: `clickhouse` (the - * root entry's functions) or `postgres` (`@maple-dev/effect-orm/postgres`). + * `/clickhouse` entry's functions) or `postgres` (`@maple-dev/effect-orm/postgres`). * A built-in function from another set fails to compile. Absent means * unchecked: a custom dialect says which set it renders, if either. */ diff --git a/src/ch/functions/builtin.ts b/src/ch/functions/builtin.ts index 9b80b3c..4ad5d68 100644 --- a/src/ch/functions/builtin.ts +++ b/src/ch/functions/builtin.ts @@ -24,7 +24,7 @@ export type FunctionSet = "clickhouse" | "postgres" export type BuiltinKind = "scalar" | "aggregate" | "window" const setLabel: Record = { - clickhouse: "a ClickHouse function (from the root entry)", + clickhouse: "a ClickHouse function (from @maple-dev/effect-orm/clickhouse)", postgres: "a Postgres function (from @maple-dev/effect-orm/postgres)", } diff --git a/src/ch/index.ts b/src/ch/index.ts index 730f3c7..557dc1a 100644 --- a/src/ch/index.ts +++ b/src/ch/index.ts @@ -1,312 +1,6 @@ -// ClickHouse Query DSL — Public API +// Internal barrel for this package's own tests: the `/clickhouse` entry plus +// the bare `table()` its `table` builds on. Not published; consumers define +// tables with `table` from `/clickhouse` or `/postgres`. -// Types -export { - type CHType, - type CHString, - type CHUInt8, - type CHUInt16, - type CHUInt32, - type CHInt32, - type CHInt64, - type CHBool, - type CHUInt64, - type CHFloat64, - type CHDateTime, - type CHDateTimeString, - type CHDateTime64String, - type CHDateTime64, - type CHMap, - type CHArray, - type CHNullable, - type InferTS, - type InferEncoded, - type ColumnDefs, - type OutputToColumnDefs, - type NullableColumnDefs, - string, - uint8, - uint16, - uint32, - bool, - uint64, - int32, - int64, - float64, - aggregateState, - untyped, - dateTime, - dateTime64, - dateTimeString, - dateTime64String, - map, - array, - nullable, - custom, - brand, -} from "./types" - -// Table -export { type SelectRowOf, type Table, type TableOptions, table } from "./table" - -// Core expression primitives -export { - type Expr, - type ColumnRef, - type Condition, - // In the signature of every comparison method and of `ColumnRef.get`, so a - // consumer writing a generic helper over `Expr` needs them at the root. - type Comparable, - type MapValueOf, - lit, - rawExpr, - untypedExpr, - rawCond, - when, - whenTrue, - inList, - inExprList, - notInList, - // Negating a condition is table stakes; it was `/expr`-only. - not, - and, - or, - outerRef, - // Reference an output alias (a GROUP BY key or aggregate) that isn't on the - // column accessor — the usual way to write a `having()` body. - dynamicColumn, -} from "./expr" - -// SQL templates inside expressions and conditions. -export { sql, type SqlIdent, type SqlRaw, type SqlTag, type SqlTemplateValue } from "./sql-template" - -// Subquery conditions. These accept a `CHQuery` as well as raw SQL, so they -// supersede the string-only `exists`/`inSubquery` still exported from `./expr` -// for direct subpath importers. -export { - type Subquery, - exists, - inSubquery, - notInSubquery, - // Splice an inner query's SQL where the builder has no syntax — compiled by - // the outer `compile`, so its failures land in the outer error channel. - subqueryExpr, - subqueryCond, - untypedSubqueryExpr, -} from "./subquery" - -// Function factories (for extensibility by package consumers) -export { - arrayOfArg, - compileFnCall, - compileFnCallCond, - compileTypedFnCall, - defineCondFn, - defineFn, - defineUntypedFn, - elementOf, - elementSchema, - firstTyped, - firstTypedNonNull, - type FnResult, - makeCond, - makeExpr, - makeUntypedExpr, - sameAs, - schemaOf, - schemaOfAny, - withoutNull, -} from "./define-fn" - -// ClickHouse functions (from category modules) -export { - // Aggregate - count, - countIf, - avg, - sum, - min_ as min, - max_ as max, - quantile, - any_ as any, - anyIf, - uniq, - uniqIf, - uniqExact, - sumIf, - avgIf, - maxIf, - minIf, - groupUniqArray, - groupUniqArrayArray, - groupUniqArrayIf, - argMin, - argMax, - argMaxMerge, - windowFunnel, - sequenceMatch, - type WindowFunnelMode, - // String - toString_ as toString, - positionCaseInsensitive, - position_ as position, - left_ as left, - length_ as length, - lower_ as lower, - domain_ as domain, - hex, - path_ as path, - cutQueryString, - replaceOne, - extract_ as extract, - match_ as match, - matchCond, - concat, - hasToken, - hasAllTokens, - // Numeric - round_ as round, - intDiv, - toFloat64OrZero, - toFloat64, - toUInt16OrZero, - toUInt64, - toInt64, - least_ as least, - greatest_ as greatest, - cityHash64, - // Date/time - toStartOfInterval, - toStartOfHour, - toStartOfMinute, - toHour, - toUnixTimestamp, - toUnixTimestamp64Nano, - intervalAdd, - intervalSub, - formatDateTime, - toDateTime, - // Conditional - if_, - multiIf, - coalesce, - ifNull, - nullIf, - ifNotFinite, - // Array - arrayOf, - arrayStringConcat, - arrayFilter, - arrayDistinct, - arrayElement, - arrayJoin, - arrayPushFront, - arrayReverseSort, - arraySort, - has, - // Map - mapContains, - mapGet, - mapKeys, - mapValues, - mapLiteral, - // JSON - toJSONString, - // Window - currentRow, - unboundedPreceding, - unboundedFollowing, - preceding, - following, - rowsBetween, - windowSpec, - over, - lagInFrame, - type CompiledWindowSpec, - type WindowFrameBound, - type WindowOrderDirection, - type WindowRowsFrame, - type WindowSpec, -} from "./functions" - -// Params -export { param, paramPlaceholder, type ParamKind, type ParamMarker } from "./param" - -// Query builder -export { - type CHQuery, - type ColumnAccessor, - type JoinedColumnAccessor, - type JoinOnCallback, - type InferOutput, - type InferQueryOutput, - type LockOptions, - from, - fromQuery, - fromUnion, -} from "./query" - -// Insert builder -export { - type CHInsert, - type CHInsertStart, - type ConflictSet, - type ConflictTarget, - type InsertRow, - type InsertRowOf, - type InsertSelectMisfits, - type InsertSelectMissing, - type InsertSettingValue, - type InsertValue, - type OnConflictDoNothing, - type OnConflictDoUpdate, - insertInto, -} from "./insert" - -// Update and delete builders -export { - type CHDelete, - type CHUpdate, - type CHUpdateStart, - type UpdateSet, - type UpdateSetOf, - deleteFrom, - update, -} from "./update" - -// Compilation -export { - // `compileCH` / `compileCHUnsafe` are the internal names; the public API is - // four, not six — `compile`/`compileUnion` and their `Unsafe` counterparts. - compileCH as compile, - compileCHUnsafe as compileUnsafe, - compileUnion, - compileUnionUnsafe, - rawCompiledQuery, - type CompiledQuery, - type CompiledQueryInput, - type CompiledQueryRowSchema, - type CHWrite, - type InsertCompileOptions, - type RowSchemaMismatch, - type TenantScope, - CompiledQueryDecodeError, - CompiledQueryEncodeError, -} from "./compile" - -// Dialects: how a compiled query's params reach the server. -export { - clickhouseDialect, - type Dialect, - type DialectClauses, - type DialectTransactions, - type IsolationLevel, - type ParamStyle, - type TransactionSettings, -} from "./dialect" - -// Failures vs defects — the rule the two classes encode is on `QueryBuilderError`. -export { QueryBuilderError, QueryBuilderDefect } from "./errors" - -// Union -export { unionAll, type CHUnionQuery, type InferUnionOutput } from "./union" +export * from "../clickhouse" +export { table, type TableOptions } from "./table" diff --git a/src/ch/insert.test-d.ts b/src/ch/insert.test-d.ts index ec8680b..54cb51f 100644 --- a/src/ch/insert.test-d.ts +++ b/src/ch/insert.test-d.ts @@ -4,7 +4,7 @@ import { Schema, type DateTime } from "effect" import { expectTypeOf } from "expect-type" import * as CH from "./index" import * as PG from "../postgres" -import * as S from "../schema" +import * as CHD from "../clickhouse" import type { RowOf } from "../database" const Events = CH.table( @@ -51,15 +51,15 @@ CH.insertInto(Branded).values({ OrgId: CH.param.string("org") }) CH.insertInto(Branded).values({ OrgId: "o" }) // defineTable: defaults are optional, computed columns are not in the row. -const Spans = S.defineTable("spans", { +const Spans = CHD.table("spans", { columns: { OrgId: CH.string, - Duration: S.column(CH.uint64, { default: 0 }), - Started: S.column(CH.dateTime, { defaultExpr: ($) => CH.rawExpr("now()", CH.dateTime) }), - Day: S.column(CH.string, { materialized: "toString(toDate(Started))" }), - Label: S.column(CH.string, { comment: "shown" }), + Duration: CHD.column(CH.uint64, { default: 0 }), + Started: CHD.column(CH.dateTime, { defaultExpr: ($) => CH.rawExpr("now()", CH.dateTime) }), + Day: CHD.column(CH.string, { materialized: "toString(toDate(Started))" }), + Label: CHD.column(CH.string, { comment: "shown" }), }, - engine: S.engine.mergeTree(), + engine: CHD.engine.mergeTree(), orderBy: ["OrgId"], }) expectTypeOf>().toEqualTypeOf<{ @@ -72,7 +72,7 @@ expectTypeOf>().toEqualTypeOf<{ CH.insertInto(Spans).values({ OrgId: "o", Label: "l", Day: "x" }) // A table with defaults still goes everywhere a table does. CH.from(Spans).select("OrgId", "Day") -expectTypeOf(S.column(CH.string)).toEqualTypeOf>() +expectTypeOf(CHD.column(CH.string)).toEqualTypeOf>() // A table typed without insert metadata reads as "no defaults, nothing computed". declare const Loose: CH.Table<"loose", { A: CH.CHString; B: CH.CHNullable }> diff --git a/src/ch/insert.test.ts b/src/ch/insert.test.ts index c65ef52..383b6eb 100644 --- a/src/ch/insert.test.ts +++ b/src/ch/insert.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from "@effect/vitest" import { DateTime, Effect, Exit } from "effect" import * as CH from "./index" import * as PG from "../postgres" -import * as S from "../schema" +import * as CHD from "../clickhouse" import { QueryBuilderDefect, QueryBuilderError } from "./errors" const Events = CH.table( @@ -387,15 +387,15 @@ describe("insertInto", () => { ) describe("defineTable", () => { - const Spans = S.defineTable("spans", { + const Spans = CHD.table("spans", { columns: { OrgId: CH.string, - Duration: S.column(CH.uint64, { default: 0 }), - Started: S.column(CH.dateTime, { defaultExpr: "now()" }), - Day: S.column(CH.string, { materialized: "toString(toDate(Started))" }), - Label: S.column(CH.string, { comment: "shown in the UI" }), + Duration: CHD.column(CH.uint64, { default: 0 }), + Started: CHD.column(CH.dateTime, { defaultExpr: "now()" }), + Day: CHD.column(CH.string, { materialized: "toString(toDate(Started))" }), + Label: CHD.column(CH.string, { comment: "shown in the UI" }), }, - engine: S.engine.mergeTree(), + engine: CHD.engine.mergeTree(), orderBy: ["OrgId"], }) diff --git a/src/ch/table.ts b/src/ch/table.ts index 4867fc6..bd7dc44 100644 --- a/src/ch/table.ts +++ b/src/ch/table.ts @@ -48,14 +48,14 @@ export interface TableOptions< /** * Columns the database fills when an insert leaves them out: a Postgres * `serial` or `DEFAULT now()`, a ClickHouse `DEFAULT`. Nullable columns are - * optional in an insert without being listed. `defineTable` works this out + * optional in an insert without being listed. each dialect's `table` works this out * from its column options. */ readonly defaults?: ReadonlyArray /** * Columns the database computes and an insert may not write: a Postgres * `GENERATED ALWAYS` column, a ClickHouse `MATERIALIZED` or `ALIAS` one. - * They stay readable. `defineTable` works this out from its column options. + * They stay readable. Each dialect's `table` works this out from its column options. */ readonly computed?: ReadonlyArray } diff --git a/src/clickhouse.ts b/src/clickhouse.ts new file mode 100644 index 0000000..2401e47 --- /dev/null +++ b/src/clickhouse.ts @@ -0,0 +1,191 @@ +// @maple-dev/effect-orm/clickhouse +// +// Everything for ClickHouse from one import: column types, tables with their +// DDL, the function catalog, the query builder and `compile`. Postgres has the +// same shape under `/postgres`. See docs/getting-started.md. + +export * from "./core" + +// Column Types +export { + type CHString, + type CHUInt8, + type CHUInt16, + type CHUInt32, + type CHInt32, + type CHInt64, + type CHBool, + type CHUInt64, + type CHFloat64, + type CHDateTime, + type CHDateTimeString, + type CHDateTime64String, + type CHDateTime64, + type CHMap, + type CHArray, + type CHNullable, + type CHStringLike, + CHNumber, + string, + uint8, + uint16, + uint32, + bool, + uint64, + int32, + int64, + float64, + aggregateState, + untyped, + dateTime, + dateTime64, + dateTimeString, + dateTime64String, + map, + array, + nullable, + custom, + brand, +} from "./ch/types" + +// Tables, materialized views and their DDL. `table` is the only way to declare +// a table: it carries the engine and keys `effect-orm generate` diffs. +export { + defineTable as table, + column, + engine, + index, + materializedView, + ttlAfterDays, + SchemaDefinitionDefect, + type ColumnInput, + type ColumnOptions, + type ColumnSpec, + type ColumnsOf, + type ComputedColumnsOf, + type DefaultedColumnsOf, + type DdlExpr, + type DdlKey, + type IndexSpec, + type MaterializedView, + type MisfitColumns, + type SchemaTable, + type TableDdl, + type TableDefinition, + type ExternalTableDefinition, +} from "./schema/define" + +// ClickHouse functions (from category modules) +export { + // Aggregate + count, + countIf, + avg, + sum, + min_ as min, + max_ as max, + quantile, + any_ as any, + anyIf, + uniq, + uniqIf, + uniqExact, + sumIf, + avgIf, + maxIf, + minIf, + groupUniqArray, + groupUniqArrayArray, + groupUniqArrayIf, + argMin, + argMax, + argMaxMerge, + windowFunnel, + sequenceMatch, + type WindowFunnelMode, + // String + toString_ as toString, + positionCaseInsensitive, + position_ as position, + left_ as left, + length_ as length, + lower_ as lower, + domain_ as domain, + hex, + path_ as path, + cutQueryString, + replaceOne, + extract_ as extract, + match_ as match, + matchCond, + concat, + hasToken, + hasAllTokens, + // Numeric + round_ as round, + intDiv, + toFloat64OrZero, + toFloat64, + toUInt16OrZero, + toUInt64, + toInt64, + least_ as least, + greatest_ as greatest, + cityHash64, + // Date/time + toStartOfInterval, + toStartOfHour, + toStartOfMinute, + toHour, + toUnixTimestamp, + toUnixTimestamp64Nano, + intervalAdd, + intervalSub, + formatDateTime, + toDateTime, + // Conditional + if_, + multiIf, + coalesce, + ifNull, + nullIf, + ifNotFinite, + // Array + arrayOf, + arrayStringConcat, + arrayFilter, + arrayDistinct, + arrayElement, + arrayJoin, + arrayPushFront, + arrayReverseSort, + arraySort, + has, + // Map + mapContains, + mapGet, + mapKeys, + mapValues, + mapLiteral, + // JSON + toJSONString, + // Window + currentRow, + unboundedPreceding, + unboundedFollowing, + preceding, + following, + rowsBetween, + windowSpec, + over, + lagInFrame, + type CompiledWindowSpec, + type WindowFrameBound, + type WindowOrderDirection, + type WindowRowsFrame, + type WindowSpec, +} from "./ch/functions" + +// Compilation, defaulting to the ClickHouse dialect +export { compileCH as compile, compileCHUnsafe as compileUnsafe, compileUnion, compileUnionUnsafe } from "./ch/compile" +export { clickhouseDialect } from "./ch/dialect" diff --git a/src/core.ts b/src/core.ts new file mode 100644 index 0000000..dd9476d --- /dev/null +++ b/src/core.ts @@ -0,0 +1,147 @@ +// The dialect-neutral query builder, shared by `/clickhouse` and `/postgres`. +// +// Not an entry point: each dialect entry re-exports this next to its own column +// types, functions, table definitions and `compile`, so one import covers +// everything for that database. Only what writes the same SQL shape on every +// dialect belongs here; a ClickHouse or Postgres function does not. + +// Column type plumbing +export { + type CHType, + type ColumnDefs, + type InferEncoded, + type InferTS, + type NullableColumnDefs, + type OutputToColumnDefs, +} from "./ch/types" + +// Tables, as the query builder sees them. Define one with the dialect's `table`. +export { type SelectRowOf, type Table } from "./ch/table" + +// Core expression primitives +export { + type Expr, + type ColumnRef, + type Condition, + type Comparable, + type MapValueOf, + lit, + rawExpr, + untypedExpr, + rawCond, + when, + whenTrue, + inList, + inExprList, + notInList, + not, + and, + or, + outerRef, + dynamicColumn, +} from "./ch/expr" + +export { sql, type SqlIdent, type SqlRaw, type SqlTag, type SqlTemplateValue } from "./ch/sql-template" + +export { + type Subquery, + exists, + inSubquery, + notInSubquery, + subqueryExpr, + subqueryCond, + untypedSubqueryExpr, +} from "./ch/subquery" + +// Function factories, for functions of your own +export { + arrayOfArg, + compileFnCall, + compileFnCallCond, + compileTypedFnCall, + defineCondFn, + defineFn, + defineUntypedFn, + elementOf, + elementSchema, + firstTyped, + firstTypedNonNull, + type FnResult, + makeCond, + makeExpr, + makeUntypedExpr, + sameAs, + schemaOf, + schemaOfAny, + withoutNull, +} from "./ch/define-fn" + +// Params +export { param, paramPlaceholder, type ParamKind, type ParamMarker } from "./ch/param" + +// Query builder +export { + type CHQuery, + type ColumnAccessor, + type JoinedColumnAccessor, + type JoinOnCallback, + type InferOutput, + type InferQueryOutput, + type LockOptions, + from, + fromQuery, + fromUnion, +} from "./ch/query" + +export { + type CHInsert, + type CHInsertStart, + type ConflictSet, + type ConflictTarget, + type InsertRow, + type InsertRowOf, + type InsertSelectMisfits, + type InsertSelectMissing, + type InsertSettingValue, + type InsertValue, + type OnConflictDoNothing, + type OnConflictDoUpdate, + insertInto, +} from "./ch/insert" + +export { + type CHDelete, + type CHUpdate, + type CHUpdateStart, + type UpdateSet, + type UpdateSetOf, + deleteFrom, + update, +} from "./ch/update" + +export { unionAll, type CHUnionQuery, type InferUnionOutput } from "./ch/union" + +// Compiled output. `compile` itself is per dialect. +export { + rawCompiledQuery, + type CompiledQuery, + type CompiledQueryInput, + type CompiledQueryRowSchema, + type CHWrite, + type InsertCompileOptions, + type RowSchemaMismatch, + type TenantScope, + CompiledQueryDecodeError, + CompiledQueryEncodeError, +} from "./ch/compile" + +export { + type Dialect, + type DialectClauses, + type DialectTransactions, + type IsolationLevel, + type ParamStyle, + type TransactionSettings, +} from "./ch/dialect" + +export { QueryBuilderError, QueryBuilderDefect } from "./ch/errors" diff --git a/src/database/database.test-d.ts b/src/database/database.test-d.ts index fa16755..8a5fbc1 100644 --- a/src/database/database.test-d.ts +++ b/src/database/database.test-d.ts @@ -2,7 +2,7 @@ import { Effect, Schema } from "effect" import { expectTypeOf } from "expect-type" -import * as CH from "../index" +import * as CH from "../ch/index" import * as PG from "../postgres" import * as Db from "../database" diff --git a/src/database/database.test.ts b/src/database/database.test.ts index f278e8d..9e2685d 100644 --- a/src/database/database.test.ts +++ b/src/database/database.test.ts @@ -7,7 +7,7 @@ import { PgliteClient } from "@effect/sql-pglite" import { assert, describe, expect, it, layer } from "@effect/vitest" import { Cause, DateTime, Deferred, Effect, Exit, Fiber, Layer, Ref, Schedule, Schema } from "effect" import * as SqlClient from "effect/sql/SqlClient" -import * as CH from "../index" +import * as CH from "../ch/index" import * as PG from "../postgres" import { clickhouseDialect } from "../ch/dialect" import { postgresDialect } from "../pg/dialect" diff --git a/src/docs-examples.test.ts b/src/docs-examples.test.ts index 8c887d2..2ae95a1 100644 --- a/src/docs-examples.test.ts +++ b/src/docs-examples.test.ts @@ -17,6 +17,7 @@ import { describe, expect, it } from "@effect/vitest" import { Effect, Exit, Option, Schema } from "effect" import * as CH from "./ch/index" import * as T from "./ch/types" +import * as CHE from "./clickhouse" import { parseStatement, renderStatement, withSettings } from "./sql/statement" import { compileCHUnsafe, compileUnionUnsafe } from "./ch/compile" import { raw as rawFragment, compile as compileFragment } from "./sql/sql-fragment" @@ -173,15 +174,15 @@ describe("docs/tables-and-types.md", () => { ) }) - it("Column types come from /types as a namespace", () => { - // The docs tell you to `import * as T from ".../types"`. Every constructor - // is on the root barrel too — this asserts the namespace form the docs - // actually show, which is the one that has to keep working. - const Counters = CH.table( - "counters", - { OrgId: T.string, Hits: T.uint16, Live: T.bool }, - { tenantColumn: "OrgId" }, - ) + it("Column types come from the dialect entry", () => { + // One import per database: types, the table and the builder all come + // from `/clickhouse`, the way the docs show them. + const Counters = CHE.table("counters", { + columns: { OrgId: CHE.string, Hits: CHE.uint16, Live: CHE.bool }, + engine: CHE.engine.mergeTree(), + orderBy: ["OrgId"], + tenantColumn: "OrgId", + }) const query = CH.from(Counters) .select(($) => ({ hits: $.Hits })) diff --git a/src/index.ts b/src/index.ts deleted file mode 100644 index ce6f4f7..0000000 --- a/src/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -// @maple-dev/effect-orm — curated public API -// -// A type-safe, immutable ClickHouse SQL query builder. The main entry point -// re-exports the DSL under friendly names. For the raw ClickHouse function -// names and the low-level fragment AST, see the `/expr`, `/types`, and `/sql` -// subpath entry points. - -export * from "./ch/index" diff --git a/src/kit/generate.ts b/src/kit/generate.ts index b3b6f50..2d07920 100644 --- a/src/kit/generate.ts +++ b/src/kit/generate.ts @@ -30,7 +30,7 @@ import { analyze, type GraphProblem } from "./graph" export interface KitConfig { /** The database this folder migrates. Default `clickhouse`. */ readonly dialect?: SchemaDialect - /** Modules whose exports include `defineTable` / `materializedView` (or `S.pg.table`) values. */ + /** Modules whose exports include `table` / `materializedView` values from `/clickhouse` or `/postgres`. */ readonly schema: string | ReadonlyArray /** The migrations folder. */ readonly out: string diff --git a/src/kit/graph.test.ts b/src/kit/graph.test.ts index c6e5f82..a58be72 100644 --- a/src/kit/graph.test.ts +++ b/src/kit/graph.test.ts @@ -1,12 +1,12 @@ import { Effect } from "effect" import { describe, expect, it } from "vitest" -import * as CH from "../ch/index" +import * as CH from "../clickhouse" import { fromRecord, type MigrationInput } from "../migrate/source" import * as S from "../schema" import { analyze } from "./graph" -const table = (name: string, extra: Record = {}) => - S.defineTable(name, { columns: { Id: CH.string, ...extra }, engine: S.engine.mergeTree(), orderBy: ["Id"] }) +const table = (name: string, extra: Record = {}) => + CH.table(name, { columns: { Id: CH.string, ...extra }, engine: CH.engine.mergeTree(), orderBy: ["Id"] }) const input = (objects: ReadonlyArray, prevIds: ReadonlyArray) => Effect.map(S.makeSnapshot(S.entitiesOf(objects), prevIds), (snapshot) => ({ diff --git a/src/kit/kit.test.ts b/src/kit/kit.test.ts index d8901d2..2f6f787 100644 --- a/src/kit/kit.test.ts +++ b/src/kit/kit.test.ts @@ -7,17 +7,17 @@ import { runCli } from "../kit" const src = resolve(import.meta.dirname, "..") const schemaModule = (extra: { column?: boolean; dropName?: boolean } = {}) => ` -import * as CH from "${src}/ch/index" +import * as CH from "${src}/clickhouse" import * as S from "${src}/schema" -export const Events = S.defineTable("events", { +export const Events = CH.table("events", { columns: { OrgId: CH.string, Timestamp: CH.dateTime64, ${extra.dropName === true ? "" : "Name: CH.string,"} - ${extra.column === true ? 'Env: S.column(CH.string, { default: "" }),' : ""} + ${extra.column === true ? 'Env: CH.column(CH.string, { default: "" }),' : ""} }, - engine: S.engine.mergeTree(), + engine: CH.engine.mergeTree(), orderBy: ["OrgId", "Timestamp"], }) ` @@ -104,15 +104,15 @@ describe("effect-orm CLI", () => { import * as PG from "${src}/postgres" import * as S from "${src}/schema" -export const Dashboards = S.pg.table("dashboards", { +export const Dashboards = PG.table("dashboards", { columns: { org_id: PG.text, id: PG.text, - status: S.pg.column(PG.text, { default: "open" }), + status: PG.column(PG.text, { default: "open" }), ${extra.owner === true ? "owner: PG.nullable(PG.text)," : ""} }, primaryKey: { columns: ["org_id", "id"], name: "dashboards_org_id_id_pk" }, - indexes: [S.pg.index("dashboards_open_idx", ["org_id"], { where: "status = 'open'" })], + indexes: [PG.index("dashboards_open_idx", ["org_id"], { where: "status = 'open'" })], }) ` const pgConfig = (schema: string, name = "effect-orm.config.ts") => diff --git a/src/migrate/pg-migrate.test.ts b/src/migrate/pg-migrate.test.ts index 7a5d498..b16f3c1 100644 --- a/src/migrate/pg-migrate.test.ts +++ b/src/migrate/pg-migrate.test.ts @@ -7,26 +7,26 @@ import * as Migrate from "../migrate" import * as PG from "../postgres" import * as S from "../schema" -const Dashboards = S.pg.table("dashboards", { +const Dashboards = PG.table("dashboards", { columns: { org_id: PG.text, id: PG.text, - status: S.pg.column(PG.text, { default: "open" }), - tags: S.pg.column(PG.array(PG.text), { default: [] }), - layout: S.pg.column(PG.jsonb(), { default: {} }), - archived: S.pg.column(PG.bool, { default: false }), - created_at: S.pg.column(PG.timestamptz, { defaultExpr: "now()" }), + status: PG.column(PG.text, { default: "open" }), + tags: PG.column(PG.array(PG.text), { default: [] }), + layout: PG.column(PG.jsonb(), { default: {} }), + archived: PG.column(PG.bool, { default: false }), + created_at: PG.column(PG.timestamptz, { defaultExpr: "now()" }), archived_at: PG.nullable(PG.timestamptz), embedding: PG.nullable(PG.array(PG.float4)), }, primaryKey: { columns: ["org_id", "id"], name: "dashboards_org_id_id_pk" }, indexes: [ - S.pg.index("dashboards_open_idx", ["org_id"], { where: ($) => $.status.in_("open", "waiting") }), - S.pg.index("dashboards_created_idx", ($) => [$.org_id, `"created_at" DESC`]), + PG.index("dashboards_open_idx", ["org_id"], { where: ($) => $.status.in_("open", "waiting") }), + PG.index("dashboards_created_idx", ($) => [$.org_id, `"created_at" DESC`]), ], }) -const Shares = S.pg.table("dashboard_shares", { +const Shares = PG.table("dashboard_shares", { columns: { org_id: PG.text, id: PG.text, @@ -36,12 +36,12 @@ const Shares = S.pg.table("dashboard_shares", { }, primaryKey: ["org_id", "id"], indexes: [ - S.pg.uniqueIndex("dashboard_shares_live_unq", ($) => [$.org_id, $.dashboard_id, CH.coalesce($.widget_id, CH.lit(""))], { + PG.uniqueIndex("dashboard_shares_live_unq", ($) => [$.org_id, $.dashboard_id, CH.coalesce($.widget_id, CH.lit(""))], { where: "revoked_at is null", }), ], foreignKeys: [ - S.pg.foreignKey({ columns: ["org_id", "dashboard_id"], references: Dashboards, foreignColumns: ["org_id", "id"], onDelete: "cascade" }), + PG.foreignKey({ columns: ["org_id", "dashboard_id"], references: Dashboards, foreignColumns: ["org_id", "id"], onDelete: "cascade" }), ], }) @@ -166,7 +166,7 @@ describe("Postgres migrations", () => { const sql = yield* SqlClient.SqlClient // What drizzle-kit (or a deploy pipeline) already applied. yield* sql.unsafe(`CREATE TABLE legacy (id text PRIMARY KEY)`) - const base = yield* S.makeSnapshot(S.pgEntitiesOf([S.pg.table("legacy", { columns: { id: PG.text }, primaryKey: ["id"] })]), [S.ORIGIN_ID], "postgres") + const base = yield* S.makeSnapshot(S.pgEntitiesOf([PG.table("legacy", { columns: { id: PG.text }, primaryKey: ["id"] })]), [S.ORIGIN_ID], "postgres") const migrations = yield* Migrate.fromRecord({ "20260101000000_drizzle_init": { kind: "sql", diff --git a/src/pg/postgres.test.ts b/src/pg/postgres.test.ts index 747a61b..c82156a 100644 --- a/src/pg/postgres.test.ts +++ b/src/pg/postgres.test.ts @@ -7,7 +7,7 @@ import { PGlite } from "@electric-sql/pglite" import { afterAll, beforeAll, describe, expect, it } from "@effect/vitest" import { DateTime, Effect } from "effect" import type { CompiledQuery } from "../ch/compile" -import * as CH from "../index" +import * as CH from "../ch/index" import * as PG from "../postgres" // PGlite 0.5 takes the session time zone from the host; the fixtures assume UTC. diff --git a/src/postgres.ts b/src/postgres.ts index 5bed08a..df7826b 100644 --- a/src/postgres.ts +++ b/src/postgres.ts @@ -1,9 +1,9 @@ // @maple-dev/effect-orm/postgres // -// Postgres for the same query builder. Build queries with the root entry -// (`table`, `from`, `param`, `unionAll`, the shared operators); declare -// columns with the types here and compile with the `compile` here, which -// defaults to the Postgres dialect. See docs/postgres.md. +// Everything for Postgres from one import: column types, tables with their +// DDL, Postgres functions, the query builder and a `compile` that defaults to +// the Postgres dialect. ClickHouse has the same shape under `/clickhouse`. See +// docs/postgres.md. import { compileCH, @@ -13,24 +13,53 @@ import { } from "./ch/compile" import { postgresDialect } from "./pg/dialect" -export { postgresDialect } +export * from "./core" export * from "./pg/types" export * from "./pg/functions" +// Portable: renders the same SQL on both dialects. +export { nullIf } from "./ch/functions" +export { postgresDialect } + +// Tables and their DDL. `table` is the only way to declare a table: it carries +// the keys, indexes and foreign keys `effect-orm generate` diffs. +export { + table, + column, + index, + uniqueIndex, + foreignKey, + defaultForeignKeyName, + type ColumnInput, + type ColumnOptions, + type ColumnSpec, + type ColumnsOf, + type DefaultedColumnsOf, + type DdlPredicate, + type ForeignKeySpec, + type IndexOptions, + type IndexSpec, + type PgSchemaTable, + type ReferentialAction, + type TableDdl, + type TableDefinition, + type ExternalTableDefinition, +} from "./schema/pg-define" +export { SchemaDefinitionDefect, type DdlExpr, type DdlKey } from "./schema/define" -/** `compile` from the root entry, for Postgres unless `options.dialect` says otherwise. */ +/** `compile`, for Postgres unless `options.dialect` says otherwise. */ // Typed through `any` and cast: `compileCH` is overloaded (queries and inserts), // and an overloaded type gives an arrow's parameters no contextual type. export const compile = ((query: any, params?: any, options?: any) => compileCH(query, params, { ...options, dialect: options?.dialect ?? postgresDialect })) as typeof compileCH -/** `compileUnsafe` from the root entry, for Postgres. */ +/** `compileUnsafe` for Postgres. */ export const compileUnsafe = ((query: any, params?: any, options?: any) => compileCHUnsafe(query, params, { ...options, dialect: options?.dialect ?? postgresDialect })) as typeof compileCHUnsafe -/** `compileUnion` from the root entry, for Postgres. */ +/** `compileUnion` for Postgres. */ export const compileUnion: typeof compileUnionCH = (union, params, options) => compileUnionCH(union, params, { ...options, dialect: options?.dialect ?? postgresDialect }) -/** `compileUnionUnsafe` from the root entry, for Postgres. */ +/** `compileUnionUnsafe` for Postgres. */ export const compileUnionUnsafe: typeof compileUnionUnsafeCH = (union, params, options) => compileUnionUnsafeCH(union, params, { ...options, dialect: options?.dialect ?? postgresDialect }) diff --git a/src/schema.ts b/src/schema.ts index da4da4d..50ca8d9 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -1,36 +1,13 @@ // @maple-dev/effect-orm/schema // -// Tables and materialized views that carry their DDL, snapshots of them, and -// the offline diff that turns two snapshots into migration ops. ClickHouse -// definitions are the top-level exports; Postgres ones live under `pg` -// (`S.pg.table`). Pure: nothing here reads files or opens a connection. See +// The tooling under migrations: schema entities, snapshots, DDL rendering and +// the offline diff that turns two snapshots into migration ops. Tables are +// declared with `table` from `/clickhouse` or `/postgres`; this entry reads +// them. Pure: nothing here reads files or opens a connection. See // docs/migrations.md. -export * as pg from "./schema/pg-define" - -export { - column, - defineTable, - engine, - index, - materializedView, - ttlAfterDays, - SchemaDefinitionDefect, - type ColumnInput, - type ColumnOptions, - type ColumnSpec, - type ColumnsOf, - type ComputedColumnsOf, - type DefaultedColumnsOf, - type DdlExpr, - type DdlKey, - type IndexSpec, - type MaterializedView, - type MisfitColumns, - type SchemaTable, - type TableDdl, - type TableDefinition, -} from "./schema/define" +export { type SchemaTable, type MaterializedView } from "./schema/define" +export { type PgSchemaTable } from "./schema/pg-define" export { ClickHouseSnapshot, ColumnDefault, diff --git a/src/schema/define.ts b/src/schema/define.ts index e49162e..d59f91d 100644 --- a/src/schema/define.ts +++ b/src/schema/define.ts @@ -1,6 +1,6 @@ // Schema definitions: tables and materialized views that carry their DDL. // -// `defineTable` returns a value that IS a `Table`, so every query API accepts it +// `table` (`defineTable` here) returns a value that IS a `Table`, so every query API accepts it // unchanged; the DDL rides beside it on `ddl`, already normalized into // entities. Expressions (keys, TTL, defaults, index and view bodies) are // written with the query DSL and rendered once, here, with the ClickHouse @@ -14,7 +14,7 @@ import { clickhouseDialect, withDialect } from "../ch/dialect" import type { Expr } from "../ch/expr" import { encodeColumnLiteral } from "../ch/literal" import { createColumnAccessor, type CHQuery, type ColumnAccessor, type NeedsSelect } from "../ch/query" -import type { Table } from "../ch/table" +import { table, type Table } from "../ch/table" import type { CHType, ColumnDefs, InferTS } from "../ch/types" import { compile as compileFragment } from "../sql/sql-fragment" import type { @@ -191,11 +191,50 @@ export interface TableDefinition> { readonly settings?: Readonly> readonly indexes?: ReadonlyArray>> readonly comment?: string - /** As for `table()`: the column carrying row-level tenancy. */ + /** The column carrying row-level tenancy; see docs/tenant-scoping.md. */ readonly tenantColumn?: keyof Columns & string } -/** The DDL a `defineTable` value carries, as entities. */ +/** + * A table this schema does not own: a system table (`system.one`), a table + * function (`numbers(10)`), a subquery, or a table another tool migrates. It + * queries like any table but carries no DDL, so `generate` never touches it, + * and its name is written verbatim as the FROM target. + */ +export interface ExternalTableDefinition> { + readonly external: true + readonly columns: Columns + readonly tenantColumn?: keyof Columns & string +} + +/** + * The query-side `Table` of an external definition. Column options still say + * which columns an insert may leave out or may not write; nothing is rendered. + */ +export const externalTable = ( + name: Name, + definition: { readonly columns: Record; readonly tenantColumn?: string }, + isSpec: (input: unknown) => input is { readonly type: CHType; readonly options: object }, + computedOptions: ReadonlyArray, +): Table => { + const inputs = Object.entries(definition.columns) + const specs = inputs.filter((entry): entry is [string, { readonly type: CHType; readonly options: object }] => + isSpec(entry[1]), + ) + const isComputed = (options: object) => computedOptions.some((key) => (options as Record)[key] !== undefined) + const given = (options: object) => Object.values(options).some((value) => value !== undefined) + return table( + name, + Object.fromEntries(inputs.map(([column, input]) => [column, isSpec(input) ? input.type : input])) as ColumnDefs, + { + ...(definition.tenantColumn !== undefined ? { tenantColumn: definition.tenantColumn } : undefined), + defaults: specs.filter(([, spec]) => given(spec.options) && !isComputed(spec.options)).map(([column]) => column), + computed: specs.filter(([, spec]) => isComputed(spec.options)).map(([column]) => column), + }, + ) +} + +/** The DDL a ClickHouse `table` value carries, as entities. */ export interface TableDdl { readonly table: TableEntity readonly columns: ReadonlyArray @@ -256,13 +295,28 @@ const columnDefault = ( } /** - * A table with its DDL. Usable everywhere a `table()` is; `generate` reads its + * A table with its DDL, published as `table` from `/clickhouse`. The value IS a + * query `Table`, so every query API accepts it; `generate` reads its * `ddl` to produce migrations. */ +export function defineTable>( + name: Name, + definition: ExternalTableDefinition, +): Table, DefaultedColumnsOf, ComputedColumnsOf> export function defineTable>( name: Name, definition: TableDefinition, -): SchemaTable, DefaultedColumnsOf, ComputedColumnsOf> { +): SchemaTable, DefaultedColumnsOf, ComputedColumnsOf> +export function defineTable>( + name: Name, + definition: TableDefinition | ExternalTableDefinition, +): SchemaTable, DefaultedColumnsOf, ComputedColumnsOf> | Table { + if ("external" in definition) { + return externalTable(name, definition, (input): input is ColumnSpec> => isColumnSpec(input as ColumnInput), [ + "materialized", + "alias", + ]) + } assertIdentifier(name, name) const inputs = Object.entries(definition.columns) if (inputs.length === 0) throw new SchemaDefinitionDefect({ object: name, message: "a table needs columns" }) diff --git a/src/schema/entities.ts b/src/schema/entities.ts index e5b087a..dbdd0e8 100644 --- a/src/schema/entities.ts +++ b/src/schema/entities.ts @@ -2,7 +2,7 @@ // and the snapshot envelope both dialects share (Postgres entities live in // `pg-entities.ts`). // -// A `defineTable` value is code; a snapshot is data. Everything downstream of +// A `table` value is code; a snapshot is data. Everything downstream of // the definitions (DDL rendering, diffing, the migrator's drift check) reads // these entities, never the definitions, so a snapshot taken months ago renders // and diffs exactly as it did when it was written. diff --git a/src/schema/external.test.ts b/src/schema/external.test.ts new file mode 100644 index 0000000..c1f466c --- /dev/null +++ b/src/schema/external.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from "vitest" +import { expectTypeOf } from "expect-type" +import * as CH from "../clickhouse" +import * as PG from "../postgres" +import * as S from "../schema" + +describe("external tables", () => { + it("query like any table, with the name written verbatim", () => { + const Numbers = CH.table("numbers(3)", { external: true, columns: { number: CH.uint64 } }) + const compiled = CH.compileUnsafe(CH.from(Numbers).select("number")) + expect(compiled.sql).toContain("FROM numbers(3)") + }) + + it("allow no columns, for a FROM that only anchors constants", () => { + const One = CH.table("system.one", { external: true, columns: {} }) + expect(CH.compileUnsafe(CH.from(One).select(() => ({ n: CH.lit(1) }))).sql).toContain("FROM system.one") + }) + + it("carry no DDL, so generate never sees them", () => { + const Clicks = CH.table("clicks", { external: true, columns: { Url: CH.string } }) + const Views = PG.table("pg_stat_user_tables", { external: true, columns: { relname: PG.text } }) + expect(S.isSchemaObject(Clicks)).toBe(false) + expect(S.isSchemaObject(Views)).toBe(false) + expect("ddl" in Clicks).toBe(false) + }) + + it("keep column options for insert typing", () => { + const Events = CH.table("events", { + external: true, + columns: { + Name: CH.string, + Status: CH.column(CH.uint16, { default: 200 }), + Day: CH.column(CH.string, { materialized: "toString(toDate(now()))" }), + }, + tenantColumn: "Name", + }) + expect(Events.defaults).toEqual(["Status"]) + expect(Events.computed).toEqual(["Day"]) + expect(Events.tenantColumn).toBe("Name") + expectTypeOf>().toEqualTypeOf< + CH.InsertRow<{ readonly Name: CH.CHString; readonly Status: CH.CHUInt16; readonly Day: CH.CHString }, "Status", "Day"> + >() + + const Users = PG.table("users", { + external: true, + columns: { id: PG.column(PG.int8, { identity: "always" }), email: PG.text }, + }) + expect(Users.defaults).toEqual(["id"]) + }) + + it("reject DDL options", () => { + // @ts-expect-error an external table has no engine + CH.table("t", { external: true, columns: { a: CH.string }, engine: CH.engine.memory() }) + // @ts-expect-error an external table has no primary key + PG.table("t", { external: true, columns: { a: PG.text }, primaryKey: ["a"] }) + }) +}) diff --git a/src/schema/pg-define.ts b/src/schema/pg-define.ts index 39fab54..c63f277 100644 --- a/src/schema/pg-define.ts +++ b/src/schema/pg-define.ts @@ -1,7 +1,7 @@ -// Postgres schema definitions: `S.pg.table` and its column, index and foreign +// Postgres schema definitions: `table` from `/postgres` and its column, index and foreign // key helpers. // -// The Postgres counterpart of `defineTable`: the value IS a `Table`, so every +// The Postgres counterpart of the ClickHouse `table`: the value IS a `Table`, so every // query API accepts it, and its DDL rides beside it on `ddl` as Postgres // entities. Expressions are written with the query DSL (or as SQL strings) and // rendered once, here, with the Postgres dialect. A definition that cannot @@ -15,7 +15,7 @@ import { createColumnAccessor, type ColumnAccessor } from "../ch/query" import type { Table } from "../ch/table" import type { CHType, ColumnDefs, InferTS } from "../ch/types" import { postgresDialect } from "../pg/dialect" -import { SchemaDefinitionDefect, type DdlExpr, type DdlKey } from "./define" +import { externalTable, SchemaDefinitionDefect, type DdlExpr, type DdlKey } from "./define" import { canonicalPgType, PG_MAX_IDENTIFIER, @@ -173,7 +173,14 @@ export interface TableDefinition> { | { readonly columns: ReadonlyArray; readonly name?: string } readonly indexes?: ReadonlyArray>> readonly foreignKeys?: ReadonlyArray> - /** As for `table()`: the column carrying row-level tenancy. */ + /** The column carrying row-level tenancy; see docs/tenant-scoping.md. */ + readonly tenantColumn?: keyof Columns & string +} + +/** A table this schema does not own: a view, a catalog table, one another tool migrates. No DDL. */ +export interface ExternalTableDefinition> { + readonly external: true + readonly columns: Columns readonly tenantColumn?: keyof Columns & string } @@ -266,15 +273,26 @@ const columnDefault = ( } /** - * A Postgres table with its DDL. Usable everywhere a `table()` is; `generate` + * A Postgres table with its DDL. The value IS a query `Table`; `generate` * reads its `ddl` when the config's dialect is `postgres`. * * A column is `NOT NULL` unless its type is `PG.nullable(...)`. */ +export function table>( + name: Name, + definition: ExternalTableDefinition, +): Table, DefaultedColumnsOf, never> export function table>( name: Name, definition: TableDefinition, -): PgSchemaTable, DefaultedColumnsOf> { +): PgSchemaTable, DefaultedColumnsOf> +export function table>( + name: Name, + definition: TableDefinition | ExternalTableDefinition, +): PgSchemaTable, DefaultedColumnsOf> | Table { + if ("external" in definition) { + return externalTable(name, definition, (input): input is ColumnSpec> => isColumnSpec(input as ColumnInput), []) + } assertIdentifier(name, name) const inputs = Object.entries(definition.columns) if (inputs.length === 0) throw new SchemaDefinitionDefect({ object: name, message: "a table needs columns" }) diff --git a/src/schema/pg-schema.test.ts b/src/schema/pg-schema.test.ts index e742b0f..1f53b32 100644 --- a/src/schema/pg-schema.test.ts +++ b/src/schema/pg-schema.test.ts @@ -4,27 +4,27 @@ import * as CH from "../ch/index" import * as PG from "../postgres" import * as S from "../schema" -const Dashboards = S.pg.table("dashboards", { +const Dashboards = PG.table("dashboards", { columns: { org_id: PG.text, id: PG.text, - name: S.pg.column(PG.text, { default: "Untitled" }), - tags: S.pg.column(PG.array(PG.text), { default: [] }), - layout: S.pg.column(PG.jsonb(), { default: {} }), + name: PG.column(PG.text, { default: "Untitled" }), + tags: PG.column(PG.array(PG.text), { default: [] }), + layout: PG.column(PG.jsonb(), { default: {} }), widgets: PG.int4, - archived: S.pg.column(PG.bool, { default: false }), - created_at: S.pg.column(PG.timestamptz, { defaultExpr: "now()" }), + archived: PG.column(PG.bool, { default: false }), + created_at: PG.column(PG.timestamptz, { defaultExpr: "now()" }), archived_at: PG.nullable(PG.timestamptz), }, primaryKey: { columns: ["org_id", "id"], name: "dashboards_org_id_id_pk" }, indexes: [ - S.pg.index("dashboards_org_idx", ["org_id"]), - S.pg.index("dashboards_live_idx", ["org_id", "created_at"], { where: ($) => $.archived_at.isNull() }), + PG.index("dashboards_org_idx", ["org_id"]), + PG.index("dashboards_live_idx", ["org_id", "created_at"], { where: ($) => $.archived_at.isNull() }), ], tenantColumn: "org_id", }) -const Shares = S.pg.table("dashboard_shares", { +const Shares = PG.table("dashboard_shares", { columns: { org_id: PG.text, id: PG.text, @@ -35,12 +35,12 @@ const Shares = S.pg.table("dashboard_shares", { }, primaryKey: ["org_id", "id"], indexes: [ - S.pg.uniqueIndex("dashboard_shares_live_unq", ($) => [$.org_id, $.dashboard_id, CH.coalesce($.widget_id, CH.lit(""))], { + PG.uniqueIndex("dashboard_shares_live_unq", ($) => [$.org_id, $.dashboard_id, CH.coalesce($.widget_id, CH.lit(""))], { where: "revoked_at is null", }), ], foreignKeys: [ - S.pg.foreignKey({ + PG.foreignKey({ columns: ["org_id", "dashboard_id"], references: Dashboards, foreignColumns: ["org_id", "id"], @@ -50,7 +50,7 @@ const Shares = S.pg.table("dashboard_shares", { ], }) -describe("S.pg.table", () => { +describe("PG.table", () => { it("is a Table the query builder accepts, with defaults optional on insert", () => { const { sql } = PG.compileUnsafe(CH.from(Dashboards).select("name").where(($) => [$.org_id.eq("o")]), {}) expect(sql).toContain('FROM "dashboards"') @@ -107,40 +107,40 @@ describe("S.pg.table", () => { }) it("names a foreign key as drizzle-kit does when no name is given", () => { - const Checks = S.pg.table("checks", { + const Checks = PG.table("checks", { columns: { id: PG.text, target_id: PG.text }, - foreignKeys: [S.pg.foreignKey({ columns: ["target_id"], references: "targets", foreignColumns: ["id"] })], + foreignKeys: [PG.foreignKey({ columns: ["target_id"], references: "targets", foreignColumns: ["id"] })], }) expect(Checks.ddl.foreignKeys[0]?.name).toBe("checks_target_id_targets_id_fk") }) it("shortens a default foreign key name past 63 characters with drizzle-kit's hash", () => { - const long = S.pg.table("organization_membership_invitations", { + const long = PG.table("organization_membership_invitations", { columns: { organization_id: PG.text, invited_by_user_id: PG.text }, foreignKeys: [ - S.pg.foreignKey({ columns: ["organization_id", "invited_by_user_id"], references: "organization_members", foreignColumns: ["organization_id", "user_id"] }), + PG.foreignKey({ columns: ["organization_id", "invited_by_user_id"], references: "organization_members", foreignColumns: ["organization_id", "user_id"] }), ], }) const name = long.ddl.foreignKeys[0]!.name expect(name).toMatch(/^organization_membership_invitations_[0-9A-Za-z]{12}_fk$/) expect(name.length).toBeLessThanOrEqual(63) // Deterministic, so a snapshot and the next generate agree. - expect(S.pg.defaultForeignKeyName("organization_membership_invitations", ["organization_id", "invited_by_user_id"], "organization_members", ["organization_id", "user_id"])).toBe(name) - expect(S.pg.defaultForeignKeyName("t".repeat(60), ["a"], "u", ["b"])).toMatch(/^[0-9A-Za-z]{12}_fk$/) + expect(PG.defaultForeignKeyName("organization_membership_invitations", ["organization_id", "invited_by_user_id"], "organization_members", ["organization_id", "user_id"])).toBe(name) + expect(PG.defaultForeignKeyName("t".repeat(60), ["a"], "u", ["b"])).toMatch(/^[0-9A-Za-z]{12}_fk$/) }) it("rejects definitions Postgres would not take as written", () => { - expect(() => S.pg.table("t", { columns: { a: PG.nullable(PG.text) }, primaryKey: ["a"] })).toThrow(/cannot be nullable/) - expect(() => S.pg.table("t".repeat(64), { columns: { a: PG.text } })).toThrow(/longer than 63/) + expect(() => PG.table("t", { columns: { a: PG.nullable(PG.text) }, primaryKey: ["a"] })).toThrow(/cannot be nullable/) + expect(() => PG.table("t".repeat(64), { columns: { a: PG.text } })).toThrow(/longer than 63/) expect(() => - S.pg.table("t", { columns: { a: PG.text }, foreignKeys: [S.pg.foreignKey({ columns: ["a"], references: "u", foreignColumns: ["x", "y"] })] }), + PG.table("t", { columns: { a: PG.text }, foreignKeys: [PG.foreignKey({ columns: ["a"], references: "u", foreignColumns: ["x", "y"] })] }), ).toThrow(/same, non-zero, length/) }) it("validates a schema as a whole", () => { - const Other = S.pg.table("other", { + const Other = PG.table("other", { columns: { a: PG.text }, - indexes: [S.pg.index("dashboards_org_idx", ["a"])], + indexes: [PG.index("dashboards_org_idx", ["a"])], }) expect(() => S.pgEntitiesOf([Dashboards, Other])).toThrow(/one namespace/) expect(() => S.pgEntitiesOf([Shares])).toThrow(/not a table in this schema/) @@ -162,19 +162,19 @@ describe("canonicalPgType", () => { describe("diffPgSchemas", () => { const v1 = S.pgEntitiesOf([Dashboards]) const v2 = S.pgEntitiesOf([ - S.pg.table("dashboards", { + PG.table("dashboards", { columns: { org_id: PG.text, id: PG.text, - name: S.pg.column(PG.text, { default: "New dashboard" }), + name: PG.column(PG.text, { default: "New dashboard" }), widgets: PG.int8, archived: PG.bool, - created_at: S.pg.column(PG.timestamptz, { defaultExpr: "now()" }), + created_at: PG.column(PG.timestamptz, { defaultExpr: "now()" }), archived_at: PG.nullable(PG.timestamptz), owner: PG.nullable(PG.text), }, primaryKey: ["org_id", "id"], - indexes: [S.pg.index("dashboards_org_idx", ["org_id", "owner"])], + indexes: [PG.index("dashboards_org_idx", ["org_id", "owner"])], }), ]) diff --git a/src/schema/schema.test.ts b/src/schema/schema.test.ts index 7953b03..9cd4dff 100644 --- a/src/schema/schema.test.ts +++ b/src/schema/schema.test.ts @@ -1,32 +1,32 @@ import { Effect } from "effect" import { describe, expect, it } from "vitest" -import * as CH from "../ch/index" +import * as CH from "../clickhouse" import * as S from "../schema" -const Spans = S.defineTable("spans", { +const Spans = CH.table("spans", { columns: { OrgId: CH.custom("LowCardinality(String)", CH.string.schema), - Timestamp: S.column(CH.dateTime64, { codec: "Delta, ZSTD(1)" }), + Timestamp: CH.column(CH.dateTime64, { codec: "Delta, ZSTD(1)" }), ServiceName: CH.string, - Duration: S.column(CH.uint64, { default: 0 }), - Day: S.column(CH.string, { materialized: ($) => CH.formatDateTime($.Timestamp, "%F") }), + Duration: CH.column(CH.uint64, { default: 0 }), + Day: CH.column(CH.string, { materialized: ($) => CH.formatDateTime($.Timestamp, "%F") }), }, - engine: S.engine.mergeTree(), + engine: CH.engine.mergeTree(), orderBy: ["OrgId", "ServiceName", "Timestamp"], partitionBy: "toDate(Timestamp)", - ttl: S.ttlAfterDays("toDate(Timestamp)", 30), + ttl: CH.ttlAfterDays("toDate(Timestamp)", 30), settings: { index_granularity: 8192 }, - indexes: [S.index("idx_duration", ($) => $.Duration, "minmax")], + indexes: [CH.index("idx_duration", ($) => $.Duration, "minmax")], tenantColumn: "OrgId", }) -const ServiceCounts = S.defineTable("service_counts", { +const ServiceCounts = CH.table("service_counts", { columns: { OrgId: CH.string, ServiceName: CH.string, Spans: CH.uint64 }, - engine: S.engine.summingMergeTree(), + engine: CH.engine.summingMergeTree(), orderBy: ["OrgId", "ServiceName"], }) -const ServiceCountsMv = S.materializedView("service_counts_mv", { +const ServiceCountsMv = CH.materializedView("service_counts_mv", { to: ServiceCounts, as: CH.from(Spans) .select(($) => ({ OrgId: $.OrgId, ServiceName: $.ServiceName, Spans: CH.count() })) @@ -75,13 +75,13 @@ describe("defineTable", () => { it("rejects a MergeTree without a sorting key", () => { expect(() => - S.defineTable("bad", { columns: { a: CH.string }, engine: S.engine.mergeTree() }), + CH.table("bad", { columns: { a: CH.string }, engine: CH.engine.mergeTree() }), ).toThrow(/needs orderBy/) }) it("rejects a non-identifier name", () => { expect(() => - S.defineTable("bad name", { columns: { a: CH.string }, engine: S.engine.null() }), + CH.table("bad name", { columns: { a: CH.string }, engine: CH.engine.null() }), ).toThrow(/plain identifier/) }) }) @@ -98,7 +98,7 @@ describe("materializedView", () => { it("rejects at the type level an output column the target lacks", () => { const misfit = () => // @ts-expect-error `Nope` is not a column of service_counts - S.materializedView("bad_mv", { + CH.materializedView("bad_mv", { to: ServiceCounts, as: CH.from(Spans).select(($) => ({ OrgId: $.OrgId, Nope: $.ServiceName })), }) @@ -134,12 +134,12 @@ describe("diffSchemas", () => { }) it("adds a column after its neighbour and recreates a changed view", () => { - const Counts2 = S.defineTable("service_counts", { - columns: { OrgId: CH.string, Env: S.column(CH.string, { default: "" }), ServiceName: CH.string, Spans: CH.uint64 }, - engine: S.engine.summingMergeTree(), + const Counts2 = CH.table("service_counts", { + columns: { OrgId: CH.string, Env: CH.column(CH.string, { default: "" }), ServiceName: CH.string, Spans: CH.uint64 }, + engine: CH.engine.summingMergeTree(), orderBy: ["OrgId", "ServiceName"], }) - const Mv2 = S.materializedView("service_counts_mv", { + const Mv2 = CH.materializedView("service_counts_mv", { to: Counts2, as: CH.from(Spans) .select(($) => ({ OrgId: $.OrgId, Env: CH.lit(""), ServiceName: $.ServiceName, Spans: CH.count() })) @@ -165,9 +165,9 @@ describe("diffSchemas", () => { }) it("reports changes ALTER cannot make", () => { - const Resorted = S.defineTable("service_counts", { + const Resorted = CH.table("service_counts", { columns: { OrgId: CH.string, ServiceName: CH.string, Spans: CH.uint32 }, - engine: S.engine.summingMergeTree(), + engine: CH.engine.summingMergeTree(), orderBy: ["ServiceName", "OrgId"], }) const { unsupported } = S.diffSchemas(S.entitiesOf([ServiceCounts]), S.entitiesOf([Resorted])) @@ -178,9 +178,9 @@ describe("diffSchemas", () => { }) it("modifies TTL without materializing it", () => { - const Shorter = S.defineTable("service_counts", { + const Shorter = CH.table("service_counts", { columns: { OrgId: CH.string, ServiceName: CH.string, Spans: CH.uint64 }, - engine: S.engine.summingMergeTree(), + engine: CH.engine.summingMergeTree(), orderBy: ["OrgId", "ServiceName"], ttl: "now() + toIntervalDay(1)", }) @@ -192,7 +192,7 @@ describe("diffSchemas", () => { it("ignores settings that only changed key order", () => { const make = (settings: Record) => - S.defineTable("ordered", { columns: { a: CH.string }, engine: S.engine.mergeTree(), orderBy: ["a"], settings }) + CH.table("ordered", { columns: { a: CH.string }, engine: CH.engine.mergeTree(), orderBy: ["a"], settings }) const before = S.entitiesOf([make({ index_granularity: 8192, merge_with_ttl_timeout: 3600 })]) const after = S.entitiesOf([make({ merge_with_ttl_timeout: 3600, index_granularity: 8192 })]) expect(S.diffSchemas(before, after).ops).toEqual([]) diff --git a/src/types.ts b/src/types.ts deleted file mode 100644 index 9a97b91..0000000 --- a/src/types.ts +++ /dev/null @@ -1,57 +0,0 @@ -// Column types — the `/types` subpath. -// -// An explicit list rather than a re-export of the implementation module, which -// is what the other three entry points do. Pointing straight at `./ch/types` -// also published `CHDateTimeUtc`, `chDateTimeLiteral` and `chDateTimeToIso`, -// which are how DateTime columns encode internally, not a consumer's tools. -export { - type CHArray, - type CHBool, - type CHDateTime, - type CHDateTime64, - type CHDateTime64String, - type CHDateTimeString, - type CHFloat64, - type CHInt32, - type CHInt64, - type CHMap, - type CHNullable, - type CHString, - type CHStringLike, - type CHType, - type CHUInt8, - type CHUInt16, - type CHUInt32, - type CHUInt64, - type ColumnDefs, - type InferEncoded, - type InferTS, - type NullableColumnDefs, - type OutputToColumnDefs, - aggregateState, - array, - bool, - /** - * The codec every numeric column type is built from: a finite number, or the - * decimal string a backend sends when it quotes 64-bit integers. Exported - * because a `custom()` type of your own almost always wants it. - */ - CHNumber, - brand, - custom, - dateTime, - dateTime64, - dateTime64String, - dateTimeString, - float64, - int32, - int64, - map, - nullable, - string, - uint8, - uint16, - uint32, - uint64, - untyped, -} from "./ch/types" diff --git a/tests/clickhouse-support.ts b/tests/clickhouse-support.ts index efa0cad..3eb5aee 100644 --- a/tests/clickhouse-support.ts +++ b/tests/clickhouse-support.ts @@ -1,7 +1,7 @@ import assert from "node:assert/strict" import { Effect, Schema } from "effect" import { HttpClient, HttpClientRequest } from "effect/http" -import type * as CH from "@maple-dev/effect-orm" +import type * as CH from "@maple-dev/effect-orm/clickhouse" // Opt in with EFFECT_ORM_CLICKHOUSE_URL, plus _USER and _PASSWORD if needed. // All fixtures are SELECTs/CTEs; this suite creates no tables and writes no data. diff --git a/tests/core-cases.ts b/tests/core-cases.ts index c060554..3388781 100644 --- a/tests/core-cases.ts +++ b/tests/core-cases.ts @@ -6,15 +6,14 @@ // holds on ClickHouse must hold on Postgres too, or the case says why not // (`expected` per target, or `rejects` for a clause a dialect refuses). import { DateTime } from "effect" -import * as CH from "@maple-dev/effect-orm" -import * as T from "@maple-dev/effect-orm/types" +import * as CH from "@maple-dev/effect-orm/clickhouse" import * as PG from "@maple-dev/effect-orm/postgres" export type DialectName = "clickhouse" | "postgres" /** ClickHouse runs twice: `join_use_nulls=0` fills a missing join row with defaults. */ export type Target = "clickhouse" | "clickhouse-join-nulls" | "postgres" -type Col = T.CHType +type Col = CH.CHType type Orders = { readonly OrgId: Col readonly Id: Col @@ -132,22 +131,22 @@ const withFixtures = (dialect: DialectName) => export const clickhouseContext: CoreContext = { dialect: "clickhouse", - orders: CH.table( - "orders", - { - OrgId: T.string, - Id: T.uint32, - Customer: T.string, - Amount: T.int64, - Status: T.string, - Note: T.nullable(T.string), - Created: T.dateTime64, + orders: CH.table("orders", { + external: true, + tenantColumn: "OrgId", + columns: { + OrgId: CH.string, + Id: CH.uint32, + Customer: CH.string, + Amount: CH.int64, + Status: CH.string, + Note: CH.nullable(CH.string), + Created: CH.dateTime64, }, - { tenantColumn: "OrgId" }, - ), - customers: CH.table("customers", { OrgId: T.string, Name: T.string, Tier: T.string }, { tenantColumn: "OrgId" }), + }), + customers: CH.table("customers", { external: true, columns: { OrgId: CH.string, Name: CH.string, Tier: CH.string }, tenantColumn: "OrgId" }), from: withFixtures("clickhouse"), - types: { text: T.string, int: T.int64 }, + types: { text: CH.string, int: CH.int64 }, fn: { count: () => CH.count(), countIf: (condition) => CH.countIf(condition), @@ -162,9 +161,10 @@ export const clickhouseContext: CoreContext = { export const postgresContext: CoreContext = { dialect: "postgres", - orders: CH.table( - "orders", - { + orders: PG.table("orders", { + external: true, + tenantColumn: "OrgId", + columns: { OrgId: PG.text, Id: PG.int4, Customer: PG.text, @@ -173,9 +173,8 @@ export const postgresContext: CoreContext = { Note: PG.nullable(PG.text), Created: PG.timestamptz, }, - { tenantColumn: "OrgId" }, - ), - customers: CH.table("customers", { OrgId: PG.text, Name: PG.text, Tier: PG.text }, { tenantColumn: "OrgId" }), + }), + customers: PG.table("customers", { external: true, columns: { OrgId: PG.text, Name: PG.text, Tier: PG.text }, tenantColumn: "OrgId" }), from: withFixtures("postgres"), types: { text: PG.text, int: PG.int8 }, fn: { @@ -528,7 +527,7 @@ export const coreCases: readonly CoreCase[] = [ id: "cte", covers: q("withCTE"), build: (ctx) => { - const paid = CH.table("paid", { Customer: ctx.types.text, Amount: ctx.types.int }) + const paid = CH.table("paid", { external: true, columns: { Customer: ctx.types.text, Amount: ctx.types.int } }) return ctx.compile( ctx .from(paid) diff --git a/tests/database.clickhouse.test.ts b/tests/database.clickhouse.test.ts index 43b6fd7..c43317c 100644 --- a/tests/database.clickhouse.test.ts +++ b/tests/database.clickhouse.test.ts @@ -4,7 +4,7 @@ import { ClickhouseClient } from "@effect/sql-clickhouse" import { DateTime, Effect, Exit } from "effect" import { describe, expect, it } from "vitest" -import * as CH from "@maple-dev/effect-orm" +import * as CH from "@maple-dev/effect-orm/clickhouse" import * as Db from "@maple-dev/effect-orm/database" import { endpoint } from "./clickhouse-support" @@ -39,7 +39,7 @@ describe("database", () => { Effect.gen(function* () { yield* db.execute(Db.sql`CREATE TABLE events (Id UInt32, Name String) ENGINE = MergeTree ORDER BY Id`) yield* db.execute(Db.sql`INSERT INTO events VALUES (${1}, ${"a"}), (${2}, ${"it's"})`) - const Events = CH.table("events", { Id: CH.uint32, Name: CH.string }) + const Events = CH.table("events", { external: true, columns: { Id: CH.uint32, Name: CH.string } }) return yield* db.run(CH.from(Events).select("Id", "Name").orderBy(["Id", "asc"])) }), ), @@ -65,18 +65,18 @@ describe("database", () => { Day String MATERIALIZED toString(toDate(At)) ) ENGINE = MergeTree ORDER BY (OrgId, Id)`, ) - const Events = CH.table( - "events", - { + const Events = CH.table("events", { + external: true, + tenantColumn: "OrgId", + columns: { OrgId: CH.string, - Id: CH.uint64, + Id: CH.column(CH.uint64, { default: 42 }), At: CH.dateTime64, Attrs: CH.map(CH.string, CH.string), Note: CH.nullable(CH.string), Tags: CH.array(CH.string), }, - { tenantColumn: "OrgId", defaults: ["Id"] }, - ) + }) const inserted = yield* db.run( CH.insertInto(Events).values([ { OrgId: CH.param.string("org"), At: new Date("2026-01-02T03:04:05.678Z"), Attrs: { a: "it's; x" }, Tags: ["t"], Note: null }, @@ -105,8 +105,8 @@ describe("database", () => { Effect.gen(function* () { yield* db.execute(Db.sql`CREATE TABLE spans (OrgId String, Name String, Ms UInt64) ENGINE = MergeTree ORDER BY OrgId`) yield* db.execute(Db.sql`CREATE TABLE daily (OrgId String, Name String, Total UInt64) ENGINE = MergeTree ORDER BY OrgId`) - const Spans = CH.table("spans", { OrgId: CH.string, Name: CH.string, Ms: CH.uint64 }, { tenantColumn: "OrgId" }) - const Daily = CH.table("daily", { OrgId: CH.string, Name: CH.string, Total: CH.uint64 }, { tenantColumn: "OrgId" }) + const Spans = CH.table("spans", { external: true, columns: { OrgId: CH.string, Name: CH.string, Ms: CH.uint64 }, tenantColumn: "OrgId" }) + const Daily = CH.table("daily", { external: true, columns: { OrgId: CH.string, Name: CH.string, Total: CH.uint64 }, tenantColumn: "OrgId" }) yield* db.run( CH.insertInto(Spans) .values([ @@ -139,7 +139,7 @@ describe("database", () => { withDatabase((db) => Effect.gen(function* () { yield* db.execute(Db.sql`CREATE TABLE jobs (OrgId String, Id UInt32, State String) ENGINE = MergeTree ORDER BY (OrgId, Id)`) - const Jobs = CH.table("jobs", { OrgId: CH.string, Id: CH.uint32, State: CH.string }, { tenantColumn: "OrgId" }) + const Jobs = CH.table("jobs", { external: true, columns: { OrgId: CH.string, Id: CH.uint32, State: CH.string }, tenantColumn: "OrgId" }) yield* db.run( CH.insertInto(Jobs).values([ { OrgId: "o", Id: 1, State: "queued" }, @@ -170,7 +170,7 @@ describe("database", () => { withDatabase((db) => Effect.gen(function* () { yield* db.execute(Db.sql`CREATE TABLE ev (OrgId String, Id UInt32, Note Nullable(String)) ENGINE = MergeTree ORDER BY (OrgId, Id)`) - const Ev = CH.table("ev", { OrgId: CH.string, Id: CH.uint32, Note: CH.nullable(CH.string) }) + const Ev = CH.table("ev", { external: true, columns: { OrgId: CH.string, Id: CH.uint32, Note: CH.nullable(CH.string) } }) yield* db.run( CH.insertInto(Ev).values([ { OrgId: "o", Id: 1, Note: null }, diff --git a/tests/deep-builder.clickhouse.test.ts b/tests/deep-builder.clickhouse.test.ts index 5d60961..0049695 100644 --- a/tests/deep-builder.clickhouse.test.ts +++ b/tests/deep-builder.clickhouse.test.ts @@ -1,14 +1,13 @@ import { Effect } from "effect" import { FetchHttpClient } from "effect/http" import { describe, expect, it } from "@effect/vitest" -import * as CH from "@maple-dev/effect-orm" -import * as T from "@maple-dev/effect-orm/types" +import * as CH from "@maple-dev/effect-orm/clickhouse" import { endpoint, execute } from "./clickhouse-support" it.layer(FetchHttpClient.layer)("builder identity regressions", (it) => { describe.skipIf(!endpoint)("live", () => { it.effect("SELECT aliases cannot replace the source tenant filter", () => Effect.gen(function* () { - const events = CH.table("(SELECT arrayJoin(['a', 'b']) AS OrgId)", { OrgId: T.string }, { tenantColumn: "OrgId" }) + const events = CH.table("(SELECT arrayJoin(['a', 'b']) AS OrgId)", { external: true, columns: { OrgId: CH.string }, tenantColumn: "OrgId" }) for (const alias of [undefined, "e"]) { const compiled = CH.compileUnsafe(CH.from(events, alias) .select(() => ({ OrgId: CH.lit("a"), total: CH.count() })) @@ -20,7 +19,10 @@ it.layer(FetchHttpClient.layer)("builder identity regressions", (it) => { it.effect("preserves DateTime and map literals through FROM and JOIN sources", () => Effect.gen(function* () { const events = CH.table("(SELECT toDateTime('2026-01-02 00:00:00', 'UTC') AS ts, map('a', 'b') AS attrs)", { - ts: T.dateTime, attrs: T.map(T.string, T.string), + external: true, + columns: { + ts: CH.dateTime, attrs: CH.map(CH.string, CH.string), + }, }) const inner = CH.from(events).select("ts", "attrs") const compiled = CH.compileUnsafe(CH.fromQuery(inner, "q") @@ -31,7 +33,7 @@ it.layer(FetchHttpClient.layer)("builder identity regressions", (it) => { })) it.effect("decodes arithmetic overflow as NaN", () => Effect.gen(function* () { - const one = CH.table("system.one", {}) + const one = CH.table("system.one", { external: true, columns: {} }) const compiled = CH.compileUnsafe(CH.from(one).select(() => ({ added: CH.lit(1e308).add(1e308), subtracted: CH.lit(-1e308).sub(1e308), diff --git a/tests/deep-codecs.clickhouse.test.ts b/tests/deep-codecs.clickhouse.test.ts index ade62d1..2450b44 100644 --- a/tests/deep-codecs.clickhouse.test.ts +++ b/tests/deep-codecs.clickhouse.test.ts @@ -1,11 +1,10 @@ import { DateTime, Effect, Schema } from "effect" import { FetchHttpClient } from "effect/http" import { describe, expect, it } from "@effect/vitest" -import * as CH from "@maple-dev/effect-orm" -import * as T from "@maple-dev/effect-orm/types" +import * as CH from "@maple-dev/effect-orm/clickhouse" import { endpoint, execute } from "./clickhouse-support" -const One = CH.table("system.one", {}) +const One = CH.table("system.one", { external: true, columns: {} }) it.layer(FetchHttpClient.layer)("composed codecs against ClickHouse", (it) => { describe.skipIf(!endpoint)("live", () => { @@ -20,13 +19,13 @@ it.layer(FetchHttpClient.layer)("composed codecs against ClickHouse", (it) => { })) it.effect("decodes a narrowed coalesce fallback", () => Effect.gen(function* () { - const a = CH.rawExpr("CAST(NULL AS Nullable(String))", T.nullable(T.custom("String", Schema.Literal("a")))) + const a = CH.rawExpr("CAST(NULL AS Nullable(String))", CH.nullable(CH.custom("String", Schema.Literal("a")))) const compiled = CH.compileUnsafe(CH.from(One).select(() => ({ result: CH.coalesce(a, CH.lit("b")) })), {}) expect((yield* execute(compiled)).rows).toEqual([{ result: "b" }]) })) it.effect("decodes overflowed aggregates and nonfinite numeric strings", () => Effect.gen(function* () { - const Numbers = CH.table("numbers(2)", {}) + const Numbers = CH.table("numbers(2)", { external: true, columns: {} }) const compiled = CH.compileUnsafe(CH.from(Numbers).select(() => ({ total: CH.sum(CH.lit(1e308)), filtered: CH.sumIf(CH.lit(1e308), CH.lit(1).eq(1)), @@ -41,8 +40,8 @@ it.layer(FetchHttpClient.layer)("composed codecs against ClickHouse", (it) => { })), { time: "2026-01-02T00:00:00.500+02:00" }) expect((yield* execute(seconds)).rows).toEqual([{ time: "2026-01-01 22:00:00" }]) const result = CH.compileUnsafe(CH.from(One).select(() => ({ - time: CH.if_(CH.lit(0).eq(1), CH.rawExpr("toDateTime('2026-01-01 00:00:00')", T.dateTime), - CH.rawExpr("toDateTime64('2026-01-02 00:00:00.789', 3)", T.dateTime64)), + time: CH.if_(CH.lit(0).eq(1), CH.rawExpr("toDateTime('2026-01-01 00:00:00')", CH.dateTime), + CH.rawExpr("toDateTime64('2026-01-02 00:00:00.789', 3)", CH.dateTime64)), })), {}) const rows = (yield* execute(result)).rows expect(DateTime.toEpochMillis(rows[0]!.time)).toBe(Date.parse("2026-01-02T00:00:00.789Z")) diff --git a/tests/dialect-cases.postgres.ts b/tests/dialect-cases.postgres.ts index fad05f7..f77aecb 100644 --- a/tests/dialect-cases.postgres.ts +++ b/tests/dialect-cases.postgres.ts @@ -3,7 +3,7 @@ // builder behaviour lives in core-cases.ts and runs on every dialect. import { DateTime, Effect, Schema } from "effect" import { expect } from "vitest" -import * as CH from "@maple-dev/effect-orm" +import * as CH from "@maple-dev/effect-orm/clickhouse" import * as PG from "@maple-dev/effect-orm/postgres" import { postgresContext as ctx } from "./core-cases" @@ -22,7 +22,9 @@ const utc = (iso: string) => DateTime.makeUnsafe(iso) // One row of every column type, read back through the declared codecs. The // `*Text` columns are text on the server, so they decode from the string wire // form a driver without type parsers sends. -const typed = CH.table("typed", { +const typed = PG.table("typed", { + external: true, + columns: { Text: PG.text, Uuid: PG.uuid, Bool: PG.bool, @@ -43,6 +45,7 @@ const typed = CH.table("typed", { Missing: PG.nullable(PG.int4), // A brand over int8, read from text: the base codec still parses the string. Branded: PG.brand(PG.int8, Schema.Number.pipe(Schema.brand("Count"))), +}, }) const typedRow = `SELECT 'a''b'::text AS "Text", diff --git a/tests/dialect-cases.ts b/tests/dialect-cases.ts index 30d0a89..406810d 100644 --- a/tests/dialect-cases.ts +++ b/tests/dialect-cases.ts @@ -1,9 +1,8 @@ // Fixtures use only public entry points, resolved through the package's built dist. // Raw SQL supplies deterministic input rows; the operation under test uses the DSL. import { DateTime, Schema } from "effect" -import * as CH from "@maple-dev/effect-orm" +import * as CH from "@maple-dev/effect-orm/clickhouse" import * as F from "@maple-dev/effect-orm/expr" -import * as T from "@maple-dev/effect-orm/types" export interface DialectCase { readonly metadata?: { readonly route: string; readonly tenantScope: CH.TenantScope } @@ -13,8 +12,8 @@ export interface DialectCase { readonly build: () => CH.CompiledQuery readonly expected: readonly unknown[] } -const one = CH.table("system.one", {}) -const n = CH.table("input", { n: T.uint8 }) +const one = CH.table("system.one", { external: true, columns: {} }) +const n = CH.table("input", { external: true, columns: { n: CH.uint8 } }) const numbers = () => CH.from(n).withCTE("input", "SELECT arrayJoin([toUInt8(1), 2, 3]) AS n") const l = CH.lit const scalar = ( @@ -350,7 +349,7 @@ export const dialectCases: readonly DialectCase[] = [ id: "aggregate-state-merge", covers: [...fn("argMaxMerge"), "type:aggregateState"], build: () => { - const states = CH.table("states", { state: T.aggregateState("argMax", "String", "UInt8") }) + const states = CH.table("states", { external: true, columns: { state: CH.aggregateState("argMax", "String", "UInt8") } }) return CH.compileUnsafe( CH.from(states) .withCTE( @@ -400,7 +399,7 @@ export const dialectCases: readonly DialectCase[] = [ hourNumber: CH.toHour(time), seconds: CH.toUnixTimestamp(CH.toDateTime(l(42))), nanos: CH.toUnixTimestamp64Nano( - CH.rawExpr("toDateTime64('1970-01-01 00:00:01.123', 3, 'UTC')", T.dateTime64), + CH.rawExpr("toDateTime64('1970-01-01 00:00:01.123', 3, 'UTC')", CH.dateTime64), ), added: CH.intervalAdd(time, 60), subtracted: CH.intervalSub(time, 60), @@ -511,8 +510,8 @@ export const dialectCases: readonly DialectCase[] = [ id: `direct-${method}`, covers: [`query:${method}`], build: () => { - const a = CH.table("a", { id: T.uint8 }), - b = CH.table("b", { id: T.uint8, name: T.string }) + const a = CH.table("a", { external: true, columns: { id: CH.uint8 } }), + b = CH.table("b", { external: true, columns: { id: CH.uint8, name: CH.string } }) const base = CH.from(a) .withCTE("a", "SELECT toUInt8(1) AS id") .withCTE("b", "SELECT toUInt8(1) AS id, 'match' AS name") @@ -630,9 +629,12 @@ export const dialectCases: readonly DialectCase[] = [ covers: [], build: () => { const rows = CH.table("structured", { - tags: T.array(T.string), - attrs: T.map(T.string, T.string), - enabled: T.bool, + external: true, + columns: { + tags: CH.array(CH.string), + attrs: CH.map(CH.string, CH.string), + enabled: CH.bool, + }, }) return CH.compileUnsafe( CH.from(rows) @@ -642,7 +644,7 @@ export const dialectCases: readonly DialectCase[] = [ ) .select("tags", "attrs", "enabled") .where(($) => [ - $.tags.eq(CH.param.of(T.array(T.string), "tags")), + $.tags.eq(CH.param.of(CH.array(CH.string), "tags")), $.attrs.eq({ key: "value" }), $.enabled.eq(CH.param.bool("enabled")), ]), @@ -676,7 +678,7 @@ export const dialectCases: readonly DialectCase[] = [ array: CH.arrayFilter("x -> x > 9", CH.arrayOf(l(1))), map: CH.mapLiteral(), missing: CH.arrayElement( - CH.rawExpr("CAST([] AS Array(Nullable(UInt8)))", T.array(T.nullable(T.uint8))), + CH.rawExpr("CAST([] AS Array(Nullable(UInt8)))", CH.array(CH.nullable(CH.uint8))), 1, ), }), @@ -699,39 +701,39 @@ export const dialectCases: readonly DialectCase[] = [ // Each descriptor is exercised on a real typed column, including compound wire values. const typeFixtures = [ - ["string", T.string, "'hello'", "hello"], - ["uint8", T.uint8, "toUInt8(255)", 255], - ["uint16", T.uint16, "toUInt16(65535)", 65535], - ["uint32", T.uint32, "toUInt32(4294967295)", 4294967295], - ["uint64", T.uint64, "toUInt64(9007199254740991)", 9007199254740991], - ["int32", T.int32, "toInt32(-2147483648)", -2147483648], - ["int64", T.int64, "toInt64(-9007199254740991)", -9007199254740991], - ["float64", T.float64, "toFloat64(1.25)", 1.25], - ["bool", T.bool, "true", true], + ["string", CH.string, "'hello'", "hello"], + ["uint8", CH.uint8, "toUInt8(255)", 255], + ["uint16", CH.uint16, "toUInt16(65535)", 65535], + ["uint32", CH.uint32, "toUInt32(4294967295)", 4294967295], + ["uint64", CH.uint64, "toUInt64(9007199254740991)", 9007199254740991], + ["int32", CH.int32, "toInt32(-2147483648)", -2147483648], + ["int64", CH.int64, "toInt64(-9007199254740991)", -9007199254740991], + ["float64", CH.float64, "toFloat64(1.25)", 1.25], + ["bool", CH.bool, "true", true], [ "dateTime", - T.dateTime, + CH.dateTime, "toDateTime('2026-01-01 00:00:00', 'UTC')", DateTime.makeUnsafe("2026-01-01T00:00:00Z"), ], [ "dateTime64", - T.dateTime64, + CH.dateTime64, "toDateTime64('2026-01-01 00:00:00.123', 3, 'UTC')", DateTime.makeUnsafe("2026-01-01T00:00:00.123Z"), ], - ["dateTimeString", T.dateTimeString, "toDateTime('2026-01-01 00:00:00', 'UTC')", "2026-01-01 00:00:00"], + ["dateTimeString", CH.dateTimeString, "toDateTime('2026-01-01 00:00:00', 'UTC')", "2026-01-01 00:00:00"], [ "dateTime64String", - T.dateTime64String, + CH.dateTime64String, "toDateTime64('2026-01-01 00:00:00.123', 3, 'UTC')", "2026-01-01 00:00:00.123", ], - ["array", T.array(T.nullable(T.int64)), "[toNullable(toInt64(42)), NULL]", [42, null]], - ["map", T.map(T.string, T.array(T.uint64)), "map('key', [toUInt64(42)])", { key: [42] }], - ["nullable", T.nullable(T.string), "CAST(NULL AS Nullable(String))", null], + ["array", CH.array(CH.nullable(CH.int64)), "[toNullable(toInt64(42)), NULL]", [42, null]], + ["map", CH.map(CH.string, CH.array(CH.uint64)), "map('key', [toUInt64(42)])", { key: [42] }], + ["nullable", CH.nullable(CH.string), "CAST(NULL AS Nullable(String))", null], // A brand over UInt64 keeps the base codec, so a quoted 64-bit value still decodes. - ["brand", T.brand(T.uint64, Schema.Number.pipe(Schema.brand("Count"))), "toUInt64(42)", 42], + ["brand", CH.brand(CH.uint64, Schema.Number.pipe(Schema.brand("Count"))), "toUInt64(42)", 42], ] as const export const typeCases: readonly DialectCase[] = typeFixtures.map(([name, type, sql, expected]) => ({ @@ -739,7 +741,7 @@ export const typeCases: readonly DialectCase[] = typeFixtures.map(([name, type, covers: [`type:${name}`], build: () => CH.compileUnsafe( - CH.from(CH.table("typed", { value: type })) + CH.from(CH.table("typed", { external: true, columns: { value: type } })) .withCTE("typed", `SELECT ${sql} AS value`) .select("value"), {}, diff --git a/tests/dialect-coverage.test.ts b/tests/dialect-coverage.test.ts index 2a85aa4..21b23c8 100644 --- a/tests/dialect-coverage.test.ts +++ b/tests/dialect-coverage.test.ts @@ -1,7 +1,6 @@ import { readFileSync } from "node:fs" import { describe, expect, it } from "vitest" -import * as CH from "@maple-dev/effect-orm" -import * as T from "@maple-dev/effect-orm/types" +import * as CH from "@maple-dev/effect-orm/clickhouse" import * as PG from "@maple-dev/effect-orm/postgres" import { coreCases, coreSkips } from "./core-cases" import { dialectCases, typeCases } from "./dialect-cases" @@ -22,8 +21,32 @@ const methods = (object: CH.CHQuery | CH.CHUnionQuery, prefi Object.entries(object) .filter(([, value]) => typeof value === "function") .map(([name]) => `${prefix}:${name}`) -const one = CH.from(CH.table("system.one", {})).select(() => ({ n: CH.lit(1) })) -const types = Object.keys(T).map((name) => `type:${name}`) +const one = CH.from(CH.table("system.one", { external: true, columns: {} })).select(() => ({ n: CH.lit(1) })) +// The column type constructors on `/clickhouse`, which also carries functions and the builder. +const typeConstructors = [ + "CHNumber", + "aggregateState", + "array", + "bool", + "brand", + "custom", + "dateTime", + "dateTime64", + "dateTime64String", + "dateTimeString", + "float64", + "int32", + "int64", + "map", + "nullable", + "string", + "uint8", + "uint16", + "uint32", + "uint64", + "untyped", +] as const satisfies ReadonlyArray +const types = typeConstructors.map((name) => `type:${name}`) const exemptions = { "type:CHNumber": "Wire codec, exercised by all numeric descriptor fixtures in both quote64 modes.", @@ -133,15 +156,31 @@ describe("core coverage manifest", () => { }) }) -// Every runtime export of the ./postgres entry. +// Every runtime export of the ./postgres entry that is its own. The shared +// builder it re-exports is the same value on /clickhouse, covered above; +// `brand` and `custom` are shared too, but are Postgres column types here. +const sharedPgTypes = new Set(["brand", "custom"]) export const postgresInventory = Object.keys(PG) + .filter( + (name) => + sharedPgTypes.has(name) || (PG as Record)[name] !== (CH as Record)[name], + ) .map((name) => `pg:${name}`) .sort() +const ddl = + "DDL definition, not a query: rendered and diffed in src/schema/pg-schema.test.ts and applied to PGlite in src/migrate/pg-migrate.test.ts." + const postgresExemptions = { "pg:PgNumber": "Wire codec behind every numeric type; the types fixture decodes it from number, bigint and string.", "pg:timestampLiteral": "Factory for timestamp literal codecs; its instances (PgTimestampLiteral, dateTimeSeconds) are exercised.", + "pg:table": ddl, + "pg:column": ddl, + "pg:index": ddl, + "pg:uniqueIndex": ddl, + "pg:foreignKey": ddl, + "pg:defaultForeignKeyName": ddl, } describe("postgres coverage manifest", () => { diff --git a/tests/migrate.clickhouse.test.ts b/tests/migrate.clickhouse.test.ts index 4e558c1..30d40fe 100644 --- a/tests/migrate.clickhouse.test.ts +++ b/tests/migrate.clickhouse.test.ts @@ -1,7 +1,7 @@ import { ClickhouseClient } from "@effect/sql-clickhouse" import { Effect, Exit, Layer } from "effect" import { describe, expect, it } from "vitest" -import * as CH from "@maple-dev/effect-orm" +import * as CH from "@maple-dev/effect-orm/clickhouse" import * as Migrate from "@maple-dev/effect-orm/migrate" import * as S from "@maple-dev/effect-orm/schema" import { endpoint } from "./clickhouse-support" @@ -33,25 +33,25 @@ const withDatabase = (body: Effect.Effect $.Name, "bloom_filter(0.01)")], + ttl: CH.ttlAfterDays("toDate(Timestamp)", 30), + indexes: [CH.index("idx_name", ($) => $.Name, "bloom_filter(0.01)")], }) -const Totals = S.defineTable("totals", { +const Totals = CH.table("totals", { columns: { OrgId: CH.string, Name: CH.string, Count: CH.uint64 }, - engine: S.engine.summingMergeTree(), + engine: CH.engine.summingMergeTree(), orderBy: ["OrgId", "Name"], }) -const TotalsMv = S.materializedView("totals_mv", { +const TotalsMv = CH.materializedView("totals_mv", { to: Totals, as: CH.from(Events) .select(($) => ({ OrgId: $.OrgId, Name: $.Name, Count: CH.sum($.Count) })) @@ -100,12 +100,12 @@ describe("migrate", () => { withDatabase( Effect.gen(function* () { const first = yield* generated([], [Events, Totals, TotalsMv], [S.ORIGIN_ID]) - const Totals2 = S.defineTable("totals", { - columns: { OrgId: CH.string, Name: CH.string, Count: CH.uint64, Events: S.column(CH.uint64, { default: 0 }) }, - engine: S.engine.summingMergeTree(), + const Totals2 = CH.table("totals", { + columns: { OrgId: CH.string, Name: CH.string, Count: CH.uint64, Events: CH.column(CH.uint64, { default: 0 }) }, + engine: CH.engine.summingMergeTree(), orderBy: ["OrgId", "Name"], }) - const Mv2 = S.materializedView("totals_mv", { + const Mv2 = CH.materializedView("totals_mv", { to: Totals2, as: CH.from(Events) .select(($) => ({ OrgId: $.OrgId, Name: $.Name, Count: CH.sum($.Count), Events: CH.count() })) diff --git a/tests/package-consumer.mts b/tests/package-consumer.mts index f485f89..67e6fa7 100644 --- a/tests/package-consumer.mts +++ b/tests/package-consumer.mts @@ -2,9 +2,8 @@ // resolve dependencies or source files from Maple's workspace. import assert from "node:assert/strict" import { Effect, Schema } from "effect" -import * as CH from "@maple-dev/effect-orm" +import * as CH from "@maple-dev/effect-orm/clickhouse" import * as F from "@maple-dev/effect-orm/expr" -import * as T from "@maple-dev/effect-orm/types" import * as Bench from "@maple-dev/effect-orm/benchmark" import { makeHttpClient } from "@maple-dev/effect-orm/benchmark/http" import { runCli } from "@maple-dev/effect-orm/benchmark/cli" @@ -14,7 +13,7 @@ import * as Migrate from "@maple-dev/effect-orm/migrate" import * as Db from "@maple-dev/effect-orm/database" import { defineConfig } from "@maple-dev/effect-orm/kit" -const events = CH.table("events", { id: T.uint64, name: T.string }) +const events = CH.table("events", { external: true, columns: { id: CH.uint64, name: CH.string } }) const query = CH.from(events) .select(($) => ({ id: CH.toString($.id), name: F.lower_($.name) })) .where(($) => [$.name.eq(CH.param.string("name"))]) @@ -26,21 +25,21 @@ const typed: readonly { readonly id: string; readonly name: string }[] = rows assert.equal(typed[0]?.id, "18446744073709551615") assert.deepEqual(await Effect.runPromise(compiled.encodeRows(rows)), rows) assert.equal(SQL.compile(SQL.str("O'Reilly")), "'O\\'Reilly'") -assert.equal(T.custom("String", Schema.String).sql, "String") -assert.equal(T.untyped("Tuple(String)").sql, "Tuple(String)") -const length = CH.defineFn<[CH.Expr], number>("length", T.uint64) +assert.equal(CH.custom("String", Schema.String).sql, "String") +assert.equal(CH.untyped("Tuple(String)").sql, "Tuple(String)") +const length = CH.defineFn<[CH.Expr], number>("length", CH.uint64) assert.equal(SQL.compile(length(CH.lit("abc")).toFragment()), "length('abc')") // @ts-expect-error -- a missing param is a type error too; this checks the runtime failure const invalid = Effect.runSync(Effect.exit(CH.compile(query, {}))) assert.equal(invalid._tag, "Failure") -const managed = S.defineTable("managed", { - columns: { id: T.uint64, name: S.column(T.string, { default: "" }) }, - engine: S.engine.mergeTree(), +const managed = CH.table("managed", { + columns: { id: CH.uint64, name: CH.column(CH.string, { default: "" }) }, + engine: CH.engine.mergeTree(), orderBy: ["id"], }) -const counts = S.defineTable("counts", { columns: { name: T.string, n: T.uint64 }, engine: S.engine.summingMergeTree(), orderBy: ["name"] }) -const countsMv = S.materializedView("counts_mv", { +const counts = CH.table("counts", { columns: { name: CH.string, n: CH.uint64 }, engine: CH.engine.summingMergeTree(), orderBy: ["name"] }) +const countsMv = CH.materializedView("counts_mv", { to: counts, as: CH.from(managed).select(($) => ({ name: $.name, n: CH.count() })).groupBy("name"), }) @@ -63,7 +62,7 @@ const checkTypes = () => { const id: number = rows[0]!.id void id // @ts-expect-error a view output column the target table lacks must not typecheck - S.materializedView("bad_mv", { to: counts, as: CH.from(managed).select(($) => ({ missing: $.name })) }) + CH.materializedView("bad_mv", { to: counts, as: CH.from(managed).select(($) => ({ missing: $.name })) }) } void checkTypes console.log("Isolated tarball imports, types, compilation and codecs passed") diff --git a/tests/postgres-support.ts b/tests/postgres-support.ts index 14457f3..d6f56cb 100644 --- a/tests/postgres-support.ts +++ b/tests/postgres-support.ts @@ -1,6 +1,6 @@ import type { PGlite } from "@electric-sql/pglite" import { Effect } from "effect" -import type * as CH from "@maple-dev/effect-orm" +import type * as CH from "@maple-dev/effect-orm/clickhouse" /** Run the compiled SQL with its bound parameters and decode through the query's own codec. */ export const runOn = Effect.fn("runOn")(function* (db: PGlite, compiled: CH.CompiledQuery) { diff --git a/tests/publish-readiness.clickhouse.test.ts b/tests/publish-readiness.clickhouse.test.ts index d20cd55..028df84 100644 --- a/tests/publish-readiness.clickhouse.test.ts +++ b/tests/publish-readiness.clickhouse.test.ts @@ -1,22 +1,17 @@ import { DateTime, Effect } from "effect" import { FetchHttpClient } from "effect/http" import { describe, expect, it } from "@effect/vitest" -import * as CH from "@maple-dev/effect-orm" -import * as T from "@maple-dev/effect-orm/types" +import * as CH from "@maple-dev/effect-orm/clickhouse" import { endpoint, execute } from "./clickhouse-support" -const One = CH.table("system.one", {}) +const One = CH.table("system.one", { external: true, columns: {} }) it.layer(FetchHttpClient.layer)("publishing regressions against ClickHouse", (it) => { describe.skipIf(!endpoint)("live", () => { it.effect("classifies scalar counts across tenants and preserves same-tenant counts", () => Effect.gen(function* () { - const events = CH.table( - "(SELECT arrayJoin(['a', 'b']) AS OrgId)", - { OrgId: T.string }, - { tenantColumn: "OrgId" }, - ) + const events = CH.table("(SELECT arrayJoin(['a', 'b']) AS OrgId)", { external: true, columns: { OrgId: CH.string }, tenantColumn: "OrgId" }) const all = CH.from(events).select(() => ({ total: CH.count() })) const own = all.where(($) => [$.OrgId.eq("a")]) for (const [inner, expectedScope, total] of [ @@ -26,7 +21,7 @@ it.layer(FetchHttpClient.layer)("publishing regressions against ClickHouse", (it const compiled = CH.compileUnsafe( CH.from(events).select(($) => ({ org: $.OrgId, - total: CH.subqueryExpr(inner, T.uint64), + total: CH.subqueryExpr(inner, CH.uint64), })).where(($) => [$.OrgId.eq("a")]), {}, ) @@ -50,8 +45,8 @@ it.layer(FetchHttpClient.layer)("publishing regressions against ClickHouse", (it it.effect("labels a join that exposes another tenant as cross-tenant", () => Effect.gen(function* () { - const a = CH.table("a", { OrgId: T.string, Id: T.uint8 }, { tenantColumn: "OrgId" }) - const b = CH.table("b", { ...a.columns, Secret: T.string }, { tenantColumn: "OrgId" }) + const a = CH.table("a", { external: true, columns: { OrgId: CH.string, Id: CH.uint8 }, tenantColumn: "OrgId" }) + const b = CH.table("b", { external: true, columns: { ...a.columns, Secret: CH.string }, tenantColumn: "OrgId" }) const compiled = CH.compileUnsafe( CH.from(a, "a") .innerJoin(b, "b", (a, b) => a.Id.eq(b.Id)) @@ -83,7 +78,7 @@ it.layer(FetchHttpClient.layer)("publishing regressions against ClickHouse", (it const union = CH.unionAll( CH.from(One).select(() => ({ value: CH.lit("ok") })), CH.from(One).select(() => ({ - value: CH.rawExpr("CAST(NULL AS Nullable(String))", T.nullable(T.string)), + value: CH.rawExpr("CAST(NULL AS Nullable(String))", CH.nullable(CH.string)), })), ) const compiled = CH.compileUnsafe(CH.fromUnion(union, "u").select("value"), {}) @@ -101,8 +96,8 @@ it.layer(FetchHttpClient.layer)("publishing regressions against ClickHouse", (it const derived = CH.fromQuery(a, "a") .leftJoinQuery(b, "b", (a, b) => a.id.eq(b.id)) .select(($) => ({ name: $.b.name })) - const A = CH.table("a", { id: T.uint8 }) - const B = CH.table("b", { id: T.uint8, name: T.string }) + const A = CH.table("a", { external: true, columns: { id: CH.uint8 } }) + const B = CH.table("b", { external: true, columns: { id: CH.uint8, name: CH.string } }) const direct = CH.from(A) .leftJoin(B, "b", (a, b) => a.id.eq(b.id)) .select(($) => ({ name: $.b.name })) @@ -169,7 +164,7 @@ it.layer(FetchHttpClient.layer)("publishing regressions against ClickHouse", (it it.effect("preserves DateTime64 bounds and round trips milliseconds", () => Effect.gen(function* () { - const ticks = CH.table("ticks", { ts: T.dateTime64 }) + const ticks = CH.table("ticks", { external: true, columns: { ts: CH.dateTime64 } }) const instant = DateTime.makeUnsafe("2026-09-07T00:00:00.789Z") const base = CH.from(ticks).withCTE( "ticks", @@ -179,7 +174,7 @@ it.layer(FetchHttpClient.layer)("publishing regressions against ClickHouse", (it instant, new Date("2026-09-07T00:00:00.789Z"), CH.param.dateTime("start"), - CH.param.of(T.dateTime64, "start"), + CH.param.of(CH.dateTime64, "start"), ]) { const compiled = CH.compileUnsafe( base.select(() => ({ count: CH.count() })).where(($) => [$.ts.gte(bound)]), diff --git a/tsdown.config.ts b/tsdown.config.ts index 2277429..e90eb1e 100644 --- a/tsdown.config.ts +++ b/tsdown.config.ts @@ -2,9 +2,8 @@ import { defineConfig } from "tsdown" export default defineConfig({ entry: { - index: "./src/index.ts", + clickhouse: "./src/clickhouse.ts", expr: "./src/expr.ts", - types: "./src/types.ts", sql: "./src/sql/index.ts", postgres: "./src/postgres.ts", schema: "./src/schema.ts",