From 4ee1b68f05b12c6c3e14dbb1bd05ae882d2fbaab Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 4 Oct 2026 16:37:57 +0200 Subject: [PATCH 1/5] Refuse invalid queries the builder used to accept - A second where()/having() ANDs with the first instead of replacing it, so a shared base keeps its tenant filter. - Comparisons refuse null (type error, and a QueryBuilderError at runtime); an empty IN list renders as 1 = 0 / 1 = 1 instead of IN (). - like/notLike/ilike accept a nullable string column. - limit/offset reject negative or fractional literals, and NaN, negative or fractional runtime values, instead of emitting or rounding them. - Join aliases must not repeat or shadow a FROM column (type error), nor repeat the FROM alias; CTE names must be unique (compile-time defect). - A query with no select() cannot be run, compiled, joined, used in FROM, a CTE, EXISTS or INSERT ... SELECT. - unionAll branches must agree on aliases and column types; an empty union is a type error. - inSubquery/notInSubquery require exactly one column of a comparable type. - update().set({}) is a type error; an UPDATE or DELETE without where() or allRows() cannot be compiled or run. Co-Authored-By: Claude Opus 5.5 --- docs/expressions.md | 10 ++- docs/queries.md | 17 ++--- docs/tenant-scoping.md | 4 +- docs/updates-and-deletes.md | 4 +- src/ch/compile.ts | 64 ++++++++++++++++-- src/ch/core-dsl.test.ts | 1 + src/ch/expr.ts | 103 +++++++++++++++++++---------- src/ch/insert.ts | 4 +- src/ch/literal.test.ts | 9 ++- src/ch/publish-readiness.test.ts | 2 +- src/ch/query.ts | 109 ++++++++++++++++++++++++++----- src/ch/soundness.test-d.ts | 98 +++++++++++++++++++++++++++ src/ch/soundness.test.ts | 107 ++++++++++++++++++++++++++++++ src/ch/subquery.ts | 19 ++++-- src/ch/union.ts | 44 +++++++++++-- src/ch/update.test.ts | 2 + src/ch/update.ts | 90 ++++++++++++++++--------- src/database/database.ts | 12 +++- src/schema/define.ts | 4 +- 19 files changed, 576 insertions(+), 127 deletions(-) create mode 100644 src/ch/soundness.test-d.ts create mode 100644 src/ch/soundness.test.ts diff --git a/docs/expressions.md b/docs/expressions.md index bf76c01..e3f3a98 100644 --- a/docs/expressions.md +++ b/docs/expressions.md @@ -19,8 +19,10 @@ $.Timestamp.gte(new Date(...)) // Timestamp >= '2026-01-01 00:00:00' ### Testing for NULL -`.eq(null)` emits `= NULL`; it does not test whether a value is missing. Use `.isNull()` (or -`.isNotNull()` for present values), which write `IS NULL` and work on every dialect: +`= NULL` is never true in SQL, so a comparison does not take `null`: `.eq(null)`, +`.in_(null)` and the like are type errors, and a `null` that arrives at runtime fails +compilation with a `QueryBuilderError`. Use `.isNull()` (or `.isNotNull()` for present +values), which write `IS NULL` and work on every dialect: ```ts title="null-filter.ts" import * as CH from "@maple-dev/effect-orm" @@ -62,7 +64,9 @@ Every `Expr` carries: Each accepts a raw value or another `Expr`. String literals are escaped; booleans emit as `1` / `0`. -`in_` carries a trailing underscore because `in` is a reserved word in JavaScript. +`in_` carries a trailing underscore because `in` is a reserved word in JavaScript. An empty +list is written as the constant it means, `1 = 0` for `in_()` and `1 = 1` for `notIn()`, since +`IN ()` is not SQL. ### String-only diff --git a/docs/queries.md b/docs/queries.md index 6b56b87..1502186 100644 --- a/docs/queries.md +++ b/docs/queries.md @@ -58,13 +58,13 @@ _(Backed by `docs/queries.md > select by column name`.)_ Entries may be `undefined`, which drops them — that is what makes optional filters clean. See [`when` / `whenTrue`](./expressions.md#optional-predicates). -**Calling `where` again replaces the previous callback.** It does not append predicates. -Put the complete filter set in one callback, including tenant and time bounds. Both flat +**Calling `where` again adds conditions**, ANDed with the earlier ones, as in Kysely. A shared +base that filters by tenant keeps that filter however many `.where(...)` calls follow. Both flat conditions and `.and()` preserve [tenant scoping](./tenant-scoping.md); `.or()` does not. +`having` accumulates the same way. -The same replacement rule applies to `select`, `groupBy`, `having`, `orderBy`, `limit`, -`offset`, and `format`. Joins and CTEs accumulate. Immutable does not mean additive: -a second `.where(...)` on a shared base can remove its tenant filter. +`select`, `groupBy`, `orderBy`, `limit`, `offset`, and `format` replace the previous value. Joins +and CTEs accumulate. ## `groupBy` @@ -127,9 +127,10 @@ Postgres wants those keys to lead the ORDER BY. Both ClickHouse and Postgres sup .limit(50).offset(100) ``` -Both take numbers, not `param.*` expressions, and are rounded with `Math.round` before -emission. That is not input validation: reject non-finite, negative, or fractional values at -your request boundary, and enforce an application maximum. Use a stable `orderBy` when paging; +Both take non-negative integers, not `param.*` expressions. A negative or fractional literal is +a type error; a value that arrives at runtime (`NaN`, `-1`, `1.5`) fails compilation with a +`QueryBuilderError` instead of being rounded. Still enforce an application maximum at your +request boundary. Use a stable `orderBy` when paging; [Recipes](./recipes.md#paginate-a-grouped-result) shows where an offset is appropriate. ## `format` diff --git a/docs/tenant-scoping.md b/docs/tenant-scoping.md index f42d57a..9de884c 100644 --- a/docs/tenant-scoping.md +++ b/docs/tenant-scoping.md @@ -180,5 +180,5 @@ inspected. Whatever you pass is taken at face value — which is why it also req `"single-tenant"` proves a structural restriction to one value, not that the requester is allowed to access that value. Resolve tenant IDs from trusted context. Treat `"untenanted"` as acceptable only for tables your application intentionally models as shared; omitting `tenantColumn` from -a real tenant table bypasses that evidence. Repeated `.where()` calls replace the earlier -filter, so assemble tenant and optional predicates in the same callback. +a real tenant table bypasses that evidence. Repeated `.where()` calls AND with the earlier +ones, so a tenant filter on a shared base query survives later filters. diff --git a/docs/updates-and-deletes.md b/docs/updates-and-deletes.md index a1b53c7..0d0d927 100644 --- a/docs/updates-and-deletes.md +++ b/docs/updates-and-deletes.md @@ -40,7 +40,9 @@ names the record type. `where` works as in a query: a list of conditions, AND-joined, with an `undefined` one skipped, so optional filters compose. A write with no `where` would change every row, so: -- compiling an UPDATE or DELETE with no `where()` is a `QueryBuilderDefect`; +- an UPDATE or DELETE with no `where()` or `allRows()` is a type error, and compiling one + that slipped past the types is a `QueryBuilderDefect`; +- calling `where` again ANDs the new conditions with the earlier ones; - a `where()` whose conditions all came out `undefined` (or render to nothing) is a `QueryBuilderError`, because that happens with data (every optional filter absent) and would otherwise widen a filtered write to the whole table; diff --git a/src/ch/compile.ts b/src/ch/compile.ts index cde91af..ae6ee0b 100644 --- a/src/ch/compile.ts +++ b/src/ch/compile.ts @@ -8,7 +8,7 @@ // 4. Assembling into SqlQuery and calling the existing compileQuery() import { custom, dateTime, dateTime64, type CHType, type ColumnDefs } from "./types" -import type { CHQuery, CHQueryState } from "./query" +import type { CHQuery, CHQueryState, NeedsSelect } from "./query" import type { CHUnionQuery } from "./union" import { isInsert, type CHInsert } from "./insert" import { isDelete, isUpdate, type CHDelete, type CHUpdate } from "./update" @@ -96,6 +96,22 @@ const orderByClause = (specs: ReadonlyArray<[string, "asc" | "desc"]>): Array { + if (value == null) return undefined + if (!Number.isSafeInteger(value) || value < 0) { + throw new QueryBuilderError({ + code: "InvalidArguments", + message: `${clause}(${value}): expected a non-negative integer`, + }) + } + return raw(String(value)) +} + /** * A `groupBy` key: the select alias where the dialect resolves aliases there, * otherwise the selected column's position. Postgres reads a bare name in @@ -559,7 +575,7 @@ export function compileCH< Params extends Record = {}, Decoded extends Output = Output, >( - query: CHQuery, + query: CHQuery & NeedsSelect, /** Values for the query's `param.*` markers. Optional when it has none. */ params?: Params, options?: { @@ -607,7 +623,7 @@ export function compileCHUnsafe< Params extends Record = {}, Decoded extends Output = Output, >( - query: CHQuery, + query: CHQuery & NeedsSelect, /** Values for the query's `param.*` markers. Optional when it has none. */ params?: Params, options?: { @@ -698,6 +714,7 @@ function compileInner< // passed `state.columns` (empty for a `fromQuery`/`fromUnion`) and no join // columns at all, so `$.p.ServiceName` and `$.bucket` compiled to correct SQL // with no schema, and the query derived nothing. + assertDistinctNames(state) const $ = makeAccessor(state) // SELECT @@ -889,8 +906,8 @@ function compileInner< .filter((c): c is NonNullable => c != null) .map((c) => c.toFragment()), orderBy: orderByClause(state.orderBySpecs).map(raw), - limit: state.limitValue != null ? raw(String(Math.round(state.limitValue))) : undefined, - offset: state.offsetValue != null ? raw(String(Math.round(state.offsetValue))) : undefined, + limit: rowCount("limit", state.limitValue), + offset: rowCount("offset", state.offsetValue), format: options?.skipFormat ? undefined : formatClause(state.formatValue), } @@ -939,6 +956,39 @@ function compileInner< ) } +/** + * Every source in a query needs its own name. Two joins under one alias, or a + * join named like the FROM source, make every qualified column ambiguous; a + * join named like a FROM column hides that column from `$`; two CTEs under one + * name are refused by both databases. The join cases are type errors too + * (`FreshAlias`); this catches the ones the type cannot see. + */ +function assertDistinctNames(state: CHQueryState): void { + const fromAlias = sourceAlias(state) + const columns = new Set(Object.keys(columnsOf(state))) + const seen = new Set([fromAlias]) + for (const join of state.typedJoins) { + if (seen.has(join.alias)) { + throw new QueryBuilderDefect({ + message: `CHQuery: join alias ${JSON.stringify(join.alias)} is already the name of another source in this query`, + }) + } + if (columns.has(join.alias)) { + throw new QueryBuilderDefect({ + message: `CHQuery: join alias ${JSON.stringify(join.alias)} is also a column of the FROM source; pick another alias`, + }) + } + seen.add(join.alias) + } + const ctes = new Set() + for (const cte of state.ctes) { + if (ctes.has(cte.name)) { + throw new QueryBuilderDefect({ message: `CHQuery: withCTE(${JSON.stringify(cte.name)}) is defined twice` }) + } + ctes.add(cte.name) + } +} + interface TenantSource { readonly column?: string readonly scope: TenantScope @@ -1226,10 +1276,10 @@ function compileUnionInner, Params extends Re sql += `\nORDER BY ${orderByClause(state.outerOrderBySpecs).join(", ")}` } if (state.outerLimitValue != null) { - sql += `\nLIMIT ${Math.round(state.outerLimitValue)}` + sql += `\nLIMIT ${compileSqlFragment(rowCount("limit", state.outerLimitValue)!)}` } if (state.outerOffsetValue != null) { - sql += `\nOFFSET ${Math.round(state.outerOffsetValue)}` + sql += `\nOFFSET ${compileSqlFragment(rowCount("offset", state.outerOffsetValue)!)}` } } diff --git a/src/ch/core-dsl.test.ts b/src/ch/core-dsl.test.ts index 70f4755..9957ae0 100644 --- a/src/ch/core-dsl.test.ts +++ b/src/ch/core-dsl.test.ts @@ -312,6 +312,7 @@ describe("condition combinators", () => { describe("compile edge cases", () => { it("throws QueryBuilderError when no select", () => { const q = CH.from(TestTable).format("JSON") + // @ts-expect-error -- a query with no select() is a type error too expect(() => compileCHUnsafe(q, {})).toThrow() }) diff --git a/src/ch/expr.ts b/src/ch/expr.ts index 0b3420e..a6e4037 100644 --- a/src/ch/expr.ts +++ b/src/ch/expr.ts @@ -12,6 +12,7 @@ import { raw, str, ident, compile, as_ as sqlAs, lazy } from "../sql/sql-fragmen import { activeSqlSyntax } from "../sql/sql-syntax" import { chDateTimeLiteral, CHFloatResult, CHNumber, string as chString, type CHType, type InferTS } from "./types" import { encodeColumnLiteral } from "./literal" +import { QueryBuilderError } from "./errors" import { markTenantColumn, markTenantPredicate, tenantColumnOf, tenantPredicatesOf } from "./tenant" // Core interfaces @@ -59,17 +60,20 @@ export interface Expr { // The widened arms sit in contravariant positions, which TypeScript's // `extends Expr` inference would prefer — the reason `InferOutput` // reads the `_phantom` property instead of structurally inferring T. - eq(other: Comparable> | Expr | Expr>): Condition - neq(other: Comparable> | Expr | Expr>): Condition - gt(other: Comparable> | Expr | Expr>): Condition - gte(other: Comparable> | Expr | Expr>): Condition - lt(other: Comparable> | Expr | Expr>): Condition - lte(other: Comparable> | Expr | Expr>): Condition - - // String operations - like(this: Expr, pattern: string): Condition - notLike(this: Expr, pattern: string): Condition - ilike(this: Expr, pattern: string): Condition + // + // A plain value is never `null`: `x = NULL` is never true in SQL, so it is + // refused here and at compile time. Use `isNull()` / `isNotNull()`. + eq(other: Operand): Condition + neq(other: Operand): Condition + gt(other: Operand): Condition + gte(other: Operand): Condition + lt(other: Operand): Condition + lte(other: Operand): Condition + + // String operations. A `Nullable(String)` matches like a `String`. + like(this: Expr, pattern: string): Condition + notLike(this: Expr, pattern: string): Condition + ilike(this: Expr, pattern: string): Condition // NULL and ranges /** `expr IS NULL`. */ @@ -77,19 +81,14 @@ export interface Expr { /** `expr IS NOT NULL`. */ isNotNull(): Condition /** `expr BETWEEN low AND high`, both ends included. */ - between( - low: Comparable> | Expr | Expr>, - high: Comparable> | Expr | Expr>, - ): Condition + between(low: Operand, high: Operand): Condition /** `expr NOT BETWEEN low AND high`. */ - notBetween( - low: Comparable> | Expr | Expr>, - high: Comparable> | Expr | Expr>, - ): Condition + notBetween(low: Operand, high: Operand): Condition - // IN / NOT IN - in_(...values: Array>>): Condition - notIn(...values: Array>>): Condition + // IN / NOT IN. An empty list is false (`IN`) or true (`NOT IN`), written + // `1 = 0` / `1 = 1`, rather than the `IN ()` no database accepts. + in_(...values: Array>>>): Condition + notIn(...values: Array>>>): Condition // JSON represents non-finite division results as null. Other arithmetic // propagates SQL NULL from either operand. @@ -109,6 +108,15 @@ export interface Expr { mod(this: Expr, n: R | Expr): Expr> } +/** + * What a comparison takes on its right: a value of the column's type (never + * `null`), or an expression of it. + */ +export type Operand = + | Comparable>> + | Expr + | Expr> + /** * What `/` and `%` decode to. A numeric literal divisor of magnitude >= 1 * cannot manufacture `inf`/`nan` from a finite dividend, so `x.div(1_000_000)` @@ -192,6 +200,37 @@ const dateTimeLiteral = (value: DateTime.Utc): string => // Expr implementation +/** + * A plain `null` (or `undefined`) on the right of a comparison. `x = NULL` is + * never true, so this is refused rather than written. A failure, not a defect: + * the value usually comes from data the types said could not be null. + */ +const refusedNull = (value: null | undefined): never => { + throw new QueryBuilderError({ + code: "InvalidArguments", + message: `compared against ${String(value)}, which SQL never matches; use isNull() / isNotNull()`, + }) +} + +/** + * `expr IN (…)` / `expr NOT IN (…)`. An empty list has no SQL spelling, so it + * is written as the constant it means: nothing is in it, everything is not. + */ +const inCond = ( + fragment: SqlFragment, + op: "IN" | "NOT IN", + values: ReadonlyArray<() => SqlFragment>, +): Condition => + makeCond( + lazy(() => + values.length === 0 + ? op === "IN" + ? "1 = 0" + : "1 = 1" + : `${compile(fragment)} ${op} (${values.map((v) => compile(v())).join(", ")})`, + ), + ) + /** Whether a codec accepts `null` — asked, not inferred from its AST, so it * stays right across Effect versions and across `T.custom` schemas. */ const acceptsNull = (schema: Schema.Codec | undefined): boolean => @@ -248,6 +287,7 @@ export function makeExpr( ): Expr { /** An operand: another expression as-is, a plain value through the codec. */ function operand(value: unknown): SqlFragment { + if (value === null || value === undefined) return refusedNull(value) return literal !== undefined && !isExprLike(value) ? literal(value) : toFragment(value) } @@ -275,14 +315,8 @@ export function makeExpr( notLike: (pattern: string) => makeCond(lazy(() => `${compile(fragment)} NOT LIKE ${compile(str(pattern))}`)), ilike: (pattern: string) => makeCond(lazy(() => `${compile(fragment)} ILIKE ${compile(str(pattern))}`)), - in_: (...values) => { - const escaped = () => values.map((v) => compile(operand(v))).join(", ") - return makeCond(lazy(() => `${compile(fragment)} IN (${escaped()})`)) - }, - notIn: (...values) => { - const escaped = () => values.map((v) => compile(operand(v))).join(", ") - return makeCond(lazy(() => `${compile(fragment)} NOT IN (${escaped()})`)) - }, + in_: (...values) => inCond(fragment, "IN", values.map((v) => () => operand(v))), + notIn: (...values) => inCond(fragment, "NOT IN", values.map((v) => () => operand(v))), // NOTE: these do NOT parenthesize their result, so chaining follows SQL // operator precedence rather than call order — `a.sub(b).div(c)` compiles @@ -452,18 +486,15 @@ export function outerRef(name: string): Expr { } export function inList(expr: Expr, values: readonly string[]): Condition { - const escaped = () => values.map((v) => compile(str(v))).join(", ") - return makeCond(lazy(() => `${compile(expr.toFragment())} IN (${escaped()})`)) + return inCond(expr.toFragment(), "IN", values.map((v) => () => str(v))) } export function inExprList(expr: Expr, values: readonly Expr[]): Condition { - const escaped = () => values.map((v) => compile(v.toFragment())).join(", ") - return makeCond(lazy(() => `${compile(expr.toFragment())} IN (${escaped()})`)) + return inCond(expr.toFragment(), "IN", values.map((v) => () => v.toFragment())) } export function notInList(expr: Expr, values: readonly string[]): Condition { - const escaped = () => values.map((v) => compile(str(v))).join(", ") - return makeCond(lazy(() => `${compile(expr.toFragment())} NOT IN (${escaped()})`)) + return inCond(expr.toFragment(), "NOT IN", values.map((v) => () => str(v))) } /** diff --git a/src/ch/insert.ts b/src/ch/insert.ts index e7b9dea..8a9b9a1 100644 --- a/src/ch/insert.ts +++ b/src/ch/insert.ts @@ -14,7 +14,7 @@ // yield* Database.run(insert, { id, orgId }) import type { Comparable, Condition, Expr, Widen } from "./expr" -import type { CHQuery, ColumnAccessor, InferOutput } from "./query" +import type { CHQuery, ColumnAccessor, InferOutput, NeedsSelect } from "./query" import type { Table } from "./table" import type { CHUnionQuery } from "./union" import type { CHType, ColumnDefs, InferTS } from "./types" @@ -198,7 +198,7 @@ export interface CHInsert< * their types. Replaces any `values`. */ select | CHUnionQuery>( - query: Q & SelectFits, Cols, Defaulted, Computed>, + query: Q & SelectFits, Cols, Defaulted, Computed> & NeedsSelect>, ): CHInsert /** diff --git a/src/ch/literal.test.ts b/src/ch/literal.test.ts index 35c78a5..32925fc 100644 --- a/src/ch/literal.test.ts +++ b/src/ch/literal.test.ts @@ -50,8 +50,11 @@ describe("literals encode through the column's type", () => { expect(whereSql(($) => [$.Live.eq(false)])).toBe("events.Live = 0") }) - it("writes null as NULL", () => { - expect(whereSql(($) => [$.Note.eq(null)])).toBe("events.Note = NULL") + it("refuses a comparison against null, which SQL never matches", () => { + // @ts-expect-error -- `null` is not an operand; use isNull() + expect(() => whereSql(($) => [$.Note.eq(null)])).toThrow(/isNull\(\)/) + expect(() => whereSql(($) => [$.Note.in_(null as unknown as string)])).toThrow(/isNull\(\)/) + expect(whereSql(($) => [$.Note.isNull()])).toBe("events.Note IS NULL") }) it("escapes strings", () => { @@ -77,7 +80,7 @@ describe("literals encode through the column's type", () => { it("refuses a value the column cannot hold", () => { expect(() => compareUnchecked("Count", "lots")).toThrow(/column Count.*Expected number/s) - expect(() => compareUnchecked("OrgId", undefined)).toThrow(/column OrgId.*undefined/s) + expect(() => compareUnchecked("OrgId", undefined)).toThrow(/compared against undefined/) }) }) diff --git a/src/ch/publish-readiness.test.ts b/src/ch/publish-readiness.test.ts index 0e2698f..3bc6d5d 100644 --- a/src/ch/publish-readiness.test.ts +++ b/src/ch/publish-readiness.test.ts @@ -156,7 +156,7 @@ describe("publishing regressions", () => { expect(compiled.sql).toMatch(/'c' AS first,\s*'d' AS last/) expect(() => CH.compileUnionUnsafe( - CH.unionAll(a, CH.from(One).select(() => ({ first: CH.lit("c") })) as never), + CH.unionAll(a, CH.from(One).select(() => ({ first: CH.lit("c") })) as unknown as typeof a), {}, ), ).toThrow("same column aliases") diff --git a/src/ch/query.ts b/src/ch/query.ts index e5dc101..6148c12 100644 --- a/src/ch/query.ts +++ b/src/ch/query.ts @@ -26,7 +26,7 @@ import type { ColumnDefs, CHType, InferTS, OutputToColumnDefs, NullableColumnDefs } from "./types" import type { Table } from "./table" -import type { Expr, Condition, ColumnRef } from "./expr" +import type { Expr, Condition, ColumnRef, Widen } from "./expr" import { makeColumnRef } from "./expr" import type { TenantScope } from "./compile" @@ -145,6 +145,53 @@ export interface CHQueryState { }> } +/** + * A join alias, refused at the type level when it is already a join alias or + * a column of the FROM source: `$.` would name two things, and SQL with + * two sources under one name is ambiguous. A clash with the FROM alias itself, + * or with a CTE, is refused when compiling. + */ +export type FreshAlias> = Alias & + (Alias extends (keyof Cols & string) | (keyof Joins & string) ? { readonly aliasAlreadyInUse: Alias } : unknown) + +/** The row a query selects. */ +export type OutputOf = Q extends { readonly _phantom?: { readonly output: infer O } } ? O : never + +/** + * `unknown` once a query has a SELECT list, otherwise a property saying so. + * Intersected with a query argument, it turns "this query selects nothing" into + * a type error wherever the query is run or used as a source. + */ +export type NeedsSelect = [keyof Output] extends [never] + ? { readonly selectRequired: "call select() first: a query with no SELECT list cannot be run or read from" } + : unknown + +/** + * Whether two column types can meet in one SQL column: one (without its + * NULL, and widened as comparisons widen) must be assignable to the other. + */ +export type CompatibleTypes = [Widen>] extends [Widen>] + ? true + : [Widen>] extends [Widen>] + ? true + : false + +/** + * `unknown` when a query selects exactly one column that can meet `T`, as + * `expr IN (subquery)` needs; otherwise a property naming what is wrong. + */ +export type SingleColumnOf = 0 extends 1 & Output + ? unknown + : [keyof Output] extends [never] + ? NeedsSelect + : IsUnion extends true + ? { readonly subqueryMustSelectOneColumn: keyof Output } + : CompatibleTypes extends true + ? unknown + : { readonly subqueryColumnTypeDiffers: Output[keyof Output] } + +type IsUnion = A extends unknown ? ([B] extends [A] ? false : true) : never + // CHQuery interface export interface CHQuery< @@ -168,6 +215,10 @@ export interface CHQuery< fn: ($: JoinedColumnAccessor) => S, ): CHQuery, Joins, Route> + /** + * Filter rows: conditions AND-joined, an `undefined` one skipped. Calling it + * again adds conditions, ANDed with the earlier ones. + */ where( fn: ($: JoinedColumnAccessor) => Array, ): CHQuery @@ -184,6 +235,8 @@ export interface CHQuery< * tenant-scoped: the rows are already aggregated by then, so the scan that * produced them crossed tenants regardless. Scope comes only from the * top-level `where` list. + * + * Calling it again adds conditions, as `where` does. */ having( fn: ($: JoinedColumnAccessor) => Array, @@ -191,9 +244,11 @@ export interface CHQuery< orderBy(...specs: Array>): CHQuery - limit(n: number): CHQuery + /** At most `n` rows: a non-negative integer. */ + limit(n: RowCount): CHQuery - offset(n: number): CHQuery + /** Skip `n` rows: a non-negative integer. */ + offset(n: RowCount): CHQuery format(fmt: "JSON" | "JSONEachRow"): CHQuery @@ -241,19 +296,19 @@ export interface CHQuery< innerJoin( table: Table, - alias: Alias, + alias: FreshAlias, on: JoinOnCallback, ): CHQuery leftJoin( table: Table, - alias: Alias, + alias: FreshAlias, on: JoinOnCallback, ): CHQuery }, Route> crossJoin( table: Table, - alias: Alias, + alias: FreshAlias, ): CHQuery // Type-safe joins with subquery (CHQuery) @@ -264,8 +319,8 @@ export interface CHQuery< JJoins extends Record, Alias extends string, >( - query: CHQuery, - alias: Alias, + query: CHQuery & NeedsSelect, + alias: FreshAlias, on: JoinOnCallback>, ): CHQuery }, Route> @@ -275,8 +330,8 @@ export interface CHQuery< JJoins extends Record, Alias extends string, >( - query: CHQuery, - alias: Alias, + query: CHQuery & NeedsSelect, + alias: FreshAlias, on: JoinOnCallback>, ): CHQuery< Cols, @@ -291,8 +346,8 @@ export interface CHQuery< JJoins extends Record, Alias extends string, >( - query: CHQuery, - alias: Alias, + query: CHQuery & NeedsSelect, + alias: FreshAlias, ): CHQuery }, Route> /** @@ -304,7 +359,10 @@ export interface CHQuery< * scope is *derived*, so a query whose only row source is a scoped CTE is * itself scoped without anyone asserting it. */ - withCTE(name: string, query: CHQuery): CHQuery + withCTE>( + name: string, + query: CHQuery & NeedsSelect, + ): CHQuery /** * Attach a CTE from pre-compiled SQL. @@ -435,6 +493,23 @@ export function createJoinedColumnAccessor Array + +/** A second `where` (or `having`) ANDs with the first, as in Kysely: replacing + * it would silently drop a filter, the tenant one included. */ +export const appendConditions = (previous: ConditionsFn | undefined, next: ConditionsFn): ConditionsFn => + previous === undefined ? next : ($) => [...previous($), ...next($)] + +/** + * A row count for `limit` / `offset`: a literal that is negative or has a + * fraction is a type error. Any other value is checked when compiling. + */ +export type RowCount = number extends N + ? N + : `${N}` extends `-${string}` | `${string}.${string}` | `${string}e${string}` + ? never + : N + function makeQuery< Cols extends ColumnDefs, Output extends Record, @@ -462,7 +537,7 @@ function makeQuery< }, where(fn) { - return makeQuery({ ...state, whereFn: fn }) + return makeQuery({ ...state, whereFn: appendConditions(state.whereFn, fn) }) }, groupBy(...keys) { @@ -470,7 +545,7 @@ function makeQuery< }, having(fn) { - return makeQuery({ ...state, havingFn: fn }) + return makeQuery({ ...state, havingFn: appendConditions(state.havingFn, fn) }) }, orderBy(...specs) { @@ -600,7 +675,7 @@ function makeQuery< name: string, sqlOrQuery: string | CHQuery, options?: { tenantScope?: TenantScope }, - ) { + ): any { // The query arm is compiled lazily in compileCH (like `fromQuery`), so // its scope is derived there rather than taken from the caller. const cte = @@ -648,7 +723,7 @@ export function fromQuery< InnerJoins extends Record, Alias extends string, >( - query: CHQuery, + query: CHQuery & NeedsSelect, alias: Alias, ): CHQuery, {}, {}, undefined> { return makeQuery({ diff --git a/src/ch/soundness.test-d.ts b/src/ch/soundness.test-d.ts new file mode 100644 index 0000000..63d15e5 --- /dev/null +++ b/src/ch/soundness.test-d.ts @@ -0,0 +1,98 @@ +// Type-level tests: invalid queries that must not type-check. +// +// Each `@ts-expect-error` is a query that is not SQL, or is SQL that cannot +// mean what it says. The compile-time half of each rule is in soundness.test.ts. + +import * as CH from "./index" +import { run } from "../database/database" + +const Users = CH.table("users", { + Id: CH.string, + Name: CH.string, + Nick: CH.nullable(CH.string), + Age: CH.uint64, +}) +const Orders = CH.table("orders", { Id: CH.string, UserId: CH.string, Amount: CH.uint64 }) +const Tags = CH.table("tags", { Id: CH.string, UserId: CH.string, Label: CH.string }) + +// Comparisons against null: `x = NULL` is never true +// @ts-expect-error -- use isNull() +CH.from(Users).where(($) => [$.Nick.eq(null)]) +// @ts-expect-error -- use isNotNull() +CH.from(Users).where(($) => [$.Nick.neq(null)]) +// @ts-expect-error -- IN (NULL) matches nothing +CH.from(Users).where(($) => [$.Nick.in_(null)]) +CH.from(Users).where(($) => [$.Nick.isNull(), $.Nick.eq("a"), $.Nick.eq($.Name)]) + +// LIKE takes a nullable string, and only a string +CH.from(Users).where(($) => [$.Nick.like("a%"), $.Name.ilike("a%")]) +// @ts-expect-error -- LIKE on a number +CH.from(Users).where(($) => [$.Age.like("1%")]) + +// limit / offset literals +CH.from(Users).select("Id").limit(10).offset(0) +const pageSize: number = 20 +CH.from(Users).select("Id").limit(pageSize) +// @ts-expect-error -- negative +CH.from(Users).select("Id").limit(-1) +// @ts-expect-error -- fractional +CH.from(Users).select("Id").offset(1.5) + +// Join aliases +// @ts-expect-error -- shadows the FROM column `Name` +CH.from(Users).innerJoin(Orders, "Name", (u, o) => u.Id.eq(o.UserId)) +CH.from(Users) + .innerJoin(Orders, "o", (u, o) => u.Id.eq(o.UserId)) + // @ts-expect-error -- `o` is already a join alias + .innerJoin(Tags, "o", (u, t) => u.Id.eq(t.UserId)) + +// A query must select something before it is run or read from +// @ts-expect-error -- no SELECT list +CH.fromQuery(CH.from(Orders), "o") +// @ts-expect-error -- no SELECT list +CH.from(Users).leftJoinQuery(CH.from(Orders), "o", (u) => u.Id.eq("x")) +// @ts-expect-error -- no SELECT list +CH.from(Users).withCTE("o", CH.from(Orders)) +// @ts-expect-error -- no SELECT list +CH.compileUnsafe(CH.from(Users)) +// @ts-expect-error -- no SELECT list +run(CH.from(Users)) +// @ts-expect-error -- no SELECT list +CH.exists(CH.from(Orders)) +run(CH.from(Users).select("Id")) + +// UNION ALL branches must agree on aliases and types +const ua = CH.from(Users).select("Id", "Name") +// @ts-expect-error -- extra column +CH.unionAll(ua, CH.from(Orders).select(($) => ({ Id: $.Id, Name: $.UserId, Extra: $.Amount }))) +// @ts-expect-error -- missing column +CH.unionAll(ua, CH.from(Orders).select(($) => ({ Id: $.Id }))) +// @ts-expect-error -- `Name` is a number in one branch +CH.unionAll(ua, CH.from(Orders).select(($) => ({ Id: $.Id, Name: $.Amount }))) +// @ts-expect-error -- no branches +CH.unionAll() +// @ts-expect-error -- no SELECT list +CH.unionAll(CH.from(Users), CH.from(Users)) +// Branch order of aliases may differ; a nullable branch widens the column +CH.unionAll(ua, CH.from(Users).select(($) => ({ Name: $.Nick, Id: $.Id }))) + +// IN (subquery) takes exactly one column of a comparable type +CH.from(Users).where(($) => [CH.inSubquery($.Id, CH.from(Orders).select("UserId"))]) +// @ts-expect-error -- two columns +CH.from(Users).where(($) => [CH.inSubquery($.Id, CH.from(Orders).select("UserId", "Amount"))]) +// @ts-expect-error -- a number column against a string +CH.from(Users).where(($) => [CH.notInSubquery($.Id, CH.from(Orders).select("Amount"))]) + +// Writes +// @ts-expect-error -- SET with no columns +CH.update(Orders).set({}).allRows() +// @ts-expect-error -- an UPDATE must say which rows +run(CH.update(Orders).set({ Amount: 1 })) +// @ts-expect-error -- a DELETE must say which rows +run(CH.deleteFrom(Orders)) +// @ts-expect-error -- not compilable either +CH.compileUnsafe(CH.deleteFrom(Orders)) +run(CH.update(Orders).set({ Amount: 1 }).where(($) => [$.Id.eq("a")])) +run(CH.deleteFrom(Orders).allRows()) +// @ts-expect-error -- INSERT ... SELECT from a query with no SELECT list +CH.insertInto(CH.table("t", { a: CH.nullable(CH.string) })).select(CH.from(Users)) diff --git a/src/ch/soundness.test.ts b/src/ch/soundness.test.ts new file mode 100644 index 0000000..5a885b5 --- /dev/null +++ b/src/ch/soundness.test.ts @@ -0,0 +1,107 @@ +// Invalid queries are refused before any SQL is sent. +// +// The type-level half is in soundness.test-d.ts. These cover what a type cannot +// see (values from data, casts, untyped callers) and pin the SQL written for +// the edge cases that do have a meaning. + +import { describe, expect, it } from "@effect/vitest" +import * as CH from "./index" +import { compileCHUnsafe, compileUnionUnsafe } from "./compile" + +const Users = CH.table("users", { Id: CH.string, Name: CH.string, Nick: CH.nullable(CH.string), Age: CH.uint64 }) +const Orders = CH.table("orders", { Id: CH.string, UserId: CH.string, Amount: CH.uint64 }) +const Tags = CH.table("tags", { Id: CH.string, UserId: CH.string, Label: CH.string }) + +const whereSql = (query: { readonly sql: string }) => query.sql.split("WHERE ")[1]?.replace(/\s+/g, " ").trim() + +describe("comparisons", () => { + it("writes an empty IN list as the constant it means", () => { + const ids: Array = [] + expect(whereSql(compileCHUnsafe(CH.from(Users).select("Id").where(($) => [$.Id.in_(...ids)])))).toBe("1 = 0") + expect(whereSql(compileCHUnsafe(CH.from(Users).select("Id").where(($) => [$.Id.notIn(...ids)])))).toBe("1 = 1") + expect(whereSql(compileCHUnsafe(CH.from(Users).select("Id").where(($) => [CH.inList($.Id, [])])))).toBe("1 = 0") + }) + + it("refuses a null that slipped past the types", () => { + const nick = null as unknown as string + expect(() => compileCHUnsafe(CH.from(Users).select("Id").where(($) => [$.Nick.eq(nick)]))).toThrow(/isNull/) + expect(() => compileCHUnsafe(CH.from(Users).select("Id").where(($) => [$.Age.between(1, nick as never)]))).toThrow( + /isNull/, + ) + }) +}) + +describe("where and having", () => { + it("AND a second where with the first, keeping the tenant filter", () => { + const Scoped = CH.table("scoped", { OrgId: CH.string, Ms: CH.uint64 }, { tenantColumn: "OrgId" }) + const compiled = compileCHUnsafe( + CH.from(Scoped) + .select("Ms") + .where(($) => [$.OrgId.eq("org")]) + .where(($) => [$.Ms.gt(1)]), + ) + expect(whereSql(compiled)).toBe("scoped.OrgId = 'org' AND scoped.Ms > 1") + expect(compiled.tenantScope).toBe("single-tenant") + }) + + it("AND a second where on an UPDATE or DELETE", () => { + const sql = compileCHUnsafe( + CH.deleteFrom(Orders) + .where(($) => [$.UserId.eq("u")]) + .where(($) => [$.Amount.gt(1)]), + ).sql + expect(sql.replace(/\s+/g, " ")).toContain("UserId = 'u' AND Amount > 1") + }) +}) + +describe("limit and offset", () => { + it.each([-1, 1.5, Number.NaN, Number.POSITIVE_INFINITY])("refuses %s", (n) => { + expect(() => compileCHUnsafe(CH.from(Users).select("Id").limit(n))).toThrow(/non-negative integer/) + expect(() => compileCHUnsafe(CH.from(Users).select("Id").offset(n))).toThrow(/non-negative integer/) + expect(() => compileUnionUnsafe(CH.unionAll(CH.from(Users).select("Id")).limit(n), {})).toThrow( + /non-negative integer/, + ) + }) + + it("writes a valid count as it is", () => { + expect(compileCHUnsafe(CH.from(Users).select("Id").limit(0).offset(20)).sql).toMatch(/LIMIT 0\s+OFFSET 20/) + }) +}) + +describe("source names", () => { + it("refuses a join alias that is the FROM alias", () => { + expect(() => + compileCHUnsafe( + CH.from(Users, "u") + .innerJoin(Orders, "u", (u, o) => u.Id.eq(o.UserId)) + .select("Id"), + ), + ).toThrow(/already the name of another source/) + }) + + it("refuses two joins under one alias, and a join alias that hides a column", () => { + const any = CH.from(Users) as any + expect(() => + compileCHUnsafe( + any + .innerJoin(Orders, "o", (u: any, o: any) => u.Id.eq(o.UserId)) + .innerJoin(Tags, "o", (u: any, t: any) => u.Id.eq(t.UserId)) + .select("Id"), + ), + ).toThrow(/already the name of another source/) + expect(() => + compileCHUnsafe(any.innerJoin(Orders, "Name", (u: any, o: any) => u.Id.eq(o.UserId)).select("Id")), + ).toThrow(/also a column of the FROM source/) + }) + + it("refuses a CTE defined twice", () => { + expect(() => + compileCHUnsafe( + CH.from(Users) + .withCTE("x", CH.from(Orders).select("Id")) + .withCTE("x", CH.from(Tags).select("Id")) + .select("Id"), + ), + ).toThrow(/defined twice/) + }) +}) diff --git a/src/ch/subquery.ts b/src/ch/subquery.ts index 53cfc90..0bc8b93 100644 --- a/src/ch/subquery.ts +++ b/src/ch/subquery.ts @@ -9,7 +9,7 @@ import { renderSubquery } from "./subquery-context" import { compileCHUnsafe } from "./compile" import { type Condition, type Expr, makeCond, makeExpr, makeUntypedExpr } from "./expr" -import type { CHQuery } from "./query" +import type { CHQuery, NeedsSelect, SingleColumnOf } from "./query" import type { CHType } from "./types" import { compile, lazy } from "../sql/sql-fragment" @@ -37,12 +37,18 @@ const toSql = (subquery: Subquery): string => // An inner tenant filter cannot confine an otherwise unfiltered outer source. /** `EXISTS (subquery)` — for correlated subqueries (see `outerRef`). */ -export function exists(subquery: Subquery): Condition { +export function exists>(subquery: string | (CHQuery & NeedsSelect)): Condition { return makeCond(lazy(() => `EXISTS (${toSql(subquery)})`)) } -/** `expr IN (subquery)`. */ -export function inSubquery(expr: Expr, subquery: Subquery): Condition { +/** + * `expr IN (subquery)`. The subquery must select exactly one column, of a type + * `expr` can be compared with. The SQL-string arm is unchecked. + */ +export function inSubquery>( + expr: Expr, + subquery: string | (CHQuery & SingleColumnOf), +): Condition { return makeCond(lazy(() => `${compile(expr.toFragment())} IN (${toSql(subquery)})`)) } @@ -52,7 +58,10 @@ export function inSubquery(expr: Expr, subquery: Subquery): Condition { * Note ClickHouse's NULL semantics: if the subquery yields any NULL, `NOT IN` * is never true. Project a non-nullable column, or filter the NULLs inside. */ -export function notInSubquery(expr: Expr, subquery: Subquery): Condition { +export function notInSubquery>( + expr: Expr, + subquery: string | (CHQuery & SingleColumnOf), +): Condition { return makeCond(lazy(() => `${compile(expr.toFragment())} NOT IN (${toSql(subquery)})`)) } diff --git a/src/ch/union.ts b/src/ch/union.ts index 45d2a41..a4b68f6 100644 --- a/src/ch/union.ts +++ b/src/ch/union.ts @@ -5,7 +5,7 @@ // OFFSET wrapping. import type { ColumnDefs } from "./types" -import type { CHQuery } from "./query" +import type { CHQuery, CompatibleTypes, OutputOf, RowCount } from "./query" // Union state (runtime) @@ -28,9 +28,9 @@ export interface CHUnionQuery = {}> { orderBy(...specs: Array<[keyof Output & string, "asc" | "desc"]>): CHUnionQuery - limit(n: number): CHUnionQuery + limit(n: RowCount): CHUnionQuery - offset(n: number): CHUnionQuery + offset(n: RowCount): CHUnionQuery format(fmt: "JSON" | "JSONEachRow"): CHUnionQuery } @@ -68,11 +68,41 @@ function makeUnionQuery>(state: CHUnionState) // Entry point -export function unionAll>( - ...queries: Array> -): CHUnionQuery { +type AnyQuery = CHQuery + +/** The aliases branch `O` disagrees with the first branch `O0` on: missing, + * extra, or of a type the first branch's column cannot hold. */ +type BranchMisfits = { + [K in keyof O0 | keyof O]: K extends keyof O0 + ? K extends keyof O + ? CompatibleTypes extends true + ? never + : K + : K + : K +}[keyof O0 | keyof O] + +/** + * `unknown` when every branch selects the first branch's aliases, of types + * that can share a column; otherwise a property naming the aliases that differ. + * Branches are matched by alias, not position, so their order may differ. + */ +export type UnionBranchesFit> = [keyof OutputOf] extends [never] + ? { readonly selectRequired: "every unionAll branch needs a select()" } + : [{ [I in keyof Q]: BranchMisfits, OutputOf> }[number]] extends [never] + ? unknown + : { readonly unionColumnsDiffer: { [I in keyof Q]: BranchMisfits, OutputOf> }[number] } + +/** The union's row: the first branch's aliases, each typed as any branch's. */ +export type UnionOutput> = { + readonly [K in keyof OutputOf]: OutputOf[K] +} + +export function unionAll]>( + ...queries: Q & UnionBranchesFit +): CHUnionQuery> { return makeUnionQuery({ - queries, + queries: queries as ReadonlyArray, outerOrderBySpecs: [], }) } diff --git a/src/ch/update.test.ts b/src/ch/update.test.ts index cc2edce..43d3fe5 100644 --- a/src/ch/update.test.ts +++ b/src/ch/update.test.ts @@ -70,6 +70,7 @@ describe("update", () => { PG.compile(CH.update(Counters).set({ search: "x" } as any).where(($) => [$.key.eq("k")])), ) expect(computed.message).toContain('sets "search"') + // @ts-expect-error -- an empty SET is a type error too const empty = yield* Effect.flip(PG.compile(CH.update(Counters).set({}).where(($) => [$.key.eq("k")]))) expect(empty.message).toContain("sets no columns") const returning = yield* Effect.exit(CH.compile(CH.update(Spans).set({ Ms: 0 }).allRows().returning())) @@ -142,6 +143,7 @@ describe("deleteFrom", () => { it.effect("refuses a delete with no where(), or whose conditions filter nothing", () => Effect.gen(function* () { + // @ts-expect-error -- a delete without where() or allRows() is a type error too expect(failure(yield* Effect.exit(CH.compile(CH.deleteFrom(Spans))))).toBeInstanceOf(QueryBuilderDefect) for (const conditions of [[], [CH.rawCond("")], [undefined, CH.rawCond(" ")]]) { const error = yield* Effect.flip(PG.compile(CH.deleteFrom(Counters).where(() => conditions))) diff --git a/src/ch/update.ts b/src/ch/update.ts index a370ac5..da84ffb 100644 --- a/src/ch/update.ts +++ b/src/ch/update.ts @@ -18,7 +18,7 @@ import type { Condition, Expr } from "./expr" import type { ConflictSet, InsertSettingValue } from "./insert" -import type { ColumnAccessor, InferOutput } from "./query" +import { appendConditions, type ColumnAccessor, type InferOutput } from "./query" import type { Table } from "./table" import type { ColumnDefs, InferTS } from "./types" @@ -53,15 +53,24 @@ export type CHDeleteState = WriteState type AllColumns = { readonly [P in keyof Cols & string]: InferTS } -/** The clauses UPDATE and DELETE share, returning `Self` with `Output` replaced. */ -interface WriteClauses { +/** + * Whether a write says which rows it changes. One without `where` or + * `allRows` cannot be compiled or run: an unfiltered UPDATE or DELETE has to + * say that it means every row. + */ +export type WriteReady = "ready" | "needs where() or allRows()" + +/** The clauses UPDATE and DELETE share: `Filtered` is the write once it says + * which rows it changes, `Self` the write as it is. */ +interface WriteClauses { /** * The rows to change, as in a query's `where`: conditions AND-joined, an - * `undefined` one skipped. Calling it again replaces them. + * `undefined` one skipped. Calling it again adds conditions, ANDed with the + * earlier ones. */ - where(fn: WhereFn): Self - /** Change every row. Without it, compiling a write with no WHERE is a defect. */ - allRows(): Self + where(fn: WhereFn): Filtered + /** Change every row. Without it (or `where`), the write cannot be compiled or run. */ + allRows(): Filtered /** * ClickHouse `SETTINGS` for this write, such as `{ mutations_sync: 2 }` so an * `ALTER TABLE ... UPDATE` waits for the mutation. Postgres refuses them. @@ -69,41 +78,59 @@ interface WriteClauses { settings(settings: Readonly>): Self } -export interface CHUpdate - extends WriteClauses> { +export interface CHUpdate< + Cols extends ColumnDefs = ColumnDefs, + Computed extends string = never, + Output = never, + Ready extends WriteReady = "ready", +> extends WriteClauses, CHUpdate> { readonly _tag: "CHUpdate" /** @internal — runtime update state */ readonly _state: CHUpdateState /** phantom. `output` is the row `Database.run` returns: none without RETURNING. */ - readonly _phantom?: { readonly cols: Cols; readonly output: Output } + readonly _phantom?: { readonly cols: Cols; readonly output: Output; readonly ready: Ready } - /** Replace the SET record. */ - set(set: UpdateSet | (($: ColumnAccessor) => UpdateSet)): CHUpdate + /** Replace the SET record. It must set at least one column. */ + set>( + set: (S & NonEmptySet) | (($: ColumnAccessor) => S & NonEmptySet), + ): CHUpdate /** The changed rows, as for an insert: every column, the named ones, or a callback. Postgres only. */ - returning(): CHUpdate> + returning(): CHUpdate, Ready> returning( ...columns: [K, ...Array] - ): CHUpdate }> - returning>>(fn: ($: ColumnAccessor) => S): CHUpdate> + ): CHUpdate }, Ready> + returning>>( + fn: ($: ColumnAccessor) => S, + ): CHUpdate, Ready> } -export interface CHDelete - extends WriteClauses> { +export interface CHDelete + extends WriteClauses, CHDelete> { readonly _tag: "CHDelete" /** @internal — runtime delete state */ readonly _state: CHDeleteState - readonly _phantom?: { readonly cols: Cols; readonly output: Output } + readonly _phantom?: { readonly cols: Cols; readonly output: Output; readonly ready: Ready } /** The deleted rows, as for an insert: every column, the named ones, or a callback. Postgres only. */ - returning(): CHDelete> - returning(...columns: [K, ...Array]): CHDelete }> - returning>>(fn: ($: ColumnAccessor) => S): CHDelete> + returning(): CHDelete, Ready> + returning( + ...columns: [K, ...Array] + ): CHDelete }, Ready> + returning>>(fn: ($: ColumnAccessor) => S): CHDelete, Ready> } +/** + * `unknown` when a SET record names at least one column. A record whose keys + * are all optional (as every SET key is) still has to set something: `SET` + * with nothing after it is not SQL. A record built at runtime is checked when + * compiling. + */ +export type NonEmptySet = [keyof S] extends [never] ? { readonly setAtLeastOneColumn: true } : unknown + /** An update with no SET yet: only `set`, so it cannot be compiled before it says what to write. */ export type CHUpdateStart = Pick< - CHUpdate, + CHUpdate, "set" > @@ -118,20 +145,21 @@ export const returningFnOf = } const writeClauses = (state: State, make: (state: State) => Self) => ({ - where: (whereFn: ($: any) => Array) => make({ ...state, whereFn }), + where: (whereFn: ($: any) => Array) => + make({ ...state, whereFn: appendConditions(state.whereFn, whereFn) }), allRows: () => make({ ...state, allRows: true }), settings: (settings: Readonly>) => make({ ...state, settings: { ...settings } }), returning: (...args: ReadonlyArray) => make({ ...state, returningFn: returningFnOf(state.table)(args) }), }) -const makeUpdate = (state: CHUpdateState): CHUpdate => ({ +const makeUpdate = (state: CHUpdateState): CHUpdate => ({ _tag: "CHUpdate", _state: state, ...writeClauses(state, makeUpdate), - set: (set) => makeUpdate({ ...state, set }), + set: (set: CHUpdateState["set"]) => makeUpdate({ ...state, set }), }) -const makeDelete = (state: CHDeleteState): CHDelete => ({ +const makeDelete = (state: CHDeleteState): CHDelete => ({ _tag: "CHDelete", _state: state, ...writeClauses(state, makeDelete), @@ -145,12 +173,14 @@ export function update(table: Table): CHDelete { - return makeDelete({ table: table as Table }) as CHDelete +export function deleteFrom( + table: Table, +): CHDelete { + return makeDelete({ table: table as Table }) as CHDelete } -export const isUpdate = (value: unknown): value is CHUpdate => +export const isUpdate = (value: unknown): value is CHUpdate => typeof value === "object" && value !== null && (value as { readonly _tag?: unknown })._tag === "CHUpdate" -export const isDelete = (value: unknown): value is CHDelete => +export const isDelete = (value: unknown): value is CHDelete => typeof value === "object" && value !== null && (value as { readonly _tag?: unknown })._tag === "CHDelete" diff --git a/src/database/database.ts b/src/database/database.ts index 3f04d7f..4465bcf 100644 --- a/src/database/database.ts +++ b/src/database/database.ts @@ -16,7 +16,7 @@ import { noTransactions, type Dialect, type IsolationLevel, type TransactionSett import type { QueryBuilderError } from "../ch/errors" import type { CHInsert } from "../ch/insert" import type { CHDelete, CHUpdate } from "../ch/update" -import type { CHQuery } from "../ch/query" +import type { CHQuery, NeedsSelect } from "../ch/query" import type { CHUnionQuery } from "../ch/union" import { DatabaseError, @@ -48,6 +48,12 @@ export type Runnable = | CHDelete | CompiledQuery +/** + * `unknown` when a `Runnable` can run as it is. A query with no SELECT list + * cannot; a write without `where`/`allRows` is not a `Runnable` at all. + */ +export type RunCheck = Q extends CHQuery ? NeedsSelect : unknown + /** The decoded row of a `Runnable`. */ export type RowOf = Q extends CompiledQuery ? Output : Q extends { readonly _phantom?: { output: infer Output } } ? Output : never @@ -97,7 +103,7 @@ export interface DatabaseApi { * runs as it is, if it was compiled for this dialect. */ readonly run: ( - query: Q, + query: Q & RunCheck, params?: Record, ) => Effect.Effect>, DatabaseError | QueryBuilderError | CompiledQueryDecodeError> /** Run a statement and return its rows, decoded through `schema` when given. */ @@ -443,7 +449,7 @@ export const layerSqlClient = (options: FromSqlClientOptions): Layer.Layer( - query: Q, + query: Q & RunCheck, params?: Record, ): Effect.Effect>, DatabaseError | QueryBuilderError | CompiledQueryDecodeError, Database> => Effect.flatMap(Effect.service(Database), (db) => db.run(query, params)) diff --git a/src/schema/define.ts b/src/schema/define.ts index 1800ffe..e49162e 100644 --- a/src/schema/define.ts +++ b/src/schema/define.ts @@ -13,7 +13,7 @@ import { compileCHUnsafe } from "../ch/compile" import { clickhouseDialect, withDialect } from "../ch/dialect" import type { Expr } from "../ch/expr" import { encodeColumnLiteral } from "../ch/literal" -import { createColumnAccessor, type CHQuery, type ColumnAccessor } from "../ch/query" +import { createColumnAccessor, type CHQuery, type ColumnAccessor, type NeedsSelect } from "../ch/query" import type { Table } from "../ch/table" import type { CHType, ColumnDefs, InferTS } from "../ch/types" import { compile as compileFragment } from "../sql/sql-fragment" @@ -392,7 +392,7 @@ export function materializedView< name: Name, options: { readonly to: SchemaTable - readonly as: CHQuery + readonly as: CHQuery & NeedsSelect } & ([MisfitColumns] extends [never] ? unknown : { readonly targetCannotTake: MisfitColumns }), From 150ad1777c82edf8b559779d1f94d7c43bb6e7f1 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 4 Oct 2026 21:11:27 +0200 Subject: [PATCH 2/5] Refuse aggregates in WHERE, ungrouped columns, and functions of another dialect Each clause is rendered under a render track that records the columns it reads outside an aggregate and whether it aggregates. compile uses it to refuse an aggregate in WHERE or a join's ON, a selected or HAVING column that is neither grouped nor aggregated, and grouping by an aggregate. Only SQL the builder writes is counted; raw SQL, templates, windows and caller-declared functions are opaque, so they can hide an error but never cause a false one. Built-in functions now belong to a function set. A ClickHouse function in a Postgres compile (count() instead of count(*)) is a defect, and the reverse; coalesce, nullIf and lower stay portable. Co-Authored-By: Claude Opus 5.5 --- docs/postgres.md | 6 +- docs/queries.md | 13 + src/ch/compile.test.ts | 2 +- src/ch/compile.ts | 531 ++++++++++++++++++-------------- src/ch/dialect.ts | 11 + src/ch/expr.ts | 50 +-- src/ch/functions/aggregate.ts | 7 +- src/ch/functions/array.ts | 7 +- src/ch/functions/builtin.ts | 83 +++++ src/ch/functions/conditional.ts | 12 +- src/ch/functions/date-time.ts | 5 +- src/ch/functions/json.ts | 4 +- src/ch/functions/map.ts | 7 +- src/ch/functions/numeric.ts | 5 +- src/ch/functions/string.ts | 10 +- src/ch/functions/window.ts | 8 +- src/ch/soundness.test.ts | 89 ++++++ src/pg/dialect.ts | 1 + src/pg/functions.ts | 42 +-- src/sql/render-tracker.ts | 87 ++++++ src/sql/sql-fragment.ts | 17 +- 21 files changed, 703 insertions(+), 294 deletions(-) create mode 100644 src/ch/functions/builtin.ts create mode 100644 src/sql/render-tracker.ts diff --git a/docs/postgres.md b/docs/postgres.md index 689d059..401dc26 100644 --- a/docs/postgres.md +++ b/docs/postgres.md @@ -109,7 +109,11 @@ which no session time zone can reinterpret; a zoneless string is read as UTC. The shared operators (`eq`, `in_`, `like`, `ilike`, `and`, `or`, `not`, arithmetic, `lit`) work unchanged. The ClickHouse function catalog on the root entry (`quantile`, `toStartOfInterval`, -map subscripts, …) writes ClickHouse SQL and will not run on Postgres. +`count()`, …) writes ClickHouse SQL, so compiling a query that uses one for Postgres is a +`QueryBuilderDefect` naming the function; the Postgres functions above fail the same way on +ClickHouse. `coalesce`, `nullIf` and `lower` from the root entry render the same on both and +are allowed on either. A custom `Dialect` opts in with `functions: "clickhouse"` or +`"postgres"`; without it, nothing is checked. ## Known differences diff --git a/docs/queries.md b/docs/queries.md index 1502186..4ef51b6 100644 --- a/docs/queries.md +++ b/docs/queries.md @@ -75,6 +75,19 @@ Takes **output keys** (the aliases from `select`), not raw column names: .groupBy("name") ``` +Once a query groups or aggregates, every column it reads outside an aggregate must be a +`groupBy` key, as both databases require. Compiling one that breaks the rule is a +`QueryBuilderDefect` naming the alias and column, instead of a server error: + +- `select(($) => ({ name: $.Name, n: CH.count() }))` with no `groupBy("name")`; +- an aggregate in `where` or a join's `on` (filter on it in `having`); +- `groupBy` naming an aggregate alias. + +An expression over a grouped column (`CH.lower($.Name)` with `Name` grouped) and a repeat of a +grouped expression are fine. Only SQL the builder writes is checked: a window (`CH.over`), a +`CH.sql` template, `rawExpr` and functions declared with `defineFn` / `makeExpr` are not looked +inside, so they can hide a mistake from this check but never trigger a false one. + ## `having` Filter groups after aggregation. The callback has the input-column accessor, so either repeat diff --git a/src/ch/compile.test.ts b/src/ch/compile.test.ts index 046e18d..bbd8dfc 100644 --- a/src/ch/compile.test.ts +++ b/src/ch/compile.test.ts @@ -337,7 +337,7 @@ describe("CompiledQuery.tenantScope", () => { // The shape that satisfied the old `sql.includes("OrgId")` guard. const compiled = compileCHUnsafe( CH.from(events) - .select(($) => ({ OrgId: $.OrgId, count: $.Count })) + .select(($) => ({ OrgId: $.OrgId, count: CH.sum($.Count) })) .groupBy("OrgId"), {}, ) diff --git a/src/ch/compile.ts b/src/ch/compile.ts index ae6ee0b..7b4fd0c 100644 --- a/src/ch/compile.ts +++ b/src/ch/compile.ts @@ -14,10 +14,11 @@ import { isInsert, type CHInsert } from "./insert" import { isDelete, isUpdate, type CHDelete, type CHUpdate } from "./update" import type { Table } from "./table" import { createColumnAccessor, createQualifiedColumnAccessor, createJoinedColumnAccessor, sourceAlias } from "./query" -import { aliased, columnTypeOf, isExprLike, type Expr } from "./expr" -import { raw, identPath, quoteIdent, quoteIdentPath, compile as compileSqlFragment } from "../sql/sql-fragment" +import { aliased, columnTypeOf, isExprLike, type Condition, type Expr } from "./expr" +import { raw, identPath, quoteIdent, quoteIdentPath, compile as compileSqlFragment, type SqlFragment } from "../sql/sql-fragment" import { splitTerminalClauses } from "../sql/terminal-clauses" import { compileQuery, type SqlQuery } from "../sql/sql-query" +import { track, untracked, type RenderTrack } from "../sql/render-tracker" import { PARAM_MARKER_PREFIX, PARAM_PLACEHOLDER_PATTERN, param, paramSchema, type ParamKind } from "./param" import { mergeResultSchemas } from "./define-fn" import { encodeValue } from "./literal" @@ -705,255 +706,333 @@ function compileInner< enclosingCtes?: ReadonlyArray }, ): CompiledQuery { - const state = query._state - const deferParams = options?.deferParams === true - - // The one accessor factory — shared with `selectExprsOf`, which reads a - // query's output schemas without compiling it. Building a second one here is - // what silently dropped every joined and subquery column's type: this path - // passed `state.columns` (empty for a `fromQuery`/`fromUnion`) and no join - // columns at all, so `$.p.ServiceName` and `$.bucket` compiled to correct SQL - // with no schema, and the query derived nothing. - assertDistinctNames(state) - const $ = makeAccessor(state) - - // SELECT - const selectExprs = state.selectFn ? state.selectFn($) : {} - const keys = Object.keys(selectExprs) - if ( - options?.selectKeys && - (keys.length !== options.selectKeys.length || - options.selectKeys.some((key) => !Object.hasOwn(selectExprs, key))) - ) { - throw new QueryBuilderDefect({ - message: "unionAll: every branch must select the same column aliases", - }) - } - const selectFragments = (options?.selectKeys ?? keys).map((alias) => aliased(selectExprs[alias], alias)) - - if (selectFragments.length === 0) { - throw new QueryBuilderDefect({ message: "CHQuery: select() is required" }) - } - - // WHERE — resolve params by injecting values into the accessor - const whereConditions = state.whereFn ? state.whereFn($) : [] - const whereFragments = whereConditions - .filter((c): c is NonNullable => c != null) - .map((c) => c.toFragment()) - - // CTEs — resolved before the FROM below, which reads their scope. A CTE given - // as a query is compiled here and its scope derived; one given as a string - // carries whatever scope the caller declared. - // Sequential, not `map`: each CTE is compiled with the ones before it in - // scope, which is the only way `WITH a AS (…), b AS (SELECT … FROM a)` can - // see that `b` reads a tenant-confined source. - const resolvedCtes: Array = [] - for (const c of state.ctes) { - if (c.query) { - const compiled = compileInner(c.query, params, { - skipFormat: true, - deferParams, - nested: true, - enclosingCtes: [...(options?.enclosingCtes ?? []), ...resolvedCtes], - }) - resolvedCtes.push({ - name: c.name, - sql: compiled.sql, - tenantScope: compiled.tenantScope, - tenantBound: tenantBoundOf(compiled), + // A nested query's columns are its own: nothing it renders reaches an + // enclosing query's render track. + return untracked(() => { + const state = query._state + const deferParams = options?.deferParams === true + + // The one accessor factory — shared with `selectExprsOf`, which reads a + // query's output schemas without compiling it. Building a second one here is + // what silently dropped every joined and subquery column's type: this path + // passed `state.columns` (empty for a `fromQuery`/`fromUnion`) and no join + // columns at all, so `$.p.ServiceName` and `$.bucket` compiled to correct SQL + // with no schema, and the query derived nothing. + assertDistinctNames(state) + const $ = makeAccessor(state) + + // SELECT + const selectExprs = state.selectFn ? state.selectFn($) : {} + const keys = Object.keys(selectExprs) + if ( + options?.selectKeys && + (keys.length !== options.selectKeys.length || + options.selectKeys.some((key) => !Object.hasOwn(selectExprs, key))) + ) { + throw new QueryBuilderDefect({ + message: "unionAll: every branch must select the same column aliases", }) - } else { - resolvedCtes.push({ name: c.name, sql: c.sql ?? "", tenantScope: c.tenantScope }) } - } + const selectKeys = options?.selectKeys ?? keys - const visibleCtes = [...resolvedCtes, ...(options?.enclosingCtes ?? [])] - const sourceForTable = (name: string, column?: string): TenantSource => { - const cte = visibleCtes.find((c) => c.name === name) - return { - // A projected CTE column need not be the original tenant key. - column: cte ? undefined : column, - scope: cte ? (cte.tenantScope ?? "cross-tenant") : column ? "cross-tenant" : "untenanted", - bound: cte?.tenantBound, + if (selectKeys.length === 0) { + throw new QueryBuilderDefect({ message: "CHQuery: select() is required" }) } - } - const sourceOf = (compiled: CompiledQuery): TenantSource => ({ - scope: compiled.tenantScope, - bound: tenantBoundOf(compiled), - }) - const mainAlias = sourceAlias(state) - const mainColumn = - state.tenantColumn === undefined - ? undefined - : `${mainAlias}.${state.tenantColumn}` - let fromFragment - let fromSource: TenantSource - if (state.fromQuery) { - const inner = compileInner(state.fromQuery, params, { - skipFormat: true, - deferParams, - nested: true, - enclosingCtes: visibleCtes, - }) - fromSource = sourceOf(inner) - fromFragment = raw(`(${inner.sql}) AS ${quoteIdent(state.fromQueryAlias ?? "")}`) - } else if (state.fromUnion) { - const inner = compileUnionInner(state.fromUnion, params, { deferParams, nested: true, enclosingCtes: visibleCtes }) - fromSource = sourceOf(inner) - const body = currentDialect().clauses.format ? splitTerminalClauses(inner.sql).body : inner.sql - fromFragment = raw(`(\n${body}\n) AS ${quoteIdent(state.fromQueryAlias ?? "")}`) - } else { - fromSource = sourceForTable(state.tableName, mainColumn) - fromFragment = mainAlias !== state.tableName - ? raw(`${quoteIdentPath(state.tableName)} AS ${quoteIdentPath(mainAlias)}`) - : identPath(state.tableName) - } - const sources: TenantSource[] = [fromSource] - const wherePredicates = whereConditions.flatMap((c) => (c ? tenantPredicatesOf(c) : [])) - const joinPredicates: Array<{ predicates: ReadonlyArray; target?: string }> = [] - let sql = withSubqueryCompiler((subquery) => { - if (typeof subquery === "string") { - sources.push({ scope: "cross-tenant" }) - return subquery - } - const compiled = compileInner(subquery, params, { - skipFormat: true, - deferParams, - nested: true, - enclosingCtes: visibleCtes, - }) - sources.push(sourceOf(compiled)) - return compiled.sql - }, () => { - const joins = state.typedJoins.map((j) => { - let tableSql: string - let source: TenantSource - if (j.innerQuery) { - const compiled = compileInner(j.innerQuery, params, { + // WHERE — resolve params by injecting values into the accessor + const whereConditions = state.whereFn ? state.whereFn($) : [] + const presentWhere = whereConditions.filter((c): c is NonNullable => c != null) + + // CTEs — resolved before the FROM below, which reads their scope. A CTE given + // as a query is compiled here and its scope derived; one given as a string + // carries whatever scope the caller declared. + // Sequential, not `map`: each CTE is compiled with the ones before it in + // scope, which is the only way `WITH a AS (…), b AS (SELECT … FROM a)` can + // see that `b` reads a tenant-confined source. + const resolvedCtes: Array = [] + for (const c of state.ctes) { + if (c.query) { + const compiled = compileInner(c.query, params, { skipFormat: true, deferParams, nested: true, - enclosingCtes: visibleCtes, + enclosingCtes: [...(options?.enclosingCtes ?? []), ...resolvedCtes], + }) + resolvedCtes.push({ + name: c.name, + sql: compiled.sql, + tenantScope: compiled.tenantScope, + tenantBound: tenantBoundOf(compiled), }) - tableSql = `(${compiled.sql})` - source = sourceOf(compiled) - } else if (j.tableName) { - tableSql = quoteIdentPath(j.tableName) - source = sourceForTable( - j.tableName, - j.tenantColumn === undefined ? undefined : `${j.alias}.${j.tenantColumn}`, - ) } else { - throw new QueryBuilderDefect({ message: "TypedJoin: missing table or query" }) - } - sources.push(source) - const on = j.on?.( - createQualifiedColumnAccessor(mainAlias, state.tenantColumn, columnsOf(state)), - createQualifiedColumnAccessor(j.alias, j.tenantColumn, joinColumnsOf(j)), - ) - if (on) { - // A LEFT JOIN's ON clause can constrain only its right side. It - // cannot remove unmatched rows from the preserved left side. - if (j.type !== "LEFT" || source.column !== undefined) { - joinPredicates.push({ - predicates: tenantPredicatesOf(on), - target: j.type === "LEFT" ? source.column : undefined, - }) - } + resolvedCtes.push({ name: c.name, sql: c.sql ?? "", tenantScope: c.tenantScope }) } + } + + const visibleCtes = [...resolvedCtes, ...(options?.enclosingCtes ?? [])] + const sourceForTable = (name: string, column?: string): TenantSource => { + const cte = visibleCtes.find((c) => c.name === name) return { - type: j.type, - table: tableSql, - alias: quoteIdent(j.alias), - on: on ? compileSqlFragment(on.toFragment()) : undefined, + // A projected CTE column need not be the original tenant key. + column: cte ? undefined : column, + scope: cte ? (cte.tenantScope ?? "cross-tenant") : column ? "cross-tenant" : "untenanted", + bound: cte?.tenantBound, } + } + const sourceOf = (compiled: CompiledQuery): TenantSource => ({ + scope: compiled.tenantScope, + bound: tenantBoundOf(compiled), }) + const mainAlias = sourceAlias(state) + const mainColumn = + state.tenantColumn === undefined + ? undefined + : `${mainAlias}.${state.tenantColumn}` + let fromFragment + let fromSource: TenantSource + if (state.fromQuery) { + const inner = compileInner(state.fromQuery, params, { + skipFormat: true, + deferParams, + nested: true, + enclosingCtes: visibleCtes, + }) + fromSource = sourceOf(inner) + fromFragment = raw(`(${inner.sql}) AS ${quoteIdent(state.fromQueryAlias ?? "")}`) + } else if (state.fromUnion) { + const inner = compileUnionInner(state.fromUnion, params, { deferParams, nested: true, enclosingCtes: visibleCtes }) + fromSource = sourceOf(inner) + const body = currentDialect().clauses.format ? splitTerminalClauses(inner.sql).body : inner.sql + fromFragment = raw(`(\n${body}\n) AS ${quoteIdent(state.fromQueryAlias ?? "")}`) + } else { + fromSource = sourceForTable(state.tableName, mainColumn) + fromFragment = mainAlias !== state.tableName + ? raw(`${quoteIdentPath(state.tableName)} AS ${quoteIdentPath(mainAlias)}`) + : identPath(state.tableName) + } - const sqlQuery: SqlQuery = { - distinct: state.distinct !== undefined, - distinctOn: Array.isArray(state.distinct) - ? (state.distinct.length === 0 - ? (() => { - throw new QueryBuilderDefect({ message: "CHQuery: distinctOn() needs at least one key" }) - })() - : state.distinct - ).map((key: string) => { - if (!(options?.selectKeys ?? keys).includes(key)) { - throw new QueryBuilderDefect({ message: `CHQuery: distinctOn(${JSON.stringify(key)}) is not a selected alias` }) - } - return raw(quoteIdent(key)) - }) - : undefined, - lock: (() => { - // Postgres refuses a lock on rows that are no longer table rows; say so here. - if (state.lock !== undefined && (state.distinct !== undefined || state.groupByKeys.length > 0 || state.havingFn !== undefined)) { - throw new QueryBuilderDefect({ - message: `CHQuery: FOR ${state.lock.strength} cannot lock rows of a query with DISTINCT, GROUP BY or HAVING`, + const sources: TenantSource[] = [fromSource] + const wherePredicates = whereConditions.flatMap((c) => (c ? tenantPredicatesOf(c) : [])) + const joinPredicates: Array<{ predicates: ReadonlyArray; target?: string }> = [] + let sql = withSubqueryCompiler((subquery) => { + if (typeof subquery === "string") { + sources.push({ scope: "cross-tenant" }) + return subquery + } + const compiled = compileInner(subquery, params, { + skipFormat: true, + deferParams, + nested: true, + enclosingCtes: visibleCtes, + }) + sources.push(sourceOf(compiled)) + return compiled.sql + }, () => { + const joins = state.typedJoins.map((j) => { + let tableSql: string + let source: TenantSource + if (j.innerQuery) { + const compiled = compileInner(j.innerQuery, params, { + skipFormat: true, + deferParams, + nested: true, + enclosingCtes: visibleCtes, }) + tableSql = `(${compiled.sql})` + source = sourceOf(compiled) + } else if (j.tableName) { + tableSql = quoteIdentPath(j.tableName) + source = sourceForTable( + j.tableName, + j.tenantColumn === undefined ? undefined : `${j.alias}.${j.tenantColumn}`, + ) + } else { + throw new QueryBuilderDefect({ message: "TypedJoin: missing table or query" }) + } + sources.push(source) + const on = j.on?.( + createQualifiedColumnAccessor(mainAlias, state.tenantColumn, columnsOf(state)), + createQualifiedColumnAccessor(j.alias, j.tenantColumn, joinColumnsOf(j)), + ) + if (on) { + // A LEFT JOIN's ON clause can constrain only its right side. It + // cannot remove unmatched rows from the preserved left side. + if (j.type !== "LEFT" || source.column !== undefined) { + joinPredicates.push({ + predicates: tenantPredicatesOf(on), + target: j.type === "LEFT" ? source.column : undefined, + }) + } + } + return { + type: j.type, + table: tableSql, + alias: quoteIdent(j.alias), + on: on ? rowFilter(`the ON clause of join ${JSON.stringify(j.alias)}`, on) : undefined, } - return lockClause(state.lock) - })(), - select: selectFragments, - from: fromFragment, - joins, - where: whereFragments, - groupBy: state.groupByKeys.map((k) => raw(groupByKey(k, options?.selectKeys ?? keys))), - // Deliberately excluded from tenant evidence: by HAVING time the - // rows are already aggregated, so the scan that produced them crossed - // tenants no matter what this filters out. - having: (state.havingFn ? state.havingFn($) : []) + }) + + // Rendered one clause at a time, each under its own track, so the + // GROUP BY rules below know what each part reads and aggregates. + const selected = selectKeys.map((alias) => { + const expr = selectExprs[alias] + const fragment = expr.toFragment() + const [sql, found] = track(() => compileSqlFragment(fragment)) + return { alias, sql, found, column: plainColumn(fragment) } + }) + const where = presentWhere.map((c) => raw(rowFilter("WHERE", c))) + const having = (state.havingFn ? state.havingFn($) : []) .filter((c): c is NonNullable => c != null) - .map((c) => c.toFragment()), - orderBy: orderByClause(state.orderBySpecs).map(raw), - limit: rowCount("limit", state.limitValue), - offset: rowCount("offset", state.offsetValue), - format: options?.skipFormat ? undefined : formatClause(state.formatValue), + .map((c) => { + const [sql, found] = track(() => compileSqlFragment(c.toFragment())) + return { sql, found } + }) + assertGrouping(selected, state.groupByKeys, having) + + const sqlQuery: SqlQuery = { + distinct: state.distinct !== undefined, + distinctOn: Array.isArray(state.distinct) + ? (state.distinct.length === 0 + ? (() => { + throw new QueryBuilderDefect({ message: "CHQuery: distinctOn() needs at least one key" }) + })() + : state.distinct + ).map((key: string) => { + if (!(options?.selectKeys ?? keys).includes(key)) { + throw new QueryBuilderDefect({ message: `CHQuery: distinctOn(${JSON.stringify(key)}) is not a selected alias` }) + } + return raw(quoteIdent(key)) + }) + : undefined, + lock: (() => { + // Postgres refuses a lock on rows that are no longer table rows; say so here. + if (state.lock !== undefined && (state.distinct !== undefined || state.groupByKeys.length > 0 || state.havingFn !== undefined)) { + throw new QueryBuilderDefect({ + message: `CHQuery: FOR ${state.lock.strength} cannot lock rows of a query with DISTINCT, GROUP BY or HAVING`, + }) + } + return lockClause(state.lock) + })(), + select: selected.map(({ alias, sql }) => raw(`${sql} AS ${quoteIdent(alias)}`)), + from: fromFragment, + joins, + where, + groupBy: state.groupByKeys.map((k) => raw(groupByKey(k, options?.selectKeys ?? keys))), + // Deliberately excluded from tenant evidence: by HAVING time the + // rows are already aggregated, so the scan that produced them crossed + // tenants no matter what this filters out. + having: having.map(({ sql }) => raw(sql)), + orderBy: orderByClause(state.orderBySpecs).map(raw), + limit: rowCount("limit", state.limitValue), + offset: rowCount("offset", state.offsetValue), + format: options?.skipFormat ? undefined : formatClause(state.formatValue), + } + + return compileQuery(sqlQuery) + }) + + // Prepend CTE definitions + if (resolvedCtes.length > 0) { + const cteDefs = resolvedCtes.map((c) => `${quoteIdent(c.name)} AS (\n${c.sql}\n)`).join(",\n") + sql = `WITH ${cteDefs}\n${sql}` } - return compileQuery(sqlQuery) + // Once, at the top: a nested query's SQL is spliced into this one, and a + // dialect that binds numbers its placeholders across the whole statement. + let parameters: ReadonlyArray = [] + if (!deferParams && options?.nested !== true) { + const rendered = renderParams(sql, params, currentDialect()) + sql = rendered.sql + parameters = rendered.parameters + } + + const scope = deriveTenantScope(sources, [{ predicates: wherePredicates }, ...joinPredicates], (value) => + deferParams ? compileSqlFragment(value) : inlineParams(compileSqlFragment(value), params), + ) + const tenantScope = state.crossTenant === true ? "cross-tenant" : scope.scope + + const derived = deriveRowSchema(selectExprs) + const derivedSchema = "schema" in derived ? derived.schema : undefined + + return withTenantBound( + makeCompiledQuery( + sql, + parameters, + tenantScope, + options?.rowSchema !== undefined ? "declared" : derivedSchema ? "derived" : "none", + () => options?.rowSchema ?? (derivedSchema as CompiledQueryRowSchema | undefined), + state.routeValue as Route, + "untyped" in derived ? derived.untyped : [], + undefined, + options?.rowSchema === undefined + ? undefined + : compareRowSchemas(options.rowSchema, derivedSchema), + currentDialect().name, + ), + tenantScope === "single-tenant" ? scope.bound : undefined, + ) }) +} - // Prepend CTE definitions - if (resolvedCtes.length > 0) { - const cteDefs = resolvedCtes.map((c) => `${quoteIdent(c.name)} AS (\n${c.sql}\n)`).join(",\n") - sql = `WITH ${cteDefs}\n${sql}` - } +/** `qualifier.name` when a fragment is a bare column, as the render tracker names it. */ +const plainColumn = (fragment: SqlFragment): string | undefined => + fragment._tag === "Ident" ? (fragment.qualifier === undefined ? fragment.name : `${fragment.qualifier}.${fragment.name}`) : undefined - // Once, at the top: a nested query's SQL is spliced into this one, and a - // dialect that binds numbers its placeholders across the whole statement. - let parameters: ReadonlyArray = [] - if (!deferParams && options?.nested !== true) { - const rendered = renderParams(sql, params, currentDialect()) - sql = rendered.sql - parameters = rendered.parameters +/** + * A condition that filters rows before they are grouped (WHERE, a join's ON). + * An aggregate has no value there yet: both databases refuse it. + */ +const rowFilter = (clause: string, condition: Condition): string => { + const [sql, found] = track(() => compileSqlFragment(condition.toFragment())) + if (found.aggregate) { + throw new QueryBuilderDefect({ + message: `CHQuery: ${clause} has an aggregate, which has no value before rows are grouped; filter on it in having()`, + }) } + return sql +} - const scope = deriveTenantScope(sources, [{ predicates: wherePredicates }, ...joinPredicates], (value) => - deferParams ? compileSqlFragment(value) : inlineParams(compileSqlFragment(value), params), - ) - const tenantScope = state.crossTenant === true ? "cross-tenant" : scope.scope - - const derived = deriveRowSchema(selectExprs) - const derivedSchema = "schema" in derived ? derived.schema : undefined - - return withTenantBound( - makeCompiledQuery( - sql, - parameters, - tenantScope, - options?.rowSchema !== undefined ? "declared" : derivedSchema ? "derived" : "none", - () => options?.rowSchema ?? (derivedSchema as CompiledQueryRowSchema | undefined), - state.routeValue as Route, - "untyped" in derived ? derived.untyped : [], - undefined, - options?.rowSchema === undefined - ? undefined - : compareRowSchemas(options.rowSchema, derivedSchema), - currentDialect().name, - ), - tenantScope === "single-tenant" ? scope.bound : undefined, - ) +/** + * The GROUP BY rule both databases enforce: once a query groups or aggregates, + * every column it reads outside an aggregate must be a grouping key. A + * selected alias may also repeat a grouped expression exactly. Only what the + * builder rendered itself is checked (see `render-tracker.ts`), so SQL it did + * not write can hide an error from this, never cause one. + */ +function assertGrouping( + selected: ReadonlyArray<{ readonly alias: string; readonly sql: string; readonly found: RenderTrack; readonly column?: string }>, + groupByKeys: ReadonlyArray, + having: ReadonlyArray<{ readonly found: RenderTrack }>, +): void { + const grouped = new Set(groupByKeys) + const aggregates = selected.some((s) => s.found.aggregate) || having.some((h) => h.found.aggregate) + if (grouped.size === 0 && !aggregates) return + for (const key of grouped) { + if (selected.find((s) => s.alias === key)?.found.aggregate === true) { + throw new QueryBuilderDefect({ message: `CHQuery: groupBy(${JSON.stringify(key)}) names an aggregate, which cannot be a grouping key` }) + } + } + const keys = selected.filter((s) => grouped.has(s.alias)) + const keyColumns = new Set(keys.flatMap((s) => (s.column === undefined ? [] : [s.column]))) + const keySql = new Set(keys.map((s) => s.sql)) + const ungrouped = (columns: ReadonlySet) => [...columns].find((column) => !keyColumns.has(column)) + for (const s of selected) { + if (grouped.has(s.alias) || keySql.has(s.sql)) continue + const column = ungrouped(s.found.columns) + if (column !== undefined) { + throw new QueryBuilderDefect({ + message: `CHQuery: select alias ${JSON.stringify(s.alias)} reads ${column}, which is neither a groupBy() key nor inside an aggregate`, + }) + } + } + for (const h of having) { + const column = ungrouped(h.found.columns) + if (column !== undefined) { + throw new QueryBuilderDefect({ + message: `CHQuery: having() reads ${column}, which is neither a groupBy() key nor inside an aggregate`, + }) + } + } } /** diff --git a/src/ch/dialect.ts b/src/ch/dialect.ts index 1574915..06c9384 100644 --- a/src/ch/dialect.ts +++ b/src/ch/dialect.ts @@ -143,6 +143,13 @@ export interface Dialect extends SqlSyntax { readonly paramCodecs?: Readonly>> /** Transaction support. Absent means none. */ readonly transactions?: DialectTransactions + /** + * Which built-in function set renders correctly here: `clickhouse` (the + * root entry's functions) or `postgres` (`@maple-dev/effect-orm/postgres`). + * A built-in function from another set fails to compile. Absent means + * unchecked: a custom dialect says which set it renders, if either. + */ + readonly functions?: string } /** ClickHouse, with params written into the SQL as literals. The default. */ @@ -164,6 +171,7 @@ export const clickhouseDialect: Dialect = { alterTableUpdate: true, }, transactions: noTransactions, + functions: "clickhouse", } // The dialect of the enclosing compile, beside the syntax installed for the @@ -173,6 +181,9 @@ let current: Dialect | undefined /** The dialect of the enclosing compile, or ClickHouse outside one. */ export const currentDialect = (): Dialect => current ?? clickhouseDialect +/** The dialect of the enclosing compile, or `undefined` outside one. */ +export const activeDialect = (): Dialect | undefined => current + /** Run `body` with `dialect`'s syntax installed, literals checked as above. */ export function withDialect(dialect: Dialect, body: () => A): A { // A nested compile for the dialect already installed keeps the checked diff --git a/src/ch/expr.ts b/src/ch/expr.ts index a6e4037..e7ec506 100644 --- a/src/ch/expr.ts +++ b/src/ch/expr.ts @@ -8,7 +8,7 @@ import { DateTime, Result, Schema } from "effect" import type { SqlFragment } from "../sql/sql-fragment" -import { raw, str, ident, compile, as_ as sqlAs, lazy } from "../sql/sql-fragment" +import { raw, str, ident, compile, as_ as sqlAs, known } from "../sql/sql-fragment" import { activeSqlSyntax } from "../sql/sql-syntax" import { chDateTimeLiteral, CHFloatResult, CHNumber, string as chString, type CHType, type InferTS } from "./types" import { encodeColumnLiteral } from "./literal" @@ -180,12 +180,12 @@ export function toFragment(value: unknown): SqlFragment { if (isExprLike(value)) return value.toFragment() if (typeof value === "string") return str(value) if (typeof value === "number") return raw(String(value)) - if (typeof value === "boolean") return lazy(() => untypedLiteral(value)) + if (typeof value === "boolean") return known(() => untypedLiteral(value)) // A DateTime column compares against a DateTime value, so the literal has to // be the dialect's own form (ClickHouse's is tz-less) rather than whatever // `String(value)` produces. - if (DateTime.isDateTime(value)) return lazy(() => dateTimeLiteral(DateTime.toUtc(value))) - if (value instanceof Date) return lazy(() => dateTimeLiteral(DateTime.makeUnsafe(value))) + if (DateTime.isDateTime(value)) return known(() => dateTimeLiteral(DateTime.toUtc(value))) + if (value instanceof Date) return known(() => dateTimeLiteral(DateTime.makeUnsafe(value))) return raw(String(value)) } @@ -222,7 +222,7 @@ const inCond = ( values: ReadonlyArray<() => SqlFragment>, ): Condition => makeCond( - lazy(() => + known(() => values.length === 0 ? op === "IN" ? "1 = 0" @@ -257,7 +257,7 @@ const arith = ( // `+`, `-`, `*` can overflow a Float64 to `inf`, sent as JSON null: NaN. const overflows = op === "+" || op === "-" || op === "*" return makeExpr( - lazy(() => `${compile(lhs)} ${op} ${compile(toFragment(rhs))}`), + known(() => `${compile(lhs)} ${op} ${compile(toFragment(rhs))}`), (nullable ? Schema.NullOr(CHNumber) : overflows ? CHFloatResult : CHNumber) as Schema.Codec, ) } @@ -297,23 +297,23 @@ export function makeExpr( ...(schema !== undefined ? { schema } : undefined), toFragment: () => fragment, - eq: (other) => makeCond(lazy(() => `${compile(fragment)} = ${compile(operand(other))}`)), - neq: (other) => makeCond(lazy(() => `${compile(fragment)} != ${compile(operand(other))}`)), - gt: (other) => makeCond(lazy(() => `${compile(fragment)} > ${compile(operand(other))}`)), - gte: (other) => makeCond(lazy(() => `${compile(fragment)} >= ${compile(operand(other))}`)), - lt: (other) => makeCond(lazy(() => `${compile(fragment)} < ${compile(operand(other))}`)), - lte: (other) => makeCond(lazy(() => `${compile(fragment)} <= ${compile(operand(other))}`)), + eq: (other) => makeCond(known(() => `${compile(fragment)} = ${compile(operand(other))}`)), + neq: (other) => makeCond(known(() => `${compile(fragment)} != ${compile(operand(other))}`)), + gt: (other) => makeCond(known(() => `${compile(fragment)} > ${compile(operand(other))}`)), + gte: (other) => makeCond(known(() => `${compile(fragment)} >= ${compile(operand(other))}`)), + lt: (other) => makeCond(known(() => `${compile(fragment)} < ${compile(operand(other))}`)), + lte: (other) => makeCond(known(() => `${compile(fragment)} <= ${compile(operand(other))}`)), - isNull: () => makeCond(lazy(() => `${compile(fragment)} IS NULL`)), - isNotNull: () => makeCond(lazy(() => `${compile(fragment)} IS NOT NULL`)), + isNull: () => makeCond(known(() => `${compile(fragment)} IS NULL`)), + isNotNull: () => makeCond(known(() => `${compile(fragment)} IS NOT NULL`)), between: (low, high) => - makeCond(lazy(() => `${compile(fragment)} BETWEEN ${compile(operand(low))} AND ${compile(operand(high))}`)), + makeCond(known(() => `${compile(fragment)} BETWEEN ${compile(operand(low))} AND ${compile(operand(high))}`)), notBetween: (low, high) => - makeCond(lazy(() => `${compile(fragment)} NOT BETWEEN ${compile(operand(low))} AND ${compile(operand(high))}`)), + makeCond(known(() => `${compile(fragment)} NOT BETWEEN ${compile(operand(low))} AND ${compile(operand(high))}`)), - like: (pattern: string) => makeCond(lazy(() => `${compile(fragment)} LIKE ${compile(str(pattern))}`)), - notLike: (pattern: string) => makeCond(lazy(() => `${compile(fragment)} NOT LIKE ${compile(str(pattern))}`)), - ilike: (pattern: string) => makeCond(lazy(() => `${compile(fragment)} ILIKE ${compile(str(pattern))}`)), + like: (pattern: string) => makeCond(known(() => `${compile(fragment)} LIKE ${compile(str(pattern))}`)), + notLike: (pattern: string) => makeCond(known(() => `${compile(fragment)} NOT LIKE ${compile(str(pattern))}`)), + ilike: (pattern: string) => makeCond(known(() => `${compile(fragment)} ILIKE ${compile(str(pattern))}`)), in_: (...values) => inCond(fragment, "IN", values.map((v) => () => operand(v))), notIn: (...values) => inCond(fragment, "NOT IN", values.map((v) => () => operand(v))), @@ -437,7 +437,7 @@ export function makeColumnRef { - return makeExpr(lazy(() => `${compile(fragment)}[${compile(str(key))}]`), columnType?.element?.schema) + return makeExpr(known(() => `${compile(fragment)}[${compile(str(key))}]`), columnType?.element?.schema) }, }, ) as ColumnRef @@ -451,11 +451,11 @@ export function makeCond(fragment: SqlFragment): Condition { toFragment: () => fragment, and(other) { return markTenantPredicate( - makeCond(lazy(() => `(${compile(fragment)} AND ${compile(other.toFragment())})`)), + makeCond(known(() => `(${compile(fragment)} AND ${compile(other.toFragment())})`)), [...tenantPredicatesOf(this), ...tenantPredicatesOf(other)], ) }, - or: (other) => makeCond(lazy(() => `(${compile(fragment)} OR ${compile(other.toFragment())})`)), + or: (other) => makeCond(known(() => `(${compile(fragment)} OR ${compile(other.toFragment())})`)), } } @@ -508,7 +508,7 @@ export function and(...conditions: ReadonlyArray): Condit const present = conditions.filter((c): c is Condition => c !== undefined) if (present.length <= 1) return present[0] return markTenantPredicate( - makeCond(lazy(() => `(${present.map((c) => compile(c.toFragment())).join(" AND ")})`)), + makeCond(known(() => `(${present.map((c) => compile(c.toFragment())).join(" AND ")})`)), present.flatMap((c) => tenantPredicatesOf(c)), ) } @@ -522,12 +522,12 @@ export function or(...conditions: ReadonlyArray): Conditi export function or(...conditions: ReadonlyArray): Condition | undefined { const present = conditions.filter((c): c is Condition => c !== undefined) if (present.length <= 1) return present[0] - return makeCond(lazy(() => `(${present.map((c) => compile(c.toFragment())).join(" OR ")})`)) + return makeCond(known(() => `(${present.map((c) => compile(c.toFragment())).join(" OR ")})`)) } /** Wrap a condition in NOT (...). */ export function not(condition: Condition): Condition { - return makeCond(lazy(() => `NOT (${compile(condition.toFragment())})`)) + return makeCond(known(() => `NOT (${compile(condition.toFragment())})`)) } // Raw expression (escape hatch) diff --git a/src/ch/functions/aggregate.ts b/src/ch/functions/aggregate.ts index 74a2753..19e389d 100644 --- a/src/ch/functions/aggregate.ts +++ b/src/ch/functions/aggregate.ts @@ -1,12 +1,15 @@ -import { defineFn, compileTypedFnCall, numericResultSchema, overflowResultSchema } from "../define-fn" +import { numericResultSchema, overflowResultSchema } from "../define-fn" import { QueryBuilderError } from "../errors" import { makeExpr } from "../expr" -import { compile, lazy } from "../../sql/sql-fragment" +import { compile } from "../../sql/sql-fragment" import type { Expr, Condition } from "../expr" import { type DateTime, Schema } from "effect" import * as T from "../types" import { arrayOfArg, sameAs, schemaOf } from "../define-fn" +import { builtins } from "./builtin" + +const { compileTypedFnCall, defineFn, lazy } = builtins("clickhouse", "aggregate") /** `groupUniqArrayIf(x, cond)` collects `x`s, so it decodes as an array of `x`. */ const arraySchemaOf = (expr: unknown) => { diff --git a/src/ch/functions/array.ts b/src/ch/functions/array.ts index 84a022e..ed343d8 100644 --- a/src/ch/functions/array.ts +++ b/src/ch/functions/array.ts @@ -1,9 +1,12 @@ import { makeCond, makeExpr, toFragment } from "../expr" -import { str, compile, lazy } from "../../sql/sql-fragment" +import { str, compile } from "../../sql/sql-fragment" import type { Condition, Expr } from "../expr" import { Schema } from "effect" import * as T from "../types" -import { defineFn, elementOf, elementSchema, mergeResultSchemas, sameAs, schemaOf } from "../define-fn" +import { elementOf, elementSchema, mergeResultSchemas, sameAs, schemaOf } from "../define-fn" +import { builtins } from "./builtin" + +const { defineFn, lazy } = builtins("clickhouse", "scalar") // Array constructors (handwritten — bracket syntax, not fn() call) diff --git a/src/ch/functions/builtin.ts b/src/ch/functions/builtin.ts new file mode 100644 index 0000000..2304235 --- /dev/null +++ b/src/ch/functions/builtin.ts @@ -0,0 +1,83 @@ +// The builder's own functions. +// +// Every built-in function renders through one of these sets, which does two +// things a caller-declared function cannot: +// +// - It refuses to render for a dialect whose function set it is not from: +// ClickHouse's `count()` is an error on Postgres, which wants `count(*)`, +// and most ClickHouse functions do not exist there at all. Outside a +// compile nothing is checked. +// - It tells the render tracker what it is (`render-tracker.ts`): a scalar +// function whose arguments it may count, an aggregate, or a window, whose +// insides it does not count. That is what lets `compile` refuse an aggregate +// in WHERE, or a column next to an aggregate that is not grouped. + +import type { Schema } from "effect" +import { compile, known, type SqlFragment } from "../../sql/sql-fragment" +import { hidden, inAggregate } from "../../sql/render-tracker" +import { activeDialect } from "../dialect" +import { QueryBuilderDefect } from "../errors" +import { type Condition, type Expr, makeCond, makeExpr, makeUntypedExpr, toFragment } from "../expr" +import type { FnResult } from "../define-fn" + +export type FunctionSet = "clickhouse" | "postgres" +export type BuiltinKind = "scalar" | "aggregate" | "window" + +const setLabel: Record = { + clickhouse: "a ClickHouse function (from the root entry)", + postgres: "a Postgres function (from @maple-dev/effect-orm/postgres)", +} + +const assertSet = (set: FunctionSet | "portable", name: string | undefined): void => { + if (set === "portable") return + const dialect = activeDialect() + if (dialect === undefined) return + if (dialect.functions !== undefined && dialect.functions !== set) { + throw new QueryBuilderDefect({ + message: `${name === undefined ? "this function" : `${name}()`} is ${setLabel[set]} and has no meaning for the ${dialect.name} dialect; use that dialect's own function`, + }) + } +} + +/** Built-in functions of one set and kind. `portable` renders for every dialect. */ +export function builtins(set: FunctionSet | "portable", kind: BuiltinKind) { + const wrap = + kind === "aggregate" ? inAggregate : kind === "window" ? hidden : (render: () => string): string => render() + + /** A built-in function's SQL, rendered lazily. */ + const lazy = (render: () => string, name?: string): SqlFragment => + known(() => { + assertSet(set, name) + return wrap(render) + }) + + const call = (name: string, args: ReadonlyArray): SqlFragment => + lazy(() => `${name}(${args.map((a) => compile(toFragment(a))).join(", ")})`, name) + + function compileTypedFnCall(name: string, schema: Schema.Codec | undefined, ...args: unknown[]): Expr { + return makeExpr(call(name, args), schema) + } + + function compileFnCall(name: string, ...args: unknown[]): Expr { + return makeUntypedExpr(call(name, args)) + } + + function compileFnCallCond(name: string, ...args: unknown[]): Condition { + return makeCond(call(name, args)) + } + + function defineFn(name: string, result: FnResult): (...args: Args) => Expr { + return (...args: Args): Expr => + compileTypedFnCall( + name, + typeof result === "function" ? result(...args) : (result.schema as Schema.Codec), + ...args, + ) + } + + function defineCondFn(name: string): (...args: Args) => Condition { + return (...args: Args): Condition => compileFnCallCond(name, ...args) + } + + return { lazy, compileTypedFnCall, compileFnCall, compileFnCallCond, defineFn, defineCondFn } +} diff --git a/src/ch/functions/conditional.ts b/src/ch/functions/conditional.ts index 6177eb6..19f4e50 100644 --- a/src/ch/functions/conditional.ts +++ b/src/ch/functions/conditional.ts @@ -1,8 +1,12 @@ import { makeExpr, toFragment } from "../expr" -import { compile, lazy } from "../../sql/sql-fragment" +import { compile } from "../../sql/sql-fragment" import type { Expr, Condition } from "../expr" import { Schema } from "effect" -import { compileTypedFnCall, defineFn, firstTypedNonNull, mergeResultSchemas, numericResultSchema, schemaOf } from "../define-fn" +import { firstTypedNonNull, mergeResultSchemas, numericResultSchema, schemaOf } from "../define-fn" +import { builtins } from "./builtin" + +const { compileTypedFnCall, defineFn, lazy } = builtins("clickhouse", "scalar") +const portable = builtins("portable", "scalar") // if / multiIf (handwritten — standard fn shape but special arg types) @@ -45,7 +49,7 @@ type Coalesced[]> = Args extends readonly [ : null export const coalesce = []>(...exprs: Args): Expr> => - defineFn>("coalesce", firstTypedNonNull())(...exprs) + portable.defineFn>("coalesce", firstTypedNonNull())(...exprs) /** * `ifNull(expr, fallback)` — `expr` unless it is NULL, else `fallback`. The @@ -57,7 +61,7 @@ export const ifNull = (expr: Expr, fallback: Expr): Expr => export function nullIf(expr: Expr, value: Expr | T): Expr { // The result is `expr` or NULL, so it decodes as `expr` does — nullably. const schema = schemaOf(expr) - return compileTypedFnCall("nullIf", schema && Schema.NullOr(schema), expr, value) + return portable.compileTypedFnCall("nullIf", schema && Schema.NullOr(schema), expr, value) } /** diff --git a/src/ch/functions/date-time.ts b/src/ch/functions/date-time.ts index 560fa00..e9f7a51 100644 --- a/src/ch/functions/date-time.ts +++ b/src/ch/functions/date-time.ts @@ -1,9 +1,12 @@ import { type DateTime, SchemaAST } from "effect" import { makeExpr } from "../expr" import { schemaOf } from "../define-fn" -import { str, compile, lazy } from "../../sql/sql-fragment" +import { str, compile } from "../../sql/sql-fragment" import type { Expr } from "../expr" import * as T from "../types" +import { builtins } from "./builtin" + +const { lazy } = builtins("clickhouse", "scalar") /** * A DateTime-valued expression: a column, a param, or `now()`. diff --git a/src/ch/functions/json.ts b/src/ch/functions/json.ts index c2a2b9e..ec55703 100644 --- a/src/ch/functions/json.ts +++ b/src/ch/functions/json.ts @@ -1,5 +1,7 @@ -import { defineFn } from "../define-fn" import type { Expr } from "../expr" import * as T from "../types" +import { builtins } from "./builtin" + +const { defineFn } = builtins("clickhouse", "scalar") export const toJSONString = defineFn<[Expr], string>("toJSONString", T.string) diff --git a/src/ch/functions/map.ts b/src/ch/functions/map.ts index b390fe3..5033d9d 100644 --- a/src/ch/functions/map.ts +++ b/src/ch/functions/map.ts @@ -1,7 +1,10 @@ import { makeCond, makeExpr } from "../expr" -import { raw, str, compile, lazy } from "../../sql/sql-fragment" +import { raw, str, compile } from "../../sql/sql-fragment" import type { Expr, Condition } from "../expr" import * as T from "../types" +import { builtins } from "./builtin" + +const { lazy } = builtins("clickhouse", "scalar") const STRINGS = T.array(T.string).schema const STRING_MAP = T.map(T.string, T.string).schema @@ -25,7 +28,7 @@ export function mapValues(mapExpr: Expr>): Expr]>): Expr> { - if (pairs.length === 0) return makeExpr(raw("map()"), STRING_MAP) + if (pairs.length === 0) return makeExpr(lazy(() => "map()", "map"), STRING_MAP) const args = () => pairs.map(([k, v]) => `${compile(str(k))}, ${compile(v.toFragment())}`).join(", ") return makeExpr(lazy(() => `map(${args()})`), STRING_MAP) } diff --git a/src/ch/functions/numeric.ts b/src/ch/functions/numeric.ts index 3cdf113..18b63d5 100644 --- a/src/ch/functions/numeric.ts +++ b/src/ch/functions/numeric.ts @@ -1,6 +1,9 @@ -import { compileTypedFnCall, defineFn, numericResultSchema, firstTypedNonNull } from "../define-fn" +import { numericResultSchema, firstTypedNonNull } from "../define-fn" import type { Expr } from "../expr" import * as T from "../types" +import { builtins } from "./builtin" + +const { compileTypedFnCall, defineFn } = builtins("clickhouse", "scalar") // Type conversion (defineFn one-liners) diff --git a/src/ch/functions/string.ts b/src/ch/functions/string.ts index 420e579..caf4421 100644 --- a/src/ch/functions/string.ts +++ b/src/ch/functions/string.ts @@ -1,9 +1,13 @@ -import { acceptsSqlNull, compileFnCall, compileTypedFnCall, defineFn } from "../define-fn" +import { acceptsSqlNull } from "../define-fn" import { Schema } from "effect" import { makeCond } from "../expr" -import { compile, str, lazy } from "../../sql/sql-fragment" +import { compile, str } from "../../sql/sql-fragment" import type { Condition, Expr } from "../expr" import * as T from "../types" +import { builtins } from "./builtin" + +const { compileFnCall, compileTypedFnCall, defineFn, lazy } = builtins("clickhouse", "scalar") +const portable = builtins("portable", "scalar") // Standard string functions (defineFn one-liners) @@ -14,7 +18,7 @@ const stringResult = (name: string, expr: Expr): Expr(expr: Expr): Expr> => stringResult("toString", expr) export const length_ = defineFn<[Expr], number>("length", T.uint64) -export const lower_ = defineFn<[Expr], string>("lower", T.string) +export const lower_ = portable.defineFn<[Expr], string>("lower", T.string) export const positionCaseInsensitive = defineFn<[Expr, Expr], number>( "positionCaseInsensitive", T.uint64, diff --git a/src/ch/functions/window.ts b/src/ch/functions/window.ts index 1378004..4fb2b7c 100644 --- a/src/ch/functions/window.ts +++ b/src/ch/functions/window.ts @@ -1,8 +1,12 @@ import { makeExpr, toFragment } from "../expr" -import { compile, lazy } from "../../sql/sql-fragment" +import { compile } from "../../sql/sql-fragment" import type { Expr } from "../expr" import { schemaOf } from "../define-fn" import { QueryBuilderError } from "../errors" +import { builtins } from "./builtin" + +const { lazy } = builtins("clickhouse", "scalar") +const window = builtins("clickhouse", "window") export type WindowOrderDirection = "asc" | "desc" @@ -82,7 +86,7 @@ function renderWindowSpec(spec: WindowSpec): string { export function over(expr: Expr, spec: CompiledWindowSpec): Expr { // A window changes which rows feed the value, never how the value decodes. - return makeExpr(lazy(() => `${compile(expr.toFragment())} OVER (${spec.sql})`), schemaOf(expr)) + return makeExpr(window.lazy(() => `${compile(expr.toFragment())} OVER (${spec.sql})`), schemaOf(expr)) } export function lagInFrame( diff --git a/src/ch/soundness.test.ts b/src/ch/soundness.test.ts index 5a885b5..4f1a7f0 100644 --- a/src/ch/soundness.test.ts +++ b/src/ch/soundness.test.ts @@ -7,6 +7,7 @@ import { describe, expect, it } from "@effect/vitest" import * as CH from "./index" import { compileCHUnsafe, compileUnionUnsafe } from "./compile" +import * as PG from "../postgres" const Users = CH.table("users", { Id: CH.string, Name: CH.string, Nick: CH.nullable(CH.string), Age: CH.uint64 }) const Orders = CH.table("orders", { Id: CH.string, UserId: CH.string, Amount: CH.uint64 }) @@ -105,3 +106,91 @@ describe("source names", () => { ).toThrow(/defined twice/) }) }) + +describe("aggregates and GROUP BY", () => { + const q = () => CH.from(Users) + + it("refuses an aggregate in WHERE or a join's ON", () => { + expect(() => compileCHUnsafe(q().select("Id").where(() => [CH.count().gt(1)]))).toThrow(/WHERE has an aggregate/) + expect(() => + compileCHUnsafe( + q() + .innerJoin(Orders, "o", (u, o) => u.Id.eq(o.UserId).and(CH.sum(o.Amount).gt(1))) + .select("Id"), + ), + ).toThrow(/ON clause of join "o" has an aggregate/) + }) + + it("refuses a column that is neither grouped nor aggregated", () => { + expect(() => compileCHUnsafe(q().select(($) => ({ Name: $.Name, n: CH.count() })))).toThrow( + /"Name" reads users.Name/, + ) + expect(() => + compileCHUnsafe(q().select(($) => ({ Name: $.Name, Age: $.Age, n: CH.count() })).groupBy("Name")), + ).toThrow(/"Age" reads users.Age/) + expect(() => compileCHUnsafe(q().select(($) => ({ x: $.Age.add(CH.count()) })))).toThrow(/"x" reads users.Age/) + expect(() => + compileCHUnsafe( + q() + .select(($) => ({ Name: $.Name, n: CH.count() })) + .groupBy("Name") + .having(($) => [$.Age.gt(1)]), + ), + ).toThrow(/having\(\) reads users.Age/) + }) + + it("refuses grouping by an aggregate", () => { + expect(() => compileCHUnsafe(q().select(() => ({ n: CH.count() })).groupBy("n"))).toThrow(/names an aggregate/) + }) + + it("accepts grouped columns, expressions over them, and repeats of a grouped expression", () => { + const grouped = q() + .select(($) => ({ Name: $.Name, upper: CH.lower($.Name), n: CH.count(), total: CH.sum($.Age) })) + .groupBy("Name") + .having(($) => [$.Name.neq(""), CH.count().gt(1)]) + expect(compileCHUnsafe(grouped).sql).toContain("GROUP BY Name") + const byExpr = q() + .select(($) => ({ k: CH.lower($.Name), again: CH.lower($.Name), n: CH.count() })) + .groupBy("k") + expect(compileCHUnsafe(byExpr).sql).toContain("GROUP BY k") + expect(compileCHUnsafe(q().select(() => ({ n: CH.count(), one: CH.lit(1) }))).sql).toContain("count()") + }) + + it("does not count what is inside a window or SQL the builder did not write", () => { + const windowed = q().select(($) => ({ + Name: $.Name, + running: CH.over(CH.sum($.Age), CH.windowSpec({ orderBy: [[$.Name, "asc"]] })), + })) + expect(compileCHUnsafe(windowed).sql).toContain("OVER") + const opaque = q() + .select(($) => ({ Name: $.Name, t: CH.sql(CH.float64)`quantileTDigest(0.9)(${$.Age})` })) + .groupBy("Name") + expect(compileCHUnsafe(opaque).sql).toContain("quantileTDigest") + }) + + it("keeps a subquery's columns out of the outer check", () => { + const inner = CH.from(Orders).select("UserId").where(($) => [$.Amount.gt(1)]) + const outer = q() + .select(($) => ({ Name: $.Name, n: CH.count() })) + .where(($) => [CH.inSubquery($.Id, inner)]) + .groupBy("Name") + expect(compileCHUnsafe(outer).sql).toContain("IN (") + }) +}) + +describe("function sets", () => { + it("refuses a ClickHouse function on Postgres, and a Postgres one on ClickHouse", () => { + expect(() => PG.compileUnsafe(CH.from(Users).select(() => ({ n: CH.count() })))).toThrow( + /count\(\) is a ClickHouse function/, + ) + expect(() => compileCHUnsafe(CH.from(Users).select(() => ({ n: PG.count() })))).toThrow(/count\(\) is a Postgres function/) + expect(PG.compileUnsafe(CH.from(Users).select(() => ({ n: PG.count() }))).sql).toContain("count(*)") + }) + + it("renders the portable ones anywhere", () => { + const sql = PG.compileUnsafe( + CH.from(Users).select(($) => ({ n: CH.coalesce($.Nick, $.Name), l: CH.lower($.Name), z: CH.nullIf($.Name, "") })), + ).sql + expect(sql).toContain("coalesce(") + }) +}) diff --git a/src/pg/dialect.ts b/src/pg/dialect.ts index 91d772e..b3f3a38 100644 --- a/src/pg/dialect.ts +++ b/src/pg/dialect.ts @@ -86,6 +86,7 @@ const placeholderCasts: Readonly> = { */ export const postgresDialect: Dialect = { name: "postgres", + functions: "postgres", quoteIdent, quoteString, literal, diff --git a/src/pg/functions.ts b/src/pg/functions.ts index 4b13813..1e72b26 100644 --- a/src/pg/functions.ts +++ b/src/pg/functions.ts @@ -10,39 +10,43 @@ import { Schema, type DateTime } from "effect" import { QueryBuilderDefect } from "../ch/errors" import { makeExpr, type Condition, type Expr } from "../ch/expr" import { schemaOf, withoutNull } from "../ch/define-fn" -import { compile, lazy, raw, str } from "../sql/sql-fragment" +import { compile, str } from "../sql/sql-fragment" +import { builtins } from "../ch/functions/builtin" import * as T from "./types" const sql = (expr: Expr | Condition): string => compile(expr.toFragment()) +const aggregate = builtins("postgres", "aggregate") +const scalar = builtins("postgres", "scalar") + const nullableNumber = Schema.NullOr(T.PgNumber) as Schema.Codec const int8 = T.int8.schema as Schema.Codec // Aggregates /** `count(*)`. */ -export const count = (): Expr => makeExpr(raw("count(*)"), int8) +export const count = (): Expr => makeExpr(aggregate.lazy(() => "count(*)", "count"), int8) /** `count(DISTINCT expr)`. */ export const countDistinct = (expr: Expr): Expr => - makeExpr(lazy(() => `count(DISTINCT ${sql(expr)})`), int8) + makeExpr(aggregate.lazy(() => `count(DISTINCT ${sql(expr)})`), int8) /** `count(*) FILTER (WHERE condition)`: ClickHouse's `countIf`. */ export const countIf = (condition: Condition): Expr => - makeExpr(lazy(() => `count(*) FILTER (WHERE ${sql(condition)})`), int8) + makeExpr(aggregate.lazy(() => `count(*) FILTER (WHERE ${sql(condition)})`), int8) /** `sum(expr)`. NULL over no rows, and a string for int8/numeric inputs on the * wire, which the result codec reads as a number. */ export const sum = (expr: Expr): Expr => - makeExpr(lazy(() => `sum(${sql(expr)})`), nullableNumber) + makeExpr(aggregate.lazy(() => `sum(${sql(expr)})`), nullableNumber) /** `sum(expr) FILTER (WHERE condition)`: ClickHouse's `sumIf`. */ export const sumIf = (expr: Expr, condition: Condition): Expr => - makeExpr(lazy(() => `sum(${sql(expr)}) FILTER (WHERE ${sql(condition)})`), nullableNumber) + makeExpr(aggregate.lazy(() => `sum(${sql(expr)}) FILTER (WHERE ${sql(condition)})`), nullableNumber) /** `avg(expr)`. NULL over no rows. */ export const avg = (expr: Expr): Expr => - makeExpr(lazy(() => `avg(${sql(expr)})`), nullableNumber) + makeExpr(aggregate.lazy(() => `avg(${sql(expr)})`), nullableNumber) const nullableOf = (expr: Expr): Schema.Codec | undefined => { const schema = schemaOf(expr) @@ -50,10 +54,10 @@ const nullableOf = (expr: Expr): Schema.Codec | undefin } /** `min(expr)`, decoding as `expr` does. NULL over no rows. */ -export const min = (expr: Expr): Expr => makeExpr(lazy(() => `min(${sql(expr)})`), nullableOf(expr)) +export const min = (expr: Expr): Expr => makeExpr(aggregate.lazy(() => `min(${sql(expr)})`), nullableOf(expr)) /** `max(expr)`, decoding as `expr` does. NULL over no rows. */ -export const max = (expr: Expr): Expr => makeExpr(lazy(() => `max(${sql(expr)})`), nullableOf(expr)) +export const max = (expr: Expr): Expr => makeExpr(aggregate.lazy(() => `max(${sql(expr)})`), nullableOf(expr)) /** `percentile_cont(fraction) WITHIN GROUP (ORDER BY expr)`: an interpolated * quantile, ClickHouse's `quantileExact` family. */ @@ -61,14 +65,14 @@ export const percentileCont = (fraction: number, expr: Expr): Exp if (!(fraction >= 0 && fraction <= 1)) { throw new QueryBuilderDefect({ message: `percentileCont: fraction must be within [0, 1], got ${fraction}` }) } - return makeExpr(lazy(() => `percentile_cont(${fraction}) WITHIN GROUP (ORDER BY ${sql(expr)})`), nullableNumber) + return makeExpr(aggregate.lazy(() => `percentile_cont(${fraction}) WITHIN GROUP (ORDER BY ${sql(expr)})`), nullableNumber) } /** `array_agg(expr)`. NULL over no rows. */ export const arrayAgg = (expr: Expr): Expr | null> => { const element = schemaOf(expr) return makeExpr( - lazy(() => `array_agg(${sql(expr)})`), + aggregate.lazy(() => `array_agg(${sql(expr)})`), element === undefined ? undefined : (Schema.NullOr(Schema.Array(element)) as Schema.Codec | null, unknown>), ) } @@ -82,7 +86,7 @@ export type DateTruncUnit = "second" | "minute" | "hour" | "day" | "week" | "mon /** `date_trunc(unit, ts, 'UTC')`: buckets in UTC whatever the session time * zone, as ClickHouse's `toStartOf*` functions do. Postgres 12+. */ export const dateTrunc = (unit: DateTruncUnit, ts: Expr): Expr => - makeExpr(lazy(() => `date_trunc(${compile(str(unit))}, ${sql(ts)}, 'UTC')`), timestamptz) + makeExpr(scalar.lazy(() => `date_trunc(${compile(str(unit))}, ${sql(ts)}, 'UTC')`), timestamptz) /** `date_bin(seconds, ts, epoch)`: fixed-width buckets aligned to the Unix * epoch, ClickHouse's `toStartOfInterval`. Postgres 14+. */ @@ -91,30 +95,30 @@ export const dateBin = (seconds: number, ts: Expr): Expr `date_bin(make_interval(secs => ${seconds}), ${sql(ts)}, TIMESTAMPTZ '1970-01-01 00:00:00+00')`), + scalar.lazy(() => `date_bin(make_interval(secs => ${seconds}), ${sql(ts)}, TIMESTAMPTZ '1970-01-01 00:00:00+00')`), timestamptz, ) } /** `now()`: the transaction's start time. */ -export const now = (): Expr => makeExpr(raw("now()"), timestamptz) +export const now = (): Expr => makeExpr(scalar.lazy(() => "now()", "now"), timestamptz) // Strings and values const text = T.text.schema as Schema.Codec -export const lower = (expr: Expr): Expr => makeExpr(lazy(() => `lower(${sql(expr)})`), text) -export const upper = (expr: Expr): Expr => makeExpr(lazy(() => `upper(${sql(expr)})`), text) +export const lower = (expr: Expr): Expr => makeExpr(scalar.lazy(() => `lower(${sql(expr)})`), text) +export const upper = (expr: Expr): Expr => makeExpr(scalar.lazy(() => `upper(${sql(expr)})`), text) export const length = (expr: Expr): Expr => - makeExpr(lazy(() => `length(${sql(expr)})`), T.int4.schema as Schema.Codec) + makeExpr(scalar.lazy(() => `length(${sql(expr)})`), T.int4.schema as Schema.Codec) /** `coalesce(expr, fallback)`, no longer nullable. */ export const coalesce = (expr: Expr, fallback: Expr): Expr => makeExpr( - lazy(() => `coalesce(${sql(expr)}, ${sql(fallback)})`), + scalar.lazy(() => `coalesce(${sql(expr)}, ${sql(fallback)})`), schemaOf(fallback) ?? withoutNull(schemaOf(expr)), ) /** `expr ->> key`: a jsonb field as text, NULL when it is absent. */ export const jsonText = (expr: Expr, key: string): Expr => - makeExpr(lazy(() => `(${sql(expr)} ->> ${compile(str(key))})`), Schema.NullOr(Schema.String) as Schema.Codec) + makeExpr(scalar.lazy(() => `(${sql(expr)} ->> ${compile(str(key))})`), Schema.NullOr(Schema.String) as Schema.Codec) diff --git a/src/sql/render-tracker.ts b/src/sql/render-tracker.ts new file mode 100644 index 0000000..314b244 --- /dev/null +++ b/src/sql/render-tracker.ts @@ -0,0 +1,87 @@ +// What a clause turned out to contain, learned while rendering it. +// +// Fragments are opaque until they render, so the only place to learn whether a +// SELECT expression aggregates, or which columns it reads outside an aggregate, +// is the render itself. `compile` renders each clause under `track` and reads +// the result; column identifiers and the builder's own functions report into +// it as they render. +// +// Only what the builder can vouch for is counted. A fragment it did not build +// (a `rawExpr`, a `sql` template, a function a caller declared with `makeExpr`) +// renders as opaque: nothing inside it is counted, so an unknown function is +// never mistaken for a scalar one wrapping a bare column. The checks built on +// this can miss an error inside opaque SQL, never invent one. +// +// Rendering is synchronous, and every entry point saves and restores, so a +// nested compile (a subquery) and a callback that throws leave no state behind. + +/** What one tracked render found. */ +export interface RenderTrack { + /** An aggregate function was rendered outside any window. */ + aggregate: boolean + /** Columns read outside any aggregate, window or opaque fragment, as + * `qualifier.name` (or `name`). */ + readonly columns: Set +} + +interface State { + readonly track: RenderTrack + aggregateDepth: number + /** Inside a window or an opaque fragment: nothing is counted. */ + hiddenDepth: number +} + +let current: State | undefined + +/** Render `body` with a fresh track, returning both. */ +export function track(body: () => A): [A, RenderTrack] { + const previous = current + const state: State = { track: { aggregate: false, columns: new Set() }, aggregateDepth: 0, hiddenDepth: 0 } + current = state + try { + return [body(), state.track] + } finally { + current = previous + } +} + +/** Render `body` with no track: a nested query's columns are its own. */ +export function untracked(body: () => A): A { + const previous = current + current = undefined + try { + return body() + } finally { + current = previous + } +} + +/** A column identifier rendered. */ +export function noteColumn(name: string): void { + if (current !== undefined && current.aggregateDepth === 0 && current.hiddenDepth === 0) current.track.columns.add(name) +} + +/** Render an aggregate call: it marks the clause, and its arguments are not bare columns. */ +export function inAggregate(render: () => string): string { + const state = current + if (state === undefined) return render() + if (state.hiddenDepth === 0) state.track.aggregate = true + state.aggregateDepth++ + try { + return render() + } finally { + state.aggregateDepth-- + } +} + +/** Render a window (`… OVER (…)`) or SQL the builder did not write: nothing inside is counted. */ +export function hidden(render: () => string): string { + const state = current + if (state === undefined) return render() + state.hiddenDepth++ + try { + return render() + } finally { + state.hiddenDepth-- + } +} diff --git a/src/sql/sql-fragment.ts b/src/sql/sql-fragment.ts index 3217776..37daba5 100644 --- a/src/sql/sql-fragment.ts +++ b/src/sql/sql-fragment.ts @@ -1,5 +1,6 @@ import { Data } from "effect" import { activeSqlSyntax } from "./sql-syntax" +import { hidden, noteColumn } from "./render-tracker" // ClickHouse string escaping @@ -74,8 +75,12 @@ export type SqlFragment = Data.TaggedEnum<{ * outer `compile` runs in, so a bad value reaches production as a synchronous * throw rather than a typed failure. Deferring the work to compile time puts * it back inside. + * + * `known` marks SQL the builder itself writes (an operator, a built-in + * function), whose parts the render tracker may count. Anything else is + * opaque to it: see `render-tracker.ts`. */ - Lazy: { readonly render: () => string } + Lazy: { readonly render: () => string; readonly known?: boolean } }> const Frag = Data.taggedEnum() @@ -98,6 +103,8 @@ export const as_ = (expr: SqlFragment, alias: string): SqlFragment => Frag.As({ export const when = (condition: boolean, fragment: SqlFragment): SqlFragment => Frag.When({ condition, fragment }) export const lazy = (render: () => string): SqlFragment => Frag.Lazy({ render }) +/** {@link lazy} for SQL the builder writes itself, which the render tracker may look inside. */ +export const known = (render: () => string): SqlFragment => Frag.Lazy({ render, known: true }) // Compiler @@ -105,10 +112,12 @@ export const compile: (fragment: SqlFragment) => string = Frag.$match({ Raw: ({ sql }) => sql, Str: ({ value }) => quoteString(value), Int: ({ value }) => String(Math.round(value)), - Ident: ({ name, qualifier }) => - qualifier === undefined ? quoteIdent(name) : `${quoteIdentPath(qualifier)}.${quoteIdent(name)}`, + Ident: ({ name, qualifier }) => { + noteColumn(qualifier === undefined ? name : `${qualifier}.${name}`) + return qualifier === undefined ? quoteIdent(name) : `${quoteIdentPath(qualifier)}.${quoteIdent(name)}` + }, Join: ({ separator, fragments }) => fragments.map(compile).filter(Boolean).join(separator), As: ({ expr, alias }) => `${compile(expr)} AS ${quoteIdent(alias)}`, When: ({ condition, fragment }) => (condition ? compile(fragment) : ""), - Lazy: ({ render }) => render(), + Lazy: ({ render, known }) => (known === true ? render() : hidden(render)), }) From 2ebe90686fe74e0fc7515493ccf7ba796d4ca6b0 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 4 Oct 2026 21:27:09 +0200 Subject: [PATCH 3/5] Track param.* placeholders in the type and require them Expr and Condition carry the params inside them (a contravariant phantom, so an Expr still goes wherever an Expr does). Queries, unions, inserts, updates and deletes collect them from every clause, subquery, join and CTE; compile, compileUnion and Database.run require each one with a value of its type. Built-in functions, defineFn/defineCondFn, subquery predicates and CH.sql pass their arguments' params on. Insert rows and SET records also reject columns the table cannot write, now that their types are inferred. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 23 +++ docs/extending.md | 18 +++ docs/params-and-compilation.md | 25 +++ src/ch/compilation-regressions.test.ts | 1 + src/ch/compile.test.ts | 1 + src/ch/compile.ts | 61 ++++--- src/ch/core-dsl.test.ts | 3 + src/ch/define-fn.ts | 28 ++-- src/ch/dialect.test.ts | 2 + src/ch/expr.ts | 169 ++++++++++++++------ src/ch/functions/aggregate.ts | 84 +++++----- src/ch/functions/array.ts | 62 ++++---- src/ch/functions/builtin.ts | 23 ++- src/ch/functions/conditional.ts | 39 ++--- src/ch/functions/date-time.ts | 48 +++--- src/ch/functions/map.ts | 14 +- src/ch/functions/numeric.ts | 26 +-- src/ch/functions/string.ts | 26 +-- src/ch/functions/window.ts | 45 ++++-- src/ch/insert.test.ts | 2 + src/ch/insert.ts | 72 ++++++--- src/ch/param.ts | 10 +- src/ch/params-propagation.test-d.ts | 211 +++++++++++++++++++++++++ src/ch/publish-readiness.test.ts | 1 + src/ch/query.ts | 121 +++++++------- src/ch/soundness.test-d.ts | 61 +++++++ src/ch/sql-template.ts | 29 +++- src/ch/subquery.ts | 35 ++-- src/ch/union.ts | 23 ++- src/ch/update.ts | 106 ++++++++----- src/database/database.test.ts | 1 + src/database/database.ts | 19 +-- src/pg/functions.ts | 35 ++-- src/pg/postgres.test.ts | 1 + 34 files changed, 1000 insertions(+), 425 deletions(-) create mode 100644 src/ch/params-propagation.test-d.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 1bb0ca3..86bed25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,29 @@ ## Unreleased +- **Breaking:** invalid queries are refused before any SQL is sent: as type errors where the + type can see them, otherwise as a `QueryBuilderError` / `QueryBuilderDefect` from `compile`. + - Params are in the query's type. `compile`, `compileUnion` and `Database.run` require every + `param.*` the query uses, with a value of its type (`CHQuery`, `CHUnionQuery`, `CHInsert`, + `CHUpdate` and `CHDelete` gain a `Params` type parameter; `Expr` and `Condition` gain `P`). + - A second `where()` / `having()` ANDs with the first instead of replacing it, on queries and + on writes. + - Comparisons refuse `null` (use `isNull()`); an empty `in_()` / `notIn()` is `1 = 0` / `1 = 1`. + `like` / `ilike` accept a nullable string. + - `limit` / `offset` refuse negative, fractional or non-finite values instead of rounding them. + - A query with no `select()` cannot be compiled, run, joined, used in `FROM`, a CTE, `EXISTS` + or `INSERT ... SELECT`. `unionAll` branches must agree on aliases and column types. + `inSubquery` / `notInSubquery` need exactly one column of a comparable type. + - Join aliases must be unique and must not shadow a FROM column or the FROM alias; CTE names + must be unique. + - `update().set({})` and a SET or insert row naming a column the table cannot write are type + errors; an UPDATE or DELETE without `where()` or `allRows()` cannot be compiled or run. + - An aggregate in WHERE or a join's ON, a column that is neither grouped nor aggregated, and + grouping by an aggregate fail to compile. SQL the builder did not write (`rawExpr`, + `CH.sql`, windows, `makeExpr`) is not looked inside. + - Built-in functions belong to a dialect: a ClickHouse function (such as `count()`) in a + Postgres compile fails, and the reverse. `coalesce`, `nullIf` and `lower` are portable. + `Dialect.functions` names a dialect's function set. - Add `CH.sql`: SQL templates inside expressions and conditions. `CH.sql(type)\`…\`` is a typed `Expr`, ``CH.sql`…` `` an untyped one, `CH.sql.cond` a `Condition`; with `sql.ident`, `sql.raw` and `sql.join`. Interpolated columns and params render as SQL and placeholders, a builder diff --git a/docs/extending.md b/docs/extending.md index 4abf0a3..05d8393 100644 --- a/docs/extending.md +++ b/docs/extending.md @@ -128,6 +128,24 @@ console.log(predicate) // Name = 'O\'Reilly' Use this for string literals only. Keep SQL structure and identifiers under application control, and validate numeric inputs such as the quantile level separately. +### Params and checks on a custom function + +`Expr` carries the `param.*` placeholders inside an expression, so `compile` can require +them. `defineFn` and `defineCondFn` pass their arguments' params on by themselves. A wrapper +built with `makeExpr` says so in its signature, with one type parameter per argument: + +```ts +const quantileExact = + (q: number) => + (expr: CH.Expr): CH.Expr => + makeExpr(raw(`quantileExact(${q})(${compile(expr.toFragment())})`), T.float64.schema) +``` + +Without it the function still works; a param inside it is then checked when compiling rather +than by the type. SQL built with `makeExpr`, `defineFn` or `CH.sql` is also opaque to the GROUP +BY checks (see [Queries](./queries.md#groupby)): a mistake inside it reaches the database, but +it never makes a valid query fail. + ## A column type of your own `T.custom(sql, schema)` is the extension point the built-in types are built from — `T.uint64` is diff --git a/docs/params-and-compilation.md b/docs/params-and-compilation.md index f919402..e307b62 100644 --- a/docs/params-and-compilation.md +++ b/docs/params-and-compilation.md @@ -29,6 +29,31 @@ Param names must be alphanumeric, optionally separated by single underscores — through the placeholder that `compile` later matches, and `__` would make its boundary ambiguous. A name that cannot round-trip is a `QueryBuilderDefect` at declaration. +## Params are in the query's type + +A query remembers the params it uses, with their types, and `compile`, `compileUnion` and +`Database.run` require them: + +```ts +const byOrg = CH.from(Events) + .select("Name") + .where(($) => [$.OrgId.eq(CH.param.string("orgId")), $.Ms.gt(CH.param.int("minMs"))]) + +CH.compile(byOrg, { orgId: "org_1", minMs: 100 }) // ok +CH.compile(byOrg, { orgId: "org_1" }) // type error: paramsRequired { orgId: string; minMs: number } +CH.compile(byOrg, { orgId: 1, minMs: 100 }) // type error: orgId is a string +``` + +Params are collected from `where`, `having`, `select`, join `on` callbacks, subqueries in +`FROM`, joins, CTEs and `EXISTS`/`IN`, union branches, and insert rows, `SET` records and +write `where`s. Extra keys are allowed, so one params object can serve several queries. A +query without params takes none. + +A function the builder does not know passes its arguments' params on only if its signature +says so: `defineFn`, `defineCondFn` and `compileTypedFnCall` from the extending API do, a +hand-written `makeExpr` does not. A param the type does not see is still checked when +compiling, as below. + ## What each kind accepts The declared kind is checked when the value arrives, so a value of the wrong shape is a diff --git a/src/ch/compilation-regressions.test.ts b/src/ch/compilation-regressions.test.ts index 63db33c..302d22c 100644 --- a/src/ch/compilation-regressions.test.ts +++ b/src/ch/compilation-regressions.test.ts @@ -82,6 +82,7 @@ describe("subquery source scope", () => { it.effect("keeps deferred failures typed and restores compilation context", () => Effect.gen(function* () { const predicate = CH.subqueryExpr(scopedCount, T.uint64).gt(0) const query = outer.select("Id").having(() => [predicate]) + // @ts-expect-error -- a missing param is a type error too const result = yield* CH.compile(query, { outer: "a" }).pipe(Effect.result) expect(result._tag).toBe("Failure") if (result._tag === "Failure") expect(result.failure.code).toBe("UnresolvedParam") diff --git a/src/ch/compile.test.ts b/src/ch/compile.test.ts index bbd8dfc..7a76034 100644 --- a/src/ch/compile.test.ts +++ b/src/ch/compile.test.ts @@ -513,6 +513,7 @@ describe("compile puts failures in the error channel", () => { // than a typed failure anyone could map to a 400. it.effect("a missing param value is a typed failure", () => Effect.gen(function* () { + // @ts-expect-error -- a missing param is a type error too const error = yield* Effect.flip(CH.compile(query, {})) expect(error._tag).toBe("@maple-dev/effect-orm/QueryBuilderError") expect(error.code).toBe("UnresolvedParam") diff --git a/src/ch/compile.ts b/src/ch/compile.ts index 7b4fd0c..788ec2e 100644 --- a/src/ch/compile.ts +++ b/src/ch/compile.ts @@ -14,7 +14,7 @@ import { isInsert, type CHInsert } from "./insert" import { isDelete, isUpdate, type CHDelete, type CHUpdate } from "./update" import type { Table } from "./table" import { createColumnAccessor, createQualifiedColumnAccessor, createJoinedColumnAccessor, sourceAlias } from "./query" -import { aliased, columnTypeOf, isExprLike, type Condition, type Expr } from "./expr" +import { aliased, columnTypeOf, isExprLike, type Condition, type Expr, type ParamsSatisfied } from "./expr" import { raw, identPath, quoteIdent, quoteIdentPath, compile as compileSqlFragment, type SqlFragment } from "../sql/sql-fragment" import { splitTerminalClauses } from "../sql/terminal-clauses" import { compileQuery, type SqlQuery } from "../sql/sql-query" @@ -573,12 +573,13 @@ export function compileCH< Output extends Record, Joins extends Record, Route extends string | undefined, - Params extends Record = {}, + Params = never, + const Given extends Record = {}, Decoded extends Output = Output, >( - query: CHQuery & NeedsSelect, - /** Values for the query's `param.*` markers. Optional when it has none. */ - params?: Params, + query: CHQuery & NeedsSelect & ParamsSatisfied, + /** Values for the query's `param.*` markers: required, and typed, when it has any. */ + params?: Given, options?: { skipFormat?: boolean rowSchema?: CompiledQueryRowSchema @@ -587,9 +588,9 @@ export function compileCH< }, ): Effect.Effect, QueryBuilderError> /** An INSERT, UPDATE or DELETE. `params` fills the `param.*` markers among its values. */ -export function compileCH( - insert: CHWrite, - params?: Record, +export function compileCH = {}>( + insert: CHWrite & ParamsSatisfied, + params?: Given, options?: InsertCompileOptions, ): Effect.Effect, QueryBuilderError> export function compileCH( @@ -601,7 +602,10 @@ export function compileCH( } /** A write statement: what `compile` takes besides a query. */ -export type CHWrite = CHInsert | CHUpdate | CHDelete +export type CHWrite = + | CHInsert + | CHUpdate + | CHDelete /** What compiling a write takes: only the dialect. */ export interface InsertCompileOptions { @@ -609,24 +613,29 @@ export interface InsertCompileOptions { } /** {@link compileCH} for a `UNION ALL`. */ -export const compileUnion = , Params extends Record>( - union: CHUnionQuery, - params: Params, +export const compileUnion = < + Output extends Record, + Params = never, + const Given extends Record = {}, +>( + union: CHUnionQuery & ParamsSatisfied, + params?: Given, options?: { rowSchema?: CompiledQueryRowSchema; deferParams?: boolean; dialect?: Dialect }, ): Effect.Effect, QueryBuilderError> => - asEffect(() => compileUnionUnsafe(union, params, options)) + asEffect(() => compileUnionUnsafe(union as CHUnionQuery, params ?? {}, options)) export function compileCHUnsafe< Cols extends ColumnDefs, Output extends Record, Joins extends Record, Route extends string | undefined, - Params extends Record = {}, + Params = never, + const Given extends Record = {}, Decoded extends Output = Output, >( - query: CHQuery & NeedsSelect, - /** Values for the query's `param.*` markers. Optional when it has none. */ - params?: Params, + query: CHQuery & NeedsSelect & ParamsSatisfied, + /** Values for the query's `param.*` markers: required, and typed, when it has any. */ + params?: Given, options?: { skipFormat?: boolean rowSchema?: CompiledQueryRowSchema @@ -639,9 +648,9 @@ export function compileCHUnsafe< }, ): CompiledQuery /** An INSERT, UPDATE or DELETE. `params` fills the `param.*` markers among its values. */ -export function compileCHUnsafe( - insert: CHWrite, - params?: Record, +export function compileCHUnsafe = {}>( + insert: CHWrite & ParamsSatisfied, + params?: Given, options?: InsertCompileOptions, ): CompiledQuery export function compileCHUnsafe( @@ -1281,16 +1290,20 @@ const unionExprsOf = ( // UNION ALL compilation -export function compileUnionUnsafe, Params extends Record>( - union: CHUnionQuery, - params: Params, +export function compileUnionUnsafe< + Output extends Record, + Params = never, + const Given extends Record = {}, +>( + union: CHUnionQuery & ParamsSatisfied, + params?: Given, options?: { rowSchema?: CompiledQueryRowSchema deferParams?: boolean dialect?: Dialect }, ): CompiledQuery { - return withDialect(options?.dialect ?? currentDialect(), () => compileUnionInner(union, params, options)) + return withDialect(options?.dialect ?? currentDialect(), () => compileUnionInner(union as CHUnionQuery, params ?? {}, options)) } /** The recursion behind {@link compileUnionUnsafe}; see {@link compileInner}. */ diff --git a/src/ch/core-dsl.test.ts b/src/ch/core-dsl.test.ts index 9957ae0..f81169e 100644 --- a/src/ch/core-dsl.test.ts +++ b/src/ch/core-dsl.test.ts @@ -443,6 +443,7 @@ describe("param resolution", () => { const q = CH.from(TestTable) .select(($) => ({ id: $.Id })) .where(($) => [$.Id.eq(CH.param.string("orgId"))]) + // @ts-expect-error -- a missing param is a type error too expect(() => compileCHUnsafe(q, {})).toThrow(/no value given for param 'orgId'/) }) @@ -450,6 +451,7 @@ describe("param resolution", () => { const q = CH.from(TestTable) .select(($) => ({ id: $.Id })) .where(($) => [$.Id.eq(CH.param.string("orgId"))]) + // @ts-expect-error -- a mistyped param is a type error too expect(() => compileCHUnsafe(q, { orgId: 42 })).toThrow(/param 'orgId' \(string\).*Expected string/) }) @@ -464,6 +466,7 @@ describe("param resolution", () => { const q = CH.from(TestTable) .select(($) => ({ id: $.Id })) .where(($) => [$.Id.eq(CH.param.string("orgId"))]) + // @ts-expect-error -- a mistyped param is a type error too expect(() => compileCHUnsafe(q, { orgId: undefined })).toThrow(/undefined is not a valid value/) }) diff --git a/src/ch/define-fn.ts b/src/ch/define-fn.ts index a864f43..2e4b374 100644 --- a/src/ch/define-fn.ts +++ b/src/ch/define-fn.ts @@ -8,7 +8,7 @@ import { Result, Schema, type SchemaAST } from "effect" import { CHFloatResult, CHNumber, dateTime64 } from "./types" import { compile, lazy } from "../sql/sql-fragment" -import type { Expr, Condition } from "./expr" +import type { Expr, Condition, ParamsIn } from "./expr" import { makeExpr, makeUntypedExpr, makeCond, toFragment } from "./expr" import type { CHType } from "./types" @@ -86,22 +86,22 @@ export { makeExpr, makeUntypedExpr, makeCond } // compileFnCall — low-level helper for handwritten generic/special functions -export function compileFnCall(name: string, ...args: unknown[]): Expr { +export function compileFnCall(name: string, ...args: A): Expr> { const compiled = () => args.map((a) => compile(toFragment(a))).join(", ") - return makeUntypedExpr(lazy(() => `${name}(${compiled()})`)) + return makeUntypedExpr(lazy(() => `${name}(${compiled()})`)) as Expr } /** `compileFnCall` for a function whose result type is known. */ -export function compileTypedFnCall( +export function compileTypedFnCall( name: string, schema: Schema.Codec | undefined, - ...args: unknown[] -): Expr { + ...args: A +): Expr> { const compiled = () => args.map((a) => compile(toFragment(a))).join(", ") - return makeExpr(lazy(() => `${name}(${compiled()})`), schema) + return makeExpr(lazy(() => `${name}(${compiled()})`), schema) as Expr } -export function compileFnCallCond(name: string, ...args: unknown[]): Condition { +export function compileFnCallCond(name: string, ...args: A): Condition> { const compiled = () => args.map((a) => compile(toFragment(a))).join(", ") return makeCond(lazy(() => `${name}(${compiled()})`)) } @@ -143,8 +143,8 @@ const resultSchema = ( export function defineFn( name: string, result: FnResult, -): (...args: Args) => Expr { - return (...args: Args): Expr => compileTypedFnCall(name, resultSchema(result, args), ...args) +): (...args: A) => Expr> { + return (...args: A) => compileTypedFnCall(name, resultSchema(result, args), ...args) } /** @@ -155,8 +155,8 @@ export function defineFn( */ export function defineUntypedFn( name: string, -): (...args: Args) => Expr { - return (...args: Args): Expr => compileFnCall(name, ...args) +): (...args: A) => Expr> { + return (...args: A) => compileFnCall(name, ...args) } // Result rules @@ -221,8 +221,8 @@ export const arrayOfArg = // Usage: // export const hasToken = defineCondFn<[Expr]>("hasToken") -export function defineCondFn(name: string): (...args: Args) => Condition { - return (...args: Args): Condition => compileFnCallCond(name, ...args) +export function defineCondFn(name: string): (...args: A) => Condition> { + return (...args: A) => compileFnCallCond(name, ...args) } /** Numeric functions preserve SQL NULL while promoting the numeric type. */ diff --git a/src/ch/dialect.test.ts b/src/ch/dialect.test.ts index 53c6cef..64f932b 100644 --- a/src/ch/dialect.test.ts +++ b/src/ch/dialect.test.ts @@ -120,9 +120,11 @@ describe("dialect params", () => { }) it("still fails a missing or ill-typed param at compile time", () => { + // @ts-expect-error -- a missing param is a type error too expect(() => compileCHUnsafe(byService, { orgId: "org_1" }, { dialect: numbered })).toThrow( /no value given for param 'service'/, ) + // @ts-expect-error -- a mistyped param is a type error too expect(() => compileCHUnsafe(byService, { orgId: 1, service: "api" }, { dialect: numbered })).toThrow( /param 'orgId'/, ) diff --git a/src/ch/expr.ts b/src/ch/expr.ts index e7ec506..25081ea 100644 --- a/src/ch/expr.ts +++ b/src/ch/expr.ts @@ -39,9 +39,73 @@ export type Comparable = TSType extends DateTime.Utc ? DateTime.Utc | Da */ export type Widen = TSType extends string ? string : TSType extends number ? number : TSType -export interface Expr { +// Params in the type +// +// An expression remembers the `param.*` placeholders it contains, so a query +// can say which params it needs and `compile` / `Database.run` can require +// them. Each one is a `ParamEntry`; an expression's are a union of them, `never` +// when it has none. +// +// The entries ride on a phantom *function parameter*, which makes them +// contravariant: an `Expr` is assignable to a plain `Expr`, so +// every function written against `Expr` still accepts one. A function that +// does not pass its arguments' entries on to its result drops them from the +// type; the param is then still checked when compiling, just not by the type. + +/** One `param.*` placeholder: its name and the value it is filled with. */ +export interface ParamEntry { + readonly name: Name + readonly value: Value +} + +/** The params of an expression, condition, or a union or array of them. */ +export type ParamsIn = 0 extends 1 & X + ? never + : X extends { readonly _params?: (entries: infer P) => void } + ? 0 extends 1 & P + ? never + : [P] extends [ParamEntry] + ? P + : never + : never + +/** The value a param needs: every entry's type for that name, intersected + * per entry (a `boolean` entry stays `boolean`, not `true & false`). */ +type ValueOf = + UnionToIntersection

extends { + readonly v: infer V + } + ? V + : never + +type UnionToIntersection = (U extends unknown ? (u: U) => void : never) extends (i: infer I) => void ? I : never + +/** + * The params object a set of entries asks for: one key per name. A name used + * with two value types needs a value of both. + */ +export type ParamsRecord

= [P] extends [never] + ? {} + : [P] extends [ParamEntry] + ? { readonly [N in P["name"]]: ValueOf } + : {} + +/** + * `unknown` when `Given` fills every param in `P` with a value of its type; + * otherwise a property spelling out the params object that is needed. Extra + * keys are allowed, so one params object can serve several queries. + */ +export type ParamsSatisfied = [P] extends [never] + ? unknown + : Given extends ParamsRecord

+ ? unknown + : { readonly paramsRequired: ParamsRecord

} + +export interface Expr { readonly _brand: "Expr" readonly _phantom?: TSType + /** phantom: the `param.*` placeholders inside this expression. */ + readonly _params?: (entries: P) => void /** * How this expression's wire value decodes, when the builder knows it. * @@ -63,59 +127,59 @@ export interface Expr { // // A plain value is never `null`: `x = NULL` is never true in SQL, so it is // refused here and at compile time. Use `isNull()` / `isNotNull()`. - eq(other: Operand): Condition - neq(other: Operand): Condition - gt(other: Operand): Condition - gte(other: Operand): Condition - lt(other: Operand): Condition - lte(other: Operand): Condition + eq(other: Operand): Condition

+ neq(other: Operand): Condition

+ gt(other: Operand): Condition

+ gte(other: Operand): Condition

+ lt(other: Operand): Condition

+ lte(other: Operand): Condition

// String operations. A `Nullable(String)` matches like a `String`. - like(this: Expr, pattern: string): Condition - notLike(this: Expr, pattern: string): Condition - ilike(this: Expr, pattern: string): Condition + like(this: Expr, pattern: string): Condition

+ notLike(this: Expr, pattern: string): Condition

+ ilike(this: Expr, pattern: string): Condition

// NULL and ranges /** `expr IS NULL`. */ - isNull(): Condition + isNull(): Condition

/** `expr IS NOT NULL`. */ - isNotNull(): Condition + isNotNull(): Condition

/** `expr BETWEEN low AND high`, both ends included. */ - between(low: Operand, high: Operand): Condition + between(low: Operand, high: Operand): Condition

/** `expr NOT BETWEEN low AND high`. */ - notBetween(low: Operand, high: Operand): Condition + notBetween(low: Operand, high: Operand): Condition

// IN / NOT IN. An empty list is false (`IN`) or true (`NOT IN`), written // `1 = 0` / `1 = 1`, rather than the `IN ()` no database accepts. - in_(...values: Array>>>): Condition - notIn(...values: Array>>>): Condition + in_(...values: Array>>>): Condition

+ notIn(...values: Array>>>): Condition

// JSON represents non-finite division results as null. Other arithmetic // propagates SQL NULL from either operand. - div(this: Expr, n: R | Expr): Expr> - mul( + div(this: Expr, n: R | Expr): Expr, P | Q> + mul( this: Expr, - n: R | Expr, - ): Expr> - add( + n: R | Expr, + ): Expr, P | Q> + add( this: Expr, - n: R | Expr, - ): Expr> - sub( + n: R | Expr, + ): Expr, P | Q> + sub( this: Expr, - n: R | Expr, - ): Expr> - mod(this: Expr, n: R | Expr): Expr> + n: R | Expr, + ): Expr, P | Q> + mod(this: Expr, n: R | Expr): Expr, P | Q> } /** * What a comparison takes on its right: a value of the column's type (never * `null`), or an expression of it. */ -export type Operand = +export type Operand = | Comparable>> - | Expr - | Expr> + | Expr + | Expr, Q> /** * What `/` and `%` decode to. A numeric literal divisor of magnitude >= 1 @@ -157,11 +221,13 @@ export interface ColumnRef = [ColType] extends [CHType<"Map", Record, any>] ? V : string -export interface Condition { +export interface Condition

{ readonly _brand: "Condition" + /** phantom: the `param.*` placeholders inside this condition. */ + readonly _params?: (entries: P) => void toFragment(): SqlFragment - and(other: Condition): Condition - or(other: Condition): Condition + and(other: Condition): Condition

+ or(other: Condition): Condition

} // Core helpers (exported for define-fn.ts and consumer extensibility) @@ -220,7 +286,7 @@ const inCond = ( fragment: SqlFragment, op: "IN" | "NOT IN", values: ReadonlyArray<() => SqlFragment>, -): Condition => +): Condition => makeCond( known(() => values.length === 0 @@ -243,7 +309,7 @@ const arith = ( op: string, rhs: number | null | Expr, lhsSchema?: Schema.Codec, -): Expr => { +): Expr => { const rhsSchema = typeof rhs === "number" || rhs === null ? undefined : rhs.schema // `x / 1000000` is finite whenever `x` is. A literal below 1 in magnitude // can overflow a large dividend (`1 / 5e-324` is `inf`), so only |d| >= 1 @@ -259,7 +325,7 @@ const arith = ( return makeExpr( known(() => `${compile(lhs)} ${op} ${compile(toFragment(rhs))}`), (nullable ? Schema.NullOr(CHNumber) : overflows ? CHFloatResult : CHNumber) as Schema.Codec, - ) + ) as Expr } /** @@ -284,7 +350,7 @@ export function makeExpr( * no type to read fall back to guessing from the JS value. */ literal?: (value: unknown) => SqlFragment, -): Expr { +): Expr { /** An operand: another expression as-is, a plain value through the codec. */ function operand(value: unknown): SqlFragment { if (value === null || value === undefined) return refusedNull(value) @@ -292,7 +358,9 @@ export function makeExpr( } // Keep operand rendering lazy so nested subqueries reach the owning compiler. - const self: Expr = { + // `any` params: the phantom is a type-level fact, and every method's result + // carries what its signature says. + const self: Expr = { _brand: "Expr" as const, ...(schema !== undefined ? { schema } : undefined), toFragment: () => fragment, @@ -350,7 +418,7 @@ export function makeExpr( export function makeUntypedExpr( fragment: SqlFragment, literal?: (value: unknown) => SqlFragment, -): Expr { +): Expr { return makeExpr(fragment, undefined, literal) } @@ -445,7 +513,7 @@ export function makeColumnRef { return { _brand: "Condition" as const, toFragment: () => fragment, @@ -502,8 +570,10 @@ export function notInList(expr: Expr, values: readonly string[]): Condit * With none left it is `undefined`, which a `where` list skips in turn. Tenant * evidence carries through, as with `.and`. */ -export function and(...conditions: ReadonlyArray): Condition -export function and(...conditions: ReadonlyArray): Condition | undefined +export function and>(...conditions: C): Condition> +export function and>( + ...conditions: C +): Condition> | undefined export function and(...conditions: ReadonlyArray): Condition | undefined { const present = conditions.filter((c): c is Condition => c !== undefined) if (present.length <= 1) return present[0] @@ -517,8 +587,10 @@ export function and(...conditions: ReadonlyArray): Condit * Conditions OR-joined, an `undefined` one skipped. With none left it is * `undefined`. An OR proves no tenant, so it carries no tenant evidence. */ -export function or(...conditions: ReadonlyArray): Condition -export function or(...conditions: ReadonlyArray): Condition | undefined +export function or>(...conditions: C): Condition> +export function or>( + ...conditions: C +): Condition> | undefined export function or(...conditions: ReadonlyArray): Condition | undefined { const present = conditions.filter((c): c is Condition => c !== undefined) if (present.length <= 1) return present[0] @@ -526,7 +598,7 @@ export function or(...conditions: ReadonlyArray): Conditi } /** Wrap a condition in NOT (...). */ -export function not(condition: Condition): Condition { +export function not

(condition: Condition

): Condition

{ return makeCond(known(() => `NOT (${compile(condition.toFragment())})`)) } @@ -588,12 +660,15 @@ export function aliased(expr: Expr, alias: string): SqlFragment { // Conditional helpers (for optional WHERE clauses) -export function when(value: T | undefined | false | null, fn: (v: T) => Condition): Condition | undefined { +export function when( + value: T | undefined | false | null, + fn: (v: T) => Condition

, +): Condition

| undefined { if (value === undefined || value === null || value === false) return undefined return fn(value) } -export function whenTrue(value: boolean | undefined, fn: () => Condition): Condition | undefined { +export function whenTrue

(value: boolean | undefined, fn: () => Condition

): Condition

| undefined { if (!value) return undefined return fn() } diff --git a/src/ch/functions/aggregate.ts b/src/ch/functions/aggregate.ts index 19e389d..47b4f76 100644 --- a/src/ch/functions/aggregate.ts +++ b/src/ch/functions/aggregate.ts @@ -2,11 +2,11 @@ import { numericResultSchema, overflowResultSchema } from "../define-fn" import { QueryBuilderError } from "../errors" import { makeExpr } from "../expr" import { compile } from "../../sql/sql-fragment" -import type { Expr, Condition } from "../expr" +import type { Expr, Condition, ParamsIn } from "../expr" import { type DateTime, Schema } from "effect" import * as T from "../types" -import { arrayOfArg, sameAs, schemaOf } from "../define-fn" +import { schemaOf } from "../define-fn" import { builtins } from "./builtin" const { compileTypedFnCall, defineFn, lazy } = builtins("clickhouse", "aggregate") @@ -23,25 +23,25 @@ export const count = defineFn<[], number>("count", T.uint64) export const avg = defineFn<[Expr], number | null>("avg", T.nullable(T.float64)) // A finite Float64 input can overflow during accumulation; the resulting // infinity decodes as NaN (see `overflowResultSchema`). -export const sum = (expr: Expr): Expr> => +export const sum = (expr: Expr): Expr, Q> => compileTypedFnCall("sum", overflowResultSchema(expr), expr) // Condition-taking aggregates export const countIf = defineFn<[Condition], number>("countIf", T.uint64) -export const sumIf = ( - expr: Expr, - condition: Condition, -): Expr> => compileTypedFnCall("sumIf", overflowResultSchema(expr), expr, condition) +export const sumIf = ( + expr: Expr, + condition: Condition, +): Expr, Q1 | Q2> => compileTypedFnCall("sumIf", overflowResultSchema(expr), expr, condition) export const avgIf = defineFn<[Expr, Condition], number | null>("avgIf", T.nullable(T.float64)) -export const maxIf = ( - expr: Expr, - condition: Condition, -): Expr> => compileTypedFnCall("maxIf", numericResultSchema(expr), expr, condition) -export const minIf = ( - expr: Expr, - condition: Condition, -): Expr> => compileTypedFnCall("minIf", numericResultSchema(expr), expr, condition) +export const maxIf = ( + expr: Expr, + condition: Condition, +): Expr, Q1 | Q2> => compileTypedFnCall("maxIf", numericResultSchema(expr), expr, condition) +export const minIf = ( + expr: Expr, + condition: Condition, +): Expr, Q1 | Q2> => compileTypedFnCall("minIf", numericResultSchema(expr), expr, condition) // Generic aggregates (compileFnCall for type preservation) @@ -53,16 +53,16 @@ export const minIf = ( // here (as an earlier version did with `NonNullable`) lied to every caller // while `sameAs(0)` kept the nullable runtime codec. -export const min_ = (expr: Expr): Expr => defineFn<[Expr], T>("min", sameAs(0))(expr) +export const min_ = (expr: Expr): Expr => compileTypedFnCall("min", schemaOf(expr), expr) -export const max_ = (expr: Expr): Expr => defineFn<[Expr], T>("max", sameAs(0))(expr) +export const max_ = (expr: Expr): Expr => compileTypedFnCall("max", schemaOf(expr), expr) -export const any_ = (expr: Expr): Expr => defineFn<[Expr], T>("any", sameAs(0))(expr) +export const any_ = (expr: Expr): Expr => compileTypedFnCall("any", schemaOf(expr), expr) -export const anyIf = (expr: Expr, cond: Condition): Expr => - defineFn<[Expr, Condition], T>("anyIf", sameAs(0))(expr, cond) +export const anyIf = (expr: Expr, cond: Condition): Expr => + compileTypedFnCall("anyIf", schemaOf(expr), expr, cond) -export const uniq = (expr: Expr): Expr => defineFn<[Expr], number>("uniq", T.uint64)(expr) +export const uniq = (expr: Expr): Expr => compileTypedFnCall("uniq", T.uint64.schema, expr) /** * `uniqIf(value, condition)` — distinct `value`s among the rows matching @@ -72,8 +72,8 @@ export const uniq = (expr: Expr): Expr => defineFn<[Expr], numb * `countIf` on a `ReplacingMergeTree`: un-merged duplicate rows for the same * key would inflate a `countIf` but not a `uniqIf` on that key. */ -export const uniqIf = (expr: Expr, cond: Condition): Expr => - defineFn<[Expr, Condition], number>("uniqIf", T.uint64)(expr, cond) +export const uniqIf = (expr: Expr, cond: Condition): Expr => + compileTypedFnCall("uniqIf", T.uint64.schema, expr, cond) /** * `uniqExact(value)` — the exact distinct count, where {@link uniq} estimates. @@ -82,11 +82,11 @@ export const uniqIf = (expr: Expr, cond: Condition): Expr => * number sits next to the rows it counts: a facet count that disagrees with the * visible list reads as a bug, not as an approximation. */ -export const uniqExact = (expr: Expr): Expr => - defineFn<[Expr], number>("uniqExact", T.uint64)(expr) +export const uniqExact = (expr: Expr): Expr => + compileTypedFnCall("uniqExact", T.uint64.schema, expr) -export const groupUniqArray = (expr: Expr): Expr> => - defineFn<[Expr], ReadonlyArray>("groupUniqArray", arrayOfArg(0))(expr) +export const groupUniqArray = (expr: Expr): Expr, Q> => + compileTypedFnCall("groupUniqArray", arraySchemaOf(expr), expr) /** * `groupUniqArrayArray(arrayColumn)` — flatten arrays across rows into one @@ -97,24 +97,24 @@ export const groupUniqArray = (expr: Expr): Expr> => * `SimpleAggregateFunction(groupUniqArrayArray, Array(T))` column is declared * with, so reading such a column back uses the same name. */ -export const groupUniqArrayArray = (expr: Expr>): Expr> => - defineFn<[Expr>], ReadonlyArray>("groupUniqArrayArray", sameAs(0))(expr) +export const groupUniqArrayArray = (expr: Expr, Q>): Expr, Q> => + compileTypedFnCall("groupUniqArrayArray", schemaOf>(expr), expr) /** `argMin(value, orderBy)` — the `value` from the row with the smallest `orderBy`. */ -export const argMin = (value: Expr, orderBy: Expr): Expr => - defineFn<[Expr, Expr], T>("argMin", sameAs(0))(value, orderBy) +export const argMin = (value: Expr, orderBy: Expr): Expr => + compileTypedFnCall("argMin", schemaOf(value), value, orderBy) /** `argMax(value, orderBy)` — the `value` from the row with the largest `orderBy`. */ -export const argMax = (value: Expr, orderBy: Expr): Expr => - defineFn<[Expr, Expr], T>("argMax", sameAs(0))(value, orderBy) +export const argMax = (value: Expr, orderBy: Expr): Expr => + compileTypedFnCall("argMax", schemaOf(value), value, orderBy) -export const argMaxMerge = (expr: Expr): Expr => - defineFn<[Expr], T>("argMaxMerge", sameAs(0))(expr) +export const argMaxMerge = (expr: Expr): Expr => + compileTypedFnCall("argMaxMerge", schemaOf(expr), expr) // Curried / parametric aggregates (handwritten — custom SQL syntax) export function quantile(q: number) { - return (expr: Expr): Expr => + return (expr: Expr): Expr => makeExpr(lazy(() => `quantile(${q})(${compile(expr.toFragment())})`), T.nullable(T.float64).schema) } @@ -126,7 +126,7 @@ export function quantile(q: number) { * curried shape: `groupUniqArrayIf(3)(x, cond)` → `groupUniqArrayIf(3)(x, cond)`. */ export function groupUniqArrayIf(maxSize: number) { - return (expr: Expr, cond: Condition): Expr> => + return (expr: Expr, cond: Condition): Expr, Q1 | Q2> => makeExpr( lazy(() => `groupUniqArrayIf(${Math.round(maxSize)})(` + @@ -160,7 +160,10 @@ export type WindowFunnelMode = "strict_order" | "strict_deduplication" | "strict */ export function windowFunnel(window: number, mode?: WindowFunnelMode) { const params = mode === undefined ? `${Math.round(window)}` : `${Math.round(window)}, '${mode}'` - return (timestamp: Expr, ...conditions: ReadonlyArray): Expr => { + return = ReadonlyArray>( + timestamp: Expr, + ...conditions: C + ): Expr> => { // Reported, not thrown: the number of conditions is the number of steps a // funnel has, and that count comes from data as often as from source. if (conditions.length === 0) { @@ -186,7 +189,10 @@ export function windowFunnel(window: number, mode?: WindowFunnelMode) { * user input, so only quote-free literals are accepted. */ export function sequenceMatch(pattern: string) { - return (timestamp: Expr, ...conditions: ReadonlyArray): Expr => { + return = ReadonlyArray>( + timestamp: Expr, + ...conditions: C + ): Expr> => { // An injection guard, so it reports rather than crashes: the pattern is // embedded verbatim, and "not user input" is a claim about the caller that // the caller is exactly who might get wrong. diff --git a/src/ch/functions/array.ts b/src/ch/functions/array.ts index ed343d8..70d8fa5 100644 --- a/src/ch/functions/array.ts +++ b/src/ch/functions/array.ts @@ -1,16 +1,18 @@ import { makeCond, makeExpr, toFragment } from "../expr" import { str, compile } from "../../sql/sql-fragment" -import type { Condition, Expr } from "../expr" +import type { Condition, Expr, ParamsIn } from "../expr" import { Schema } from "effect" import * as T from "../types" -import { elementOf, elementSchema, mergeResultSchemas, sameAs, schemaOf } from "../define-fn" +import { elementSchema, mergeResultSchemas, schemaOf } from "../define-fn" import { builtins } from "./builtin" -const { defineFn, lazy } = builtins("clickhouse", "scalar") +const { compileTypedFnCall, lazy } = builtins("clickhouse", "scalar") // Array constructors (handwritten — bracket syntax, not fn() call) -export function arrayOf(...exprs: Expr[]): Expr> { +export function arrayOf> = ReadonlyArray>>( + ...exprs: E & ReadonlyArray> +): Expr, ParamsIn> { const args = () => exprs.map((e) => compile(e.toFragment())).join(", ") const element = mergeResultSchemas(exprs.map((expr) => expr.schema)) return makeExpr(lazy(() => `[${args()}]`), element && Schema.Array(element)) @@ -18,56 +20,58 @@ export function arrayOf(...exprs: Expr[]): Expr> { // Array functions (handwritten — polymorphic or special syntax) -export function arrayStringConcat( - parts: Expr[] | Expr>, +export function arrayStringConcat> | Expr>>( + parts: Parts, sep: string, -): Expr { +): Expr> ? Parts[number] : Parts>> { if (Array.isArray(parts)) { const arr = () => parts.map((p: Expr) => compile(p.toFragment())).join(", ") return makeExpr(lazy(() => `arrayStringConcat([${arr()}], ${compile(str(sep))})`), T.string.schema) } + const expr = parts as Expr> return makeExpr( - lazy(() => `arrayStringConcat(${compile(parts.toFragment())}, ${compile(str(sep))})`), + lazy(() => `arrayStringConcat(${compile(expr.toFragment())}, ${compile(str(sep))})`), T.string.schema, ) } -export function arrayFilter(fn: string, arr: Expr>): Expr> { +export function arrayFilter(fn: string, arr: Expr, Q>): Expr, Q> { return makeExpr(lazy(() => `arrayFilter(${fn}, ${compile(arr.toFragment())})`), schemaOf>(arr)) } /** `arrayJoin` unnests, so the row value is one element of the array. */ -export const arrayJoin = (arr: Expr>): Expr => - defineFn<[Expr>], T>("arrayJoin", elementOf(0))(arr) +export const arrayJoin = (arr: Expr, Q>): Expr => + compileTypedFnCall("arrayJoin", elementSchema(schemaOf>(arr)), arr) /** * Array functions that hand back the array they were given, reordered or - * filtered — so the result decodes exactly as the input does. `sameAs(0)` says - * that once instead of once per function. + * filtered — so the result decodes exactly as the input does. */ -type ArrayFn = [Expr>] +export const arraySort = (arr: Expr, Q>): Expr, Q> => + compileTypedFnCall("arraySort", schemaOf>(arr), arr) -export const arraySort = (arr: Expr>): Expr> => - defineFn, ReadonlyArray>("arraySort", sameAs(0))(arr) +export const arrayReverseSort = (arr: Expr, Q>): Expr, Q> => + compileTypedFnCall("arrayReverseSort", schemaOf>(arr), arr) -export const arrayReverseSort = (arr: Expr>): Expr> => - defineFn, ReadonlyArray>("arrayReverseSort", sameAs(0))(arr) +export const arrayDistinct = (arr: Expr, Q>): Expr, Q> => + compileTypedFnCall("arrayDistinct", schemaOf>(arr), arr) -export const arrayDistinct = (arr: Expr>): Expr> => - defineFn, ReadonlyArray>("arrayDistinct", sameAs(0))(arr) - -export const arrayPushFront = (arr: Expr>, element: Expr): Expr> => - defineFn<[Expr>, Expr], ReadonlyArray>("arrayPushFront", (arr, element) => { - const item = mergeResultSchemas([elementSchema(arr.schema), element.schema]) - return item && Schema.Array(item) - })(arr, element) +export const arrayPushFront = ( + arr: Expr, Q1>, + element: Expr, +): Expr, Q1 | Q2> => { + const item = mergeResultSchemas([elementSchema(arr.schema), element.schema]) + return compileTypedFnCall("arrayPushFront", item && Schema.Array(item), arr, element) +} /** `arrayElement(arr, n)` — ClickHouse's 1-indexed subscript. The result is one * element, so it decodes as the array's element type. */ -export const arrayElement = (arr: Expr>, index: number | Expr): Expr => - defineFn<[Expr>, number | Expr], T>("arrayElement", elementOf(0))(arr, index) +export const arrayElement = ( + arr: Expr, Q1>, + index: number | Expr, +): Expr => compileTypedFnCall("arrayElement", elementSchema(schemaOf>(arr)), arr, index) -export function has(arr: Expr>, value: Expr | T): Condition { +export function has(arr: Expr, Q1>, value: Expr | T): Condition { const valueFragment = toFragment(value) return makeCond(lazy(() => `has(${compile(arr.toFragment())}, ${compile(valueFragment)})`)) } diff --git a/src/ch/functions/builtin.ts b/src/ch/functions/builtin.ts index 2304235..9b80b3c 100644 --- a/src/ch/functions/builtin.ts +++ b/src/ch/functions/builtin.ts @@ -17,7 +17,7 @@ import { compile, known, type SqlFragment } from "../../sql/sql-fragment" import { hidden, inAggregate } from "../../sql/render-tracker" import { activeDialect } from "../dialect" import { QueryBuilderDefect } from "../errors" -import { type Condition, type Expr, makeCond, makeExpr, makeUntypedExpr, toFragment } from "../expr" +import { type Condition, type Expr, makeCond, makeExpr, makeUntypedExpr, type ParamsIn, toFragment } from "../expr" import type { FnResult } from "../define-fn" export type FunctionSet = "clickhouse" | "postgres" @@ -54,20 +54,27 @@ export function builtins(set: FunctionSet | "portable", kind: BuiltinKind) { const call = (name: string, args: ReadonlyArray): SqlFragment => lazy(() => `${name}(${args.map((a) => compile(toFragment(a))).join(", ")})`, name) - function compileTypedFnCall(name: string, schema: Schema.Codec | undefined, ...args: unknown[]): Expr { + // The call helpers leave params to the signature of the function that uses + // them (`any` here); `defineFn` and `defineCondFn` carry their arguments' + // params on to their result. + + function compileTypedFnCall(name: string, schema: Schema.Codec | undefined, ...args: unknown[]): Expr { return makeExpr(call(name, args), schema) } - function compileFnCall(name: string, ...args: unknown[]): Expr { + function compileFnCall(name: string, ...args: unknown[]): Expr { return makeUntypedExpr(call(name, args)) } - function compileFnCallCond(name: string, ...args: unknown[]): Condition { + function compileFnCallCond(name: string, ...args: unknown[]): Condition { return makeCond(call(name, args)) } - function defineFn(name: string, result: FnResult): (...args: Args) => Expr { - return (...args: Args): Expr => + function defineFn( + name: string, + result: FnResult, + ): (...args: A) => Expr> { + return (...args: A) => compileTypedFnCall( name, typeof result === "function" ? result(...args) : (result.schema as Schema.Codec), @@ -75,8 +82,8 @@ export function builtins(set: FunctionSet | "portable", kind: BuiltinKind) { ) } - function defineCondFn(name: string): (...args: Args) => Condition { - return (...args: Args): Condition => compileFnCallCond(name, ...args) + function defineCondFn(name: string): (...args: A) => Condition> { + return (...args: A) => compileFnCallCond(name, ...args) } return { lazy, compileTypedFnCall, compileFnCall, compileFnCallCond, defineFn, defineCondFn } diff --git a/src/ch/functions/conditional.ts b/src/ch/functions/conditional.ts index 19f4e50..a434ac1 100644 --- a/src/ch/functions/conditional.ts +++ b/src/ch/functions/conditional.ts @@ -1,24 +1,27 @@ import { makeExpr, toFragment } from "../expr" import { compile } from "../../sql/sql-fragment" -import type { Expr, Condition } from "../expr" +import type { Expr, Condition, ParamsIn } from "../expr" import { Schema } from "effect" import { firstTypedNonNull, mergeResultSchemas, numericResultSchema, schemaOf } from "../define-fn" import { builtins } from "./builtin" -const { compileTypedFnCall, defineFn, lazy } = builtins("clickhouse", "scalar") +const { compileTypedFnCall, lazy } = builtins("clickhouse", "scalar") const portable = builtins("portable", "scalar") // if / multiIf (handwritten — standard fn shape but special arg types) /** Either branch can produce the result, including a nullable branch. */ -export const if_ = (cond: Condition, then_: Expr, else_: Expr): Expr => - defineFn<[Condition, Expr, Expr], T>("if", (_, then_, else_) => branchSchema(then_, else_))( - cond, - then_, - else_, - ) +export const if_ = ( + cond: Condition, + then_: Expr, + else_: Expr, +): Expr => compileTypedFnCall("if", branchSchema(then_, else_), cond, then_, else_) -export function multiIf(cases: Array<[Condition, Expr]>, else_: Expr): Expr { +export function multiIf< + T, + const Cases extends ReadonlyArray]> = ReadonlyArray]>, + Q = never, +>(cases: Cases & ReadonlyArray]>, else_: Expr): Expr | Q> { const parts = () => cases .map(([cond, val]) => `${compile(cond.toFragment())}, ${compile(val.toFragment())}`) .join(", ") @@ -48,17 +51,17 @@ type Coalesced[]> = Args extends readonly [ : ExprValue : null -export const coalesce = []>(...exprs: Args): Expr> => - portable.defineFn>("coalesce", firstTypedNonNull())(...exprs) +export const coalesce = []>(...exprs: Args): Expr, ParamsIn> => + portable.compileTypedFnCall("coalesce", firstTypedNonNull>()(...exprs), ...exprs) /** * `ifNull(expr, fallback)` — `expr` unless it is NULL, else `fallback`. The * two-argument coalesce; a non-nullable fallback strips the `| null`. */ -export const ifNull = (expr: Expr, fallback: Expr): Expr => - defineFn<[Expr, Expr], T>("ifNull", firstTypedNonNull())(expr, fallback) +export const ifNull = (expr: Expr, fallback: Expr): Expr => + compileTypedFnCall("ifNull", firstTypedNonNull<[Expr, Expr], T>()(expr, fallback), expr, fallback) -export function nullIf(expr: Expr, value: Expr | T): Expr { +export function nullIf(expr: Expr, value: Expr | T): Expr { // The result is `expr` or NULL, so it decodes as `expr` does — nullably. const schema = schemaOf(expr) return portable.compileTypedFnCall("nullIf", schema && Schema.NullOr(schema), expr, value) @@ -71,10 +74,10 @@ export function nullIf(expr: Expr, value: Expr | T): Expr { * SQL NULL passes through unchanged. For a guaranteed numeric result use * `ifNull(ifNotFinite(expr, 0), lit(0))`. */ -export function ifNotFinite( - expr: Expr, - fallback: number | Expr, -): Expr> { +export function ifNotFinite( + expr: Expr, + fallback: number | Expr, +): Expr, Q1 | Q2> { return makeExpr>( lazy(() => `ifNotFinite(${compile(expr.toFragment())}, ${compile(toFragment(fallback))})`), numericResultSchema(expr), diff --git a/src/ch/functions/date-time.ts b/src/ch/functions/date-time.ts index e9f7a51..a1d00b7 100644 --- a/src/ch/functions/date-time.ts +++ b/src/ch/functions/date-time.ts @@ -16,7 +16,7 @@ const { lazy } = builtins("clickhouse", "scalar") * `dateTimeString` and you get the string ClickHouse sent. */ type DateTimeValue = DateTime.Utc | string -type DateTimeExpr = Expr +type DateTimeExpr = Expr /** Keep the input's own decoding; fall back to parsed UTC for an untyped one. */ const sameDateTime = (col: Expr) => @@ -24,10 +24,10 @@ const sameDateTime = (col: Expr) => // Date/time functions (handwritten — custom INTERVAL syntax) -export function toStartOfInterval( - col: DateTimeExpr, - seconds: number | Expr, -): DateTimeExpr { +export function toStartOfInterval( + col: DateTimeExpr, + seconds: number | Expr, +): DateTimeExpr { const secStr = () => typeof seconds === "number" ? String(Math.round(seconds)) @@ -45,7 +45,9 @@ export function toStartOfInterval( * service-map edge rollup, and the dependencies tab all read from * `*_hourly` tables on this exact boundary). */ -export function toStartOfHour(col: DateTimeExpr): DateTimeExpr { +export function toStartOfHour( + col: DateTimeExpr, +): DateTimeExpr { return makeExpr(lazy(() => `toStartOfHour(${compile(col.toFragment())})`), sameDateTime(col)) } @@ -54,9 +56,9 @@ export function toStartOfHour(col: DateT * minute-grain counterpart of {@link toStartOfHour}, for queries spliced against * a `*_minutely` rollup. */ -export function toStartOfMinute( - col: DateTimeExpr, -): DateTimeExpr { +export function toStartOfMinute( + col: DateTimeExpr, +): DateTimeExpr { return makeExpr(lazy(() => `toStartOfMinute(${compile(col.toFragment())})`), sameDateTime(col)) } @@ -65,7 +67,7 @@ export function toStartOfMinute( * anomaly detector's seasonal-naive baseline to select "matched hours" (same * hour-of-day ±1) across the trailing week without storing baselines anywhere. */ -export function toHour(col: DateTimeExpr): Expr { +export function toHour(col: DateTimeExpr): Expr { return makeExpr(lazy(() => `toHour(${compile(col.toFragment())})`), T.uint8.schema) } @@ -75,7 +77,7 @@ export function toHour(col: DateTimeExpr): Expr { * "have we already sealed this hour" check) without forcing the consumer to * parse RFC3339. */ -export function toUnixTimestamp(col: DateTimeExpr): Expr { +export function toUnixTimestamp(col: DateTimeExpr): Expr { return makeExpr(lazy(() => `toUnixTimestamp(${compile(col.toFragment())})`), T.uint32.schema) } @@ -83,14 +85,14 @@ export function toUnixTimestamp(col: DateTimeExpr): Expr * `toUnixTimestamp64Nano(expr)` — convert DateTime64 to a nanosecond epoch. * Used for counter-rate delta windows where sub-second scrape spacing matters. */ -export function toUnixTimestamp64Nano(col: DateTimeExpr): Expr { +export function toUnixTimestamp64Nano(col: DateTimeExpr): Expr { return makeExpr(lazy(() => `toUnixTimestamp64Nano(${compile(col.toFragment())})`), T.uint64.schema) } -export function intervalSub( - col: DateTimeExpr, - seconds: number | Expr, -): DateTimeExpr { +export function intervalSub( + col: DateTimeExpr, + seconds: number | Expr, +): DateTimeExpr { const secStr = () => typeof seconds === "number" ? String(Math.round(seconds)) @@ -99,10 +101,10 @@ export function intervalSub( } /** The other half of {@link intervalSub} — `expr + INTERVAL n SECOND`. */ -export function intervalAdd( - col: DateTimeExpr, - seconds: number | Expr, -): DateTimeExpr { +export function intervalAdd( + col: DateTimeExpr, + seconds: number | Expr, +): DateTimeExpr { const secStr = () => typeof seconds === "number" ? String(Math.round(seconds)) @@ -111,7 +113,7 @@ export function intervalAdd( } /** `formatDateTime(expr, 'format')` — format a DateTime/DateTime64 as a string. */ -export function formatDateTime(col: DateTimeExpr, format: string): Expr { +export function formatDateTime(col: DateTimeExpr, format: string): Expr { return makeExpr( lazy(() => `formatDateTime(${compile(col.toFragment())}, ${compile(str(format))})`), T.string.schema, @@ -124,8 +126,8 @@ export function formatDateTime(col: DateTimeExpr, format: string) * require a Date/DateTime/DateTime64 argument and won't implicitly parse a * string literal. */ -export function toDateTime(col: Expr): DateTimeExpr -export function toDateTime(col: Expr): DateTimeExpr +export function toDateTime(col: Expr): DateTimeExpr +export function toDateTime(col: Expr): DateTimeExpr export function toDateTime(col: Expr): Expr { // String inputs retain the string flavour; numeric epoch inputs decode to UTC. const input = schemaOf(col) diff --git a/src/ch/functions/map.ts b/src/ch/functions/map.ts index 5033d9d..19448d2 100644 --- a/src/ch/functions/map.ts +++ b/src/ch/functions/map.ts @@ -1,6 +1,6 @@ import { makeCond, makeExpr } from "../expr" import { raw, str, compile } from "../../sql/sql-fragment" -import type { Expr, Condition } from "../expr" +import type { Expr, Condition, ParamsIn } from "../expr" import * as T from "../types" import { builtins } from "./builtin" @@ -11,23 +11,25 @@ const STRING_MAP = T.map(T.string, T.string).schema // Map functions (handwritten — bracket syntax or custom assembly) -export function mapContains(mapExpr: Expr>, key: string): Condition { +export function mapContains(mapExpr: Expr, Q>, key: string): Condition { return makeCond(lazy(() => `mapContains(${compile(mapExpr.toFragment())}, ${compile(str(key))})`)) } -export function mapGet(mapExpr: Expr>, key: string): Expr { +export function mapGet(mapExpr: Expr, Q>, key: string): Expr { return makeExpr(lazy(() => `${compile(mapExpr.toFragment())}[${compile(str(key))}]`), T.string.schema) } -export function mapKeys(mapExpr: Expr>): Expr> { +export function mapKeys(mapExpr: Expr, Q>): Expr, Q> { return makeExpr(lazy(() => `mapKeys(${compile(mapExpr.toFragment())})`), STRINGS) } -export function mapValues(mapExpr: Expr>): Expr> { +export function mapValues(mapExpr: Expr, Q>): Expr, Q> { return makeExpr(lazy(() => `mapValues(${compile(mapExpr.toFragment())})`), STRINGS) } -export function mapLiteral(...pairs: Array<[string, Expr]>): Expr> { +export function mapLiteral< + Pairs extends ReadonlyArray]> = ReadonlyArray]>, +>(...pairs: Pairs): Expr, ParamsIn> { if (pairs.length === 0) return makeExpr(lazy(() => "map()", "map"), STRING_MAP) const args = () => pairs.map(([k, v]) => `${compile(str(k))}, ${compile(v.toFragment())}`).join(", ") return makeExpr(lazy(() => `map(${args()})`), STRING_MAP) diff --git a/src/ch/functions/numeric.ts b/src/ch/functions/numeric.ts index 18b63d5..1cc94e3 100644 --- a/src/ch/functions/numeric.ts +++ b/src/ch/functions/numeric.ts @@ -1,5 +1,5 @@ import { numericResultSchema, firstTypedNonNull } from "../define-fn" -import type { Expr } from "../expr" +import type { Expr, ParamsIn } from "../expr" import * as T from "../types" import { builtins } from "./builtin" @@ -8,25 +8,25 @@ const { compileTypedFnCall, defineFn } = builtins("clickhouse", "scalar") // Type conversion (defineFn one-liners) // Inf/NaN and overflowing numeric strings parse successfully and decode as NaN. -export const toFloat64OrZero = (expr: Expr): Expr => +export const toFloat64OrZero = (expr: Expr): Expr => compileTypedFnCall("toFloat64OrZero", T.CHFloatResult, expr) -export const toFloat64 = (expr: Expr): Expr> => +export const toFloat64 = (expr: Expr): Expr, Q> => compileTypedFnCall("toFloat64", numericResultSchema(expr), expr) export const toUInt16OrZero = defineFn<[Expr], number>("toUInt16OrZero", T.uint16) export const toUInt64 = defineFn<[Expr | Expr], number>("toUInt64", T.uint64) -export const toInt64 = (expr: Expr): Expr> => +export const toInt64 = (expr: Expr): Expr, Q> => compileTypedFnCall("toInt64", numericResultSchema(expr), expr) // Arithmetic (compileFnCall wrappers for mixed arg types) -export function intDiv(a: Expr, b: number | Expr): Expr { +export function intDiv(a: Expr, b: number | Expr): Expr { return compileTypedFnCall("intDiv", T.int64.schema, a, b) } -export function round_( - expr: Expr, +export function round_( + expr: Expr, decimals?: number, -): Expr> { +): Expr, Q> { return decimals != null ? compileTypedFnCall("round", numericResultSchema(expr), expr, decimals) : compileTypedFnCall("round", numericResultSchema(expr), expr) @@ -37,14 +37,14 @@ export function round_( type Extremum[]> = Extract> extends never ? number | null : number -export function least_[]>(...exprs: Args): Expr> { - return defineFn>("least", firstTypedNonNull())(...exprs) +export function least_[]>(...exprs: Args): Expr, ParamsIn> { + return compileTypedFnCall("least", firstTypedNonNull>()(...exprs), ...exprs) } -export function greatest_[]>(...exprs: Args): Expr> { - return defineFn>("greatest", firstTypedNonNull())(...exprs) +export function greatest_[]>(...exprs: Args): Expr, ParamsIn> { + return compileTypedFnCall("greatest", firstTypedNonNull>()(...exprs), ...exprs) } -export function cityHash64(...exprs: Expr[]): Expr { +export function cityHash64[]>(...exprs: Args): Expr> { return compileTypedFnCall("cityHash64", T.uint64.schema, ...exprs) } diff --git a/src/ch/functions/string.ts b/src/ch/functions/string.ts index caf4421..9b04fd5 100644 --- a/src/ch/functions/string.ts +++ b/src/ch/functions/string.ts @@ -2,7 +2,7 @@ import { acceptsSqlNull } from "../define-fn" import { Schema } from "effect" import { makeCond } from "../expr" import { compile, str } from "../../sql/sql-fragment" -import type { Condition, Expr } from "../expr" +import type { Condition, Expr, ParamsIn } from "../expr" import * as T from "../types" import { builtins } from "./builtin" @@ -11,12 +11,12 @@ const portable = builtins("portable", "scalar") // Standard string functions (defineFn one-liners) -const stringResult = (name: string, expr: Expr): Expr> => +const stringResult = (name: string, expr: Expr): Expr, Q> => compileTypedFnCall(name, (expr.schema && acceptsSqlNull(expr.schema) ? Schema.NullOr(T.string.schema) : T.string.schema) as Schema.Codec, any>, expr) -export const toString_ = (expr: Expr): Expr> => stringResult("toString", expr) +export const toString_ = (expr: Expr): Expr, Q> => stringResult("toString", expr) export const length_ = defineFn<[Expr], number>("length", T.uint64) export const lower_ = portable.defineFn<[Expr], string>("lower", T.string) export const positionCaseInsensitive = defineFn<[Expr, Expr], number>( @@ -36,7 +36,7 @@ export const left_ = defineFn<[Expr, Expr], string>("left", T.st /** `hex(x)` — the hex rendering of any value's bytes, as a String. The usual * reason to reach for it is making a hash printable. */ -export const hex = (expr: Expr): Expr> => stringResult("hex", expr) +export const hex = (expr: Expr): Expr, Q> => stringResult("hex", expr) export const domain_ = defineFn<[Expr], string>("domain", T.string) export const path_ = defineFn<[Expr], string>("path", T.string) @@ -44,15 +44,15 @@ export const cutQueryString = defineFn<[Expr], string>("cutQueryString", // Mixed Expr + literal args (compileFnCall wrappers) -export function position_(haystack: Expr, needle: string): Expr { +export function position_(haystack: Expr, needle: string): Expr { return compileTypedFnCall("position", T.uint64.schema, haystack, needle) } -export function extract_(expr: Expr, pattern: string): Expr { +export function extract_(expr: Expr, pattern: string): Expr { return compileTypedFnCall("extract", T.string.schema, expr, pattern) } -export function replaceOne(haystack: Expr, pattern: string, replacement: string): Expr { +export function replaceOne(haystack: Expr, pattern: string, replacement: string): Expr { return compileTypedFnCall("replaceOne", T.string.schema, haystack, pattern, replacement) } @@ -64,18 +64,18 @@ export function replaceOne(haystack: Expr, pattern: string, replacement: * where a predicate is wanted, so the SQL reads as a condition rather than * `match(…) = 1`. */ -export function match_(haystack: Expr, pattern: string): Expr { +export function match_(haystack: Expr, pattern: string): Expr { return compileTypedFnCall("match", T.uint8.schema, haystack, pattern) } /** `match(haystack, pattern)` as a predicate — see {@link match_}. */ -export function matchCond(haystack: Expr, pattern: string): Condition { +export function matchCond(haystack: Expr, pattern: string): Condition { return makeCond(lazy(() => `match(${compile(haystack.toFragment())}, ${compile(str(pattern))})`)) } // Variadic string functions -export function concat(...exprs: Array | string>): Expr { +export function concat | string>>(...exprs: Args): Expr> { return compileTypedFnCall("concat", T.string.schema, ...exprs) } @@ -92,17 +92,17 @@ export function concat(...exprs: Array | string>): Expr { * Needles are literals by design — the multi-search family requires a constant * array, so there is no expression-valued overload to offer. */ -export function multiSearchAnyCaseInsensitive(haystack: Expr, needles: readonly string[]): Condition { +export function multiSearchAnyCaseInsensitive(haystack: Expr, needles: readonly string[]): Condition { const array = needles.map((needle) => compile(str(needle))).join(", ") return makeCond(lazy(() => `multiSearchAnyCaseInsensitive(${compile(haystack.toFragment())}, [${array}])`)) } -export function hasToken(haystack: Expr, token: Expr | string): Condition { +export function hasToken(haystack: Expr, token: Expr | string): Condition { const call = compileFnCall("hasToken", haystack, token) return makeCond(call.toFragment()) } -export function hasAllTokens(haystack: Expr, tokens: Expr | string): Condition { +export function hasAllTokens(haystack: Expr, tokens: Expr | string): Condition { const call = compileFnCall("hasAllTokens", haystack, tokens) return makeCond(call.toFragment()) } diff --git a/src/ch/functions/window.ts b/src/ch/functions/window.ts index 4fb2b7c..f4d3d1a 100644 --- a/src/ch/functions/window.ts +++ b/src/ch/functions/window.ts @@ -1,6 +1,6 @@ import { makeExpr, toFragment } from "../expr" import { compile } from "../../sql/sql-fragment" -import type { Expr } from "../expr" +import type { Expr, ParamsIn } from "../expr" import { schemaOf } from "../define-fn" import { QueryBuilderError } from "../errors" import { builtins } from "./builtin" @@ -10,17 +10,19 @@ const window = builtins("clickhouse", "window") export type WindowOrderDirection = "asc" | "desc" -export type WindowFrameBound = +export type WindowFrameBound

= | { readonly type: "CurrentRow" } | { readonly type: "UnboundedPreceding" } | { readonly type: "UnboundedFollowing" } - | { readonly type: "Preceding"; readonly value: number | Expr } - | { readonly type: "Following"; readonly value: number | Expr } + | { readonly type: "Preceding"; readonly value: number | Expr } + | { readonly type: "Following"; readonly value: number | Expr } -export interface WindowRowsFrame { +export interface WindowRowsFrame

{ readonly type: "RowsBetween" readonly start: WindowFrameBound readonly end: WindowFrameBound + /** phantom: the `param.*` placeholders in the bounds. */ + readonly _params?: (entries: P) => void } export interface WindowSpec { @@ -29,28 +31,39 @@ export interface WindowSpec { readonly frame?: WindowRowsFrame } -export interface CompiledWindowSpec { +export interface CompiledWindowSpec

{ readonly _brand: "WindowSpec" readonly sql: string + /** phantom: the `param.*` placeholders in the spec. */ + readonly _params?: (entries: P) => void } +/** The params of a window spec's partition, order and frame expressions. */ +type SpecParams = + | ParamsIn[number]> + | ParamsIn[number][0]> + | ParamsIn + export const currentRow: WindowFrameBound = { type: "CurrentRow" } export const unboundedPreceding: WindowFrameBound = { type: "UnboundedPreceding" } export const unboundedFollowing: WindowFrameBound = { type: "UnboundedFollowing" } -export function preceding(value: number | Expr): WindowFrameBound { +export function preceding(value: number | Expr): WindowFrameBound { return { type: "Preceding", value } } -export function following(value: number | Expr): WindowFrameBound { +export function following(value: number | Expr): WindowFrameBound { return { type: "Following", value } } -export function rowsBetween(start: WindowFrameBound, end: WindowFrameBound): WindowRowsFrame { +export function rowsBetween( + start: WindowFrameBound, + end: WindowFrameBound, +): WindowRowsFrame { return { type: "RowsBetween", start, end } } -export function windowSpec(spec: WindowSpec): CompiledWindowSpec { +export function windowSpec(spec: S): CompiledWindowSpec> { if (!spec.partitionBy?.length && !spec.orderBy?.length && !spec.frame) { throw new QueryBuilderError({ code: "InvalidArguments", @@ -84,16 +97,16 @@ function renderWindowSpec(spec: WindowSpec): string { return parts.join(" ") } -export function over(expr: Expr, spec: CompiledWindowSpec): Expr { +export function over(expr: Expr, spec: CompiledWindowSpec): Expr { // A window changes which rows feed the value, never how the value decodes. return makeExpr(window.lazy(() => `${compile(expr.toFragment())} OVER (${spec.sql})`), schemaOf(expr)) } -export function lagInFrame( - expr: Expr, - offset: number | Expr, - defaultValue: T | Expr, -): Expr { +export function lagInFrame( + expr: Expr, + offset: number | Expr, + defaultValue: T | Expr, +): Expr { return makeExpr( lazy(() => `lagInFrame(${compile(expr.toFragment())}, ${compile(toFragment(offset))}, ${compile(toFragment(defaultValue))})`, diff --git a/src/ch/insert.test.ts b/src/ch/insert.test.ts index 1162279..c65ef52 100644 --- a/src/ch/insert.test.ts +++ b/src/ch/insert.test.ts @@ -337,10 +337,12 @@ describe("insertInto", () => { const Plain = CH.table("plain", { A: CH.uint32, B: CH.nullable(CH.string) }) const errors = yield* Effect.all( [ + // @ts-expect-error -- no rows is a type error too CH.compile(CH.insertInto(Plain).values([])), CH.compile(CH.insertInto(Plain).values({ A: 1, C: 2 } as any)), CH.compile(CH.insertInto(Plain).values([{ B: undefined } as any])), CH.compile(CH.insertInto(Plain).values({ A: null as any })), + // @ts-expect-error -- a missing param is a type error too CH.compile(CH.insertInto(Plain).values({ A: CH.param.int("a") })), ].map(Effect.flip), ) diff --git a/src/ch/insert.ts b/src/ch/insert.ts index 8a9b9a1..98fc6b2 100644 --- a/src/ch/insert.ts +++ b/src/ch/insert.ts @@ -13,10 +13,10 @@ // }) // yield* Database.run(insert, { id, orgId }) -import type { Comparable, Condition, Expr, Widen } from "./expr" +import type { Comparable, Condition, Expr, ParamsIn, Widen } from "./expr" import type { CHQuery, ColumnAccessor, InferOutput, NeedsSelect } from "./query" import type { Table } from "./table" -import type { CHUnionQuery } from "./union" +import type { CHUnionQuery, QueryParams } from "./union" import type { CHType, ColumnDefs, InferTS } from "./types" /** @@ -134,7 +134,29 @@ export interface OnConflictDoNothing { readonly targetWhere?: ($: ColumnAccessor) => Condition } -export interface OnConflictDoUpdate { +/** + * `unknown` when a row or SET record names only columns of the table; + * otherwise a property naming the others. Needed where the record's type is + * inferred (to read its params), since inference lets extra keys through. + */ +export type OnlyColumns = [R] extends [never] + ? { readonly atLeastOneRow: "values() needs a row" } + : [Exclude] extends [never] + ? unknown + : { readonly notWritableColumns: Exclude } + +/** The columns an insert or update may write: every column but the computed ones. */ +export type WritableColumns = Exclude> + +/** The params of the values in a record (a row, a SET), or an array of them. */ +export type RecordParams = R extends ReadonlyArray ? ParamsIn : ParamsIn + +export interface OnConflictDoUpdate< + Cols extends ColumnDefs, + Computed extends string = never, + S extends ConflictSet = ConflictSet, + W = never, +> { /** Required: Postgres must know which index the update is for. */ readonly target: ConflictTarget readonly targetWhere?: ($: ColumnAccessor) => Condition @@ -144,16 +166,16 @@ export interface OnConflictDoUpdate ({ count: $.count.add(excluded.count) }))`. */ readonly set: - | ConflictSet - | (($: ColumnAccessor, excluded: ColumnAccessor) => ConflictSet) + | (S & OnlyColumns>) + | (($: ColumnAccessor, excluded: ColumnAccessor) => S & OnlyColumns>) /** Update only the existing rows this holds for; the others are skipped. */ - readonly where?: ($: ColumnAccessor, excluded: ColumnAccessor) => Condition + readonly where?: ($: ColumnAccessor, excluded: ColumnAccessor) => Condition } /** @internal — what an insert does on conflict. */ export type ConflictClause = | ({ readonly action: "nothing" } & OnConflictDoNothing) - | ({ readonly action: "update" } & OnConflictDoUpdate) + | ({ readonly action: "update" } & OnConflictDoUpdate) /** @internal — runtime insert state */ export interface CHInsertState { @@ -175,21 +197,22 @@ export interface CHInsert< Defaulted extends string = never, Computed extends string = never, Output = never, + Params = never, > { readonly _tag: "CHInsert" /** @internal — runtime insert state */ readonly _state: CHInsertState /** phantom. `output` is the row `Database.run` returns: none without RETURNING. */ - readonly _phantom?: { readonly cols: Cols; readonly output: Output } + readonly _phantom?: { readonly cols: Cols; readonly output: Output; readonly params: (entries: Params) => void } /** * The rows to insert: one row or an array. Calling it again replaces the * rows. Columns are written in table order whatever the key order, and a * column some rows leave out is written as `DEFAULT` in those rows. */ - values( - rows: InsertRow | ReadonlyArray>, - ): CHInsert + values>( + rows: (R | ReadonlyArray) & OnlyColumns>, + ): CHInsert> /** * `INSERT ... SELECT`: insert the rows a query (or union) selects. Each @@ -199,7 +222,7 @@ export interface CHInsert< */ select | CHUnionQuery>( query: Q & SelectFits, Cols, Defaulted, Computed> & NeedsSelect>, - ): CHInsert + ): CHInsert> /** * ClickHouse `SETTINGS` for this insert, such as @@ -207,7 +230,7 @@ export interface CHInsert< * identifiers; values are written as literals. Calling it again replaces * them. On a dialect without insert settings (Postgres) compiling is a defect. */ - settings(settings: Readonly>): CHInsert + settings(settings: Readonly>): CHInsert /** * Return the inserted rows: every column with no arguments (Drizzle's bare @@ -216,34 +239,36 @@ export interface CHInsert< * derived row schema. Postgres only; on a dialect without RETURNING * (ClickHouse) compiling is a defect. Calling it again replaces the list. */ - returning(): CHInsert }> + returning(): CHInsert }, Params> returning( ...columns: [K, ...Array] - ): CHInsert }> + ): CHInsert }, Params> returning>>( fn: ($: ColumnAccessor) => S, - ): CHInsert> + ): CHInsert, Params | ParamsIn> /** * `ON CONFLICT DO NOTHING`: skip a row that conflicts. With `returning`, a * skipped row returns nothing. Postgres only; replaces any earlier * `onConflict*`. */ - onConflictDoNothing(options?: OnConflictDoNothing): CHInsert + onConflictDoNothing(options?: OnConflictDoNothing): CHInsert /** * `ON CONFLICT (target) DO UPDATE SET ...`: an upsert. Postgres only; * replaces any earlier `onConflict*`. */ - onConflictDoUpdate(options: OnConflictDoUpdate): CHInsert + onConflictDoUpdate, W = never>( + options: OnConflictDoUpdate, + ): CHInsert | W> } const makeInsert = ( state: CHInsertState, -): CHInsert => ({ +): CHInsert => ({ _tag: "CHInsert", _state: state, - values: (rows) => + values: (rows: unknown) => makeInsert({ ...state, selectQuery: undefined, @@ -260,9 +285,10 @@ const makeInsert = Record>) : ($: any) => Object.fromEntries(columns.map((column) => [column, $[column]])) return makeInsert({ ...state, returningFn }) - }) as CHInsert["returning"], + }) as CHInsert["returning"], onConflictDoNothing: (options = {}) => makeInsert({ ...state, conflict: { action: "nothing", ...options } }), - onConflictDoUpdate: (options) => makeInsert({ ...state, conflict: { action: "update", ...options } }), + onConflictDoUpdate: (options: OnConflictDoUpdate) => + makeInsert({ ...state, conflict: { action: "update", ...options } }), }) /** @@ -282,5 +308,5 @@ export function insertInto({ table: table as Table }) } -export const isInsert = (value: unknown): value is CHInsert => +export const isInsert = (value: unknown): value is CHInsert => typeof value === "object" && value !== null && (value as { readonly _tag?: unknown })._tag === "CHInsert" diff --git a/src/ch/param.ts b/src/ch/param.ts index bb6ff17..de2291f 100644 --- a/src/ch/param.ts +++ b/src/ch/param.ts @@ -7,7 +7,7 @@ import { type DateTime, Schema } from "effect" import type { SqlFragment } from "../sql/sql-fragment" import { raw } from "../sql/sql-fragment" -import type { Expr } from "./expr" +import type { Comparable, Expr, ParamEntry } from "./expr" import { QueryBuilderDefect } from "./errors" import * as T from "./types" import type { CHType } from "./types" @@ -63,7 +63,13 @@ function assertValidParamName(name: string): void { // Param marker — used during query definition (before compilation) -export interface ParamMarker extends Expr { +/** + * What a param named in a query is filled with: a value of its type, and for a + * DateTime param a `Date` or a `'YYYY-MM-DD hh:mm:ss'` string as well. + */ +export type ParamValue = Comparable + +export interface ParamMarker extends Expr>> { readonly _paramName: N readonly _paramType?: T } diff --git a/src/ch/params-propagation.test-d.ts b/src/ch/params-propagation.test-d.ts new file mode 100644 index 0000000..a13f21f --- /dev/null +++ b/src/ch/params-propagation.test-d.ts @@ -0,0 +1,211 @@ +// Type-level tests: a `param.*` used inside a function call is a param of the +// query that uses the call. + +import type { DateTime } from "effect" +import { expectTypeOf } from "expect-type" +import * as CH from "./index" +import * as PG from "../postgres" +import type { InferQueryOutput } from "./query" +import type { Condition, Expr } from "./expr" + +const Users = CH.table("users", { + Id: CH.string, + Name: CH.string, + Age: CH.uint64, + Score: CH.float64, + Nick: CH.nullable(CH.string), + Attrs: CH.map(CH.string, CH.string), + Tags: CH.array(CH.string), + Nums: CH.array(CH.uint64), + CreatedAt: CH.dateTime64, +}) + +const PgUsers = CH.table("users", { id: PG.text, age: PG.int4, at: PG.timestamptz }) + +// aggregate.ts +const agg = CH.from(Users).select(($) => ({ n: CH.sumIf($.Age, $.Name.eq(CH.param.string("name"))) })) +// @ts-expect-error -- `name` is missing +CH.compileUnsafe(agg, {}) +CH.compileUnsafe(agg, { name: "a" }) + +// array.ts +const arr = CH.from(Users) + .select("Id") + .where(($) => [CH.has($.Tags, CH.param.string("tag"))]) +// @ts-expect-error -- `tag` is missing +CH.compileUnsafe(arr, {}) +CH.compileUnsafe(arr, { tag: "a" }) +const arrOf = CH.from(Users).select(($) => ({ xs: CH.arrayOf($.Name, CH.param.string("extra")) })) +expectTypeOf>().toEqualTypeOf<{ readonly xs: ReadonlyArray }>() +// @ts-expect-error -- `extra` is missing +CH.compileUnsafe(arrOf, {}) +CH.compileUnsafe(arrOf, { extra: "a" }) +const elem = CH.from(Users).select(($) => ({ x: CH.arrayElement($.Nums, CH.param.int("i")) })) +expectTypeOf>().toEqualTypeOf<{ readonly x: number }>() +// @ts-expect-error -- `i` is missing +CH.compileUnsafe(elem, {}) +CH.compileUnsafe(elem, { i: 1 }) + +// conditional.ts +const cond = CH.from(Users).select(($) => ({ + label: CH.if_($.Age.gt(CH.param.int("adult")), $.Name, CH.param.string("fallback")), +})) +expectTypeOf>().toEqualTypeOf<{ readonly label: string }>() +// @ts-expect-error -- `fallback` is missing +CH.compileUnsafe(cond, { adult: 18 }) +CH.compileUnsafe(cond, { adult: 18, fallback: "x" }) +const multi = CH.from(Users).select(($) => ({ + band: CH.multiIf( + [ + [$.Age.lt(CH.param.int("young")), CH.lit("young")], + [$.Age.lt(60), CH.lit("adult")], + ], + CH.lit("old"), + ), +})) +expectTypeOf>().toEqualTypeOf<{ readonly band: string }>() +// @ts-expect-error -- `young` is missing +CH.compileUnsafe(multi, {}) +CH.compileUnsafe(multi, { young: 18 }) +const coalesced = CH.from(Users).select(($) => ({ nick: CH.coalesce($.Nick, CH.param.string("anon")) })) +expectTypeOf>().toEqualTypeOf<{ readonly nick: string }>() +// @ts-expect-error -- `anon` is missing +CH.compileUnsafe(coalesced, {}) +CH.compileUnsafe(coalesced, { anon: "a" }) +expectTypeOf(CH.coalesce(CH.nullIf(CH.lit(1), 1), CH.lit(2))).toMatchTypeOf>() + +// date-time.ts +const bucketed = CH.from(Users).select(($) => ({ bucket: CH.toStartOfInterval($.CreatedAt, CH.param.int("step")) })) +expectTypeOf>().toEqualTypeOf<{ readonly bucket: DateTime.Utc }>() +// @ts-expect-error -- `step` is missing +CH.compileUnsafe(bucketed, {}) +CH.compileUnsafe(bucketed, { step: 60 }) +expectTypeOf(CH.toStartOfInterval(CH.param.dateTime("ts"), 60)).toMatchTypeOf>() + +// map.ts +const mapped = CH.from(Users) + .select("Id") + .where(($) => [CH.mapGet($.Attrs, "k").eq(CH.param.string("v")), CH.mapContains($.Attrs, "k")]) +// @ts-expect-error -- `v` is missing +CH.compileUnsafe(mapped, {}) +CH.compileUnsafe(mapped, { v: "a" }) +const mapLit = CH.from(Users).select(() => ({ m: CH.mapLiteral(["k", CH.param.string("mv")]) })) +// @ts-expect-error -- `mv` is missing +CH.compileUnsafe(mapLit, {}) +CH.compileUnsafe(mapLit, { mv: "a" }) + +// numeric.ts +const numeric = CH.from(Users).select(($) => ({ d: CH.intDiv($.Age, CH.param.int("by")) })) +// @ts-expect-error -- `by` is missing +CH.compileUnsafe(numeric, {}) +CH.compileUnsafe(numeric, { by: 2 }) + +// string.ts +const str = CH.from(Users).select(($) => ({ s: CH.concat($.Name, CH.param.string("suffix")) })) +expectTypeOf>().toEqualTypeOf<{ readonly s: string }>() +// @ts-expect-error -- `suffix` is missing +CH.compileUnsafe(str, {}) +CH.compileUnsafe(str, { suffix: "!" }) + +// window.ts +const windowed = CH.from(Users).select(($) => ({ + prev: CH.over( + CH.lagInFrame($.Score, 1, CH.param.float("dflt")), + CH.windowSpec({ partitionBy: [$.Name], orderBy: [[$.CreatedAt, "asc"]] }), + ), +})) +expectTypeOf>().toEqualTypeOf<{ readonly prev: number }>() +// @ts-expect-error -- `dflt` is missing +CH.compileUnsafe(windowed, {}) +CH.compileUnsafe(windowed, { dflt: 0 }) +const framed = CH.from(Users).select(($) => ({ + total: CH.over( + CH.sum($.Age), + CH.windowSpec({ + orderBy: [[$.CreatedAt, "asc"]], + frame: CH.rowsBetween(CH.preceding(CH.param.int("back")), CH.currentRow), + }), + ), +})) +// @ts-expect-error -- `back` is missing +CH.compileUnsafe(framed, {}) +CH.compileUnsafe(framed, { back: 3 }) +const partitioned = CH.from(Users).select(($) => ({ + n: CH.over(CH.count(), CH.windowSpec({ partitionBy: [CH.concat($.Name, CH.param.string("p"))] })), +})) +// @ts-expect-error -- `p` is missing +CH.compileUnsafe(partitioned, {}) +CH.compileUnsafe(partitioned, { p: "x" }) + +// subquery.ts +const inner = CH.from(Users) + .select("Id") + .where(($) => [$.Age.gt(CH.param.int("minAge"))]) +const viaIn = CH.from(Users) + .select("Id") + .where(($) => [CH.inSubquery($.Id, inner)]) +// @ts-expect-error -- the subquery's `minAge` is missing +CH.compileUnsafe(viaIn, {}) +CH.compileUnsafe(viaIn, { minAge: 1 }) +const viaExists = CH.from(Users) + .select("Id") + .where(() => [CH.exists(inner)]) +// @ts-expect-error -- the subquery's `minAge` is missing +CH.compileUnsafe(viaExists, {}) +CH.compileUnsafe(viaExists, { minAge: 1 }) +const viaExpr = CH.from(Users).select(() => ({ n: CH.subqueryExpr(inner, CH.uint64, (sql) => `(SELECT count() FROM (${sql}))`) })) +expectTypeOf>().toEqualTypeOf<{ readonly n: number }>() +// @ts-expect-error -- the subquery's `minAge` is missing +CH.compileUnsafe(viaExpr, {}) +CH.compileUnsafe(viaExpr, { minAge: 1 }) +// The SQL-string arm carries none. +CH.compileUnsafe(CH.from(Users).select("Id").where(() => [CH.subqueryCond("SELECT 1", (sql) => `1 IN (${sql})`)]), {}) + +// sql-template.ts +const typed = CH.sql(CH.float64)`1` +expectTypeOf(typed).toEqualTypeOf>() +const templated = CH.from(Users).select(($) => ({ x: CH.sql(CH.float64)`${$.Score} * ${CH.param.float("factor")}` })) +expectTypeOf>().toEqualTypeOf<{ readonly x: number }>() +// @ts-expect-error -- `factor` is missing +CH.compileUnsafe(templated, {}) +CH.compileUnsafe(templated, { factor: 2 }) +const tcond = CH.from(Users) + .select("Id") + .where(($) => [CH.sql.cond`${$.Name} = ${CH.param.string("who")}`]) +// @ts-expect-error -- `who` is missing +CH.compileUnsafe(tcond, {}) +CH.compileUnsafe(tcond, { who: "a" }) +expectTypeOf(CH.sql.cond`1 = 1`).toEqualTypeOf>() +const joined = CH.from(Users) + .select("Id") + .where(($) => [CH.sql.cond`${$.Age} IN (${CH.sql.join([1, CH.param.int("other")])})`]) +// @ts-expect-error -- `other` is missing +CH.compileUnsafe(joined, {}) +CH.compileUnsafe(joined, { other: 2 }) +const tsub = CH.from(Users) + .select("Id") + .where(($) => [CH.sql.cond`${$.Id} IN ${inner}`]) +// @ts-expect-error -- the subquery's `minAge` is missing +CH.compileUnsafe(tsub, {}) +CH.compileUnsafe(tsub, { minAge: 1 }) + +// pg/functions.ts +const pg = CH.from(PgUsers).select(($) => ({ n: PG.sumIf($.age, $.id.eq(CH.param.string("pgId"))) })) +// @ts-expect-error -- `pgId` is missing +PG.compileUnsafe(pg, {}) +PG.compileUnsafe(pg, { pgId: "a" }) +const pgBin = CH.from(PgUsers).select(($) => ({ day: PG.dateTrunc("day", PG.coalesce($.at, CH.param.dateTime("dflt"))) })) +expectTypeOf>().toEqualTypeOf<{ readonly day: DateTime.Utc }>() +// @ts-expect-error -- `dflt` is missing +PG.compileUnsafe(pgBin, {}) +PG.compileUnsafe(pgBin, { dflt: new Date() }) + +// An array literal keeps every element's params. +const joinedStr = CH.from(Users).select(($) => ({ + s: CH.arrayStringConcat([$.Name, CH.param.string("sepPart"), $.Id], ","), +})) +expectTypeOf>().toEqualTypeOf<{ readonly s: string }>() +// @ts-expect-error -- `sepPart` is missing +CH.compileUnsafe(joinedStr, {}) +CH.compileUnsafe(joinedStr, { sepPart: "-" }) +CH.compileUnsafe(CH.from(Users).select(($) => ({ s: CH.arrayStringConcat($.Tags, ",") })), {}) diff --git a/src/ch/publish-readiness.test.ts b/src/ch/publish-readiness.test.ts index 3bc6d5d..2b50a2f 100644 --- a/src/ch/publish-readiness.test.ts +++ b/src/ch/publish-readiness.test.ts @@ -244,6 +244,7 @@ describe("publishing regressions", () => { const B = T.custom("String", Schema.Literal("b")) const query = CH.from(One).select(() => ({ a: CH.param.of(A, "a"), b: CH.param.of(B, "b") })) expect(CH.compileUnsafe(query, { a: "a", b: "b" }).sql).toContain("'b' AS b") + // @ts-expect-error -- a mistyped param is a type error too expect(() => CH.compileUnsafe(query, { a: "b", b: "a" })).toThrow("not a valid value") }) diff --git a/src/ch/query.ts b/src/ch/query.ts index 6148c12..9520232 100644 --- a/src/ch/query.ts +++ b/src/ch/query.ts @@ -26,7 +26,7 @@ import type { ColumnDefs, CHType, InferTS, OutputToColumnDefs, NullableColumnDefs } from "./types" import type { Table } from "./table" -import type { Expr, Condition, ColumnRef, Widen } from "./expr" +import type { Expr, Condition, ColumnRef, ParamsIn, Widen } from "./expr" import { makeColumnRef } from "./expr" import type { TenantScope } from "./compile" @@ -80,10 +80,10 @@ export interface LockClause extends LockOptions { } /** Callback for ON conditions — receives main and joined column accessors. */ -export type JoinOnCallback = ( +export type JoinOnCallback = ( main: ColumnAccessor, joined: ColumnAccessor, -) => Condition +) => Condition

// Query state (runtime storage) @@ -107,11 +107,11 @@ export interface CHQueryState { readonly tableAlias?: string readonly columns: ColumnDefs readonly selectFn?: ($: any) => SelectRecord - readonly whereFn?: ($: any) => Array + readonly whereFn?: ($: any) => ReadonlyArray readonly groupByKeys: string[] /** Post-aggregation filter. Deliberately NOT consulted when deriving tenant * scope — see `having()` on the interface. */ - readonly havingFn?: ($: any) => Array + readonly havingFn?: ($: any) => ReadonlyArray readonly orderBySpecs: Array<[string, "asc" | "desc"]> readonly limitValue?: number readonly offsetValue?: number @@ -199,31 +199,33 @@ export interface CHQuery< Output extends Record = {}, Joins extends Record = {}, Route extends string | undefined = string | undefined, + /** The `param.*` placeholders the query uses, as `ParamEntry`s. */ + Params = never, > { /** @internal — runtime query state */ readonly _state: CHQueryState /** phantom */ - readonly _phantom?: { cols: Cols; output: Output; joins: Joins; route: Route } + readonly _phantom?: { cols: Cols; output: Output; joins: Joins; route: Route; params: (entries: Params) => void } /** Select specific columns by name. Output keys match column names. */ select( ...columns: K[] - ): CHQuery }, Joins, Route> + ): CHQuery }, Joins, Route, Params> /** Select computed expressions via callback. */ select( fn: ($: JoinedColumnAccessor) => S, - ): CHQuery, Joins, Route> + ): CHQuery, Joins, Route, Params | ParamsIn> /** * Filter rows: conditions AND-joined, an `undefined` one skipped. Calling it * again adds conditions, ANDed with the earlier ones. */ - where( - fn: ($: JoinedColumnAccessor) => Array, - ): CHQuery + where>( + fn: ($: JoinedColumnAccessor) => C, + ): CHQuery> - groupBy(...keys: Array): CHQuery + groupBy(...keys: Array): CHQuery /** * Post-aggregation filter, applied after `GROUP BY`. @@ -238,48 +240,48 @@ export interface CHQuery< * * Calling it again adds conditions, as `where` does. */ - having( - fn: ($: JoinedColumnAccessor) => Array, - ): CHQuery + having>( + fn: ($: JoinedColumnAccessor) => C, + ): CHQuery> - orderBy(...specs: Array>): CHQuery + orderBy(...specs: Array>): CHQuery /** At most `n` rows: a non-negative integer. */ - limit(n: RowCount): CHQuery + limit(n: RowCount): CHQuery /** Skip `n` rows: a non-negative integer. */ - offset(n: RowCount): CHQuery + offset(n: RowCount): CHQuery - format(fmt: "JSON" | "JSONEachRow"): CHQuery + format(fmt: "JSON" | "JSONEachRow"): CHQuery /** `SELECT DISTINCT`: drop duplicate output rows. */ - distinct(): CHQuery + distinct(): CHQuery /** * `SELECT DISTINCT ON (keys)`: keep the first row of each group of these * output aliases, in ORDER BY order (Postgres wants the keys to lead the * ORDER BY). Replaces `distinct()`. */ - distinctOn(...keys: [keyof Output & string, ...Array]): CHQuery + distinctOn(...keys: [keyof Output & string, ...Array]): CHQuery /** * `FOR UPDATE`: lock the selected rows until the transaction ends. Run it * inside `Database.transaction`. Postgres only; replaces any earlier lock. */ - forUpdate(options?: LockOptions): CHQuery + forUpdate(options?: LockOptions): CHQuery /** `FOR NO KEY UPDATE`: as `forUpdate`, without blocking inserts that reference the rows. */ - forNoKeyUpdate(options?: LockOptions): CHQuery + forNoKeyUpdate(options?: LockOptions): CHQuery /** `FOR SHARE`: a shared lock, which blocks writers but not other sharers. */ - forShare(options?: LockOptions): CHQuery + forShare(options?: LockOptions): CHQuery /** `FOR KEY SHARE`: the weakest lock, blocking only deletes and key updates. */ - forKeyShare(options?: LockOptions): CHQuery + forKeyShare(options?: LockOptions): CHQuery /** * Tag this query with an execution route, carried through to the compiled * query as a type-level fact. The tag is opaque to the builder: what routes * exist, and what an executor does with one, is the caller's vocabulary. */ - route(route: Route): CHQuery + route(route: NewRoute): CHQuery /** * Declare that this query deliberately reads across every tenant, forcing @@ -290,26 +292,26 @@ export interface CHQuery< * "someone forgot the tenant filter" until an author says which. Executors * are expected to refuse these on the ordinary read path. */ - crossTenant(): CHQuery + crossTenant(): CHQuery // Type-safe joins with Table - innerJoin( + innerJoin( table: Table, alias: FreshAlias, - on: JoinOnCallback, - ): CHQuery + on: JoinOnCallback, + ): CHQuery - leftJoin( + leftJoin( table: Table, alias: FreshAlias, - on: JoinOnCallback, - ): CHQuery }, Route> + on: JoinOnCallback, + ): CHQuery }, Route, Params | OnParams> crossJoin( table: Table, alias: FreshAlias, - ): CHQuery + ): CHQuery // Type-safe joins with subquery (CHQuery) @@ -318,26 +320,37 @@ export interface CHQuery< JOutput extends Record, JJoins extends Record, Alias extends string, + JParams = never, + OnParams = never, >( - query: CHQuery & NeedsSelect, + query: CHQuery & NeedsSelect, alias: FreshAlias, - on: JoinOnCallback>, - ): CHQuery }, Route> + on: JoinOnCallback, OnParams>, + ): CHQuery< + Cols, + Output, + Joins & { readonly [K in Alias]: OutputToColumnDefs }, + Route, + Params | JParams | OnParams + > leftJoinQuery< JCols extends ColumnDefs, JOutput extends Record, JJoins extends Record, Alias extends string, + JParams = never, + OnParams = never, >( - query: CHQuery & NeedsSelect, + query: CHQuery & NeedsSelect, alias: FreshAlias, - on: JoinOnCallback>, + on: JoinOnCallback, OnParams>, ): CHQuery< Cols, Output, Joins & { readonly [K in Alias]: NullableColumnDefs> }, - Route + Route, + Params | JParams | OnParams > crossJoinQuery< @@ -345,10 +358,11 @@ export interface CHQuery< JOutput extends Record, JJoins extends Record, Alias extends string, + JParams = never, >( - query: CHQuery & NeedsSelect, + query: CHQuery & NeedsSelect, alias: FreshAlias, - ): CHQuery }, Route> + ): CHQuery }, Route, Params | JParams> /** * Add a CTE (WITH clause). The CTE is prepended to the compiled query, and @@ -359,10 +373,10 @@ export interface CHQuery< * scope is *derived*, so a query whose only row source is a scoped CTE is * itself scoped without anyone asserting it. */ - withCTE>( + withCTE, CTEParams = never>( name: string, - query: CHQuery & NeedsSelect, - ): CHQuery + query: CHQuery & NeedsSelect, + ): CHQuery /** * Attach a CTE from pre-compiled SQL. @@ -377,7 +391,7 @@ export interface CHQuery< name: string, sql: string, options?: { readonly tenantScope?: TenantScope }, - ): CHQuery + ): CHQuery } // Type utilities for extracting output types from queries @@ -493,7 +507,7 @@ export function createJoinedColumnAccessor Array +type ConditionsFn = ($: any) => ReadonlyArray /** A second `where` (or `having`) ANDs with the first, as in Kysely: replacing * it would silently drop a filter, the tenant one included. */ @@ -515,7 +529,7 @@ function makeQuery< Output extends Record, Joins extends Record, Route extends string | undefined, ->(state: CHQueryState): CHQuery { +>(state: CHQueryState): CHQuery { return { _state: state, @@ -722,10 +736,11 @@ export function fromQuery< InnerOutput extends Record, InnerJoins extends Record, Alias extends string, + InnerParams = never, >( - query: CHQuery & NeedsSelect, + query: CHQuery & NeedsSelect, alias: Alias, -): CHQuery, {}, {}, undefined> { +): CHQuery, {}, {}, undefined, InnerParams> { return makeQuery({ tableName: alias, columns: {}, @@ -756,10 +771,10 @@ export function fromQuery< * .select($ => ({ ..., total: CH.sum($.edges.partial) })) * .groupBy("...") */ -export function fromUnion, Alias extends string>( - union: import("./union").CHUnionQuery, +export function fromUnion, Alias extends string, UnionParams = never>( + union: import("./union").CHUnionQuery, alias: Alias, -): CHQuery, {}, {}, undefined> { +): CHQuery, {}, {}, undefined, UnionParams> { return makeQuery({ tableName: alias, columns: {}, diff --git a/src/ch/soundness.test-d.ts b/src/ch/soundness.test-d.ts index 63d15e5..a4e0b5e 100644 --- a/src/ch/soundness.test-d.ts +++ b/src/ch/soundness.test-d.ts @@ -96,3 +96,64 @@ run(CH.update(Orders).set({ Amount: 1 }).where(($) => [$.Id.eq("a")])) run(CH.deleteFrom(Orders).allRows()) // @ts-expect-error -- INSERT ... SELECT from a query with no SELECT list CH.insertInto(CH.table("t", { a: CH.nullable(CH.string) })).select(CH.from(Users)) + +// Params: a query's `param.*` placeholders are in its type, and compile / run +// require them, with values of their types. +const byId = CH.from(Users) + .select("Id") + .where(($) => [$.Id.eq(CH.param.string("id"))]) + .where(($) => [$.Age.between(CH.param.int("minAge"), 99)]) +CH.compileUnsafe(byId, { id: "a", minAge: 1 }) +CH.compileUnsafe(byId, { id: "a", minAge: 1, unrelated: true }) +// @ts-expect-error -- `minAge` is missing +CH.compileUnsafe(byId, { id: "a" }) +// @ts-expect-error -- `id` is a string param +CH.compileUnsafe(byId, { id: 1, minAge: 1 }) +// @ts-expect-error -- no params at all +CH.compileUnsafe(byId) +// @ts-expect-error -- run checks them too +run(byId, {}) +run(byId, { id: "a", minAge: 1 }) +CH.compileUnsafe(CH.from(Users).select("Id")) + +// Through and/or/not, select, having, joins, subqueries in FROM, unions +const combined = CH.from(Users) + .innerJoin(Orders, "o", (u, o) => u.Id.eq(o.UserId).and(o.Amount.gt(CH.param.int("minAmount")))) + .select(($) => ({ Id: $.Id, scaled: $.Age.mul(CH.param.float("scale")) })) + .where(($) => [CH.or($.Name.eq(CH.param.string("name")), CH.not($.Nick.isNull()))]) +// @ts-expect-error -- needs minAmount, scale and name +CH.compileUnsafe(combined, { minAmount: 1, scale: 2 }) +CH.compileUnsafe(combined, { minAmount: 1, scale: 2, name: "n" }) +const outer = CH.fromQuery(byId, "b").select("Id") +// @ts-expect-error -- the subquery's params are the outer query's +CH.compileUnsafe(outer, {}) +const both = CH.unionAll(byId, CH.from(Users).select("Id").where(($) => [$.Name.eq(CH.param.string("name"))])) +// @ts-expect-error -- every branch's params +CH.compileUnionUnsafe(both, { id: "a", minAge: 1 }) +CH.compileUnionUnsafe(both, { id: "a", minAge: 1, name: "n" }) + +// A DateTime param takes a Date or a string as well +const Events = CH.table("events", { At: CH.dateTime64 }) +CH.compileUnsafe(CH.from(Events).select("At").where(($) => [$.At.gte(CH.param.dateTime("since"))]), { + since: new Date(), +}) + +// Writes +const ins = CH.insertInto(Orders).values({ Id: CH.param.string("id"), UserId: "u", Amount: 1 }) +// @ts-expect-error -- `id` is missing +CH.compileUnsafe(ins, {}) +CH.compileUnsafe(ins, { id: "x" }) +// @ts-expect-error -- not a column of the table +CH.insertInto(Orders).values({ Id: "a", UserId: "u", Amount: 1, Bogus: 1 }) +const upd = CH.update(Orders) + .set({ Amount: CH.param.int("amount") }) + .where(($) => [$.Id.eq(CH.param.string("id"))]) +// @ts-expect-error -- `amount` is missing +run(upd, { id: "a" }) +run(upd, { id: "a", amount: 1 }) +// @ts-expect-error -- not a column of the table +CH.update(Orders).set({ Bogus: 1 }).allRows() +const del = CH.deleteFrom(Orders).where(($) => [$.Id.eq(CH.param.string("id"))]) +// @ts-expect-error -- `id` is missing +run(del) +run(del, { id: "a" }) diff --git a/src/ch/sql-template.ts b/src/ch/sql-template.ts index b682f64..2f69fdf 100644 --- a/src/ch/sql-template.ts +++ b/src/ch/sql-template.ts @@ -24,9 +24,10 @@ import { DateTime } from "effect" import { currentDialect } from "./dialect" import { QueryBuilderError } from "./errors" -import { type Condition, type Expr, isExprLike, makeCond, makeExpr, makeUntypedExpr, toFragment } from "./expr" +import { type Condition, type Expr, isExprLike, makeCond, makeExpr, makeUntypedExpr, type ParamsIn, toFragment } from "./expr" import { compileCHUnsafe } from "./compile" import type { CHQuery } from "./query" +import type { QueryParams } from "./union" import { renderSubquery } from "./subquery-context" import type { CHType } from "./types" import { compile, lazy, quoteIdentPath, type SqlFragment } from "../sql/sql-fragment" @@ -153,21 +154,30 @@ const renderValueRaw = (value: unknown): string => { const fragmentOf = (strings: ReadonlyArray, values: ReadonlyArray): SqlFragment => lazy(() => `(${strings.reduce((text, part, index) => (index === 0 ? part : text + renderValue(values[index - 1]) + part), "")})`) -type Tag = (strings: TemplateStringsArray, ...values: ReadonlyArray) => A +/** The params of interpolated values: an expression's, a condition's, or a subquery's. */ +type ValueParams = ParamsIn | QueryParams + +type Tag = >( + strings: TemplateStringsArray, + ...values: V +) => Kind extends "cond" ? Condition> : Expr> export interface SqlTag { /** * An expression of `type`: `CH.sql(PG.text)\`...\``. The type decodes the * value when it is selected, so the query keeps its row schema. */ - (type: CHType): Tag> + (type: CHType): Tag<"expr", T> /** * An expression with no declared type. Selecting one costs the query its row * schema, as `untypedExpr` does; give a type where the value is selected. */ - (strings: TemplateStringsArray, ...values: ReadonlyArray): Expr + >( + strings: TemplateStringsArray, + ...values: V + ): Expr> /** A condition, for `where`, `having`, a join's ON, or `CH.and` / `CH.or`. */ - readonly cond: Tag + readonly cond: Tag<"cond"> /** SQL text spliced as-is. Only for text under your control, never for input. */ readonly raw: (sql: string) => SqlRaw /** A table or column name, quoted by the dialect. Plain names only, dotted for `schema.table`. */ @@ -177,7 +187,10 @@ export interface SqlTag { * interpolation is. Not parenthesized, so it fits a list: `IN (${sql.join(xs)})`. * An empty list fails the compile, since `IN ()` is not SQL. */ - readonly join: (values: ReadonlyArray, separator?: string) => Expr + readonly join: >( + values: V, + separator?: string, + ) => Expr> } const isTemplateStrings = (value: unknown): value is TemplateStringsArray => @@ -195,11 +208,11 @@ export const sql: SqlTag = Object.assign( makeExpr(fragmentOf(strings, inner), type.schema) }, { - cond: (strings: TemplateStringsArray, ...values: ReadonlyArray): Condition => + cond: (strings: TemplateStringsArray, ...values: ReadonlyArray): Condition => makeCond(fragmentOf(strings, values)), raw: (text: string): SqlRaw => mint({ _tag: RawTag, sql: text }), ident: (name: string): SqlIdent => mint({ _tag: IdentTag, name }), - join: (values: ReadonlyArray, separator = ", "): Expr => + join: (values: ReadonlyArray, separator = ", "): Expr => makeUntypedExpr( lazy(() => { if (values.length === 0) { diff --git a/src/ch/subquery.ts b/src/ch/subquery.ts index 0bc8b93..e05c236 100644 --- a/src/ch/subquery.ts +++ b/src/ch/subquery.ts @@ -10,6 +10,7 @@ import { renderSubquery } from "./subquery-context" import { compileCHUnsafe } from "./compile" import { type Condition, type Expr, makeCond, makeExpr, makeUntypedExpr } from "./expr" import type { CHQuery, NeedsSelect, SingleColumnOf } from "./query" +import type { QueryParams } from "./union" import type { CHType } from "./types" import { compile, lazy } from "../sql/sql-fragment" @@ -37,7 +38,9 @@ const toSql = (subquery: Subquery): string => // An inner tenant filter cannot confine an otherwise unfiltered outer source. /** `EXISTS (subquery)` — for correlated subqueries (see `outerRef`). */ -export function exists>(subquery: string | (CHQuery & NeedsSelect)): Condition { +export function exists, SP = never>( + subquery: string | (CHQuery & NeedsSelect), +): Condition { return makeCond(lazy(() => `EXISTS (${toSql(subquery)})`)) } @@ -45,10 +48,10 @@ export function exists>(subquery: string | (CHQuer * `expr IN (subquery)`. The subquery must select exactly one column, of a type * `expr` can be compared with. The SQL-string arm is unchecked. */ -export function inSubquery>( - expr: Expr, - subquery: string | (CHQuery & SingleColumnOf), -): Condition { +export function inSubquery, Q = never, SP = never>( + expr: Expr, + subquery: string | (CHQuery & SingleColumnOf), +): Condition { return makeCond(lazy(() => `${compile(expr.toFragment())} IN (${toSql(subquery)})`)) } @@ -58,10 +61,10 @@ export function inSubquery>( * Note ClickHouse's NULL semantics: if the subquery yields any NULL, `NOT IN` * is never true. Project a non-nullable column, or filter the NULLs inside. */ -export function notInSubquery>( - expr: Expr, - subquery: string | (CHQuery & SingleColumnOf), -): Condition { +export function notInSubquery, Q = never, SP = never>( + expr: Expr, + subquery: string | (CHQuery & SingleColumnOf), +): Condition { return makeCond(lazy(() => `${compile(expr.toFragment())} NOT IN (${toSql(subquery)})`)) } @@ -97,11 +100,11 @@ export function notInSubquery>( * Expression composition preserves deferred rendering, including when a caller * builds the expression or condition before the outer query. */ -export function subqueryExpr( - subquery: Subquery, +export function subqueryExpr( + subquery: S, type: CHType, wrap: (sql: string) => string = (sql) => `(${sql})`, -): Expr { +): Expr> { return makeExpr( lazy(() => wrap(toSql(subquery))), type.schema, @@ -111,15 +114,15 @@ export function subqueryExpr( /** {@link subqueryExpr} for a spliced value with no declared result type — a * sort tuple, an `argMin` tiebreaker. Selecting one costs the query its row * schema, the same as `untypedExpr`. */ -export function untypedSubqueryExpr( - subquery: Subquery, +export function untypedSubqueryExpr( + subquery: S, wrap: (sql: string) => string = (sql) => `(${sql})`, -): Expr { +): Expr> { return makeUntypedExpr(lazy(() => wrap(toSql(subquery)))) } /** {@link subqueryExpr} as a predicate — for the `IN`/`EXISTS` shapes the three * conditions above do not cover, such as `x IN (SELECT k FROM ())`. */ -export function subqueryCond(subquery: Subquery, wrap: (sql: string) => string): Condition { +export function subqueryCond(subquery: S, wrap: (sql: string) => string): Condition> { return makeCond(lazy(() => wrap(toSql(subquery)))) } diff --git a/src/ch/union.ts b/src/ch/union.ts index a4b68f6..32700b9 100644 --- a/src/ch/union.ts +++ b/src/ch/union.ts @@ -19,20 +19,20 @@ interface CHUnionState { // CHUnionQuery interface -export interface CHUnionQuery = {}> { +export interface CHUnionQuery = {}, Params = never> { readonly _tag: "CHUnionQuery" /** @internal — runtime union state */ readonly _state: CHUnionState /** phantom */ - readonly _phantom?: { output: Output } + readonly _phantom?: { output: Output; params: (entries: Params) => void } - orderBy(...specs: Array<[keyof Output & string, "asc" | "desc"]>): CHUnionQuery + orderBy(...specs: Array<[keyof Output & string, "asc" | "desc"]>): CHUnionQuery - limit(n: RowCount): CHUnionQuery + limit(n: RowCount): CHUnionQuery - offset(n: RowCount): CHUnionQuery + offset(n: RowCount): CHUnionQuery - format(fmt: "JSON" | "JSONEachRow"): CHUnionQuery + format(fmt: "JSON" | "JSONEachRow"): CHUnionQuery } /** Extract the Output type from a CHUnionQuery. */ @@ -40,7 +40,7 @@ export type InferUnionOutput = Q extends CHUnionQuery ? O : never // Implementation -function makeUnionQuery>(state: CHUnionState): CHUnionQuery { +function makeUnionQuery>(state: CHUnionState): CHUnionQuery { return { _tag: "CHUnionQuery" as const, _state: state, @@ -93,6 +93,13 @@ export type UnionBranchesFit> = [keyof OutputO ? unknown : { readonly unionColumnsDiffer: { [I in keyof Q]: BranchMisfits, OutputOf> }[number] } +/** The `ParamEntry`s of a query, union or write; `never` when it has none. */ +export type QueryParams = Q extends { readonly _phantom?: { readonly params: (entries: infer P) => void } } + ? 0 extends 1 & P + ? never + : P + : never + /** The union's row: the first branch's aliases, each typed as any branch's. */ export type UnionOutput> = { readonly [K in keyof OutputOf]: OutputOf[K] @@ -100,7 +107,7 @@ export type UnionOutput> = { export function unionAll]>( ...queries: Q & UnionBranchesFit -): CHUnionQuery> { +): CHUnionQuery, QueryParams> { return makeUnionQuery({ queries: queries as ReadonlyArray, outerOrderBySpecs: [], diff --git a/src/ch/update.ts b/src/ch/update.ts index da84ffb..d0bac02 100644 --- a/src/ch/update.ts +++ b/src/ch/update.ts @@ -16,8 +16,8 @@ // // CH.deleteFrom(ApiKeys).where(($) => [$.orgId.eq(CH.param.string("orgId"))]) -import type { Condition, Expr } from "./expr" -import type { ConflictSet, InsertSettingValue } from "./insert" +import type { Condition, Expr, ParamsIn } from "./expr" +import type { ConflictSet, InsertSettingValue, OnlyColumns, WritableColumns } from "./insert" import { appendConditions, type ColumnAccessor, type InferOutput } from "./query" import type { Table } from "./table" import type { ColumnDefs, InferTS } from "./types" @@ -31,12 +31,12 @@ export type UpdateSet /** The SET record of a table value: `UpdateSetOf`. */ export type UpdateSetOf = T extends Table ? UpdateSet : never -type WhereFn = ($: ColumnAccessor) => Array +type WhereFn = ($: ColumnAccessor) => C /** @internal — what UPDATE and DELETE share. */ interface WriteState { readonly table: Table - readonly whereFn?: ($: any) => Array + readonly whereFn?: ($: any) => ReadonlyArray /** Set by `allRows()`: the write is meant to touch every row. */ readonly allRows?: boolean readonly returningFn?: ($: any) => Record> @@ -60,64 +60,90 @@ type AllColumns = { readonly [P in keyof Cols & string] */ export type WriteReady = "ready" | "needs where() or allRows()" -/** The clauses UPDATE and DELETE share: `Filtered` is the write once it says - * which rows it changes, `Self` the write as it is. */ -interface WriteClauses { - /** - * The rows to change, as in a query's `where`: conditions AND-joined, an - * `undefined` one skipped. Calling it again adds conditions, ANDed with the - * earlier ones. - */ - where(fn: WhereFn): Filtered - /** Change every row. Without it (or `where`), the write cannot be compiled or run. */ - allRows(): Filtered - /** - * ClickHouse `SETTINGS` for this write, such as `{ mutations_sync: 2 }` so an - * `ALTER TABLE ... UPDATE` waits for the mutation. Postgres refuses them. - */ - settings(settings: Readonly>): Self -} +type Conditions = ReadonlyArray export interface CHUpdate< Cols extends ColumnDefs = ColumnDefs, Computed extends string = never, Output = never, Ready extends WriteReady = "ready", -> extends WriteClauses, CHUpdate> { + Params = never, +> { readonly _tag: "CHUpdate" /** @internal — runtime update state */ readonly _state: CHUpdateState /** phantom. `output` is the row `Database.run` returns: none without RETURNING. */ - readonly _phantom?: { readonly cols: Cols; readonly output: Output; readonly ready: Ready } + readonly _phantom?: { + readonly cols: Cols + readonly output: Output + readonly ready: Ready + readonly params: (entries: Params) => void + } - /** Replace the SET record. It must set at least one column. */ + /** Replace the SET record. It must set at least one column, and only columns the table can write. */ set>( - set: (S & NonEmptySet) | (($: ColumnAccessor) => S & NonEmptySet), - ): CHUpdate + set: + | (S & NonEmptySet & OnlyColumns>) + | (($: ColumnAccessor) => S & NonEmptySet & OnlyColumns>), + ): CHUpdate> + + /** + * The rows to change, as in a query's `where`: conditions AND-joined, an + * `undefined` one skipped. Calling it again adds conditions, ANDed with the + * earlier ones. + */ + where( + fn: WhereFn, + ): CHUpdate> + /** Change every row. Without it (or `where`), the write cannot be compiled or run. */ + allRows(): CHUpdate + /** + * ClickHouse `SETTINGS` for this write, such as `{ mutations_sync: 2 }` so an + * `ALTER TABLE ... UPDATE` waits for the mutation. Postgres refuses them. + */ + settings(settings: Readonly>): CHUpdate /** The changed rows, as for an insert: every column, the named ones, or a callback. Postgres only. */ - returning(): CHUpdate, Ready> + returning(): CHUpdate, Ready, Params> returning( ...columns: [K, ...Array] - ): CHUpdate }, Ready> + ): CHUpdate }, Ready, Params> returning>>( fn: ($: ColumnAccessor) => S, - ): CHUpdate, Ready> + ): CHUpdate, Ready, Params | ParamsIn> } -export interface CHDelete - extends WriteClauses, CHDelete> { +export interface CHDelete< + Cols extends ColumnDefs = ColumnDefs, + Output = never, + Ready extends WriteReady = "ready", + Params = never, +> { readonly _tag: "CHDelete" /** @internal — runtime delete state */ readonly _state: CHDeleteState - readonly _phantom?: { readonly cols: Cols; readonly output: Output; readonly ready: Ready } + readonly _phantom?: { + readonly cols: Cols + readonly output: Output + readonly ready: Ready + readonly params: (entries: Params) => void + } + + /** The rows to delete, as for an UPDATE. Calling it again ANDs the conditions. */ + where(fn: WhereFn): CHDelete> + /** Delete every row. Without it (or `where`), the delete cannot be compiled or run. */ + allRows(): CHDelete + /** ClickHouse `SETTINGS`, such as `{ lightweight_deletes_sync: 2 }`. Postgres refuses them. */ + settings(settings: Readonly>): CHDelete /** The deleted rows, as for an insert: every column, the named ones, or a callback. Postgres only. */ - returning(): CHDelete, Ready> + returning(): CHDelete, Ready, Params> returning( ...columns: [K, ...Array] - ): CHDelete }, Ready> - returning>>(fn: ($: ColumnAccessor) => S): CHDelete, Ready> + ): CHDelete }, Ready, Params> + returning>>( + fn: ($: ColumnAccessor) => S, + ): CHDelete, Ready, Params | ParamsIn> } /** @@ -145,21 +171,21 @@ export const returningFnOf = } const writeClauses = (state: State, make: (state: State) => Self) => ({ - where: (whereFn: ($: any) => Array) => + where: (whereFn: ($: any) => ReadonlyArray) => make({ ...state, whereFn: appendConditions(state.whereFn, whereFn) }), allRows: () => make({ ...state, allRows: true }), settings: (settings: Readonly>) => make({ ...state, settings: { ...settings } }), returning: (...args: ReadonlyArray) => make({ ...state, returningFn: returningFnOf(state.table)(args) }), }) -const makeUpdate = (state: CHUpdateState): CHUpdate => ({ +const makeUpdate = (state: CHUpdateState): CHUpdate => ({ _tag: "CHUpdate", _state: state, ...writeClauses(state, makeUpdate), set: (set: CHUpdateState["set"]) => makeUpdate({ ...state, set }), }) -const makeDelete = (state: CHDeleteState): CHDelete => ({ +const makeDelete = (state: CHDeleteState): CHDelete => ({ _tag: "CHDelete", _state: state, ...writeClauses(state, makeDelete), @@ -179,8 +205,8 @@ export function deleteFrom( return makeDelete({ table: table as Table }) as CHDelete } -export const isUpdate = (value: unknown): value is CHUpdate => +export const isUpdate = (value: unknown): value is CHUpdate => typeof value === "object" && value !== null && (value as { readonly _tag?: unknown })._tag === "CHUpdate" -export const isDelete = (value: unknown): value is CHDelete => +export const isDelete = (value: unknown): value is CHDelete => typeof value === "object" && value !== null && (value as { readonly _tag?: unknown })._tag === "CHDelete" diff --git a/src/database/database.test.ts b/src/database/database.test.ts index e1648a4..f278e8d 100644 --- a/src/database/database.test.ts +++ b/src/database/database.test.ts @@ -67,6 +67,7 @@ layer(Live, { excludeTestServices: true })("Database on PGlite", (it) => { .select("id") .where(($) => [$.note.eq(CH.param.string("note"))]) expect(yield* Db.run(byNote, { note: "b" })).toEqual([{ id: 2 }]) + // @ts-expect-error -- a missing param is a type error too const error = yield* Effect.flip(Db.run(byNote)) expect(error).toBeInstanceOf(CH.QueryBuilderError) }), diff --git a/src/database/database.ts b/src/database/database.ts index 4465bcf..cedb3e5 100644 --- a/src/database/database.ts +++ b/src/database/database.ts @@ -17,7 +17,8 @@ import type { QueryBuilderError } from "../ch/errors" import type { CHInsert } from "../ch/insert" import type { CHDelete, CHUpdate } from "../ch/update" import type { CHQuery, NeedsSelect } from "../ch/query" -import type { CHUnionQuery } from "../ch/union" +import type { CHUnionQuery, QueryParams } from "../ch/union" +import type { ParamsSatisfied } from "../ch/expr" import { DatabaseError, TransactionClosed, @@ -102,9 +103,9 @@ export interface DatabaseApi { * `params` fills the query's `param.*` markers. A query compiled elsewhere * runs as it is, if it was compiled for this dialect. */ - readonly run: ( - query: Q & RunCheck, - params?: Record, + readonly run: = {}>( + query: Q & RunCheck & ParamsSatisfied, Given>, + params?: Given, ) => Effect.Effect>, DatabaseError | QueryBuilderError | CompiledQueryDecodeError> /** Run a statement and return its rows, decoded through `schema` when given. */ readonly query: { @@ -281,8 +282,8 @@ export const fromSqlClient = (sql: SqlClient.SqlClient, options: FromSqlClientOp // A write without RETURNING sends back no rows, so it runs the way `execute` // does: through `command`, which a ClickHouse client needs for a statement // with no result set. - const run: DatabaseApi["run"] = (runnable, params = {}) => - Effect.flatMap(compileFor(runnable, params), (compiled) => + const run: DatabaseApi["run"] = (runnable, params) => + Effect.flatMap(compileFor(runnable, params ?? {}), (compiled) => compiled.kind !== "select" && compiled.returning === undefined ? Effect.as(execute(compiled), []) : Effect.flatMap(rows(compiled), (wire) => compiled.decodeRows(wire)), @@ -448,9 +449,9 @@ export const layerSqlClient = (options: FromSqlClientOptions): Layer.Layer( - query: Q & RunCheck, - params?: Record, +export const run = = {}>( + query: Q & RunCheck & ParamsSatisfied, Given>, + params?: Given, ): Effect.Effect>, DatabaseError | QueryBuilderError | CompiledQueryDecodeError, Database> => Effect.flatMap(Effect.service(Database), (db) => db.run(query, params)) diff --git a/src/pg/functions.ts b/src/pg/functions.ts index 1e72b26..819b858 100644 --- a/src/pg/functions.ts +++ b/src/pg/functions.ts @@ -28,24 +28,27 @@ const int8 = T.int8.schema as Schema.Codec export const count = (): Expr => makeExpr(aggregate.lazy(() => "count(*)", "count"), int8) /** `count(DISTINCT expr)`. */ -export const countDistinct = (expr: Expr): Expr => +export const countDistinct = (expr: Expr): Expr => makeExpr(aggregate.lazy(() => `count(DISTINCT ${sql(expr)})`), int8) /** `count(*) FILTER (WHERE condition)`: ClickHouse's `countIf`. */ -export const countIf = (condition: Condition): Expr => +export const countIf = (condition: Condition): Expr => makeExpr(aggregate.lazy(() => `count(*) FILTER (WHERE ${sql(condition)})`), int8) /** `sum(expr)`. NULL over no rows, and a string for int8/numeric inputs on the * wire, which the result codec reads as a number. */ -export const sum = (expr: Expr): Expr => +export const sum = (expr: Expr): Expr => makeExpr(aggregate.lazy(() => `sum(${sql(expr)})`), nullableNumber) /** `sum(expr) FILTER (WHERE condition)`: ClickHouse's `sumIf`. */ -export const sumIf = (expr: Expr, condition: Condition): Expr => +export const sumIf = ( + expr: Expr, + condition: Condition, +): Expr => makeExpr(aggregate.lazy(() => `sum(${sql(expr)}) FILTER (WHERE ${sql(condition)})`), nullableNumber) /** `avg(expr)`. NULL over no rows. */ -export const avg = (expr: Expr): Expr => +export const avg = (expr: Expr): Expr => makeExpr(aggregate.lazy(() => `avg(${sql(expr)})`), nullableNumber) const nullableOf = (expr: Expr): Schema.Codec | undefined => { @@ -54,14 +57,14 @@ const nullableOf = (expr: Expr): Schema.Codec | undefin } /** `min(expr)`, decoding as `expr` does. NULL over no rows. */ -export const min = (expr: Expr): Expr => makeExpr(aggregate.lazy(() => `min(${sql(expr)})`), nullableOf(expr)) +export const min = (expr: Expr): Expr => makeExpr(aggregate.lazy(() => `min(${sql(expr)})`), nullableOf(expr)) /** `max(expr)`, decoding as `expr` does. NULL over no rows. */ -export const max = (expr: Expr): Expr => makeExpr(aggregate.lazy(() => `max(${sql(expr)})`), nullableOf(expr)) +export const max = (expr: Expr): Expr => makeExpr(aggregate.lazy(() => `max(${sql(expr)})`), nullableOf(expr)) /** `percentile_cont(fraction) WITHIN GROUP (ORDER BY expr)`: an interpolated * quantile, ClickHouse's `quantileExact` family. */ -export const percentileCont = (fraction: number, expr: Expr): Expr => { +export const percentileCont = (fraction: number, expr: Expr): Expr => { if (!(fraction >= 0 && fraction <= 1)) { throw new QueryBuilderDefect({ message: `percentileCont: fraction must be within [0, 1], got ${fraction}` }) } @@ -69,7 +72,7 @@ export const percentileCont = (fraction: number, expr: Expr): Exp } /** `array_agg(expr)`. NULL over no rows. */ -export const arrayAgg = (expr: Expr): Expr | null> => { +export const arrayAgg = (expr: Expr): Expr | null, Q> => { const element = schemaOf(expr) return makeExpr( aggregate.lazy(() => `array_agg(${sql(expr)})`), @@ -85,12 +88,12 @@ export type DateTruncUnit = "second" | "minute" | "hour" | "day" | "week" | "mon /** `date_trunc(unit, ts, 'UTC')`: buckets in UTC whatever the session time * zone, as ClickHouse's `toStartOf*` functions do. Postgres 12+. */ -export const dateTrunc = (unit: DateTruncUnit, ts: Expr): Expr => +export const dateTrunc = (unit: DateTruncUnit, ts: Expr): Expr => makeExpr(scalar.lazy(() => `date_trunc(${compile(str(unit))}, ${sql(ts)}, 'UTC')`), timestamptz) /** `date_bin(seconds, ts, epoch)`: fixed-width buckets aligned to the Unix * epoch, ClickHouse's `toStartOfInterval`. Postgres 14+. */ -export const dateBin = (seconds: number, ts: Expr): Expr => { +export const dateBin = (seconds: number, ts: Expr): Expr => { if (!(Number.isSafeInteger(seconds) && seconds > 0)) { throw new QueryBuilderDefect({ message: `dateBin: bucket width must be a positive whole number of seconds, got ${seconds}` }) } @@ -107,18 +110,18 @@ export const now = (): Expr => makeExpr(scalar.lazy(() => "now()", const text = T.text.schema as Schema.Codec -export const lower = (expr: Expr): Expr => makeExpr(scalar.lazy(() => `lower(${sql(expr)})`), text) -export const upper = (expr: Expr): Expr => makeExpr(scalar.lazy(() => `upper(${sql(expr)})`), text) -export const length = (expr: Expr): Expr => +export const lower = (expr: Expr): Expr => makeExpr(scalar.lazy(() => `lower(${sql(expr)})`), text) +export const upper = (expr: Expr): Expr => makeExpr(scalar.lazy(() => `upper(${sql(expr)})`), text) +export const length = (expr: Expr): Expr => makeExpr(scalar.lazy(() => `length(${sql(expr)})`), T.int4.schema as Schema.Codec) /** `coalesce(expr, fallback)`, no longer nullable. */ -export const coalesce = (expr: Expr, fallback: Expr): Expr => +export const coalesce = (expr: Expr, fallback: Expr): Expr => makeExpr( scalar.lazy(() => `coalesce(${sql(expr)}, ${sql(fallback)})`), schemaOf(fallback) ?? withoutNull(schemaOf(expr)), ) /** `expr ->> key`: a jsonb field as text, NULL when it is absent. */ -export const jsonText = (expr: Expr, key: string): Expr => +export const jsonText = (expr: Expr, key: string): Expr => makeExpr(scalar.lazy(() => `(${sql(expr)} ->> ${compile(str(key))})`), Schema.NullOr(Schema.String) as Schema.Codec) diff --git a/src/pg/postgres.test.ts b/src/pg/postgres.test.ts index df788be..747a61b 100644 --- a/src/pg/postgres.test.ts +++ b/src/pg/postgres.test.ts @@ -197,6 +197,7 @@ describe("postgres dialect", () => { .select(($) => ({ count: $.Count })) .where(($) => [$.OrgId.eq(CH.param.string("orgId"))]) expect(() => PG.compileUnsafe(query.format("JSON"), { orgId: "o" })).toThrow(/no FORMAT clause/) + // @ts-expect-error -- a missing param is a type error too expect(() => PG.compileUnsafe(query, {})).toThrow(/no value given for param 'orgId'/) }) }) From e1bf55b640f592f897441a9ee0780a310410cf3f Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 4 Oct 2026 22:59:57 +0200 Subject: [PATCH 4/5] Close the remaining ways a param could leave the type - makeExpr / makeUntypedExpr / makeCond take the expressions they interpolate as `uses` and carry their params; compiling one whose SQL holds a param no `uses` entry carries is a defect, so a param can no longer reach a query without being in its type. - An explicit type argument on makeExpr, subqueryExpr or compileTypedFnCall is now an error (the inferred parameter comes first) instead of silently dropping params; untypedSubqueryExpr returns Expr. - inSubquery / notInSubquery check at compile time that the subquery selects exactly one column, for callers past the types. - param.dateTimeString / dateTimeSeconds accept a Date or DateTime.Utc in the type, as they do at runtime. - Integration fixtures and the docs behaviour check no longer rely on a second where() replacing the first. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 +++ docs/extending.md | 34 +++++++------- docs/params-and-compilation.md | 1 + docs/reference.md | 8 ++-- scripts/check-doc-examples.mjs | 7 +-- src/ch/compile.ts | 6 +++ src/ch/custom-expr.ts | 86 ++++++++++++++++++++++++++++++++++ src/ch/define-fn.ts | 9 ++-- src/ch/param.ts | 12 ++--- src/ch/soundness.test-d.ts | 25 ++++++++++ src/ch/soundness.test.ts | 29 ++++++++++++ src/ch/subquery.ts | 37 +++++++++++---- src/docs-examples.test.ts | 2 +- src/expr.ts | 5 +- tests/core-cases.ts | 16 +++---- 15 files changed, 230 insertions(+), 53 deletions(-) create mode 100644 src/ch/custom-expr.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 86bed25..f31eaec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,12 @@ - Built-in functions belong to a dialect: a ClickHouse function (such as `count()`) in a Postgres compile fails, and the reverse. `coalesce`, `nullIf` and `lower` are portable. `Dialect.functions` names a dialect's function set. + - `makeExpr`, `makeUntypedExpr` and `makeCond` take the expressions they interpolate as + `uses`, whose params the result carries; a param in the SQL that no `uses` entry carries + fails to compile. Their value type comes from the schema: explicit type arguments + (`makeExpr`, `subqueryExpr`, `compileTypedFnCall`) are errors, so they cannot + silently drop params. `untypedSubqueryExpr` returns `Expr`. + - `inSubquery` / `notInSubquery` check at compile time that the subquery selects one column. - Add `CH.sql`: SQL templates inside expressions and conditions. `CH.sql(type)\`…\`` is a typed `Expr`, ``CH.sql`…` `` an untyped one, `CH.sql.cond` a `Condition`; with `sql.ident`, `sql.raw` and `sql.join`. Interpolated columns and params render as SQL and placeholders, a builder diff --git a/docs/extending.md b/docs/extending.md index 05d8393..8c9de15 100644 --- a/docs/extending.md +++ b/docs/extending.md @@ -78,7 +78,7 @@ import * as CH from "@maple-dev/effect-orm" import * as T from "@maple-dev/effect-orm/types" const greatestOf = (first: CH.Expr, ...rest: CH.Expr[]) => - CH.compileTypedFnCall("greatest", T.float64.schema, first, ...rest) + CH.compileTypedFnCall("greatest", T.float64.schema, first, ...rest) const Events = CH.table("events", { Name: T.string, DurationMs: T.uint64 }) export const compiled = CH.compileUnsafe( @@ -104,11 +104,16 @@ anything bespoke: import { makeExpr } from "@maple-dev/effect-orm" import { raw, compile } from "@maple-dev/effect-orm/sql" -const quantileExact = (q: number) => (expr: CH.Expr) => - makeExpr(raw(`quantileExact(${q})(${compile(expr.toFragment())})`), T.float64.schema) +const quantileExact = + (q: number) => + (expr: CH.Expr) => + makeExpr(raw(`quantileExact(${q})(${compile(expr.toFragment())})`), T.float64.schema, undefined, [expr]) ``` -This is how the bundled `quantile` is built. Note the second argument: `makeExpr` requires a +This is how the bundled `quantile` is built. The last argument, `uses`, lists the expressions +the fragment interpolates (see [below](#params-and-checks-on-a-custom-function)). The value +type comes from the schema; `makeExpr(…)` with an explicit type argument does not +type-check. Note the second argument: `makeExpr` requires a schema — passing `undefined` is how a wrapper _forwards_ the untypedness of its own argument (`schemaOf(arg)`), not something to write. For an expression that genuinely has no type, use `makeUntypedExpr`, which says so and costs the query its row schema knowingly. @@ -131,18 +136,15 @@ control, and validate numeric inputs such as the quantile level separately. ### Params and checks on a custom function `Expr` carries the `param.*` placeholders inside an expression, so `compile` can require -them. `defineFn` and `defineCondFn` pass their arguments' params on by themselves. A wrapper -built with `makeExpr` says so in its signature, with one type parameter per argument: - -```ts -const quantileExact = - (q: number) => - (expr: CH.Expr): CH.Expr => - makeExpr(raw(`quantileExact(${q})(${compile(expr.toFragment())})`), T.float64.schema) -``` - -Without it the function still works; a param inside it is then checked when compiling rather -than by the type. SQL built with `makeExpr`, `defineFn` or `CH.sql` is also opaque to the GROUP +them. `defineFn`, `defineCondFn` and `compileTypedFnCall` pass their arguments' params on by +themselves. `makeExpr`, `makeUntypedExpr` and `makeCond` cannot see inside the SQL you build, +so they take the expressions you interpolate as `uses` (the last argument): the result carries +their params, and compiling fails with a `QueryBuilderDefect` if the SQL holds a param that no +`uses` entry carries. A param can therefore not reach a query without being in its type. + +Generic functions take their params as one type parameter per argument (`Expr` +above); a parameter written as a plain `Expr` accepts any expression but drops its params +from the type, so they are then checked only when compiling. SQL built with `makeExpr`, `defineFn` or `CH.sql` is also opaque to the GROUP BY checks (see [Queries](./queries.md#groupby)): a mistake inside it reaches the database, but it never makes a valid query fail. diff --git a/docs/params-and-compilation.md b/docs/params-and-compilation.md index e307b62..70c1450 100644 --- a/docs/params-and-compilation.md +++ b/docs/params-and-compilation.md @@ -305,6 +305,7 @@ const query = CH.from(Events) .where(($) => [$.Name.eq(CH.param.string("name"))]) export const outcome = await Effect.runPromise( + // @ts-expect-error -- a missing param is a type error too; this shows the runtime failure CH.compile(query, {}).pipe( Effect.map((compiled) => ({ ok: true as const, sql: compiled.sql })), Effect.catchTag("@maple-dev/effect-orm/QueryBuilderError", (error) => diff --git a/docs/reference.md b/docs/reference.md index bbc62c1..a6c7c97 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -152,7 +152,7 @@ outer set and its failures land in the outer error channel. See | Export | Purpose | | ---------------------------------- | ----------------------------------------- | | `subqueryExpr(q, type, wrap?)` | Inner SQL as an `Expr` of a declared type | -| `untypedSubqueryExpr(q, wrap?)` | Same with no type — costs the row schema | +| `untypedSubqueryExpr(q, wrap?)` | Same with no type — costs the row schema | | `subqueryCond(q, wrap)` | Inner SQL as a `Condition` | `wrap` receives the inner SQL and returns the text to emit. It defaults to wrapping the SQL in @@ -177,9 +177,9 @@ parentheses, which is the plain "this value is a sub-SELECT" case. | `compileFnCall(name, ...args)` | Variadic/generic wrapper (untyped result) | | `compileTypedFnCall(name, schema,)` | Same, with the result codec | | `compileFnCallCond(name, ...args)` | Same, returning `Condition` | -| `makeExpr(fragment, schema)` | Build an `Expr` from a fragment and its codec | -| `makeUntypedExpr(fragment)` | Same with no codec — costs the row schema | -| `makeCond(fragment)` | Build a `Condition` from a fragment | +| `makeExpr(fragment, schema, literal?, uses?)` | Build an `Expr` from a fragment and its codec; `uses` carries params | +| `makeUntypedExpr(fragment, literal?, uses?)` | Same with no codec — costs the row schema | +| `makeCond(fragment, uses?)` | Build a `Condition` from a fragment | | `schemaOf(expr)` | An expression's codec, or `undefined` | | `schemaOfAny(...exprs)` | The first codec among several | | `elementSchema(expr)` | The element codec of an array expression | diff --git a/scripts/check-doc-examples.mjs b/scripts/check-doc-examples.mjs index c566632..345a3cf 100644 --- a/scripts/check-doc-examples.mjs +++ b/scripts/check-doc-examples.mjs @@ -76,11 +76,12 @@ assert.match(sql(page.compiled), /ORDER BY count DESC, name ASC LIMIT 25 OFFSET const ids = await import("./large-ids") assert.match(sql(ids.compiled), /toString\\(records.Id\\) AS id/) assert.equal(ids.rows[0]?.id, "18446744073709551615") -const replaced = CH.compileUnsafe(CH.from(Events).select("Name") +const anded = CH.compileUnsafe(CH.from(Events).select("Name") .where(($) => [$.OrgId.eq("org_123")]) .where(($) => [$.Name.eq("checkout")]), {}) -assert.equal(replaced.tenantScope, "cross-tenant") -assert.doesNotMatch(replaced.sql, /OrgId =/) +assert.equal(anded.tenantScope, "single-tenant") +assert.match(anded.sql, /OrgId = 'org_123'/) +assert.match(anded.sql, /Name = 'checkout'/) const benchmark = await import("./benchmark-suite") const suite = await Effect.runPromise(benchmark.default) assert.equal(suite.source, "events") diff --git a/src/ch/compile.ts b/src/ch/compile.ts index 788ec2e..31bed69 100644 --- a/src/ch/compile.ts +++ b/src/ch/compile.ts @@ -1221,6 +1221,12 @@ function derivedLiteralSchema(schema: Schema.Codec): Schema.Codec): ReadonlyArray | undefined { + const exprs = selectExprsOf(query) + return exprs === undefined ? undefined : Object.keys(exprs) +} + /** Evaluate a query's SELECT callback without compiling it. */ function selectExprsOf(query: CHQuery): Record | undefined { const state = query._state diff --git a/src/ch/custom-expr.ts b/src/ch/custom-expr.ts new file mode 100644 index 0000000..8e284f3 --- /dev/null +++ b/src/ch/custom-expr.ts @@ -0,0 +1,86 @@ +// Expressions a caller builds from SQL fragments. +// +// The public `makeExpr` / `makeUntypedExpr` / `makeCond`. The builder's own +// functions use the unchecked ones in `./expr`, whose signatures carry their +// arguments' params; a caller's fragment is a closure the type cannot see +// into, so these take the expressions it interpolates as `uses`, carry their +// params in the type, and check when compiling that the rendered SQL holds no +// param beyond theirs. A param can then never be in a query without being in +// its type: either `uses` names it, or compiling fails and says so. + +import { type Condition, type Expr, makeCond as makeCondUnchecked, makeExpr as makeExprUnchecked, type ParamsIn } from "./expr" +import { QueryBuilderDefect } from "./errors" +import { PARAM_PLACEHOLDER_PATTERN } from "./param" +import { compile, lazy, type SqlFragment } from "../sql/sql-fragment" +import { untracked } from "../sql/render-tracker" +import type { Schema } from "effect" + +/** The `param.*` placeholders in rendered SQL. */ +const placeholders = (sql: string): ReadonlySet => + new Set([...sql.matchAll(new RegExp(PARAM_PLACEHOLDER_PATTERN.source, "g"))].map((match) => match[0])) + +const fragmentOf = (value: unknown): SqlFragment | undefined => + typeof value === "object" && value !== null && "toFragment" in value && typeof value.toFragment === "function" + ? (value as { toFragment(): SqlFragment }).toFragment() + : undefined + +/** `fragment`, refusing at render a param that none of `uses` holds. */ +const checked = (what: string, fragment: SqlFragment, uses: ReadonlyArray): SqlFragment => + lazy(() => { + const sql = compile(fragment) + const found = placeholders(sql) + if (found.size === 0) return sql + // Rendered again only to read their placeholders, outside the render track. + const declared = new Set( + untracked(() => + uses.flatMap((use) => { + const f = fragmentOf(use) + return f === undefined ? [] : [...placeholders(compile(f))] + }), + ), + ) + const undeclared = [...found].filter((p) => !declared.has(p)) + if (undeclared.length > 0) { + throw new QueryBuilderDefect({ + message: `${what}: the SQL holds a param (${undeclared.join(", ")}) that no expression in \`uses\` carries; pass the expressions you interpolate as \`uses\` so the query's type requires their params`, + }) + } + return sql + }) + +/** + * An expression from a fragment and the codec its value decodes with. + * + * `uses` lists the expressions the fragment interpolates: their params become + * the result's, so `compile` requires them. A param in the SQL that no `uses` + * entry carries fails to compile. + * + * `uses` is the first type parameter so that it is always inferred: the value + * type comes from `schema`, and an explicit `makeExpr(…)` is an error + * rather than a call that silently stops reading `uses`. + */ +export function makeExpr = [], T = unknown>( + fragment: SqlFragment, + schema: Schema.Codec | undefined, + literal?: (value: unknown) => SqlFragment, + uses?: U, +): Expr> { + return makeExprUnchecked(checked("makeExpr", fragment, uses ?? []), schema, literal) +} + +/** {@link makeExpr} with no declared result type: selecting it costs the query its row schema. */ +export function makeUntypedExpr = [], T = unknown>( + fragment: SqlFragment, + literal?: (value: unknown) => SqlFragment, + uses?: U, +): Expr> { + return makeExprUnchecked(checked("makeUntypedExpr", fragment, uses ?? []), undefined, literal) +} + +/** A condition from a fragment. `uses` as for {@link makeExpr}. */ +export function makeCond = []>( + fragment: SqlFragment, + uses?: U, +): Condition> { + return makeCondUnchecked(checked("makeCond", fragment, uses ?? [])) +} diff --git a/src/ch/define-fn.ts b/src/ch/define-fn.ts index 2e4b374..6b9b04d 100644 --- a/src/ch/define-fn.ts +++ b/src/ch/define-fn.ts @@ -82,7 +82,7 @@ export const withoutNull = ( } // Re-export for consumer convenience -export { makeExpr, makeUntypedExpr, makeCond } +export { makeExpr, makeUntypedExpr, makeCond } from "./custom-expr" // compileFnCall — low-level helper for handwritten generic/special functions @@ -92,7 +92,10 @@ export function compileFnCall(name: string, } /** `compileFnCall` for a function whose result type is known. */ -export function compileTypedFnCall( +// `A` first, and inferred: `R` comes from `schema`, so an explicit +// `compileTypedFnCall(…)` is an error rather than a call that stops +// reading its arguments' params. +export function compileTypedFnCall( name: string, schema: Schema.Codec | undefined, ...args: A @@ -144,7 +147,7 @@ export function defineFn( name: string, result: FnResult, ): (...args: A) => Expr> { - return (...args: A) => compileTypedFnCall(name, resultSchema(result, args), ...args) + return (...args: A) => compileTypedFnCall(name, resultSchema(result, args), ...args) } /** diff --git a/src/ch/param.ts b/src/ch/param.ts index de2291f..3ae466d 100644 --- a/src/ch/param.ts +++ b/src/ch/param.ts @@ -69,7 +69,7 @@ function assertValidParamName(name: string): void { */ export type ParamValue = Comparable -export interface ParamMarker extends Expr>> { +export interface ParamMarker> extends Expr> { readonly _paramName: N readonly _paramType?: T } @@ -156,10 +156,10 @@ const paramTypes = new Map>([ export const paramSchema = (kind: ParamKind): Schema.Codec | undefined => paramTypes.get(kind) const makeParam = - (kind: ParamKind, schema: Schema.Codec) => - (name: N): ParamMarker => { + >(kind: ParamKind, schema: Schema.Codec) => + (name: N): ParamMarker => { assertValidParamName(name) - return makeParamMarker(name, raw(paramPlaceholder(kind, name)), schema) + return makeParamMarker(name, raw(paramPlaceholder(kind, name)), schema) as ParamMarker } const customKinds = new WeakMap, ParamKind>() @@ -204,7 +204,7 @@ export const param = { * Identical at runtime — the flavours differ only in what the row decodes to, * and a param has to agree with the column it bounds. */ - dateTimeString: makeParam("dateTime", T.dateTimeString.schema), + dateTimeString: makeParam>("dateTime", T.dateTimeString.schema), /** * The same bound, floored to whole seconds. @@ -220,7 +220,7 @@ export const param = { * Widening is safe where these appear: they bound a partition/index key for * pruning, and the exact `DateTime64` predicate still decides the result. */ - dateTimeSeconds: makeParam("dateTimeSeconds", T.dateTimeString.schema), + dateTimeSeconds: makeParam>("dateTimeSeconds", T.dateTimeString.schema), /** * A param of any column type, resolved through that type's own codec. diff --git a/src/ch/soundness.test-d.ts b/src/ch/soundness.test-d.ts index a4e0b5e..5d777ae 100644 --- a/src/ch/soundness.test-d.ts +++ b/src/ch/soundness.test-d.ts @@ -157,3 +157,28 @@ const del = CH.deleteFrom(Orders).where(($) => [$.Id.eq(CH.param.string("id"))]) // @ts-expect-error -- `id` is missing run(del) run(del, { id: "a" }) + +// A custom expression carries the params of the expressions it declares in `uses` +{ + const { raw } = { raw: (sql: string) => CH.untypedExpr(sql).toFragment() } + const scaled = CH.from(Users).select(($) => { + const factor = CH.param.float("factor") + return { x: CH.makeExpr(raw("x"), CH.float64.schema, undefined, [$.Age, factor]) } + }) + // @ts-expect-error -- `factor` comes from `uses` + CH.compileUnsafe(scaled, {}) + CH.compileUnsafe(scaled, { factor: 2 }) +} + +// @ts-expect-error -- the value type comes from the schema; an explicit one would stop `uses` being read +CH.makeExpr(CH.untypedExpr("1").toFragment(), CH.float64.schema) + +// An explicit type argument on a subquery expression is an error, not a silent loss of params +const scoped = CH.from(Orders).select(() => ({ n: CH.count() })).where(($) => [$.UserId.eq(CH.param.string("u"))]) +// @ts-expect-error -- the subquery is inferred; give the type as a column type +CH.subqueryExpr(scoped, CH.uint64) +// @ts-expect-error -- untypedSubqueryExpr takes no value type +CH.untypedSubqueryExpr(scoped) +const withScalar = CH.from(Users).select(() => ({ n: CH.subqueryExpr(scoped, CH.uint64) })) +// @ts-expect-error -- the subquery's `u` is required +CH.compileUnsafe(withScalar, {}) diff --git a/src/ch/soundness.test.ts b/src/ch/soundness.test.ts index 4f1a7f0..72a438e 100644 --- a/src/ch/soundness.test.ts +++ b/src/ch/soundness.test.ts @@ -194,3 +194,32 @@ describe("function sets", () => { expect(sql).toContain("coalesce(") }) }) + +describe("what the types cannot see", () => { + it("refuses a custom expression that interpolates a param it does not declare in uses", () => { + const param = CH.param.string("secret") + const hidden = CH.makeExpr( + CH.untypedExpr(`concat('a', ${"__x__"})`).toFragment(), + CH.string.schema, + ) + expect(compileCHUnsafe(CH.from(Users).select(() => ({ h: hidden }))).sql).toContain("concat") + const sneaky = CH.makeCond(CH.sql.cond`${param} = 'x'`.toFragment()) + expect(() => compileCHUnsafe(CH.from(Users).select("Id").where(() => [sneaky]), { secret: "s" })).toThrow( + /no expression in `uses` carries/, + ) + const declared = CH.makeCond(CH.sql.cond`${param} = 'x'`.toFragment(), [param]) + expect(compileCHUnsafe(CH.from(Users).select("Id").where(() => [declared]), { secret: "s" }).sql).toContain( + "('s') = 'x'", + ) + }) + + it("refuses an IN subquery that selects other than one column, past the types", () => { + const two = CH.from(Orders).select("UserId", "Amount") as any + expect(() => compileCHUnsafe(CH.from(Users).select("Id").where(($) => [CH.inSubquery($.Id, two)]))).toThrow( + /exactly one column, not UserId, Amount/, + ) + expect(() => compileCHUnsafe(CH.from(Users).select("Id").where(($) => [CH.notInSubquery($.Id, two)]))).toThrow( + /exactly one column/, + ) + }) +}) diff --git a/src/ch/subquery.ts b/src/ch/subquery.ts index e05c236..c4c344d 100644 --- a/src/ch/subquery.ts +++ b/src/ch/subquery.ts @@ -7,7 +7,8 @@ // — one top-level `const` away from a TDZ crash in the bundle. import { renderSubquery } from "./subquery-context" -import { compileCHUnsafe } from "./compile" +import { compileCHUnsafe, selectedAliasesOf } from "./compile" +import { QueryBuilderDefect } from "./errors" import { type Condition, type Expr, makeCond, makeExpr, makeUntypedExpr } from "./expr" import type { CHQuery, NeedsSelect, SingleColumnOf } from "./query" import type { QueryParams } from "./union" @@ -34,6 +35,22 @@ const toSql = (subquery: Subquery): string => : compileCHUnsafe(query, {}, { skipFormat: true, deferParams: true }).sql, ) +/** + * `expr IN (subquery)` compares against one column. The type says so; this + * says so to a caller past the types (a cast, an untyped query), where the + * database would otherwise fail on, or ClickHouse silently compare, a tuple. + */ +const oneColumn = (what: string, subquery: Subquery): Subquery => { + if (typeof subquery === "string") return subquery + const aliases = selectedAliasesOf(subquery) + if (aliases !== undefined && aliases.length !== 1) { + throw new QueryBuilderDefect({ + message: `${what}: the subquery must select exactly one column, not ${aliases.length === 0 ? "none" : aliases.join(", ")}`, + }) + } + return subquery +} + // Subqueries contribute their own source scope, never a binding on outer rows. // An inner tenant filter cannot confine an otherwise unfiltered outer source. @@ -52,7 +69,7 @@ export function inSubquery, Q = never, SP = nev expr: Expr, subquery: string | (CHQuery & SingleColumnOf), ): Condition { - return makeCond(lazy(() => `${compile(expr.toFragment())} IN (${toSql(subquery)})`)) + return makeCond(lazy(() => `${compile(expr.toFragment())} IN (${toSql(oneColumn("inSubquery", subquery))})`)) } /** @@ -65,7 +82,7 @@ export function notInSubquery, Q = never, SP = expr: Expr, subquery: string | (CHQuery & SingleColumnOf), ): Condition { - return makeCond(lazy(() => `${compile(expr.toFragment())} NOT IN (${toSql(subquery)})`)) + return makeCond(lazy(() => `${compile(expr.toFragment())} NOT IN (${toSql(oneColumn("notInSubquery", subquery))})`)) } // Spliced sub-SELECTs @@ -100,7 +117,10 @@ export function notInSubquery, Q = never, SP = * Expression composition preserves deferred rendering, including when a caller * builds the expression or condition before the outer query. */ -export function subqueryExpr( +// The subquery is the first type parameter, and inferred: an explicit type +// argument (`subqueryExpr(…)`) is then an error rather than a call that +// stops inferring the subquery and drops its params from the type. +export function subqueryExpr( subquery: S, type: CHType, wrap: (sql: string) => string = (sql) => `(${sql})`, @@ -113,12 +133,13 @@ export function subqueryExpr( /** {@link subqueryExpr} for a spliced value with no declared result type — a * sort tuple, an `argMin` tiebreaker. Selecting one costs the query its row - * schema, the same as `untypedExpr`. */ -export function untypedSubqueryExpr( + * schema, the same as `untypedExpr`. Its value is `unknown`; for a value of a + * type, use `subqueryExpr` with that type. */ +export function untypedSubqueryExpr( subquery: S, wrap: (sql: string) => string = (sql) => `(${sql})`, -): Expr> { - return makeUntypedExpr(lazy(() => wrap(toSql(subquery)))) +): Expr> { + return makeUntypedExpr(lazy(() => wrap(toSql(subquery)))) } /** {@link subqueryExpr} as a predicate — for the `IN`/`EXISTS` shapes the three diff --git a/src/docs-examples.test.ts b/src/docs-examples.test.ts index 0b91b53..8c887d2 100644 --- a/src/docs-examples.test.ts +++ b/src/docs-examples.test.ts @@ -848,7 +848,7 @@ describe("docs/extending.md", () => { it("makeExpr builds custom call syntax", () => { const quantileExact = (q: number) => (expr: CH.Expr) => - CH.makeExpr( + CH.makeExpr( rawFragment(`quantileExact(${q})(${compileFragment(expr.toFragment())})`), T.float64.schema, ) diff --git a/src/expr.ts b/src/expr.ts index 58f898c..863419c 100644 --- a/src/expr.ts +++ b/src/expr.ts @@ -11,9 +11,6 @@ export { inExprList, inList, lit, - makeCond, - makeExpr, - makeUntypedExpr, not, notInList, or, @@ -27,6 +24,8 @@ export { when, whenTrue, } from "./ch/expr" +// The checked constructors: a param interpolated without `uses` fails to compile. +export { makeCond, makeExpr, makeUntypedExpr } from "./ch/custom-expr" export * from "./ch/functions" // The factories behind `./ch/functions`, so a consumer can declare a function // this package does not model and have it carry a result type like a built-in. diff --git a/tests/core-cases.ts b/tests/core-cases.ts index 043d250..c060554 100644 --- a/tests/core-cases.ts +++ b/tests/core-cases.ts @@ -376,7 +376,7 @@ export const coreCases: readonly CoreCase[] = [ ctx.compile( orgOrders(ctx) .select(($) => ({ id: $.Id, flagged: CH.param.bool("flag") })) - .where(($) => [$.OrgId.eq(CH.param.string("orgId")), $.Id.eq(1)]), + .where(($) => [$.Id.eq(1)]), { ...org, flag: true }, ), expected: [{ id: 1, flagged: true }], @@ -406,7 +406,7 @@ export const coreCases: readonly CoreCase[] = [ orgOrders(ctx) .leftJoin(ctx.customers, "c", (o, c) => o.Customer.eq(c.Name).and(o.OrgId.eq(c.OrgId))) .select(($) => ({ id: $.Id, tier: $.c.Tier })) - .where(($) => [$.OrgId.eq(CH.param.string("orgId")), $.Id.gte(3)]) + .where(($) => [$.Id.gte(3)]) .orderBy(["id", "asc"]), org, ), @@ -492,7 +492,7 @@ export const coreCases: readonly CoreCase[] = [ orgOrders(ctx) .crossJoinQuery(customers, "k") .select(($) => ({ id: $.Id, customers: $.k.customers })) - .where(($) => [$.OrgId.eq(CH.param.string("orgId")), $.Id.lte(2)]) + .where(($) => [$.Id.lte(2)]) .orderBy(["id", "asc"]), org, ) @@ -591,7 +591,7 @@ export const coreCases: readonly CoreCase[] = [ ctx.compile( orgOrders(ctx) .select(($) => ({ id: $.Id })) - .where(($) => [$.OrgId.eq(CH.param.string("orgId")), $.Id.eq(1)]) + .where(($) => [$.Id.eq(1)]) .format("JSON"), org, ), @@ -606,7 +606,7 @@ export const coreCases: readonly CoreCase[] = [ const one = (id: number) => orgOrders(ctx) .select(($) => ({ id: $.Id })) - .where(($) => [$.OrgId.eq(CH.param.string("orgId")), $.Id.eq(id)]) + .where(($) => [$.Id.eq(id)]) return ctx.compileUnion(CH.unionAll(one(1), one(2)).orderBy(["id", "asc"]).format("JSON"), org) }, expected: [{ id: 1 }, { id: 2 }], @@ -620,7 +620,6 @@ export const coreCases: readonly CoreCase[] = [ orgOrders(ctx) .select(($) => ({ id: $.Id })) .where(($) => [ - $.OrgId.eq(CH.param.string("orgId")), $.Note.isNull(), $.Amount.between(5, CH.param.int("hi")), $.Id.notBetween(3, 3), @@ -634,7 +633,7 @@ export const coreCases: readonly CoreCase[] = [ id: "is-not-null", covers: e("isNotNull"), build: (ctx) => - ctx.compile(orgOrders(ctx).select(($) => ({ id: $.Id })).where(($) => [$.OrgId.eq(CH.param.string("orgId")), $.Note.isNotNull()]).orderBy(["id", "asc"]), org), + ctx.compile(orgOrders(ctx).select(($) => ({ id: $.Id })).where(($) => [$.Note.isNotNull()]).orderBy(["id", "asc"]), org), expected: [{ id: 1 }, { id: 4 }], }, { @@ -645,7 +644,6 @@ export const coreCases: readonly CoreCase[] = [ orgOrders(ctx) .select(($) => ({ id: $.Id })) .where(($) => [ - $.OrgId.eq(CH.param.string("orgId")), CH.or(CH.and($.Status.eq("paid"), $.Amount.gt(15)), undefined, $.Customer.eq("globex")), ]) .orderBy(["id", "asc"]), @@ -683,7 +681,7 @@ export const coreCases: readonly CoreCase[] = [ rejects: { clickhouse: /no row locks/ }, build: (ctx) => { // Each strength compiles; the one sent is FOR UPDATE SKIP LOCKED. - const base = orgOrders(ctx).select(($) => ({ id: $.Id })).where(($) => [$.OrgId.eq(CH.param.string("orgId")), $.Id.eq(1)]) + const base = orgOrders(ctx).select(($) => ({ id: $.Id })).where(($) => [$.Id.eq(1)]) for (const locked of [base.forNoKeyUpdate({ noWait: true }), base.forShare(), base.forKeyShare()]) ctx.compile(locked, org) return ctx.compile(base.forUpdate({ skipLocked: true }), org) }, From 93a02b360d062d6efaf265df45435446b1d0e76a Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 4 Oct 2026 23:02:28 +0200 Subject: [PATCH 5/5] Mark the package consumer's missing-param check as a type error The tarball check compiles a query without its param to exercise the runtime failure; params are now in the type, so that call is a type error too. Co-Authored-By: Claude Opus 5.5 --- tests/package-consumer.mts | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/package-consumer.mts b/tests/package-consumer.mts index 895d5a9..f485f89 100644 --- a/tests/package-consumer.mts +++ b/tests/package-consumer.mts @@ -30,6 +30,7 @@ assert.equal(T.custom("String", Schema.String).sql, "String") assert.equal(T.untyped("Tuple(String)").sql, "Tuple(String)") const length = CH.defineFn<[CH.Expr], number>("length", T.uint64) assert.equal(SQL.compile(length(CH.lit("abc")).toFragment()), "length('abc')") +// @ts-expect-error -- a missing param is a type error too; this checks the runtime failure const invalid = Effect.runSync(Effect.exit(CH.compile(query, {}))) assert.equal(invalid._tag, "Failure")