diff --git a/design/dialects.md b/design/dialects.md new file mode 100644 index 0000000..f1ddc1c --- /dev/null +++ b/design/dialects.md @@ -0,0 +1,69 @@ +# Dialects: one builder, several databases + +Status: steps 1 and 2 landed (params and literals go through a `Dialect`). Steps 3 to 6 are open. + +## Goal + +Keep one query builder, one tenant-scope analysis, and one row-decoding model, and let the +database-specific parts (SQL syntax, literals, param binding, column wire formats, the +function catalog) vary per dialect. ClickHouse output stays byte-identical at every step; the +exact-SQL tests in `src/ch/compile.test.ts` are the guard. + +## What we took from Kysely and Drizzle + +Both were read from source (Kysely 0.28.17, Drizzle 1.0.0-rc.5). + +| Idea | Where it comes from | How it lands here | +| --- | --- | --- | +| Builders produce a config object; a dialect turns it into SQL | Drizzle `PgDialect.buildSelectQuery(config)` | `CHQueryState` already is that config. `compile.ts` becomes the ClickHouse dialect's `buildSelect` | +| SQL as a chunk tree rendered at the end with `escapeName` / `escapeParam` / `escapeString` | Drizzle `SQL` chunks + `BuildQueryConfig` | `SqlFragment` gets a real `Param` chunk; `Str`/`Lazy` stop rendering ClickHouse text early | +| Inline vs bound params as a switch | Drizzle `inlineParams` | `ParamStyle`: `inline` (ClickHouse today) or `bind` | +| A small override surface per dialect | Kysely `DefaultQueryCompiler` hooks (MySQL overrides ~10 methods) | The `Dialect` interface grows hook by hook, never a copy of the compiler | +| Capability flags instead of dialect checks | Kysely `DialectAdapter` (`supportsReturning`, ...) | Flags such as `supportsFilterClause`, `aliasInWhere`, `limitBy` | +| A compiled query that keeps its structure | Kysely `CompiledQuery.query` | `CompiledQuery` already carries tenant scope and row schema; `parameters` added in step 1 | +| Rewrites as passes over the tree | Kysely plugins (`transformQuery` / `transformResult`) | Tenant-scope proof and empty-`IN` handling as passes over the state | +| Logical type separate from how a driver sends it | Drizzle v1 codecs, `refineGenericPgCodecs` per driver | `CHType` splits into a type and a transport codec (step 3) | +| Shared operators, per-dialect function catalogs | Drizzle `sql/expressions` vs `pg-core` / `mysql-core` | `eq`, `and`, `in_` in core; `countIf`, `percentileCont` per dialect | + +What we deliberately do not copy: + +- Kysely decodes nothing at runtime; its row types are a promise. Our schema-backed + `decodeRows` stays. +- Drizzle copies the whole query builder per dialect package. We share the builder and vary + only types and functions. +- Tenant scoping stays a proof, not a plugin that injects `OrgId = ...`. Injection would hide a + missing tenant condition instead of reporting it. + +## Steps + +1. **Params through a dialect (done).** `renderParams` resolves placeholders once, at the top + of the statement, so a binding dialect can number them across unions and subqueries. + `CompiledQuery.parameters` holds the bound values. Tenant bounds still render as ClickHouse + literals, since they are compared as text and never sent. +2. **Literals behind the dialect (done).** `Dialect.quoteString` and `Dialect.literal` write + every `Str` fragment, column literal, and inline param. `compile` installs the dialect for + the length of the compile (`withDialect`, the same save/restore pattern as + `withSubqueryCompiler`), because literals are written inside the query callbacks, which + have no dialect argument. The `__PARAM_` safety is now enforced rather than assumed: a + literal that contains the marker fails with `InvalidLiteral`, so a dialect with weaker + escaping cannot turn a value into a placeholder. +3. **Identifier quoting.** Postgres folds unquoted names to lower case, so `OrgId` must be + written `"OrgId"`. Column refs, aliases, qualified names, `groupBy` keys and `orderBy` + specs are raw text today (`raw(name)` in `expr.ts`, `orderByClause` in `compile.ts`), so + this is its own step: column refs become `Ident` fragments that carry their qualifier. +4. **Split `CHType`.** A logical type (`sql` name, TS type) plus a codec for the transport. The + UInt64-as-string rule belongs to ClickHouse's `FORMAT JSON` over HTTP, not to ClickHouse; + the native client sends something else, and Postgres drivers send int8 as a string and + timestamptz as a `Date`. +5. **Move `compile.ts` into `ClickHouseDialect.buildSelect(state)`.** `compileQuery` and the + terminal clauses (`FORMAT`, `SETTINGS`, `LIMIT BY`) become ClickHouse-only. +6. **Postgres dialect.** `pg.T` column types, `$n` binding, `"ident"` quoting, a function + catalog covering the common cases (`FILTER (WHERE ...)`, `date_bin`, `percentile_cont`, + jsonb access), and capability flags for what ClickHouse allows and Postgres does not + (select aliases in `WHERE`/`HAVING`, default values instead of `NULL` in outer joins). + +## Open questions + +- ClickHouse also supports server-side binding (`{name:Type}` with `query_params`). Adding it + needs the param kind to name a ClickHouse type, which `param.of` already has. +- Whether the package splits (`core`, `clickhouse`, `postgres` entry points) at step 4 or 5. diff --git a/docs/params-and-compilation.md b/docs/params-and-compilation.md index 73834e5..92c1c7b 100644 --- a/docs/params-and-compilation.md +++ b/docs/params-and-compilation.md @@ -147,6 +147,7 @@ CH.compileUnsafe(query, params, options?) // CompiledQuery, throws | `options.rowSchema` | Effect `Schema` used by `decodeRows` / `decodeFirstRow` | | `options.skipFormat` | Omit a trailing `FORMAT` clause (used internally for subqueries) | | `options.deferParams` | Leave placeholders unresolved, for SQL spliced into an outer compile | +| `options.dialect` | How params reach the server; `clickhouseDialect` when omitted | `compileCH` is the internal name; the package exports it as `compile`. Unions use `compileUnion(union, params)`. @@ -156,6 +157,7 @@ CH.compileUnsafe(query, params, options?) // CompiledQuery, throws ```ts interface CompiledQuery { readonly sql: string + readonly parameters: ReadonlyArray readonly tenantScope: "single-tenant" | "cross-tenant" | "untenanted" readonly rowSchemaSource: "declared" | "derived" | "none" readonly rowSchema: CompiledQueryRowSchema | undefined @@ -172,6 +174,7 @@ interface CompiledQuery { | Field | Purpose | | ------------------------------- | ---------------------------------------------------------------------------------------- | | `sql` | The statement to execute. The builder never runs it. | +| `parameters` | Values a binding dialect sends beside `sql`, in placeholder order; empty by default | | `tenantScope` | Whether the query pins a single tenant — see [Tenant scoping](./tenant-scoping.md) | | `rowSchemaSource` | Where the row schema came from, so a caller can tell real validation from a pass-through | | `rowSchema` | The codec itself, for a caller that needs a `Schema` rather than a call | @@ -184,6 +187,40 @@ interface CompiledQuery { Use `decodeRows` to validate wire values against the row schema. +## Dialects + +A `Dialect` decides how literals are written and how resolved params reach the server. Its +`quoteString` and `literal` write every string fragment, every value compared against a column, +and every inline param, for the length of the compile. The default, `clickhouseDialect`, +writes each value into the SQL as a ClickHouse literal and leaves +`parameters` empty. A dialect whose `params` style is `bind` leaves a placeholder instead and +returns the encoded values in `parameters`, numbered once across the whole statement, unions +and subqueries included: + +```ts +const numbered: CH.Dialect = { + ...CH.clickhouseDialect, + name: "numbered", + params: { _tag: "bind", placeholder: (index) => `$${index}`, reuse: true }, +} + +const compiled = CH.compileUnsafe(query, { orgId: "org_1" }, { dialect: numbered }) +// compiled.sql: ... WHERE OrgId = $1 +// compiled.parameters: ["org_1"] +``` + +`reuse: true` lets one numbered placeholder stand for every use of a param; set it to `false` +for positional `?` placeholders, which bind a value each time they appear. Either way a bound +value is the column codec's wire form, the same value an inline literal is written from, and a +missing or ill-typed param still fails the compile. + +Params are resolved by rewriting placeholders in the finished SQL, so a dialect's literals must +never spell the param marker `__PARAM_`. ClickHouse writes it as `\x5F_PARAM_`; a literal that +does contain the marker fails the compile with `InvalidLiteral` rather than being rewritten. + +Identifiers, clauses, and functions are still ClickHouse SQL; a dialect changes literals and +param binding today. + ## Handwritten SQL When you need SQL the builder cannot express, `rawCompiledQuery` wraps a string in the same diff --git a/docs/reference.md b/docs/reference.md index 852a7ee..d767975 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -88,6 +88,10 @@ Note `/sql` exports a `compile` (fragment → string) distinct from the root `co | `compileUnion` | `(union, params, options?) => Effect, QueryBuilderError>` | | `compileUnionUnsafe` | The same, throwing instead | | `rawCompiledQuery` | `({ sql, tenantScope, reason, justification, rowSchema?, route? }) => CompiledQuery` | +| `clickhouseDialect` | The default `Dialect`: params written into the SQL as ClickHouse literals | + +`Dialect` and `ParamStyle` describe how params reach the server; pass one as +`options.dialect`. See [Params and compilation](./params-and-compilation.md#dialects). ### Params @@ -269,7 +273,7 @@ Types: `WindowSpec`, `CompiledWindowSpec`, `WindowFrameBound`, `WindowRowsFrame` **Everything else** — `Table`, `TableOptions`, `Expr`, `ColumnRef`, `Condition`, `Comparable` (what a value of a type may be compared against), `MapValueOf`, `Subquery`, `ParamMarker`, `ParamKind`, `CHQuery`, `CHUnionQuery`, `ColumnAccessor`, `JoinedColumnAccessor`, -`JoinOnCallback`, `CompiledQuery`, `CompiledQueryInput`, `CompiledQueryRowSchema`, `RowSchemaMismatch`, `TenantScope`, `FnResult`, +`JoinOnCallback`, `CompiledQuery`, `CompiledQueryInput`, `CompiledQueryRowSchema`, `RowSchemaMismatch`, `TenantScope`, `Dialect`, `ParamStyle`, `FnResult`, `WindowFunnelMode`, `WindowSpec`, `WindowRowsFrame`, `WindowFrameBound`, `WindowOrderDirection`, `CompiledWindowSpec`. diff --git a/src/ch/compile.ts b/src/ch/compile.ts index 17c7bd5..4708634 100644 --- a/src/ch/compile.ts +++ b/src/ch/compile.ts @@ -12,12 +12,13 @@ import type { CHQuery, CHQueryState } from "./query" import type { CHUnionQuery } from "./union" import { createQualifiedColumnAccessor, createJoinedColumnAccessor, sourceAlias } from "./query" import { aliased, columnTypeOf } from "./expr" -import { raw, ident, escapeClickHouseString, compile as compileSqlFragment } from "../sql/sql-fragment" +import { raw, ident, compile as compileSqlFragment } from "../sql/sql-fragment" import { splitTerminalClauses } from "../sql/terminal-clauses" import { compileQuery, type SqlQuery } from "../sql/sql-query" import { PARAM_MARKER_PREFIX, PARAM_PLACEHOLDER_PATTERN, paramSchema, type ParamKind } from "./param" import { mergeResultSchemas } from "./define-fn" -import { encodeLiteral } from "./literal" +import { encodeValue } from "./literal" +import { checkedLiteral, clickhouseDialect, currentDialect, withDialect, type Dialect } from "./dialect" import { Effect, Option, Schema } from "effect" import { QueryBuilderDefect, QueryBuilderError } from "./errors" import { withSubqueryCompiler } from "./subquery-context" @@ -101,6 +102,13 @@ interface ResolvedCte { interface CompiledQueryBase { readonly sql: string + /** + * The values a binding dialect sends beside `sql`, in placeholder order. + * + * Empty when the dialect writes params into the SQL as literals, which is + * what ClickHouse (the default) does. + */ + readonly parameters: ReadonlyArray readonly tenantScope: TenantScope /** * Where the query's row schema came from, or `"none"` if it has none. @@ -282,6 +290,7 @@ const compareRowSchemas = (declared: unknown, derived: unknown): RowSchemaMismat const makeCompiledQuery = ( sql: string, + parameters: ReadonlyArray, tenantScope: TenantScope, rowSchemaSource: "declared" | "derived" | "none", /** Built on first decode: a derived schema costs a `Schema.Struct` per @@ -352,6 +361,7 @@ const makeCompiledQuery = ( return { sql, + parameters, tenantScope, // Resolved eagerly only here, where the getter is already memoised by // `decodeRow`/`encodeRow` below; reading it does not build a second one. @@ -412,6 +422,7 @@ export const rawCompiledQuery = < }): CompiledQuery => makeCompiledQuery( args.sql, + [], args.tenantScope, args.rowSchema === undefined ? "none" : "declared", () => args.rowSchema, @@ -468,6 +479,7 @@ export const compileCH = < skipFormat?: boolean rowSchema?: CompiledQueryRowSchema deferParams?: boolean + dialect?: Dialect }, ): Effect.Effect, QueryBuilderError> => asEffect(() => compileCHUnsafe(query, params, options)) @@ -476,7 +488,7 @@ export const compileCH = < export const compileUnion = , Params extends Record>( union: CHUnionQuery, params: Params, - options?: { rowSchema?: CompiledQueryRowSchema; deferParams?: boolean }, + options?: { rowSchema?: CompiledQueryRowSchema; deferParams?: boolean; dialect?: Dialect }, ): Effect.Effect, QueryBuilderError> => asEffect(() => compileUnionUnsafe(union, params, options)) @@ -497,9 +509,11 @@ export function compileCHUnsafe< * For fragments spliced into a larger query — a subquery condition — whose * params are resolved by the outer compilation pass. */ deferParams?: boolean + /** How params reach the server. ClickHouse literals when omitted. */ + dialect?: Dialect }, ): CompiledQuery { - return compileInner(query, params, options) + return withDialect(options?.dialect ?? currentDialect(), () => compileInner(query, params, options)) } /** @@ -531,6 +545,12 @@ function compileInner< * For fragments spliced into a larger query — a subquery condition — whose * params are resolved by the outer compilation pass. */ deferParams?: boolean + /** + * Set by a compile that splices this query's SQL into its own: the + * outer one resolves params once over the whole statement, which a + * dialect that numbers its placeholders depends on. + */ + nested?: boolean /** * The tenant scopes of CTEs an enclosing query has already resolved. * @@ -591,6 +611,7 @@ function compileInner< const compiled = compileInner(c.query, params, { skipFormat: true, deferParams, + nested: true, enclosingCtes: [...(options?.enclosingCtes ?? []), ...resolvedCtes], }) resolvedCtes.push({ @@ -629,12 +650,13 @@ function compileInner< const inner = compileInner(state.fromQuery, params, { skipFormat: true, deferParams, + nested: true, enclosingCtes: visibleCtes, }) fromSource = sourceOf(inner) fromFragment = raw(`(${inner.sql}) AS ${state.fromQueryAlias}`) } else if (state.fromUnion) { - const inner = compileUnionUnsafe(state.fromUnion, params, { deferParams, enclosingCtes: visibleCtes }) + const inner = compileUnionInner(state.fromUnion, params, { deferParams, nested: true, enclosingCtes: visibleCtes }) fromSource = sourceOf(inner) fromFragment = raw(`(\n${splitTerminalClauses(inner.sql).body}\n) AS ${state.fromQueryAlias}`) } else { @@ -655,6 +677,7 @@ function compileInner< const compiled = compileInner(subquery, params, { skipFormat: true, deferParams, + nested: true, enclosingCtes: visibleCtes, }) sources.push(sourceOf(compiled)) @@ -667,6 +690,7 @@ function compileInner< const compiled = compileInner(j.innerQuery, params, { skipFormat: true, deferParams, + nested: true, enclosingCtes: visibleCtes, }) tableSql = `(${compiled.sql})` @@ -730,10 +754,17 @@ function compileInner< sql = `WITH ${cteDefs}\n${sql}` } - if (!deferParams) sql = resolveParams(sql, params) + // Once, at the top: a nested query's SQL is spliced into this one, and a + // dialect that binds numbers its placeholders across the whole statement. + let parameters: ReadonlyArray = [] + if (!deferParams && options?.nested !== true) { + const rendered = renderParams(sql, params, currentDialect()) + sql = rendered.sql + parameters = rendered.parameters + } const scope = deriveTenantScope(sources, [{ predicates: wherePredicates }, ...joinPredicates], (value) => - deferParams ? compileSqlFragment(value) : resolveParams(compileSqlFragment(value), params), + deferParams ? compileSqlFragment(value) : inlineParams(compileSqlFragment(value), params), ) const tenantScope = state.crossTenant === true ? "cross-tenant" : scope.scope @@ -743,6 +774,7 @@ function compileInner< return withTenantBound( makeCompiledQuery( sql, + parameters, tenantScope, options?.rowSchema !== undefined ? "declared" : derivedSchema ? "derived" : "none", () => options?.rowSchema ?? (derivedSchema as CompiledQueryRowSchema | undefined), @@ -976,8 +1008,24 @@ export function compileUnionUnsafe, Params ex options?: { rowSchema?: CompiledQueryRowSchema deferParams?: boolean - /** Internal — see `compileInner`'s option of the same name. A union in a - * later CTE's FROM must still see its earlier scoped siblings. */ + dialect?: Dialect + }, +): CompiledQuery { + return withDialect(options?.dialect ?? currentDialect(), () => compileUnionInner(union, params, options)) +} + +/** The recursion behind {@link compileUnionUnsafe}; see {@link compileInner}. */ +function compileUnionInner, Params extends Record>( + union: CHUnionQuery, + params: Params, + options?: { + rowSchema?: CompiledQueryRowSchema + deferParams?: boolean + /** Set by a compile that splices this union's SQL into its own, and so + * resolves its params itself. */ + nested?: boolean + /** A union in a later CTE's FROM must still see its earlier scoped + * siblings. See `compileInner`'s option of the same name. */ enclosingCtes?: ReadonlyArray }, ): CompiledQuery { @@ -990,7 +1038,7 @@ export function compileUnionUnsafe, Params ex if (first === undefined) throw new QueryBuilderDefect({ message: "unionAll requires at least one query" }) const selectKeys = Object.keys(selectExprsOf(first) ?? {}) const subQueries = state.queries.map((q) => - compileInner(q, params, { skipFormat: true, deferParams, selectKeys, enclosingCtes }), + compileInner(q, params, { skipFormat: true, deferParams, nested: true, selectKeys, enclosingCtes }), ) const bounds = new Set( subQueries.flatMap((q) => { @@ -1035,6 +1083,13 @@ export function compileUnionUnsafe, Params ex sql += `\nFORMAT ${state.formatValue}` } + let parameters: ReadonlyArray = [] + if (!deferParams && options?.nested !== true) { + const rendered = renderParams(sql, params, currentDialect()) + sql = rendered.sql + parameters = rendered.parameters + } + // A union decodes as its branches do — but not as its FIRST branch does. // The branches share an Output *shape*, not a column type: ClickHouse widens // across them, so a column that is `String` in one branch and nullable in @@ -1048,6 +1103,7 @@ export function compileUnionUnsafe, Params ex return withTenantBound( makeCompiledQuery( sql, + parameters, tenantScope, options?.rowSchema !== undefined ? "declared" : derivedSchema ? "derived" : "none", () => options?.rowSchema ?? (derivedSchema as CompiledQueryRowSchema | undefined), @@ -1063,26 +1119,47 @@ export function compileUnionUnsafe, Params ex } /** - * Substitute every `__PARAM____` placeholder with its value. + * Substitute every `__PARAM____` placeholder, the way the dialect + * sends params: as literals written into the SQL, or as bind placeholders whose + * encoded values come back in `parameters`. * - * Params are resolved here rather than sent as ClickHouse query parameters, so - * a value that never arrives, or arrives as the wrong type, would otherwise + * Params are resolved here rather than handed to the driver unchecked, so a + * value that never arrives, or arrives as the wrong type, would otherwise * become part of the SQL text: a missing param used to ship the placeholder * itself to the server, and a `Date` handed to a dateTime param used to - * stringify as `Thu Jan 01 2026 …`. Both are now compile-time failures. + * stringify as `Thu Jan 01 2026 …`. Both are compile-time failures, whichever + * way the dialect sends values. * * Params the query doesn't mention are ignored — one bag of params is commonly * shared across a family of queries. */ -function resolveParams(sql: string, params: Record): string { +function renderParams( + sql: string, + params: Record, + dialect: Dialect, +): { readonly sql: string; readonly parameters: ReadonlyArray } { const missing: Array = [] + const parameters: Array = [] + // Keyed by kind and name: `dateTime` and `dateTimeSeconds` read one value + // and encode it differently, so they are two bound values, not one. + const bound = new Map() + const style = dialect.params const resolved = sql.replace(PARAM_PLACEHOLDER_PATTERN, (placeholder, kind: string, name: string) => { if (!(name in params)) { missing.push(name) return placeholder } - return resolveParam(kind as ParamKind, name, params[name]) + const value = encodeParam(kind as ParamKind, name, params[name]) + if (style._tag === "inline") return checkedLiteral(dialect, value, paramContext(kind, name)) + + const key = `${kind}\0${name}` + const existing = style.reuse ? bound.get(key) : undefined + if (existing !== undefined) return existing + parameters.push(value) + const marker = style.placeholder(parameters.length) + bound.set(key, marker) + return marker }) if (missing.length > 0) { @@ -1105,17 +1182,28 @@ function resolveParams(sql: string, params: Record): string { }) } - return resolved + return { sql: resolved, parameters } } /** - * A param value as a ClickHouse literal, through its declared type's codec. + * A fragment with its params written in as ClickHouse literals. + * + * Only for tenant bounds, which are compared with each other as text and never + * sent anywhere, so they render the same whatever the query's dialect. + */ +const inlineParams = (sql: string, params: Record): string => + renderParams(sql, params, clickhouseDialect).sql + +const paramContext = (kind: string, name: string): string => `param '${name}' (${kind})` + +/** + * A param value encoded to its wire form, through its declared type's codec. * * The same schema that decodes a column of that type runs backwards here, so * the two directions cannot drift: a `DateTime` param and a `DateTime` column * agree on the literal by construction, not by two functions being kept in sync. */ -function resolveParam(kind: ParamKind, name: string, value: unknown): string { +function encodeParam(kind: ParamKind, name: string, value: unknown): unknown { const schema = paramSchema(kind) if (schema === undefined) { // Only reachable from a hand-written placeholder naming a kind nothing @@ -1125,5 +1213,5 @@ function resolveParam(kind: ParamKind, name: string, value: unknown): string { message: `compile: param '${name}' has an unknown type '${kind}'`, }) } - return encodeLiteral(schema, value, `param '${name}' (${kind})`) + return encodeValue(schema, value, paramContext(kind, name)) } diff --git a/src/ch/dialect.test.ts b/src/ch/dialect.test.ts new file mode 100644 index 0000000..842f59b --- /dev/null +++ b/src/ch/dialect.test.ts @@ -0,0 +1,184 @@ +import { describe, expect, it } from "@effect/vitest" +import { compile as compileFragment, str } from "../sql/sql-fragment" +import { compileCHUnsafe, compileUnionUnsafe } from "./compile" +import type { Dialect } from "./dialect" +import * as CH from "./index" + +const numbered: Dialect = { + ...CH.clickhouseDialect, + name: "numbered", + params: { _tag: "bind", placeholder: (index) => `$${index}`, reuse: true }, +} + +const positional: Dialect = { + ...CH.clickhouseDialect, + name: "positional", + params: { _tag: "bind", placeholder: () => "?", reuse: false }, +} + +// Standard SQL strings: a quote is doubled, a backslash is literal text. +const standardQuote = (value: string) => `'${value.replace(/'/g, "''")}'` +const literalDialect = (name: string, quoteString: (value: string) => string): Dialect => ({ + name, + quoteString, + literal: (value, context) => { + if (typeof value === "string") return quoteString(value) + if (typeof value === "boolean") return value ? "TRUE" : "FALSE" + return CH.clickhouseDialect.literal(value, context) + }, + params: { _tag: "inline" }, +}) + +// Splits the marker across two concatenated literals so no value spells it. +const standard = literalDialect("standard", (value) => + standardQuote(value).replace(/__PARAM_/g, "_' || '_PARAM_"), +) + +// Quotes correctly but never escapes the param marker. +const naive = literalDialect("naive", standardQuote) + +const events = CH.table( + "events", + { OrgId: CH.string, Service: CH.string, Count: CH.uint64, Timestamp: CH.dateTime64 }, + { tenantColumn: "OrgId" }, +) + +const byService = CH.from(events) + .select(($) => ({ count: $.Count })) + .where(($) => [$.OrgId.eq(CH.param.string("orgId")), $.Service.eq(CH.param.string("service"))]) + +describe("dialect params", () => { + it("ClickHouse writes params as literals and binds nothing", () => { + const compiled = compileCHUnsafe(byService, { orgId: "org_1", service: "api" }) + expect(compiled.sql).toContain("OrgId = 'org_1'") + expect(compiled.parameters).toEqual([]) + // Passing the default explicitly is the same compile. + expect(compileCHUnsafe(byService, { orgId: "org_1", service: "api" }, { dialect: CH.clickhouseDialect }).sql).toBe( + compiled.sql, + ) + }) + + it("a binding dialect leaves placeholders and returns values in order", () => { + const compiled = compileCHUnsafe(byService, { orgId: "org_1", service: "api" }, { dialect: numbered }) + expect(compiled.sql).toContain("OrgId = $1") + expect(compiled.sql).toContain("Service = $2") + expect(compiled.sql).not.toContain("org_1") + expect(compiled.parameters).toEqual(["org_1", "api"]) + }) + + // The value is the column codec's wire form, the same thing an inline + // literal is written from, not the JS value the caller passed. + it("binds the encoded wire value", () => { + const query = CH.from(events) + .select(($) => ({ count: $.Count })) + .where(($) => [$.OrgId.eq("org"), $.Timestamp.gte(CH.param.dateTime("start"))]) + const compiled = compileCHUnsafe(query, { start: new Date("2026-01-01T00:00:00.250Z") }, { dialect: numbered }) + expect(compiled.parameters).toEqual(["2026-01-01 00:00:00.250"]) + }) + + it("numbered placeholders reuse one slot for a repeated param; positional ones bind it again", () => { + const twice = CH.from(events) + .select(($) => ({ count: $.Count })) + .where(($) => [$.OrgId.eq(CH.param.string("orgId")), $.Service.neq(CH.param.string("orgId"))]) + const params = { orgId: "org_1" } + + const reused = compileCHUnsafe(twice, params, { dialect: numbered }) + expect(reused.sql).toContain("OrgId = $1") + expect(reused.sql).toContain("Service != $1") + expect(reused.parameters).toEqual(["org_1"]) + + const repeated = compileCHUnsafe(twice, params, { dialect: positional }) + expect(repeated.parameters).toEqual(["org_1", "org_1"]) + }) + + // Nested queries are spliced in as text, so placeholders have to be numbered + // once across the whole statement, not restarted in each branch. + it("numbers placeholders across union branches and subqueries", () => { + const branch = (service: string) => + CH.from(events) + .select(($) => ({ count: $.Count })) + .where(($) => [$.OrgId.eq(CH.param.string("orgId")), $.Service.eq(CH.param.string(service))]) + + const union = compileUnionUnsafe( + CH.unionAll(branch("a"), branch("b")), + { orgId: "org_1", a: "api", b: "web" }, + { dialect: numbered }, + ) + expect(union.parameters).toEqual(["org_1", "api", "web"]) + expect(union.sql.match(/\$\d/g)).toEqual(["$1", "$2", "$1", "$3"]) + + const outer = compileCHUnsafe( + CH.fromQuery(byService, "i") + .select(($) => ({ total: CH.sum($.count) })) + .where(($) => [$.count.gt(CH.param.int("min"))]), + { orgId: "org_1", service: "api", min: 5 }, + { dialect: numbered }, + ) + expect(outer.parameters).toEqual(["org_1", "api", 5]) + expect(outer.tenantScope).toBe("single-tenant") + }) + + it("still fails a missing or ill-typed param at compile time", () => { + expect(() => compileCHUnsafe(byService, { orgId: "org_1" }, { dialect: numbered })).toThrow( + /no value given for param 'service'/, + ) + expect(() => compileCHUnsafe(byService, { orgId: 1, service: "api" }, { dialect: numbered })).toThrow( + /param 'orgId'/, + ) + }) +}) + +describe("dialect literal syntax", () => { + it("writes column literals, string fragments and inline params in the dialect's syntax", () => { + const query = CH.from(events) + .select(($) => ({ count: $.Count })) + .where(($) => [ + $.OrgId.eq(CH.param.string("orgId")), + $.Service.eq("O'Reilly\\"), + $.Service.like("%it's%"), + ]) + const compiled = compileCHUnsafe(query, { orgId: "a'b" }, { dialect: standard }) + expect(compiled.sql).toContain("OrgId = 'a''b'") + expect(compiled.sql).toContain("Service = 'O''Reilly\\'") + expect(compiled.sql).toContain("Service LIKE '%it''s%'") + + // The same query for ClickHouse keeps its backslash escapes. + const clickhouse = compileCHUnsafe(query, { orgId: "a'b" }) + expect(clickhouse.sql).toContain("OrgId = 'a\\'b'") + expect(clickhouse.sql).toContain("Service = 'O\\'Reilly\\\\'") + }) + + it("reaches union branches and subqueries", () => { + const branch = CH.from(events) + .select(($) => ({ count: $.Count })) + .where(($) => [$.OrgId.eq("it's")]) + const union = compileUnionUnsafe(CH.unionAll(branch, branch), {}, { dialect: standard }) + expect(union.sql.match(/'it''s'/g)).toHaveLength(2) + + const outer = compileCHUnsafe( + CH.fromQuery(branch, "i").select(($) => ({ total: CH.sum($.count) })), + {}, + { dialect: standard }, + ) + expect(outer.sql).toContain("OrgId = 'it''s'") + }) + + it("is only installed for the compile", () => { + compileCHUnsafe(byService, { orgId: "o", service: "s" }, { dialect: standard }) + expect(compileFragment(str("it's"))).toBe("'it\\'s'") + }) + + // Params are resolved by rewriting the finished SQL, so a literal that spells + // a placeholder would be rewritten too. A dialect that fails to escape the + // marker is refused instead of trusted. + it("refuses a literal that spells the param marker", () => { + const smuggle = CH.from(events) + .select(($) => ({ count: $.Count })) + .where(($) => [$.OrgId.eq(CH.param.string("orgId")), $.Service.eq("__PARAM_string_orgId__")]) + + expect(() => compileCHUnsafe(smuggle, { orgId: "o" }, { dialect: naive })).toThrow(/reserved param marker/) + expect(compileCHUnsafe(smuggle, { orgId: "o" }, { dialect: standard }).sql).toContain( + "Service = '_' || '_PARAM_string_orgId__'", + ) + }) +}) diff --git a/src/ch/dialect.ts b/src/ch/dialect.ts new file mode 100644 index 0000000..28523d0 --- /dev/null +++ b/src/ch/dialect.ts @@ -0,0 +1,102 @@ +// SQL Dialects +// +// What a compiled query needs to know about the database it is written for. +// The builder, the tenant analysis, and row decoding are the same for every +// database; a dialect is the part that is not: how literals are written and how +// resolved param values reach the server. Identifier quoting, clause rendering, +// and column wire formats move behind it in later steps (see +// `design/dialects.md`). + +import { quoteClickHouseString } from "../sql/sql-fragment" +import { activeLiteralSyntax, withLiteralSyntax, type LiteralSyntax } from "../sql/literal-syntax" +import { QueryBuilderError } from "./errors" +import { sqlLiteral } from "./literal" +import { PARAM_MARKER_PREFIX } from "./param" + +/** + * How resolved param values reach the server. + * + * `inline` writes each value into the SQL text with the dialect's `literal`, + * which is what ClickHouse over HTTP has always done here. `bind` leaves a + * placeholder in the SQL and returns the encoded values in + * `CompiledQuery.parameters`, in placeholder order, for a driver that binds + * them (`$1` for Postgres, `?` for MySQL and SQLite). + */ +export type ParamStyle = + | { readonly _tag: "inline" } + | { + readonly _tag: "bind" + /** The placeholder for the value at 1-based `index`. */ + readonly placeholder: (index: number) => string + /** + * Whether one placeholder may stand for every use of the same param. + * + * True for numbered placeholders (`$1` can appear twice); false for + * positional ones (`?` binds the next value each time it appears), where + * a param used twice is bound twice. + */ + readonly reuse: boolean + } + +/** + * A database the builder writes SQL for. + * + * `quoteString` and `literal` must escape so that no value can spell the param + * marker (`__PARAM_`) in the rendered SQL: params are resolved by rewriting the + * finished text, so a value that spelled one would be rewritten too. A literal + * that does is refused at compile time rather than trusted. + */ +export interface Dialect extends LiteralSyntax { + /** Shown in errors. */ + readonly name: string + readonly params: ParamStyle +} + +/** ClickHouse, with params written into the SQL as literals. The default. */ +export const clickhouseDialect: Dialect = { + name: "clickhouse", + quoteString: quoteClickHouseString, + literal: sqlLiteral, + params: { _tag: "inline" }, +} + +// The dialect of the enclosing compile, beside the syntax installed for the +// fragment renderer. Same save/restore discipline as `withLiteralSyntax`. +let current: Dialect | undefined + +/** The dialect of the enclosing compile, or ClickHouse outside one. */ +export const currentDialect = (): Dialect => current ?? clickhouseDialect + +/** Run `body` with `dialect`'s literal syntax installed, checked as above. */ +export function withDialect(dialect: Dialect, body: () => A): A { + // A nested compile for the dialect already installed keeps the checked + // syntax that is there instead of wrapping it again. + if (current === dialect && activeLiteralSyntax() !== undefined) return body() + const previous = current + current = dialect + try { + return withLiteralSyntax(checkedSyntax(dialect), body) + } finally { + current = previous + } +} + +/** `dialect.literal`, checked as above. For params written in after the + * callbacks have run, outside any installed syntax. */ +export const checkedLiteral = (dialect: Dialect, value: unknown, context: string): string => + checked(dialect, dialect.literal(value, context), context) + +const checkedSyntax = (dialect: Dialect): LiteralSyntax => ({ + quoteString: (value) => checked(dialect, dialect.quoteString(value), "a string literal"), + literal: (value, context) => checkedLiteral(dialect, value, context), +}) + +const checked = (dialect: Dialect, sql: string, context: string): string => { + if (sql.includes(PARAM_MARKER_PREFIX)) { + throw new QueryBuilderError({ + code: "InvalidLiteral", + message: `${context}: the ${dialect.name} dialect wrote a literal containing the reserved param marker \`${PARAM_MARKER_PREFIX}\``, + }) + } + return sql +} diff --git a/src/ch/index.ts b/src/ch/index.ts index 815a712..c66e169 100644 --- a/src/ch/index.ts +++ b/src/ch/index.ts @@ -257,6 +257,9 @@ export { CompiledQueryEncodeError, } from "./compile" +// Dialects: how a compiled query's params reach the server. +export { clickhouseDialect, type Dialect, type ParamStyle } from "./dialect" + // Failures vs defects — the rule the two classes encode is on `QueryBuilderError`. export { QueryBuilderError, QueryBuilderDefect } from "./errors" diff --git a/src/ch/literal.ts b/src/ch/literal.ts index b76e6ee..09a5612 100644 --- a/src/ch/literal.ts +++ b/src/ch/literal.ts @@ -13,7 +13,8 @@ import { Result, Schema } from "effect" import { QueryBuilderError } from "./errors" -import { escapeClickHouseString } from "../sql/sql-fragment" +import { quoteClickHouseString } from "../sql/sql-fragment" +import { activeLiteralSyntax } from "../sql/literal-syntax" import type { CHType } from "./types" const isPlainObject = (value: unknown): value is Record => @@ -48,7 +49,7 @@ const oneLine = (failure: unknown): string => */ export function sqlLiteral(value: unknown, context: string): string { if (value === null) return "NULL" - if (typeof value === "string") return `'${escapeClickHouseString(value)}'` + if (typeof value === "string") return quoteClickHouseString(value) if (typeof value === "bigint") return String(value) if (typeof value === "boolean") return value ? "1" : "0" @@ -88,6 +89,16 @@ export function sqlLiteral(value: unknown, context: string): string { * fails here — while building the SQL — instead of becoming part of it. */ export function encodeLiteral(schema: Schema.Codec, value: unknown, context: string): string { + return (activeLiteralSyntax()?.literal ?? sqlLiteral)(encodeValue(schema, value, context), context) +} + +/** + * Encode a value through a schema to its wire form, without writing SQL. + * + * The half of {@link encodeLiteral} a dialect that binds params needs: the + * driver sends the wire value itself, so there is no literal to write. + */ +export function encodeValue(schema: Schema.Codec, value: unknown, context: string): unknown { const encoded = Schema.encodeUnknownResult(schema)(value) if (Result.isFailure(encoded)) { throw new QueryBuilderError({ @@ -95,7 +106,7 @@ export function encodeLiteral(schema: Schema.Codec, value: unknown, c message: `${context}: ${describe(value)} is not a valid value — ${oneLine(encoded.failure)}`, }) } - return sqlLiteral(encoded.success, context) + return encoded.success } /** `encodeLiteral` against a column type, naming the column in any failure. */ diff --git a/src/sql/literal-syntax.ts b/src/sql/literal-syntax.ts new file mode 100644 index 0000000..c714a2c --- /dev/null +++ b/src/sql/literal-syntax.ts @@ -0,0 +1,32 @@ +// Literal syntax of the dialect being compiled for. +// +// Literals are written while a query's callbacks run, deep inside expression +// code that has no dialect argument to read. `compile` installs the dialect's +// syntax here for the duration of the compile instead, the same way +// `withSubqueryCompiler` hands down the subquery renderer. A leaf module so the +// fragment renderer can read it without importing the builder. + +export interface LiteralSyntax { + /** A string as a quoted literal. */ + readonly quoteString: (value: string) => string + /** An encoded wire value (string, number, boolean, null, arrays and records + * of those) as a literal. Throws for a value it cannot write. */ + readonly literal: (value: unknown, context: string) => string +} + +// Compilation is synchronous. Save/restore makes nested compilations +// independent, including when a callback throws. +let current: LiteralSyntax | undefined + +export function withLiteralSyntax(syntax: LiteralSyntax, body: () => A): A { + const previous = current + current = syntax + try { + return body() + } finally { + current = previous + } +} + +/** The syntax installed by the enclosing compile, if there is one. */ +export const activeLiteralSyntax = (): LiteralSyntax | undefined => current diff --git a/src/sql/sql-fragment.ts b/src/sql/sql-fragment.ts index a6102a6..366e8ce 100644 --- a/src/sql/sql-fragment.ts +++ b/src/sql/sql-fragment.ts @@ -1,4 +1,5 @@ import { Data } from "effect" +import { activeLiteralSyntax } from "./literal-syntax" // ClickHouse string escaping @@ -27,12 +28,19 @@ export function escapeClickHouseString(value: string): string { .replace(/__PARAM_/g, "\\x5F_PARAM_") } +/** A string as a ClickHouse literal: quoted, and escaped as above. */ +export const quoteClickHouseString = (value: string): string => `'${escapeClickHouseString(value)}'` + +/** A string literal in the syntax of the dialect being compiled for, or + * ClickHouse's outside a compile. */ +const quoteString = (value: string): string => (activeLiteralSyntax()?.quoteString ?? quoteClickHouseString)(value) + // SQL Fragment AST export type SqlFragment = Data.TaggedEnum<{ /** Raw SQL string — no escaping. For ClickHouse-specific syntax. */ Raw: { readonly sql: string } - /** Auto-escaped string parameter: produces 'escaped_value' */ + /** A string literal, quoted and escaped by the dialect being compiled for */ Str: { readonly value: string } /** Integer parameter: produces the number as string, rounded */ Int: { readonly value: number } @@ -76,7 +84,7 @@ export const lazy = (render: () => string): SqlFragment => Frag.Lazy({ render }) export const compile: (fragment: SqlFragment) => string = Frag.$match({ Raw: ({ sql }) => sql, - Str: ({ value }) => `'${escapeClickHouseString(value)}'`, + Str: ({ value }) => quoteString(value), Int: ({ value }) => String(Math.round(value)), Ident: ({ name }) => name, Join: ({ separator, fragments }) => fragments.map(compile).filter(Boolean).join(separator),