diff --git a/packages/opencode/src/altimate/native/dbt/project-lineage.ts b/packages/opencode/src/altimate/native/dbt/project-lineage.ts index ab22c8dc07..4f1acbf3b6 100644 --- a/packages/opencode/src/altimate/native/dbt/project-lineage.ts +++ b/packages/opencode/src/altimate/native/dbt/project-lineage.ts @@ -136,9 +136,7 @@ export function selectItems(stripped: string): string[] { if (start < 0) { if (boundary && word === "select") { start = i + 6 - // A whole word only: `allocation` and `distinct_users` are column names, not modifiers. - const lead = /^\s+(?:distinct\s+on\s*\([^)]*\)|distinct|all|top\s+\d+)(?![\w$])/i.exec(text.slice(start)) - if (lead) start += lead[0].length + start += modifierLength(text, start) from = start i = start - 1 } @@ -179,20 +177,140 @@ export function itemName(item: string): string | null { return ENDS_EXPRESSION.test(plain[2]!) ? null : unquoteName(plain[2]!) } -/** The column a SELECT item casts, unaliased (`amount::numeric`, `cast(t.amount as numeric)`); null otherwise. */ -function castColumn(item: string): string | null { - const ref = `(?:(?:${QUOTED}|[\\w$]+)\\.)*(${QUOTED}|[\\w$]+)` - // A type name: words (`double precision`, `pg_catalog.int4`), a size (`numeric(10, 2)`), array marks (`int[]`, - // `int[3]`), a quoted name, a further `::` cast. - const type = `((?:[\\w\\s(),:.]|\\[\\d*\\]|"(?:[^"]|"")*")+)` - const m = new RegExp(`^${ref}\\s*::\\s*${type}$`, "i").exec(item) ?? new RegExp(`^cast\\s*\\(\\s*${ref}\\s+as\\s+${type}\\)$`, "i").exec(item) - if (!m || NOT_TYPE.test(m[2]!.replace(new RegExp(QUOTED, "g"), " "))) return null - return unquoteName(m[1]!) +/** + * The length of the modifier a SELECT list starts with at `at` in quote-blanked text: `distinct on (…)` up to its + * matching `)`, `top N`, or the whole word `distinct` or `all` (`allocation` and `distinct_users` are names); 0 when + * there is none. + */ +function modifierLength(text: string, at: number): number { + const rest = text.slice(at) + const on = /^\s+distinct\s+on\s*\(/i.exec(rest) + if (on) { + const end = closingParen(rest, on[0].length - 1) + return end < 0 ? 0 : end + 1 + } + // The same name characters as the cast reader's, so `allé` is a name too. + return /^\s+(?:top\s+\d+|(?:distinct|all)(?![\p{L}\p{N}_$]))/iu.exec(rest)?.[0].length ?? 0 } -/** Words that make the rest of an item more than a type (`amount::numeric is not null`, `amount::int as x`). */ +/** The index of the `)` that closes the `(` at `open` in quote-blanked text, -1 when it isn't closed. */ +function closingParen(text: string, open: number): number { + let depth = 0 + for (let i = open; i < text.length; i++) { + if (text[i] === "(") depth++ + else if (text[i] === ")" && --depth === 0) return i + } + return -1 +} + +/** A token of a SELECT item for the cast reader. `other` is any character the reader never accepts (`+`, `-`, `|`, …). */ +type CastToken = { kind: "name" | "quoted" | "number" | "string" | "punct" | "other"; text: string } + +const CAST_TOKEN = + /\s+|("(?:[^"]|"")*"|`(?:[^`]|``)*`)|('(?:[^']|'')*')|(\d+(?:\.\d*)?(?:e[+-]?\d+)?|\.\d+(?:e[+-]?\d+)?)|([\p{L}_$][\p{L}\p{N}_$]*)|(::|[().,[\]])|([^])/iuy + +/** The tokens of a literal- and comment-stripped SELECT item, whitespace dropped. */ +function castTokens(item: string): CastToken[] { + const out: CastToken[] = [] + CAST_TOKEN.lastIndex = 0 + for (let m = CAST_TOKEN.exec(item); m; m = CAST_TOKEN.exec(item)) { + const kind = m[1] ? "quoted" : m[2] ? "string" : m[3] ? "number" : m[4] ? "name" : m[5] ? "punct" : m[6] ? "other" : undefined + if (kind) out.push({ kind, text: m[0] }) + } + return out +} + +/** Words a type name never holds: an item with one after `::` is more than a cast (`amount::numeric is not null`). */ const NOT_TYPE = - /(? t[k]?.kind === "punct" && t[k]!.text === text + const word = (k: number, text: string) => t[k]?.kind === "name" && t[k]!.text.toLowerCase() === text + const named = (k: number) => t[k]?.kind === "quoted" || t[k]?.kind === "name" + // A type modifier: a number (`numeric(10, 2)`) or a name (`varchar(max)`, `geometry(Point, 4326)`). + const modifier = (k: number) => t[k]?.kind === "number" || named(k) + + const type = (): boolean => { + let words = 0 + for (;;) { + const k = t[at] + if (k && (k.kind === "quoted" || (k.kind === "name" && !NOT_TYPE.test(k.text)))) at++, words++ + else if (words && punct(at, ".")) at++ + else if (words && punct(at, "(") && modifier(at + 1)) { + at += 2 + while (punct(at, ",") && modifier(at + 1)) at += 2 + if (!punct(at, ")")) return false + at++ + } else if (words && punct(at, "[")) { + at += t[at + 1]?.kind === "number" ? 2 : 1 + if (!punct(at, "]")) return false + at++ + } else return words > 0 + } + } + const column = (): string | null => { + if (!named(at)) return null + const start = at++ + while (punct(at, ".") && named(at + 1)) at += 2 + const last = t[at - 1]! + // Only the token's own quote is escaped by doubling: `"a``b"` is a``b. + if (last.kind === "quoted") { + const q = last.text[0]! + return last.text.slice(1, -1).replaceAll(q + q, q).toLowerCase() + } + return at - start === 1 && ENDS_EXPRESSION.test(last.text) ? null : last.text.toLowerCase() + } + const operand = (): string | null => { + if (punct(at, "(")) { + at++ + const name = chain() + if (name === null || !punct(at, ")")) return null + at++ + shaped = true + return name + } + if (word(at, "cast") && punct(at + 1, "(")) { + at += 2 + const name = chain() + if (name === null || !word(at, "as")) return null + at++ + if (!type() || !punct(at, ")")) return null + at++ + shaped = true + return name + } + return column() + } + const chain = (): string | null => { + const name = operand() + if (name === null) return null + while (punct(at, "::")) { + at++ + if (!type()) return null + shaped = true + } + return name + } + + const name = chain() + return name !== null && shaped && at === t.length ? name : null +} /** * The alias templates `COLUMNS(...) AS "…"` items give (DuckDB's `"col_\0"`, which hold a `\N` marker), lower-cased. diff --git a/packages/opencode/test/altimate/column-answer.test.ts b/packages/opencode/test/altimate/column-answer.test.ts index c2bb551bdc..8cbc27fa1e 100644 --- a/packages/opencode/test/altimate/column-answer.test.ts +++ b/packages/opencode/test/altimate/column-answer.test.ts @@ -5,7 +5,7 @@ import { join } from "node:path" import { ENGINE, MAX_ANSWER_COLUMNS, columnAnswer, describeAnswer, type ColumnAnswer } from "../../src/altimate/native/dbt/column-answer" import { columnReach, namesColumn } from "../../src/altimate/native/dbt/column-reach" import { dbtLineage } from "../../src/altimate/native/dbt/lineage" -import { clearLineageCaches, hasNestedWith, howOf, itemName, quotingIsClear, relationUse, shapeFlags, stripAggFilter, stripSql } from "../../src/altimate/native/dbt/project-lineage" +import { castColumn, clearLineageCaches, hasNestedWith, howOf, itemName, quotingIsClear, relationUse, shapeFlags, stripAggFilter, stripSql } from "../../src/altimate/native/dbt/project-lineage" const dirs: string[] = [] afterEach(() => { @@ -1303,7 +1303,7 @@ describe("both tools: names at the edges", () => { }) test("a column name that starts with all or distinct isn't read as a SELECT modifier", () => { - for (const name of ["allocation", "distinct_users", "all_time"]) { + for (const name of ["allocation", "distinct_users", "all_time", "all$x"]) { const path = write(nodes("postgres", [ model("f", [], `select 2.5 as ${name}`, [name]), model("r", ["f"], `with q as (select * from ${T("f")}) select ${name}::numeric from q`), @@ -1316,8 +1316,90 @@ describe("both tools: names at the edges", () => { } }) + test("a cast after a SELECT modifier is named after its column, so a settled model has it", () => { + // Most throw "r has no column amount" on main, where the modifier or the parentheses hid the cast. + // `distinct on (id) id, …` and `top 5 amount…` already answer there; they stay as controls. + for (const dialect of ["postgres", "redshift"]) { + const lists = ["distinct(amount)::numeric, id", "all(amount)::numeric, id", "distinct ((amount))::numeric, id", "id, (amount)::numeric", + "id, cast((amount) as numeric)"] + // DISTINCT ON is Postgres only; TOP is Redshift's. + if (dialect === "postgres") + lists.push("distinct on(id)amount::numeric, id", "distinct on (coalesce(id, 0)) id, amount::numeric", "distinct on (id) id, amount::numeric") + if (dialect === "redshift") lists.push("top 5amount::numeric, id", "top 5 amount::numeric, id") + for (const list of lists) { + const path = write(nodes(dialect, [ + model("f", [], "select 1 as id, 2.5 as amount", ["id", "amount"]), + model("r", ["f"], `with q as (select * from ${T("f")}) select ${list} from q`), + model("child", ["r"], `select * from ${T("r")}`), + ])) + expect(() => columnReach(path, "r", "amount")).not.toThrow() + expect(carried(columnAnswer(path, "r", "amount"))).toEqual(["child.amount"]) + } + } + }) + + test("a qualified column named like a literal keeps its name through a cast", () => { + for (const dialect of ["postgres", "redshift"]) { + for (const [name, cast] of [["end", "q.end::int"], ["end", "cast(q.end as int)"], ["end", "(q.end)::int"], ["null", "q.null::int"], ["true", "q.true::int"]]) { + const path = write(nodes(dialect, [ + model("f", [], `select 1 as id, 2 as "${name}"`, ["id", name!]), + model("r", ["f"], `with q as (select * from ${T("f")}) select id, ${cast} from q`), + model("child", ["r"], `select * from ${T("r")}`), + ])) + expect(() => columnReach(path, "r", name!)).not.toThrow() + expect(carried(columnAnswer(path, "r", name!))).toEqual([`child.${name}`]) + } + } + }) + + // Every cast form the reviews of this reader found, with the name Postgres gives it (null: not a cast of a column, so + // the engine's `_col_N` stays). Items are as `selectItems` hands them over: string literals already stripped to `''`. + const CAST_FORMS: [string, string | null][] = [ + // Whole-item casts of a column. + ["amount::numeric", "amount"], ["cast(amount as numeric)", "amount"], ["t.amount::numeric", "amount"], + ["cast(t.amount as numeric)", "amount"], ["amount::int[]", "amount"], ["cast(amount as int[])", "amount"], + ["amount::int[3]", "amount"], ['amount::"numeric"', "amount"], ["amount::pg_catalog.numeric", "amount"], + ["amount::double precision", "amount"], ["amount::numeric(10, 2)", "amount"], ["cast(amount as numeric(10, 2))", "amount"], + ["amount::timestamp(3) with time zone", "amount"], ["amount::geometry(Point,4326)", "amount"], + ["cast(amount as geometry(Point, 4326))", "amount"], ["amount::varchar(max)", "amount"], + // Chains and parentheses. + ["amount::numeric::int", "amount"], ["cast(amount as numeric)::int", "amount"], ["cast(cast(amount as numeric) as int)", "amount"], + ["(amount::numeric)::int", "amount"], ["(amount)::numeric", "amount"], ["((amount))::numeric", "amount"], + ["cast((amount) as numeric)", "amount"], ["(amount)", "amount"], + // No space beside `as`, and quoted names. + ["cast((amount)as numeric)", "amount"], ['cast("amount"as numeric)', "amount"], ['cast(amount as"numeric")', "amount"], + ['"Amount"::int', "amount"], ['("a""b")::numeric', 'a"b'], ['("null")::int', "null"], ['"a``b"::int', "a``b"], ['`a""b`::int', 'a""b'], + // Qualified names that read like literals or keywords. + ["q.end::int", "end"], ["cast(q.end as int)", "end"], ["(q.end)::int", "end"], ["q.null::int", "null"], ["q.true::int", "true"], + ["cast(t.has as int)", "has"], ["allé::numeric", "allé"], + // Not a cast of a column. + ["amount", null], ["t.amount", null], ["(null)::int", null], ["(true)::boolean", null], ["(1)::int", null], ["1::int", null], + ["cast(1.5 as numeric)", null], ["(1.0)::numeric", null], ["''::int", null], ["coalesce(amount)::numeric", null], + ["-(amount)::numeric", null], ["(amount)::numeric * 2", null], ["(amount + 1)::int", null], ["amount::numeric + 1", null], + ["cast(amount as int) * 2", null], ["amount::text || ''", null], ["cast(amount as int) between 1 and 2", null], + ["amount::numeric is not null", null], ["amount::int in (1, 2)", null], ["amount::numeric as _col_1", null], + ["amount::timestamp at time zone ''", null], ["amount::text collate \"C\"", null], ["arr[1]::int", null], ["cast(amount)", null], + ["cast(amount as)", null], ["amount::", null], ["(amount", null], + ] + + test("castColumn reads every cast form found so far", () => { + const wrong = CAST_FORMS.filter(([item, name]) => castColumn(item) !== name).map(([item, name]) => `${item} → ${castColumn(item)} (want ${name})`) + expect(wrong).toEqual([]) + }) + + test("mixed and nested cast chains of a column are named after it", () => { + for (const cast of ["cast(amount as numeric)::int", "cast(cast(amount as numeric) as int)", "(amount::numeric)::int"]) { + const path = write(nodes("postgres", [ + model("orders", [], "select 1 as order_id, 2.5 as amount", ["order_id", "amount"]), + model("r", ["orders"], `select order_id, ${cast} from ${T("orders")}`), + ])) + expect(carried(columnAnswer(path, "orders", "amount"))).toEqual(["r.amount"]) + } + }) + test("an expression that only starts with a cast isn't named after the column, and a _col_N the SQL writes stays", () => { - for (const item of ["amount::numeric is not null", "amount::int in (1, 2)", "cast(amount as int) between 1 and 2", "amount::numeric as _col_1"]) { + for (const item of ["amount::numeric is not null", "amount::int in (1, 2)", "cast(amount as int) between 1 and 2", "amount::numeric as _col_1", + "(amount + 1)::int", "amount::numeric + 1", "cast(amount as int) * 2"]) { const path = write(nodes("postgres", [ model("orders", [], "select 1 as order_id, 2.5 as amount", ["order_id", "amount"]), model("r", ["orders"], `select order_id, ${item} from ${T("orders")}`), @@ -1352,6 +1434,14 @@ describe("both tools: names at the edges", () => { ])) const named = dbtLineage({ manifest_path: clash, model: "child" }) expect(Object.keys((named.raw_lineage as any).column_dict ?? {}).sort()).toEqual(["id", "id_1", "id_1_1"]) + // Names repeat regardless of case: `Val, val` is `val, val_1`. + const cased = write(nodes("duckdb", [ + model("f", [], "select 1 as id, 2.5 as amount", ["id", "amount"]), + model("r", ["f"], `select id, amount as Val, id as val from ${T("f")}`), + model("child", ["r"], `select * from ${T("r")}`), + ])) + const folded = dbtLineage({ manifest_path: cased, model: "child" }) + expect(Object.keys((folded.raw_lineage as any).column_dict ?? {}).sort()).toEqual(["id", "val", "val_1"]) }) test("the text line says how many may-carry models are only not traced, without suggesting no effect", () => {