diff --git a/packages/opencode/src/altimate/tools/dbt-map-relations.ts b/packages/opencode/src/altimate/tools/dbt-map-relations.ts index f949274620..cafbf26adf 100644 --- a/packages/opencode/src/altimate/tools/dbt-map-relations.ts +++ b/packages/opencode/src/altimate/tools/dbt-map-relations.ts @@ -1,21 +1,762 @@ import z from "zod" +import fs from "fs" +import path from "path" import { Tool } from "../../tool/tool" +import { Dispatcher } from "../native" +import { loadRawManifest, detectDialect } from "../native/dbt/helpers" +import { projectRoot } from "../native/connections/registry" + +export type RelationMatch = "exact" | "case_insensitive" | "partial" | "none" +export type RelationKind = "model" | "snapshot" | "seed" | "source" + +export type RelationCandidate = { + replaceWith: string + kind: RelationKind + name: string + uniqueId: string + relationName: string + match: Exclude +} + +export type RelationMapping = { + /** The table as written in the SQL, qualifiers included. */ + raw: string + /** A `ref()` or `source()` call, or `raw` itself when nothing matched. */ + replaceWith: string + match: RelationMatch + /** Every manifest relation the table matched, best first. */ + candidates: RelationCandidate[] +} + +type Relation = { + kind: RelationKind + name: string + uniqueId: string + relationName: string + parts: string[] + call: string + /** False for a model that comes from an installed package; a bare ref() resolves to the project's own first. */ + root: boolean +} + +const RANK: Record, number> = { exact: 0, case_insensitive: 1, partial: 2 } +const KIND_RANK: Record = { model: 0, snapshot: 1, seed: 2, source: 3 } + +/** + * Split a dotted name into its parts, honouring "double", `back` and [bracket] quotes. Dots inside + * backticks split the name when `splitBacktickDots` is set: BigQuery reads `proj.ds.tbl` as a path. + */ +export function splitQualified(text: string, splitBacktickDots = false): string[] { + const parts: string[] = [] + let cur = "" + let i = 0 + const push = () => { + parts.push(cur) + cur = "" + } + while (i < text.length) { + const ch = text[i] + if (ch === '"' || ch === "`" || ch === "[") { + const close = ch === "[" ? "]" : ch + let j = i + 1 + let body = "" + while (j < text.length) { + if (text[j] === close) { + if (text[j + 1] === close && close !== "]") { + body += close + j += 2 + continue + } + break + } + body += text[j++] + } + if (ch === "`" && splitBacktickDots) { + const sub = body.split(".") + cur += sub[0] + for (const s of sub.slice(1)) { + push() + cur = s + } + } else cur += body + i = j + 1 + } else if (ch === ".") { + push() + i++ + } else { + cur += ch + i++ + } + } + push() + return parts +} + +/** A table reference found in the SQL text: the text as written and its unquoted parts. */ +type Reference = { raw: string; parts: string[]; start?: number; /** The last part was written in double quotes. */ quoted?: boolean } + +/** What the scan of a query found: its table references in text order, and the names its CTEs define. */ +export type Scan = { + refs: Reference[] + /** By identity (see `identityOf`): the text range each definition of the name is visible in (its statement or enclosing parentheses). */ + ctes: Map +} + +// Dialects whose double quotes (and, for some, backslashes) belong to string literals. +const DOUBLE_QUOTE_STRING = new Set(["bigquery", "databricks", "spark", "mysql", "hive"]) +const BACKSLASH_ESCAPE = new Set(["bigquery", "databricks", "spark", "mysql", "hive", "clickhouse", "snowflake"]) +const DOLLAR_QUOTE = new Set(["snowflake", "postgres", "redshift", "duckdb"]) +const IMPLICIT_RECURSION = new Set(["snowflake", "tsql", "sqlserver", "mssql", "fabric", "synapse", "oracle"]) +// Dialects whose block comments nest: an inner comment does not end the outer one. +const NESTED_COMMENTS = new Set(["postgres", "postgresql", "cockroachdb", "duckdb", "tsql", "sqlserver", "mssql", "fabric", "synapse", "clickhouse", "spark", "databricks"]) +const ESCAPE_STRING = new Set(["postgres", "postgresql", "redshift", "cockroachdb", "duckdb"]) +const HASH_COMMENT = new Set(["bigquery", "mysql", "mariadb", "clickhouse"]) +// Dialects where a double-quoted name keeps its case while an unquoted one is folded, so `"orders"` and `ORDERS` differ. +const FOLDS_UNQUOTED: Record = { snowflake: "upper", oracle: "upper", postgres: "lower", postgresql: "lower", redshift: "lower", trino: "lower" } +const BRACKET_QUOTE = new Set(["tsql", "sqlserver", "mssql", "fabric", "synapse"]) + +const RESERVED = new Set([ + "select", + "from", + "where", + "join", + "inner", + "left", + "right", + "full", + "outer", + "cross", + "natural", + "lateral", + "on", + "using", + "group", + "order", + "having", + "limit", + "offset", + "union", + "intersect", + "except", + "minus", + "qualify", + "window", + "set", + "when", + "then", + "else", + "end", + "as", + "values", + "returning", + "fetch", + "for", + "pivot", + "unpivot", + "tablesample", + "straight_join", + "asof", + "with", + "and", + "or", + "not", + "is", + "in", +]) +// Clause words that start a list of tables. `from` inside these calls reads a column or a string instead. +const TABLE_KEYWORDS = new Set(["from", "join", "straight_join", "into", "update", "using"]) +// Words that end a join's ON condition: the next join, or the next clause of the query. +const JOIN_CONDITION_END = new Set([ + "join", "straight_join", "inner", "left", "right", "full", "cross", "natural", "outer", "where", "group", "order", "having", + "union", "intersect", "except", "minus", "limit", "offset", "fetch", "window", "qualify", "for", "returning", + "when", "set", "do", "select", "lateral", "distribute", "cluster", "sort", "settings", +]) +// Words that can stand between `SELECT` and a `STRAIGHT_JOIN` select option; there it names no table. +const SELECT_OPTIONS = new Set(["select", "all", "distinct", "distinctrow", "high_priority", "sql_small_result", "sql_big_result", "sql_buffer_result", "sql_no_cache", "sql_calc_found_rows"]) +// Words that stand before `TABLE` and name a statement that creates or overwrites one. +const DDL_TABLE_PREFIX = new Set(["overwrite", "create", "replace", "temporary", "temp", "transient", "external", "volatile", "unlogged", "global", "local"]) +const CREATE_TABLE_MODIFIERS = new Set(["if", "not", "exists"]) +// After these a BigQuery project id may carry hyphens. +const HYPHEN_AFTER = new Set([...TABLE_KEYWORDS, "table", "exists"]) +const SUBQUERY_START = new Set(["select", "with", "values", "table", "from"]) +const FROM_INSIDE_CALL = new Set(["extract", "trim", "substring", "overlay", "position"]) + +type Token = { kind: "word" | "quoted" | "punct" | "other"; start: number; end: number; text: string } + +/** The tokens of the SQL with comments and string literals dropped; a quoted name stays one token. */ +function tokenize(sql: string, dialect: string): Token[] { + const out: Token[] = [] + const n = sql.length + let i = 0 + const doubleIsString = DOUBLE_QUOTE_STRING.has(dialect) + const backslash = BACKSLASH_ESCAPE.has(dialect) + const push = (kind: Token["kind"], start: number, end: number) => + out.push({ kind, start, end, text: sql.slice(start, end) }) + const skipString = (quote: string, from: number, escaped = false) => { + let j = from + while (j < n) { + if ((backslash || escaped) && sql[j] === "\\") j += 2 + else if (sql[j] === quote) { + if (sql[j + 1] === quote) j += 2 + else return j + 1 + } else j++ + } + return n + } + const lastWord = () => { + const last = out[out.length - 1] + return last?.kind === "word" ? last.text.toLowerCase() : "" + } + while (i < n) { + const ch = sql[i] + const two = sql.slice(i, i + 2) + if (/\s/.test(ch)) i++ + else if (two === "--" || (ch === "#" && HASH_COMMENT.has(dialect)) || (two === "//" && (dialect === "snowflake" || dialect === "clickhouse"))) { + while (i < n && sql[i] !== "\n") i++ + } else if (two === "/*") { + if (NESTED_COMMENTS.has(dialect)) { + let depth = 1 + let j = i + 2 + while (j < n && depth > 0) { + if (sql.startsWith("/*", j)) (depth++, (j += 2)) + else if (sql.startsWith("*/", j)) (depth--, (j += 2)) + else j++ + } + i = j + } else { + const end = sql.indexOf("*/", i + 2) + i = end < 0 ? n : end + 2 + } + } else if (ch === "'" || (ch === '"' && doubleIsString)) { + const triple = dialect === "bigquery" && sql.startsWith(ch.repeat(3), i) + if (triple) { + const end = sql.indexOf(ch.repeat(3), i + 3) + i = end < 0 ? n : end + 3 + } else { + // PostgreSQL's E'...' string reads a backslash as an escape. + const prefix = out[out.length - 1] + const escapeString = ch === "'" && ESCAPE_STRING.has(dialect) && prefix?.kind === "word" && prefix.end === i && /^e$/i.test(prefix.text) + i = skipString(ch, i + 1, escapeString) + } + } else if ( + ch === "$" && + DOLLAR_QUOTE.has(dialect) && + /^\$(?:[A-Za-z_][A-Za-z0-9_]*)?\$/.test(sql.slice(i, i + 40)) + ) { + const tag = /^\$(?:[A-Za-z_][A-Za-z0-9_]*)?\$/.exec(sql.slice(i, i + 40))![0] + const end = sql.indexOf(tag, i + tag.length) + i = end < 0 ? n : end + tag.length + } else if (ch === '"' || ch === "`" || (ch === "[" && BRACKET_QUOTE.has(dialect))) { + const close = ch === "[" ? "]" : ch + let j = i + 1 + while (j < n) { + if (sql[j] === close) { + if (sql[j + 1] === close && close !== "]") j += 2 + else break + } else j++ + } + const end = Math.min(j + 1, n) + push("quoted", i, end) + i = end + } else if (/[\p{L}_]/u.test(ch)) { + let j = i + 1 + while (j < n && /[\p{L}\p{M}\p{N}_$]/u.test(sql[j])) j++ + // BigQuery project ids may contain hyphens (`my-proj.ds.tbl` unquoted), but only where a table is named. + if (dialect === "bigquery" && (HYPHEN_AFTER.has(lastWord()) || out[out.length - 1]?.text === ",")) { + while (sql[j] === "-" && /[A-Za-z0-9_]/.test(sql[j + 1] ?? "")) { + j++ + while (j < n && /[\p{L}\p{N}_$]/u.test(sql[j])) j++ + } + } + push("word", i, j) + i = j + } else if (/[0-9]/.test(ch)) { + let j = i + 1 + while (j < n && /[A-Za-z0-9_]/.test(sql[j])) j++ + push("other", i, j) + i = j + } else { + push(/[.,()]/.test(ch) ? "punct" : "other", i, i + 1) + i++ + } + } + return out +} + +/** The name a quoted token or word holds: quotes removed, doubled quotes undone. */ +function nameOf(token: Token, dialect: string): string[] { + if (token.kind !== "quoted") return [token.text] + const open = token.text[0] + const close = open === "[" ? "]" : open + const body = token.text.slice(1, token.text.endsWith(close) && token.text.length > 1 ? -1 : undefined) + const name = close === "]" ? body : body.split(close + close).join(close) + // BigQuery reads `proj.ds.tbl` in one pair of backticks as a path. + return open === "`" && dialect === "bigquery" ? name.split(".") : [name] +} + +/** Index of the token after the `)` that closes the `(` at `at`. */ +function afterParens(tokens: Token[], at: number): number { + let depth = 0 + for (let i = at; i < tokens.length; i++) { + if (tokens[i].text === "(" && tokens[i].kind === "punct") depth++ + else if (tokens[i].text === ")" && tokens[i].kind === "punct" && --depth === 0) return i + 1 + } + return tokens.length +} + +/** + * What a name written once refers to: its lower-cased text, except where the dialect keeps the case of a + * double-quoted name and folds an unquoted one, which there is the folded text and the quoted one its own text. + */ +function identityOf(name: string, quoted: boolean, dialect: string): string { + const fold = FOLDS_UNQUOTED[dialect] + if (!fold) return name.toLowerCase() + if (quoted) return name + return fold === "upper" ? name.toUpperCase() : name.toLowerCase() +} + +const isPunct = (token: Token | undefined, text: string) => token?.kind === "punct" && token.text === text + +/** + * The table references of FROM, JOIN, INTO, UPDATE and USING clauses, in text order, written as in the SQL, + * and the names the query's CTEs define. altimate-core's table list keeps only the last name part and leaves + * out any table named like a CTE, so the schema and database a person wrote are read back from the text here. + */ +export function scanTableReferences(sql: string, dialect: string): Scan { + const tokens = tokenize(sql, dialect) + const refs: Reference[] = [] + const ctes: Scan["ctes"] = new Map() + const calls: string[] = [] + + // `name [(columns)] as [not] [materialized] (`, then `,` for the next CTE. + const readCtes = (start: number) => { + let i = start + // Snowflake, T-SQL and Oracle recurse without the keyword: a CTE that names itself is the recursion there. + const keyword = tokens[i]?.text.toLowerCase() === "recursive" + if (keyword) i++ + const recursive = keyword || IMPLICIT_RECURSION.has(dialect) + // A CTE is visible to the end of its statement, or of the parentheses around its WITH. + let depth = 0 + let scopeEnd = sql.length + for (let k = start; k < tokens.length; k++) { + if (isPunct(tokens[k], "(")) depth++ + else if (isPunct(tokens[k], ")") && depth-- === 0) (scopeEnd = tokens[k].start), (k = tokens.length) + else if (tokens[k].text === ";" && depth === 0) (scopeEnd = tokens[k].start), (k = tokens.length) + } + while (tokens[i] && (tokens[i].kind === "word" || tokens[i].kind === "quoted")) { + const name = identityOf(nameOf(tokens[i], dialect)[0], tokens[i].kind === "quoted" && tokens[i].text[0] === '"', dialect) + let j = i + 1 + if (isPunct(tokens[j], "(")) j = afterParens(tokens, j) + if (tokens[j]?.text.toLowerCase() !== "as") return + j++ + if (tokens[j]?.text.toLowerCase() === "not") j++ + if (tokens[j]?.text.toLowerCase() === "materialized") j++ + if (!isPunct(tokens[j], "(")) return + j = afterParens(tokens, j) + const known = ctes.get(name) ?? { scopes: [] } + // Visible after its own definition, so an earlier CTE or its own body names the real table; a recursive one sees every CTE of its WITH, later ones included. + known.scopes.push([recursive ? tokens[start - 1].start : tokens[j - 1].end, scopeEnd]) + ctes.set(name, known) + if (!isPunct(tokens[j], ",")) return + i = j + 1 + } + } + + const isIndexHint = (at: number) => + tokens[at]?.kind === "word" && ["use", "force", "ignore"].includes(tokens[at].text.toLowerCase()) && ["index", "key"].includes(tokens[at + 1]?.text.toLowerCase() ?? "") + + const readList = (start: number, strict: boolean) => { + let i = start + // FROM and JOIN lists may hold a parenthesised group at any position. After USING a `(` is a column list + // unless a JOIN sits inside it. + const using = tokens[start - 1]?.text.toLowerCase() === "using" + const groups = strict + const isGroup = (at: number) => + !using || tokens.slice(at, afterParens(tokens, at)).some((t) => t.kind === "word" && t.text.toLowerCase() === "join") + let groupEnd = -1 + for (;;) { + // LATERAL prefixes a table expression in the list; the list goes on after it. + if (tokens[i]?.kind === "word" && tokens[i].text.toLowerCase() === "lateral") i++ + // `(a join b on ...)` groups tables, at any position in the list: its first table is read like any other, and the list goes on after the + // group's `)`. The tables after the first are found by the JOIN in the group. Any other `(` is a subquery. + while ( + groups && + isPunct(tokens[i], "(") && + isGroup(i) && + !SUBQUERY_START.has(tokens[i + 1]?.text.toLowerCase() ?? "") && + (["word", "quoted"].includes(tokens[i + 1]?.kind ?? "") || isPunct(tokens[i + 1], "(")) + ) { + if (groupEnd < 0) groupEnd = afterParens(tokens, i) + i++ + } + if (isPunct(tokens[i], "(")) { + i = afterParens(tokens, i) + if (groupEnd >= 0) i = Math.max(i, groupEnd) + groupEnd = -1 + } else { + const first = tokens[i] + if (!first || (first.kind !== "word" && first.kind !== "quoted")) return + if (first.kind === "word" && RESERVED.has(first.text.toLowerCase())) return + // PostgreSQL's `FROM ONLY name` leaves out the table's children; the name follows. + if (dialect.startsWith("postgres") && first.kind === "word" && first.text.toLowerCase() === "only" && (["word", "quoted"].includes(tokens[i + 1]?.kind ?? "") || (isPunct(tokens[i + 1], "(") && ["word", "quoted"].includes(tokens[i + 2]?.kind ?? "")))) { + i++ + continue + } + const parts: string[] = [...nameOf(first, dialect)] + let at = i + for (;;) { + // T-SQL's `db..table` leaves the schema out: the gap is kept, so the name stays as written and matches nothing it should not. + const gap = isPunct(tokens[at + 1], ".") && isPunct(tokens[at + 2], ".") && ["word", "quoted"].includes(tokens[at + 3]?.kind ?? "") + if (gap) { + parts.push("") + at += 1 + } + if (!isPunct(tokens[at + 1], ".") || !["word", "quoted"].includes(tokens[at + 2]?.kind ?? "")) break + at += 2 + parts.push(...nameOf(tokens[at], dialect)) + } + const last = tokens[at] + i = at + 1 + if (isPunct(tokens[i], "(") && strict) i = afterParens(tokens, i) + else + refs.push({ + raw: sql.slice(first.start, last.end), + parts, + start: first.start, + quoted: last.kind === "quoted" && last.text[0] === '"', + }) + if (groupEnd >= 0) i = groupEnd + groupEnd = -1 + } + // An alias with or without AS, then its column list, then a comma continues a FROM list. + if (tokens[i]?.text.toLowerCase() === "as") i++ + const alias = tokens[i] + const startsHint = isIndexHint(i) + const startsSample = startsHint || alias?.kind === "word" && ["sample", "tablesample", "pivot", "unpivot"].includes(alias.text.toLowerCase()) && (isPunct(tokens[i + 1], "(") || ["bernoulli", "system", "block", "row"].includes(tokens[i + 1]?.text.toLowerCase() ?? "")) + if (alias && !startsSample && (alias.kind === "quoted" || (alias.kind === "word" && !RESERVED.has(alias.text.toLowerCase())))) { + i++ + if (isPunct(tokens[i], "(")) i = afterParens(tokens, i) + } + // A sample clause or a join condition sits between a table and the comma that goes on with the list. + i = skipSampleAndCondition(i) + if (isPunct(tokens[i], ",")) i++ + else return + } + } + + // After a table and its alias: `TABLESAMPLE [method] (n) [REPEATABLE (n)]`, then a join's `ON ...` up to a comma. + const skipSampleAndCondition = (start: number) => { + let i = start + const word = () => (tokens[i]?.kind === "word" ? tokens[i].text.toLowerCase() : "") + // MySQL's `USE|FORCE|IGNORE INDEX|KEY [FOR ...] (...)` also sits there. + const indexHint = () => isIndexHint(i) + while ( + indexHint() || + ["tablesample", "sample", "repeatable", "seed", "bernoulli", "system", "block", "row"].includes(word()) || + (["with", "pivot", "unpivot"].includes(word()) && isPunct(tokens[i + 1], "(")) + ) { + // T-SQL table hints `WITH (...)` and PIVOT / UNPIVOT (with their alias) sit between a table and the comma that continues the list. + const pivot = ["pivot", "unpivot"].includes(word()) + const hint = indexHint() + i++ + if (hint) { + i++ + while (tokens[i] && ["for", "join", "order", "group", "by"].includes(word())) i++ + } + if (isPunct(tokens[i], "(")) i = afterParens(tokens, i) + if (pivot) { + if (tokens[i]?.text.toLowerCase() === "as") i++ + if (tokens[i] && (tokens[i].kind === "quoted" || (tokens[i].kind === "word" && !RESERVED.has(word())))) i++ + } + } + if (word() !== "on") return i + i++ + let cases = 0 + while (i < tokens.length) { + if (tokens[i].kind === "word" && word() === "case") cases++ + else if (tokens[i].kind === "word" && word() === "end" && cases > 0) cases-- + if (isPunct(tokens[i], "(")) i = afterParens(tokens, i) + else if (isPunct(tokens[i], ",")) return i + else if (isPunct(tokens[i], ")") || tokens[i].text === ";" || (tokens[i].kind === "word" && cases === 0 && JOIN_CONDITION_END.has(word()))) return i + else i++ + } + return i + } + + for (let i = 0; i < tokens.length; i++) { + const token = tokens[i] + if (isPunct(token, "(")) { + const before = tokens[i - 1] + calls.push(before?.kind === "word" ? before.text.toLowerCase() : "") + } else if (isPunct(token, ")")) calls.pop() + else if (token.kind === "word") { + const word = token.text.toLowerCase() + if (word === "with") readCtes(i + 1) + else if (word === "table" && DDL_TABLE_PREFIX.has(tokens[i - 1]?.text.toLowerCase() ?? "")) { + // `INSERT OVERWRITE TABLE t` and `CREATE [OR REPLACE] TABLE t`: the target is read with the qualifiers written. + let at = i + 1 + while (CREATE_TABLE_MODIFIERS.has(tokens[at]?.text.toLowerCase() ?? "")) at++ + readList(at, false) + } else if (word === "overwrite" && tokens[i - 1]?.text.toLowerCase() === "insert" && !["table", "into", "directory", "local"].includes(tokens[i + 1]?.text.toLowerCase() ?? "")) { + // Databricks' `INSERT OVERWRITE name`, without the optional TABLE. + readList(i + 1, false) + } else if (TABLE_KEYWORDS.has(word)) { + if ( + word === "from" && + (FROM_INSIDE_CALL.has(calls[calls.length - 1] ?? "") || tokens[i - 1]?.text.toLowerCase() === "distinct") + ) + continue + // `FOR JOIN` inside a MySQL index hint names no table. + if (word === "join" && tokens[i - 1]?.text.toLowerCase() === "for") continue + // `INSERT OVERWRITE DIRECTORY 'path' USING format` names a file format. + if (word === "using" && tokens[i - 1]?.text.toLowerCase() === "directory") continue + // `SELECT STRAIGHT_JOIN ` is a select option, not a join. + if (word === "straight_join" && SELECT_OPTIONS.has(tokens[i - 1]?.text.toLowerCase() ?? "")) continue + // A name followed by "(" after FROM, JOIN or USING is a table function; after INTO or UPDATE it is a table. + readList( + word === "into" && tokens[i + 1]?.text.toLowerCase() === "table" ? i + 2 : i + 1, + word === "from" || word === "join" || word === "straight_join" || word === "using", + ) + } + } + } + // An outer list is read before the subquery inside it: put them back in text order. + refs.sort((a, b) => a.start! - b.start!) + return { refs, ctes } +} + +/** A single-quoted Jinja string literal. */ +function jinjaString(value: string): string { + return `'${value.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'` +} + +/** The model, snapshot, seed and source nodes of a manifest, each with its relation's name parts. */ +// The manifest is parsed JSON of a shape dbt owns and versions; its fields are checked where they are read. +export function manifestRelations(manifest: any, dialect?: string): Relation[] { + const rootProject: string | undefined = manifest?.metadata?.project_name + // Only BigQuery names put dots inside backticks; elsewhere a dot in a quoted name is part of the name. + const splitDots = (dialect ?? detectDialect(manifest ?? {})).toLowerCase() === "bigquery" + const out: Relation[] = [] + const relationOf = (node: any): string | undefined => { + if (typeof node?.relation_name === "string" && node.relation_name) return node.relation_name + const db = node?.database + const schema = node?.schema + const name = node?.alias ?? node?.identifier ?? node?.name + if (typeof schema === "string" && schema && typeof name === "string" && name) { + return [db, schema, name].filter((p) => typeof p === "string" && p).join(".") + } + return undefined + } + const nodes: Record = manifest?.nodes ?? {} + for (const [uniqueId, node] of Object.entries(nodes)) { + const kind = node?.resource_type + if (kind !== "model" && kind !== "snapshot" && kind !== "seed") continue + // Ephemeral models are inlined by dbt and have no relation to match. + if (node?.config?.materialized === "ephemeral") continue + const relationName = relationOf(node) + if (!relationName || typeof node.name !== "string") continue + const pkg = + typeof node.package_name === "string" && rootProject && node.package_name !== rootProject + ? node.package_name + : undefined + // Every version of a model shares its name; a bare ref() resolves to the latest one, so a version is named. + const version = + node.version === null || node.version === undefined || node.version === "" + ? "" + : `, v=${JSON.stringify(node.version)}` + const call = pkg + ? `{{ ref(${jinjaString(pkg)}, ${jinjaString(node.name)}${version}) }}` + : `{{ ref(${jinjaString(node.name)}${version}) }}` + out.push({ + kind, + name: node.name, + uniqueId, + relationName, + parts: splitQualified(relationName, splitDots), + call, + root: !pkg, + }) + } + const sources: Record = manifest?.sources ?? {} + for (const [uniqueId, node] of Object.entries(sources)) { + const relationName = relationOf(node) + if (!relationName || typeof node?.source_name !== "string" || typeof node?.name !== "string") continue + out.push({ + kind: "source", + name: node.name, + uniqueId, + relationName, + parts: splitQualified(relationName, splitDots), + call: `{{ source(${jinjaString(node.source_name)}, ${jinjaString(node.name)}) }}`, + root: true, + }) + } + return out +} + +/** How a table written as `parts` matches a relation, or null when it does not. */ +export function matchParts(parts: string[], relation: string[]): Exclude | null { + if (parts.length === 0 || parts.length > relation.length) return null + const tail = relation.slice(relation.length - parts.length) + // T-SQL's `db..table` leaves the schema out: the empty part stands for whatever schema the relation has, so it is only a partial match. + const gap = parts.includes("") + const same = parts.every((p, i) => p === tail[i]) + const fold = parts.every((p, i) => p === "" || p.toLowerCase() === tail[i].toLowerCase()) + if (parts.length < relation.length || gap) return fold ? "partial" : null + if (same) return "exact" + if (fold) return "case_insensitive" + return null +} + +/** Map each table of `sql` to the manifest's ref() or source() call that builds it. */ +export function mapRelations(tables: string[], sql: string, relations: Relation[], dialect: string): RelationMapping[] { + const { refs, ctes } = scanTableReferences(sql, dialect) + // Only tables altimate-core reported count, plus qualified ones its list leaves out because a CTE shares their name. + const wanted = new Set(tables.map((t) => t.toLowerCase())) + const entries: Reference[] = [] + const seen = new Set() + const add = (r: Reference) => { + const key = r.raw + if (seen.has(key)) return + seen.add(key) + entries.push(r) + } + const covered = new Set() + const cteNames = new Set([...ctes.keys()].map((k) => k.toLowerCase())) + for (const r of refs) { + const last = r.parts[r.parts.length - 1] + const whole = r.parts.join(".") + const named = ctes.get(identityOf(last, !!r.quoted, dialect)) + // A CTE hides a table only where it is visible: a later statement can name a real table the same way. + const cte = named && named.scopes.some(([from, to]) => r.start! >= from && r.start! < to) ? named : undefined + if (r.parts.length === 1 && cte) continue + if (wanted.has(last.toLowerCase()) || wanted.has(whole.toLowerCase())) { + add(r) + covered.add(last.toLowerCase()) + covered.add(whole.toLowerCase()) + } else if (cteNames.has(last.toLowerCase())) add(r) + } + for (const t of tables) { + if (covered.has(t.toLowerCase()) || cteNames.has(t.toLowerCase())) continue + add({ raw: t, parts: splitQualified(t, dialect === "bigquery") }) + } + return entries.map((entry) => { + const hits: (RelationCandidate & { root: boolean; folds: boolean })[] = [] + // Where an unquoted name is folded (Snowflake reads `orders` as ORDERS), the relation it really names comes first. + const direction = FOLDS_UNQUOTED[dialect] + const written = entry.parts[entry.parts.length - 1] + const folded = direction && !entry.quoted && written ? (direction === "upper" ? written.toUpperCase() : written.toLowerCase()) : null + for (const rel of relations) { + const how = matchParts(entry.parts, rel.parts) + if (!how) continue + hits.push({ + replaceWith: rel.call, + kind: rel.kind, + name: rel.name, + uniqueId: rel.uniqueId, + relationName: rel.relationName, + match: how, + root: rel.root, + // A relation written without quotes in the manifest is folded by the warehouse like the name in the query, so it is the one named. + folds: folded === null || rel.parts[rel.parts.length - 1] === folded || !/["`\]]$/.test(rel.relationName), + }) + } + hits.sort( + (a, b) => + Number(!a.folds) - Number(!b.folds) || + RANK[a.match] - RANK[b.match] || KIND_RANK[a.kind] - KIND_RANK[b.kind] || + Number(b.root) - Number(a.root) || + a.uniqueId.localeCompare(b.uniqueId), + ) + const best = hits[0] + return { + raw: entry.raw, + replaceWith: best ? best.replaceWith : entry.raw, + match: best ? best.match : "none", + candidates: hits.map(({ root: _root, folds: _folds, ...candidate }) => candidate), + } + }) +} + +export function formatMapping(rows: RelationMapping[]): string { + if (rows.length === 0) return "No tables found in the SQL." + const order: RelationMatch[] = ["none", "partial", "case_insensitive", "exact"] + const lines: string[] = [] + for (const match of order) { + for (const r of rows.filter((x) => x.match === match)) { + const more = r.candidates.length > 1 ? ` (${r.candidates.length} candidates)` : "" + lines.push( + match === "none" ? `none: ${r.raw} -> no model or source` : `${match}: ${r.raw} -> ${r.replaceWith}${more}`, + ) + } + } + return lines.join("\n") +} + +type MapRelationsMetadata = { + success: boolean + mappings: RelationMapping[] + error?: string + dialect?: string + manifest_path?: string +} export const DbtMapRelationsTool = Tool.define("dbt_map_relations", { - description: "Map each table a raw SQL query reads to the dbt ref() or source() for it, from the manifest.", + description: + "Map the tables in a SQL query to dbt ref() and source() calls, from the project's manifest: each table is matched to the relation_name of a model, snapshot, seed or source. Returns, per table, the replacement, how it matched (exact, case_insensitive, partial, none) and the candidates. Read-only.", parameters: z.object({ - sql: z.string().describe("Raw SQL to map"), - dialect: z.string().optional().describe("SQL dialect of the query"), + sql: z.string().describe("The raw SQL to map"), + dialect: z.string().optional().describe("SQL dialect (default: the manifest's adapter)"), manifest_path: z .string() .optional() .describe("manifest.json to match against (default: the dbt project's target/manifest.json)"), + project_dir: z + .string() + .optional() + .describe("dbt project folder whose manifest to use, found as dbt finds it (DBT_TARGET_PATH, then target-path); ignored when manifest_path is given"), }), - async execute() { - return { - title: "dbt_map_relations: not implemented", - metadata: { success: false, error: "not implemented" }, - output: "dbt_map_relations is not implemented in this build of altimate-code yet.", + async execute(args): Promise<{ title: string; metadata: MapRelationsMetadata; output: string }> { + const fail = (msg: string) => ({ + title: "dbt Map Relations: ERROR", + metadata: { success: false, error: msg, mappings: [] } as MapRelationsMetadata, + output: msg, + }) + try { + const manifestPath = args.manifest_path ?? (await defaultManifestPath(args.project_dir)) + if (!manifestPath) return fail("No dbt project found here. Run from a dbt project or pass manifest_path.") + const manifest = loadRawManifest(manifestPath) + if (!manifest) return fail(`No manifest at ${manifestPath}. Run \`dbt parse\` or pass manifest_path.`) + const dialect = (args.dialect || detectDialect(manifest)).toLowerCase() + const result = await Dispatcher.call("altimate_core.metadata", { sql: args.sql, dialect }) + const data = (result.data ?? {}) as Record + const error = result.error ?? data.error + if (!result.success || error) return fail(`Could not read the tables of the SQL: ${error ?? "parse failed"}`) + const tables: string[] = Array.isArray(data.tables) + ? data.tables.filter((t: unknown) => typeof t === "string") + : [] + const mappings = mapRelations(tables, args.sql, manifestRelations(manifest, dialect), dialect) + const count = (m: RelationMatch) => mappings.filter((r) => r.match === m).length + return { + title: `dbt Map Relations: ${mappings.length} tables, ${count("none")} unmatched`, + metadata: { success: true, dialect, manifest_path: manifestPath, mappings }, + output: formatMapping(mappings), + } + } catch (e) { + return fail(e instanceof Error ? e.message : String(e)) } }, }) + +async function defaultManifestPath(projectDir?: string): Promise { + const { findProjectRoot } = await import("../../../../dbt-tools/src/config") + const found = projectDir?.trim() || findProjectRoot(projectRoot()) + const root = found ?? (await (await import("../native/connections/register")).getDbtProjectRoot()) + if (!root) return null + let real = root + try { + real = await fs.promises.realpath(root) + } catch {} + const { resolveDbtTargetPath } = await import("../validators/validator-utils") + return path.join(await resolveDbtTargetPath(real), "manifest.json") +} diff --git a/packages/opencode/test/altimate/dbt-map-relations.test.ts b/packages/opencode/test/altimate/dbt-map-relations.test.ts new file mode 100644 index 0000000000..8f878a36e0 --- /dev/null +++ b/packages/opencode/test/altimate/dbt-map-relations.test.ts @@ -0,0 +1,843 @@ +/** + * dbt_map_relations: maps the tables of a raw SQL query to ref() and source() calls. + * + * The first block pins what altimate-core's extractMetadata returns for quoted, upper-case and + * qualified names: it keeps only the last name part (quotes removed, case kept) and merges tables + * that share it, so the matching rules read the qualifiers back from the SQL text. + */ +import { describe, test, expect, afterEach, beforeAll } from "bun:test" +import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from "fs" +import { tmpdir } from "os" +import { join } from "path" +import { Effect } from "effect" +import type { Tool } from "@/tool/tool" +import { SessionID, MessageID } from "@/session/schema" +import { Dispatcher } from "../../src/altimate/native" +import { registerAll as registerAltimateCore } from "../../src/altimate/native/altimate-core" +import { initTool } from "./tool-fixture" +import { + DbtMapRelationsTool, + manifestRelations, + mapRelations, + matchParts, + scanTableReferences, + splitQualified, +} from "../../src/altimate/tools/dbt-map-relations" + +// Other test files reset the dispatcher; register the parser handlers this file depends on. +beforeAll(() => registerAltimateCore()) + +async function tablesOf(sql: string, dialect: string): Promise { + const result = await Dispatcher.call("altimate_core.metadata", { sql, dialect }) + expect(result.success).toBe(true) + return ((result.data ?? {}) as { tables?: string[] }).tables ?? [] +} + +describe("altimate-core extractMetadata table names", () => { + test("Snowflake: last name part only, quotes removed, case kept, one entry per bare name", async () => { + expect( + await tablesOf( + 'select * from RAW.JAFFLE.Customers c join "raw"."jaffle"."orders" o on 1=1 join analytics.stg_x x on 1=1', + "snowflake", + ), + ).toEqual(["Customers", "orders", "stg_x"]) + expect(await tablesOf("select * from db.sch.orders o join other.orders o2 on 1=1", "snowflake")).toEqual(["orders"]) + expect( + await tablesOf("select * from RAW.JAFFLE.CUSTOMERS a join raw.jaffle.customers b on 1=1", "snowflake"), + ).toEqual(["CUSTOMERS", "customers"]) + expect(await tablesOf('select * from "My Schema"."My Table" t', "snowflake")).toEqual(["My Table"]) + }) + + test("BigQuery: a single backtick path comes back whole, a split one by its last part", async () => { + expect( + await tablesOf( + "select * from `proj.ds.Customers` a join `proj`.`ds`.`Orders` b on 1=1 join ds.x c on 1=1", + "bigquery", + ), + ).toEqual(["Orders", "proj.ds.Customers", "x"]) + expect(await tablesOf("select * from proj-1.ds.Customers a", "bigquery")).toEqual(["Customers"]) + }) + + test("Databricks: three-part and backticked names reduce to the last part", async () => { + expect( + await tablesOf( + "select * from main.raw.Customers a join raw.orders b on 1=1 join `main`.`raw`.`items` c on 1=1", + "databricks", + ), + ).toEqual(["Customers", "items", "orders"]) + }) + + test("CTE names are not tables", async () => { + expect(await tablesOf("with a as (select * from raw.customers) select * from a", "snowflake")).toEqual([ + "customers", + ]) + }) +}) + +describe("splitQualified", () => { + test("splits on dots outside quotes", () => { + expect(splitQualified('"RAW"."JAFFLE"."CUSTOMERS"')).toEqual(["RAW", "JAFFLE", "CUSTOMERS"]) + expect(splitQualified('"a.b"."c"')).toEqual(["a.b", "c"]) + expect(splitQualified("`proj`.`ds`.`t`")).toEqual(["proj", "ds", "t"]) + expect(splitQualified("[db].[dbo].[t]")).toEqual(["db", "dbo", "t"]) + }) + test("backtick dots split only when asked", () => { + expect(splitQualified("`proj.ds.t`", true)).toEqual(["proj", "ds", "t"]) + expect(splitQualified("`proj.ds.t`")).toEqual(["proj.ds.t"]) + }) +}) + +describe("scanTableReferences", () => { + const raws = (sql: string, dialect = "snowflake") => scanTableReferences(sql, dialect).refs.map((r) => r.raw) + + test("reads qualified names from FROM, JOIN and comma lists, written as in the SQL", () => { + const { refs } = scanTableReferences( + 'select * from raw.jaffle.customers c, "RAW"."JAFFLE"."orders" as o left join other.payments p on 1=1', + "snowflake", + ) + expect(refs.map((r) => r.raw)).toEqual(["raw.jaffle.customers", '"RAW"."JAFFLE"."orders"', "other.payments"]) + expect(refs[1].parts).toEqual(["RAW", "JAFFLE", "orders"]) + }) + test("ignores table functions, subqueries, comments and strings", () => { + expect( + raws( + "select 'from fake.t' as s from table(flatten(x)) f join (select 1) q on 1=1 /* from c.d */ -- from e.f\n join real.t t on 1=1", + ), + ).toEqual(["real.t"]) + }) + test("reads hyphenated BigQuery project ids", () => { + expect(scanTableReferences("select * from my-proj.ds.tbl", "bigquery").refs[0].parts).toEqual([ + "my-proj", + "ds", + "tbl", + ]) + }) + test("whitespace or a comment around a dot does not drop the qualifier", () => { + expect(raws("select * from db . right . orders")).toEqual(["db . right . orders"]) + expect(scanTableReferences("select * from other/*gap*/.marts.customers", "snowflake").refs[0].parts).toEqual([ + "other", + "marts", + "customers", + ]) + }) + test("a table after a subquery or table function in a comma list is still read", () => { + expect(raws("select * from (select 1) x, db.right.orders")).toEqual(["db.right.orders"]) + expect(raws("select * from table(generator(rowcount => 1)) x, db.right.orders")).toEqual(["db.right.orders"]) + expect(raws("select * from (select * from db.wrong.orders) x, db.right.orders")).toEqual([ + "db.wrong.orders", + "db.right.orders", + ]) + }) + test("brackets quote names for SQL Server dialects only; ClickHouse double quotes are identifiers", () => { + expect(raws("select * from [db].[s].[t]", "tsql")).toEqual(["[db].[s].[t]"]) + expect(raws('select * from "db"."orders"', "clickhouse")).toEqual(['"db"."orders"']) + }) + test("a hyphenated BigQuery project id is read after a comma and `insert into table` names its table", () => { + expect(raws("select * from a, my-proj.ds.t", "bigquery")).toEqual(["a", "my-proj.ds.t"]) + expect(raws("insert into table db.t select 1", "databricks")).toEqual(["db.t"]) + }) + test("clause words inside quoted names, strings and dollar quotes are not clauses", () => { + expect(raws('select "from orders", "join orders" as "using orders" from db.right.orders')).toEqual([ + "db.right.orders", + ]) + expect(raws("select $$from orders $$ as s from db.right.orders")).toEqual(["db.right.orders"]) + expect(raws("select $q1$from orders $q1$ as s from db.right.orders", "postgres")).toEqual(["db.right.orders"]) + expect(raws('select "from orders", r"from orders " from db.right.orders', "bigquery")).toEqual(["db.right.orders"]) + expect(raws('select "from orders" from db.right.orders', "databricks")).toEqual(["db.right.orders"]) + }) + test("EXTRACT, TRIM and IS DISTINCT FROM are not table clauses", () => { + expect(raws("select extract(year from d), trim(both 'x' from s) from t where a is distinct from orders")).toEqual([ + "t", + ]) + }) + test("a backslash ends a string in dialects where it does not escape", () => { + expect(raws("select 'C:\\' as p from db.right.orders", "postgres")).toEqual(["db.right.orders"]) + expect(raws("select 'it\\'s' as p from db.right.orders", "snowflake")).toEqual(["db.right.orders"]) + }) + test("CTE names are collected, and a table named like a CTE keeps its qualifiers", () => { + const { refs, ctes } = scanTableReferences( + "with orders as (select * from db.raw.orders), b(x) as materialized (select 1) select * from orders", + "snowflake", + ) + // Snowflake folds an unquoted name to upper case, so that is the name a CTE is known by. + expect([...ctes.keys()]).toEqual(["ORDERS", "B"]) + expect(refs.map((r) => r.raw)).toEqual(["db.raw.orders", "orders"]) + }) +}) + +describe("matchParts", () => { + const rel = ["RAW", "JAFFLE", "CUSTOMERS"] + test("exact, case_insensitive, partial and no match", () => { + expect(matchParts(["RAW", "JAFFLE", "CUSTOMERS"], rel)).toBe("exact") + expect(matchParts(["raw", "jaffle", "customers"], rel)).toBe("case_insensitive") + expect(matchParts(["jaffle", "customers"], rel)).toBe("partial") + expect(matchParts(["customers"], rel)).toBe("partial") + expect(matchParts(["other", "jaffle", "customers"], rel)).toBeNull() + expect(matchParts(["x", "raw", "jaffle", "customers"], rel)).toBeNull() + }) +}) + +const MANIFEST = { + metadata: { adapter_type: "snowflake", project_name: "shop" }, + nodes: { + "model.shop.customers": { + resource_type: "model", + name: "customers", + package_name: "shop", + relation_name: '"ANALYTICS"."MARTS"."CUSTOMERS"', + config: { materialized: "table" }, + }, + "model.shop.stg_orders": { + resource_type: "model", + name: "stg_orders", + package_name: "shop", + relation_name: '"ANALYTICS"."STAGING"."STG_ORDERS"', + config: { materialized: "view" }, + }, + "model.shop.inlined": { + resource_type: "model", + name: "inlined", + package_name: "shop", + relation_name: null, + config: { materialized: "ephemeral" }, + }, + "model.utils.date_spine": { + resource_type: "model", + name: "date_spine", + package_name: "utils", + relation_name: '"ANALYTICS"."UTILS"."DATE_SPINE"', + config: { materialized: "table" }, + }, + "seed.shop.countries": { + resource_type: "seed", + name: "countries", + package_name: "shop", + relation_name: '"ANALYTICS"."SEEDS"."COUNTRIES"', + }, + "test.shop.not_null": { resource_type: "test", name: "not_null", relation_name: '"X"."Y"."Z"' }, + }, + sources: { + "source.shop.jaffle.customers": { + resource_type: "source", + source_name: "jaffle", + name: "customers", + relation_name: '"RAW"."JAFFLE"."CUSTOMERS"', + }, + "source.shop.jaffle.payments": { + resource_type: "source", + source_name: "jaffle", + name: "payments", + relation_name: '"RAW"."JAFFLE"."PAYMENTS"', + }, + }, +} + +describe("mapRelations", () => { + const relations = manifestRelations(MANIFEST) + const map = (sql: string, tables: string[], dialect = "snowflake") => mapRelations(tables, sql, relations, dialect) + + test("manifestRelations skips ephemeral models and non-relations, and names other packages' models", () => { + const names = relations.map((r) => r.uniqueId).sort() + expect(names).not.toContain("model.shop.inlined") + expect(names).not.toContain("test.shop.not_null") + expect(relations.find((r) => r.name === "date_spine")?.call).toBe("{{ ref('utils', 'date_spine') }}") + }) + + test("exact: every part written, same case", () => { + const [row] = map('select * from "RAW"."JAFFLE"."PAYMENTS"', ["PAYMENTS"]) + expect(row.match).toBe("exact") + expect(row.replaceWith).toBe("{{ source('jaffle', 'payments') }}") + expect(row.raw).toBe('"RAW"."JAFFLE"."PAYMENTS"') + expect(row.candidates).toHaveLength(1) + }) + + test("case_insensitive: every part written, different case", () => { + const [row] = map("select * from raw.jaffle.payments", ["payments"]) + expect(row.match).toBe("case_insensitive") + expect(row.replaceWith).toBe("{{ source('jaffle', 'payments') }}") + }) + + test("partial: a missing database or schema, a model preferred over a source of the same name", () => { + const [row] = map("select * from customers", ["customers"]) + expect(row.match).toBe("partial") + expect(row.replaceWith).toBe("{{ ref('customers') }}") + expect(row.candidates.map((c) => c.kind)).toEqual(["model", "source"]) + }) + + test("a written schema narrows the match to one relation", () => { + const [row] = map("select * from jaffle.customers", ["customers"]) + expect(row.match).toBe("partial") + expect(row.replaceWith).toBe("{{ source('jaffle', 'customers') }}") + expect(row.candidates).toHaveLength(1) + }) + + test("a fully written name picks the source, not the model that shares its last part", () => { + const [row] = map("select * from raw.jaffle.customers", ["customers"]) + expect(row.match).toBe("case_insensitive") + expect(row.replaceWith).toBe("{{ source('jaffle', 'customers') }}") + }) + + test("a better match beats a preferred kind", () => { + const [row] = map('select * from "RAW"."JAFFLE"."CUSTOMERS"', ["CUSTOMERS"]) + expect(row.match).toBe("exact") + expect(row.replaceWith).toBe("{{ source('jaffle', 'customers') }}") + expect(row.candidates).toHaveLength(1) + }) + + test("none: keeps the table as written, no candidates", () => { + const [row] = map("select * from legacy.orders_v1", ["orders_v1"]) + expect(row).toEqual({ raw: "legacy.orders_v1", replaceWith: "legacy.orders_v1", match: "none", candidates: [] }) + }) + + test("a seed is matched with ref()", () => { + const [row] = map("select * from analytics.seeds.countries", ["countries"]) + expect(row.replaceWith).toBe("{{ ref('countries') }}") + expect(row.candidates[0].kind).toBe("seed") + }) + + test("tables that share a last part in different schemas stay separate", () => { + const rows = map("select * from raw.jaffle.customers a join analytics.marts.customers b on 1=1", ["customers"]) + expect(rows.map((r) => [r.raw, r.replaceWith, r.match])).toEqual([ + ["raw.jaffle.customers", "{{ source('jaffle', 'customers') }}", "case_insensitive"], + ["analytics.marts.customers", "{{ ref('customers') }}", "case_insensitive"], + ]) + }) + + test("BigQuery: a backtick path in one piece is split into its parts", () => { + const bq = manifestRelations({ + metadata: { adapter_type: "bigquery", project_name: "shop" }, + nodes: {}, + sources: { + "source.shop.raw.orders": { + resource_type: "source", + source_name: "raw", + name: "orders", + relation_name: "`proj`.`ds`.`orders`", + }, + }, + }) + const [row] = mapRelations(["proj.ds.orders"], "select * from `proj.ds.orders`", bq, "bigquery") + expect(row.match).toBe("exact") + expect(row.replaceWith).toBe("{{ source('raw', 'orders') }}") + }) + + test("a table the text scan does not find is matched by its bare name", () => { + const [row] = map("select 1", ["stg_orders"]) + expect(row.match).toBe("partial") + expect(row.replaceWith).toBe("{{ ref('stg_orders') }}") + }) + + test("a CTE named like a table is not mapped, and the physical table it reads keeps its mapping", () => { + const sql = "with customers as (select * from raw.jaffle.customers) select * from customers" + expect(map(sql, []).map((r) => [r.raw, r.match])).toEqual([["raw.jaffle.customers", "case_insensitive"]]) + expect(map(sql, ["customers"]).map((r) => [r.raw, r.match])).toEqual([["raw.jaffle.customers", "case_insensitive"]]) + }) + + test("a table named like its own CTE, read inside that CTE, is the real table; a recursive CTE's own name is not", () => { + const sql = "with orders as (select * from orders) select * from orders" + expect(mapRelations([], sql, [], "postgres").map((r) => r.raw)).toEqual(["orders"]) + expect( + map("with recursive customers as (select 1 union all select * from customers) select * from customers", []), + ).toEqual([]) + }) + + test("differently written forms of one table stay separate rows; an identical repeat does not", () => { + const sql = + 'select * from raw.jaffle.payments a join "RAW"."JAFFLE"."PAYMENTS" b on 1=1 join raw.jaffle.payments c on 1=1' + expect(map(sql, ["PAYMENTS"]).map((r) => r.raw)).toEqual(["raw.jaffle.payments", '"RAW"."JAFFLE"."PAYMENTS"']) + }) + + test("a versioned model is referenced by its version", () => { + const versioned = manifestRelations({ + metadata: { project_name: "shop" }, + nodes: { + "model.shop.orders.v1": { + resource_type: "model", + name: "orders", + package_name: "shop", + version: 1, + relation_name: '"A"."B"."ORDERS_V1"', + config: {}, + }, + "model.shop.orders.v2": { + resource_type: "model", + name: "orders", + package_name: "shop", + version: "2", + relation_name: '"A"."B"."ORDERS"', + config: {}, + }, + "model.pkg.dim.v3": { + resource_type: "model", + name: "dim", + package_name: "pkg", + version: 3, + relation_name: '"A"."B"."DIM"', + config: {}, + }, + }, + }) + const rows = mapRelations( + ["ORDERS_V1", "ORDERS", "DIM"], + 'select * from "A"."B"."ORDERS_V1", "A"."B"."ORDERS", "A"."B"."DIM"', + versioned, + "snowflake", + ) + expect(rows.map((r) => r.replaceWith)).toEqual([ + "{{ ref('orders', v=1) }}", + "{{ ref('orders', v=\"2\") }}", + "{{ ref('pkg', 'dim', v=3) }}", + ]) + }) + + test("a name altimate-core did not report is not a table", () => { + expect(map("with customers as (select 1) select * from customers", [])).toEqual([]) + }) +}) + +const ctx: Tool.Context = { + sessionID: SessionID.make("ses_map_relations"), + messageID: MessageID.make("msg_map_relations"), + agent: "analyst", + abort: new AbortController().signal, + messages: [], + metadata: () => Effect.void, + ask: () => Effect.void, + extra: { forceReadOnly: true }, +} + +describe("dbt_map_relations tool", () => { + let dirs: string[] = [] + afterEach(() => { + for (const d of dirs) rmSync(d, { recursive: true, force: true }) + dirs = [] + }) + function manifestPath(manifest: unknown = MANIFEST): string { + const dir = mkdtempSync(join(tmpdir(), "map-relations-")) + dirs.push(dir) + const p = join(dir, "manifest.json") + writeFileSync(p, JSON.stringify(manifest)) + return p + } + /** Runs `fn` with a temp dir as the working directory and HOME and no DBT_TARGET_PATH, all restored after; a dbt project with a manifest is written there when `manifest` is given. */ + async function inTempProject(manifest: unknown | null, fn: (dir: string) => Promise): Promise { + const dir = realpathSync(mkdtempSync(join(tmpdir(), "map-relations-project-"))) + dirs.push(dir) + if (manifest) { + writeFileSync(join(dir, "dbt_project.yml"), "name: jaffle\n") + mkdirSync(join(dir, "target")) + writeFileSync(join(dir, "target", "manifest.json"), JSON.stringify(manifest)) + } + const cwd = process.cwd() + const home = process.env.HOME + const target = process.env.DBT_TARGET_PATH + delete process.env.DBT_TARGET_PATH + process.env.HOME = dir + process.chdir(dir) + try { + return await fn(dir) + } finally { + process.chdir(cwd) + if (home === undefined) delete process.env.HOME + else process.env.HOME = home + if (target === undefined) delete process.env.DBT_TARGET_PATH + else process.env.DBT_TARGET_PATH = target + } + } + + async function run(args: Record) { + const tool = await initTool(DbtMapRelationsTool) + return tool.execute(args, ctx) + } + + test("maps every table, takes the dialect from the manifest, returns rows in metadata", async () => { + const result = await run({ + sql: 'select * from raw.jaffle.customers c join "ANALYTICS"."STAGING"."STG_ORDERS" o on 1=1 join legacy.thing t on 1=1 join payments p on 1=1', + manifest_path: manifestPath(), + }) + expect(result.metadata.success).toBe(true) + expect(result.metadata.dialect).toBe("snowflake") + const rows = result.metadata.mappings as { raw: string; replaceWith: string; match: string }[] + expect(rows.map((r) => [r.raw, r.match])).toEqual([ + ["raw.jaffle.customers", "case_insensitive"], + ['"ANALYTICS"."STAGING"."STG_ORDERS"', "exact"], + ["legacy.thing", "none"], + ["payments", "partial"], + ]) + expect(result.output.split("\n")[0]).toBe("none: legacy.thing -> no model or source") + expect(result.title).toContain("1 unmatched") + }) + + test("a missing manifest is an error that names the path", async () => { + const result = await run({ sql: "select * from t", manifest_path: "/nonexistent/manifest.json" }) + expect(result.metadata.success).toBe(false) + expect(result.output).toContain("/nonexistent/manifest.json") + }) + + test("without manifest_path the dbt project around the working directory supplies target/manifest.json", async () => { + await inTempProject(MANIFEST, async (dir) => { + const result = await run({ sql: "select * from legacy.thing" }) + expect(result.metadata.success).toBe(true) + expect(result.metadata.manifest_path).toBe(join(dir, "target", "manifest.json")) + }) + }) + + test("project_dir names the project, and its manifest is found as dbt finds it: DBT_TARGET_PATH first, then a templated target-path at its env_var default", async () => { + const dir = realpathSync(mkdtempSync(join(tmpdir(), "map-relations-pd-"))) + dirs.push(dir) + writeFileSync(join(dir, "dbt_project.yml"), "name: jaffle\ntarget-path: \"{{ env_var('MAP_REL_TARGET', 'built') }}\"\n") + for (const folder of ["built", "ci_out"]) { + mkdirSync(join(dir, folder)) + writeFileSync(join(dir, folder, "manifest.json"), JSON.stringify(MANIFEST)) + } + const saved = { target: process.env.DBT_TARGET_PATH, own: process.env.MAP_REL_TARGET } + try { + delete process.env.DBT_TARGET_PATH + delete process.env.MAP_REL_TARGET + expect((await run({ sql: "select * from legacy.thing", project_dir: dir })).metadata.manifest_path).toBe(join(dir, "built", "manifest.json")) + process.env.MAP_REL_TARGET = "ci_out" + expect((await run({ sql: "select * from legacy.thing", project_dir: dir })).metadata.manifest_path).toBe(join(dir, "ci_out", "manifest.json")) + process.env.DBT_TARGET_PATH = "built" + expect((await run({ sql: "select * from legacy.thing", project_dir: dir })).metadata.manifest_path).toBe(join(dir, "built", "manifest.json")) + } finally { + for (const [key, value] of [["DBT_TARGET_PATH", saved.target], ["MAP_REL_TARGET", saved.own]] as const) { + if (value === undefined) delete process.env[key] + else process.env[key] = value + } + } + }) + + test("an empty or blank project_dir is absent: the project around the working directory is found", async () => { + await inTempProject(MANIFEST, async (dir) => { + for (const blank of ["", " "]) { + const result = await run({ sql: "select * from legacy.thing", project_dir: blank }) + expect(result.metadata.success).toBe(true) + expect(result.metadata.manifest_path).toBe(join(dir, "target", "manifest.json")) + } + }) + }) + + test("outside a dbt project, without manifest_path, the error says so", async () => { + await inTempProject(null, async () => { + const result = await run({ sql: "select * from t" }) + expect(result.metadata.success).toBe(false) + expect(result.output).toBe("No dbt project found here. Run from a dbt project or pass manifest_path.") + }) + }) + + test("with the real parser, a table a CTE shares its name with and a dot spaced out are both mapped", async () => { + const result = await run({ + sql: "with customers as (select * from raw.jaffle.customers) select * from customers c join analytics . staging . stg_orders o on 1=1", + manifest_path: manifestPath(), + }) + const rows = result.metadata.mappings as { raw: string; match: string }[] + expect(rows.map((r) => [r.raw, r.match])).toEqual([ + ["raw.jaffle.customers", "case_insensitive"], + ["analytics . staging . stg_orders", "case_insensitive"], + ]) + }) + + test("SQL that does not parse is an error", async () => { + const result = await run({ sql: "select * from {{ ref('x') }}", manifest_path: manifestPath() }) + expect(result.metadata.success).toBe(false) + expect(result.output).toContain("Could not read the tables") + }) + + test("SQL without tables returns no rows", async () => { + const result = await run({ sql: "select 1", manifest_path: manifestPath() }) + expect(result.metadata.success).toBe(true) + expect(result.metadata.mappings).toEqual([]) + expect(result.output).toBe("No tables found in the SQL.") + }) +}) + +describe("review regressions", () => { + test("a name with a quote or backslash is escaped in the generated call", () => { + const rel = manifestRelations({ + nodes: { "model.p.o'x": { resource_type: "model", name: "o'x", schema: "s", database: "d", package_name: "p" } }, + sources: { "source.p.a.b": { source_name: "a\\b", name: "b'c", schema: "s", database: "d" } }, + }) + expect(rel.find((r) => r.kind === "model")?.call).toBe("{{ ref('o\\'x') }}") + expect(rel.find((r) => r.kind === "source")?.call).toBe("{{ source('a\\\\b', 'b\\'c') }}") + }) + + test("a dot inside backticks is split for BigQuery manifests only", () => { + const node = { resource_type: "model", name: "orders", relation_name: "`main`.`raw`.`orders.v1`", schema: "raw" } + const dbx = manifestRelations({ metadata: { adapter_type: "databricks" }, nodes: { "model.p.orders": node } }) + expect(dbx[0].parts).toEqual(["main", "raw", "orders.v1"]) + const bq = manifestRelations({ metadata: { adapter_type: "bigquery" }, nodes: { "model.p.orders": node } }) + expect(bq[0].parts).toEqual(["main", "raw", "orders", "v1"]) + }) + + test("an explicit dialect decides the backtick split, and a null manifest does not throw", () => { + const node = { resource_type: "model", name: "orders", relation_name: "`p.ds.orders`", schema: "ds" } + const rel = manifestRelations({ nodes: { "model.p.orders": node } }, "bigquery") + expect(rel[0].parts).toEqual(["p", "ds", "orders"]) + expect(manifestRelations(null)).toEqual([]) + }) + + test("the table list goes on after LATERAL", () => { + const { refs } = scanTableReferences( + "select * from db.one.orders a, lateral flatten(input => a.items) f, db.two.orders b", + "snowflake", + ) + expect(refs.map((r) => r.raw)).toEqual(["db.one.orders", "db.two.orders"]) + }) + + test("quoted names that differ only in inner whitespace are different tables", () => { + const rows = mapRelations(["A B", "A B"], 'select * from "A B", "A B"', [], "snowflake") + expect(rows.map((r) => r.raw)).toEqual(['"A B"', '"A B"']) + }) + test("a CTE that names itself is the recursion in Snowflake and T-SQL without the RECURSIVE keyword", () => { + const sql = "with c as (select 0 n union all select n + 1 from c where n < 5) select * from c" + for (const dialect of ["snowflake", "tsql"]) { + expect(mapRelations(["c"], sql, [], dialect)).toEqual([]) + } + expect(mapRelations(["c"], sql, [], "postgres").map((r) => r.raw)).toEqual(["c"]) + }) + + test("a model of the project outranks a same-named model of an installed package", () => { + const node = (pkg: string) => ({ + resource_type: "model", + name: "date_spine", + package_name: pkg, + relation_name: `"db"."s"."date_spine"`, + schema: "s", + }) + const manifest = { + metadata: { project_name: "shop" }, + nodes: { "model.dbt_utils.date_spine": node("dbt_utils"), "model.shop.date_spine": node("shop") }, + } + const rows = mapRelations(["date_spine"], "select * from date_spine", manifestRelations(manifest, "snowflake"), "snowflake") + expect(rows[0].replaceWith).toBe("{{ ref('date_spine') }}") + expect(rows[0].candidates.length).toBe(2) + expect(Object.keys(rows[0].candidates[0])).not.toContain("root") + }) + + test("# and // line comments are skipped where the dialect has them", () => { + const bq = scanTableReferences("select * from a.orders # don't remove\njoin a.payments", "bigquery") + expect(bq.refs.map((r) => r.raw)).toEqual(["a.orders", "a.payments"]) + const sf = scanTableReferences("select * from a.orders // it's here\njoin a.payments", "snowflake") + expect(sf.refs.map((r) => r.raw)).toEqual(["a.orders", "a.payments"]) + }) + + test("a backticked path and a bare name of the same table add no row the SQL cannot show", () => { + const rows = mapRelations(["orders", "proj.ds.orders"], "select * from orders o join `proj.ds.orders`", [], "bigquery") + expect(rows.map((r) => r.raw)).toEqual(["orders", "`proj.ds.orders`"]) + }) +}) + +describe("table lists after a join condition, a sample clause or ONLY", () => { + const raws = (sql: string, dialect = "snowflake") => scanTableReferences(sql, dialect).refs.map((r) => r.raw) + + test("a comma after a join's ON condition goes on with the list", () => { + expect(raws("select * from db.z.items i join db.a.orders a on i.id = a.id, db.z.orders z")).toEqual([ + "db.z.items", + "db.a.orders", + "db.z.orders", + ]) + expect(raws("select * from a join b on f(a.x, b.y) = 1, c where c.k = 1")).toEqual(["a", "b", "c"]) + expect(raws("select * from a join b using (id), c")).toEqual(["a", "b", "c"]) + }) + + test("a join condition ends at the next join or clause, not at a comma inside the select list", () => { + expect(raws("select * from a join b on a.id = b.id join c on c.id = b.id where a.x in (1, 2)")).toEqual(["a", "b", "c"]) + }) + + test("the first table of a parenthesized join is read; a subquery is not a table", () => { + expect(raws("select * from (raw.jaffle.a x join raw.jaffle.b y on x.id = y.id)")).toEqual(["raw.jaffle.a", "raw.jaffle.b"]) + expect(raws("select * from (select * from raw.s.t) q, raw.s.u")).toEqual(["raw.s.t", "raw.s.u"]) + }) + + test("a table after a parenthesized join is still read, however deep the group", () => { + expect(raws("select * from (a join b on a.id = b.id), c")).toEqual(["a", "b", "c"]) + expect(raws("select * from (a join b on a.id = b.id) j, c")).toEqual(["a", "b", "c"]) + expect(raws("select * from ((a join b on a.id = b.id) join c on c.id = a.id), d")).toEqual(["a", "b", "c", "d"]) + }) + + test("a group that starts with a subquery or ONLY, and a CASE in a join condition, do not cut the list", () => { + expect(raws("select * from ((select * from x) q join b on 1=1), c")).toEqual(["x", "b", "c"]) + expect(raws("select * from (only a join b on 1=1), c", "postgres")).toEqual(["a", "b", "c"]) + expect(raws("select * from a join b on case when a.x = 1 then 1 else 0 end = 1, c")).toEqual(["a", "b", "c"]) + }) + + test("a join condition does not run on into MERGE, ON CONFLICT or later statements", () => { + expect(raws("merge into t using s on t.id = s.id when matched then update set a = s.a, b = s.b", "snowflake")).toEqual(["t", "s"]) + expect(raws("insert into t select * from s on conflict (id) do update set a = excluded.a, b = excluded.b", "postgres")).toEqual(["t", "s"]) + expect(raws("select * from a join b on a.id = b.id; select x, y from c")).toEqual(["a", "b", "c"]) + }) + + test("ONLY is skipped for PostgreSQL only", () => { + expect(raws("select * from only join b on 1=1", "snowflake")).toEqual(["only", "b"]) + }) + + test("a sample clause does not end the list", () => { + expect(raws("select * from db.z.items tablesample (10), db.z.orders")).toEqual(["db.z.items", "db.z.orders"]) + expect(raws("select * from db.z.items i tablesample bernoulli (10) repeatable (3), db.z.orders", "postgres")).toEqual([ + "db.z.items", + "db.z.orders", + ]) + }) + + test("PostgreSQL's ONLY is not a table name", () => { + expect(raws("select * from only db.z.orders", "postgres")).toEqual(["db.z.orders"]) + expect(raws("select * from only orders o, only db.z.items", "postgres")).toEqual(["orders", "db.z.items"]) + }) + + test("a parenthesized join after a comma, a USING group and a bare SAMPLE clause keep every table", () => { + expect(raws("select * from s.items i, (one.orders a join two.orders b on a.id = b.id)")).toEqual(["s.items", "one.orders", "two.orders"]) + expect(raws("merge into t using (one.s a join two.s b on a.id = b.id) on 1=1")).toEqual(["t", "one.s", "two.s"]) + expect(raws("select * from db.z.items sample (10), db.z.orders")).toEqual(["db.z.items", "db.z.orders"]) + expect(raws("select * from db.z.items sample join db.z.orders on 1=1")).toEqual(["db.z.items", "db.z.orders"]) + }) + + test("ClickHouse comments, and the mssql and redshift spellings, follow their dialect", () => { + expect(raws("select * from a.orders # it's fine\n join a.payments", "clickhouse")).toEqual(["a.orders", "a.payments"]) + expect(raws("select * from a.orders // from legacy.x\n join a.payments", "clickhouse")).toEqual(["a.orders", "a.payments"]) + expect(mapRelations(["c"], "with c as (select 0 union all select n + 1 from c) select * from c", [], "mssql")).toEqual([]) + expect(raws("select * from [db].[s].[t]", "mssql")).toEqual(["[db].[s].[t]"]) + expect(mapRelations(["orders"], 'with "Orders" as (select 1) select * from orders', [], "redshift").map((r) => r.raw)).toEqual(["orders"]) + }) + + test("a CTE hides a table only inside its own statement", () => { + const rows = mapRelations(["orders"], "with orders as (select 1) select * from orders; select * from orders", [], "postgres") + expect(rows.map((r) => r.raw)).toEqual(["orders"]) + expect(mapRelations(["orders"], "with orders as (select 1) select * from orders", [], "postgres")).toEqual([]) + }) + + test("a CTE hides a table only after its own definition", () => { + expect(mapRelations(["orders"], "with a as (select * from orders), orders as (select 1) select * from a", [], "postgres").map((r) => r.raw)).toEqual(["orders"]) + expect(mapRelations(["orders"], "with recursive orders as (select 1 union all select * from orders) select * from orders", [], "postgres")).toEqual([]) + expect( + mapRelations(["orders"], "with recursive orders as (select 1 union all select * from orders) select 1; with orders as (select * from orders) select * from orders", [], "postgres").map((r) => r.raw), + ).toEqual(["orders"]) + }) + + test("an escape string, a parenthesized ONLY, an omitted schema and non-ASCII names are read as written", () => { + expect(raws("select E'it\\'s from wrong.orders' as s from real.orders", "postgres")).toEqual(["real.orders"]) + expect(raws("select * from only (raw.orders) a join other.orders b on true", "postgres")).toEqual(["raw.orders", "other.orders"]) + expect(raws("select * from db..orders a join other.orders b on 1=1", "mssql")).toEqual(["db..orders", "other.orders"]) + expect(raws("select * from public.café", "postgres")).toEqual(["public.café"]) + }) + + test("a recursive WITH sees every CTE of its list; MySQL index hints and STRAIGHT_JOIN keep their qualifiers", () => { + expect(mapRelations(["orders"], "with recursive a as (select * from orders), orders as (select 1) select * from a", [], "postgres")).toEqual([]) + expect(raws("select * from raw.items use index (idx_id), archive.orders", "mysql")).toEqual(["raw.items", "archive.orders"]) + expect(raws("select * from raw.items a force index for join (i) straight_join correct.orders b on a.id = b.id", "mysql")).toEqual(["raw.items", "correct.orders"]) + }) + + test("an unquoted name is matched as the dialect folds it", () => { + const rel = (name: string, id: string) => ({ parts: ["DB", "S", name], call: `{{ ref('${id}') }}`, kind: "model" as const, name: id, uniqueId: id, relationName: `"DB"."S"."${name}"`, root: true }) + const rows = mapRelations(["orders"], "select * from DB.S.orders", [rel("orders", "a_lower"), rel("ORDERS", "b_upper")], "snowflake") + expect(rows[0].replaceWith).toBe("{{ ref('b_upper') }}") + const mixed = mapRelations(["orders"], 'select * from "DB"."S".orders', [rel("orders", "a_lower"), rel("ORDERS", "b_upper")], "snowflake") + expect(mixed[0].replaceWith).toBe("{{ ref('b_upper') }}") + }) + + test("the target of INSERT OVERWRITE TABLE and CREATE TABLE keeps its qualifiers", () => { + expect(raws("insert overwrite table main.live.orders select * from main.raw.input", "databricks")).toEqual(["main.live.orders", "main.raw.input"]) + expect(raws("insert overwrite main.live.orders select * from main.raw.input", "databricks")).toEqual(["main.live.orders", "main.raw.input"]) + expect(raws("create or replace table main.live.orders as select * from main.raw.input", "snowflake")).toEqual(["main.live.orders", "main.raw.input"]) + expect(raws("create table if not exists main.live.orders as select * from main.raw.input", "postgres")).toEqual(["main.live.orders", "main.raw.input"]) + }) + + test("INSERT OVERWRITE skips DIRECTORY, LOCAL DIRECTORY and INTO, and only follows INSERT", () => { + expect(raws("insert overwrite directory '/tmp/x' using parquet select * from main.raw.input", "databricks")).toEqual(["main.raw.input"]) + expect(raws("insert overwrite local directory '/tmp/x' select * from main.raw.input", "databricks")).toEqual(["main.raw.input"]) + expect(raws("insert overwrite into main.live.orders select * from main.raw.input", "snowflake")).toEqual(["main.live.orders", "main.raw.input"]) + expect(raws("select * from main.raw.input order by overwrite desc", "databricks")).toEqual(["main.raw.input"]) + }) + + test("SELECT STRAIGHT_JOIN is an option and a bare STRAIGHT_JOIN joins a table", () => { + expect(raws("select straight_join orders, count(*) from db.orders", "mysql")).toEqual(["db.orders"]) + expect(raws("select distinct straight_join orders from db.orders", "mysql")).toEqual(["db.orders"]) + expect(raws("select * from db.one.orders o straight_join db.two.payments p on o.id = p.id", "mysql")).toEqual(["db.one.orders", "db.two.payments"]) + }) + + test("BigQuery hyphenated project ids survive in a CREATE TABLE target", () => { + expect(raws("create table my-proj.ds.orders as select * from my-proj.ds.input", "bigquery")).toEqual(["my-proj.ds.orders", "my-proj.ds.input"]) + expect(raws("create table if not exists my-proj.ds.orders as select 1", "bigquery")).toEqual(["my-proj.ds.orders"]) + }) + + test("a name with a combining mark stays one name", () => { + expect(raws("select * from raw.cafe\u0301", "snowflake")).toEqual(["raw.cafe\u0301"]) + }) + + test("E strings with an escaped quote hide nothing in DuckDB; block comments nest as sql-tokenize says", () => { + expect(raws("select E'it\\'s' as p from db.right.orders", "duckdb")).toEqual(["db.right.orders"]) + expect(raws("select * from db.a.orders /* outer /* nested */ from db.wrong.orders */", "spark")).toEqual(["db.a.orders"]) + expect(raws("select * from db.a.orders /* outer /* nested */ from db.wrong.orders */", "clickhouse")).toEqual(["db.a.orders"]) + }) + + test("Trino folds an unquoted name to lower case, so a quoted CTE of another case hides nothing", () => { + expect(mapRelations(["orders"], 'with "Orders" as (select 1) select * from orders', [], "trino").map((r) => r.raw)).toEqual(["orders"]) + }) + + test("T-SQL db..table matches the relation of any schema as a partial match", () => { + const rel = { parts: ["db", "dbo", "orders"], call: "{{ ref('orders') }}", kind: "model" as const, name: "orders", uniqueId: "model.p.orders", relationName: "db.dbo.orders", root: true } + const rows = mapRelations(["orders"], "select * from db..orders", [rel], "mssql") + expect(rows.map((r) => [r.raw, r.match])).toEqual([["db..orders", "partial"]]) + expect(matchParts(["db", "", "orders"], ["other", "dbo", "orders"])).toBeNull() + }) + + test("an unquoted manifest relation outranks a quoted one of another case when the match is otherwise the same", () => { + const rel = (relationName: string, parts: string[], id: string) => ({ parts, call: `{{ ref('${id}') }}`, kind: "model" as const, name: id, uniqueId: id, relationName, root: true }) + // Both are a case-insensitive match for `db.s.orders`; Snowflake reads that as ORDERS, which only the unquoted relation is. + const rows = mapRelations( + ["orders"], + "select * from db.s.orders", + [rel('"DB"."S"."orders"', ["DB", "S", "orders"], "a_quoted_lower"), rel("DB.S.orders", ["DB", "S", "orders"], "b_unquoted")], + "snowflake", + ) + expect(rows[0].replaceWith).toBe("{{ ref('b_unquoted') }}") + }) + + test("T-SQL table hints and PIVOT do not end the list", () => { + expect(raws("select * from db.a.orders with (nolock), db.z.orders", "mssql")).toEqual(["db.a.orders", "db.z.orders"]) + expect(raws("select * from db.a.orders pivot (sum(x) for y in ([a])) p, db.z.orders", "mssql")).toEqual(["db.a.orders", "db.z.orders"]) + }) + + test("block comments nest where the dialect says so", () => { + expect(raws("select * from db.a.orders /* outer /* nested */ from db.wrong.orders */", "postgres")).toEqual(["db.a.orders"]) + expect(raws("select /* a /* b */ from fake.orders */ * from real.orders", "mssql")).toEqual(["real.orders"]) + expect(raws("select * from a.orders /* x /* y */ join a.payments", "mysql")).toEqual(["a.orders", "a.payments"]) + }) + + test("each of them reaches the mapped rows with the schema as written", () => { + const relations = manifestRelations(MANIFEST) + const rows = mapRelations( + ["items", "orders"], + "select * from db.z.items i join db.a.orders a on i.id = a.id, db.z.orders z", + relations, + "snowflake", + ) + expect(rows.map((r) => r.raw)).toEqual(["db.z.items", "db.a.orders", "db.z.orders"]) + }) +}) + +describe("a quoted CTE name and a table that differs only in case", () => { + const rows = (sql: string, dialect: string) => mapRelations(["orders"], sql, [], dialect).map((r) => r.raw) + + test("Snowflake: a quoted lower-case CTE is not the unquoted table", () => { + expect(rows('with "orders" as (select 1) select * from ORDERS', "snowflake")).toEqual(["ORDERS"]) + expect(rows('with "orders" as (select 1) select * from "orders"', "snowflake")).toEqual([]) + expect(rows("with orders as (select 1) select * from ORDERS", "snowflake")).toEqual([]) + }) + + test("PostgreSQL: a quoted upper-case CTE is not the unquoted table", () => { + expect(rows('with "ORDERS" as (select 1) select * from orders', "postgres")).toEqual(["orders"]) + expect(rows("with orders as (select 1) select * from ORDERS", "postgres")).toEqual([]) + }) + + test("with the real parser: a quoted CTE name beside a physical table of another case", async () => { + // altimate-core reports the table list the mapper starts from; the physical table must survive both steps. + const sql = 'with "orders" as (select * from raw."orders") select * from ORDERS o join "orders" x on 1=1' + const tables = await tablesOf(sql, "snowflake") + expect(mapRelations(tables, sql, [], "snowflake").map((r) => r.raw)).toEqual(expect.arrayContaining(["ORDERS"])) + }) + + test("elsewhere quoting does not change the case a name is compared in", () => { + expect(rows("with `Orders` as (select 1) select * from orders", "databricks")).toEqual([]) + expect(rows("with [Orders] as (select 1) select * from orders", "tsql")).toEqual([]) + }) +})