From 0cab4069bd922633f9847036b755ee53eccbf468 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Thu, 19 Nov 2026 17:43:18 +0530 Subject: [PATCH 1/8] fix(dbt-cost-estimate): count IDENTIFIER() as a possible file read on Databricks scanSql blanks quoted text, so a source named inside IDENTIFIER('...') never reached the read_files / cloud_files / format.`path` checks and EXPLAIN COST was sent while Databricks read files to plan it. A lone string literal in IDENTIFIER() is now read as code; any other argument (parameter, variable, concatenation, adjacent literals, escapes) means no estimate. Co-Authored-By: Claude Sonnet 5.5 --- .../src/altimate/tools/dbt-cost-estimate.ts | 37 +++++++++++--- .../test/altimate/dbt-cost-estimate.test.ts | 48 ++++++++++++++++++- 2 files changed, 78 insertions(+), 7 deletions(-) diff --git a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts index b9b3b21b68..3de7970845 100644 --- a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts +++ b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts @@ -170,8 +170,23 @@ export function parseDatabricksExplainCost(plan: unknown): PlanEstimate | { noSt * does not read counts. */ export function readsFiles(sql: string): boolean { - const { script, code } = scanSql(sql, "databricks") + const { script, code, literals } = scanSql(sql, "databricks") if (script) return true + if (fileSource(code)) return true + // IDENTIFIER(expr) names a function or relation from a string, which the scan blanked: read a lone string literal as + // code, and count any other argument (a variable, :param, concatenation, adjacent literals, escapes) as a read. + const calls = code.match(/(^|[^a-z0-9_.$`])identifier\s*\(/gi)?.length ?? 0 + let plain = 0 + for (const lit of literals) { + plain++ + if (lit.text.includes("\\")) return true + if (/(^|[^a-z0-9_])`?(read_files|cloud_files)`?\s*$/i.test(lit.text) || fileSource(lit.text)) return true + } + return calls > plain +} + +/** Whether code (as `scanSql` leaves it) names a file source: a read_files or cloud_files call, a format or path over backticks. */ +function fileSource(code: string): boolean { // A function call, qualified or not (system.builtin.read_files). if (/(^|[^a-z0-9_])`?(read_files|cloud_files)`?\s*\(/i.test(code)) return true // A known format over any backtick-quoted name, unless the format word is the middle of a dotted name (main.json.`t`). @@ -222,16 +237,19 @@ export function mayBeScript(input: string, type: string): boolean { /** * The one reading of the SQL both checks share: whether it may be a script (see `mayBeScript`), and its code with - * comments and quoted text as spaces and Databricks' backtick-quoted names kept as written. + * comments and quoted text as spaces and Databricks' backtick-quoted names kept as written, and (Databricks) the lone + * string literals an IDENTIFIER( call holds. */ -export function scanSql(input: string, type: string): { script: boolean; code: string } { - const script = { script: true, code: "" } +export function scanSql(input: string, type: string): { script: boolean; code: string; literals: { text: string }[] } { + const script = { script: true, code: "", literals: [] } // Read with the line breaks Jinja renders: every \r\n and lone \r is \n. const sql = input.replace(/\r\n?/g, "\n") const databricks = type === "databricks" // The Snowflake splitter yields a line starting with "!" as a command of its own, wherever it sits: never read. if (!databricks && /(^|\n)!/.test(sql)) return script let code = "" + // Databricks: the text of each string literal that opens an IDENTIFIER( call, and where it sits in `code`. + const literals: { text: string }[] = [] // Past a semicolon only whitespace and comments may follow: anything else, quoted or not, is a second statement. let ended = false let i = 0 @@ -269,15 +287,22 @@ export function scanSql(input: string, type: string): { script: boolean; code: s let j = i + 1 while (j < sql.length && sql[j] !== close) j += escapes && sql[j] === "\\" ? 2 : 1 if (j >= sql.length) return script + let blank = " " + if (databricks && c !== "`" && /(^|[^a-z0-9_.$`])identifier\s*\(\s*$/i.test(code)) { + // Marked, so a second literal after it does not read as the call's first. + blank = "\u0000" + // Only a literal the call's parenthesis closes is plain: a following literal, operator or name is not. + if (/^(?:\s|--[^\n]*(?:\n|$)|\/\*[\s\S]*?\*\/)*\)/.test(sql.slice(j + 1))) literals.push({ text: sql.slice(i + 1, j) }) + } // A backtick-quoted name is code; quoted text is not. - code += c === "`" ? sql.slice(i, j + 1) : " " + code += c === "`" ? sql.slice(i, j + 1) : blank i = j + 1 continue } code += c i++ } - return { script: /\bexecute\s+immediate\b/i.test(code), code } + return { script: /\bexecute\s+immediate\b/i.test(code), code, literals } } /** Starts `start` and waits for it, but gives up as soon as the call is cancelled; a cancelled call starts nothing. */ diff --git a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts index a2b54ca496..1751c6b98f 100644 --- a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts +++ b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts @@ -885,7 +885,7 @@ describe("review follow-ups, round eight", () => { }) test("both checks read the SQL the same way: one scan", () => { - expect(scanSql("select `json`.`x` from t -- c\n", "databricks")).toEqual({ script: false, code: "select `json`.`x` from t " }) + expect(scanSql("select `json`.`x` from t -- c\n", "databricks")).toEqual({ script: false, code: "select `json`.`x` from t ", literals: [] }) expect(scanSql("select 'a' from t", "databricks").code).toBe("select from t") }) }) @@ -995,3 +995,49 @@ describe("review follow-ups, round twelve", () => { } }) }) + +describe("IDENTIFIER() on Databricks", () => { + test.each([ + "SELECT * FROM IDENTIFIER('read_files')('/Volumes/c/s/v/raw', format => 'json')", + "SELECT * FROM IDENTIFIER('system.builtin.read_files')('s3://b/x')", + "SELECT * FROM IDENTIFIER('json.`/Volumes/c/s/v/raw`')", + "select * from identifier ( 'READ_FILES' ) ('/x')", + "select * from IDENTIFIER(\"Cloud_Files\")('/x')", + "select * from IDENTIFIER('system . builtin . `read_files`')('/x')", + "select * from IDENTIFIER /* c */ ( /* d */ 'CSV.`/x`')", + "select * from IDENTIFIER('`parquet`.`s3://b/x`')", + "select * from IDENTIFIER('read' || '_files')('/x')", + "select * from IDENTIFIER('read_' '' 'files')('/x')", + "select * from IDENTIFIER(:tbl)", + "select * from IDENTIFIER(concat('json.`', :p, '`'))", + "select * from IDENTIFIER(tbl_var)", + "select * from IDENTIFIER('cat.sch.tbl') a join IDENTIFIER(:other) b on a.id = b.id", + "select * from IDENTIFIER('read\\_files')('/x')", + ])("%p may read files", (sql) => { + expect(readsFiles(sql)).toBe(true) + }) + + test.each([ + "select * from IDENTIFIER('cat.sch.tbl')", + "select * from identifier ( \"cat.sch.tbl\" ) t where t.x = 'read_files'", + "select IDENTIFIER('col') from main.sales.orders", + "select * from IDENTIFIER('main.json.t')", + "select * from `identifier`('x')", + ])("%p reads tables", (sql) => { + expect(readsFiles(sql)).toBe(false) + }) + + test.each([ + ["SELECT * FROM IDENTIFIER('system.builtin.read_files')('s3://b/x')", false], + ["SELECT * FROM IDENTIFIER(:tbl)", false], + ["SELECT * FROM IDENTIFIER('cat.sch.tbl')", true], + ])("%p: EXPLAIN sent is %p", async (compiled, explained) => { + useAdapter(calls, { type: "databricks", compile: async () => compiled, explain: async () => JSON.stringify([DATABRICKS_PLAN]) }) + await (await initTool(DbtCostEstimateTool)).execute({ model: "orders" }, ctx) + expect(calls.queries.length > 0).toBe(explained) + }) + + test("leaves Snowflake and BigQuery alone", async () => { + expect(scanSql("select * from IDENTIFIER('cat.sch.tbl')", "snowflake").literals).toEqual([]) + }) +}) From 9559b8386c5c6fa8e8771ee13a29262ee658633b Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Thu, 19 Nov 2026 17:48:29 +0530 Subject: [PATCH 2/8] fix(dbt-cost-estimate): read nested, qualified and commented IDENTIFIER() names conservatively Co-Authored-By: Claude Sonnet 5.5 --- .../opencode/src/altimate/tools/dbt-cost-estimate.ts | 12 +++++++++--- .../opencode/test/altimate/dbt-cost-estimate.test.ts | 11 +++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts index 3de7970845..46922544dc 100644 --- a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts +++ b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts @@ -175,11 +175,17 @@ export function readsFiles(sql: string): boolean { if (fileSource(code)) return true // IDENTIFIER(expr) names a function or relation from a string, which the scan blanked: read a lone string literal as // code, and count any other argument (a variable, :param, concatenation, adjacent literals, escapes) as a read. - const calls = code.match(/(^|[^a-z0-9_.$`])identifier\s*\(/gi)?.length ?? 0 + // Counted outside backtick-quoted names, and with no boundary consumed, so IDENTIFIER(IDENTIFIER('f')('x')) is two. + const bare = code.replace(/`[^`]*`/g, " ") + const calls = bare.match(/(? plain @@ -288,7 +294,7 @@ export function scanSql(input: string, type: string): { script: boolean; code: s while (j < sql.length && sql[j] !== close) j += escapes && sql[j] === "\\" ? 2 : 1 if (j >= sql.length) return script let blank = " " - if (databricks && c !== "`" && /(^|[^a-z0-9_.$`])identifier\s*\(\s*$/i.test(code)) { + if (databricks && c !== "`" && /(? { "select * from IDENTIFIER(tbl_var)", "select * from IDENTIFIER('cat.sch.tbl') a join IDENTIFIER(:other) b on a.id = b.id", "select * from IDENTIFIER('read\\_files')('/x')", + "SELECT * FROM IDENTIFIER(IDENTIFIER('lower')('READ_FILES'))('/Volumes/c/s/v/raw')", + "SELECT * FROM IDENTIFIER( IDENTIFIER('concat')('read_', 'files'))('/Volumes/c/s/v/raw')", + "SELECT * FROM IDENTIFIER('read_files /* c */')('/Volumes/c/s/v/raw')", + "SELECT * FROM IDENTIFIER('read_files/**/')('/Volumes/c/s/v/raw')", + "SELECT * FROM IDENTIFIER('json/**/.`/Volumes/c/s/v/raw`')", + "SELECT * FROM IDENTIFIER('read_files -- c\n')('/x')", + "SELECT * FROM system.builtin.IDENTIFIER('read_files')('/Volumes/c/s/v/raw')", + "SELECT * FROM json.IDENTIFIER('`/Volumes/c/s/v/raw`')", + "SELECT * FROM IDENTIFIER('json').`data.json`", + "SELECT * FROM IDENTIFIER('json') . `data`", ])("%p may read files", (sql) => { expect(readsFiles(sql)).toBe(true) }) @@ -1023,6 +1033,7 @@ describe("IDENTIFIER() on Databricks", () => { "select IDENTIFIER('col') from main.sales.orders", "select * from IDENTIFIER('main.json.t')", "select * from `identifier`('x')", + "SELECT `note identifier(:x)` FROM main.sales.orders", ])("%p reads tables", (sql) => { expect(readsFiles(sql)).toBe(false) }) From b0986906c523584a21475c080531d1653738d322 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Thu, 19 Nov 2026 17:48:39 +0530 Subject: [PATCH 3/8] test(dbt-cost-estimate): quoted column named like an IDENTIFIER call Co-Authored-By: Claude Sonnet 5.5 --- packages/opencode/test/altimate/dbt-cost-estimate.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts index 09585e83c5..1a3bb02e35 100644 --- a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts +++ b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts @@ -1033,7 +1033,7 @@ describe("IDENTIFIER() on Databricks", () => { "select IDENTIFIER('col') from main.sales.orders", "select * from IDENTIFIER('main.json.t')", "select * from `identifier`('x')", - "SELECT `note identifier(:x)` FROM main.sales.orders", + "SELECT `note identifier(x)` FROM main.sales.orders", ])("%p reads tables", (sql) => { expect(readsFiles(sql)).toBe(false) }) From c305810b6452679588991590b56dc35b9642d260 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Thu, 19 Nov 2026 18:04:13 +0530 Subject: [PATCH 4/8] fix(dbt-cost-estimate): refuse double-quoted, keyword-folded and nested IDENTIFIER() names, and scan literals in linear time Co-Authored-By: Claude Sonnet 5.5 --- .../src/altimate/tools/dbt-cost-estimate.ts | 24 ++++++++++++++----- .../test/altimate/dbt-cost-estimate.test.ts | 14 ++++++++++- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts index 46922544dc..21f35554e4 100644 --- a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts +++ b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts @@ -176,16 +176,18 @@ export function readsFiles(sql: string): boolean { // IDENTIFIER(expr) names a function or relation from a string, which the scan blanked: read a lone string literal as // code, and count any other argument (a variable, :param, concatenation, adjacent literals, escapes) as a read. // Counted outside backtick-quoted names, and with no boundary consumed, so IDENTIFIER(IDENTIFIER('f')('x')) is two. - const bare = code.replace(/`[^`]*`/g, " ") - const calls = bare.match(/(? plain @@ -294,11 +296,12 @@ export function scanSql(input: string, type: string): { script: boolean; code: s while (j < sql.length && sql[j] !== close) j += escapes && sql[j] === "\\" ? 2 : 1 if (j >= sql.length) return script let blank = " " - if (databricks && c !== "`" && /(? 0 && /\s/.test(code[end - 1])) end-- + const start = Math.max(0, end - 80) + const at = code.slice(start, end).toUpperCase().search(/IDENTIFIER\s*\($/) + return at >= 0 && (start + at === 0 || !/[A-Za-z0-9_$]/.test(code[start + at - 1])) +} + /** Starts `start` and waits for it, but gives up as soon as the call is cancelled; a cancelled call starts nothing. */ export function untilAbort(start: () => Promise, signal: AbortSignal): Promise { if (signal.aborted) return Promise.reject(signal.reason) diff --git a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts index 1a3bb02e35..e4279c0281 100644 --- a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts +++ b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts @@ -1023,13 +1023,18 @@ describe("IDENTIFIER() on Databricks", () => { "SELECT * FROM json.IDENTIFIER('`/Volumes/c/s/v/raw`')", "SELECT * FROM IDENTIFIER('json').`data.json`", "SELECT * FROM IDENTIFIER('json') . `data`", + "SELECT * FROM IDENTIFIER('\"read_files\"')('/Volumes/c/s/v/raw', format => 'json')", + "SELECT * FROM IDENTIFIER(\"fn\")('/Volumes/c/s/v/raw')", + "SELECT * FROM \u0131DENTIFIER('read_files')('/Volumes/c/s/v/raw')", + "SELECT * FROM IDENTIFIER(\"IDENTIFIER('read_files')\")('/Volumes/c/s/v/raw')", + "SELECT * FROM IDENTIFIER('identifier(''x'')')('/x')", ])("%p may read files", (sql) => { expect(readsFiles(sql)).toBe(true) }) test.each([ "select * from IDENTIFIER('cat.sch.tbl')", - "select * from identifier ( \"cat.sch.tbl\" ) t where t.x = 'read_files'", + "select * from identifier ( 'cat.sch.tbl' ) t where t.x = 'read_files'", "select IDENTIFIER('col') from main.sales.orders", "select * from IDENTIFIER('main.json.t')", "select * from `identifier`('x')", @@ -1048,6 +1053,13 @@ describe("IDENTIFIER() on Databricks", () => { expect(calls.queries.length > 0).toBe(explained) }) + test("many ordinary literals scan in linear time", () => { + const sql = `select ${Array.from({ length: 20000 }, (_, n) => `'v${n}'`).join(", ")} from main.s.t` + const started = performance.now() + expect(readsFiles(sql)).toBe(false) + expect(performance.now() - started).toBeLessThan(1000) + }) + test("leaves Snowflake and BigQuery alone", async () => { expect(scanSql("select * from IDENTIFIER('cat.sch.tbl')", "snowflake").literals).toEqual([]) }) From 2888cdb7d92e47474e1925bd3d72ea4157eb482c Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Thu, 19 Nov 2026 18:22:40 +0530 Subject: [PATCH 5/8] fix(dbt-cost-estimate): scan IDENTIFIER() arguments in one pass, with no backtracking or case-folded offsets Co-Authored-By: Claude Sonnet 5.5 --- .../src/altimate/tools/dbt-cost-estimate.ts | 29 ++++++++++++++++--- .../test/altimate/dbt-cost-estimate.test.ts | 12 ++++++++ 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts index 21f35554e4..07197dc72e 100644 --- a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts +++ b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts @@ -301,7 +301,7 @@ export function scanSql(input: string, type: string): { script: boolean; code: s // Marked, so a second literal after it does not read as the call's first. blank = "\u0000" // Only a literal the call's parenthesis closes is plain: a following literal, operator or name is not. - if (c === "'" && /^(?:\s|--[^\n]*(?:\n|$)|\/\*[\s\S]*?\*\/)*\)/.test(sql.slice(j + 1))) literals.push({ text: sql.slice(i + 1, j) }) + if (c === "'" && closesCall(sql, j + 1)) literals.push({ text: sql.slice(i + 1, j) }) } // A backtick-quoted name is code; quoted text is not. code += c === "`" ? sql.slice(i, j + 1) : blank @@ -318,9 +318,30 @@ export function scanSql(input: string, type: string): { script: boolean; code: s function opensIdentifier(code: string): boolean { let end = code.length while (end > 0 && /\s/.test(code[end - 1])) end-- - const start = Math.max(0, end - 80) - const at = code.slice(start, end).toUpperCase().search(/IDENTIFIER\s*\($/) - return at >= 0 && (start + at === 0 || !/[A-Za-z0-9_$]/.test(code[start + at - 1])) + if (code[end - 1] !== "(") return false + end-- + while (end > 0 && /\s/.test(code[end - 1])) end-- + // Upper-cased as Spark's lexer folds keywords (a dotless ı spells IDENTIFIER too). + if (end < 10 || code.slice(end - 10, end).toUpperCase() !== "IDENTIFIER") return false + return end === 10 || !/[A-Za-z0-9_$]/.test(code[end - 11]) +} + +/** Whether, past whitespace and comments, the next character of `sql` from `at` is `)`. One pass, no backtracking. */ +function closesCall(sql: string, at: number): boolean { + let i = at + while (i < sql.length) { + if (/\s/.test(sql[i])) i++ + else if (sql.startsWith("--", i)) { + const end = sql.indexOf("\n", i) + if (end < 0) return false + i = end + 1 + } else if (sql.startsWith("/*", i)) { + const end = sql.indexOf("*/", i + 2) + if (end < 0) return false + i = end + 2 + } else return sql[i] === ")" + } + return false } /** Starts `start` and waits for it, but gives up as soon as the call is cancelled; a cancelled call starts nothing. */ diff --git a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts index e4279c0281..35ea385c61 100644 --- a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts +++ b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts @@ -1028,6 +1028,9 @@ describe("IDENTIFIER() on Databricks", () => { "SELECT * FROM \u0131DENTIFIER('read_files')('/Volumes/c/s/v/raw')", "SELECT * FROM IDENTIFIER(\"IDENTIFIER('read_files')\")('/Volumes/c/s/v/raw')", "SELECT * FROM IDENTIFIER('identifier(''x'')')('/x')", + "SELECT * FROM IDENTIFIER('read_' /*a*/ || 'files' /*b*/)('/Volumes/c/s/v/raw', format => 'json')", + "SELECT * FROM IDENTIFIER('js' /*a*/ || 'on.`/Volumes/c/s/v/raw`' /*b*/)", + "SELECT 1 AS `\u00df\u00df\u00df`, XIDENTIFIER('safe') AS n FROM IDENTIFIER(concat('read_', 'files'))('/x')", ])("%p may read files", (sql) => { expect(readsFiles(sql)).toBe(true) }) @@ -1039,6 +1042,8 @@ describe("IDENTIFIER() on Databricks", () => { "select * from IDENTIFIER('main.json.t')", "select * from `identifier`('x')", "SELECT `note identifier(x)` FROM main.sales.orders", + `select * from IDENTIFIER${" ".repeat(120)}( 'cat.sch.tbl' )`, + "SELECT 1 AS `\u00df\u00df`, XIDENTIFIER('safe') AS n FROM main.sales.orders", ])("%p reads tables", (sql) => { expect(readsFiles(sql)).toBe(false) }) @@ -1053,6 +1058,13 @@ describe("IDENTIFIER() on Databricks", () => { expect(calls.queries.length > 0).toBe(explained) }) + test("repeated comments before a concatenation scan in linear time", () => { + const sql = "SELECT * FROM IDENTIFIER('read_' " + "/**/".repeat(30) + " || 'files')('/Volumes/c/s/v/raw')" + const started = performance.now() + expect(readsFiles(sql)).toBe(true) + expect(performance.now() - started).toBeLessThan(200) + }) + test("many ordinary literals scan in linear time", () => { const sql = `select ${Array.from({ length: 20000 }, (_, n) => `'v${n}'`).join(", ")} from main.s.t` const started = performance.now() From b9e79e57110e7c92d4f7ac88e4d5867761c128c9 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Thu, 19 Nov 2026 18:39:25 +0530 Subject: [PATCH 6/8] fix(dbt-cost-estimate): track the IDENTIFIER( context without rescanning, refuse non-ASCII names, keep identifier-named tables estimable Co-Authored-By: Claude Sonnet 5.5 --- .../src/altimate/tools/dbt-cost-estimate.ts | 36 ++++++++++--------- .../test/altimate/dbt-cost-estimate.test.ts | 9 +++++ 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts index 07197dc72e..e96115c713 100644 --- a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts +++ b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts @@ -186,8 +186,8 @@ export function readsFiles(sql: string): boolean { for (const lit of literals) { plain++ // Spark reads the string as a name, ignoring comments and unescaping backslashes: not read here. - // A double quote or a nested IDENTIFIER names a quoting mode or clause of its own. - if (/[\\"]|\/\*|--/.test(lit.text) || /IDENTIFIER/.test(lit.text.toUpperCase())) return true + // A double quote, a nested IDENTIFIER or a non-ASCII letter (Spark folds ſ to s, so jſon is json) is not read. + if (/[\\"]|\/\*|--/.test(lit.text) || /IDENTIFIER\s*\(/.test(lit.text.toUpperCase()) || /[^\x00-\x7f]/.test(lit.text)) return true if (/(^|[^a-z0-9_])`?(read_files|cloud_files)`?\s*$/i.test(lit.text) || fileSource(lit.text)) return true } return calls > plain @@ -256,6 +256,12 @@ export function scanSql(input: string, type: string): { script: boolean; code: s // The Snowflake splitter yields a line starting with "!" as a command of its own, wherever it sits: never read. if (!databricks && /(^|\n)!/.test(sql)) return script let code = "" + // The last of `code` with its whitespace collapsed to one space: what an IDENTIFIER( call is read from. + let tail = "" + const emit = (text: string) => { + code += text + tail = (tail + text.slice(-40)).replace(/\s+/g, " ").slice(-40) + } // Databricks: the text of each string literal that opens an IDENTIFIER( call, and where it sits in `code`. const literals: { text: string }[] = [] // Past a semicolon only whitespace and comments may follow: anything else, quoted or not, is a second statement. @@ -270,7 +276,7 @@ export function scanSql(input: string, type: string): { script: boolean; code: s if (databricks && end >= 0 && sql[end - 1] === "\\") return script if (end < 0) break i = end + 1 - code += " " + emit(" ") continue } if (two === "/*") { @@ -279,7 +285,7 @@ export function scanSql(input: string, type: string): { script: boolean; code: s // The closer's star counts too: in "/*/*/" an opener overlaps it. if (end < 0 || sql.slice(i + 2, end + 1).includes("/*")) return script i = end + 2 - code += " " + emit(" ") continue } if (two === "$$" || (!databricks && two === "//")) return script @@ -297,33 +303,31 @@ export function scanSql(input: string, type: string): { script: boolean; code: s if (j >= sql.length) return script let blank = " " // Only a single-quoted string is a literal name: "…" may be a quoted identifier or a variable. - if (databricks && c !== "`" && opensIdentifier(code)) { + if (databricks && c !== "`" && opensIdentifier(tail)) { // Marked, so a second literal after it does not read as the call's first. blank = "\u0000" // Only a literal the call's parenthesis closes is plain: a following literal, operator or name is not. if (c === "'" && closesCall(sql, j + 1)) literals.push({ text: sql.slice(i + 1, j) }) } // A backtick-quoted name is code; quoted text is not. - code += c === "`" ? sql.slice(i, j + 1) : blank + emit(c === "`" ? sql.slice(i, j + 1) : blank) i = j + 1 continue } - code += c + emit(c) i++ } return { script: /\bexecute\s+immediate\b/i.test(code), code, literals } } -/** Whether `code` (as `scanSql` builds it) ends in `IDENTIFIER(`, spacing aside. */ -function opensIdentifier(code: string): boolean { - let end = code.length - while (end > 0 && /\s/.test(code[end - 1])) end-- - if (code[end - 1] !== "(") return false - end-- - while (end > 0 && /\s/.test(code[end - 1])) end-- +/** Whether `tail` (the last of `scanSql`'s code, whitespace collapsed to one space) ends in `IDENTIFIER(`. */ +function opensIdentifier(tail: string): boolean { + let t = tail.replace(/ $/, "") + if (!t.endsWith("(")) return false + t = t.slice(0, -1).replace(/ $/, "") // Upper-cased as Spark's lexer folds keywords (a dotless ı spells IDENTIFIER too). - if (end < 10 || code.slice(end - 10, end).toUpperCase() !== "IDENTIFIER") return false - return end === 10 || !/[A-Za-z0-9_$]/.test(code[end - 11]) + if (t.length < 10 || t.slice(-10).toUpperCase() !== "IDENTIFIER") return false + return t.length === 10 || !/[A-Za-z0-9_$]/.test(t[t.length - 11]) } /** Whether, past whitespace and comments, the next character of `sql` from `at` is `)`. One pass, no backtracking. */ diff --git a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts index 35ea385c61..9acdf1915e 100644 --- a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts +++ b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts @@ -1029,6 +1029,7 @@ describe("IDENTIFIER() on Databricks", () => { "SELECT * FROM IDENTIFIER(\"IDENTIFIER('read_files')\")('/Volumes/c/s/v/raw')", "SELECT * FROM IDENTIFIER('identifier(''x'')')('/x')", "SELECT * FROM IDENTIFIER('read_' /*a*/ || 'files' /*b*/)('/Volumes/c/s/v/raw', format => 'json')", + "SELECT * FROM IDENTIFIER('`j\u017fon`.`data.json`')", "SELECT * FROM IDENTIFIER('js' /*a*/ || 'on.`/Volumes/c/s/v/raw`' /*b*/)", "SELECT 1 AS `\u00df\u00df\u00df`, XIDENTIFIER('safe') AS n FROM IDENTIFIER(concat('read_', 'files'))('/x')", ])("%p may read files", (sql) => { @@ -1043,6 +1044,7 @@ describe("IDENTIFIER() on Databricks", () => { "select * from `identifier`('x')", "SELECT `note identifier(x)` FROM main.sales.orders", `select * from IDENTIFIER${" ".repeat(120)}( 'cat.sch.tbl' )`, + "select * from IDENTIFIER('main.analytics.identifier_map')", "SELECT 1 AS `\u00df\u00df`, XIDENTIFIER('safe') AS n FROM main.sales.orders", ])("%p reads tables", (sql) => { expect(readsFiles(sql)).toBe(false) @@ -1065,6 +1067,13 @@ describe("IDENTIFIER() on Databricks", () => { expect(performance.now() - started).toBeLessThan(200) }) + test("adjacent ordinary literals scan in linear time", () => { + const sql = "SELECT " + "'a' ".repeat(64000) + "FROM main.s.t" + const started = performance.now() + expect(readsFiles(sql)).toBe(false) + expect(performance.now() - started).toBeLessThan(1000) + }) + test("many ordinary literals scan in linear time", () => { const sql = `select ${Array.from({ length: 20000 }, (_, n) => `'v${n}'`).join(", ")} from main.s.t` const started = performance.now() From 44096f1cf55fd09b1372be2acc7f7ec577ba547a Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Thu, 19 Nov 2026 18:55:10 +0530 Subject: [PATCH 7/8] fix(dbt-cost-estimate): count IDENTIFIER() calls and literals with one rule, refuse substitution and format-prefixed names Co-Authored-By: Claude Sonnet 5.5 --- .../src/altimate/tools/dbt-cost-estimate.ts | 25 +++++++++++-------- .../test/altimate/dbt-cost-estimate.test.ts | 8 +++++- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts index e96115c713..7b5c22357c 100644 --- a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts +++ b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts @@ -163,6 +163,9 @@ export function parseDatabricksExplainCost(plan: unknown): PlanEstimate | { noSt return { bytes: Math.round(bytes) } } +/** The data source formats Spark reads a path with (`json.`/path``). */ +const FILE_FORMATS = "json|csv|parquet|avro|orc|text|binaryfile|xml|libsvm|image" + /** * Whether a Databricks query reads files rather than tables (read_files, cloud_files, a path such as json.`…`): * planning one can read the files to infer their schema. Read on the code `scanSql` leaves (comments and quoted text @@ -170,15 +173,13 @@ export function parseDatabricksExplainCost(plan: unknown): PlanEstimate | { noSt * does not read counts. */ export function readsFiles(sql: string): boolean { - const { script, code, literals } = scanSql(sql, "databricks") + const { script, code, literals, calls } = scanSql(sql, "databricks") if (script) return true if (fileSource(code)) return true // IDENTIFIER(expr) names a function or relation from a string, which the scan blanked: read a lone string literal as // code, and count any other argument (a variable, :param, concatenation, adjacent literals, escapes) as a read. - // Counted outside backtick-quoted names, and with no boundary consumed, so IDENTIFIER(IDENTIFIER('f')('x')) is two. - // Upper-cased as Spark's lexer folds keywords (a dotless ı spells IDENTIFIER too). + // Upper-cased as Spark's lexer folds keywords (a dotless ı spells IDENTIFIER too). const bare = code.replace(/`[^`]*`/g, " ").toUpperCase() - const calls = bare.match(/(? plain @@ -200,8 +202,7 @@ function fileSource(code: string): boolean { // A known format over any backtick-quoted name, unless the format word is the middle of a dotted name (main.json.`t`). // Where a relation goes cannot be told from a column without parsing (FROM (json.`x`), FROM`json`.`x`), so a // column qualified by an alias named like a format (json.`order-id`) is refused too: no estimate, never a missed read. - const formats = "json|csv|parquet|avro|orc|text|binaryfile|xml|libsvm|image" - if (new RegExp(`(?:(^|[^a-z0-9_.\`])(${formats})|(^|[^.\`])\`(${formats})\`)\\s*\\.\\s*\``, "i").test(code)) return true + if (new RegExp(`(?:(^|[^a-z0-9_.\`])(${FILE_FORMATS})|(^|[^.\`])\`(${FILE_FORMATS})\`)\\s*\\.\\s*\``, "i").test(code)) return true // Any backtick-quoted name that looks like a path ("/" or ":", as in s3://…, dbfs:/…, /Volumes/…), whatever the source // before it (a format, a provider class) or a doubled backtick inside it, unless it directly follows Delta's own // provider: a source over files can infer its schema from them. @@ -248,8 +249,8 @@ export function mayBeScript(input: string, type: string): boolean { * comments and quoted text as spaces and Databricks' backtick-quoted names kept as written, and (Databricks) the lone * string literals an IDENTIFIER( call holds. */ -export function scanSql(input: string, type: string): { script: boolean; code: string; literals: { text: string }[] } { - const script = { script: true, code: "", literals: [] } +export function scanSql(input: string, type: string): { script: boolean; code: string; literals: { text: string }[]; calls: number } { + const script = { script: true, code: "", literals: [], calls: 0 } // Read with the line breaks Jinja renders: every \r\n and lone \r is \n. const sql = input.replace(/\r\n?/g, "\n") const databricks = type === "databricks" @@ -264,6 +265,7 @@ export function scanSql(input: string, type: string): { script: boolean; code: s } // Databricks: the text of each string literal that opens an IDENTIFIER( call, and where it sits in `code`. const literals: { text: string }[] = [] + let calls = 0 // Past a semicolon only whitespace and comments may follow: anything else, quoted or not, is a second statement. let ended = false let i = 0 @@ -315,9 +317,10 @@ export function scanSql(input: string, type: string): { script: boolean; code: s continue } emit(c) + if (databricks && c === "(" && opensIdentifier(tail)) calls++ i++ } - return { script: /\bexecute\s+immediate\b/i.test(code), code, literals } + return { script: /\bexecute\s+immediate\b/i.test(code), code, literals, calls } } /** Whether `tail` (the last of `scanSql`'s code, whitespace collapsed to one space) ends in `IDENTIFIER(`. */ diff --git a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts index 9acdf1915e..a8b7dea06a 100644 --- a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts +++ b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts @@ -885,7 +885,7 @@ describe("review follow-ups, round eight", () => { }) test("both checks read the SQL the same way: one scan", () => { - expect(scanSql("select `json`.`x` from t -- c\n", "databricks")).toEqual({ script: false, code: "select `json`.`x` from t ", literals: [] }) + expect(scanSql("select `json`.`x` from t -- c\n", "databricks")).toEqual({ script: false, code: "select `json`.`x` from t ", literals: [], calls: 0 }) expect(scanSql("select 'a' from t", "databricks").code).toBe("select from t") }) }) @@ -1030,6 +1030,11 @@ describe("IDENTIFIER() on Databricks", () => { "SELECT * FROM IDENTIFIER('identifier(''x'')')('/x')", "SELECT * FROM IDENTIFIER('read_' /*a*/ || 'files' /*b*/)('/Volumes/c/s/v/raw', format => 'json')", "SELECT * FROM IDENTIFIER('`j\u017fon`.`data.json`')", + "SELECT * FROM IDENTIFIER('${hiveconf:source}')('/Volumes/c/s/v/raw')", + "SELECT * FROM IDENTIFIER('json.raw')", + "SELECT * FROM IDENTIFIER(' CSV . raw')", + "SELECT \u017fIDENTIFIER('safe') FROM IDENTIFIER(concat('read_', 'files'))('/x')", + "SELECT \u0131DENTIFIER('safe') FROM IDENTIFIER(concat('read_', 'files'))('/x')", "SELECT * FROM IDENTIFIER('js' /*a*/ || 'on.`/Volumes/c/s/v/raw`' /*b*/)", "SELECT 1 AS `\u00df\u00df\u00df`, XIDENTIFIER('safe') AS n FROM IDENTIFIER(concat('read_', 'files'))('/x')", ])("%p may read files", (sql) => { @@ -1083,5 +1088,6 @@ describe("IDENTIFIER() on Databricks", () => { test("leaves Snowflake and BigQuery alone", async () => { expect(scanSql("select * from IDENTIFIER('cat.sch.tbl')", "snowflake").literals).toEqual([]) + expect(scanSql("select * from IDENTIFIER('cat.sch.tbl')", "snowflake").calls).toBe(0) }) }) From afcc90c2e6d6977443ae256748bc18152f23052c Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Thu, 19 Nov 2026 19:10:55 +0530 Subject: [PATCH 8/8] fix(dbt-cost-estimate): allow only a plain dotted table name in IDENTIFIER('...'); refuse every other literal Co-Authored-By: Claude Sonnet 5.5 --- .../src/altimate/tools/dbt-cost-estimate.ts | 18 ++++++++++-------- .../test/altimate/dbt-cost-estimate.test.ts | 9 +++++++++ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts index 7b5c22357c..9cf1b66c66 100644 --- a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts +++ b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts @@ -176,21 +176,23 @@ export function readsFiles(sql: string): boolean { const { script, code, literals, calls } = scanSql(sql, "databricks") if (script) return true if (fileSource(code)) return true - // IDENTIFIER(expr) names a function or relation from a string, which the scan blanked: read a lone string literal as - // code, and count any other argument (a variable, :param, concatenation, adjacent literals, escapes) as a read. - // Upper-cased as Spark's lexer folds keywords (a dotless ı spells IDENTIFIER too). + // IDENTIFIER(expr) names a function or relation from a string, which the scan blanked: allow a lone string literal + // that is a plain table name, and count any other argument (a variable, :param, concatenation, adjacent literals, escapes) as a read. + // Upper-cased as Spark's lexer folds keywords (a dotless ı spells IDENTIFIER too). const bare = code.replace(/`[^`]*`/g, " ").toUpperCase() // A qualified clause (system.builtin.IDENTIFIER('read_files')) or one a name continues past (IDENTIFIER('json').`p`) // is assembled from parts this does not read. if (/\.\s*IDENTIFIER\s*\(/.test(bare) || /IDENTIFIER\s*\(\s*\u0000\s*\)\s*\./.test(bare)) return true let plain = 0 + const format = new RegExp(`^(${FILE_FORMATS})$`, "i") for (const lit of literals) { plain++ - // Spark reads the string as a name, ignoring comments and unescaping backslashes: not read here. - // A double quote, a `${…}` substitution, a nested IDENTIFIER, a non-ASCII letter (Spark folds ſ to s, so jſon is json) or - // a format before the path (json.raw) is not read. - if (/[\\"$]|\/\*|--/.test(lit.text) || new RegExp(`^\\s*\`?(${FILE_FORMATS})\`?\\s*\\.`, "i").test(lit.text) || /IDENTIFIER\s*\(/.test(lit.text.toUpperCase()) || /[^\x00-\x7f]/.test(lit.text)) return true - if (/(^|[^a-z0-9_])`?(read_files|cloud_files)`?\s*$/i.test(lit.text) || fileSource(lit.text)) return true + // Only a plain dotted name of up to three parts is a table: anything quoted, escaped, commented, substituted, + // non-ASCII or longer is not read (Spark resolves it by rules this does not model), nor is a read_files or + // cloud_files part, or a format before the path (json.raw). + const parts = lit.text.split(".") + if (parts.length > 3 || !parts.every((part) => /^[A-Za-z_][A-Za-z0-9_]*$/.test(part))) return true + if (parts.some((part) => /^(read_files|cloud_files)$/i.test(part)) || format.test(parts[0])) return true } return calls > plain } diff --git a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts index a8b7dea06a..d0ed7b3e80 100644 --- a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts +++ b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts @@ -1035,6 +1035,9 @@ describe("IDENTIFIER() on Databricks", () => { "SELECT * FROM IDENTIFIER(' CSV . raw')", "SELECT \u017fIDENTIFIER('safe') FROM IDENTIFIER(concat('read_', 'files'))('/x')", "SELECT \u0131DENTIFIER('safe') FROM IDENTIFIER(concat('read_', 'files'))('/x')", + "SELECT * FROM IDENTIFIER('`org.apache.spark.sql.json.DefaultSource`.`data.json`')", + "SELECT * FROM IDENTIFIER('`org.apache.spark.sql.json.DefaultSource`.`data.json`')", + "SELECT * FROM IDENTIFIER('a.b.c.d')", "SELECT * FROM IDENTIFIER('js' /*a*/ || 'on.`/Volumes/c/s/v/raw`' /*b*/)", "SELECT 1 AS `\u00df\u00df\u00df`, XIDENTIFIER('safe') AS n FROM IDENTIFIER(concat('read_', 'files'))('/x')", ])("%p may read files", (sql) => { @@ -1079,6 +1082,12 @@ describe("IDENTIFIER() on Databricks", () => { expect(performance.now() - started).toBeLessThan(1000) }) + test("an unbalanced backtick over many slashes scans in linear time", () => { + const started = performance.now() + expect(readsFiles("SELECT * FROM IDENTIFIER('`" + "/".repeat(128000) + "')")).toBe(true) + expect(performance.now() - started).toBeLessThan(1000) + }) + test("many ordinary literals scan in linear time", () => { const sql = `select ${Array.from({ length: 20000 }, (_, n) => `'v${n}'`).join(", ")} from main.s.t` const started = performance.now()