From 7f037cd06acad561eacb4d8845340d5034b4f2a2 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Thu, 19 Nov 2026 12:35:36 +0530 Subject: [PATCH 01/14] feat: dbt_cost_estimate on Snowflake and Databricks from the planner's EXPLAIN Snowflake and Databricks have no dry run, so the estimate there comes from the planner without running the query: - Snowflake: EXPLAIN USING JSON, GlobalStats.bytesAssigned (bytes left after compile-time pruning; documented as an upper bound). - Databricks: EXPLAIN COST, the sum of Statistics(sizeInBytes) over the leaves of the optimized logical plan (table sizes before column pruning and file skipping). A leaf with no size, or Spark's default for a table without statistics, is no estimate. The EXPLAIN runs through dbt's own run_query inside an inline compile, since dbt show wraps the query in a limit. Results carry source "explain" and bound "upper" or "estimate". BigQuery is unchanged. Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/tools/dbt-cost-estimate.ts | 184 +++++++++++- .../test/altimate/dbt-cost-estimate.test.ts | 262 +++++++++++++++++- 2 files changed, 439 insertions(+), 7 deletions(-) diff --git a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts index 758dfc2631..4f297b29a4 100644 --- a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts +++ b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts @@ -54,6 +54,121 @@ export function parseBytesProcessed(display: unknown): { bytes: number; maxBytes return { bytes, maxBytes: Math.max(bytes, maxBytes) } } +/** Bytes in the units the BigQuery dry run writes them in: "4.2 TiB". */ +export function formatBytes(bytes: number): string { + let n = bytes + let i = 0 + while (n >= 1024 && i < UNITS.length - 1) { + n /= 1024 + i++ + } + return `${n.toFixed(1)} ${UNITS[i]}` +} + +/** A scan estimate from the warehouse's query planner: what EXPLAIN says the query reads, without running it. */ +export type PlanEstimate = { bytes: number; partitionsAssigned?: number; partitionsTotal?: number } + +const count = (value: unknown) => (typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : undefined) + +/** + * The bytes Snowflake's `EXPLAIN USING JSON` says the query will scan: `GlobalStats.bytesAssigned`, the + * micro-partitions left after compile-time pruning. Takes the `content` column as text or parsed JSON. + * Undefined for anything without a readable `bytesAssigned`, so a changed format is reported, never read as a count. + */ +export function parseSnowflakeExplain(content: unknown): PlanEstimate | undefined { + let plan = content + if (typeof plan === "string") { + try { + plan = JSON.parse(plan) + } catch { + return undefined + } + } + const stats = (plan as { GlobalStats?: Record } | null)?.GlobalStats + if (!stats || typeof stats !== "object") return undefined + const bytes = count(stats.bytesAssigned) + if (bytes === undefined) return undefined + const partitionsAssigned = count(stats.partitionsAssigned) + const partitionsTotal = count(stats.partitionsTotal) + return { + bytes: Math.round(bytes), + ...(partitionsAssigned !== undefined && { partitionsAssigned }), + ...(partitionsTotal !== undefined && { partitionsTotal }), + } +} + +/** Spark's `Utils.bytesToString` units, each 1024 times the one before; Spark 2 wrote "KB" for "KiB". */ +const SPARK_UNITS: Record = { B: 0, KIB: 1, MIB: 2, GIB: 3, TIB: 4, PIB: 5, EIB: 6, KB: 1, MB: 2, GB: 3, TB: 4, PB: 5, EB: 6 } +/** + * A relation Spark has no size for is given `spark.sql.defaultSizeInBytes`, Long.MaxValue unless set ("8.0 EiB"): + * a leaf at or past this is no size at all. + */ +const SPARK_NO_SIZE = 2 ** 62 + +/** A `sizeInBytes=` figure read back into bytes: "12.3 MiB", "16.0 B", or past a long "1.23E+21 B". */ +function sparkBytes(figure: string): number | undefined { + const match = /^(\d+(?:\.\d+)?(?:E[+-]?\d+)?)\s*([A-Za-z]+)$/i.exec(figure.trim()) + if (!match) return undefined + const unit = SPARK_UNITS[match[2].toUpperCase()] + if (unit === undefined) return undefined + const value = Number(match[1]) * 1024 ** unit + return Number.isFinite(value) ? value : undefined +} + +/** + * The bytes Databricks' `EXPLAIN COST` says the query reads: the sum of `Statistics(sizeInBytes=…)` over the + * leaves of the Optimized Logical Plan, the relations it scans. Their sizes are table (or file) sizes from the + * catalog and the Delta log, before column pruning and runtime file skipping. `noStats` when a leaf carries no + * size or Spark's default for a relation without statistics; undefined when there is no plan to read. + */ +export function parseDatabricksExplainCost(plan: unknown): (PlanEstimate & { leaves: number }) | { noStats: true } | undefined { + if (typeof plan !== "string") return undefined + const lines = plan.replace(/\r\n?/g, "\n").split("\n") + const start = lines.findIndex((line) => /^\s*==\s*Optimized Logical Plan\s*==\s*$/i.test(line)) + if (start < 0) return undefined + const nodes: { depth: number; text: string }[] = [] + for (const line of lines.slice(start + 1)) { + if (/^\s*==.*==\s*$/.test(line)) break + if (!line.trim()) continue + // Each level of the tree is three characters: ": " or " " for an ancestor, ":- " or "+- " for the node itself. + const prefix = /^(?:: | )*(?:[:+]- )?/.exec(line)![0] + nodes.push({ depth: prefix.length / 3, text: line.slice(prefix.length) }) + } + if (nodes.length === 0) return undefined + let bytes = 0 + let leaves = 0 + for (let i = 0; i < nodes.length; i++) { + // A leaf is a node the next line does not go below. + if (i + 1 < nodes.length && nodes[i + 1].depth > nodes[i].depth) continue + leaves++ + const figure = /Statistics\(sizeInBytes=([^,)]+)/.exec(nodes[i].text)?.[1] + const size = figure === undefined ? undefined : sparkBytes(figure) + if (size === undefined || size >= SPARK_NO_SIZE) return { noStats: true } + bytes += size + } + return { bytes: Math.round(bytes), leaves } +} + +/** + * The warehouses with no dry run whose planner still says what a query reads, the EXPLAIN that asks it, and the + * reader of its one row. + */ +const PLANNERS: Record PlanEstimate | { noStats: true } | undefined }> = { + snowflake: { name: "Snowflake", explain: "EXPLAIN USING JSON", read: parseSnowflakeExplain }, + databricks: { name: "Databricks", explain: "EXPLAIN COST", read: parseDatabricksExplainCost }, +} + +/** + * Inline Jinja that runs the EXPLAIN through dbt's own connection (`run_query`) while dbt compiles it, and + * renders the first column of its answer as JSON. Compiling, not `dbt show`, since show wraps the query in a + * limit, which an EXPLAIN can't take. The SQL is written into a Jinja string, so its quotes and backslashes are escaped. + */ +export function explainJinja(explain: string, sql: string): string { + const statement = `${explain}\n${sql.replace(/[\s;]+$/, "")}` + const literal = `'${statement.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'` + return `{%- set altimate_plan = run_query(${literal}) -%}{{ tojson(altimate_plan.columns[0].values() | list) }}` +} + /** Starts `start` and waits for it, but gives up as soon as the call is cancelled; a cancelled call starts nothing. */ export function untilAbort(start: () => Promise, signal: AbortSignal): Promise { if (signal.aborted) return Promise.reject(signal.reason) @@ -71,9 +186,68 @@ function failure(title: string, message: string, extra: Record return { title: `dbt_cost_estimate: ${title}`, metadata: { success: false, error: message, ...extra }, output: message } } +/** The planner's estimate of a compiled query on a warehouse with no dry run: an EXPLAIN, which compiles the query and runs none of it. */ +async function explainEstimate( + adapter: any, + type: string, + planner: (typeof PLANNERS)[string], + compiled: string, + model: string | undefined, + signal: AbortSignal, +) { + const extra = { adapter: type, source: "explain" } + let row: unknown + try { + const rendered = await untilAbort(() => adapter.unsafeCompileQuery(explainJinja(planner.explain, compiled), undefined), signal) + const rows = typeof rendered === "string" ? JSON.parse(rendered.trim()) : undefined + if (!Array.isArray(rows) || rows.length === 0) throw new Error(`${planner.explain} returned no plan`) + row = rows[0] + } catch (e) { + if (signal.aborted) throw e + return failure("explain failed", `No estimate: ${planner.name}'s ${planner.explain} failed: ${e instanceof Error ? e.message : String(e)}`, extra) + } + const parsed = planner.read(row) + if (parsed && "noStats" in parsed) { + return failure( + "no estimate", + `No estimate: ${planner.name}'s planner has no size for a table this query reads (it shows Spark's default for a table without statistics). ANALYZE TABLE COMPUTE STATISTICS gives it one.`, + extra, + ) + } + if (!parsed) { + return failure("unreadable estimate", `No estimate: ${planner.name}'s ${planner.explain} answered a plan this tool can't read.`, extra) + } + const shown = formatBytes(parsed.bytes) + const partitions = + parsed.partitionsAssigned !== undefined && parsed.partitionsTotal !== undefined + ? `${parsed.partitionsAssigned} of ${parsed.partitionsTotal} partitions after pruning; ` + : "" + const accuracy = + type === "snowflake" + ? `${partitions}an upper bound: runtime pruning can scan less, and a cached result none` + : "table sizes before column pruning and file skipping, which usually read less; not a bound either way" + return { + title: `dbt_cost_estimate: ${shown}`, + metadata: { + success: true, + supported: true, + adapter: type, + source: "explain", + // Snowflake documents bytesAssigned as an upper bound; Spark's sizes are the planner's guess, either way. + bound: type === "snowflake" ? "upper" : "estimate", + bytes: parsed.bytes, + display: shown, + ...(parsed.partitionsAssigned !== undefined && { partitionsAssigned: parsed.partitionsAssigned }), + ...(parsed.partitionsTotal !== undefined && { partitionsTotal: parsed.partitionsTotal }), + ...(model && { model }), + }, + output: `${model ? `${model} ` : "The query "}would scan about ${shown}, by ${planner.name}'s plan estimate (${accuracy}).`, + } +} + export const DbtCostEstimateTool = Tool.define("dbt_cost_estimate", { description: - "Estimate the bytes a dbt model or a SQL query would scan, from a warehouse dry run. Only BigQuery has a dry run: any other warehouse answers `supported: false` and nothing is compiled. Starts the dbt adapter (which runs `dbt deps`), so it needs write permission (`--allow-write` on `tool run` or the MCP server). The dry run never executes the final query, but compiling either input can render Jinja, and macros such as `run_query` can run real queries that are billed. Pass `model` (a model name, compiled as dbt would) or `sql` (may use Jinja).", + "Estimate the bytes a dbt model or a SQL query would scan, without running it. BigQuery answers from a dry run (`maxBytes` bounds it, or `lowerBound` marks a floor); Snowflake (`EXPLAIN USING JSON`, bytes left after pruning) and Databricks (`EXPLAIN COST`, the sizes of the tables it reads) answer from the planner (`source: \"explain\"`; `bound` is \"upper\" on Snowflake, \"estimate\" on Databricks), and none when Databricks has no table statistics. EXPLAIN compiles the query on the warehouse and scans no data. Any other warehouse answers `supported: false` and nothing is compiled. Starts the dbt adapter (which runs `dbt deps`), so it needs write permission (`--allow-write` on `tool run` or the MCP server). The dry run never executes the final query, but compiling either input can render Jinja, and macros such as `run_query` can run real queries that are billed. Pass `model` (a model name, compiled as dbt would) or `sql` (may use Jinja).", parameters: z.object({ model: z.string().optional().describe("Model name to compile and estimate"), sql: z.string().optional().describe("SQL (may use Jinja) to compile and estimate"), @@ -85,13 +259,14 @@ export const DbtCostEstimateTool = Tool.define("dbt_cost_estimate", { if (!access.adapter) return failure("not available", `No estimate: ${access.reason}.`, { refusal: access.refusal }) const adapter = access.adapter const type: string | undefined = adapter.getAdapterType?.() - // Every dbt-integration refuses the dry run off BigQuery, so nothing is compiled there. - if (type !== "bigquery") { + // Every dbt-integration refuses the dry run off BigQuery; elsewhere only a planner EXPLAIN tells the scan. Nothing is compiled for any other warehouse. + const planner = type ? PLANNERS[type] : undefined + if (type !== "bigquery" && !planner) { const adapterName = type ?? "unknown" return { title: "dbt_cost_estimate: no estimate", metadata: { success: true, supported: false, adapter: adapterName }, - output: `No cost estimate: dry runs are only available on BigQuery (this project uses ${adapterName}).`, + output: `No cost estimate: scan estimates are only available on BigQuery, Snowflake and Databricks (this project uses ${adapterName}).`, } } const signal = ctx.abort @@ -109,6 +284,7 @@ export const DbtCostEstimateTool = Tool.define("dbt_cost_estimate", { if (typeof compiled !== "string" || !compiled.trim()) { return failure("compile failed", "The query compiled to nothing.", { adapter: type }) } + if (planner) return explainEstimate(adapter, type!, planner, compiled, args.model, signal) let display: unknown try { const integration = await untilAbort(async () => { diff --git a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts index 885e982b95..bd3e81983b 100644 --- a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts +++ b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts @@ -9,7 +9,14 @@ import { Effect } from "effect" import type { Tool } from "@/tool/tool" import { SessionID, MessageID } from "@/session/schema" import { dbtAdapterInternals, resetDbtAdapter } from "@/altimate/native/connections/register" -import { DbtCostEstimateTool, parseBytesProcessed, untilAbort } from "@/altimate/tools/dbt-cost-estimate" +import { + DbtCostEstimateTool, + explainJinja, + parseBytesProcessed, + parseDatabricksExplainCost, + parseSnowflakeExplain, + untilAbort, +} from "@/altimate/tools/dbt-cost-estimate" import { initTool } from "./tool-fixture" const ctx: Tool.Context = { @@ -28,7 +35,15 @@ type Calls = { nodes: string[]; queries: [string, string | undefined][]; dryRuns function fakeAdapter( calls: Calls, - opts: { type?: string; bytes?: unknown; compileFails?: boolean; dryRunFails?: boolean; compile?: () => Promise } = {}, + opts: { + type?: string + bytes?: unknown + compileFails?: boolean + dryRunFails?: boolean + compile?: () => Promise + /** What compiling the EXPLAIN Jinja renders (the plan's first column as JSON), or a failure to throw. */ + explain?: () => Promise + } = {}, ) { return { isInitialized: () => true, @@ -42,6 +57,7 @@ function fakeAdapter( }, unsafeCompileQuery: async (query: string, name?: string) => { calls.queries.push([query, name]) + if (opts.explain && query.startsWith("{%- set altimate_plan")) return opts.explain() if (opts.compileFails) throw new Error("compilation failed") return `compiled(${query})` }, @@ -208,7 +224,7 @@ describe("dbt_cost_estimate", () => { } }) - test.each(["snowflake", "databricks", "duckdb"])("answers supported:false on %s and compiles nothing", async (type) => { + test.each(["duckdb", "redshift", "postgres"])("answers supported:false on %s and compiles nothing", async (type) => { useAdapter(calls, { type }) const result = await (await initTool(DbtCostEstimateTool)).execute({ model: "orders" }, ctx) expect(result.metadata).toMatchObject({ success: true, supported: false, adapter: type }) @@ -340,3 +356,243 @@ describe("dbt_cost_estimate", () => { expect(calls.dryRuns).toEqual(["compiled(where s = 'a')"]) }) }) + +// Snowflake's EXPLAIN USING JSON answers one row, column `content`: the shape documented for the command, with a +// table scan per object read. Databricks' EXPLAIN COST answers one row, column `plan`: Spark's tree string, each node +// suffixed with `Statistics(sizeInBytes=…[, rowCount=…])` as Statistics.simpleString and Utils.bytesToString write it. +const SNOWFLAKE_PLAN = JSON.stringify({ + GlobalStats: { partitionsTotal: 1240, partitionsAssigned: 37, bytesAssigned: 2_147_483_648 }, + Operations: [ + [ + { id: 0, operation: "Result", expressions: ["ORDERS.O_ORDERKEY", "CUSTOMER.C_NAME"] }, + { id: 1, parentOperators: [0], operation: "InnerJoin", expressions: ["joinKey: (CUSTOMER.C_CUSTKEY = ORDERS.O_CUSTKEY)"] }, + { + id: 2, + parentOperators: [1], + operation: "TableScan", + objects: ["ANALYTICS.PUBLIC.ORDERS"], + expressions: ["O_ORDERKEY", "O_CUSTKEY", "O_ORDERDATE"], + partitionsAssigned: 30, + partitionsTotal: 1200, + bytesAssigned: 2_000_000_000, + }, + { id: 3, parentOperators: [1], operation: "JoinFilter", expressions: ["joinKey: (CUSTOMER.C_CUSTKEY = ORDERS.O_CUSTKEY)"] }, + { + id: 4, + parentOperators: [3], + operation: "TableScan", + objects: ["ANALYTICS.PUBLIC.CUSTOMER"], + expressions: ["C_CUSTKEY", "C_NAME"], + partitionsAssigned: 7, + partitionsTotal: 40, + bytesAssigned: 147_483_648, + }, + ], + ], +}) + +const DATABRICKS_PLAN = [ + "== Optimized Logical Plan ==", + "Aggregate [o_custkey#12L], [o_custkey#12L, sum(o_totalprice#15) AS revenue#40], Statistics(sizeInBytes=1.2 MiB, rowCount=5.00E+4)", + "+- Project [o_custkey#12L, o_totalprice#15], Statistics(sizeInBytes=1.6 GiB)", + " +- Join Inner, (o_custkey#12L = c_custkey#30L), Statistics(sizeInBytes=3.0 GiB)", + " :- Filter (isnotnull(o_orderdate#16) AND (o_orderdate#16 >= 2024-01-01)), Statistics(sizeInBytes=2.5 GiB)", + " : +- Relation main.sales.orders[o_orderkey#11L,o_custkey#12L,o_totalprice#15,o_orderdate#16] parquet, Statistics(sizeInBytes=2.5 GiB, rowCount=1.50E+7)", + " +- Relation main.sales.customer[c_custkey#30L,c_name#31] parquet, Statistics(sizeInBytes=512.0 MiB)", + "", + "== Physical Plan ==", + "AdaptiveSparkPlan isFinalPlan=false", + "+- HashAggregate(keys=[o_custkey#12L], functions=[sum(o_totalprice#15)])", + " +- FileScan parquet main.sales.orders[o_custkey#12L] Batched: true, Statistics(sizeInBytes=99.0 EiB)", +].join("\n") + +describe("parseSnowflakeExplain", () => { + test("reads GlobalStats: the bytes and partitions left after pruning", () => { + expect(parseSnowflakeExplain(SNOWFLAKE_PLAN)).toEqual({ bytes: 2_147_483_648, partitionsAssigned: 37, partitionsTotal: 1240 }) + }) + + test("takes the content already parsed", () => { + expect(parseSnowflakeExplain(JSON.parse(SNOWFLAKE_PLAN))).toEqual({ bytes: 2_147_483_648, partitionsAssigned: 37, partitionsTotal: 1240 }) + }) + + test("a query that reads no table is zero bytes", () => { + expect(parseSnowflakeExplain('{"GlobalStats":{"partitionsTotal":0,"partitionsAssigned":0,"bytesAssigned":0},"Operations":[[{"id":0,"operation":"Result"}]]}')).toEqual({ + bytes: 0, + partitionsAssigned: 0, + partitionsTotal: 0, + }) + }) + + test("bytes without partition counts are still an estimate", () => { + expect(parseSnowflakeExplain({ GlobalStats: { bytesAssigned: 1024 } })).toEqual({ bytes: 1024 }) + }) + + test.each([ + ["not JSON", "GlobalStats: 1024"], + ["no GlobalStats", '{"Operations":[]}'], + ["no bytesAssigned", '{"GlobalStats":{"partitionsTotal":3}}'], + ["text bytes", '{"GlobalStats":{"bytesAssigned":"1024"}}'], + ["negative bytes", '{"GlobalStats":{"bytesAssigned":-1}}'], + ["the tabular format", "step\tid\tparent\toperation"], + ["null", null], + ["a number", 1024], + ])("does not read %s", (_, content) => { + expect(parseSnowflakeExplain(content)).toBeUndefined() + }) +}) + +describe("parseDatabricksExplainCost", () => { + test("sums the leaves of the optimized plan: the relations it scans, not the operators above them", () => { + expect(parseDatabricksExplainCost(DATABRICKS_PLAN)).toEqual({ bytes: Math.round(2.5 * 1024 ** 3 + 512 * 1024 ** 2), leaves: 2 }) + }) + + test("a single relation and Spark's B unit", () => { + const plan = "== Optimized Logical Plan ==\nProject [id#0L], Statistics(sizeInBytes=12.0 B)\n+- Relation main.t[id#0L] parquet, Statistics(sizeInBytes=16.0 B, rowCount=2)\n\n== Physical Plan ==\n*(1) ColumnarToRow" + expect(parseDatabricksExplainCost(plan)).toEqual({ bytes: 16, leaves: 1 }) + }) + + test("a scalar subquery's plan is printed inside the tree, and its relation counts too", () => { + const plan = [ + "== Optimized Logical Plan ==", + "Project [id#1L, scalar-subquery#5 [] AS m#6L], Statistics(sizeInBytes=2.0 MiB)", + ": +- Aggregate [max(v#3L) AS max(v)#8L], Statistics(sizeInBytes=16.0 B, rowCount=1)", + ": +- Relation main.s.dim[v#3L] parquet, Statistics(sizeInBytes=3.0 MiB)", + "+- Relation main.s.fact[id#1L] parquet, Statistics(sizeInBytes=1.5 GiB)", + "", + "== Physical Plan ==", + ].join("\n") + expect(parseDatabricksExplainCost(plan)).toEqual({ bytes: Math.round(1.5 * 1024 ** 3 + 3 * 1024 ** 2), leaves: 2 }) + }) + + test("reads Spark 2's KB/MB units and the scientific form past a long", () => { + expect(parseDatabricksExplainCost("== Optimized Logical Plan ==\nRelation[a#0] parquet, Statistics(sizeInBytes=2.0 MB, hints=none)")).toEqual({ + bytes: 2 * 1024 ** 2, + leaves: 1, + }) + // A size past 2^62 is Spark's "no statistics" default or larger: not a size. + expect(parseDatabricksExplainCost("== Optimized Logical Plan ==\nRelation[a#0] parquet, Statistics(sizeInBytes=1.23E+21 B)")).toEqual({ noStats: true }) + }) + + test("a relation without statistics shows Spark's default size, Long.MaxValue: no estimate, not 8 EiB", () => { + const plan = "== Optimized Logical Plan ==\nJoin Inner, Statistics(sizeInBytes=8.0 EiB)\n:- Relation main.a[x#1] parquet, Statistics(sizeInBytes=1.0 GiB)\n+- Relation jdbc.b[x#2] JDBCRelation, Statistics(sizeInBytes=8.0 EiB)" + expect(parseDatabricksExplainCost(plan)).toEqual({ noStats: true }) + }) + + test("a leaf with no statistics at all is no estimate", () => { + expect(parseDatabricksExplainCost("== Optimized Logical Plan ==\nProject [a#0], Statistics(sizeInBytes=1.0 KiB)\n+- Relation main.t[a#0] parquet")).toEqual({ noStats: true }) + }) + + test.each([ + ["no optimized plan section", "== Physical Plan ==\n*(1) Scan"], + ["an empty section", "== Optimized Logical Plan ==\n\n== Physical Plan ==\nScan"], + ["not text", { plan: "x" }], + ["null", null], + ])("does not read %s", (_, plan) => { + expect(parseDatabricksExplainCost(plan)).toBeUndefined() + }) + + test("reads Windows line endings", () => { + expect(parseDatabricksExplainCost("== Optimized Logical Plan ==\r\nRelation main.t[a#0] parquet, Statistics(sizeInBytes=4.0 KiB)\r\n")).toEqual({ bytes: 4096, leaves: 1 }) + }) +}) + +describe("explainJinja", () => { + test("runs the EXPLAIN through dbt's run_query and renders its first column as JSON", () => { + expect(explainJinja("EXPLAIN COST", "select 1")).toBe( + "{%- set altimate_plan = run_query('EXPLAIN COST\nselect 1') -%}{{ tojson(altimate_plan.columns[0].values() | list) }}", + ) + }) + + test("escapes quotes and backslashes for the Jinja string, and drops a trailing semicolon", () => { + expect(explainJinja("EXPLAIN USING JSON", "select * from t where s = 'a\\b';\n")).toContain("run_query('EXPLAIN USING JSON\nselect * from t where s = \\'a\\\\b\\'')") + }) +}) + +describe("dbt_cost_estimate on a warehouse with a planner", () => { + const rendered = (row: unknown) => async () => JSON.stringify([row]) + + test("Snowflake: compiles the model, then EXPLAINs it, and answers the bytes assigned as an upper bound", async () => { + useAdapter(calls, { type: "snowflake", explain: rendered(SNOWFLAKE_PLAN) }) + const result = await (await initTool(DbtCostEstimateTool)).execute({ model: "orders" }, ctx) + expect(calls.nodes).toEqual(["orders"]) + expect(calls.queries).toEqual([[explainJinja("EXPLAIN USING JSON", "compiled(orders)"), undefined]]) + expect(calls.dryRuns).toEqual([]) + expect(result.metadata).toEqual({ + success: true, + supported: true, + adapter: "snowflake", + source: "explain", + bound: "upper", + bytes: 2_147_483_648, + display: "2.0 GiB", + partitionsAssigned: 37, + partitionsTotal: 1240, + model: "orders", + truncated: false, + }) + expect(result.output).toContain("orders would scan about 2.0 GiB, by Snowflake's plan estimate (37 of 1240 partitions after pruning; an upper bound") + }) + + test("Databricks: estimates inline SQL from EXPLAIN COST, as an estimate, not a bound", async () => { + useAdapter(calls, { type: "databricks", explain: rendered(DATABRICKS_PLAN) }) + const result = await (await initTool(DbtCostEstimateTool)).execute({ sql: "select * from {{ ref('orders') }}" }, ctx) + expect(calls.queries).toEqual([ + ["select * from {{ ref('orders') }}", undefined], + [explainJinja("EXPLAIN COST", "compiled(select * from {{ ref('orders') }})"), undefined], + ]) + expect(result.metadata).toEqual({ + success: true, + supported: true, + adapter: "databricks", + source: "explain", + bound: "estimate", + bytes: Math.round(2.5 * 1024 ** 3 + 512 * 1024 ** 2), + display: "3.0 GiB", + truncated: false, + }) + expect(result.output).toContain("not a bound either way") + }) + + test("Databricks without table statistics: no estimate, and says why", async () => { + useAdapter(calls, { type: "databricks", explain: rendered("== Optimized Logical Plan ==\nRelation x.y[a#1] parquet, Statistics(sizeInBytes=8.0 EiB)") }) + const result = await (await initTool(DbtCostEstimateTool)).execute({ model: "orders" }, ctx) + expect(result.metadata).toEqual({ success: false, adapter: "databricks", source: "explain", error: expect.stringContaining("no size for a table"), truncated: false }) + expect(result.metadata.bytes).toBeUndefined() + expect(result.output).toContain("ANALYZE TABLE") + }) + + test("an EXPLAIN the warehouse refuses is no estimate, with the warehouse's error", async () => { + useAdapter(calls, { type: "snowflake", explain: async () => Promise.reject(new Error("SQL compilation error: Object 'ORDERS' does not exist")) }) + const result = await (await initTool(DbtCostEstimateTool)).execute({ model: "orders" }, ctx) + expect(result.title).toBe("dbt_cost_estimate: explain failed") + expect(result.metadata).toMatchObject({ success: false, adapter: "snowflake", source: "explain" }) + expect(result.output).toContain("does not exist") + }) + + test.each([ + ["a plan it can't read", JSON.stringify(['{"Operations":[]}'])], + ["no rows", "[]"], + ["text that is not JSON", "EXPLAIN output"], + ])("%s is no estimate, never a count", async (_, out) => { + useAdapter(calls, { type: "snowflake", explain: async () => out }) + const result = await (await initTool(DbtCostEstimateTool)).execute({ model: "orders" }, ctx) + expect(result.metadata.success).toBe(false) + expect(result.metadata.bytes).toBeUndefined() + }) + + test("a model that doesn't compile is not EXPLAINed", async () => { + useAdapter(calls, { type: "snowflake", compileFails: true, explain: rendered(SNOWFLAKE_PLAN) }) + const result = await (await initTool(DbtCostEstimateTool)).execute({ model: "nope" }, ctx) + expect(result.title).toBe("dbt_cost_estimate: compile failed") + expect(calls.queries).toEqual([]) + }) + + test("a cancelled call stops waiting for the EXPLAIN", async () => { + const abort = new AbortController() + useAdapter(calls, { type: "databricks", explain: () => new Promise(() => {}) }) + const run = (await initTool(DbtCostEstimateTool)).execute({ model: "orders" }, { ...ctx, abort: abort.signal }) + await Bun.sleep(20) + abort.abort(new Error("cancelled")) + await expect(run).rejects.toThrow("cancelled") + }) +}) From e573cd8a1538c9045db73757bd08611c2fb302da Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Thu, 19 Nov 2026 12:36:41 +0530 Subject: [PATCH 02/14] fix: plain EXPLAIN failure reasons and possessive warehouse names Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/tools/dbt-cost-estimate.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts index 4f297b29a4..1036820574 100644 --- a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts +++ b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts @@ -153,9 +153,9 @@ export function parseDatabricksExplainCost(plan: unknown): (PlanEstimate & { lea * The warehouses with no dry run whose planner still says what a query reads, the EXPLAIN that asks it, and the * reader of its one row. */ -const PLANNERS: Record PlanEstimate | { noStats: true } | undefined }> = { - snowflake: { name: "Snowflake", explain: "EXPLAIN USING JSON", read: parseSnowflakeExplain }, - databricks: { name: "Databricks", explain: "EXPLAIN COST", read: parseDatabricksExplainCost }, +const PLANNERS: Record PlanEstimate | { noStats: true } | undefined }> = { + snowflake: { owner: "Snowflake's", explain: "EXPLAIN USING JSON", read: parseSnowflakeExplain }, + databricks: { owner: "Databricks'", explain: "EXPLAIN COST", read: parseDatabricksExplainCost }, } /** @@ -204,18 +204,18 @@ async function explainEstimate( row = rows[0] } catch (e) { if (signal.aborted) throw e - return failure("explain failed", `No estimate: ${planner.name}'s ${planner.explain} failed: ${e instanceof Error ? e.message : String(e)}`, extra) + return failure("explain failed", `${planner.owner} ${planner.explain} failed: ${e instanceof Error ? e.message : String(e)}`, extra) } const parsed = planner.read(row) if (parsed && "noStats" in parsed) { return failure( "no estimate", - `No estimate: ${planner.name}'s planner has no size for a table this query reads (it shows Spark's default for a table without statistics). ANALYZE TABLE
COMPUTE STATISTICS gives it one.`, + `${planner.owner} planner has no size for a table this query reads (it shows Spark's default for a table without statistics). ANALYZE TABLE
COMPUTE STATISTICS gives it one.`, extra, ) } if (!parsed) { - return failure("unreadable estimate", `No estimate: ${planner.name}'s ${planner.explain} answered a plan this tool can't read.`, extra) + return failure("unreadable estimate", `${planner.owner} ${planner.explain} answered a plan this tool can't read.`, extra) } const shown = formatBytes(parsed.bytes) const partitions = @@ -241,7 +241,7 @@ async function explainEstimate( ...(parsed.partitionsTotal !== undefined && { partitionsTotal: parsed.partitionsTotal }), ...(model && { model }), }, - output: `${model ? `${model} ` : "The query "}would scan about ${shown}, by ${planner.name}'s plan estimate (${accuracy}).`, + output: `${model ? `${model} ` : "The query "}would scan about ${shown}, by ${planner.owner} plan estimate (${accuracy}).`, } } From c98261b23eea0d52f23230314e675d84096d8258 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Thu, 19 Nov 2026 12:50:26 +0530 Subject: [PATCH 03/14] fix: EXPLAIN only at dbt's execute pass, and never a script - dbt compile --inline renders the query at parse first, where execute is false and run_query returns None: the template now renders [] there and runs nothing, instead of failing on the missing table. - dbt-snowflake runs every statement it splits off, so a second statement after the EXPLAIN would run for real. A query that may be more than one statement (after setting aside quoted text, identifiers and comments; a doubt counts) gets no estimate and is never sent. Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/tools/dbt-cost-estimate.ts | 26 ++++++++- .../test/altimate/dbt-cost-estimate.test.ts | 56 ++++++++++++++++++- 2 files changed, 78 insertions(+), 4 deletions(-) diff --git a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts index 1036820574..002bfd4530 100644 --- a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts +++ b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts @@ -161,12 +161,27 @@ const PLANNERS: Record adapter.unsafeCompileQuery(explainJinja(planner.explain, compiled), undefined), signal) diff --git a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts index bd3e81983b..8bb771f1dc 100644 --- a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts +++ b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts @@ -12,6 +12,7 @@ import { dbtAdapterInternals, resetDbtAdapter } from "@/altimate/native/connecti import { DbtCostEstimateTool, explainJinja, + mayBeScript, parseBytesProcessed, parseDatabricksExplainCost, parseSnowflakeExplain, @@ -57,7 +58,7 @@ function fakeAdapter( }, unsafeCompileQuery: async (query: string, name?: string) => { calls.queries.push([query, name]) - if (opts.explain && query.startsWith("{%- set altimate_plan")) return opts.explain() + if (opts.explain && query.startsWith("{%- if execute -%}{%- set altimate_plan")) return opts.explain() if (opts.compileFails) throw new Error("compilation failed") return `compiled(${query})` }, @@ -499,10 +500,18 @@ describe("parseDatabricksExplainCost", () => { describe("explainJinja", () => { test("runs the EXPLAIN through dbt's run_query and renders its first column as JSON", () => { expect(explainJinja("EXPLAIN COST", "select 1")).toBe( - "{%- set altimate_plan = run_query('EXPLAIN COST\nselect 1') -%}{{ tojson(altimate_plan.columns[0].values() | list) }}", + "{%- if execute -%}{%- set altimate_plan = run_query('EXPLAIN COST\nselect 1') -%}{{ tojson(altimate_plan.columns[0].values() | list) }}{%- else -%}[]{%- endif -%}", ) }) + test("at dbt's parse pass (execute false) it runs nothing and renders an empty list, not an error", () => { + // dbt compile --inline renders the query at parse first, where run_query returns None: reading its columns there failed. + const template = explainJinja("EXPLAIN USING JSON", "select 1") + const parsePass = /^\{%- if execute -%\}[\s\S]*\{%- else -%\}([\s\S]*)\{%- endif -%\}$/.exec(template) + expect(parsePass?.[1]).toBe("[]") + expect(template.indexOf("run_query")).toBeLessThan(template.indexOf("{%- else -%}")) + }) + test("escapes quotes and backslashes for the Jinja string, and drops a trailing semicolon", () => { expect(explainJinja("EXPLAIN USING JSON", "select * from t where s = 'a\\b';\n")).toContain("run_query('EXPLAIN USING JSON\nselect * from t where s = \\'a\\\\b\\'')") }) @@ -596,3 +605,46 @@ describe("dbt_cost_estimate on a warehouse with a planner", () => { await expect(run).rejects.toThrow("cancelled") }) }) + +describe("mayBeScript", () => { + test.each([ + ["snowflake", "select 1; delete from orders"], + ["snowflake", "select 1;\ndelete from orders;"], + ["snowflake", "select * from t where s = 'a'; drop table t"], + ["snowflake", "select $$a$$; delete from t"], + // Snowflake's own comment and dollar-quoted text are not set aside: a doubt counts as a script. + ["snowflake", "select 1 // note; delete from t"], + ["snowflake", "select $$a;b$$"], + ["snowflake", "execute immediate 'select 1'"], + ["snowflake", "select `a; b` from t"], + ["snowflake", "select 'unclosed; delete from t"], + ["databricks", "select 1; delete from orders"], + ["databricks", "select 1 # c; delete from t"], + ])("on %s, %p may be more than one statement", (type, sql) => { + expect(mayBeScript(sql, type)).toBe(true) + }) + + test.each([ + ["snowflake", "select 1"], + ["snowflake", "select 1;\n "], + ["snowflake", "select 'a;b', \"x;y\" from t -- one; two\n/* three; */"], + ["snowflake", "select 'it''s; fine', 'a\\'; b' from t"], + ["databricks", "select `p;q`.c from t where s = 'a;b'"], + ["databricks", "with x as (select 1) select * from x;"], + ])("on %s, %p is one statement", (type, sql) => { + expect(mayBeScript(sql, type)).toBe(false) + }) +}) + +describe("dbt_cost_estimate refuses to EXPLAIN a script", () => { + for (const type of ["snowflake", "databricks"]) { + test(`on ${type}, a second statement is never sent: no estimate, and only the compile ran`, async () => { + useAdapter(calls, { type, explain: async () => JSON.stringify([SNOWFLAKE_PLAN]) }) + const result = await (await initTool(DbtCostEstimateTool)).execute({ sql: "select 1; delete from orders" }, ctx) + expect(calls.queries).toEqual([["select 1; delete from orders", undefined]]) + expect(result.metadata).toMatchObject({ success: false, adapter: type, source: "explain" }) + expect(result.metadata.bytes).toBeUndefined() + expect(result.output).toContain("more than one statement") + }) + } +}) From eb7bedc84f78c6fcff455130bf9b95a950aad90c Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Thu, 19 Nov 2026 12:58:14 +0530 Subject: [PATCH 04/14] fix: a one-pass script check that matches the Snowflake splitter, and only Spark's own statistics suffix - The script check now reads the SQL left to right, as the Snowflake connector's splitter does (a backslash escapes in both quote kinds), and counts anything it does not model ($$ text, // comments, raw strings, nested or unclosed comments) as a script: text inside one could hide a separator the driver sees. - A Databricks leaf's size is read only from the Statistics suffix Spark appends, so a column named like it is neither a size nor stands in for missing statistics. Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/tools/dbt-cost-estimate.ts | 57 ++++++++++++++++--- .../test/altimate/dbt-cost-estimate.test.ts | 26 +++++++++ 2 files changed, 75 insertions(+), 8 deletions(-) diff --git a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts index 002bfd4530..ab63cc4c4a 100644 --- a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts +++ b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts @@ -141,7 +141,8 @@ export function parseDatabricksExplainCost(plan: unknown): (PlanEstimate & { lea // A leaf is a node the next line does not go below. if (i + 1 < nodes.length && nodes[i + 1].depth > nodes[i].depth) continue leaves++ - const figure = /Statistics\(sizeInBytes=([^,)]+)/.exec(nodes[i].text)?.[1] + // Only the suffix Spark appends to the node: a column can be named anything, "Statistics(sizeInBytes=…)" too. + const figure = /, Statistics\(sizeInBytes=([^,()]+)(?:, [^()]*)?\)\s*$/.exec(nodes[i].text)?.[1] const size = figure === undefined ? undefined : sparkBytes(figure) if (size === undefined || size >= SPARK_NO_SIZE) return { noStats: true } bytes += size @@ -173,15 +174,55 @@ export function explainJinja(explain: string, sql: string): string { /** * Whether the SQL may be more than one statement, as the warehouse's driver splits it: dbt-snowflake runs each - * statement it splits off, so a second one after the EXPLAIN would run for real. Only what certainly hides a - * semicolon is set aside (quoted text and identifiers, `--` and block comments, backticks on Databricks); anything - * else, dollar-quoted text and `//` comments included, counts, so a doubt is no estimate, never a run. + * statement it splits off, so a second one after the EXPLAIN would run for real. One pass, left to right, that + * reads only what it models exactly: single- and double-quoted text (a backslash escapes, as the Snowflake + * connector's splitter reads it), backticks on Databricks, `--` and block comments. Anything else that could hide or fake a separator (Snowflake's `$$` text and `//` comments, + * Databricks' raw strings and nested comments, an unclosed quote or comment, EXECUTE IMMEDIATE) counts as a + * script, as does a semicolon with anything but whitespace after it: a doubt is no estimate, never a run. */ export function mayBeScript(sql: string, type: string): boolean { - const quoted = type === "databricks" ? /--[^\n]*|\/\*[\s\S]*?\*\/|'(?:[^'\\]|\\.)*'|"(?:[^"\\]|\\.)*"|`[^`]*`/g : /--[^\n]*|\/\*[\s\S]*?\*\/|'(?:[^'\\]|\\.)*'|"[^"]*"/g - const bare = sql.replace(quoted, " ").replace(/[\s;]+$/, "") - // An unclosed quote or comment is left in place, and so is any semicolon after it. - return bare.includes(";") || /\bexecute\s+immediate\b/i.test(bare) + const databricks = type === "databricks" + let code = "" + let i = 0 + while (i < sql.length) { + const c = sql[i] + const two = sql.slice(i, i + 2) + if (two === "--") { + const end = sql.indexOf("\n", i) + if (end < 0) break + i = end + 1 + code += " " + continue + } + if (two === "/*") { + const end = sql.indexOf("*/", i + 2) + // Databricks nests block comments, and an unclosed one hides the rest: neither is read. + if (end < 0 || (databricks && sql.slice(i + 2, end).includes("/*"))) return true + i = end + 2 + code += " " + continue + } + if (two === "$$" || (!databricks && two === "//")) return true + // Databricks raw strings (r'…', R"…") do not escape with a backslash: not read. + if (databricks && /[rR]/.test(c) && /['"]/.test(sql[i + 1] ?? "") && !/[A-Za-z0-9_]/.test(sql[i - 1] ?? "")) return true + const close = c === "'" || c === '"' || (databricks && c === "`") ? c : null + if (close) { + // A backslash escapes in quotes, as the splitter has it (only before a quote or a backslash, which skipping the + // next character always matches); Databricks' backticks take none. + const escapes = c !== "`" + let j = i + 1 + while (j < sql.length && sql[j] !== close) j += escapes && sql[j] === "\\" ? 2 : 1 + if (j >= sql.length) return true + i = j + 1 + code += " " + continue + } + code += c + i++ + } + const semicolon = code.indexOf(";") + if (semicolon >= 0 && /[^\s;]/.test(code.slice(semicolon))) return true + return /\bexecute\s+immediate\b/i.test(code) } /** Starts `start` and waits for it, but gives up as soon as the call is cancelled; a cancelled call starts nothing. */ diff --git a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts index 8bb771f1dc..1298a75837 100644 --- a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts +++ b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts @@ -620,6 +620,15 @@ describe("mayBeScript", () => { ["snowflake", "select 'unclosed; delete from t"], ["databricks", "select 1; delete from orders"], ["databricks", "select 1 # c; delete from t"], + // Found in review: text the scan does not model must not hide a separator the driver sees. + ["snowflake", "select $$--$$; delete from orders"], + ["snowflake", "select $$'$$; delete from orders -- '"], + ["snowflake", "select 1 // '\n; delete from orders -- '"], + ["snowflake", 'select "a\\" x" ; delete from orders; "'], + ["snowflake", "select 1 /* unclosed ; delete from orders"], + ["databricks", "select r'\\'; delete from orders -- '"], + ["databricks", "select 1 /* a /* b */ ; delete from orders */"], + ["snowflake", "select 1; -- one\ndelete from orders"], ])("on %s, %p may be more than one statement", (type, sql) => { expect(mayBeScript(sql, type)).toBe(true) }) @@ -631,6 +640,9 @@ describe("mayBeScript", () => { ["snowflake", "select 'it''s; fine', 'a\\'; b' from t"], ["databricks", "select `p;q`.c from t where s = 'a;b'"], ["databricks", "with x as (select 1) select * from x;"], + ["snowflake", "select 1; -- done"], + ["snowflake", "select 'a\\'; b', \"c\\\"; d\" from t"], + ["databricks", "select array_join(arr, ';') from t -- x; y"], ])("on %s, %p is one statement", (type, sql) => { expect(mayBeScript(sql, type)).toBe(false) }) @@ -648,3 +660,17 @@ describe("dbt_cost_estimate refuses to EXPLAIN a script", () => { }) } }) + +describe("parseDatabricksExplainCost reads only the statistics Spark appends", () => { + test("a column named like statistics is not a size", () => { + expect(parseDatabricksExplainCost("== Optimized Logical Plan ==\nRelation main.t[Statistics(sizeInBytes=0.0 B)#0] parquet, Statistics(sizeInBytes=1.0 GiB)")).toEqual({ + bytes: 1024 ** 3, + leaves: 1, + }) + }) + + test("nor stands in for missing statistics, or for Spark's default size", () => { + expect(parseDatabricksExplainCost("== Optimized Logical Plan ==\nRelation main.t[Statistics(sizeInBytes=0.0 B)#0] parquet")).toEqual({ noStats: true }) + expect(parseDatabricksExplainCost("== Optimized Logical Plan ==\nRelation [Statistics(sizeInBytes=0 B)#0] parquet, Statistics(sizeInBytes=8.0 EiB)")).toEqual({ noStats: true }) + }) +}) From de598ebb5e6f1587e620ec469c2d77d675231163 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Thu, 19 Nov 2026 13:08:31 +0530 Subject: [PATCH 05/14] fix: check the SQL with Jinja's line breaks, and the review's follow-ups - Jinja turns a lone \r (and \r\n) in a string into \n, which can end a -- comment the check read as running on: the SQL is normalized first, and the same text is checked and sent. - Only Spark's own section headers end the optimized plan. - No size for a table names it, and the advice is ANALYZE ... NOSCAN, which reads file metadata; a full ANALYZE scans the table. - The description says Databricks runs EXPLAIN on, and may start, the profile's warehouse or cluster. - The parser no longer returns an unused leaf count. Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/tools/dbt-cost-estimate.ts | 34 ++++++---- .../test/altimate/dbt-cost-estimate.test.ts | 63 ++++++++++++++----- 2 files changed, 71 insertions(+), 26 deletions(-) diff --git a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts index ab63cc4c4a..2ef061a6b8 100644 --- a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts +++ b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts @@ -105,6 +105,9 @@ const SPARK_UNITS: Record = { B: 0, KIB: 1, MIB: 2, GIB: 3, TIB: */ const SPARK_NO_SIZE = 2 ** 62 +/** The headers Spark writes between the sections of an EXPLAIN. */ +const SPARK_SECTION = /^== (Parsed Logical Plan|Analyzed Logical Plan|Optimized Logical Plan|Physical Plan) ==\s*$/ + /** A `sizeInBytes=` figure read back into bytes: "12.3 MiB", "16.0 B", or past a long "1.23E+21 B". */ function sparkBytes(figure: string): number | undefined { const match = /^(\d+(?:\.\d+)?(?:E[+-]?\d+)?)\s*([A-Za-z]+)$/i.exec(figure.trim()) @@ -119,16 +122,18 @@ function sparkBytes(figure: string): number | undefined { * The bytes Databricks' `EXPLAIN COST` says the query reads: the sum of `Statistics(sizeInBytes=…)` over the * leaves of the Optimized Logical Plan, the relations it scans. Their sizes are table (or file) sizes from the * catalog and the Delta log, before column pruning and runtime file skipping. `noStats` when a leaf carries no - * size or Spark's default for a relation without statistics; undefined when there is no plan to read. + * size or Spark's default for a relation without statistics (with the relation's name when the leaf gives one); + * undefined when there is no plan to read. */ -export function parseDatabricksExplainCost(plan: unknown): (PlanEstimate & { leaves: number }) | { noStats: true } | undefined { +export function parseDatabricksExplainCost(plan: unknown): PlanEstimate | { noStats: true; relation?: string } | undefined { if (typeof plan !== "string") return undefined const lines = plan.replace(/\r\n?/g, "\n").split("\n") const start = lines.findIndex((line) => /^\s*==\s*Optimized Logical Plan\s*==\s*$/i.test(line)) if (start < 0) return undefined const nodes: { depth: number; text: string }[] = [] for (const line of lines.slice(start + 1)) { - if (/^\s*==.*==\s*$/.test(line)) break + // Only Spark's own section headers end the plan: a line of a multi-line node could look like one. + if (SPARK_SECTION.test(line)) break if (!line.trim()) continue // Each level of the tree is three characters: ": " or " " for an ancestor, ":- " or "+- " for the node itself. const prefix = /^(?:: | )*(?:[:+]- )?/.exec(line)![0] @@ -136,25 +141,27 @@ export function parseDatabricksExplainCost(plan: unknown): (PlanEstimate & { lea } if (nodes.length === 0) return undefined let bytes = 0 - let leaves = 0 for (let i = 0; i < nodes.length; i++) { // A leaf is a node the next line does not go below. if (i + 1 < nodes.length && nodes[i + 1].depth > nodes[i].depth) continue - leaves++ // Only the suffix Spark appends to the node: a column can be named anything, "Statistics(sizeInBytes=…)" too. const figure = /, Statistics\(sizeInBytes=([^,()]+)(?:, [^()]*)?\)\s*$/.exec(nodes[i].text)?.[1] const size = figure === undefined ? undefined : sparkBytes(figure) - if (size === undefined || size >= SPARK_NO_SIZE) return { noStats: true } + if (size === undefined || size >= SPARK_NO_SIZE) { + // "Relation main.sales.orders[…] parquet": the table's name, when the leaf is one. + const relation = /^(?:Relation|HiveTableRelation|LogicalRelation)\s+\[?`?([A-Za-z0-9_.`]+?)`?[\[,\s]/.exec(nodes[i].text)?.[1] + return relation ? { noStats: true, relation: relation.replace(/`/g, "") } : { noStats: true } + } bytes += size } - return { bytes: Math.round(bytes), leaves } + return { bytes: Math.round(bytes) } } /** * The warehouses with no dry run whose planner still says what a query reads, the EXPLAIN that asks it, and the * reader of its one row. */ -const PLANNERS: Record PlanEstimate | { noStats: true } | undefined }> = { +const PLANNERS: Record PlanEstimate | { noStats: true; relation?: string } | undefined }> = { snowflake: { owner: "Snowflake's", explain: "EXPLAIN USING JSON", read: parseSnowflakeExplain }, databricks: { owner: "Databricks'", explain: "EXPLAIN COST", read: parseDatabricksExplainCost }, } @@ -252,7 +259,9 @@ async function explainEstimate( signal: AbortSignal, ) { const extra = { adapter: type, source: "explain" } - if (mayBeScript(compiled, type)) { + // Jinja turns every \r\n and lone \r in a string into \n: the check reads the SQL exactly as the driver will get it. + const sql = compiled.replace(/\r\n?/g, "\n") + if (mayBeScript(sql, type)) { return failure( "no estimate", `The query is more than one statement (or may be): ${planner.explain} takes one, and a second would run for real, so it is not estimated.`, @@ -261,7 +270,7 @@ async function explainEstimate( } let row: unknown try { - const rendered = await untilAbort(() => adapter.unsafeCompileQuery(explainJinja(planner.explain, compiled), undefined), signal) + const rendered = await untilAbort(() => adapter.unsafeCompileQuery(explainJinja(planner.explain, sql), undefined), signal) const rows = typeof rendered === "string" ? JSON.parse(rendered.trim()) : undefined if (!Array.isArray(rows) || rows.length === 0) throw new Error(`${planner.explain} returned no plan`) row = rows[0] @@ -271,9 +280,10 @@ async function explainEstimate( } const parsed = planner.read(row) if (parsed && "noStats" in parsed) { + const table = parsed.relation ?? "
" return failure( "no estimate", - `${planner.owner} planner has no size for a table this query reads (it shows Spark's default for a table without statistics). ANALYZE TABLE
COMPUTE STATISTICS gives it one.`, + `${planner.owner} planner has no size for ${parsed.relation ?? "a table this query reads"} (it shows Spark's default for a table without statistics). ANALYZE TABLE ${table} COMPUTE STATISTICS NOSCAN records its size from file metadata without reading the data; without NOSCAN, ANALYZE scans the whole table.`, extra, ) } @@ -310,7 +320,7 @@ async function explainEstimate( export const DbtCostEstimateTool = Tool.define("dbt_cost_estimate", { description: - "Estimate the bytes a dbt model or a SQL query would scan, without running it. BigQuery answers from a dry run (`maxBytes` bounds it, or `lowerBound` marks a floor); Snowflake (`EXPLAIN USING JSON`, bytes left after pruning) and Databricks (`EXPLAIN COST`, the sizes of the tables it reads) answer from the planner (`source: \"explain\"`; `bound` is \"upper\" on Snowflake, \"estimate\" on Databricks), and none when Databricks has no table statistics. EXPLAIN compiles the query on the warehouse and scans no data. Any other warehouse answers `supported: false` and nothing is compiled. Starts the dbt adapter (which runs `dbt deps`), so it needs write permission (`--allow-write` on `tool run` or the MCP server). The dry run never executes the final query, but compiling either input can render Jinja, and macros such as `run_query` can run real queries that are billed. Pass `model` (a model name, compiled as dbt would) or `sql` (may use Jinja).", + "Estimate the bytes a dbt model or a SQL query would scan, without running it. BigQuery answers from a dry run (`maxBytes` bounds it, or `lowerBound` marks a floor); Snowflake (`EXPLAIN USING JSON`, bytes left after pruning) and Databricks (`EXPLAIN COST`, the sizes of the tables it reads) answer from the planner (`source: \"explain\"`; `bound` is \"upper\" on Snowflake, \"estimate\" on Databricks), and none when Databricks has no table statistics. EXPLAIN compiles the query on the warehouse and scans no data: on Snowflake it needs no running warehouse (cloud-services time only); on Databricks it runs on the SQL warehouse or cluster the profile connects to, and starts it (billed while up) if it is stopped. A query that may be more than one statement is never EXPLAINed. Any other warehouse answers `supported: false` and nothing is compiled. Starts the dbt adapter (which runs `dbt deps`), so it needs write permission (`--allow-write` on `tool run` or the MCP server). The dry run never executes the final query, but compiling either input can render Jinja, and macros such as `run_query` can run real queries that are billed. Pass `model` (a model name, compiled as dbt would) or `sql` (may use Jinja).", parameters: z.object({ model: z.string().optional().describe("Model name to compile and estimate"), sql: z.string().optional().describe("SQL (may use Jinja) to compile and estimate"), diff --git a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts index 1298a75837..afc4f6d7c1 100644 --- a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts +++ b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts @@ -230,7 +230,7 @@ describe("dbt_cost_estimate", () => { const result = await (await initTool(DbtCostEstimateTool)).execute({ model: "orders" }, ctx) expect(result.metadata).toMatchObject({ success: true, supported: false, adapter: type }) expect(result.metadata.bytes).toBeUndefined() - expect(result.output).toContain("only available on BigQuery") + expect(result.output).toBe(`No cost estimate: scan estimates are only available on BigQuery, Snowflake and Databricks (this project uses ${type}).`) expect(calls).toEqual({ nodes: [], queries: [], dryRuns: [] }) }) @@ -444,12 +444,12 @@ describe("parseSnowflakeExplain", () => { describe("parseDatabricksExplainCost", () => { test("sums the leaves of the optimized plan: the relations it scans, not the operators above them", () => { - expect(parseDatabricksExplainCost(DATABRICKS_PLAN)).toEqual({ bytes: Math.round(2.5 * 1024 ** 3 + 512 * 1024 ** 2), leaves: 2 }) + expect(parseDatabricksExplainCost(DATABRICKS_PLAN)).toEqual({ bytes: Math.round(2.5 * 1024 ** 3 + 512 * 1024 ** 2) }) }) test("a single relation and Spark's B unit", () => { const plan = "== Optimized Logical Plan ==\nProject [id#0L], Statistics(sizeInBytes=12.0 B)\n+- Relation main.t[id#0L] parquet, Statistics(sizeInBytes=16.0 B, rowCount=2)\n\n== Physical Plan ==\n*(1) ColumnarToRow" - expect(parseDatabricksExplainCost(plan)).toEqual({ bytes: 16, leaves: 1 }) + expect(parseDatabricksExplainCost(plan)).toEqual({ bytes: 16 }) }) test("a scalar subquery's plan is printed inside the tree, and its relation counts too", () => { @@ -462,13 +462,12 @@ describe("parseDatabricksExplainCost", () => { "", "== Physical Plan ==", ].join("\n") - expect(parseDatabricksExplainCost(plan)).toEqual({ bytes: Math.round(1.5 * 1024 ** 3 + 3 * 1024 ** 2), leaves: 2 }) + expect(parseDatabricksExplainCost(plan)).toEqual({ bytes: Math.round(1.5 * 1024 ** 3 + 3 * 1024 ** 2) }) }) test("reads Spark 2's KB/MB units and the scientific form past a long", () => { expect(parseDatabricksExplainCost("== Optimized Logical Plan ==\nRelation[a#0] parquet, Statistics(sizeInBytes=2.0 MB, hints=none)")).toEqual({ - bytes: 2 * 1024 ** 2, - leaves: 1, + bytes: 2 * 1024 ** 2 , }) // A size past 2^62 is Spark's "no statistics" default or larger: not a size. expect(parseDatabricksExplainCost("== Optimized Logical Plan ==\nRelation[a#0] parquet, Statistics(sizeInBytes=1.23E+21 B)")).toEqual({ noStats: true }) @@ -476,11 +475,11 @@ describe("parseDatabricksExplainCost", () => { test("a relation without statistics shows Spark's default size, Long.MaxValue: no estimate, not 8 EiB", () => { const plan = "== Optimized Logical Plan ==\nJoin Inner, Statistics(sizeInBytes=8.0 EiB)\n:- Relation main.a[x#1] parquet, Statistics(sizeInBytes=1.0 GiB)\n+- Relation jdbc.b[x#2] JDBCRelation, Statistics(sizeInBytes=8.0 EiB)" - expect(parseDatabricksExplainCost(plan)).toEqual({ noStats: true }) + expect(parseDatabricksExplainCost(plan)).toEqual({ noStats: true, relation: "jdbc.b" }) }) test("a leaf with no statistics at all is no estimate", () => { - expect(parseDatabricksExplainCost("== Optimized Logical Plan ==\nProject [a#0], Statistics(sizeInBytes=1.0 KiB)\n+- Relation main.t[a#0] parquet")).toEqual({ noStats: true }) + expect(parseDatabricksExplainCost("== Optimized Logical Plan ==\nProject [a#0], Statistics(sizeInBytes=1.0 KiB)\n+- Relation main.t[a#0] parquet")).toEqual({ noStats: true, relation: "main.t" }) }) test.each([ @@ -493,7 +492,7 @@ describe("parseDatabricksExplainCost", () => { }) test("reads Windows line endings", () => { - expect(parseDatabricksExplainCost("== Optimized Logical Plan ==\r\nRelation main.t[a#0] parquet, Statistics(sizeInBytes=4.0 KiB)\r\n")).toEqual({ bytes: 4096, leaves: 1 }) + expect(parseDatabricksExplainCost("== Optimized Logical Plan ==\r\nRelation main.t[a#0] parquet, Statistics(sizeInBytes=4.0 KiB)\r\n")).toEqual({ bytes: 4096 }) }) }) @@ -565,9 +564,10 @@ describe("dbt_cost_estimate on a warehouse with a planner", () => { test("Databricks without table statistics: no estimate, and says why", async () => { useAdapter(calls, { type: "databricks", explain: rendered("== Optimized Logical Plan ==\nRelation x.y[a#1] parquet, Statistics(sizeInBytes=8.0 EiB)") }) const result = await (await initTool(DbtCostEstimateTool)).execute({ model: "orders" }, ctx) - expect(result.metadata).toEqual({ success: false, adapter: "databricks", source: "explain", error: expect.stringContaining("no size for a table"), truncated: false }) + expect(result.metadata).toEqual({ success: false, adapter: "databricks", source: "explain", error: expect.stringContaining("no size for x.y"), truncated: false }) expect(result.metadata.bytes).toBeUndefined() - expect(result.output).toContain("ANALYZE TABLE") + expect(result.output).toContain("ANALYZE TABLE x.y COMPUTE STATISTICS NOSCAN") + expect(result.output).toContain("no size for x.y") }) test("an EXPLAIN the warehouse refuses is no estimate, with the warehouse's error", async () => { @@ -664,13 +664,48 @@ describe("dbt_cost_estimate refuses to EXPLAIN a script", () => { describe("parseDatabricksExplainCost reads only the statistics Spark appends", () => { test("a column named like statistics is not a size", () => { expect(parseDatabricksExplainCost("== Optimized Logical Plan ==\nRelation main.t[Statistics(sizeInBytes=0.0 B)#0] parquet, Statistics(sizeInBytes=1.0 GiB)")).toEqual({ - bytes: 1024 ** 3, - leaves: 1, + bytes: 1024 ** 3 , }) }) test("nor stands in for missing statistics, or for Spark's default size", () => { - expect(parseDatabricksExplainCost("== Optimized Logical Plan ==\nRelation main.t[Statistics(sizeInBytes=0.0 B)#0] parquet")).toEqual({ noStats: true }) + expect(parseDatabricksExplainCost("== Optimized Logical Plan ==\nRelation main.t[Statistics(sizeInBytes=0.0 B)#0] parquet")).toEqual({ noStats: true, relation: "main.t" }) expect(parseDatabricksExplainCost("== Optimized Logical Plan ==\nRelation [Statistics(sizeInBytes=0 B)#0] parquet, Statistics(sizeInBytes=8.0 EiB)")).toEqual({ noStats: true }) }) }) + +describe("review follow-ups", () => { + test("a lone carriage return, which Jinja turns into a line break, cannot end a comment the check reads as running on", async () => { + for (const type of ["snowflake", "databricks"]) { + expect(mayBeScript("select 1 -- c\n; delete from orders", type)).toBe(true) + const local: Calls = { nodes: [], queries: [], dryRuns: [] } + useAdapter(local, { type, compile: async () => "select 1 -- comment\r; delete from orders", explain: async () => JSON.stringify([SNOWFLAKE_PLAN]) }) + const result = await (await initTool(DbtCostEstimateTool)).execute({ model: "orders" }, ctx) + expect(local.queries).toEqual([]) + expect(result.metadata).toMatchObject({ success: false, source: "explain" }) + expect(result.output).toContain("more than one statement") + } + }) + + test("the SQL sent is the one checked: line endings are already line breaks", async () => { + useAdapter(calls, { type: "snowflake", compile: async () => "select 1\r\nfrom t\rwhere x", explain: async () => JSON.stringify([SNOWFLAKE_PLAN]) }) + await (await initTool(DbtCostEstimateTool)).execute({ model: "orders" }, ctx) + expect(calls.queries).toEqual([[explainJinja("EXPLAIN USING JSON", "select 1\nfrom t\nwhere x"), undefined]]) + }) + + test("only Spark's own section headers end the optimized plan", () => { + const plan = [ + "== Optimized Logical Plan ==", + "Union, Statistics(sizeInBytes=2.0 GiB)", + ":- Project [concat(a#1, ==, note ==) AS x#3], Statistics(sizeInBytes=1.0 GiB)", + "== note ==", + ": +- Relation main.s.a[a#1] parquet, Statistics(sizeInBytes=1.0 GiB)", + "+- Relation main.s.b[b#2] parquet, Statistics(sizeInBytes=1.0 GiB)", + "", + "== Physical Plan ==", + "Union", + ].join("\n") + // The stray line is read as a node, not as the end: both relations count, never fewer. + expect((parseDatabricksExplainCost(plan) as { bytes: number }).bytes).toBeGreaterThanOrEqual(2 * 1024 ** 3) + }) +}) From bee287bd99f48031481b31baf947646405a97ec0 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Thu, 19 Nov 2026 13:20:17 +0530 Subject: [PATCH 06/14] fix: a line starting with ! (a Snowflake client command) and a nested comment are never EXPLAINed Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/tools/dbt-cost-estimate.ts | 12 ++++---- .../test/altimate/dbt-cost-estimate.test.ts | 30 +++++++++++++++++++ 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts index 2ef061a6b8..e7fea8b980 100644 --- a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts +++ b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts @@ -183,12 +183,14 @@ export function explainJinja(explain: string, sql: string): string { * Whether the SQL may be more than one statement, as the warehouse's driver splits it: dbt-snowflake runs each * statement it splits off, so a second one after the EXPLAIN would run for real. One pass, left to right, that * reads only what it models exactly: single- and double-quoted text (a backslash escapes, as the Snowflake - * connector's splitter reads it), backticks on Databricks, `--` and block comments. Anything else that could hide or fake a separator (Snowflake's `$$` text and `//` comments, - * Databricks' raw strings and nested comments, an unclosed quote or comment, EXECUTE IMMEDIATE) counts as a - * script, as does a semicolon with anything but whitespace after it: a doubt is no estimate, never a run. + * connector's splitter reads it), backticks on Databricks, `--` and block comments. Anything else that could hide or + * fake a separator (Snowflake's `$$` text, `//` comments and `!` commands at a line's start, raw strings, nested + * comments, an unclosed quote or comment, EXECUTE IMMEDIATE) counts as a script, as does a semicolon with anything but whitespace after it: a doubt is no estimate, never a run. */ export function mayBeScript(sql: string, type: string): boolean { const databricks = type === "databricks" + // The Snowflake splitter yields a line starting with "!" as a command of its own, wherever it sits: never read. + if (!databricks && /(^|\n)!/.test(sql)) return true let code = "" let i = 0 while (i < sql.length) { @@ -203,8 +205,8 @@ export function mayBeScript(sql: string, type: string): boolean { } if (two === "/*") { const end = sql.indexOf("*/", i + 2) - // Databricks nests block comments, and an unclosed one hides the rest: neither is read. - if (end < 0 || (databricks && sql.slice(i + 2, end).includes("/*"))) return true + // Databricks nests block comments, and an unclosed one hides the rest: neither is read, on either warehouse. + if (end < 0 || sql.slice(i + 2, end).includes("/*")) return true i = end + 2 code += " " continue diff --git a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts index afc4f6d7c1..d6d93c7a37 100644 --- a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts +++ b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts @@ -709,3 +709,33 @@ describe("review follow-ups", () => { expect((parseDatabricksExplainCost(plan) as { bytes: number }).bytes).toBeGreaterThanOrEqual(2 * 1024 ** 3) }) }) + +describe("review follow-ups, round four", () => { + test.each([ + ["snowflake", "select 1\n!foo"], + ["snowflake", "!set x=1\nselect 1"], + ["snowflake", "select 1 /* outer /* inner */ */"], + ["databricks", "select 1 /* outer /* inner */ */"], + ])("on %s, %p is not read: no estimate", (type, sql) => { + expect(mayBeScript(sql, type)).toBe(true) + }) + + for (const [type, compiled] of [ + ["snowflake", "select 1\n!foo"], + ["snowflake", "select 1\r!foo"], + ["snowflake", "select 1\r\n!foo"], + ["snowflake", "select 1 /* outer /* inner */ */"], + ] as const) { + test(`on ${type}, ${JSON.stringify(compiled)} never reaches the EXPLAIN`, async () => { + const local: Calls = { nodes: [], queries: [], dryRuns: [] } + useAdapter(local, { type, compile: async () => compiled, explain: async () => JSON.stringify([SNOWFLAKE_PLAN]) }) + const result = await (await initTool(DbtCostEstimateTool)).execute({ model: "orders" }, ctx) + expect(local.queries).toEqual([]) + expect(result.metadata).toMatchObject({ success: false, source: "explain" }) + }) + } + + test("a ! inside a line is an operator, not a command", () => { + expect(mayBeScript("select a from t where a != 1", "snowflake")).toBe(false) + }) +}) From 5c9d024b40474509377dbf647d2f23b5cb283f90 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Thu, 19 Nov 2026 13:27:21 +0530 Subject: [PATCH 07/14] fix: nothing but comments after a semicolon, overlapping comment openers, and only a plain relation name in the advice - Past a semicolon only whitespace, comments and more semicolons may follow: quoted text there is a second statement too. - "/*/*/" opens a nested comment on Databricks: an opener overlapping the closer counts as nested. - The script check reads the line breaks Jinja renders itself. - Spark prints a relation's name unquoted, so the advice names it only when it is plain (letters, digits, underscores, dots) before the column list. - The cancellation test aborts once the EXPLAIN has started. Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/tools/dbt-cost-estimate.ts | 20 ++++--- .../test/altimate/dbt-cost-estimate.test.ts | 56 ++++++++++++++++++- 2 files changed, 67 insertions(+), 9 deletions(-) diff --git a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts index e7fea8b980..f47989dba5 100644 --- a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts +++ b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts @@ -148,9 +148,10 @@ export function parseDatabricksExplainCost(plan: unknown): PlanEstimate | { noSt const figure = /, Statistics\(sizeInBytes=([^,()]+)(?:, [^()]*)?\)\s*$/.exec(nodes[i].text)?.[1] const size = figure === undefined ? undefined : sparkBytes(figure) if (size === undefined || size >= SPARK_NO_SIZE) { - // "Relation main.sales.orders[…] parquet": the table's name, when the leaf is one. - const relation = /^(?:Relation|HiveTableRelation|LogicalRelation)\s+\[?`?([A-Za-z0-9_.`]+?)`?[\[,\s]/.exec(nodes[i].text)?.[1] - return relation ? { noStats: true, relation: relation.replace(/`/g, "") } : { noStats: true } + // "Relation main.sales.orders[…] parquet": Spark prints the name unquoted, so only a plain one (letters, digits, + // underscores, dots) right before the column list is surely the whole name; any other gives none. + const relation = /^Relation ([A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*)\[/.exec(nodes[i].text)?.[1] + return relation ? { noStats: true, relation } : { noStats: true } } bytes += size } @@ -187,11 +188,15 @@ export function explainJinja(explain: string, sql: string): string { * fake a separator (Snowflake's `$$` text, `//` comments and `!` commands at a line's start, raw strings, nested * comments, an unclosed quote or comment, EXECUTE IMMEDIATE) counts as a script, as does a semicolon with anything but whitespace after it: a doubt is no estimate, never a run. */ -export function mayBeScript(sql: string, type: string): boolean { +export function mayBeScript(input: string, type: string): boolean { + // Read with the line breaks Jinja renders: every \r\n and lone \r is \n. + const sql = input.replace(/\r\n?/g, "\n") const databricks = type === "databricks" // The Snowflake splitter yields a line starting with "!" as a command of its own, wherever it sits: never read. if (!databricks && /(^|\n)!/.test(sql)) return true let code = "" + // Past a semicolon only whitespace and comments may follow: anything else, quoted or not, is a second statement. + let ended = false let i = 0 while (i < sql.length) { const c = sql[i] @@ -206,7 +211,8 @@ export function mayBeScript(sql: string, type: string): boolean { if (two === "/*") { const end = sql.indexOf("*/", i + 2) // Databricks nests block comments, and an unclosed one hides the rest: neither is read, on either warehouse. - if (end < 0 || sql.slice(i + 2, end).includes("/*")) return true + // The closer's star counts too: in "/*/*/" an opener overlaps it. + if (end < 0 || sql.slice(i + 2, end + 1).includes("/*")) return true i = end + 2 code += " " continue @@ -214,6 +220,8 @@ export function mayBeScript(sql: string, type: string): boolean { if (two === "$$" || (!databricks && two === "//")) return true // Databricks raw strings (r'…', R"…") do not escape with a backslash: not read. if (databricks && /[rR]/.test(c) && /['"]/.test(sql[i + 1] ?? "") && !/[A-Za-z0-9_]/.test(sql[i - 1] ?? "")) return true + if (ended && !/[\s;]/.test(c)) return true + if (c === ";") ended = true const close = c === "'" || c === '"' || (databricks && c === "`") ? c : null if (close) { // A backslash escapes in quotes, as the splitter has it (only before a quote or a backslash, which skipping the @@ -229,8 +237,6 @@ export function mayBeScript(sql: string, type: string): boolean { code += c i++ } - const semicolon = code.indexOf(";") - if (semicolon >= 0 && /[^\s;]/.test(code.slice(semicolon))) return true return /\bexecute\s+immediate\b/i.test(code) } diff --git a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts index d6d93c7a37..ad98bc277a 100644 --- a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts +++ b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts @@ -598,9 +598,18 @@ describe("dbt_cost_estimate on a warehouse with a planner", () => { test("a cancelled call stops waiting for the EXPLAIN", async () => { const abort = new AbortController() - useAdapter(calls, { type: "databricks", explain: () => new Promise(() => {}) }) + let started!: () => void + const explaining = new Promise((resolve) => (started = resolve)) + useAdapter(calls, { + type: "databricks", + explain: () => { + started() + return new Promise(() => {}) + }, + }) const run = (await initTool(DbtCostEstimateTool)).execute({ model: "orders" }, { ...ctx, abort: abort.signal }) - await Bun.sleep(20) + // Cancelled once the EXPLAIN is under way, not before it starts. + await explaining abort.abort(new Error("cancelled")) await expect(run).rejects.toThrow("cancelled") }) @@ -739,3 +748,46 @@ describe("review follow-ups, round four", () => { expect(mayBeScript("select a from t where a != 1", "snowflake")).toBe(false) }) }) + +describe("review follow-ups, round five", () => { + test.each([ + ["snowflake", "select 1; 'second\nstatement'"], + ["snowflake", 'select 1; "x"'], + ["databricks", "select 1; `x`"], + ["snowflake", "select 1 /*/*/"], + ["databricks", "select 1 /*/*/ ' */ */ ; delete from t -- '"], + ["snowflake", "select 1 -- c\r; delete from orders"], + ])("on %s, %p is not read: no estimate", (type, sql) => { + expect(mayBeScript(sql, type)).toBe(true) + }) + + test.each([ + ["snowflake", "select 1;;\n"], + ["snowflake", "select 1; /* done */ -- really"], + ])("on %s, %p is one statement", (type, sql) => { + expect(mayBeScript(sql, type)).toBe(false) + }) + + test("quoted text after a semicolon never reaches the EXPLAIN", async () => { + useAdapter(calls, { type: "snowflake", compile: async () => "select 1; 'second\nstatement'", explain: async () => JSON.stringify([SNOWFLAKE_PLAN]) }) + const result = await (await initTool(DbtCostEstimateTool)).execute({ model: "orders" }, ctx) + expect(calls.queries).toEqual([]) + expect(result.metadata).toMatchObject({ success: false, source: "explain" }) + }) + + test.each([ + ["a comma", "Relation main.sales.orders,archive[id#0] parquet, Statistics(sizeInBytes=8.0 EiB)"], + ["a hyphen", "Relation main.sales.my-table[id#0] parquet, Statistics(sizeInBytes=8.0 EiB)"], + ["a space", "Relation main.sales.my table[id#0] parquet, Statistics(sizeInBytes=8.0 EiB)"], + ["a Hive relation", "HiveTableRelation [`spark_catalog`.`default`.`t`, org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe], Statistics(sizeInBytes=8.0 EiB)"], + ])("a relation whose printed name has %s is not named: Spark prints it unquoted, so it may not be the whole name", (_, leaf) => { + expect(parseDatabricksExplainCost(`== Optimized Logical Plan ==\n${leaf}`)).toEqual({ noStats: true }) + }) + + test("a plain relation name is named in the advice", () => { + expect(parseDatabricksExplainCost("== Optimized Logical Plan ==\nRelation main.sales.orders[id#0] parquet, Statistics(sizeInBytes=8.0 EiB)")).toEqual({ + noStats: true, + relation: "main.sales.orders", + }) + }) +}) From e67e6ad4440c3d299455a7fb1c5db6dc3402f9f2 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Thu, 19 Nov 2026 13:33:53 +0530 Subject: [PATCH 08/14] fix: a plan node continued on its own line is one node, and no EXPLAIN for a Databricks query over files - A tree has one root, so a later line with no prefix is the rest of the node above it (a literal with a line break): it is joined, not counted as a leaf. - Databricks infers the schema of files a query reads directly (read_files, cloud_files, json.`path`) by reading them while it plans, so such a query gets no estimate and no EXPLAIN. The description says so. Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/tools/dbt-cost-estimate.ts | 18 ++++++++- .../test/altimate/dbt-cost-estimate.test.ts | 40 ++++++++++++++++++- 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts index f47989dba5..c401589dd4 100644 --- a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts +++ b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts @@ -137,6 +137,11 @@ export function parseDatabricksExplainCost(plan: unknown): PlanEstimate | { noSt if (!line.trim()) continue // Each level of the tree is three characters: ": " or " " for an ancestor, ":- " or "+- " for the node itself. const prefix = /^(?:: | )*(?:[:+]- )?/.exec(line)![0] + // A tree has one root: a later line with no prefix is the rest of the node above it (a literal with a line break). + if (!prefix && nodes.length > 0) { + nodes[nodes.length - 1].text += `\n${line}` + continue + } nodes.push({ depth: prefix.length / 3, text: line.slice(prefix.length) }) } if (nodes.length === 0) return undefined @@ -158,6 +163,9 @@ export function parseDatabricksExplainCost(plan: unknown): PlanEstimate | { noSt return { bytes: Math.round(bytes) } } +/** Databricks queries over files rather than tables: planning one can read the files to infer their schema. */ +const READS_FILES = /\b(read_files|cloud_files)\s*\(|\b(json|csv|parquet|avro|orc|text|binaryFile|xml)\s*\.\s*`/i + /** * The warehouses with no dry run whose planner still says what a query reads, the EXPLAIN that asks it, and the * reader of its one row. @@ -269,6 +277,14 @@ async function explainEstimate( const extra = { adapter: type, source: "explain" } // Jinja turns every \r\n and lone \r in a string into \n: the check reads the SQL exactly as the driver will get it. const sql = compiled.replace(/\r\n?/g, "\n") + // Databricks infers the schema of files a query reads directly by reading them, while it plans: an EXPLAIN would. + if (type === "databricks" && READS_FILES.test(sql)) { + return failure( + "no estimate", + "The query reads files directly (read_files, cloud_files or a path such as json.`…`): Databricks infers their schema by reading them while it plans, so an EXPLAIN is not run for it.", + extra, + ) + } if (mayBeScript(sql, type)) { return failure( "no estimate", @@ -328,7 +344,7 @@ async function explainEstimate( export const DbtCostEstimateTool = Tool.define("dbt_cost_estimate", { description: - "Estimate the bytes a dbt model or a SQL query would scan, without running it. BigQuery answers from a dry run (`maxBytes` bounds it, or `lowerBound` marks a floor); Snowflake (`EXPLAIN USING JSON`, bytes left after pruning) and Databricks (`EXPLAIN COST`, the sizes of the tables it reads) answer from the planner (`source: \"explain\"`; `bound` is \"upper\" on Snowflake, \"estimate\" on Databricks), and none when Databricks has no table statistics. EXPLAIN compiles the query on the warehouse and scans no data: on Snowflake it needs no running warehouse (cloud-services time only); on Databricks it runs on the SQL warehouse or cluster the profile connects to, and starts it (billed while up) if it is stopped. A query that may be more than one statement is never EXPLAINed. Any other warehouse answers `supported: false` and nothing is compiled. Starts the dbt adapter (which runs `dbt deps`), so it needs write permission (`--allow-write` on `tool run` or the MCP server). The dry run never executes the final query, but compiling either input can render Jinja, and macros such as `run_query` can run real queries that are billed. Pass `model` (a model name, compiled as dbt would) or `sql` (may use Jinja).", + "Estimate the bytes a dbt model or a SQL query would scan, without running it. BigQuery answers from a dry run (`maxBytes` bounds it, or `lowerBound` marks a floor); Snowflake (`EXPLAIN USING JSON`, bytes left after pruning) and Databricks (`EXPLAIN COST`, the sizes of the tables it reads) answer from the planner (`source: \"explain\"`; `bound` is \"upper\" on Snowflake, \"estimate\" on Databricks), and none when Databricks has no table statistics. EXPLAIN compiles the query on the warehouse and scans no data: on Snowflake it needs no running warehouse (cloud-services time only); on Databricks it runs on the SQL warehouse or cluster the profile connects to, and starts it (billed while up) if it is stopped. A query that may be more than one statement is never EXPLAINed, nor on Databricks one that reads files directly (read_files, cloud_files, json.`path`), whose planning reads the files to infer a schema. Any other warehouse answers `supported: false` and nothing is compiled. Starts the dbt adapter (which runs `dbt deps`), so it needs write permission (`--allow-write` on `tool run` or the MCP server). The dry run never executes the final query, but compiling either input can render Jinja, and macros such as `run_query` can run real queries that are billed. Pass `model` (a model name, compiled as dbt would) or `sql` (may use Jinja).", parameters: z.object({ model: z.string().optional().describe("Model name to compile and estimate"), sql: z.string().optional().describe("SQL (may use Jinja) to compile and estimate"), diff --git a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts index ad98bc277a..df40598dda 100644 --- a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts +++ b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts @@ -714,8 +714,8 @@ describe("review follow-ups", () => { "== Physical Plan ==", "Union", ].join("\n") - // The stray line is read as a node, not as the end: both relations count, never fewer. - expect((parseDatabricksExplainCost(plan) as { bytes: number }).bytes).toBeGreaterThanOrEqual(2 * 1024 ** 3) + // The stray line is the rest of the Project above it, not the end of the plan and not a node: the two relations, exactly. + expect(parseDatabricksExplainCost(plan)).toEqual({ bytes: 2 * 1024 ** 3 }) }) }) @@ -791,3 +791,39 @@ describe("review follow-ups, round five", () => { }) }) }) + +describe("review follow-ups, round six", () => { + test("a node continued on a line of its own is one node: its size is not added as a leaf", () => { + const plan = [ + "== Optimized Logical Plan ==", + "Project [concat(a#1, line one", + "line two) AS x#3], Statistics(sizeInBytes=5.0 GiB)", + "+- Relation main.s.a[a#1] parquet, Statistics(sizeInBytes=1.0 GiB)", + ].join("\n") + expect(parseDatabricksExplainCost(plan)).toEqual({ bytes: 1024 ** 3 }) + }) + + test.each([ + "select * from read_files('/Volumes/c/s/v/raw', format => 'json')", + "select * from READ_FILES ('/Volumes/c/s/v/raw')", + "select * from cloud_files('/mnt/raw', 'json')", + "select * from json.`/Volumes/c/s/v/raw`", + "select * from parquet . `s3://bucket/path`", + ])("on Databricks, %p reads files to plan: no EXPLAIN is sent", async (compiled) => { + const local: Calls = { nodes: [], queries: [], dryRuns: [] } + useAdapter(local, { type: "databricks", compile: async () => compiled, explain: async () => JSON.stringify([DATABRICKS_PLAN]) }) + const result = await (await initTool(DbtCostEstimateTool)).execute({ model: "orders" }, ctx) + expect(local.queries).toEqual([]) + expect(result.metadata).toMatchObject({ success: false, adapter: "databricks", source: "explain" }) + expect(result.output).toContain("reads files directly") + }) + + test("a Delta path and a table are still estimated", async () => { + for (const compiled of ["select * from delta.`/Volumes/c/s/v/t`", "select * from main.sales.orders"]) { + const local: Calls = { nodes: [], queries: [], dryRuns: [] } + useAdapter(local, { type: "databricks", compile: async () => compiled, explain: async () => JSON.stringify([DATABRICKS_PLAN]) }) + const result = await (await initTool(DbtCostEstimateTool)).execute({ model: "orders" }, ctx) + expect(result.metadata).toMatchObject({ success: true, source: "explain" }) + } + }) +}) From 4e9a2496b7b395e880aea7dd18e5e8e8fc3233dd Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Thu, 19 Nov 2026 13:39:31 +0530 Subject: [PATCH 09/14] fix: a file read split by a comment or with backtick-quoted names is still found The check for a Databricks query over files runs after the script check (every quote and comment closed, none nested), on the text with its comments as spaces, and takes backtick-quoted function and format names. Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/tools/dbt-cost-estimate.ts | 24 +++++++++---- .../test/altimate/dbt-cost-estimate.test.ts | 34 +++++++++++++++++++ 2 files changed, 51 insertions(+), 7 deletions(-) diff --git a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts index c401589dd4..88429e8047 100644 --- a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts +++ b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts @@ -163,8 +163,18 @@ export function parseDatabricksExplainCost(plan: unknown): PlanEstimate | { noSt return { bytes: Math.round(bytes) } } -/** Databricks queries over files rather than tables: planning one can read the files to infer their schema. */ -const READS_FILES = /\b(read_files|cloud_files)\s*\(|\b(json|csv|parquet|avro|orc|text|binaryFile|xml)\s*\.\s*`/i +/** + * Whether a Databricks query reads files rather than tables (read_files, cloud_files, a path such as json.`…`): + * planning one can read the files to infer their schema. Read on the text with its comments as spaces (quoted text + * kept, so a comment marker inside it stays) and any spacing between the name and what follows; the names may be + * backtick-quoted. Called once `mayBeScript` has passed: every quote and comment is closed and none nests. + */ +export function readsFiles(sql: string): boolean { + const bare = sql + .replace(/'(?:[^'\\]|\\.)*'|"(?:[^"\\]|\\.)*"|`[^`]*`|--[^\n]*|\/\*[\s\S]*?\*\//g, (token) => (token.startsWith("--") || token.startsWith("/*") ? " " : token)) + .toLowerCase() + return /(^|[^a-z0-9_])`?(read_files|cloud_files)`?\s*\(|(^|[^a-z0-9_])`?(json|csv|parquet|avro|orc|text|binaryfile|xml)`?\s*\.\s*`/.test(bare) +} /** * The warehouses with no dry run whose planner still says what a query reads, the EXPLAIN that asks it, and the @@ -277,18 +287,18 @@ async function explainEstimate( const extra = { adapter: type, source: "explain" } // Jinja turns every \r\n and lone \r in a string into \n: the check reads the SQL exactly as the driver will get it. const sql = compiled.replace(/\r\n?/g, "\n") - // Databricks infers the schema of files a query reads directly by reading them, while it plans: an EXPLAIN would. - if (type === "databricks" && READS_FILES.test(sql)) { + if (mayBeScript(sql, type)) { return failure( "no estimate", - "The query reads files directly (read_files, cloud_files or a path such as json.`…`): Databricks infers their schema by reading them while it plans, so an EXPLAIN is not run for it.", + `The query is more than one statement (or may be): ${planner.explain} takes one, and a second would run for real, so it is not estimated.`, extra, ) } - if (mayBeScript(sql, type)) { + // Databricks infers the schema of files a query reads directly by reading them, while it plans: an EXPLAIN would. + if (type === "databricks" && readsFiles(sql)) { return failure( "no estimate", - `The query is more than one statement (or may be): ${planner.explain} takes one, and a second would run for real, so it is not estimated.`, + "The query reads files directly (read_files, cloud_files or a path such as json.`…`): Databricks infers their schema by reading them while it plans, so an EXPLAIN is not run for it.", extra, ) } diff --git a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts index df40598dda..cf31aacd59 100644 --- a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts +++ b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts @@ -14,6 +14,7 @@ import { explainJinja, mayBeScript, parseBytesProcessed, + readsFiles, parseDatabricksExplainCost, parseSnowflakeExplain, untilAbort, @@ -827,3 +828,36 @@ describe("review follow-ups, round six", () => { } }) }) + +describe("review follow-ups, round seven", () => { + test.each([ + "select * from read_files/**/('/Volumes/c/s/v/raw', format => 'json')", + "select * from `read_files`('/x')", + "select * from read_files -- why\n ('/x')", + "select * from `json`.`/Volumes/c/s/v/raw`", + "select * from json/**/.`/x`", + "select '--', * from read_files('/x')", + "select * from system.builtin.read_files('/x')", + ])("%p reads files", (sql) => { + expect(readsFiles(sql)).toBe(true) + }) + + test.each([ + "select * from main.sales.orders", + "select t.json.field, text.col from t", + "select * from delta.`/Volumes/c/s/v/t`", + "select 'read_files' as note from t", + ])("%p reads tables", (sql) => { + expect(readsFiles(sql)).toBe(false) + }) + + test("on Databricks, a comment-split or quoted file read never reaches the EXPLAIN", async () => { + for (const compiled of ["select * from read_files/**/('/x')", "select * from `json`.`/x`"]) { + const local: Calls = { nodes: [], queries: [], dryRuns: [] } + useAdapter(local, { type: "databricks", compile: async () => compiled, explain: async () => JSON.stringify([DATABRICKS_PLAN]) }) + const result = await (await initTool(DbtCostEstimateTool)).execute({ model: "orders" }, ctx) + expect(local.queries).toEqual([]) + expect(result.metadata).toMatchObject({ success: false, adapter: "databricks", source: "explain" }) + } + }) +}) From 167342de69543a962f654acf786743fc0bbcfbc8 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Thu, 19 Nov 2026 13:44:22 +0530 Subject: [PATCH 10/14] fix: one scan of the SQL for both checks, so a backslash before a line break is read the same way The file-read check read quoted text with its own regex, which did not take a backslash before a line break as an escape: a "--" after it was taken as a comment hiding read_files. Both checks now use the code one scanner leaves (comments and quoted text as spaces, backtick-quoted names kept). Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/tools/dbt-cost-estimate.ts | 39 ++++++++++++------- .../test/altimate/dbt-cost-estimate.test.ts | 28 +++++++++++++ 2 files changed, 52 insertions(+), 15 deletions(-) diff --git a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts index 88429e8047..91edba6e77 100644 --- a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts +++ b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts @@ -165,15 +165,14 @@ export function parseDatabricksExplainCost(plan: unknown): PlanEstimate | { noSt /** * Whether a Databricks query reads files rather than tables (read_files, cloud_files, a path such as json.`…`): - * planning one can read the files to infer their schema. Read on the text with its comments as spaces (quoted text - * kept, so a comment marker inside it stays) and any spacing between the name and what follows; the names may be - * backtick-quoted. Called once `mayBeScript` has passed: every quote and comment is closed and none nests. + * planning one can read the files to infer their schema. Read on the code `scanSql` leaves (comments and quoted text + * as spaces, backtick-quoted names kept), with any spacing between a name and what follows. Anything the scanner + * does not read counts. */ export function readsFiles(sql: string): boolean { - const bare = sql - .replace(/'(?:[^'\\]|\\.)*'|"(?:[^"\\]|\\.)*"|`[^`]*`|--[^\n]*|\/\*[\s\S]*?\*\//g, (token) => (token.startsWith("--") || token.startsWith("/*") ? " " : token)) - .toLowerCase() - return /(^|[^a-z0-9_])`?(read_files|cloud_files)`?\s*\(|(^|[^a-z0-9_])`?(json|csv|parquet|avro|orc|text|binaryfile|xml)`?\s*\.\s*`/.test(bare) + const { script, code } = scanSql(sql, "databricks") + if (script) return true + return /(^|[^a-z0-9_])`?(read_files|cloud_files)`?\s*\(|(^|[^a-z0-9_])`?(json|csv|parquet|avro|orc|text|binaryfile|xml)`?\s*\.\s*`/i.test(code) } /** @@ -207,11 +206,20 @@ export function explainJinja(explain: string, sql: string): string { * comments, an unclosed quote or comment, EXECUTE IMMEDIATE) counts as a script, as does a semicolon with anything but whitespace after it: a doubt is no estimate, never a run. */ export function mayBeScript(input: string, type: string): boolean { + return scanSql(input, type).script +} + +/** + * The one reading of the SQL both checks share: whether it may be a script (see `mayBeScript`), and its code with + * comments and quoted text as spaces and Databricks' backtick-quoted names kept as written. + */ +export function scanSql(input: string, type: string): { script: boolean; code: string } { + const script = { script: true, code: "" } // Read with the line breaks Jinja renders: every \r\n and lone \r is \n. const sql = input.replace(/\r\n?/g, "\n") const databricks = type === "databricks" // The Snowflake splitter yields a line starting with "!" as a command of its own, wherever it sits: never read. - if (!databricks && /(^|\n)!/.test(sql)) return true + if (!databricks && /(^|\n)!/.test(sql)) return script let code = "" // Past a semicolon only whitespace and comments may follow: anything else, quoted or not, is a second statement. let ended = false @@ -230,15 +238,15 @@ export function mayBeScript(input: string, type: string): boolean { const end = sql.indexOf("*/", i + 2) // Databricks nests block comments, and an unclosed one hides the rest: neither is read, on either warehouse. // The closer's star counts too: in "/*/*/" an opener overlaps it. - if (end < 0 || sql.slice(i + 2, end + 1).includes("/*")) return true + if (end < 0 || sql.slice(i + 2, end + 1).includes("/*")) return script i = end + 2 code += " " continue } - if (two === "$$" || (!databricks && two === "//")) return true + if (two === "$$" || (!databricks && two === "//")) return script // Databricks raw strings (r'…', R"…") do not escape with a backslash: not read. - if (databricks && /[rR]/.test(c) && /['"]/.test(sql[i + 1] ?? "") && !/[A-Za-z0-9_]/.test(sql[i - 1] ?? "")) return true - if (ended && !/[\s;]/.test(c)) return true + if (databricks && /[rR]/.test(c) && /['"]/.test(sql[i + 1] ?? "") && !/[A-Za-z0-9_]/.test(sql[i - 1] ?? "")) return script + if (ended && !/[\s;]/.test(c)) return script if (c === ";") ended = true const close = c === "'" || c === '"' || (databricks && c === "`") ? c : null if (close) { @@ -247,15 +255,16 @@ export function mayBeScript(input: string, type: string): boolean { const escapes = c !== "`" let j = i + 1 while (j < sql.length && sql[j] !== close) j += escapes && sql[j] === "\\" ? 2 : 1 - if (j >= sql.length) return true + if (j >= sql.length) return script + // A backtick-quoted name is code; quoted text is not. + code += c === "`" ? sql.slice(i, j + 1) : " " i = j + 1 - code += " " continue } code += c i++ } - return /\bexecute\s+immediate\b/i.test(code) + return { script: /\bexecute\s+immediate\b/i.test(code), code } } /** Starts `start` and waits for it, but gives up as soon as the call is cancelled; a cancelled call starts nothing. */ diff --git a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts index cf31aacd59..ed3f621b25 100644 --- a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts +++ b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts @@ -15,6 +15,7 @@ import { mayBeScript, parseBytesProcessed, readsFiles, + scanSql, parseDatabricksExplainCost, parseSnowflakeExplain, untilAbort, @@ -861,3 +862,30 @@ describe("review follow-ups, round seven", () => { } }) }) + +describe("review follow-ups, round eight", () => { + const escapedNewline = [ + "WITH unused AS (SELECT 'a\\\n-- x' AS s) SELECT * FROM read_files('/x')", + "SELECT length('a\\\n--') AS n, * FROM read_files('/x', format => 'json')", + ] + + test.each(escapedNewline)("a backslash before a line break stays inside the text: %p reads files", (sql) => { + expect(mayBeScript(sql, "databricks")).toBe(false) + expect(readsFiles(sql)).toBe(true) + }) + + test("so it never reaches the EXPLAIN", async () => { + for (const compiled of escapedNewline) { + const local: Calls = { nodes: [], queries: [], dryRuns: [] } + useAdapter(local, { type: "databricks", compile: async () => compiled, explain: async () => JSON.stringify([DATABRICKS_PLAN]) }) + const result = await (await initTool(DbtCostEstimateTool)).execute({ model: "orders" }, ctx) + expect(local.queries).toEqual([]) + expect(result.metadata).toMatchObject({ success: false, adapter: "databricks", source: "explain" }) + } + }) + + test("both checks read the SQL the same way: one scan", () => { + expect(scanSql("select `json`.`x` from t -- c\n", "databricks")).toEqual({ script: false, code: "select `json`.`x` from t " }) + expect(scanSql("select 'a' from t", "databricks").code).toBe("select from t") + }) +}) From fa165e2fbbbe4c4d18c78f5914a485eb14006832 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Thu, 19 Nov 2026 13:50:34 +0530 Subject: [PATCH 11/14] fix: a Databricks line comment continued past a backslash, and any non-Delta source over a path - Spark continues a -- comment past a trailing backslash: such a comment is not read, so the query gets no estimate. - Besides the known formats (now with libsvm and image), any source over a path-like backtick name (a "/" or ":" in it), Delta aside, counts as a query over files. Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/tools/dbt-cost-estimate.ts | 9 ++++- .../test/altimate/dbt-cost-estimate.test.ts | 34 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts index 91edba6e77..65eefd8716 100644 --- a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts +++ b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts @@ -172,7 +172,12 @@ export function parseDatabricksExplainCost(plan: unknown): PlanEstimate | { noSt export function readsFiles(sql: string): boolean { const { script, code } = scanSql(sql, "databricks") if (script) return true - return /(^|[^a-z0-9_])`?(read_files|cloud_files)`?\s*\(|(^|[^a-z0-9_])`?(json|csv|parquet|avro|orc|text|binaryfile|xml)`?\s*\.\s*`/i.test(code) + if (/(^|[^a-z0-9_])`?(read_files|cloud_files)`?\s*\(|(^|[^a-z0-9_])`?(json|csv|parquet|avro|orc|text|binaryfile|xml|libsvm|image)`?\s*\.\s*`/i.test(code)) return true + // Any other source over a path (`provider`.`/a/path`, s3://…, dbfs:/…), Delta's log aside: its schema can come from the files too. + for (const match of code.matchAll(/(^|[^a-z0-9_.`])`?([a-z_][a-z0-9_]*)`?\s*\.\s*`([^`]*)`/gi)) { + if (match[2].toLowerCase() !== "delta" && /[/:]/.test(match[3])) return true + } + return false } /** @@ -229,6 +234,8 @@ export function scanSql(input: string, type: string): { script: boolean; code: s const two = sql.slice(i, i + 2) if (two === "--") { const end = sql.indexOf("\n", i) + // Spark continues a line comment past a backslash at its end: not read. + if (databricks && end >= 0 && sql[end - 1] === "\\") return script if (end < 0) break i = end + 1 code += " " diff --git a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts index ed3f621b25..706d0c3a34 100644 --- a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts +++ b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts @@ -889,3 +889,37 @@ describe("review follow-ups, round eight", () => { expect(scanSql("select 'a' from t", "databricks").code).toBe("select from t") }) }) + +describe("review follow-ups, round nine", () => { + test("a Databricks line comment continued past a backslash is not read", () => { + const sql = "SELECT 1 -- \\\n'ignored\n, * FROM read_files('/Volumes/c/s/v/raw', format => 'json') -- '" + expect(mayBeScript(sql, "databricks")).toBe(true) + expect(readsFiles(sql)).toBe(true) + }) + + test.each([ + "SELECT * FROM libsvm.`s3://bucket/train`", + "SELECT * FROM somefmt.`dbfs:/mnt/raw`", + "SELECT * FROM `binaryFile`.`/Volumes/c/s/v/raw`", + ])("%p reads files", (sql) => { + expect(readsFiles(sql)).toBe(true) + }) + + test.each([ + "SELECT * FROM delta.`/Volumes/c/s/v/t`", + "SELECT * FROM sales.`order items`", + "SELECT * FROM main.sales.`orders`", + ])("%p reads a table", (sql) => { + expect(readsFiles(sql)).toBe(false) + }) + + test("a continued comment or a LIBSVM path never reaches the EXPLAIN", async () => { + for (const compiled of ["SELECT 1 -- \\\n'ignored\n, * FROM read_files('/x') -- '", "SELECT * FROM libsvm.`s3://bucket/train`"]) { + const local: Calls = { nodes: [], queries: [], dryRuns: [] } + useAdapter(local, { type: "databricks", compile: async () => compiled, explain: async () => JSON.stringify([DATABRICKS_PLAN]) }) + const result = await (await initTool(DbtCostEstimateTool)).execute({ model: "orders" }, ctx) + expect(local.queries).toEqual([]) + expect(result.metadata).toMatchObject({ success: false, adapter: "databricks", source: "explain" }) + } + }) +}) From 2ef754e9776b198a1ea974427f77bbfd00226586 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Thu, 19 Nov 2026 13:56:44 +0530 Subject: [PATCH 12/14] fix: any backtick-quoted path-like name, whatever the source before it, is a read of files A provider class name (`org.apache...json.DefaultSource`) or a doubled backtick in the path no longer hides a query over files: any backtick-quoted name with a "/" or ":" counts, unless it follows Delta's own provider. Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/tools/dbt-cost-estimate.ts | 8 ++++--- .../test/altimate/dbt-cost-estimate.test.ts | 24 +++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts index 65eefd8716..44f97d2f19 100644 --- a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts +++ b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts @@ -173,9 +173,11 @@ export function readsFiles(sql: string): boolean { const { script, code } = scanSql(sql, "databricks") if (script) return true if (/(^|[^a-z0-9_])`?(read_files|cloud_files)`?\s*\(|(^|[^a-z0-9_])`?(json|csv|parquet|avro|orc|text|binaryfile|xml|libsvm|image)`?\s*\.\s*`/i.test(code)) return true - // Any other source over a path (`provider`.`/a/path`, s3://…, dbfs:/…), Delta's log aside: its schema can come from the files too. - for (const match of code.matchAll(/(^|[^a-z0-9_.`])`?([a-z_][a-z0-9_]*)`?\s*\.\s*`([^`]*)`/gi)) { - if (match[2].toLowerCase() !== "delta" && /[/:]/.test(match[3])) return true + // Any backtick-quoted name that looks like a path ("/" or ":", as in s3://…, dbfs:/…, /Volumes/…), whatever the source + // before it (a format, a provider class) or a doubled backtick inside it, unless it follows Delta's own provider: + // a source over files can infer its schema from them. + for (const match of code.matchAll(/`[^`]*`/g)) { + if (/[/:]/.test(match[0]) && !/(^|[^a-z0-9_.`])`?delta`?\s*\.\s*$/i.test(code.slice(0, match.index))) return true } return false } diff --git a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts index 706d0c3a34..cf5c3454ab 100644 --- a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts +++ b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts @@ -923,3 +923,27 @@ describe("review follow-ups, round nine", () => { } }) }) + +describe("review follow-ups, round ten", () => { + test.each([ + "SELECT * FROM `org.apache.spark.sql.json.DefaultSource`.`/Volumes/c/s/v/raw`", + "SELECT * FROM `org.apache.spark.sql.execution.datasources.json.JsonFileFormat`.`/Volumes/c/s/v/raw`", + "SELECT * FROM somefmt.`prefix``/raw`", + "SELECT * FROM delta.`a``/b`", + ])("%p reads files", (sql) => { + expect(readsFiles(sql)).toBe(true) + }) + + test("Delta's own provider over a path is still a table", () => { + expect(readsFiles("SELECT * FROM `delta`.`/Volumes/c/s/v/t`")).toBe(false) + expect(readsFiles("SELECT * FROM delta . `s3://bucket/t`")).toBe(false) + }) + + test("a provider class name over a path never reaches the EXPLAIN", async () => { + const local: Calls = { nodes: [], queries: [], dryRuns: [] } + useAdapter(local, { type: "databricks", compile: async () => "SELECT * FROM `org.apache.spark.sql.json.DefaultSource`.`/Volumes/c/s/v/raw`", explain: async () => JSON.stringify([DATABRICKS_PLAN]) }) + const result = await (await initTool(DbtCostEstimateTool)).execute({ model: "orders" }, ctx) + expect(local.queries).toEqual([]) + expect(result.metadata).toMatchObject({ success: false, adapter: "databricks", source: "explain" }) + }) +}) From 4824fd690b2ff523a782291aa100134da547704f Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Thu, 19 Nov 2026 14:02:53 +0530 Subject: [PATCH 13/14] fix: the file-read check takes formats only where a relation goes, in one linear pass - A known format name over a backtick-quoted name counts only after FROM, JOIN or a comma, so json.`order-id` as a column, or main.json.`orders`, is a table again. - Delta's own paths are set aside in one replace, not by re-reading the text before every backtick-quoted name. - explainEstimate takes the one adapter method it calls, typed. Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/tools/dbt-cost-estimate.ts | 17 ++++++------ .../test/altimate/dbt-cost-estimate.test.ts | 27 +++++++++++++++++++ 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts index 44f97d2f19..e879e20a48 100644 --- a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts +++ b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts @@ -172,14 +172,15 @@ export function parseDatabricksExplainCost(plan: unknown): PlanEstimate | { noSt export function readsFiles(sql: string): boolean { const { script, code } = scanSql(sql, "databricks") if (script) return true - if (/(^|[^a-z0-9_])`?(read_files|cloud_files)`?\s*\(|(^|[^a-z0-9_])`?(json|csv|parquet|avro|orc|text|binaryfile|xml|libsvm|image)`?\s*\.\s*`/i.test(code)) return true + // A function call, qualified or not (system.builtin.read_files). + if (/(^|[^a-z0-9_])`?(read_files|cloud_files)`?\s*\(/i.test(code)) return true + // A known format over any name, where a relation goes (after FROM, JOIN or a comma), so json.`order-id` as a column is not one. + if (/(\b(from|join)\s+|,\s*)`?(json|csv|parquet|avro|orc|text|binaryfile|xml|libsvm|image)`?\s*\.\s*`/i.test(code)) return true // Any backtick-quoted name that looks like a path ("/" or ":", as in s3://…, dbfs:/…, /Volumes/…), whatever the source - // before it (a format, a provider class) or a doubled backtick inside it, unless it follows Delta's own provider: - // a source over files can infer its schema from them. - for (const match of code.matchAll(/`[^`]*`/g)) { - if (/[/:]/.test(match[0]) && !/(^|[^a-z0-9_.`])`?delta`?\s*\.\s*$/i.test(code.slice(0, match.index))) return true - } - return false + // before it (a format, a provider class) or a doubled backtick inside it, unless it directly follows Delta's own + // provider: a source over files can infer its schema from them. + const rest = code.replace(/(^|[^a-z0-9_.`])`?delta`?\s*\.\s*`[^`]*`/gi, "$1 ") + return /`[^`]*[/:][^`]*`/.test(rest) } /** @@ -295,7 +296,7 @@ function failure(title: string, message: string, extra: Record /** The planner's estimate of a compiled query on a warehouse with no dry run: an EXPLAIN, which compiles the query and runs none of it. */ async function explainEstimate( - adapter: any, + adapter: { unsafeCompileQuery(sql: string, model?: string): Promise }, type: string, planner: (typeof PLANNERS)[string], compiled: string, diff --git a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts index cf5c3454ab..f4d658464b 100644 --- a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts +++ b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts @@ -947,3 +947,30 @@ describe("review follow-ups, round ten", () => { expect(result.metadata).toMatchObject({ success: false, adapter: "databricks", source: "explain" }) }) }) + +describe("bot review follow-ups", () => { + test.each([ + "select * from query_audit where sql_text like '%read_files(%'", + "SELECT json.`order-id` FROM main.sales.orders AS json", + "SELECT * FROM main.json.`orders`", + "SELECT * FROM main.image.`thumbnails`", + ])("%p reads a table", (sql) => { + expect(readsFiles(sql)).toBe(false) + }) + + test.each([ + "SELECT * FROM json.`data.json`", + "SELECT * FROM t JOIN csv.`rel` ON true", + "SELECT * FROM a, parquet.`p`", + "SELECT * FROM system.builtin.read_files('/x')", + ])("%p reads files", (sql) => { + expect(readsFiles(sql)).toBe(true) + }) + + test("many backtick-quoted names are read in one pass", () => { + const sql = `select ${Array.from({ length: 5000 }, (_, i) => `\`c${i}\``).join(", ")} from delta.\`/Volumes/c/s/v/t\`` + const started = performance.now() + expect(readsFiles(sql)).toBe(false) + expect(performance.now() - started).toBeLessThan(500) + }) +}) From c3e8f88e032990399918228bc01e3bb0096fc7c4 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Thu, 19 Nov 2026 14:07:36 +0530 Subject: [PATCH 14/14] fix: a format over a backtick-quoted name counts wherever it is not the middle of a dotted name Limiting it to after FROM, JOIN or a comma missed FROM (json.`x`) and FROM`json`.`x`. Where a relation goes cannot be told from a column without parsing, so the check stays on the side of no estimate: a column qualified by an alias named like a format is refused too. Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/tools/dbt-cost-estimate.ts | 7 ++++-- .../test/altimate/dbt-cost-estimate.test.ts | 23 ++++++++++++++++++- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts index e879e20a48..b9b3b21b68 100644 --- a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts +++ b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts @@ -174,8 +174,11 @@ export function readsFiles(sql: string): boolean { if (script) return true // A function call, qualified or not (system.builtin.read_files). if (/(^|[^a-z0-9_])`?(read_files|cloud_files)`?\s*\(/i.test(code)) return true - // A known format over any name, where a relation goes (after FROM, JOIN or a comma), so json.`order-id` as a column is not one. - if (/(\b(from|join)\s+|,\s*)`?(json|csv|parquet|avro|orc|text|binaryfile|xml|libsvm|image)`?\s*\.\s*`/i.test(code)) return true + // A known format over any backtick-quoted name, unless the format word is the middle of a dotted name (main.json.`t`). + // Where a relation goes cannot be told from a column without parsing (FROM (json.`x`), FROM`json`.`x`), so a + // column qualified by an alias named like a format (json.`order-id`) is refused too: no estimate, never a missed read. + const formats = "json|csv|parquet|avro|orc|text|binaryfile|xml|libsvm|image" + if (new RegExp(`(?:(^|[^a-z0-9_.\`])(${formats})|(^|[^.\`])\`(${formats})\`)\\s*\\.\\s*\``, "i").test(code)) return true // Any backtick-quoted name that looks like a path ("/" or ":", as in s3://…, dbfs:/…, /Volumes/…), whatever the source // before it (a format, a provider class) or a doubled backtick inside it, unless it directly follows Delta's own // provider: a source over files can infer its schema from them. diff --git a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts index f4d658464b..a2b54ca496 100644 --- a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts +++ b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts @@ -951,7 +951,6 @@ describe("review follow-ups, round ten", () => { describe("bot review follow-ups", () => { test.each([ "select * from query_audit where sql_text like '%read_files(%'", - "SELECT json.`order-id` FROM main.sales.orders AS json", "SELECT * FROM main.json.`orders`", "SELECT * FROM main.image.`thumbnails`", ])("%p reads a table", (sql) => { @@ -974,3 +973,25 @@ describe("bot review follow-ups", () => { expect(performance.now() - started).toBeLessThan(500) }) }) + +describe("review follow-ups, round twelve", () => { + test.each([ + "SELECT * FROM (json.`data.json`)", + "SELECT * FROM`json`.`data.json`", + "SELECT * FROM json . `data.json`", + // A column qualified by an alias named like a format cannot be told from a path query without parsing: refused. + "SELECT json.`order-id` FROM main.sales.orders AS json", + ])("%p reads files (or may)", (sql) => { + expect(readsFiles(sql)).toBe(true) + }) + + test("a parenthesised or unspaced file read never reaches the EXPLAIN", async () => { + for (const compiled of ["SELECT * FROM (json.`data.json`)", "SELECT * FROM`json`.`data.json`"]) { + const local: Calls = { nodes: [], queries: [], dryRuns: [] } + useAdapter(local, { type: "databricks", compile: async () => compiled, explain: async () => JSON.stringify([DATABRICKS_PLAN]) }) + const result = await (await initTool(DbtCostEstimateTool)).execute({ model: "orders" }, ctx) + expect(local.queries).toEqual([]) + expect(result.metadata).toMatchObject({ success: false, adapter: "databricks", source: "explain" }) + } + }) +})