diff --git a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts index 1b113f1994..758dfc2631 100644 --- a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts +++ b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts @@ -1,17 +1,156 @@ import z from "zod" import { Tool } from "../../tool/tool" +import { dbtAdapterFor } from "./dbt-adapter-access" + +/** + * Every dbt integration but the Python bridge writes the SQL into a `{{ validate_sql('...') }}` Jinja + * argument, so a quote or backslash in it must be escaped or any filter on a string breaks the dry run. + * The bridge, which a config without an integration also means, binds the SQL as a parameter and takes it as is. + */ +export function forDryRun(sql: string, integration: unknown): string { + return integration == null || integration === "core" ? sql : sql.replace(/\\/g, "\\\\").replace(/'/g, "\\'") +} + +/** The units dbt-bigquery's `format_bytes` writes, smallest first; each is 1024 times the one before. */ +const UNITS = ["Bytes", "KiB", "MiB", "GiB", "TiB", "PiB"] as const + +/** + * The byte count in a dry run's `bytes_processed`, which dbt-bigquery formats as text ("4.2 TiB", + * "512.0 Bytes"), and the most it can be. The text carries one decimal, so a figure past a byte is + * rounded: the true count is within half a tenth of its unit. Undefined for anything else, so a changed + * format is reported, never read as a count. + */ +/** Whether the SQL is a script (more than one statement, or EXECUTE IMMEDIATE): a dry run only sees its first statements. */ +export function isScript(sql: string): boolean { + // One pass over comments and quoted text together: a quote inside a comment, or a comment marker inside a quote, is not either. + const bare = sql + .replace(/--[^\n]*|#[^\n]*|\/\*[\s\S]*?\*\/|'(?:[^'\\]|\\.)*'|"(?:[^"\\]|\\.)*"|`[^`]*`/g, " ") + .replace(/;\s*$/, "") + return bare.includes(";") || /\bexecute\s+immediate\b/i.test(bare) +} + +export function parseBytesProcessed(display: unknown): { bytes: number; maxBytes: number } | undefined { + if (typeof display === "number") { + if (!Number.isFinite(display) || display < 0) return undefined + const bytes = Math.round(display) + return { bytes, maxBytes: bytes } + } + if (typeof display !== "string") return undefined + const match = /^\s*(\d+(?:\.\d+)?)\s*(Bytes|KiB|MiB|GiB|TiB|PiB)\s*$/i.exec(display) + if (!match) return undefined + const unit = UNITS.findIndex((u) => u.toLowerCase() === match[2].toLowerCase()) + const scale = 1024 ** unit + const bytes = Math.round(Number(match[1]) * scale) + if (unit === 0) return { bytes, maxBytes: bytes } + // Exact in integers: the shown figure is rounded to its last digit, so the true count is at most half + // of that digit above it. A float product is off by whole bytes past a few TiB. + const [whole, fraction = ""] = match[1].split(".") + const digits = BigInt(whole + fraction) + let upper = ((digits * 2n + 1n) * 1024n ** BigInt(unit) + 2n * 10n ** BigInt(fraction.length) - 1n) / (2n * 10n ** BigInt(fraction.length)) + // Python divides in floats, so a count within a float's last place of a tie is shown rounded down. + upper += upper >> 50n + let maxBytes = Number(upper) + if (BigInt(Math.trunc(maxBytes)) < upper) maxBytes *= 1 + Number.EPSILON + return { bytes, maxBytes: Math.max(bytes, maxBytes) } +} + +/** Starts `start` and waits for it, but gives up as soon as the call is cancelled; a cancelled call starts nothing. */ +export function untilAbort(start: () => Promise, signal: AbortSignal): Promise { + if (signal.aborted) return Promise.reject(signal.reason) + return new Promise((resolve, reject) => { + const stop = () => reject(signal.reason) + signal.addEventListener("abort", stop, { once: true }) + Promise.resolve() + .then(() => (signal.aborted ? Promise.reject(signal.reason) : start())) + .then(resolve, reject) + .finally(() => signal.removeEventListener("abort", stop)) + }) +} + +function failure(title: string, message: string, extra: Record = {}) { + return { title: `dbt_cost_estimate: ${title}`, metadata: { success: false, error: message, ...extra }, output: message } +} export const DbtCostEstimateTool = Tool.define("dbt_cost_estimate", { - description: "Estimate the bytes a dbt model or a SQL query would scan, from a warehouse dry run (BigQuery).", + description: + "Estimate the bytes a dbt model or a SQL query would scan, from a warehouse dry run. Only BigQuery has a dry run: any other warehouse answers `supported: false` and nothing is compiled. Starts the dbt adapter (which runs `dbt deps`), so it needs write permission (`--allow-write` on `tool run` or the MCP server). The dry run never executes the final query, but compiling either input can render Jinja, and macros such as `run_query` can run real queries that are billed. Pass `model` (a model name, compiled as dbt would) or `sql` (may use Jinja).", parameters: z.object({ model: z.string().optional().describe("Model name to compile and estimate"), sql: z.string().optional().describe("SQL (may use Jinja) to compile and estimate"), }), - async execute() { + async execute(args, ctx) { + if (!args.model && !args.sql) return failure("nothing to estimate", "Pass a model name or SQL.") + if (args.model && args.sql) return failure("ambiguous input", "Pass a model name or SQL, not both.") + const access = await dbtAdapterFor(ctx) + if (!access.adapter) return failure("not available", `No estimate: ${access.reason}.`, { refusal: access.refusal }) + const adapter = access.adapter + const type: string | undefined = adapter.getAdapterType?.() + // Every dbt-integration refuses the dry run off BigQuery, so nothing is compiled there. + if (type !== "bigquery") { + const adapterName = type ?? "unknown" + return { + title: "dbt_cost_estimate: no estimate", + metadata: { success: true, supported: false, adapter: adapterName }, + output: `No cost estimate: dry runs are only available on BigQuery (this project uses ${adapterName}).`, + } + } + const signal = ctx.abort + let compiled: unknown + try { + // A model compiles by name, so its node context (`this`, config) is kept; SQL compiles inline. + compiled = await untilAbort( + () => (args.model ? adapter.unsafeCompileNode(args.model) : adapter.unsafeCompileQuery(args.sql!, undefined)), + signal, + ) + } catch (e) { + if (signal.aborted) throw e + return failure("compile failed", e instanceof Error ? e.message : String(e), { adapter: type }) + } + if (typeof compiled !== "string" || !compiled.trim()) { + return failure("compile failed", "The query compiled to nothing.", { adapter: type }) + } + let display: unknown + try { + const integration = await untilAbort(async () => { + const { dbtAdapterInternals } = await import("../native/connections/register") + const read = dbtAdapterInternals.readConfig ?? (await import("../../../../dbt-tools/src/config")).read + return ((await read()) as { dbtIntegration?: unknown } | null)?.dbtIntegration + }, signal) + const result = await untilAbort(() => adapter.validateSQLDryRun(forDryRun(compiled as string, integration)), signal) + if (result == null) { + return failure("dry run failed", "The dry run returned no estimate: dbt-core before 1.6 can't dry run.", { adapter: type }) + } + display = (result as { bytes_processed?: unknown }).bytes_processed + } catch (e) { + if (signal.aborted) throw e + return failure("dry run failed", e instanceof Error ? e.message : String(e), { adapter: type }) + } + const parsed = parseBytesProcessed(display) + if (!parsed) { + return failure("unreadable estimate", `The dry run returned an amount this tool can't read: ${JSON.stringify(display)}.`, { + adapter: type, + }) + } + const name = args.model + const shown = typeof display === "number" ? `${parsed.bytes} Bytes` : String(display) + // A dry run can report 0 for a query that still reads data (BigQuery federated sources), and it skips + // what a script runs after its first statement or inside EXECUTE IMMEDIATE: those figures are lower + // bounds and carry no upper bound. + const zero = parsed.bytes === 0 + const lower = zero || isScript(compiled as string) return { - title: "dbt_cost_estimate: not implemented", - metadata: { success: false, error: "not implemented" }, - output: "dbt_cost_estimate is not implemented in this build of altimate-code yet.", + title: `dbt_cost_estimate: ${shown}`, + metadata: { + success: true, + supported: true, + adapter: type, + ...(lower ? { bytes: parsed.bytes, lowerBound: true } : parsed), + display: shown, + ...(name && { model: name }), + }, + output: `${name ? `${name} ` : "The query "}would process ${shown} ${ + lower ? "(a dry run can undercount a script or a zero estimate, so this is a lower bound, not a cap)." : `(at most ${parsed.maxBytes} bytes).` + }`, } }, }) diff --git a/packages/opencode/test/altimate/dbt-cost-estimate.test.ts b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts new file mode 100644 index 0000000000..885e982b95 --- /dev/null +++ b/packages/opencode/test/altimate/dbt-cost-estimate.test.ts @@ -0,0 +1,342 @@ +// dbt_cost_estimate: the dry run's formatted size read back into bytes (and its upper bound) for each unit, +// the answer for a warehouse with no dry run, a read-only call, and a model compiled by name. The dbt adapter is +// replaced through `dbtAdapterInternals`, so these tests need no Python, dbt or warehouse. +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test" +import { mkdtemp, rm, writeFile } from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { Effect } from "effect" +import type { Tool } from "@/tool/tool" +import { SessionID, MessageID } from "@/session/schema" +import { dbtAdapterInternals, resetDbtAdapter } from "@/altimate/native/connections/register" +import { DbtCostEstimateTool, parseBytesProcessed, untilAbort } from "@/altimate/tools/dbt-cost-estimate" +import { initTool } from "./tool-fixture" + +const ctx: Tool.Context = { + sessionID: SessionID.make("ses_dbt_cost_estimate"), + messageID: MessageID.make("msg_dbt_cost_estimate"), + agent: "analyst", + abort: new AbortController().signal, + messages: [], + metadata: () => Effect.void, + ask: () => Effect.void, +} + +let project: string + +type Calls = { nodes: string[]; queries: [string, string | undefined][]; dryRuns: string[] } + +function fakeAdapter( + calls: Calls, + opts: { type?: string; bytes?: unknown; compileFails?: boolean; dryRunFails?: boolean; compile?: () => Promise } = {}, +) { + return { + isInitialized: () => true, + dispose: async () => {}, + getAdapterType: () => opts.type ?? "bigquery", + unsafeCompileNode: async (model: string) => { + calls.nodes.push(model) + if (opts.compile) return opts.compile() + if (opts.compileFails) throw new Error("compilation failed") + return `compiled(${model})` + }, + unsafeCompileQuery: async (query: string, name?: string) => { + calls.queries.push([query, name]) + if (opts.compileFails) throw new Error("compilation failed") + return `compiled(${query})` + }, + validateSQLDryRun: async (query: string) => { + calls.dryRuns.push(query) + if (opts.dryRunFails) throw new Error("Access Denied: BigQuery job") + return "bytes" in opts && opts.bytes === null ? null : { bytes_processed: "bytes" in opts ? opts.bytes : "4.2 TiB" } + }, + } +} + +function useAdapter(calls: Calls, opts?: Parameters[1]) { + dbtAdapterInternals.readConfig = async () => ({ + projectRoot: project, + pythonPath: "python3", + dbtIntegration: "corecommand", + queryLimit: 500, + }) + dbtAdapterInternals.create = async () => fakeAdapter(calls, opts) +} + +beforeAll(async () => { + project = await mkdtemp(path.join(os.tmpdir(), "dbt-cost-estimate-")) + await writeFile(path.join(project, "dbt_project.yml"), "name: shop\n") +}) + +afterAll(async () => { + await rm(project, { recursive: true, force: true }) +}) + +let calls: Calls +beforeEach(() => { + resetDbtAdapter() + calls = { nodes: [], queries: [], dryRuns: [] } +}) +afterEach(() => { + resetDbtAdapter() + delete dbtAdapterInternals.readConfig + delete dbtAdapterInternals.create +}) + +describe("parseBytesProcessed", () => { + test.each([ + ["0.0 Bytes", 0, 0], + ["512.0 Bytes", 512, 512], + ["1.0 KiB", 1024, Math.ceil(1.05 * 1024)], + ["1.5 KiB", 1536, Math.ceil(1.55 * 1024)], + ["2.0 MiB", 2 * 1024 ** 2, Math.ceil(2.05 * 1024 ** 2)], + ["4.2 GiB", Math.round(4.2 * 1024 ** 3), Math.ceil(4.25 * 1024 ** 3)], + ["4.2 TiB", Math.round(4.2 * 1024 ** 4), Math.ceil(4.25 * 1024 ** 4)], + ["1.0 PiB", 1024 ** 5, Math.ceil(1.05 * 1024 ** 5)], + ["10.5 PiB", Math.round(10.5 * 1024 ** 5), Math.ceil(10.55 * 1024 ** 5)], + [" 7.0 MiB ", 7 * 1024 ** 2, Math.ceil(7.05 * 1024 ** 2)], + ])("reads %s as %d bytes, at most about %d", (display, bytes, maxBytes) => { + const parsed = parseBytesProcessed(display)! + expect(parsed.bytes).toBe(bytes) + // The bound is the figure plus half its last digit; past a PiB it carries a few bytes of float margin. + expect(parsed.maxBytes).toBeGreaterThanOrEqual(maxBytes) + expect(parsed.maxBytes).toBeLessThanOrEqual(maxBytes + maxBytes * 1e-14 + 1) + }) + + test("a rounded figure's upper bound covers a scan just over a limit", () => { + // 10 GiB + 20 MiB is shown as "10.0 GiB": the point estimate sits on the limit, the bound above it. + const over = 10 * 1024 ** 3 + 20 * 1024 ** 2 + const parsed = parseBytesProcessed("10.0 GiB")! + expect(parsed.bytes).toBe(10 * 1024 ** 3) + expect(parsed.maxBytes).toBeGreaterThanOrEqual(over) + }) + + test("the upper bound reaches a count just past a tie that the dry run shows rounded down", () => { + // 9.25 PiB is a tie in the dry run's float formatting, which shows it as "9.2": the bound sits above the tie. + const tie = 37n * 1024n ** 5n / 4n + expect(BigInt(parseBytesProcessed("9.2 PiB")!.maxBytes)).toBeGreaterThan(tie) + }) + + test("takes a number as the exact count", () => { + expect(parseBytesProcessed(2048)).toEqual({ bytes: 2048, maxBytes: 2048 }) + }) + + test.each([["4.2 EiB"], ["4.2"], ["TiB"], [""], ["about 4 GiB"], ["-1.0 KiB"], [null], [undefined], [{}], [Number.NaN], [-5]])( + "does not read %p", + (value) => { + expect(parseBytesProcessed(value)).toBeUndefined() + }, + ) +}) + +describe("dbt_cost_estimate", () => { + test("estimates a model compiled by name", async () => { + useAdapter(calls) + const result = await (await initTool(DbtCostEstimateTool)).execute({ model: "orders" }, ctx) + expect(calls).toEqual({ nodes: ["orders"], queries: [], dryRuns: ["compiled(orders)"] }) + expect(result.metadata).toEqual({ + success: true, + supported: true, + adapter: "bigquery", + bytes: Math.round(4.2 * 1024 ** 4), + maxBytes: Math.ceil(4.25 * 1024 ** 4), + display: "4.2 TiB", + model: "orders", + truncated: false, + }) + expect(result.output).toContain("orders would process 4.2 TiB") + }) + + test("estimates SQL as given", async () => { + useAdapter(calls, { bytes: "1.0 GiB" }) + const result = await (await initTool(DbtCostEstimateTool)).execute({ sql: "select 1" }, ctx) + expect(calls).toEqual({ nodes: [], queries: [["select 1", undefined]], dryRuns: ["compiled(select 1)"] }) + expect(result.metadata).toEqual({ + success: true, + supported: true, + adapter: "bigquery", + bytes: 1024 ** 3, + maxBytes: Math.ceil(1.05 * 1024 ** 3), + display: "1.0 GiB", + truncated: false, + }) + }) + + test("a number from the dry run is shown with its unit and is exact", async () => { + useAdapter(calls, { bytes: 2048 }) + const result = await (await initTool(DbtCostEstimateTool)).execute({ sql: "select 1" }, ctx) + expect(result.metadata).toMatchObject({ bytes: 2048, maxBytes: 2048, display: "2048 Bytes" }) + expect(result.output).toContain("2048 Bytes") + }) + + test("a zero from the dry run is a lower bound, not an upper bound", async () => { + useAdapter(calls, { bytes: 0 }) + const result = await (await initTool(DbtCostEstimateTool)).execute({ sql: "select 1" }, ctx) + expect(result.metadata).toEqual({ + success: true, + supported: true, + adapter: "bigquery", + bytes: 0, + lowerBound: true, + display: "0 Bytes", + truncated: false, + }) + expect(result.output).toContain("lower bound") + expect(result.output).not.toContain("at most") + }) + + test.each([ + "select 1; select * from big", + "select 1; execute immediate 'select * from big'", + "execute immediate 'select * from big'", + "select * from t where m = '--'; execute immediate 'select * from big'", + "select * from t where m = '#sale'; select * from big", + ])("a script (%s) is a lower bound, not an upper bound", async (sql) => { + useAdapter(calls, { bytes: "1.0 GiB" }) + const result = await (await initTool(DbtCostEstimateTool)).execute({ sql }, ctx) + expect(result.metadata).toMatchObject({ bytes: 1024 ** 3, lowerBound: true }) + expect(result.metadata).not.toHaveProperty("maxBytes") + expect(result.output).not.toContain("at most") + }) + + test("a semicolon in a string or a comment is not a script", async () => { + useAdapter(calls, { bytes: "1.0 GiB" }) + const tool = await initTool(DbtCostEstimateTool) + for (const sql of ["select 'a;b' from t", "select 1 -- x; y\nfrom t", "select `p.d;t`.c from `p.d.t`"]) { + const result = await tool.execute({ sql }, ctx) + expect(result.metadata).toHaveProperty("maxBytes") + } + }) + + test.each(["snowflake", "databricks", "duckdb"])("answers supported:false on %s and compiles nothing", async (type) => { + useAdapter(calls, { type }) + const result = await (await initTool(DbtCostEstimateTool)).execute({ model: "orders" }, ctx) + expect(result.metadata).toMatchObject({ success: true, supported: false, adapter: type }) + expect(result.metadata.bytes).toBeUndefined() + expect(result.output).toContain("only available on BigQuery") + expect(calls).toEqual({ nodes: [], queries: [], dryRuns: [] }) + }) + + test("refuses a read-only call and starts no adapter", async () => { + let created = 0 + useAdapter(calls) + const make = dbtAdapterInternals.create! + dbtAdapterInternals.create = async (config) => { + created += 1 + return make(config) + } + const result = await (await initTool(DbtCostEstimateTool)).execute({ model: "orders" }, { ...ctx, extra: { forceReadOnly: true } }) + expect(created).toBe(0) + expect(result.metadata).toMatchObject({ success: false, refusal: "read_only" }) + expect(result.output).toContain("--allow-write") + }) + + test("the tool description says compiling can run billed queries and the adapter start runs dbt deps", async () => { + const tool = await initTool(DbtCostEstimateTool) + expect(tool.description).toContain("runs `dbt deps`") + expect(tool.description).toContain("compiling either input can render Jinja, and macros such as `run_query` can run real queries that are billed") + }) + + test("the tool description names the MCP server as a way to grant write", async () => { + const tool = await initTool(DbtCostEstimateTool) + expect(tool.description).toContain("`--allow-write` on `tool run` or the MCP server") + }) + + test("rejects a call that passes both a model and SQL, estimating neither", async () => { + useAdapter(calls) + const result = await (await initTool(DbtCostEstimateTool)).execute({ model: "orders", sql: "select 1" }, ctx) + expect(result.metadata.success).toBe(false) + expect(result.output).toContain("not both") + expect(calls).toEqual({ nodes: [], queries: [], dryRuns: [] }) + }) + + test("asks for something to estimate", async () => { + useAdapter(calls) + const result = await (await initTool(DbtCostEstimateTool)).execute({}, ctx) + expect(result.metadata.success).toBe(false) + expect(calls).toEqual({ nodes: [], queries: [], dryRuns: [] }) + }) + + test("reports a model dbt can't compile, as a compile failure, without a dry run", async () => { + useAdapter(calls, { compileFails: true }) + const result = await (await initTool(DbtCostEstimateTool)).execute({ model: "nope" }, ctx) + expect(result.title).toBe("dbt_cost_estimate: compile failed") + expect(result.metadata).toMatchObject({ success: false, error: "compilation failed" }) + expect(calls.dryRuns).toEqual([]) + }) + + test("reports a compile that produced nothing, without a dry run", async () => { + useAdapter(calls, { compile: async () => " " }) + const result = await (await initTool(DbtCostEstimateTool)).execute({ model: "orders" }, ctx) + expect(result.metadata).toMatchObject({ success: false, error: "The query compiled to nothing." }) + expect(calls.dryRuns).toEqual([]) + }) + + test("reports a dry run that returns nothing, as dbt-core before 1.6 does", async () => { + useAdapter(calls, { bytes: null }) + const result = await (await initTool(DbtCostEstimateTool)).execute({ sql: "select 1" }, ctx) + expect(result.metadata).toMatchObject({ success: false }) + expect(result.output).toContain("before 1.6") + }) + + test("a cancelled call stops waiting for a compile that never finishes", async () => { + const abort = new AbortController() + useAdapter(calls, { compile: () => new Promise(() => {}) }) + const run = (await initTool(DbtCostEstimateTool)).execute({ model: "orders" }, { ...ctx, abort: abort.signal }) + await Bun.sleep(20) + abort.abort(new Error("cancelled")) + await expect(run).rejects.toThrow("cancelled") + expect(calls.dryRuns).toEqual([]) + }) + + test("work queued before a cancellation is not started by it", async () => { + // The signal reads as live at the first check and cancelled once the queued work comes to run. + let reads = 0 + const signal = { + get aborted() { + return ++reads > 1 + }, + reason: new Error("cancelled"), + addEventListener() {}, + removeEventListener() {}, + } as unknown as AbortSignal + let started = 0 + await expect(untilAbort(async () => void started++, signal)).rejects.toThrow("cancelled") + expect(started).toBe(0) + }) + + test("reports a dry run the warehouse refused", async () => { + useAdapter(calls, { dryRunFails: true }) + const result = await (await initTool(DbtCostEstimateTool)).execute({ sql: "select 1" }, ctx) + expect(result.title).toBe("dbt_cost_estimate: dry run failed") + expect(result.metadata).toMatchObject({ success: false, error: "Access Denied: BigQuery job" }) + }) + + test("reports a size it can't read instead of guessing", async () => { + useAdapter(calls, { bytes: "lots" }) + const result = await (await initTool(DbtCostEstimateTool)).execute({ sql: "select 1" }, ctx) + expect(result.metadata.success).toBe(false) + expect(result.metadata.bytes).toBeUndefined() + expect(result.output).toContain('"lots"') + }) + + test("escapes quotes and backslashes for the integrations that write the SQL into a Jinja argument", async () => { + useAdapter(calls) + await (await initTool(DbtCostEstimateTool)).execute({ sql: "where s = 'a\\b'" }, ctx) + expect(calls.dryRuns).toEqual(["compiled(where s = \\'a\\\\b\\')"]) + }) + + test("hands the SQL to the Python bridge as is", async () => { + useAdapter(calls) + dbtAdapterInternals.readConfig = async () => ({ projectRoot: project, pythonPath: "python3", dbtIntegration: "core", queryLimit: 500 }) + await (await initTool(DbtCostEstimateTool)).execute({ sql: "where s = 'a'" }, ctx) + expect(calls.dryRuns).toEqual(["compiled(where s = 'a')"]) + }) + + test("a config that names no integration is the Python bridge: the SQL goes as is", async () => { + useAdapter(calls) + dbtAdapterInternals.readConfig = async () => ({ projectRoot: project, pythonPath: "python3", queryLimit: 500 }) + await (await initTool(DbtCostEstimateTool)).execute({ sql: "where s = 'a'" }, ctx) + expect(calls.dryRuns).toEqual(["compiled(where s = 'a')"]) + }) +})