diff --git a/plugins/altimate-code/.claude-plugin/plugin.json b/plugins/altimate-code/.claude-plugin/plugin.json index 971d4eaa..ff11cbb0 100644 --- a/plugins/altimate-code/.claude-plugin/plugin.json +++ b/plugins/altimate-code/.claude-plugin/plugin.json @@ -30,8 +30,8 @@ }, "maxScanBytes": { "type": "number", - "title": "Largest BigQuery scan without asking", - "description": "A dbt run, build or show that a BigQuery dry run estimates would scan more bytes than this is held until you approve it with /allow-scan; 0 never holds, and other warehouses are never held. Optional; default: 10737418240 (10 GiB).", + "title": "Largest dbt scan without asking", + "description": "A dbt run, build or show estimated to scan more bytes than this is held until you approve it with /allow-scan; 0 never holds. BigQuery's estimate is its dry run; Snowflake's and Databricks' is the query planner's (EXPLAIN), which can differ from what the run scans. A query with no estimate, and other warehouses, are never held. Optional; default: 10737418240 (10 GiB).", "default": 10737418240, "min": 0 }, diff --git a/plugins/altimate-code/README.md b/plugins/altimate-code/README.md index 46af70b1..0a86ac8a 100644 --- a/plugins/altimate-code/README.md +++ b/plugins/altimate-code/README.md @@ -154,12 +154,12 @@ Compile and validate runs a query on your warehouse after every model edit (`dbt ## Settings -The plugin has seven settings, shown when you enable it (`userConfig` in `.claude-plugin/plugin.json`). All are optional: Claude Code may report them as "not set", and each then takes the default below. `dbt_mods` is used by the dbt helpers, `maxScanBytes` by the BigQuery scan check, `healthBlocksPush` and `healthConfigPath` by the project health check, the two comments settings by team comments, and `show_edit_impact` by the per-edit impact note. +The plugin has seven settings, shown when you enable it (`userConfig` in `.claude-plugin/plugin.json`). All are optional: Claude Code may report them as "not set", and each then takes the default below. `dbt_mods` is used by the dbt helpers, `maxScanBytes` by the scan check (BigQuery, Snowflake, Databricks), `healthBlocksPush` and `healthConfigPath` by the project health check, the two comments settings by team comments, and `show_edit_impact` by the per-edit impact note. | Setting | Default | What it controls | | --- | --- | --- | | `dbt_mods` | on | Turns all of the dbt helpers (above) off when false. | -| `maxScanBytes` | 10737418240 (10 GiB) | A dbt run, build or show that a BigQuery dry run estimates would scan more bytes than this is held until you approve it with `/allow-scan` (0 never holds; at least 0). Other warehouses are never held. | +| `maxScanBytes` | 10737418240 (10 GiB) | A dbt run, build or show estimated to scan more bytes than this is held until you approve it with `/allow-scan` (0 never holds; at least 0). On BigQuery the estimate is a dry run; on Snowflake and Databricks it is the query planner's (`EXPLAIN`): Snowflake's bytes after pruning, an upper bound, and Databricks' table sizes, which a run usually reads less of. A query with no estimate (a Databricks table without statistics) runs, with a note saying why; other warehouses are never held. | | `healthBlocksPush` | off | When on, `git push` is refused while the project health check finds new issues in the models you changed, and while the check cannot run (altimate-code missing, or without `dbt_health`) or has not finished. Off, the issues are only shown. If the first check of a session lands after you changed a model, that model's existing issues count as new (the baseline then lacks them). The reverse also holds: a model you drop from a schema file before that first check lands has its new issues taken as existing, so a push is not held for them. | | `healthConfigPath` | empty | The DataPilot health check config file, relative to the dbt project. Empty uses the default checks. | | `commentsPollSeconds` | 900 | How often, in seconds, the team's comments on shared docs are fetched again in an interactive session (30 to 86400). | diff --git a/plugins/altimate-code/hooks/features/cost.test.tsx b/plugins/altimate-code/hooks/features/cost.test.tsx index 36f48a43..b17fbfd8 100644 --- a/plugins/altimate-code/hooks/features/cost.test.tsx +++ b/plugins/altimate-code/hooks/features/cost.test.tsx @@ -409,13 +409,13 @@ test('commands that scan nothing are not looked at', async ($, on) => { }) test('another warehouse is told once, never held, and not asked again', async ($, on) => { - const { clock, estimates, ran } = await start($, on, { fallback: { success: true, supported: false, adapter: 'snowflake' } }) + const { clock, estimates, ran } = await start($, on, { fallback: { success: true, supported: false, adapter: 'duckdb' } }) const first: any = await $.tool.call(bash('dbt run -s orders')) const second: any = await $.tool.call(bash('dbt run -s customers')) await clock.settle() expect(first.deny).toBeUndefined() - expect(first.context).toEqual(['cost · no estimate on snowflake: only BigQuery has a dry run']) + expect(first.context).toEqual(['cost · no estimate on duckdb: scans are estimated on BigQuery, Snowflake and Databricks only']) expect(second.deny).toBeUndefined() expect(estimates).toEqual(['orders']) expect(ran).toEqual(['dbt run -s orders', 'dbt run -s customers']) @@ -1524,3 +1524,155 @@ test('a parallel call whose estimate fails is held by the hold another call reco expect(results.every(r => typeof r?.deny === 'string')).toBe(true) expect(ran).toEqual([]) }) + +// Snowflake and Databricks: dbt_cost_estimate answers the planner's EXPLAIN estimate (`source: "explain"`), with no rounding bound. +const planned = (adapter: string, bytes: number, display: string, bound = adapter === 'snowflake' ? 'upper' : 'estimate') => ({ + success: true, + supported: true, + adapter, + source: 'explain', + bound, + bytes, + display, +}) + +for (const [adapter, name] of [ + ['snowflake', 'Snowflake'], + ['databricks', 'Databricks'], +] as const) { + test(`on ${adapter}, a model whose plan estimate is over the limit is held, with the estimate and where it came from`, async ($, on) => { + const { clock, estimates, ran } = await start($, on, { answers: { orders: planned(adapter, 20 * GIB, '20.0 GiB') } }) + const result: any = await $.tool.call(bash('dbt run -s orders')) + await clock.settle() + + expect(estimates).toEqual(['orders']) + expect(ran).toEqual([]) + expect(result.deny).toBe(`≈ 20.0 GiB would be scanned (${name} plan estimate), over the 10.0 GiB limit. Narrow the query, or ask the user to run /allow-scan and then run the same command again.`) + }) + + test(`on ${adapter}, a model whose plan estimate is under the limit runs without a hold, saying where the estimate came from`, async ($, on) => { + const { clock, ran } = await start($, on, { answers: { orders: planned(adapter, 2 * GIB, '2.0 GiB') } }) + const result: any = await $.tool.call(bash('dbt run -s orders')) + await clock.settle() + + expect(result.deny).toBeUndefined() + expect(ran).toEqual(['dbt run -s orders']) + expect(result.context).toEqual([`cost · orders would scan 2.0 GiB (${name} plan estimate)`]) + }) + + test(`on ${adapter}, an altimate-code that estimates BigQuery only says a newer one is needed, and holds nothing`, async ($, on) => { + const { clock, ran } = await start($, on, { fallback: { success: true, supported: false, adapter } }) + const result: any = await $.tool.call(bash('dbt run -s orders')) + await clock.settle() + + expect(result.deny).toBeUndefined() + expect(ran).toEqual(['dbt run -s orders']) + expect(result.context).toEqual([`cost · no estimate on ${adapter}: this altimate-code estimates BigQuery only; ${name} needs a newer altimate-code (the release with Snowflake and Databricks estimates)`]) + }) +} + +test('on Snowflake, a held command is allowed once by /allow-scan, and its note says the estimate is the plan\'s', async ($, on) => { + const { clock, ran } = await start($, on, { answers: { orders: planned('snowflake', 20 * GIB, '20.0 GiB') } }) + expect(((await $.tool.call(bash('dbt build -s orders'))) as any).deny).toBeDefined() + expect((await $.command.run({ command: 'allow-scan', args: '' } as any)).text).toContain('orders (≈ 20.0 GiB, Snowflake plan estimate)') + const allowed: any = await $.tool.call(bash('dbt build -s orders')) + await clock.settle() + + expect(ran).toEqual(['dbt build -s orders']) + expect(allowed.context).toEqual(['cost · orders would scan 20.0 GiB (Snowflake plan estimate), over the 10.0 GiB limit: allowed once']) +}) + +test('on Databricks, a table without statistics gives no estimate: the command runs, with the reason', async ($, on) => { + const reason = "Databricks' planner has no size for a table this query reads (it shows Spark's default for a table without statistics). ANALYZE TABLE COMPUTE STATISTICS gives it one." + const { clock, ran } = await start($, on, { answers: { orders: { success: false, adapter: 'databricks', source: 'explain', error: reason } } }) + const result: any = await $.tool.call(bash('dbt run -s orders')) + await clock.settle() + + expect(result.deny).toBeUndefined() + expect(ran).toEqual(['dbt run -s orders']) + expect(result.context).toEqual([`cost · no estimate for orders: ${reason}`]) +}) + +test('on Databricks, a model with no estimate is asked about again on the next command: one table without statistics does not stop the check', async ($, on) => { + const { clock, estimates, ran } = await start($, on, { + answers: { orders: { success: false, adapter: 'databricks', source: 'explain', error: 'no size' }, customers: planned('databricks', 20 * GIB, '20.0 GiB') }, + }) + await $.tool.call(bash('dbt run -s orders')) + const second: any = await $.tool.call(bash('dbt run -s customers')) + await clock.settle() + + expect(estimates).toEqual(['orders', 'customers']) + expect(ran).toEqual(['dbt run -s orders']) + expect(second.deny).toContain('(Databricks plan estimate)') +}) + +test('/scan-cost on Snowflake names the plan estimate and the verdict', async ($, on) => { + await start($, on, { answers: { orders: planned('snowflake', 20 * GIB, '20.0 GiB') } }) + expect((await $.command.run({ command: 'scan-cost', args: 'orders' } as any)).text).toBe( + 'orders would scan 20.0 GiB (Snowflake plan estimate), over the 10.0 GiB limit: a dbt run of it is held until /allow-scan.', + ) +}) + +test('/scan-cost on an older altimate-code over Databricks says a newer one is needed', async ($, on) => { + await start($, on, { fallback: { success: true, supported: false, adapter: 'databricks' } }) + expect((await $.command.run({ command: 'scan-cost', args: 'orders' } as any)).text).toBe( + 'No estimate: this altimate-code estimates BigQuery only; Databricks needs a newer altimate-code (the release with Snowflake and Databricks estimates).', + ) +}) + +test('a BigQuery dry run is shown as before, with no estimate source', async ($, on) => { + const { clock } = await start($, on, { answers: { orders: UNDER } }) + const result: any = await $.tool.call(bash('dbt run -s orders')) + await clock.settle() + expect(result.context).toEqual(['cost · orders would scan 2.0 GiB']) +}) + +test('on Snowflake, a hold kept while the estimate fails on the retry still names the plan estimate, and so does /allow-scan', async ($, on) => { + const { clock, ran } = await start($, on, { + sequence: { orders: [planned('snowflake', 20 * GIB, '20.0 GiB'), { success: false, adapter: 'snowflake', source: 'explain', error: 'EXPLAIN failed' }] }, + }) + expect(((await $.tool.call(bash('dbt run -s orders'))) as any).deny).toContain('(Snowflake plan estimate)') + const retry: any = await $.tool.call(bash('dbt run -s orders')) + await clock.settle() + + expect(ran).toEqual([]) + expect(retry.deny).toBe('≈ 20.0 GiB would be scanned (Snowflake plan estimate), over the 10.0 GiB limit. Narrow the query, or ask the user to run /allow-scan and then run the same command again.') + expect((await $.command.run({ command: 'allow-scan', args: '' } as any)).text).toContain('orders (≈ 20.0 GiB, Snowflake plan estimate)') +}) + +/** The band above the prompt, as health's tests mount it. */ +const BAND = { + plugin: 'altimate-code', + component: 'AbovePrompt', + props: { hasSurvey: false, isCollapsed: false, isWorking: false, maxRows: 10, bodyColumns: 120, scroll: { offset: 0, bodyRows: 10 }, view: {} }, +} as const + +/** Whether the mounted band shows this exact line. */ +async function bandShows($: any, text: string): Promise { + const band = await $.ui.mount({ ...BAND, surface: 'terminal' }) + const found = await band.find({ type: 'Text', text }) + await band.unmount() + return found !== undefined && found !== null +} + +for (const [adapter, name] of [ + ['snowflake', 'Snowflake'], + ['databricks', 'Databricks'], +] as const) { + test(`on ${adapter}, the band's held line names the plan estimate, and still does once the retry's estimate fails`, async ($, on) => { + const { clock } = await start($, on, { sequence: { orders: [planned(adapter, 20 * GIB, '20.0 GiB'), { success: false, adapter, source: 'explain', error: 'EXPLAIN failed' }] } }) + await $.tool.call(bash('dbt run -s orders')) + await clock.settle() + expect(await bandShows($, `⛔ held: orders would scan 20.0 GiB (${name} plan estimate) (limit 10.0 GiB) · /allow-scan`)).toBe(true) + await $.tool.call(bash('dbt run -s orders')) + await clock.settle() + expect(await bandShows($, `⛔ held: orders would scan 20.0 GiB (${name} plan estimate) (limit 10.0 GiB) · /allow-scan`)).toBe(true) + }) +} + +test("on BigQuery, the band's held line is as before", async ($, on) => { + const { clock } = await start($, on, { answers: { orders: OVER } }) + await $.tool.call(bash('dbt run -s orders')) + await clock.settle() + expect(await bandShows($, '⛔ held: orders would scan 20.0 GiB (limit 10.0 GiB) · /allow-scan')).toBe(true) +}) diff --git a/plugins/altimate-code/hooks/features/cost.tsx b/plugins/altimate-code/hooks/features/cost.tsx index f36ff11d..1303438f 100644 --- a/plugins/altimate-code/hooks/features/cost.tsx +++ b/plugins/altimate-code/hooks/features/cost.tsx @@ -1,9 +1,11 @@ -// Cost check: BigQuery scans over the limit are held until allowed. +// Cost check: dbt scans over the limit are held until allowed. // // Before a `dbt run`, `dbt build` or `dbt show` Claude runs, the gate asks altimate-code's -// dbt_cost_estimate (a BigQuery dry run) what the query would scan. Over `maxScanBytes` the command is -// refused with the estimate until the user runs /allow-scan, which lets that same command through once. -// Other warehouses have no dry run: the first answer says so, and nothing is estimated or held after that. +// dbt_cost_estimate what the query would scan: a dry run on BigQuery, the planner's EXPLAIN estimate on +// Snowflake and Databricks. Over `maxScanBytes` the command is refused with the estimate until the user runs +// /allow-scan, which lets that same command through once. A query with no estimate (a Databricks table without +// statistics, an EXPLAIN that failed) runs, with a line saying why. Other warehouses have no estimate: the first +// answer says so, and nothing is estimated or held after that. // Only what names its query is estimated: `dbt show --inline ` and a single plain model name // (`-s orders`); any other selection, and any command that sets its own target, defer state, vars or // project, runs as it is, with one line saying so under the Bash row. @@ -34,6 +36,10 @@ const PLAIN_MODEL = /^[A-Za-z_][A-Za-z0-9_]*$/ const UNKNOWN_TOOL = /Unknown tool/i /** The altimate-code that has dbt_cost_estimate; changing it is the one edit a release needs. */ const NEEDED_VERSION = 'a newer altimate-code (the release with dbt_cost_estimate)' +/** The altimate-code whose dbt_cost_estimate reads Snowflake's and Databricks' EXPLAIN. */ +const NEEDED_PLANNER_VERSION = 'a newer altimate-code (the release with Snowflake and Databricks estimates)' +/** The warehouses whose estimate is the planner's (EXPLAIN), by adapter type: how the estimate is named to the user. */ +const PLANNER_NAMES: Record = { snowflake: 'Snowflake', databricks: 'Databricks' } const costAtom = atom({ plugin: 'altimate-code', key: 'cost' } as const, null as CostState | null) @@ -144,7 +150,7 @@ export function register(on: On, context: FeatureContext) { }) if (!held) return { text: 'Nothing is held: no dbt command is waiting on its scan size.' } if (!allowed) return { text: 'The held command changed meanwhile: run /allow-scan again to allow the one now held.' } - return { text: `Allowed: ${held.target} (≈ ${held.display}) runs once when Claude runs the same command again; after that it is held again.` } + return { text: `Allowed: ${held.target} (≈ ${held.display}${held.planner ? `, ${held.planner} plan estimate` : ''}) runs once when Claude runs the same command again; after that it is held again.` } }) on('command.run', { command: 'scan-cost' }, async ($, e) => { @@ -163,9 +169,9 @@ export function register(on: On, context: FeatureContext) { if (answer.kind === 'bytes') { await forgetUnsupported($) await rememberEstimate($, target, answer.bytes, answer.display) - return { text: `${target} would scan ${answer.display}${verdict(answer.maxBytes)}` } + return { text: `${target} would scan ${answer.display}${basisOf(answer)}${verdict(answer.maxBytes)}` } } - return { text: answer.kind === 'unsupported' ? `No estimate: ${answer.adapter} has no dry run (only BigQuery has).` : `No estimate: ${answer.reason}` } + return { text: answer.kind === 'unsupported' ? `No estimate: ${unsupportedWhy(answer.adapter)}.` : `No estimate: ${answer.reason}` } }) } @@ -180,8 +186,8 @@ export async function sessionStart($: EngineInterface, e: Args<'session.start'>) active.clear() await update($, costAtom, () => ({ ...FRESH })) await setStatusPart($, 'cost', null) - await $.command.register({ name: 'allow-scan', description: 'Let the dbt command held for its BigQuery scan size run once, when run again unchanged' }) - await $.command.register({ name: 'scan-cost', description: 'Estimate the BigQuery scan of a dbt model before running it', argumentHint: '' }) + await $.command.register({ name: 'allow-scan', description: 'Let the dbt command held for its scan size run once, when run again unchanged' }) + await $.command.register({ name: 'scan-cost', description: 'Estimate the scan of a dbt model before running it (BigQuery, Snowflake, Databricks)', argumentHint: '' }) } /** Also called for a subagent's turns, which carry `e.agentId`. */ @@ -196,7 +202,7 @@ export async function sessionEnd($: EngineInterface, e: Args<'session.end'>) {} export async function bandLine($: EngineInterface, ui: BandUi): Promise { const held = (await read($, costAtom))?.held if (!held) return null - return {`⛔ held: ${held.target} would scan ${held.display} (limit ${formatBytes(limitBytes)}) · /allow-scan`} + return {`⛔ held: ${held.target} would scan ${held.display}${basisOf(held)} (limit ${formatBytes(limitBytes)}) · /allow-scan`} } /** The command to run in place of `e.command`; null keeps it. */ @@ -228,7 +234,7 @@ async function leaveProject($: EngineInterface) { } /** Per command: the over-limit answer a pass has in hand but has not yet recorded as the hold. */ -const overNow = new Map() +const overNow = new Map() async function gate($: EngineInterface, e: BashCall, mine: { note?: object }): Promise<{ deny: string } | null> { if (limitBytes <= 0) return null @@ -264,7 +270,7 @@ async function gate($: EngineInterface, e: BashCall, mine: { note?: object }): P const lines: string[] = [] let estimated = false - const over: { label: string; bytes: number; point: number; display: string }[] = [] + const over: { label: string; bytes: number; point: number; display: string; planner?: string }[] = [] // One wait for the whole command, however many queries it names. const waitUntil = (await $.clock.now()) + GATE_WAIT_MS const pending: string[] = [] @@ -279,7 +285,7 @@ async function gate($: EngineInterface, e: BashCall, mine: { note?: object }): P await update($, costAtom, prev => { const current = prev ?? FRESH const same = current.held?.command === e.command - return { ...current, allowOnce: same && current.allowOnce, held: { command: e.command, target: worst.label, bytes: worst.bytes, display: worst.display } } + return { ...current, allowOnce: same && current.allowOnce, held: { command: e.command, target: worst.label, bytes: worst.bytes, display: worst.display, ...(worst.planner && { planner: worst.planner }) } } }) } catch { // The refusal stands whether or not the hold could be recorded: a gate that throws lets the call run. @@ -309,9 +315,9 @@ async function gate($: EngineInterface, e: BashCall, mine: { note?: object }): P // The answer in hand is still the project's: one over the limit is kept, so the refusal holds the command. if (answer !== 'pending' && answer.kind === 'bytes' && answer.maxBytes > limitBytes) { // Noted before any await, as on the normal path: a parallel pass for this command whose own estimate fails must still see it. - mine.note = { label, bytes: answer.maxBytes, display: answer.display } - overNow.set(e.command, mine.note as { label: string; bytes: number; display: string }) - over.push({ label, bytes: answer.maxBytes, point: answer.bytes, display: answer.display }) + mine.note = { label, bytes: answer.maxBytes, display: answer.display, planner: answer.planner } + overNow.set(e.command, mine.note as { label: string; bytes: number; display: string; planner?: string }) + over.push({ label, bytes: answer.maxBytes, point: answer.bytes, display: answer.display, planner: answer.planner }) } return unreadable() } @@ -331,7 +337,7 @@ async function gate($: EngineInterface, e: BashCall, mine: { note?: object }): P if (generation !== startedAt) break await update($, costAtom, prev => ({ ...(prev ?? FRESH), unsupportedAdapter: answer.adapter, held: null, allowOnce: false })) await showStatus($, null) - lines.push(`cost · no estimate on ${answer.adapter}: only BigQuery has a dry run`) + lines.push(`cost · no estimate on ${answer.adapter}: ${unsupportedWhy(answer.adapter)}`) over.length = 0 pending.length = 0 break @@ -344,12 +350,12 @@ async function gate($: EngineInterface, e: BashCall, mine: { note?: object }): P estimated = true // Noted before any await: a parallel pass for this command whose own estimate fails must still see it. if (answer.maxBytes > limitBytes) { - mine.note = { label, bytes: answer.maxBytes, display: answer.display } - overNow.set(e.command, mine.note as { label: string; bytes: number; display: string }) + mine.note = { label, bytes: answer.maxBytes, display: answer.display, planner: answer.planner } + overNow.set(e.command, mine.note as { label: string; bytes: number; display: string; planner?: string }) } await rememberEstimate($, label, answer.bytes, answer.display) - if (answer.maxBytes <= limitBytes) lines.push(`cost · ${label} would scan ${answer.display}`) - else over.push({ label, bytes: answer.maxBytes, point: answer.bytes, display: answer.display }) + if (answer.maxBytes <= limitBytes) lines.push(`cost · ${label} would scan ${answer.display}${basisOf(answer)}`) + else over.push({ label, bytes: answer.maxBytes, point: answer.bytes, display: answer.display, planner: answer.planner }) } if (generation !== startedAt) { @@ -370,9 +376,9 @@ async function gate($: EngineInterface, e: BashCall, mine: { note?: object }): P // The hold is read now, not from the state taken before the estimates: a parallel call may have recorded it meanwhile. // Or from an over-limit answer a parallel pass for the same command has in hand and has not recorded yet. const stored = over.length === 0 && failed ? (await read($, costAtom))?.held : null - const heldNow = stored && stored.command === e.command ? { label: stored.target, bytes: stored.bytes, display: stored.display } : overNow.get(e.command) + const heldNow = stored && stored.command === e.command ? { label: stored.target, bytes: stored.bytes, display: stored.display, planner: stored.planner } : overNow.get(e.command) if (over.length === 0 && failed && heldNow && heldNow.bytes > limitBytes) { - over.push({ label: heldNow.label, bytes: heldNow.bytes, point: heldNow.bytes, display: heldNow.display }) + over.push({ label: heldNow.label, bytes: heldNow.bytes, point: heldNow.bytes, display: heldNow.display, planner: heldNow.planner }) } if (over.length > 0) { @@ -385,15 +391,15 @@ async function gate($: EngineInterface, e: BashCall, mine: { note?: object }): P granted = current.allowOnce && current.held?.command === e.command return granted ? { ...current, allowOnce: false, held: null } - : { ...current, allowOnce: false, held: { command: e.command, target: worst.label, bytes: worst.bytes, display: worst.display } } + : { ...current, allowOnce: false, held: { command: e.command, target: worst.label, bytes: worst.bytes, display: worst.display, ...(worst.planner && { planner: worst.planner }) } } }) } catch { // A hold that cannot be recorded must not become a pass: a gate that throws lets the call run. notes.delete(e.command) - return { deny: `${worst.display} would be scanned, over the ${formatBytes(limitBytes)} limit, and the hold could not be recorded. Run the same command again.` } + return { deny: `${worst.display} would be scanned${basisOf(worst)}, over the ${formatBytes(limitBytes)} limit, and the hold could not be recorded. Run the same command again.` } } if (granted) { - for (const one of over) lines.push(`cost · ${one.label} would scan ${one.display}, over the ${formatBytes(limitBytes)} limit: allowed once`) + for (const one of over) lines.push(`cost · ${one.label} would scan ${one.display}${basisOf(one)}, over the ${formatBytes(limitBytes)} limit: allowed once`) keepNotes(e.command, lines) // The hold is spent: the status line stops showing it. await showLastEstimate($) @@ -411,7 +417,7 @@ async function gate($: EngineInterface, e: BashCall, mine: { note?: object }): P ? `just over ${formatBytes(limitBytes)}` : `≈ ${worst.display}` return { - deny: `${scanned} would be scanned, over the ${formatBytes(limitBytes)} limit. Narrow the query, or ask the user to run /allow-scan and then run the same command again.`, + deny: `${scanned} would be scanned${basisOf(worst)}, over the ${formatBytes(limitBytes)} limit. Narrow the query, or ask the user to run /allow-scan and then run the same command again.`, } } // A command that was estimated and is under the limit ends an earlier hold, and its allowance. One that @@ -657,14 +663,28 @@ function shellWords(text: string): string[] { return words } -type Answer = { kind: 'bytes'; bytes: number; maxBytes: number; display: string } | { kind: 'unsupported'; adapter: string } | { kind: 'none'; reason: string } +/** `planner` names the warehouse when the estimate is its planner's (an EXPLAIN), not a dry run. */ +type Answer = { kind: 'bytes'; bytes: number; maxBytes: number; display: string; planner?: string } | { kind: 'unsupported'; adapter: string } | { kind: 'none'; reason: string } + +/** How the estimate was made, as a tail for the number: nothing for a dry run, the planner's name for an EXPLAIN. */ +function basisOf(answer: { planner?: string }): string { + return answer.planner ? ` (${answer.planner} plan estimate)` : '' +} + +/** Why a warehouse has no estimate: an altimate-code too old for its planner, or one altimate-code cannot estimate. */ +function unsupportedWhy(adapter: string): string { + const planner = PLANNER_NAMES[adapter] + return planner + ? `this altimate-code estimates BigQuery only; ${planner} needs ${NEEDED_PLANNER_VERSION}` + : 'scans are estimated on BigQuery, Snowflake and Databricks only' +} /** Asks altimate-code for the scan of a model or SQL; never throws: a failure is `none` with why. */ async function estimate($: EngineInterface, args: { model?: string; sql?: string }, until: number): Promise { try { const run = await runTool($, 'dbt_cost_estimate', args, { write: true, until }) if (!run.ok && UNKNOWN_TOOL.test(run.text)) return { kind: 'none', reason: `this altimate-code has no cost estimate: it needs ${NEEDED_VERSION}` } - const data = run.metadata as { supported?: unknown; adapter?: unknown; bytes?: unknown; maxBytes?: unknown; lowerBound?: unknown; display?: unknown; error?: unknown } | null + const data = run.metadata as { supported?: unknown; adapter?: unknown; source?: unknown; bytes?: unknown; maxBytes?: unknown; lowerBound?: unknown; display?: unknown; error?: unknown } | null if (data?.supported === false) return { kind: 'unsupported', adapter: typeof data.adapter === 'string' ? data.adapter : 'this warehouse' } // A dry run that can undercount (a zero, a script) gives no upper bound to hold a command on. if (data?.supported === true && data.lowerBound === true) { @@ -677,7 +697,9 @@ async function estimate($: EngineInterface, args: { model?: string; sql?: string if (data?.supported === true && typeof data.bytes === 'number' && Number.isFinite(data.bytes)) { // The tool's upper bound when it gives one: the display is rounded, and the limit is a hard one. const maxBytes = typeof data.maxBytes === 'number' && Number.isFinite(data.maxBytes) && data.maxBytes >= data.bytes ? data.maxBytes : data.bytes - return { kind: 'bytes', bytes: data.bytes, maxBytes, display: typeof data.display === 'string' ? data.display : formatBytes(data.bytes) } + // The planner's estimate (Snowflake, Databricks) has no rounding bound: the limit is held against the estimate itself. + const planner = data.source === 'explain' ? (typeof data.adapter === 'string' ? (PLANNER_NAMES[data.adapter] ?? data.adapter) : 'the warehouse') : undefined + return { kind: 'bytes', bytes: data.bytes, maxBytes, display: typeof data.display === 'string' ? data.display : formatBytes(data.bytes), ...(planner && { planner }) } } const why = typeof data?.error === 'string' ? data.error : run.text.trim().split('\n')[0] || 'altimate-code gave no estimate' return { kind: 'none', reason: why } diff --git a/plugins/altimate-code/types/index.d.ts b/plugins/altimate-code/types/index.d.ts index 7c4a338a..3b9c021a 100644 --- a/plugins/altimate-code/types/index.d.ts +++ b/plugins/altimate-code/types/index.d.ts @@ -501,12 +501,12 @@ export interface CommentsState { // Filled in by that feature alone (hooks/features/); the other features never read or write it. // --------------------------------------------------------------------------------------------------- export interface CostState { - /** The warehouse that has no dry run (any but BigQuery), once the tool said so: nothing is estimated after that. */ + /** The warehouse with no estimate (not BigQuery, Snowflake or Databricks, or an altimate-code too old for it), once the tool said so: nothing is estimated after that. */ unsupportedAdapter: string | null - /** The newest estimate: what was estimated, in bytes, as the dry run wrote it. */ + /** The newest estimate: what was estimated, in bytes, as the dry run or the planner gave it. */ lastEstimate: { target: string; bytes: number; display: string } | null - /** The command held for being over the limit, until it is allowed or a later dbt command is estimated under the limit; `bytes` is its scan's upper bound. */ - held: { command: string; target: string; bytes: number; display: string } | null + /** The command held for being over the limit, until it is allowed or a later dbt command is estimated under the limit; `bytes` is its scan's upper bound, `planner` the warehouse whose planner made the estimate (none for a dry run). */ + held: { command: string; target: string; bytes: number; display: string; planner?: string } | null /** /allow-scan was given for the held command: that same command runs once. */ allowOnce: boolean }